1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * CDX bus driver.
  4 *
  5 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
  6 */
  7
  8/*
  9 * Architecture Overview
 10 * =====================
 11 * CDX is a Hardware Architecture designed for AMD FPGA devices. It
 12 * consists of sophisticated mechanism for interaction between FPGA,
 13 * Firmware and the APUs (Application CPUs).
 14 *
 15 * Firmware resides on RPU (Realtime CPUs) which interacts with
 16 * the FPGA program manager and the APUs. The RPU provides memory-mapped
 17 * interface (RPU if) which is used to communicate with APUs.
 18 *
 19 * The diagram below shows an overview of the CDX architecture:
 20 *
 21 *          +--------------------------------------+
 22 *          |    Application CPUs (APU)            |
 23 *          |                                      |
 24 *          |                    CDX device drivers|
 25 *          |     Linux OS                |        |
 26 *          |                        CDX bus       |
 27 *          |                             |        |
 28 *          |                     CDX controller   |
 29 *          |                             |        |
 30 *          +-----------------------------|--------+
 31 *                                        | (discover, config,
 32 *                                        |  reset, rescan)
 33 *                                        |
 34 *          +------------------------| RPU if |----+
 35 *          |                             |        |
 36 *          |                             V        |
 37 *          |          Realtime CPUs (RPU)         |
 38 *          |                                      |
 39 *          +--------------------------------------+
 40 *                                |
 41 *          +---------------------|----------------+
 42 *          |  FPGA               |                |
 43 *          |      +-----------------------+       |
 44 *          |      |           |           |       |
 45 *          | +-------+    +-------+   +-------+   |
 46 *          | | dev 1 |    | dev 2 |   | dev 3 |   |
 47 *          | +-------+    +-------+   +-------+   |
 48 *          +--------------------------------------+
 49 *
 50 * The RPU firmware extracts the device information from the loaded FPGA
 51 * image and implements a mechanism that allows the APU drivers to
 52 * enumerate such devices (device personality and resource details) via
 53 * a dedicated communication channel. RPU mediates operations such as
 54 * discover, reset and rescan of the FPGA devices for the APU. This is
 55 * done using memory mapped interface provided by the RPU to APU.
 56 */
 57
 58#include <linux/init.h>
 59#include <linux/irqdomain.h>
 60#include <linux/kernel.h>
 61#include <linux/of.h>
 62#include <linux/of_device.h>
 63#include <linux/of_platform.h>
 64#include <linux/platform_device.h>
 65#include <linux/slab.h>
 66#include <linux/mm.h>
 67#include <linux/idr.h>
 68#include <linux/cdx/cdx_bus.h>
 69#include <linux/iommu.h>
 70#include <linux/dma-map-ops.h>
 71#include <linux/debugfs.h>
 72#include "cdx.h"
 73
 74/* Default DMA mask for devices on a CDX bus */
 75#define CDX_DEFAULT_DMA_MASK	(~0ULL)
 76#define MAX_CDX_CONTROLLERS 16
 77
 78/* IDA for CDX controllers registered with the CDX bus */
 79static DEFINE_IDA(cdx_controller_ida);
 80/* Lock to protect controller ops */
 81static DEFINE_MUTEX(cdx_controller_lock);
 82/* Debugfs dir for cdx bus */
 83static struct dentry *cdx_debugfs_dir;
 84
 85static char *compat_node_name = "xlnx,versal-net-cdx";
 86
 87static void cdx_destroy_res_attr(struct cdx_device *cdx_dev, int num);
 88
 89/**
 90 * cdx_dev_reset - Reset a CDX device
 91 * @dev: CDX device
 92 *
 93 * Return: -errno on failure, 0 on success.
 94 */
 95int cdx_dev_reset(struct device *dev)
 96{
 97	struct cdx_device *cdx_dev = to_cdx_device(dev);
 98	struct cdx_controller *cdx = cdx_dev->cdx;
 99	struct cdx_device_config dev_config = {0};
100	struct cdx_driver *cdx_drv;
101	int ret;
102
103	cdx_drv = to_cdx_driver(dev->driver);
104	/* Notify driver that device is being reset */
105	if (cdx_drv && cdx_drv->reset_prepare)
106		cdx_drv->reset_prepare(cdx_dev);
107
108	dev_config.type = CDX_DEV_RESET_CONF;
109	ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
110				      cdx_dev->dev_num, &dev_config);
111	if (ret)
112		dev_err(dev, "cdx device reset failed\n");
113
114	/* Notify driver that device reset is complete */
115	if (cdx_drv && cdx_drv->reset_done)
116		cdx_drv->reset_done(cdx_dev);
117
118	return ret;
119}
120EXPORT_SYMBOL_GPL(cdx_dev_reset);
121
122/**
123 * reset_cdx_device - Reset a CDX device
124 * @dev: CDX device
125 * @data: This is always passed as NULL, and is not used in this API,
126 *    but is required here as the device_for_each_child() API expects
127 *    the passed function to have this as an argument.
128 *
129 * Return: -errno on failure, 0 on success.
130 */
131static int reset_cdx_device(struct device *dev, void *data)
132{
133	return cdx_dev_reset(dev);
134}
135
136/**
137 * cdx_unregister_device - Unregister a CDX device
138 * @dev: CDX device
139 * @data: This is always passed as NULL, and is not used in this API,
140 *	  but is required here as the bus_for_each_dev() API expects
141 *	  the passed function (cdx_unregister_device) to have this
142 *	  as an argument.
143 *
144 * Return: 0 on success.
145 */
146static int cdx_unregister_device(struct device *dev,
147				 void *data)
148{
149	struct cdx_device *cdx_dev = to_cdx_device(dev);
150	struct cdx_controller *cdx = cdx_dev->cdx;
151
152	if (cdx_dev->is_bus) {
153		device_for_each_child(dev, NULL, cdx_unregister_device);
154		if (cdx_dev->enabled && cdx->ops->bus_disable)
155			cdx->ops->bus_disable(cdx, cdx_dev->bus_num);
156	} else {
157		cdx_destroy_res_attr(cdx_dev, MAX_CDX_DEV_RESOURCES);
158		debugfs_remove_recursive(cdx_dev->debugfs_dir);
159		kfree(cdx_dev->driver_override);
160		cdx_dev->driver_override = NULL;
161	}
162
163	/*
164	 * Do not free cdx_dev here as it would be freed in
165	 * cdx_device_release() called from within put_device().
166	 */
167	device_del(&cdx_dev->dev);
168	put_device(&cdx_dev->dev);
169
170	return 0;
171}
172
173static void cdx_unregister_devices(struct bus_type *bus)
174{
175	/* Reset all the devices attached to cdx bus */
176	bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
177}
178
179/**
180 * cdx_match_one_device - Tell if a CDX device structure has a matching
181 *			  CDX device id structure
182 * @id: single CDX device id structure to match
183 * @dev: the CDX device structure to match against
184 *
185 * Return: matching cdx_device_id structure or NULL if there is no match.
186 */
187static inline const struct cdx_device_id *
188cdx_match_one_device(const struct cdx_device_id *id,
189		     const struct cdx_device *dev)
190{
191	/* Use vendor ID and device ID for matching */
192	if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) &&
193	    (id->device == CDX_ANY_ID || id->device == dev->device) &&
194	    (id->subvendor == CDX_ANY_ID || id->subvendor == dev->subsystem_vendor) &&
195	    (id->subdevice == CDX_ANY_ID || id->subdevice == dev->subsystem_device) &&
196	    !((id->class ^ dev->class) & id->class_mask))
197		return id;
198	return NULL;
199}
200
201/**
202 * cdx_match_id - See if a CDX device matches a given cdx_id table
203 * @ids: array of CDX device ID structures to search in
204 * @dev: the CDX device structure to match against.
205 *
206 * Used by a driver to check whether a CDX device is in its list of
207 * supported devices. Returns the matching cdx_device_id structure or
208 * NULL if there is no match.
209 *
210 * Return: matching cdx_device_id structure or NULL if there is no match.
211 */
212static inline const struct cdx_device_id *
213cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
214{
215	if (ids) {
216		while (ids->vendor || ids->device) {
217			if (cdx_match_one_device(ids, dev))
218				return ids;
219			ids++;
220		}
221	}
222	return NULL;
223}
224
225int cdx_set_master(struct cdx_device *cdx_dev)
226{
227	struct cdx_controller *cdx = cdx_dev->cdx;
228	struct cdx_device_config dev_config;
229	int ret = -EOPNOTSUPP;
230
231	dev_config.type = CDX_DEV_BUS_MASTER_CONF;
232	dev_config.bus_master_enable = true;
233	if (cdx->ops->dev_configure)
234		ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
235					      cdx_dev->dev_num, &dev_config);
236
237	return ret;
238}
239EXPORT_SYMBOL_GPL(cdx_set_master);
240
241int cdx_clear_master(struct cdx_device *cdx_dev)
242{
243	struct cdx_controller *cdx = cdx_dev->cdx;
244	struct cdx_device_config dev_config;
245	int ret = -EOPNOTSUPP;
246
247	dev_config.type = CDX_DEV_BUS_MASTER_CONF;
248	dev_config.bus_master_enable = false;
249	if (cdx->ops->dev_configure)
250		ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num,
251					      cdx_dev->dev_num, &dev_config);
252
253	return ret;
254}
255EXPORT_SYMBOL_GPL(cdx_clear_master);
256
257/**
258 * cdx_bus_match - device to driver matching callback
259 * @dev: the cdx device to match against
260 * @drv: the device driver to search for matching cdx device
261 * structures
262 *
263 * Return: true on success, false otherwise.
264 */
265static int cdx_bus_match(struct device *dev, const struct device_driver *drv)
266{
267	struct cdx_device *cdx_dev = to_cdx_device(dev);
268	const struct cdx_driver *cdx_drv = to_cdx_driver(drv);
269	const struct cdx_device_id *found_id = NULL;
270	const struct cdx_device_id *ids;
271
272	if (cdx_dev->is_bus)
273		return false;
274
275	ids = cdx_drv->match_id_table;
276
277	/* When driver_override is set, only bind to the matching driver */
278	if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name))
279		return false;
280
281	found_id = cdx_match_id(ids, cdx_dev);
282	if (!found_id)
283		return false;
284
285	do {
286		/*
287		 * In case override_only was set, enforce driver_override
288		 * matching.
289		 */
290		if (!found_id->override_only)
291			return true;
292		if (cdx_dev->driver_override)
293			return true;
294
295		ids = found_id + 1;
296		found_id = cdx_match_id(ids, cdx_dev);
297	} while (found_id);
298
299	return false;
300}
301
302static int cdx_probe(struct device *dev)
303{
304	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
305	struct cdx_device *cdx_dev = to_cdx_device(dev);
306	struct cdx_controller *cdx = cdx_dev->cdx;
307	int error;
308
309	/*
310	 * Setup MSI device data so that generic MSI alloc/free can
311	 * be used by the device driver.
312	 */
313	if (cdx->msi_domain) {
314		error = msi_setup_device_data(&cdx_dev->dev);
315		if (error)
316			return error;
317	}
318
319	error = cdx_drv->probe(cdx_dev);
320	if (error) {
321		dev_err_probe(dev, error, "%s failed\n", __func__);
322		return error;
323	}
324
325	return 0;
326}
327
328static void cdx_remove(struct device *dev)
329{
330	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
331	struct cdx_device *cdx_dev = to_cdx_device(dev);
332
333	if (cdx_drv && cdx_drv->remove)
334		cdx_drv->remove(cdx_dev);
335}
336
337static void cdx_shutdown(struct device *dev)
338{
339	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
340	struct cdx_device *cdx_dev = to_cdx_device(dev);
341
342	if (cdx_drv && cdx_drv->shutdown)
343		cdx_drv->shutdown(cdx_dev);
344}
345
346static int cdx_dma_configure(struct device *dev)
347{
348	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
349	struct cdx_device *cdx_dev = to_cdx_device(dev);
350	struct cdx_controller *cdx = cdx_dev->cdx;
351	u32 input_id = cdx_dev->req_id;
352	int ret;
353
354	ret = of_dma_configure_id(dev, cdx->dev->of_node, 0, &input_id);
355	if (ret && ret != -EPROBE_DEFER) {
356		dev_err(dev, "of_dma_configure_id() failed\n");
357		return ret;
358	}
359
360	if (!ret && !cdx_drv->driver_managed_dma) {
361		ret = iommu_device_use_default_domain(dev);
362		if (ret)
363			arch_teardown_dma_ops(dev);
364	}
365
366	return 0;
367}
368
369static void cdx_dma_cleanup(struct device *dev)
370{
371	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
372
373	if (!cdx_drv->driver_managed_dma)
374		iommu_device_unuse_default_domain(dev);
375}
376
377/* show configuration fields */
378#define cdx_config_attr(field, format_string)	\
379static ssize_t	\
380field##_show(struct device *dev, struct device_attribute *attr, char *buf)	\
381{	\
382	struct cdx_device *cdx_dev = to_cdx_device(dev);	\
383	return sysfs_emit(buf, format_string, cdx_dev->field);	\
384}	\
385static DEVICE_ATTR_RO(field)
386
387cdx_config_attr(vendor, "0x%04x\n");
388cdx_config_attr(device, "0x%04x\n");
389cdx_config_attr(subsystem_vendor, "0x%04x\n");
390cdx_config_attr(subsystem_device, "0x%04x\n");
391cdx_config_attr(revision, "0x%02x\n");
392cdx_config_attr(class, "0x%06x\n");
393
394static ssize_t remove_store(struct device *dev,
395			    struct device_attribute *attr,
396			    const char *buf, size_t count)
397{
398	bool val;
399
400	if (kstrtobool(buf, &val) < 0)
401		return -EINVAL;
402
403	if (!val)
404		return -EINVAL;
405
406	if (device_remove_file_self(dev, attr)) {
407		int ret;
408
409		ret = cdx_unregister_device(dev, NULL);
410		if (ret)
411			return ret;
412	}
413
414	return count;
415}
416static DEVICE_ATTR_WO(remove);
417
418static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
419			   const char *buf, size_t count)
420{
421	struct cdx_device *cdx_dev = to_cdx_device(dev);
422	bool val;
423	int ret;
424
425	if (kstrtobool(buf, &val) < 0)
426		return -EINVAL;
427
428	if (!val)
429		return -EINVAL;
430
431	if (cdx_dev->is_bus)
432		/* Reset all the devices attached to cdx bus */
433		ret = device_for_each_child(dev, NULL, reset_cdx_device);
434	else
435		ret = cdx_dev_reset(dev);
436
437	return ret < 0 ? ret : count;
438}
439static DEVICE_ATTR_WO(reset);
440
441static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
442			     char *buf)
443{
444	struct cdx_device *cdx_dev = to_cdx_device(dev);
445
446	return sprintf(buf, "cdx:v%04Xd%04Xsv%04Xsd%04Xc%06X\n", cdx_dev->vendor,
447			cdx_dev->device, cdx_dev->subsystem_vendor, cdx_dev->subsystem_device,
448			cdx_dev->class);
449}
450static DEVICE_ATTR_RO(modalias);
451
452static ssize_t driver_override_store(struct device *dev,
453				     struct device_attribute *attr,
454				     const char *buf, size_t count)
455{
456	struct cdx_device *cdx_dev = to_cdx_device(dev);
457	int ret;
458
459	if (WARN_ON(dev->bus != &cdx_bus_type))
460		return -EINVAL;
461
462	ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count);
463	if (ret)
464		return ret;
465
466	return count;
467}
468
469static ssize_t driver_override_show(struct device *dev,
470				    struct device_attribute *attr, char *buf)
471{
472	struct cdx_device *cdx_dev = to_cdx_device(dev);
473	ssize_t len;
474
475	device_lock(dev);
476	len = sysfs_emit(buf, "%s\n", cdx_dev->driver_override);
477	device_unlock(dev);
478	return len;
479}
480static DEVICE_ATTR_RW(driver_override);
481
482static ssize_t enable_store(struct device *dev, struct device_attribute *attr,
483			    const char *buf, size_t count)
484{
485	struct cdx_device *cdx_dev = to_cdx_device(dev);
486	struct cdx_controller *cdx = cdx_dev->cdx;
487	bool enable;
488	int ret;
489
490	if (kstrtobool(buf, &enable) < 0)
491		return -EINVAL;
492
493	if (enable == cdx_dev->enabled)
494		return count;
495
496	if (enable && cdx->ops->bus_enable)
497		ret = cdx->ops->bus_enable(cdx, cdx_dev->bus_num);
498	else if (!enable && cdx->ops->bus_disable)
499		ret = cdx->ops->bus_disable(cdx, cdx_dev->bus_num);
500	else
501		ret = -EOPNOTSUPP;
502
503	if (!ret)
504		cdx_dev->enabled = enable;
505
506	return ret < 0 ? ret : count;
507}
508
509static ssize_t enable_show(struct device *dev, struct device_attribute *attr, char *buf)
510{
511	struct cdx_device *cdx_dev = to_cdx_device(dev);
512
513	return sysfs_emit(buf, "%u\n", cdx_dev->enabled);
514}
515static DEVICE_ATTR_RW(enable);
516
517static umode_t cdx_dev_attrs_are_visible(struct kobject *kobj, struct attribute *a, int n)
518{
519	struct device *dev = kobj_to_dev(kobj);
520	struct cdx_device *cdx_dev;
521
522	cdx_dev = to_cdx_device(dev);
523	if (!cdx_dev->is_bus)
524		return a->mode;
525
526	return 0;
527}
528
529static umode_t cdx_bus_attrs_are_visible(struct kobject *kobj, struct attribute *a, int n)
530{
531	struct device *dev = kobj_to_dev(kobj);
532	struct cdx_device *cdx_dev;
533
534	cdx_dev = to_cdx_device(dev);
535	if (cdx_dev->is_bus)
536		return a->mode;
537
538	return 0;
539}
540
541static struct attribute *cdx_dev_attrs[] = {
542	&dev_attr_remove.attr,
543	&dev_attr_reset.attr,
544	&dev_attr_vendor.attr,
545	&dev_attr_device.attr,
546	&dev_attr_subsystem_vendor.attr,
547	&dev_attr_subsystem_device.attr,
548	&dev_attr_class.attr,
549	&dev_attr_revision.attr,
550	&dev_attr_modalias.attr,
551	&dev_attr_driver_override.attr,
552	NULL,
553};
554
555static const struct attribute_group cdx_dev_group = {
556	.attrs = cdx_dev_attrs,
557	.is_visible = cdx_dev_attrs_are_visible,
558};
559
560static struct attribute *cdx_bus_dev_attrs[] = {
561	&dev_attr_enable.attr,
562	&dev_attr_reset.attr,
563	NULL,
564};
565
566static const struct attribute_group cdx_bus_dev_group = {
567	.attrs = cdx_bus_dev_attrs,
568	.is_visible = cdx_bus_attrs_are_visible,
569};
570
571static const struct attribute_group *cdx_dev_groups[] = {
572	&cdx_dev_group,
573	&cdx_bus_dev_group,
574	NULL,
575};
576
577static int cdx_debug_resource_show(struct seq_file *s, void *data)
578{
579	struct cdx_device *cdx_dev = s->private;
580	int i;
581
582	for (i = 0; i < MAX_CDX_DEV_RESOURCES; i++) {
583		struct resource *res =  &cdx_dev->res[i];
584
585		seq_printf(s, "%pr\n", res);
586	}
587
588	return 0;
589}
590DEFINE_SHOW_ATTRIBUTE(cdx_debug_resource);
591
592static void cdx_device_debugfs_init(struct cdx_device *cdx_dev)
593{
594	cdx_dev->debugfs_dir = debugfs_create_dir(dev_name(&cdx_dev->dev), cdx_debugfs_dir);
595	if (IS_ERR(cdx_dev->debugfs_dir))
596		return;
597
598	debugfs_create_file("resource", 0444, cdx_dev->debugfs_dir, cdx_dev,
599			    &cdx_debug_resource_fops);
600}
601
602static ssize_t rescan_store(const struct bus_type *bus,
603			    const char *buf, size_t count)
604{
605	struct cdx_controller *cdx;
606	struct platform_device *pd;
607	struct device_node *np;
608	bool val;
609
610	if (kstrtobool(buf, &val) < 0)
611		return -EINVAL;
612
613	if (!val)
614		return -EINVAL;
615
616	mutex_lock(&cdx_controller_lock);
617
618	/* Unregister all the devices on the bus */
619	cdx_unregister_devices(&cdx_bus_type);
620
621	/* Rescan all the devices */
622	for_each_compatible_node(np, NULL, compat_node_name) {
623		pd = of_find_device_by_node(np);
624		if (!pd) {
625			of_node_put(np);
626			count = -EINVAL;
627			goto unlock;
628		}
629
630		cdx = platform_get_drvdata(pd);
631		if (cdx && cdx->controller_registered && cdx->ops->scan)
632			cdx->ops->scan(cdx);
633
634		put_device(&pd->dev);
635	}
636
637unlock:
638	mutex_unlock(&cdx_controller_lock);
639
640	return count;
641}
642static BUS_ATTR_WO(rescan);
643
644static struct attribute *cdx_bus_attrs[] = {
645	&bus_attr_rescan.attr,
646	NULL,
647};
648ATTRIBUTE_GROUPS(cdx_bus);
649
650struct bus_type cdx_bus_type = {
651	.name		= "cdx",
652	.match		= cdx_bus_match,
653	.probe		= cdx_probe,
654	.remove		= cdx_remove,
655	.shutdown	= cdx_shutdown,
656	.dma_configure	= cdx_dma_configure,
657	.dma_cleanup	= cdx_dma_cleanup,
658	.bus_groups	= cdx_bus_groups,
659	.dev_groups	= cdx_dev_groups,
660};
661EXPORT_SYMBOL_GPL(cdx_bus_type);
662
663int __cdx_driver_register(struct cdx_driver *cdx_driver,
664			  struct module *owner)
665{
666	int error;
667
668	cdx_driver->driver.owner = owner;
669	cdx_driver->driver.bus = &cdx_bus_type;
670
671	error = driver_register(&cdx_driver->driver);
672	if (error) {
673		pr_err("driver_register() failed for %s: %d\n",
674		       cdx_driver->driver.name, error);
675		return error;
676	}
677
678	return 0;
679}
680EXPORT_SYMBOL_GPL(__cdx_driver_register);
681
682void cdx_driver_unregister(struct cdx_driver *cdx_driver)
683{
684	driver_unregister(&cdx_driver->driver);
685}
686EXPORT_SYMBOL_GPL(cdx_driver_unregister);
687
688static void cdx_device_release(struct device *dev)
689{
690	struct cdx_device *cdx_dev = to_cdx_device(dev);
691
692	kfree(cdx_dev);
693}
694
695static const struct vm_operations_struct cdx_phys_vm_ops = {
696#ifdef CONFIG_HAVE_IOREMAP_PROT
697	.access = generic_access_phys,
698#endif
699};
700
701/**
702 * cdx_mmap_resource - map a CDX resource into user memory space
703 * @fp: File pointer. Not used in this function, but required where
704 *      this API is registered as a callback.
705 * @kobj: kobject for mapping
706 * @attr: struct bin_attribute for the file being mapped
707 * @vma: struct vm_area_struct passed into the mmap
708 *
709 * Use the regular CDX mapping routines to map a CDX resource into userspace.
710 *
711 * Return: true on success, false otherwise.
712 */
713static int cdx_mmap_resource(struct file *fp, struct kobject *kobj,
714			     const struct bin_attribute *attr,
715			     struct vm_area_struct *vma)
716{
717	struct cdx_device *cdx_dev = to_cdx_device(kobj_to_dev(kobj));
718	int num = (unsigned long)attr->private;
719	struct resource *res;
720	unsigned long size;
721
722	res = &cdx_dev->res[num];
723	if (iomem_is_exclusive(res->start))
724		return -EINVAL;
725
726	/* Make sure the caller is mapping a valid resource for this device */
727	size = ((cdx_resource_len(cdx_dev, num) - 1) >> PAGE_SHIFT) + 1;
728	if (vma->vm_pgoff + vma_pages(vma) > size)
729		return -EINVAL;
730
731	/*
732	 * Map memory region and vm->vm_pgoff is expected to be an
733	 * offset within that region.
734	 */
735	vma->vm_page_prot = pgprot_device(vma->vm_page_prot);
736	vma->vm_pgoff += (cdx_resource_start(cdx_dev, num) >> PAGE_SHIFT);
737	vma->vm_ops = &cdx_phys_vm_ops;
738	return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
739				  vma->vm_end - vma->vm_start,
740				  vma->vm_page_prot);
741}
742
743static void cdx_destroy_res_attr(struct cdx_device *cdx_dev, int num)
744{
745	int i;
746
747	/* removing the bin attributes */
748	for (i = 0; i < num; i++) {
749		struct bin_attribute *res_attr;
750
751		res_attr = cdx_dev->res_attr[i];
752		if (res_attr) {
753			sysfs_remove_bin_file(&cdx_dev->dev.kobj, res_attr);
754			kfree(res_attr);
755		}
756	}
757}
758
759#define CDX_RES_ATTR_NAME_LEN	10
760static int cdx_create_res_attr(struct cdx_device *cdx_dev, int num)
761{
762	struct bin_attribute *res_attr;
763	char *res_attr_name;
764	int ret;
765
766	res_attr = kzalloc(sizeof(*res_attr) + CDX_RES_ATTR_NAME_LEN, GFP_ATOMIC);
767	if (!res_attr)
768		return -ENOMEM;
769
770	res_attr_name = (char *)(res_attr + 1);
771
772	sysfs_bin_attr_init(res_attr);
773
774	cdx_dev->res_attr[num] = res_attr;
775	sprintf(res_attr_name, "resource%d", num);
776
777	res_attr->mmap = cdx_mmap_resource;
778	res_attr->attr.name = res_attr_name;
779	res_attr->attr.mode = 0600;
780	res_attr->size = cdx_resource_len(cdx_dev, num);
781	res_attr->private = (void *)(unsigned long)num;
782	ret = sysfs_create_bin_file(&cdx_dev->dev.kobj, res_attr);
783	if (ret)
784		kfree(res_attr);
785
786	return ret;
787}
788
789int cdx_device_add(struct cdx_dev_params *dev_params)
790{
791	struct cdx_controller *cdx = dev_params->cdx;
792	struct cdx_device *cdx_dev;
793	int ret, i;
794
795	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
796	if (!cdx_dev)
797		return -ENOMEM;
798
799	/* Populate resource */
800	memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
801		dev_params->res_count);
802	cdx_dev->res_count = dev_params->res_count;
803
804	/* Populate CDX dev params */
805	cdx_dev->req_id = dev_params->req_id;
806	cdx_dev->msi_dev_id = dev_params->msi_dev_id;
807	cdx_dev->vendor = dev_params->vendor;
808	cdx_dev->device = dev_params->device;
809	cdx_dev->subsystem_vendor = dev_params->subsys_vendor;
810	cdx_dev->subsystem_device = dev_params->subsys_device;
811	cdx_dev->class = dev_params->class;
812	cdx_dev->revision = dev_params->revision;
813	cdx_dev->bus_num = dev_params->bus_num;
814	cdx_dev->dev_num = dev_params->dev_num;
815	cdx_dev->cdx = dev_params->cdx;
816	cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
817
818	/* Initialize generic device */
819	device_initialize(&cdx_dev->dev);
820	cdx_dev->dev.parent = dev_params->parent;
821	cdx_dev->dev.bus = &cdx_bus_type;
822	cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
823	cdx_dev->dev.release = cdx_device_release;
824	cdx_dev->msi_write_pending = false;
825	mutex_init(&cdx_dev->irqchip_lock);
826
827	/* Set Name */
828	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x",
829		     ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)),
830		     cdx_dev->dev_num);
831
832	if (cdx->msi_domain) {
833		cdx_dev->num_msi = dev_params->num_msi;
834		dev_set_msi_domain(&cdx_dev->dev, cdx->msi_domain);
835	}
836
837	ret = device_add(&cdx_dev->dev);
838	if (ret) {
839		dev_err(&cdx_dev->dev,
840			"cdx device add failed: %d", ret);
841		goto fail;
842	}
843
844	/* Create resource<N> attributes */
845	for (i = 0; i < MAX_CDX_DEV_RESOURCES; i++) {
846		if (cdx_resource_flags(cdx_dev, i) & IORESOURCE_MEM) {
847			/* skip empty resources */
848			if (!cdx_resource_len(cdx_dev, i))
849				continue;
850
851			ret = cdx_create_res_attr(cdx_dev, i);
852			if (ret != 0) {
853				dev_err(&cdx_dev->dev,
854					"cdx device resource<%d> file creation failed: %d", i, ret);
855				goto resource_create_fail;
856			}
857		}
858	}
859
860	cdx_device_debugfs_init(cdx_dev);
861
862	return 0;
863resource_create_fail:
864	cdx_destroy_res_attr(cdx_dev, i);
865	device_del(&cdx_dev->dev);
866fail:
867	/*
868	 * Do not free cdx_dev here as it would be freed in
869	 * cdx_device_release() called from put_device().
870	 */
871	put_device(&cdx_dev->dev);
872
873	return ret;
874}
875EXPORT_SYMBOL_NS_GPL(cdx_device_add, "CDX_BUS_CONTROLLER");
876
877struct device *cdx_bus_add(struct cdx_controller *cdx, u8 bus_num)
878{
879	struct cdx_device *cdx_dev;
880	int ret;
881
882	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
883	if (!cdx_dev)
884		return NULL;
885
886	device_initialize(&cdx_dev->dev);
887	cdx_dev->cdx = cdx;
888
889	cdx_dev->dev.parent = cdx->dev;
890	cdx_dev->dev.bus = &cdx_bus_type;
891	cdx_dev->dev.release = cdx_device_release;
892	cdx_dev->is_bus = true;
893	cdx_dev->bus_num = bus_num;
894
895	dev_set_name(&cdx_dev->dev, "cdx-%02x",
896		     ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (bus_num & CDX_BUS_NUM_MASK)));
897
898	ret = device_add(&cdx_dev->dev);
899	if (ret) {
900		dev_err(&cdx_dev->dev, "cdx bus device add failed: %d\n", ret);
901		goto device_add_fail;
902	}
903
904	if (cdx->ops->bus_enable) {
905		ret = cdx->ops->bus_enable(cdx, bus_num);
906		if (ret && ret != -EALREADY) {
907			dev_err(cdx->dev, "cdx bus enable failed: %d\n", ret);
908			goto bus_enable_fail;
909		}
910	}
911
912	cdx_dev->enabled = true;
913	return &cdx_dev->dev;
914
915bus_enable_fail:
916	device_del(&cdx_dev->dev);
917device_add_fail:
918	put_device(&cdx_dev->dev);
919
920	return NULL;
921}
922EXPORT_SYMBOL_NS_GPL(cdx_bus_add, "CDX_BUS_CONTROLLER");
923
924int cdx_register_controller(struct cdx_controller *cdx)
925{
926	int ret;
927
928	ret = ida_alloc_range(&cdx_controller_ida, 0,  MAX_CDX_CONTROLLERS - 1, GFP_KERNEL);
929	if (ret < 0) {
930		dev_err(cdx->dev,
931			"No free index available. Maximum controllers already registered\n");
932		cdx->id = (u8)MAX_CDX_CONTROLLERS;
933		return ret;
934	}
935
936	mutex_lock(&cdx_controller_lock);
937	cdx->id = ret;
938
939	/* Scan all the devices */
940	if (cdx->ops->scan)
941		cdx->ops->scan(cdx);
942	cdx->controller_registered = true;
943	mutex_unlock(&cdx_controller_lock);
944
945	return 0;
946}
947EXPORT_SYMBOL_NS_GPL(cdx_register_controller, "CDX_BUS_CONTROLLER");
948
949void cdx_unregister_controller(struct cdx_controller *cdx)
950{
951	if (cdx->id >= MAX_CDX_CONTROLLERS)
952		return;
953
954	mutex_lock(&cdx_controller_lock);
955
956	cdx->controller_registered = false;
957	device_for_each_child(cdx->dev, NULL, cdx_unregister_device);
958	ida_free(&cdx_controller_ida, cdx->id);
959
960	mutex_unlock(&cdx_controller_lock);
961}
962EXPORT_SYMBOL_NS_GPL(cdx_unregister_controller, "CDX_BUS_CONTROLLER");
963
964static int __init cdx_bus_init(void)
965{
966	int ret;
967
968	ret = bus_register(&cdx_bus_type);
969	if (!ret)
970		cdx_debugfs_dir = debugfs_create_dir(cdx_bus_type.name, NULL);
971
972	return ret;
973}
974postcore_initcall(cdx_bus_init);