1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
   4 */
   5
   6/*
   7 * This code implements the DMA subsystem. It provides a HW-neutral interface
   8 * for other kernel code to use asynchronous memory copy capabilities,
   9 * if present, and allows different HW DMA drivers to register as providing
  10 * this capability.
  11 *
  12 * Due to the fact we are accelerating what is already a relatively fast
  13 * operation, the code goes to great lengths to avoid additional overhead,
  14 * such as locking.
  15 *
  16 * LOCKING:
  17 *
  18 * The subsystem keeps a global list of dma_device structs it is protected by a
  19 * mutex, dma_list_mutex.
  20 *
  21 * A subsystem can get access to a channel by calling dmaengine_get() followed
  22 * by dma_find_channel(), or if it has need for an exclusive channel it can call
  23 * dma_request_channel().  Once a channel is allocated a reference is taken
  24 * against its corresponding driver to disable removal.
  25 *
  26 * Each device has a channels list, which runs unlocked but is never modified
  27 * once the device is registered, it's just setup by the driver.
  28 *
  29 * See Documentation/driver-api/dmaengine for more details
  30 */
  31
  32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  33
  34#include <linux/platform_device.h>
  35#include <linux/dma-mapping.h>
  36#include <linux/init.h>
  37#include <linux/module.h>
  38#include <linux/mm.h>
  39#include <linux/device.h>
  40#include <linux/dmaengine.h>
  41#include <linux/hardirq.h>
  42#include <linux/spinlock.h>
  43#include <linux/percpu.h>
  44#include <linux/rcupdate.h>
  45#include <linux/mutex.h>
  46#include <linux/jiffies.h>
  47#include <linux/rculist.h>
  48#include <linux/idr.h>
  49#include <linux/slab.h>
  50#include <linux/acpi.h>
  51#include <linux/acpi_dma.h>
  52#include <linux/of_dma.h>
  53#include <linux/mempool.h>
  54#include <linux/numa.h>
  55
  56#include "dmaengine.h"
  57
  58static DEFINE_MUTEX(dma_list_mutex);
  59static DEFINE_IDA(dma_ida);
  60static LIST_HEAD(dma_device_list);
  61static long dmaengine_ref_count;
  62
  63/* --- debugfs implementation --- */
  64#ifdef CONFIG_DEBUG_FS
  65#include <linux/debugfs.h>
  66
  67static struct dentry *rootdir;
  68
  69static void dmaengine_debug_register(struct dma_device *dma_dev)
  70{
  71	dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev),
  72						   rootdir);
  73	if (IS_ERR(dma_dev->dbg_dev_root))
  74		dma_dev->dbg_dev_root = NULL;
  75}
  76
  77static void dmaengine_debug_unregister(struct dma_device *dma_dev)
  78{
  79	debugfs_remove_recursive(dma_dev->dbg_dev_root);
  80	dma_dev->dbg_dev_root = NULL;
  81}
  82
  83static void dmaengine_dbg_summary_show(struct seq_file *s,
  84				       struct dma_device *dma_dev)
  85{
  86	struct dma_chan *chan;
  87
  88	list_for_each_entry(chan, &dma_dev->channels, device_node) {
  89		if (chan->client_count) {
  90			seq_printf(s, " %-13s| %s", dma_chan_name(chan),
  91				   chan->dbg_client_name ?: "in-use");
  92
  93			if (chan->router)
  94				seq_printf(s, " (via router: %s)\n",
  95					dev_name(chan->router->dev));
  96			else
  97				seq_puts(s, "\n");
  98		}
  99	}
 100}
 101
 102static int dmaengine_summary_show(struct seq_file *s, void *data)
 103{
 104	struct dma_device *dma_dev = NULL;
 105
 106	mutex_lock(&dma_list_mutex);
 107	list_for_each_entry(dma_dev, &dma_device_list, global_node) {
 108		seq_printf(s, "dma%d (%s): number of channels: %u\n",
 109			   dma_dev->dev_id, dev_name(dma_dev->dev),
 110			   dma_dev->chancnt);
 111
 112		if (dma_dev->dbg_summary_show)
 113			dma_dev->dbg_summary_show(s, dma_dev);
 114		else
 115			dmaengine_dbg_summary_show(s, dma_dev);
 116
 117		if (!list_is_last(&dma_dev->global_node, &dma_device_list))
 118			seq_puts(s, "\n");
 119	}
 120	mutex_unlock(&dma_list_mutex);
 121
 122	return 0;
 123}
 124DEFINE_SHOW_ATTRIBUTE(dmaengine_summary);
 125
 126static void __init dmaengine_debugfs_init(void)
 127{
 128	rootdir = debugfs_create_dir("dmaengine", NULL);
 129
 130	/* /sys/kernel/debug/dmaengine/summary */
 131	debugfs_create_file("summary", 0444, rootdir, NULL,
 132			    &dmaengine_summary_fops);
 133}
 134#else
 135static inline void dmaengine_debugfs_init(void) { }
 136static inline int dmaengine_debug_register(struct dma_device *dma_dev)
 137{
 138	return 0;
 139}
 140
 141static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { }
 142#endif	/* DEBUG_FS */
 143
 144/* --- sysfs implementation --- */
 145
 146#define DMA_SLAVE_NAME	"slave"
 147
 148/**
 149 * dev_to_dma_chan - convert a device pointer to its sysfs container object
 150 * @dev:	device node
 151 *
 152 * Must be called under dma_list_mutex.
 153 */
 154static struct dma_chan *dev_to_dma_chan(struct device *dev)
 155{
 156	struct dma_chan_dev *chan_dev;
 157
 158	chan_dev = container_of(dev, typeof(*chan_dev), device);
 159	return chan_dev->chan;
 160}
 161
 162static ssize_t memcpy_count_show(struct device *dev,
 163				 struct device_attribute *attr, char *buf)
 164{
 165	struct dma_chan *chan;
 166	unsigned long count = 0;
 167	int i;
 168	int err;
 169
 170	mutex_lock(&dma_list_mutex);
 171	chan = dev_to_dma_chan(dev);
 172	if (chan) {
 173		for_each_possible_cpu(i)
 174			count += per_cpu_ptr(chan->local, i)->memcpy_count;
 175		err = sprintf(buf, "%lu\n", count);
 176	} else
 177		err = -ENODEV;
 178	mutex_unlock(&dma_list_mutex);
 179
 180	return err;
 181}
 182static DEVICE_ATTR_RO(memcpy_count);
 183
 184static ssize_t bytes_transferred_show(struct device *dev,
 185				      struct device_attribute *attr, char *buf)
 186{
 187	struct dma_chan *chan;
 188	unsigned long count = 0;
 189	int i;
 190	int err;
 191
 192	mutex_lock(&dma_list_mutex);
 193	chan = dev_to_dma_chan(dev);
 194	if (chan) {
 195		for_each_possible_cpu(i)
 196			count += per_cpu_ptr(chan->local, i)->bytes_transferred;
 197		err = sprintf(buf, "%lu\n", count);
 198	} else
 199		err = -ENODEV;
 200	mutex_unlock(&dma_list_mutex);
 201
 202	return err;
 203}
 204static DEVICE_ATTR_RO(bytes_transferred);
 205
 206static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,
 207			   char *buf)
 208{
 209	struct dma_chan *chan;
 210	int err;
 211
 212	mutex_lock(&dma_list_mutex);
 213	chan = dev_to_dma_chan(dev);
 214	if (chan)
 215		err = sprintf(buf, "%d\n", chan->client_count);
 216	else
 217		err = -ENODEV;
 218	mutex_unlock(&dma_list_mutex);
 219
 220	return err;
 221}
 222static DEVICE_ATTR_RO(in_use);
 223
 224static struct attribute *dma_dev_attrs[] = {
 225	&dev_attr_memcpy_count.attr,
 226	&dev_attr_bytes_transferred.attr,
 227	&dev_attr_in_use.attr,
 228	NULL,
 229};
 230ATTRIBUTE_GROUPS(dma_dev);
 231
 232static void chan_dev_release(struct device *dev)
 233{
 234	struct dma_chan_dev *chan_dev;
 235
 236	chan_dev = container_of(dev, typeof(*chan_dev), device);
 237	kfree(chan_dev);
 238}
 239
 240static struct class dma_devclass = {
 241	.name		= "dma",
 242	.dev_groups	= dma_dev_groups,
 243	.dev_release	= chan_dev_release,
 244};
 245
 246/* --- client and device registration --- */
 247
 248/* enable iteration over all operation types */
 249static dma_cap_mask_t dma_cap_mask_all;
 250
 251/**
 252 * struct dma_chan_tbl_ent - tracks channel allocations per core/operation
 253 * @chan:	associated channel for this entry
 254 */
 255struct dma_chan_tbl_ent {
 256	struct dma_chan *chan;
 257};
 258
 259/* percpu lookup table for memory-to-memory offload providers */
 260static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
 261
 262static int __init dma_channel_table_init(void)
 263{
 264	enum dma_transaction_type cap;
 265	int err = 0;
 266
 267	bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
 268
 269	/* 'interrupt', 'private', and 'slave' are channel capabilities,
 270	 * but are not associated with an operation so they do not need
 271	 * an entry in the channel_table
 272	 */
 273	clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
 274	clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
 275	clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
 276
 277	for_each_dma_cap_mask(cap, dma_cap_mask_all) {
 278		channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
 279		if (!channel_table[cap]) {
 280			err = -ENOMEM;
 281			break;
 282		}
 283	}
 284
 285	if (err) {
 286		pr_err("dmaengine dma_channel_table_init failure: %d\n", err);
 287		for_each_dma_cap_mask(cap, dma_cap_mask_all)
 288			free_percpu(channel_table[cap]);
 289	}
 290
 291	return err;
 292}
 293arch_initcall(dma_channel_table_init);
 294
 295/**
 296 * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU
 297 * @chan:	DMA channel to test
 298 * @cpu:	CPU index which the channel should be close to
 299 *
 300 * Returns true if the channel is in the same NUMA-node as the CPU.
 301 */
 302static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
 303{
 304	int node = dev_to_node(chan->device->dev);
 305	return node == NUMA_NO_NODE ||
 306		cpumask_test_cpu(cpu, cpumask_of_node(node));
 307}
 308
 309/**
 310 * min_chan - finds the channel with min count and in the same NUMA-node as the CPU
 311 * @cap:	capability to match
 312 * @cpu:	CPU index which the channel should be close to
 313 *
 314 * If some channels are close to the given CPU, the one with the lowest
 315 * reference count is returned. Otherwise, CPU is ignored and only the
 316 * reference count is taken into account.
 317 *
 318 * Must be called under dma_list_mutex.
 319 */
 320static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
 321{
 322	struct dma_device *device;
 323	struct dma_chan *chan;
 324	struct dma_chan *min = NULL;
 325	struct dma_chan *localmin = NULL;
 326
 327	list_for_each_entry(device, &dma_device_list, global_node) {
 328		if (!dma_has_cap(cap, device->cap_mask) ||
 329		    dma_has_cap(DMA_PRIVATE, device->cap_mask))
 330			continue;
 331		list_for_each_entry(chan, &device->channels, device_node) {
 332			if (!chan->client_count)
 333				continue;
 334			if (!min || chan->table_count < min->table_count)
 335				min = chan;
 336
 337			if (dma_chan_is_local(chan, cpu))
 338				if (!localmin ||
 339				    chan->table_count < localmin->table_count)
 340					localmin = chan;
 341		}
 342	}
 343
 344	chan = localmin ? localmin : min;
 345
 346	if (chan)
 347		chan->table_count++;
 348
 349	return chan;
 350}
 351
 352/**
 353 * dma_channel_rebalance - redistribute the available channels
 354 *
 355 * Optimize for CPU isolation (each CPU gets a dedicated channel for an
 356 * operation type) in the SMP case, and operation isolation (avoid
 357 * multi-tasking channels) in the non-SMP case.
 358 *
 359 * Must be called under dma_list_mutex.
 360 */
 361static void dma_channel_rebalance(void)
 362{
 363	struct dma_chan *chan;
 364	struct dma_device *device;
 365	int cpu;
 366	int cap;
 367
 368	/* undo the last distribution */
 369	for_each_dma_cap_mask(cap, dma_cap_mask_all)
 370		for_each_possible_cpu(cpu)
 371			per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
 372
 373	list_for_each_entry(device, &dma_device_list, global_node) {
 374		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
 375			continue;
 376		list_for_each_entry(chan, &device->channels, device_node)
 377			chan->table_count = 0;
 378	}
 379
 380	/* don't populate the channel_table if no clients are available */
 381	if (!dmaengine_ref_count)
 382		return;
 383
 384	/* redistribute available channels */
 385	for_each_dma_cap_mask(cap, dma_cap_mask_all)
 386		for_each_online_cpu(cpu) {
 387			chan = min_chan(cap, cpu);
 388			per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
 389		}
 390}
 391
 392static int dma_device_satisfies_mask(struct dma_device *device,
 393				     const dma_cap_mask_t *want)
 394{
 395	dma_cap_mask_t has;
 396
 397	bitmap_and(has.bits, want->bits, device->cap_mask.bits,
 398		DMA_TX_TYPE_END);
 399	return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
 400}
 401
 402static struct module *dma_chan_to_owner(struct dma_chan *chan)
 403{
 404	return chan->device->owner;
 405}
 406
 407/**
 408 * balance_ref_count - catch up the channel reference count
 409 * @chan:	channel to balance ->client_count versus dmaengine_ref_count
 410 *
 411 * Must be called under dma_list_mutex.
 412 */
 413static void balance_ref_count(struct dma_chan *chan)
 414{
 415	struct module *owner = dma_chan_to_owner(chan);
 416
 417	while (chan->client_count < dmaengine_ref_count) {
 418		__module_get(owner);
 419		chan->client_count++;
 420	}
 421}
 422
 423static void dma_device_release(struct kref *ref)
 424{
 425	struct dma_device *device = container_of(ref, struct dma_device, ref);
 426
 427	list_del_rcu(&device->global_node);
 428	dma_channel_rebalance();
 429
 430	if (device->device_release)
 431		device->device_release(device);
 432}
 433
 434static void dma_device_put(struct dma_device *device)
 435{
 436	lockdep_assert_held(&dma_list_mutex);
 437	kref_put(&device->ref, dma_device_release);
 438}
 439
 440/**
 441 * dma_chan_get - try to grab a DMA channel's parent driver module
 442 * @chan:	channel to grab
 443 *
 444 * Must be called under dma_list_mutex.
 445 */
 446static int dma_chan_get(struct dma_chan *chan)
 447{
 448	struct module *owner = dma_chan_to_owner(chan);
 449	int ret;
 450
 451	/* The channel is already in use, update client count */
 452	if (chan->client_count) {
 453		__module_get(owner);
 454		goto out;
 
 455	}
 456
 457	if (!try_module_get(owner))
 458		return -ENODEV;
 459
 460	ret = kref_get_unless_zero(&chan->device->ref);
 461	if (!ret) {
 462		ret = -ENODEV;
 463		goto module_put_out;
 464	}
 465
 466	/* allocate upon first client reference */
 467	if (chan->device->device_alloc_chan_resources) {
 468		ret = chan->device->device_alloc_chan_resources(chan);
 469		if (ret < 0)
 470			goto err_out;
 471	}
 472
 
 
 473	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
 474		balance_ref_count(chan);
 475
 476out:
 477	chan->client_count++;
 478	return 0;
 479
 480err_out:
 481	dma_device_put(chan->device);
 482module_put_out:
 483	module_put(owner);
 484	return ret;
 485}
 486
 487/**
 488 * dma_chan_put - drop a reference to a DMA channel's parent driver module
 489 * @chan:	channel to release
 490 *
 491 * Must be called under dma_list_mutex.
 492 */
 493static void dma_chan_put(struct dma_chan *chan)
 494{
 495	/* This channel is not in use, bail out */
 496	if (!chan->client_count)
 497		return;
 498
 499	chan->client_count--;
 500
 501	/* This channel is not in use anymore, free it */
 502	if (!chan->client_count && chan->device->device_free_chan_resources) {
 503		/* Make sure all operations have completed */
 504		dmaengine_synchronize(chan);
 505		chan->device->device_free_chan_resources(chan);
 506	}
 507
 508	/* If the channel is used via a DMA request router, free the mapping */
 509	if (chan->router && chan->router->route_free) {
 510		chan->router->route_free(chan->router->dev, chan->route_data);
 511		chan->router = NULL;
 512		chan->route_data = NULL;
 513	}
 514
 515	dma_device_put(chan->device);
 516	module_put(dma_chan_to_owner(chan));
 517}
 518
 519enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
 520{
 521	enum dma_status status;
 522	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
 523
 524	dma_async_issue_pending(chan);
 525	do {
 526		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 527		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
 528			dev_err(chan->device->dev, "%s: timeout!\n", __func__);
 529			return DMA_ERROR;
 530		}
 531		if (status != DMA_IN_PROGRESS)
 532			break;
 533		cpu_relax();
 534	} while (1);
 535
 536	return status;
 537}
 538EXPORT_SYMBOL(dma_sync_wait);
 539
 540/**
 541 * dma_find_channel - find a channel to carry out the operation
 542 * @tx_type:	transaction type
 543 */
 544struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
 545{
 546	return this_cpu_read(channel_table[tx_type]->chan);
 547}
 548EXPORT_SYMBOL(dma_find_channel);
 549
 550/**
 551 * dma_issue_pending_all - flush all pending operations across all channels
 552 */
 553void dma_issue_pending_all(void)
 554{
 555	struct dma_device *device;
 556	struct dma_chan *chan;
 557
 558	rcu_read_lock();
 559	list_for_each_entry_rcu(device, &dma_device_list, global_node) {
 560		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
 561			continue;
 562		list_for_each_entry(chan, &device->channels, device_node)
 563			if (chan->client_count)
 564				device->device_issue_pending(chan);
 565	}
 566	rcu_read_unlock();
 567}
 568EXPORT_SYMBOL(dma_issue_pending_all);
 569
 570int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
 571{
 572	struct dma_device *device;
 573
 574	if (!chan || !caps)
 575		return -EINVAL;
 576
 577	device = chan->device;
 578
 579	/* check if the channel supports slave transactions */
 580	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
 581	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))
 582		return -ENXIO;
 583
 584	/*
 585	 * Check whether it reports it uses the generic slave
 586	 * capabilities, if not, that means it doesn't support any
 587	 * kind of slave capabilities reporting.
 588	 */
 589	if (!device->directions)
 590		return -ENXIO;
 591
 592	caps->src_addr_widths = device->src_addr_widths;
 593	caps->dst_addr_widths = device->dst_addr_widths;
 594	caps->directions = device->directions;
 595	caps->min_burst = device->min_burst;
 596	caps->max_burst = device->max_burst;
 597	caps->max_sg_burst = device->max_sg_burst;
 598	caps->residue_granularity = device->residue_granularity;
 599	caps->descriptor_reuse = device->descriptor_reuse;
 600	caps->cmd_pause = !!device->device_pause;
 601	caps->cmd_resume = !!device->device_resume;
 602	caps->cmd_terminate = !!device->device_terminate_all;
 603
 604	/*
 605	 * DMA engine device might be configured with non-uniformly
 606	 * distributed slave capabilities per device channels. In this
 607	 * case the corresponding driver may provide the device_caps
 608	 * callback to override the generic capabilities with
 609	 * channel-specific ones.
 610	 */
 611	if (device->device_caps)
 612		device->device_caps(chan, caps);
 613
 614	return 0;
 615}
 616EXPORT_SYMBOL_GPL(dma_get_slave_caps);
 617
 618static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
 619					  struct dma_device *dev,
 620					  dma_filter_fn fn, void *fn_param)
 621{
 622	struct dma_chan *chan;
 623
 624	if (mask && !dma_device_satisfies_mask(dev, mask)) {
 625		dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
 626		return NULL;
 627	}
 628	/* devices with multiple channels need special handling as we need to
 629	 * ensure that all channels are either private or public.
 630	 */
 631	if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
 632		list_for_each_entry(chan, &dev->channels, device_node) {
 633			/* some channels are already publicly allocated */
 634			if (chan->client_count)
 635				return NULL;
 636		}
 637
 638	list_for_each_entry(chan, &dev->channels, device_node) {
 639		if (chan->client_count) {
 640			dev_dbg(dev->dev, "%s: %s busy\n",
 641				 __func__, dma_chan_name(chan));
 642			continue;
 643		}
 644		if (fn && !fn(chan, fn_param)) {
 645			dev_dbg(dev->dev, "%s: %s filter said false\n",
 646				 __func__, dma_chan_name(chan));
 647			continue;
 648		}
 649		return chan;
 650	}
 651
 652	return NULL;
 653}
 654
 655static struct dma_chan *find_candidate(struct dma_device *device,
 656				       const dma_cap_mask_t *mask,
 657				       dma_filter_fn fn, void *fn_param)
 658{
 659	struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);
 660	int err;
 661
 662	if (chan) {
 663		/* Found a suitable channel, try to grab, prep, and return it.
 664		 * We first set DMA_PRIVATE to disable balance_ref_count as this
 665		 * channel will not be published in the general-purpose
 666		 * allocator
 667		 */
 668		dma_cap_set(DMA_PRIVATE, device->cap_mask);
 669		device->privatecnt++;
 670		err = dma_chan_get(chan);
 671
 672		if (err) {
 673			if (err == -ENODEV) {
 674				dev_dbg(device->dev, "%s: %s module removed\n",
 675					__func__, dma_chan_name(chan));
 676				list_del_rcu(&device->global_node);
 677			} else
 678				dev_dbg(device->dev,
 679					"%s: failed to get %s: (%d)\n",
 680					 __func__, dma_chan_name(chan), err);
 681
 682			if (--device->privatecnt == 0)
 683				dma_cap_clear(DMA_PRIVATE, device->cap_mask);
 684
 685			chan = ERR_PTR(err);
 686		}
 687	}
 688
 689	return chan ? chan : ERR_PTR(-EPROBE_DEFER);
 690}
 691
 692/**
 693 * dma_get_slave_channel - try to get specific channel exclusively
 694 * @chan:	target channel
 695 */
 696struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
 697{
 698	int err = -EBUSY;
 699
 700	/* lock against __dma_request_channel */
 701	mutex_lock(&dma_list_mutex);
 702
 703	if (chan->client_count == 0) {
 704		struct dma_device *device = chan->device;
 
 705
 706		dma_cap_set(DMA_PRIVATE, device->cap_mask);
 707		device->privatecnt++;
 708		err = dma_chan_get(chan);
 709		if (err) {
 710			dev_dbg(chan->device->dev,
 711				"%s: failed to get %s: (%d)\n",
 712				__func__, dma_chan_name(chan), err);
 713			chan = NULL;
 714			if (--device->privatecnt == 0)
 715				dma_cap_clear(DMA_PRIVATE, device->cap_mask);
 716		}
 717	} else
 718		chan = NULL;
 719
 720	mutex_unlock(&dma_list_mutex);
 721
 722
 723	return chan;
 724}
 725EXPORT_SYMBOL_GPL(dma_get_slave_channel);
 726
 727struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
 728{
 729	dma_cap_mask_t mask;
 730	struct dma_chan *chan;
 731
 732	dma_cap_zero(mask);
 733	dma_cap_set(DMA_SLAVE, mask);
 734
 735	/* lock against __dma_request_channel */
 736	mutex_lock(&dma_list_mutex);
 737
 738	chan = find_candidate(device, &mask, NULL, NULL);
 739
 740	mutex_unlock(&dma_list_mutex);
 741
 742	return IS_ERR(chan) ? NULL : chan;
 743}
 744EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
 745
 746/**
 747 * __dma_request_channel - try to allocate an exclusive channel
 748 * @mask:	capabilities that the channel must satisfy
 749 * @fn:		optional callback to disposition available channels
 750 * @fn_param:	opaque parameter to pass to dma_filter_fn()
 751 * @np:		device node to look for DMA channels
 752 *
 753 * Returns pointer to appropriate DMA channel on success or NULL.
 754 */
 755struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
 756				       dma_filter_fn fn, void *fn_param,
 757				       struct device_node *np)
 758{
 759	struct dma_device *device, *_d;
 760	struct dma_chan *chan = NULL;
 761
 762	/* Find a channel */
 763	mutex_lock(&dma_list_mutex);
 764	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
 765		/* Finds a DMA controller with matching device node */
 766		if (np && device->dev->of_node && np != device->dev->of_node)
 767			continue;
 768
 769		chan = find_candidate(device, mask, fn, fn_param);
 770		if (!IS_ERR(chan))
 771			break;
 772
 773		chan = NULL;
 774	}
 775	mutex_unlock(&dma_list_mutex);
 776
 777	pr_debug("%s: %s (%s)\n",
 778		 __func__,
 779		 chan ? "success" : "fail",
 780		 chan ? dma_chan_name(chan) : NULL);
 781
 782	return chan;
 783}
 784EXPORT_SYMBOL_GPL(__dma_request_channel);
 785
 786static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
 787						    const char *name,
 788						    struct device *dev)
 789{
 790	int i;
 791
 792	if (!device->filter.mapcnt)
 793		return NULL;
 794
 795	for (i = 0; i < device->filter.mapcnt; i++) {
 796		const struct dma_slave_map *map = &device->filter.map[i];
 797
 798		if (!strcmp(map->devname, dev_name(dev)) &&
 799		    !strcmp(map->slave, name))
 800			return map;
 801	}
 802
 803	return NULL;
 804}
 805
 806/**
 807 * dma_request_chan - try to allocate an exclusive slave channel
 808 * @dev:	pointer to client device structure
 809 * @name:	slave channel name
 810 *
 811 * Returns pointer to appropriate DMA channel on success or an error pointer.
 812 */
 813struct dma_chan *dma_request_chan(struct device *dev, const char *name)
 814{
 815	struct dma_device *d, *_d;
 816	struct dma_chan *chan = NULL;
 817
 818	/* If device-tree is present get slave info from here */
 819	if (dev->of_node)
 820		chan = of_dma_request_slave_channel(dev->of_node, name);
 821
 822	/* If device was enumerated by ACPI get slave info from here */
 823	if (has_acpi_companion(dev) && !chan)
 824		chan = acpi_dma_request_slave_chan_by_name(dev, name);
 825
 826	if (PTR_ERR(chan) == -EPROBE_DEFER)
 827		return chan;
 828
 829	if (!IS_ERR_OR_NULL(chan))
 830		goto found;
 831
 832	/* Try to find the channel via the DMA filter map(s) */
 833	mutex_lock(&dma_list_mutex);
 834	list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {
 835		dma_cap_mask_t mask;
 836		const struct dma_slave_map *map = dma_filter_match(d, name, dev);
 837
 838		if (!map)
 839			continue;
 840
 841		dma_cap_zero(mask);
 842		dma_cap_set(DMA_SLAVE, mask);
 843
 844		chan = find_candidate(d, &mask, d->filter.fn, map->param);
 845		if (!IS_ERR(chan))
 846			break;
 847	}
 848	mutex_unlock(&dma_list_mutex);
 849
 850	if (IS_ERR_OR_NULL(chan))
 851		return chan ? chan : ERR_PTR(-EPROBE_DEFER);
 
 
 852
 853found:
 854#ifdef CONFIG_DEBUG_FS
 855	chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev),
 856					  name);
 857#endif
 858
 859	chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);
 860	if (!chan->name)
 861		return chan;
 862	chan->slave = dev;
 863
 864	if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,
 865			      DMA_SLAVE_NAME))
 866		dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);
 867	if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))
 868		dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);
 869
 870	return chan;
 871}
 872EXPORT_SYMBOL_GPL(dma_request_chan);
 873
 874/**
 875 * dma_request_slave_channel - try to allocate an exclusive slave channel
 876 * @dev:	pointer to client device structure
 877 * @name:	slave channel name
 878 *
 879 * Returns pointer to appropriate DMA channel on success or NULL.
 880 */
 881struct dma_chan *dma_request_slave_channel(struct device *dev,
 882					   const char *name)
 883{
 884	struct dma_chan *ch = dma_request_chan(dev, name);
 885	if (IS_ERR(ch))
 886		return NULL;
 887
 888	return ch;
 889}
 890EXPORT_SYMBOL_GPL(dma_request_slave_channel);
 891
 892/**
 893 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
 894 * @mask:	capabilities that the channel must satisfy
 895 *
 896 * Returns pointer to appropriate DMA channel on success or an error pointer.
 897 */
 898struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
 899{
 900	struct dma_chan *chan;
 901
 902	if (!mask)
 903		return ERR_PTR(-ENODEV);
 904
 905	chan = __dma_request_channel(mask, NULL, NULL, NULL);
 906	if (!chan) {
 907		mutex_lock(&dma_list_mutex);
 908		if (list_empty(&dma_device_list))
 909			chan = ERR_PTR(-EPROBE_DEFER);
 910		else
 911			chan = ERR_PTR(-ENODEV);
 912		mutex_unlock(&dma_list_mutex);
 913	}
 914
 915	return chan;
 916}
 917EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);
 918
 919void dma_release_channel(struct dma_chan *chan)
 920{
 921	mutex_lock(&dma_list_mutex);
 922	WARN_ONCE(chan->client_count != 1,
 923		  "chan reference count %d != 1\n", chan->client_count);
 924	dma_chan_put(chan);
 925	/* drop PRIVATE cap enabled by __dma_request_channel() */
 926	if (--chan->device->privatecnt == 0)
 927		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
 928
 929	if (chan->slave) {
 930		sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);
 931		sysfs_remove_link(&chan->slave->kobj, chan->name);
 932		kfree(chan->name);
 933		chan->name = NULL;
 934		chan->slave = NULL;
 935	}
 936
 937#ifdef CONFIG_DEBUG_FS
 938	kfree(chan->dbg_client_name);
 939	chan->dbg_client_name = NULL;
 940#endif
 941	mutex_unlock(&dma_list_mutex);
 942}
 943EXPORT_SYMBOL_GPL(dma_release_channel);
 944
 945/**
 946 * dmaengine_get - register interest in dma_channels
 947 */
 948void dmaengine_get(void)
 949{
 950	struct dma_device *device, *_d;
 951	struct dma_chan *chan;
 952	int err;
 953
 954	mutex_lock(&dma_list_mutex);
 955	dmaengine_ref_count++;
 956
 957	/* try to grab channels */
 958	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
 959		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
 960			continue;
 961		list_for_each_entry(chan, &device->channels, device_node) {
 962			err = dma_chan_get(chan);
 963			if (err == -ENODEV) {
 964				/* module removed before we could use it */
 965				list_del_rcu(&device->global_node);
 966				break;
 967			} else if (err)
 968				dev_dbg(chan->device->dev,
 969					"%s: failed to get %s: (%d)\n",
 970					__func__, dma_chan_name(chan), err);
 971		}
 972	}
 973
 974	/* if this is the first reference and there were channels
 975	 * waiting we need to rebalance to get those channels
 976	 * incorporated into the channel table
 977	 */
 978	if (dmaengine_ref_count == 1)
 979		dma_channel_rebalance();
 980	mutex_unlock(&dma_list_mutex);
 981}
 982EXPORT_SYMBOL(dmaengine_get);
 983
 984/**
 985 * dmaengine_put - let DMA drivers be removed when ref_count == 0
 986 */
 987void dmaengine_put(void)
 988{
 989	struct dma_device *device, *_d;
 990	struct dma_chan *chan;
 991
 992	mutex_lock(&dma_list_mutex);
 993	dmaengine_ref_count--;
 994	BUG_ON(dmaengine_ref_count < 0);
 995	/* drop channel references */
 996	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
 997		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
 998			continue;
 999		list_for_each_entry(chan, &device->channels, device_node)
1000			dma_chan_put(chan);
1001	}
1002	mutex_unlock(&dma_list_mutex);
1003}
1004EXPORT_SYMBOL(dmaengine_put);
1005
1006static bool device_has_all_tx_types(struct dma_device *device)
1007{
1008	/* A device that satisfies this test has channels that will never cause
1009	 * an async_tx channel switch event as all possible operation types can
1010	 * be handled.
1011	 */
1012	#ifdef CONFIG_ASYNC_TX_DMA
1013	if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
1014		return false;
1015	#endif
1016
1017	#if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
1018	if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
1019		return false;
1020	#endif
1021
1022	#if IS_ENABLED(CONFIG_ASYNC_XOR)
1023	if (!dma_has_cap(DMA_XOR, device->cap_mask))
1024		return false;
1025
1026	#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
1027	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
1028		return false;
1029	#endif
1030	#endif
1031
1032	#if IS_ENABLED(CONFIG_ASYNC_PQ)
1033	if (!dma_has_cap(DMA_PQ, device->cap_mask))
1034		return false;
1035
1036	#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
1037	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
1038		return false;
1039	#endif
1040	#endif
1041
1042	return true;
1043}
1044
1045static int get_dma_id(struct dma_device *device)
1046{
1047	int rc = ida_alloc(&dma_ida, GFP_KERNEL);
1048
1049	if (rc < 0)
1050		return rc;
1051	device->dev_id = rc;
1052	return 0;
1053}
1054
1055static int __dma_async_device_channel_register(struct dma_device *device,
1056					       struct dma_chan *chan)
 
1057{
1058	int rc = 0;
1059
1060	chan->local = alloc_percpu(typeof(*chan->local));
1061	if (!chan->local)
1062		goto err_out;
1063	chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
1064	if (!chan->dev) {
1065		free_percpu(chan->local);
1066		chan->local = NULL;
1067		goto err_out;
1068	}
1069
1070	/*
1071	 * When the chan_id is a negative value, we are dynamically adding
1072	 * the channel. Otherwise we are static enumerating.
1073	 */
1074	mutex_lock(&device->chan_mutex);
1075	chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);
1076	mutex_unlock(&device->chan_mutex);
1077	if (chan->chan_id < 0) {
1078		pr_err("%s: unable to alloc ida for chan: %d\n",
1079		       __func__, chan->chan_id);
1080		goto err_out;
 
1081	}
1082
1083	chan->dev->device.class = &dma_devclass;
1084	chan->dev->device.parent = device->dev;
1085	chan->dev->chan = chan;
1086	chan->dev->dev_id = device->dev_id;
1087	dev_set_name(&chan->dev->device, "dma%dchan%d",
1088		     device->dev_id, chan->chan_id);
 
 
1089	rc = device_register(&chan->dev->device);
1090	if (rc)
1091		goto err_out_ida;
1092	chan->client_count = 0;
1093	device->chancnt++;
1094
1095	return 0;
1096
1097 err_out_ida:
1098	mutex_lock(&device->chan_mutex);
1099	ida_free(&device->chan_ida, chan->chan_id);
1100	mutex_unlock(&device->chan_mutex);
1101 err_out:
1102	free_percpu(chan->local);
1103	kfree(chan->dev);
 
 
 
1104	return rc;
1105}
1106
1107int dma_async_device_channel_register(struct dma_device *device,
1108				      struct dma_chan *chan)
 
1109{
1110	int rc;
1111
1112	rc = __dma_async_device_channel_register(device, chan);
1113	if (rc < 0)
1114		return rc;
1115
1116	dma_channel_rebalance();
1117	return 0;
1118}
1119EXPORT_SYMBOL_GPL(dma_async_device_channel_register);
1120
1121static void __dma_async_device_channel_unregister(struct dma_device *device,
1122						  struct dma_chan *chan)
1123{
 
 
 
1124	WARN_ONCE(!device->device_release && chan->client_count,
1125		  "%s called while %d clients hold a reference\n",
1126		  __func__, chan->client_count);
1127	mutex_lock(&dma_list_mutex);
1128	list_del(&chan->device_node);
1129	device->chancnt--;
1130	chan->dev->chan = NULL;
1131	mutex_unlock(&dma_list_mutex);
1132	mutex_lock(&device->chan_mutex);
1133	ida_free(&device->chan_ida, chan->chan_id);
1134	mutex_unlock(&device->chan_mutex);
1135	device_unregister(&chan->dev->device);
1136	free_percpu(chan->local);
1137}
1138
1139void dma_async_device_channel_unregister(struct dma_device *device,
1140					 struct dma_chan *chan)
1141{
1142	__dma_async_device_channel_unregister(device, chan);
1143	dma_channel_rebalance();
1144}
1145EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
1146
1147/**
1148 * dma_async_device_register - registers DMA devices found
1149 * @device:	pointer to &struct dma_device
1150 *
1151 * After calling this routine the structure should not be freed except in the
1152 * device_release() callback which will be called after
1153 * dma_async_device_unregister() is called and no further references are taken.
1154 */
1155int dma_async_device_register(struct dma_device *device)
1156{
1157	int rc;
1158	struct dma_chan* chan;
1159
1160	if (!device)
1161		return -ENODEV;
1162
1163	/* validate device routines */
1164	if (!device->dev) {
1165		pr_err("DMAdevice must have dev\n");
1166		return -EIO;
1167	}
1168
1169	device->owner = device->dev->driver->owner;
1170
1171	if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) {
1172		dev_err(device->dev,
1173			"Device claims capability %s, but op is not defined\n",
1174			"DMA_MEMCPY");
1175		return -EIO;
1176	}
1177
1178	if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) {
1179		dev_err(device->dev,
1180			"Device claims capability %s, but op is not defined\n",
1181			"DMA_XOR");
1182		return -EIO;
1183	}
1184
1185	if (dma_has_cap(DMA_XOR_VAL, device->cap_mask) && !device->device_prep_dma_xor_val) {
1186		dev_err(device->dev,
1187			"Device claims capability %s, but op is not defined\n",
1188			"DMA_XOR_VAL");
1189		return -EIO;
1190	}
1191
1192	if (dma_has_cap(DMA_PQ, device->cap_mask) && !device->device_prep_dma_pq) {
1193		dev_err(device->dev,
1194			"Device claims capability %s, but op is not defined\n",
1195			"DMA_PQ");
1196		return -EIO;
1197	}
1198
1199	if (dma_has_cap(DMA_PQ_VAL, device->cap_mask) && !device->device_prep_dma_pq_val) {
1200		dev_err(device->dev,
1201			"Device claims capability %s, but op is not defined\n",
1202			"DMA_PQ_VAL");
1203		return -EIO;
1204	}
1205
1206	if (dma_has_cap(DMA_MEMSET, device->cap_mask) && !device->device_prep_dma_memset) {
1207		dev_err(device->dev,
1208			"Device claims capability %s, but op is not defined\n",
1209			"DMA_MEMSET");
1210		return -EIO;
1211	}
1212
1213	if (dma_has_cap(DMA_INTERRUPT, device->cap_mask) && !device->device_prep_dma_interrupt) {
1214		dev_err(device->dev,
1215			"Device claims capability %s, but op is not defined\n",
1216			"DMA_INTERRUPT");
1217		return -EIO;
1218	}
1219
1220	if (dma_has_cap(DMA_CYCLIC, device->cap_mask) && !device->device_prep_dma_cyclic) {
1221		dev_err(device->dev,
1222			"Device claims capability %s, but op is not defined\n",
1223			"DMA_CYCLIC");
1224		return -EIO;
1225	}
1226
1227	if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) {
1228		dev_err(device->dev,
1229			"Device claims capability %s, but op is not defined\n",
1230			"DMA_INTERLEAVE");
1231		return -EIO;
1232	}
1233
 
1234
1235	if (!device->device_tx_status) {
1236		dev_err(device->dev, "Device tx_status is not defined\n");
1237		return -EIO;
1238	}
1239
1240
1241	if (!device->device_issue_pending) {
1242		dev_err(device->dev, "Device issue_pending is not defined\n");
1243		return -EIO;
1244	}
1245
1246	if (!device->device_release)
1247		dev_dbg(device->dev,
1248			 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");
1249
1250	kref_init(&device->ref);
1251
1252	/* note: this only matters in the
1253	 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
1254	 */
1255	if (device_has_all_tx_types(device))
1256		dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
1257
1258	rc = get_dma_id(device);
1259	if (rc != 0)
1260		return rc;
1261
1262	mutex_init(&device->chan_mutex);
1263	ida_init(&device->chan_ida);
1264
1265	/* represent channels in sysfs. Probably want devs too */
1266	list_for_each_entry(chan, &device->channels, device_node) {
1267		rc = __dma_async_device_channel_register(device, chan);
1268		if (rc < 0)
1269			goto err_out;
1270	}
1271
1272	mutex_lock(&dma_list_mutex);
1273	/* take references on public channels */
1274	if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
1275		list_for_each_entry(chan, &device->channels, device_node) {
1276			/* if clients are already waiting for channels we need
1277			 * to take references on their behalf
1278			 */
1279			if (dma_chan_get(chan) == -ENODEV) {
1280				/* note we can only get here for the first
1281				 * channel as the remaining channels are
1282				 * guaranteed to get a reference
1283				 */
1284				rc = -ENODEV;
1285				mutex_unlock(&dma_list_mutex);
1286				goto err_out;
1287			}
1288		}
1289	list_add_tail_rcu(&device->global_node, &dma_device_list);
1290	if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
1291		device->privatecnt++;	/* Always private */
1292	dma_channel_rebalance();
1293	mutex_unlock(&dma_list_mutex);
1294
1295	dmaengine_debug_register(device);
1296
1297	return 0;
1298
1299err_out:
1300	/* if we never registered a channel just release the idr */
1301	if (!device->chancnt) {
1302		ida_free(&dma_ida, device->dev_id);
1303		return rc;
1304	}
1305
1306	list_for_each_entry(chan, &device->channels, device_node) {
1307		if (chan->local == NULL)
1308			continue;
1309		mutex_lock(&dma_list_mutex);
1310		chan->dev->chan = NULL;
1311		mutex_unlock(&dma_list_mutex);
1312		device_unregister(&chan->dev->device);
1313		free_percpu(chan->local);
1314	}
1315	return rc;
1316}
1317EXPORT_SYMBOL(dma_async_device_register);
1318
1319/**
1320 * dma_async_device_unregister - unregister a DMA device
1321 * @device:	pointer to &struct dma_device
1322 *
1323 * This routine is called by dma driver exit routines, dmaengine holds module
1324 * references to prevent it being called while channels are in use.
1325 */
1326void dma_async_device_unregister(struct dma_device *device)
1327{
1328	struct dma_chan *chan, *n;
1329
1330	dmaengine_debug_unregister(device);
1331
1332	list_for_each_entry_safe(chan, n, &device->channels, device_node)
1333		__dma_async_device_channel_unregister(device, chan);
1334
1335	mutex_lock(&dma_list_mutex);
1336	/*
1337	 * setting DMA_PRIVATE ensures the device being torn down will not
1338	 * be used in the channel_table
1339	 */
1340	dma_cap_set(DMA_PRIVATE, device->cap_mask);
1341	dma_channel_rebalance();
1342	ida_free(&dma_ida, device->dev_id);
1343	dma_device_put(device);
1344	mutex_unlock(&dma_list_mutex);
1345}
1346EXPORT_SYMBOL(dma_async_device_unregister);
1347
1348static void dmam_device_release(struct device *dev, void *res)
1349{
1350	struct dma_device *device;
1351
1352	device = *(struct dma_device **)res;
1353	dma_async_device_unregister(device);
1354}
1355
1356/**
1357 * dmaenginem_async_device_register - registers DMA devices found
1358 * @device:	pointer to &struct dma_device
1359 *
1360 * The operation is managed and will be undone on driver detach.
1361 */
1362int dmaenginem_async_device_register(struct dma_device *device)
1363{
1364	void *p;
1365	int ret;
1366
1367	p = devres_alloc(dmam_device_release, sizeof(void *), GFP_KERNEL);
1368	if (!p)
1369		return -ENOMEM;
1370
1371	ret = dma_async_device_register(device);
1372	if (!ret) {
1373		*(struct dma_device **)p = device;
1374		devres_add(device->dev, p);
1375	} else {
1376		devres_free(p);
1377	}
1378
1379	return ret;
1380}
1381EXPORT_SYMBOL(dmaenginem_async_device_register);
1382
1383struct dmaengine_unmap_pool {
1384	struct kmem_cache *cache;
1385	const char *name;
1386	mempool_t *pool;
1387	size_t size;
1388};
1389
1390#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
1391static struct dmaengine_unmap_pool unmap_pool[] = {
1392	__UNMAP_POOL(2),
1393	#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1394	__UNMAP_POOL(16),
1395	__UNMAP_POOL(128),
1396	__UNMAP_POOL(256),
1397	#endif
1398};
1399
1400static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
1401{
1402	int order = get_count_order(nr);
1403
1404	switch (order) {
1405	case 0 ... 1:
1406		return &unmap_pool[0];
1407#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1408	case 2 ... 4:
1409		return &unmap_pool[1];
1410	case 5 ... 7:
1411		return &unmap_pool[2];
1412	case 8:
1413		return &unmap_pool[3];
1414#endif
1415	default:
1416		BUG();
1417		return NULL;
1418	}
1419}
1420
1421static void dmaengine_unmap(struct kref *kref)
1422{
1423	struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
1424	struct device *dev = unmap->dev;
1425	int cnt, i;
1426
1427	cnt = unmap->to_cnt;
1428	for (i = 0; i < cnt; i++)
1429		dma_unmap_page(dev, unmap->addr[i], unmap->len,
1430			       DMA_TO_DEVICE);
1431	cnt += unmap->from_cnt;
1432	for (; i < cnt; i++)
1433		dma_unmap_page(dev, unmap->addr[i], unmap->len,
1434			       DMA_FROM_DEVICE);
1435	cnt += unmap->bidi_cnt;
1436	for (; i < cnt; i++) {
1437		if (unmap->addr[i] == 0)
1438			continue;
1439		dma_unmap_page(dev, unmap->addr[i], unmap->len,
1440			       DMA_BIDIRECTIONAL);
1441	}
1442	cnt = unmap->map_cnt;
1443	mempool_free(unmap, __get_unmap_pool(cnt)->pool);
1444}
1445
1446void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
1447{
1448	if (unmap)
1449		kref_put(&unmap->kref, dmaengine_unmap);
1450}
1451EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
1452
1453static void dmaengine_destroy_unmap_pool(void)
1454{
1455	int i;
1456
1457	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
1458		struct dmaengine_unmap_pool *p = &unmap_pool[i];
1459
1460		mempool_destroy(p->pool);
1461		p->pool = NULL;
1462		kmem_cache_destroy(p->cache);
1463		p->cache = NULL;
1464	}
1465}
1466
1467static int __init dmaengine_init_unmap_pool(void)
1468{
1469	int i;
1470
1471	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
1472		struct dmaengine_unmap_pool *p = &unmap_pool[i];
1473		size_t size;
1474
1475		size = sizeof(struct dmaengine_unmap_data) +
1476		       sizeof(dma_addr_t) * p->size;
1477
1478		p->cache = kmem_cache_create(p->name, size, 0,
1479					     SLAB_HWCACHE_ALIGN, NULL);
1480		if (!p->cache)
1481			break;
1482		p->pool = mempool_create_slab_pool(1, p->cache);
1483		if (!p->pool)
1484			break;
1485	}
1486
1487	if (i == ARRAY_SIZE(unmap_pool))
1488		return 0;
1489
1490	dmaengine_destroy_unmap_pool();
1491	return -ENOMEM;
1492}
1493
1494struct dmaengine_unmap_data *
1495dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
1496{
1497	struct dmaengine_unmap_data *unmap;
1498
1499	unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
1500	if (!unmap)
1501		return NULL;
1502
1503	memset(unmap, 0, sizeof(*unmap));
1504	kref_init(&unmap->kref);
1505	unmap->dev = dev;
1506	unmap->map_cnt = nr;
1507
1508	return unmap;
1509}
1510EXPORT_SYMBOL(dmaengine_get_unmap_data);
1511
1512void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
1513	struct dma_chan *chan)
1514{
1515	tx->chan = chan;
1516	#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
1517	spin_lock_init(&tx->lock);
1518	#endif
1519}
1520EXPORT_SYMBOL(dma_async_tx_descriptor_init);
1521
1522static inline int desc_check_and_set_metadata_mode(
1523	struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)
1524{
1525	/* Make sure that the metadata mode is not mixed */
1526	if (!desc->desc_metadata_mode) {
1527		if (dmaengine_is_metadata_mode_supported(desc->chan, mode))
1528			desc->desc_metadata_mode = mode;
1529		else
1530			return -ENOTSUPP;
1531	} else if (desc->desc_metadata_mode != mode) {
1532		return -EINVAL;
1533	}
1534
1535	return 0;
1536}
1537
1538int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
1539				   void *data, size_t len)
1540{
1541	int ret;
1542
1543	if (!desc)
1544		return -EINVAL;
1545
1546	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);
1547	if (ret)
1548		return ret;
1549
1550	if (!desc->metadata_ops || !desc->metadata_ops->attach)
1551		return -ENOTSUPP;
1552
1553	return desc->metadata_ops->attach(desc, data, len);
1554}
1555EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);
1556
1557void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
1558				      size_t *payload_len, size_t *max_len)
1559{
1560	int ret;
1561
1562	if (!desc)
1563		return ERR_PTR(-EINVAL);
1564
1565	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
1566	if (ret)
1567		return ERR_PTR(ret);
1568
1569	if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)
1570		return ERR_PTR(-ENOTSUPP);
1571
1572	return desc->metadata_ops->get_ptr(desc, payload_len, max_len);
1573}
1574EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);
1575
1576int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
1577				    size_t payload_len)
1578{
1579	int ret;
1580
1581	if (!desc)
1582		return -EINVAL;
1583
1584	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
1585	if (ret)
1586		return ret;
1587
1588	if (!desc->metadata_ops || !desc->metadata_ops->set_len)
1589		return -ENOTSUPP;
1590
1591	return desc->metadata_ops->set_len(desc, payload_len);
1592}
1593EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
1594
1595/**
1596 * dma_wait_for_async_tx - spin wait for a transaction to complete
1597 * @tx:		in-flight transaction to wait on
1598 */
1599enum dma_status
1600dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
1601{
1602	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
1603
1604	if (!tx)
1605		return DMA_COMPLETE;
1606
1607	while (tx->cookie == -EBUSY) {
1608		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
1609			dev_err(tx->chan->device->dev,
1610				"%s timeout waiting for descriptor submission\n",
1611				__func__);
1612			return DMA_ERROR;
1613		}
1614		cpu_relax();
1615	}
1616	return dma_sync_wait(tx->chan, tx->cookie);
1617}
1618EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
1619
1620/**
1621 * dma_run_dependencies - process dependent operations on the target channel
1622 * @tx:		transaction with dependencies
1623 *
1624 * Helper routine for DMA drivers to process (start) dependent operations
1625 * on their target channel.
1626 */
1627void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
1628{
1629	struct dma_async_tx_descriptor *dep = txd_next(tx);
1630	struct dma_async_tx_descriptor *dep_next;
1631	struct dma_chan *chan;
1632
1633	if (!dep)
1634		return;
1635
1636	/* we'll submit tx->next now, so clear the link */
1637	txd_clear_next(tx);
1638	chan = dep->chan;
1639
1640	/* keep submitting up until a channel switch is detected
1641	 * in that case we will be called again as a result of
1642	 * processing the interrupt from async_tx_channel_switch
1643	 */
1644	for (; dep; dep = dep_next) {
1645		txd_lock(dep);
1646		txd_clear_parent(dep);
1647		dep_next = txd_next(dep);
1648		if (dep_next && dep_next->chan == chan)
1649			txd_clear_next(dep); /* ->next will be submitted */
1650		else
1651			dep_next = NULL; /* submit current dep and terminate */
1652		txd_unlock(dep);
1653
1654		dep->tx_submit(dep);
1655	}
1656
1657	chan->device->device_issue_pending(chan);
1658}
1659EXPORT_SYMBOL_GPL(dma_run_dependencies);
1660
1661static int __init dma_bus_init(void)
1662{
1663	int err = dmaengine_init_unmap_pool();
1664
1665	if (err)
1666		return err;
1667
1668	err = class_register(&dma_devclass);
1669	if (!err)
1670		dmaengine_debugfs_init();
1671
1672	return err;
1673}
1674arch_initcall(dma_bus_init);