Linux Audio

Check our new training course

Loading...
v6.8
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * DMA Engine test module
   4 *
   5 * Copyright (C) 2007 Atmel Corporation
   6 * Copyright (C) 2013 Intel Corporation
 
 
 
 
   7 */
   8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10#include <linux/err.h>
  11#include <linux/delay.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/dmaengine.h>
  14#include <linux/freezer.h>
  15#include <linux/init.h>
  16#include <linux/kthread.h>
  17#include <linux/sched/task.h>
  18#include <linux/module.h>
  19#include <linux/moduleparam.h>
  20#include <linux/random.h>
  21#include <linux/slab.h>
  22#include <linux/wait.h>
  23
  24static bool nobounce;
  25module_param(nobounce, bool, 0644);
  26MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)");
  27
  28static unsigned int test_buf_size = 16384;
  29module_param(test_buf_size, uint, 0644);
  30MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  31
 
 
 
 
 
  32static char test_device[32];
  33module_param_string(device, test_device, sizeof(test_device), 0644);
 
  34MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  35
  36static unsigned int threads_per_chan = 1;
  37module_param(threads_per_chan, uint, 0644);
  38MODULE_PARM_DESC(threads_per_chan,
  39		"Number of threads to start per channel (default: 1)");
  40
  41static unsigned int max_channels;
  42module_param(max_channels, uint, 0644);
  43MODULE_PARM_DESC(max_channels,
  44		"Maximum number of channels to use (default: all)");
  45
  46static unsigned int iterations;
  47module_param(iterations, uint, 0644);
  48MODULE_PARM_DESC(iterations,
  49		"Iterations before stopping test (default: infinite)");
  50
 
 
 
 
 
  51static unsigned int dmatest;
  52module_param(dmatest, uint, 0644);
  53MODULE_PARM_DESC(dmatest,
  54		"dmatest 0-memcpy 1-memset (default: 0)");
  55
  56static unsigned int xor_sources = 3;
  57module_param(xor_sources, uint, 0644);
  58MODULE_PARM_DESC(xor_sources,
  59		"Number of xor source buffers (default: 3)");
  60
  61static unsigned int pq_sources = 3;
  62module_param(pq_sources, uint, 0644);
  63MODULE_PARM_DESC(pq_sources,
  64		"Number of p+q source buffers (default: 3)");
  65
  66static int timeout = 3000;
  67module_param(timeout, int, 0644);
  68MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  69		 "Pass -1 for infinite timeout");
  70
  71static bool noverify;
  72module_param(noverify, bool, 0644);
  73MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
  74
  75static bool norandom;
  76module_param(norandom, bool, 0644);
  77MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
  78
  79static bool verbose;
  80module_param(verbose, bool, 0644);
  81MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
  82
  83static int alignment = -1;
  84module_param(alignment, int, 0644);
  85MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
  86
  87static unsigned int transfer_size;
  88module_param(transfer_size, uint, 0644);
  89MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
  90
  91static bool polled;
  92module_param(polled, bool, 0644);
  93MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
  94
  95/**
  96 * struct dmatest_params - test parameters.
  97 * @nobounce:		prevent using swiotlb buffer
  98 * @buf_size:		size of the memcpy test buffer
  99 * @channel:		bus ID of the channel to test
 100 * @device:		bus ID of the DMA Engine to test
 101 * @threads_per_chan:	number of threads to start per channel
 102 * @max_channels:	maximum number of channels to use
 103 * @iterations:		iterations before stopping test
 104 * @xor_sources:	number of xor source buffers
 105 * @pq_sources:		number of p+q source buffers
 106 * @timeout:		transfer timeout in msec, -1 for infinite timeout
 107 * @noverify:		disable data verification
 108 * @norandom:		disable random offset setup
 109 * @alignment:		custom data address alignment taken as 2^alignment
 110 * @transfer_size:	custom transfer size in bytes
 111 * @polled:		use polling for completion instead of interrupts
 112 */
 113struct dmatest_params {
 114	bool		nobounce;
 115	unsigned int	buf_size;
 116	char		channel[20];
 117	char		device[32];
 118	unsigned int	threads_per_chan;
 119	unsigned int	max_channels;
 120	unsigned int	iterations;
 121	unsigned int	xor_sources;
 122	unsigned int	pq_sources;
 123	int		timeout;
 124	bool		noverify;
 125	bool		norandom;
 126	int		alignment;
 127	unsigned int	transfer_size;
 128	bool		polled;
 129};
 130
 131/**
 132 * struct dmatest_info - test information.
 133 * @params:		test parameters
 134 * @channels:		channels under test
 135 * @nr_channels:	number of channels under test
 136 * @lock:		access protection to the fields of this structure
 137 * @did_init:		module has been initialized completely
 138 * @last_error:		test has faced configuration issues
 139 */
 140static struct dmatest_info {
 141	/* Test parameters */
 142	struct dmatest_params	params;
 143
 144	/* Internal state */
 145	struct list_head	channels;
 146	unsigned int		nr_channels;
 147	int			last_error;
 148	struct mutex		lock;
 149	bool			did_init;
 150} test_info = {
 151	.channels = LIST_HEAD_INIT(test_info.channels),
 152	.lock = __MUTEX_INITIALIZER(test_info.lock),
 153};
 154
 155static int dmatest_run_set(const char *val, const struct kernel_param *kp);
 156static int dmatest_run_get(char *val, const struct kernel_param *kp);
 157static const struct kernel_param_ops run_ops = {
 158	.set = dmatest_run_set,
 159	.get = dmatest_run_get,
 160};
 161static bool dmatest_run;
 162module_param_cb(run, &run_ops, &dmatest_run, 0644);
 163MODULE_PARM_DESC(run, "Run the test (default: false)");
 164
 165static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
 166static int dmatest_chan_get(char *val, const struct kernel_param *kp);
 167static const struct kernel_param_ops multi_chan_ops = {
 168	.set = dmatest_chan_set,
 169	.get = dmatest_chan_get,
 170};
 171
 172static char test_channel[20];
 173static struct kparam_string newchan_kps = {
 174	.string = test_channel,
 175	.maxlen = 20,
 176};
 177module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
 178MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
 179
 180static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
 181static const struct kernel_param_ops test_list_ops = {
 182	.get = dmatest_test_list_get,
 183};
 184module_param_cb(test_list, &test_list_ops, NULL, 0444);
 185MODULE_PARM_DESC(test_list, "Print current test list");
 186
 187/* Maximum amount of mismatched bytes in buffer to print */
 188#define MAX_ERROR_COUNT		32
 189
 190/*
 191 * Initialization patterns. All bytes in the source buffer has bit 7
 192 * set, all bytes in the destination buffer has bit 7 cleared.
 193 *
 194 * Bit 6 is set for all bytes which are to be copied by the DMA
 195 * engine. Bit 5 is set for all bytes which are to be overwritten by
 196 * the DMA engine.
 197 *
 198 * The remaining bits are the inverse of a counter which increments by
 199 * one for each byte address.
 200 */
 201#define PATTERN_SRC		0x80
 202#define PATTERN_DST		0x00
 203#define PATTERN_COPY		0x40
 204#define PATTERN_OVERWRITE	0x20
 205#define PATTERN_COUNT_MASK	0x1f
 206#define PATTERN_MEMSET_IDX	0x01
 207
 208/* Fixed point arithmetic ops */
 209#define FIXPT_SHIFT		8
 210#define FIXPNT_MASK		0xFF
 211#define FIXPT_TO_INT(a)	((a) >> FIXPT_SHIFT)
 212#define INT_TO_FIXPT(a)	((a) << FIXPT_SHIFT)
 213#define FIXPT_GET_FRAC(a)	((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
 214
 215/* poor man's completion - we want to use wait_event_freezable() on it */
 216struct dmatest_done {
 217	bool			done;
 218	wait_queue_head_t	*wait;
 219};
 220
 221struct dmatest_data {
 222	u8		**raw;
 223	u8		**aligned;
 224	gfp_t		gfp_flags;
 225	unsigned int	cnt;
 226	unsigned int	off;
 227};
 228
 229struct dmatest_thread {
 230	struct list_head	node;
 231	struct dmatest_info	*info;
 232	struct task_struct	*task;
 233	struct dma_chan		*chan;
 234	struct dmatest_data	src;
 235	struct dmatest_data	dst;
 
 
 236	enum dma_transaction_type type;
 237	wait_queue_head_t done_wait;
 238	struct dmatest_done test_done;
 239	bool			done;
 240	bool			pending;
 241};
 242
 243struct dmatest_chan {
 244	struct list_head	node;
 245	struct dma_chan		*chan;
 246	struct list_head	threads;
 247};
 248
 249static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
 250static bool wait;
 251
 252static bool is_threaded_test_run(struct dmatest_info *info)
 253{
 254	struct dmatest_chan *dtc;
 255
 256	list_for_each_entry(dtc, &info->channels, node) {
 257		struct dmatest_thread *thread;
 258
 259		list_for_each_entry(thread, &dtc->threads, node) {
 260			if (!thread->done && !thread->pending)
 261				return true;
 262		}
 263	}
 264
 265	return false;
 266}
 267
 268static bool is_threaded_test_pending(struct dmatest_info *info)
 269{
 270	struct dmatest_chan *dtc;
 271
 272	list_for_each_entry(dtc, &info->channels, node) {
 273		struct dmatest_thread *thread;
 274
 275		list_for_each_entry(thread, &dtc->threads, node) {
 276			if (thread->pending)
 277				return true;
 278		}
 279	}
 280
 281	return false;
 282}
 283
 284static int dmatest_wait_get(char *val, const struct kernel_param *kp)
 285{
 286	struct dmatest_info *info = &test_info;
 287	struct dmatest_params *params = &info->params;
 288
 289	if (params->iterations)
 290		wait_event(thread_wait, !is_threaded_test_run(info));
 291	wait = true;
 292	return param_get_bool(val, kp);
 293}
 294
 295static const struct kernel_param_ops wait_ops = {
 296	.get = dmatest_wait_get,
 297	.set = param_set_bool,
 298};
 299module_param_cb(wait, &wait_ops, &wait, 0444);
 300MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
 301
 302static bool dmatest_match_channel(struct dmatest_params *params,
 303		struct dma_chan *chan)
 304{
 305	if (params->channel[0] == '\0')
 306		return true;
 307	return strcmp(dma_chan_name(chan), params->channel) == 0;
 308}
 309
 310static bool dmatest_match_device(struct dmatest_params *params,
 311		struct dma_device *device)
 312{
 313	if (params->device[0] == '\0')
 314		return true;
 315	return strcmp(dev_name(device->dev), params->device) == 0;
 316}
 317
 318static unsigned long dmatest_random(void)
 319{
 320	unsigned long buf;
 321
 322	get_random_bytes(&buf, sizeof(buf));
 323	return buf;
 324}
 325
 326static inline u8 gen_inv_idx(u8 index, bool is_memset)
 327{
 328	u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
 329
 330	return ~val & PATTERN_COUNT_MASK;
 331}
 332
 333static inline u8 gen_src_value(u8 index, bool is_memset)
 334{
 335	return PATTERN_SRC | gen_inv_idx(index, is_memset);
 336}
 337
 338static inline u8 gen_dst_value(u8 index, bool is_memset)
 339{
 340	return PATTERN_DST | gen_inv_idx(index, is_memset);
 341}
 342
 343static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 344		unsigned int buf_size, bool is_memset)
 345{
 346	unsigned int i;
 347	u8 *buf;
 348
 349	for (; (buf = *bufs); bufs++) {
 350		for (i = 0; i < start; i++)
 351			buf[i] = gen_src_value(i, is_memset);
 352		for ( ; i < start + len; i++)
 353			buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
 
 354		for ( ; i < buf_size; i++)
 355			buf[i] = gen_src_value(i, is_memset);
 356		buf++;
 357	}
 358}
 359
 360static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 361		unsigned int buf_size, bool is_memset)
 362{
 363	unsigned int i;
 364	u8 *buf;
 365
 366	for (; (buf = *bufs); bufs++) {
 367		for (i = 0; i < start; i++)
 368			buf[i] = gen_dst_value(i, is_memset);
 369		for ( ; i < start + len; i++)
 370			buf[i] = gen_dst_value(i, is_memset) |
 371						PATTERN_OVERWRITE;
 372		for ( ; i < buf_size; i++)
 373			buf[i] = gen_dst_value(i, is_memset);
 374	}
 375}
 376
 377static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 378		unsigned int counter, bool is_srcbuf, bool is_memset)
 379{
 380	u8		diff = actual ^ pattern;
 381	u8		expected = pattern | gen_inv_idx(counter, is_memset);
 382	const char	*thread_name = current->comm;
 383
 384	if (is_srcbuf)
 385		pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 386			thread_name, index, expected, actual);
 387	else if ((pattern & PATTERN_COPY)
 388			&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 389		pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 390			thread_name, index, expected, actual);
 391	else if (diff & PATTERN_SRC)
 392		pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 393			thread_name, index, expected, actual);
 394	else
 395		pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 396			thread_name, index, expected, actual);
 397}
 398
 399static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 400		unsigned int end, unsigned int counter, u8 pattern,
 401		bool is_srcbuf, bool is_memset)
 402{
 403	unsigned int i;
 404	unsigned int error_count = 0;
 405	u8 actual;
 406	u8 expected;
 407	u8 *buf;
 408	unsigned int counter_orig = counter;
 409
 410	for (; (buf = *bufs); bufs++) {
 411		counter = counter_orig;
 412		for (i = start; i < end; i++) {
 413			actual = buf[i];
 414			expected = pattern | gen_inv_idx(counter, is_memset);
 415			if (actual != expected) {
 416				if (error_count < MAX_ERROR_COUNT)
 417					dmatest_mismatch(actual, pattern, i,
 418							 counter, is_srcbuf,
 419							 is_memset);
 420				error_count++;
 421			}
 422			counter++;
 423		}
 424	}
 425
 426	if (error_count > MAX_ERROR_COUNT)
 427		pr_warn("%s: %u errors suppressed\n",
 428			current->comm, error_count - MAX_ERROR_COUNT);
 429
 430	return error_count;
 431}
 432
 
 
 
 
 
 433
 434static void dmatest_callback(void *arg)
 435{
 436	struct dmatest_done *done = arg;
 437	struct dmatest_thread *thread =
 438		container_of(done, struct dmatest_thread, test_done);
 439	if (!thread->done) {
 440		done->done = true;
 441		wake_up_all(done->wait);
 442	} else {
 443		/*
 444		 * If thread->done, it means that this callback occurred
 445		 * after the parent thread has cleaned up. This can
 446		 * happen in the case that driver doesn't implement
 447		 * the terminate_all() functionality and a dma operation
 448		 * did not occur within the timeout period
 449		 */
 450		WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
 451	}
 452}
 453
 454static unsigned int min_odd(unsigned int x, unsigned int y)
 455{
 456	unsigned int val = min(x, y);
 457
 458	return val % 2 ? val : val - 1;
 459}
 460
 461static void result(const char *err, unsigned int n, unsigned int src_off,
 462		   unsigned int dst_off, unsigned int len, unsigned long data)
 463{
 464	if (IS_ERR_VALUE(data)) {
 465		pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",
 466			current->comm, n, err, src_off, dst_off, len, data);
 467	} else {
 468		pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 469			current->comm, n, err, src_off, dst_off, len, data);
 470	}
 471}
 472
 473static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
 474		       unsigned int dst_off, unsigned int len,
 475		       unsigned long data)
 476{
 477	pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 478		 current->comm, n, err, src_off, dst_off, len, data);
 479}
 480
 481#define verbose_result(err, n, src_off, dst_off, len, data) ({	\
 482	if (verbose)						\
 483		result(err, n, src_off, dst_off, len, data);	\
 484	else							\
 485		dbg_result(err, n, src_off, dst_off, len, data);\
 486})
 487
 488static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
 489{
 490	unsigned long long per_sec = 1000000;
 491
 492	if (runtime <= 0)
 493		return 0;
 494
 495	/* drop precision until runtime is 32-bits */
 496	while (runtime > UINT_MAX) {
 497		runtime >>= 1;
 498		per_sec <<= 1;
 499	}
 500
 501	per_sec *= val;
 502	per_sec = INT_TO_FIXPT(per_sec);
 503	do_div(per_sec, runtime);
 504
 505	return per_sec;
 506}
 507
 508static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
 509{
 510	return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
 511}
 512
 513static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
 514{
 515	unsigned int i;
 516
 517	for (i = 0; i < cnt; i++)
 518		kfree(d->raw[i]);
 519
 520	kfree(d->aligned);
 521	kfree(d->raw);
 522}
 523
 524static void dmatest_free_test_data(struct dmatest_data *d)
 525{
 526	__dmatest_free_test_data(d, d->cnt);
 527}
 528
 529static int dmatest_alloc_test_data(struct dmatest_data *d,
 530		unsigned int buf_size, u8 align)
 531{
 532	unsigned int i = 0;
 533
 534	d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
 535	if (!d->raw)
 536		return -ENOMEM;
 537
 538	d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
 539	if (!d->aligned)
 540		goto err;
 541
 542	for (i = 0; i < d->cnt; i++) {
 543		d->raw[i] = kmalloc(buf_size + align, d->gfp_flags);
 544		if (!d->raw[i])
 545			goto err;
 546
 547		/* align to alignment restriction */
 548		if (align)
 549			d->aligned[i] = PTR_ALIGN(d->raw[i], align);
 550		else
 551			d->aligned[i] = d->raw[i];
 552	}
 553
 554	return 0;
 555err:
 556	__dmatest_free_test_data(d, i);
 557	return -ENOMEM;
 558}
 559
 560/*
 561 * This function repeatedly tests DMA transfers of various lengths and
 562 * offsets for a given operation type until it is told to exit by
 563 * kthread_stop(). There may be multiple threads running this function
 564 * in parallel for a single channel, and there may be multiple channels
 565 * being tested in parallel.
 566 *
 567 * Before each test, the source and destination buffer is initialized
 568 * with a known pattern. This pattern is different depending on
 569 * whether it's in an area which is supposed to be copied or
 570 * overwritten, and different in the source and destination buffers.
 571 * So if the DMA engine doesn't copy exactly what we tell it to copy,
 572 * we'll notice.
 573 */
 574static int dmatest_func(void *data)
 575{
 
 576	struct dmatest_thread	*thread = data;
 577	struct dmatest_done	*done = &thread->test_done;
 578	struct dmatest_info	*info;
 579	struct dmatest_params	*params;
 580	struct dma_chan		*chan;
 581	struct dma_device	*dev;
 582	struct device		*dma_dev;
 583	unsigned int		error_count;
 584	unsigned int		failed_tests = 0;
 585	unsigned int		total_tests = 0;
 586	dma_cookie_t		cookie;
 587	enum dma_status		status;
 588	enum dma_ctrl_flags	flags;
 589	u8			*pq_coefs = NULL;
 590	int			ret;
 591	unsigned int		buf_size;
 592	struct dmatest_data	*src;
 593	struct dmatest_data	*dst;
 594	int			i;
 595	ktime_t			ktime, start, diff;
 596	ktime_t			filltime = 0;
 597	ktime_t			comparetime = 0;
 598	s64			runtime = 0;
 599	unsigned long long	total_len = 0;
 600	unsigned long long	iops = 0;
 601	u8			align = 0;
 602	bool			is_memset = false;
 603	dma_addr_t		*srcs;
 604	dma_addr_t		*dma_pq;
 605
 606	set_freezable();
 607
 608	ret = -ENOMEM;
 609
 610	smp_rmb();
 611	thread->pending = false;
 612	info = thread->info;
 613	params = &info->params;
 614	chan = thread->chan;
 615	dev = chan->device;
 616	dma_dev = dmaengine_get_dma_device(chan);
 617
 618	src = &thread->src;
 619	dst = &thread->dst;
 620	if (thread->type == DMA_MEMCPY) {
 621		align = params->alignment < 0 ? dev->copy_align :
 622						params->alignment;
 623		src->cnt = dst->cnt = 1;
 624	} else if (thread->type == DMA_MEMSET) {
 625		align = params->alignment < 0 ? dev->fill_align :
 626						params->alignment;
 627		src->cnt = dst->cnt = 1;
 628		is_memset = true;
 629	} else if (thread->type == DMA_XOR) {
 630		/* force odd to ensure dst = src */
 631		src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 632		dst->cnt = 1;
 633		align = params->alignment < 0 ? dev->xor_align :
 634						params->alignment;
 635	} else if (thread->type == DMA_PQ) {
 636		/* force odd to ensure dst = src */
 637		src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 638		dst->cnt = 2;
 639		align = params->alignment < 0 ? dev->pq_align :
 640						params->alignment;
 641
 642		pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
 643		if (!pq_coefs)
 644			goto err_thread_type;
 645
 646		for (i = 0; i < src->cnt; i++)
 647			pq_coefs[i] = 1;
 648	} else
 649		goto err_thread_type;
 650
 651	/* Check if buffer count fits into map count variable (u8) */
 652	if ((src->cnt + dst->cnt) >= 255) {
 653		pr_err("too many buffers (%d of 255 supported)\n",
 654		       src->cnt + dst->cnt);
 655		goto err_free_coefs;
 656	}
 
 
 
 
 
 
 
 657
 658	buf_size = params->buf_size;
 659	if (1 << align > buf_size) {
 660		pr_err("%u-byte buffer too small for %d-byte alignment\n",
 661		       buf_size, 1 << align);
 662		goto err_free_coefs;
 663	}
 
 664
 665	src->gfp_flags = GFP_KERNEL;
 666	dst->gfp_flags = GFP_KERNEL;
 667	if (params->nobounce) {
 668		src->gfp_flags = GFP_DMA;
 669		dst->gfp_flags = GFP_DMA;
 670	}
 671
 672	if (dmatest_alloc_test_data(src, buf_size, align) < 0)
 673		goto err_free_coefs;
 
 674
 675	if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
 676		goto err_src;
 
 
 
 677
 678	set_user_nice(current, 10);
 
 
 
 
 
 
 679
 680	srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL);
 681	if (!srcs)
 682		goto err_dst;
 683
 684	dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL);
 685	if (!dma_pq)
 686		goto err_srcs_array;
 687
 688	/*
 689	 * src and dst buffers are freed by ourselves below
 690	 */
 691	if (params->polled)
 692		flags = DMA_CTRL_ACK;
 693	else
 694		flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 695
 696	ktime = ktime_get();
 697	while (!(kthread_should_stop() ||
 698	       (params->iterations && total_tests >= params->iterations))) {
 699		struct dma_async_tx_descriptor *tx = NULL;
 700		struct dmaengine_unmap_data *um;
 
 701		dma_addr_t *dsts;
 702		unsigned int len;
 
 
 703
 704		total_tests++;
 705
 706		if (params->transfer_size) {
 707			if (params->transfer_size >= buf_size) {
 708				pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
 709				       params->transfer_size, buf_size);
 710				break;
 711			}
 712			len = params->transfer_size;
 713		} else if (params->norandom) {
 714			len = buf_size;
 715		} else {
 716			len = dmatest_random() % buf_size + 1;
 717		}
 718
 719		/* Do not alter transfer size explicitly defined by user */
 720		if (!params->transfer_size) {
 721			len = (len >> align) << align;
 722			if (!len)
 723				len = 1 << align;
 724		}
 
 
 
 725		total_len += len;
 726
 727		if (params->norandom) {
 728			src->off = 0;
 729			dst->off = 0;
 730		} else {
 731			src->off = dmatest_random() % (buf_size - len + 1);
 732			dst->off = dmatest_random() % (buf_size - len + 1);
 
 733
 734			src->off = (src->off >> align) << align;
 735			dst->off = (dst->off >> align) << align;
 736		}
 737
 738		if (!params->noverify) {
 739			start = ktime_get();
 740			dmatest_init_srcs(src->aligned, src->off, len,
 741					  buf_size, is_memset);
 742			dmatest_init_dsts(dst->aligned, dst->off, len,
 743					  buf_size, is_memset);
 744
 745			diff = ktime_sub(ktime_get(), start);
 746			filltime = ktime_add(filltime, diff);
 747		}
 748
 749		um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
 750					      GFP_KERNEL);
 751		if (!um) {
 752			failed_tests++;
 753			result("unmap data NULL", total_tests,
 754			       src->off, dst->off, len, ret);
 755			continue;
 756		}
 757
 758		um->len = buf_size;
 759		for (i = 0; i < src->cnt; i++) {
 760			void *buf = src->aligned[i];
 761			struct page *pg = virt_to_page(buf);
 762			unsigned long pg_off = offset_in_page(buf);
 763
 764			um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
 765						   um->len, DMA_TO_DEVICE);
 766			srcs[i] = um->addr[i] + src->off;
 767			ret = dma_mapping_error(dma_dev, um->addr[i]);
 768			if (ret) {
 
 769				result("src mapping error", total_tests,
 770				       src->off, dst->off, len, ret);
 771				goto error_unmap_continue;
 
 772			}
 773			um->to_cnt++;
 774		}
 775		/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 776		dsts = &um->addr[src->cnt];
 777		for (i = 0; i < dst->cnt; i++) {
 778			void *buf = dst->aligned[i];
 779			struct page *pg = virt_to_page(buf);
 780			unsigned long pg_off = offset_in_page(buf);
 781
 782			dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
 783					       DMA_BIDIRECTIONAL);
 784			ret = dma_mapping_error(dma_dev, dsts[i]);
 785			if (ret) {
 
 786				result("dst mapping error", total_tests,
 787				       src->off, dst->off, len, ret);
 788				goto error_unmap_continue;
 
 789			}
 790			um->bidi_cnt++;
 791		}
 792
 
 
 
 
 
 
 
 
 
 793		if (thread->type == DMA_MEMCPY)
 794			tx = dev->device_prep_dma_memcpy(chan,
 795							 dsts[0] + dst->off,
 796							 srcs[0], len, flags);
 797		else if (thread->type == DMA_MEMSET)
 798			tx = dev->device_prep_dma_memset(chan,
 799						dsts[0] + dst->off,
 800						*(src->aligned[0] + src->off),
 801						len, flags);
 802		else if (thread->type == DMA_XOR)
 803			tx = dev->device_prep_dma_xor(chan,
 804						      dsts[0] + dst->off,
 805						      srcs, src->cnt,
 806						      len, flags);
 807		else if (thread->type == DMA_PQ) {
 808			for (i = 0; i < dst->cnt; i++)
 809				dma_pq[i] = dsts[i] + dst->off;
 
 
 810			tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
 811						     src->cnt, pq_coefs,
 812						     len, flags);
 813		}
 814
 815		if (!tx) {
 816			result("prep error", total_tests, src->off,
 817			       dst->off, len, ret);
 
 818			msleep(100);
 819			goto error_unmap_continue;
 
 820		}
 821
 822		done->done = false;
 823		if (!params->polled) {
 824			tx->callback = dmatest_callback;
 825			tx->callback_param = done;
 826		}
 827		cookie = tx->tx_submit(tx);
 828
 829		if (dma_submit_error(cookie)) {
 830			result("submit error", total_tests, src->off,
 831			       dst->off, len, ret);
 
 832			msleep(100);
 833			goto error_unmap_continue;
 
 834		}
 
 835
 836		if (params->polled) {
 837			status = dma_sync_wait(chan, cookie);
 838			dmaengine_terminate_sync(chan);
 839			if (status == DMA_COMPLETE)
 840				done->done = true;
 841		} else {
 842			dma_async_issue_pending(chan);
 843
 844			wait_event_freezable_timeout(thread->done_wait,
 845					done->done,
 846					msecs_to_jiffies(params->timeout));
 847
 848			status = dma_async_is_tx_complete(chan, cookie, NULL,
 849							  NULL);
 850		}
 851
 852		if (!done->done) {
 853			result("test timed out", total_tests, src->off, dst->off,
 
 
 
 
 
 
 
 
 
 854			       len, 0);
 855			goto error_unmap_continue;
 856		} else if (status != DMA_COMPLETE &&
 857			   !(dma_has_cap(DMA_COMPLETION_NO_ORDER,
 858					 dev->cap_mask) &&
 859			     status == DMA_OUT_OF_ORDER)) {
 860			result(status == DMA_ERROR ?
 861			       "completion error status" :
 862			       "completion busy status", total_tests, src->off,
 863			       dst->off, len, ret);
 864			goto error_unmap_continue;
 
 865		}
 866
 867		dmaengine_unmap_put(um);
 868
 869		if (params->noverify) {
 870			verbose_result("test passed", total_tests, src->off,
 871				       dst->off, len, 0);
 872			continue;
 873		}
 874
 875		start = ktime_get();
 876		pr_debug("%s: verifying source buffer...\n", current->comm);
 877		error_count = dmatest_verify(src->aligned, 0, src->off,
 878				0, PATTERN_SRC, true, is_memset);
 879		error_count += dmatest_verify(src->aligned, src->off,
 880				src->off + len, src->off,
 881				PATTERN_SRC | PATTERN_COPY, true, is_memset);
 882		error_count += dmatest_verify(src->aligned, src->off + len,
 883				buf_size, src->off + len,
 884				PATTERN_SRC, true, is_memset);
 885
 886		pr_debug("%s: verifying dest buffer...\n", current->comm);
 887		error_count += dmatest_verify(dst->aligned, 0, dst->off,
 888				0, PATTERN_DST, false, is_memset);
 889
 890		error_count += dmatest_verify(dst->aligned, dst->off,
 891				dst->off + len, src->off,
 892				PATTERN_SRC | PATTERN_COPY, false, is_memset);
 893
 894		error_count += dmatest_verify(dst->aligned, dst->off + len,
 895				buf_size, dst->off + len,
 896				PATTERN_DST, false, is_memset);
 897
 898		diff = ktime_sub(ktime_get(), start);
 899		comparetime = ktime_add(comparetime, diff);
 900
 901		if (error_count) {
 902			result("data error", total_tests, src->off, dst->off,
 903			       len, error_count);
 904			failed_tests++;
 905		} else {
 906			verbose_result("test passed", total_tests, src->off,
 907				       dst->off, len, 0);
 908		}
 909
 910		continue;
 911
 912error_unmap_continue:
 913		dmaengine_unmap_put(um);
 914		failed_tests++;
 915	}
 916	ktime = ktime_sub(ktime_get(), ktime);
 917	ktime = ktime_sub(ktime, comparetime);
 918	ktime = ktime_sub(ktime, filltime);
 919	runtime = ktime_to_us(ktime);
 920
 921	ret = 0;
 922	kfree(dma_pq);
 923err_srcs_array:
 924	kfree(srcs);
 925err_dst:
 926	dmatest_free_test_data(dst);
 927err_src:
 928	dmatest_free_test_data(src);
 929err_free_coefs:
 
 
 
 
 
 
 930	kfree(pq_coefs);
 931err_thread_type:
 932	iops = dmatest_persec(runtime, total_tests);
 933	pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
 934		current->comm, total_tests, failed_tests,
 935		FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
 936		dmatest_KBs(runtime, total_len), ret);
 937
 938	/* terminate all transfers on specified channels */
 939	if (ret || failed_tests)
 940		dmaengine_terminate_sync(chan);
 941
 942	thread->done = true;
 943	wake_up(&thread_wait);
 944
 945	return ret;
 946}
 947
 948static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 949{
 950	struct dmatest_thread	*thread;
 951	struct dmatest_thread	*_thread;
 952	int			ret;
 953
 954	list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 955		ret = kthread_stop(thread->task);
 956		pr_debug("thread %s exited with status %d\n",
 957			 thread->task->comm, ret);
 958		list_del(&thread->node);
 959		put_task_struct(thread->task);
 960		kfree(thread);
 961	}
 962
 963	/* terminate all transfers on specified channels */
 964	dmaengine_terminate_sync(dtc->chan);
 965
 966	kfree(dtc);
 967}
 968
 969static int dmatest_add_threads(struct dmatest_info *info,
 970		struct dmatest_chan *dtc, enum dma_transaction_type type)
 971{
 972	struct dmatest_params *params = &info->params;
 973	struct dmatest_thread *thread;
 974	struct dma_chan *chan = dtc->chan;
 975	char *op;
 976	unsigned int i;
 977
 978	if (type == DMA_MEMCPY)
 979		op = "copy";
 980	else if (type == DMA_MEMSET)
 981		op = "set";
 982	else if (type == DMA_XOR)
 983		op = "xor";
 984	else if (type == DMA_PQ)
 985		op = "pq";
 986	else
 987		return -EINVAL;
 988
 989	for (i = 0; i < params->threads_per_chan; i++) {
 990		thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 991		if (!thread) {
 992			pr_warn("No memory for %s-%s%u\n",
 993				dma_chan_name(chan), op, i);
 994			break;
 995		}
 996		thread->info = info;
 997		thread->chan = dtc->chan;
 998		thread->type = type;
 999		thread->test_done.wait = &thread->done_wait;
1000		init_waitqueue_head(&thread->done_wait);
1001		smp_wmb();
1002		thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
1003				dma_chan_name(chan), op, i);
1004		if (IS_ERR(thread->task)) {
1005			pr_warn("Failed to create thread %s-%s%u\n",
1006				dma_chan_name(chan), op, i);
1007			kfree(thread);
1008			break;
1009		}
1010
1011		/* srcbuf and dstbuf are allocated by the thread itself */
1012		get_task_struct(thread->task);
1013		list_add_tail(&thread->node, &dtc->threads);
1014		thread->pending = true;
1015	}
1016
1017	return i;
1018}
1019
1020static int dmatest_add_channel(struct dmatest_info *info,
1021		struct dma_chan *chan)
1022{
1023	struct dmatest_chan	*dtc;
1024	struct dma_device	*dma_dev = chan->device;
1025	unsigned int		thread_count = 0;
1026	int cnt;
1027
1028	dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
1029	if (!dtc) {
1030		pr_warn("No memory for %s\n", dma_chan_name(chan));
1031		return -ENOMEM;
1032	}
1033
1034	dtc->chan = chan;
1035	INIT_LIST_HEAD(&dtc->threads);
1036
1037	if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) &&
1038	    info->params.polled) {
1039		info->params.polled = false;
1040		pr_warn("DMA_COMPLETION_NO_ORDER, polled disabled\n");
1041	}
1042
1043	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1044		if (dmatest == 0) {
1045			cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
1046			thread_count += cnt > 0 ? cnt : 0;
1047		}
1048	}
1049
1050	if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1051		if (dmatest == 1) {
1052			cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
1053			thread_count += cnt > 0 ? cnt : 0;
1054		}
1055	}
1056
1057	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1058		cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1059		thread_count += cnt > 0 ? cnt : 0;
1060	}
1061	if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1062		cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1063		thread_count += cnt > 0 ? cnt : 0;
1064	}
1065
1066	pr_info("Added %u threads using %s\n",
1067		thread_count, dma_chan_name(chan));
1068
1069	list_add_tail(&dtc->node, &info->channels);
1070	info->nr_channels++;
1071
1072	return 0;
1073}
1074
1075static bool filter(struct dma_chan *chan, void *param)
1076{
1077	return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device);
 
 
 
 
 
 
1078}
1079
1080static void request_channels(struct dmatest_info *info,
1081			     enum dma_transaction_type type)
1082{
1083	dma_cap_mask_t mask;
1084
1085	dma_cap_zero(mask);
1086	dma_cap_set(type, mask);
1087	for (;;) {
1088		struct dmatest_params *params = &info->params;
1089		struct dma_chan *chan;
1090
1091		chan = dma_request_channel(mask, filter, params);
1092		if (chan) {
1093			if (dmatest_add_channel(info, chan)) {
1094				dma_release_channel(chan);
1095				break; /* add_channel failed, punt */
1096			}
1097		} else
1098			break; /* no more channels available */
1099		if (params->max_channels &&
1100		    info->nr_channels >= params->max_channels)
1101			break; /* we have all we need */
1102	}
1103}
1104
1105static void add_threaded_test(struct dmatest_info *info)
1106{
1107	struct dmatest_params *params = &info->params;
1108
1109	/* Copy test parameters */
1110	params->nobounce = nobounce;
1111	params->buf_size = test_buf_size;
1112	strscpy(params->channel, strim(test_channel), sizeof(params->channel));
1113	strscpy(params->device, strim(test_device), sizeof(params->device));
1114	params->threads_per_chan = threads_per_chan;
1115	params->max_channels = max_channels;
1116	params->iterations = iterations;
1117	params->xor_sources = xor_sources;
1118	params->pq_sources = pq_sources;
1119	params->timeout = timeout;
1120	params->noverify = noverify;
1121	params->norandom = norandom;
1122	params->alignment = alignment;
1123	params->transfer_size = transfer_size;
1124	params->polled = polled;
1125
1126	request_channels(info, DMA_MEMCPY);
1127	request_channels(info, DMA_MEMSET);
1128	request_channels(info, DMA_XOR);
 
1129	request_channels(info, DMA_PQ);
1130}
1131
1132static void run_pending_tests(struct dmatest_info *info)
1133{
1134	struct dmatest_chan *dtc;
1135	unsigned int thread_count = 0;
1136
1137	list_for_each_entry(dtc, &info->channels, node) {
1138		struct dmatest_thread *thread;
1139
1140		thread_count = 0;
1141		list_for_each_entry(thread, &dtc->threads, node) {
1142			wake_up_process(thread->task);
1143			thread_count++;
1144		}
1145		pr_info("Started %u threads using %s\n",
1146			thread_count, dma_chan_name(dtc->chan));
1147	}
1148}
1149
1150static void stop_threaded_test(struct dmatest_info *info)
1151{
1152	struct dmatest_chan *dtc, *_dtc;
1153	struct dma_chan *chan;
1154
1155	list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1156		list_del(&dtc->node);
1157		chan = dtc->chan;
1158		dmatest_cleanup_channel(dtc);
1159		pr_debug("dropped channel %s\n", dma_chan_name(chan));
1160		dma_release_channel(chan);
1161	}
1162
1163	info->nr_channels = 0;
1164}
1165
1166static void start_threaded_tests(struct dmatest_info *info)
1167{
1168	/* we might be called early to set run=, defer running until all
1169	 * parameters have been evaluated
1170	 */
1171	if (!info->did_init)
1172		return;
1173
1174	run_pending_tests(info);
 
 
 
 
1175}
1176
1177static int dmatest_run_get(char *val, const struct kernel_param *kp)
1178{
1179	struct dmatest_info *info = &test_info;
1180
1181	mutex_lock(&info->lock);
1182	if (is_threaded_test_run(info)) {
1183		dmatest_run = true;
1184	} else {
1185		if (!is_threaded_test_pending(info))
1186			stop_threaded_test(info);
1187		dmatest_run = false;
1188	}
1189	mutex_unlock(&info->lock);
1190
1191	return param_get_bool(val, kp);
1192}
1193
1194static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1195{
1196	struct dmatest_info *info = &test_info;
1197	int ret;
1198
1199	mutex_lock(&info->lock);
1200	ret = param_set_bool(val, kp);
1201	if (ret) {
1202		mutex_unlock(&info->lock);
1203		return ret;
1204	} else if (dmatest_run) {
1205		if (!is_threaded_test_pending(info)) {
1206			/*
1207			 * We have nothing to run. This can be due to:
1208			 */
1209			ret = info->last_error;
1210			if (ret) {
1211				/* 1) Misconfiguration */
1212				pr_err("Channel misconfigured, can't continue\n");
1213				mutex_unlock(&info->lock);
1214				return ret;
1215			} else {
1216				/* 2) We rely on defaults */
1217				pr_info("No channels configured, continue with any\n");
1218				if (!is_threaded_test_run(info))
1219					stop_threaded_test(info);
1220				add_threaded_test(info);
1221			}
1222		}
1223		start_threaded_tests(info);
1224	} else {
1225		stop_threaded_test(info);
1226	}
1227
1228	mutex_unlock(&info->lock);
1229
1230	return ret;
1231}
1232
1233static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1234{
1235	struct dmatest_info *info = &test_info;
1236	struct dmatest_chan *dtc;
1237	char chan_reset_val[20];
1238	int ret;
1239
1240	mutex_lock(&info->lock);
1241	ret = param_set_copystring(val, kp);
1242	if (ret) {
1243		mutex_unlock(&info->lock);
1244		return ret;
1245	}
1246	/*Clear any previously run threads */
1247	if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1248		stop_threaded_test(info);
1249	/* Reject channels that are already registered */
1250	if (is_threaded_test_pending(info)) {
1251		list_for_each_entry(dtc, &info->channels, node) {
1252			if (strcmp(dma_chan_name(dtc->chan),
1253				   strim(test_channel)) == 0) {
1254				dtc = list_last_entry(&info->channels,
1255						      struct dmatest_chan,
1256						      node);
1257				strscpy(chan_reset_val,
1258					dma_chan_name(dtc->chan),
1259					sizeof(chan_reset_val));
1260				ret = -EBUSY;
1261				goto add_chan_err;
1262			}
1263		}
1264	}
1265
1266	add_threaded_test(info);
1267
1268	/* Check if channel was added successfully */
1269	if (!list_empty(&info->channels)) {
1270		/*
1271		 * if new channel was not successfully added, revert the
1272		 * "test_channel" string to the name of the last successfully
1273		 * added channel. exception for when users issues empty string
1274		 * to channel parameter.
1275		 */
1276		dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1277		if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1278		    && (strcmp("", strim(test_channel)) != 0)) {
1279			ret = -EINVAL;
1280			strscpy(chan_reset_val, dma_chan_name(dtc->chan),
1281				sizeof(chan_reset_val));
1282			goto add_chan_err;
1283		}
1284
1285	} else {
1286		/* Clear test_channel if no channels were added successfully */
1287		strscpy(chan_reset_val, "", sizeof(chan_reset_val));
1288		ret = -EBUSY;
1289		goto add_chan_err;
1290	}
1291
1292	info->last_error = ret;
1293	mutex_unlock(&info->lock);
1294
1295	return ret;
1296
1297add_chan_err:
1298	param_set_copystring(chan_reset_val, kp);
1299	info->last_error = ret;
1300	mutex_unlock(&info->lock);
1301
1302	return ret;
1303}
1304
1305static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1306{
1307	struct dmatest_info *info = &test_info;
1308
1309	mutex_lock(&info->lock);
1310	if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1311		stop_threaded_test(info);
1312		strscpy(test_channel, "", sizeof(test_channel));
1313	}
1314	mutex_unlock(&info->lock);
1315
1316	return param_get_string(val, kp);
1317}
1318
1319static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1320{
1321	struct dmatest_info *info = &test_info;
1322	struct dmatest_chan *dtc;
1323	unsigned int thread_count = 0;
1324
1325	list_for_each_entry(dtc, &info->channels, node) {
1326		struct dmatest_thread *thread;
1327
1328		thread_count = 0;
1329		list_for_each_entry(thread, &dtc->threads, node) {
1330			thread_count++;
1331		}
1332		pr_info("%u threads using %s\n",
1333			thread_count, dma_chan_name(dtc->chan));
1334	}
1335
1336	return 0;
1337}
1338
1339static int __init dmatest_init(void)
1340{
1341	struct dmatest_info *info = &test_info;
1342	struct dmatest_params *params = &info->params;
1343
1344	if (dmatest_run) {
1345		mutex_lock(&info->lock);
1346		add_threaded_test(info);
1347		run_pending_tests(info);
1348		mutex_unlock(&info->lock);
1349	}
1350
1351	if (params->iterations && wait)
1352		wait_event(thread_wait, !is_threaded_test_run(info));
1353
1354	/* module parameters are stable, inittime tests are started,
1355	 * let userspace take over 'run' control
1356	 */
1357	info->did_init = true;
1358
1359	return 0;
1360}
1361/* when compiled-in wait for drivers to load first */
1362late_initcall(dmatest_init);
1363
1364static void __exit dmatest_exit(void)
1365{
1366	struct dmatest_info *info = &test_info;
1367
1368	mutex_lock(&info->lock);
1369	stop_threaded_test(info);
1370	mutex_unlock(&info->lock);
1371}
1372module_exit(dmatest_exit);
1373
1374MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1375MODULE_LICENSE("GPL v2");
v4.10.11
 
   1/*
   2 * DMA Engine test module
   3 *
   4 * Copyright (C) 2007 Atmel Corporation
   5 * Copyright (C) 2013 Intel Corporation
   6 *
   7 * This program is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License version 2 as
   9 * published by the Free Software Foundation.
  10 */
  11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
 
  13#include <linux/delay.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/dmaengine.h>
  16#include <linux/freezer.h>
  17#include <linux/init.h>
  18#include <linux/kthread.h>
 
  19#include <linux/module.h>
  20#include <linux/moduleparam.h>
  21#include <linux/random.h>
  22#include <linux/slab.h>
  23#include <linux/wait.h>
  24
 
 
 
 
  25static unsigned int test_buf_size = 16384;
  26module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
  27MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
  28
  29static char test_channel[20];
  30module_param_string(channel, test_channel, sizeof(test_channel),
  31		S_IRUGO | S_IWUSR);
  32MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
  33
  34static char test_device[32];
  35module_param_string(device, test_device, sizeof(test_device),
  36		S_IRUGO | S_IWUSR);
  37MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
  38
  39static unsigned int threads_per_chan = 1;
  40module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
  41MODULE_PARM_DESC(threads_per_chan,
  42		"Number of threads to start per channel (default: 1)");
  43
  44static unsigned int max_channels;
  45module_param(max_channels, uint, S_IRUGO | S_IWUSR);
  46MODULE_PARM_DESC(max_channels,
  47		"Maximum number of channels to use (default: all)");
  48
  49static unsigned int iterations;
  50module_param(iterations, uint, S_IRUGO | S_IWUSR);
  51MODULE_PARM_DESC(iterations,
  52		"Iterations before stopping test (default: infinite)");
  53
  54static unsigned int sg_buffers = 1;
  55module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
  56MODULE_PARM_DESC(sg_buffers,
  57		"Number of scatter gather buffers (default: 1)");
  58
  59static unsigned int dmatest;
  60module_param(dmatest, uint, S_IRUGO | S_IWUSR);
  61MODULE_PARM_DESC(dmatest,
  62		"dmatest 0-memcpy 1-slave_sg (default: 0)");
  63
  64static unsigned int xor_sources = 3;
  65module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
  66MODULE_PARM_DESC(xor_sources,
  67		"Number of xor source buffers (default: 3)");
  68
  69static unsigned int pq_sources = 3;
  70module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
  71MODULE_PARM_DESC(pq_sources,
  72		"Number of p+q source buffers (default: 3)");
  73
  74static int timeout = 3000;
  75module_param(timeout, uint, S_IRUGO | S_IWUSR);
  76MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
  77		 "Pass -1 for infinite timeout");
  78
  79static bool noverify;
  80module_param(noverify, bool, S_IRUGO | S_IWUSR);
  81MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
 
 
 
 
  82
  83static bool verbose;
  84module_param(verbose, bool, S_IRUGO | S_IWUSR);
  85MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
  86
 
 
 
 
 
 
 
 
 
 
 
 
  87/**
  88 * struct dmatest_params - test parameters.
 
  89 * @buf_size:		size of the memcpy test buffer
  90 * @channel:		bus ID of the channel to test
  91 * @device:		bus ID of the DMA Engine to test
  92 * @threads_per_chan:	number of threads to start per channel
  93 * @max_channels:	maximum number of channels to use
  94 * @iterations:		iterations before stopping test
  95 * @xor_sources:	number of xor source buffers
  96 * @pq_sources:		number of p+q source buffers
  97 * @timeout:		transfer timeout in msec, -1 for infinite timeout
 
 
 
 
 
  98 */
  99struct dmatest_params {
 
 100	unsigned int	buf_size;
 101	char		channel[20];
 102	char		device[32];
 103	unsigned int	threads_per_chan;
 104	unsigned int	max_channels;
 105	unsigned int	iterations;
 106	unsigned int	xor_sources;
 107	unsigned int	pq_sources;
 108	int		timeout;
 109	bool		noverify;
 
 
 
 
 110};
 111
 112/**
 113 * struct dmatest_info - test information.
 114 * @params:		test parameters
 
 
 115 * @lock:		access protection to the fields of this structure
 
 
 116 */
 117static struct dmatest_info {
 118	/* Test parameters */
 119	struct dmatest_params	params;
 120
 121	/* Internal state */
 122	struct list_head	channels;
 123	unsigned int		nr_channels;
 
 124	struct mutex		lock;
 125	bool			did_init;
 126} test_info = {
 127	.channels = LIST_HEAD_INIT(test_info.channels),
 128	.lock = __MUTEX_INITIALIZER(test_info.lock),
 129};
 130
 131static int dmatest_run_set(const char *val, const struct kernel_param *kp);
 132static int dmatest_run_get(char *val, const struct kernel_param *kp);
 133static const struct kernel_param_ops run_ops = {
 134	.set = dmatest_run_set,
 135	.get = dmatest_run_get,
 136};
 137static bool dmatest_run;
 138module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
 139MODULE_PARM_DESC(run, "Run the test (default: false)");
 140
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 141/* Maximum amount of mismatched bytes in buffer to print */
 142#define MAX_ERROR_COUNT		32
 143
 144/*
 145 * Initialization patterns. All bytes in the source buffer has bit 7
 146 * set, all bytes in the destination buffer has bit 7 cleared.
 147 *
 148 * Bit 6 is set for all bytes which are to be copied by the DMA
 149 * engine. Bit 5 is set for all bytes which are to be overwritten by
 150 * the DMA engine.
 151 *
 152 * The remaining bits are the inverse of a counter which increments by
 153 * one for each byte address.
 154 */
 155#define PATTERN_SRC		0x80
 156#define PATTERN_DST		0x00
 157#define PATTERN_COPY		0x40
 158#define PATTERN_OVERWRITE	0x20
 159#define PATTERN_COUNT_MASK	0x1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 160
 161struct dmatest_thread {
 162	struct list_head	node;
 163	struct dmatest_info	*info;
 164	struct task_struct	*task;
 165	struct dma_chan		*chan;
 166	u8			**srcs;
 167	u8			**usrcs;
 168	u8			**dsts;
 169	u8			**udsts;
 170	enum dma_transaction_type type;
 
 
 171	bool			done;
 
 172};
 173
 174struct dmatest_chan {
 175	struct list_head	node;
 176	struct dma_chan		*chan;
 177	struct list_head	threads;
 178};
 179
 180static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
 181static bool wait;
 182
 183static bool is_threaded_test_run(struct dmatest_info *info)
 184{
 185	struct dmatest_chan *dtc;
 186
 187	list_for_each_entry(dtc, &info->channels, node) {
 188		struct dmatest_thread *thread;
 189
 190		list_for_each_entry(thread, &dtc->threads, node) {
 191			if (!thread->done)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 192				return true;
 193		}
 194	}
 195
 196	return false;
 197}
 198
 199static int dmatest_wait_get(char *val, const struct kernel_param *kp)
 200{
 201	struct dmatest_info *info = &test_info;
 202	struct dmatest_params *params = &info->params;
 203
 204	if (params->iterations)
 205		wait_event(thread_wait, !is_threaded_test_run(info));
 206	wait = true;
 207	return param_get_bool(val, kp);
 208}
 209
 210static const struct kernel_param_ops wait_ops = {
 211	.get = dmatest_wait_get,
 212	.set = param_set_bool,
 213};
 214module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
 215MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
 216
 217static bool dmatest_match_channel(struct dmatest_params *params,
 218		struct dma_chan *chan)
 219{
 220	if (params->channel[0] == '\0')
 221		return true;
 222	return strcmp(dma_chan_name(chan), params->channel) == 0;
 223}
 224
 225static bool dmatest_match_device(struct dmatest_params *params,
 226		struct dma_device *device)
 227{
 228	if (params->device[0] == '\0')
 229		return true;
 230	return strcmp(dev_name(device->dev), params->device) == 0;
 231}
 232
 233static unsigned long dmatest_random(void)
 234{
 235	unsigned long buf;
 236
 237	prandom_bytes(&buf, sizeof(buf));
 238	return buf;
 239}
 240
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 241static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
 242		unsigned int buf_size)
 243{
 244	unsigned int i;
 245	u8 *buf;
 246
 247	for (; (buf = *bufs); bufs++) {
 248		for (i = 0; i < start; i++)
 249			buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 250		for ( ; i < start + len; i++)
 251			buf[i] = PATTERN_SRC | PATTERN_COPY
 252				| (~i & PATTERN_COUNT_MASK);
 253		for ( ; i < buf_size; i++)
 254			buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
 255		buf++;
 256	}
 257}
 258
 259static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
 260		unsigned int buf_size)
 261{
 262	unsigned int i;
 263	u8 *buf;
 264
 265	for (; (buf = *bufs); bufs++) {
 266		for (i = 0; i < start; i++)
 267			buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 268		for ( ; i < start + len; i++)
 269			buf[i] = PATTERN_DST | PATTERN_OVERWRITE
 270				| (~i & PATTERN_COUNT_MASK);
 271		for ( ; i < buf_size; i++)
 272			buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
 273	}
 274}
 275
 276static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
 277		unsigned int counter, bool is_srcbuf)
 278{
 279	u8		diff = actual ^ pattern;
 280	u8		expected = pattern | (~counter & PATTERN_COUNT_MASK);
 281	const char	*thread_name = current->comm;
 282
 283	if (is_srcbuf)
 284		pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
 285			thread_name, index, expected, actual);
 286	else if ((pattern & PATTERN_COPY)
 287			&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
 288		pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
 289			thread_name, index, expected, actual);
 290	else if (diff & PATTERN_SRC)
 291		pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
 292			thread_name, index, expected, actual);
 293	else
 294		pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
 295			thread_name, index, expected, actual);
 296}
 297
 298static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
 299		unsigned int end, unsigned int counter, u8 pattern,
 300		bool is_srcbuf)
 301{
 302	unsigned int i;
 303	unsigned int error_count = 0;
 304	u8 actual;
 305	u8 expected;
 306	u8 *buf;
 307	unsigned int counter_orig = counter;
 308
 309	for (; (buf = *bufs); bufs++) {
 310		counter = counter_orig;
 311		for (i = start; i < end; i++) {
 312			actual = buf[i];
 313			expected = pattern | (~counter & PATTERN_COUNT_MASK);
 314			if (actual != expected) {
 315				if (error_count < MAX_ERROR_COUNT)
 316					dmatest_mismatch(actual, pattern, i,
 317							 counter, is_srcbuf);
 
 318				error_count++;
 319			}
 320			counter++;
 321		}
 322	}
 323
 324	if (error_count > MAX_ERROR_COUNT)
 325		pr_warn("%s: %u errors suppressed\n",
 326			current->comm, error_count - MAX_ERROR_COUNT);
 327
 328	return error_count;
 329}
 330
 331/* poor man's completion - we want to use wait_event_freezable() on it */
 332struct dmatest_done {
 333	bool			done;
 334	wait_queue_head_t	*wait;
 335};
 336
 337static void dmatest_callback(void *arg)
 338{
 339	struct dmatest_done *done = arg;
 340
 341	done->done = true;
 342	wake_up_all(done->wait);
 
 
 
 
 
 
 
 
 
 
 
 
 343}
 344
 345static unsigned int min_odd(unsigned int x, unsigned int y)
 346{
 347	unsigned int val = min(x, y);
 348
 349	return val % 2 ? val : val - 1;
 350}
 351
 352static void result(const char *err, unsigned int n, unsigned int src_off,
 353		   unsigned int dst_off, unsigned int len, unsigned long data)
 354{
 355	pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 356		current->comm, n, err, src_off, dst_off, len, data);
 
 
 
 
 
 357}
 358
 359static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
 360		       unsigned int dst_off, unsigned int len,
 361		       unsigned long data)
 362{
 363	pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
 364		 current->comm, n, err, src_off, dst_off, len, data);
 365}
 366
 367#define verbose_result(err, n, src_off, dst_off, len, data) ({	\
 368	if (verbose)						\
 369		result(err, n, src_off, dst_off, len, data);	\
 370	else							\
 371		dbg_result(err, n, src_off, dst_off, len, data);\
 372})
 373
 374static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
 375{
 376	unsigned long long per_sec = 1000000;
 377
 378	if (runtime <= 0)
 379		return 0;
 380
 381	/* drop precision until runtime is 32-bits */
 382	while (runtime > UINT_MAX) {
 383		runtime >>= 1;
 384		per_sec <<= 1;
 385	}
 386
 387	per_sec *= val;
 
 388	do_div(per_sec, runtime);
 
 389	return per_sec;
 390}
 391
 392static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
 393{
 394	return dmatest_persec(runtime, len >> 10);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 395}
 396
 397/*
 398 * This function repeatedly tests DMA transfers of various lengths and
 399 * offsets for a given operation type until it is told to exit by
 400 * kthread_stop(). There may be multiple threads running this function
 401 * in parallel for a single channel, and there may be multiple channels
 402 * being tested in parallel.
 403 *
 404 * Before each test, the source and destination buffer is initialized
 405 * with a known pattern. This pattern is different depending on
 406 * whether it's in an area which is supposed to be copied or
 407 * overwritten, and different in the source and destination buffers.
 408 * So if the DMA engine doesn't copy exactly what we tell it to copy,
 409 * we'll notice.
 410 */
 411static int dmatest_func(void *data)
 412{
 413	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
 414	struct dmatest_thread	*thread = data;
 415	struct dmatest_done	done = { .wait = &done_wait };
 416	struct dmatest_info	*info;
 417	struct dmatest_params	*params;
 418	struct dma_chan		*chan;
 419	struct dma_device	*dev;
 
 420	unsigned int		error_count;
 421	unsigned int		failed_tests = 0;
 422	unsigned int		total_tests = 0;
 423	dma_cookie_t		cookie;
 424	enum dma_status		status;
 425	enum dma_ctrl_flags 	flags;
 426	u8			*pq_coefs = NULL;
 427	int			ret;
 428	int			src_cnt;
 429	int			dst_cnt;
 
 430	int			i;
 431	ktime_t			ktime, start, diff;
 432	ktime_t			filltime = 0;
 433	ktime_t			comparetime = 0;
 434	s64			runtime = 0;
 435	unsigned long long	total_len = 0;
 
 436	u8			align = 0;
 
 
 
 437
 438	set_freezable();
 439
 440	ret = -ENOMEM;
 441
 442	smp_rmb();
 
 443	info = thread->info;
 444	params = &info->params;
 445	chan = thread->chan;
 446	dev = chan->device;
 
 
 
 
 447	if (thread->type == DMA_MEMCPY) {
 448		align = dev->copy_align;
 449		src_cnt = dst_cnt = 1;
 450	} else if (thread->type == DMA_SG) {
 451		align = dev->copy_align;
 452		src_cnt = dst_cnt = sg_buffers;
 
 
 
 453	} else if (thread->type == DMA_XOR) {
 454		/* force odd to ensure dst = src */
 455		src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
 456		dst_cnt = 1;
 457		align = dev->xor_align;
 
 458	} else if (thread->type == DMA_PQ) {
 459		/* force odd to ensure dst = src */
 460		src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
 461		dst_cnt = 2;
 462		align = dev->pq_align;
 
 463
 464		pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
 465		if (!pq_coefs)
 466			goto err_thread_type;
 467
 468		for (i = 0; i < src_cnt; i++)
 469			pq_coefs[i] = 1;
 470	} else
 471		goto err_thread_type;
 472
 473	thread->srcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
 474	if (!thread->srcs)
 475		goto err_srcs;
 476
 477	thread->usrcs = kcalloc(src_cnt + 1, sizeof(u8 *), GFP_KERNEL);
 478	if (!thread->usrcs)
 479		goto err_usrcs;
 480
 481	for (i = 0; i < src_cnt; i++) {
 482		thread->usrcs[i] = kmalloc(params->buf_size + align,
 483					   GFP_KERNEL);
 484		if (!thread->usrcs[i])
 485			goto err_srcbuf;
 486
 487		/* align srcs to alignment restriction */
 488		if (align)
 489			thread->srcs[i] = PTR_ALIGN(thread->usrcs[i], align);
 490		else
 491			thread->srcs[i] = thread->usrcs[i];
 492	}
 493	thread->srcs[i] = NULL;
 494
 495	thread->dsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
 496	if (!thread->dsts)
 497		goto err_dsts;
 
 
 
 498
 499	thread->udsts = kcalloc(dst_cnt + 1, sizeof(u8 *), GFP_KERNEL);
 500	if (!thread->udsts)
 501		goto err_udsts;
 502
 503	for (i = 0; i < dst_cnt; i++) {
 504		thread->udsts[i] = kmalloc(params->buf_size + align,
 505					   GFP_KERNEL);
 506		if (!thread->udsts[i])
 507			goto err_dstbuf;
 508
 509		/* align dsts to alignment restriction */
 510		if (align)
 511			thread->dsts[i] = PTR_ALIGN(thread->udsts[i], align);
 512		else
 513			thread->dsts[i] = thread->udsts[i];
 514	}
 515	thread->dsts[i] = NULL;
 516
 517	set_user_nice(current, 10);
 
 
 
 
 
 
 518
 519	/*
 520	 * src and dst buffers are freed by ourselves below
 521	 */
 522	flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
 
 
 
 523
 524	ktime = ktime_get();
 525	while (!kthread_should_stop()
 526	       && !(params->iterations && total_tests >= params->iterations)) {
 527		struct dma_async_tx_descriptor *tx = NULL;
 528		struct dmaengine_unmap_data *um;
 529		dma_addr_t srcs[src_cnt];
 530		dma_addr_t *dsts;
 531		unsigned int src_off, dst_off, len;
 532		struct scatterlist tx_sg[src_cnt];
 533		struct scatterlist rx_sg[src_cnt];
 534
 535		total_tests++;
 536
 537		if (1 << align > params->buf_size) {
 538			pr_err("%u-byte buffer too small for %d-byte alignment\n",
 539			       params->buf_size, 1 << align);
 540			break;
 
 
 
 
 
 
 
 541		}
 542
 543		if (params->noverify)
 544			len = params->buf_size;
 545		else
 546			len = dmatest_random() % params->buf_size + 1;
 547
 548		len = (len >> align) << align;
 549		if (!len)
 550			len = 1 << align;
 551
 552		total_len += len;
 553
 554		if (params->noverify) {
 555			src_off = 0;
 556			dst_off = 0;
 557		} else {
 558			start = ktime_get();
 559			src_off = dmatest_random() % (params->buf_size - len + 1);
 560			dst_off = dmatest_random() % (params->buf_size - len + 1);
 561
 562			src_off = (src_off >> align) << align;
 563			dst_off = (dst_off >> align) << align;
 
 564
 565			dmatest_init_srcs(thread->srcs, src_off, len,
 566					  params->buf_size);
 567			dmatest_init_dsts(thread->dsts, dst_off, len,
 568					  params->buf_size);
 
 
 569
 570			diff = ktime_sub(ktime_get(), start);
 571			filltime = ktime_add(filltime, diff);
 572		}
 573
 574		um = dmaengine_get_unmap_data(dev->dev, src_cnt + dst_cnt,
 575					      GFP_KERNEL);
 576		if (!um) {
 577			failed_tests++;
 578			result("unmap data NULL", total_tests,
 579			       src_off, dst_off, len, ret);
 580			continue;
 581		}
 582
 583		um->len = params->buf_size;
 584		for (i = 0; i < src_cnt; i++) {
 585			void *buf = thread->srcs[i];
 586			struct page *pg = virt_to_page(buf);
 587			unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
 588
 589			um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
 590						   um->len, DMA_TO_DEVICE);
 591			srcs[i] = um->addr[i] + src_off;
 592			ret = dma_mapping_error(dev->dev, um->addr[i]);
 593			if (ret) {
 594				dmaengine_unmap_put(um);
 595				result("src mapping error", total_tests,
 596				       src_off, dst_off, len, ret);
 597				failed_tests++;
 598				continue;
 599			}
 600			um->to_cnt++;
 601		}
 602		/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
 603		dsts = &um->addr[src_cnt];
 604		for (i = 0; i < dst_cnt; i++) {
 605			void *buf = thread->dsts[i];
 606			struct page *pg = virt_to_page(buf);
 607			unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
 608
 609			dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
 610					       DMA_BIDIRECTIONAL);
 611			ret = dma_mapping_error(dev->dev, dsts[i]);
 612			if (ret) {
 613				dmaengine_unmap_put(um);
 614				result("dst mapping error", total_tests,
 615				       src_off, dst_off, len, ret);
 616				failed_tests++;
 617				continue;
 618			}
 619			um->bidi_cnt++;
 620		}
 621
 622		sg_init_table(tx_sg, src_cnt);
 623		sg_init_table(rx_sg, src_cnt);
 624		for (i = 0; i < src_cnt; i++) {
 625			sg_dma_address(&rx_sg[i]) = srcs[i];
 626			sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
 627			sg_dma_len(&tx_sg[i]) = len;
 628			sg_dma_len(&rx_sg[i]) = len;
 629		}
 630
 631		if (thread->type == DMA_MEMCPY)
 632			tx = dev->device_prep_dma_memcpy(chan,
 633							 dsts[0] + dst_off,
 634							 srcs[0], len, flags);
 635		else if (thread->type == DMA_SG)
 636			tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
 637						     rx_sg, src_cnt, flags);
 
 
 638		else if (thread->type == DMA_XOR)
 639			tx = dev->device_prep_dma_xor(chan,
 640						      dsts[0] + dst_off,
 641						      srcs, src_cnt,
 642						      len, flags);
 643		else if (thread->type == DMA_PQ) {
 644			dma_addr_t dma_pq[dst_cnt];
 645
 646			for (i = 0; i < dst_cnt; i++)
 647				dma_pq[i] = dsts[i] + dst_off;
 648			tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
 649						     src_cnt, pq_coefs,
 650						     len, flags);
 651		}
 652
 653		if (!tx) {
 654			dmaengine_unmap_put(um);
 655			result("prep error", total_tests, src_off,
 656			       dst_off, len, ret);
 657			msleep(100);
 658			failed_tests++;
 659			continue;
 660		}
 661
 662		done.done = false;
 663		tx->callback = dmatest_callback;
 664		tx->callback_param = &done;
 
 
 665		cookie = tx->tx_submit(tx);
 666
 667		if (dma_submit_error(cookie)) {
 668			dmaengine_unmap_put(um);
 669			result("submit error", total_tests, src_off,
 670			       dst_off, len, ret);
 671			msleep(100);
 672			failed_tests++;
 673			continue;
 674		}
 675		dma_async_issue_pending(chan);
 676
 677		wait_event_freezable_timeout(done_wait, done.done,
 678					     msecs_to_jiffies(params->timeout));
 
 
 
 
 
 
 
 
 
 679
 680		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 
 
 681
 682		if (!done.done) {
 683			/*
 684			 * We're leaving the timed out dma operation with
 685			 * dangling pointer to done_wait.  To make this
 686			 * correct, we'll need to allocate wait_done for
 687			 * each test iteration and perform "who's gonna
 688			 * free it this time?" dancing.  For now, just
 689			 * leave it dangling.
 690			 */
 691			dmaengine_unmap_put(um);
 692			result("test timed out", total_tests, src_off, dst_off,
 693			       len, 0);
 694			failed_tests++;
 695			continue;
 696		} else if (status != DMA_COMPLETE) {
 697			dmaengine_unmap_put(um);
 
 698			result(status == DMA_ERROR ?
 699			       "completion error status" :
 700			       "completion busy status", total_tests, src_off,
 701			       dst_off, len, ret);
 702			failed_tests++;
 703			continue;
 704		}
 705
 706		dmaengine_unmap_put(um);
 707
 708		if (params->noverify) {
 709			verbose_result("test passed", total_tests, src_off,
 710				       dst_off, len, 0);
 711			continue;
 712		}
 713
 714		start = ktime_get();
 715		pr_debug("%s: verifying source buffer...\n", current->comm);
 716		error_count = dmatest_verify(thread->srcs, 0, src_off,
 717				0, PATTERN_SRC, true);
 718		error_count += dmatest_verify(thread->srcs, src_off,
 719				src_off + len, src_off,
 720				PATTERN_SRC | PATTERN_COPY, true);
 721		error_count += dmatest_verify(thread->srcs, src_off + len,
 722				params->buf_size, src_off + len,
 723				PATTERN_SRC, true);
 724
 725		pr_debug("%s: verifying dest buffer...\n", current->comm);
 726		error_count += dmatest_verify(thread->dsts, 0, dst_off,
 727				0, PATTERN_DST, false);
 728		error_count += dmatest_verify(thread->dsts, dst_off,
 729				dst_off + len, src_off,
 730				PATTERN_SRC | PATTERN_COPY, false);
 731		error_count += dmatest_verify(thread->dsts, dst_off + len,
 732				params->buf_size, dst_off + len,
 733				PATTERN_DST, false);
 
 
 734
 735		diff = ktime_sub(ktime_get(), start);
 736		comparetime = ktime_add(comparetime, diff);
 737
 738		if (error_count) {
 739			result("data error", total_tests, src_off, dst_off,
 740			       len, error_count);
 741			failed_tests++;
 742		} else {
 743			verbose_result("test passed", total_tests, src_off,
 744				       dst_off, len, 0);
 745		}
 
 
 
 
 
 
 746	}
 747	ktime = ktime_sub(ktime_get(), ktime);
 748	ktime = ktime_sub(ktime, comparetime);
 749	ktime = ktime_sub(ktime, filltime);
 750	runtime = ktime_to_us(ktime);
 751
 752	ret = 0;
 753err_dstbuf:
 754	for (i = 0; thread->udsts[i]; i++)
 755		kfree(thread->udsts[i]);
 756	kfree(thread->udsts);
 757err_udsts:
 758	kfree(thread->dsts);
 759err_dsts:
 760err_srcbuf:
 761	for (i = 0; thread->usrcs[i]; i++)
 762		kfree(thread->usrcs[i]);
 763	kfree(thread->usrcs);
 764err_usrcs:
 765	kfree(thread->srcs);
 766err_srcs:
 767	kfree(pq_coefs);
 768err_thread_type:
 769	pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
 
 770		current->comm, total_tests, failed_tests,
 771		dmatest_persec(runtime, total_tests),
 772		dmatest_KBs(runtime, total_len), ret);
 773
 774	/* terminate all transfers on specified channels */
 775	if (ret)
 776		dmaengine_terminate_all(chan);
 777
 778	thread->done = true;
 779	wake_up(&thread_wait);
 780
 781	return ret;
 782}
 783
 784static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
 785{
 786	struct dmatest_thread	*thread;
 787	struct dmatest_thread	*_thread;
 788	int			ret;
 789
 790	list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
 791		ret = kthread_stop(thread->task);
 792		pr_debug("thread %s exited with status %d\n",
 793			 thread->task->comm, ret);
 794		list_del(&thread->node);
 795		put_task_struct(thread->task);
 796		kfree(thread);
 797	}
 798
 799	/* terminate all transfers on specified channels */
 800	dmaengine_terminate_all(dtc->chan);
 801
 802	kfree(dtc);
 803}
 804
 805static int dmatest_add_threads(struct dmatest_info *info,
 806		struct dmatest_chan *dtc, enum dma_transaction_type type)
 807{
 808	struct dmatest_params *params = &info->params;
 809	struct dmatest_thread *thread;
 810	struct dma_chan *chan = dtc->chan;
 811	char *op;
 812	unsigned int i;
 813
 814	if (type == DMA_MEMCPY)
 815		op = "copy";
 816	else if (type == DMA_SG)
 817		op = "sg";
 818	else if (type == DMA_XOR)
 819		op = "xor";
 820	else if (type == DMA_PQ)
 821		op = "pq";
 822	else
 823		return -EINVAL;
 824
 825	for (i = 0; i < params->threads_per_chan; i++) {
 826		thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
 827		if (!thread) {
 828			pr_warn("No memory for %s-%s%u\n",
 829				dma_chan_name(chan), op, i);
 830			break;
 831		}
 832		thread->info = info;
 833		thread->chan = dtc->chan;
 834		thread->type = type;
 
 
 835		smp_wmb();
 836		thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
 837				dma_chan_name(chan), op, i);
 838		if (IS_ERR(thread->task)) {
 839			pr_warn("Failed to create thread %s-%s%u\n",
 840				dma_chan_name(chan), op, i);
 841			kfree(thread);
 842			break;
 843		}
 844
 845		/* srcbuf and dstbuf are allocated by the thread itself */
 846		get_task_struct(thread->task);
 847		list_add_tail(&thread->node, &dtc->threads);
 848		wake_up_process(thread->task);
 849	}
 850
 851	return i;
 852}
 853
 854static int dmatest_add_channel(struct dmatest_info *info,
 855		struct dma_chan *chan)
 856{
 857	struct dmatest_chan	*dtc;
 858	struct dma_device	*dma_dev = chan->device;
 859	unsigned int		thread_count = 0;
 860	int cnt;
 861
 862	dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
 863	if (!dtc) {
 864		pr_warn("No memory for %s\n", dma_chan_name(chan));
 865		return -ENOMEM;
 866	}
 867
 868	dtc->chan = chan;
 869	INIT_LIST_HEAD(&dtc->threads);
 870
 
 
 
 
 
 
 871	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
 872		if (dmatest == 0) {
 873			cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
 874			thread_count += cnt > 0 ? cnt : 0;
 875		}
 876	}
 877
 878	if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
 879		if (dmatest == 1) {
 880			cnt = dmatest_add_threads(info, dtc, DMA_SG);
 881			thread_count += cnt > 0 ? cnt : 0;
 882		}
 883	}
 884
 885	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
 886		cnt = dmatest_add_threads(info, dtc, DMA_XOR);
 887		thread_count += cnt > 0 ? cnt : 0;
 888	}
 889	if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
 890		cnt = dmatest_add_threads(info, dtc, DMA_PQ);
 891		thread_count += cnt > 0 ? cnt : 0;
 892	}
 893
 894	pr_info("Started %u threads using %s\n",
 895		thread_count, dma_chan_name(chan));
 896
 897	list_add_tail(&dtc->node, &info->channels);
 898	info->nr_channels++;
 899
 900	return 0;
 901}
 902
 903static bool filter(struct dma_chan *chan, void *param)
 904{
 905	struct dmatest_params *params = param;
 906
 907	if (!dmatest_match_channel(params, chan) ||
 908	    !dmatest_match_device(params, chan->device))
 909		return false;
 910	else
 911		return true;
 912}
 913
 914static void request_channels(struct dmatest_info *info,
 915			     enum dma_transaction_type type)
 916{
 917	dma_cap_mask_t mask;
 918
 919	dma_cap_zero(mask);
 920	dma_cap_set(type, mask);
 921	for (;;) {
 922		struct dmatest_params *params = &info->params;
 923		struct dma_chan *chan;
 924
 925		chan = dma_request_channel(mask, filter, params);
 926		if (chan) {
 927			if (dmatest_add_channel(info, chan)) {
 928				dma_release_channel(chan);
 929				break; /* add_channel failed, punt */
 930			}
 931		} else
 932			break; /* no more channels available */
 933		if (params->max_channels &&
 934		    info->nr_channels >= params->max_channels)
 935			break; /* we have all we need */
 936	}
 937}
 938
 939static void run_threaded_test(struct dmatest_info *info)
 940{
 941	struct dmatest_params *params = &info->params;
 942
 943	/* Copy test parameters */
 
 944	params->buf_size = test_buf_size;
 945	strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
 946	strlcpy(params->device, strim(test_device), sizeof(params->device));
 947	params->threads_per_chan = threads_per_chan;
 948	params->max_channels = max_channels;
 949	params->iterations = iterations;
 950	params->xor_sources = xor_sources;
 951	params->pq_sources = pq_sources;
 952	params->timeout = timeout;
 953	params->noverify = noverify;
 
 
 
 
 954
 955	request_channels(info, DMA_MEMCPY);
 
 956	request_channels(info, DMA_XOR);
 957	request_channels(info, DMA_SG);
 958	request_channels(info, DMA_PQ);
 959}
 960
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 961static void stop_threaded_test(struct dmatest_info *info)
 962{
 963	struct dmatest_chan *dtc, *_dtc;
 964	struct dma_chan *chan;
 965
 966	list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
 967		list_del(&dtc->node);
 968		chan = dtc->chan;
 969		dmatest_cleanup_channel(dtc);
 970		pr_debug("dropped channel %s\n", dma_chan_name(chan));
 971		dma_release_channel(chan);
 972	}
 973
 974	info->nr_channels = 0;
 975}
 976
 977static void restart_threaded_test(struct dmatest_info *info, bool run)
 978{
 979	/* we might be called early to set run=, defer running until all
 980	 * parameters have been evaluated
 981	 */
 982	if (!info->did_init)
 983		return;
 984
 985	/* Stop any running test first */
 986	stop_threaded_test(info);
 987
 988	/* Run test with new parameters */
 989	run_threaded_test(info);
 990}
 991
 992static int dmatest_run_get(char *val, const struct kernel_param *kp)
 993{
 994	struct dmatest_info *info = &test_info;
 995
 996	mutex_lock(&info->lock);
 997	if (is_threaded_test_run(info)) {
 998		dmatest_run = true;
 999	} else {
1000		stop_threaded_test(info);
 
1001		dmatest_run = false;
1002	}
1003	mutex_unlock(&info->lock);
1004
1005	return param_get_bool(val, kp);
1006}
1007
1008static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1009{
1010	struct dmatest_info *info = &test_info;
1011	int ret;
1012
1013	mutex_lock(&info->lock);
1014	ret = param_set_bool(val, kp);
1015	if (ret) {
1016		mutex_unlock(&info->lock);
1017		return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1018	}
1019
1020	if (is_threaded_test_run(info))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1021		ret = -EBUSY;
1022	else if (dmatest_run)
1023		restart_threaded_test(info, dmatest_run);
 
 
 
 
 
1024
 
 
 
1025	mutex_unlock(&info->lock);
1026
1027	return ret;
1028}
1029
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1030static int __init dmatest_init(void)
1031{
1032	struct dmatest_info *info = &test_info;
1033	struct dmatest_params *params = &info->params;
1034
1035	if (dmatest_run) {
1036		mutex_lock(&info->lock);
1037		run_threaded_test(info);
 
1038		mutex_unlock(&info->lock);
1039	}
1040
1041	if (params->iterations && wait)
1042		wait_event(thread_wait, !is_threaded_test_run(info));
1043
1044	/* module parameters are stable, inittime tests are started,
1045	 * let userspace take over 'run' control
1046	 */
1047	info->did_init = true;
1048
1049	return 0;
1050}
1051/* when compiled-in wait for drivers to load first */
1052late_initcall(dmatest_init);
1053
1054static void __exit dmatest_exit(void)
1055{
1056	struct dmatest_info *info = &test_info;
1057
1058	mutex_lock(&info->lock);
1059	stop_threaded_test(info);
1060	mutex_unlock(&info->lock);
1061}
1062module_exit(dmatest_exit);
1063
1064MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1065MODULE_LICENSE("GPL v2");