Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2007-2008 Pierre Ossman
4 */
5
6#include <linux/mmc/core.h>
7#include <linux/mmc/card.h>
8#include <linux/mmc/host.h>
9#include <linux/mmc/mmc.h>
10#include <linux/slab.h>
11
12#include <linux/scatterlist.h>
13#include <linux/list.h>
14
15#include <linux/debugfs.h>
16#include <linux/uaccess.h>
17#include <linux/seq_file.h>
18#include <linux/module.h>
19
20#include "core.h"
21#include "card.h"
22#include "host.h"
23#include "bus.h"
24#include "mmc_ops.h"
25
26#define RESULT_OK 0
27#define RESULT_FAIL 1
28#define RESULT_UNSUP_HOST 2
29#define RESULT_UNSUP_CARD 3
30
31#define BUFFER_ORDER 2
32#define BUFFER_SIZE (PAGE_SIZE << BUFFER_ORDER)
33
34#define TEST_ALIGN_END 8
35
36/*
37 * Limit the test area size to the maximum MMC HC erase group size. Note that
38 * the maximum SD allocation unit size is just 4MiB.
39 */
40#define TEST_AREA_MAX_SIZE (128 * 1024 * 1024)
41
42/**
43 * struct mmc_test_pages - pages allocated by 'alloc_pages()'.
44 * @page: first page in the allocation
45 * @order: order of the number of pages allocated
46 */
47struct mmc_test_pages {
48 struct page *page;
49 unsigned int order;
50};
51
52/**
53 * struct mmc_test_mem - allocated memory.
54 * @arr: array of allocations
55 * @cnt: number of allocations
56 */
57struct mmc_test_mem {
58 struct mmc_test_pages *arr;
59 unsigned int cnt;
60};
61
62/**
63 * struct mmc_test_area - information for performance tests.
64 * @max_sz: test area size (in bytes)
65 * @dev_addr: address on card at which to do performance tests
66 * @max_tfr: maximum transfer size allowed by driver (in bytes)
67 * @max_segs: maximum segments allowed by driver in scatterlist @sg
68 * @max_seg_sz: maximum segment size allowed by driver
69 * @blocks: number of (512 byte) blocks currently mapped by @sg
70 * @sg_len: length of currently mapped scatterlist @sg
71 * @mem: allocated memory
72 * @sg: scatterlist
73 * @sg_areq: scatterlist for non-blocking request
74 */
75struct mmc_test_area {
76 unsigned long max_sz;
77 unsigned int dev_addr;
78 unsigned int max_tfr;
79 unsigned int max_segs;
80 unsigned int max_seg_sz;
81 unsigned int blocks;
82 unsigned int sg_len;
83 struct mmc_test_mem *mem;
84 struct scatterlist *sg;
85 struct scatterlist *sg_areq;
86};
87
88/**
89 * struct mmc_test_transfer_result - transfer results for performance tests.
90 * @link: double-linked list
91 * @count: amount of group of sectors to check
92 * @sectors: amount of sectors to check in one group
93 * @ts: time values of transfer
94 * @rate: calculated transfer rate
95 * @iops: I/O operations per second (times 100)
96 */
97struct mmc_test_transfer_result {
98 struct list_head link;
99 unsigned int count;
100 unsigned int sectors;
101 struct timespec64 ts;
102 unsigned int rate;
103 unsigned int iops;
104};
105
106/**
107 * struct mmc_test_general_result - results for tests.
108 * @link: double-linked list
109 * @card: card under test
110 * @testcase: number of test case
111 * @result: result of test run
112 * @tr_lst: transfer measurements if any as mmc_test_transfer_result
113 */
114struct mmc_test_general_result {
115 struct list_head link;
116 struct mmc_card *card;
117 int testcase;
118 int result;
119 struct list_head tr_lst;
120};
121
122/**
123 * struct mmc_test_dbgfs_file - debugfs related file.
124 * @link: double-linked list
125 * @card: card under test
126 * @file: file created under debugfs
127 */
128struct mmc_test_dbgfs_file {
129 struct list_head link;
130 struct mmc_card *card;
131 struct dentry *file;
132};
133
134/**
135 * struct mmc_test_card - test information.
136 * @card: card under test
137 * @scratch: transfer buffer
138 * @buffer: transfer buffer
139 * @highmem: buffer for highmem tests
140 * @area: information for performance tests
141 * @gr: pointer to results of current testcase
142 */
143struct mmc_test_card {
144 struct mmc_card *card;
145
146 u8 scratch[BUFFER_SIZE];
147 u8 *buffer;
148#ifdef CONFIG_HIGHMEM
149 struct page *highmem;
150#endif
151 struct mmc_test_area area;
152 struct mmc_test_general_result *gr;
153};
154
155enum mmc_test_prep_media {
156 MMC_TEST_PREP_NONE = 0,
157 MMC_TEST_PREP_WRITE_FULL = 1 << 0,
158 MMC_TEST_PREP_ERASE = 1 << 1,
159};
160
161struct mmc_test_multiple_rw {
162 unsigned int *sg_len;
163 unsigned int *bs;
164 unsigned int len;
165 unsigned int size;
166 bool do_write;
167 bool do_nonblock_req;
168 enum mmc_test_prep_media prepare;
169};
170
171/*******************************************************************/
172/* General helper functions */
173/*******************************************************************/
174
175/*
176 * Configure correct block size in card
177 */
178static int mmc_test_set_blksize(struct mmc_test_card *test, unsigned size)
179{
180 return mmc_set_blocklen(test->card, size);
181}
182
183static bool mmc_test_card_cmd23(struct mmc_card *card)
184{
185 return mmc_card_mmc(card) ||
186 (mmc_card_sd(card) && card->scr.cmds & SD_SCR_CMD23_SUPPORT);
187}
188
189static void mmc_test_prepare_sbc(struct mmc_test_card *test,
190 struct mmc_request *mrq, unsigned int blocks)
191{
192 struct mmc_card *card = test->card;
193
194 if (!mrq->sbc || !mmc_host_cmd23(card->host) ||
195 !mmc_test_card_cmd23(card) || !mmc_op_multi(mrq->cmd->opcode) ||
196 (card->quirks & MMC_QUIRK_BLK_NO_CMD23)) {
197 mrq->sbc = NULL;
198 return;
199 }
200
201 mrq->sbc->opcode = MMC_SET_BLOCK_COUNT;
202 mrq->sbc->arg = blocks;
203 mrq->sbc->flags = MMC_RSP_R1 | MMC_CMD_AC;
204}
205
206/*
207 * Fill in the mmc_request structure given a set of transfer parameters.
208 */
209static void mmc_test_prepare_mrq(struct mmc_test_card *test,
210 struct mmc_request *mrq, struct scatterlist *sg, unsigned sg_len,
211 unsigned dev_addr, unsigned blocks, unsigned blksz, int write)
212{
213 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data || !mrq->stop))
214 return;
215
216 if (blocks > 1) {
217 mrq->cmd->opcode = write ?
218 MMC_WRITE_MULTIPLE_BLOCK : MMC_READ_MULTIPLE_BLOCK;
219 } else {
220 mrq->cmd->opcode = write ?
221 MMC_WRITE_BLOCK : MMC_READ_SINGLE_BLOCK;
222 }
223
224 mrq->cmd->arg = dev_addr;
225 if (!mmc_card_blockaddr(test->card))
226 mrq->cmd->arg <<= 9;
227
228 mrq->cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC;
229
230 if (blocks == 1)
231 mrq->stop = NULL;
232 else {
233 mrq->stop->opcode = MMC_STOP_TRANSMISSION;
234 mrq->stop->arg = 0;
235 mrq->stop->flags = MMC_RSP_R1B | MMC_CMD_AC;
236 }
237
238 mrq->data->blksz = blksz;
239 mrq->data->blocks = blocks;
240 mrq->data->flags = write ? MMC_DATA_WRITE : MMC_DATA_READ;
241 mrq->data->sg = sg;
242 mrq->data->sg_len = sg_len;
243
244 mmc_test_prepare_sbc(test, mrq, blocks);
245
246 mmc_set_data_timeout(mrq->data, test->card);
247}
248
249static int mmc_test_busy(struct mmc_command *cmd)
250{
251 return !(cmd->resp[0] & R1_READY_FOR_DATA) ||
252 (R1_CURRENT_STATE(cmd->resp[0]) == R1_STATE_PRG);
253}
254
255/*
256 * Wait for the card to finish the busy state
257 */
258static int mmc_test_wait_busy(struct mmc_test_card *test)
259{
260 int ret, busy;
261 struct mmc_command cmd = {};
262
263 busy = 0;
264 do {
265 memset(&cmd, 0, sizeof(struct mmc_command));
266
267 cmd.opcode = MMC_SEND_STATUS;
268 cmd.arg = test->card->rca << 16;
269 cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
270
271 ret = mmc_wait_for_cmd(test->card->host, &cmd, 0);
272 if (ret)
273 break;
274
275 if (!busy && mmc_test_busy(&cmd)) {
276 busy = 1;
277 if (test->card->host->caps & MMC_CAP_WAIT_WHILE_BUSY)
278 pr_info("%s: Warning: Host did not wait for busy state to end.\n",
279 mmc_hostname(test->card->host));
280 }
281 } while (mmc_test_busy(&cmd));
282
283 return ret;
284}
285
286/*
287 * Transfer a single sector of kernel addressable data
288 */
289static int mmc_test_buffer_transfer(struct mmc_test_card *test,
290 u8 *buffer, unsigned addr, unsigned blksz, int write)
291{
292 struct mmc_request mrq = {};
293 struct mmc_command cmd = {};
294 struct mmc_command stop = {};
295 struct mmc_data data = {};
296
297 struct scatterlist sg;
298
299 mrq.cmd = &cmd;
300 mrq.data = &data;
301 mrq.stop = &stop;
302
303 sg_init_one(&sg, buffer, blksz);
304
305 mmc_test_prepare_mrq(test, &mrq, &sg, 1, addr, 1, blksz, write);
306
307 mmc_wait_for_req(test->card->host, &mrq);
308
309 if (cmd.error)
310 return cmd.error;
311 if (data.error)
312 return data.error;
313
314 return mmc_test_wait_busy(test);
315}
316
317static void mmc_test_free_mem(struct mmc_test_mem *mem)
318{
319 if (!mem)
320 return;
321 while (mem->cnt--)
322 __free_pages(mem->arr[mem->cnt].page,
323 mem->arr[mem->cnt].order);
324 kfree(mem->arr);
325 kfree(mem);
326}
327
328/*
329 * Allocate a lot of memory, preferably max_sz but at least min_sz. In case
330 * there isn't much memory do not exceed 1/16th total lowmem pages. Also do
331 * not exceed a maximum number of segments and try not to make segments much
332 * bigger than maximum segment size.
333 */
334static struct mmc_test_mem *mmc_test_alloc_mem(unsigned long min_sz,
335 unsigned long max_sz,
336 unsigned int max_segs,
337 unsigned int max_seg_sz)
338{
339 unsigned long max_page_cnt = DIV_ROUND_UP(max_sz, PAGE_SIZE);
340 unsigned long min_page_cnt = DIV_ROUND_UP(min_sz, PAGE_SIZE);
341 unsigned long max_seg_page_cnt = DIV_ROUND_UP(max_seg_sz, PAGE_SIZE);
342 unsigned long page_cnt = 0;
343 unsigned long limit = nr_free_buffer_pages() >> 4;
344 struct mmc_test_mem *mem;
345
346 if (max_page_cnt > limit)
347 max_page_cnt = limit;
348 if (min_page_cnt > max_page_cnt)
349 min_page_cnt = max_page_cnt;
350
351 if (max_seg_page_cnt > max_page_cnt)
352 max_seg_page_cnt = max_page_cnt;
353
354 if (max_segs > max_page_cnt)
355 max_segs = max_page_cnt;
356
357 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
358 if (!mem)
359 return NULL;
360
361 mem->arr = kcalloc(max_segs, sizeof(*mem->arr), GFP_KERNEL);
362 if (!mem->arr)
363 goto out_free;
364
365 while (max_page_cnt) {
366 struct page *page;
367 unsigned int order;
368 gfp_t flags = GFP_KERNEL | GFP_DMA | __GFP_NOWARN |
369 __GFP_NORETRY;
370
371 order = get_order(max_seg_page_cnt << PAGE_SHIFT);
372 while (1) {
373 page = alloc_pages(flags, order);
374 if (page || !order)
375 break;
376 order -= 1;
377 }
378 if (!page) {
379 if (page_cnt < min_page_cnt)
380 goto out_free;
381 break;
382 }
383 mem->arr[mem->cnt].page = page;
384 mem->arr[mem->cnt].order = order;
385 mem->cnt += 1;
386 if (max_page_cnt <= (1UL << order))
387 break;
388 max_page_cnt -= 1UL << order;
389 page_cnt += 1UL << order;
390 if (mem->cnt >= max_segs) {
391 if (page_cnt < min_page_cnt)
392 goto out_free;
393 break;
394 }
395 }
396
397 return mem;
398
399out_free:
400 mmc_test_free_mem(mem);
401 return NULL;
402}
403
404/*
405 * Map memory into a scatterlist. Optionally allow the same memory to be
406 * mapped more than once.
407 */
408static int mmc_test_map_sg(struct mmc_test_mem *mem, unsigned long size,
409 struct scatterlist *sglist, int repeat,
410 unsigned int max_segs, unsigned int max_seg_sz,
411 unsigned int *sg_len, int min_sg_len)
412{
413 struct scatterlist *sg = NULL;
414 unsigned int i;
415 unsigned long sz = size;
416
417 sg_init_table(sglist, max_segs);
418 if (min_sg_len > max_segs)
419 min_sg_len = max_segs;
420
421 *sg_len = 0;
422 do {
423 for (i = 0; i < mem->cnt; i++) {
424 unsigned long len = PAGE_SIZE << mem->arr[i].order;
425
426 if (min_sg_len && (size / min_sg_len < len))
427 len = ALIGN(size / min_sg_len, 512);
428 if (len > sz)
429 len = sz;
430 if (len > max_seg_sz)
431 len = max_seg_sz;
432 if (sg)
433 sg = sg_next(sg);
434 else
435 sg = sglist;
436 if (!sg)
437 return -EINVAL;
438 sg_set_page(sg, mem->arr[i].page, len, 0);
439 sz -= len;
440 *sg_len += 1;
441 if (!sz)
442 break;
443 }
444 } while (sz && repeat);
445
446 if (sz)
447 return -EINVAL;
448
449 if (sg)
450 sg_mark_end(sg);
451
452 return 0;
453}
454
455/*
456 * Map memory into a scatterlist so that no pages are contiguous. Allow the
457 * same memory to be mapped more than once.
458 */
459static int mmc_test_map_sg_max_scatter(struct mmc_test_mem *mem,
460 unsigned long sz,
461 struct scatterlist *sglist,
462 unsigned int max_segs,
463 unsigned int max_seg_sz,
464 unsigned int *sg_len)
465{
466 struct scatterlist *sg = NULL;
467 unsigned int i = mem->cnt, cnt;
468 unsigned long len;
469 void *base, *addr, *last_addr = NULL;
470
471 sg_init_table(sglist, max_segs);
472
473 *sg_len = 0;
474 while (sz) {
475 base = page_address(mem->arr[--i].page);
476 cnt = 1 << mem->arr[i].order;
477 while (sz && cnt) {
478 addr = base + PAGE_SIZE * --cnt;
479 if (last_addr && last_addr + PAGE_SIZE == addr)
480 continue;
481 last_addr = addr;
482 len = PAGE_SIZE;
483 if (len > max_seg_sz)
484 len = max_seg_sz;
485 if (len > sz)
486 len = sz;
487 if (sg)
488 sg = sg_next(sg);
489 else
490 sg = sglist;
491 if (!sg)
492 return -EINVAL;
493 sg_set_page(sg, virt_to_page(addr), len, 0);
494 sz -= len;
495 *sg_len += 1;
496 }
497 if (i == 0)
498 i = mem->cnt;
499 }
500
501 if (sg)
502 sg_mark_end(sg);
503
504 return 0;
505}
506
507/*
508 * Calculate transfer rate in bytes per second.
509 */
510static unsigned int mmc_test_rate(uint64_t bytes, struct timespec64 *ts)
511{
512 uint64_t ns;
513
514 ns = timespec64_to_ns(ts);
515 bytes *= 1000000000;
516
517 while (ns > UINT_MAX) {
518 bytes >>= 1;
519 ns >>= 1;
520 }
521
522 if (!ns)
523 return 0;
524
525 do_div(bytes, (uint32_t)ns);
526
527 return bytes;
528}
529
530/*
531 * Save transfer results for future usage
532 */
533static void mmc_test_save_transfer_result(struct mmc_test_card *test,
534 unsigned int count, unsigned int sectors, struct timespec64 ts,
535 unsigned int rate, unsigned int iops)
536{
537 struct mmc_test_transfer_result *tr;
538
539 if (!test->gr)
540 return;
541
542 tr = kmalloc(sizeof(*tr), GFP_KERNEL);
543 if (!tr)
544 return;
545
546 tr->count = count;
547 tr->sectors = sectors;
548 tr->ts = ts;
549 tr->rate = rate;
550 tr->iops = iops;
551
552 list_add_tail(&tr->link, &test->gr->tr_lst);
553}
554
555/*
556 * Print the transfer rate.
557 */
558static void mmc_test_print_rate(struct mmc_test_card *test, uint64_t bytes,
559 struct timespec64 *ts1, struct timespec64 *ts2)
560{
561 unsigned int rate, iops, sectors = bytes >> 9;
562 struct timespec64 ts;
563
564 ts = timespec64_sub(*ts2, *ts1);
565
566 rate = mmc_test_rate(bytes, &ts);
567 iops = mmc_test_rate(100, &ts); /* I/O ops per sec x 100 */
568
569 pr_info("%s: Transfer of %u sectors (%u%s KiB) took %llu.%09u "
570 "seconds (%u kB/s, %u KiB/s, %u.%02u IOPS)\n",
571 mmc_hostname(test->card->host), sectors, sectors >> 1,
572 (sectors & 1 ? ".5" : ""), (u64)ts.tv_sec,
573 (u32)ts.tv_nsec, rate / 1000, rate / 1024,
574 iops / 100, iops % 100);
575
576 mmc_test_save_transfer_result(test, 1, sectors, ts, rate, iops);
577}
578
579/*
580 * Print the average transfer rate.
581 */
582static void mmc_test_print_avg_rate(struct mmc_test_card *test, uint64_t bytes,
583 unsigned int count, struct timespec64 *ts1,
584 struct timespec64 *ts2)
585{
586 unsigned int rate, iops, sectors = bytes >> 9;
587 uint64_t tot = bytes * count;
588 struct timespec64 ts;
589
590 ts = timespec64_sub(*ts2, *ts1);
591
592 rate = mmc_test_rate(tot, &ts);
593 iops = mmc_test_rate(count * 100, &ts); /* I/O ops per sec x 100 */
594
595 pr_info("%s: Transfer of %u x %u sectors (%u x %u%s KiB) took "
596 "%llu.%09u seconds (%u kB/s, %u KiB/s, "
597 "%u.%02u IOPS, sg_len %d)\n",
598 mmc_hostname(test->card->host), count, sectors, count,
599 sectors >> 1, (sectors & 1 ? ".5" : ""),
600 (u64)ts.tv_sec, (u32)ts.tv_nsec,
601 rate / 1000, rate / 1024, iops / 100, iops % 100,
602 test->area.sg_len);
603
604 mmc_test_save_transfer_result(test, count, sectors, ts, rate, iops);
605}
606
607/*
608 * Return the card size in sectors.
609 */
610static unsigned int mmc_test_capacity(struct mmc_card *card)
611{
612 if (!mmc_card_sd(card) && mmc_card_blockaddr(card))
613 return card->ext_csd.sectors;
614 else
615 return card->csd.capacity << (card->csd.read_blkbits - 9);
616}
617
618/*******************************************************************/
619/* Test preparation and cleanup */
620/*******************************************************************/
621
622/*
623 * Fill the first couple of sectors of the card with known data
624 * so that bad reads/writes can be detected
625 */
626static int __mmc_test_prepare(struct mmc_test_card *test, int write, int val)
627{
628 int ret, i;
629
630 ret = mmc_test_set_blksize(test, 512);
631 if (ret)
632 return ret;
633
634 if (write)
635 memset(test->buffer, val, 512);
636 else {
637 for (i = 0; i < 512; i++)
638 test->buffer[i] = i;
639 }
640
641 for (i = 0; i < BUFFER_SIZE / 512; i++) {
642 ret = mmc_test_buffer_transfer(test, test->buffer, i, 512, 1);
643 if (ret)
644 return ret;
645 }
646
647 return 0;
648}
649
650static int mmc_test_prepare_write(struct mmc_test_card *test)
651{
652 return __mmc_test_prepare(test, 1, 0xDF);
653}
654
655static int mmc_test_prepare_read(struct mmc_test_card *test)
656{
657 return __mmc_test_prepare(test, 0, 0);
658}
659
660static int mmc_test_cleanup(struct mmc_test_card *test)
661{
662 return __mmc_test_prepare(test, 1, 0);
663}
664
665/*******************************************************************/
666/* Test execution helpers */
667/*******************************************************************/
668
669/*
670 * Modifies the mmc_request to perform the "short transfer" tests
671 */
672static void mmc_test_prepare_broken_mrq(struct mmc_test_card *test,
673 struct mmc_request *mrq, int write)
674{
675 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
676 return;
677
678 if (mrq->data->blocks > 1) {
679 mrq->cmd->opcode = write ?
680 MMC_WRITE_BLOCK : MMC_READ_SINGLE_BLOCK;
681 mrq->stop = NULL;
682 } else {
683 mrq->cmd->opcode = MMC_SEND_STATUS;
684 mrq->cmd->arg = test->card->rca << 16;
685 }
686}
687
688/*
689 * Checks that a normal transfer didn't have any errors
690 */
691static int mmc_test_check_result(struct mmc_test_card *test,
692 struct mmc_request *mrq)
693{
694 int ret;
695
696 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
697 return -EINVAL;
698
699 ret = 0;
700
701 if (mrq->sbc && mrq->sbc->error)
702 ret = mrq->sbc->error;
703 if (!ret && mrq->cmd->error)
704 ret = mrq->cmd->error;
705 if (!ret && mrq->data->error)
706 ret = mrq->data->error;
707 if (!ret && mrq->stop && mrq->stop->error)
708 ret = mrq->stop->error;
709 if (!ret && mrq->data->bytes_xfered !=
710 mrq->data->blocks * mrq->data->blksz)
711 ret = RESULT_FAIL;
712
713 if (ret == -EINVAL)
714 ret = RESULT_UNSUP_HOST;
715
716 return ret;
717}
718
719/*
720 * Checks that a "short transfer" behaved as expected
721 */
722static int mmc_test_check_broken_result(struct mmc_test_card *test,
723 struct mmc_request *mrq)
724{
725 int ret;
726
727 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
728 return -EINVAL;
729
730 ret = 0;
731
732 if (!ret && mrq->cmd->error)
733 ret = mrq->cmd->error;
734 if (!ret && mrq->data->error == 0)
735 ret = RESULT_FAIL;
736 if (!ret && mrq->data->error != -ETIMEDOUT)
737 ret = mrq->data->error;
738 if (!ret && mrq->stop && mrq->stop->error)
739 ret = mrq->stop->error;
740 if (mrq->data->blocks > 1) {
741 if (!ret && mrq->data->bytes_xfered > mrq->data->blksz)
742 ret = RESULT_FAIL;
743 } else {
744 if (!ret && mrq->data->bytes_xfered > 0)
745 ret = RESULT_FAIL;
746 }
747
748 if (ret == -EINVAL)
749 ret = RESULT_UNSUP_HOST;
750
751 return ret;
752}
753
754struct mmc_test_req {
755 struct mmc_request mrq;
756 struct mmc_command sbc;
757 struct mmc_command cmd;
758 struct mmc_command stop;
759 struct mmc_command status;
760 struct mmc_data data;
761};
762
763/*
764 * Tests nonblock transfer with certain parameters
765 */
766static void mmc_test_req_reset(struct mmc_test_req *rq)
767{
768 memset(rq, 0, sizeof(struct mmc_test_req));
769
770 rq->mrq.cmd = &rq->cmd;
771 rq->mrq.data = &rq->data;
772 rq->mrq.stop = &rq->stop;
773}
774
775static struct mmc_test_req *mmc_test_req_alloc(void)
776{
777 struct mmc_test_req *rq = kmalloc(sizeof(*rq), GFP_KERNEL);
778
779 if (rq)
780 mmc_test_req_reset(rq);
781
782 return rq;
783}
784
785static void mmc_test_wait_done(struct mmc_request *mrq)
786{
787 complete(&mrq->completion);
788}
789
790static int mmc_test_start_areq(struct mmc_test_card *test,
791 struct mmc_request *mrq,
792 struct mmc_request *prev_mrq)
793{
794 struct mmc_host *host = test->card->host;
795 int err = 0;
796
797 if (mrq) {
798 init_completion(&mrq->completion);
799 mrq->done = mmc_test_wait_done;
800 mmc_pre_req(host, mrq);
801 }
802
803 if (prev_mrq) {
804 wait_for_completion(&prev_mrq->completion);
805 err = mmc_test_wait_busy(test);
806 if (!err)
807 err = mmc_test_check_result(test, prev_mrq);
808 }
809
810 if (!err && mrq) {
811 err = mmc_start_request(host, mrq);
812 if (err)
813 mmc_retune_release(host);
814 }
815
816 if (prev_mrq)
817 mmc_post_req(host, prev_mrq, 0);
818
819 if (err && mrq)
820 mmc_post_req(host, mrq, err);
821
822 return err;
823}
824
825static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
826 unsigned int dev_addr, int write,
827 int count)
828{
829 struct mmc_test_req *rq1, *rq2;
830 struct mmc_request *mrq, *prev_mrq;
831 int i;
832 int ret = RESULT_OK;
833 struct mmc_test_area *t = &test->area;
834 struct scatterlist *sg = t->sg;
835 struct scatterlist *sg_areq = t->sg_areq;
836
837 rq1 = mmc_test_req_alloc();
838 rq2 = mmc_test_req_alloc();
839 if (!rq1 || !rq2) {
840 ret = RESULT_FAIL;
841 goto err;
842 }
843
844 mrq = &rq1->mrq;
845 prev_mrq = NULL;
846
847 for (i = 0; i < count; i++) {
848 mmc_test_req_reset(container_of(mrq, struct mmc_test_req, mrq));
849 mmc_test_prepare_mrq(test, mrq, sg, t->sg_len, dev_addr,
850 t->blocks, 512, write);
851 ret = mmc_test_start_areq(test, mrq, prev_mrq);
852 if (ret)
853 goto err;
854
855 if (!prev_mrq)
856 prev_mrq = &rq2->mrq;
857
858 swap(mrq, prev_mrq);
859 swap(sg, sg_areq);
860 dev_addr += t->blocks;
861 }
862
863 ret = mmc_test_start_areq(test, NULL, prev_mrq);
864err:
865 kfree(rq1);
866 kfree(rq2);
867 return ret;
868}
869
870/*
871 * Tests a basic transfer with certain parameters
872 */
873static int mmc_test_simple_transfer(struct mmc_test_card *test,
874 struct scatterlist *sg, unsigned sg_len, unsigned dev_addr,
875 unsigned blocks, unsigned blksz, int write)
876{
877 struct mmc_request mrq = {};
878 struct mmc_command cmd = {};
879 struct mmc_command stop = {};
880 struct mmc_data data = {};
881
882 mrq.cmd = &cmd;
883 mrq.data = &data;
884 mrq.stop = &stop;
885
886 mmc_test_prepare_mrq(test, &mrq, sg, sg_len, dev_addr,
887 blocks, blksz, write);
888
889 mmc_wait_for_req(test->card->host, &mrq);
890
891 mmc_test_wait_busy(test);
892
893 return mmc_test_check_result(test, &mrq);
894}
895
896/*
897 * Tests a transfer where the card will fail completely or partly
898 */
899static int mmc_test_broken_transfer(struct mmc_test_card *test,
900 unsigned blocks, unsigned blksz, int write)
901{
902 struct mmc_request mrq = {};
903 struct mmc_command cmd = {};
904 struct mmc_command stop = {};
905 struct mmc_data data = {};
906
907 struct scatterlist sg;
908
909 mrq.cmd = &cmd;
910 mrq.data = &data;
911 mrq.stop = &stop;
912
913 sg_init_one(&sg, test->buffer, blocks * blksz);
914
915 mmc_test_prepare_mrq(test, &mrq, &sg, 1, 0, blocks, blksz, write);
916 mmc_test_prepare_broken_mrq(test, &mrq, write);
917
918 mmc_wait_for_req(test->card->host, &mrq);
919
920 mmc_test_wait_busy(test);
921
922 return mmc_test_check_broken_result(test, &mrq);
923}
924
925/*
926 * Does a complete transfer test where data is also validated
927 *
928 * Note: mmc_test_prepare() must have been done before this call
929 */
930static int mmc_test_transfer(struct mmc_test_card *test,
931 struct scatterlist *sg, unsigned sg_len, unsigned dev_addr,
932 unsigned blocks, unsigned blksz, int write)
933{
934 int ret, i;
935
936 if (write) {
937 for (i = 0; i < blocks * blksz; i++)
938 test->scratch[i] = i;
939 } else {
940 memset(test->scratch, 0, BUFFER_SIZE);
941 }
942 sg_copy_from_buffer(sg, sg_len, test->scratch, BUFFER_SIZE);
943
944 ret = mmc_test_set_blksize(test, blksz);
945 if (ret)
946 return ret;
947
948 ret = mmc_test_simple_transfer(test, sg, sg_len, dev_addr,
949 blocks, blksz, write);
950 if (ret)
951 return ret;
952
953 if (write) {
954 int sectors;
955
956 ret = mmc_test_set_blksize(test, 512);
957 if (ret)
958 return ret;
959
960 sectors = (blocks * blksz + 511) / 512;
961 if ((sectors * 512) == (blocks * blksz))
962 sectors++;
963
964 if ((sectors * 512) > BUFFER_SIZE)
965 return -EINVAL;
966
967 memset(test->buffer, 0, sectors * 512);
968
969 for (i = 0; i < sectors; i++) {
970 ret = mmc_test_buffer_transfer(test,
971 test->buffer + i * 512,
972 dev_addr + i, 512, 0);
973 if (ret)
974 return ret;
975 }
976
977 for (i = 0; i < blocks * blksz; i++) {
978 if (test->buffer[i] != (u8)i)
979 return RESULT_FAIL;
980 }
981
982 for (; i < sectors * 512; i++) {
983 if (test->buffer[i] != 0xDF)
984 return RESULT_FAIL;
985 }
986 } else {
987 sg_copy_to_buffer(sg, sg_len, test->scratch, BUFFER_SIZE);
988 for (i = 0; i < blocks * blksz; i++) {
989 if (test->scratch[i] != (u8)i)
990 return RESULT_FAIL;
991 }
992 }
993
994 return 0;
995}
996
997/*******************************************************************/
998/* Tests */
999/*******************************************************************/
1000
1001struct mmc_test_case {
1002 const char *name;
1003
1004 int (*prepare)(struct mmc_test_card *);
1005 int (*run)(struct mmc_test_card *);
1006 int (*cleanup)(struct mmc_test_card *);
1007};
1008
1009static int mmc_test_basic_write(struct mmc_test_card *test)
1010{
1011 int ret;
1012 struct scatterlist sg;
1013
1014 ret = mmc_test_set_blksize(test, 512);
1015 if (ret)
1016 return ret;
1017
1018 sg_init_one(&sg, test->buffer, 512);
1019
1020 return mmc_test_simple_transfer(test, &sg, 1, 0, 1, 512, 1);
1021}
1022
1023static int mmc_test_basic_read(struct mmc_test_card *test)
1024{
1025 int ret;
1026 struct scatterlist sg;
1027
1028 ret = mmc_test_set_blksize(test, 512);
1029 if (ret)
1030 return ret;
1031
1032 sg_init_one(&sg, test->buffer, 512);
1033
1034 return mmc_test_simple_transfer(test, &sg, 1, 0, 1, 512, 0);
1035}
1036
1037static int mmc_test_verify_write(struct mmc_test_card *test)
1038{
1039 struct scatterlist sg;
1040
1041 sg_init_one(&sg, test->buffer, 512);
1042
1043 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 1);
1044}
1045
1046static int mmc_test_verify_read(struct mmc_test_card *test)
1047{
1048 struct scatterlist sg;
1049
1050 sg_init_one(&sg, test->buffer, 512);
1051
1052 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 0);
1053}
1054
1055static int mmc_test_multi_write(struct mmc_test_card *test)
1056{
1057 unsigned int size;
1058 struct scatterlist sg;
1059
1060 if (test->card->host->max_blk_count == 1)
1061 return RESULT_UNSUP_HOST;
1062
1063 size = PAGE_SIZE * 2;
1064 size = min(size, test->card->host->max_req_size);
1065 size = min(size, test->card->host->max_seg_size);
1066 size = min(size, test->card->host->max_blk_count * 512);
1067
1068 if (size < 1024)
1069 return RESULT_UNSUP_HOST;
1070
1071 sg_init_one(&sg, test->buffer, size);
1072
1073 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 1);
1074}
1075
1076static int mmc_test_multi_read(struct mmc_test_card *test)
1077{
1078 unsigned int size;
1079 struct scatterlist sg;
1080
1081 if (test->card->host->max_blk_count == 1)
1082 return RESULT_UNSUP_HOST;
1083
1084 size = PAGE_SIZE * 2;
1085 size = min(size, test->card->host->max_req_size);
1086 size = min(size, test->card->host->max_seg_size);
1087 size = min(size, test->card->host->max_blk_count * 512);
1088
1089 if (size < 1024)
1090 return RESULT_UNSUP_HOST;
1091
1092 sg_init_one(&sg, test->buffer, size);
1093
1094 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 0);
1095}
1096
1097static int mmc_test_pow2_write(struct mmc_test_card *test)
1098{
1099 int ret, i;
1100 struct scatterlist sg;
1101
1102 if (!test->card->csd.write_partial)
1103 return RESULT_UNSUP_CARD;
1104
1105 for (i = 1; i < 512; i <<= 1) {
1106 sg_init_one(&sg, test->buffer, i);
1107 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 1);
1108 if (ret)
1109 return ret;
1110 }
1111
1112 return 0;
1113}
1114
1115static int mmc_test_pow2_read(struct mmc_test_card *test)
1116{
1117 int ret, i;
1118 struct scatterlist sg;
1119
1120 if (!test->card->csd.read_partial)
1121 return RESULT_UNSUP_CARD;
1122
1123 for (i = 1; i < 512; i <<= 1) {
1124 sg_init_one(&sg, test->buffer, i);
1125 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 0);
1126 if (ret)
1127 return ret;
1128 }
1129
1130 return 0;
1131}
1132
1133static int mmc_test_weird_write(struct mmc_test_card *test)
1134{
1135 int ret, i;
1136 struct scatterlist sg;
1137
1138 if (!test->card->csd.write_partial)
1139 return RESULT_UNSUP_CARD;
1140
1141 for (i = 3; i < 512; i += 7) {
1142 sg_init_one(&sg, test->buffer, i);
1143 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 1);
1144 if (ret)
1145 return ret;
1146 }
1147
1148 return 0;
1149}
1150
1151static int mmc_test_weird_read(struct mmc_test_card *test)
1152{
1153 int ret, i;
1154 struct scatterlist sg;
1155
1156 if (!test->card->csd.read_partial)
1157 return RESULT_UNSUP_CARD;
1158
1159 for (i = 3; i < 512; i += 7) {
1160 sg_init_one(&sg, test->buffer, i);
1161 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 0);
1162 if (ret)
1163 return ret;
1164 }
1165
1166 return 0;
1167}
1168
1169static int mmc_test_align_write(struct mmc_test_card *test)
1170{
1171 int ret, i;
1172 struct scatterlist sg;
1173
1174 for (i = 1; i < TEST_ALIGN_END; i++) {
1175 sg_init_one(&sg, test->buffer + i, 512);
1176 ret = mmc_test_transfer(test, &sg, 1, 0, 1, 512, 1);
1177 if (ret)
1178 return ret;
1179 }
1180
1181 return 0;
1182}
1183
1184static int mmc_test_align_read(struct mmc_test_card *test)
1185{
1186 int ret, i;
1187 struct scatterlist sg;
1188
1189 for (i = 1; i < TEST_ALIGN_END; i++) {
1190 sg_init_one(&sg, test->buffer + i, 512);
1191 ret = mmc_test_transfer(test, &sg, 1, 0, 1, 512, 0);
1192 if (ret)
1193 return ret;
1194 }
1195
1196 return 0;
1197}
1198
1199static int mmc_test_align_multi_write(struct mmc_test_card *test)
1200{
1201 int ret, i;
1202 unsigned int size;
1203 struct scatterlist sg;
1204
1205 if (test->card->host->max_blk_count == 1)
1206 return RESULT_UNSUP_HOST;
1207
1208 size = PAGE_SIZE * 2;
1209 size = min(size, test->card->host->max_req_size);
1210 size = min(size, test->card->host->max_seg_size);
1211 size = min(size, test->card->host->max_blk_count * 512);
1212
1213 if (size < 1024)
1214 return RESULT_UNSUP_HOST;
1215
1216 for (i = 1; i < TEST_ALIGN_END; i++) {
1217 sg_init_one(&sg, test->buffer + i, size);
1218 ret = mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 1);
1219 if (ret)
1220 return ret;
1221 }
1222
1223 return 0;
1224}
1225
1226static int mmc_test_align_multi_read(struct mmc_test_card *test)
1227{
1228 int ret, i;
1229 unsigned int size;
1230 struct scatterlist sg;
1231
1232 if (test->card->host->max_blk_count == 1)
1233 return RESULT_UNSUP_HOST;
1234
1235 size = PAGE_SIZE * 2;
1236 size = min(size, test->card->host->max_req_size);
1237 size = min(size, test->card->host->max_seg_size);
1238 size = min(size, test->card->host->max_blk_count * 512);
1239
1240 if (size < 1024)
1241 return RESULT_UNSUP_HOST;
1242
1243 for (i = 1; i < TEST_ALIGN_END; i++) {
1244 sg_init_one(&sg, test->buffer + i, size);
1245 ret = mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 0);
1246 if (ret)
1247 return ret;
1248 }
1249
1250 return 0;
1251}
1252
1253static int mmc_test_xfersize_write(struct mmc_test_card *test)
1254{
1255 int ret;
1256
1257 ret = mmc_test_set_blksize(test, 512);
1258 if (ret)
1259 return ret;
1260
1261 return mmc_test_broken_transfer(test, 1, 512, 1);
1262}
1263
1264static int mmc_test_xfersize_read(struct mmc_test_card *test)
1265{
1266 int ret;
1267
1268 ret = mmc_test_set_blksize(test, 512);
1269 if (ret)
1270 return ret;
1271
1272 return mmc_test_broken_transfer(test, 1, 512, 0);
1273}
1274
1275static int mmc_test_multi_xfersize_write(struct mmc_test_card *test)
1276{
1277 int ret;
1278
1279 if (test->card->host->max_blk_count == 1)
1280 return RESULT_UNSUP_HOST;
1281
1282 ret = mmc_test_set_blksize(test, 512);
1283 if (ret)
1284 return ret;
1285
1286 return mmc_test_broken_transfer(test, 2, 512, 1);
1287}
1288
1289static int mmc_test_multi_xfersize_read(struct mmc_test_card *test)
1290{
1291 int ret;
1292
1293 if (test->card->host->max_blk_count == 1)
1294 return RESULT_UNSUP_HOST;
1295
1296 ret = mmc_test_set_blksize(test, 512);
1297 if (ret)
1298 return ret;
1299
1300 return mmc_test_broken_transfer(test, 2, 512, 0);
1301}
1302
1303#ifdef CONFIG_HIGHMEM
1304
1305static int mmc_test_write_high(struct mmc_test_card *test)
1306{
1307 struct scatterlist sg;
1308
1309 sg_init_table(&sg, 1);
1310 sg_set_page(&sg, test->highmem, 512, 0);
1311
1312 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 1);
1313}
1314
1315static int mmc_test_read_high(struct mmc_test_card *test)
1316{
1317 struct scatterlist sg;
1318
1319 sg_init_table(&sg, 1);
1320 sg_set_page(&sg, test->highmem, 512, 0);
1321
1322 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 0);
1323}
1324
1325static int mmc_test_multi_write_high(struct mmc_test_card *test)
1326{
1327 unsigned int size;
1328 struct scatterlist sg;
1329
1330 if (test->card->host->max_blk_count == 1)
1331 return RESULT_UNSUP_HOST;
1332
1333 size = PAGE_SIZE * 2;
1334 size = min(size, test->card->host->max_req_size);
1335 size = min(size, test->card->host->max_seg_size);
1336 size = min(size, test->card->host->max_blk_count * 512);
1337
1338 if (size < 1024)
1339 return RESULT_UNSUP_HOST;
1340
1341 sg_init_table(&sg, 1);
1342 sg_set_page(&sg, test->highmem, size, 0);
1343
1344 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 1);
1345}
1346
1347static int mmc_test_multi_read_high(struct mmc_test_card *test)
1348{
1349 unsigned int size;
1350 struct scatterlist sg;
1351
1352 if (test->card->host->max_blk_count == 1)
1353 return RESULT_UNSUP_HOST;
1354
1355 size = PAGE_SIZE * 2;
1356 size = min(size, test->card->host->max_req_size);
1357 size = min(size, test->card->host->max_seg_size);
1358 size = min(size, test->card->host->max_blk_count * 512);
1359
1360 if (size < 1024)
1361 return RESULT_UNSUP_HOST;
1362
1363 sg_init_table(&sg, 1);
1364 sg_set_page(&sg, test->highmem, size, 0);
1365
1366 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 0);
1367}
1368
1369#else
1370
1371static int mmc_test_no_highmem(struct mmc_test_card *test)
1372{
1373 pr_info("%s: Highmem not configured - test skipped\n",
1374 mmc_hostname(test->card->host));
1375 return 0;
1376}
1377
1378#endif /* CONFIG_HIGHMEM */
1379
1380/*
1381 * Map sz bytes so that it can be transferred.
1382 */
1383static int mmc_test_area_map(struct mmc_test_card *test, unsigned long sz,
1384 int max_scatter, int min_sg_len, bool nonblock)
1385{
1386 struct mmc_test_area *t = &test->area;
1387 int err;
1388 unsigned int sg_len = 0;
1389
1390 t->blocks = sz >> 9;
1391
1392 if (max_scatter) {
1393 err = mmc_test_map_sg_max_scatter(t->mem, sz, t->sg,
1394 t->max_segs, t->max_seg_sz,
1395 &t->sg_len);
1396 } else {
1397 err = mmc_test_map_sg(t->mem, sz, t->sg, 1, t->max_segs,
1398 t->max_seg_sz, &t->sg_len, min_sg_len);
1399 }
1400
1401 if (err || !nonblock)
1402 goto err;
1403
1404 if (max_scatter) {
1405 err = mmc_test_map_sg_max_scatter(t->mem, sz, t->sg_areq,
1406 t->max_segs, t->max_seg_sz,
1407 &sg_len);
1408 } else {
1409 err = mmc_test_map_sg(t->mem, sz, t->sg_areq, 1, t->max_segs,
1410 t->max_seg_sz, &sg_len, min_sg_len);
1411 }
1412 if (!err && sg_len != t->sg_len)
1413 err = -EINVAL;
1414
1415err:
1416 if (err)
1417 pr_info("%s: Failed to map sg list\n",
1418 mmc_hostname(test->card->host));
1419 return err;
1420}
1421
1422/*
1423 * Transfer bytes mapped by mmc_test_area_map().
1424 */
1425static int mmc_test_area_transfer(struct mmc_test_card *test,
1426 unsigned int dev_addr, int write)
1427{
1428 struct mmc_test_area *t = &test->area;
1429
1430 return mmc_test_simple_transfer(test, t->sg, t->sg_len, dev_addr,
1431 t->blocks, 512, write);
1432}
1433
1434/*
1435 * Map and transfer bytes for multiple transfers.
1436 */
1437static int mmc_test_area_io_seq(struct mmc_test_card *test, unsigned long sz,
1438 unsigned int dev_addr, int write,
1439 int max_scatter, int timed, int count,
1440 bool nonblock, int min_sg_len)
1441{
1442 struct timespec64 ts1, ts2;
1443 int ret = 0;
1444 int i;
1445
1446 /*
1447 * In the case of a maximally scattered transfer, the maximum transfer
1448 * size is further limited by using PAGE_SIZE segments.
1449 */
1450 if (max_scatter) {
1451 struct mmc_test_area *t = &test->area;
1452 unsigned long max_tfr;
1453
1454 if (t->max_seg_sz >= PAGE_SIZE)
1455 max_tfr = t->max_segs * PAGE_SIZE;
1456 else
1457 max_tfr = t->max_segs * t->max_seg_sz;
1458 if (sz > max_tfr)
1459 sz = max_tfr;
1460 }
1461
1462 ret = mmc_test_area_map(test, sz, max_scatter, min_sg_len, nonblock);
1463 if (ret)
1464 return ret;
1465
1466 if (timed)
1467 ktime_get_ts64(&ts1);
1468 if (nonblock)
1469 ret = mmc_test_nonblock_transfer(test, dev_addr, write, count);
1470 else
1471 for (i = 0; i < count && ret == 0; i++) {
1472 ret = mmc_test_area_transfer(test, dev_addr, write);
1473 dev_addr += sz >> 9;
1474 }
1475
1476 if (ret)
1477 return ret;
1478
1479 if (timed)
1480 ktime_get_ts64(&ts2);
1481
1482 if (timed)
1483 mmc_test_print_avg_rate(test, sz, count, &ts1, &ts2);
1484
1485 return 0;
1486}
1487
1488static int mmc_test_area_io(struct mmc_test_card *test, unsigned long sz,
1489 unsigned int dev_addr, int write, int max_scatter,
1490 int timed)
1491{
1492 return mmc_test_area_io_seq(test, sz, dev_addr, write, max_scatter,
1493 timed, 1, false, 0);
1494}
1495
1496/*
1497 * Write the test area entirely.
1498 */
1499static int mmc_test_area_fill(struct mmc_test_card *test)
1500{
1501 struct mmc_test_area *t = &test->area;
1502
1503 return mmc_test_area_io(test, t->max_tfr, t->dev_addr, 1, 0, 0);
1504}
1505
1506/*
1507 * Erase the test area entirely.
1508 */
1509static int mmc_test_area_erase(struct mmc_test_card *test)
1510{
1511 struct mmc_test_area *t = &test->area;
1512
1513 if (!mmc_can_erase(test->card))
1514 return 0;
1515
1516 return mmc_erase(test->card, t->dev_addr, t->max_sz >> 9,
1517 MMC_ERASE_ARG);
1518}
1519
1520/*
1521 * Cleanup struct mmc_test_area.
1522 */
1523static int mmc_test_area_cleanup(struct mmc_test_card *test)
1524{
1525 struct mmc_test_area *t = &test->area;
1526
1527 kfree(t->sg);
1528 kfree(t->sg_areq);
1529 mmc_test_free_mem(t->mem);
1530
1531 return 0;
1532}
1533
1534/*
1535 * Initialize an area for testing large transfers. The test area is set to the
1536 * middle of the card because cards may have different characteristics at the
1537 * front (for FAT file system optimization). Optionally, the area is erased
1538 * (if the card supports it) which may improve write performance. Optionally,
1539 * the area is filled with data for subsequent read tests.
1540 */
1541static int mmc_test_area_init(struct mmc_test_card *test, int erase, int fill)
1542{
1543 struct mmc_test_area *t = &test->area;
1544 unsigned long min_sz = 64 * 1024, sz;
1545 int ret;
1546
1547 ret = mmc_test_set_blksize(test, 512);
1548 if (ret)
1549 return ret;
1550
1551 /* Make the test area size about 4MiB */
1552 sz = (unsigned long)test->card->pref_erase << 9;
1553 t->max_sz = sz;
1554 while (t->max_sz < 4 * 1024 * 1024)
1555 t->max_sz += sz;
1556 while (t->max_sz > TEST_AREA_MAX_SIZE && t->max_sz > sz)
1557 t->max_sz -= sz;
1558
1559 t->max_segs = test->card->host->max_segs;
1560 t->max_seg_sz = test->card->host->max_seg_size;
1561 t->max_seg_sz -= t->max_seg_sz % 512;
1562
1563 t->max_tfr = t->max_sz;
1564 if (t->max_tfr >> 9 > test->card->host->max_blk_count)
1565 t->max_tfr = test->card->host->max_blk_count << 9;
1566 if (t->max_tfr > test->card->host->max_req_size)
1567 t->max_tfr = test->card->host->max_req_size;
1568 if (t->max_tfr / t->max_seg_sz > t->max_segs)
1569 t->max_tfr = t->max_segs * t->max_seg_sz;
1570
1571 /*
1572 * Try to allocate enough memory for a max. sized transfer. Less is OK
1573 * because the same memory can be mapped into the scatterlist more than
1574 * once. Also, take into account the limits imposed on scatterlist
1575 * segments by the host driver.
1576 */
1577 t->mem = mmc_test_alloc_mem(min_sz, t->max_tfr, t->max_segs,
1578 t->max_seg_sz);
1579 if (!t->mem)
1580 return -ENOMEM;
1581
1582 t->sg = kmalloc_array(t->max_segs, sizeof(*t->sg), GFP_KERNEL);
1583 if (!t->sg) {
1584 ret = -ENOMEM;
1585 goto out_free;
1586 }
1587
1588 t->sg_areq = kmalloc_array(t->max_segs, sizeof(*t->sg_areq),
1589 GFP_KERNEL);
1590 if (!t->sg_areq) {
1591 ret = -ENOMEM;
1592 goto out_free;
1593 }
1594
1595 t->dev_addr = mmc_test_capacity(test->card) / 2;
1596 t->dev_addr -= t->dev_addr % (t->max_sz >> 9);
1597
1598 if (erase) {
1599 ret = mmc_test_area_erase(test);
1600 if (ret)
1601 goto out_free;
1602 }
1603
1604 if (fill) {
1605 ret = mmc_test_area_fill(test);
1606 if (ret)
1607 goto out_free;
1608 }
1609
1610 return 0;
1611
1612out_free:
1613 mmc_test_area_cleanup(test);
1614 return ret;
1615}
1616
1617/*
1618 * Prepare for large transfers. Do not erase the test area.
1619 */
1620static int mmc_test_area_prepare(struct mmc_test_card *test)
1621{
1622 return mmc_test_area_init(test, 0, 0);
1623}
1624
1625/*
1626 * Prepare for large transfers. Do erase the test area.
1627 */
1628static int mmc_test_area_prepare_erase(struct mmc_test_card *test)
1629{
1630 return mmc_test_area_init(test, 1, 0);
1631}
1632
1633/*
1634 * Prepare for large transfers. Erase and fill the test area.
1635 */
1636static int mmc_test_area_prepare_fill(struct mmc_test_card *test)
1637{
1638 return mmc_test_area_init(test, 1, 1);
1639}
1640
1641/*
1642 * Test best-case performance. Best-case performance is expected from
1643 * a single large transfer.
1644 *
1645 * An additional option (max_scatter) allows the measurement of the same
1646 * transfer but with no contiguous pages in the scatter list. This tests
1647 * the efficiency of DMA to handle scattered pages.
1648 */
1649static int mmc_test_best_performance(struct mmc_test_card *test, int write,
1650 int max_scatter)
1651{
1652 struct mmc_test_area *t = &test->area;
1653
1654 return mmc_test_area_io(test, t->max_tfr, t->dev_addr, write,
1655 max_scatter, 1);
1656}
1657
1658/*
1659 * Best-case read performance.
1660 */
1661static int mmc_test_best_read_performance(struct mmc_test_card *test)
1662{
1663 return mmc_test_best_performance(test, 0, 0);
1664}
1665
1666/*
1667 * Best-case write performance.
1668 */
1669static int mmc_test_best_write_performance(struct mmc_test_card *test)
1670{
1671 return mmc_test_best_performance(test, 1, 0);
1672}
1673
1674/*
1675 * Best-case read performance into scattered pages.
1676 */
1677static int mmc_test_best_read_perf_max_scatter(struct mmc_test_card *test)
1678{
1679 return mmc_test_best_performance(test, 0, 1);
1680}
1681
1682/*
1683 * Best-case write performance from scattered pages.
1684 */
1685static int mmc_test_best_write_perf_max_scatter(struct mmc_test_card *test)
1686{
1687 return mmc_test_best_performance(test, 1, 1);
1688}
1689
1690/*
1691 * Single read performance by transfer size.
1692 */
1693static int mmc_test_profile_read_perf(struct mmc_test_card *test)
1694{
1695 struct mmc_test_area *t = &test->area;
1696 unsigned long sz;
1697 unsigned int dev_addr;
1698 int ret;
1699
1700 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1701 dev_addr = t->dev_addr + (sz >> 9);
1702 ret = mmc_test_area_io(test, sz, dev_addr, 0, 0, 1);
1703 if (ret)
1704 return ret;
1705 }
1706 sz = t->max_tfr;
1707 dev_addr = t->dev_addr;
1708 return mmc_test_area_io(test, sz, dev_addr, 0, 0, 1);
1709}
1710
1711/*
1712 * Single write performance by transfer size.
1713 */
1714static int mmc_test_profile_write_perf(struct mmc_test_card *test)
1715{
1716 struct mmc_test_area *t = &test->area;
1717 unsigned long sz;
1718 unsigned int dev_addr;
1719 int ret;
1720
1721 ret = mmc_test_area_erase(test);
1722 if (ret)
1723 return ret;
1724 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1725 dev_addr = t->dev_addr + (sz >> 9);
1726 ret = mmc_test_area_io(test, sz, dev_addr, 1, 0, 1);
1727 if (ret)
1728 return ret;
1729 }
1730 ret = mmc_test_area_erase(test);
1731 if (ret)
1732 return ret;
1733 sz = t->max_tfr;
1734 dev_addr = t->dev_addr;
1735 return mmc_test_area_io(test, sz, dev_addr, 1, 0, 1);
1736}
1737
1738/*
1739 * Single trim performance by transfer size.
1740 */
1741static int mmc_test_profile_trim_perf(struct mmc_test_card *test)
1742{
1743 struct mmc_test_area *t = &test->area;
1744 unsigned long sz;
1745 unsigned int dev_addr;
1746 struct timespec64 ts1, ts2;
1747 int ret;
1748
1749 if (!mmc_can_trim(test->card))
1750 return RESULT_UNSUP_CARD;
1751
1752 if (!mmc_can_erase(test->card))
1753 return RESULT_UNSUP_HOST;
1754
1755 for (sz = 512; sz < t->max_sz; sz <<= 1) {
1756 dev_addr = t->dev_addr + (sz >> 9);
1757 ktime_get_ts64(&ts1);
1758 ret = mmc_erase(test->card, dev_addr, sz >> 9, MMC_TRIM_ARG);
1759 if (ret)
1760 return ret;
1761 ktime_get_ts64(&ts2);
1762 mmc_test_print_rate(test, sz, &ts1, &ts2);
1763 }
1764 dev_addr = t->dev_addr;
1765 ktime_get_ts64(&ts1);
1766 ret = mmc_erase(test->card, dev_addr, sz >> 9, MMC_TRIM_ARG);
1767 if (ret)
1768 return ret;
1769 ktime_get_ts64(&ts2);
1770 mmc_test_print_rate(test, sz, &ts1, &ts2);
1771 return 0;
1772}
1773
1774static int mmc_test_seq_read_perf(struct mmc_test_card *test, unsigned long sz)
1775{
1776 struct mmc_test_area *t = &test->area;
1777 unsigned int dev_addr, i, cnt;
1778 struct timespec64 ts1, ts2;
1779 int ret;
1780
1781 cnt = t->max_sz / sz;
1782 dev_addr = t->dev_addr;
1783 ktime_get_ts64(&ts1);
1784 for (i = 0; i < cnt; i++) {
1785 ret = mmc_test_area_io(test, sz, dev_addr, 0, 0, 0);
1786 if (ret)
1787 return ret;
1788 dev_addr += (sz >> 9);
1789 }
1790 ktime_get_ts64(&ts2);
1791 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1792 return 0;
1793}
1794
1795/*
1796 * Consecutive read performance by transfer size.
1797 */
1798static int mmc_test_profile_seq_read_perf(struct mmc_test_card *test)
1799{
1800 struct mmc_test_area *t = &test->area;
1801 unsigned long sz;
1802 int ret;
1803
1804 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1805 ret = mmc_test_seq_read_perf(test, sz);
1806 if (ret)
1807 return ret;
1808 }
1809 sz = t->max_tfr;
1810 return mmc_test_seq_read_perf(test, sz);
1811}
1812
1813static int mmc_test_seq_write_perf(struct mmc_test_card *test, unsigned long sz)
1814{
1815 struct mmc_test_area *t = &test->area;
1816 unsigned int dev_addr, i, cnt;
1817 struct timespec64 ts1, ts2;
1818 int ret;
1819
1820 ret = mmc_test_area_erase(test);
1821 if (ret)
1822 return ret;
1823 cnt = t->max_sz / sz;
1824 dev_addr = t->dev_addr;
1825 ktime_get_ts64(&ts1);
1826 for (i = 0; i < cnt; i++) {
1827 ret = mmc_test_area_io(test, sz, dev_addr, 1, 0, 0);
1828 if (ret)
1829 return ret;
1830 dev_addr += (sz >> 9);
1831 }
1832 ktime_get_ts64(&ts2);
1833 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1834 return 0;
1835}
1836
1837/*
1838 * Consecutive write performance by transfer size.
1839 */
1840static int mmc_test_profile_seq_write_perf(struct mmc_test_card *test)
1841{
1842 struct mmc_test_area *t = &test->area;
1843 unsigned long sz;
1844 int ret;
1845
1846 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1847 ret = mmc_test_seq_write_perf(test, sz);
1848 if (ret)
1849 return ret;
1850 }
1851 sz = t->max_tfr;
1852 return mmc_test_seq_write_perf(test, sz);
1853}
1854
1855/*
1856 * Consecutive trim performance by transfer size.
1857 */
1858static int mmc_test_profile_seq_trim_perf(struct mmc_test_card *test)
1859{
1860 struct mmc_test_area *t = &test->area;
1861 unsigned long sz;
1862 unsigned int dev_addr, i, cnt;
1863 struct timespec64 ts1, ts2;
1864 int ret;
1865
1866 if (!mmc_can_trim(test->card))
1867 return RESULT_UNSUP_CARD;
1868
1869 if (!mmc_can_erase(test->card))
1870 return RESULT_UNSUP_HOST;
1871
1872 for (sz = 512; sz <= t->max_sz; sz <<= 1) {
1873 ret = mmc_test_area_erase(test);
1874 if (ret)
1875 return ret;
1876 ret = mmc_test_area_fill(test);
1877 if (ret)
1878 return ret;
1879 cnt = t->max_sz / sz;
1880 dev_addr = t->dev_addr;
1881 ktime_get_ts64(&ts1);
1882 for (i = 0; i < cnt; i++) {
1883 ret = mmc_erase(test->card, dev_addr, sz >> 9,
1884 MMC_TRIM_ARG);
1885 if (ret)
1886 return ret;
1887 dev_addr += (sz >> 9);
1888 }
1889 ktime_get_ts64(&ts2);
1890 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1891 }
1892 return 0;
1893}
1894
1895static unsigned int rnd_next = 1;
1896
1897static unsigned int mmc_test_rnd_num(unsigned int rnd_cnt)
1898{
1899 uint64_t r;
1900
1901 rnd_next = rnd_next * 1103515245 + 12345;
1902 r = (rnd_next >> 16) & 0x7fff;
1903 return (r * rnd_cnt) >> 15;
1904}
1905
1906static int mmc_test_rnd_perf(struct mmc_test_card *test, int write, int print,
1907 unsigned long sz, int secs, int force_retuning)
1908{
1909 unsigned int dev_addr, cnt, rnd_addr, range1, range2, last_ea = 0, ea;
1910 unsigned int ssz;
1911 struct timespec64 ts1, ts2, ts;
1912 int ret;
1913
1914 ssz = sz >> 9;
1915
1916 rnd_addr = mmc_test_capacity(test->card) / 4;
1917 range1 = rnd_addr / test->card->pref_erase;
1918 range2 = range1 / ssz;
1919
1920 ktime_get_ts64(&ts1);
1921 for (cnt = 0; cnt < UINT_MAX; cnt++) {
1922 ktime_get_ts64(&ts2);
1923 ts = timespec64_sub(ts2, ts1);
1924 if (ts.tv_sec >= secs)
1925 break;
1926 ea = mmc_test_rnd_num(range1);
1927 if (ea == last_ea)
1928 ea -= 1;
1929 last_ea = ea;
1930 dev_addr = rnd_addr + test->card->pref_erase * ea +
1931 ssz * mmc_test_rnd_num(range2);
1932 if (force_retuning)
1933 mmc_retune_needed(test->card->host);
1934 ret = mmc_test_area_io(test, sz, dev_addr, write, 0, 0);
1935 if (ret)
1936 return ret;
1937 }
1938 if (print)
1939 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1940 return 0;
1941}
1942
1943static int mmc_test_random_perf(struct mmc_test_card *test, int write)
1944{
1945 struct mmc_test_area *t = &test->area;
1946 unsigned int next;
1947 unsigned long sz;
1948 int ret;
1949
1950 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1951 /*
1952 * When writing, try to get more consistent results by running
1953 * the test twice with exactly the same I/O but outputting the
1954 * results only for the 2nd run.
1955 */
1956 if (write) {
1957 next = rnd_next;
1958 ret = mmc_test_rnd_perf(test, write, 0, sz, 10, 0);
1959 if (ret)
1960 return ret;
1961 rnd_next = next;
1962 }
1963 ret = mmc_test_rnd_perf(test, write, 1, sz, 10, 0);
1964 if (ret)
1965 return ret;
1966 }
1967 sz = t->max_tfr;
1968 if (write) {
1969 next = rnd_next;
1970 ret = mmc_test_rnd_perf(test, write, 0, sz, 10, 0);
1971 if (ret)
1972 return ret;
1973 rnd_next = next;
1974 }
1975 return mmc_test_rnd_perf(test, write, 1, sz, 10, 0);
1976}
1977
1978static int mmc_test_retuning(struct mmc_test_card *test)
1979{
1980 if (!mmc_can_retune(test->card->host)) {
1981 pr_info("%s: No retuning - test skipped\n",
1982 mmc_hostname(test->card->host));
1983 return RESULT_UNSUP_HOST;
1984 }
1985
1986 return mmc_test_rnd_perf(test, 0, 0, 8192, 30, 1);
1987}
1988
1989/*
1990 * Random read performance by transfer size.
1991 */
1992static int mmc_test_random_read_perf(struct mmc_test_card *test)
1993{
1994 return mmc_test_random_perf(test, 0);
1995}
1996
1997/*
1998 * Random write performance by transfer size.
1999 */
2000static int mmc_test_random_write_perf(struct mmc_test_card *test)
2001{
2002 return mmc_test_random_perf(test, 1);
2003}
2004
2005static int mmc_test_seq_perf(struct mmc_test_card *test, int write,
2006 unsigned int tot_sz, int max_scatter)
2007{
2008 struct mmc_test_area *t = &test->area;
2009 unsigned int dev_addr, i, cnt, sz, ssz;
2010 struct timespec64 ts1, ts2;
2011 int ret;
2012
2013 sz = t->max_tfr;
2014
2015 /*
2016 * In the case of a maximally scattered transfer, the maximum transfer
2017 * size is further limited by using PAGE_SIZE segments.
2018 */
2019 if (max_scatter) {
2020 unsigned long max_tfr;
2021
2022 if (t->max_seg_sz >= PAGE_SIZE)
2023 max_tfr = t->max_segs * PAGE_SIZE;
2024 else
2025 max_tfr = t->max_segs * t->max_seg_sz;
2026 if (sz > max_tfr)
2027 sz = max_tfr;
2028 }
2029
2030 ssz = sz >> 9;
2031 dev_addr = mmc_test_capacity(test->card) / 4;
2032 if (tot_sz > dev_addr << 9)
2033 tot_sz = dev_addr << 9;
2034 cnt = tot_sz / sz;
2035 dev_addr &= 0xffff0000; /* Round to 64MiB boundary */
2036
2037 ktime_get_ts64(&ts1);
2038 for (i = 0; i < cnt; i++) {
2039 ret = mmc_test_area_io(test, sz, dev_addr, write,
2040 max_scatter, 0);
2041 if (ret)
2042 return ret;
2043 dev_addr += ssz;
2044 }
2045 ktime_get_ts64(&ts2);
2046
2047 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
2048
2049 return 0;
2050}
2051
2052static int mmc_test_large_seq_perf(struct mmc_test_card *test, int write)
2053{
2054 int ret, i;
2055
2056 for (i = 0; i < 10; i++) {
2057 ret = mmc_test_seq_perf(test, write, 10 * 1024 * 1024, 1);
2058 if (ret)
2059 return ret;
2060 }
2061 for (i = 0; i < 5; i++) {
2062 ret = mmc_test_seq_perf(test, write, 100 * 1024 * 1024, 1);
2063 if (ret)
2064 return ret;
2065 }
2066 for (i = 0; i < 3; i++) {
2067 ret = mmc_test_seq_perf(test, write, 1000 * 1024 * 1024, 1);
2068 if (ret)
2069 return ret;
2070 }
2071
2072 return ret;
2073}
2074
2075/*
2076 * Large sequential read performance.
2077 */
2078static int mmc_test_large_seq_read_perf(struct mmc_test_card *test)
2079{
2080 return mmc_test_large_seq_perf(test, 0);
2081}
2082
2083/*
2084 * Large sequential write performance.
2085 */
2086static int mmc_test_large_seq_write_perf(struct mmc_test_card *test)
2087{
2088 return mmc_test_large_seq_perf(test, 1);
2089}
2090
2091static int mmc_test_rw_multiple(struct mmc_test_card *test,
2092 struct mmc_test_multiple_rw *tdata,
2093 unsigned int reqsize, unsigned int size,
2094 int min_sg_len)
2095{
2096 unsigned int dev_addr;
2097 struct mmc_test_area *t = &test->area;
2098 int ret = 0;
2099
2100 /* Set up test area */
2101 if (size > mmc_test_capacity(test->card) / 2 * 512)
2102 size = mmc_test_capacity(test->card) / 2 * 512;
2103 if (reqsize > t->max_tfr)
2104 reqsize = t->max_tfr;
2105 dev_addr = mmc_test_capacity(test->card) / 4;
2106 if ((dev_addr & 0xffff0000))
2107 dev_addr &= 0xffff0000; /* Round to 64MiB boundary */
2108 else
2109 dev_addr &= 0xfffff800; /* Round to 1MiB boundary */
2110 if (!dev_addr)
2111 goto err;
2112
2113 if (reqsize > size)
2114 return 0;
2115
2116 /* prepare test area */
2117 if (mmc_can_erase(test->card) &&
2118 tdata->prepare & MMC_TEST_PREP_ERASE) {
2119 ret = mmc_erase(test->card, dev_addr,
2120 size / 512, test->card->erase_arg);
2121 if (ret)
2122 ret = mmc_erase(test->card, dev_addr,
2123 size / 512, MMC_ERASE_ARG);
2124 if (ret)
2125 goto err;
2126 }
2127
2128 /* Run test */
2129 ret = mmc_test_area_io_seq(test, reqsize, dev_addr,
2130 tdata->do_write, 0, 1, size / reqsize,
2131 tdata->do_nonblock_req, min_sg_len);
2132 if (ret)
2133 goto err;
2134
2135 return ret;
2136 err:
2137 pr_info("[%s] error\n", __func__);
2138 return ret;
2139}
2140
2141static int mmc_test_rw_multiple_size(struct mmc_test_card *test,
2142 struct mmc_test_multiple_rw *rw)
2143{
2144 int ret = 0;
2145 int i;
2146 void *pre_req = test->card->host->ops->pre_req;
2147 void *post_req = test->card->host->ops->post_req;
2148
2149 if (rw->do_nonblock_req &&
2150 ((!pre_req && post_req) || (pre_req && !post_req))) {
2151 pr_info("error: only one of pre/post is defined\n");
2152 return -EINVAL;
2153 }
2154
2155 for (i = 0 ; i < rw->len && ret == 0; i++) {
2156 ret = mmc_test_rw_multiple(test, rw, rw->bs[i], rw->size, 0);
2157 if (ret)
2158 break;
2159 }
2160 return ret;
2161}
2162
2163static int mmc_test_rw_multiple_sg_len(struct mmc_test_card *test,
2164 struct mmc_test_multiple_rw *rw)
2165{
2166 int ret = 0;
2167 int i;
2168
2169 for (i = 0 ; i < rw->len && ret == 0; i++) {
2170 ret = mmc_test_rw_multiple(test, rw, 512 * 1024, rw->size,
2171 rw->sg_len[i]);
2172 if (ret)
2173 break;
2174 }
2175 return ret;
2176}
2177
2178/*
2179 * Multiple blocking write 4k to 4 MB chunks
2180 */
2181static int mmc_test_profile_mult_write_blocking_perf(struct mmc_test_card *test)
2182{
2183 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2184 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2185 struct mmc_test_multiple_rw test_data = {
2186 .bs = bs,
2187 .size = TEST_AREA_MAX_SIZE,
2188 .len = ARRAY_SIZE(bs),
2189 .do_write = true,
2190 .do_nonblock_req = false,
2191 .prepare = MMC_TEST_PREP_ERASE,
2192 };
2193
2194 return mmc_test_rw_multiple_size(test, &test_data);
2195};
2196
2197/*
2198 * Multiple non-blocking write 4k to 4 MB chunks
2199 */
2200static int mmc_test_profile_mult_write_nonblock_perf(struct mmc_test_card *test)
2201{
2202 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2203 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2204 struct mmc_test_multiple_rw test_data = {
2205 .bs = bs,
2206 .size = TEST_AREA_MAX_SIZE,
2207 .len = ARRAY_SIZE(bs),
2208 .do_write = true,
2209 .do_nonblock_req = true,
2210 .prepare = MMC_TEST_PREP_ERASE,
2211 };
2212
2213 return mmc_test_rw_multiple_size(test, &test_data);
2214}
2215
2216/*
2217 * Multiple blocking read 4k to 4 MB chunks
2218 */
2219static int mmc_test_profile_mult_read_blocking_perf(struct mmc_test_card *test)
2220{
2221 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2222 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2223 struct mmc_test_multiple_rw test_data = {
2224 .bs = bs,
2225 .size = TEST_AREA_MAX_SIZE,
2226 .len = ARRAY_SIZE(bs),
2227 .do_write = false,
2228 .do_nonblock_req = false,
2229 .prepare = MMC_TEST_PREP_NONE,
2230 };
2231
2232 return mmc_test_rw_multiple_size(test, &test_data);
2233}
2234
2235/*
2236 * Multiple non-blocking read 4k to 4 MB chunks
2237 */
2238static int mmc_test_profile_mult_read_nonblock_perf(struct mmc_test_card *test)
2239{
2240 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2241 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2242 struct mmc_test_multiple_rw test_data = {
2243 .bs = bs,
2244 .size = TEST_AREA_MAX_SIZE,
2245 .len = ARRAY_SIZE(bs),
2246 .do_write = false,
2247 .do_nonblock_req = true,
2248 .prepare = MMC_TEST_PREP_NONE,
2249 };
2250
2251 return mmc_test_rw_multiple_size(test, &test_data);
2252}
2253
2254/*
2255 * Multiple blocking write 1 to 512 sg elements
2256 */
2257static int mmc_test_profile_sglen_wr_blocking_perf(struct mmc_test_card *test)
2258{
2259 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2260 1 << 7, 1 << 8, 1 << 9};
2261 struct mmc_test_multiple_rw test_data = {
2262 .sg_len = sg_len,
2263 .size = TEST_AREA_MAX_SIZE,
2264 .len = ARRAY_SIZE(sg_len),
2265 .do_write = true,
2266 .do_nonblock_req = false,
2267 .prepare = MMC_TEST_PREP_ERASE,
2268 };
2269
2270 return mmc_test_rw_multiple_sg_len(test, &test_data);
2271};
2272
2273/*
2274 * Multiple non-blocking write 1 to 512 sg elements
2275 */
2276static int mmc_test_profile_sglen_wr_nonblock_perf(struct mmc_test_card *test)
2277{
2278 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2279 1 << 7, 1 << 8, 1 << 9};
2280 struct mmc_test_multiple_rw test_data = {
2281 .sg_len = sg_len,
2282 .size = TEST_AREA_MAX_SIZE,
2283 .len = ARRAY_SIZE(sg_len),
2284 .do_write = true,
2285 .do_nonblock_req = true,
2286 .prepare = MMC_TEST_PREP_ERASE,
2287 };
2288
2289 return mmc_test_rw_multiple_sg_len(test, &test_data);
2290}
2291
2292/*
2293 * Multiple blocking read 1 to 512 sg elements
2294 */
2295static int mmc_test_profile_sglen_r_blocking_perf(struct mmc_test_card *test)
2296{
2297 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2298 1 << 7, 1 << 8, 1 << 9};
2299 struct mmc_test_multiple_rw test_data = {
2300 .sg_len = sg_len,
2301 .size = TEST_AREA_MAX_SIZE,
2302 .len = ARRAY_SIZE(sg_len),
2303 .do_write = false,
2304 .do_nonblock_req = false,
2305 .prepare = MMC_TEST_PREP_NONE,
2306 };
2307
2308 return mmc_test_rw_multiple_sg_len(test, &test_data);
2309}
2310
2311/*
2312 * Multiple non-blocking read 1 to 512 sg elements
2313 */
2314static int mmc_test_profile_sglen_r_nonblock_perf(struct mmc_test_card *test)
2315{
2316 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2317 1 << 7, 1 << 8, 1 << 9};
2318 struct mmc_test_multiple_rw test_data = {
2319 .sg_len = sg_len,
2320 .size = TEST_AREA_MAX_SIZE,
2321 .len = ARRAY_SIZE(sg_len),
2322 .do_write = false,
2323 .do_nonblock_req = true,
2324 .prepare = MMC_TEST_PREP_NONE,
2325 };
2326
2327 return mmc_test_rw_multiple_sg_len(test, &test_data);
2328}
2329
2330/*
2331 * eMMC hardware reset.
2332 */
2333static int mmc_test_reset(struct mmc_test_card *test)
2334{
2335 struct mmc_card *card = test->card;
2336 int err;
2337
2338 err = mmc_hw_reset(card);
2339 if (!err) {
2340 /*
2341 * Reset will re-enable the card's command queue, but tests
2342 * expect it to be disabled.
2343 */
2344 if (card->ext_csd.cmdq_en)
2345 mmc_cmdq_disable(card);
2346 return RESULT_OK;
2347 } else if (err == -EOPNOTSUPP) {
2348 return RESULT_UNSUP_HOST;
2349 }
2350
2351 return RESULT_FAIL;
2352}
2353
2354static int mmc_test_send_status(struct mmc_test_card *test,
2355 struct mmc_command *cmd)
2356{
2357 memset(cmd, 0, sizeof(*cmd));
2358
2359 cmd->opcode = MMC_SEND_STATUS;
2360 if (!mmc_host_is_spi(test->card->host))
2361 cmd->arg = test->card->rca << 16;
2362 cmd->flags = MMC_RSP_SPI_R2 | MMC_RSP_R1 | MMC_CMD_AC;
2363
2364 return mmc_wait_for_cmd(test->card->host, cmd, 0);
2365}
2366
2367static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
2368 unsigned int dev_addr, int use_sbc,
2369 int repeat_cmd, int write, int use_areq)
2370{
2371 struct mmc_test_req *rq = mmc_test_req_alloc();
2372 struct mmc_host *host = test->card->host;
2373 struct mmc_test_area *t = &test->area;
2374 struct mmc_request *mrq;
2375 unsigned long timeout;
2376 bool expired = false;
2377 int ret = 0, cmd_ret;
2378 u32 status = 0;
2379 int count = 0;
2380
2381 if (!rq)
2382 return -ENOMEM;
2383
2384 mrq = &rq->mrq;
2385 if (use_sbc)
2386 mrq->sbc = &rq->sbc;
2387 mrq->cap_cmd_during_tfr = true;
2388
2389 mmc_test_prepare_mrq(test, mrq, t->sg, t->sg_len, dev_addr, t->blocks,
2390 512, write);
2391
2392 if (use_sbc && t->blocks > 1 && !mrq->sbc) {
2393 ret = mmc_host_cmd23(host) ?
2394 RESULT_UNSUP_CARD :
2395 RESULT_UNSUP_HOST;
2396 goto out_free;
2397 }
2398
2399 /* Start ongoing data request */
2400 if (use_areq) {
2401 ret = mmc_test_start_areq(test, mrq, NULL);
2402 if (ret)
2403 goto out_free;
2404 } else {
2405 mmc_wait_for_req(host, mrq);
2406 }
2407
2408 timeout = jiffies + msecs_to_jiffies(3000);
2409 do {
2410 count += 1;
2411
2412 /* Send status command while data transfer in progress */
2413 cmd_ret = mmc_test_send_status(test, &rq->status);
2414 if (cmd_ret)
2415 break;
2416
2417 status = rq->status.resp[0];
2418 if (status & R1_ERROR) {
2419 cmd_ret = -EIO;
2420 break;
2421 }
2422
2423 if (mmc_is_req_done(host, mrq))
2424 break;
2425
2426 expired = time_after(jiffies, timeout);
2427 if (expired) {
2428 pr_info("%s: timeout waiting for Tran state status %#x\n",
2429 mmc_hostname(host), status);
2430 cmd_ret = -ETIMEDOUT;
2431 break;
2432 }
2433 } while (repeat_cmd && R1_CURRENT_STATE(status) != R1_STATE_TRAN);
2434
2435 /* Wait for data request to complete */
2436 if (use_areq) {
2437 ret = mmc_test_start_areq(test, NULL, mrq);
2438 } else {
2439 mmc_wait_for_req_done(test->card->host, mrq);
2440 }
2441
2442 /*
2443 * For cap_cmd_during_tfr request, upper layer must send stop if
2444 * required.
2445 */
2446 if (mrq->data->stop && (mrq->data->error || !mrq->sbc)) {
2447 if (ret)
2448 mmc_wait_for_cmd(host, mrq->data->stop, 0);
2449 else
2450 ret = mmc_wait_for_cmd(host, mrq->data->stop, 0);
2451 }
2452
2453 if (ret)
2454 goto out_free;
2455
2456 if (cmd_ret) {
2457 pr_info("%s: Send Status failed: status %#x, error %d\n",
2458 mmc_hostname(test->card->host), status, cmd_ret);
2459 }
2460
2461 ret = mmc_test_check_result(test, mrq);
2462 if (ret)
2463 goto out_free;
2464
2465 ret = mmc_test_wait_busy(test);
2466 if (ret)
2467 goto out_free;
2468
2469 if (repeat_cmd && (t->blocks + 1) << 9 > t->max_tfr)
2470 pr_info("%s: %d commands completed during transfer of %u blocks\n",
2471 mmc_hostname(test->card->host), count, t->blocks);
2472
2473 if (cmd_ret)
2474 ret = cmd_ret;
2475out_free:
2476 kfree(rq);
2477
2478 return ret;
2479}
2480
2481static int __mmc_test_cmds_during_tfr(struct mmc_test_card *test,
2482 unsigned long sz, int use_sbc, int write,
2483 int use_areq)
2484{
2485 struct mmc_test_area *t = &test->area;
2486 int ret;
2487
2488 if (!(test->card->host->caps & MMC_CAP_CMD_DURING_TFR))
2489 return RESULT_UNSUP_HOST;
2490
2491 ret = mmc_test_area_map(test, sz, 0, 0, use_areq);
2492 if (ret)
2493 return ret;
2494
2495 ret = mmc_test_ongoing_transfer(test, t->dev_addr, use_sbc, 0, write,
2496 use_areq);
2497 if (ret)
2498 return ret;
2499
2500 return mmc_test_ongoing_transfer(test, t->dev_addr, use_sbc, 1, write,
2501 use_areq);
2502}
2503
2504static int mmc_test_cmds_during_tfr(struct mmc_test_card *test, int use_sbc,
2505 int write, int use_areq)
2506{
2507 struct mmc_test_area *t = &test->area;
2508 unsigned long sz;
2509 int ret;
2510
2511 for (sz = 512; sz <= t->max_tfr; sz += 512) {
2512 ret = __mmc_test_cmds_during_tfr(test, sz, use_sbc, write,
2513 use_areq);
2514 if (ret)
2515 return ret;
2516 }
2517 return 0;
2518}
2519
2520/*
2521 * Commands during read - no Set Block Count (CMD23).
2522 */
2523static int mmc_test_cmds_during_read(struct mmc_test_card *test)
2524{
2525 return mmc_test_cmds_during_tfr(test, 0, 0, 0);
2526}
2527
2528/*
2529 * Commands during write - no Set Block Count (CMD23).
2530 */
2531static int mmc_test_cmds_during_write(struct mmc_test_card *test)
2532{
2533 return mmc_test_cmds_during_tfr(test, 0, 1, 0);
2534}
2535
2536/*
2537 * Commands during read - use Set Block Count (CMD23).
2538 */
2539static int mmc_test_cmds_during_read_cmd23(struct mmc_test_card *test)
2540{
2541 return mmc_test_cmds_during_tfr(test, 1, 0, 0);
2542}
2543
2544/*
2545 * Commands during write - use Set Block Count (CMD23).
2546 */
2547static int mmc_test_cmds_during_write_cmd23(struct mmc_test_card *test)
2548{
2549 return mmc_test_cmds_during_tfr(test, 1, 1, 0);
2550}
2551
2552/*
2553 * Commands during non-blocking read - use Set Block Count (CMD23).
2554 */
2555static int mmc_test_cmds_during_read_cmd23_nonblock(struct mmc_test_card *test)
2556{
2557 return mmc_test_cmds_during_tfr(test, 1, 0, 1);
2558}
2559
2560/*
2561 * Commands during non-blocking write - use Set Block Count (CMD23).
2562 */
2563static int mmc_test_cmds_during_write_cmd23_nonblock(struct mmc_test_card *test)
2564{
2565 return mmc_test_cmds_during_tfr(test, 1, 1, 1);
2566}
2567
2568static const struct mmc_test_case mmc_test_cases[] = {
2569 {
2570 .name = "Basic write (no data verification)",
2571 .run = mmc_test_basic_write,
2572 },
2573
2574 {
2575 .name = "Basic read (no data verification)",
2576 .run = mmc_test_basic_read,
2577 },
2578
2579 {
2580 .name = "Basic write (with data verification)",
2581 .prepare = mmc_test_prepare_write,
2582 .run = mmc_test_verify_write,
2583 .cleanup = mmc_test_cleanup,
2584 },
2585
2586 {
2587 .name = "Basic read (with data verification)",
2588 .prepare = mmc_test_prepare_read,
2589 .run = mmc_test_verify_read,
2590 .cleanup = mmc_test_cleanup,
2591 },
2592
2593 {
2594 .name = "Multi-block write",
2595 .prepare = mmc_test_prepare_write,
2596 .run = mmc_test_multi_write,
2597 .cleanup = mmc_test_cleanup,
2598 },
2599
2600 {
2601 .name = "Multi-block read",
2602 .prepare = mmc_test_prepare_read,
2603 .run = mmc_test_multi_read,
2604 .cleanup = mmc_test_cleanup,
2605 },
2606
2607 {
2608 .name = "Power of two block writes",
2609 .prepare = mmc_test_prepare_write,
2610 .run = mmc_test_pow2_write,
2611 .cleanup = mmc_test_cleanup,
2612 },
2613
2614 {
2615 .name = "Power of two block reads",
2616 .prepare = mmc_test_prepare_read,
2617 .run = mmc_test_pow2_read,
2618 .cleanup = mmc_test_cleanup,
2619 },
2620
2621 {
2622 .name = "Weird sized block writes",
2623 .prepare = mmc_test_prepare_write,
2624 .run = mmc_test_weird_write,
2625 .cleanup = mmc_test_cleanup,
2626 },
2627
2628 {
2629 .name = "Weird sized block reads",
2630 .prepare = mmc_test_prepare_read,
2631 .run = mmc_test_weird_read,
2632 .cleanup = mmc_test_cleanup,
2633 },
2634
2635 {
2636 .name = "Badly aligned write",
2637 .prepare = mmc_test_prepare_write,
2638 .run = mmc_test_align_write,
2639 .cleanup = mmc_test_cleanup,
2640 },
2641
2642 {
2643 .name = "Badly aligned read",
2644 .prepare = mmc_test_prepare_read,
2645 .run = mmc_test_align_read,
2646 .cleanup = mmc_test_cleanup,
2647 },
2648
2649 {
2650 .name = "Badly aligned multi-block write",
2651 .prepare = mmc_test_prepare_write,
2652 .run = mmc_test_align_multi_write,
2653 .cleanup = mmc_test_cleanup,
2654 },
2655
2656 {
2657 .name = "Badly aligned multi-block read",
2658 .prepare = mmc_test_prepare_read,
2659 .run = mmc_test_align_multi_read,
2660 .cleanup = mmc_test_cleanup,
2661 },
2662
2663 {
2664 .name = "Proper xfer_size at write (start failure)",
2665 .run = mmc_test_xfersize_write,
2666 },
2667
2668 {
2669 .name = "Proper xfer_size at read (start failure)",
2670 .run = mmc_test_xfersize_read,
2671 },
2672
2673 {
2674 .name = "Proper xfer_size at write (midway failure)",
2675 .run = mmc_test_multi_xfersize_write,
2676 },
2677
2678 {
2679 .name = "Proper xfer_size at read (midway failure)",
2680 .run = mmc_test_multi_xfersize_read,
2681 },
2682
2683#ifdef CONFIG_HIGHMEM
2684
2685 {
2686 .name = "Highmem write",
2687 .prepare = mmc_test_prepare_write,
2688 .run = mmc_test_write_high,
2689 .cleanup = mmc_test_cleanup,
2690 },
2691
2692 {
2693 .name = "Highmem read",
2694 .prepare = mmc_test_prepare_read,
2695 .run = mmc_test_read_high,
2696 .cleanup = mmc_test_cleanup,
2697 },
2698
2699 {
2700 .name = "Multi-block highmem write",
2701 .prepare = mmc_test_prepare_write,
2702 .run = mmc_test_multi_write_high,
2703 .cleanup = mmc_test_cleanup,
2704 },
2705
2706 {
2707 .name = "Multi-block highmem read",
2708 .prepare = mmc_test_prepare_read,
2709 .run = mmc_test_multi_read_high,
2710 .cleanup = mmc_test_cleanup,
2711 },
2712
2713#else
2714
2715 {
2716 .name = "Highmem write",
2717 .run = mmc_test_no_highmem,
2718 },
2719
2720 {
2721 .name = "Highmem read",
2722 .run = mmc_test_no_highmem,
2723 },
2724
2725 {
2726 .name = "Multi-block highmem write",
2727 .run = mmc_test_no_highmem,
2728 },
2729
2730 {
2731 .name = "Multi-block highmem read",
2732 .run = mmc_test_no_highmem,
2733 },
2734
2735#endif /* CONFIG_HIGHMEM */
2736
2737 {
2738 .name = "Best-case read performance",
2739 .prepare = mmc_test_area_prepare_fill,
2740 .run = mmc_test_best_read_performance,
2741 .cleanup = mmc_test_area_cleanup,
2742 },
2743
2744 {
2745 .name = "Best-case write performance",
2746 .prepare = mmc_test_area_prepare_erase,
2747 .run = mmc_test_best_write_performance,
2748 .cleanup = mmc_test_area_cleanup,
2749 },
2750
2751 {
2752 .name = "Best-case read performance into scattered pages",
2753 .prepare = mmc_test_area_prepare_fill,
2754 .run = mmc_test_best_read_perf_max_scatter,
2755 .cleanup = mmc_test_area_cleanup,
2756 },
2757
2758 {
2759 .name = "Best-case write performance from scattered pages",
2760 .prepare = mmc_test_area_prepare_erase,
2761 .run = mmc_test_best_write_perf_max_scatter,
2762 .cleanup = mmc_test_area_cleanup,
2763 },
2764
2765 {
2766 .name = "Single read performance by transfer size",
2767 .prepare = mmc_test_area_prepare_fill,
2768 .run = mmc_test_profile_read_perf,
2769 .cleanup = mmc_test_area_cleanup,
2770 },
2771
2772 {
2773 .name = "Single write performance by transfer size",
2774 .prepare = mmc_test_area_prepare,
2775 .run = mmc_test_profile_write_perf,
2776 .cleanup = mmc_test_area_cleanup,
2777 },
2778
2779 {
2780 .name = "Single trim performance by transfer size",
2781 .prepare = mmc_test_area_prepare_fill,
2782 .run = mmc_test_profile_trim_perf,
2783 .cleanup = mmc_test_area_cleanup,
2784 },
2785
2786 {
2787 .name = "Consecutive read performance by transfer size",
2788 .prepare = mmc_test_area_prepare_fill,
2789 .run = mmc_test_profile_seq_read_perf,
2790 .cleanup = mmc_test_area_cleanup,
2791 },
2792
2793 {
2794 .name = "Consecutive write performance by transfer size",
2795 .prepare = mmc_test_area_prepare,
2796 .run = mmc_test_profile_seq_write_perf,
2797 .cleanup = mmc_test_area_cleanup,
2798 },
2799
2800 {
2801 .name = "Consecutive trim performance by transfer size",
2802 .prepare = mmc_test_area_prepare,
2803 .run = mmc_test_profile_seq_trim_perf,
2804 .cleanup = mmc_test_area_cleanup,
2805 },
2806
2807 {
2808 .name = "Random read performance by transfer size",
2809 .prepare = mmc_test_area_prepare,
2810 .run = mmc_test_random_read_perf,
2811 .cleanup = mmc_test_area_cleanup,
2812 },
2813
2814 {
2815 .name = "Random write performance by transfer size",
2816 .prepare = mmc_test_area_prepare,
2817 .run = mmc_test_random_write_perf,
2818 .cleanup = mmc_test_area_cleanup,
2819 },
2820
2821 {
2822 .name = "Large sequential read into scattered pages",
2823 .prepare = mmc_test_area_prepare,
2824 .run = mmc_test_large_seq_read_perf,
2825 .cleanup = mmc_test_area_cleanup,
2826 },
2827
2828 {
2829 .name = "Large sequential write from scattered pages",
2830 .prepare = mmc_test_area_prepare,
2831 .run = mmc_test_large_seq_write_perf,
2832 .cleanup = mmc_test_area_cleanup,
2833 },
2834
2835 {
2836 .name = "Write performance with blocking req 4k to 4MB",
2837 .prepare = mmc_test_area_prepare,
2838 .run = mmc_test_profile_mult_write_blocking_perf,
2839 .cleanup = mmc_test_area_cleanup,
2840 },
2841
2842 {
2843 .name = "Write performance with non-blocking req 4k to 4MB",
2844 .prepare = mmc_test_area_prepare,
2845 .run = mmc_test_profile_mult_write_nonblock_perf,
2846 .cleanup = mmc_test_area_cleanup,
2847 },
2848
2849 {
2850 .name = "Read performance with blocking req 4k to 4MB",
2851 .prepare = mmc_test_area_prepare,
2852 .run = mmc_test_profile_mult_read_blocking_perf,
2853 .cleanup = mmc_test_area_cleanup,
2854 },
2855
2856 {
2857 .name = "Read performance with non-blocking req 4k to 4MB",
2858 .prepare = mmc_test_area_prepare,
2859 .run = mmc_test_profile_mult_read_nonblock_perf,
2860 .cleanup = mmc_test_area_cleanup,
2861 },
2862
2863 {
2864 .name = "Write performance blocking req 1 to 512 sg elems",
2865 .prepare = mmc_test_area_prepare,
2866 .run = mmc_test_profile_sglen_wr_blocking_perf,
2867 .cleanup = mmc_test_area_cleanup,
2868 },
2869
2870 {
2871 .name = "Write performance non-blocking req 1 to 512 sg elems",
2872 .prepare = mmc_test_area_prepare,
2873 .run = mmc_test_profile_sglen_wr_nonblock_perf,
2874 .cleanup = mmc_test_area_cleanup,
2875 },
2876
2877 {
2878 .name = "Read performance blocking req 1 to 512 sg elems",
2879 .prepare = mmc_test_area_prepare,
2880 .run = mmc_test_profile_sglen_r_blocking_perf,
2881 .cleanup = mmc_test_area_cleanup,
2882 },
2883
2884 {
2885 .name = "Read performance non-blocking req 1 to 512 sg elems",
2886 .prepare = mmc_test_area_prepare,
2887 .run = mmc_test_profile_sglen_r_nonblock_perf,
2888 .cleanup = mmc_test_area_cleanup,
2889 },
2890
2891 {
2892 .name = "Reset test",
2893 .run = mmc_test_reset,
2894 },
2895
2896 {
2897 .name = "Commands during read - no Set Block Count (CMD23)",
2898 .prepare = mmc_test_area_prepare,
2899 .run = mmc_test_cmds_during_read,
2900 .cleanup = mmc_test_area_cleanup,
2901 },
2902
2903 {
2904 .name = "Commands during write - no Set Block Count (CMD23)",
2905 .prepare = mmc_test_area_prepare,
2906 .run = mmc_test_cmds_during_write,
2907 .cleanup = mmc_test_area_cleanup,
2908 },
2909
2910 {
2911 .name = "Commands during read - use Set Block Count (CMD23)",
2912 .prepare = mmc_test_area_prepare,
2913 .run = mmc_test_cmds_during_read_cmd23,
2914 .cleanup = mmc_test_area_cleanup,
2915 },
2916
2917 {
2918 .name = "Commands during write - use Set Block Count (CMD23)",
2919 .prepare = mmc_test_area_prepare,
2920 .run = mmc_test_cmds_during_write_cmd23,
2921 .cleanup = mmc_test_area_cleanup,
2922 },
2923
2924 {
2925 .name = "Commands during non-blocking read - use Set Block Count (CMD23)",
2926 .prepare = mmc_test_area_prepare,
2927 .run = mmc_test_cmds_during_read_cmd23_nonblock,
2928 .cleanup = mmc_test_area_cleanup,
2929 },
2930
2931 {
2932 .name = "Commands during non-blocking write - use Set Block Count (CMD23)",
2933 .prepare = mmc_test_area_prepare,
2934 .run = mmc_test_cmds_during_write_cmd23_nonblock,
2935 .cleanup = mmc_test_area_cleanup,
2936 },
2937
2938 {
2939 .name = "Re-tuning reliability",
2940 .prepare = mmc_test_area_prepare,
2941 .run = mmc_test_retuning,
2942 .cleanup = mmc_test_area_cleanup,
2943 },
2944
2945};
2946
2947static DEFINE_MUTEX(mmc_test_lock);
2948
2949static LIST_HEAD(mmc_test_result);
2950
2951static void mmc_test_run(struct mmc_test_card *test, int testcase)
2952{
2953 int i, ret;
2954
2955 pr_info("%s: Starting tests of card %s...\n",
2956 mmc_hostname(test->card->host), mmc_card_id(test->card));
2957
2958 mmc_claim_host(test->card->host);
2959
2960 for (i = 0; i < ARRAY_SIZE(mmc_test_cases); i++) {
2961 struct mmc_test_general_result *gr;
2962
2963 if (testcase && ((i + 1) != testcase))
2964 continue;
2965
2966 pr_info("%s: Test case %d. %s...\n",
2967 mmc_hostname(test->card->host), i + 1,
2968 mmc_test_cases[i].name);
2969
2970 if (mmc_test_cases[i].prepare) {
2971 ret = mmc_test_cases[i].prepare(test);
2972 if (ret) {
2973 pr_info("%s: Result: Prepare stage failed! (%d)\n",
2974 mmc_hostname(test->card->host),
2975 ret);
2976 continue;
2977 }
2978 }
2979
2980 gr = kzalloc(sizeof(*gr), GFP_KERNEL);
2981 if (gr) {
2982 INIT_LIST_HEAD(&gr->tr_lst);
2983
2984 /* Assign data what we know already */
2985 gr->card = test->card;
2986 gr->testcase = i;
2987
2988 /* Append container to global one */
2989 list_add_tail(&gr->link, &mmc_test_result);
2990
2991 /*
2992 * Save the pointer to created container in our private
2993 * structure.
2994 */
2995 test->gr = gr;
2996 }
2997
2998 ret = mmc_test_cases[i].run(test);
2999 switch (ret) {
3000 case RESULT_OK:
3001 pr_info("%s: Result: OK\n",
3002 mmc_hostname(test->card->host));
3003 break;
3004 case RESULT_FAIL:
3005 pr_info("%s: Result: FAILED\n",
3006 mmc_hostname(test->card->host));
3007 break;
3008 case RESULT_UNSUP_HOST:
3009 pr_info("%s: Result: UNSUPPORTED (by host)\n",
3010 mmc_hostname(test->card->host));
3011 break;
3012 case RESULT_UNSUP_CARD:
3013 pr_info("%s: Result: UNSUPPORTED (by card)\n",
3014 mmc_hostname(test->card->host));
3015 break;
3016 default:
3017 pr_info("%s: Result: ERROR (%d)\n",
3018 mmc_hostname(test->card->host), ret);
3019 }
3020
3021 /* Save the result */
3022 if (gr)
3023 gr->result = ret;
3024
3025 if (mmc_test_cases[i].cleanup) {
3026 ret = mmc_test_cases[i].cleanup(test);
3027 if (ret) {
3028 pr_info("%s: Warning: Cleanup stage failed! (%d)\n",
3029 mmc_hostname(test->card->host),
3030 ret);
3031 }
3032 }
3033 }
3034
3035 mmc_release_host(test->card->host);
3036
3037 pr_info("%s: Tests completed.\n",
3038 mmc_hostname(test->card->host));
3039}
3040
3041static void mmc_test_free_result(struct mmc_card *card)
3042{
3043 struct mmc_test_general_result *gr, *grs;
3044
3045 mutex_lock(&mmc_test_lock);
3046
3047 list_for_each_entry_safe(gr, grs, &mmc_test_result, link) {
3048 struct mmc_test_transfer_result *tr, *trs;
3049
3050 if (card && gr->card != card)
3051 continue;
3052
3053 list_for_each_entry_safe(tr, trs, &gr->tr_lst, link) {
3054 list_del(&tr->link);
3055 kfree(tr);
3056 }
3057
3058 list_del(&gr->link);
3059 kfree(gr);
3060 }
3061
3062 mutex_unlock(&mmc_test_lock);
3063}
3064
3065static LIST_HEAD(mmc_test_file_test);
3066
3067static int mtf_test_show(struct seq_file *sf, void *data)
3068{
3069 struct mmc_card *card = sf->private;
3070 struct mmc_test_general_result *gr;
3071
3072 mutex_lock(&mmc_test_lock);
3073
3074 list_for_each_entry(gr, &mmc_test_result, link) {
3075 struct mmc_test_transfer_result *tr;
3076
3077 if (gr->card != card)
3078 continue;
3079
3080 seq_printf(sf, "Test %d: %d\n", gr->testcase + 1, gr->result);
3081
3082 list_for_each_entry(tr, &gr->tr_lst, link) {
3083 seq_printf(sf, "%u %d %llu.%09u %u %u.%02u\n",
3084 tr->count, tr->sectors,
3085 (u64)tr->ts.tv_sec, (u32)tr->ts.tv_nsec,
3086 tr->rate, tr->iops / 100, tr->iops % 100);
3087 }
3088 }
3089
3090 mutex_unlock(&mmc_test_lock);
3091
3092 return 0;
3093}
3094
3095static int mtf_test_open(struct inode *inode, struct file *file)
3096{
3097 return single_open(file, mtf_test_show, inode->i_private);
3098}
3099
3100static ssize_t mtf_test_write(struct file *file, const char __user *buf,
3101 size_t count, loff_t *pos)
3102{
3103 struct seq_file *sf = file->private_data;
3104 struct mmc_card *card = sf->private;
3105 struct mmc_test_card *test;
3106 long testcase;
3107 int ret;
3108
3109 ret = kstrtol_from_user(buf, count, 10, &testcase);
3110 if (ret)
3111 return ret;
3112
3113 test = kzalloc(sizeof(*test), GFP_KERNEL);
3114 if (!test)
3115 return -ENOMEM;
3116
3117 /*
3118 * Remove all test cases associated with given card. Thus we have only
3119 * actual data of the last run.
3120 */
3121 mmc_test_free_result(card);
3122
3123 test->card = card;
3124
3125 test->buffer = kzalloc(BUFFER_SIZE, GFP_KERNEL);
3126#ifdef CONFIG_HIGHMEM
3127 test->highmem = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM, BUFFER_ORDER);
3128#endif
3129
3130#ifdef CONFIG_HIGHMEM
3131 if (test->buffer && test->highmem) {
3132#else
3133 if (test->buffer) {
3134#endif
3135 mutex_lock(&mmc_test_lock);
3136 mmc_test_run(test, testcase);
3137 mutex_unlock(&mmc_test_lock);
3138 }
3139
3140#ifdef CONFIG_HIGHMEM
3141 __free_pages(test->highmem, BUFFER_ORDER);
3142#endif
3143 kfree(test->buffer);
3144 kfree(test);
3145
3146 return count;
3147}
3148
3149static const struct file_operations mmc_test_fops_test = {
3150 .open = mtf_test_open,
3151 .read = seq_read,
3152 .write = mtf_test_write,
3153 .llseek = seq_lseek,
3154 .release = single_release,
3155};
3156
3157static int mtf_testlist_show(struct seq_file *sf, void *data)
3158{
3159 int i;
3160
3161 mutex_lock(&mmc_test_lock);
3162
3163 seq_puts(sf, "0:\tRun all tests\n");
3164 for (i = 0; i < ARRAY_SIZE(mmc_test_cases); i++)
3165 seq_printf(sf, "%d:\t%s\n", i + 1, mmc_test_cases[i].name);
3166
3167 mutex_unlock(&mmc_test_lock);
3168
3169 return 0;
3170}
3171
3172DEFINE_SHOW_ATTRIBUTE(mtf_testlist);
3173
3174static void mmc_test_free_dbgfs_file(struct mmc_card *card)
3175{
3176 struct mmc_test_dbgfs_file *df, *dfs;
3177
3178 mutex_lock(&mmc_test_lock);
3179
3180 list_for_each_entry_safe(df, dfs, &mmc_test_file_test, link) {
3181 if (card && df->card != card)
3182 continue;
3183 debugfs_remove(df->file);
3184 list_del(&df->link);
3185 kfree(df);
3186 }
3187
3188 mutex_unlock(&mmc_test_lock);
3189}
3190
3191static int __mmc_test_register_dbgfs_file(struct mmc_card *card,
3192 const char *name, umode_t mode, const struct file_operations *fops)
3193{
3194 struct dentry *file = NULL;
3195 struct mmc_test_dbgfs_file *df;
3196
3197 if (card->debugfs_root)
3198 file = debugfs_create_file(name, mode, card->debugfs_root,
3199 card, fops);
3200
3201 df = kmalloc(sizeof(*df), GFP_KERNEL);
3202 if (!df) {
3203 debugfs_remove(file);
3204 return -ENOMEM;
3205 }
3206
3207 df->card = card;
3208 df->file = file;
3209
3210 list_add(&df->link, &mmc_test_file_test);
3211 return 0;
3212}
3213
3214static int mmc_test_register_dbgfs_file(struct mmc_card *card)
3215{
3216 int ret;
3217
3218 mutex_lock(&mmc_test_lock);
3219
3220 ret = __mmc_test_register_dbgfs_file(card, "test", S_IWUSR | S_IRUGO,
3221 &mmc_test_fops_test);
3222 if (ret)
3223 goto err;
3224
3225 ret = __mmc_test_register_dbgfs_file(card, "testlist", S_IRUGO,
3226 &mtf_testlist_fops);
3227 if (ret)
3228 goto err;
3229
3230err:
3231 mutex_unlock(&mmc_test_lock);
3232
3233 return ret;
3234}
3235
3236static int mmc_test_probe(struct mmc_card *card)
3237{
3238 int ret;
3239
3240 if (!mmc_card_mmc(card) && !mmc_card_sd(card))
3241 return -ENODEV;
3242
3243 ret = mmc_test_register_dbgfs_file(card);
3244 if (ret)
3245 return ret;
3246
3247 if (card->ext_csd.cmdq_en) {
3248 mmc_claim_host(card->host);
3249 ret = mmc_cmdq_disable(card);
3250 mmc_release_host(card->host);
3251 if (ret)
3252 return ret;
3253 }
3254
3255 dev_info(&card->dev, "Card claimed for testing.\n");
3256
3257 return 0;
3258}
3259
3260static void mmc_test_remove(struct mmc_card *card)
3261{
3262 if (card->reenable_cmdq) {
3263 mmc_claim_host(card->host);
3264 mmc_cmdq_enable(card);
3265 mmc_release_host(card->host);
3266 }
3267 mmc_test_free_result(card);
3268 mmc_test_free_dbgfs_file(card);
3269}
3270
3271static struct mmc_driver mmc_driver = {
3272 .drv = {
3273 .name = "mmc_test",
3274 },
3275 .probe = mmc_test_probe,
3276 .remove = mmc_test_remove,
3277};
3278
3279static int __init mmc_test_init(void)
3280{
3281 return mmc_register_driver(&mmc_driver);
3282}
3283
3284static void __exit mmc_test_exit(void)
3285{
3286 /* Clear stalled data if card is still plugged */
3287 mmc_test_free_result(NULL);
3288 mmc_test_free_dbgfs_file(NULL);
3289
3290 mmc_unregister_driver(&mmc_driver);
3291}
3292
3293module_init(mmc_test_init);
3294module_exit(mmc_test_exit);
3295
3296MODULE_LICENSE("GPL");
3297MODULE_DESCRIPTION("Multimedia Card (MMC) host test driver");
3298MODULE_AUTHOR("Pierre Ossman");
1/*
2 * Copyright 2007-2008 Pierre Ossman
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or (at
7 * your option) any later version.
8 */
9
10#include <linux/mmc/core.h>
11#include <linux/mmc/card.h>
12#include <linux/mmc/host.h>
13#include <linux/mmc/mmc.h>
14#include <linux/slab.h>
15
16#include <linux/scatterlist.h>
17#include <linux/swap.h> /* For nr_free_buffer_pages() */
18#include <linux/list.h>
19
20#include <linux/debugfs.h>
21#include <linux/uaccess.h>
22#include <linux/seq_file.h>
23#include <linux/module.h>
24
25#include "core.h"
26#include "card.h"
27#include "host.h"
28#include "bus.h"
29#include "mmc_ops.h"
30
31#define RESULT_OK 0
32#define RESULT_FAIL 1
33#define RESULT_UNSUP_HOST 2
34#define RESULT_UNSUP_CARD 3
35
36#define BUFFER_ORDER 2
37#define BUFFER_SIZE (PAGE_SIZE << BUFFER_ORDER)
38
39#define TEST_ALIGN_END 8
40
41/*
42 * Limit the test area size to the maximum MMC HC erase group size. Note that
43 * the maximum SD allocation unit size is just 4MiB.
44 */
45#define TEST_AREA_MAX_SIZE (128 * 1024 * 1024)
46
47/**
48 * struct mmc_test_pages - pages allocated by 'alloc_pages()'.
49 * @page: first page in the allocation
50 * @order: order of the number of pages allocated
51 */
52struct mmc_test_pages {
53 struct page *page;
54 unsigned int order;
55};
56
57/**
58 * struct mmc_test_mem - allocated memory.
59 * @arr: array of allocations
60 * @cnt: number of allocations
61 */
62struct mmc_test_mem {
63 struct mmc_test_pages *arr;
64 unsigned int cnt;
65};
66
67/**
68 * struct mmc_test_area - information for performance tests.
69 * @max_sz: test area size (in bytes)
70 * @dev_addr: address on card at which to do performance tests
71 * @max_tfr: maximum transfer size allowed by driver (in bytes)
72 * @max_segs: maximum segments allowed by driver in scatterlist @sg
73 * @max_seg_sz: maximum segment size allowed by driver
74 * @blocks: number of (512 byte) blocks currently mapped by @sg
75 * @sg_len: length of currently mapped scatterlist @sg
76 * @mem: allocated memory
77 * @sg: scatterlist
78 */
79struct mmc_test_area {
80 unsigned long max_sz;
81 unsigned int dev_addr;
82 unsigned int max_tfr;
83 unsigned int max_segs;
84 unsigned int max_seg_sz;
85 unsigned int blocks;
86 unsigned int sg_len;
87 struct mmc_test_mem *mem;
88 struct scatterlist *sg;
89};
90
91/**
92 * struct mmc_test_transfer_result - transfer results for performance tests.
93 * @link: double-linked list
94 * @count: amount of group of sectors to check
95 * @sectors: amount of sectors to check in one group
96 * @ts: time values of transfer
97 * @rate: calculated transfer rate
98 * @iops: I/O operations per second (times 100)
99 */
100struct mmc_test_transfer_result {
101 struct list_head link;
102 unsigned int count;
103 unsigned int sectors;
104 struct timespec64 ts;
105 unsigned int rate;
106 unsigned int iops;
107};
108
109/**
110 * struct mmc_test_general_result - results for tests.
111 * @link: double-linked list
112 * @card: card under test
113 * @testcase: number of test case
114 * @result: result of test run
115 * @tr_lst: transfer measurements if any as mmc_test_transfer_result
116 */
117struct mmc_test_general_result {
118 struct list_head link;
119 struct mmc_card *card;
120 int testcase;
121 int result;
122 struct list_head tr_lst;
123};
124
125/**
126 * struct mmc_test_dbgfs_file - debugfs related file.
127 * @link: double-linked list
128 * @card: card under test
129 * @file: file created under debugfs
130 */
131struct mmc_test_dbgfs_file {
132 struct list_head link;
133 struct mmc_card *card;
134 struct dentry *file;
135};
136
137/**
138 * struct mmc_test_card - test information.
139 * @card: card under test
140 * @scratch: transfer buffer
141 * @buffer: transfer buffer
142 * @highmem: buffer for highmem tests
143 * @area: information for performance tests
144 * @gr: pointer to results of current testcase
145 */
146struct mmc_test_card {
147 struct mmc_card *card;
148
149 u8 scratch[BUFFER_SIZE];
150 u8 *buffer;
151#ifdef CONFIG_HIGHMEM
152 struct page *highmem;
153#endif
154 struct mmc_test_area area;
155 struct mmc_test_general_result *gr;
156};
157
158enum mmc_test_prep_media {
159 MMC_TEST_PREP_NONE = 0,
160 MMC_TEST_PREP_WRITE_FULL = 1 << 0,
161 MMC_TEST_PREP_ERASE = 1 << 1,
162};
163
164struct mmc_test_multiple_rw {
165 unsigned int *sg_len;
166 unsigned int *bs;
167 unsigned int len;
168 unsigned int size;
169 bool do_write;
170 bool do_nonblock_req;
171 enum mmc_test_prep_media prepare;
172};
173
174/*******************************************************************/
175/* General helper functions */
176/*******************************************************************/
177
178/*
179 * Configure correct block size in card
180 */
181static int mmc_test_set_blksize(struct mmc_test_card *test, unsigned size)
182{
183 return mmc_set_blocklen(test->card, size);
184}
185
186static bool mmc_test_card_cmd23(struct mmc_card *card)
187{
188 return mmc_card_mmc(card) ||
189 (mmc_card_sd(card) && card->scr.cmds & SD_SCR_CMD23_SUPPORT);
190}
191
192static void mmc_test_prepare_sbc(struct mmc_test_card *test,
193 struct mmc_request *mrq, unsigned int blocks)
194{
195 struct mmc_card *card = test->card;
196
197 if (!mrq->sbc || !mmc_host_cmd23(card->host) ||
198 !mmc_test_card_cmd23(card) || !mmc_op_multi(mrq->cmd->opcode) ||
199 (card->quirks & MMC_QUIRK_BLK_NO_CMD23)) {
200 mrq->sbc = NULL;
201 return;
202 }
203
204 mrq->sbc->opcode = MMC_SET_BLOCK_COUNT;
205 mrq->sbc->arg = blocks;
206 mrq->sbc->flags = MMC_RSP_R1 | MMC_CMD_AC;
207}
208
209/*
210 * Fill in the mmc_request structure given a set of transfer parameters.
211 */
212static void mmc_test_prepare_mrq(struct mmc_test_card *test,
213 struct mmc_request *mrq, struct scatterlist *sg, unsigned sg_len,
214 unsigned dev_addr, unsigned blocks, unsigned blksz, int write)
215{
216 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data || !mrq->stop))
217 return;
218
219 if (blocks > 1) {
220 mrq->cmd->opcode = write ?
221 MMC_WRITE_MULTIPLE_BLOCK : MMC_READ_MULTIPLE_BLOCK;
222 } else {
223 mrq->cmd->opcode = write ?
224 MMC_WRITE_BLOCK : MMC_READ_SINGLE_BLOCK;
225 }
226
227 mrq->cmd->arg = dev_addr;
228 if (!mmc_card_blockaddr(test->card))
229 mrq->cmd->arg <<= 9;
230
231 mrq->cmd->flags = MMC_RSP_R1 | MMC_CMD_ADTC;
232
233 if (blocks == 1)
234 mrq->stop = NULL;
235 else {
236 mrq->stop->opcode = MMC_STOP_TRANSMISSION;
237 mrq->stop->arg = 0;
238 mrq->stop->flags = MMC_RSP_R1B | MMC_CMD_AC;
239 }
240
241 mrq->data->blksz = blksz;
242 mrq->data->blocks = blocks;
243 mrq->data->flags = write ? MMC_DATA_WRITE : MMC_DATA_READ;
244 mrq->data->sg = sg;
245 mrq->data->sg_len = sg_len;
246
247 mmc_test_prepare_sbc(test, mrq, blocks);
248
249 mmc_set_data_timeout(mrq->data, test->card);
250}
251
252static int mmc_test_busy(struct mmc_command *cmd)
253{
254 return !(cmd->resp[0] & R1_READY_FOR_DATA) ||
255 (R1_CURRENT_STATE(cmd->resp[0]) == R1_STATE_PRG);
256}
257
258/*
259 * Wait for the card to finish the busy state
260 */
261static int mmc_test_wait_busy(struct mmc_test_card *test)
262{
263 int ret, busy;
264 struct mmc_command cmd = {};
265
266 busy = 0;
267 do {
268 memset(&cmd, 0, sizeof(struct mmc_command));
269
270 cmd.opcode = MMC_SEND_STATUS;
271 cmd.arg = test->card->rca << 16;
272 cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
273
274 ret = mmc_wait_for_cmd(test->card->host, &cmd, 0);
275 if (ret)
276 break;
277
278 if (!busy && mmc_test_busy(&cmd)) {
279 busy = 1;
280 if (test->card->host->caps & MMC_CAP_WAIT_WHILE_BUSY)
281 pr_info("%s: Warning: Host did not wait for busy state to end.\n",
282 mmc_hostname(test->card->host));
283 }
284 } while (mmc_test_busy(&cmd));
285
286 return ret;
287}
288
289/*
290 * Transfer a single sector of kernel addressable data
291 */
292static int mmc_test_buffer_transfer(struct mmc_test_card *test,
293 u8 *buffer, unsigned addr, unsigned blksz, int write)
294{
295 struct mmc_request mrq = {};
296 struct mmc_command cmd = {};
297 struct mmc_command stop = {};
298 struct mmc_data data = {};
299
300 struct scatterlist sg;
301
302 mrq.cmd = &cmd;
303 mrq.data = &data;
304 mrq.stop = &stop;
305
306 sg_init_one(&sg, buffer, blksz);
307
308 mmc_test_prepare_mrq(test, &mrq, &sg, 1, addr, 1, blksz, write);
309
310 mmc_wait_for_req(test->card->host, &mrq);
311
312 if (cmd.error)
313 return cmd.error;
314 if (data.error)
315 return data.error;
316
317 return mmc_test_wait_busy(test);
318}
319
320static void mmc_test_free_mem(struct mmc_test_mem *mem)
321{
322 if (!mem)
323 return;
324 while (mem->cnt--)
325 __free_pages(mem->arr[mem->cnt].page,
326 mem->arr[mem->cnt].order);
327 kfree(mem->arr);
328 kfree(mem);
329}
330
331/*
332 * Allocate a lot of memory, preferably max_sz but at least min_sz. In case
333 * there isn't much memory do not exceed 1/16th total lowmem pages. Also do
334 * not exceed a maximum number of segments and try not to make segments much
335 * bigger than maximum segment size.
336 */
337static struct mmc_test_mem *mmc_test_alloc_mem(unsigned long min_sz,
338 unsigned long max_sz,
339 unsigned int max_segs,
340 unsigned int max_seg_sz)
341{
342 unsigned long max_page_cnt = DIV_ROUND_UP(max_sz, PAGE_SIZE);
343 unsigned long min_page_cnt = DIV_ROUND_UP(min_sz, PAGE_SIZE);
344 unsigned long max_seg_page_cnt = DIV_ROUND_UP(max_seg_sz, PAGE_SIZE);
345 unsigned long page_cnt = 0;
346 unsigned long limit = nr_free_buffer_pages() >> 4;
347 struct mmc_test_mem *mem;
348
349 if (max_page_cnt > limit)
350 max_page_cnt = limit;
351 if (min_page_cnt > max_page_cnt)
352 min_page_cnt = max_page_cnt;
353
354 if (max_seg_page_cnt > max_page_cnt)
355 max_seg_page_cnt = max_page_cnt;
356
357 if (max_segs > max_page_cnt)
358 max_segs = max_page_cnt;
359
360 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
361 if (!mem)
362 return NULL;
363
364 mem->arr = kcalloc(max_segs, sizeof(*mem->arr), GFP_KERNEL);
365 if (!mem->arr)
366 goto out_free;
367
368 while (max_page_cnt) {
369 struct page *page;
370 unsigned int order;
371 gfp_t flags = GFP_KERNEL | GFP_DMA | __GFP_NOWARN |
372 __GFP_NORETRY;
373
374 order = get_order(max_seg_page_cnt << PAGE_SHIFT);
375 while (1) {
376 page = alloc_pages(flags, order);
377 if (page || !order)
378 break;
379 order -= 1;
380 }
381 if (!page) {
382 if (page_cnt < min_page_cnt)
383 goto out_free;
384 break;
385 }
386 mem->arr[mem->cnt].page = page;
387 mem->arr[mem->cnt].order = order;
388 mem->cnt += 1;
389 if (max_page_cnt <= (1UL << order))
390 break;
391 max_page_cnt -= 1UL << order;
392 page_cnt += 1UL << order;
393 if (mem->cnt >= max_segs) {
394 if (page_cnt < min_page_cnt)
395 goto out_free;
396 break;
397 }
398 }
399
400 return mem;
401
402out_free:
403 mmc_test_free_mem(mem);
404 return NULL;
405}
406
407/*
408 * Map memory into a scatterlist. Optionally allow the same memory to be
409 * mapped more than once.
410 */
411static int mmc_test_map_sg(struct mmc_test_mem *mem, unsigned long size,
412 struct scatterlist *sglist, int repeat,
413 unsigned int max_segs, unsigned int max_seg_sz,
414 unsigned int *sg_len, int min_sg_len)
415{
416 struct scatterlist *sg = NULL;
417 unsigned int i;
418 unsigned long sz = size;
419
420 sg_init_table(sglist, max_segs);
421 if (min_sg_len > max_segs)
422 min_sg_len = max_segs;
423
424 *sg_len = 0;
425 do {
426 for (i = 0; i < mem->cnt; i++) {
427 unsigned long len = PAGE_SIZE << mem->arr[i].order;
428
429 if (min_sg_len && (size / min_sg_len < len))
430 len = ALIGN(size / min_sg_len, 512);
431 if (len > sz)
432 len = sz;
433 if (len > max_seg_sz)
434 len = max_seg_sz;
435 if (sg)
436 sg = sg_next(sg);
437 else
438 sg = sglist;
439 if (!sg)
440 return -EINVAL;
441 sg_set_page(sg, mem->arr[i].page, len, 0);
442 sz -= len;
443 *sg_len += 1;
444 if (!sz)
445 break;
446 }
447 } while (sz && repeat);
448
449 if (sz)
450 return -EINVAL;
451
452 if (sg)
453 sg_mark_end(sg);
454
455 return 0;
456}
457
458/*
459 * Map memory into a scatterlist so that no pages are contiguous. Allow the
460 * same memory to be mapped more than once.
461 */
462static int mmc_test_map_sg_max_scatter(struct mmc_test_mem *mem,
463 unsigned long sz,
464 struct scatterlist *sglist,
465 unsigned int max_segs,
466 unsigned int max_seg_sz,
467 unsigned int *sg_len)
468{
469 struct scatterlist *sg = NULL;
470 unsigned int i = mem->cnt, cnt;
471 unsigned long len;
472 void *base, *addr, *last_addr = NULL;
473
474 sg_init_table(sglist, max_segs);
475
476 *sg_len = 0;
477 while (sz) {
478 base = page_address(mem->arr[--i].page);
479 cnt = 1 << mem->arr[i].order;
480 while (sz && cnt) {
481 addr = base + PAGE_SIZE * --cnt;
482 if (last_addr && last_addr + PAGE_SIZE == addr)
483 continue;
484 last_addr = addr;
485 len = PAGE_SIZE;
486 if (len > max_seg_sz)
487 len = max_seg_sz;
488 if (len > sz)
489 len = sz;
490 if (sg)
491 sg = sg_next(sg);
492 else
493 sg = sglist;
494 if (!sg)
495 return -EINVAL;
496 sg_set_page(sg, virt_to_page(addr), len, 0);
497 sz -= len;
498 *sg_len += 1;
499 }
500 if (i == 0)
501 i = mem->cnt;
502 }
503
504 if (sg)
505 sg_mark_end(sg);
506
507 return 0;
508}
509
510/*
511 * Calculate transfer rate in bytes per second.
512 */
513static unsigned int mmc_test_rate(uint64_t bytes, struct timespec64 *ts)
514{
515 uint64_t ns;
516
517 ns = timespec64_to_ns(ts);
518 bytes *= 1000000000;
519
520 while (ns > UINT_MAX) {
521 bytes >>= 1;
522 ns >>= 1;
523 }
524
525 if (!ns)
526 return 0;
527
528 do_div(bytes, (uint32_t)ns);
529
530 return bytes;
531}
532
533/*
534 * Save transfer results for future usage
535 */
536static void mmc_test_save_transfer_result(struct mmc_test_card *test,
537 unsigned int count, unsigned int sectors, struct timespec64 ts,
538 unsigned int rate, unsigned int iops)
539{
540 struct mmc_test_transfer_result *tr;
541
542 if (!test->gr)
543 return;
544
545 tr = kmalloc(sizeof(*tr), GFP_KERNEL);
546 if (!tr)
547 return;
548
549 tr->count = count;
550 tr->sectors = sectors;
551 tr->ts = ts;
552 tr->rate = rate;
553 tr->iops = iops;
554
555 list_add_tail(&tr->link, &test->gr->tr_lst);
556}
557
558/*
559 * Print the transfer rate.
560 */
561static void mmc_test_print_rate(struct mmc_test_card *test, uint64_t bytes,
562 struct timespec64 *ts1, struct timespec64 *ts2)
563{
564 unsigned int rate, iops, sectors = bytes >> 9;
565 struct timespec64 ts;
566
567 ts = timespec64_sub(*ts2, *ts1);
568
569 rate = mmc_test_rate(bytes, &ts);
570 iops = mmc_test_rate(100, &ts); /* I/O ops per sec x 100 */
571
572 pr_info("%s: Transfer of %u sectors (%u%s KiB) took %llu.%09u "
573 "seconds (%u kB/s, %u KiB/s, %u.%02u IOPS)\n",
574 mmc_hostname(test->card->host), sectors, sectors >> 1,
575 (sectors & 1 ? ".5" : ""), (u64)ts.tv_sec,
576 (u32)ts.tv_nsec, rate / 1000, rate / 1024,
577 iops / 100, iops % 100);
578
579 mmc_test_save_transfer_result(test, 1, sectors, ts, rate, iops);
580}
581
582/*
583 * Print the average transfer rate.
584 */
585static void mmc_test_print_avg_rate(struct mmc_test_card *test, uint64_t bytes,
586 unsigned int count, struct timespec64 *ts1,
587 struct timespec64 *ts2)
588{
589 unsigned int rate, iops, sectors = bytes >> 9;
590 uint64_t tot = bytes * count;
591 struct timespec64 ts;
592
593 ts = timespec64_sub(*ts2, *ts1);
594
595 rate = mmc_test_rate(tot, &ts);
596 iops = mmc_test_rate(count * 100, &ts); /* I/O ops per sec x 100 */
597
598 pr_info("%s: Transfer of %u x %u sectors (%u x %u%s KiB) took "
599 "%llu.%09u seconds (%u kB/s, %u KiB/s, "
600 "%u.%02u IOPS, sg_len %d)\n",
601 mmc_hostname(test->card->host), count, sectors, count,
602 sectors >> 1, (sectors & 1 ? ".5" : ""),
603 (u64)ts.tv_sec, (u32)ts.tv_nsec,
604 rate / 1000, rate / 1024, iops / 100, iops % 100,
605 test->area.sg_len);
606
607 mmc_test_save_transfer_result(test, count, sectors, ts, rate, iops);
608}
609
610/*
611 * Return the card size in sectors.
612 */
613static unsigned int mmc_test_capacity(struct mmc_card *card)
614{
615 if (!mmc_card_sd(card) && mmc_card_blockaddr(card))
616 return card->ext_csd.sectors;
617 else
618 return card->csd.capacity << (card->csd.read_blkbits - 9);
619}
620
621/*******************************************************************/
622/* Test preparation and cleanup */
623/*******************************************************************/
624
625/*
626 * Fill the first couple of sectors of the card with known data
627 * so that bad reads/writes can be detected
628 */
629static int __mmc_test_prepare(struct mmc_test_card *test, int write)
630{
631 int ret, i;
632
633 ret = mmc_test_set_blksize(test, 512);
634 if (ret)
635 return ret;
636
637 if (write)
638 memset(test->buffer, 0xDF, 512);
639 else {
640 for (i = 0; i < 512; i++)
641 test->buffer[i] = i;
642 }
643
644 for (i = 0; i < BUFFER_SIZE / 512; i++) {
645 ret = mmc_test_buffer_transfer(test, test->buffer, i, 512, 1);
646 if (ret)
647 return ret;
648 }
649
650 return 0;
651}
652
653static int mmc_test_prepare_write(struct mmc_test_card *test)
654{
655 return __mmc_test_prepare(test, 1);
656}
657
658static int mmc_test_prepare_read(struct mmc_test_card *test)
659{
660 return __mmc_test_prepare(test, 0);
661}
662
663static int mmc_test_cleanup(struct mmc_test_card *test)
664{
665 int ret, i;
666
667 ret = mmc_test_set_blksize(test, 512);
668 if (ret)
669 return ret;
670
671 memset(test->buffer, 0, 512);
672
673 for (i = 0; i < BUFFER_SIZE / 512; i++) {
674 ret = mmc_test_buffer_transfer(test, test->buffer, i, 512, 1);
675 if (ret)
676 return ret;
677 }
678
679 return 0;
680}
681
682/*******************************************************************/
683/* Test execution helpers */
684/*******************************************************************/
685
686/*
687 * Modifies the mmc_request to perform the "short transfer" tests
688 */
689static void mmc_test_prepare_broken_mrq(struct mmc_test_card *test,
690 struct mmc_request *mrq, int write)
691{
692 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
693 return;
694
695 if (mrq->data->blocks > 1) {
696 mrq->cmd->opcode = write ?
697 MMC_WRITE_BLOCK : MMC_READ_SINGLE_BLOCK;
698 mrq->stop = NULL;
699 } else {
700 mrq->cmd->opcode = MMC_SEND_STATUS;
701 mrq->cmd->arg = test->card->rca << 16;
702 }
703}
704
705/*
706 * Checks that a normal transfer didn't have any errors
707 */
708static int mmc_test_check_result(struct mmc_test_card *test,
709 struct mmc_request *mrq)
710{
711 int ret;
712
713 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
714 return -EINVAL;
715
716 ret = 0;
717
718 if (mrq->sbc && mrq->sbc->error)
719 ret = mrq->sbc->error;
720 if (!ret && mrq->cmd->error)
721 ret = mrq->cmd->error;
722 if (!ret && mrq->data->error)
723 ret = mrq->data->error;
724 if (!ret && mrq->stop && mrq->stop->error)
725 ret = mrq->stop->error;
726 if (!ret && mrq->data->bytes_xfered !=
727 mrq->data->blocks * mrq->data->blksz)
728 ret = RESULT_FAIL;
729
730 if (ret == -EINVAL)
731 ret = RESULT_UNSUP_HOST;
732
733 return ret;
734}
735
736/*
737 * Checks that a "short transfer" behaved as expected
738 */
739static int mmc_test_check_broken_result(struct mmc_test_card *test,
740 struct mmc_request *mrq)
741{
742 int ret;
743
744 if (WARN_ON(!mrq || !mrq->cmd || !mrq->data))
745 return -EINVAL;
746
747 ret = 0;
748
749 if (!ret && mrq->cmd->error)
750 ret = mrq->cmd->error;
751 if (!ret && mrq->data->error == 0)
752 ret = RESULT_FAIL;
753 if (!ret && mrq->data->error != -ETIMEDOUT)
754 ret = mrq->data->error;
755 if (!ret && mrq->stop && mrq->stop->error)
756 ret = mrq->stop->error;
757 if (mrq->data->blocks > 1) {
758 if (!ret && mrq->data->bytes_xfered > mrq->data->blksz)
759 ret = RESULT_FAIL;
760 } else {
761 if (!ret && mrq->data->bytes_xfered > 0)
762 ret = RESULT_FAIL;
763 }
764
765 if (ret == -EINVAL)
766 ret = RESULT_UNSUP_HOST;
767
768 return ret;
769}
770
771struct mmc_test_req {
772 struct mmc_request mrq;
773 struct mmc_command sbc;
774 struct mmc_command cmd;
775 struct mmc_command stop;
776 struct mmc_command status;
777 struct mmc_data data;
778};
779
780/*
781 * Tests nonblock transfer with certain parameters
782 */
783static void mmc_test_req_reset(struct mmc_test_req *rq)
784{
785 memset(rq, 0, sizeof(struct mmc_test_req));
786
787 rq->mrq.cmd = &rq->cmd;
788 rq->mrq.data = &rq->data;
789 rq->mrq.stop = &rq->stop;
790}
791
792static struct mmc_test_req *mmc_test_req_alloc(void)
793{
794 struct mmc_test_req *rq = kmalloc(sizeof(*rq), GFP_KERNEL);
795
796 if (rq)
797 mmc_test_req_reset(rq);
798
799 return rq;
800}
801
802static void mmc_test_wait_done(struct mmc_request *mrq)
803{
804 complete(&mrq->completion);
805}
806
807static int mmc_test_start_areq(struct mmc_test_card *test,
808 struct mmc_request *mrq,
809 struct mmc_request *prev_mrq)
810{
811 struct mmc_host *host = test->card->host;
812 int err = 0;
813
814 if (mrq) {
815 init_completion(&mrq->completion);
816 mrq->done = mmc_test_wait_done;
817 mmc_pre_req(host, mrq);
818 }
819
820 if (prev_mrq) {
821 wait_for_completion(&prev_mrq->completion);
822 err = mmc_test_wait_busy(test);
823 if (!err)
824 err = mmc_test_check_result(test, prev_mrq);
825 }
826
827 if (!err && mrq) {
828 err = mmc_start_request(host, mrq);
829 if (err)
830 mmc_retune_release(host);
831 }
832
833 if (prev_mrq)
834 mmc_post_req(host, prev_mrq, 0);
835
836 if (err && mrq)
837 mmc_post_req(host, mrq, err);
838
839 return err;
840}
841
842static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
843 struct scatterlist *sg, unsigned sg_len,
844 unsigned dev_addr, unsigned blocks,
845 unsigned blksz, int write, int count)
846{
847 struct mmc_test_req *rq1, *rq2;
848 struct mmc_request *mrq, *prev_mrq;
849 int i;
850 int ret = RESULT_OK;
851
852 rq1 = mmc_test_req_alloc();
853 rq2 = mmc_test_req_alloc();
854 if (!rq1 || !rq2) {
855 ret = RESULT_FAIL;
856 goto err;
857 }
858
859 mrq = &rq1->mrq;
860 prev_mrq = NULL;
861
862 for (i = 0; i < count; i++) {
863 mmc_test_req_reset(container_of(mrq, struct mmc_test_req, mrq));
864 mmc_test_prepare_mrq(test, mrq, sg, sg_len, dev_addr, blocks,
865 blksz, write);
866 ret = mmc_test_start_areq(test, mrq, prev_mrq);
867 if (ret)
868 goto err;
869
870 if (!prev_mrq)
871 prev_mrq = &rq2->mrq;
872
873 swap(mrq, prev_mrq);
874 dev_addr += blocks;
875 }
876
877 ret = mmc_test_start_areq(test, NULL, prev_mrq);
878err:
879 kfree(rq1);
880 kfree(rq2);
881 return ret;
882}
883
884/*
885 * Tests a basic transfer with certain parameters
886 */
887static int mmc_test_simple_transfer(struct mmc_test_card *test,
888 struct scatterlist *sg, unsigned sg_len, unsigned dev_addr,
889 unsigned blocks, unsigned blksz, int write)
890{
891 struct mmc_request mrq = {};
892 struct mmc_command cmd = {};
893 struct mmc_command stop = {};
894 struct mmc_data data = {};
895
896 mrq.cmd = &cmd;
897 mrq.data = &data;
898 mrq.stop = &stop;
899
900 mmc_test_prepare_mrq(test, &mrq, sg, sg_len, dev_addr,
901 blocks, blksz, write);
902
903 mmc_wait_for_req(test->card->host, &mrq);
904
905 mmc_test_wait_busy(test);
906
907 return mmc_test_check_result(test, &mrq);
908}
909
910/*
911 * Tests a transfer where the card will fail completely or partly
912 */
913static int mmc_test_broken_transfer(struct mmc_test_card *test,
914 unsigned blocks, unsigned blksz, int write)
915{
916 struct mmc_request mrq = {};
917 struct mmc_command cmd = {};
918 struct mmc_command stop = {};
919 struct mmc_data data = {};
920
921 struct scatterlist sg;
922
923 mrq.cmd = &cmd;
924 mrq.data = &data;
925 mrq.stop = &stop;
926
927 sg_init_one(&sg, test->buffer, blocks * blksz);
928
929 mmc_test_prepare_mrq(test, &mrq, &sg, 1, 0, blocks, blksz, write);
930 mmc_test_prepare_broken_mrq(test, &mrq, write);
931
932 mmc_wait_for_req(test->card->host, &mrq);
933
934 mmc_test_wait_busy(test);
935
936 return mmc_test_check_broken_result(test, &mrq);
937}
938
939/*
940 * Does a complete transfer test where data is also validated
941 *
942 * Note: mmc_test_prepare() must have been done before this call
943 */
944static int mmc_test_transfer(struct mmc_test_card *test,
945 struct scatterlist *sg, unsigned sg_len, unsigned dev_addr,
946 unsigned blocks, unsigned blksz, int write)
947{
948 int ret, i;
949 unsigned long flags;
950
951 if (write) {
952 for (i = 0; i < blocks * blksz; i++)
953 test->scratch[i] = i;
954 } else {
955 memset(test->scratch, 0, BUFFER_SIZE);
956 }
957 local_irq_save(flags);
958 sg_copy_from_buffer(sg, sg_len, test->scratch, BUFFER_SIZE);
959 local_irq_restore(flags);
960
961 ret = mmc_test_set_blksize(test, blksz);
962 if (ret)
963 return ret;
964
965 ret = mmc_test_simple_transfer(test, sg, sg_len, dev_addr,
966 blocks, blksz, write);
967 if (ret)
968 return ret;
969
970 if (write) {
971 int sectors;
972
973 ret = mmc_test_set_blksize(test, 512);
974 if (ret)
975 return ret;
976
977 sectors = (blocks * blksz + 511) / 512;
978 if ((sectors * 512) == (blocks * blksz))
979 sectors++;
980
981 if ((sectors * 512) > BUFFER_SIZE)
982 return -EINVAL;
983
984 memset(test->buffer, 0, sectors * 512);
985
986 for (i = 0; i < sectors; i++) {
987 ret = mmc_test_buffer_transfer(test,
988 test->buffer + i * 512,
989 dev_addr + i, 512, 0);
990 if (ret)
991 return ret;
992 }
993
994 for (i = 0; i < blocks * blksz; i++) {
995 if (test->buffer[i] != (u8)i)
996 return RESULT_FAIL;
997 }
998
999 for (; i < sectors * 512; i++) {
1000 if (test->buffer[i] != 0xDF)
1001 return RESULT_FAIL;
1002 }
1003 } else {
1004 local_irq_save(flags);
1005 sg_copy_to_buffer(sg, sg_len, test->scratch, BUFFER_SIZE);
1006 local_irq_restore(flags);
1007 for (i = 0; i < blocks * blksz; i++) {
1008 if (test->scratch[i] != (u8)i)
1009 return RESULT_FAIL;
1010 }
1011 }
1012
1013 return 0;
1014}
1015
1016/*******************************************************************/
1017/* Tests */
1018/*******************************************************************/
1019
1020struct mmc_test_case {
1021 const char *name;
1022
1023 int (*prepare)(struct mmc_test_card *);
1024 int (*run)(struct mmc_test_card *);
1025 int (*cleanup)(struct mmc_test_card *);
1026};
1027
1028static int mmc_test_basic_write(struct mmc_test_card *test)
1029{
1030 int ret;
1031 struct scatterlist sg;
1032
1033 ret = mmc_test_set_blksize(test, 512);
1034 if (ret)
1035 return ret;
1036
1037 sg_init_one(&sg, test->buffer, 512);
1038
1039 return mmc_test_simple_transfer(test, &sg, 1, 0, 1, 512, 1);
1040}
1041
1042static int mmc_test_basic_read(struct mmc_test_card *test)
1043{
1044 int ret;
1045 struct scatterlist sg;
1046
1047 ret = mmc_test_set_blksize(test, 512);
1048 if (ret)
1049 return ret;
1050
1051 sg_init_one(&sg, test->buffer, 512);
1052
1053 return mmc_test_simple_transfer(test, &sg, 1, 0, 1, 512, 0);
1054}
1055
1056static int mmc_test_verify_write(struct mmc_test_card *test)
1057{
1058 struct scatterlist sg;
1059
1060 sg_init_one(&sg, test->buffer, 512);
1061
1062 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 1);
1063}
1064
1065static int mmc_test_verify_read(struct mmc_test_card *test)
1066{
1067 struct scatterlist sg;
1068
1069 sg_init_one(&sg, test->buffer, 512);
1070
1071 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 0);
1072}
1073
1074static int mmc_test_multi_write(struct mmc_test_card *test)
1075{
1076 unsigned int size;
1077 struct scatterlist sg;
1078
1079 if (test->card->host->max_blk_count == 1)
1080 return RESULT_UNSUP_HOST;
1081
1082 size = PAGE_SIZE * 2;
1083 size = min(size, test->card->host->max_req_size);
1084 size = min(size, test->card->host->max_seg_size);
1085 size = min(size, test->card->host->max_blk_count * 512);
1086
1087 if (size < 1024)
1088 return RESULT_UNSUP_HOST;
1089
1090 sg_init_one(&sg, test->buffer, size);
1091
1092 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 1);
1093}
1094
1095static int mmc_test_multi_read(struct mmc_test_card *test)
1096{
1097 unsigned int size;
1098 struct scatterlist sg;
1099
1100 if (test->card->host->max_blk_count == 1)
1101 return RESULT_UNSUP_HOST;
1102
1103 size = PAGE_SIZE * 2;
1104 size = min(size, test->card->host->max_req_size);
1105 size = min(size, test->card->host->max_seg_size);
1106 size = min(size, test->card->host->max_blk_count * 512);
1107
1108 if (size < 1024)
1109 return RESULT_UNSUP_HOST;
1110
1111 sg_init_one(&sg, test->buffer, size);
1112
1113 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 0);
1114}
1115
1116static int mmc_test_pow2_write(struct mmc_test_card *test)
1117{
1118 int ret, i;
1119 struct scatterlist sg;
1120
1121 if (!test->card->csd.write_partial)
1122 return RESULT_UNSUP_CARD;
1123
1124 for (i = 1; i < 512; i <<= 1) {
1125 sg_init_one(&sg, test->buffer, i);
1126 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 1);
1127 if (ret)
1128 return ret;
1129 }
1130
1131 return 0;
1132}
1133
1134static int mmc_test_pow2_read(struct mmc_test_card *test)
1135{
1136 int ret, i;
1137 struct scatterlist sg;
1138
1139 if (!test->card->csd.read_partial)
1140 return RESULT_UNSUP_CARD;
1141
1142 for (i = 1; i < 512; i <<= 1) {
1143 sg_init_one(&sg, test->buffer, i);
1144 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 0);
1145 if (ret)
1146 return ret;
1147 }
1148
1149 return 0;
1150}
1151
1152static int mmc_test_weird_write(struct mmc_test_card *test)
1153{
1154 int ret, i;
1155 struct scatterlist sg;
1156
1157 if (!test->card->csd.write_partial)
1158 return RESULT_UNSUP_CARD;
1159
1160 for (i = 3; i < 512; i += 7) {
1161 sg_init_one(&sg, test->buffer, i);
1162 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 1);
1163 if (ret)
1164 return ret;
1165 }
1166
1167 return 0;
1168}
1169
1170static int mmc_test_weird_read(struct mmc_test_card *test)
1171{
1172 int ret, i;
1173 struct scatterlist sg;
1174
1175 if (!test->card->csd.read_partial)
1176 return RESULT_UNSUP_CARD;
1177
1178 for (i = 3; i < 512; i += 7) {
1179 sg_init_one(&sg, test->buffer, i);
1180 ret = mmc_test_transfer(test, &sg, 1, 0, 1, i, 0);
1181 if (ret)
1182 return ret;
1183 }
1184
1185 return 0;
1186}
1187
1188static int mmc_test_align_write(struct mmc_test_card *test)
1189{
1190 int ret, i;
1191 struct scatterlist sg;
1192
1193 for (i = 1; i < TEST_ALIGN_END; i++) {
1194 sg_init_one(&sg, test->buffer + i, 512);
1195 ret = mmc_test_transfer(test, &sg, 1, 0, 1, 512, 1);
1196 if (ret)
1197 return ret;
1198 }
1199
1200 return 0;
1201}
1202
1203static int mmc_test_align_read(struct mmc_test_card *test)
1204{
1205 int ret, i;
1206 struct scatterlist sg;
1207
1208 for (i = 1; i < TEST_ALIGN_END; i++) {
1209 sg_init_one(&sg, test->buffer + i, 512);
1210 ret = mmc_test_transfer(test, &sg, 1, 0, 1, 512, 0);
1211 if (ret)
1212 return ret;
1213 }
1214
1215 return 0;
1216}
1217
1218static int mmc_test_align_multi_write(struct mmc_test_card *test)
1219{
1220 int ret, i;
1221 unsigned int size;
1222 struct scatterlist sg;
1223
1224 if (test->card->host->max_blk_count == 1)
1225 return RESULT_UNSUP_HOST;
1226
1227 size = PAGE_SIZE * 2;
1228 size = min(size, test->card->host->max_req_size);
1229 size = min(size, test->card->host->max_seg_size);
1230 size = min(size, test->card->host->max_blk_count * 512);
1231
1232 if (size < 1024)
1233 return RESULT_UNSUP_HOST;
1234
1235 for (i = 1; i < TEST_ALIGN_END; i++) {
1236 sg_init_one(&sg, test->buffer + i, size);
1237 ret = mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 1);
1238 if (ret)
1239 return ret;
1240 }
1241
1242 return 0;
1243}
1244
1245static int mmc_test_align_multi_read(struct mmc_test_card *test)
1246{
1247 int ret, i;
1248 unsigned int size;
1249 struct scatterlist sg;
1250
1251 if (test->card->host->max_blk_count == 1)
1252 return RESULT_UNSUP_HOST;
1253
1254 size = PAGE_SIZE * 2;
1255 size = min(size, test->card->host->max_req_size);
1256 size = min(size, test->card->host->max_seg_size);
1257 size = min(size, test->card->host->max_blk_count * 512);
1258
1259 if (size < 1024)
1260 return RESULT_UNSUP_HOST;
1261
1262 for (i = 1; i < TEST_ALIGN_END; i++) {
1263 sg_init_one(&sg, test->buffer + i, size);
1264 ret = mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 0);
1265 if (ret)
1266 return ret;
1267 }
1268
1269 return 0;
1270}
1271
1272static int mmc_test_xfersize_write(struct mmc_test_card *test)
1273{
1274 int ret;
1275
1276 ret = mmc_test_set_blksize(test, 512);
1277 if (ret)
1278 return ret;
1279
1280 return mmc_test_broken_transfer(test, 1, 512, 1);
1281}
1282
1283static int mmc_test_xfersize_read(struct mmc_test_card *test)
1284{
1285 int ret;
1286
1287 ret = mmc_test_set_blksize(test, 512);
1288 if (ret)
1289 return ret;
1290
1291 return mmc_test_broken_transfer(test, 1, 512, 0);
1292}
1293
1294static int mmc_test_multi_xfersize_write(struct mmc_test_card *test)
1295{
1296 int ret;
1297
1298 if (test->card->host->max_blk_count == 1)
1299 return RESULT_UNSUP_HOST;
1300
1301 ret = mmc_test_set_blksize(test, 512);
1302 if (ret)
1303 return ret;
1304
1305 return mmc_test_broken_transfer(test, 2, 512, 1);
1306}
1307
1308static int mmc_test_multi_xfersize_read(struct mmc_test_card *test)
1309{
1310 int ret;
1311
1312 if (test->card->host->max_blk_count == 1)
1313 return RESULT_UNSUP_HOST;
1314
1315 ret = mmc_test_set_blksize(test, 512);
1316 if (ret)
1317 return ret;
1318
1319 return mmc_test_broken_transfer(test, 2, 512, 0);
1320}
1321
1322#ifdef CONFIG_HIGHMEM
1323
1324static int mmc_test_write_high(struct mmc_test_card *test)
1325{
1326 struct scatterlist sg;
1327
1328 sg_init_table(&sg, 1);
1329 sg_set_page(&sg, test->highmem, 512, 0);
1330
1331 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 1);
1332}
1333
1334static int mmc_test_read_high(struct mmc_test_card *test)
1335{
1336 struct scatterlist sg;
1337
1338 sg_init_table(&sg, 1);
1339 sg_set_page(&sg, test->highmem, 512, 0);
1340
1341 return mmc_test_transfer(test, &sg, 1, 0, 1, 512, 0);
1342}
1343
1344static int mmc_test_multi_write_high(struct mmc_test_card *test)
1345{
1346 unsigned int size;
1347 struct scatterlist sg;
1348
1349 if (test->card->host->max_blk_count == 1)
1350 return RESULT_UNSUP_HOST;
1351
1352 size = PAGE_SIZE * 2;
1353 size = min(size, test->card->host->max_req_size);
1354 size = min(size, test->card->host->max_seg_size);
1355 size = min(size, test->card->host->max_blk_count * 512);
1356
1357 if (size < 1024)
1358 return RESULT_UNSUP_HOST;
1359
1360 sg_init_table(&sg, 1);
1361 sg_set_page(&sg, test->highmem, size, 0);
1362
1363 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 1);
1364}
1365
1366static int mmc_test_multi_read_high(struct mmc_test_card *test)
1367{
1368 unsigned int size;
1369 struct scatterlist sg;
1370
1371 if (test->card->host->max_blk_count == 1)
1372 return RESULT_UNSUP_HOST;
1373
1374 size = PAGE_SIZE * 2;
1375 size = min(size, test->card->host->max_req_size);
1376 size = min(size, test->card->host->max_seg_size);
1377 size = min(size, test->card->host->max_blk_count * 512);
1378
1379 if (size < 1024)
1380 return RESULT_UNSUP_HOST;
1381
1382 sg_init_table(&sg, 1);
1383 sg_set_page(&sg, test->highmem, size, 0);
1384
1385 return mmc_test_transfer(test, &sg, 1, 0, size / 512, 512, 0);
1386}
1387
1388#else
1389
1390static int mmc_test_no_highmem(struct mmc_test_card *test)
1391{
1392 pr_info("%s: Highmem not configured - test skipped\n",
1393 mmc_hostname(test->card->host));
1394 return 0;
1395}
1396
1397#endif /* CONFIG_HIGHMEM */
1398
1399/*
1400 * Map sz bytes so that it can be transferred.
1401 */
1402static int mmc_test_area_map(struct mmc_test_card *test, unsigned long sz,
1403 int max_scatter, int min_sg_len)
1404{
1405 struct mmc_test_area *t = &test->area;
1406 int err;
1407
1408 t->blocks = sz >> 9;
1409
1410 if (max_scatter) {
1411 err = mmc_test_map_sg_max_scatter(t->mem, sz, t->sg,
1412 t->max_segs, t->max_seg_sz,
1413 &t->sg_len);
1414 } else {
1415 err = mmc_test_map_sg(t->mem, sz, t->sg, 1, t->max_segs,
1416 t->max_seg_sz, &t->sg_len, min_sg_len);
1417 }
1418 if (err)
1419 pr_info("%s: Failed to map sg list\n",
1420 mmc_hostname(test->card->host));
1421 return err;
1422}
1423
1424/*
1425 * Transfer bytes mapped by mmc_test_area_map().
1426 */
1427static int mmc_test_area_transfer(struct mmc_test_card *test,
1428 unsigned int dev_addr, int write)
1429{
1430 struct mmc_test_area *t = &test->area;
1431
1432 return mmc_test_simple_transfer(test, t->sg, t->sg_len, dev_addr,
1433 t->blocks, 512, write);
1434}
1435
1436/*
1437 * Map and transfer bytes for multiple transfers.
1438 */
1439static int mmc_test_area_io_seq(struct mmc_test_card *test, unsigned long sz,
1440 unsigned int dev_addr, int write,
1441 int max_scatter, int timed, int count,
1442 bool nonblock, int min_sg_len)
1443{
1444 struct timespec64 ts1, ts2;
1445 int ret = 0;
1446 int i;
1447 struct mmc_test_area *t = &test->area;
1448
1449 /*
1450 * In the case of a maximally scattered transfer, the maximum transfer
1451 * size is further limited by using PAGE_SIZE segments.
1452 */
1453 if (max_scatter) {
1454 struct mmc_test_area *t = &test->area;
1455 unsigned long max_tfr;
1456
1457 if (t->max_seg_sz >= PAGE_SIZE)
1458 max_tfr = t->max_segs * PAGE_SIZE;
1459 else
1460 max_tfr = t->max_segs * t->max_seg_sz;
1461 if (sz > max_tfr)
1462 sz = max_tfr;
1463 }
1464
1465 ret = mmc_test_area_map(test, sz, max_scatter, min_sg_len);
1466 if (ret)
1467 return ret;
1468
1469 if (timed)
1470 ktime_get_ts64(&ts1);
1471 if (nonblock)
1472 ret = mmc_test_nonblock_transfer(test, t->sg, t->sg_len,
1473 dev_addr, t->blocks, 512, write, count);
1474 else
1475 for (i = 0; i < count && ret == 0; i++) {
1476 ret = mmc_test_area_transfer(test, dev_addr, write);
1477 dev_addr += sz >> 9;
1478 }
1479
1480 if (ret)
1481 return ret;
1482
1483 if (timed)
1484 ktime_get_ts64(&ts2);
1485
1486 if (timed)
1487 mmc_test_print_avg_rate(test, sz, count, &ts1, &ts2);
1488
1489 return 0;
1490}
1491
1492static int mmc_test_area_io(struct mmc_test_card *test, unsigned long sz,
1493 unsigned int dev_addr, int write, int max_scatter,
1494 int timed)
1495{
1496 return mmc_test_area_io_seq(test, sz, dev_addr, write, max_scatter,
1497 timed, 1, false, 0);
1498}
1499
1500/*
1501 * Write the test area entirely.
1502 */
1503static int mmc_test_area_fill(struct mmc_test_card *test)
1504{
1505 struct mmc_test_area *t = &test->area;
1506
1507 return mmc_test_area_io(test, t->max_tfr, t->dev_addr, 1, 0, 0);
1508}
1509
1510/*
1511 * Erase the test area entirely.
1512 */
1513static int mmc_test_area_erase(struct mmc_test_card *test)
1514{
1515 struct mmc_test_area *t = &test->area;
1516
1517 if (!mmc_can_erase(test->card))
1518 return 0;
1519
1520 return mmc_erase(test->card, t->dev_addr, t->max_sz >> 9,
1521 MMC_ERASE_ARG);
1522}
1523
1524/*
1525 * Cleanup struct mmc_test_area.
1526 */
1527static int mmc_test_area_cleanup(struct mmc_test_card *test)
1528{
1529 struct mmc_test_area *t = &test->area;
1530
1531 kfree(t->sg);
1532 mmc_test_free_mem(t->mem);
1533
1534 return 0;
1535}
1536
1537/*
1538 * Initialize an area for testing large transfers. The test area is set to the
1539 * middle of the card because cards may have different characteristics at the
1540 * front (for FAT file system optimization). Optionally, the area is erased
1541 * (if the card supports it) which may improve write performance. Optionally,
1542 * the area is filled with data for subsequent read tests.
1543 */
1544static int mmc_test_area_init(struct mmc_test_card *test, int erase, int fill)
1545{
1546 struct mmc_test_area *t = &test->area;
1547 unsigned long min_sz = 64 * 1024, sz;
1548 int ret;
1549
1550 ret = mmc_test_set_blksize(test, 512);
1551 if (ret)
1552 return ret;
1553
1554 /* Make the test area size about 4MiB */
1555 sz = (unsigned long)test->card->pref_erase << 9;
1556 t->max_sz = sz;
1557 while (t->max_sz < 4 * 1024 * 1024)
1558 t->max_sz += sz;
1559 while (t->max_sz > TEST_AREA_MAX_SIZE && t->max_sz > sz)
1560 t->max_sz -= sz;
1561
1562 t->max_segs = test->card->host->max_segs;
1563 t->max_seg_sz = test->card->host->max_seg_size;
1564 t->max_seg_sz -= t->max_seg_sz % 512;
1565
1566 t->max_tfr = t->max_sz;
1567 if (t->max_tfr >> 9 > test->card->host->max_blk_count)
1568 t->max_tfr = test->card->host->max_blk_count << 9;
1569 if (t->max_tfr > test->card->host->max_req_size)
1570 t->max_tfr = test->card->host->max_req_size;
1571 if (t->max_tfr / t->max_seg_sz > t->max_segs)
1572 t->max_tfr = t->max_segs * t->max_seg_sz;
1573
1574 /*
1575 * Try to allocate enough memory for a max. sized transfer. Less is OK
1576 * because the same memory can be mapped into the scatterlist more than
1577 * once. Also, take into account the limits imposed on scatterlist
1578 * segments by the host driver.
1579 */
1580 t->mem = mmc_test_alloc_mem(min_sz, t->max_tfr, t->max_segs,
1581 t->max_seg_sz);
1582 if (!t->mem)
1583 return -ENOMEM;
1584
1585 t->sg = kmalloc_array(t->max_segs, sizeof(*t->sg), GFP_KERNEL);
1586 if (!t->sg) {
1587 ret = -ENOMEM;
1588 goto out_free;
1589 }
1590
1591 t->dev_addr = mmc_test_capacity(test->card) / 2;
1592 t->dev_addr -= t->dev_addr % (t->max_sz >> 9);
1593
1594 if (erase) {
1595 ret = mmc_test_area_erase(test);
1596 if (ret)
1597 goto out_free;
1598 }
1599
1600 if (fill) {
1601 ret = mmc_test_area_fill(test);
1602 if (ret)
1603 goto out_free;
1604 }
1605
1606 return 0;
1607
1608out_free:
1609 mmc_test_area_cleanup(test);
1610 return ret;
1611}
1612
1613/*
1614 * Prepare for large transfers. Do not erase the test area.
1615 */
1616static int mmc_test_area_prepare(struct mmc_test_card *test)
1617{
1618 return mmc_test_area_init(test, 0, 0);
1619}
1620
1621/*
1622 * Prepare for large transfers. Do erase the test area.
1623 */
1624static int mmc_test_area_prepare_erase(struct mmc_test_card *test)
1625{
1626 return mmc_test_area_init(test, 1, 0);
1627}
1628
1629/*
1630 * Prepare for large transfers. Erase and fill the test area.
1631 */
1632static int mmc_test_area_prepare_fill(struct mmc_test_card *test)
1633{
1634 return mmc_test_area_init(test, 1, 1);
1635}
1636
1637/*
1638 * Test best-case performance. Best-case performance is expected from
1639 * a single large transfer.
1640 *
1641 * An additional option (max_scatter) allows the measurement of the same
1642 * transfer but with no contiguous pages in the scatter list. This tests
1643 * the efficiency of DMA to handle scattered pages.
1644 */
1645static int mmc_test_best_performance(struct mmc_test_card *test, int write,
1646 int max_scatter)
1647{
1648 struct mmc_test_area *t = &test->area;
1649
1650 return mmc_test_area_io(test, t->max_tfr, t->dev_addr, write,
1651 max_scatter, 1);
1652}
1653
1654/*
1655 * Best-case read performance.
1656 */
1657static int mmc_test_best_read_performance(struct mmc_test_card *test)
1658{
1659 return mmc_test_best_performance(test, 0, 0);
1660}
1661
1662/*
1663 * Best-case write performance.
1664 */
1665static int mmc_test_best_write_performance(struct mmc_test_card *test)
1666{
1667 return mmc_test_best_performance(test, 1, 0);
1668}
1669
1670/*
1671 * Best-case read performance into scattered pages.
1672 */
1673static int mmc_test_best_read_perf_max_scatter(struct mmc_test_card *test)
1674{
1675 return mmc_test_best_performance(test, 0, 1);
1676}
1677
1678/*
1679 * Best-case write performance from scattered pages.
1680 */
1681static int mmc_test_best_write_perf_max_scatter(struct mmc_test_card *test)
1682{
1683 return mmc_test_best_performance(test, 1, 1);
1684}
1685
1686/*
1687 * Single read performance by transfer size.
1688 */
1689static int mmc_test_profile_read_perf(struct mmc_test_card *test)
1690{
1691 struct mmc_test_area *t = &test->area;
1692 unsigned long sz;
1693 unsigned int dev_addr;
1694 int ret;
1695
1696 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1697 dev_addr = t->dev_addr + (sz >> 9);
1698 ret = mmc_test_area_io(test, sz, dev_addr, 0, 0, 1);
1699 if (ret)
1700 return ret;
1701 }
1702 sz = t->max_tfr;
1703 dev_addr = t->dev_addr;
1704 return mmc_test_area_io(test, sz, dev_addr, 0, 0, 1);
1705}
1706
1707/*
1708 * Single write performance by transfer size.
1709 */
1710static int mmc_test_profile_write_perf(struct mmc_test_card *test)
1711{
1712 struct mmc_test_area *t = &test->area;
1713 unsigned long sz;
1714 unsigned int dev_addr;
1715 int ret;
1716
1717 ret = mmc_test_area_erase(test);
1718 if (ret)
1719 return ret;
1720 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1721 dev_addr = t->dev_addr + (sz >> 9);
1722 ret = mmc_test_area_io(test, sz, dev_addr, 1, 0, 1);
1723 if (ret)
1724 return ret;
1725 }
1726 ret = mmc_test_area_erase(test);
1727 if (ret)
1728 return ret;
1729 sz = t->max_tfr;
1730 dev_addr = t->dev_addr;
1731 return mmc_test_area_io(test, sz, dev_addr, 1, 0, 1);
1732}
1733
1734/*
1735 * Single trim performance by transfer size.
1736 */
1737static int mmc_test_profile_trim_perf(struct mmc_test_card *test)
1738{
1739 struct mmc_test_area *t = &test->area;
1740 unsigned long sz;
1741 unsigned int dev_addr;
1742 struct timespec64 ts1, ts2;
1743 int ret;
1744
1745 if (!mmc_can_trim(test->card))
1746 return RESULT_UNSUP_CARD;
1747
1748 if (!mmc_can_erase(test->card))
1749 return RESULT_UNSUP_HOST;
1750
1751 for (sz = 512; sz < t->max_sz; sz <<= 1) {
1752 dev_addr = t->dev_addr + (sz >> 9);
1753 ktime_get_ts64(&ts1);
1754 ret = mmc_erase(test->card, dev_addr, sz >> 9, MMC_TRIM_ARG);
1755 if (ret)
1756 return ret;
1757 ktime_get_ts64(&ts2);
1758 mmc_test_print_rate(test, sz, &ts1, &ts2);
1759 }
1760 dev_addr = t->dev_addr;
1761 ktime_get_ts64(&ts1);
1762 ret = mmc_erase(test->card, dev_addr, sz >> 9, MMC_TRIM_ARG);
1763 if (ret)
1764 return ret;
1765 ktime_get_ts64(&ts2);
1766 mmc_test_print_rate(test, sz, &ts1, &ts2);
1767 return 0;
1768}
1769
1770static int mmc_test_seq_read_perf(struct mmc_test_card *test, unsigned long sz)
1771{
1772 struct mmc_test_area *t = &test->area;
1773 unsigned int dev_addr, i, cnt;
1774 struct timespec64 ts1, ts2;
1775 int ret;
1776
1777 cnt = t->max_sz / sz;
1778 dev_addr = t->dev_addr;
1779 ktime_get_ts64(&ts1);
1780 for (i = 0; i < cnt; i++) {
1781 ret = mmc_test_area_io(test, sz, dev_addr, 0, 0, 0);
1782 if (ret)
1783 return ret;
1784 dev_addr += (sz >> 9);
1785 }
1786 ktime_get_ts64(&ts2);
1787 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1788 return 0;
1789}
1790
1791/*
1792 * Consecutive read performance by transfer size.
1793 */
1794static int mmc_test_profile_seq_read_perf(struct mmc_test_card *test)
1795{
1796 struct mmc_test_area *t = &test->area;
1797 unsigned long sz;
1798 int ret;
1799
1800 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1801 ret = mmc_test_seq_read_perf(test, sz);
1802 if (ret)
1803 return ret;
1804 }
1805 sz = t->max_tfr;
1806 return mmc_test_seq_read_perf(test, sz);
1807}
1808
1809static int mmc_test_seq_write_perf(struct mmc_test_card *test, unsigned long sz)
1810{
1811 struct mmc_test_area *t = &test->area;
1812 unsigned int dev_addr, i, cnt;
1813 struct timespec64 ts1, ts2;
1814 int ret;
1815
1816 ret = mmc_test_area_erase(test);
1817 if (ret)
1818 return ret;
1819 cnt = t->max_sz / sz;
1820 dev_addr = t->dev_addr;
1821 ktime_get_ts64(&ts1);
1822 for (i = 0; i < cnt; i++) {
1823 ret = mmc_test_area_io(test, sz, dev_addr, 1, 0, 0);
1824 if (ret)
1825 return ret;
1826 dev_addr += (sz >> 9);
1827 }
1828 ktime_get_ts64(&ts2);
1829 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1830 return 0;
1831}
1832
1833/*
1834 * Consecutive write performance by transfer size.
1835 */
1836static int mmc_test_profile_seq_write_perf(struct mmc_test_card *test)
1837{
1838 struct mmc_test_area *t = &test->area;
1839 unsigned long sz;
1840 int ret;
1841
1842 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1843 ret = mmc_test_seq_write_perf(test, sz);
1844 if (ret)
1845 return ret;
1846 }
1847 sz = t->max_tfr;
1848 return mmc_test_seq_write_perf(test, sz);
1849}
1850
1851/*
1852 * Consecutive trim performance by transfer size.
1853 */
1854static int mmc_test_profile_seq_trim_perf(struct mmc_test_card *test)
1855{
1856 struct mmc_test_area *t = &test->area;
1857 unsigned long sz;
1858 unsigned int dev_addr, i, cnt;
1859 struct timespec64 ts1, ts2;
1860 int ret;
1861
1862 if (!mmc_can_trim(test->card))
1863 return RESULT_UNSUP_CARD;
1864
1865 if (!mmc_can_erase(test->card))
1866 return RESULT_UNSUP_HOST;
1867
1868 for (sz = 512; sz <= t->max_sz; sz <<= 1) {
1869 ret = mmc_test_area_erase(test);
1870 if (ret)
1871 return ret;
1872 ret = mmc_test_area_fill(test);
1873 if (ret)
1874 return ret;
1875 cnt = t->max_sz / sz;
1876 dev_addr = t->dev_addr;
1877 ktime_get_ts64(&ts1);
1878 for (i = 0; i < cnt; i++) {
1879 ret = mmc_erase(test->card, dev_addr, sz >> 9,
1880 MMC_TRIM_ARG);
1881 if (ret)
1882 return ret;
1883 dev_addr += (sz >> 9);
1884 }
1885 ktime_get_ts64(&ts2);
1886 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1887 }
1888 return 0;
1889}
1890
1891static unsigned int rnd_next = 1;
1892
1893static unsigned int mmc_test_rnd_num(unsigned int rnd_cnt)
1894{
1895 uint64_t r;
1896
1897 rnd_next = rnd_next * 1103515245 + 12345;
1898 r = (rnd_next >> 16) & 0x7fff;
1899 return (r * rnd_cnt) >> 15;
1900}
1901
1902static int mmc_test_rnd_perf(struct mmc_test_card *test, int write, int print,
1903 unsigned long sz)
1904{
1905 unsigned int dev_addr, cnt, rnd_addr, range1, range2, last_ea = 0, ea;
1906 unsigned int ssz;
1907 struct timespec64 ts1, ts2, ts;
1908 int ret;
1909
1910 ssz = sz >> 9;
1911
1912 rnd_addr = mmc_test_capacity(test->card) / 4;
1913 range1 = rnd_addr / test->card->pref_erase;
1914 range2 = range1 / ssz;
1915
1916 ktime_get_ts64(&ts1);
1917 for (cnt = 0; cnt < UINT_MAX; cnt++) {
1918 ktime_get_ts64(&ts2);
1919 ts = timespec64_sub(ts2, ts1);
1920 if (ts.tv_sec >= 10)
1921 break;
1922 ea = mmc_test_rnd_num(range1);
1923 if (ea == last_ea)
1924 ea -= 1;
1925 last_ea = ea;
1926 dev_addr = rnd_addr + test->card->pref_erase * ea +
1927 ssz * mmc_test_rnd_num(range2);
1928 ret = mmc_test_area_io(test, sz, dev_addr, write, 0, 0);
1929 if (ret)
1930 return ret;
1931 }
1932 if (print)
1933 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
1934 return 0;
1935}
1936
1937static int mmc_test_random_perf(struct mmc_test_card *test, int write)
1938{
1939 struct mmc_test_area *t = &test->area;
1940 unsigned int next;
1941 unsigned long sz;
1942 int ret;
1943
1944 for (sz = 512; sz < t->max_tfr; sz <<= 1) {
1945 /*
1946 * When writing, try to get more consistent results by running
1947 * the test twice with exactly the same I/O but outputting the
1948 * results only for the 2nd run.
1949 */
1950 if (write) {
1951 next = rnd_next;
1952 ret = mmc_test_rnd_perf(test, write, 0, sz);
1953 if (ret)
1954 return ret;
1955 rnd_next = next;
1956 }
1957 ret = mmc_test_rnd_perf(test, write, 1, sz);
1958 if (ret)
1959 return ret;
1960 }
1961 sz = t->max_tfr;
1962 if (write) {
1963 next = rnd_next;
1964 ret = mmc_test_rnd_perf(test, write, 0, sz);
1965 if (ret)
1966 return ret;
1967 rnd_next = next;
1968 }
1969 return mmc_test_rnd_perf(test, write, 1, sz);
1970}
1971
1972/*
1973 * Random read performance by transfer size.
1974 */
1975static int mmc_test_random_read_perf(struct mmc_test_card *test)
1976{
1977 return mmc_test_random_perf(test, 0);
1978}
1979
1980/*
1981 * Random write performance by transfer size.
1982 */
1983static int mmc_test_random_write_perf(struct mmc_test_card *test)
1984{
1985 return mmc_test_random_perf(test, 1);
1986}
1987
1988static int mmc_test_seq_perf(struct mmc_test_card *test, int write,
1989 unsigned int tot_sz, int max_scatter)
1990{
1991 struct mmc_test_area *t = &test->area;
1992 unsigned int dev_addr, i, cnt, sz, ssz;
1993 struct timespec64 ts1, ts2;
1994 int ret;
1995
1996 sz = t->max_tfr;
1997
1998 /*
1999 * In the case of a maximally scattered transfer, the maximum transfer
2000 * size is further limited by using PAGE_SIZE segments.
2001 */
2002 if (max_scatter) {
2003 unsigned long max_tfr;
2004
2005 if (t->max_seg_sz >= PAGE_SIZE)
2006 max_tfr = t->max_segs * PAGE_SIZE;
2007 else
2008 max_tfr = t->max_segs * t->max_seg_sz;
2009 if (sz > max_tfr)
2010 sz = max_tfr;
2011 }
2012
2013 ssz = sz >> 9;
2014 dev_addr = mmc_test_capacity(test->card) / 4;
2015 if (tot_sz > dev_addr << 9)
2016 tot_sz = dev_addr << 9;
2017 cnt = tot_sz / sz;
2018 dev_addr &= 0xffff0000; /* Round to 64MiB boundary */
2019
2020 ktime_get_ts64(&ts1);
2021 for (i = 0; i < cnt; i++) {
2022 ret = mmc_test_area_io(test, sz, dev_addr, write,
2023 max_scatter, 0);
2024 if (ret)
2025 return ret;
2026 dev_addr += ssz;
2027 }
2028 ktime_get_ts64(&ts2);
2029
2030 mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
2031
2032 return 0;
2033}
2034
2035static int mmc_test_large_seq_perf(struct mmc_test_card *test, int write)
2036{
2037 int ret, i;
2038
2039 for (i = 0; i < 10; i++) {
2040 ret = mmc_test_seq_perf(test, write, 10 * 1024 * 1024, 1);
2041 if (ret)
2042 return ret;
2043 }
2044 for (i = 0; i < 5; i++) {
2045 ret = mmc_test_seq_perf(test, write, 100 * 1024 * 1024, 1);
2046 if (ret)
2047 return ret;
2048 }
2049 for (i = 0; i < 3; i++) {
2050 ret = mmc_test_seq_perf(test, write, 1000 * 1024 * 1024, 1);
2051 if (ret)
2052 return ret;
2053 }
2054
2055 return ret;
2056}
2057
2058/*
2059 * Large sequential read performance.
2060 */
2061static int mmc_test_large_seq_read_perf(struct mmc_test_card *test)
2062{
2063 return mmc_test_large_seq_perf(test, 0);
2064}
2065
2066/*
2067 * Large sequential write performance.
2068 */
2069static int mmc_test_large_seq_write_perf(struct mmc_test_card *test)
2070{
2071 return mmc_test_large_seq_perf(test, 1);
2072}
2073
2074static int mmc_test_rw_multiple(struct mmc_test_card *test,
2075 struct mmc_test_multiple_rw *tdata,
2076 unsigned int reqsize, unsigned int size,
2077 int min_sg_len)
2078{
2079 unsigned int dev_addr;
2080 struct mmc_test_area *t = &test->area;
2081 int ret = 0;
2082
2083 /* Set up test area */
2084 if (size > mmc_test_capacity(test->card) / 2 * 512)
2085 size = mmc_test_capacity(test->card) / 2 * 512;
2086 if (reqsize > t->max_tfr)
2087 reqsize = t->max_tfr;
2088 dev_addr = mmc_test_capacity(test->card) / 4;
2089 if ((dev_addr & 0xffff0000))
2090 dev_addr &= 0xffff0000; /* Round to 64MiB boundary */
2091 else
2092 dev_addr &= 0xfffff800; /* Round to 1MiB boundary */
2093 if (!dev_addr)
2094 goto err;
2095
2096 if (reqsize > size)
2097 return 0;
2098
2099 /* prepare test area */
2100 if (mmc_can_erase(test->card) &&
2101 tdata->prepare & MMC_TEST_PREP_ERASE) {
2102 ret = mmc_erase(test->card, dev_addr,
2103 size / 512, MMC_SECURE_ERASE_ARG);
2104 if (ret)
2105 ret = mmc_erase(test->card, dev_addr,
2106 size / 512, MMC_ERASE_ARG);
2107 if (ret)
2108 goto err;
2109 }
2110
2111 /* Run test */
2112 ret = mmc_test_area_io_seq(test, reqsize, dev_addr,
2113 tdata->do_write, 0, 1, size / reqsize,
2114 tdata->do_nonblock_req, min_sg_len);
2115 if (ret)
2116 goto err;
2117
2118 return ret;
2119 err:
2120 pr_info("[%s] error\n", __func__);
2121 return ret;
2122}
2123
2124static int mmc_test_rw_multiple_size(struct mmc_test_card *test,
2125 struct mmc_test_multiple_rw *rw)
2126{
2127 int ret = 0;
2128 int i;
2129 void *pre_req = test->card->host->ops->pre_req;
2130 void *post_req = test->card->host->ops->post_req;
2131
2132 if (rw->do_nonblock_req &&
2133 ((!pre_req && post_req) || (pre_req && !post_req))) {
2134 pr_info("error: only one of pre/post is defined\n");
2135 return -EINVAL;
2136 }
2137
2138 for (i = 0 ; i < rw->len && ret == 0; i++) {
2139 ret = mmc_test_rw_multiple(test, rw, rw->bs[i], rw->size, 0);
2140 if (ret)
2141 break;
2142 }
2143 return ret;
2144}
2145
2146static int mmc_test_rw_multiple_sg_len(struct mmc_test_card *test,
2147 struct mmc_test_multiple_rw *rw)
2148{
2149 int ret = 0;
2150 int i;
2151
2152 for (i = 0 ; i < rw->len && ret == 0; i++) {
2153 ret = mmc_test_rw_multiple(test, rw, 512 * 1024, rw->size,
2154 rw->sg_len[i]);
2155 if (ret)
2156 break;
2157 }
2158 return ret;
2159}
2160
2161/*
2162 * Multiple blocking write 4k to 4 MB chunks
2163 */
2164static int mmc_test_profile_mult_write_blocking_perf(struct mmc_test_card *test)
2165{
2166 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2167 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2168 struct mmc_test_multiple_rw test_data = {
2169 .bs = bs,
2170 .size = TEST_AREA_MAX_SIZE,
2171 .len = ARRAY_SIZE(bs),
2172 .do_write = true,
2173 .do_nonblock_req = false,
2174 .prepare = MMC_TEST_PREP_ERASE,
2175 };
2176
2177 return mmc_test_rw_multiple_size(test, &test_data);
2178};
2179
2180/*
2181 * Multiple non-blocking write 4k to 4 MB chunks
2182 */
2183static int mmc_test_profile_mult_write_nonblock_perf(struct mmc_test_card *test)
2184{
2185 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2186 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2187 struct mmc_test_multiple_rw test_data = {
2188 .bs = bs,
2189 .size = TEST_AREA_MAX_SIZE,
2190 .len = ARRAY_SIZE(bs),
2191 .do_write = true,
2192 .do_nonblock_req = true,
2193 .prepare = MMC_TEST_PREP_ERASE,
2194 };
2195
2196 return mmc_test_rw_multiple_size(test, &test_data);
2197}
2198
2199/*
2200 * Multiple blocking read 4k to 4 MB chunks
2201 */
2202static int mmc_test_profile_mult_read_blocking_perf(struct mmc_test_card *test)
2203{
2204 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2205 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2206 struct mmc_test_multiple_rw test_data = {
2207 .bs = bs,
2208 .size = TEST_AREA_MAX_SIZE,
2209 .len = ARRAY_SIZE(bs),
2210 .do_write = false,
2211 .do_nonblock_req = false,
2212 .prepare = MMC_TEST_PREP_NONE,
2213 };
2214
2215 return mmc_test_rw_multiple_size(test, &test_data);
2216}
2217
2218/*
2219 * Multiple non-blocking read 4k to 4 MB chunks
2220 */
2221static int mmc_test_profile_mult_read_nonblock_perf(struct mmc_test_card *test)
2222{
2223 unsigned int bs[] = {1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
2224 1 << 17, 1 << 18, 1 << 19, 1 << 20, 1 << 22};
2225 struct mmc_test_multiple_rw test_data = {
2226 .bs = bs,
2227 .size = TEST_AREA_MAX_SIZE,
2228 .len = ARRAY_SIZE(bs),
2229 .do_write = false,
2230 .do_nonblock_req = true,
2231 .prepare = MMC_TEST_PREP_NONE,
2232 };
2233
2234 return mmc_test_rw_multiple_size(test, &test_data);
2235}
2236
2237/*
2238 * Multiple blocking write 1 to 512 sg elements
2239 */
2240static int mmc_test_profile_sglen_wr_blocking_perf(struct mmc_test_card *test)
2241{
2242 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2243 1 << 7, 1 << 8, 1 << 9};
2244 struct mmc_test_multiple_rw test_data = {
2245 .sg_len = sg_len,
2246 .size = TEST_AREA_MAX_SIZE,
2247 .len = ARRAY_SIZE(sg_len),
2248 .do_write = true,
2249 .do_nonblock_req = false,
2250 .prepare = MMC_TEST_PREP_ERASE,
2251 };
2252
2253 return mmc_test_rw_multiple_sg_len(test, &test_data);
2254};
2255
2256/*
2257 * Multiple non-blocking write 1 to 512 sg elements
2258 */
2259static int mmc_test_profile_sglen_wr_nonblock_perf(struct mmc_test_card *test)
2260{
2261 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2262 1 << 7, 1 << 8, 1 << 9};
2263 struct mmc_test_multiple_rw test_data = {
2264 .sg_len = sg_len,
2265 .size = TEST_AREA_MAX_SIZE,
2266 .len = ARRAY_SIZE(sg_len),
2267 .do_write = true,
2268 .do_nonblock_req = true,
2269 .prepare = MMC_TEST_PREP_ERASE,
2270 };
2271
2272 return mmc_test_rw_multiple_sg_len(test, &test_data);
2273}
2274
2275/*
2276 * Multiple blocking read 1 to 512 sg elements
2277 */
2278static int mmc_test_profile_sglen_r_blocking_perf(struct mmc_test_card *test)
2279{
2280 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2281 1 << 7, 1 << 8, 1 << 9};
2282 struct mmc_test_multiple_rw test_data = {
2283 .sg_len = sg_len,
2284 .size = TEST_AREA_MAX_SIZE,
2285 .len = ARRAY_SIZE(sg_len),
2286 .do_write = false,
2287 .do_nonblock_req = false,
2288 .prepare = MMC_TEST_PREP_NONE,
2289 };
2290
2291 return mmc_test_rw_multiple_sg_len(test, &test_data);
2292}
2293
2294/*
2295 * Multiple non-blocking read 1 to 512 sg elements
2296 */
2297static int mmc_test_profile_sglen_r_nonblock_perf(struct mmc_test_card *test)
2298{
2299 unsigned int sg_len[] = {1, 1 << 3, 1 << 4, 1 << 5, 1 << 6,
2300 1 << 7, 1 << 8, 1 << 9};
2301 struct mmc_test_multiple_rw test_data = {
2302 .sg_len = sg_len,
2303 .size = TEST_AREA_MAX_SIZE,
2304 .len = ARRAY_SIZE(sg_len),
2305 .do_write = false,
2306 .do_nonblock_req = true,
2307 .prepare = MMC_TEST_PREP_NONE,
2308 };
2309
2310 return mmc_test_rw_multiple_sg_len(test, &test_data);
2311}
2312
2313/*
2314 * eMMC hardware reset.
2315 */
2316static int mmc_test_reset(struct mmc_test_card *test)
2317{
2318 struct mmc_card *card = test->card;
2319 struct mmc_host *host = card->host;
2320 int err;
2321
2322 err = mmc_hw_reset(host);
2323 if (!err) {
2324 /*
2325 * Reset will re-enable the card's command queue, but tests
2326 * expect it to be disabled.
2327 */
2328 if (card->ext_csd.cmdq_en)
2329 mmc_cmdq_disable(card);
2330 return RESULT_OK;
2331 } else if (err == -EOPNOTSUPP) {
2332 return RESULT_UNSUP_HOST;
2333 }
2334
2335 return RESULT_FAIL;
2336}
2337
2338static int mmc_test_send_status(struct mmc_test_card *test,
2339 struct mmc_command *cmd)
2340{
2341 memset(cmd, 0, sizeof(*cmd));
2342
2343 cmd->opcode = MMC_SEND_STATUS;
2344 if (!mmc_host_is_spi(test->card->host))
2345 cmd->arg = test->card->rca << 16;
2346 cmd->flags = MMC_RSP_SPI_R2 | MMC_RSP_R1 | MMC_CMD_AC;
2347
2348 return mmc_wait_for_cmd(test->card->host, cmd, 0);
2349}
2350
2351static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
2352 unsigned int dev_addr, int use_sbc,
2353 int repeat_cmd, int write, int use_areq)
2354{
2355 struct mmc_test_req *rq = mmc_test_req_alloc();
2356 struct mmc_host *host = test->card->host;
2357 struct mmc_test_area *t = &test->area;
2358 struct mmc_request *mrq;
2359 unsigned long timeout;
2360 bool expired = false;
2361 int ret = 0, cmd_ret;
2362 u32 status = 0;
2363 int count = 0;
2364
2365 if (!rq)
2366 return -ENOMEM;
2367
2368 mrq = &rq->mrq;
2369 if (use_sbc)
2370 mrq->sbc = &rq->sbc;
2371 mrq->cap_cmd_during_tfr = true;
2372
2373 mmc_test_prepare_mrq(test, mrq, t->sg, t->sg_len, dev_addr, t->blocks,
2374 512, write);
2375
2376 if (use_sbc && t->blocks > 1 && !mrq->sbc) {
2377 ret = mmc_host_cmd23(host) ?
2378 RESULT_UNSUP_CARD :
2379 RESULT_UNSUP_HOST;
2380 goto out_free;
2381 }
2382
2383 /* Start ongoing data request */
2384 if (use_areq) {
2385 ret = mmc_test_start_areq(test, mrq, NULL);
2386 if (ret)
2387 goto out_free;
2388 } else {
2389 mmc_wait_for_req(host, mrq);
2390 }
2391
2392 timeout = jiffies + msecs_to_jiffies(3000);
2393 do {
2394 count += 1;
2395
2396 /* Send status command while data transfer in progress */
2397 cmd_ret = mmc_test_send_status(test, &rq->status);
2398 if (cmd_ret)
2399 break;
2400
2401 status = rq->status.resp[0];
2402 if (status & R1_ERROR) {
2403 cmd_ret = -EIO;
2404 break;
2405 }
2406
2407 if (mmc_is_req_done(host, mrq))
2408 break;
2409
2410 expired = time_after(jiffies, timeout);
2411 if (expired) {
2412 pr_info("%s: timeout waiting for Tran state status %#x\n",
2413 mmc_hostname(host), status);
2414 cmd_ret = -ETIMEDOUT;
2415 break;
2416 }
2417 } while (repeat_cmd && R1_CURRENT_STATE(status) != R1_STATE_TRAN);
2418
2419 /* Wait for data request to complete */
2420 if (use_areq) {
2421 ret = mmc_test_start_areq(test, NULL, mrq);
2422 } else {
2423 mmc_wait_for_req_done(test->card->host, mrq);
2424 }
2425
2426 /*
2427 * For cap_cmd_during_tfr request, upper layer must send stop if
2428 * required.
2429 */
2430 if (mrq->data->stop && (mrq->data->error || !mrq->sbc)) {
2431 if (ret)
2432 mmc_wait_for_cmd(host, mrq->data->stop, 0);
2433 else
2434 ret = mmc_wait_for_cmd(host, mrq->data->stop, 0);
2435 }
2436
2437 if (ret)
2438 goto out_free;
2439
2440 if (cmd_ret) {
2441 pr_info("%s: Send Status failed: status %#x, error %d\n",
2442 mmc_hostname(test->card->host), status, cmd_ret);
2443 }
2444
2445 ret = mmc_test_check_result(test, mrq);
2446 if (ret)
2447 goto out_free;
2448
2449 ret = mmc_test_wait_busy(test);
2450 if (ret)
2451 goto out_free;
2452
2453 if (repeat_cmd && (t->blocks + 1) << 9 > t->max_tfr)
2454 pr_info("%s: %d commands completed during transfer of %u blocks\n",
2455 mmc_hostname(test->card->host), count, t->blocks);
2456
2457 if (cmd_ret)
2458 ret = cmd_ret;
2459out_free:
2460 kfree(rq);
2461
2462 return ret;
2463}
2464
2465static int __mmc_test_cmds_during_tfr(struct mmc_test_card *test,
2466 unsigned long sz, int use_sbc, int write,
2467 int use_areq)
2468{
2469 struct mmc_test_area *t = &test->area;
2470 int ret;
2471
2472 if (!(test->card->host->caps & MMC_CAP_CMD_DURING_TFR))
2473 return RESULT_UNSUP_HOST;
2474
2475 ret = mmc_test_area_map(test, sz, 0, 0);
2476 if (ret)
2477 return ret;
2478
2479 ret = mmc_test_ongoing_transfer(test, t->dev_addr, use_sbc, 0, write,
2480 use_areq);
2481 if (ret)
2482 return ret;
2483
2484 return mmc_test_ongoing_transfer(test, t->dev_addr, use_sbc, 1, write,
2485 use_areq);
2486}
2487
2488static int mmc_test_cmds_during_tfr(struct mmc_test_card *test, int use_sbc,
2489 int write, int use_areq)
2490{
2491 struct mmc_test_area *t = &test->area;
2492 unsigned long sz;
2493 int ret;
2494
2495 for (sz = 512; sz <= t->max_tfr; sz += 512) {
2496 ret = __mmc_test_cmds_during_tfr(test, sz, use_sbc, write,
2497 use_areq);
2498 if (ret)
2499 return ret;
2500 }
2501 return 0;
2502}
2503
2504/*
2505 * Commands during read - no Set Block Count (CMD23).
2506 */
2507static int mmc_test_cmds_during_read(struct mmc_test_card *test)
2508{
2509 return mmc_test_cmds_during_tfr(test, 0, 0, 0);
2510}
2511
2512/*
2513 * Commands during write - no Set Block Count (CMD23).
2514 */
2515static int mmc_test_cmds_during_write(struct mmc_test_card *test)
2516{
2517 return mmc_test_cmds_during_tfr(test, 0, 1, 0);
2518}
2519
2520/*
2521 * Commands during read - use Set Block Count (CMD23).
2522 */
2523static int mmc_test_cmds_during_read_cmd23(struct mmc_test_card *test)
2524{
2525 return mmc_test_cmds_during_tfr(test, 1, 0, 0);
2526}
2527
2528/*
2529 * Commands during write - use Set Block Count (CMD23).
2530 */
2531static int mmc_test_cmds_during_write_cmd23(struct mmc_test_card *test)
2532{
2533 return mmc_test_cmds_during_tfr(test, 1, 1, 0);
2534}
2535
2536/*
2537 * Commands during non-blocking read - use Set Block Count (CMD23).
2538 */
2539static int mmc_test_cmds_during_read_cmd23_nonblock(struct mmc_test_card *test)
2540{
2541 return mmc_test_cmds_during_tfr(test, 1, 0, 1);
2542}
2543
2544/*
2545 * Commands during non-blocking write - use Set Block Count (CMD23).
2546 */
2547static int mmc_test_cmds_during_write_cmd23_nonblock(struct mmc_test_card *test)
2548{
2549 return mmc_test_cmds_during_tfr(test, 1, 1, 1);
2550}
2551
2552static const struct mmc_test_case mmc_test_cases[] = {
2553 {
2554 .name = "Basic write (no data verification)",
2555 .run = mmc_test_basic_write,
2556 },
2557
2558 {
2559 .name = "Basic read (no data verification)",
2560 .run = mmc_test_basic_read,
2561 },
2562
2563 {
2564 .name = "Basic write (with data verification)",
2565 .prepare = mmc_test_prepare_write,
2566 .run = mmc_test_verify_write,
2567 .cleanup = mmc_test_cleanup,
2568 },
2569
2570 {
2571 .name = "Basic read (with data verification)",
2572 .prepare = mmc_test_prepare_read,
2573 .run = mmc_test_verify_read,
2574 .cleanup = mmc_test_cleanup,
2575 },
2576
2577 {
2578 .name = "Multi-block write",
2579 .prepare = mmc_test_prepare_write,
2580 .run = mmc_test_multi_write,
2581 .cleanup = mmc_test_cleanup,
2582 },
2583
2584 {
2585 .name = "Multi-block read",
2586 .prepare = mmc_test_prepare_read,
2587 .run = mmc_test_multi_read,
2588 .cleanup = mmc_test_cleanup,
2589 },
2590
2591 {
2592 .name = "Power of two block writes",
2593 .prepare = mmc_test_prepare_write,
2594 .run = mmc_test_pow2_write,
2595 .cleanup = mmc_test_cleanup,
2596 },
2597
2598 {
2599 .name = "Power of two block reads",
2600 .prepare = mmc_test_prepare_read,
2601 .run = mmc_test_pow2_read,
2602 .cleanup = mmc_test_cleanup,
2603 },
2604
2605 {
2606 .name = "Weird sized block writes",
2607 .prepare = mmc_test_prepare_write,
2608 .run = mmc_test_weird_write,
2609 .cleanup = mmc_test_cleanup,
2610 },
2611
2612 {
2613 .name = "Weird sized block reads",
2614 .prepare = mmc_test_prepare_read,
2615 .run = mmc_test_weird_read,
2616 .cleanup = mmc_test_cleanup,
2617 },
2618
2619 {
2620 .name = "Badly aligned write",
2621 .prepare = mmc_test_prepare_write,
2622 .run = mmc_test_align_write,
2623 .cleanup = mmc_test_cleanup,
2624 },
2625
2626 {
2627 .name = "Badly aligned read",
2628 .prepare = mmc_test_prepare_read,
2629 .run = mmc_test_align_read,
2630 .cleanup = mmc_test_cleanup,
2631 },
2632
2633 {
2634 .name = "Badly aligned multi-block write",
2635 .prepare = mmc_test_prepare_write,
2636 .run = mmc_test_align_multi_write,
2637 .cleanup = mmc_test_cleanup,
2638 },
2639
2640 {
2641 .name = "Badly aligned multi-block read",
2642 .prepare = mmc_test_prepare_read,
2643 .run = mmc_test_align_multi_read,
2644 .cleanup = mmc_test_cleanup,
2645 },
2646
2647 {
2648 .name = "Correct xfer_size at write (start failure)",
2649 .run = mmc_test_xfersize_write,
2650 },
2651
2652 {
2653 .name = "Correct xfer_size at read (start failure)",
2654 .run = mmc_test_xfersize_read,
2655 },
2656
2657 {
2658 .name = "Correct xfer_size at write (midway failure)",
2659 .run = mmc_test_multi_xfersize_write,
2660 },
2661
2662 {
2663 .name = "Correct xfer_size at read (midway failure)",
2664 .run = mmc_test_multi_xfersize_read,
2665 },
2666
2667#ifdef CONFIG_HIGHMEM
2668
2669 {
2670 .name = "Highmem write",
2671 .prepare = mmc_test_prepare_write,
2672 .run = mmc_test_write_high,
2673 .cleanup = mmc_test_cleanup,
2674 },
2675
2676 {
2677 .name = "Highmem read",
2678 .prepare = mmc_test_prepare_read,
2679 .run = mmc_test_read_high,
2680 .cleanup = mmc_test_cleanup,
2681 },
2682
2683 {
2684 .name = "Multi-block highmem write",
2685 .prepare = mmc_test_prepare_write,
2686 .run = mmc_test_multi_write_high,
2687 .cleanup = mmc_test_cleanup,
2688 },
2689
2690 {
2691 .name = "Multi-block highmem read",
2692 .prepare = mmc_test_prepare_read,
2693 .run = mmc_test_multi_read_high,
2694 .cleanup = mmc_test_cleanup,
2695 },
2696
2697#else
2698
2699 {
2700 .name = "Highmem write",
2701 .run = mmc_test_no_highmem,
2702 },
2703
2704 {
2705 .name = "Highmem read",
2706 .run = mmc_test_no_highmem,
2707 },
2708
2709 {
2710 .name = "Multi-block highmem write",
2711 .run = mmc_test_no_highmem,
2712 },
2713
2714 {
2715 .name = "Multi-block highmem read",
2716 .run = mmc_test_no_highmem,
2717 },
2718
2719#endif /* CONFIG_HIGHMEM */
2720
2721 {
2722 .name = "Best-case read performance",
2723 .prepare = mmc_test_area_prepare_fill,
2724 .run = mmc_test_best_read_performance,
2725 .cleanup = mmc_test_area_cleanup,
2726 },
2727
2728 {
2729 .name = "Best-case write performance",
2730 .prepare = mmc_test_area_prepare_erase,
2731 .run = mmc_test_best_write_performance,
2732 .cleanup = mmc_test_area_cleanup,
2733 },
2734
2735 {
2736 .name = "Best-case read performance into scattered pages",
2737 .prepare = mmc_test_area_prepare_fill,
2738 .run = mmc_test_best_read_perf_max_scatter,
2739 .cleanup = mmc_test_area_cleanup,
2740 },
2741
2742 {
2743 .name = "Best-case write performance from scattered pages",
2744 .prepare = mmc_test_area_prepare_erase,
2745 .run = mmc_test_best_write_perf_max_scatter,
2746 .cleanup = mmc_test_area_cleanup,
2747 },
2748
2749 {
2750 .name = "Single read performance by transfer size",
2751 .prepare = mmc_test_area_prepare_fill,
2752 .run = mmc_test_profile_read_perf,
2753 .cleanup = mmc_test_area_cleanup,
2754 },
2755
2756 {
2757 .name = "Single write performance by transfer size",
2758 .prepare = mmc_test_area_prepare,
2759 .run = mmc_test_profile_write_perf,
2760 .cleanup = mmc_test_area_cleanup,
2761 },
2762
2763 {
2764 .name = "Single trim performance by transfer size",
2765 .prepare = mmc_test_area_prepare_fill,
2766 .run = mmc_test_profile_trim_perf,
2767 .cleanup = mmc_test_area_cleanup,
2768 },
2769
2770 {
2771 .name = "Consecutive read performance by transfer size",
2772 .prepare = mmc_test_area_prepare_fill,
2773 .run = mmc_test_profile_seq_read_perf,
2774 .cleanup = mmc_test_area_cleanup,
2775 },
2776
2777 {
2778 .name = "Consecutive write performance by transfer size",
2779 .prepare = mmc_test_area_prepare,
2780 .run = mmc_test_profile_seq_write_perf,
2781 .cleanup = mmc_test_area_cleanup,
2782 },
2783
2784 {
2785 .name = "Consecutive trim performance by transfer size",
2786 .prepare = mmc_test_area_prepare,
2787 .run = mmc_test_profile_seq_trim_perf,
2788 .cleanup = mmc_test_area_cleanup,
2789 },
2790
2791 {
2792 .name = "Random read performance by transfer size",
2793 .prepare = mmc_test_area_prepare,
2794 .run = mmc_test_random_read_perf,
2795 .cleanup = mmc_test_area_cleanup,
2796 },
2797
2798 {
2799 .name = "Random write performance by transfer size",
2800 .prepare = mmc_test_area_prepare,
2801 .run = mmc_test_random_write_perf,
2802 .cleanup = mmc_test_area_cleanup,
2803 },
2804
2805 {
2806 .name = "Large sequential read into scattered pages",
2807 .prepare = mmc_test_area_prepare,
2808 .run = mmc_test_large_seq_read_perf,
2809 .cleanup = mmc_test_area_cleanup,
2810 },
2811
2812 {
2813 .name = "Large sequential write from scattered pages",
2814 .prepare = mmc_test_area_prepare,
2815 .run = mmc_test_large_seq_write_perf,
2816 .cleanup = mmc_test_area_cleanup,
2817 },
2818
2819 {
2820 .name = "Write performance with blocking req 4k to 4MB",
2821 .prepare = mmc_test_area_prepare,
2822 .run = mmc_test_profile_mult_write_blocking_perf,
2823 .cleanup = mmc_test_area_cleanup,
2824 },
2825
2826 {
2827 .name = "Write performance with non-blocking req 4k to 4MB",
2828 .prepare = mmc_test_area_prepare,
2829 .run = mmc_test_profile_mult_write_nonblock_perf,
2830 .cleanup = mmc_test_area_cleanup,
2831 },
2832
2833 {
2834 .name = "Read performance with blocking req 4k to 4MB",
2835 .prepare = mmc_test_area_prepare,
2836 .run = mmc_test_profile_mult_read_blocking_perf,
2837 .cleanup = mmc_test_area_cleanup,
2838 },
2839
2840 {
2841 .name = "Read performance with non-blocking req 4k to 4MB",
2842 .prepare = mmc_test_area_prepare,
2843 .run = mmc_test_profile_mult_read_nonblock_perf,
2844 .cleanup = mmc_test_area_cleanup,
2845 },
2846
2847 {
2848 .name = "Write performance blocking req 1 to 512 sg elems",
2849 .prepare = mmc_test_area_prepare,
2850 .run = mmc_test_profile_sglen_wr_blocking_perf,
2851 .cleanup = mmc_test_area_cleanup,
2852 },
2853
2854 {
2855 .name = "Write performance non-blocking req 1 to 512 sg elems",
2856 .prepare = mmc_test_area_prepare,
2857 .run = mmc_test_profile_sglen_wr_nonblock_perf,
2858 .cleanup = mmc_test_area_cleanup,
2859 },
2860
2861 {
2862 .name = "Read performance blocking req 1 to 512 sg elems",
2863 .prepare = mmc_test_area_prepare,
2864 .run = mmc_test_profile_sglen_r_blocking_perf,
2865 .cleanup = mmc_test_area_cleanup,
2866 },
2867
2868 {
2869 .name = "Read performance non-blocking req 1 to 512 sg elems",
2870 .prepare = mmc_test_area_prepare,
2871 .run = mmc_test_profile_sglen_r_nonblock_perf,
2872 .cleanup = mmc_test_area_cleanup,
2873 },
2874
2875 {
2876 .name = "Reset test",
2877 .run = mmc_test_reset,
2878 },
2879
2880 {
2881 .name = "Commands during read - no Set Block Count (CMD23)",
2882 .prepare = mmc_test_area_prepare,
2883 .run = mmc_test_cmds_during_read,
2884 .cleanup = mmc_test_area_cleanup,
2885 },
2886
2887 {
2888 .name = "Commands during write - no Set Block Count (CMD23)",
2889 .prepare = mmc_test_area_prepare,
2890 .run = mmc_test_cmds_during_write,
2891 .cleanup = mmc_test_area_cleanup,
2892 },
2893
2894 {
2895 .name = "Commands during read - use Set Block Count (CMD23)",
2896 .prepare = mmc_test_area_prepare,
2897 .run = mmc_test_cmds_during_read_cmd23,
2898 .cleanup = mmc_test_area_cleanup,
2899 },
2900
2901 {
2902 .name = "Commands during write - use Set Block Count (CMD23)",
2903 .prepare = mmc_test_area_prepare,
2904 .run = mmc_test_cmds_during_write_cmd23,
2905 .cleanup = mmc_test_area_cleanup,
2906 },
2907
2908 {
2909 .name = "Commands during non-blocking read - use Set Block Count (CMD23)",
2910 .prepare = mmc_test_area_prepare,
2911 .run = mmc_test_cmds_during_read_cmd23_nonblock,
2912 .cleanup = mmc_test_area_cleanup,
2913 },
2914
2915 {
2916 .name = "Commands during non-blocking write - use Set Block Count (CMD23)",
2917 .prepare = mmc_test_area_prepare,
2918 .run = mmc_test_cmds_during_write_cmd23_nonblock,
2919 .cleanup = mmc_test_area_cleanup,
2920 },
2921};
2922
2923static DEFINE_MUTEX(mmc_test_lock);
2924
2925static LIST_HEAD(mmc_test_result);
2926
2927static void mmc_test_run(struct mmc_test_card *test, int testcase)
2928{
2929 int i, ret;
2930
2931 pr_info("%s: Starting tests of card %s...\n",
2932 mmc_hostname(test->card->host), mmc_card_id(test->card));
2933
2934 mmc_claim_host(test->card->host);
2935
2936 for (i = 0; i < ARRAY_SIZE(mmc_test_cases); i++) {
2937 struct mmc_test_general_result *gr;
2938
2939 if (testcase && ((i + 1) != testcase))
2940 continue;
2941
2942 pr_info("%s: Test case %d. %s...\n",
2943 mmc_hostname(test->card->host), i + 1,
2944 mmc_test_cases[i].name);
2945
2946 if (mmc_test_cases[i].prepare) {
2947 ret = mmc_test_cases[i].prepare(test);
2948 if (ret) {
2949 pr_info("%s: Result: Prepare stage failed! (%d)\n",
2950 mmc_hostname(test->card->host),
2951 ret);
2952 continue;
2953 }
2954 }
2955
2956 gr = kzalloc(sizeof(*gr), GFP_KERNEL);
2957 if (gr) {
2958 INIT_LIST_HEAD(&gr->tr_lst);
2959
2960 /* Assign data what we know already */
2961 gr->card = test->card;
2962 gr->testcase = i;
2963
2964 /* Append container to global one */
2965 list_add_tail(&gr->link, &mmc_test_result);
2966
2967 /*
2968 * Save the pointer to created container in our private
2969 * structure.
2970 */
2971 test->gr = gr;
2972 }
2973
2974 ret = mmc_test_cases[i].run(test);
2975 switch (ret) {
2976 case RESULT_OK:
2977 pr_info("%s: Result: OK\n",
2978 mmc_hostname(test->card->host));
2979 break;
2980 case RESULT_FAIL:
2981 pr_info("%s: Result: FAILED\n",
2982 mmc_hostname(test->card->host));
2983 break;
2984 case RESULT_UNSUP_HOST:
2985 pr_info("%s: Result: UNSUPPORTED (by host)\n",
2986 mmc_hostname(test->card->host));
2987 break;
2988 case RESULT_UNSUP_CARD:
2989 pr_info("%s: Result: UNSUPPORTED (by card)\n",
2990 mmc_hostname(test->card->host));
2991 break;
2992 default:
2993 pr_info("%s: Result: ERROR (%d)\n",
2994 mmc_hostname(test->card->host), ret);
2995 }
2996
2997 /* Save the result */
2998 if (gr)
2999 gr->result = ret;
3000
3001 if (mmc_test_cases[i].cleanup) {
3002 ret = mmc_test_cases[i].cleanup(test);
3003 if (ret) {
3004 pr_info("%s: Warning: Cleanup stage failed! (%d)\n",
3005 mmc_hostname(test->card->host),
3006 ret);
3007 }
3008 }
3009 }
3010
3011 mmc_release_host(test->card->host);
3012
3013 pr_info("%s: Tests completed.\n",
3014 mmc_hostname(test->card->host));
3015}
3016
3017static void mmc_test_free_result(struct mmc_card *card)
3018{
3019 struct mmc_test_general_result *gr, *grs;
3020
3021 mutex_lock(&mmc_test_lock);
3022
3023 list_for_each_entry_safe(gr, grs, &mmc_test_result, link) {
3024 struct mmc_test_transfer_result *tr, *trs;
3025
3026 if (card && gr->card != card)
3027 continue;
3028
3029 list_for_each_entry_safe(tr, trs, &gr->tr_lst, link) {
3030 list_del(&tr->link);
3031 kfree(tr);
3032 }
3033
3034 list_del(&gr->link);
3035 kfree(gr);
3036 }
3037
3038 mutex_unlock(&mmc_test_lock);
3039}
3040
3041static LIST_HEAD(mmc_test_file_test);
3042
3043static int mtf_test_show(struct seq_file *sf, void *data)
3044{
3045 struct mmc_card *card = (struct mmc_card *)sf->private;
3046 struct mmc_test_general_result *gr;
3047
3048 mutex_lock(&mmc_test_lock);
3049
3050 list_for_each_entry(gr, &mmc_test_result, link) {
3051 struct mmc_test_transfer_result *tr;
3052
3053 if (gr->card != card)
3054 continue;
3055
3056 seq_printf(sf, "Test %d: %d\n", gr->testcase + 1, gr->result);
3057
3058 list_for_each_entry(tr, &gr->tr_lst, link) {
3059 seq_printf(sf, "%u %d %llu.%09u %u %u.%02u\n",
3060 tr->count, tr->sectors,
3061 (u64)tr->ts.tv_sec, (u32)tr->ts.tv_nsec,
3062 tr->rate, tr->iops / 100, tr->iops % 100);
3063 }
3064 }
3065
3066 mutex_unlock(&mmc_test_lock);
3067
3068 return 0;
3069}
3070
3071static int mtf_test_open(struct inode *inode, struct file *file)
3072{
3073 return single_open(file, mtf_test_show, inode->i_private);
3074}
3075
3076static ssize_t mtf_test_write(struct file *file, const char __user *buf,
3077 size_t count, loff_t *pos)
3078{
3079 struct seq_file *sf = (struct seq_file *)file->private_data;
3080 struct mmc_card *card = (struct mmc_card *)sf->private;
3081 struct mmc_test_card *test;
3082 long testcase;
3083 int ret;
3084
3085 ret = kstrtol_from_user(buf, count, 10, &testcase);
3086 if (ret)
3087 return ret;
3088
3089 test = kzalloc(sizeof(*test), GFP_KERNEL);
3090 if (!test)
3091 return -ENOMEM;
3092
3093 /*
3094 * Remove all test cases associated with given card. Thus we have only
3095 * actual data of the last run.
3096 */
3097 mmc_test_free_result(card);
3098
3099 test->card = card;
3100
3101 test->buffer = kzalloc(BUFFER_SIZE, GFP_KERNEL);
3102#ifdef CONFIG_HIGHMEM
3103 test->highmem = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM, BUFFER_ORDER);
3104#endif
3105
3106#ifdef CONFIG_HIGHMEM
3107 if (test->buffer && test->highmem) {
3108#else
3109 if (test->buffer) {
3110#endif
3111 mutex_lock(&mmc_test_lock);
3112 mmc_test_run(test, testcase);
3113 mutex_unlock(&mmc_test_lock);
3114 }
3115
3116#ifdef CONFIG_HIGHMEM
3117 __free_pages(test->highmem, BUFFER_ORDER);
3118#endif
3119 kfree(test->buffer);
3120 kfree(test);
3121
3122 return count;
3123}
3124
3125static const struct file_operations mmc_test_fops_test = {
3126 .open = mtf_test_open,
3127 .read = seq_read,
3128 .write = mtf_test_write,
3129 .llseek = seq_lseek,
3130 .release = single_release,
3131};
3132
3133static int mtf_testlist_show(struct seq_file *sf, void *data)
3134{
3135 int i;
3136
3137 mutex_lock(&mmc_test_lock);
3138
3139 seq_puts(sf, "0:\tRun all tests\n");
3140 for (i = 0; i < ARRAY_SIZE(mmc_test_cases); i++)
3141 seq_printf(sf, "%d:\t%s\n", i + 1, mmc_test_cases[i].name);
3142
3143 mutex_unlock(&mmc_test_lock);
3144
3145 return 0;
3146}
3147
3148static int mtf_testlist_open(struct inode *inode, struct file *file)
3149{
3150 return single_open(file, mtf_testlist_show, inode->i_private);
3151}
3152
3153static const struct file_operations mmc_test_fops_testlist = {
3154 .open = mtf_testlist_open,
3155 .read = seq_read,
3156 .llseek = seq_lseek,
3157 .release = single_release,
3158};
3159
3160static void mmc_test_free_dbgfs_file(struct mmc_card *card)
3161{
3162 struct mmc_test_dbgfs_file *df, *dfs;
3163
3164 mutex_lock(&mmc_test_lock);
3165
3166 list_for_each_entry_safe(df, dfs, &mmc_test_file_test, link) {
3167 if (card && df->card != card)
3168 continue;
3169 debugfs_remove(df->file);
3170 list_del(&df->link);
3171 kfree(df);
3172 }
3173
3174 mutex_unlock(&mmc_test_lock);
3175}
3176
3177static int __mmc_test_register_dbgfs_file(struct mmc_card *card,
3178 const char *name, umode_t mode, const struct file_operations *fops)
3179{
3180 struct dentry *file = NULL;
3181 struct mmc_test_dbgfs_file *df;
3182
3183 if (card->debugfs_root)
3184 file = debugfs_create_file(name, mode, card->debugfs_root,
3185 card, fops);
3186
3187 if (IS_ERR_OR_NULL(file)) {
3188 dev_err(&card->dev,
3189 "Can't create %s. Perhaps debugfs is disabled.\n",
3190 name);
3191 return -ENODEV;
3192 }
3193
3194 df = kmalloc(sizeof(*df), GFP_KERNEL);
3195 if (!df) {
3196 debugfs_remove(file);
3197 return -ENOMEM;
3198 }
3199
3200 df->card = card;
3201 df->file = file;
3202
3203 list_add(&df->link, &mmc_test_file_test);
3204 return 0;
3205}
3206
3207static int mmc_test_register_dbgfs_file(struct mmc_card *card)
3208{
3209 int ret;
3210
3211 mutex_lock(&mmc_test_lock);
3212
3213 ret = __mmc_test_register_dbgfs_file(card, "test", S_IWUSR | S_IRUGO,
3214 &mmc_test_fops_test);
3215 if (ret)
3216 goto err;
3217
3218 ret = __mmc_test_register_dbgfs_file(card, "testlist", S_IRUGO,
3219 &mmc_test_fops_testlist);
3220 if (ret)
3221 goto err;
3222
3223err:
3224 mutex_unlock(&mmc_test_lock);
3225
3226 return ret;
3227}
3228
3229static int mmc_test_probe(struct mmc_card *card)
3230{
3231 int ret;
3232
3233 if (!mmc_card_mmc(card) && !mmc_card_sd(card))
3234 return -ENODEV;
3235
3236 ret = mmc_test_register_dbgfs_file(card);
3237 if (ret)
3238 return ret;
3239
3240 if (card->ext_csd.cmdq_en) {
3241 mmc_claim_host(card->host);
3242 ret = mmc_cmdq_disable(card);
3243 mmc_release_host(card->host);
3244 if (ret)
3245 return ret;
3246 }
3247
3248 dev_info(&card->dev, "Card claimed for testing.\n");
3249
3250 return 0;
3251}
3252
3253static void mmc_test_remove(struct mmc_card *card)
3254{
3255 if (card->reenable_cmdq) {
3256 mmc_claim_host(card->host);
3257 mmc_cmdq_enable(card);
3258 mmc_release_host(card->host);
3259 }
3260 mmc_test_free_result(card);
3261 mmc_test_free_dbgfs_file(card);
3262}
3263
3264static void mmc_test_shutdown(struct mmc_card *card)
3265{
3266}
3267
3268static struct mmc_driver mmc_driver = {
3269 .drv = {
3270 .name = "mmc_test",
3271 },
3272 .probe = mmc_test_probe,
3273 .remove = mmc_test_remove,
3274 .shutdown = mmc_test_shutdown,
3275};
3276
3277static int __init mmc_test_init(void)
3278{
3279 return mmc_register_driver(&mmc_driver);
3280}
3281
3282static void __exit mmc_test_exit(void)
3283{
3284 /* Clear stalled data if card is still plugged */
3285 mmc_test_free_result(NULL);
3286 mmc_test_free_dbgfs_file(NULL);
3287
3288 mmc_unregister_driver(&mmc_driver);
3289}
3290
3291module_init(mmc_test_init);
3292module_exit(mmc_test_exit);
3293
3294MODULE_LICENSE("GPL");
3295MODULE_DESCRIPTION("Multimedia Card (MMC) host test driver");
3296MODULE_AUTHOR("Pierre Ossman");