Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Swap block device support for MTDs
4 * Turns an MTD device into a swap device with block wear leveling
5 *
6 * Copyright © 2007,2011 Nokia Corporation. All rights reserved.
7 *
8 * Authors: Jarkko Lavinen <jarkko.lavinen@nokia.com>
9 *
10 * Based on Richard Purdie's earlier implementation in 2007. Background
11 * support and lock-less operation written by Adrian Hunter.
12 */
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/mtd/mtd.h>
17#include <linux/mtd/blktrans.h>
18#include <linux/rbtree.h>
19#include <linux/sched.h>
20#include <linux/slab.h>
21#include <linux/vmalloc.h>
22#include <linux/blkdev.h>
23#include <linux/swap.h>
24#include <linux/debugfs.h>
25#include <linux/seq_file.h>
26#include <linux/device.h>
27#include <linux/math64.h>
28
29#define MTDSWAP_PREFIX "mtdswap"
30
31/*
32 * The number of free eraseblocks when GC should stop
33 */
34#define CLEAN_BLOCK_THRESHOLD 20
35
36/*
37 * Number of free eraseblocks below which GC can also collect low frag
38 * blocks.
39 */
40#define LOW_FRAG_GC_THRESHOLD 5
41
42/*
43 * Wear level cost amortization. We want to do wear leveling on the background
44 * without disturbing gc too much. This is made by defining max GC frequency.
45 * Frequency value 6 means 1/6 of the GC passes will pick an erase block based
46 * on the biggest wear difference rather than the biggest dirtiness.
47 *
48 * The lower freq2 should be chosen so that it makes sure the maximum erase
49 * difference will decrease even if a malicious application is deliberately
50 * trying to make erase differences large.
51 */
52#define MAX_ERASE_DIFF 4000
53#define COLLECT_NONDIRTY_BASE MAX_ERASE_DIFF
54#define COLLECT_NONDIRTY_FREQ1 6
55#define COLLECT_NONDIRTY_FREQ2 4
56
57#define PAGE_UNDEF UINT_MAX
58#define BLOCK_UNDEF UINT_MAX
59#define BLOCK_ERROR (UINT_MAX - 1)
60#define BLOCK_MAX (UINT_MAX - 2)
61
62#define EBLOCK_BAD (1 << 0)
63#define EBLOCK_NOMAGIC (1 << 1)
64#define EBLOCK_BITFLIP (1 << 2)
65#define EBLOCK_FAILED (1 << 3)
66#define EBLOCK_READERR (1 << 4)
67#define EBLOCK_IDX_SHIFT 5
68
69struct swap_eb {
70 struct rb_node rb;
71 struct rb_root *root;
72
73 unsigned int flags;
74 unsigned int active_count;
75 unsigned int erase_count;
76 unsigned int pad; /* speeds up pointer decrement */
77};
78
79#define MTDSWAP_ECNT_MIN(rbroot) (rb_entry(rb_first(rbroot), struct swap_eb, \
80 rb)->erase_count)
81#define MTDSWAP_ECNT_MAX(rbroot) (rb_entry(rb_last(rbroot), struct swap_eb, \
82 rb)->erase_count)
83
84struct mtdswap_tree {
85 struct rb_root root;
86 unsigned int count;
87};
88
89enum {
90 MTDSWAP_CLEAN,
91 MTDSWAP_USED,
92 MTDSWAP_LOWFRAG,
93 MTDSWAP_HIFRAG,
94 MTDSWAP_DIRTY,
95 MTDSWAP_BITFLIP,
96 MTDSWAP_FAILING,
97 MTDSWAP_TREE_CNT,
98};
99
100struct mtdswap_dev {
101 struct mtd_blktrans_dev *mbd_dev;
102 struct mtd_info *mtd;
103 struct device *dev;
104
105 unsigned int *page_data;
106 unsigned int *revmap;
107
108 unsigned int eblks;
109 unsigned int spare_eblks;
110 unsigned int pages_per_eblk;
111 unsigned int max_erase_count;
112 struct swap_eb *eb_data;
113
114 struct mtdswap_tree trees[MTDSWAP_TREE_CNT];
115
116 unsigned long long sect_read_count;
117 unsigned long long sect_write_count;
118 unsigned long long mtd_write_count;
119 unsigned long long mtd_read_count;
120 unsigned long long discard_count;
121 unsigned long long discard_page_count;
122
123 unsigned int curr_write_pos;
124 struct swap_eb *curr_write;
125
126 char *page_buf;
127 char *oob_buf;
128};
129
130struct mtdswap_oobdata {
131 __le16 magic;
132 __le32 count;
133} __packed;
134
135#define MTDSWAP_MAGIC_CLEAN 0x2095
136#define MTDSWAP_MAGIC_DIRTY (MTDSWAP_MAGIC_CLEAN + 1)
137#define MTDSWAP_TYPE_CLEAN 0
138#define MTDSWAP_TYPE_DIRTY 1
139#define MTDSWAP_OOBSIZE sizeof(struct mtdswap_oobdata)
140
141#define MTDSWAP_ERASE_RETRIES 3 /* Before marking erase block bad */
142#define MTDSWAP_IO_RETRIES 3
143
144enum {
145 MTDSWAP_SCANNED_CLEAN,
146 MTDSWAP_SCANNED_DIRTY,
147 MTDSWAP_SCANNED_BITFLIP,
148 MTDSWAP_SCANNED_BAD,
149};
150
151/*
152 * In the worst case mtdswap_writesect() has allocated the last clean
153 * page from the current block and is then pre-empted by the GC
154 * thread. The thread can consume a full erase block when moving a
155 * block.
156 */
157#define MIN_SPARE_EBLOCKS 2
158#define MIN_ERASE_BLOCKS (MIN_SPARE_EBLOCKS + 1)
159
160#define TREE_ROOT(d, name) (&d->trees[MTDSWAP_ ## name].root)
161#define TREE_EMPTY(d, name) (TREE_ROOT(d, name)->rb_node == NULL)
162#define TREE_NONEMPTY(d, name) (!TREE_EMPTY(d, name))
163#define TREE_COUNT(d, name) (d->trees[MTDSWAP_ ## name].count)
164
165#define MTDSWAP_MBD_TO_MTDSWAP(dev) ((struct mtdswap_dev *)dev->priv)
166
167static char partitions[128] = "";
168module_param_string(partitions, partitions, sizeof(partitions), 0444);
169MODULE_PARM_DESC(partitions, "MTD partition numbers to use as swap "
170 "partitions=\"1,3,5\"");
171
172static unsigned int spare_eblocks = 10;
173module_param(spare_eblocks, uint, 0444);
174MODULE_PARM_DESC(spare_eblocks, "Percentage of spare erase blocks for "
175 "garbage collection (default 10%)");
176
177static bool header; /* false */
178module_param(header, bool, 0444);
179MODULE_PARM_DESC(header,
180 "Include builtin swap header (default 0, without header)");
181
182static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background);
183
184static loff_t mtdswap_eb_offset(struct mtdswap_dev *d, struct swap_eb *eb)
185{
186 return (loff_t)(eb - d->eb_data) * d->mtd->erasesize;
187}
188
189static void mtdswap_eb_detach(struct mtdswap_dev *d, struct swap_eb *eb)
190{
191 unsigned int oldidx;
192 struct mtdswap_tree *tp;
193
194 if (eb->root) {
195 tp = container_of(eb->root, struct mtdswap_tree, root);
196 oldidx = tp - &d->trees[0];
197
198 d->trees[oldidx].count--;
199 rb_erase(&eb->rb, eb->root);
200 }
201}
202
203static void __mtdswap_rb_add(struct rb_root *root, struct swap_eb *eb)
204{
205 struct rb_node **p, *parent = NULL;
206 struct swap_eb *cur;
207
208 p = &root->rb_node;
209 while (*p) {
210 parent = *p;
211 cur = rb_entry(parent, struct swap_eb, rb);
212 if (eb->erase_count > cur->erase_count)
213 p = &(*p)->rb_right;
214 else
215 p = &(*p)->rb_left;
216 }
217
218 rb_link_node(&eb->rb, parent, p);
219 rb_insert_color(&eb->rb, root);
220}
221
222static void mtdswap_rb_add(struct mtdswap_dev *d, struct swap_eb *eb, int idx)
223{
224 struct rb_root *root;
225
226 if (eb->root == &d->trees[idx].root)
227 return;
228
229 mtdswap_eb_detach(d, eb);
230 root = &d->trees[idx].root;
231 __mtdswap_rb_add(root, eb);
232 eb->root = root;
233 d->trees[idx].count++;
234}
235
236static struct rb_node *mtdswap_rb_index(struct rb_root *root, unsigned int idx)
237{
238 struct rb_node *p;
239 unsigned int i;
240
241 p = rb_first(root);
242 i = 0;
243 while (i < idx && p) {
244 p = rb_next(p);
245 i++;
246 }
247
248 return p;
249}
250
251static int mtdswap_handle_badblock(struct mtdswap_dev *d, struct swap_eb *eb)
252{
253 int ret;
254 loff_t offset;
255
256 d->spare_eblks--;
257 eb->flags |= EBLOCK_BAD;
258 mtdswap_eb_detach(d, eb);
259 eb->root = NULL;
260
261 /* badblocks not supported */
262 if (!mtd_can_have_bb(d->mtd))
263 return 1;
264
265 offset = mtdswap_eb_offset(d, eb);
266 dev_warn(d->dev, "Marking bad block at %08llx\n", offset);
267 ret = mtd_block_markbad(d->mtd, offset);
268
269 if (ret) {
270 dev_warn(d->dev, "Mark block bad failed for block at %08llx "
271 "error %d\n", offset, ret);
272 return ret;
273 }
274
275 return 1;
276
277}
278
279static int mtdswap_handle_write_error(struct mtdswap_dev *d, struct swap_eb *eb)
280{
281 unsigned int marked = eb->flags & EBLOCK_FAILED;
282 struct swap_eb *curr_write = d->curr_write;
283
284 eb->flags |= EBLOCK_FAILED;
285 if (curr_write == eb) {
286 d->curr_write = NULL;
287
288 if (!marked && d->curr_write_pos != 0) {
289 mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
290 return 0;
291 }
292 }
293
294 return mtdswap_handle_badblock(d, eb);
295}
296
297static int mtdswap_read_oob(struct mtdswap_dev *d, loff_t from,
298 struct mtd_oob_ops *ops)
299{
300 int ret = mtd_read_oob(d->mtd, from, ops);
301
302 if (mtd_is_bitflip(ret))
303 return ret;
304
305 if (ret) {
306 dev_warn(d->dev, "Read OOB failed %d for block at %08llx\n",
307 ret, from);
308 return ret;
309 }
310
311 if (ops->oobretlen < ops->ooblen) {
312 dev_warn(d->dev, "Read OOB return short read (%zd bytes not "
313 "%zd) for block at %08llx\n",
314 ops->oobretlen, ops->ooblen, from);
315 return -EIO;
316 }
317
318 return 0;
319}
320
321static int mtdswap_read_markers(struct mtdswap_dev *d, struct swap_eb *eb)
322{
323 struct mtdswap_oobdata *data, *data2;
324 int ret;
325 loff_t offset;
326 struct mtd_oob_ops ops = { };
327
328 offset = mtdswap_eb_offset(d, eb);
329
330 /* Check first if the block is bad. */
331 if (mtd_can_have_bb(d->mtd) && mtd_block_isbad(d->mtd, offset))
332 return MTDSWAP_SCANNED_BAD;
333
334 ops.ooblen = 2 * d->mtd->oobavail;
335 ops.oobbuf = d->oob_buf;
336 ops.ooboffs = 0;
337 ops.datbuf = NULL;
338 ops.mode = MTD_OPS_AUTO_OOB;
339
340 ret = mtdswap_read_oob(d, offset, &ops);
341
342 if (ret && !mtd_is_bitflip(ret))
343 return ret;
344
345 data = (struct mtdswap_oobdata *)d->oob_buf;
346 data2 = (struct mtdswap_oobdata *)
347 (d->oob_buf + d->mtd->oobavail);
348
349 if (le16_to_cpu(data->magic) == MTDSWAP_MAGIC_CLEAN) {
350 eb->erase_count = le32_to_cpu(data->count);
351 if (mtd_is_bitflip(ret))
352 ret = MTDSWAP_SCANNED_BITFLIP;
353 else {
354 if (le16_to_cpu(data2->magic) == MTDSWAP_MAGIC_DIRTY)
355 ret = MTDSWAP_SCANNED_DIRTY;
356 else
357 ret = MTDSWAP_SCANNED_CLEAN;
358 }
359 } else {
360 eb->flags |= EBLOCK_NOMAGIC;
361 ret = MTDSWAP_SCANNED_DIRTY;
362 }
363
364 return ret;
365}
366
367static int mtdswap_write_marker(struct mtdswap_dev *d, struct swap_eb *eb,
368 u16 marker)
369{
370 struct mtdswap_oobdata n;
371 int ret;
372 loff_t offset;
373 struct mtd_oob_ops ops = { };
374
375 ops.ooboffs = 0;
376 ops.oobbuf = (uint8_t *)&n;
377 ops.mode = MTD_OPS_AUTO_OOB;
378 ops.datbuf = NULL;
379
380 if (marker == MTDSWAP_TYPE_CLEAN) {
381 n.magic = cpu_to_le16(MTDSWAP_MAGIC_CLEAN);
382 n.count = cpu_to_le32(eb->erase_count);
383 ops.ooblen = MTDSWAP_OOBSIZE;
384 offset = mtdswap_eb_offset(d, eb);
385 } else {
386 n.magic = cpu_to_le16(MTDSWAP_MAGIC_DIRTY);
387 ops.ooblen = sizeof(n.magic);
388 offset = mtdswap_eb_offset(d, eb) + d->mtd->writesize;
389 }
390
391 ret = mtd_write_oob(d->mtd, offset, &ops);
392
393 if (ret) {
394 dev_warn(d->dev, "Write OOB failed for block at %08llx "
395 "error %d\n", offset, ret);
396 if (ret == -EIO || mtd_is_eccerr(ret))
397 mtdswap_handle_write_error(d, eb);
398 return ret;
399 }
400
401 if (ops.oobretlen != ops.ooblen) {
402 dev_warn(d->dev, "Short OOB write for block at %08llx: "
403 "%zd not %zd\n",
404 offset, ops.oobretlen, ops.ooblen);
405 return ret;
406 }
407
408 return 0;
409}
410
411/*
412 * Are there any erase blocks without MAGIC_CLEAN header, presumably
413 * because power was cut off after erase but before header write? We
414 * need to guestimate the erase count.
415 */
416static void mtdswap_check_counts(struct mtdswap_dev *d)
417{
418 struct rb_root hist_root = RB_ROOT;
419 struct rb_node *medrb;
420 struct swap_eb *eb;
421 unsigned int i, cnt, median;
422
423 cnt = 0;
424 for (i = 0; i < d->eblks; i++) {
425 eb = d->eb_data + i;
426
427 if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
428 continue;
429
430 __mtdswap_rb_add(&hist_root, eb);
431 cnt++;
432 }
433
434 if (cnt == 0)
435 return;
436
437 medrb = mtdswap_rb_index(&hist_root, cnt / 2);
438 median = rb_entry(medrb, struct swap_eb, rb)->erase_count;
439
440 d->max_erase_count = MTDSWAP_ECNT_MAX(&hist_root);
441
442 for (i = 0; i < d->eblks; i++) {
443 eb = d->eb_data + i;
444
445 if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_READERR))
446 eb->erase_count = median;
447
448 if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
449 continue;
450
451 rb_erase(&eb->rb, &hist_root);
452 }
453}
454
455static void mtdswap_scan_eblks(struct mtdswap_dev *d)
456{
457 int status;
458 unsigned int i, idx;
459 struct swap_eb *eb;
460
461 for (i = 0; i < d->eblks; i++) {
462 eb = d->eb_data + i;
463
464 status = mtdswap_read_markers(d, eb);
465 if (status < 0)
466 eb->flags |= EBLOCK_READERR;
467 else if (status == MTDSWAP_SCANNED_BAD) {
468 eb->flags |= EBLOCK_BAD;
469 continue;
470 }
471
472 switch (status) {
473 case MTDSWAP_SCANNED_CLEAN:
474 idx = MTDSWAP_CLEAN;
475 break;
476 case MTDSWAP_SCANNED_DIRTY:
477 case MTDSWAP_SCANNED_BITFLIP:
478 idx = MTDSWAP_DIRTY;
479 break;
480 default:
481 idx = MTDSWAP_FAILING;
482 }
483
484 eb->flags |= (idx << EBLOCK_IDX_SHIFT);
485 }
486
487 mtdswap_check_counts(d);
488
489 for (i = 0; i < d->eblks; i++) {
490 eb = d->eb_data + i;
491
492 if (eb->flags & EBLOCK_BAD)
493 continue;
494
495 idx = eb->flags >> EBLOCK_IDX_SHIFT;
496 mtdswap_rb_add(d, eb, idx);
497 }
498}
499
500/*
501 * Place eblk into a tree corresponding to its number of active blocks
502 * it contains.
503 */
504static void mtdswap_store_eb(struct mtdswap_dev *d, struct swap_eb *eb)
505{
506 unsigned int weight = eb->active_count;
507 unsigned int maxweight = d->pages_per_eblk;
508
509 if (eb == d->curr_write)
510 return;
511
512 if (eb->flags & EBLOCK_BITFLIP)
513 mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
514 else if (eb->flags & (EBLOCK_READERR | EBLOCK_FAILED))
515 mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
516 if (weight == maxweight)
517 mtdswap_rb_add(d, eb, MTDSWAP_USED);
518 else if (weight == 0)
519 mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
520 else if (weight > (maxweight/2))
521 mtdswap_rb_add(d, eb, MTDSWAP_LOWFRAG);
522 else
523 mtdswap_rb_add(d, eb, MTDSWAP_HIFRAG);
524}
525
526static int mtdswap_erase_block(struct mtdswap_dev *d, struct swap_eb *eb)
527{
528 struct mtd_info *mtd = d->mtd;
529 struct erase_info erase;
530 unsigned int retries = 0;
531 int ret;
532
533 eb->erase_count++;
534 if (eb->erase_count > d->max_erase_count)
535 d->max_erase_count = eb->erase_count;
536
537retry:
538 memset(&erase, 0, sizeof(struct erase_info));
539 erase.addr = mtdswap_eb_offset(d, eb);
540 erase.len = mtd->erasesize;
541
542 ret = mtd_erase(mtd, &erase);
543 if (ret) {
544 if (retries++ < MTDSWAP_ERASE_RETRIES) {
545 dev_warn(d->dev,
546 "erase of erase block %#llx on %s failed",
547 erase.addr, mtd->name);
548 yield();
549 goto retry;
550 }
551
552 dev_err(d->dev, "Cannot erase erase block %#llx on %s\n",
553 erase.addr, mtd->name);
554
555 mtdswap_handle_badblock(d, eb);
556 return -EIO;
557 }
558
559 return 0;
560}
561
562static int mtdswap_map_free_block(struct mtdswap_dev *d, unsigned int page,
563 unsigned int *block)
564{
565 int ret;
566 struct swap_eb *old_eb = d->curr_write;
567 struct rb_root *clean_root;
568 struct swap_eb *eb;
569
570 if (old_eb == NULL || d->curr_write_pos >= d->pages_per_eblk) {
571 do {
572 if (TREE_EMPTY(d, CLEAN))
573 return -ENOSPC;
574
575 clean_root = TREE_ROOT(d, CLEAN);
576 eb = rb_entry(rb_first(clean_root), struct swap_eb, rb);
577 rb_erase(&eb->rb, clean_root);
578 eb->root = NULL;
579 TREE_COUNT(d, CLEAN)--;
580
581 ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_DIRTY);
582 } while (ret == -EIO || mtd_is_eccerr(ret));
583
584 if (ret)
585 return ret;
586
587 d->curr_write_pos = 0;
588 d->curr_write = eb;
589 if (old_eb)
590 mtdswap_store_eb(d, old_eb);
591 }
592
593 *block = (d->curr_write - d->eb_data) * d->pages_per_eblk +
594 d->curr_write_pos;
595
596 d->curr_write->active_count++;
597 d->revmap[*block] = page;
598 d->curr_write_pos++;
599
600 return 0;
601}
602
603static unsigned int mtdswap_free_page_cnt(struct mtdswap_dev *d)
604{
605 return TREE_COUNT(d, CLEAN) * d->pages_per_eblk +
606 d->pages_per_eblk - d->curr_write_pos;
607}
608
609static unsigned int mtdswap_enough_free_pages(struct mtdswap_dev *d)
610{
611 return mtdswap_free_page_cnt(d) > d->pages_per_eblk;
612}
613
614static int mtdswap_write_block(struct mtdswap_dev *d, char *buf,
615 unsigned int page, unsigned int *bp, int gc_context)
616{
617 struct mtd_info *mtd = d->mtd;
618 struct swap_eb *eb;
619 size_t retlen;
620 loff_t writepos;
621 int ret;
622
623retry:
624 if (!gc_context)
625 while (!mtdswap_enough_free_pages(d))
626 if (mtdswap_gc(d, 0) > 0)
627 return -ENOSPC;
628
629 ret = mtdswap_map_free_block(d, page, bp);
630 eb = d->eb_data + (*bp / d->pages_per_eblk);
631
632 if (ret == -EIO || mtd_is_eccerr(ret)) {
633 d->curr_write = NULL;
634 eb->active_count--;
635 d->revmap[*bp] = PAGE_UNDEF;
636 goto retry;
637 }
638
639 if (ret < 0)
640 return ret;
641
642 writepos = (loff_t)*bp << PAGE_SHIFT;
643 ret = mtd_write(mtd, writepos, PAGE_SIZE, &retlen, buf);
644 if (ret == -EIO || mtd_is_eccerr(ret)) {
645 d->curr_write_pos--;
646 eb->active_count--;
647 d->revmap[*bp] = PAGE_UNDEF;
648 mtdswap_handle_write_error(d, eb);
649 goto retry;
650 }
651
652 if (ret < 0) {
653 dev_err(d->dev, "Write to MTD device failed: %d (%zd written)",
654 ret, retlen);
655 goto err;
656 }
657
658 if (retlen != PAGE_SIZE) {
659 dev_err(d->dev, "Short write to MTD device: %zd written",
660 retlen);
661 ret = -EIO;
662 goto err;
663 }
664
665 return ret;
666
667err:
668 d->curr_write_pos--;
669 eb->active_count--;
670 d->revmap[*bp] = PAGE_UNDEF;
671
672 return ret;
673}
674
675static int mtdswap_move_block(struct mtdswap_dev *d, unsigned int oldblock,
676 unsigned int *newblock)
677{
678 struct mtd_info *mtd = d->mtd;
679 struct swap_eb *eb, *oldeb;
680 int ret;
681 size_t retlen;
682 unsigned int page, retries;
683 loff_t readpos;
684
685 page = d->revmap[oldblock];
686 readpos = (loff_t) oldblock << PAGE_SHIFT;
687 retries = 0;
688
689retry:
690 ret = mtd_read(mtd, readpos, PAGE_SIZE, &retlen, d->page_buf);
691
692 if (ret < 0 && !mtd_is_bitflip(ret)) {
693 oldeb = d->eb_data + oldblock / d->pages_per_eblk;
694 oldeb->flags |= EBLOCK_READERR;
695
696 dev_err(d->dev, "Read Error: %d (block %u)\n", ret,
697 oldblock);
698 retries++;
699 if (retries < MTDSWAP_IO_RETRIES)
700 goto retry;
701
702 goto read_error;
703 }
704
705 if (retlen != PAGE_SIZE) {
706 dev_err(d->dev, "Short read: %zd (block %u)\n", retlen,
707 oldblock);
708 ret = -EIO;
709 goto read_error;
710 }
711
712 ret = mtdswap_write_block(d, d->page_buf, page, newblock, 1);
713 if (ret < 0) {
714 d->page_data[page] = BLOCK_ERROR;
715 dev_err(d->dev, "Write error: %d\n", ret);
716 return ret;
717 }
718
719 d->page_data[page] = *newblock;
720 d->revmap[oldblock] = PAGE_UNDEF;
721 eb = d->eb_data + oldblock / d->pages_per_eblk;
722 eb->active_count--;
723
724 return 0;
725
726read_error:
727 d->page_data[page] = BLOCK_ERROR;
728 d->revmap[oldblock] = PAGE_UNDEF;
729 return ret;
730}
731
732static int mtdswap_gc_eblock(struct mtdswap_dev *d, struct swap_eb *eb)
733{
734 unsigned int i, block, eblk_base, newblock;
735 int ret, errcode;
736
737 errcode = 0;
738 eblk_base = (eb - d->eb_data) * d->pages_per_eblk;
739
740 for (i = 0; i < d->pages_per_eblk; i++) {
741 if (d->spare_eblks < MIN_SPARE_EBLOCKS)
742 return -ENOSPC;
743
744 block = eblk_base + i;
745 if (d->revmap[block] == PAGE_UNDEF)
746 continue;
747
748 ret = mtdswap_move_block(d, block, &newblock);
749 if (ret < 0 && !errcode)
750 errcode = ret;
751 }
752
753 return errcode;
754}
755
756static int __mtdswap_choose_gc_tree(struct mtdswap_dev *d)
757{
758 int idx, stopat;
759
760 if (TREE_COUNT(d, CLEAN) < LOW_FRAG_GC_THRESHOLD)
761 stopat = MTDSWAP_LOWFRAG;
762 else
763 stopat = MTDSWAP_HIFRAG;
764
765 for (idx = MTDSWAP_BITFLIP; idx >= stopat; idx--)
766 if (d->trees[idx].root.rb_node != NULL)
767 return idx;
768
769 return -1;
770}
771
772static int mtdswap_wlfreq(unsigned int maxdiff)
773{
774 unsigned int h, x, y, dist, base;
775
776 /*
777 * Calculate linear ramp down from f1 to f2 when maxdiff goes from
778 * MAX_ERASE_DIFF to MAX_ERASE_DIFF + COLLECT_NONDIRTY_BASE. Similar
779 * to triangle with height f1 - f1 and width COLLECT_NONDIRTY_BASE.
780 */
781
782 dist = maxdiff - MAX_ERASE_DIFF;
783 if (dist > COLLECT_NONDIRTY_BASE)
784 dist = COLLECT_NONDIRTY_BASE;
785
786 /*
787 * Modelling the slop as right angular triangle with base
788 * COLLECT_NONDIRTY_BASE and height freq1 - freq2. The ratio y/x is
789 * equal to the ratio h/base.
790 */
791 h = COLLECT_NONDIRTY_FREQ1 - COLLECT_NONDIRTY_FREQ2;
792 base = COLLECT_NONDIRTY_BASE;
793
794 x = dist - base;
795 y = (x * h + base / 2) / base;
796
797 return COLLECT_NONDIRTY_FREQ2 + y;
798}
799
800static int mtdswap_choose_wl_tree(struct mtdswap_dev *d)
801{
802 static unsigned int pick_cnt;
803 unsigned int i, idx = -1, wear, max;
804 struct rb_root *root;
805
806 max = 0;
807 for (i = 0; i <= MTDSWAP_DIRTY; i++) {
808 root = &d->trees[i].root;
809 if (root->rb_node == NULL)
810 continue;
811
812 wear = d->max_erase_count - MTDSWAP_ECNT_MIN(root);
813 if (wear > max) {
814 max = wear;
815 idx = i;
816 }
817 }
818
819 if (max > MAX_ERASE_DIFF && pick_cnt >= mtdswap_wlfreq(max) - 1) {
820 pick_cnt = 0;
821 return idx;
822 }
823
824 pick_cnt++;
825 return -1;
826}
827
828static int mtdswap_choose_gc_tree(struct mtdswap_dev *d,
829 unsigned int background)
830{
831 int idx;
832
833 if (TREE_NONEMPTY(d, FAILING) &&
834 (background || (TREE_EMPTY(d, CLEAN) && TREE_EMPTY(d, DIRTY))))
835 return MTDSWAP_FAILING;
836
837 idx = mtdswap_choose_wl_tree(d);
838 if (idx >= MTDSWAP_CLEAN)
839 return idx;
840
841 return __mtdswap_choose_gc_tree(d);
842}
843
844static struct swap_eb *mtdswap_pick_gc_eblk(struct mtdswap_dev *d,
845 unsigned int background)
846{
847 struct rb_root *rp = NULL;
848 struct swap_eb *eb = NULL;
849 int idx;
850
851 if (background && TREE_COUNT(d, CLEAN) > CLEAN_BLOCK_THRESHOLD &&
852 TREE_EMPTY(d, DIRTY) && TREE_EMPTY(d, FAILING))
853 return NULL;
854
855 idx = mtdswap_choose_gc_tree(d, background);
856 if (idx < 0)
857 return NULL;
858
859 rp = &d->trees[idx].root;
860 eb = rb_entry(rb_first(rp), struct swap_eb, rb);
861
862 rb_erase(&eb->rb, rp);
863 eb->root = NULL;
864 d->trees[idx].count--;
865 return eb;
866}
867
868static unsigned int mtdswap_test_patt(unsigned int i)
869{
870 return i % 2 ? 0x55555555 : 0xAAAAAAAA;
871}
872
873static unsigned int mtdswap_eblk_passes(struct mtdswap_dev *d,
874 struct swap_eb *eb)
875{
876 struct mtd_info *mtd = d->mtd;
877 unsigned int test, i, j, patt, mtd_pages;
878 loff_t base, pos;
879 unsigned int *p1 = (unsigned int *)d->page_buf;
880 unsigned char *p2 = (unsigned char *)d->oob_buf;
881 struct mtd_oob_ops ops = { };
882 int ret;
883
884 ops.mode = MTD_OPS_AUTO_OOB;
885 ops.len = mtd->writesize;
886 ops.ooblen = mtd->oobavail;
887 ops.ooboffs = 0;
888 ops.datbuf = d->page_buf;
889 ops.oobbuf = d->oob_buf;
890 base = mtdswap_eb_offset(d, eb);
891 mtd_pages = d->pages_per_eblk * PAGE_SIZE / mtd->writesize;
892
893 for (test = 0; test < 2; test++) {
894 pos = base;
895 for (i = 0; i < mtd_pages; i++) {
896 patt = mtdswap_test_patt(test + i);
897 memset(d->page_buf, patt, mtd->writesize);
898 memset(d->oob_buf, patt, mtd->oobavail);
899 ret = mtd_write_oob(mtd, pos, &ops);
900 if (ret)
901 goto error;
902
903 pos += mtd->writesize;
904 }
905
906 pos = base;
907 for (i = 0; i < mtd_pages; i++) {
908 ret = mtd_read_oob(mtd, pos, &ops);
909 if (ret)
910 goto error;
911
912 patt = mtdswap_test_patt(test + i);
913 for (j = 0; j < mtd->writesize/sizeof(int); j++)
914 if (p1[j] != patt)
915 goto error;
916
917 for (j = 0; j < mtd->oobavail; j++)
918 if (p2[j] != (unsigned char)patt)
919 goto error;
920
921 pos += mtd->writesize;
922 }
923
924 ret = mtdswap_erase_block(d, eb);
925 if (ret)
926 goto error;
927 }
928
929 eb->flags &= ~EBLOCK_READERR;
930 return 1;
931
932error:
933 mtdswap_handle_badblock(d, eb);
934 return 0;
935}
936
937static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background)
938{
939 struct swap_eb *eb;
940 int ret;
941
942 if (d->spare_eblks < MIN_SPARE_EBLOCKS)
943 return 1;
944
945 eb = mtdswap_pick_gc_eblk(d, background);
946 if (!eb)
947 return 1;
948
949 ret = mtdswap_gc_eblock(d, eb);
950 if (ret == -ENOSPC)
951 return 1;
952
953 if (eb->flags & EBLOCK_FAILED) {
954 mtdswap_handle_badblock(d, eb);
955 return 0;
956 }
957
958 eb->flags &= ~EBLOCK_BITFLIP;
959 ret = mtdswap_erase_block(d, eb);
960 if ((eb->flags & EBLOCK_READERR) &&
961 (ret || !mtdswap_eblk_passes(d, eb)))
962 return 0;
963
964 if (ret == 0)
965 ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_CLEAN);
966
967 if (ret == 0)
968 mtdswap_rb_add(d, eb, MTDSWAP_CLEAN);
969 else if (ret != -EIO && !mtd_is_eccerr(ret))
970 mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
971
972 return 0;
973}
974
975static void mtdswap_background(struct mtd_blktrans_dev *dev)
976{
977 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
978 int ret;
979
980 while (1) {
981 ret = mtdswap_gc(d, 1);
982 if (ret || mtd_blktrans_cease_background(dev))
983 return;
984 }
985}
986
987static void mtdswap_cleanup(struct mtdswap_dev *d)
988{
989 vfree(d->eb_data);
990 vfree(d->revmap);
991 vfree(d->page_data);
992 kfree(d->oob_buf);
993 kfree(d->page_buf);
994}
995
996static int mtdswap_flush(struct mtd_blktrans_dev *dev)
997{
998 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
999
1000 mtd_sync(d->mtd);
1001 return 0;
1002}
1003
1004static unsigned int mtdswap_badblocks(struct mtd_info *mtd, uint64_t size)
1005{
1006 loff_t offset;
1007 unsigned int badcnt;
1008
1009 badcnt = 0;
1010
1011 if (mtd_can_have_bb(mtd))
1012 for (offset = 0; offset < size; offset += mtd->erasesize)
1013 if (mtd_block_isbad(mtd, offset))
1014 badcnt++;
1015
1016 return badcnt;
1017}
1018
1019static int mtdswap_writesect(struct mtd_blktrans_dev *dev,
1020 unsigned long page, char *buf)
1021{
1022 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1023 unsigned int newblock, mapped;
1024 struct swap_eb *eb;
1025 int ret;
1026
1027 d->sect_write_count++;
1028
1029 if (d->spare_eblks < MIN_SPARE_EBLOCKS)
1030 return -ENOSPC;
1031
1032 if (header) {
1033 /* Ignore writes to the header page */
1034 if (unlikely(page == 0))
1035 return 0;
1036
1037 page--;
1038 }
1039
1040 mapped = d->page_data[page];
1041 if (mapped <= BLOCK_MAX) {
1042 eb = d->eb_data + (mapped / d->pages_per_eblk);
1043 eb->active_count--;
1044 mtdswap_store_eb(d, eb);
1045 d->page_data[page] = BLOCK_UNDEF;
1046 d->revmap[mapped] = PAGE_UNDEF;
1047 }
1048
1049 ret = mtdswap_write_block(d, buf, page, &newblock, 0);
1050 d->mtd_write_count++;
1051
1052 if (ret < 0)
1053 return ret;
1054
1055 d->page_data[page] = newblock;
1056
1057 return 0;
1058}
1059
1060/* Provide a dummy swap header for the kernel */
1061static int mtdswap_auto_header(struct mtdswap_dev *d, char *buf)
1062{
1063 union swap_header *hd = (union swap_header *)(buf);
1064
1065 memset(buf, 0, PAGE_SIZE - 10);
1066
1067 hd->info.version = 1;
1068 hd->info.last_page = d->mbd_dev->size - 1;
1069 hd->info.nr_badpages = 0;
1070
1071 memcpy(buf + PAGE_SIZE - 10, "SWAPSPACE2", 10);
1072
1073 return 0;
1074}
1075
1076static int mtdswap_readsect(struct mtd_blktrans_dev *dev,
1077 unsigned long page, char *buf)
1078{
1079 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1080 struct mtd_info *mtd = d->mtd;
1081 unsigned int realblock, retries;
1082 loff_t readpos;
1083 struct swap_eb *eb;
1084 size_t retlen;
1085 int ret;
1086
1087 d->sect_read_count++;
1088
1089 if (header) {
1090 if (unlikely(page == 0))
1091 return mtdswap_auto_header(d, buf);
1092
1093 page--;
1094 }
1095
1096 realblock = d->page_data[page];
1097 if (realblock > BLOCK_MAX) {
1098 memset(buf, 0x0, PAGE_SIZE);
1099 if (realblock == BLOCK_UNDEF)
1100 return 0;
1101 else
1102 return -EIO;
1103 }
1104
1105 eb = d->eb_data + (realblock / d->pages_per_eblk);
1106 BUG_ON(d->revmap[realblock] == PAGE_UNDEF);
1107
1108 readpos = (loff_t)realblock << PAGE_SHIFT;
1109 retries = 0;
1110
1111retry:
1112 ret = mtd_read(mtd, readpos, PAGE_SIZE, &retlen, buf);
1113
1114 d->mtd_read_count++;
1115 if (mtd_is_bitflip(ret)) {
1116 eb->flags |= EBLOCK_BITFLIP;
1117 mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
1118 ret = 0;
1119 }
1120
1121 if (ret < 0) {
1122 dev_err(d->dev, "Read error %d\n", ret);
1123 eb->flags |= EBLOCK_READERR;
1124 mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
1125 retries++;
1126 if (retries < MTDSWAP_IO_RETRIES)
1127 goto retry;
1128
1129 return ret;
1130 }
1131
1132 if (retlen != PAGE_SIZE) {
1133 dev_err(d->dev, "Short read %zd\n", retlen);
1134 return -EIO;
1135 }
1136
1137 return 0;
1138}
1139
1140static int mtdswap_discard(struct mtd_blktrans_dev *dev, unsigned long first,
1141 unsigned nr_pages)
1142{
1143 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1144 unsigned long page;
1145 struct swap_eb *eb;
1146 unsigned int mapped;
1147
1148 d->discard_count++;
1149
1150 for (page = first; page < first + nr_pages; page++) {
1151 mapped = d->page_data[page];
1152 if (mapped <= BLOCK_MAX) {
1153 eb = d->eb_data + (mapped / d->pages_per_eblk);
1154 eb->active_count--;
1155 mtdswap_store_eb(d, eb);
1156 d->page_data[page] = BLOCK_UNDEF;
1157 d->revmap[mapped] = PAGE_UNDEF;
1158 d->discard_page_count++;
1159 } else if (mapped == BLOCK_ERROR) {
1160 d->page_data[page] = BLOCK_UNDEF;
1161 d->discard_page_count++;
1162 }
1163 }
1164
1165 return 0;
1166}
1167
1168static int mtdswap_show(struct seq_file *s, void *data)
1169{
1170 struct mtdswap_dev *d = (struct mtdswap_dev *) s->private;
1171 unsigned long sum;
1172 unsigned int count[MTDSWAP_TREE_CNT];
1173 unsigned int min[MTDSWAP_TREE_CNT];
1174 unsigned int max[MTDSWAP_TREE_CNT];
1175 unsigned int i, cw = 0, cwp = 0, cwecount = 0, bb_cnt, mapped, pages;
1176 uint64_t use_size;
1177 static const char * const name[] = {
1178 "clean", "used", "low", "high", "dirty", "bitflip", "failing"
1179 };
1180
1181 mutex_lock(&d->mbd_dev->lock);
1182
1183 for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
1184 struct rb_root *root = &d->trees[i].root;
1185
1186 if (root->rb_node) {
1187 count[i] = d->trees[i].count;
1188 min[i] = MTDSWAP_ECNT_MIN(root);
1189 max[i] = MTDSWAP_ECNT_MAX(root);
1190 } else
1191 count[i] = 0;
1192 }
1193
1194 if (d->curr_write) {
1195 cw = 1;
1196 cwp = d->curr_write_pos;
1197 cwecount = d->curr_write->erase_count;
1198 }
1199
1200 sum = 0;
1201 for (i = 0; i < d->eblks; i++)
1202 sum += d->eb_data[i].erase_count;
1203
1204 use_size = (uint64_t)d->eblks * d->mtd->erasesize;
1205 bb_cnt = mtdswap_badblocks(d->mtd, use_size);
1206
1207 mapped = 0;
1208 pages = d->mbd_dev->size;
1209 for (i = 0; i < pages; i++)
1210 if (d->page_data[i] != BLOCK_UNDEF)
1211 mapped++;
1212
1213 mutex_unlock(&d->mbd_dev->lock);
1214
1215 for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
1216 if (!count[i])
1217 continue;
1218
1219 if (min[i] != max[i])
1220 seq_printf(s, "%s:\t%5d erase blocks, erased min %d, "
1221 "max %d times\n",
1222 name[i], count[i], min[i], max[i]);
1223 else
1224 seq_printf(s, "%s:\t%5d erase blocks, all erased %d "
1225 "times\n", name[i], count[i], min[i]);
1226 }
1227
1228 if (bb_cnt)
1229 seq_printf(s, "bad:\t%5u erase blocks\n", bb_cnt);
1230
1231 if (cw)
1232 seq_printf(s, "current erase block: %u pages used, %u free, "
1233 "erased %u times\n",
1234 cwp, d->pages_per_eblk - cwp, cwecount);
1235
1236 seq_printf(s, "total erasures: %lu\n", sum);
1237
1238 seq_puts(s, "\n");
1239
1240 seq_printf(s, "mtdswap_readsect count: %llu\n", d->sect_read_count);
1241 seq_printf(s, "mtdswap_writesect count: %llu\n", d->sect_write_count);
1242 seq_printf(s, "mtdswap_discard count: %llu\n", d->discard_count);
1243 seq_printf(s, "mtd read count: %llu\n", d->mtd_read_count);
1244 seq_printf(s, "mtd write count: %llu\n", d->mtd_write_count);
1245 seq_printf(s, "discarded pages count: %llu\n", d->discard_page_count);
1246
1247 seq_puts(s, "\n");
1248 seq_printf(s, "total pages: %u\n", pages);
1249 seq_printf(s, "pages mapped: %u\n", mapped);
1250
1251 return 0;
1252}
1253DEFINE_SHOW_ATTRIBUTE(mtdswap);
1254
1255static int mtdswap_add_debugfs(struct mtdswap_dev *d)
1256{
1257 struct dentry *root = d->mtd->dbg.dfs_dir;
1258
1259 if (!IS_ENABLED(CONFIG_DEBUG_FS))
1260 return 0;
1261
1262 if (IS_ERR_OR_NULL(root))
1263 return -1;
1264
1265 debugfs_create_file("mtdswap_stats", S_IRUSR, root, d, &mtdswap_fops);
1266
1267 return 0;
1268}
1269
1270static int mtdswap_init(struct mtdswap_dev *d, unsigned int eblocks,
1271 unsigned int spare_cnt)
1272{
1273 struct mtd_info *mtd = d->mbd_dev->mtd;
1274 unsigned int i, eblk_bytes, pages, blocks;
1275 int ret = -ENOMEM;
1276
1277 d->mtd = mtd;
1278 d->eblks = eblocks;
1279 d->spare_eblks = spare_cnt;
1280 d->pages_per_eblk = mtd->erasesize >> PAGE_SHIFT;
1281
1282 pages = d->mbd_dev->size;
1283 blocks = eblocks * d->pages_per_eblk;
1284
1285 for (i = 0; i < MTDSWAP_TREE_CNT; i++)
1286 d->trees[i].root = RB_ROOT;
1287
1288 d->page_data = vmalloc(array_size(pages, sizeof(int)));
1289 if (!d->page_data)
1290 goto page_data_fail;
1291
1292 d->revmap = vmalloc(array_size(blocks, sizeof(int)));
1293 if (!d->revmap)
1294 goto revmap_fail;
1295
1296 eblk_bytes = sizeof(struct swap_eb)*d->eblks;
1297 d->eb_data = vzalloc(eblk_bytes);
1298 if (!d->eb_data)
1299 goto eb_data_fail;
1300
1301 for (i = 0; i < pages; i++)
1302 d->page_data[i] = BLOCK_UNDEF;
1303
1304 for (i = 0; i < blocks; i++)
1305 d->revmap[i] = PAGE_UNDEF;
1306
1307 d->page_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1308 if (!d->page_buf)
1309 goto page_buf_fail;
1310
1311 d->oob_buf = kmalloc_array(2, mtd->oobavail, GFP_KERNEL);
1312 if (!d->oob_buf)
1313 goto oob_buf_fail;
1314
1315 mtdswap_scan_eblks(d);
1316
1317 return 0;
1318
1319oob_buf_fail:
1320 kfree(d->page_buf);
1321page_buf_fail:
1322 vfree(d->eb_data);
1323eb_data_fail:
1324 vfree(d->revmap);
1325revmap_fail:
1326 vfree(d->page_data);
1327page_data_fail:
1328 printk(KERN_ERR "%s: init failed (%d)\n", MTDSWAP_PREFIX, ret);
1329 return ret;
1330}
1331
1332static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
1333{
1334 struct mtdswap_dev *d;
1335 struct mtd_blktrans_dev *mbd_dev;
1336 char *parts;
1337 char *this_opt;
1338 unsigned long part;
1339 unsigned int eblocks, eavailable, bad_blocks, spare_cnt;
1340 uint64_t swap_size, use_size, size_limit;
1341 int ret;
1342
1343 parts = &partitions[0];
1344 if (!*parts)
1345 return;
1346
1347 while ((this_opt = strsep(&parts, ",")) != NULL) {
1348 if (kstrtoul(this_opt, 0, &part) < 0)
1349 return;
1350
1351 if (mtd->index == part)
1352 break;
1353 }
1354
1355 if (mtd->index != part)
1356 return;
1357
1358 if (mtd->erasesize < PAGE_SIZE || mtd->erasesize % PAGE_SIZE) {
1359 printk(KERN_ERR "%s: Erase size %u not multiple of PAGE_SIZE "
1360 "%lu\n", MTDSWAP_PREFIX, mtd->erasesize, PAGE_SIZE);
1361 return;
1362 }
1363
1364 if (PAGE_SIZE % mtd->writesize || mtd->writesize > PAGE_SIZE) {
1365 printk(KERN_ERR "%s: PAGE_SIZE %lu not multiple of write size"
1366 " %u\n", MTDSWAP_PREFIX, PAGE_SIZE, mtd->writesize);
1367 return;
1368 }
1369
1370 if (!mtd->oobsize || mtd->oobavail < MTDSWAP_OOBSIZE) {
1371 printk(KERN_ERR "%s: Not enough free bytes in OOB, "
1372 "%d available, %zu needed.\n",
1373 MTDSWAP_PREFIX, mtd->oobavail, MTDSWAP_OOBSIZE);
1374 return;
1375 }
1376
1377 if (spare_eblocks > 100)
1378 spare_eblocks = 100;
1379
1380 use_size = mtd->size;
1381 size_limit = (uint64_t) BLOCK_MAX * PAGE_SIZE;
1382
1383 if (mtd->size > size_limit) {
1384 printk(KERN_WARNING "%s: Device too large. Limiting size to "
1385 "%llu bytes\n", MTDSWAP_PREFIX, size_limit);
1386 use_size = size_limit;
1387 }
1388
1389 eblocks = mtd_div_by_eb(use_size, mtd);
1390 use_size = (uint64_t)eblocks * mtd->erasesize;
1391 bad_blocks = mtdswap_badblocks(mtd, use_size);
1392 eavailable = eblocks - bad_blocks;
1393
1394 if (eavailable < MIN_ERASE_BLOCKS) {
1395 printk(KERN_ERR "%s: Not enough erase blocks. %u available, "
1396 "%d needed\n", MTDSWAP_PREFIX, eavailable,
1397 MIN_ERASE_BLOCKS);
1398 return;
1399 }
1400
1401 spare_cnt = div_u64((uint64_t)eavailable * spare_eblocks, 100);
1402
1403 if (spare_cnt < MIN_SPARE_EBLOCKS)
1404 spare_cnt = MIN_SPARE_EBLOCKS;
1405
1406 if (spare_cnt > eavailable - 1)
1407 spare_cnt = eavailable - 1;
1408
1409 swap_size = (uint64_t)(eavailable - spare_cnt) * mtd->erasesize +
1410 (header ? PAGE_SIZE : 0);
1411
1412 printk(KERN_INFO "%s: Enabling MTD swap on device %lu, size %llu KB, "
1413 "%u spare, %u bad blocks\n",
1414 MTDSWAP_PREFIX, part, swap_size / 1024, spare_cnt, bad_blocks);
1415
1416 d = kzalloc(sizeof(struct mtdswap_dev), GFP_KERNEL);
1417 if (!d)
1418 return;
1419
1420 mbd_dev = kzalloc(sizeof(struct mtd_blktrans_dev), GFP_KERNEL);
1421 if (!mbd_dev) {
1422 kfree(d);
1423 return;
1424 }
1425
1426 d->mbd_dev = mbd_dev;
1427 mbd_dev->priv = d;
1428
1429 mbd_dev->mtd = mtd;
1430 mbd_dev->devnum = mtd->index;
1431 mbd_dev->size = swap_size >> PAGE_SHIFT;
1432 mbd_dev->tr = tr;
1433
1434 if (!(mtd->flags & MTD_WRITEABLE))
1435 mbd_dev->readonly = 1;
1436
1437 if (mtdswap_init(d, eblocks, spare_cnt) < 0)
1438 goto init_failed;
1439
1440 if (add_mtd_blktrans_dev(mbd_dev) < 0)
1441 goto cleanup;
1442
1443 d->dev = disk_to_dev(mbd_dev->disk);
1444
1445 ret = mtdswap_add_debugfs(d);
1446 if (ret < 0)
1447 goto debugfs_failed;
1448
1449 return;
1450
1451debugfs_failed:
1452 del_mtd_blktrans_dev(mbd_dev);
1453
1454cleanup:
1455 mtdswap_cleanup(d);
1456
1457init_failed:
1458 kfree(mbd_dev);
1459 kfree(d);
1460}
1461
1462static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
1463{
1464 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1465
1466 del_mtd_blktrans_dev(dev);
1467 mtdswap_cleanup(d);
1468 kfree(d);
1469}
1470
1471static struct mtd_blktrans_ops mtdswap_ops = {
1472 .name = "mtdswap",
1473 .major = 0,
1474 .part_bits = 0,
1475 .blksize = PAGE_SIZE,
1476 .flush = mtdswap_flush,
1477 .readsect = mtdswap_readsect,
1478 .writesect = mtdswap_writesect,
1479 .discard = mtdswap_discard,
1480 .background = mtdswap_background,
1481 .add_mtd = mtdswap_add_mtd,
1482 .remove_dev = mtdswap_remove_dev,
1483 .owner = THIS_MODULE,
1484};
1485
1486module_mtd_blktrans(mtdswap_ops);
1487
1488MODULE_LICENSE("GPL");
1489MODULE_AUTHOR("Jarkko Lavinen <jarkko.lavinen@nokia.com>");
1490MODULE_DESCRIPTION("Block device access to an MTD suitable for using as "
1491 "swap space");
1/*
2 * Swap block device support for MTDs
3 * Turns an MTD device into a swap device with block wear leveling
4 *
5 * Copyright © 2007,2011 Nokia Corporation. All rights reserved.
6 *
7 * Authors: Jarkko Lavinen <jarkko.lavinen@nokia.com>
8 *
9 * Based on Richard Purdie's earlier implementation in 2007. Background
10 * support and lock-less operation written by Adrian Hunter.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * version 2 as published by the Free Software Foundation.
15 *
16 * This program is distributed in the hope that it will be useful, but
17 * WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
24 * 02110-1301 USA
25 */
26
27#include <linux/kernel.h>
28#include <linux/module.h>
29#include <linux/mtd/mtd.h>
30#include <linux/mtd/blktrans.h>
31#include <linux/rbtree.h>
32#include <linux/sched.h>
33#include <linux/slab.h>
34#include <linux/vmalloc.h>
35#include <linux/genhd.h>
36#include <linux/swap.h>
37#include <linux/debugfs.h>
38#include <linux/seq_file.h>
39#include <linux/device.h>
40#include <linux/math64.h>
41
42#define MTDSWAP_PREFIX "mtdswap"
43
44/*
45 * The number of free eraseblocks when GC should stop
46 */
47#define CLEAN_BLOCK_THRESHOLD 20
48
49/*
50 * Number of free eraseblocks below which GC can also collect low frag
51 * blocks.
52 */
53#define LOW_FRAG_GC_TRESHOLD 5
54
55/*
56 * Wear level cost amortization. We want to do wear leveling on the background
57 * without disturbing gc too much. This is made by defining max GC frequency.
58 * Frequency value 6 means 1/6 of the GC passes will pick an erase block based
59 * on the biggest wear difference rather than the biggest dirtiness.
60 *
61 * The lower freq2 should be chosen so that it makes sure the maximum erase
62 * difference will decrease even if a malicious application is deliberately
63 * trying to make erase differences large.
64 */
65#define MAX_ERASE_DIFF 4000
66#define COLLECT_NONDIRTY_BASE MAX_ERASE_DIFF
67#define COLLECT_NONDIRTY_FREQ1 6
68#define COLLECT_NONDIRTY_FREQ2 4
69
70#define PAGE_UNDEF UINT_MAX
71#define BLOCK_UNDEF UINT_MAX
72#define BLOCK_ERROR (UINT_MAX - 1)
73#define BLOCK_MAX (UINT_MAX - 2)
74
75#define EBLOCK_BAD (1 << 0)
76#define EBLOCK_NOMAGIC (1 << 1)
77#define EBLOCK_BITFLIP (1 << 2)
78#define EBLOCK_FAILED (1 << 3)
79#define EBLOCK_READERR (1 << 4)
80#define EBLOCK_IDX_SHIFT 5
81
82struct swap_eb {
83 struct rb_node rb;
84 struct rb_root *root;
85
86 unsigned int flags;
87 unsigned int active_count;
88 unsigned int erase_count;
89 unsigned int pad; /* speeds up pointer decremtnt */
90};
91
92#define MTDSWAP_ECNT_MIN(rbroot) (rb_entry(rb_first(rbroot), struct swap_eb, \
93 rb)->erase_count)
94#define MTDSWAP_ECNT_MAX(rbroot) (rb_entry(rb_last(rbroot), struct swap_eb, \
95 rb)->erase_count)
96
97struct mtdswap_tree {
98 struct rb_root root;
99 unsigned int count;
100};
101
102enum {
103 MTDSWAP_CLEAN,
104 MTDSWAP_USED,
105 MTDSWAP_LOWFRAG,
106 MTDSWAP_HIFRAG,
107 MTDSWAP_DIRTY,
108 MTDSWAP_BITFLIP,
109 MTDSWAP_FAILING,
110 MTDSWAP_TREE_CNT,
111};
112
113struct mtdswap_dev {
114 struct mtd_blktrans_dev *mbd_dev;
115 struct mtd_info *mtd;
116 struct device *dev;
117
118 unsigned int *page_data;
119 unsigned int *revmap;
120
121 unsigned int eblks;
122 unsigned int spare_eblks;
123 unsigned int pages_per_eblk;
124 unsigned int max_erase_count;
125 struct swap_eb *eb_data;
126
127 struct mtdswap_tree trees[MTDSWAP_TREE_CNT];
128
129 unsigned long long sect_read_count;
130 unsigned long long sect_write_count;
131 unsigned long long mtd_write_count;
132 unsigned long long mtd_read_count;
133 unsigned long long discard_count;
134 unsigned long long discard_page_count;
135
136 unsigned int curr_write_pos;
137 struct swap_eb *curr_write;
138
139 char *page_buf;
140 char *oob_buf;
141
142 struct dentry *debugfs_root;
143};
144
145struct mtdswap_oobdata {
146 __le16 magic;
147 __le32 count;
148} __attribute__((packed));
149
150#define MTDSWAP_MAGIC_CLEAN 0x2095
151#define MTDSWAP_MAGIC_DIRTY (MTDSWAP_MAGIC_CLEAN + 1)
152#define MTDSWAP_TYPE_CLEAN 0
153#define MTDSWAP_TYPE_DIRTY 1
154#define MTDSWAP_OOBSIZE sizeof(struct mtdswap_oobdata)
155
156#define MTDSWAP_ERASE_RETRIES 3 /* Before marking erase block bad */
157#define MTDSWAP_IO_RETRIES 3
158
159enum {
160 MTDSWAP_SCANNED_CLEAN,
161 MTDSWAP_SCANNED_DIRTY,
162 MTDSWAP_SCANNED_BITFLIP,
163 MTDSWAP_SCANNED_BAD,
164};
165
166/*
167 * In the worst case mtdswap_writesect() has allocated the last clean
168 * page from the current block and is then pre-empted by the GC
169 * thread. The thread can consume a full erase block when moving a
170 * block.
171 */
172#define MIN_SPARE_EBLOCKS 2
173#define MIN_ERASE_BLOCKS (MIN_SPARE_EBLOCKS + 1)
174
175#define TREE_ROOT(d, name) (&d->trees[MTDSWAP_ ## name].root)
176#define TREE_EMPTY(d, name) (TREE_ROOT(d, name)->rb_node == NULL)
177#define TREE_NONEMPTY(d, name) (!TREE_EMPTY(d, name))
178#define TREE_COUNT(d, name) (d->trees[MTDSWAP_ ## name].count)
179
180#define MTDSWAP_MBD_TO_MTDSWAP(dev) ((struct mtdswap_dev *)dev->priv)
181
182static char partitions[128] = "";
183module_param_string(partitions, partitions, sizeof(partitions), 0444);
184MODULE_PARM_DESC(partitions, "MTD partition numbers to use as swap "
185 "partitions=\"1,3,5\"");
186
187static unsigned int spare_eblocks = 10;
188module_param(spare_eblocks, uint, 0444);
189MODULE_PARM_DESC(spare_eblocks, "Percentage of spare erase blocks for "
190 "garbage collection (default 10%)");
191
192static bool header; /* false */
193module_param(header, bool, 0444);
194MODULE_PARM_DESC(header,
195 "Include builtin swap header (default 0, without header)");
196
197static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background);
198
199static loff_t mtdswap_eb_offset(struct mtdswap_dev *d, struct swap_eb *eb)
200{
201 return (loff_t)(eb - d->eb_data) * d->mtd->erasesize;
202}
203
204static void mtdswap_eb_detach(struct mtdswap_dev *d, struct swap_eb *eb)
205{
206 unsigned int oldidx;
207 struct mtdswap_tree *tp;
208
209 if (eb->root) {
210 tp = container_of(eb->root, struct mtdswap_tree, root);
211 oldidx = tp - &d->trees[0];
212
213 d->trees[oldidx].count--;
214 rb_erase(&eb->rb, eb->root);
215 }
216}
217
218static void __mtdswap_rb_add(struct rb_root *root, struct swap_eb *eb)
219{
220 struct rb_node **p, *parent = NULL;
221 struct swap_eb *cur;
222
223 p = &root->rb_node;
224 while (*p) {
225 parent = *p;
226 cur = rb_entry(parent, struct swap_eb, rb);
227 if (eb->erase_count > cur->erase_count)
228 p = &(*p)->rb_right;
229 else
230 p = &(*p)->rb_left;
231 }
232
233 rb_link_node(&eb->rb, parent, p);
234 rb_insert_color(&eb->rb, root);
235}
236
237static void mtdswap_rb_add(struct mtdswap_dev *d, struct swap_eb *eb, int idx)
238{
239 struct rb_root *root;
240
241 if (eb->root == &d->trees[idx].root)
242 return;
243
244 mtdswap_eb_detach(d, eb);
245 root = &d->trees[idx].root;
246 __mtdswap_rb_add(root, eb);
247 eb->root = root;
248 d->trees[idx].count++;
249}
250
251static struct rb_node *mtdswap_rb_index(struct rb_root *root, unsigned int idx)
252{
253 struct rb_node *p;
254 unsigned int i;
255
256 p = rb_first(root);
257 i = 0;
258 while (i < idx && p) {
259 p = rb_next(p);
260 i++;
261 }
262
263 return p;
264}
265
266static int mtdswap_handle_badblock(struct mtdswap_dev *d, struct swap_eb *eb)
267{
268 int ret;
269 loff_t offset;
270
271 d->spare_eblks--;
272 eb->flags |= EBLOCK_BAD;
273 mtdswap_eb_detach(d, eb);
274 eb->root = NULL;
275
276 /* badblocks not supported */
277 if (!d->mtd->block_markbad)
278 return 1;
279
280 offset = mtdswap_eb_offset(d, eb);
281 dev_warn(d->dev, "Marking bad block at %08llx\n", offset);
282 ret = d->mtd->block_markbad(d->mtd, offset);
283
284 if (ret) {
285 dev_warn(d->dev, "Mark block bad failed for block at %08llx "
286 "error %d\n", offset, ret);
287 return ret;
288 }
289
290 return 1;
291
292}
293
294static int mtdswap_handle_write_error(struct mtdswap_dev *d, struct swap_eb *eb)
295{
296 unsigned int marked = eb->flags & EBLOCK_FAILED;
297 struct swap_eb *curr_write = d->curr_write;
298
299 eb->flags |= EBLOCK_FAILED;
300 if (curr_write == eb) {
301 d->curr_write = NULL;
302
303 if (!marked && d->curr_write_pos != 0) {
304 mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
305 return 0;
306 }
307 }
308
309 return mtdswap_handle_badblock(d, eb);
310}
311
312static int mtdswap_read_oob(struct mtdswap_dev *d, loff_t from,
313 struct mtd_oob_ops *ops)
314{
315 int ret = d->mtd->read_oob(d->mtd, from, ops);
316
317 if (ret == -EUCLEAN)
318 return ret;
319
320 if (ret) {
321 dev_warn(d->dev, "Read OOB failed %d for block at %08llx\n",
322 ret, from);
323 return ret;
324 }
325
326 if (ops->oobretlen < ops->ooblen) {
327 dev_warn(d->dev, "Read OOB return short read (%zd bytes not "
328 "%zd) for block at %08llx\n",
329 ops->oobretlen, ops->ooblen, from);
330 return -EIO;
331 }
332
333 return 0;
334}
335
336static int mtdswap_read_markers(struct mtdswap_dev *d, struct swap_eb *eb)
337{
338 struct mtdswap_oobdata *data, *data2;
339 int ret;
340 loff_t offset;
341 struct mtd_oob_ops ops;
342
343 offset = mtdswap_eb_offset(d, eb);
344
345 /* Check first if the block is bad. */
346 if (d->mtd->block_isbad && d->mtd->block_isbad(d->mtd, offset))
347 return MTDSWAP_SCANNED_BAD;
348
349 ops.ooblen = 2 * d->mtd->ecclayout->oobavail;
350 ops.oobbuf = d->oob_buf;
351 ops.ooboffs = 0;
352 ops.datbuf = NULL;
353 ops.mode = MTD_OOB_AUTO;
354
355 ret = mtdswap_read_oob(d, offset, &ops);
356
357 if (ret && ret != -EUCLEAN)
358 return ret;
359
360 data = (struct mtdswap_oobdata *)d->oob_buf;
361 data2 = (struct mtdswap_oobdata *)
362 (d->oob_buf + d->mtd->ecclayout->oobavail);
363
364 if (le16_to_cpu(data->magic) == MTDSWAP_MAGIC_CLEAN) {
365 eb->erase_count = le32_to_cpu(data->count);
366 if (ret == -EUCLEAN)
367 ret = MTDSWAP_SCANNED_BITFLIP;
368 else {
369 if (le16_to_cpu(data2->magic) == MTDSWAP_MAGIC_DIRTY)
370 ret = MTDSWAP_SCANNED_DIRTY;
371 else
372 ret = MTDSWAP_SCANNED_CLEAN;
373 }
374 } else {
375 eb->flags |= EBLOCK_NOMAGIC;
376 ret = MTDSWAP_SCANNED_DIRTY;
377 }
378
379 return ret;
380}
381
382static int mtdswap_write_marker(struct mtdswap_dev *d, struct swap_eb *eb,
383 u16 marker)
384{
385 struct mtdswap_oobdata n;
386 int ret;
387 loff_t offset;
388 struct mtd_oob_ops ops;
389
390 ops.ooboffs = 0;
391 ops.oobbuf = (uint8_t *)&n;
392 ops.mode = MTD_OOB_AUTO;
393 ops.datbuf = NULL;
394
395 if (marker == MTDSWAP_TYPE_CLEAN) {
396 n.magic = cpu_to_le16(MTDSWAP_MAGIC_CLEAN);
397 n.count = cpu_to_le32(eb->erase_count);
398 ops.ooblen = MTDSWAP_OOBSIZE;
399 offset = mtdswap_eb_offset(d, eb);
400 } else {
401 n.magic = cpu_to_le16(MTDSWAP_MAGIC_DIRTY);
402 ops.ooblen = sizeof(n.magic);
403 offset = mtdswap_eb_offset(d, eb) + d->mtd->writesize;
404 }
405
406 ret = d->mtd->write_oob(d->mtd, offset , &ops);
407
408 if (ret) {
409 dev_warn(d->dev, "Write OOB failed for block at %08llx "
410 "error %d\n", offset, ret);
411 if (ret == -EIO || ret == -EBADMSG)
412 mtdswap_handle_write_error(d, eb);
413 return ret;
414 }
415
416 if (ops.oobretlen != ops.ooblen) {
417 dev_warn(d->dev, "Short OOB write for block at %08llx: "
418 "%zd not %zd\n",
419 offset, ops.oobretlen, ops.ooblen);
420 return ret;
421 }
422
423 return 0;
424}
425
426/*
427 * Are there any erase blocks without MAGIC_CLEAN header, presumably
428 * because power was cut off after erase but before header write? We
429 * need to guestimate the erase count.
430 */
431static void mtdswap_check_counts(struct mtdswap_dev *d)
432{
433 struct rb_root hist_root = RB_ROOT;
434 struct rb_node *medrb;
435 struct swap_eb *eb;
436 unsigned int i, cnt, median;
437
438 cnt = 0;
439 for (i = 0; i < d->eblks; i++) {
440 eb = d->eb_data + i;
441
442 if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
443 continue;
444
445 __mtdswap_rb_add(&hist_root, eb);
446 cnt++;
447 }
448
449 if (cnt == 0)
450 return;
451
452 medrb = mtdswap_rb_index(&hist_root, cnt / 2);
453 median = rb_entry(medrb, struct swap_eb, rb)->erase_count;
454
455 d->max_erase_count = MTDSWAP_ECNT_MAX(&hist_root);
456
457 for (i = 0; i < d->eblks; i++) {
458 eb = d->eb_data + i;
459
460 if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_READERR))
461 eb->erase_count = median;
462
463 if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
464 continue;
465
466 rb_erase(&eb->rb, &hist_root);
467 }
468}
469
470static void mtdswap_scan_eblks(struct mtdswap_dev *d)
471{
472 int status;
473 unsigned int i, idx;
474 struct swap_eb *eb;
475
476 for (i = 0; i < d->eblks; i++) {
477 eb = d->eb_data + i;
478
479 status = mtdswap_read_markers(d, eb);
480 if (status < 0)
481 eb->flags |= EBLOCK_READERR;
482 else if (status == MTDSWAP_SCANNED_BAD) {
483 eb->flags |= EBLOCK_BAD;
484 continue;
485 }
486
487 switch (status) {
488 case MTDSWAP_SCANNED_CLEAN:
489 idx = MTDSWAP_CLEAN;
490 break;
491 case MTDSWAP_SCANNED_DIRTY:
492 case MTDSWAP_SCANNED_BITFLIP:
493 idx = MTDSWAP_DIRTY;
494 break;
495 default:
496 idx = MTDSWAP_FAILING;
497 }
498
499 eb->flags |= (idx << EBLOCK_IDX_SHIFT);
500 }
501
502 mtdswap_check_counts(d);
503
504 for (i = 0; i < d->eblks; i++) {
505 eb = d->eb_data + i;
506
507 if (eb->flags & EBLOCK_BAD)
508 continue;
509
510 idx = eb->flags >> EBLOCK_IDX_SHIFT;
511 mtdswap_rb_add(d, eb, idx);
512 }
513}
514
515/*
516 * Place eblk into a tree corresponding to its number of active blocks
517 * it contains.
518 */
519static void mtdswap_store_eb(struct mtdswap_dev *d, struct swap_eb *eb)
520{
521 unsigned int weight = eb->active_count;
522 unsigned int maxweight = d->pages_per_eblk;
523
524 if (eb == d->curr_write)
525 return;
526
527 if (eb->flags & EBLOCK_BITFLIP)
528 mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
529 else if (eb->flags & (EBLOCK_READERR | EBLOCK_FAILED))
530 mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
531 if (weight == maxweight)
532 mtdswap_rb_add(d, eb, MTDSWAP_USED);
533 else if (weight == 0)
534 mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
535 else if (weight > (maxweight/2))
536 mtdswap_rb_add(d, eb, MTDSWAP_LOWFRAG);
537 else
538 mtdswap_rb_add(d, eb, MTDSWAP_HIFRAG);
539}
540
541
542static void mtdswap_erase_callback(struct erase_info *done)
543{
544 wait_queue_head_t *wait_q = (wait_queue_head_t *)done->priv;
545 wake_up(wait_q);
546}
547
548static int mtdswap_erase_block(struct mtdswap_dev *d, struct swap_eb *eb)
549{
550 struct mtd_info *mtd = d->mtd;
551 struct erase_info erase;
552 wait_queue_head_t wq;
553 unsigned int retries = 0;
554 int ret;
555
556 eb->erase_count++;
557 if (eb->erase_count > d->max_erase_count)
558 d->max_erase_count = eb->erase_count;
559
560retry:
561 init_waitqueue_head(&wq);
562 memset(&erase, 0, sizeof(struct erase_info));
563
564 erase.mtd = mtd;
565 erase.callback = mtdswap_erase_callback;
566 erase.addr = mtdswap_eb_offset(d, eb);
567 erase.len = mtd->erasesize;
568 erase.priv = (u_long)&wq;
569
570 ret = mtd->erase(mtd, &erase);
571 if (ret) {
572 if (retries++ < MTDSWAP_ERASE_RETRIES) {
573 dev_warn(d->dev,
574 "erase of erase block %#llx on %s failed",
575 erase.addr, mtd->name);
576 yield();
577 goto retry;
578 }
579
580 dev_err(d->dev, "Cannot erase erase block %#llx on %s\n",
581 erase.addr, mtd->name);
582
583 mtdswap_handle_badblock(d, eb);
584 return -EIO;
585 }
586
587 ret = wait_event_interruptible(wq, erase.state == MTD_ERASE_DONE ||
588 erase.state == MTD_ERASE_FAILED);
589 if (ret) {
590 dev_err(d->dev, "Interrupted erase block %#llx erassure on %s",
591 erase.addr, mtd->name);
592 return -EINTR;
593 }
594
595 if (erase.state == MTD_ERASE_FAILED) {
596 if (retries++ < MTDSWAP_ERASE_RETRIES) {
597 dev_warn(d->dev,
598 "erase of erase block %#llx on %s failed",
599 erase.addr, mtd->name);
600 yield();
601 goto retry;
602 }
603
604 mtdswap_handle_badblock(d, eb);
605 return -EIO;
606 }
607
608 return 0;
609}
610
611static int mtdswap_map_free_block(struct mtdswap_dev *d, unsigned int page,
612 unsigned int *block)
613{
614 int ret;
615 struct swap_eb *old_eb = d->curr_write;
616 struct rb_root *clean_root;
617 struct swap_eb *eb;
618
619 if (old_eb == NULL || d->curr_write_pos >= d->pages_per_eblk) {
620 do {
621 if (TREE_EMPTY(d, CLEAN))
622 return -ENOSPC;
623
624 clean_root = TREE_ROOT(d, CLEAN);
625 eb = rb_entry(rb_first(clean_root), struct swap_eb, rb);
626 rb_erase(&eb->rb, clean_root);
627 eb->root = NULL;
628 TREE_COUNT(d, CLEAN)--;
629
630 ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_DIRTY);
631 } while (ret == -EIO || ret == -EBADMSG);
632
633 if (ret)
634 return ret;
635
636 d->curr_write_pos = 0;
637 d->curr_write = eb;
638 if (old_eb)
639 mtdswap_store_eb(d, old_eb);
640 }
641
642 *block = (d->curr_write - d->eb_data) * d->pages_per_eblk +
643 d->curr_write_pos;
644
645 d->curr_write->active_count++;
646 d->revmap[*block] = page;
647 d->curr_write_pos++;
648
649 return 0;
650}
651
652static unsigned int mtdswap_free_page_cnt(struct mtdswap_dev *d)
653{
654 return TREE_COUNT(d, CLEAN) * d->pages_per_eblk +
655 d->pages_per_eblk - d->curr_write_pos;
656}
657
658static unsigned int mtdswap_enough_free_pages(struct mtdswap_dev *d)
659{
660 return mtdswap_free_page_cnt(d) > d->pages_per_eblk;
661}
662
663static int mtdswap_write_block(struct mtdswap_dev *d, char *buf,
664 unsigned int page, unsigned int *bp, int gc_context)
665{
666 struct mtd_info *mtd = d->mtd;
667 struct swap_eb *eb;
668 size_t retlen;
669 loff_t writepos;
670 int ret;
671
672retry:
673 if (!gc_context)
674 while (!mtdswap_enough_free_pages(d))
675 if (mtdswap_gc(d, 0) > 0)
676 return -ENOSPC;
677
678 ret = mtdswap_map_free_block(d, page, bp);
679 eb = d->eb_data + (*bp / d->pages_per_eblk);
680
681 if (ret == -EIO || ret == -EBADMSG) {
682 d->curr_write = NULL;
683 eb->active_count--;
684 d->revmap[*bp] = PAGE_UNDEF;
685 goto retry;
686 }
687
688 if (ret < 0)
689 return ret;
690
691 writepos = (loff_t)*bp << PAGE_SHIFT;
692 ret = mtd->write(mtd, writepos, PAGE_SIZE, &retlen, buf);
693 if (ret == -EIO || ret == -EBADMSG) {
694 d->curr_write_pos--;
695 eb->active_count--;
696 d->revmap[*bp] = PAGE_UNDEF;
697 mtdswap_handle_write_error(d, eb);
698 goto retry;
699 }
700
701 if (ret < 0) {
702 dev_err(d->dev, "Write to MTD device failed: %d (%zd written)",
703 ret, retlen);
704 goto err;
705 }
706
707 if (retlen != PAGE_SIZE) {
708 dev_err(d->dev, "Short write to MTD device: %zd written",
709 retlen);
710 ret = -EIO;
711 goto err;
712 }
713
714 return ret;
715
716err:
717 d->curr_write_pos--;
718 eb->active_count--;
719 d->revmap[*bp] = PAGE_UNDEF;
720
721 return ret;
722}
723
724static int mtdswap_move_block(struct mtdswap_dev *d, unsigned int oldblock,
725 unsigned int *newblock)
726{
727 struct mtd_info *mtd = d->mtd;
728 struct swap_eb *eb, *oldeb;
729 int ret;
730 size_t retlen;
731 unsigned int page, retries;
732 loff_t readpos;
733
734 page = d->revmap[oldblock];
735 readpos = (loff_t) oldblock << PAGE_SHIFT;
736 retries = 0;
737
738retry:
739 ret = mtd->read(mtd, readpos, PAGE_SIZE, &retlen, d->page_buf);
740
741 if (ret < 0 && ret != -EUCLEAN) {
742 oldeb = d->eb_data + oldblock / d->pages_per_eblk;
743 oldeb->flags |= EBLOCK_READERR;
744
745 dev_err(d->dev, "Read Error: %d (block %u)\n", ret,
746 oldblock);
747 retries++;
748 if (retries < MTDSWAP_IO_RETRIES)
749 goto retry;
750
751 goto read_error;
752 }
753
754 if (retlen != PAGE_SIZE) {
755 dev_err(d->dev, "Short read: %zd (block %u)\n", retlen,
756 oldblock);
757 ret = -EIO;
758 goto read_error;
759 }
760
761 ret = mtdswap_write_block(d, d->page_buf, page, newblock, 1);
762 if (ret < 0) {
763 d->page_data[page] = BLOCK_ERROR;
764 dev_err(d->dev, "Write error: %d\n", ret);
765 return ret;
766 }
767
768 eb = d->eb_data + *newblock / d->pages_per_eblk;
769 d->page_data[page] = *newblock;
770 d->revmap[oldblock] = PAGE_UNDEF;
771 eb = d->eb_data + oldblock / d->pages_per_eblk;
772 eb->active_count--;
773
774 return 0;
775
776read_error:
777 d->page_data[page] = BLOCK_ERROR;
778 d->revmap[oldblock] = PAGE_UNDEF;
779 return ret;
780}
781
782static int mtdswap_gc_eblock(struct mtdswap_dev *d, struct swap_eb *eb)
783{
784 unsigned int i, block, eblk_base, newblock;
785 int ret, errcode;
786
787 errcode = 0;
788 eblk_base = (eb - d->eb_data) * d->pages_per_eblk;
789
790 for (i = 0; i < d->pages_per_eblk; i++) {
791 if (d->spare_eblks < MIN_SPARE_EBLOCKS)
792 return -ENOSPC;
793
794 block = eblk_base + i;
795 if (d->revmap[block] == PAGE_UNDEF)
796 continue;
797
798 ret = mtdswap_move_block(d, block, &newblock);
799 if (ret < 0 && !errcode)
800 errcode = ret;
801 }
802
803 return errcode;
804}
805
806static int __mtdswap_choose_gc_tree(struct mtdswap_dev *d)
807{
808 int idx, stopat;
809
810 if (TREE_COUNT(d, CLEAN) < LOW_FRAG_GC_TRESHOLD)
811 stopat = MTDSWAP_LOWFRAG;
812 else
813 stopat = MTDSWAP_HIFRAG;
814
815 for (idx = MTDSWAP_BITFLIP; idx >= stopat; idx--)
816 if (d->trees[idx].root.rb_node != NULL)
817 return idx;
818
819 return -1;
820}
821
822static int mtdswap_wlfreq(unsigned int maxdiff)
823{
824 unsigned int h, x, y, dist, base;
825
826 /*
827 * Calculate linear ramp down from f1 to f2 when maxdiff goes from
828 * MAX_ERASE_DIFF to MAX_ERASE_DIFF + COLLECT_NONDIRTY_BASE. Similar
829 * to triangle with height f1 - f1 and width COLLECT_NONDIRTY_BASE.
830 */
831
832 dist = maxdiff - MAX_ERASE_DIFF;
833 if (dist > COLLECT_NONDIRTY_BASE)
834 dist = COLLECT_NONDIRTY_BASE;
835
836 /*
837 * Modelling the slop as right angular triangle with base
838 * COLLECT_NONDIRTY_BASE and height freq1 - freq2. The ratio y/x is
839 * equal to the ratio h/base.
840 */
841 h = COLLECT_NONDIRTY_FREQ1 - COLLECT_NONDIRTY_FREQ2;
842 base = COLLECT_NONDIRTY_BASE;
843
844 x = dist - base;
845 y = (x * h + base / 2) / base;
846
847 return COLLECT_NONDIRTY_FREQ2 + y;
848}
849
850static int mtdswap_choose_wl_tree(struct mtdswap_dev *d)
851{
852 static unsigned int pick_cnt;
853 unsigned int i, idx = -1, wear, max;
854 struct rb_root *root;
855
856 max = 0;
857 for (i = 0; i <= MTDSWAP_DIRTY; i++) {
858 root = &d->trees[i].root;
859 if (root->rb_node == NULL)
860 continue;
861
862 wear = d->max_erase_count - MTDSWAP_ECNT_MIN(root);
863 if (wear > max) {
864 max = wear;
865 idx = i;
866 }
867 }
868
869 if (max > MAX_ERASE_DIFF && pick_cnt >= mtdswap_wlfreq(max) - 1) {
870 pick_cnt = 0;
871 return idx;
872 }
873
874 pick_cnt++;
875 return -1;
876}
877
878static int mtdswap_choose_gc_tree(struct mtdswap_dev *d,
879 unsigned int background)
880{
881 int idx;
882
883 if (TREE_NONEMPTY(d, FAILING) &&
884 (background || (TREE_EMPTY(d, CLEAN) && TREE_EMPTY(d, DIRTY))))
885 return MTDSWAP_FAILING;
886
887 idx = mtdswap_choose_wl_tree(d);
888 if (idx >= MTDSWAP_CLEAN)
889 return idx;
890
891 return __mtdswap_choose_gc_tree(d);
892}
893
894static struct swap_eb *mtdswap_pick_gc_eblk(struct mtdswap_dev *d,
895 unsigned int background)
896{
897 struct rb_root *rp = NULL;
898 struct swap_eb *eb = NULL;
899 int idx;
900
901 if (background && TREE_COUNT(d, CLEAN) > CLEAN_BLOCK_THRESHOLD &&
902 TREE_EMPTY(d, DIRTY) && TREE_EMPTY(d, FAILING))
903 return NULL;
904
905 idx = mtdswap_choose_gc_tree(d, background);
906 if (idx < 0)
907 return NULL;
908
909 rp = &d->trees[idx].root;
910 eb = rb_entry(rb_first(rp), struct swap_eb, rb);
911
912 rb_erase(&eb->rb, rp);
913 eb->root = NULL;
914 d->trees[idx].count--;
915 return eb;
916}
917
918static unsigned int mtdswap_test_patt(unsigned int i)
919{
920 return i % 2 ? 0x55555555 : 0xAAAAAAAA;
921}
922
923static unsigned int mtdswap_eblk_passes(struct mtdswap_dev *d,
924 struct swap_eb *eb)
925{
926 struct mtd_info *mtd = d->mtd;
927 unsigned int test, i, j, patt, mtd_pages;
928 loff_t base, pos;
929 unsigned int *p1 = (unsigned int *)d->page_buf;
930 unsigned char *p2 = (unsigned char *)d->oob_buf;
931 struct mtd_oob_ops ops;
932 int ret;
933
934 ops.mode = MTD_OOB_AUTO;
935 ops.len = mtd->writesize;
936 ops.ooblen = mtd->ecclayout->oobavail;
937 ops.ooboffs = 0;
938 ops.datbuf = d->page_buf;
939 ops.oobbuf = d->oob_buf;
940 base = mtdswap_eb_offset(d, eb);
941 mtd_pages = d->pages_per_eblk * PAGE_SIZE / mtd->writesize;
942
943 for (test = 0; test < 2; test++) {
944 pos = base;
945 for (i = 0; i < mtd_pages; i++) {
946 patt = mtdswap_test_patt(test + i);
947 memset(d->page_buf, patt, mtd->writesize);
948 memset(d->oob_buf, patt, mtd->ecclayout->oobavail);
949 ret = mtd->write_oob(mtd, pos, &ops);
950 if (ret)
951 goto error;
952
953 pos += mtd->writesize;
954 }
955
956 pos = base;
957 for (i = 0; i < mtd_pages; i++) {
958 ret = mtd->read_oob(mtd, pos, &ops);
959 if (ret)
960 goto error;
961
962 patt = mtdswap_test_patt(test + i);
963 for (j = 0; j < mtd->writesize/sizeof(int); j++)
964 if (p1[j] != patt)
965 goto error;
966
967 for (j = 0; j < mtd->ecclayout->oobavail; j++)
968 if (p2[j] != (unsigned char)patt)
969 goto error;
970
971 pos += mtd->writesize;
972 }
973
974 ret = mtdswap_erase_block(d, eb);
975 if (ret)
976 goto error;
977 }
978
979 eb->flags &= ~EBLOCK_READERR;
980 return 1;
981
982error:
983 mtdswap_handle_badblock(d, eb);
984 return 0;
985}
986
987static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background)
988{
989 struct swap_eb *eb;
990 int ret;
991
992 if (d->spare_eblks < MIN_SPARE_EBLOCKS)
993 return 1;
994
995 eb = mtdswap_pick_gc_eblk(d, background);
996 if (!eb)
997 return 1;
998
999 ret = mtdswap_gc_eblock(d, eb);
1000 if (ret == -ENOSPC)
1001 return 1;
1002
1003 if (eb->flags & EBLOCK_FAILED) {
1004 mtdswap_handle_badblock(d, eb);
1005 return 0;
1006 }
1007
1008 eb->flags &= ~EBLOCK_BITFLIP;
1009 ret = mtdswap_erase_block(d, eb);
1010 if ((eb->flags & EBLOCK_READERR) &&
1011 (ret || !mtdswap_eblk_passes(d, eb)))
1012 return 0;
1013
1014 if (ret == 0)
1015 ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_CLEAN);
1016
1017 if (ret == 0)
1018 mtdswap_rb_add(d, eb, MTDSWAP_CLEAN);
1019 else if (ret != -EIO && ret != -EBADMSG)
1020 mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
1021
1022 return 0;
1023}
1024
1025static void mtdswap_background(struct mtd_blktrans_dev *dev)
1026{
1027 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1028 int ret;
1029
1030 while (1) {
1031 ret = mtdswap_gc(d, 1);
1032 if (ret || mtd_blktrans_cease_background(dev))
1033 return;
1034 }
1035}
1036
1037static void mtdswap_cleanup(struct mtdswap_dev *d)
1038{
1039 vfree(d->eb_data);
1040 vfree(d->revmap);
1041 vfree(d->page_data);
1042 kfree(d->oob_buf);
1043 kfree(d->page_buf);
1044}
1045
1046static int mtdswap_flush(struct mtd_blktrans_dev *dev)
1047{
1048 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1049
1050 if (d->mtd->sync)
1051 d->mtd->sync(d->mtd);
1052 return 0;
1053}
1054
1055static unsigned int mtdswap_badblocks(struct mtd_info *mtd, uint64_t size)
1056{
1057 loff_t offset;
1058 unsigned int badcnt;
1059
1060 badcnt = 0;
1061
1062 if (mtd->block_isbad)
1063 for (offset = 0; offset < size; offset += mtd->erasesize)
1064 if (mtd->block_isbad(mtd, offset))
1065 badcnt++;
1066
1067 return badcnt;
1068}
1069
1070static int mtdswap_writesect(struct mtd_blktrans_dev *dev,
1071 unsigned long page, char *buf)
1072{
1073 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1074 unsigned int newblock, mapped;
1075 struct swap_eb *eb;
1076 int ret;
1077
1078 d->sect_write_count++;
1079
1080 if (d->spare_eblks < MIN_SPARE_EBLOCKS)
1081 return -ENOSPC;
1082
1083 if (header) {
1084 /* Ignore writes to the header page */
1085 if (unlikely(page == 0))
1086 return 0;
1087
1088 page--;
1089 }
1090
1091 mapped = d->page_data[page];
1092 if (mapped <= BLOCK_MAX) {
1093 eb = d->eb_data + (mapped / d->pages_per_eblk);
1094 eb->active_count--;
1095 mtdswap_store_eb(d, eb);
1096 d->page_data[page] = BLOCK_UNDEF;
1097 d->revmap[mapped] = PAGE_UNDEF;
1098 }
1099
1100 ret = mtdswap_write_block(d, buf, page, &newblock, 0);
1101 d->mtd_write_count++;
1102
1103 if (ret < 0)
1104 return ret;
1105
1106 eb = d->eb_data + (newblock / d->pages_per_eblk);
1107 d->page_data[page] = newblock;
1108
1109 return 0;
1110}
1111
1112/* Provide a dummy swap header for the kernel */
1113static int mtdswap_auto_header(struct mtdswap_dev *d, char *buf)
1114{
1115 union swap_header *hd = (union swap_header *)(buf);
1116
1117 memset(buf, 0, PAGE_SIZE - 10);
1118
1119 hd->info.version = 1;
1120 hd->info.last_page = d->mbd_dev->size - 1;
1121 hd->info.nr_badpages = 0;
1122
1123 memcpy(buf + PAGE_SIZE - 10, "SWAPSPACE2", 10);
1124
1125 return 0;
1126}
1127
1128static int mtdswap_readsect(struct mtd_blktrans_dev *dev,
1129 unsigned long page, char *buf)
1130{
1131 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1132 struct mtd_info *mtd = d->mtd;
1133 unsigned int realblock, retries;
1134 loff_t readpos;
1135 struct swap_eb *eb;
1136 size_t retlen;
1137 int ret;
1138
1139 d->sect_read_count++;
1140
1141 if (header) {
1142 if (unlikely(page == 0))
1143 return mtdswap_auto_header(d, buf);
1144
1145 page--;
1146 }
1147
1148 realblock = d->page_data[page];
1149 if (realblock > BLOCK_MAX) {
1150 memset(buf, 0x0, PAGE_SIZE);
1151 if (realblock == BLOCK_UNDEF)
1152 return 0;
1153 else
1154 return -EIO;
1155 }
1156
1157 eb = d->eb_data + (realblock / d->pages_per_eblk);
1158 BUG_ON(d->revmap[realblock] == PAGE_UNDEF);
1159
1160 readpos = (loff_t)realblock << PAGE_SHIFT;
1161 retries = 0;
1162
1163retry:
1164 ret = mtd->read(mtd, readpos, PAGE_SIZE, &retlen, buf);
1165
1166 d->mtd_read_count++;
1167 if (ret == -EUCLEAN) {
1168 eb->flags |= EBLOCK_BITFLIP;
1169 mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
1170 ret = 0;
1171 }
1172
1173 if (ret < 0) {
1174 dev_err(d->dev, "Read error %d\n", ret);
1175 eb->flags |= EBLOCK_READERR;
1176 mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
1177 retries++;
1178 if (retries < MTDSWAP_IO_RETRIES)
1179 goto retry;
1180
1181 return ret;
1182 }
1183
1184 if (retlen != PAGE_SIZE) {
1185 dev_err(d->dev, "Short read %zd\n", retlen);
1186 return -EIO;
1187 }
1188
1189 return 0;
1190}
1191
1192static int mtdswap_discard(struct mtd_blktrans_dev *dev, unsigned long first,
1193 unsigned nr_pages)
1194{
1195 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1196 unsigned long page;
1197 struct swap_eb *eb;
1198 unsigned int mapped;
1199
1200 d->discard_count++;
1201
1202 for (page = first; page < first + nr_pages; page++) {
1203 mapped = d->page_data[page];
1204 if (mapped <= BLOCK_MAX) {
1205 eb = d->eb_data + (mapped / d->pages_per_eblk);
1206 eb->active_count--;
1207 mtdswap_store_eb(d, eb);
1208 d->page_data[page] = BLOCK_UNDEF;
1209 d->revmap[mapped] = PAGE_UNDEF;
1210 d->discard_page_count++;
1211 } else if (mapped == BLOCK_ERROR) {
1212 d->page_data[page] = BLOCK_UNDEF;
1213 d->discard_page_count++;
1214 }
1215 }
1216
1217 return 0;
1218}
1219
1220static int mtdswap_show(struct seq_file *s, void *data)
1221{
1222 struct mtdswap_dev *d = (struct mtdswap_dev *) s->private;
1223 unsigned long sum;
1224 unsigned int count[MTDSWAP_TREE_CNT];
1225 unsigned int min[MTDSWAP_TREE_CNT];
1226 unsigned int max[MTDSWAP_TREE_CNT];
1227 unsigned int i, cw = 0, cwp = 0, cwecount = 0, bb_cnt, mapped, pages;
1228 uint64_t use_size;
1229 char *name[] = {"clean", "used", "low", "high", "dirty", "bitflip",
1230 "failing"};
1231
1232 mutex_lock(&d->mbd_dev->lock);
1233
1234 for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
1235 struct rb_root *root = &d->trees[i].root;
1236
1237 if (root->rb_node) {
1238 count[i] = d->trees[i].count;
1239 min[i] = rb_entry(rb_first(root), struct swap_eb,
1240 rb)->erase_count;
1241 max[i] = rb_entry(rb_last(root), struct swap_eb,
1242 rb)->erase_count;
1243 } else
1244 count[i] = 0;
1245 }
1246
1247 if (d->curr_write) {
1248 cw = 1;
1249 cwp = d->curr_write_pos;
1250 cwecount = d->curr_write->erase_count;
1251 }
1252
1253 sum = 0;
1254 for (i = 0; i < d->eblks; i++)
1255 sum += d->eb_data[i].erase_count;
1256
1257 use_size = (uint64_t)d->eblks * d->mtd->erasesize;
1258 bb_cnt = mtdswap_badblocks(d->mtd, use_size);
1259
1260 mapped = 0;
1261 pages = d->mbd_dev->size;
1262 for (i = 0; i < pages; i++)
1263 if (d->page_data[i] != BLOCK_UNDEF)
1264 mapped++;
1265
1266 mutex_unlock(&d->mbd_dev->lock);
1267
1268 for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
1269 if (!count[i])
1270 continue;
1271
1272 if (min[i] != max[i])
1273 seq_printf(s, "%s:\t%5d erase blocks, erased min %d, "
1274 "max %d times\n",
1275 name[i], count[i], min[i], max[i]);
1276 else
1277 seq_printf(s, "%s:\t%5d erase blocks, all erased %d "
1278 "times\n", name[i], count[i], min[i]);
1279 }
1280
1281 if (bb_cnt)
1282 seq_printf(s, "bad:\t%5u erase blocks\n", bb_cnt);
1283
1284 if (cw)
1285 seq_printf(s, "current erase block: %u pages used, %u free, "
1286 "erased %u times\n",
1287 cwp, d->pages_per_eblk - cwp, cwecount);
1288
1289 seq_printf(s, "total erasures: %lu\n", sum);
1290
1291 seq_printf(s, "\n");
1292
1293 seq_printf(s, "mtdswap_readsect count: %llu\n", d->sect_read_count);
1294 seq_printf(s, "mtdswap_writesect count: %llu\n", d->sect_write_count);
1295 seq_printf(s, "mtdswap_discard count: %llu\n", d->discard_count);
1296 seq_printf(s, "mtd read count: %llu\n", d->mtd_read_count);
1297 seq_printf(s, "mtd write count: %llu\n", d->mtd_write_count);
1298 seq_printf(s, "discarded pages count: %llu\n", d->discard_page_count);
1299
1300 seq_printf(s, "\n");
1301 seq_printf(s, "total pages: %u\n", pages);
1302 seq_printf(s, "pages mapped: %u\n", mapped);
1303
1304 return 0;
1305}
1306
1307static int mtdswap_open(struct inode *inode, struct file *file)
1308{
1309 return single_open(file, mtdswap_show, inode->i_private);
1310}
1311
1312static const struct file_operations mtdswap_fops = {
1313 .open = mtdswap_open,
1314 .read = seq_read,
1315 .llseek = seq_lseek,
1316 .release = single_release,
1317};
1318
1319static int mtdswap_add_debugfs(struct mtdswap_dev *d)
1320{
1321 struct gendisk *gd = d->mbd_dev->disk;
1322 struct device *dev = disk_to_dev(gd);
1323
1324 struct dentry *root;
1325 struct dentry *dent;
1326
1327 root = debugfs_create_dir(gd->disk_name, NULL);
1328 if (IS_ERR(root))
1329 return 0;
1330
1331 if (!root) {
1332 dev_err(dev, "failed to initialize debugfs\n");
1333 return -1;
1334 }
1335
1336 d->debugfs_root = root;
1337
1338 dent = debugfs_create_file("stats", S_IRUSR, root, d,
1339 &mtdswap_fops);
1340 if (!dent) {
1341 dev_err(d->dev, "debugfs_create_file failed\n");
1342 debugfs_remove_recursive(root);
1343 d->debugfs_root = NULL;
1344 return -1;
1345 }
1346
1347 return 0;
1348}
1349
1350static int mtdswap_init(struct mtdswap_dev *d, unsigned int eblocks,
1351 unsigned int spare_cnt)
1352{
1353 struct mtd_info *mtd = d->mbd_dev->mtd;
1354 unsigned int i, eblk_bytes, pages, blocks;
1355 int ret = -ENOMEM;
1356
1357 d->mtd = mtd;
1358 d->eblks = eblocks;
1359 d->spare_eblks = spare_cnt;
1360 d->pages_per_eblk = mtd->erasesize >> PAGE_SHIFT;
1361
1362 pages = d->mbd_dev->size;
1363 blocks = eblocks * d->pages_per_eblk;
1364
1365 for (i = 0; i < MTDSWAP_TREE_CNT; i++)
1366 d->trees[i].root = RB_ROOT;
1367
1368 d->page_data = vmalloc(sizeof(int)*pages);
1369 if (!d->page_data)
1370 goto page_data_fail;
1371
1372 d->revmap = vmalloc(sizeof(int)*blocks);
1373 if (!d->revmap)
1374 goto revmap_fail;
1375
1376 eblk_bytes = sizeof(struct swap_eb)*d->eblks;
1377 d->eb_data = vmalloc(eblk_bytes);
1378 if (!d->eb_data)
1379 goto eb_data_fail;
1380
1381 memset(d->eb_data, 0, eblk_bytes);
1382 for (i = 0; i < pages; i++)
1383 d->page_data[i] = BLOCK_UNDEF;
1384
1385 for (i = 0; i < blocks; i++)
1386 d->revmap[i] = PAGE_UNDEF;
1387
1388 d->page_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1389 if (!d->page_buf)
1390 goto page_buf_fail;
1391
1392 d->oob_buf = kmalloc(2 * mtd->ecclayout->oobavail, GFP_KERNEL);
1393 if (!d->oob_buf)
1394 goto oob_buf_fail;
1395
1396 mtdswap_scan_eblks(d);
1397
1398 return 0;
1399
1400oob_buf_fail:
1401 kfree(d->page_buf);
1402page_buf_fail:
1403 vfree(d->eb_data);
1404eb_data_fail:
1405 vfree(d->revmap);
1406revmap_fail:
1407 vfree(d->page_data);
1408page_data_fail:
1409 printk(KERN_ERR "%s: init failed (%d)\n", MTDSWAP_PREFIX, ret);
1410 return ret;
1411}
1412
1413static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
1414{
1415 struct mtdswap_dev *d;
1416 struct mtd_blktrans_dev *mbd_dev;
1417 char *parts;
1418 char *this_opt;
1419 unsigned long part;
1420 unsigned int eblocks, eavailable, bad_blocks, spare_cnt;
1421 uint64_t swap_size, use_size, size_limit;
1422 struct nand_ecclayout *oinfo;
1423 int ret;
1424
1425 parts = &partitions[0];
1426 if (!*parts)
1427 return;
1428
1429 while ((this_opt = strsep(&parts, ",")) != NULL) {
1430 if (strict_strtoul(this_opt, 0, &part) < 0)
1431 return;
1432
1433 if (mtd->index == part)
1434 break;
1435 }
1436
1437 if (mtd->index != part)
1438 return;
1439
1440 if (mtd->erasesize < PAGE_SIZE || mtd->erasesize % PAGE_SIZE) {
1441 printk(KERN_ERR "%s: Erase size %u not multiple of PAGE_SIZE "
1442 "%lu\n", MTDSWAP_PREFIX, mtd->erasesize, PAGE_SIZE);
1443 return;
1444 }
1445
1446 if (PAGE_SIZE % mtd->writesize || mtd->writesize > PAGE_SIZE) {
1447 printk(KERN_ERR "%s: PAGE_SIZE %lu not multiple of write size"
1448 " %u\n", MTDSWAP_PREFIX, PAGE_SIZE, mtd->writesize);
1449 return;
1450 }
1451
1452 oinfo = mtd->ecclayout;
1453 if (!oinfo) {
1454 printk(KERN_ERR "%s: mtd%d does not have OOB\n",
1455 MTDSWAP_PREFIX, mtd->index);
1456 return;
1457 }
1458
1459 if (!mtd->oobsize || oinfo->oobavail < MTDSWAP_OOBSIZE) {
1460 printk(KERN_ERR "%s: Not enough free bytes in OOB, "
1461 "%d available, %zu needed.\n",
1462 MTDSWAP_PREFIX, oinfo->oobavail, MTDSWAP_OOBSIZE);
1463 return;
1464 }
1465
1466 if (spare_eblocks > 100)
1467 spare_eblocks = 100;
1468
1469 use_size = mtd->size;
1470 size_limit = (uint64_t) BLOCK_MAX * PAGE_SIZE;
1471
1472 if (mtd->size > size_limit) {
1473 printk(KERN_WARNING "%s: Device too large. Limiting size to "
1474 "%llu bytes\n", MTDSWAP_PREFIX, size_limit);
1475 use_size = size_limit;
1476 }
1477
1478 eblocks = mtd_div_by_eb(use_size, mtd);
1479 use_size = eblocks * mtd->erasesize;
1480 bad_blocks = mtdswap_badblocks(mtd, use_size);
1481 eavailable = eblocks - bad_blocks;
1482
1483 if (eavailable < MIN_ERASE_BLOCKS) {
1484 printk(KERN_ERR "%s: Not enough erase blocks. %u available, "
1485 "%d needed\n", MTDSWAP_PREFIX, eavailable,
1486 MIN_ERASE_BLOCKS);
1487 return;
1488 }
1489
1490 spare_cnt = div_u64((uint64_t)eavailable * spare_eblocks, 100);
1491
1492 if (spare_cnt < MIN_SPARE_EBLOCKS)
1493 spare_cnt = MIN_SPARE_EBLOCKS;
1494
1495 if (spare_cnt > eavailable - 1)
1496 spare_cnt = eavailable - 1;
1497
1498 swap_size = (uint64_t)(eavailable - spare_cnt) * mtd->erasesize +
1499 (header ? PAGE_SIZE : 0);
1500
1501 printk(KERN_INFO "%s: Enabling MTD swap on device %lu, size %llu KB, "
1502 "%u spare, %u bad blocks\n",
1503 MTDSWAP_PREFIX, part, swap_size / 1024, spare_cnt, bad_blocks);
1504
1505 d = kzalloc(sizeof(struct mtdswap_dev), GFP_KERNEL);
1506 if (!d)
1507 return;
1508
1509 mbd_dev = kzalloc(sizeof(struct mtd_blktrans_dev), GFP_KERNEL);
1510 if (!mbd_dev) {
1511 kfree(d);
1512 return;
1513 }
1514
1515 d->mbd_dev = mbd_dev;
1516 mbd_dev->priv = d;
1517
1518 mbd_dev->mtd = mtd;
1519 mbd_dev->devnum = mtd->index;
1520 mbd_dev->size = swap_size >> PAGE_SHIFT;
1521 mbd_dev->tr = tr;
1522
1523 if (!(mtd->flags & MTD_WRITEABLE))
1524 mbd_dev->readonly = 1;
1525
1526 if (mtdswap_init(d, eblocks, spare_cnt) < 0)
1527 goto init_failed;
1528
1529 if (add_mtd_blktrans_dev(mbd_dev) < 0)
1530 goto cleanup;
1531
1532 d->dev = disk_to_dev(mbd_dev->disk);
1533
1534 ret = mtdswap_add_debugfs(d);
1535 if (ret < 0)
1536 goto debugfs_failed;
1537
1538 return;
1539
1540debugfs_failed:
1541 del_mtd_blktrans_dev(mbd_dev);
1542
1543cleanup:
1544 mtdswap_cleanup(d);
1545
1546init_failed:
1547 kfree(mbd_dev);
1548 kfree(d);
1549}
1550
1551static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
1552{
1553 struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
1554
1555 debugfs_remove_recursive(d->debugfs_root);
1556 del_mtd_blktrans_dev(dev);
1557 mtdswap_cleanup(d);
1558 kfree(d);
1559}
1560
1561static struct mtd_blktrans_ops mtdswap_ops = {
1562 .name = "mtdswap",
1563 .major = 0,
1564 .part_bits = 0,
1565 .blksize = PAGE_SIZE,
1566 .flush = mtdswap_flush,
1567 .readsect = mtdswap_readsect,
1568 .writesect = mtdswap_writesect,
1569 .discard = mtdswap_discard,
1570 .background = mtdswap_background,
1571 .add_mtd = mtdswap_add_mtd,
1572 .remove_dev = mtdswap_remove_dev,
1573 .owner = THIS_MODULE,
1574};
1575
1576static int __init mtdswap_modinit(void)
1577{
1578 return register_mtd_blktrans(&mtdswap_ops);
1579}
1580
1581static void __exit mtdswap_modexit(void)
1582{
1583 deregister_mtd_blktrans(&mtdswap_ops);
1584}
1585
1586module_init(mtdswap_modinit);
1587module_exit(mtdswap_modexit);
1588
1589
1590MODULE_LICENSE("GPL");
1591MODULE_AUTHOR("Jarkko Lavinen <jarkko.lavinen@nokia.com>");
1592MODULE_DESCRIPTION("Block device access to an MTD suitable for using as "
1593 "swap space");