cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

mtdswap.c (33802B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Swap block device support for MTDs
      4 * Turns an MTD device into a swap device with block wear leveling
      5 *
      6 * Copyright © 2007,2011 Nokia Corporation. All rights reserved.
      7 *
      8 * Authors: Jarkko Lavinen <jarkko.lavinen@nokia.com>
      9 *
     10 * Based on Richard Purdie's earlier implementation in 2007. Background
     11 * support and lock-less operation written by Adrian Hunter.
     12 */
     13
     14#include <linux/kernel.h>
     15#include <linux/module.h>
     16#include <linux/mtd/mtd.h>
     17#include <linux/mtd/blktrans.h>
     18#include <linux/rbtree.h>
     19#include <linux/sched.h>
     20#include <linux/slab.h>
     21#include <linux/vmalloc.h>
     22#include <linux/blkdev.h>
     23#include <linux/swap.h>
     24#include <linux/debugfs.h>
     25#include <linux/seq_file.h>
     26#include <linux/device.h>
     27#include <linux/math64.h>
     28
     29#define MTDSWAP_PREFIX "mtdswap"
     30
     31/*
     32 * The number of free eraseblocks when GC should stop
     33 */
     34#define CLEAN_BLOCK_THRESHOLD	20
     35
     36/*
     37 * Number of free eraseblocks below which GC can also collect low frag
     38 * blocks.
     39 */
     40#define LOW_FRAG_GC_THRESHOLD	5
     41
     42/*
     43 * Wear level cost amortization. We want to do wear leveling on the background
     44 * without disturbing gc too much. This is made by defining max GC frequency.
     45 * Frequency value 6 means 1/6 of the GC passes will pick an erase block based
     46 * on the biggest wear difference rather than the biggest dirtiness.
     47 *
     48 * The lower freq2 should be chosen so that it makes sure the maximum erase
     49 * difference will decrease even if a malicious application is deliberately
     50 * trying to make erase differences large.
     51 */
     52#define MAX_ERASE_DIFF		4000
     53#define COLLECT_NONDIRTY_BASE	MAX_ERASE_DIFF
     54#define COLLECT_NONDIRTY_FREQ1	6
     55#define COLLECT_NONDIRTY_FREQ2	4
     56
     57#define PAGE_UNDEF		UINT_MAX
     58#define BLOCK_UNDEF		UINT_MAX
     59#define BLOCK_ERROR		(UINT_MAX - 1)
     60#define BLOCK_MAX		(UINT_MAX - 2)
     61
     62#define EBLOCK_BAD		(1 << 0)
     63#define EBLOCK_NOMAGIC		(1 << 1)
     64#define EBLOCK_BITFLIP		(1 << 2)
     65#define EBLOCK_FAILED		(1 << 3)
     66#define EBLOCK_READERR		(1 << 4)
     67#define EBLOCK_IDX_SHIFT	5
     68
     69struct swap_eb {
     70	struct rb_node rb;
     71	struct rb_root *root;
     72
     73	unsigned int flags;
     74	unsigned int active_count;
     75	unsigned int erase_count;
     76	unsigned int pad;		/* speeds up pointer decrement */
     77};
     78
     79#define MTDSWAP_ECNT_MIN(rbroot) (rb_entry(rb_first(rbroot), struct swap_eb, \
     80				rb)->erase_count)
     81#define MTDSWAP_ECNT_MAX(rbroot) (rb_entry(rb_last(rbroot), struct swap_eb, \
     82				rb)->erase_count)
     83
     84struct mtdswap_tree {
     85	struct rb_root root;
     86	unsigned int count;
     87};
     88
     89enum {
     90	MTDSWAP_CLEAN,
     91	MTDSWAP_USED,
     92	MTDSWAP_LOWFRAG,
     93	MTDSWAP_HIFRAG,
     94	MTDSWAP_DIRTY,
     95	MTDSWAP_BITFLIP,
     96	MTDSWAP_FAILING,
     97	MTDSWAP_TREE_CNT,
     98};
     99
    100struct mtdswap_dev {
    101	struct mtd_blktrans_dev *mbd_dev;
    102	struct mtd_info *mtd;
    103	struct device *dev;
    104
    105	unsigned int *page_data;
    106	unsigned int *revmap;
    107
    108	unsigned int eblks;
    109	unsigned int spare_eblks;
    110	unsigned int pages_per_eblk;
    111	unsigned int max_erase_count;
    112	struct swap_eb *eb_data;
    113
    114	struct mtdswap_tree trees[MTDSWAP_TREE_CNT];
    115
    116	unsigned long long sect_read_count;
    117	unsigned long long sect_write_count;
    118	unsigned long long mtd_write_count;
    119	unsigned long long mtd_read_count;
    120	unsigned long long discard_count;
    121	unsigned long long discard_page_count;
    122
    123	unsigned int curr_write_pos;
    124	struct swap_eb *curr_write;
    125
    126	char *page_buf;
    127	char *oob_buf;
    128};
    129
    130struct mtdswap_oobdata {
    131	__le16 magic;
    132	__le32 count;
    133} __packed;
    134
    135#define MTDSWAP_MAGIC_CLEAN	0x2095
    136#define MTDSWAP_MAGIC_DIRTY	(MTDSWAP_MAGIC_CLEAN + 1)
    137#define MTDSWAP_TYPE_CLEAN	0
    138#define MTDSWAP_TYPE_DIRTY	1
    139#define MTDSWAP_OOBSIZE		sizeof(struct mtdswap_oobdata)
    140
    141#define MTDSWAP_ERASE_RETRIES	3 /* Before marking erase block bad */
    142#define MTDSWAP_IO_RETRIES	3
    143
    144enum {
    145	MTDSWAP_SCANNED_CLEAN,
    146	MTDSWAP_SCANNED_DIRTY,
    147	MTDSWAP_SCANNED_BITFLIP,
    148	MTDSWAP_SCANNED_BAD,
    149};
    150
    151/*
    152 * In the worst case mtdswap_writesect() has allocated the last clean
    153 * page from the current block and is then pre-empted by the GC
    154 * thread. The thread can consume a full erase block when moving a
    155 * block.
    156 */
    157#define MIN_SPARE_EBLOCKS	2
    158#define MIN_ERASE_BLOCKS	(MIN_SPARE_EBLOCKS + 1)
    159
    160#define TREE_ROOT(d, name) (&d->trees[MTDSWAP_ ## name].root)
    161#define TREE_EMPTY(d, name) (TREE_ROOT(d, name)->rb_node == NULL)
    162#define TREE_NONEMPTY(d, name) (!TREE_EMPTY(d, name))
    163#define TREE_COUNT(d, name) (d->trees[MTDSWAP_ ## name].count)
    164
    165#define MTDSWAP_MBD_TO_MTDSWAP(dev) ((struct mtdswap_dev *)dev->priv)
    166
    167static char partitions[128] = "";
    168module_param_string(partitions, partitions, sizeof(partitions), 0444);
    169MODULE_PARM_DESC(partitions, "MTD partition numbers to use as swap "
    170		"partitions=\"1,3,5\"");
    171
    172static unsigned int spare_eblocks = 10;
    173module_param(spare_eblocks, uint, 0444);
    174MODULE_PARM_DESC(spare_eblocks, "Percentage of spare erase blocks for "
    175		"garbage collection (default 10%)");
    176
    177static bool header; /* false */
    178module_param(header, bool, 0444);
    179MODULE_PARM_DESC(header,
    180		"Include builtin swap header (default 0, without header)");
    181
    182static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background);
    183
    184static loff_t mtdswap_eb_offset(struct mtdswap_dev *d, struct swap_eb *eb)
    185{
    186	return (loff_t)(eb - d->eb_data) * d->mtd->erasesize;
    187}
    188
    189static void mtdswap_eb_detach(struct mtdswap_dev *d, struct swap_eb *eb)
    190{
    191	unsigned int oldidx;
    192	struct mtdswap_tree *tp;
    193
    194	if (eb->root) {
    195		tp = container_of(eb->root, struct mtdswap_tree, root);
    196		oldidx = tp - &d->trees[0];
    197
    198		d->trees[oldidx].count--;
    199		rb_erase(&eb->rb, eb->root);
    200	}
    201}
    202
    203static void __mtdswap_rb_add(struct rb_root *root, struct swap_eb *eb)
    204{
    205	struct rb_node **p, *parent = NULL;
    206	struct swap_eb *cur;
    207
    208	p = &root->rb_node;
    209	while (*p) {
    210		parent = *p;
    211		cur = rb_entry(parent, struct swap_eb, rb);
    212		if (eb->erase_count > cur->erase_count)
    213			p = &(*p)->rb_right;
    214		else
    215			p = &(*p)->rb_left;
    216	}
    217
    218	rb_link_node(&eb->rb, parent, p);
    219	rb_insert_color(&eb->rb, root);
    220}
    221
    222static void mtdswap_rb_add(struct mtdswap_dev *d, struct swap_eb *eb, int idx)
    223{
    224	struct rb_root *root;
    225
    226	if (eb->root == &d->trees[idx].root)
    227		return;
    228
    229	mtdswap_eb_detach(d, eb);
    230	root = &d->trees[idx].root;
    231	__mtdswap_rb_add(root, eb);
    232	eb->root = root;
    233	d->trees[idx].count++;
    234}
    235
    236static struct rb_node *mtdswap_rb_index(struct rb_root *root, unsigned int idx)
    237{
    238	struct rb_node *p;
    239	unsigned int i;
    240
    241	p = rb_first(root);
    242	i = 0;
    243	while (i < idx && p) {
    244		p = rb_next(p);
    245		i++;
    246	}
    247
    248	return p;
    249}
    250
    251static int mtdswap_handle_badblock(struct mtdswap_dev *d, struct swap_eb *eb)
    252{
    253	int ret;
    254	loff_t offset;
    255
    256	d->spare_eblks--;
    257	eb->flags |= EBLOCK_BAD;
    258	mtdswap_eb_detach(d, eb);
    259	eb->root = NULL;
    260
    261	/* badblocks not supported */
    262	if (!mtd_can_have_bb(d->mtd))
    263		return 1;
    264
    265	offset = mtdswap_eb_offset(d, eb);
    266	dev_warn(d->dev, "Marking bad block at %08llx\n", offset);
    267	ret = mtd_block_markbad(d->mtd, offset);
    268
    269	if (ret) {
    270		dev_warn(d->dev, "Mark block bad failed for block at %08llx "
    271			"error %d\n", offset, ret);
    272		return ret;
    273	}
    274
    275	return 1;
    276
    277}
    278
    279static int mtdswap_handle_write_error(struct mtdswap_dev *d, struct swap_eb *eb)
    280{
    281	unsigned int marked = eb->flags & EBLOCK_FAILED;
    282	struct swap_eb *curr_write = d->curr_write;
    283
    284	eb->flags |= EBLOCK_FAILED;
    285	if (curr_write == eb) {
    286		d->curr_write = NULL;
    287
    288		if (!marked && d->curr_write_pos != 0) {
    289			mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
    290			return 0;
    291		}
    292	}
    293
    294	return mtdswap_handle_badblock(d, eb);
    295}
    296
    297static int mtdswap_read_oob(struct mtdswap_dev *d, loff_t from,
    298			struct mtd_oob_ops *ops)
    299{
    300	int ret = mtd_read_oob(d->mtd, from, ops);
    301
    302	if (mtd_is_bitflip(ret))
    303		return ret;
    304
    305	if (ret) {
    306		dev_warn(d->dev, "Read OOB failed %d for block at %08llx\n",
    307			ret, from);
    308		return ret;
    309	}
    310
    311	if (ops->oobretlen < ops->ooblen) {
    312		dev_warn(d->dev, "Read OOB return short read (%zd bytes not "
    313			"%zd) for block at %08llx\n",
    314			ops->oobretlen, ops->ooblen, from);
    315		return -EIO;
    316	}
    317
    318	return 0;
    319}
    320
    321static int mtdswap_read_markers(struct mtdswap_dev *d, struct swap_eb *eb)
    322{
    323	struct mtdswap_oobdata *data, *data2;
    324	int ret;
    325	loff_t offset;
    326	struct mtd_oob_ops ops;
    327
    328	offset = mtdswap_eb_offset(d, eb);
    329
    330	/* Check first if the block is bad. */
    331	if (mtd_can_have_bb(d->mtd) && mtd_block_isbad(d->mtd, offset))
    332		return MTDSWAP_SCANNED_BAD;
    333
    334	ops.ooblen = 2 * d->mtd->oobavail;
    335	ops.oobbuf = d->oob_buf;
    336	ops.ooboffs = 0;
    337	ops.datbuf = NULL;
    338	ops.mode = MTD_OPS_AUTO_OOB;
    339
    340	ret = mtdswap_read_oob(d, offset, &ops);
    341
    342	if (ret && !mtd_is_bitflip(ret))
    343		return ret;
    344
    345	data = (struct mtdswap_oobdata *)d->oob_buf;
    346	data2 = (struct mtdswap_oobdata *)
    347		(d->oob_buf + d->mtd->oobavail);
    348
    349	if (le16_to_cpu(data->magic) == MTDSWAP_MAGIC_CLEAN) {
    350		eb->erase_count = le32_to_cpu(data->count);
    351		if (mtd_is_bitflip(ret))
    352			ret = MTDSWAP_SCANNED_BITFLIP;
    353		else {
    354			if (le16_to_cpu(data2->magic) == MTDSWAP_MAGIC_DIRTY)
    355				ret = MTDSWAP_SCANNED_DIRTY;
    356			else
    357				ret = MTDSWAP_SCANNED_CLEAN;
    358		}
    359	} else {
    360		eb->flags |= EBLOCK_NOMAGIC;
    361		ret = MTDSWAP_SCANNED_DIRTY;
    362	}
    363
    364	return ret;
    365}
    366
    367static int mtdswap_write_marker(struct mtdswap_dev *d, struct swap_eb *eb,
    368				u16 marker)
    369{
    370	struct mtdswap_oobdata n;
    371	int ret;
    372	loff_t offset;
    373	struct mtd_oob_ops ops;
    374
    375	ops.ooboffs = 0;
    376	ops.oobbuf = (uint8_t *)&n;
    377	ops.mode = MTD_OPS_AUTO_OOB;
    378	ops.datbuf = NULL;
    379
    380	if (marker == MTDSWAP_TYPE_CLEAN) {
    381		n.magic = cpu_to_le16(MTDSWAP_MAGIC_CLEAN);
    382		n.count = cpu_to_le32(eb->erase_count);
    383		ops.ooblen = MTDSWAP_OOBSIZE;
    384		offset = mtdswap_eb_offset(d, eb);
    385	} else {
    386		n.magic = cpu_to_le16(MTDSWAP_MAGIC_DIRTY);
    387		ops.ooblen = sizeof(n.magic);
    388		offset = mtdswap_eb_offset(d, eb) + d->mtd->writesize;
    389	}
    390
    391	ret = mtd_write_oob(d->mtd, offset, &ops);
    392
    393	if (ret) {
    394		dev_warn(d->dev, "Write OOB failed for block at %08llx "
    395			"error %d\n", offset, ret);
    396		if (ret == -EIO || mtd_is_eccerr(ret))
    397			mtdswap_handle_write_error(d, eb);
    398		return ret;
    399	}
    400
    401	if (ops.oobretlen != ops.ooblen) {
    402		dev_warn(d->dev, "Short OOB write for block at %08llx: "
    403			"%zd not %zd\n",
    404			offset, ops.oobretlen, ops.ooblen);
    405		return ret;
    406	}
    407
    408	return 0;
    409}
    410
    411/*
    412 * Are there any erase blocks without MAGIC_CLEAN header, presumably
    413 * because power was cut off after erase but before header write? We
    414 * need to guestimate the erase count.
    415 */
    416static void mtdswap_check_counts(struct mtdswap_dev *d)
    417{
    418	struct rb_root hist_root = RB_ROOT;
    419	struct rb_node *medrb;
    420	struct swap_eb *eb;
    421	unsigned int i, cnt, median;
    422
    423	cnt = 0;
    424	for (i = 0; i < d->eblks; i++) {
    425		eb = d->eb_data + i;
    426
    427		if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
    428			continue;
    429
    430		__mtdswap_rb_add(&hist_root, eb);
    431		cnt++;
    432	}
    433
    434	if (cnt == 0)
    435		return;
    436
    437	medrb = mtdswap_rb_index(&hist_root, cnt / 2);
    438	median = rb_entry(medrb, struct swap_eb, rb)->erase_count;
    439
    440	d->max_erase_count = MTDSWAP_ECNT_MAX(&hist_root);
    441
    442	for (i = 0; i < d->eblks; i++) {
    443		eb = d->eb_data + i;
    444
    445		if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_READERR))
    446			eb->erase_count = median;
    447
    448		if (eb->flags & (EBLOCK_NOMAGIC | EBLOCK_BAD | EBLOCK_READERR))
    449			continue;
    450
    451		rb_erase(&eb->rb, &hist_root);
    452	}
    453}
    454
    455static void mtdswap_scan_eblks(struct mtdswap_dev *d)
    456{
    457	int status;
    458	unsigned int i, idx;
    459	struct swap_eb *eb;
    460
    461	for (i = 0; i < d->eblks; i++) {
    462		eb = d->eb_data + i;
    463
    464		status = mtdswap_read_markers(d, eb);
    465		if (status < 0)
    466			eb->flags |= EBLOCK_READERR;
    467		else if (status == MTDSWAP_SCANNED_BAD) {
    468			eb->flags |= EBLOCK_BAD;
    469			continue;
    470		}
    471
    472		switch (status) {
    473		case MTDSWAP_SCANNED_CLEAN:
    474			idx = MTDSWAP_CLEAN;
    475			break;
    476		case MTDSWAP_SCANNED_DIRTY:
    477		case MTDSWAP_SCANNED_BITFLIP:
    478			idx = MTDSWAP_DIRTY;
    479			break;
    480		default:
    481			idx = MTDSWAP_FAILING;
    482		}
    483
    484		eb->flags |= (idx << EBLOCK_IDX_SHIFT);
    485	}
    486
    487	mtdswap_check_counts(d);
    488
    489	for (i = 0; i < d->eblks; i++) {
    490		eb = d->eb_data + i;
    491
    492		if (eb->flags & EBLOCK_BAD)
    493			continue;
    494
    495		idx = eb->flags >> EBLOCK_IDX_SHIFT;
    496		mtdswap_rb_add(d, eb, idx);
    497	}
    498}
    499
    500/*
    501 * Place eblk into a tree corresponding to its number of active blocks
    502 * it contains.
    503 */
    504static void mtdswap_store_eb(struct mtdswap_dev *d, struct swap_eb *eb)
    505{
    506	unsigned int weight = eb->active_count;
    507	unsigned int maxweight = d->pages_per_eblk;
    508
    509	if (eb == d->curr_write)
    510		return;
    511
    512	if (eb->flags & EBLOCK_BITFLIP)
    513		mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
    514	else if (eb->flags & (EBLOCK_READERR | EBLOCK_FAILED))
    515		mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
    516	if (weight == maxweight)
    517		mtdswap_rb_add(d, eb, MTDSWAP_USED);
    518	else if (weight == 0)
    519		mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
    520	else if (weight > (maxweight/2))
    521		mtdswap_rb_add(d, eb, MTDSWAP_LOWFRAG);
    522	else
    523		mtdswap_rb_add(d, eb, MTDSWAP_HIFRAG);
    524}
    525
    526static int mtdswap_erase_block(struct mtdswap_dev *d, struct swap_eb *eb)
    527{
    528	struct mtd_info *mtd = d->mtd;
    529	struct erase_info erase;
    530	unsigned int retries = 0;
    531	int ret;
    532
    533	eb->erase_count++;
    534	if (eb->erase_count > d->max_erase_count)
    535		d->max_erase_count = eb->erase_count;
    536
    537retry:
    538	memset(&erase, 0, sizeof(struct erase_info));
    539	erase.addr	= mtdswap_eb_offset(d, eb);
    540	erase.len	= mtd->erasesize;
    541
    542	ret = mtd_erase(mtd, &erase);
    543	if (ret) {
    544		if (retries++ < MTDSWAP_ERASE_RETRIES) {
    545			dev_warn(d->dev,
    546				"erase of erase block %#llx on %s failed",
    547				erase.addr, mtd->name);
    548			yield();
    549			goto retry;
    550		}
    551
    552		dev_err(d->dev, "Cannot erase erase block %#llx on %s\n",
    553			erase.addr, mtd->name);
    554
    555		mtdswap_handle_badblock(d, eb);
    556		return -EIO;
    557	}
    558
    559	return 0;
    560}
    561
    562static int mtdswap_map_free_block(struct mtdswap_dev *d, unsigned int page,
    563				unsigned int *block)
    564{
    565	int ret;
    566	struct swap_eb *old_eb = d->curr_write;
    567	struct rb_root *clean_root;
    568	struct swap_eb *eb;
    569
    570	if (old_eb == NULL || d->curr_write_pos >= d->pages_per_eblk) {
    571		do {
    572			if (TREE_EMPTY(d, CLEAN))
    573				return -ENOSPC;
    574
    575			clean_root = TREE_ROOT(d, CLEAN);
    576			eb = rb_entry(rb_first(clean_root), struct swap_eb, rb);
    577			rb_erase(&eb->rb, clean_root);
    578			eb->root = NULL;
    579			TREE_COUNT(d, CLEAN)--;
    580
    581			ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_DIRTY);
    582		} while (ret == -EIO || mtd_is_eccerr(ret));
    583
    584		if (ret)
    585			return ret;
    586
    587		d->curr_write_pos = 0;
    588		d->curr_write = eb;
    589		if (old_eb)
    590			mtdswap_store_eb(d, old_eb);
    591	}
    592
    593	*block = (d->curr_write - d->eb_data) * d->pages_per_eblk +
    594		d->curr_write_pos;
    595
    596	d->curr_write->active_count++;
    597	d->revmap[*block] = page;
    598	d->curr_write_pos++;
    599
    600	return 0;
    601}
    602
    603static unsigned int mtdswap_free_page_cnt(struct mtdswap_dev *d)
    604{
    605	return TREE_COUNT(d, CLEAN) * d->pages_per_eblk +
    606		d->pages_per_eblk - d->curr_write_pos;
    607}
    608
    609static unsigned int mtdswap_enough_free_pages(struct mtdswap_dev *d)
    610{
    611	return mtdswap_free_page_cnt(d) > d->pages_per_eblk;
    612}
    613
    614static int mtdswap_write_block(struct mtdswap_dev *d, char *buf,
    615			unsigned int page, unsigned int *bp, int gc_context)
    616{
    617	struct mtd_info *mtd = d->mtd;
    618	struct swap_eb *eb;
    619	size_t retlen;
    620	loff_t writepos;
    621	int ret;
    622
    623retry:
    624	if (!gc_context)
    625		while (!mtdswap_enough_free_pages(d))
    626			if (mtdswap_gc(d, 0) > 0)
    627				return -ENOSPC;
    628
    629	ret = mtdswap_map_free_block(d, page, bp);
    630	eb = d->eb_data + (*bp / d->pages_per_eblk);
    631
    632	if (ret == -EIO || mtd_is_eccerr(ret)) {
    633		d->curr_write = NULL;
    634		eb->active_count--;
    635		d->revmap[*bp] = PAGE_UNDEF;
    636		goto retry;
    637	}
    638
    639	if (ret < 0)
    640		return ret;
    641
    642	writepos = (loff_t)*bp << PAGE_SHIFT;
    643	ret =  mtd_write(mtd, writepos, PAGE_SIZE, &retlen, buf);
    644	if (ret == -EIO || mtd_is_eccerr(ret)) {
    645		d->curr_write_pos--;
    646		eb->active_count--;
    647		d->revmap[*bp] = PAGE_UNDEF;
    648		mtdswap_handle_write_error(d, eb);
    649		goto retry;
    650	}
    651
    652	if (ret < 0) {
    653		dev_err(d->dev, "Write to MTD device failed: %d (%zd written)",
    654			ret, retlen);
    655		goto err;
    656	}
    657
    658	if (retlen != PAGE_SIZE) {
    659		dev_err(d->dev, "Short write to MTD device: %zd written",
    660			retlen);
    661		ret = -EIO;
    662		goto err;
    663	}
    664
    665	return ret;
    666
    667err:
    668	d->curr_write_pos--;
    669	eb->active_count--;
    670	d->revmap[*bp] = PAGE_UNDEF;
    671
    672	return ret;
    673}
    674
    675static int mtdswap_move_block(struct mtdswap_dev *d, unsigned int oldblock,
    676		unsigned int *newblock)
    677{
    678	struct mtd_info *mtd = d->mtd;
    679	struct swap_eb *eb, *oldeb;
    680	int ret;
    681	size_t retlen;
    682	unsigned int page, retries;
    683	loff_t readpos;
    684
    685	page = d->revmap[oldblock];
    686	readpos = (loff_t) oldblock << PAGE_SHIFT;
    687	retries = 0;
    688
    689retry:
    690	ret = mtd_read(mtd, readpos, PAGE_SIZE, &retlen, d->page_buf);
    691
    692	if (ret < 0 && !mtd_is_bitflip(ret)) {
    693		oldeb = d->eb_data + oldblock / d->pages_per_eblk;
    694		oldeb->flags |= EBLOCK_READERR;
    695
    696		dev_err(d->dev, "Read Error: %d (block %u)\n", ret,
    697			oldblock);
    698		retries++;
    699		if (retries < MTDSWAP_IO_RETRIES)
    700			goto retry;
    701
    702		goto read_error;
    703	}
    704
    705	if (retlen != PAGE_SIZE) {
    706		dev_err(d->dev, "Short read: %zd (block %u)\n", retlen,
    707		       oldblock);
    708		ret = -EIO;
    709		goto read_error;
    710	}
    711
    712	ret = mtdswap_write_block(d, d->page_buf, page, newblock, 1);
    713	if (ret < 0) {
    714		d->page_data[page] = BLOCK_ERROR;
    715		dev_err(d->dev, "Write error: %d\n", ret);
    716		return ret;
    717	}
    718
    719	d->page_data[page] = *newblock;
    720	d->revmap[oldblock] = PAGE_UNDEF;
    721	eb = d->eb_data + oldblock / d->pages_per_eblk;
    722	eb->active_count--;
    723
    724	return 0;
    725
    726read_error:
    727	d->page_data[page] = BLOCK_ERROR;
    728	d->revmap[oldblock] = PAGE_UNDEF;
    729	return ret;
    730}
    731
    732static int mtdswap_gc_eblock(struct mtdswap_dev *d, struct swap_eb *eb)
    733{
    734	unsigned int i, block, eblk_base, newblock;
    735	int ret, errcode;
    736
    737	errcode = 0;
    738	eblk_base = (eb - d->eb_data) * d->pages_per_eblk;
    739
    740	for (i = 0; i < d->pages_per_eblk; i++) {
    741		if (d->spare_eblks < MIN_SPARE_EBLOCKS)
    742			return -ENOSPC;
    743
    744		block = eblk_base + i;
    745		if (d->revmap[block] == PAGE_UNDEF)
    746			continue;
    747
    748		ret = mtdswap_move_block(d, block, &newblock);
    749		if (ret < 0 && !errcode)
    750			errcode = ret;
    751	}
    752
    753	return errcode;
    754}
    755
    756static int __mtdswap_choose_gc_tree(struct mtdswap_dev *d)
    757{
    758	int idx, stopat;
    759
    760	if (TREE_COUNT(d, CLEAN) < LOW_FRAG_GC_THRESHOLD)
    761		stopat = MTDSWAP_LOWFRAG;
    762	else
    763		stopat = MTDSWAP_HIFRAG;
    764
    765	for (idx = MTDSWAP_BITFLIP; idx >= stopat; idx--)
    766		if (d->trees[idx].root.rb_node != NULL)
    767			return idx;
    768
    769	return -1;
    770}
    771
    772static int mtdswap_wlfreq(unsigned int maxdiff)
    773{
    774	unsigned int h, x, y, dist, base;
    775
    776	/*
    777	 * Calculate linear ramp down from f1 to f2 when maxdiff goes from
    778	 * MAX_ERASE_DIFF to MAX_ERASE_DIFF + COLLECT_NONDIRTY_BASE.  Similar
    779	 * to triangle with height f1 - f1 and width COLLECT_NONDIRTY_BASE.
    780	 */
    781
    782	dist = maxdiff - MAX_ERASE_DIFF;
    783	if (dist > COLLECT_NONDIRTY_BASE)
    784		dist = COLLECT_NONDIRTY_BASE;
    785
    786	/*
    787	 * Modelling the slop as right angular triangle with base
    788	 * COLLECT_NONDIRTY_BASE and height freq1 - freq2. The ratio y/x is
    789	 * equal to the ratio h/base.
    790	 */
    791	h = COLLECT_NONDIRTY_FREQ1 - COLLECT_NONDIRTY_FREQ2;
    792	base = COLLECT_NONDIRTY_BASE;
    793
    794	x = dist - base;
    795	y = (x * h + base / 2) / base;
    796
    797	return COLLECT_NONDIRTY_FREQ2 + y;
    798}
    799
    800static int mtdswap_choose_wl_tree(struct mtdswap_dev *d)
    801{
    802	static unsigned int pick_cnt;
    803	unsigned int i, idx = -1, wear, max;
    804	struct rb_root *root;
    805
    806	max = 0;
    807	for (i = 0; i <= MTDSWAP_DIRTY; i++) {
    808		root = &d->trees[i].root;
    809		if (root->rb_node == NULL)
    810			continue;
    811
    812		wear = d->max_erase_count - MTDSWAP_ECNT_MIN(root);
    813		if (wear > max) {
    814			max = wear;
    815			idx = i;
    816		}
    817	}
    818
    819	if (max > MAX_ERASE_DIFF && pick_cnt >= mtdswap_wlfreq(max) - 1) {
    820		pick_cnt = 0;
    821		return idx;
    822	}
    823
    824	pick_cnt++;
    825	return -1;
    826}
    827
    828static int mtdswap_choose_gc_tree(struct mtdswap_dev *d,
    829				unsigned int background)
    830{
    831	int idx;
    832
    833	if (TREE_NONEMPTY(d, FAILING) &&
    834		(background || (TREE_EMPTY(d, CLEAN) && TREE_EMPTY(d, DIRTY))))
    835		return MTDSWAP_FAILING;
    836
    837	idx = mtdswap_choose_wl_tree(d);
    838	if (idx >= MTDSWAP_CLEAN)
    839		return idx;
    840
    841	return __mtdswap_choose_gc_tree(d);
    842}
    843
    844static struct swap_eb *mtdswap_pick_gc_eblk(struct mtdswap_dev *d,
    845					unsigned int background)
    846{
    847	struct rb_root *rp = NULL;
    848	struct swap_eb *eb = NULL;
    849	int idx;
    850
    851	if (background && TREE_COUNT(d, CLEAN) > CLEAN_BLOCK_THRESHOLD &&
    852		TREE_EMPTY(d, DIRTY) && TREE_EMPTY(d, FAILING))
    853		return NULL;
    854
    855	idx = mtdswap_choose_gc_tree(d, background);
    856	if (idx < 0)
    857		return NULL;
    858
    859	rp = &d->trees[idx].root;
    860	eb = rb_entry(rb_first(rp), struct swap_eb, rb);
    861
    862	rb_erase(&eb->rb, rp);
    863	eb->root = NULL;
    864	d->trees[idx].count--;
    865	return eb;
    866}
    867
    868static unsigned int mtdswap_test_patt(unsigned int i)
    869{
    870	return i % 2 ? 0x55555555 : 0xAAAAAAAA;
    871}
    872
    873static unsigned int mtdswap_eblk_passes(struct mtdswap_dev *d,
    874					struct swap_eb *eb)
    875{
    876	struct mtd_info *mtd = d->mtd;
    877	unsigned int test, i, j, patt, mtd_pages;
    878	loff_t base, pos;
    879	unsigned int *p1 = (unsigned int *)d->page_buf;
    880	unsigned char *p2 = (unsigned char *)d->oob_buf;
    881	struct mtd_oob_ops ops;
    882	int ret;
    883
    884	ops.mode = MTD_OPS_AUTO_OOB;
    885	ops.len = mtd->writesize;
    886	ops.ooblen = mtd->oobavail;
    887	ops.ooboffs = 0;
    888	ops.datbuf = d->page_buf;
    889	ops.oobbuf = d->oob_buf;
    890	base = mtdswap_eb_offset(d, eb);
    891	mtd_pages = d->pages_per_eblk * PAGE_SIZE / mtd->writesize;
    892
    893	for (test = 0; test < 2; test++) {
    894		pos = base;
    895		for (i = 0; i < mtd_pages; i++) {
    896			patt = mtdswap_test_patt(test + i);
    897			memset(d->page_buf, patt, mtd->writesize);
    898			memset(d->oob_buf, patt, mtd->oobavail);
    899			ret = mtd_write_oob(mtd, pos, &ops);
    900			if (ret)
    901				goto error;
    902
    903			pos += mtd->writesize;
    904		}
    905
    906		pos = base;
    907		for (i = 0; i < mtd_pages; i++) {
    908			ret = mtd_read_oob(mtd, pos, &ops);
    909			if (ret)
    910				goto error;
    911
    912			patt = mtdswap_test_patt(test + i);
    913			for (j = 0; j < mtd->writesize/sizeof(int); j++)
    914				if (p1[j] != patt)
    915					goto error;
    916
    917			for (j = 0; j < mtd->oobavail; j++)
    918				if (p2[j] != (unsigned char)patt)
    919					goto error;
    920
    921			pos += mtd->writesize;
    922		}
    923
    924		ret = mtdswap_erase_block(d, eb);
    925		if (ret)
    926			goto error;
    927	}
    928
    929	eb->flags &= ~EBLOCK_READERR;
    930	return 1;
    931
    932error:
    933	mtdswap_handle_badblock(d, eb);
    934	return 0;
    935}
    936
    937static int mtdswap_gc(struct mtdswap_dev *d, unsigned int background)
    938{
    939	struct swap_eb *eb;
    940	int ret;
    941
    942	if (d->spare_eblks < MIN_SPARE_EBLOCKS)
    943		return 1;
    944
    945	eb = mtdswap_pick_gc_eblk(d, background);
    946	if (!eb)
    947		return 1;
    948
    949	ret = mtdswap_gc_eblock(d, eb);
    950	if (ret == -ENOSPC)
    951		return 1;
    952
    953	if (eb->flags & EBLOCK_FAILED) {
    954		mtdswap_handle_badblock(d, eb);
    955		return 0;
    956	}
    957
    958	eb->flags &= ~EBLOCK_BITFLIP;
    959	ret = mtdswap_erase_block(d, eb);
    960	if ((eb->flags & EBLOCK_READERR) &&
    961		(ret || !mtdswap_eblk_passes(d, eb)))
    962		return 0;
    963
    964	if (ret == 0)
    965		ret = mtdswap_write_marker(d, eb, MTDSWAP_TYPE_CLEAN);
    966
    967	if (ret == 0)
    968		mtdswap_rb_add(d, eb, MTDSWAP_CLEAN);
    969	else if (ret != -EIO && !mtd_is_eccerr(ret))
    970		mtdswap_rb_add(d, eb, MTDSWAP_DIRTY);
    971
    972	return 0;
    973}
    974
    975static void mtdswap_background(struct mtd_blktrans_dev *dev)
    976{
    977	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
    978	int ret;
    979
    980	while (1) {
    981		ret = mtdswap_gc(d, 1);
    982		if (ret || mtd_blktrans_cease_background(dev))
    983			return;
    984	}
    985}
    986
    987static void mtdswap_cleanup(struct mtdswap_dev *d)
    988{
    989	vfree(d->eb_data);
    990	vfree(d->revmap);
    991	vfree(d->page_data);
    992	kfree(d->oob_buf);
    993	kfree(d->page_buf);
    994}
    995
    996static int mtdswap_flush(struct mtd_blktrans_dev *dev)
    997{
    998	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
    999
   1000	mtd_sync(d->mtd);
   1001	return 0;
   1002}
   1003
   1004static unsigned int mtdswap_badblocks(struct mtd_info *mtd, uint64_t size)
   1005{
   1006	loff_t offset;
   1007	unsigned int badcnt;
   1008
   1009	badcnt = 0;
   1010
   1011	if (mtd_can_have_bb(mtd))
   1012		for (offset = 0; offset < size; offset += mtd->erasesize)
   1013			if (mtd_block_isbad(mtd, offset))
   1014				badcnt++;
   1015
   1016	return badcnt;
   1017}
   1018
   1019static int mtdswap_writesect(struct mtd_blktrans_dev *dev,
   1020			unsigned long page, char *buf)
   1021{
   1022	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
   1023	unsigned int newblock, mapped;
   1024	struct swap_eb *eb;
   1025	int ret;
   1026
   1027	d->sect_write_count++;
   1028
   1029	if (d->spare_eblks < MIN_SPARE_EBLOCKS)
   1030		return -ENOSPC;
   1031
   1032	if (header) {
   1033		/* Ignore writes to the header page */
   1034		if (unlikely(page == 0))
   1035			return 0;
   1036
   1037		page--;
   1038	}
   1039
   1040	mapped = d->page_data[page];
   1041	if (mapped <= BLOCK_MAX) {
   1042		eb = d->eb_data + (mapped / d->pages_per_eblk);
   1043		eb->active_count--;
   1044		mtdswap_store_eb(d, eb);
   1045		d->page_data[page] = BLOCK_UNDEF;
   1046		d->revmap[mapped] = PAGE_UNDEF;
   1047	}
   1048
   1049	ret = mtdswap_write_block(d, buf, page, &newblock, 0);
   1050	d->mtd_write_count++;
   1051
   1052	if (ret < 0)
   1053		return ret;
   1054
   1055	d->page_data[page] = newblock;
   1056
   1057	return 0;
   1058}
   1059
   1060/* Provide a dummy swap header for the kernel */
   1061static int mtdswap_auto_header(struct mtdswap_dev *d, char *buf)
   1062{
   1063	union swap_header *hd = (union swap_header *)(buf);
   1064
   1065	memset(buf, 0, PAGE_SIZE - 10);
   1066
   1067	hd->info.version = 1;
   1068	hd->info.last_page = d->mbd_dev->size - 1;
   1069	hd->info.nr_badpages = 0;
   1070
   1071	memcpy(buf + PAGE_SIZE - 10, "SWAPSPACE2", 10);
   1072
   1073	return 0;
   1074}
   1075
   1076static int mtdswap_readsect(struct mtd_blktrans_dev *dev,
   1077			unsigned long page, char *buf)
   1078{
   1079	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
   1080	struct mtd_info *mtd = d->mtd;
   1081	unsigned int realblock, retries;
   1082	loff_t readpos;
   1083	struct swap_eb *eb;
   1084	size_t retlen;
   1085	int ret;
   1086
   1087	d->sect_read_count++;
   1088
   1089	if (header) {
   1090		if (unlikely(page == 0))
   1091			return mtdswap_auto_header(d, buf);
   1092
   1093		page--;
   1094	}
   1095
   1096	realblock = d->page_data[page];
   1097	if (realblock > BLOCK_MAX) {
   1098		memset(buf, 0x0, PAGE_SIZE);
   1099		if (realblock == BLOCK_UNDEF)
   1100			return 0;
   1101		else
   1102			return -EIO;
   1103	}
   1104
   1105	eb = d->eb_data + (realblock / d->pages_per_eblk);
   1106	BUG_ON(d->revmap[realblock] == PAGE_UNDEF);
   1107
   1108	readpos = (loff_t)realblock << PAGE_SHIFT;
   1109	retries = 0;
   1110
   1111retry:
   1112	ret = mtd_read(mtd, readpos, PAGE_SIZE, &retlen, buf);
   1113
   1114	d->mtd_read_count++;
   1115	if (mtd_is_bitflip(ret)) {
   1116		eb->flags |= EBLOCK_BITFLIP;
   1117		mtdswap_rb_add(d, eb, MTDSWAP_BITFLIP);
   1118		ret = 0;
   1119	}
   1120
   1121	if (ret < 0) {
   1122		dev_err(d->dev, "Read error %d\n", ret);
   1123		eb->flags |= EBLOCK_READERR;
   1124		mtdswap_rb_add(d, eb, MTDSWAP_FAILING);
   1125		retries++;
   1126		if (retries < MTDSWAP_IO_RETRIES)
   1127			goto retry;
   1128
   1129		return ret;
   1130	}
   1131
   1132	if (retlen != PAGE_SIZE) {
   1133		dev_err(d->dev, "Short read %zd\n", retlen);
   1134		return -EIO;
   1135	}
   1136
   1137	return 0;
   1138}
   1139
   1140static int mtdswap_discard(struct mtd_blktrans_dev *dev, unsigned long first,
   1141			unsigned nr_pages)
   1142{
   1143	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
   1144	unsigned long page;
   1145	struct swap_eb *eb;
   1146	unsigned int mapped;
   1147
   1148	d->discard_count++;
   1149
   1150	for (page = first; page < first + nr_pages; page++) {
   1151		mapped = d->page_data[page];
   1152		if (mapped <= BLOCK_MAX) {
   1153			eb = d->eb_data + (mapped / d->pages_per_eblk);
   1154			eb->active_count--;
   1155			mtdswap_store_eb(d, eb);
   1156			d->page_data[page] = BLOCK_UNDEF;
   1157			d->revmap[mapped] = PAGE_UNDEF;
   1158			d->discard_page_count++;
   1159		} else if (mapped == BLOCK_ERROR) {
   1160			d->page_data[page] = BLOCK_UNDEF;
   1161			d->discard_page_count++;
   1162		}
   1163	}
   1164
   1165	return 0;
   1166}
   1167
   1168static int mtdswap_show(struct seq_file *s, void *data)
   1169{
   1170	struct mtdswap_dev *d = (struct mtdswap_dev *) s->private;
   1171	unsigned long sum;
   1172	unsigned int count[MTDSWAP_TREE_CNT];
   1173	unsigned int min[MTDSWAP_TREE_CNT];
   1174	unsigned int max[MTDSWAP_TREE_CNT];
   1175	unsigned int i, cw = 0, cwp = 0, cwecount = 0, bb_cnt, mapped, pages;
   1176	uint64_t use_size;
   1177	static const char * const name[] = {
   1178		"clean", "used", "low", "high", "dirty", "bitflip", "failing"
   1179	};
   1180
   1181	mutex_lock(&d->mbd_dev->lock);
   1182
   1183	for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
   1184		struct rb_root *root = &d->trees[i].root;
   1185
   1186		if (root->rb_node) {
   1187			count[i] = d->trees[i].count;
   1188			min[i] = MTDSWAP_ECNT_MIN(root);
   1189			max[i] = MTDSWAP_ECNT_MAX(root);
   1190		} else
   1191			count[i] = 0;
   1192	}
   1193
   1194	if (d->curr_write) {
   1195		cw = 1;
   1196		cwp = d->curr_write_pos;
   1197		cwecount = d->curr_write->erase_count;
   1198	}
   1199
   1200	sum = 0;
   1201	for (i = 0; i < d->eblks; i++)
   1202		sum += d->eb_data[i].erase_count;
   1203
   1204	use_size = (uint64_t)d->eblks * d->mtd->erasesize;
   1205	bb_cnt = mtdswap_badblocks(d->mtd, use_size);
   1206
   1207	mapped = 0;
   1208	pages = d->mbd_dev->size;
   1209	for (i = 0; i < pages; i++)
   1210		if (d->page_data[i] != BLOCK_UNDEF)
   1211			mapped++;
   1212
   1213	mutex_unlock(&d->mbd_dev->lock);
   1214
   1215	for (i = 0; i < MTDSWAP_TREE_CNT; i++) {
   1216		if (!count[i])
   1217			continue;
   1218
   1219		if (min[i] != max[i])
   1220			seq_printf(s, "%s:\t%5d erase blocks, erased min %d, "
   1221				"max %d times\n",
   1222				name[i], count[i], min[i], max[i]);
   1223		else
   1224			seq_printf(s, "%s:\t%5d erase blocks, all erased %d "
   1225				"times\n", name[i], count[i], min[i]);
   1226	}
   1227
   1228	if (bb_cnt)
   1229		seq_printf(s, "bad:\t%5u erase blocks\n", bb_cnt);
   1230
   1231	if (cw)
   1232		seq_printf(s, "current erase block: %u pages used, %u free, "
   1233			"erased %u times\n",
   1234			cwp, d->pages_per_eblk - cwp, cwecount);
   1235
   1236	seq_printf(s, "total erasures: %lu\n", sum);
   1237
   1238	seq_puts(s, "\n");
   1239
   1240	seq_printf(s, "mtdswap_readsect count: %llu\n", d->sect_read_count);
   1241	seq_printf(s, "mtdswap_writesect count: %llu\n", d->sect_write_count);
   1242	seq_printf(s, "mtdswap_discard count: %llu\n", d->discard_count);
   1243	seq_printf(s, "mtd read count: %llu\n", d->mtd_read_count);
   1244	seq_printf(s, "mtd write count: %llu\n", d->mtd_write_count);
   1245	seq_printf(s, "discarded pages count: %llu\n", d->discard_page_count);
   1246
   1247	seq_puts(s, "\n");
   1248	seq_printf(s, "total pages: %u\n", pages);
   1249	seq_printf(s, "pages mapped: %u\n", mapped);
   1250
   1251	return 0;
   1252}
   1253DEFINE_SHOW_ATTRIBUTE(mtdswap);
   1254
   1255static int mtdswap_add_debugfs(struct mtdswap_dev *d)
   1256{
   1257	struct dentry *root = d->mtd->dbg.dfs_dir;
   1258
   1259	if (!IS_ENABLED(CONFIG_DEBUG_FS))
   1260		return 0;
   1261
   1262	if (IS_ERR_OR_NULL(root))
   1263		return -1;
   1264
   1265	debugfs_create_file("mtdswap_stats", S_IRUSR, root, d, &mtdswap_fops);
   1266
   1267	return 0;
   1268}
   1269
   1270static int mtdswap_init(struct mtdswap_dev *d, unsigned int eblocks,
   1271			unsigned int spare_cnt)
   1272{
   1273	struct mtd_info *mtd = d->mbd_dev->mtd;
   1274	unsigned int i, eblk_bytes, pages, blocks;
   1275	int ret = -ENOMEM;
   1276
   1277	d->mtd = mtd;
   1278	d->eblks = eblocks;
   1279	d->spare_eblks = spare_cnt;
   1280	d->pages_per_eblk = mtd->erasesize >> PAGE_SHIFT;
   1281
   1282	pages = d->mbd_dev->size;
   1283	blocks = eblocks * d->pages_per_eblk;
   1284
   1285	for (i = 0; i < MTDSWAP_TREE_CNT; i++)
   1286		d->trees[i].root = RB_ROOT;
   1287
   1288	d->page_data = vmalloc(array_size(pages, sizeof(int)));
   1289	if (!d->page_data)
   1290		goto page_data_fail;
   1291
   1292	d->revmap = vmalloc(array_size(blocks, sizeof(int)));
   1293	if (!d->revmap)
   1294		goto revmap_fail;
   1295
   1296	eblk_bytes = sizeof(struct swap_eb)*d->eblks;
   1297	d->eb_data = vzalloc(eblk_bytes);
   1298	if (!d->eb_data)
   1299		goto eb_data_fail;
   1300
   1301	for (i = 0; i < pages; i++)
   1302		d->page_data[i] = BLOCK_UNDEF;
   1303
   1304	for (i = 0; i < blocks; i++)
   1305		d->revmap[i] = PAGE_UNDEF;
   1306
   1307	d->page_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
   1308	if (!d->page_buf)
   1309		goto page_buf_fail;
   1310
   1311	d->oob_buf = kmalloc_array(2, mtd->oobavail, GFP_KERNEL);
   1312	if (!d->oob_buf)
   1313		goto oob_buf_fail;
   1314
   1315	mtdswap_scan_eblks(d);
   1316
   1317	return 0;
   1318
   1319oob_buf_fail:
   1320	kfree(d->page_buf);
   1321page_buf_fail:
   1322	vfree(d->eb_data);
   1323eb_data_fail:
   1324	vfree(d->revmap);
   1325revmap_fail:
   1326	vfree(d->page_data);
   1327page_data_fail:
   1328	printk(KERN_ERR "%s: init failed (%d)\n", MTDSWAP_PREFIX, ret);
   1329	return ret;
   1330}
   1331
   1332static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
   1333{
   1334	struct mtdswap_dev *d;
   1335	struct mtd_blktrans_dev *mbd_dev;
   1336	char *parts;
   1337	char *this_opt;
   1338	unsigned long part;
   1339	unsigned int eblocks, eavailable, bad_blocks, spare_cnt;
   1340	uint64_t swap_size, use_size, size_limit;
   1341	int ret;
   1342
   1343	parts = &partitions[0];
   1344	if (!*parts)
   1345		return;
   1346
   1347	while ((this_opt = strsep(&parts, ",")) != NULL) {
   1348		if (kstrtoul(this_opt, 0, &part) < 0)
   1349			return;
   1350
   1351		if (mtd->index == part)
   1352			break;
   1353	}
   1354
   1355	if (mtd->index != part)
   1356		return;
   1357
   1358	if (mtd->erasesize < PAGE_SIZE || mtd->erasesize % PAGE_SIZE) {
   1359		printk(KERN_ERR "%s: Erase size %u not multiple of PAGE_SIZE "
   1360			"%lu\n", MTDSWAP_PREFIX, mtd->erasesize, PAGE_SIZE);
   1361		return;
   1362	}
   1363
   1364	if (PAGE_SIZE % mtd->writesize || mtd->writesize > PAGE_SIZE) {
   1365		printk(KERN_ERR "%s: PAGE_SIZE %lu not multiple of write size"
   1366			" %u\n", MTDSWAP_PREFIX, PAGE_SIZE, mtd->writesize);
   1367		return;
   1368	}
   1369
   1370	if (!mtd->oobsize || mtd->oobavail < MTDSWAP_OOBSIZE) {
   1371		printk(KERN_ERR "%s: Not enough free bytes in OOB, "
   1372			"%d available, %zu needed.\n",
   1373			MTDSWAP_PREFIX, mtd->oobavail, MTDSWAP_OOBSIZE);
   1374		return;
   1375	}
   1376
   1377	if (spare_eblocks > 100)
   1378		spare_eblocks = 100;
   1379
   1380	use_size = mtd->size;
   1381	size_limit = (uint64_t) BLOCK_MAX * PAGE_SIZE;
   1382
   1383	if (mtd->size > size_limit) {
   1384		printk(KERN_WARNING "%s: Device too large. Limiting size to "
   1385			"%llu bytes\n", MTDSWAP_PREFIX, size_limit);
   1386		use_size = size_limit;
   1387	}
   1388
   1389	eblocks = mtd_div_by_eb(use_size, mtd);
   1390	use_size = (uint64_t)eblocks * mtd->erasesize;
   1391	bad_blocks = mtdswap_badblocks(mtd, use_size);
   1392	eavailable = eblocks - bad_blocks;
   1393
   1394	if (eavailable < MIN_ERASE_BLOCKS) {
   1395		printk(KERN_ERR "%s: Not enough erase blocks. %u available, "
   1396			"%d needed\n", MTDSWAP_PREFIX, eavailable,
   1397			MIN_ERASE_BLOCKS);
   1398		return;
   1399	}
   1400
   1401	spare_cnt = div_u64((uint64_t)eavailable * spare_eblocks, 100);
   1402
   1403	if (spare_cnt < MIN_SPARE_EBLOCKS)
   1404		spare_cnt = MIN_SPARE_EBLOCKS;
   1405
   1406	if (spare_cnt > eavailable - 1)
   1407		spare_cnt = eavailable - 1;
   1408
   1409	swap_size = (uint64_t)(eavailable - spare_cnt) * mtd->erasesize +
   1410		(header ? PAGE_SIZE : 0);
   1411
   1412	printk(KERN_INFO "%s: Enabling MTD swap on device %lu, size %llu KB, "
   1413		"%u spare, %u bad blocks\n",
   1414		MTDSWAP_PREFIX, part, swap_size / 1024, spare_cnt, bad_blocks);
   1415
   1416	d = kzalloc(sizeof(struct mtdswap_dev), GFP_KERNEL);
   1417	if (!d)
   1418		return;
   1419
   1420	mbd_dev = kzalloc(sizeof(struct mtd_blktrans_dev), GFP_KERNEL);
   1421	if (!mbd_dev) {
   1422		kfree(d);
   1423		return;
   1424	}
   1425
   1426	d->mbd_dev = mbd_dev;
   1427	mbd_dev->priv = d;
   1428
   1429	mbd_dev->mtd = mtd;
   1430	mbd_dev->devnum = mtd->index;
   1431	mbd_dev->size = swap_size >> PAGE_SHIFT;
   1432	mbd_dev->tr = tr;
   1433
   1434	if (!(mtd->flags & MTD_WRITEABLE))
   1435		mbd_dev->readonly = 1;
   1436
   1437	if (mtdswap_init(d, eblocks, spare_cnt) < 0)
   1438		goto init_failed;
   1439
   1440	if (add_mtd_blktrans_dev(mbd_dev) < 0)
   1441		goto cleanup;
   1442
   1443	d->dev = disk_to_dev(mbd_dev->disk);
   1444
   1445	ret = mtdswap_add_debugfs(d);
   1446	if (ret < 0)
   1447		goto debugfs_failed;
   1448
   1449	return;
   1450
   1451debugfs_failed:
   1452	del_mtd_blktrans_dev(mbd_dev);
   1453
   1454cleanup:
   1455	mtdswap_cleanup(d);
   1456
   1457init_failed:
   1458	kfree(mbd_dev);
   1459	kfree(d);
   1460}
   1461
   1462static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev)
   1463{
   1464	struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev);
   1465
   1466	del_mtd_blktrans_dev(dev);
   1467	mtdswap_cleanup(d);
   1468	kfree(d);
   1469}
   1470
   1471static struct mtd_blktrans_ops mtdswap_ops = {
   1472	.name		= "mtdswap",
   1473	.major		= 0,
   1474	.part_bits	= 0,
   1475	.blksize	= PAGE_SIZE,
   1476	.flush		= mtdswap_flush,
   1477	.readsect	= mtdswap_readsect,
   1478	.writesect	= mtdswap_writesect,
   1479	.discard	= mtdswap_discard,
   1480	.background	= mtdswap_background,
   1481	.add_mtd	= mtdswap_add_mtd,
   1482	.remove_dev	= mtdswap_remove_dev,
   1483	.owner		= THIS_MODULE,
   1484};
   1485
   1486module_mtd_blktrans(mtdswap_ops);
   1487
   1488MODULE_LICENSE("GPL");
   1489MODULE_AUTHOR("Jarkko Lavinen <jarkko.lavinen@nokia.com>");
   1490MODULE_DESCRIPTION("Block device access to an MTD suitable for using as "
   1491		"swap space");