cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

mtk_nand.c (41838B)


      1// SPDX-License-Identifier: GPL-2.0 OR MIT
      2/*
      3 * MTK NAND Flash controller driver.
      4 * Copyright (C) 2016 MediaTek Inc.
      5 * Authors:	Xiaolei Li		<xiaolei.li@mediatek.com>
      6 *		Jorge Ramirez-Ortiz	<jorge.ramirez-ortiz@linaro.org>
      7 */
      8
      9#include <linux/platform_device.h>
     10#include <linux/dma-mapping.h>
     11#include <linux/interrupt.h>
     12#include <linux/delay.h>
     13#include <linux/clk.h>
     14#include <linux/mtd/rawnand.h>
     15#include <linux/mtd/mtd.h>
     16#include <linux/module.h>
     17#include <linux/iopoll.h>
     18#include <linux/of.h>
     19#include <linux/of_device.h>
     20#include <linux/mtd/nand-ecc-mtk.h>
     21
     22/* NAND controller register definition */
     23#define NFI_CNFG		(0x00)
     24#define		CNFG_AHB		BIT(0)
     25#define		CNFG_READ_EN		BIT(1)
     26#define		CNFG_DMA_BURST_EN	BIT(2)
     27#define		CNFG_BYTE_RW		BIT(6)
     28#define		CNFG_HW_ECC_EN		BIT(8)
     29#define		CNFG_AUTO_FMT_EN	BIT(9)
     30#define		CNFG_OP_CUST		(6 << 12)
     31#define NFI_PAGEFMT		(0x04)
     32#define		PAGEFMT_FDM_ECC_SHIFT	(12)
     33#define		PAGEFMT_FDM_SHIFT	(8)
     34#define		PAGEFMT_SEC_SEL_512	BIT(2)
     35#define		PAGEFMT_512_2K		(0)
     36#define		PAGEFMT_2K_4K		(1)
     37#define		PAGEFMT_4K_8K		(2)
     38#define		PAGEFMT_8K_16K		(3)
     39/* NFI control */
     40#define NFI_CON			(0x08)
     41#define		CON_FIFO_FLUSH		BIT(0)
     42#define		CON_NFI_RST		BIT(1)
     43#define		CON_BRD			BIT(8)  /* burst  read */
     44#define		CON_BWR			BIT(9)	/* burst  write */
     45#define		CON_SEC_SHIFT		(12)
     46/* Timming control register */
     47#define NFI_ACCCON		(0x0C)
     48#define NFI_INTR_EN		(0x10)
     49#define		INTR_AHB_DONE_EN	BIT(6)
     50#define NFI_INTR_STA		(0x14)
     51#define NFI_CMD			(0x20)
     52#define NFI_ADDRNOB		(0x30)
     53#define NFI_COLADDR		(0x34)
     54#define NFI_ROWADDR		(0x38)
     55#define NFI_STRDATA		(0x40)
     56#define		STAR_EN			(1)
     57#define		STAR_DE			(0)
     58#define NFI_CNRNB		(0x44)
     59#define NFI_DATAW		(0x50)
     60#define NFI_DATAR		(0x54)
     61#define NFI_PIO_DIRDY		(0x58)
     62#define		PIO_DI_RDY		(0x01)
     63#define NFI_STA			(0x60)
     64#define		STA_CMD			BIT(0)
     65#define		STA_ADDR		BIT(1)
     66#define		STA_BUSY		BIT(8)
     67#define		STA_EMP_PAGE		BIT(12)
     68#define		NFI_FSM_CUSTDATA	(0xe << 16)
     69#define		NFI_FSM_MASK		(0xf << 16)
     70#define NFI_ADDRCNTR		(0x70)
     71#define		CNTR_MASK		GENMASK(16, 12)
     72#define		ADDRCNTR_SEC_SHIFT	(12)
     73#define		ADDRCNTR_SEC(val) \
     74		(((val) & CNTR_MASK) >> ADDRCNTR_SEC_SHIFT)
     75#define NFI_STRADDR		(0x80)
     76#define NFI_BYTELEN		(0x84)
     77#define NFI_CSEL		(0x90)
     78#define NFI_FDML(x)		(0xA0 + (x) * sizeof(u32) * 2)
     79#define NFI_FDMM(x)		(0xA4 + (x) * sizeof(u32) * 2)
     80#define NFI_FDM_MAX_SIZE	(8)
     81#define NFI_FDM_MIN_SIZE	(1)
     82#define NFI_DEBUG_CON1		(0x220)
     83#define		STROBE_MASK		GENMASK(4, 3)
     84#define		STROBE_SHIFT		(3)
     85#define		MAX_STROBE_DLY		(3)
     86#define NFI_MASTER_STA		(0x224)
     87#define		MASTER_STA_MASK		(0x0FFF)
     88#define NFI_EMPTY_THRESH	(0x23C)
     89
     90#define MTK_NAME		"mtk-nand"
     91#define KB(x)			((x) * 1024UL)
     92#define MB(x)			(KB(x) * 1024UL)
     93
     94#define MTK_TIMEOUT		(500000)
     95#define MTK_RESET_TIMEOUT	(1000000)
     96#define MTK_NAND_MAX_NSELS	(2)
     97#define MTK_NFC_MIN_SPARE	(16)
     98#define ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt) \
     99	((tpoecs) << 28 | (tprecs) << 22 | (tc2r) << 16 | \
    100	(tw2r) << 12 | (twh) << 8 | (twst) << 4 | (trlt))
    101
    102struct mtk_nfc_caps {
    103	const u8 *spare_size;
    104	u8 num_spare_size;
    105	u8 pageformat_spare_shift;
    106	u8 nfi_clk_div;
    107	u8 max_sector;
    108	u32 max_sector_size;
    109};
    110
    111struct mtk_nfc_bad_mark_ctl {
    112	void (*bm_swap)(struct mtd_info *, u8 *buf, int raw);
    113	u32 sec;
    114	u32 pos;
    115};
    116
    117/*
    118 * FDM: region used to store free OOB data
    119 */
    120struct mtk_nfc_fdm {
    121	u32 reg_size;
    122	u32 ecc_size;
    123};
    124
    125struct mtk_nfc_nand_chip {
    126	struct list_head node;
    127	struct nand_chip nand;
    128
    129	struct mtk_nfc_bad_mark_ctl bad_mark;
    130	struct mtk_nfc_fdm fdm;
    131	u32 spare_per_sector;
    132
    133	int nsels;
    134	u8 sels[];
    135	/* nothing after this field */
    136};
    137
    138struct mtk_nfc_clk {
    139	struct clk *nfi_clk;
    140	struct clk *pad_clk;
    141};
    142
    143struct mtk_nfc {
    144	struct nand_controller controller;
    145	struct mtk_ecc_config ecc_cfg;
    146	struct mtk_nfc_clk clk;
    147	struct mtk_ecc *ecc;
    148
    149	struct device *dev;
    150	const struct mtk_nfc_caps *caps;
    151	void __iomem *regs;
    152
    153	struct completion done;
    154	struct list_head chips;
    155
    156	u8 *buffer;
    157
    158	unsigned long assigned_cs;
    159};
    160
    161/*
    162 * supported spare size of each IP.
    163 * order should be the same with the spare size bitfiled defination of
    164 * register NFI_PAGEFMT.
    165 */
    166static const u8 spare_size_mt2701[] = {
    167	16, 26, 27, 28, 32, 36, 40, 44,	48, 49, 50, 51, 52, 62, 63, 64
    168};
    169
    170static const u8 spare_size_mt2712[] = {
    171	16, 26, 27, 28, 32, 36, 40, 44, 48, 49, 50, 51, 52, 62, 61, 63, 64, 67,
    172	74
    173};
    174
    175static const u8 spare_size_mt7622[] = {
    176	16, 26, 27, 28
    177};
    178
    179static inline struct mtk_nfc_nand_chip *to_mtk_nand(struct nand_chip *nand)
    180{
    181	return container_of(nand, struct mtk_nfc_nand_chip, nand);
    182}
    183
    184static inline u8 *data_ptr(struct nand_chip *chip, const u8 *p, int i)
    185{
    186	return (u8 *)p + i * chip->ecc.size;
    187}
    188
    189static inline u8 *oob_ptr(struct nand_chip *chip, int i)
    190{
    191	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    192	u8 *poi;
    193
    194	/* map the sector's FDM data to free oob:
    195	 * the beginning of the oob area stores the FDM data of bad mark sectors
    196	 */
    197
    198	if (i < mtk_nand->bad_mark.sec)
    199		poi = chip->oob_poi + (i + 1) * mtk_nand->fdm.reg_size;
    200	else if (i == mtk_nand->bad_mark.sec)
    201		poi = chip->oob_poi;
    202	else
    203		poi = chip->oob_poi + i * mtk_nand->fdm.reg_size;
    204
    205	return poi;
    206}
    207
    208static inline int mtk_data_len(struct nand_chip *chip)
    209{
    210	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    211
    212	return chip->ecc.size + mtk_nand->spare_per_sector;
    213}
    214
    215static inline u8 *mtk_data_ptr(struct nand_chip *chip,  int i)
    216{
    217	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    218
    219	return nfc->buffer + i * mtk_data_len(chip);
    220}
    221
    222static inline u8 *mtk_oob_ptr(struct nand_chip *chip, int i)
    223{
    224	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    225
    226	return nfc->buffer + i * mtk_data_len(chip) + chip->ecc.size;
    227}
    228
    229static inline void nfi_writel(struct mtk_nfc *nfc, u32 val, u32 reg)
    230{
    231	writel(val, nfc->regs + reg);
    232}
    233
    234static inline void nfi_writew(struct mtk_nfc *nfc, u16 val, u32 reg)
    235{
    236	writew(val, nfc->regs + reg);
    237}
    238
    239static inline void nfi_writeb(struct mtk_nfc *nfc, u8 val, u32 reg)
    240{
    241	writeb(val, nfc->regs + reg);
    242}
    243
    244static inline u32 nfi_readl(struct mtk_nfc *nfc, u32 reg)
    245{
    246	return readl_relaxed(nfc->regs + reg);
    247}
    248
    249static inline u16 nfi_readw(struct mtk_nfc *nfc, u32 reg)
    250{
    251	return readw_relaxed(nfc->regs + reg);
    252}
    253
    254static inline u8 nfi_readb(struct mtk_nfc *nfc, u32 reg)
    255{
    256	return readb_relaxed(nfc->regs + reg);
    257}
    258
    259static void mtk_nfc_hw_reset(struct mtk_nfc *nfc)
    260{
    261	struct device *dev = nfc->dev;
    262	u32 val;
    263	int ret;
    264
    265	/* reset all registers and force the NFI master to terminate */
    266	nfi_writel(nfc, CON_FIFO_FLUSH | CON_NFI_RST, NFI_CON);
    267
    268	/* wait for the master to finish the last transaction */
    269	ret = readl_poll_timeout(nfc->regs + NFI_MASTER_STA, val,
    270				 !(val & MASTER_STA_MASK), 50,
    271				 MTK_RESET_TIMEOUT);
    272	if (ret)
    273		dev_warn(dev, "master active in reset [0x%x] = 0x%x\n",
    274			 NFI_MASTER_STA, val);
    275
    276	/* ensure any status register affected by the NFI master is reset */
    277	nfi_writel(nfc, CON_FIFO_FLUSH | CON_NFI_RST, NFI_CON);
    278	nfi_writew(nfc, STAR_DE, NFI_STRDATA);
    279}
    280
    281static int mtk_nfc_send_command(struct mtk_nfc *nfc, u8 command)
    282{
    283	struct device *dev = nfc->dev;
    284	u32 val;
    285	int ret;
    286
    287	nfi_writel(nfc, command, NFI_CMD);
    288
    289	ret = readl_poll_timeout_atomic(nfc->regs + NFI_STA, val,
    290					!(val & STA_CMD), 10,  MTK_TIMEOUT);
    291	if (ret) {
    292		dev_warn(dev, "nfi core timed out entering command mode\n");
    293		return -EIO;
    294	}
    295
    296	return 0;
    297}
    298
    299static int mtk_nfc_send_address(struct mtk_nfc *nfc, int addr)
    300{
    301	struct device *dev = nfc->dev;
    302	u32 val;
    303	int ret;
    304
    305	nfi_writel(nfc, addr, NFI_COLADDR);
    306	nfi_writel(nfc, 0, NFI_ROWADDR);
    307	nfi_writew(nfc, 1, NFI_ADDRNOB);
    308
    309	ret = readl_poll_timeout_atomic(nfc->regs + NFI_STA, val,
    310					!(val & STA_ADDR), 10, MTK_TIMEOUT);
    311	if (ret) {
    312		dev_warn(dev, "nfi core timed out entering address mode\n");
    313		return -EIO;
    314	}
    315
    316	return 0;
    317}
    318
    319static int mtk_nfc_hw_runtime_config(struct mtd_info *mtd)
    320{
    321	struct nand_chip *chip = mtd_to_nand(mtd);
    322	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    323	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    324	u32 fmt, spare, i;
    325
    326	if (!mtd->writesize)
    327		return 0;
    328
    329	spare = mtk_nand->spare_per_sector;
    330
    331	switch (mtd->writesize) {
    332	case 512:
    333		fmt = PAGEFMT_512_2K | PAGEFMT_SEC_SEL_512;
    334		break;
    335	case KB(2):
    336		if (chip->ecc.size == 512)
    337			fmt = PAGEFMT_2K_4K | PAGEFMT_SEC_SEL_512;
    338		else
    339			fmt = PAGEFMT_512_2K;
    340		break;
    341	case KB(4):
    342		if (chip->ecc.size == 512)
    343			fmt = PAGEFMT_4K_8K | PAGEFMT_SEC_SEL_512;
    344		else
    345			fmt = PAGEFMT_2K_4K;
    346		break;
    347	case KB(8):
    348		if (chip->ecc.size == 512)
    349			fmt = PAGEFMT_8K_16K | PAGEFMT_SEC_SEL_512;
    350		else
    351			fmt = PAGEFMT_4K_8K;
    352		break;
    353	case KB(16):
    354		fmt = PAGEFMT_8K_16K;
    355		break;
    356	default:
    357		dev_err(nfc->dev, "invalid page len: %d\n", mtd->writesize);
    358		return -EINVAL;
    359	}
    360
    361	/*
    362	 * the hardware will double the value for this eccsize, so we need to
    363	 * halve it
    364	 */
    365	if (chip->ecc.size == 1024)
    366		spare >>= 1;
    367
    368	for (i = 0; i < nfc->caps->num_spare_size; i++) {
    369		if (nfc->caps->spare_size[i] == spare)
    370			break;
    371	}
    372
    373	if (i == nfc->caps->num_spare_size) {
    374		dev_err(nfc->dev, "invalid spare size %d\n", spare);
    375		return -EINVAL;
    376	}
    377
    378	fmt |= i << nfc->caps->pageformat_spare_shift;
    379
    380	fmt |= mtk_nand->fdm.reg_size << PAGEFMT_FDM_SHIFT;
    381	fmt |= mtk_nand->fdm.ecc_size << PAGEFMT_FDM_ECC_SHIFT;
    382	nfi_writel(nfc, fmt, NFI_PAGEFMT);
    383
    384	nfc->ecc_cfg.strength = chip->ecc.strength;
    385	nfc->ecc_cfg.len = chip->ecc.size + mtk_nand->fdm.ecc_size;
    386
    387	return 0;
    388}
    389
    390static inline void mtk_nfc_wait_ioready(struct mtk_nfc *nfc)
    391{
    392	int rc;
    393	u8 val;
    394
    395	rc = readb_poll_timeout_atomic(nfc->regs + NFI_PIO_DIRDY, val,
    396				       val & PIO_DI_RDY, 10, MTK_TIMEOUT);
    397	if (rc < 0)
    398		dev_err(nfc->dev, "data not ready\n");
    399}
    400
    401static inline u8 mtk_nfc_read_byte(struct nand_chip *chip)
    402{
    403	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    404	u32 reg;
    405
    406	/* after each byte read, the NFI_STA reg is reset by the hardware */
    407	reg = nfi_readl(nfc, NFI_STA) & NFI_FSM_MASK;
    408	if (reg != NFI_FSM_CUSTDATA) {
    409		reg = nfi_readw(nfc, NFI_CNFG);
    410		reg |= CNFG_BYTE_RW | CNFG_READ_EN;
    411		nfi_writew(nfc, reg, NFI_CNFG);
    412
    413		/*
    414		 * set to max sector to allow the HW to continue reading over
    415		 * unaligned accesses
    416		 */
    417		reg = (nfc->caps->max_sector << CON_SEC_SHIFT) | CON_BRD;
    418		nfi_writel(nfc, reg, NFI_CON);
    419
    420		/* trigger to fetch data */
    421		nfi_writew(nfc, STAR_EN, NFI_STRDATA);
    422	}
    423
    424	mtk_nfc_wait_ioready(nfc);
    425
    426	return nfi_readb(nfc, NFI_DATAR);
    427}
    428
    429static void mtk_nfc_read_buf(struct nand_chip *chip, u8 *buf, int len)
    430{
    431	int i;
    432
    433	for (i = 0; i < len; i++)
    434		buf[i] = mtk_nfc_read_byte(chip);
    435}
    436
    437static void mtk_nfc_write_byte(struct nand_chip *chip, u8 byte)
    438{
    439	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    440	u32 reg;
    441
    442	reg = nfi_readl(nfc, NFI_STA) & NFI_FSM_MASK;
    443
    444	if (reg != NFI_FSM_CUSTDATA) {
    445		reg = nfi_readw(nfc, NFI_CNFG) | CNFG_BYTE_RW;
    446		nfi_writew(nfc, reg, NFI_CNFG);
    447
    448		reg = nfc->caps->max_sector << CON_SEC_SHIFT | CON_BWR;
    449		nfi_writel(nfc, reg, NFI_CON);
    450
    451		nfi_writew(nfc, STAR_EN, NFI_STRDATA);
    452	}
    453
    454	mtk_nfc_wait_ioready(nfc);
    455	nfi_writeb(nfc, byte, NFI_DATAW);
    456}
    457
    458static void mtk_nfc_write_buf(struct nand_chip *chip, const u8 *buf, int len)
    459{
    460	int i;
    461
    462	for (i = 0; i < len; i++)
    463		mtk_nfc_write_byte(chip, buf[i]);
    464}
    465
    466static int mtk_nfc_exec_instr(struct nand_chip *chip,
    467			      const struct nand_op_instr *instr)
    468{
    469	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    470	unsigned int i;
    471	u32 status;
    472
    473	switch (instr->type) {
    474	case NAND_OP_CMD_INSTR:
    475		mtk_nfc_send_command(nfc, instr->ctx.cmd.opcode);
    476		return 0;
    477	case NAND_OP_ADDR_INSTR:
    478		for (i = 0; i < instr->ctx.addr.naddrs; i++)
    479			mtk_nfc_send_address(nfc, instr->ctx.addr.addrs[i]);
    480		return 0;
    481	case NAND_OP_DATA_IN_INSTR:
    482		mtk_nfc_read_buf(chip, instr->ctx.data.buf.in,
    483				 instr->ctx.data.len);
    484		return 0;
    485	case NAND_OP_DATA_OUT_INSTR:
    486		mtk_nfc_write_buf(chip, instr->ctx.data.buf.out,
    487				  instr->ctx.data.len);
    488		return 0;
    489	case NAND_OP_WAITRDY_INSTR:
    490		return readl_poll_timeout(nfc->regs + NFI_STA, status,
    491					  !(status & STA_BUSY), 20,
    492					  instr->ctx.waitrdy.timeout_ms * 1000);
    493	default:
    494		break;
    495	}
    496
    497	return -EINVAL;
    498}
    499
    500static void mtk_nfc_select_target(struct nand_chip *nand, unsigned int cs)
    501{
    502	struct mtk_nfc *nfc = nand_get_controller_data(nand);
    503	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(nand);
    504
    505	mtk_nfc_hw_runtime_config(nand_to_mtd(nand));
    506
    507	nfi_writel(nfc, mtk_nand->sels[cs], NFI_CSEL);
    508}
    509
    510static int mtk_nfc_exec_op(struct nand_chip *chip,
    511			   const struct nand_operation *op,
    512			   bool check_only)
    513{
    514	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    515	unsigned int i;
    516	int ret = 0;
    517
    518	if (check_only)
    519		return 0;
    520
    521	mtk_nfc_hw_reset(nfc);
    522	nfi_writew(nfc, CNFG_OP_CUST, NFI_CNFG);
    523	mtk_nfc_select_target(chip, op->cs);
    524
    525	for (i = 0; i < op->ninstrs; i++) {
    526		ret = mtk_nfc_exec_instr(chip, &op->instrs[i]);
    527		if (ret)
    528			break;
    529	}
    530
    531	return ret;
    532}
    533
    534static int mtk_nfc_setup_interface(struct nand_chip *chip, int csline,
    535				   const struct nand_interface_config *conf)
    536{
    537	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    538	const struct nand_sdr_timings *timings;
    539	u32 rate, tpoecs, tprecs, tc2r, tw2r, twh, twst = 0, trlt = 0;
    540	u32 temp, tsel = 0;
    541
    542	timings = nand_get_sdr_timings(conf);
    543	if (IS_ERR(timings))
    544		return -ENOTSUPP;
    545
    546	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
    547		return 0;
    548
    549	rate = clk_get_rate(nfc->clk.nfi_clk);
    550	/* There is a frequency divider in some IPs */
    551	rate /= nfc->caps->nfi_clk_div;
    552
    553	/* turn clock rate into KHZ */
    554	rate /= 1000;
    555
    556	tpoecs = max(timings->tALH_min, timings->tCLH_min) / 1000;
    557	tpoecs = DIV_ROUND_UP(tpoecs * rate, 1000000);
    558	tpoecs &= 0xf;
    559
    560	tprecs = max(timings->tCLS_min, timings->tALS_min) / 1000;
    561	tprecs = DIV_ROUND_UP(tprecs * rate, 1000000);
    562	tprecs &= 0x3f;
    563
    564	/* sdr interface has no tCR which means CE# low to RE# low */
    565	tc2r = 0;
    566
    567	tw2r = timings->tWHR_min / 1000;
    568	tw2r = DIV_ROUND_UP(tw2r * rate, 1000000);
    569	tw2r = DIV_ROUND_UP(tw2r - 1, 2);
    570	tw2r &= 0xf;
    571
    572	twh = max(timings->tREH_min, timings->tWH_min) / 1000;
    573	twh = DIV_ROUND_UP(twh * rate, 1000000) - 1;
    574	twh &= 0xf;
    575
    576	/* Calculate real WE#/RE# hold time in nanosecond */
    577	temp = (twh + 1) * 1000000 / rate;
    578	/* nanosecond to picosecond */
    579	temp *= 1000;
    580
    581	/*
    582	 * WE# low level time should be expaned to meet WE# pulse time
    583	 * and WE# cycle time at the same time.
    584	 */
    585	if (temp < timings->tWC_min)
    586		twst = timings->tWC_min - temp;
    587	twst = max(timings->tWP_min, twst) / 1000;
    588	twst = DIV_ROUND_UP(twst * rate, 1000000) - 1;
    589	twst &= 0xf;
    590
    591	/*
    592	 * RE# low level time should be expaned to meet RE# pulse time
    593	 * and RE# cycle time at the same time.
    594	 */
    595	if (temp < timings->tRC_min)
    596		trlt = timings->tRC_min - temp;
    597	trlt = max(trlt, timings->tRP_min) / 1000;
    598	trlt = DIV_ROUND_UP(trlt * rate, 1000000) - 1;
    599	trlt &= 0xf;
    600
    601	/* Calculate RE# pulse time in nanosecond. */
    602	temp = (trlt + 1) * 1000000 / rate;
    603	/* nanosecond to picosecond */
    604	temp *= 1000;
    605	/*
    606	 * If RE# access time is bigger than RE# pulse time,
    607	 * delay sampling data timing.
    608	 */
    609	if (temp < timings->tREA_max) {
    610		tsel = timings->tREA_max / 1000;
    611		tsel = DIV_ROUND_UP(tsel * rate, 1000000);
    612		tsel -= (trlt + 1);
    613		if (tsel > MAX_STROBE_DLY) {
    614			trlt += tsel - MAX_STROBE_DLY;
    615			tsel = MAX_STROBE_DLY;
    616		}
    617	}
    618	temp = nfi_readl(nfc, NFI_DEBUG_CON1);
    619	temp &= ~STROBE_MASK;
    620	temp |= tsel << STROBE_SHIFT;
    621	nfi_writel(nfc, temp, NFI_DEBUG_CON1);
    622
    623	/*
    624	 * ACCON: access timing control register
    625	 * -------------------------------------
    626	 * 31:28: tpoecs, minimum required time for CS post pulling down after
    627	 *        accessing the device
    628	 * 27:22: tprecs, minimum required time for CS pre pulling down before
    629	 *        accessing the device
    630	 * 21:16: tc2r, minimum required time from NCEB low to NREB low
    631	 * 15:12: tw2r, minimum required time from NWEB high to NREB low.
    632	 * 11:08: twh, write enable hold time
    633	 * 07:04: twst, write wait states
    634	 * 03:00: trlt, read wait states
    635	 */
    636	trlt = ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt);
    637	nfi_writel(nfc, trlt, NFI_ACCCON);
    638
    639	return 0;
    640}
    641
    642static int mtk_nfc_sector_encode(struct nand_chip *chip, u8 *data)
    643{
    644	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    645	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    646	int size = chip->ecc.size + mtk_nand->fdm.reg_size;
    647
    648	nfc->ecc_cfg.mode = ECC_DMA_MODE;
    649	nfc->ecc_cfg.op = ECC_ENCODE;
    650
    651	return mtk_ecc_encode(nfc->ecc, &nfc->ecc_cfg, data, size);
    652}
    653
    654static void mtk_nfc_no_bad_mark_swap(struct mtd_info *a, u8 *b, int c)
    655{
    656	/* nop */
    657}
    658
    659static void mtk_nfc_bad_mark_swap(struct mtd_info *mtd, u8 *buf, int raw)
    660{
    661	struct nand_chip *chip = mtd_to_nand(mtd);
    662	struct mtk_nfc_nand_chip *nand = to_mtk_nand(chip);
    663	u32 bad_pos = nand->bad_mark.pos;
    664
    665	if (raw)
    666		bad_pos += nand->bad_mark.sec * mtk_data_len(chip);
    667	else
    668		bad_pos += nand->bad_mark.sec * chip->ecc.size;
    669
    670	swap(chip->oob_poi[0], buf[bad_pos]);
    671}
    672
    673static int mtk_nfc_format_subpage(struct mtd_info *mtd, u32 offset,
    674				  u32 len, const u8 *buf)
    675{
    676	struct nand_chip *chip = mtd_to_nand(mtd);
    677	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    678	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    679	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
    680	u32 start, end;
    681	int i, ret;
    682
    683	start = offset / chip->ecc.size;
    684	end = DIV_ROUND_UP(offset + len, chip->ecc.size);
    685
    686	memset(nfc->buffer, 0xff, mtd->writesize + mtd->oobsize);
    687	for (i = 0; i < chip->ecc.steps; i++) {
    688		memcpy(mtk_data_ptr(chip, i), data_ptr(chip, buf, i),
    689		       chip->ecc.size);
    690
    691		if (start > i || i >= end)
    692			continue;
    693
    694		if (i == mtk_nand->bad_mark.sec)
    695			mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, 1);
    696
    697		memcpy(mtk_oob_ptr(chip, i), oob_ptr(chip, i), fdm->reg_size);
    698
    699		/* program the CRC back to the OOB */
    700		ret = mtk_nfc_sector_encode(chip, mtk_data_ptr(chip, i));
    701		if (ret < 0)
    702			return ret;
    703	}
    704
    705	return 0;
    706}
    707
    708static void mtk_nfc_format_page(struct mtd_info *mtd, const u8 *buf)
    709{
    710	struct nand_chip *chip = mtd_to_nand(mtd);
    711	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    712	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    713	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
    714	u32 i;
    715
    716	memset(nfc->buffer, 0xff, mtd->writesize + mtd->oobsize);
    717	for (i = 0; i < chip->ecc.steps; i++) {
    718		if (buf)
    719			memcpy(mtk_data_ptr(chip, i), data_ptr(chip, buf, i),
    720			       chip->ecc.size);
    721
    722		if (i == mtk_nand->bad_mark.sec)
    723			mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, 1);
    724
    725		memcpy(mtk_oob_ptr(chip, i), oob_ptr(chip, i), fdm->reg_size);
    726	}
    727}
    728
    729static inline void mtk_nfc_read_fdm(struct nand_chip *chip, u32 start,
    730				    u32 sectors)
    731{
    732	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    733	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    734	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
    735	u32 vall, valm;
    736	u8 *oobptr;
    737	int i, j;
    738
    739	for (i = 0; i < sectors; i++) {
    740		oobptr = oob_ptr(chip, start + i);
    741		vall = nfi_readl(nfc, NFI_FDML(i));
    742		valm = nfi_readl(nfc, NFI_FDMM(i));
    743
    744		for (j = 0; j < fdm->reg_size; j++)
    745			oobptr[j] = (j >= 4 ? valm : vall) >> ((j % 4) * 8);
    746	}
    747}
    748
    749static inline void mtk_nfc_write_fdm(struct nand_chip *chip)
    750{
    751	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    752	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    753	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
    754	u32 vall, valm;
    755	u8 *oobptr;
    756	int i, j;
    757
    758	for (i = 0; i < chip->ecc.steps; i++) {
    759		oobptr = oob_ptr(chip, i);
    760		vall = 0;
    761		valm = 0;
    762		for (j = 0; j < 8; j++) {
    763			if (j < 4)
    764				vall |= (j < fdm->reg_size ? oobptr[j] : 0xff)
    765						<< (j * 8);
    766			else
    767				valm |= (j < fdm->reg_size ? oobptr[j] : 0xff)
    768						<< ((j - 4) * 8);
    769		}
    770		nfi_writel(nfc, vall, NFI_FDML(i));
    771		nfi_writel(nfc, valm, NFI_FDMM(i));
    772	}
    773}
    774
    775static int mtk_nfc_do_write_page(struct mtd_info *mtd, struct nand_chip *chip,
    776				 const u8 *buf, int page, int len)
    777{
    778	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    779	struct device *dev = nfc->dev;
    780	dma_addr_t addr;
    781	u32 reg;
    782	int ret;
    783
    784	addr = dma_map_single(dev, (void *)buf, len, DMA_TO_DEVICE);
    785	ret = dma_mapping_error(nfc->dev, addr);
    786	if (ret) {
    787		dev_err(nfc->dev, "dma mapping error\n");
    788		return -EINVAL;
    789	}
    790
    791	reg = nfi_readw(nfc, NFI_CNFG) | CNFG_AHB | CNFG_DMA_BURST_EN;
    792	nfi_writew(nfc, reg, NFI_CNFG);
    793
    794	nfi_writel(nfc, chip->ecc.steps << CON_SEC_SHIFT, NFI_CON);
    795	nfi_writel(nfc, lower_32_bits(addr), NFI_STRADDR);
    796	nfi_writew(nfc, INTR_AHB_DONE_EN, NFI_INTR_EN);
    797
    798	init_completion(&nfc->done);
    799
    800	reg = nfi_readl(nfc, NFI_CON) | CON_BWR;
    801	nfi_writel(nfc, reg, NFI_CON);
    802	nfi_writew(nfc, STAR_EN, NFI_STRDATA);
    803
    804	ret = wait_for_completion_timeout(&nfc->done, msecs_to_jiffies(500));
    805	if (!ret) {
    806		dev_err(dev, "program ahb done timeout\n");
    807		nfi_writew(nfc, 0, NFI_INTR_EN);
    808		ret = -ETIMEDOUT;
    809		goto timeout;
    810	}
    811
    812	ret = readl_poll_timeout_atomic(nfc->regs + NFI_ADDRCNTR, reg,
    813					ADDRCNTR_SEC(reg) >= chip->ecc.steps,
    814					10, MTK_TIMEOUT);
    815	if (ret)
    816		dev_err(dev, "hwecc write timeout\n");
    817
    818timeout:
    819
    820	dma_unmap_single(nfc->dev, addr, len, DMA_TO_DEVICE);
    821	nfi_writel(nfc, 0, NFI_CON);
    822
    823	return ret;
    824}
    825
    826static int mtk_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
    827			      const u8 *buf, int page, int raw)
    828{
    829	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    830	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    831	size_t len;
    832	const u8 *bufpoi;
    833	u32 reg;
    834	int ret;
    835
    836	mtk_nfc_select_target(chip, chip->cur_cs);
    837	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
    838
    839	if (!raw) {
    840		/* OOB => FDM: from register,  ECC: from HW */
    841		reg = nfi_readw(nfc, NFI_CNFG) | CNFG_AUTO_FMT_EN;
    842		nfi_writew(nfc, reg | CNFG_HW_ECC_EN, NFI_CNFG);
    843
    844		nfc->ecc_cfg.op = ECC_ENCODE;
    845		nfc->ecc_cfg.mode = ECC_NFI_MODE;
    846		ret = mtk_ecc_enable(nfc->ecc, &nfc->ecc_cfg);
    847		if (ret) {
    848			/* clear NFI config */
    849			reg = nfi_readw(nfc, NFI_CNFG);
    850			reg &= ~(CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN);
    851			nfi_writew(nfc, reg, NFI_CNFG);
    852
    853			return ret;
    854		}
    855
    856		memcpy(nfc->buffer, buf, mtd->writesize);
    857		mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, raw);
    858		bufpoi = nfc->buffer;
    859
    860		/* write OOB into the FDM registers (OOB area in MTK NAND) */
    861		mtk_nfc_write_fdm(chip);
    862	} else {
    863		bufpoi = buf;
    864	}
    865
    866	len = mtd->writesize + (raw ? mtd->oobsize : 0);
    867	ret = mtk_nfc_do_write_page(mtd, chip, bufpoi, page, len);
    868
    869	if (!raw)
    870		mtk_ecc_disable(nfc->ecc);
    871
    872	if (ret)
    873		return ret;
    874
    875	return nand_prog_page_end_op(chip);
    876}
    877
    878static int mtk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf,
    879				    int oob_on, int page)
    880{
    881	return mtk_nfc_write_page(nand_to_mtd(chip), chip, buf, page, 0);
    882}
    883
    884static int mtk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf,
    885				  int oob_on, int pg)
    886{
    887	struct mtd_info *mtd = nand_to_mtd(chip);
    888	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    889
    890	mtk_nfc_format_page(mtd, buf);
    891	return mtk_nfc_write_page(mtd, chip, nfc->buffer, pg, 1);
    892}
    893
    894static int mtk_nfc_write_subpage_hwecc(struct nand_chip *chip, u32 offset,
    895				       u32 data_len, const u8 *buf,
    896				       int oob_on, int page)
    897{
    898	struct mtd_info *mtd = nand_to_mtd(chip);
    899	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    900	int ret;
    901
    902	ret = mtk_nfc_format_subpage(mtd, offset, data_len, buf);
    903	if (ret < 0)
    904		return ret;
    905
    906	/* use the data in the private buffer (now with FDM and CRC) */
    907	return mtk_nfc_write_page(mtd, chip, nfc->buffer, page, 1);
    908}
    909
    910static int mtk_nfc_write_oob_std(struct nand_chip *chip, int page)
    911{
    912	return mtk_nfc_write_page_raw(chip, NULL, 1, page);
    913}
    914
    915static int mtk_nfc_update_ecc_stats(struct mtd_info *mtd, u8 *buf, u32 start,
    916				    u32 sectors)
    917{
    918	struct nand_chip *chip = mtd_to_nand(mtd);
    919	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    920	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    921	struct mtk_ecc_stats stats;
    922	u32 reg_size = mtk_nand->fdm.reg_size;
    923	int rc, i;
    924
    925	rc = nfi_readl(nfc, NFI_STA) & STA_EMP_PAGE;
    926	if (rc) {
    927		memset(buf, 0xff, sectors * chip->ecc.size);
    928		for (i = 0; i < sectors; i++)
    929			memset(oob_ptr(chip, start + i), 0xff, reg_size);
    930		return 0;
    931	}
    932
    933	mtk_ecc_get_stats(nfc->ecc, &stats, sectors);
    934	mtd->ecc_stats.corrected += stats.corrected;
    935	mtd->ecc_stats.failed += stats.failed;
    936
    937	return stats.bitflips;
    938}
    939
    940static int mtk_nfc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
    941				u32 data_offs, u32 readlen,
    942				u8 *bufpoi, int page, int raw)
    943{
    944	struct mtk_nfc *nfc = nand_get_controller_data(chip);
    945	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
    946	u32 spare = mtk_nand->spare_per_sector;
    947	u32 column, sectors, start, end, reg;
    948	dma_addr_t addr;
    949	int bitflips = 0;
    950	size_t len;
    951	u8 *buf;
    952	int rc;
    953
    954	mtk_nfc_select_target(chip, chip->cur_cs);
    955	start = data_offs / chip->ecc.size;
    956	end = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
    957
    958	sectors = end - start;
    959	column = start * (chip->ecc.size + spare);
    960
    961	len = sectors * chip->ecc.size + (raw ? sectors * spare : 0);
    962	buf = bufpoi + start * chip->ecc.size;
    963
    964	nand_read_page_op(chip, page, column, NULL, 0);
    965
    966	addr = dma_map_single(nfc->dev, buf, len, DMA_FROM_DEVICE);
    967	rc = dma_mapping_error(nfc->dev, addr);
    968	if (rc) {
    969		dev_err(nfc->dev, "dma mapping error\n");
    970
    971		return -EINVAL;
    972	}
    973
    974	reg = nfi_readw(nfc, NFI_CNFG);
    975	reg |= CNFG_READ_EN | CNFG_DMA_BURST_EN | CNFG_AHB;
    976	if (!raw) {
    977		reg |= CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN;
    978		nfi_writew(nfc, reg, NFI_CNFG);
    979
    980		nfc->ecc_cfg.mode = ECC_NFI_MODE;
    981		nfc->ecc_cfg.sectors = sectors;
    982		nfc->ecc_cfg.op = ECC_DECODE;
    983		rc = mtk_ecc_enable(nfc->ecc, &nfc->ecc_cfg);
    984		if (rc) {
    985			dev_err(nfc->dev, "ecc enable\n");
    986			/* clear NFI_CNFG */
    987			reg &= ~(CNFG_DMA_BURST_EN | CNFG_AHB | CNFG_READ_EN |
    988				CNFG_AUTO_FMT_EN | CNFG_HW_ECC_EN);
    989			nfi_writew(nfc, reg, NFI_CNFG);
    990			dma_unmap_single(nfc->dev, addr, len, DMA_FROM_DEVICE);
    991
    992			return rc;
    993		}
    994	} else {
    995		nfi_writew(nfc, reg, NFI_CNFG);
    996	}
    997
    998	nfi_writel(nfc, sectors << CON_SEC_SHIFT, NFI_CON);
    999	nfi_writew(nfc, INTR_AHB_DONE_EN, NFI_INTR_EN);
   1000	nfi_writel(nfc, lower_32_bits(addr), NFI_STRADDR);
   1001
   1002	init_completion(&nfc->done);
   1003	reg = nfi_readl(nfc, NFI_CON) | CON_BRD;
   1004	nfi_writel(nfc, reg, NFI_CON);
   1005	nfi_writew(nfc, STAR_EN, NFI_STRDATA);
   1006
   1007	rc = wait_for_completion_timeout(&nfc->done, msecs_to_jiffies(500));
   1008	if (!rc)
   1009		dev_warn(nfc->dev, "read ahb/dma done timeout\n");
   1010
   1011	rc = readl_poll_timeout_atomic(nfc->regs + NFI_BYTELEN, reg,
   1012				       ADDRCNTR_SEC(reg) >= sectors, 10,
   1013				       MTK_TIMEOUT);
   1014	if (rc < 0) {
   1015		dev_err(nfc->dev, "subpage done timeout\n");
   1016		bitflips = -EIO;
   1017	} else if (!raw) {
   1018		rc = mtk_ecc_wait_done(nfc->ecc, ECC_DECODE);
   1019		bitflips = rc < 0 ? -ETIMEDOUT :
   1020			mtk_nfc_update_ecc_stats(mtd, buf, start, sectors);
   1021		mtk_nfc_read_fdm(chip, start, sectors);
   1022	}
   1023
   1024	dma_unmap_single(nfc->dev, addr, len, DMA_FROM_DEVICE);
   1025
   1026	if (raw)
   1027		goto done;
   1028
   1029	mtk_ecc_disable(nfc->ecc);
   1030
   1031	if (clamp(mtk_nand->bad_mark.sec, start, end) == mtk_nand->bad_mark.sec)
   1032		mtk_nand->bad_mark.bm_swap(mtd, bufpoi, raw);
   1033done:
   1034	nfi_writel(nfc, 0, NFI_CON);
   1035
   1036	return bitflips;
   1037}
   1038
   1039static int mtk_nfc_read_subpage_hwecc(struct nand_chip *chip, u32 off,
   1040				      u32 len, u8 *p, int pg)
   1041{
   1042	return mtk_nfc_read_subpage(nand_to_mtd(chip), chip, off, len, p, pg,
   1043				    0);
   1044}
   1045
   1046static int mtk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *p, int oob_on,
   1047				   int pg)
   1048{
   1049	struct mtd_info *mtd = nand_to_mtd(chip);
   1050
   1051	return mtk_nfc_read_subpage(mtd, chip, 0, mtd->writesize, p, pg, 0);
   1052}
   1053
   1054static int mtk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on,
   1055				 int page)
   1056{
   1057	struct mtd_info *mtd = nand_to_mtd(chip);
   1058	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
   1059	struct mtk_nfc *nfc = nand_get_controller_data(chip);
   1060	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
   1061	int i, ret;
   1062
   1063	memset(nfc->buffer, 0xff, mtd->writesize + mtd->oobsize);
   1064	ret = mtk_nfc_read_subpage(mtd, chip, 0, mtd->writesize, nfc->buffer,
   1065				   page, 1);
   1066	if (ret < 0)
   1067		return ret;
   1068
   1069	for (i = 0; i < chip->ecc.steps; i++) {
   1070		memcpy(oob_ptr(chip, i), mtk_oob_ptr(chip, i), fdm->reg_size);
   1071
   1072		if (i == mtk_nand->bad_mark.sec)
   1073			mtk_nand->bad_mark.bm_swap(mtd, nfc->buffer, 1);
   1074
   1075		if (buf)
   1076			memcpy(data_ptr(chip, buf, i), mtk_data_ptr(chip, i),
   1077			       chip->ecc.size);
   1078	}
   1079
   1080	return ret;
   1081}
   1082
   1083static int mtk_nfc_read_oob_std(struct nand_chip *chip, int page)
   1084{
   1085	return mtk_nfc_read_page_raw(chip, NULL, 1, page);
   1086}
   1087
   1088static inline void mtk_nfc_hw_init(struct mtk_nfc *nfc)
   1089{
   1090	/*
   1091	 * CNRNB: nand ready/busy register
   1092	 * -------------------------------
   1093	 * 7:4: timeout register for polling the NAND busy/ready signal
   1094	 * 0  : poll the status of the busy/ready signal after [7:4]*16 cycles.
   1095	 */
   1096	nfi_writew(nfc, 0xf1, NFI_CNRNB);
   1097	nfi_writel(nfc, PAGEFMT_8K_16K, NFI_PAGEFMT);
   1098
   1099	mtk_nfc_hw_reset(nfc);
   1100
   1101	nfi_readl(nfc, NFI_INTR_STA);
   1102	nfi_writel(nfc, 0, NFI_INTR_EN);
   1103}
   1104
   1105static irqreturn_t mtk_nfc_irq(int irq, void *id)
   1106{
   1107	struct mtk_nfc *nfc = id;
   1108	u16 sta, ien;
   1109
   1110	sta = nfi_readw(nfc, NFI_INTR_STA);
   1111	ien = nfi_readw(nfc, NFI_INTR_EN);
   1112
   1113	if (!(sta & ien))
   1114		return IRQ_NONE;
   1115
   1116	nfi_writew(nfc, ~sta & ien, NFI_INTR_EN);
   1117	complete(&nfc->done);
   1118
   1119	return IRQ_HANDLED;
   1120}
   1121
   1122static int mtk_nfc_enable_clk(struct device *dev, struct mtk_nfc_clk *clk)
   1123{
   1124	int ret;
   1125
   1126	ret = clk_prepare_enable(clk->nfi_clk);
   1127	if (ret) {
   1128		dev_err(dev, "failed to enable nfi clk\n");
   1129		return ret;
   1130	}
   1131
   1132	ret = clk_prepare_enable(clk->pad_clk);
   1133	if (ret) {
   1134		dev_err(dev, "failed to enable pad clk\n");
   1135		clk_disable_unprepare(clk->nfi_clk);
   1136		return ret;
   1137	}
   1138
   1139	return 0;
   1140}
   1141
   1142static void mtk_nfc_disable_clk(struct mtk_nfc_clk *clk)
   1143{
   1144	clk_disable_unprepare(clk->nfi_clk);
   1145	clk_disable_unprepare(clk->pad_clk);
   1146}
   1147
   1148static int mtk_nfc_ooblayout_free(struct mtd_info *mtd, int section,
   1149				  struct mtd_oob_region *oob_region)
   1150{
   1151	struct nand_chip *chip = mtd_to_nand(mtd);
   1152	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
   1153	struct mtk_nfc_fdm *fdm = &mtk_nand->fdm;
   1154	u32 eccsteps;
   1155
   1156	eccsteps = mtd->writesize / chip->ecc.size;
   1157
   1158	if (section >= eccsteps)
   1159		return -ERANGE;
   1160
   1161	oob_region->length = fdm->reg_size - fdm->ecc_size;
   1162	oob_region->offset = section * fdm->reg_size + fdm->ecc_size;
   1163
   1164	return 0;
   1165}
   1166
   1167static int mtk_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
   1168				 struct mtd_oob_region *oob_region)
   1169{
   1170	struct nand_chip *chip = mtd_to_nand(mtd);
   1171	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
   1172	u32 eccsteps;
   1173
   1174	if (section)
   1175		return -ERANGE;
   1176
   1177	eccsteps = mtd->writesize / chip->ecc.size;
   1178	oob_region->offset = mtk_nand->fdm.reg_size * eccsteps;
   1179	oob_region->length = mtd->oobsize - oob_region->offset;
   1180
   1181	return 0;
   1182}
   1183
   1184static const struct mtd_ooblayout_ops mtk_nfc_ooblayout_ops = {
   1185	.free = mtk_nfc_ooblayout_free,
   1186	.ecc = mtk_nfc_ooblayout_ecc,
   1187};
   1188
   1189static void mtk_nfc_set_fdm(struct mtk_nfc_fdm *fdm, struct mtd_info *mtd)
   1190{
   1191	struct nand_chip *nand = mtd_to_nand(mtd);
   1192	struct mtk_nfc_nand_chip *chip = to_mtk_nand(nand);
   1193	struct mtk_nfc *nfc = nand_get_controller_data(nand);
   1194	u32 ecc_bytes;
   1195
   1196	ecc_bytes = DIV_ROUND_UP(nand->ecc.strength *
   1197				 mtk_ecc_get_parity_bits(nfc->ecc), 8);
   1198
   1199	fdm->reg_size = chip->spare_per_sector - ecc_bytes;
   1200	if (fdm->reg_size > NFI_FDM_MAX_SIZE)
   1201		fdm->reg_size = NFI_FDM_MAX_SIZE;
   1202
   1203	/* bad block mark storage */
   1204	fdm->ecc_size = 1;
   1205}
   1206
   1207static void mtk_nfc_set_bad_mark_ctl(struct mtk_nfc_bad_mark_ctl *bm_ctl,
   1208				     struct mtd_info *mtd)
   1209{
   1210	struct nand_chip *nand = mtd_to_nand(mtd);
   1211
   1212	if (mtd->writesize == 512) {
   1213		bm_ctl->bm_swap = mtk_nfc_no_bad_mark_swap;
   1214	} else {
   1215		bm_ctl->bm_swap = mtk_nfc_bad_mark_swap;
   1216		bm_ctl->sec = mtd->writesize / mtk_data_len(nand);
   1217		bm_ctl->pos = mtd->writesize % mtk_data_len(nand);
   1218	}
   1219}
   1220
   1221static int mtk_nfc_set_spare_per_sector(u32 *sps, struct mtd_info *mtd)
   1222{
   1223	struct nand_chip *nand = mtd_to_nand(mtd);
   1224	struct mtk_nfc *nfc = nand_get_controller_data(nand);
   1225	const u8 *spare = nfc->caps->spare_size;
   1226	u32 eccsteps, i, closest_spare = 0;
   1227
   1228	eccsteps = mtd->writesize / nand->ecc.size;
   1229	*sps = mtd->oobsize / eccsteps;
   1230
   1231	if (nand->ecc.size == 1024)
   1232		*sps >>= 1;
   1233
   1234	if (*sps < MTK_NFC_MIN_SPARE)
   1235		return -EINVAL;
   1236
   1237	for (i = 0; i < nfc->caps->num_spare_size; i++) {
   1238		if (*sps >= spare[i] && spare[i] >= spare[closest_spare]) {
   1239			closest_spare = i;
   1240			if (*sps == spare[i])
   1241				break;
   1242		}
   1243	}
   1244
   1245	*sps = spare[closest_spare];
   1246
   1247	if (nand->ecc.size == 1024)
   1248		*sps <<= 1;
   1249
   1250	return 0;
   1251}
   1252
   1253static int mtk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
   1254{
   1255	struct nand_chip *nand = mtd_to_nand(mtd);
   1256	const struct nand_ecc_props *requirements =
   1257		nanddev_get_ecc_requirements(&nand->base);
   1258	struct mtk_nfc *nfc = nand_get_controller_data(nand);
   1259	u32 spare;
   1260	int free, ret;
   1261
   1262	/* support only ecc hw mode */
   1263	if (nand->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST) {
   1264		dev_err(dev, "ecc.engine_type not supported\n");
   1265		return -EINVAL;
   1266	}
   1267
   1268	/* if optional dt settings not present */
   1269	if (!nand->ecc.size || !nand->ecc.strength) {
   1270		/* use datasheet requirements */
   1271		nand->ecc.strength = requirements->strength;
   1272		nand->ecc.size = requirements->step_size;
   1273
   1274		/*
   1275		 * align eccstrength and eccsize
   1276		 * this controller only supports 512 and 1024 sizes
   1277		 */
   1278		if (nand->ecc.size < 1024) {
   1279			if (mtd->writesize > 512 &&
   1280			    nfc->caps->max_sector_size > 512) {
   1281				nand->ecc.size = 1024;
   1282				nand->ecc.strength <<= 1;
   1283			} else {
   1284				nand->ecc.size = 512;
   1285			}
   1286		} else {
   1287			nand->ecc.size = 1024;
   1288		}
   1289
   1290		ret = mtk_nfc_set_spare_per_sector(&spare, mtd);
   1291		if (ret)
   1292			return ret;
   1293
   1294		/* calculate oob bytes except ecc parity data */
   1295		free = (nand->ecc.strength * mtk_ecc_get_parity_bits(nfc->ecc)
   1296			+ 7) >> 3;
   1297		free = spare - free;
   1298
   1299		/*
   1300		 * enhance ecc strength if oob left is bigger than max FDM size
   1301		 * or reduce ecc strength if oob size is not enough for ecc
   1302		 * parity data.
   1303		 */
   1304		if (free > NFI_FDM_MAX_SIZE) {
   1305			spare -= NFI_FDM_MAX_SIZE;
   1306			nand->ecc.strength = (spare << 3) /
   1307					     mtk_ecc_get_parity_bits(nfc->ecc);
   1308		} else if (free < 0) {
   1309			spare -= NFI_FDM_MIN_SIZE;
   1310			nand->ecc.strength = (spare << 3) /
   1311					     mtk_ecc_get_parity_bits(nfc->ecc);
   1312		}
   1313	}
   1314
   1315	mtk_ecc_adjust_strength(nfc->ecc, &nand->ecc.strength);
   1316
   1317	dev_info(dev, "eccsize %d eccstrength %d\n",
   1318		 nand->ecc.size, nand->ecc.strength);
   1319
   1320	return 0;
   1321}
   1322
   1323static int mtk_nfc_attach_chip(struct nand_chip *chip)
   1324{
   1325	struct mtd_info *mtd = nand_to_mtd(chip);
   1326	struct device *dev = mtd->dev.parent;
   1327	struct mtk_nfc *nfc = nand_get_controller_data(chip);
   1328	struct mtk_nfc_nand_chip *mtk_nand = to_mtk_nand(chip);
   1329	int len;
   1330	int ret;
   1331
   1332	if (chip->options & NAND_BUSWIDTH_16) {
   1333		dev_err(dev, "16bits buswidth not supported");
   1334		return -EINVAL;
   1335	}
   1336
   1337	/* store bbt magic in page, cause OOB is not protected */
   1338	if (chip->bbt_options & NAND_BBT_USE_FLASH)
   1339		chip->bbt_options |= NAND_BBT_NO_OOB;
   1340
   1341	ret = mtk_nfc_ecc_init(dev, mtd);
   1342	if (ret)
   1343		return ret;
   1344
   1345	ret = mtk_nfc_set_spare_per_sector(&mtk_nand->spare_per_sector, mtd);
   1346	if (ret)
   1347		return ret;
   1348
   1349	mtk_nfc_set_fdm(&mtk_nand->fdm, mtd);
   1350	mtk_nfc_set_bad_mark_ctl(&mtk_nand->bad_mark, mtd);
   1351
   1352	len = mtd->writesize + mtd->oobsize;
   1353	nfc->buffer = devm_kzalloc(dev, len, GFP_KERNEL);
   1354	if (!nfc->buffer)
   1355		return  -ENOMEM;
   1356
   1357	return 0;
   1358}
   1359
   1360static const struct nand_controller_ops mtk_nfc_controller_ops = {
   1361	.attach_chip = mtk_nfc_attach_chip,
   1362	.setup_interface = mtk_nfc_setup_interface,
   1363	.exec_op = mtk_nfc_exec_op,
   1364};
   1365
   1366static int mtk_nfc_nand_chip_init(struct device *dev, struct mtk_nfc *nfc,
   1367				  struct device_node *np)
   1368{
   1369	struct mtk_nfc_nand_chip *chip;
   1370	struct nand_chip *nand;
   1371	struct mtd_info *mtd;
   1372	int nsels;
   1373	u32 tmp;
   1374	int ret;
   1375	int i;
   1376
   1377	if (!of_get_property(np, "reg", &nsels))
   1378		return -ENODEV;
   1379
   1380	nsels /= sizeof(u32);
   1381	if (!nsels || nsels > MTK_NAND_MAX_NSELS) {
   1382		dev_err(dev, "invalid reg property size %d\n", nsels);
   1383		return -EINVAL;
   1384	}
   1385
   1386	chip = devm_kzalloc(dev, sizeof(*chip) + nsels * sizeof(u8),
   1387			    GFP_KERNEL);
   1388	if (!chip)
   1389		return -ENOMEM;
   1390
   1391	chip->nsels = nsels;
   1392	for (i = 0; i < nsels; i++) {
   1393		ret = of_property_read_u32_index(np, "reg", i, &tmp);
   1394		if (ret) {
   1395			dev_err(dev, "reg property failure : %d\n", ret);
   1396			return ret;
   1397		}
   1398
   1399		if (tmp >= MTK_NAND_MAX_NSELS) {
   1400			dev_err(dev, "invalid CS: %u\n", tmp);
   1401			return -EINVAL;
   1402		}
   1403
   1404		if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
   1405			dev_err(dev, "CS %u already assigned\n", tmp);
   1406			return -EINVAL;
   1407		}
   1408
   1409		chip->sels[i] = tmp;
   1410	}
   1411
   1412	nand = &chip->nand;
   1413	nand->controller = &nfc->controller;
   1414
   1415	nand_set_flash_node(nand, np);
   1416	nand_set_controller_data(nand, nfc);
   1417
   1418	nand->options |= NAND_USES_DMA | NAND_SUBPAGE_READ;
   1419
   1420	/* set default mode in case dt entry is missing */
   1421	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
   1422
   1423	nand->ecc.write_subpage = mtk_nfc_write_subpage_hwecc;
   1424	nand->ecc.write_page_raw = mtk_nfc_write_page_raw;
   1425	nand->ecc.write_page = mtk_nfc_write_page_hwecc;
   1426	nand->ecc.write_oob_raw = mtk_nfc_write_oob_std;
   1427	nand->ecc.write_oob = mtk_nfc_write_oob_std;
   1428
   1429	nand->ecc.read_subpage = mtk_nfc_read_subpage_hwecc;
   1430	nand->ecc.read_page_raw = mtk_nfc_read_page_raw;
   1431	nand->ecc.read_page = mtk_nfc_read_page_hwecc;
   1432	nand->ecc.read_oob_raw = mtk_nfc_read_oob_std;
   1433	nand->ecc.read_oob = mtk_nfc_read_oob_std;
   1434
   1435	mtd = nand_to_mtd(nand);
   1436	mtd->owner = THIS_MODULE;
   1437	mtd->dev.parent = dev;
   1438	mtd->name = MTK_NAME;
   1439	mtd_set_ooblayout(mtd, &mtk_nfc_ooblayout_ops);
   1440
   1441	mtk_nfc_hw_init(nfc);
   1442
   1443	ret = nand_scan(nand, nsels);
   1444	if (ret)
   1445		return ret;
   1446
   1447	ret = mtd_device_register(mtd, NULL, 0);
   1448	if (ret) {
   1449		dev_err(dev, "mtd parse partition error\n");
   1450		nand_cleanup(nand);
   1451		return ret;
   1452	}
   1453
   1454	list_add_tail(&chip->node, &nfc->chips);
   1455
   1456	return 0;
   1457}
   1458
   1459static int mtk_nfc_nand_chips_init(struct device *dev, struct mtk_nfc *nfc)
   1460{
   1461	struct device_node *np = dev->of_node;
   1462	struct device_node *nand_np;
   1463	int ret;
   1464
   1465	for_each_child_of_node(np, nand_np) {
   1466		ret = mtk_nfc_nand_chip_init(dev, nfc, nand_np);
   1467		if (ret) {
   1468			of_node_put(nand_np);
   1469			return ret;
   1470		}
   1471	}
   1472
   1473	return 0;
   1474}
   1475
   1476static const struct mtk_nfc_caps mtk_nfc_caps_mt2701 = {
   1477	.spare_size = spare_size_mt2701,
   1478	.num_spare_size = 16,
   1479	.pageformat_spare_shift = 4,
   1480	.nfi_clk_div = 1,
   1481	.max_sector = 16,
   1482	.max_sector_size = 1024,
   1483};
   1484
   1485static const struct mtk_nfc_caps mtk_nfc_caps_mt2712 = {
   1486	.spare_size = spare_size_mt2712,
   1487	.num_spare_size = 19,
   1488	.pageformat_spare_shift = 16,
   1489	.nfi_clk_div = 2,
   1490	.max_sector = 16,
   1491	.max_sector_size = 1024,
   1492};
   1493
   1494static const struct mtk_nfc_caps mtk_nfc_caps_mt7622 = {
   1495	.spare_size = spare_size_mt7622,
   1496	.num_spare_size = 4,
   1497	.pageformat_spare_shift = 4,
   1498	.nfi_clk_div = 1,
   1499	.max_sector = 8,
   1500	.max_sector_size = 512,
   1501};
   1502
   1503static const struct of_device_id mtk_nfc_id_table[] = {
   1504	{
   1505		.compatible = "mediatek,mt2701-nfc",
   1506		.data = &mtk_nfc_caps_mt2701,
   1507	}, {
   1508		.compatible = "mediatek,mt2712-nfc",
   1509		.data = &mtk_nfc_caps_mt2712,
   1510	}, {
   1511		.compatible = "mediatek,mt7622-nfc",
   1512		.data = &mtk_nfc_caps_mt7622,
   1513	},
   1514	{}
   1515};
   1516MODULE_DEVICE_TABLE(of, mtk_nfc_id_table);
   1517
   1518static int mtk_nfc_probe(struct platform_device *pdev)
   1519{
   1520	struct device *dev = &pdev->dev;
   1521	struct device_node *np = dev->of_node;
   1522	struct mtk_nfc *nfc;
   1523	int ret, irq;
   1524
   1525	nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
   1526	if (!nfc)
   1527		return -ENOMEM;
   1528
   1529	nand_controller_init(&nfc->controller);
   1530	INIT_LIST_HEAD(&nfc->chips);
   1531	nfc->controller.ops = &mtk_nfc_controller_ops;
   1532
   1533	/* probe defer if not ready */
   1534	nfc->ecc = of_mtk_ecc_get(np);
   1535	if (IS_ERR(nfc->ecc))
   1536		return PTR_ERR(nfc->ecc);
   1537	else if (!nfc->ecc)
   1538		return -ENODEV;
   1539
   1540	nfc->caps = of_device_get_match_data(dev);
   1541	nfc->dev = dev;
   1542
   1543	nfc->regs = devm_platform_ioremap_resource(pdev, 0);
   1544	if (IS_ERR(nfc->regs)) {
   1545		ret = PTR_ERR(nfc->regs);
   1546		goto release_ecc;
   1547	}
   1548
   1549	nfc->clk.nfi_clk = devm_clk_get(dev, "nfi_clk");
   1550	if (IS_ERR(nfc->clk.nfi_clk)) {
   1551		dev_err(dev, "no clk\n");
   1552		ret = PTR_ERR(nfc->clk.nfi_clk);
   1553		goto release_ecc;
   1554	}
   1555
   1556	nfc->clk.pad_clk = devm_clk_get(dev, "pad_clk");
   1557	if (IS_ERR(nfc->clk.pad_clk)) {
   1558		dev_err(dev, "no pad clk\n");
   1559		ret = PTR_ERR(nfc->clk.pad_clk);
   1560		goto release_ecc;
   1561	}
   1562
   1563	ret = mtk_nfc_enable_clk(dev, &nfc->clk);
   1564	if (ret)
   1565		goto release_ecc;
   1566
   1567	irq = platform_get_irq(pdev, 0);
   1568	if (irq < 0) {
   1569		ret = -EINVAL;
   1570		goto clk_disable;
   1571	}
   1572
   1573	ret = devm_request_irq(dev, irq, mtk_nfc_irq, 0x0, "mtk-nand", nfc);
   1574	if (ret) {
   1575		dev_err(dev, "failed to request nfi irq\n");
   1576		goto clk_disable;
   1577	}
   1578
   1579	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
   1580	if (ret) {
   1581		dev_err(dev, "failed to set dma mask\n");
   1582		goto clk_disable;
   1583	}
   1584
   1585	platform_set_drvdata(pdev, nfc);
   1586
   1587	ret = mtk_nfc_nand_chips_init(dev, nfc);
   1588	if (ret) {
   1589		dev_err(dev, "failed to init nand chips\n");
   1590		goto clk_disable;
   1591	}
   1592
   1593	return 0;
   1594
   1595clk_disable:
   1596	mtk_nfc_disable_clk(&nfc->clk);
   1597
   1598release_ecc:
   1599	mtk_ecc_release(nfc->ecc);
   1600
   1601	return ret;
   1602}
   1603
   1604static int mtk_nfc_remove(struct platform_device *pdev)
   1605{
   1606	struct mtk_nfc *nfc = platform_get_drvdata(pdev);
   1607	struct mtk_nfc_nand_chip *mtk_chip;
   1608	struct nand_chip *chip;
   1609	int ret;
   1610
   1611	while (!list_empty(&nfc->chips)) {
   1612		mtk_chip = list_first_entry(&nfc->chips,
   1613					    struct mtk_nfc_nand_chip, node);
   1614		chip = &mtk_chip->nand;
   1615		ret = mtd_device_unregister(nand_to_mtd(chip));
   1616		WARN_ON(ret);
   1617		nand_cleanup(chip);
   1618		list_del(&mtk_chip->node);
   1619	}
   1620
   1621	mtk_ecc_release(nfc->ecc);
   1622	mtk_nfc_disable_clk(&nfc->clk);
   1623
   1624	return 0;
   1625}
   1626
   1627#ifdef CONFIG_PM_SLEEP
   1628static int mtk_nfc_suspend(struct device *dev)
   1629{
   1630	struct mtk_nfc *nfc = dev_get_drvdata(dev);
   1631
   1632	mtk_nfc_disable_clk(&nfc->clk);
   1633
   1634	return 0;
   1635}
   1636
   1637static int mtk_nfc_resume(struct device *dev)
   1638{
   1639	struct mtk_nfc *nfc = dev_get_drvdata(dev);
   1640	struct mtk_nfc_nand_chip *chip;
   1641	struct nand_chip *nand;
   1642	int ret;
   1643	u32 i;
   1644
   1645	udelay(200);
   1646
   1647	ret = mtk_nfc_enable_clk(dev, &nfc->clk);
   1648	if (ret)
   1649		return ret;
   1650
   1651	/* reset NAND chip if VCC was powered off */
   1652	list_for_each_entry(chip, &nfc->chips, node) {
   1653		nand = &chip->nand;
   1654		for (i = 0; i < chip->nsels; i++)
   1655			nand_reset(nand, i);
   1656	}
   1657
   1658	return 0;
   1659}
   1660
   1661static SIMPLE_DEV_PM_OPS(mtk_nfc_pm_ops, mtk_nfc_suspend, mtk_nfc_resume);
   1662#endif
   1663
   1664static struct platform_driver mtk_nfc_driver = {
   1665	.probe  = mtk_nfc_probe,
   1666	.remove = mtk_nfc_remove,
   1667	.driver = {
   1668		.name  = MTK_NAME,
   1669		.of_match_table = mtk_nfc_id_table,
   1670#ifdef CONFIG_PM_SLEEP
   1671		.pm = &mtk_nfc_pm_ops,
   1672#endif
   1673	},
   1674};
   1675
   1676module_platform_driver(mtk_nfc_driver);
   1677
   1678MODULE_LICENSE("Dual MIT/GPL");
   1679MODULE_AUTHOR("Xiaolei Li <xiaolei.li@mediatek.com>");
   1680MODULE_DESCRIPTION("MTK Nand Flash Controller Driver");