cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

amba-pl08x.c (82236B)


      1// SPDX-License-Identifier: GPL-2.0-or-later
      2/*
      3 * Copyright (c) 2006 ARM Ltd.
      4 * Copyright (c) 2010 ST-Ericsson SA
      5 * Copyirght (c) 2017 Linaro Ltd.
      6 *
      7 * Author: Peter Pearse <peter.pearse@arm.com>
      8 * Author: Linus Walleij <linus.walleij@linaro.org>
      9 *
     10 * Documentation: ARM DDI 0196G == PL080
     11 * Documentation: ARM DDI 0218E == PL081
     12 * Documentation: S3C6410 User's Manual == PL080S
     13 *
     14 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
     15 * channel.
     16 *
     17 * The PL080 has 8 channels available for simultaneous use, and the PL081
     18 * has only two channels. So on these DMA controllers the number of channels
     19 * and the number of incoming DMA signals are two totally different things.
     20 * It is usually not possible to theoretically handle all physical signals,
     21 * so a multiplexing scheme with possible denial of use is necessary.
     22 *
     23 * The PL080 has a dual bus master, PL081 has a single master.
     24 *
     25 * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
     26 * It differs in following aspects:
     27 * - CH_CONFIG register at different offset,
     28 * - separate CH_CONTROL2 register for transfer size,
     29 * - bigger maximum transfer size,
     30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
     31 * - no support for peripheral flow control.
     32 *
     33 * Memory to peripheral transfer may be visualized as
     34 *	Get data from memory to DMAC
     35 *	Until no data left
     36 *		On burst request from peripheral
     37 *			Destination burst from DMAC to peripheral
     38 *			Clear burst request
     39 *	Raise terminal count interrupt
     40 *
     41 * For peripherals with a FIFO:
     42 * Source      burst size == half the depth of the peripheral FIFO
     43 * Destination burst size == the depth of the peripheral FIFO
     44 *
     45 * (Bursts are irrelevant for mem to mem transfers - there are no burst
     46 * signals, the DMA controller will simply facilitate its AHB master.)
     47 *
     48 * ASSUMES default (little) endianness for DMA transfers
     49 *
     50 * The PL08x has two flow control settings:
     51 *  - DMAC flow control: the transfer size defines the number of transfers
     52 *    which occur for the current LLI entry, and the DMAC raises TC at the
     53 *    end of every LLI entry.  Observed behaviour shows the DMAC listening
     54 *    to both the BREQ and SREQ signals (contrary to documented),
     55 *    transferring data if either is active.  The LBREQ and LSREQ signals
     56 *    are ignored.
     57 *
     58 *  - Peripheral flow control: the transfer size is ignored (and should be
     59 *    zero).  The data is transferred from the current LLI entry, until
     60 *    after the final transfer signalled by LBREQ or LSREQ.  The DMAC
     61 *    will then move to the next LLI entry. Unsupported by PL080S.
     62 */
     63#include <linux/amba/bus.h>
     64#include <linux/amba/pl08x.h>
     65#include <linux/debugfs.h>
     66#include <linux/delay.h>
     67#include <linux/device.h>
     68#include <linux/dmaengine.h>
     69#include <linux/dmapool.h>
     70#include <linux/dma-mapping.h>
     71#include <linux/export.h>
     72#include <linux/init.h>
     73#include <linux/interrupt.h>
     74#include <linux/module.h>
     75#include <linux/of.h>
     76#include <linux/of_dma.h>
     77#include <linux/pm_runtime.h>
     78#include <linux/seq_file.h>
     79#include <linux/slab.h>
     80#include <linux/amba/pl080.h>
     81
     82#include "dmaengine.h"
     83#include "virt-dma.h"
     84
     85#define DRIVER_NAME	"pl08xdmac"
     86
     87#define PL80X_DMA_BUSWIDTHS \
     88	BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
     89	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
     90	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
     91	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
     92
     93static struct amba_driver pl08x_amba_driver;
     94struct pl08x_driver_data;
     95
     96/**
     97 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
     98 * @config_offset: offset to the configuration register
     99 * @channels: the number of channels available in this variant
    100 * @signals: the number of request signals available from the hardware
    101 * @dualmaster: whether this version supports dual AHB masters or not.
    102 * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the
    103 *	channels have Nomadik security extension bits that need to be checked
    104 *	for permission before use and some registers are missing
    105 * @pl080s: whether this variant is a Samsung PL080S, which has separate
    106 *	register and LLI word for transfer size.
    107 * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020
    108 * @max_transfer_size: the maximum single element transfer size for this
    109 *	PL08x variant.
    110 */
    111struct vendor_data {
    112	u8 config_offset;
    113	u8 channels;
    114	u8 signals;
    115	bool dualmaster;
    116	bool nomadik;
    117	bool pl080s;
    118	bool ftdmac020;
    119	u32 max_transfer_size;
    120};
    121
    122/**
    123 * struct pl08x_bus_data - information of source or destination
    124 * busses for a transfer
    125 * @addr: current address
    126 * @maxwidth: the maximum width of a transfer on this bus
    127 * @buswidth: the width of this bus in bytes: 1, 2 or 4
    128 */
    129struct pl08x_bus_data {
    130	dma_addr_t addr;
    131	u8 maxwidth;
    132	u8 buswidth;
    133};
    134
    135#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
    136
    137/**
    138 * struct pl08x_phy_chan - holder for the physical channels
    139 * @id: physical index to this channel
    140 * @base: memory base address for this physical channel
    141 * @reg_config: configuration address for this physical channel
    142 * @reg_control: control address for this physical channel
    143 * @reg_src: transfer source address register
    144 * @reg_dst: transfer destination address register
    145 * @reg_lli: transfer LLI address register
    146 * @reg_busy: if the variant has a special per-channel busy register,
    147 * this contains a pointer to it
    148 * @lock: a lock to use when altering an instance of this struct
    149 * @serving: the virtual channel currently being served by this physical
    150 * channel
    151 * @locked: channel unavailable for the system, e.g. dedicated to secure
    152 * world
    153 * @ftdmac020: channel is on a FTDMAC020
    154 * @pl080s: channel is on a PL08s
    155 */
    156struct pl08x_phy_chan {
    157	unsigned int id;
    158	void __iomem *base;
    159	void __iomem *reg_config;
    160	void __iomem *reg_control;
    161	void __iomem *reg_src;
    162	void __iomem *reg_dst;
    163	void __iomem *reg_lli;
    164	void __iomem *reg_busy;
    165	spinlock_t lock;
    166	struct pl08x_dma_chan *serving;
    167	bool locked;
    168	bool ftdmac020;
    169	bool pl080s;
    170};
    171
    172/**
    173 * struct pl08x_sg - structure containing data per sg
    174 * @src_addr: src address of sg
    175 * @dst_addr: dst address of sg
    176 * @len: transfer len in bytes
    177 * @node: node for txd's dsg_list
    178 */
    179struct pl08x_sg {
    180	dma_addr_t src_addr;
    181	dma_addr_t dst_addr;
    182	size_t len;
    183	struct list_head node;
    184};
    185
    186/**
    187 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
    188 * @vd: virtual DMA descriptor
    189 * @dsg_list: list of children sg's
    190 * @llis_bus: DMA memory address (physical) start for the LLIs
    191 * @llis_va: virtual memory address start for the LLIs
    192 * @cctl: control reg values for current txd
    193 * @ccfg: config reg values for current txd
    194 * @done: this marks completed descriptors, which should not have their
    195 *   mux released.
    196 * @cyclic: indicate cyclic transfers
    197 */
    198struct pl08x_txd {
    199	struct virt_dma_desc vd;
    200	struct list_head dsg_list;
    201	dma_addr_t llis_bus;
    202	u32 *llis_va;
    203	/* Default cctl value for LLIs */
    204	u32 cctl;
    205	/*
    206	 * Settings to be put into the physical channel when we
    207	 * trigger this txd.  Other registers are in llis_va[0].
    208	 */
    209	u32 ccfg;
    210	bool done;
    211	bool cyclic;
    212};
    213
    214/**
    215 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
    216 * states
    217 * @PL08X_CHAN_IDLE: the channel is idle
    218 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
    219 * channel and is running a transfer on it
    220 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
    221 * channel, but the transfer is currently paused
    222 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
    223 * channel to become available (only pertains to memcpy channels)
    224 */
    225enum pl08x_dma_chan_state {
    226	PL08X_CHAN_IDLE,
    227	PL08X_CHAN_RUNNING,
    228	PL08X_CHAN_PAUSED,
    229	PL08X_CHAN_WAITING,
    230};
    231
    232/**
    233 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
    234 * @vc: wrappped virtual channel
    235 * @phychan: the physical channel utilized by this channel, if there is one
    236 * @name: name of channel
    237 * @cd: channel platform data
    238 * @cfg: slave configuration
    239 * @at: active transaction on this channel
    240 * @host: a pointer to the host (internal use)
    241 * @state: whether the channel is idle, paused, running etc
    242 * @slave: whether this channel is a device (slave) or for memcpy
    243 * @signal: the physical DMA request signal which this channel is using
    244 * @mux_use: count of descriptors using this DMA request signal setting
    245 * @waiting_at: time in jiffies when this channel moved to waiting state
    246 */
    247struct pl08x_dma_chan {
    248	struct virt_dma_chan vc;
    249	struct pl08x_phy_chan *phychan;
    250	const char *name;
    251	struct pl08x_channel_data *cd;
    252	struct dma_slave_config cfg;
    253	struct pl08x_txd *at;
    254	struct pl08x_driver_data *host;
    255	enum pl08x_dma_chan_state state;
    256	bool slave;
    257	int signal;
    258	unsigned mux_use;
    259	unsigned long waiting_at;
    260};
    261
    262/**
    263 * struct pl08x_driver_data - the local state holder for the PL08x
    264 * @slave: optional slave engine for this instance
    265 * @memcpy: memcpy engine for this instance
    266 * @has_slave: the PL08x has a slave engine (routed signals)
    267 * @base: virtual memory base (remapped) for the PL08x
    268 * @adev: the corresponding AMBA (PrimeCell) bus entry
    269 * @vd: vendor data for this PL08x variant
    270 * @pd: platform data passed in from the platform/machine
    271 * @phy_chans: array of data for the physical channels
    272 * @pool: a pool for the LLI descriptors
    273 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
    274 * fetches
    275 * @mem_buses: set to indicate memory transfers on AHB2.
    276 * @lli_words: how many words are used in each LLI item for this variant
    277 */
    278struct pl08x_driver_data {
    279	struct dma_device slave;
    280	struct dma_device memcpy;
    281	bool has_slave;
    282	void __iomem *base;
    283	struct amba_device *adev;
    284	const struct vendor_data *vd;
    285	struct pl08x_platform_data *pd;
    286	struct pl08x_phy_chan *phy_chans;
    287	struct dma_pool *pool;
    288	u8 lli_buses;
    289	u8 mem_buses;
    290	u8 lli_words;
    291};
    292
    293/*
    294 * PL08X specific defines
    295 */
    296
    297/* The order of words in an LLI. */
    298#define PL080_LLI_SRC		0
    299#define PL080_LLI_DST		1
    300#define PL080_LLI_LLI		2
    301#define PL080_LLI_CCTL		3
    302#define PL080S_LLI_CCTL2	4
    303
    304/* Total words in an LLI. */
    305#define PL080_LLI_WORDS		4
    306#define PL080S_LLI_WORDS	8
    307
    308/*
    309 * Number of LLIs in each LLI buffer allocated for one transfer
    310 * (maximum times we call dma_pool_alloc on this pool without freeing)
    311 */
    312#define MAX_NUM_TSFR_LLIS	512
    313#define PL08X_ALIGN		8
    314
    315static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
    316{
    317	return container_of(chan, struct pl08x_dma_chan, vc.chan);
    318}
    319
    320static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
    321{
    322	return container_of(tx, struct pl08x_txd, vd.tx);
    323}
    324
    325/*
    326 * Mux handling.
    327 *
    328 * This gives us the DMA request input to the PL08x primecell which the
    329 * peripheral described by the channel data will be routed to, possibly
    330 * via a board/SoC specific external MUX.  One important point to note
    331 * here is that this does not depend on the physical channel.
    332 */
    333static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
    334{
    335	const struct pl08x_platform_data *pd = plchan->host->pd;
    336	int ret;
    337
    338	if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
    339		ret = pd->get_xfer_signal(plchan->cd);
    340		if (ret < 0) {
    341			plchan->mux_use = 0;
    342			return ret;
    343		}
    344
    345		plchan->signal = ret;
    346	}
    347	return 0;
    348}
    349
    350static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
    351{
    352	const struct pl08x_platform_data *pd = plchan->host->pd;
    353
    354	if (plchan->signal >= 0) {
    355		WARN_ON(plchan->mux_use == 0);
    356
    357		if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
    358			pd->put_xfer_signal(plchan->cd, plchan->signal);
    359			plchan->signal = -1;
    360		}
    361	}
    362}
    363
    364/*
    365 * Physical channel handling
    366 */
    367
    368/* Whether a certain channel is busy or not */
    369static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
    370{
    371	unsigned int val;
    372
    373	/* If we have a special busy register, take a shortcut */
    374	if (ch->reg_busy) {
    375		val = readl(ch->reg_busy);
    376		return !!(val & BIT(ch->id));
    377	}
    378	val = readl(ch->reg_config);
    379	return val & PL080_CONFIG_ACTIVE;
    380}
    381
    382/*
    383 * pl08x_write_lli() - Write an LLI into the DMA controller.
    384 *
    385 * The PL08x derivatives support linked lists, but the first item of the
    386 * list containing the source, destination, control word and next LLI is
    387 * ignored. Instead the driver has to write those values directly into the
    388 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE
    389 * register need to be set up for the first transfer.
    390 */
    391static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
    392		struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
    393{
    394	if (pl08x->vd->pl080s)
    395		dev_vdbg(&pl08x->adev->dev,
    396			"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
    397			"clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
    398			phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
    399			lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
    400			lli[PL080S_LLI_CCTL2], ccfg);
    401	else
    402		dev_vdbg(&pl08x->adev->dev,
    403			"WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
    404			"clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
    405			phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
    406			lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
    407
    408	writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src);
    409	writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst);
    410	writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli);
    411
    412	/*
    413	 * The FTMAC020 has a different layout in the CCTL word of the LLI
    414	 * and the CCTL register which is split in CSR and SIZE registers.
    415	 * Convert the LLI item CCTL into the proper values to write into
    416	 * the CSR and SIZE registers.
    417	 */
    418	if (phychan->ftdmac020) {
    419		u32 llictl = lli[PL080_LLI_CCTL];
    420		u32 val = 0;
    421
    422		/* Write the transfer size (12 bits) to the size register */
    423		writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK,
    424			       phychan->base + FTDMAC020_CH_SIZE);
    425		/*
    426		 * Then write the control bits 28..16 to the control register
    427		 * by shuffleing the bits around to where they are in the
    428		 * main register. The mapping is as follows:
    429		 * Bit 28: TC_MSK - mask on all except last LLI
    430		 * Bit 27..25: SRC_WIDTH
    431		 * Bit 24..22: DST_WIDTH
    432		 * Bit 21..20: SRCAD_CTRL
    433		 * Bit 19..17: DSTAD_CTRL
    434		 * Bit 17: SRC_SEL
    435		 * Bit 16: DST_SEL
    436		 */
    437		if (llictl & FTDMAC020_LLI_TC_MSK)
    438			val |= FTDMAC020_CH_CSR_TC_MSK;
    439		val |= ((llictl  & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
    440			(FTDMAC020_LLI_SRC_WIDTH_SHIFT -
    441			 FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT));
    442		val |= ((llictl  & FTDMAC020_LLI_DST_WIDTH_MSK) >>
    443			(FTDMAC020_LLI_DST_WIDTH_SHIFT -
    444			 FTDMAC020_CH_CSR_DST_WIDTH_SHIFT));
    445		val |= ((llictl  & FTDMAC020_LLI_SRCAD_CTL_MSK) >>
    446			(FTDMAC020_LLI_SRCAD_CTL_SHIFT -
    447			 FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT));
    448		val |= ((llictl  & FTDMAC020_LLI_DSTAD_CTL_MSK) >>
    449			(FTDMAC020_LLI_DSTAD_CTL_SHIFT -
    450			 FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT));
    451		if (llictl & FTDMAC020_LLI_SRC_SEL)
    452			val |= FTDMAC020_CH_CSR_SRC_SEL;
    453		if (llictl & FTDMAC020_LLI_DST_SEL)
    454			val |= FTDMAC020_CH_CSR_DST_SEL;
    455
    456		/*
    457		 * Set up the bits that exist in the CSR but are not
    458		 * part the LLI, i.e. only gets written to the control
    459		 * register right here.
    460		 *
    461		 * FIXME: do not just handle memcpy, also handle slave DMA.
    462		 */
    463		switch (pl08x->pd->memcpy_burst_size) {
    464		default:
    465		case PL08X_BURST_SZ_1:
    466			val |= PL080_BSIZE_1 <<
    467				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    468			break;
    469		case PL08X_BURST_SZ_4:
    470			val |= PL080_BSIZE_4 <<
    471				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    472			break;
    473		case PL08X_BURST_SZ_8:
    474			val |= PL080_BSIZE_8 <<
    475				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    476			break;
    477		case PL08X_BURST_SZ_16:
    478			val |= PL080_BSIZE_16 <<
    479				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    480			break;
    481		case PL08X_BURST_SZ_32:
    482			val |= PL080_BSIZE_32 <<
    483				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    484			break;
    485		case PL08X_BURST_SZ_64:
    486			val |= PL080_BSIZE_64 <<
    487				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    488			break;
    489		case PL08X_BURST_SZ_128:
    490			val |= PL080_BSIZE_128 <<
    491				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    492			break;
    493		case PL08X_BURST_SZ_256:
    494			val |= PL080_BSIZE_256 <<
    495				FTDMAC020_CH_CSR_SRC_SIZE_SHIFT;
    496			break;
    497		}
    498
    499		/* Protection flags */
    500		if (pl08x->pd->memcpy_prot_buff)
    501			val |= FTDMAC020_CH_CSR_PROT2;
    502		if (pl08x->pd->memcpy_prot_cache)
    503			val |= FTDMAC020_CH_CSR_PROT3;
    504		/* We are the kernel, so we are in privileged mode */
    505		val |= FTDMAC020_CH_CSR_PROT1;
    506
    507		writel_relaxed(val, phychan->reg_control);
    508	} else {
    509		/* Bits are just identical */
    510		writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control);
    511	}
    512
    513	/* Second control word on the PL080s */
    514	if (pl08x->vd->pl080s)
    515		writel_relaxed(lli[PL080S_LLI_CCTL2],
    516				phychan->base + PL080S_CH_CONTROL2);
    517
    518	writel(ccfg, phychan->reg_config);
    519}
    520
    521/*
    522 * Set the initial DMA register values i.e. those for the first LLI
    523 * The next LLI pointer and the configuration interrupt bit have
    524 * been set when the LLIs were constructed.  Poke them into the hardware
    525 * and start the transfer.
    526 */
    527static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
    528{
    529	struct pl08x_driver_data *pl08x = plchan->host;
    530	struct pl08x_phy_chan *phychan = plchan->phychan;
    531	struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
    532	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
    533	u32 val;
    534
    535	list_del(&txd->vd.node);
    536
    537	plchan->at = txd;
    538
    539	/* Wait for channel inactive */
    540	while (pl08x_phy_channel_busy(phychan))
    541		cpu_relax();
    542
    543	pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
    544
    545	/* Enable the DMA channel */
    546	/* Do not access config register until channel shows as disabled */
    547	while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id))
    548		cpu_relax();
    549
    550	/* Do not access config register until channel shows as inactive */
    551	if (phychan->ftdmac020) {
    552		val = readl(phychan->reg_config);
    553		while (val & FTDMAC020_CH_CFG_BUSY)
    554			val = readl(phychan->reg_config);
    555
    556		val = readl(phychan->reg_control);
    557		while (val & FTDMAC020_CH_CSR_EN)
    558			val = readl(phychan->reg_control);
    559
    560		writel(val | FTDMAC020_CH_CSR_EN,
    561		       phychan->reg_control);
    562	} else {
    563		val = readl(phychan->reg_config);
    564		while ((val & PL080_CONFIG_ACTIVE) ||
    565		       (val & PL080_CONFIG_ENABLE))
    566			val = readl(phychan->reg_config);
    567
    568		writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
    569	}
    570}
    571
    572/*
    573 * Pause the channel by setting the HALT bit.
    574 *
    575 * For M->P transfers, pause the DMAC first and then stop the peripheral -
    576 * the FIFO can only drain if the peripheral is still requesting data.
    577 * (note: this can still timeout if the DMAC FIFO never drains of data.)
    578 *
    579 * For P->M transfers, disable the peripheral first to stop it filling
    580 * the DMAC FIFO, and then pause the DMAC.
    581 */
    582static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
    583{
    584	u32 val;
    585	int timeout;
    586
    587	if (ch->ftdmac020) {
    588		/* Use the enable bit on the FTDMAC020 */
    589		val = readl(ch->reg_control);
    590		val &= ~FTDMAC020_CH_CSR_EN;
    591		writel(val, ch->reg_control);
    592		return;
    593	}
    594
    595	/* Set the HALT bit and wait for the FIFO to drain */
    596	val = readl(ch->reg_config);
    597	val |= PL080_CONFIG_HALT;
    598	writel(val, ch->reg_config);
    599
    600	/* Wait for channel inactive */
    601	for (timeout = 1000; timeout; timeout--) {
    602		if (!pl08x_phy_channel_busy(ch))
    603			break;
    604		udelay(1);
    605	}
    606	if (pl08x_phy_channel_busy(ch))
    607		pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
    608}
    609
    610static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
    611{
    612	u32 val;
    613
    614	/* Use the enable bit on the FTDMAC020 */
    615	if (ch->ftdmac020) {
    616		val = readl(ch->reg_control);
    617		val |= FTDMAC020_CH_CSR_EN;
    618		writel(val, ch->reg_control);
    619		return;
    620	}
    621
    622	/* Clear the HALT bit */
    623	val = readl(ch->reg_config);
    624	val &= ~PL080_CONFIG_HALT;
    625	writel(val, ch->reg_config);
    626}
    627
    628/*
    629 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
    630 * clears any pending interrupt status.  This should not be used for
    631 * an on-going transfer, but as a method of shutting down a channel
    632 * (eg, when it's no longer used) or terminating a transfer.
    633 */
    634static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
    635	struct pl08x_phy_chan *ch)
    636{
    637	u32 val;
    638
    639	/* The layout for the FTDMAC020 is different */
    640	if (ch->ftdmac020) {
    641		/* Disable all interrupts */
    642		val = readl(ch->reg_config);
    643		val |= (FTDMAC020_CH_CFG_INT_ABT_MASK |
    644			FTDMAC020_CH_CFG_INT_ERR_MASK |
    645			FTDMAC020_CH_CFG_INT_TC_MASK);
    646		writel(val, ch->reg_config);
    647
    648		/* Abort and disable channel */
    649		val = readl(ch->reg_control);
    650		val &= ~FTDMAC020_CH_CSR_EN;
    651		val |= FTDMAC020_CH_CSR_ABT;
    652		writel(val, ch->reg_control);
    653
    654		/* Clear ABT and ERR interrupt flags */
    655		writel(BIT(ch->id) | BIT(ch->id + 16),
    656		       pl08x->base + PL080_ERR_CLEAR);
    657		writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
    658
    659		return;
    660	}
    661
    662	val = readl(ch->reg_config);
    663	val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
    664		 PL080_CONFIG_TC_IRQ_MASK);
    665	writel(val, ch->reg_config);
    666
    667	writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR);
    668	writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR);
    669}
    670
    671static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch)
    672{
    673	u32 val;
    674	u32 bytes;
    675
    676	if (ch->ftdmac020) {
    677		bytes = readl(ch->base + FTDMAC020_CH_SIZE);
    678
    679		val = readl(ch->reg_control);
    680		val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK;
    681		val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT;
    682	} else if (ch->pl080s) {
    683		val = readl(ch->base + PL080S_CH_CONTROL2);
    684		bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
    685
    686		val = readl(ch->reg_control);
    687		val &= PL080_CONTROL_SWIDTH_MASK;
    688		val >>= PL080_CONTROL_SWIDTH_SHIFT;
    689	} else {
    690		/* Plain PL08x */
    691		val = readl(ch->reg_control);
    692		bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
    693
    694		val &= PL080_CONTROL_SWIDTH_MASK;
    695		val >>= PL080_CONTROL_SWIDTH_SHIFT;
    696	}
    697
    698	switch (val) {
    699	case PL080_WIDTH_8BIT:
    700		break;
    701	case PL080_WIDTH_16BIT:
    702		bytes *= 2;
    703		break;
    704	case PL080_WIDTH_32BIT:
    705		bytes *= 4;
    706		break;
    707	}
    708	return bytes;
    709}
    710
    711static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va)
    712{
    713	u32 val;
    714	u32 bytes;
    715
    716	if (ch->ftdmac020) {
    717		val = llis_va[PL080_LLI_CCTL];
    718		bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK;
    719
    720		val = llis_va[PL080_LLI_CCTL];
    721		val &= FTDMAC020_LLI_SRC_WIDTH_MSK;
    722		val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT;
    723	} else if (ch->pl080s) {
    724		val = llis_va[PL080S_LLI_CCTL2];
    725		bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK;
    726
    727		val = llis_va[PL080_LLI_CCTL];
    728		val &= PL080_CONTROL_SWIDTH_MASK;
    729		val >>= PL080_CONTROL_SWIDTH_SHIFT;
    730	} else {
    731		/* Plain PL08x */
    732		val = llis_va[PL080_LLI_CCTL];
    733		bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK;
    734
    735		val &= PL080_CONTROL_SWIDTH_MASK;
    736		val >>= PL080_CONTROL_SWIDTH_SHIFT;
    737	}
    738
    739	switch (val) {
    740	case PL080_WIDTH_8BIT:
    741		break;
    742	case PL080_WIDTH_16BIT:
    743		bytes *= 2;
    744		break;
    745	case PL080_WIDTH_32BIT:
    746		bytes *= 4;
    747		break;
    748	}
    749	return bytes;
    750}
    751
    752/* The channel should be paused when calling this */
    753static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
    754{
    755	struct pl08x_driver_data *pl08x = plchan->host;
    756	const u32 *llis_va, *llis_va_limit;
    757	struct pl08x_phy_chan *ch;
    758	dma_addr_t llis_bus;
    759	struct pl08x_txd *txd;
    760	u32 llis_max_words;
    761	size_t bytes;
    762	u32 clli;
    763
    764	ch = plchan->phychan;
    765	txd = plchan->at;
    766
    767	if (!ch || !txd)
    768		return 0;
    769
    770	/*
    771	 * Follow the LLIs to get the number of remaining
    772	 * bytes in the currently active transaction.
    773	 */
    774	clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2;
    775
    776	/* First get the remaining bytes in the active transfer */
    777	bytes = get_bytes_in_phy_channel(ch);
    778
    779	if (!clli)
    780		return bytes;
    781
    782	llis_va = txd->llis_va;
    783	llis_bus = txd->llis_bus;
    784
    785	llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
    786	BUG_ON(clli < llis_bus || clli >= llis_bus +
    787						sizeof(u32) * llis_max_words);
    788
    789	/*
    790	 * Locate the next LLI - as this is an array,
    791	 * it's simple maths to find.
    792	 */
    793	llis_va += (clli - llis_bus) / sizeof(u32);
    794
    795	llis_va_limit = llis_va + llis_max_words;
    796
    797	for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
    798		bytes += get_bytes_in_lli(ch, llis_va);
    799
    800		/*
    801		 * A LLI pointer going backward terminates the LLI list
    802		 */
    803		if (llis_va[PL080_LLI_LLI] <= clli)
    804			break;
    805	}
    806
    807	return bytes;
    808}
    809
    810/*
    811 * Allocate a physical channel for a virtual channel
    812 *
    813 * Try to locate a physical channel to be used for this transfer. If all
    814 * are taken return NULL and the requester will have to cope by using
    815 * some fallback PIO mode or retrying later.
    816 */
    817static struct pl08x_phy_chan *
    818pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
    819		      struct pl08x_dma_chan *virt_chan)
    820{
    821	struct pl08x_phy_chan *ch = NULL;
    822	unsigned long flags;
    823	int i;
    824
    825	for (i = 0; i < pl08x->vd->channels; i++) {
    826		ch = &pl08x->phy_chans[i];
    827
    828		spin_lock_irqsave(&ch->lock, flags);
    829
    830		if (!ch->locked && !ch->serving) {
    831			ch->serving = virt_chan;
    832			spin_unlock_irqrestore(&ch->lock, flags);
    833			break;
    834		}
    835
    836		spin_unlock_irqrestore(&ch->lock, flags);
    837	}
    838
    839	if (i == pl08x->vd->channels) {
    840		/* No physical channel available, cope with it */
    841		return NULL;
    842	}
    843
    844	return ch;
    845}
    846
    847/* Mark the physical channel as free.  Note, this write is atomic. */
    848static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
    849					 struct pl08x_phy_chan *ch)
    850{
    851	ch->serving = NULL;
    852}
    853
    854/*
    855 * Try to allocate a physical channel.  When successful, assign it to
    856 * this virtual channel, and initiate the next descriptor.  The
    857 * virtual channel lock must be held at this point.
    858 */
    859static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
    860{
    861	struct pl08x_driver_data *pl08x = plchan->host;
    862	struct pl08x_phy_chan *ch;
    863
    864	ch = pl08x_get_phy_channel(pl08x, plchan);
    865	if (!ch) {
    866		dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
    867		plchan->state = PL08X_CHAN_WAITING;
    868		plchan->waiting_at = jiffies;
    869		return;
    870	}
    871
    872	dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
    873		ch->id, plchan->name);
    874
    875	plchan->phychan = ch;
    876	plchan->state = PL08X_CHAN_RUNNING;
    877	pl08x_start_next_txd(plchan);
    878}
    879
    880static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
    881	struct pl08x_dma_chan *plchan)
    882{
    883	struct pl08x_driver_data *pl08x = plchan->host;
    884
    885	dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
    886		ch->id, plchan->name);
    887
    888	/*
    889	 * We do this without taking the lock; we're really only concerned
    890	 * about whether this pointer is NULL or not, and we're guaranteed
    891	 * that this will only be called when it _already_ is non-NULL.
    892	 */
    893	ch->serving = plchan;
    894	plchan->phychan = ch;
    895	plchan->state = PL08X_CHAN_RUNNING;
    896	pl08x_start_next_txd(plchan);
    897}
    898
    899/*
    900 * Free a physical DMA channel, potentially reallocating it to another
    901 * virtual channel if we have any pending.
    902 */
    903static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
    904{
    905	struct pl08x_driver_data *pl08x = plchan->host;
    906	struct pl08x_dma_chan *p, *next;
    907	unsigned long waiting_at;
    908 retry:
    909	next = NULL;
    910	waiting_at = jiffies;
    911
    912	/*
    913	 * Find a waiting virtual channel for the next transfer.
    914	 * To be fair, time when each channel reached waiting state is compared
    915	 * to select channel that is waiting for the longest time.
    916	 */
    917	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
    918		if (p->state == PL08X_CHAN_WAITING &&
    919		    p->waiting_at <= waiting_at) {
    920			next = p;
    921			waiting_at = p->waiting_at;
    922		}
    923
    924	if (!next && pl08x->has_slave) {
    925		list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
    926			if (p->state == PL08X_CHAN_WAITING &&
    927			    p->waiting_at <= waiting_at) {
    928				next = p;
    929				waiting_at = p->waiting_at;
    930			}
    931	}
    932
    933	/* Ensure that the physical channel is stopped */
    934	pl08x_terminate_phy_chan(pl08x, plchan->phychan);
    935
    936	if (next) {
    937		bool success;
    938
    939		/*
    940		 * Eww.  We know this isn't going to deadlock
    941		 * but lockdep probably doesn't.
    942		 */
    943		spin_lock(&next->vc.lock);
    944		/* Re-check the state now that we have the lock */
    945		success = next->state == PL08X_CHAN_WAITING;
    946		if (success)
    947			pl08x_phy_reassign_start(plchan->phychan, next);
    948		spin_unlock(&next->vc.lock);
    949
    950		/* If the state changed, try to find another channel */
    951		if (!success)
    952			goto retry;
    953	} else {
    954		/* No more jobs, so free up the physical channel */
    955		pl08x_put_phy_channel(pl08x, plchan->phychan);
    956	}
    957
    958	plchan->phychan = NULL;
    959	plchan->state = PL08X_CHAN_IDLE;
    960}
    961
    962/*
    963 * LLI handling
    964 */
    965
    966static inline unsigned int
    967pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x,
    968			u32 cctl,
    969			bool source)
    970{
    971	u32 val;
    972
    973	if (pl08x->vd->ftdmac020) {
    974		if (source)
    975			val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >>
    976				FTDMAC020_LLI_SRC_WIDTH_SHIFT;
    977		else
    978			val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >>
    979				FTDMAC020_LLI_DST_WIDTH_SHIFT;
    980	} else {
    981		if (source)
    982			val = (cctl & PL080_CONTROL_SWIDTH_MASK) >>
    983				PL080_CONTROL_SWIDTH_SHIFT;
    984		else
    985			val = (cctl & PL080_CONTROL_DWIDTH_MASK) >>
    986				PL080_CONTROL_DWIDTH_SHIFT;
    987	}
    988
    989	switch (val) {
    990	case PL080_WIDTH_8BIT:
    991		return 1;
    992	case PL080_WIDTH_16BIT:
    993		return 2;
    994	case PL080_WIDTH_32BIT:
    995		return 4;
    996	default:
    997		break;
    998	}
    999	BUG();
   1000	return 0;
   1001}
   1002
   1003static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x,
   1004					 u32 cctl,
   1005					 u8 srcwidth, u8 dstwidth,
   1006					 size_t tsize)
   1007{
   1008	u32 retbits = cctl;
   1009
   1010	/*
   1011	 * Remove all src, dst and transfer size bits, then set the
   1012	 * width and size according to the parameters. The bit offsets
   1013	 * are different in the FTDMAC020 so we need to accound for this.
   1014	 */
   1015	if (pl08x->vd->ftdmac020) {
   1016		retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK;
   1017		retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK;
   1018		retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK;
   1019
   1020		switch (srcwidth) {
   1021		case 1:
   1022			retbits |= PL080_WIDTH_8BIT <<
   1023				FTDMAC020_LLI_SRC_WIDTH_SHIFT;
   1024			break;
   1025		case 2:
   1026			retbits |= PL080_WIDTH_16BIT <<
   1027				FTDMAC020_LLI_SRC_WIDTH_SHIFT;
   1028			break;
   1029		case 4:
   1030			retbits |= PL080_WIDTH_32BIT <<
   1031				FTDMAC020_LLI_SRC_WIDTH_SHIFT;
   1032			break;
   1033		default:
   1034			BUG();
   1035			break;
   1036		}
   1037
   1038		switch (dstwidth) {
   1039		case 1:
   1040			retbits |= PL080_WIDTH_8BIT <<
   1041				FTDMAC020_LLI_DST_WIDTH_SHIFT;
   1042			break;
   1043		case 2:
   1044			retbits |= PL080_WIDTH_16BIT <<
   1045				FTDMAC020_LLI_DST_WIDTH_SHIFT;
   1046			break;
   1047		case 4:
   1048			retbits |= PL080_WIDTH_32BIT <<
   1049				FTDMAC020_LLI_DST_WIDTH_SHIFT;
   1050			break;
   1051		default:
   1052			BUG();
   1053			break;
   1054		}
   1055
   1056		tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK;
   1057		retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT;
   1058	} else {
   1059		retbits &= ~PL080_CONTROL_DWIDTH_MASK;
   1060		retbits &= ~PL080_CONTROL_SWIDTH_MASK;
   1061		retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
   1062
   1063		switch (srcwidth) {
   1064		case 1:
   1065			retbits |= PL080_WIDTH_8BIT <<
   1066				PL080_CONTROL_SWIDTH_SHIFT;
   1067			break;
   1068		case 2:
   1069			retbits |= PL080_WIDTH_16BIT <<
   1070				PL080_CONTROL_SWIDTH_SHIFT;
   1071			break;
   1072		case 4:
   1073			retbits |= PL080_WIDTH_32BIT <<
   1074				PL080_CONTROL_SWIDTH_SHIFT;
   1075			break;
   1076		default:
   1077			BUG();
   1078			break;
   1079		}
   1080
   1081		switch (dstwidth) {
   1082		case 1:
   1083			retbits |= PL080_WIDTH_8BIT <<
   1084				PL080_CONTROL_DWIDTH_SHIFT;
   1085			break;
   1086		case 2:
   1087			retbits |= PL080_WIDTH_16BIT <<
   1088				PL080_CONTROL_DWIDTH_SHIFT;
   1089			break;
   1090		case 4:
   1091			retbits |= PL080_WIDTH_32BIT <<
   1092				PL080_CONTROL_DWIDTH_SHIFT;
   1093			break;
   1094		default:
   1095			BUG();
   1096			break;
   1097		}
   1098
   1099		tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
   1100		retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
   1101	}
   1102
   1103	return retbits;
   1104}
   1105
   1106struct pl08x_lli_build_data {
   1107	struct pl08x_txd *txd;
   1108	struct pl08x_bus_data srcbus;
   1109	struct pl08x_bus_data dstbus;
   1110	size_t remainder;
   1111	u32 lli_bus;
   1112};
   1113
   1114/*
   1115 * Autoselect a master bus to use for the transfer. Slave will be the chosen as
   1116 * victim in case src & dest are not similarly aligned. i.e. If after aligning
   1117 * masters address with width requirements of transfer (by sending few byte by
   1118 * byte data), slave is still not aligned, then its width will be reduced to
   1119 * BYTE.
   1120 * - prefers the destination bus if both available
   1121 * - prefers bus with fixed address (i.e. peripheral)
   1122 */
   1123static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x,
   1124				    struct pl08x_lli_build_data *bd,
   1125				    struct pl08x_bus_data **mbus,
   1126				    struct pl08x_bus_data **sbus,
   1127				    u32 cctl)
   1128{
   1129	bool dst_incr;
   1130	bool src_incr;
   1131
   1132	/*
   1133	 * The FTDMAC020 only supports memory-to-memory transfer, so
   1134	 * source and destination always increase.
   1135	 */
   1136	if (pl08x->vd->ftdmac020) {
   1137		dst_incr = true;
   1138		src_incr = true;
   1139	} else {
   1140		dst_incr = !!(cctl & PL080_CONTROL_DST_INCR);
   1141		src_incr = !!(cctl & PL080_CONTROL_SRC_INCR);
   1142	}
   1143
   1144	/*
   1145	 * If either bus is not advancing, i.e. it is a peripheral, that
   1146	 * one becomes master
   1147	 */
   1148	if (!dst_incr) {
   1149		*mbus = &bd->dstbus;
   1150		*sbus = &bd->srcbus;
   1151	} else if (!src_incr) {
   1152		*mbus = &bd->srcbus;
   1153		*sbus = &bd->dstbus;
   1154	} else {
   1155		if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
   1156			*mbus = &bd->dstbus;
   1157			*sbus = &bd->srcbus;
   1158		} else {
   1159			*mbus = &bd->srcbus;
   1160			*sbus = &bd->dstbus;
   1161		}
   1162	}
   1163}
   1164
   1165/*
   1166 * Fills in one LLI for a certain transfer descriptor and advance the counter
   1167 */
   1168static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
   1169				    struct pl08x_lli_build_data *bd,
   1170				    int num_llis, int len, u32 cctl, u32 cctl2)
   1171{
   1172	u32 offset = num_llis * pl08x->lli_words;
   1173	u32 *llis_va = bd->txd->llis_va + offset;
   1174	dma_addr_t llis_bus = bd->txd->llis_bus;
   1175
   1176	BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
   1177
   1178	/* Advance the offset to next LLI. */
   1179	offset += pl08x->lli_words;
   1180
   1181	llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
   1182	llis_va[PL080_LLI_DST] = bd->dstbus.addr;
   1183	llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
   1184	llis_va[PL080_LLI_LLI] |= bd->lli_bus;
   1185	llis_va[PL080_LLI_CCTL] = cctl;
   1186	if (pl08x->vd->pl080s)
   1187		llis_va[PL080S_LLI_CCTL2] = cctl2;
   1188
   1189	if (pl08x->vd->ftdmac020) {
   1190		/* FIXME: only memcpy so far so both increase */
   1191		bd->srcbus.addr += len;
   1192		bd->dstbus.addr += len;
   1193	} else {
   1194		if (cctl & PL080_CONTROL_SRC_INCR)
   1195			bd->srcbus.addr += len;
   1196		if (cctl & PL080_CONTROL_DST_INCR)
   1197			bd->dstbus.addr += len;
   1198	}
   1199
   1200	BUG_ON(bd->remainder < len);
   1201
   1202	bd->remainder -= len;
   1203}
   1204
   1205static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
   1206			struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
   1207			int num_llis, size_t *total_bytes)
   1208{
   1209	*cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len);
   1210	pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
   1211	(*total_bytes) += len;
   1212}
   1213
   1214#if 1
   1215static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
   1216			   const u32 *llis_va, int num_llis)
   1217{
   1218	int i;
   1219
   1220	if (pl08x->vd->pl080s) {
   1221		dev_vdbg(&pl08x->adev->dev,
   1222			"%-3s %-9s  %-10s %-10s %-10s %-10s %s\n",
   1223			"lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
   1224		for (i = 0; i < num_llis; i++) {
   1225			dev_vdbg(&pl08x->adev->dev,
   1226				"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
   1227				i, llis_va, llis_va[PL080_LLI_SRC],
   1228				llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
   1229				llis_va[PL080_LLI_CCTL],
   1230				llis_va[PL080S_LLI_CCTL2]);
   1231			llis_va += pl08x->lli_words;
   1232		}
   1233	} else {
   1234		dev_vdbg(&pl08x->adev->dev,
   1235			"%-3s %-9s  %-10s %-10s %-10s %s\n",
   1236			"lli", "", "csrc", "cdst", "clli", "cctl");
   1237		for (i = 0; i < num_llis; i++) {
   1238			dev_vdbg(&pl08x->adev->dev,
   1239				"%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
   1240				i, llis_va, llis_va[PL080_LLI_SRC],
   1241				llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
   1242				llis_va[PL080_LLI_CCTL]);
   1243			llis_va += pl08x->lli_words;
   1244		}
   1245	}
   1246}
   1247#else
   1248static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
   1249				  const u32 *llis_va, int num_llis) {}
   1250#endif
   1251
   1252/*
   1253 * This fills in the table of LLIs for the transfer descriptor
   1254 * Note that we assume we never have to change the burst sizes
   1255 * Return 0 for error
   1256 */
   1257static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
   1258			      struct pl08x_txd *txd)
   1259{
   1260	struct pl08x_bus_data *mbus, *sbus;
   1261	struct pl08x_lli_build_data bd;
   1262	int num_llis = 0;
   1263	u32 cctl, early_bytes = 0;
   1264	size_t max_bytes_per_lli, total_bytes;
   1265	u32 *llis_va, *last_lli;
   1266	struct pl08x_sg *dsg;
   1267
   1268	txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
   1269	if (!txd->llis_va) {
   1270		dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
   1271		return 0;
   1272	}
   1273
   1274	bd.txd = txd;
   1275	bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
   1276	cctl = txd->cctl;
   1277
   1278	/* Find maximum width of the source bus */
   1279	bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true);
   1280
   1281	/* Find maximum width of the destination bus */
   1282	bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false);
   1283
   1284	list_for_each_entry(dsg, &txd->dsg_list, node) {
   1285		total_bytes = 0;
   1286		cctl = txd->cctl;
   1287
   1288		bd.srcbus.addr = dsg->src_addr;
   1289		bd.dstbus.addr = dsg->dst_addr;
   1290		bd.remainder = dsg->len;
   1291		bd.srcbus.buswidth = bd.srcbus.maxwidth;
   1292		bd.dstbus.buswidth = bd.dstbus.maxwidth;
   1293
   1294		pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl);
   1295
   1296		dev_vdbg(&pl08x->adev->dev,
   1297			"src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
   1298			(u64)bd.srcbus.addr,
   1299			cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
   1300			bd.srcbus.buswidth,
   1301			(u64)bd.dstbus.addr,
   1302			cctl & PL080_CONTROL_DST_INCR ? "+" : "",
   1303			bd.dstbus.buswidth,
   1304			bd.remainder);
   1305		dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
   1306			mbus == &bd.srcbus ? "src" : "dst",
   1307			sbus == &bd.srcbus ? "src" : "dst");
   1308
   1309		/*
   1310		 * Zero length is only allowed if all these requirements are
   1311		 * met:
   1312		 * - flow controller is peripheral.
   1313		 * - src.addr is aligned to src.width
   1314		 * - dst.addr is aligned to dst.width
   1315		 *
   1316		 * sg_len == 1 should be true, as there can be two cases here:
   1317		 *
   1318		 * - Memory addresses are contiguous and are not scattered.
   1319		 *   Here, Only one sg will be passed by user driver, with
   1320		 *   memory address and zero length. We pass this to controller
   1321		 *   and after the transfer it will receive the last burst
   1322		 *   request from peripheral and so transfer finishes.
   1323		 *
   1324		 * - Memory addresses are scattered and are not contiguous.
   1325		 *   Here, Obviously as DMA controller doesn't know when a lli's
   1326		 *   transfer gets over, it can't load next lli. So in this
   1327		 *   case, there has to be an assumption that only one lli is
   1328		 *   supported. Thus, we can't have scattered addresses.
   1329		 */
   1330		if (!bd.remainder) {
   1331			u32 fc;
   1332
   1333			/* FTDMAC020 only does memory-to-memory */
   1334			if (pl08x->vd->ftdmac020)
   1335				fc = PL080_FLOW_MEM2MEM;
   1336			else
   1337				fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
   1338					PL080_CONFIG_FLOW_CONTROL_SHIFT;
   1339			if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
   1340					(fc <= PL080_FLOW_SRC2DST_SRC))) {
   1341				dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
   1342					__func__);
   1343				return 0;
   1344			}
   1345
   1346			if (!IS_BUS_ALIGNED(&bd.srcbus) ||
   1347				!IS_BUS_ALIGNED(&bd.dstbus)) {
   1348				dev_err(&pl08x->adev->dev,
   1349					"%s src & dst address must be aligned to src"
   1350					" & dst width if peripheral is flow controller",
   1351					__func__);
   1352				return 0;
   1353			}
   1354
   1355			cctl = pl08x_lli_control_bits(pl08x, cctl,
   1356					bd.srcbus.buswidth, bd.dstbus.buswidth,
   1357					0);
   1358			pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
   1359					0, cctl, 0);
   1360			break;
   1361		}
   1362
   1363		/*
   1364		 * Send byte by byte for following cases
   1365		 * - Less than a bus width available
   1366		 * - until master bus is aligned
   1367		 */
   1368		if (bd.remainder < mbus->buswidth)
   1369			early_bytes = bd.remainder;
   1370		else if (!IS_BUS_ALIGNED(mbus)) {
   1371			early_bytes = mbus->buswidth -
   1372				(mbus->addr & (mbus->buswidth - 1));
   1373			if ((bd.remainder - early_bytes) < mbus->buswidth)
   1374				early_bytes = bd.remainder;
   1375		}
   1376
   1377		if (early_bytes) {
   1378			dev_vdbg(&pl08x->adev->dev,
   1379				"%s byte width LLIs (remain 0x%08zx)\n",
   1380				__func__, bd.remainder);
   1381			prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
   1382				num_llis++, &total_bytes);
   1383		}
   1384
   1385		if (bd.remainder) {
   1386			/*
   1387			 * Master now aligned
   1388			 * - if slave is not then we must set its width down
   1389			 */
   1390			if (!IS_BUS_ALIGNED(sbus)) {
   1391				dev_dbg(&pl08x->adev->dev,
   1392					"%s set down bus width to one byte\n",
   1393					__func__);
   1394
   1395				sbus->buswidth = 1;
   1396			}
   1397
   1398			/*
   1399			 * Bytes transferred = tsize * src width, not
   1400			 * MIN(buswidths)
   1401			 */
   1402			max_bytes_per_lli = bd.srcbus.buswidth *
   1403						pl08x->vd->max_transfer_size;
   1404			dev_vdbg(&pl08x->adev->dev,
   1405				"%s max bytes per lli = %zu\n",
   1406				__func__, max_bytes_per_lli);
   1407
   1408			/*
   1409			 * Make largest possible LLIs until less than one bus
   1410			 * width left
   1411			 */
   1412			while (bd.remainder > (mbus->buswidth - 1)) {
   1413				size_t lli_len, tsize, width;
   1414
   1415				/*
   1416				 * If enough left try to send max possible,
   1417				 * otherwise try to send the remainder
   1418				 */
   1419				lli_len = min(bd.remainder, max_bytes_per_lli);
   1420
   1421				/*
   1422				 * Check against maximum bus alignment:
   1423				 * Calculate actual transfer size in relation to
   1424				 * bus width an get a maximum remainder of the
   1425				 * highest bus width - 1
   1426				 */
   1427				width = max(mbus->buswidth, sbus->buswidth);
   1428				lli_len = (lli_len / width) * width;
   1429				tsize = lli_len / bd.srcbus.buswidth;
   1430
   1431				dev_vdbg(&pl08x->adev->dev,
   1432					"%s fill lli with single lli chunk of "
   1433					"size 0x%08zx (remainder 0x%08zx)\n",
   1434					__func__, lli_len, bd.remainder);
   1435
   1436				cctl = pl08x_lli_control_bits(pl08x, cctl,
   1437					bd.srcbus.buswidth, bd.dstbus.buswidth,
   1438					tsize);
   1439				pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
   1440						lli_len, cctl, tsize);
   1441				total_bytes += lli_len;
   1442			}
   1443
   1444			/*
   1445			 * Send any odd bytes
   1446			 */
   1447			if (bd.remainder) {
   1448				dev_vdbg(&pl08x->adev->dev,
   1449					"%s align with boundary, send odd bytes (remain %zu)\n",
   1450					__func__, bd.remainder);
   1451				prep_byte_width_lli(pl08x, &bd, &cctl,
   1452					bd.remainder, num_llis++, &total_bytes);
   1453			}
   1454		}
   1455
   1456		if (total_bytes != dsg->len) {
   1457			dev_err(&pl08x->adev->dev,
   1458				"%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
   1459				__func__, total_bytes, dsg->len);
   1460			return 0;
   1461		}
   1462
   1463		if (num_llis >= MAX_NUM_TSFR_LLIS) {
   1464			dev_err(&pl08x->adev->dev,
   1465				"%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
   1466				__func__, MAX_NUM_TSFR_LLIS);
   1467			return 0;
   1468		}
   1469	}
   1470
   1471	llis_va = txd->llis_va;
   1472	last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
   1473
   1474	if (txd->cyclic) {
   1475		/* Link back to the first LLI. */
   1476		last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
   1477	} else {
   1478		/* The final LLI terminates the LLI. */
   1479		last_lli[PL080_LLI_LLI] = 0;
   1480		/* The final LLI element shall also fire an interrupt. */
   1481		if (pl08x->vd->ftdmac020)
   1482			last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK;
   1483		else
   1484			last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
   1485	}
   1486
   1487	pl08x_dump_lli(pl08x, llis_va, num_llis);
   1488
   1489	return num_llis;
   1490}
   1491
   1492static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
   1493			   struct pl08x_txd *txd)
   1494{
   1495	struct pl08x_sg *dsg, *_dsg;
   1496
   1497	if (txd->llis_va)
   1498		dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
   1499
   1500	list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
   1501		list_del(&dsg->node);
   1502		kfree(dsg);
   1503	}
   1504
   1505	kfree(txd);
   1506}
   1507
   1508static void pl08x_desc_free(struct virt_dma_desc *vd)
   1509{
   1510	struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
   1511	struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
   1512
   1513	dma_descriptor_unmap(&vd->tx);
   1514	if (!txd->done)
   1515		pl08x_release_mux(plchan);
   1516
   1517	pl08x_free_txd(plchan->host, txd);
   1518}
   1519
   1520static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
   1521				struct pl08x_dma_chan *plchan)
   1522{
   1523	LIST_HEAD(head);
   1524
   1525	vchan_get_all_descriptors(&plchan->vc, &head);
   1526	vchan_dma_desc_free_list(&plchan->vc, &head);
   1527}
   1528
   1529/*
   1530 * The DMA ENGINE API
   1531 */
   1532static void pl08x_free_chan_resources(struct dma_chan *chan)
   1533{
   1534	/* Ensure all queued descriptors are freed */
   1535	vchan_free_chan_resources(to_virt_chan(chan));
   1536}
   1537
   1538/*
   1539 * Code accessing dma_async_is_complete() in a tight loop may give problems.
   1540 * If slaves are relying on interrupts to signal completion this function
   1541 * must not be called with interrupts disabled.
   1542 */
   1543static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
   1544		dma_cookie_t cookie, struct dma_tx_state *txstate)
   1545{
   1546	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   1547	struct virt_dma_desc *vd;
   1548	unsigned long flags;
   1549	enum dma_status ret;
   1550	size_t bytes = 0;
   1551
   1552	ret = dma_cookie_status(chan, cookie, txstate);
   1553	if (ret == DMA_COMPLETE)
   1554		return ret;
   1555
   1556	/*
   1557	 * There's no point calculating the residue if there's
   1558	 * no txstate to store the value.
   1559	 */
   1560	if (!txstate) {
   1561		if (plchan->state == PL08X_CHAN_PAUSED)
   1562			ret = DMA_PAUSED;
   1563		return ret;
   1564	}
   1565
   1566	spin_lock_irqsave(&plchan->vc.lock, flags);
   1567	ret = dma_cookie_status(chan, cookie, txstate);
   1568	if (ret != DMA_COMPLETE) {
   1569		vd = vchan_find_desc(&plchan->vc, cookie);
   1570		if (vd) {
   1571			/* On the issued list, so hasn't been processed yet */
   1572			struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
   1573			struct pl08x_sg *dsg;
   1574
   1575			list_for_each_entry(dsg, &txd->dsg_list, node)
   1576				bytes += dsg->len;
   1577		} else {
   1578			bytes = pl08x_getbytes_chan(plchan);
   1579		}
   1580	}
   1581	spin_unlock_irqrestore(&plchan->vc.lock, flags);
   1582
   1583	/*
   1584	 * This cookie not complete yet
   1585	 * Get number of bytes left in the active transactions and queue
   1586	 */
   1587	dma_set_residue(txstate, bytes);
   1588
   1589	if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
   1590		ret = DMA_PAUSED;
   1591
   1592	/* Whether waiting or running, we're in progress */
   1593	return ret;
   1594}
   1595
   1596/* PrimeCell DMA extension */
   1597struct burst_table {
   1598	u32 burstwords;
   1599	u32 reg;
   1600};
   1601
   1602static const struct burst_table burst_sizes[] = {
   1603	{
   1604		.burstwords = 256,
   1605		.reg = PL080_BSIZE_256,
   1606	},
   1607	{
   1608		.burstwords = 128,
   1609		.reg = PL080_BSIZE_128,
   1610	},
   1611	{
   1612		.burstwords = 64,
   1613		.reg = PL080_BSIZE_64,
   1614	},
   1615	{
   1616		.burstwords = 32,
   1617		.reg = PL080_BSIZE_32,
   1618	},
   1619	{
   1620		.burstwords = 16,
   1621		.reg = PL080_BSIZE_16,
   1622	},
   1623	{
   1624		.burstwords = 8,
   1625		.reg = PL080_BSIZE_8,
   1626	},
   1627	{
   1628		.burstwords = 4,
   1629		.reg = PL080_BSIZE_4,
   1630	},
   1631	{
   1632		.burstwords = 0,
   1633		.reg = PL080_BSIZE_1,
   1634	},
   1635};
   1636
   1637/*
   1638 * Given the source and destination available bus masks, select which
   1639 * will be routed to each port.  We try to have source and destination
   1640 * on separate ports, but always respect the allowable settings.
   1641 */
   1642static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst)
   1643{
   1644	u32 cctl = 0;
   1645	u32 dst_ahb2;
   1646	u32 src_ahb2;
   1647
   1648	/* The FTDMAC020 use different bits to indicate src/dst bus */
   1649	if (ftdmac020) {
   1650		dst_ahb2 = FTDMAC020_LLI_DST_SEL;
   1651		src_ahb2 = FTDMAC020_LLI_SRC_SEL;
   1652	} else {
   1653		dst_ahb2 = PL080_CONTROL_DST_AHB2;
   1654		src_ahb2 = PL080_CONTROL_SRC_AHB2;
   1655	}
   1656
   1657	if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
   1658		cctl |= dst_ahb2;
   1659	if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
   1660		cctl |= src_ahb2;
   1661
   1662	return cctl;
   1663}
   1664
   1665static u32 pl08x_cctl(u32 cctl)
   1666{
   1667	cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
   1668		  PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
   1669		  PL080_CONTROL_PROT_MASK);
   1670
   1671	/* Access the cell in privileged mode, non-bufferable, non-cacheable */
   1672	return cctl | PL080_CONTROL_PROT_SYS;
   1673}
   1674
   1675static u32 pl08x_width(enum dma_slave_buswidth width)
   1676{
   1677	switch (width) {
   1678	case DMA_SLAVE_BUSWIDTH_1_BYTE:
   1679		return PL080_WIDTH_8BIT;
   1680	case DMA_SLAVE_BUSWIDTH_2_BYTES:
   1681		return PL080_WIDTH_16BIT;
   1682	case DMA_SLAVE_BUSWIDTH_4_BYTES:
   1683		return PL080_WIDTH_32BIT;
   1684	default:
   1685		return ~0;
   1686	}
   1687}
   1688
   1689static u32 pl08x_burst(u32 maxburst)
   1690{
   1691	int i;
   1692
   1693	for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
   1694		if (burst_sizes[i].burstwords <= maxburst)
   1695			break;
   1696
   1697	return burst_sizes[i].reg;
   1698}
   1699
   1700static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
   1701	enum dma_slave_buswidth addr_width, u32 maxburst)
   1702{
   1703	u32 width, burst, cctl = 0;
   1704
   1705	width = pl08x_width(addr_width);
   1706	if (width == ~0)
   1707		return ~0;
   1708
   1709	cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
   1710	cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
   1711
   1712	/*
   1713	 * If this channel will only request single transfers, set this
   1714	 * down to ONE element.  Also select one element if no maxburst
   1715	 * is specified.
   1716	 */
   1717	if (plchan->cd->single)
   1718		maxburst = 1;
   1719
   1720	burst = pl08x_burst(maxburst);
   1721	cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
   1722	cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
   1723
   1724	return pl08x_cctl(cctl);
   1725}
   1726
   1727/*
   1728 * Slave transactions callback to the slave device to allow
   1729 * synchronization of slave DMA signals with the DMAC enable
   1730 */
   1731static void pl08x_issue_pending(struct dma_chan *chan)
   1732{
   1733	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   1734	unsigned long flags;
   1735
   1736	spin_lock_irqsave(&plchan->vc.lock, flags);
   1737	if (vchan_issue_pending(&plchan->vc)) {
   1738		if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
   1739			pl08x_phy_alloc_and_start(plchan);
   1740	}
   1741	spin_unlock_irqrestore(&plchan->vc.lock, flags);
   1742}
   1743
   1744static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
   1745{
   1746	struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
   1747
   1748	if (txd)
   1749		INIT_LIST_HEAD(&txd->dsg_list);
   1750	return txd;
   1751}
   1752
   1753static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x)
   1754{
   1755	u32 cctl = 0;
   1756
   1757	/* Conjure cctl */
   1758	switch (pl08x->pd->memcpy_burst_size) {
   1759	default:
   1760		dev_err(&pl08x->adev->dev,
   1761			"illegal burst size for memcpy, set to 1\n");
   1762		fallthrough;
   1763	case PL08X_BURST_SZ_1:
   1764		cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |
   1765			PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;
   1766		break;
   1767	case PL08X_BURST_SZ_4:
   1768		cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |
   1769			PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;
   1770		break;
   1771	case PL08X_BURST_SZ_8:
   1772		cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |
   1773			PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;
   1774		break;
   1775	case PL08X_BURST_SZ_16:
   1776		cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |
   1777			PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;
   1778		break;
   1779	case PL08X_BURST_SZ_32:
   1780		cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |
   1781			PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;
   1782		break;
   1783	case PL08X_BURST_SZ_64:
   1784		cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |
   1785			PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;
   1786		break;
   1787	case PL08X_BURST_SZ_128:
   1788		cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |
   1789			PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;
   1790		break;
   1791	case PL08X_BURST_SZ_256:
   1792		cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |
   1793			PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;
   1794		break;
   1795	}
   1796
   1797	switch (pl08x->pd->memcpy_bus_width) {
   1798	default:
   1799		dev_err(&pl08x->adev->dev,
   1800			"illegal bus width for memcpy, set to 8 bits\n");
   1801		fallthrough;
   1802	case PL08X_BUS_WIDTH_8_BITS:
   1803		cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |
   1804			PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
   1805		break;
   1806	case PL08X_BUS_WIDTH_16_BITS:
   1807		cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |
   1808			PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
   1809		break;
   1810	case PL08X_BUS_WIDTH_32_BITS:
   1811		cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |
   1812			PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
   1813		break;
   1814	}
   1815
   1816	/* Protection flags */
   1817	if (pl08x->pd->memcpy_prot_buff)
   1818		cctl |= PL080_CONTROL_PROT_BUFF;
   1819	if (pl08x->pd->memcpy_prot_cache)
   1820		cctl |= PL080_CONTROL_PROT_CACHE;
   1821
   1822	/* We are the kernel, so we are in privileged mode */
   1823	cctl |= PL080_CONTROL_PROT_SYS;
   1824
   1825	/* Both to be incremented or the code will break */
   1826	cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
   1827
   1828	if (pl08x->vd->dualmaster)
   1829		cctl |= pl08x_select_bus(false,
   1830					 pl08x->mem_buses,
   1831					 pl08x->mem_buses);
   1832
   1833	return cctl;
   1834}
   1835
   1836static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x)
   1837{
   1838	u32 cctl = 0;
   1839
   1840	/* Conjure cctl */
   1841	switch (pl08x->pd->memcpy_bus_width) {
   1842	default:
   1843		dev_err(&pl08x->adev->dev,
   1844			"illegal bus width for memcpy, set to 8 bits\n");
   1845		fallthrough;
   1846	case PL08X_BUS_WIDTH_8_BITS:
   1847		cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
   1848			PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
   1849		break;
   1850	case PL08X_BUS_WIDTH_16_BITS:
   1851		cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
   1852			PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
   1853		break;
   1854	case PL08X_BUS_WIDTH_32_BITS:
   1855		cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT |
   1856			PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT;
   1857		break;
   1858	}
   1859
   1860	/*
   1861	 * By default mask the TC IRQ on all LLIs, it will be unmasked on
   1862	 * the last LLI item by other code.
   1863	 */
   1864	cctl |= FTDMAC020_LLI_TC_MSK;
   1865
   1866	/*
   1867	 * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL
   1868	 * and FTDMAC020_LLI_DSTAD_CTL as zero
   1869	 */
   1870	if (pl08x->vd->dualmaster)
   1871		cctl |= pl08x_select_bus(true,
   1872					 pl08x->mem_buses,
   1873					 pl08x->mem_buses);
   1874
   1875	return cctl;
   1876}
   1877
   1878/*
   1879 * Initialize a descriptor to be used by memcpy submit
   1880 */
   1881static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
   1882		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
   1883		size_t len, unsigned long flags)
   1884{
   1885	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   1886	struct pl08x_driver_data *pl08x = plchan->host;
   1887	struct pl08x_txd *txd;
   1888	struct pl08x_sg *dsg;
   1889	int ret;
   1890
   1891	txd = pl08x_get_txd(plchan);
   1892	if (!txd) {
   1893		dev_err(&pl08x->adev->dev,
   1894			"%s no memory for descriptor\n", __func__);
   1895		return NULL;
   1896	}
   1897
   1898	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
   1899	if (!dsg) {
   1900		pl08x_free_txd(pl08x, txd);
   1901		return NULL;
   1902	}
   1903	list_add_tail(&dsg->node, &txd->dsg_list);
   1904
   1905	dsg->src_addr = src;
   1906	dsg->dst_addr = dest;
   1907	dsg->len = len;
   1908	if (pl08x->vd->ftdmac020) {
   1909		/* Writing CCFG zero ENABLES all interrupts */
   1910		txd->ccfg = 0;
   1911		txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x);
   1912	} else {
   1913		txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
   1914			PL080_CONFIG_TC_IRQ_MASK |
   1915			PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
   1916		txd->cctl = pl08x_memcpy_cctl(pl08x);
   1917	}
   1918
   1919	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
   1920	if (!ret) {
   1921		pl08x_free_txd(pl08x, txd);
   1922		return NULL;
   1923	}
   1924
   1925	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
   1926}
   1927
   1928static struct pl08x_txd *pl08x_init_txd(
   1929		struct dma_chan *chan,
   1930		enum dma_transfer_direction direction,
   1931		dma_addr_t *slave_addr)
   1932{
   1933	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   1934	struct pl08x_driver_data *pl08x = plchan->host;
   1935	struct pl08x_txd *txd;
   1936	enum dma_slave_buswidth addr_width;
   1937	int ret, tmp;
   1938	u8 src_buses, dst_buses;
   1939	u32 maxburst, cctl;
   1940
   1941	txd = pl08x_get_txd(plchan);
   1942	if (!txd) {
   1943		dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
   1944		return NULL;
   1945	}
   1946
   1947	/*
   1948	 * Set up addresses, the PrimeCell configured address
   1949	 * will take precedence since this may configure the
   1950	 * channel target address dynamically at runtime.
   1951	 */
   1952	if (direction == DMA_MEM_TO_DEV) {
   1953		cctl = PL080_CONTROL_SRC_INCR;
   1954		*slave_addr = plchan->cfg.dst_addr;
   1955		addr_width = plchan->cfg.dst_addr_width;
   1956		maxburst = plchan->cfg.dst_maxburst;
   1957		src_buses = pl08x->mem_buses;
   1958		dst_buses = plchan->cd->periph_buses;
   1959	} else if (direction == DMA_DEV_TO_MEM) {
   1960		cctl = PL080_CONTROL_DST_INCR;
   1961		*slave_addr = plchan->cfg.src_addr;
   1962		addr_width = plchan->cfg.src_addr_width;
   1963		maxburst = plchan->cfg.src_maxburst;
   1964		src_buses = plchan->cd->periph_buses;
   1965		dst_buses = pl08x->mem_buses;
   1966	} else {
   1967		pl08x_free_txd(pl08x, txd);
   1968		dev_err(&pl08x->adev->dev,
   1969			"%s direction unsupported\n", __func__);
   1970		return NULL;
   1971	}
   1972
   1973	cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
   1974	if (cctl == ~0) {
   1975		pl08x_free_txd(pl08x, txd);
   1976		dev_err(&pl08x->adev->dev,
   1977			"DMA slave configuration botched?\n");
   1978		return NULL;
   1979	}
   1980
   1981	txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses);
   1982
   1983	if (plchan->cfg.device_fc)
   1984		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
   1985			PL080_FLOW_PER2MEM_PER;
   1986	else
   1987		tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
   1988			PL080_FLOW_PER2MEM;
   1989
   1990	txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
   1991		PL080_CONFIG_TC_IRQ_MASK |
   1992		tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
   1993
   1994	ret = pl08x_request_mux(plchan);
   1995	if (ret < 0) {
   1996		pl08x_free_txd(pl08x, txd);
   1997		dev_dbg(&pl08x->adev->dev,
   1998			"unable to mux for transfer on %s due to platform restrictions\n",
   1999			plchan->name);
   2000		return NULL;
   2001	}
   2002
   2003	dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
   2004		 plchan->signal, plchan->name);
   2005
   2006	/* Assign the flow control signal to this channel */
   2007	if (direction == DMA_MEM_TO_DEV)
   2008		txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
   2009	else
   2010		txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
   2011
   2012	return txd;
   2013}
   2014
   2015static int pl08x_tx_add_sg(struct pl08x_txd *txd,
   2016			   enum dma_transfer_direction direction,
   2017			   dma_addr_t slave_addr,
   2018			   dma_addr_t buf_addr,
   2019			   unsigned int len)
   2020{
   2021	struct pl08x_sg *dsg;
   2022
   2023	dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
   2024	if (!dsg)
   2025		return -ENOMEM;
   2026
   2027	list_add_tail(&dsg->node, &txd->dsg_list);
   2028
   2029	dsg->len = len;
   2030	if (direction == DMA_MEM_TO_DEV) {
   2031		dsg->src_addr = buf_addr;
   2032		dsg->dst_addr = slave_addr;
   2033	} else {
   2034		dsg->src_addr = slave_addr;
   2035		dsg->dst_addr = buf_addr;
   2036	}
   2037
   2038	return 0;
   2039}
   2040
   2041static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
   2042		struct dma_chan *chan, struct scatterlist *sgl,
   2043		unsigned int sg_len, enum dma_transfer_direction direction,
   2044		unsigned long flags, void *context)
   2045{
   2046	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2047	struct pl08x_driver_data *pl08x = plchan->host;
   2048	struct pl08x_txd *txd;
   2049	struct scatterlist *sg;
   2050	int ret, tmp;
   2051	dma_addr_t slave_addr;
   2052
   2053	dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
   2054			__func__, sg_dma_len(sgl), plchan->name);
   2055
   2056	txd = pl08x_init_txd(chan, direction, &slave_addr);
   2057	if (!txd)
   2058		return NULL;
   2059
   2060	for_each_sg(sgl, sg, sg_len, tmp) {
   2061		ret = pl08x_tx_add_sg(txd, direction, slave_addr,
   2062				      sg_dma_address(sg),
   2063				      sg_dma_len(sg));
   2064		if (ret) {
   2065			pl08x_release_mux(plchan);
   2066			pl08x_free_txd(pl08x, txd);
   2067			dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
   2068					__func__);
   2069			return NULL;
   2070		}
   2071	}
   2072
   2073	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
   2074	if (!ret) {
   2075		pl08x_release_mux(plchan);
   2076		pl08x_free_txd(pl08x, txd);
   2077		return NULL;
   2078	}
   2079
   2080	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
   2081}
   2082
   2083static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
   2084		struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
   2085		size_t period_len, enum dma_transfer_direction direction,
   2086		unsigned long flags)
   2087{
   2088	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2089	struct pl08x_driver_data *pl08x = plchan->host;
   2090	struct pl08x_txd *txd;
   2091	int ret, tmp;
   2092	dma_addr_t slave_addr;
   2093
   2094	dev_dbg(&pl08x->adev->dev,
   2095		"%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
   2096		__func__, period_len, buf_len,
   2097		direction == DMA_MEM_TO_DEV ? "to" : "from",
   2098		plchan->name);
   2099
   2100	txd = pl08x_init_txd(chan, direction, &slave_addr);
   2101	if (!txd)
   2102		return NULL;
   2103
   2104	txd->cyclic = true;
   2105	txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
   2106	for (tmp = 0; tmp < buf_len; tmp += period_len) {
   2107		ret = pl08x_tx_add_sg(txd, direction, slave_addr,
   2108				      buf_addr + tmp, period_len);
   2109		if (ret) {
   2110			pl08x_release_mux(plchan);
   2111			pl08x_free_txd(pl08x, txd);
   2112			return NULL;
   2113		}
   2114	}
   2115
   2116	ret = pl08x_fill_llis_for_desc(plchan->host, txd);
   2117	if (!ret) {
   2118		pl08x_release_mux(plchan);
   2119		pl08x_free_txd(pl08x, txd);
   2120		return NULL;
   2121	}
   2122
   2123	return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
   2124}
   2125
   2126static int pl08x_config(struct dma_chan *chan,
   2127			struct dma_slave_config *config)
   2128{
   2129	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2130	struct pl08x_driver_data *pl08x = plchan->host;
   2131
   2132	if (!plchan->slave)
   2133		return -EINVAL;
   2134
   2135	/* Reject definitely invalid configurations */
   2136	if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
   2137	    config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
   2138		return -EINVAL;
   2139
   2140	if (config->device_fc && pl08x->vd->pl080s) {
   2141		dev_err(&pl08x->adev->dev,
   2142			"%s: PL080S does not support peripheral flow control\n",
   2143			__func__);
   2144		return -EINVAL;
   2145	}
   2146
   2147	plchan->cfg = *config;
   2148
   2149	return 0;
   2150}
   2151
   2152static int pl08x_terminate_all(struct dma_chan *chan)
   2153{
   2154	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2155	struct pl08x_driver_data *pl08x = plchan->host;
   2156	unsigned long flags;
   2157
   2158	spin_lock_irqsave(&plchan->vc.lock, flags);
   2159	if (!plchan->phychan && !plchan->at) {
   2160		spin_unlock_irqrestore(&plchan->vc.lock, flags);
   2161		return 0;
   2162	}
   2163
   2164	plchan->state = PL08X_CHAN_IDLE;
   2165
   2166	if (plchan->phychan) {
   2167		/*
   2168		 * Mark physical channel as free and free any slave
   2169		 * signal
   2170		 */
   2171		pl08x_phy_free(plchan);
   2172	}
   2173	/* Dequeue jobs and free LLIs */
   2174	if (plchan->at) {
   2175		vchan_terminate_vdesc(&plchan->at->vd);
   2176		plchan->at = NULL;
   2177	}
   2178	/* Dequeue jobs not yet fired as well */
   2179	pl08x_free_txd_list(pl08x, plchan);
   2180
   2181	spin_unlock_irqrestore(&plchan->vc.lock, flags);
   2182
   2183	return 0;
   2184}
   2185
   2186static void pl08x_synchronize(struct dma_chan *chan)
   2187{
   2188	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2189
   2190	vchan_synchronize(&plchan->vc);
   2191}
   2192
   2193static int pl08x_pause(struct dma_chan *chan)
   2194{
   2195	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2196	unsigned long flags;
   2197
   2198	/*
   2199	 * Anything succeeds on channels with no physical allocation and
   2200	 * no queued transfers.
   2201	 */
   2202	spin_lock_irqsave(&plchan->vc.lock, flags);
   2203	if (!plchan->phychan && !plchan->at) {
   2204		spin_unlock_irqrestore(&plchan->vc.lock, flags);
   2205		return 0;
   2206	}
   2207
   2208	pl08x_pause_phy_chan(plchan->phychan);
   2209	plchan->state = PL08X_CHAN_PAUSED;
   2210
   2211	spin_unlock_irqrestore(&plchan->vc.lock, flags);
   2212
   2213	return 0;
   2214}
   2215
   2216static int pl08x_resume(struct dma_chan *chan)
   2217{
   2218	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2219	unsigned long flags;
   2220
   2221	/*
   2222	 * Anything succeeds on channels with no physical allocation and
   2223	 * no queued transfers.
   2224	 */
   2225	spin_lock_irqsave(&plchan->vc.lock, flags);
   2226	if (!plchan->phychan && !plchan->at) {
   2227		spin_unlock_irqrestore(&plchan->vc.lock, flags);
   2228		return 0;
   2229	}
   2230
   2231	pl08x_resume_phy_chan(plchan->phychan);
   2232	plchan->state = PL08X_CHAN_RUNNING;
   2233
   2234	spin_unlock_irqrestore(&plchan->vc.lock, flags);
   2235
   2236	return 0;
   2237}
   2238
   2239bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
   2240{
   2241	struct pl08x_dma_chan *plchan;
   2242	char *name = chan_id;
   2243
   2244	/* Reject channels for devices not bound to this driver */
   2245	if (chan->device->dev->driver != &pl08x_amba_driver.drv)
   2246		return false;
   2247
   2248	plchan = to_pl08x_chan(chan);
   2249
   2250	/* Check that the channel is not taken! */
   2251	if (!strcmp(plchan->name, name))
   2252		return true;
   2253
   2254	return false;
   2255}
   2256EXPORT_SYMBOL_GPL(pl08x_filter_id);
   2257
   2258static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id)
   2259{
   2260	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
   2261
   2262	return plchan->cd == chan_id;
   2263}
   2264
   2265/*
   2266 * Just check that the device is there and active
   2267 * TODO: turn this bit on/off depending on the number of physical channels
   2268 * actually used, if it is zero... well shut it off. That will save some
   2269 * power. Cut the clock at the same time.
   2270 */
   2271static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
   2272{
   2273	/* The Nomadik variant does not have the config register */
   2274	if (pl08x->vd->nomadik)
   2275		return;
   2276	/* The FTDMAC020 variant does this in another register */
   2277	if (pl08x->vd->ftdmac020) {
   2278		writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR);
   2279		return;
   2280	}
   2281	writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
   2282}
   2283
   2284static irqreturn_t pl08x_irq(int irq, void *dev)
   2285{
   2286	struct pl08x_driver_data *pl08x = dev;
   2287	u32 mask = 0, err, tc, i;
   2288
   2289	/* check & clear - ERR & TC interrupts */
   2290	err = readl(pl08x->base + PL080_ERR_STATUS);
   2291	if (err) {
   2292		dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
   2293			__func__, err);
   2294		writel(err, pl08x->base + PL080_ERR_CLEAR);
   2295	}
   2296	tc = readl(pl08x->base + PL080_TC_STATUS);
   2297	if (tc)
   2298		writel(tc, pl08x->base + PL080_TC_CLEAR);
   2299
   2300	if (!err && !tc)
   2301		return IRQ_NONE;
   2302
   2303	for (i = 0; i < pl08x->vd->channels; i++) {
   2304		if ((BIT(i) & err) || (BIT(i) & tc)) {
   2305			/* Locate physical channel */
   2306			struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
   2307			struct pl08x_dma_chan *plchan = phychan->serving;
   2308			struct pl08x_txd *tx;
   2309
   2310			if (!plchan) {
   2311				dev_err(&pl08x->adev->dev,
   2312					"%s Error TC interrupt on unused channel: 0x%08x\n",
   2313					__func__, i);
   2314				continue;
   2315			}
   2316
   2317			spin_lock(&plchan->vc.lock);
   2318			tx = plchan->at;
   2319			if (tx && tx->cyclic) {
   2320				vchan_cyclic_callback(&tx->vd);
   2321			} else if (tx) {
   2322				plchan->at = NULL;
   2323				/*
   2324				 * This descriptor is done, release its mux
   2325				 * reservation.
   2326				 */
   2327				pl08x_release_mux(plchan);
   2328				tx->done = true;
   2329				vchan_cookie_complete(&tx->vd);
   2330
   2331				/*
   2332				 * And start the next descriptor (if any),
   2333				 * otherwise free this channel.
   2334				 */
   2335				if (vchan_next_desc(&plchan->vc))
   2336					pl08x_start_next_txd(plchan);
   2337				else
   2338					pl08x_phy_free(plchan);
   2339			}
   2340			spin_unlock(&plchan->vc.lock);
   2341
   2342			mask |= BIT(i);
   2343		}
   2344	}
   2345
   2346	return mask ? IRQ_HANDLED : IRQ_NONE;
   2347}
   2348
   2349static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
   2350{
   2351	chan->slave = true;
   2352	chan->name = chan->cd->bus_id;
   2353	chan->cfg.src_addr = chan->cd->addr;
   2354	chan->cfg.dst_addr = chan->cd->addr;
   2355}
   2356
   2357/*
   2358 * Initialise the DMAC memcpy/slave channels.
   2359 * Make a local wrapper to hold required data
   2360 */
   2361static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
   2362		struct dma_device *dmadev, unsigned int channels, bool slave)
   2363{
   2364	struct pl08x_dma_chan *chan;
   2365	int i;
   2366
   2367	INIT_LIST_HEAD(&dmadev->channels);
   2368
   2369	/*
   2370	 * Register as many many memcpy as we have physical channels,
   2371	 * we won't always be able to use all but the code will have
   2372	 * to cope with that situation.
   2373	 */
   2374	for (i = 0; i < channels; i++) {
   2375		chan = kzalloc(sizeof(*chan), GFP_KERNEL);
   2376		if (!chan)
   2377			return -ENOMEM;
   2378
   2379		chan->host = pl08x;
   2380		chan->state = PL08X_CHAN_IDLE;
   2381		chan->signal = -1;
   2382
   2383		if (slave) {
   2384			chan->cd = &pl08x->pd->slave_channels[i];
   2385			/*
   2386			 * Some implementations have muxed signals, whereas some
   2387			 * use a mux in front of the signals and need dynamic
   2388			 * assignment of signals.
   2389			 */
   2390			chan->signal = i;
   2391			pl08x_dma_slave_init(chan);
   2392		} else {
   2393			chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL);
   2394			if (!chan->cd) {
   2395				kfree(chan);
   2396				return -ENOMEM;
   2397			}
   2398			chan->cd->bus_id = "memcpy";
   2399			chan->cd->periph_buses = pl08x->pd->mem_buses;
   2400			chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
   2401			if (!chan->name) {
   2402				kfree(chan->cd);
   2403				kfree(chan);
   2404				return -ENOMEM;
   2405			}
   2406		}
   2407		dev_dbg(&pl08x->adev->dev,
   2408			 "initialize virtual channel \"%s\"\n",
   2409			 chan->name);
   2410
   2411		chan->vc.desc_free = pl08x_desc_free;
   2412		vchan_init(&chan->vc, dmadev);
   2413	}
   2414	dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
   2415		 i, slave ? "slave" : "memcpy");
   2416	return i;
   2417}
   2418
   2419static void pl08x_free_virtual_channels(struct dma_device *dmadev)
   2420{
   2421	struct pl08x_dma_chan *chan = NULL;
   2422	struct pl08x_dma_chan *next;
   2423
   2424	list_for_each_entry_safe(chan,
   2425				 next, &dmadev->channels, vc.chan.device_node) {
   2426		list_del(&chan->vc.chan.device_node);
   2427		kfree(chan);
   2428	}
   2429}
   2430
   2431#ifdef CONFIG_DEBUG_FS
   2432static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
   2433{
   2434	switch (state) {
   2435	case PL08X_CHAN_IDLE:
   2436		return "idle";
   2437	case PL08X_CHAN_RUNNING:
   2438		return "running";
   2439	case PL08X_CHAN_PAUSED:
   2440		return "paused";
   2441	case PL08X_CHAN_WAITING:
   2442		return "waiting";
   2443	default:
   2444		break;
   2445	}
   2446	return "UNKNOWN STATE";
   2447}
   2448
   2449static int pl08x_debugfs_show(struct seq_file *s, void *data)
   2450{
   2451	struct pl08x_driver_data *pl08x = s->private;
   2452	struct pl08x_dma_chan *chan;
   2453	struct pl08x_phy_chan *ch;
   2454	unsigned long flags;
   2455	int i;
   2456
   2457	seq_printf(s, "PL08x physical channels:\n");
   2458	seq_printf(s, "CHANNEL:\tUSER:\n");
   2459	seq_printf(s, "--------\t-----\n");
   2460	for (i = 0; i < pl08x->vd->channels; i++) {
   2461		struct pl08x_dma_chan *virt_chan;
   2462
   2463		ch = &pl08x->phy_chans[i];
   2464
   2465		spin_lock_irqsave(&ch->lock, flags);
   2466		virt_chan = ch->serving;
   2467
   2468		seq_printf(s, "%d\t\t%s%s\n",
   2469			   ch->id,
   2470			   virt_chan ? virt_chan->name : "(none)",
   2471			   ch->locked ? " LOCKED" : "");
   2472
   2473		spin_unlock_irqrestore(&ch->lock, flags);
   2474	}
   2475
   2476	seq_printf(s, "\nPL08x virtual memcpy channels:\n");
   2477	seq_printf(s, "CHANNEL:\tSTATE:\n");
   2478	seq_printf(s, "--------\t------\n");
   2479	list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
   2480		seq_printf(s, "%s\t\t%s\n", chan->name,
   2481			   pl08x_state_str(chan->state));
   2482	}
   2483
   2484	if (pl08x->has_slave) {
   2485		seq_printf(s, "\nPL08x virtual slave channels:\n");
   2486		seq_printf(s, "CHANNEL:\tSTATE:\n");
   2487		seq_printf(s, "--------\t------\n");
   2488		list_for_each_entry(chan, &pl08x->slave.channels,
   2489				    vc.chan.device_node) {
   2490			seq_printf(s, "%s\t\t%s\n", chan->name,
   2491				   pl08x_state_str(chan->state));
   2492		}
   2493	}
   2494
   2495	return 0;
   2496}
   2497
   2498DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs);
   2499
   2500static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
   2501{
   2502	/* Expose a simple debugfs interface to view all clocks */
   2503	debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO,
   2504			    NULL, pl08x, &pl08x_debugfs_fops);
   2505}
   2506
   2507#else
   2508static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
   2509{
   2510}
   2511#endif
   2512
   2513#ifdef CONFIG_OF
   2514static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,
   2515					 u32 id)
   2516{
   2517	struct pl08x_dma_chan *chan;
   2518
   2519	/* Trying to get a slave channel from something with no slave support */
   2520	if (!pl08x->has_slave)
   2521		return NULL;
   2522
   2523	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
   2524		if (chan->signal == id)
   2525			return &chan->vc.chan;
   2526	}
   2527
   2528	return NULL;
   2529}
   2530
   2531static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
   2532				       struct of_dma *ofdma)
   2533{
   2534	struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
   2535	struct dma_chan *dma_chan;
   2536	struct pl08x_dma_chan *plchan;
   2537
   2538	if (!pl08x)
   2539		return NULL;
   2540
   2541	if (dma_spec->args_count != 2) {
   2542		dev_err(&pl08x->adev->dev,
   2543			"DMA channel translation requires two cells\n");
   2544		return NULL;
   2545	}
   2546
   2547	dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);
   2548	if (!dma_chan) {
   2549		dev_err(&pl08x->adev->dev,
   2550			"DMA slave channel not found\n");
   2551		return NULL;
   2552	}
   2553
   2554	plchan = to_pl08x_chan(dma_chan);
   2555	dev_dbg(&pl08x->adev->dev,
   2556		"translated channel for signal %d\n",
   2557		dma_spec->args[0]);
   2558
   2559	/* Augment channel data for applicable AHB buses */
   2560	plchan->cd->periph_buses = dma_spec->args[1];
   2561	return dma_get_slave_channel(dma_chan);
   2562}
   2563
   2564static int pl08x_of_probe(struct amba_device *adev,
   2565			  struct pl08x_driver_data *pl08x,
   2566			  struct device_node *np)
   2567{
   2568	struct pl08x_platform_data *pd;
   2569	struct pl08x_channel_data *chanp = NULL;
   2570	u32 val;
   2571	int ret;
   2572	int i;
   2573
   2574	pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);
   2575	if (!pd)
   2576		return -ENOMEM;
   2577
   2578	/* Eligible bus masters for fetching LLIs */
   2579	if (of_property_read_bool(np, "lli-bus-interface-ahb1"))
   2580		pd->lli_buses |= PL08X_AHB1;
   2581	if (of_property_read_bool(np, "lli-bus-interface-ahb2"))
   2582		pd->lli_buses |= PL08X_AHB2;
   2583	if (!pd->lli_buses) {
   2584		dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");
   2585		pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;
   2586	}
   2587
   2588	/* Eligible bus masters for memory access */
   2589	if (of_property_read_bool(np, "mem-bus-interface-ahb1"))
   2590		pd->mem_buses |= PL08X_AHB1;
   2591	if (of_property_read_bool(np, "mem-bus-interface-ahb2"))
   2592		pd->mem_buses |= PL08X_AHB2;
   2593	if (!pd->mem_buses) {
   2594		dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");
   2595		pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;
   2596	}
   2597
   2598	/* Parse the memcpy channel properties */
   2599	ret = of_property_read_u32(np, "memcpy-burst-size", &val);
   2600	if (ret) {
   2601		dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");
   2602		val = 1;
   2603	}
   2604	switch (val) {
   2605	default:
   2606		dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");
   2607		fallthrough;
   2608	case 1:
   2609		pd->memcpy_burst_size = PL08X_BURST_SZ_1;
   2610		break;
   2611	case 4:
   2612		pd->memcpy_burst_size = PL08X_BURST_SZ_4;
   2613		break;
   2614	case 8:
   2615		pd->memcpy_burst_size = PL08X_BURST_SZ_8;
   2616		break;
   2617	case 16:
   2618		pd->memcpy_burst_size = PL08X_BURST_SZ_16;
   2619		break;
   2620	case 32:
   2621		pd->memcpy_burst_size = PL08X_BURST_SZ_32;
   2622		break;
   2623	case 64:
   2624		pd->memcpy_burst_size = PL08X_BURST_SZ_64;
   2625		break;
   2626	case 128:
   2627		pd->memcpy_burst_size = PL08X_BURST_SZ_128;
   2628		break;
   2629	case 256:
   2630		pd->memcpy_burst_size = PL08X_BURST_SZ_256;
   2631		break;
   2632	}
   2633
   2634	ret = of_property_read_u32(np, "memcpy-bus-width", &val);
   2635	if (ret) {
   2636		dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");
   2637		val = 8;
   2638	}
   2639	switch (val) {
   2640	default:
   2641		dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");
   2642		fallthrough;
   2643	case 8:
   2644		pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS;
   2645		break;
   2646	case 16:
   2647		pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS;
   2648		break;
   2649	case 32:
   2650		pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS;
   2651		break;
   2652	}
   2653
   2654	/*
   2655	 * Allocate channel data for all possible slave channels (one
   2656	 * for each possible signal), channels will then be allocated
   2657	 * for a device and have it's AHB interfaces set up at
   2658	 * translation time.
   2659	 */
   2660	if (pl08x->vd->signals) {
   2661		chanp = devm_kcalloc(&adev->dev,
   2662				     pl08x->vd->signals,
   2663				     sizeof(struct pl08x_channel_data),
   2664				     GFP_KERNEL);
   2665		if (!chanp)
   2666			return -ENOMEM;
   2667
   2668		pd->slave_channels = chanp;
   2669		for (i = 0; i < pl08x->vd->signals; i++) {
   2670			/*
   2671			 * chanp->periph_buses will be assigned at translation
   2672			 */
   2673			chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i);
   2674			chanp++;
   2675		}
   2676		pd->num_slave_channels = pl08x->vd->signals;
   2677	}
   2678
   2679	pl08x->pd = pd;
   2680
   2681	return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,
   2682					  pl08x);
   2683}
   2684#else
   2685static inline int pl08x_of_probe(struct amba_device *adev,
   2686				 struct pl08x_driver_data *pl08x,
   2687				 struct device_node *np)
   2688{
   2689	return -EINVAL;
   2690}
   2691#endif
   2692
   2693static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
   2694{
   2695	struct pl08x_driver_data *pl08x;
   2696	struct vendor_data *vd = id->data;
   2697	struct device_node *np = adev->dev.of_node;
   2698	u32 tsfr_size;
   2699	int ret = 0;
   2700	int i;
   2701
   2702	ret = amba_request_regions(adev, NULL);
   2703	if (ret)
   2704		return ret;
   2705
   2706	/* Ensure that we can do DMA */
   2707	ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
   2708	if (ret)
   2709		goto out_no_pl08x;
   2710
   2711	/* Create the driver state holder */
   2712	pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
   2713	if (!pl08x) {
   2714		ret = -ENOMEM;
   2715		goto out_no_pl08x;
   2716	}
   2717
   2718	/* Assign useful pointers to the driver state */
   2719	pl08x->adev = adev;
   2720	pl08x->vd = vd;
   2721
   2722	pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
   2723	if (!pl08x->base) {
   2724		ret = -ENOMEM;
   2725		goto out_no_ioremap;
   2726	}
   2727
   2728	if (vd->ftdmac020) {
   2729		u32 val;
   2730
   2731		val = readl(pl08x->base + FTDMAC020_REVISION);
   2732		dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n",
   2733			 (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);
   2734		val = readl(pl08x->base + FTDMAC020_FEATURE);
   2735		dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, "
   2736			 "%s built-in bridge, %s, %s linked lists\n",
   2737			 (val >> 12) & 0x0f,
   2738			 (val & BIT(10)) ? "no" : "has",
   2739			 (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0",
   2740			 (val & BIT(8)) ? "supports" : "does not support");
   2741
   2742		/* Vendor data from feature register */
   2743		if (!(val & BIT(8)))
   2744			dev_warn(&pl08x->adev->dev,
   2745				 "linked lists not supported, required\n");
   2746		vd->channels = (val >> 12) & 0x0f;
   2747		vd->dualmaster = !!(val & BIT(9));
   2748	}
   2749
   2750	/* Initialize memcpy engine */
   2751	dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
   2752	pl08x->memcpy.dev = &adev->dev;
   2753	pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
   2754	pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
   2755	pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
   2756	pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
   2757	pl08x->memcpy.device_config = pl08x_config;
   2758	pl08x->memcpy.device_pause = pl08x_pause;
   2759	pl08x->memcpy.device_resume = pl08x_resume;
   2760	pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
   2761	pl08x->memcpy.device_synchronize = pl08x_synchronize;
   2762	pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;
   2763	pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
   2764	pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);
   2765	pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
   2766	if (vd->ftdmac020)
   2767		pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;
   2768
   2769
   2770	/*
   2771	 * Initialize slave engine, if the block has no signals, that means
   2772	 * we have no slave support.
   2773	 */
   2774	if (vd->signals) {
   2775		pl08x->has_slave = true;
   2776		dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
   2777		dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
   2778		pl08x->slave.dev = &adev->dev;
   2779		pl08x->slave.device_free_chan_resources =
   2780			pl08x_free_chan_resources;
   2781		pl08x->slave.device_tx_status = pl08x_dma_tx_status;
   2782		pl08x->slave.device_issue_pending = pl08x_issue_pending;
   2783		pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
   2784		pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
   2785		pl08x->slave.device_config = pl08x_config;
   2786		pl08x->slave.device_pause = pl08x_pause;
   2787		pl08x->slave.device_resume = pl08x_resume;
   2788		pl08x->slave.device_terminate_all = pl08x_terminate_all;
   2789		pl08x->slave.device_synchronize = pl08x_synchronize;
   2790		pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;
   2791		pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
   2792		pl08x->slave.directions =
   2793			BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
   2794		pl08x->slave.residue_granularity =
   2795			DMA_RESIDUE_GRANULARITY_SEGMENT;
   2796	}
   2797
   2798	/* Get the platform data */
   2799	pl08x->pd = dev_get_platdata(&adev->dev);
   2800	if (!pl08x->pd) {
   2801		if (np) {
   2802			ret = pl08x_of_probe(adev, pl08x, np);
   2803			if (ret)
   2804				goto out_no_platdata;
   2805		} else {
   2806			dev_err(&adev->dev, "no platform data supplied\n");
   2807			ret = -EINVAL;
   2808			goto out_no_platdata;
   2809		}
   2810	} else {
   2811		pl08x->slave.filter.map = pl08x->pd->slave_map;
   2812		pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len;
   2813		pl08x->slave.filter.fn = pl08x_filter_fn;
   2814	}
   2815
   2816	/* By default, AHB1 only.  If dualmaster, from platform */
   2817	pl08x->lli_buses = PL08X_AHB1;
   2818	pl08x->mem_buses = PL08X_AHB1;
   2819	if (pl08x->vd->dualmaster) {
   2820		pl08x->lli_buses = pl08x->pd->lli_buses;
   2821		pl08x->mem_buses = pl08x->pd->mem_buses;
   2822	}
   2823
   2824	if (vd->pl080s)
   2825		pl08x->lli_words = PL080S_LLI_WORDS;
   2826	else
   2827		pl08x->lli_words = PL080_LLI_WORDS;
   2828	tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
   2829
   2830	/* A DMA memory pool for LLIs, align on 1-byte boundary */
   2831	pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
   2832						tsfr_size, PL08X_ALIGN, 0);
   2833	if (!pl08x->pool) {
   2834		ret = -ENOMEM;
   2835		goto out_no_lli_pool;
   2836	}
   2837
   2838	/* Turn on the PL08x */
   2839	pl08x_ensure_on(pl08x);
   2840
   2841	/* Clear any pending interrupts */
   2842	if (vd->ftdmac020)
   2843		/* This variant has error IRQs in bits 16-19 */
   2844		writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR);
   2845	else
   2846		writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
   2847	writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
   2848
   2849	/* Attach the interrupt handler */
   2850	ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);
   2851	if (ret) {
   2852		dev_err(&adev->dev, "%s failed to request interrupt %d\n",
   2853			__func__, adev->irq[0]);
   2854		goto out_no_irq;
   2855	}
   2856
   2857	/* Initialize physical channels */
   2858	pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
   2859			GFP_KERNEL);
   2860	if (!pl08x->phy_chans) {
   2861		ret = -ENOMEM;
   2862		goto out_no_phychans;
   2863	}
   2864
   2865	for (i = 0; i < vd->channels; i++) {
   2866		struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
   2867
   2868		ch->id = i;
   2869		ch->base = pl08x->base + PL080_Cx_BASE(i);
   2870		if (vd->ftdmac020) {
   2871			/* FTDMA020 has a special channel busy register */
   2872			ch->reg_busy = ch->base + FTDMAC020_CH_BUSY;
   2873			ch->reg_config = ch->base + FTDMAC020_CH_CFG;
   2874			ch->reg_control = ch->base + FTDMAC020_CH_CSR;
   2875			ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR;
   2876			ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR;
   2877			ch->reg_lli = ch->base + FTDMAC020_CH_LLP;
   2878			ch->ftdmac020 = true;
   2879		} else {
   2880			ch->reg_config = ch->base + vd->config_offset;
   2881			ch->reg_control = ch->base + PL080_CH_CONTROL;
   2882			ch->reg_src = ch->base + PL080_CH_SRC_ADDR;
   2883			ch->reg_dst = ch->base + PL080_CH_DST_ADDR;
   2884			ch->reg_lli = ch->base + PL080_CH_LLI;
   2885		}
   2886		if (vd->pl080s)
   2887			ch->pl080s = true;
   2888
   2889		spin_lock_init(&ch->lock);
   2890
   2891		/*
   2892		 * Nomadik variants can have channels that are locked
   2893		 * down for the secure world only. Lock up these channels
   2894		 * by perpetually serving a dummy virtual channel.
   2895		 */
   2896		if (vd->nomadik) {
   2897			u32 val;
   2898
   2899			val = readl(ch->reg_config);
   2900			if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
   2901				dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
   2902				ch->locked = true;
   2903			}
   2904		}
   2905
   2906		dev_dbg(&adev->dev, "physical channel %d is %s\n",
   2907			i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
   2908	}
   2909
   2910	/* Register as many memcpy channels as there are physical channels */
   2911	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
   2912					      pl08x->vd->channels, false);
   2913	if (ret <= 0) {
   2914		dev_warn(&pl08x->adev->dev,
   2915			 "%s failed to enumerate memcpy channels - %d\n",
   2916			 __func__, ret);
   2917		goto out_no_memcpy;
   2918	}
   2919
   2920	/* Register slave channels */
   2921	if (pl08x->has_slave) {
   2922		ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
   2923					pl08x->pd->num_slave_channels, true);
   2924		if (ret < 0) {
   2925			dev_warn(&pl08x->adev->dev,
   2926				 "%s failed to enumerate slave channels - %d\n",
   2927				 __func__, ret);
   2928			goto out_no_slave;
   2929		}
   2930	}
   2931
   2932	ret = dma_async_device_register(&pl08x->memcpy);
   2933	if (ret) {
   2934		dev_warn(&pl08x->adev->dev,
   2935			"%s failed to register memcpy as an async device - %d\n",
   2936			__func__, ret);
   2937		goto out_no_memcpy_reg;
   2938	}
   2939
   2940	if (pl08x->has_slave) {
   2941		ret = dma_async_device_register(&pl08x->slave);
   2942		if (ret) {
   2943			dev_warn(&pl08x->adev->dev,
   2944			"%s failed to register slave as an async device - %d\n",
   2945			__func__, ret);
   2946			goto out_no_slave_reg;
   2947		}
   2948	}
   2949
   2950	amba_set_drvdata(adev, pl08x);
   2951	init_pl08x_debugfs(pl08x);
   2952	dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",
   2953		 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),
   2954		 (unsigned long long)adev->res.start, adev->irq[0]);
   2955
   2956	return 0;
   2957
   2958out_no_slave_reg:
   2959	dma_async_device_unregister(&pl08x->memcpy);
   2960out_no_memcpy_reg:
   2961	if (pl08x->has_slave)
   2962		pl08x_free_virtual_channels(&pl08x->slave);
   2963out_no_slave:
   2964	pl08x_free_virtual_channels(&pl08x->memcpy);
   2965out_no_memcpy:
   2966	kfree(pl08x->phy_chans);
   2967out_no_phychans:
   2968	free_irq(adev->irq[0], pl08x);
   2969out_no_irq:
   2970	dma_pool_destroy(pl08x->pool);
   2971out_no_lli_pool:
   2972out_no_platdata:
   2973	iounmap(pl08x->base);
   2974out_no_ioremap:
   2975	kfree(pl08x);
   2976out_no_pl08x:
   2977	amba_release_regions(adev);
   2978	return ret;
   2979}
   2980
   2981/* PL080 has 8 channels and the PL080 have just 2 */
   2982static struct vendor_data vendor_pl080 = {
   2983	.config_offset = PL080_CH_CONFIG,
   2984	.channels = 8,
   2985	.signals = 16,
   2986	.dualmaster = true,
   2987	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
   2988};
   2989
   2990static struct vendor_data vendor_nomadik = {
   2991	.config_offset = PL080_CH_CONFIG,
   2992	.channels = 8,
   2993	.signals = 32,
   2994	.dualmaster = true,
   2995	.nomadik = true,
   2996	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
   2997};
   2998
   2999static struct vendor_data vendor_pl080s = {
   3000	.config_offset = PL080S_CH_CONFIG,
   3001	.channels = 8,
   3002	.signals = 32,
   3003	.pl080s = true,
   3004	.max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
   3005};
   3006
   3007static struct vendor_data vendor_pl081 = {
   3008	.config_offset = PL080_CH_CONFIG,
   3009	.channels = 2,
   3010	.signals = 16,
   3011	.dualmaster = false,
   3012	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
   3013};
   3014
   3015static struct vendor_data vendor_ftdmac020 = {
   3016	.config_offset = PL080_CH_CONFIG,
   3017	.ftdmac020 = true,
   3018	.max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
   3019};
   3020
   3021static const struct amba_id pl08x_ids[] = {
   3022	/* Samsung PL080S variant */
   3023	{
   3024		.id	= 0x0a141080,
   3025		.mask	= 0xffffffff,
   3026		.data	= &vendor_pl080s,
   3027	},
   3028	/* PL080 */
   3029	{
   3030		.id	= 0x00041080,
   3031		.mask	= 0x000fffff,
   3032		.data	= &vendor_pl080,
   3033	},
   3034	/* PL081 */
   3035	{
   3036		.id	= 0x00041081,
   3037		.mask	= 0x000fffff,
   3038		.data	= &vendor_pl081,
   3039	},
   3040	/* Nomadik 8815 PL080 variant */
   3041	{
   3042		.id	= 0x00280080,
   3043		.mask	= 0x00ffffff,
   3044		.data	= &vendor_nomadik,
   3045	},
   3046	/* Faraday Technology FTDMAC020 */
   3047	{
   3048		.id	= 0x0003b080,
   3049		.mask	= 0x000fffff,
   3050		.data	= &vendor_ftdmac020,
   3051	},
   3052	{ 0, 0 },
   3053};
   3054
   3055MODULE_DEVICE_TABLE(amba, pl08x_ids);
   3056
   3057static struct amba_driver pl08x_amba_driver = {
   3058	.drv.name	= DRIVER_NAME,
   3059	.id_table	= pl08x_ids,
   3060	.probe		= pl08x_probe,
   3061};
   3062
   3063static int __init pl08x_init(void)
   3064{
   3065	int retval;
   3066	retval = amba_driver_register(&pl08x_amba_driver);
   3067	if (retval)
   3068		printk(KERN_WARNING DRIVER_NAME
   3069		       "failed to register as an AMBA device (%d)\n",
   3070		       retval);
   3071	return retval;
   3072}
   3073subsys_initcall(pl08x_init);