cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

spi-qcom-qspi.c (16637B)


      1// SPDX-License-Identifier: GPL-2.0
      2// Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
      3
      4#include <linux/clk.h>
      5#include <linux/interconnect.h>
      6#include <linux/interrupt.h>
      7#include <linux/io.h>
      8#include <linux/module.h>
      9#include <linux/of.h>
     10#include <linux/of_platform.h>
     11#include <linux/pm_runtime.h>
     12#include <linux/pm_opp.h>
     13#include <linux/spi/spi.h>
     14#include <linux/spi/spi-mem.h>
     15
     16
     17#define QSPI_NUM_CS		2
     18#define QSPI_BYTES_PER_WORD	4
     19
     20#define MSTR_CONFIG		0x0000
     21#define FULL_CYCLE_MODE		BIT(3)
     22#define FB_CLK_EN		BIT(4)
     23#define PIN_HOLDN		BIT(6)
     24#define PIN_WPN			BIT(7)
     25#define DMA_ENABLE		BIT(8)
     26#define BIG_ENDIAN_MODE		BIT(9)
     27#define SPI_MODE_MSK		0xc00
     28#define SPI_MODE_SHFT		10
     29#define CHIP_SELECT_NUM		BIT(12)
     30#define SBL_EN			BIT(13)
     31#define LPA_BASE_MSK		0x3c000
     32#define LPA_BASE_SHFT		14
     33#define TX_DATA_DELAY_MSK	0xc0000
     34#define TX_DATA_DELAY_SHFT	18
     35#define TX_CLK_DELAY_MSK	0x300000
     36#define TX_CLK_DELAY_SHFT	20
     37#define TX_CS_N_DELAY_MSK	0xc00000
     38#define TX_CS_N_DELAY_SHFT	22
     39#define TX_DATA_OE_DELAY_MSK	0x3000000
     40#define TX_DATA_OE_DELAY_SHFT	24
     41
     42#define AHB_MASTER_CFG				0x0004
     43#define HMEM_TYPE_START_MID_TRANS_MSK		0x7
     44#define HMEM_TYPE_START_MID_TRANS_SHFT		0
     45#define HMEM_TYPE_LAST_TRANS_MSK		0x38
     46#define HMEM_TYPE_LAST_TRANS_SHFT		3
     47#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK	0xc0
     48#define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT	6
     49#define HMEMTYPE_READ_TRANS_MSK			0x700
     50#define HMEMTYPE_READ_TRANS_SHFT		8
     51#define HSHARED					BIT(11)
     52#define HINNERSHARED				BIT(12)
     53
     54#define MSTR_INT_EN		0x000C
     55#define MSTR_INT_STATUS		0x0010
     56#define RESP_FIFO_UNDERRUN	BIT(0)
     57#define RESP_FIFO_NOT_EMPTY	BIT(1)
     58#define RESP_FIFO_RDY		BIT(2)
     59#define HRESP_FROM_NOC_ERR	BIT(3)
     60#define WR_FIFO_EMPTY		BIT(9)
     61#define WR_FIFO_FULL		BIT(10)
     62#define WR_FIFO_OVERRUN		BIT(11)
     63#define TRANSACTION_DONE	BIT(16)
     64#define QSPI_ERR_IRQS		(RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
     65				 WR_FIFO_OVERRUN)
     66#define QSPI_ALL_IRQS		(QSPI_ERR_IRQS | RESP_FIFO_RDY | \
     67				 WR_FIFO_EMPTY | WR_FIFO_FULL | \
     68				 TRANSACTION_DONE)
     69
     70#define PIO_XFER_CTRL		0x0014
     71#define REQUEST_COUNT_MSK	0xffff
     72
     73#define PIO_XFER_CFG		0x0018
     74#define TRANSFER_DIRECTION	BIT(0)
     75#define MULTI_IO_MODE_MSK	0xe
     76#define MULTI_IO_MODE_SHFT	1
     77#define TRANSFER_FRAGMENT	BIT(8)
     78#define SDR_1BIT		1
     79#define SDR_2BIT		2
     80#define SDR_4BIT		3
     81#define DDR_1BIT		5
     82#define DDR_2BIT		6
     83#define DDR_4BIT		7
     84#define DMA_DESC_SINGLE_SPI	1
     85#define DMA_DESC_DUAL_SPI	2
     86#define DMA_DESC_QUAD_SPI	3
     87
     88#define PIO_XFER_STATUS		0x001c
     89#define WR_FIFO_BYTES_MSK	0xffff0000
     90#define WR_FIFO_BYTES_SHFT	16
     91
     92#define PIO_DATAOUT_1B		0x0020
     93#define PIO_DATAOUT_4B		0x0024
     94
     95#define RD_FIFO_CFG		0x0028
     96#define CONTINUOUS_MODE		BIT(0)
     97
     98#define RD_FIFO_STATUS	0x002c
     99#define FIFO_EMPTY	BIT(11)
    100#define WR_CNTS_MSK	0x7f0
    101#define WR_CNTS_SHFT	4
    102#define RDY_64BYTE	BIT(3)
    103#define RDY_32BYTE	BIT(2)
    104#define RDY_16BYTE	BIT(1)
    105#define FIFO_RDY	BIT(0)
    106
    107#define RD_FIFO_RESET		0x0030
    108#define RESET_FIFO		BIT(0)
    109
    110#define CUR_MEM_ADDR		0x0048
    111#define HW_VERSION		0x004c
    112#define RD_FIFO			0x0050
    113#define SAMPLING_CLK_CFG	0x0090
    114#define SAMPLING_CLK_STATUS	0x0094
    115
    116
    117enum qspi_dir {
    118	QSPI_READ,
    119	QSPI_WRITE,
    120};
    121
    122struct qspi_xfer {
    123	union {
    124		const void *tx_buf;
    125		void *rx_buf;
    126	};
    127	unsigned int rem_bytes;
    128	unsigned int buswidth;
    129	enum qspi_dir dir;
    130	bool is_last;
    131};
    132
    133enum qspi_clocks {
    134	QSPI_CLK_CORE,
    135	QSPI_CLK_IFACE,
    136	QSPI_NUM_CLKS
    137};
    138
    139struct qcom_qspi {
    140	void __iomem *base;
    141	struct device *dev;
    142	struct clk_bulk_data *clks;
    143	struct qspi_xfer xfer;
    144	struct icc_path *icc_path_cpu_to_qspi;
    145	unsigned long last_speed;
    146	/* Lock to protect data accessed by IRQs */
    147	spinlock_t lock;
    148};
    149
    150static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
    151				   unsigned int buswidth)
    152{
    153	switch (buswidth) {
    154	case 1:
    155		return SDR_1BIT << MULTI_IO_MODE_SHFT;
    156	case 2:
    157		return SDR_2BIT << MULTI_IO_MODE_SHFT;
    158	case 4:
    159		return SDR_4BIT << MULTI_IO_MODE_SHFT;
    160	default:
    161		dev_warn_once(ctrl->dev,
    162				"Unexpected bus width: %u\n", buswidth);
    163		return SDR_1BIT << MULTI_IO_MODE_SHFT;
    164	}
    165}
    166
    167static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
    168{
    169	u32 pio_xfer_cfg;
    170	const struct qspi_xfer *xfer;
    171
    172	xfer = &ctrl->xfer;
    173	pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
    174	pio_xfer_cfg &= ~TRANSFER_DIRECTION;
    175	pio_xfer_cfg |= xfer->dir;
    176	if (xfer->is_last)
    177		pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
    178	else
    179		pio_xfer_cfg |= TRANSFER_FRAGMENT;
    180	pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
    181	pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
    182
    183	writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
    184}
    185
    186static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
    187{
    188	u32 pio_xfer_ctrl;
    189
    190	pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
    191	pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
    192	pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
    193	writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
    194}
    195
    196static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
    197{
    198	u32 ints;
    199
    200	qcom_qspi_pio_xfer_cfg(ctrl);
    201
    202	/* Ack any previous interrupts that might be hanging around */
    203	writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
    204
    205	/* Setup new interrupts */
    206	if (ctrl->xfer.dir == QSPI_WRITE)
    207		ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
    208	else
    209		ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
    210	writel(ints, ctrl->base + MSTR_INT_EN);
    211
    212	/* Kick off the transfer */
    213	qcom_qspi_pio_xfer_ctrl(ctrl);
    214}
    215
    216static void qcom_qspi_handle_err(struct spi_master *master,
    217				 struct spi_message *msg)
    218{
    219	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
    220	unsigned long flags;
    221
    222	spin_lock_irqsave(&ctrl->lock, flags);
    223	writel(0, ctrl->base + MSTR_INT_EN);
    224	ctrl->xfer.rem_bytes = 0;
    225	spin_unlock_irqrestore(&ctrl->lock, flags);
    226}
    227
    228static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
    229{
    230	int ret;
    231	unsigned int avg_bw_cpu;
    232
    233	if (speed_hz == ctrl->last_speed)
    234		return 0;
    235
    236	/* In regular operation (SBL_EN=1) core must be 4x transfer clock */
    237	ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
    238	if (ret) {
    239		dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
    240		return ret;
    241	}
    242
    243	/*
    244	 * Set BW quota for CPU as driver supports FIFO mode only.
    245	 * We don't have explicit peak requirement so keep it equal to avg_bw.
    246	 */
    247	avg_bw_cpu = Bps_to_icc(speed_hz);
    248	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
    249	if (ret) {
    250		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
    251			__func__, ret);
    252		return ret;
    253	}
    254
    255	ctrl->last_speed = speed_hz;
    256
    257	return 0;
    258}
    259
    260static int qcom_qspi_transfer_one(struct spi_master *master,
    261				  struct spi_device *slv,
    262				  struct spi_transfer *xfer)
    263{
    264	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
    265	int ret;
    266	unsigned long speed_hz;
    267	unsigned long flags;
    268
    269	speed_hz = slv->max_speed_hz;
    270	if (xfer->speed_hz)
    271		speed_hz = xfer->speed_hz;
    272
    273	ret = qcom_qspi_set_speed(ctrl, speed_hz);
    274	if (ret)
    275		return ret;
    276
    277	spin_lock_irqsave(&ctrl->lock, flags);
    278
    279	/* We are half duplex, so either rx or tx will be set */
    280	if (xfer->rx_buf) {
    281		ctrl->xfer.dir = QSPI_READ;
    282		ctrl->xfer.buswidth = xfer->rx_nbits;
    283		ctrl->xfer.rx_buf = xfer->rx_buf;
    284	} else {
    285		ctrl->xfer.dir = QSPI_WRITE;
    286		ctrl->xfer.buswidth = xfer->tx_nbits;
    287		ctrl->xfer.tx_buf = xfer->tx_buf;
    288	}
    289	ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
    290					  &master->cur_msg->transfers);
    291	ctrl->xfer.rem_bytes = xfer->len;
    292	qcom_qspi_pio_xfer(ctrl);
    293
    294	spin_unlock_irqrestore(&ctrl->lock, flags);
    295
    296	/* We'll call spi_finalize_current_transfer() when done */
    297	return 1;
    298}
    299
    300static int qcom_qspi_prepare_message(struct spi_master *master,
    301				     struct spi_message *message)
    302{
    303	u32 mstr_cfg;
    304	struct qcom_qspi *ctrl;
    305	int tx_data_oe_delay = 1;
    306	int tx_data_delay = 1;
    307	unsigned long flags;
    308
    309	ctrl = spi_master_get_devdata(master);
    310	spin_lock_irqsave(&ctrl->lock, flags);
    311
    312	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
    313	mstr_cfg &= ~CHIP_SELECT_NUM;
    314	if (message->spi->chip_select)
    315		mstr_cfg |= CHIP_SELECT_NUM;
    316
    317	mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
    318	mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
    319	mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
    320	mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
    321	mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
    322	mstr_cfg &= ~DMA_ENABLE;
    323
    324	writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
    325	spin_unlock_irqrestore(&ctrl->lock, flags);
    326
    327	return 0;
    328}
    329
    330static irqreturn_t pio_read(struct qcom_qspi *ctrl)
    331{
    332	u32 rd_fifo_status;
    333	u32 rd_fifo;
    334	unsigned int wr_cnts;
    335	unsigned int bytes_to_read;
    336	unsigned int words_to_read;
    337	u32 *word_buf;
    338	u8 *byte_buf;
    339	int i;
    340
    341	rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
    342
    343	if (!(rd_fifo_status & FIFO_RDY)) {
    344		dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
    345		return IRQ_NONE;
    346	}
    347
    348	wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
    349	wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
    350
    351	words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
    352	bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
    353
    354	if (words_to_read) {
    355		word_buf = ctrl->xfer.rx_buf;
    356		ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
    357		ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
    358		ctrl->xfer.rx_buf = word_buf + words_to_read;
    359	}
    360
    361	if (bytes_to_read) {
    362		byte_buf = ctrl->xfer.rx_buf;
    363		rd_fifo = readl(ctrl->base + RD_FIFO);
    364		ctrl->xfer.rem_bytes -= bytes_to_read;
    365		for (i = 0; i < bytes_to_read; i++)
    366			*byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
    367		ctrl->xfer.rx_buf = byte_buf;
    368	}
    369
    370	return IRQ_HANDLED;
    371}
    372
    373static irqreturn_t pio_write(struct qcom_qspi *ctrl)
    374{
    375	const void *xfer_buf = ctrl->xfer.tx_buf;
    376	const int *word_buf;
    377	const char *byte_buf;
    378	unsigned int wr_fifo_bytes;
    379	unsigned int wr_fifo_words;
    380	unsigned int wr_size;
    381	unsigned int rem_words;
    382
    383	wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
    384	wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
    385
    386	if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
    387		/* Process the last 1-3 bytes */
    388		wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
    389		ctrl->xfer.rem_bytes -= wr_size;
    390
    391		byte_buf = xfer_buf;
    392		while (wr_size--)
    393			writel(*byte_buf++,
    394			       ctrl->base + PIO_DATAOUT_1B);
    395		ctrl->xfer.tx_buf = byte_buf;
    396	} else {
    397		/*
    398		 * Process all the whole words; to keep things simple we'll
    399		 * just wait for the next interrupt to handle the last 1-3
    400		 * bytes if we don't have an even number of words.
    401		 */
    402		rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
    403		wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
    404
    405		wr_size = min(rem_words, wr_fifo_words);
    406		ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
    407
    408		word_buf = xfer_buf;
    409		iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
    410		ctrl->xfer.tx_buf = word_buf + wr_size;
    411
    412	}
    413
    414	return IRQ_HANDLED;
    415}
    416
    417static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
    418{
    419	u32 int_status;
    420	struct qcom_qspi *ctrl = dev_id;
    421	irqreturn_t ret = IRQ_NONE;
    422
    423	spin_lock(&ctrl->lock);
    424
    425	int_status = readl(ctrl->base + MSTR_INT_STATUS);
    426	writel(int_status, ctrl->base + MSTR_INT_STATUS);
    427
    428	if (ctrl->xfer.dir == QSPI_WRITE) {
    429		if (int_status & WR_FIFO_EMPTY)
    430			ret = pio_write(ctrl);
    431	} else {
    432		if (int_status & RESP_FIFO_RDY)
    433			ret = pio_read(ctrl);
    434	}
    435
    436	if (int_status & QSPI_ERR_IRQS) {
    437		if (int_status & RESP_FIFO_UNDERRUN)
    438			dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
    439		if (int_status & WR_FIFO_OVERRUN)
    440			dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
    441		if (int_status & HRESP_FROM_NOC_ERR)
    442			dev_err(ctrl->dev, "IRQ error: NOC response error\n");
    443		ret = IRQ_HANDLED;
    444	}
    445
    446	if (!ctrl->xfer.rem_bytes) {
    447		writel(0, ctrl->base + MSTR_INT_EN);
    448		spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
    449	}
    450
    451	spin_unlock(&ctrl->lock);
    452	return ret;
    453}
    454
    455static int qcom_qspi_probe(struct platform_device *pdev)
    456{
    457	int ret;
    458	struct device *dev;
    459	struct spi_master *master;
    460	struct qcom_qspi *ctrl;
    461
    462	dev = &pdev->dev;
    463
    464	master = devm_spi_alloc_master(dev, sizeof(*ctrl));
    465	if (!master)
    466		return -ENOMEM;
    467
    468	platform_set_drvdata(pdev, master);
    469
    470	ctrl = spi_master_get_devdata(master);
    471
    472	spin_lock_init(&ctrl->lock);
    473	ctrl->dev = dev;
    474	ctrl->base = devm_platform_ioremap_resource(pdev, 0);
    475	if (IS_ERR(ctrl->base))
    476		return PTR_ERR(ctrl->base);
    477
    478	ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
    479				  sizeof(*ctrl->clks), GFP_KERNEL);
    480	if (!ctrl->clks)
    481		return -ENOMEM;
    482
    483	ctrl->clks[QSPI_CLK_CORE].id = "core";
    484	ctrl->clks[QSPI_CLK_IFACE].id = "iface";
    485	ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
    486	if (ret)
    487		return ret;
    488
    489	ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
    490	if (IS_ERR(ctrl->icc_path_cpu_to_qspi))
    491		return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
    492				     "Failed to get cpu path\n");
    493
    494	/* Set BW vote for register access */
    495	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
    496				Bps_to_icc(1000));
    497	if (ret) {
    498		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
    499				__func__, ret);
    500		return ret;
    501	}
    502
    503	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
    504	if (ret) {
    505		dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
    506				__func__, ret);
    507		return ret;
    508	}
    509
    510	ret = platform_get_irq(pdev, 0);
    511	if (ret < 0)
    512		return ret;
    513	ret = devm_request_irq(dev, ret, qcom_qspi_irq, 0, dev_name(dev), ctrl);
    514	if (ret) {
    515		dev_err(dev, "Failed to request irq %d\n", ret);
    516		return ret;
    517	}
    518
    519	master->max_speed_hz = 300000000;
    520	master->num_chipselect = QSPI_NUM_CS;
    521	master->bus_num = -1;
    522	master->dev.of_node = pdev->dev.of_node;
    523	master->mode_bits = SPI_MODE_0 |
    524			    SPI_TX_DUAL | SPI_RX_DUAL |
    525			    SPI_TX_QUAD | SPI_RX_QUAD;
    526	master->flags = SPI_MASTER_HALF_DUPLEX;
    527	master->prepare_message = qcom_qspi_prepare_message;
    528	master->transfer_one = qcom_qspi_transfer_one;
    529	master->handle_err = qcom_qspi_handle_err;
    530	master->auto_runtime_pm = true;
    531
    532	ret = devm_pm_opp_set_clkname(&pdev->dev, "core");
    533	if (ret)
    534		return ret;
    535	/* OPP table is optional */
    536	ret = devm_pm_opp_of_add_table(&pdev->dev);
    537	if (ret && ret != -ENODEV) {
    538		dev_err(&pdev->dev, "invalid OPP table in device tree\n");
    539		return ret;
    540	}
    541
    542	pm_runtime_use_autosuspend(dev);
    543	pm_runtime_set_autosuspend_delay(dev, 250);
    544	pm_runtime_enable(dev);
    545
    546	ret = spi_register_master(master);
    547	if (!ret)
    548		return 0;
    549
    550	pm_runtime_disable(dev);
    551
    552	return ret;
    553}
    554
    555static int qcom_qspi_remove(struct platform_device *pdev)
    556{
    557	struct spi_master *master = platform_get_drvdata(pdev);
    558
    559	/* Unregister _before_ disabling pm_runtime() so we stop transfers */
    560	spi_unregister_master(master);
    561
    562	pm_runtime_disable(&pdev->dev);
    563
    564	return 0;
    565}
    566
    567static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
    568{
    569	struct spi_master *master = dev_get_drvdata(dev);
    570	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
    571	int ret;
    572
    573	/* Drop the performance state vote */
    574	dev_pm_opp_set_rate(dev, 0);
    575	clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
    576
    577	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
    578	if (ret) {
    579		dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
    580			__func__, ret);
    581		return ret;
    582	}
    583
    584	return 0;
    585}
    586
    587static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
    588{
    589	struct spi_master *master = dev_get_drvdata(dev);
    590	struct qcom_qspi *ctrl = spi_master_get_devdata(master);
    591	int ret;
    592
    593	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
    594	if (ret) {
    595		dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
    596			__func__, ret);
    597		return ret;
    598	}
    599
    600	ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
    601	if (ret)
    602		return ret;
    603
    604	return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
    605}
    606
    607static int __maybe_unused qcom_qspi_suspend(struct device *dev)
    608{
    609	struct spi_master *master = dev_get_drvdata(dev);
    610	int ret;
    611
    612	ret = spi_master_suspend(master);
    613	if (ret)
    614		return ret;
    615
    616	ret = pm_runtime_force_suspend(dev);
    617	if (ret)
    618		spi_master_resume(master);
    619
    620	return ret;
    621}
    622
    623static int __maybe_unused qcom_qspi_resume(struct device *dev)
    624{
    625	struct spi_master *master = dev_get_drvdata(dev);
    626	int ret;
    627
    628	ret = pm_runtime_force_resume(dev);
    629	if (ret)
    630		return ret;
    631
    632	ret = spi_master_resume(master);
    633	if (ret)
    634		pm_runtime_force_suspend(dev);
    635
    636	return ret;
    637}
    638
    639static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
    640	SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
    641			   qcom_qspi_runtime_resume, NULL)
    642	SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
    643};
    644
    645static const struct of_device_id qcom_qspi_dt_match[] = {
    646	{ .compatible = "qcom,qspi-v1", },
    647	{ }
    648};
    649MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
    650
    651static struct platform_driver qcom_qspi_driver = {
    652	.driver = {
    653		.name		= "qcom_qspi",
    654		.pm		= &qcom_qspi_dev_pm_ops,
    655		.of_match_table = qcom_qspi_dt_match,
    656	},
    657	.probe = qcom_qspi_probe,
    658	.remove = qcom_qspi_remove,
    659};
    660module_platform_driver(qcom_qspi_driver);
    661
    662MODULE_DESCRIPTION("SPI driver for QSPI cores");
    663MODULE_LICENSE("GPL v2");