cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

svc-i3c-master.c (44039B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * Silvaco dual-role I3C master driver
      4 *
      5 * Copyright (C) 2020 Silvaco
      6 * Author: Miquel RAYNAL <miquel.raynal@bootlin.com>
      7 * Based on a work from: Conor Culhane <conor.culhane@silvaco.com>
      8 */
      9
     10#include <linux/bitfield.h>
     11#include <linux/clk.h>
     12#include <linux/completion.h>
     13#include <linux/errno.h>
     14#include <linux/i3c/master.h>
     15#include <linux/interrupt.h>
     16#include <linux/iopoll.h>
     17#include <linux/list.h>
     18#include <linux/module.h>
     19#include <linux/of.h>
     20#include <linux/pinctrl/consumer.h>
     21#include <linux/platform_device.h>
     22#include <linux/pm_runtime.h>
     23
     24/* Master Mode Registers */
     25#define SVC_I3C_MCONFIG      0x000
     26#define   SVC_I3C_MCONFIG_MASTER_EN BIT(0)
     27#define   SVC_I3C_MCONFIG_DISTO(x) FIELD_PREP(BIT(3), (x))
     28#define   SVC_I3C_MCONFIG_HKEEP(x) FIELD_PREP(GENMASK(5, 4), (x))
     29#define   SVC_I3C_MCONFIG_ODSTOP(x) FIELD_PREP(BIT(6), (x))
     30#define   SVC_I3C_MCONFIG_PPBAUD(x) FIELD_PREP(GENMASK(11, 8), (x))
     31#define   SVC_I3C_MCONFIG_PPLOW(x) FIELD_PREP(GENMASK(15, 12), (x))
     32#define   SVC_I3C_MCONFIG_ODBAUD(x) FIELD_PREP(GENMASK(23, 16), (x))
     33#define   SVC_I3C_MCONFIG_ODHPP(x) FIELD_PREP(BIT(24), (x))
     34#define   SVC_I3C_MCONFIG_SKEW(x) FIELD_PREP(GENMASK(27, 25), (x))
     35#define   SVC_I3C_MCONFIG_I2CBAUD(x) FIELD_PREP(GENMASK(31, 28), (x))
     36
     37#define SVC_I3C_MCTRL        0x084
     38#define   SVC_I3C_MCTRL_REQUEST_MASK GENMASK(2, 0)
     39#define   SVC_I3C_MCTRL_REQUEST_NONE 0
     40#define   SVC_I3C_MCTRL_REQUEST_START_ADDR 1
     41#define   SVC_I3C_MCTRL_REQUEST_STOP 2
     42#define   SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK 3
     43#define   SVC_I3C_MCTRL_REQUEST_PROC_DAA 4
     44#define   SVC_I3C_MCTRL_REQUEST_AUTO_IBI 7
     45#define   SVC_I3C_MCTRL_TYPE_I3C 0
     46#define   SVC_I3C_MCTRL_TYPE_I2C BIT(4)
     47#define   SVC_I3C_MCTRL_IBIRESP_AUTO 0
     48#define   SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE 0
     49#define   SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE BIT(7)
     50#define   SVC_I3C_MCTRL_IBIRESP_NACK BIT(6)
     51#define   SVC_I3C_MCTRL_IBIRESP_MANUAL GENMASK(7, 6)
     52#define   SVC_I3C_MCTRL_DIR(x) FIELD_PREP(BIT(8), (x))
     53#define   SVC_I3C_MCTRL_DIR_WRITE 0
     54#define   SVC_I3C_MCTRL_DIR_READ 1
     55#define   SVC_I3C_MCTRL_ADDR(x) FIELD_PREP(GENMASK(15, 9), (x))
     56#define   SVC_I3C_MCTRL_RDTERM(x) FIELD_PREP(GENMASK(23, 16), (x))
     57
     58#define SVC_I3C_MSTATUS      0x088
     59#define   SVC_I3C_MSTATUS_STATE(x) FIELD_GET(GENMASK(2, 0), (x))
     60#define   SVC_I3C_MSTATUS_STATE_DAA(x) (SVC_I3C_MSTATUS_STATE(x) == 5)
     61#define   SVC_I3C_MSTATUS_STATE_IDLE(x) (SVC_I3C_MSTATUS_STATE(x) == 0)
     62#define   SVC_I3C_MSTATUS_BETWEEN(x) FIELD_GET(BIT(4), (x))
     63#define   SVC_I3C_MSTATUS_NACKED(x) FIELD_GET(BIT(5), (x))
     64#define   SVC_I3C_MSTATUS_IBITYPE(x) FIELD_GET(GENMASK(7, 6), (x))
     65#define   SVC_I3C_MSTATUS_IBITYPE_IBI 1
     66#define   SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST 2
     67#define   SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN 3
     68#define   SVC_I3C_MINT_SLVSTART BIT(8)
     69#define   SVC_I3C_MINT_MCTRLDONE BIT(9)
     70#define   SVC_I3C_MINT_COMPLETE BIT(10)
     71#define   SVC_I3C_MINT_RXPEND BIT(11)
     72#define   SVC_I3C_MINT_TXNOTFULL BIT(12)
     73#define   SVC_I3C_MINT_IBIWON BIT(13)
     74#define   SVC_I3C_MINT_ERRWARN BIT(15)
     75#define   SVC_I3C_MSTATUS_SLVSTART(x) FIELD_GET(SVC_I3C_MINT_SLVSTART, (x))
     76#define   SVC_I3C_MSTATUS_MCTRLDONE(x) FIELD_GET(SVC_I3C_MINT_MCTRLDONE, (x))
     77#define   SVC_I3C_MSTATUS_COMPLETE(x) FIELD_GET(SVC_I3C_MINT_COMPLETE, (x))
     78#define   SVC_I3C_MSTATUS_RXPEND(x) FIELD_GET(SVC_I3C_MINT_RXPEND, (x))
     79#define   SVC_I3C_MSTATUS_TXNOTFULL(x) FIELD_GET(SVC_I3C_MINT_TXNOTFULL, (x))
     80#define   SVC_I3C_MSTATUS_IBIWON(x) FIELD_GET(SVC_I3C_MINT_IBIWON, (x))
     81#define   SVC_I3C_MSTATUS_ERRWARN(x) FIELD_GET(SVC_I3C_MINT_ERRWARN, (x))
     82#define   SVC_I3C_MSTATUS_IBIADDR(x) FIELD_GET(GENMASK(30, 24), (x))
     83
     84#define SVC_I3C_IBIRULES     0x08C
     85#define   SVC_I3C_IBIRULES_ADDR(slot, addr) FIELD_PREP(GENMASK(29, 0), \
     86						       ((addr) & 0x3F) << ((slot) * 6))
     87#define   SVC_I3C_IBIRULES_ADDRS 5
     88#define   SVC_I3C_IBIRULES_MSB0 BIT(30)
     89#define   SVC_I3C_IBIRULES_NOBYTE BIT(31)
     90#define   SVC_I3C_IBIRULES_MANDBYTE 0
     91#define SVC_I3C_MINTSET      0x090
     92#define SVC_I3C_MINTCLR      0x094
     93#define SVC_I3C_MINTMASKED   0x098
     94#define SVC_I3C_MERRWARN     0x09C
     95#define SVC_I3C_MDMACTRL     0x0A0
     96#define SVC_I3C_MDATACTRL    0x0AC
     97#define   SVC_I3C_MDATACTRL_FLUSHTB BIT(0)
     98#define   SVC_I3C_MDATACTRL_FLUSHRB BIT(1)
     99#define   SVC_I3C_MDATACTRL_UNLOCK_TRIG BIT(3)
    100#define   SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL GENMASK(5, 4)
    101#define   SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY 0
    102#define   SVC_I3C_MDATACTRL_RXCOUNT(x) FIELD_GET(GENMASK(28, 24), (x))
    103#define   SVC_I3C_MDATACTRL_TXFULL BIT(30)
    104#define   SVC_I3C_MDATACTRL_RXEMPTY BIT(31)
    105
    106#define SVC_I3C_MWDATAB      0x0B0
    107#define   SVC_I3C_MWDATAB_END BIT(8)
    108
    109#define SVC_I3C_MWDATABE     0x0B4
    110#define SVC_I3C_MWDATAH      0x0B8
    111#define SVC_I3C_MWDATAHE     0x0BC
    112#define SVC_I3C_MRDATAB      0x0C0
    113#define SVC_I3C_MRDATAH      0x0C8
    114#define SVC_I3C_MWMSG_SDR    0x0D0
    115#define SVC_I3C_MRMSG_SDR    0x0D4
    116#define SVC_I3C_MWMSG_DDR    0x0D8
    117#define SVC_I3C_MRMSG_DDR    0x0DC
    118
    119#define SVC_I3C_MDYNADDR     0x0E4
    120#define   SVC_MDYNADDR_VALID BIT(0)
    121#define   SVC_MDYNADDR_ADDR(x) FIELD_PREP(GENMASK(7, 1), (x))
    122
    123#define SVC_I3C_MAX_DEVS 32
    124#define SVC_I3C_PM_TIMEOUT_MS 1000
    125
    126/* This parameter depends on the implementation and may be tuned */
    127#define SVC_I3C_FIFO_SIZE 16
    128
    129struct svc_i3c_cmd {
    130	u8 addr;
    131	bool rnw;
    132	u8 *in;
    133	const void *out;
    134	unsigned int len;
    135	unsigned int read_len;
    136	bool continued;
    137};
    138
    139struct svc_i3c_xfer {
    140	struct list_head node;
    141	struct completion comp;
    142	int ret;
    143	unsigned int type;
    144	unsigned int ncmds;
    145	struct svc_i3c_cmd cmds[];
    146};
    147
    148/**
    149 * struct svc_i3c_master - Silvaco I3C Master structure
    150 * @base: I3C master controller
    151 * @dev: Corresponding device
    152 * @regs: Memory mapping
    153 * @free_slots: Bit array of available slots
    154 * @addrs: Array containing the dynamic addresses of each attached device
    155 * @descs: Array of descriptors, one per attached device
    156 * @hj_work: Hot-join work
    157 * @ibi_work: IBI work
    158 * @irq: Main interrupt
    159 * @pclk: System clock
    160 * @fclk: Fast clock (bus)
    161 * @sclk: Slow clock (other events)
    162 * @xferqueue: Transfer queue structure
    163 * @xferqueue.list: List member
    164 * @xferqueue.cur: Current ongoing transfer
    165 * @xferqueue.lock: Queue lock
    166 * @ibi: IBI structure
    167 * @ibi.num_slots: Number of slots available in @ibi.slots
    168 * @ibi.slots: Available IBI slots
    169 * @ibi.tbq_slot: To be queued IBI slot
    170 * @ibi.lock: IBI lock
    171 */
    172struct svc_i3c_master {
    173	struct i3c_master_controller base;
    174	struct device *dev;
    175	void __iomem *regs;
    176	u32 free_slots;
    177	u8 addrs[SVC_I3C_MAX_DEVS];
    178	struct i3c_dev_desc *descs[SVC_I3C_MAX_DEVS];
    179	struct work_struct hj_work;
    180	struct work_struct ibi_work;
    181	int irq;
    182	struct clk *pclk;
    183	struct clk *fclk;
    184	struct clk *sclk;
    185	struct {
    186		struct list_head list;
    187		struct svc_i3c_xfer *cur;
    188		/* Prevent races between transfers */
    189		spinlock_t lock;
    190	} xferqueue;
    191	struct {
    192		unsigned int num_slots;
    193		struct i3c_dev_desc **slots;
    194		struct i3c_ibi_slot *tbq_slot;
    195		/* Prevent races within IBI handlers */
    196		spinlock_t lock;
    197	} ibi;
    198};
    199
    200/**
    201 * struct svc_i3c_i2c_dev_data - Device specific data
    202 * @index: Index in the master tables corresponding to this device
    203 * @ibi: IBI slot index in the master structure
    204 * @ibi_pool: IBI pool associated to this device
    205 */
    206struct svc_i3c_i2c_dev_data {
    207	u8 index;
    208	int ibi;
    209	struct i3c_generic_ibi_pool *ibi_pool;
    210};
    211
    212static bool svc_i3c_master_error(struct svc_i3c_master *master)
    213{
    214	u32 mstatus, merrwarn;
    215
    216	mstatus = readl(master->regs + SVC_I3C_MSTATUS);
    217	if (SVC_I3C_MSTATUS_ERRWARN(mstatus)) {
    218		merrwarn = readl(master->regs + SVC_I3C_MERRWARN);
    219		writel(merrwarn, master->regs + SVC_I3C_MERRWARN);
    220		dev_err(master->dev,
    221			"Error condition: MSTATUS 0x%08x, MERRWARN 0x%08x\n",
    222			mstatus, merrwarn);
    223
    224		return true;
    225	}
    226
    227	return false;
    228}
    229
    230static void svc_i3c_master_enable_interrupts(struct svc_i3c_master *master, u32 mask)
    231{
    232	writel(mask, master->regs + SVC_I3C_MINTSET);
    233}
    234
    235static void svc_i3c_master_disable_interrupts(struct svc_i3c_master *master)
    236{
    237	u32 mask = readl(master->regs + SVC_I3C_MINTSET);
    238
    239	writel(mask, master->regs + SVC_I3C_MINTCLR);
    240}
    241
    242static void svc_i3c_master_clear_merrwarn(struct svc_i3c_master *master)
    243{
    244	/* Clear pending warnings */
    245	writel(readl(master->regs + SVC_I3C_MERRWARN),
    246	       master->regs + SVC_I3C_MERRWARN);
    247}
    248
    249static void svc_i3c_master_flush_fifo(struct svc_i3c_master *master)
    250{
    251	/* Flush FIFOs */
    252	writel(SVC_I3C_MDATACTRL_FLUSHTB | SVC_I3C_MDATACTRL_FLUSHRB,
    253	       master->regs + SVC_I3C_MDATACTRL);
    254}
    255
    256static void svc_i3c_master_reset_fifo_trigger(struct svc_i3c_master *master)
    257{
    258	u32 reg;
    259
    260	/* Set RX and TX tigger levels, flush FIFOs */
    261	reg = SVC_I3C_MDATACTRL_FLUSHTB |
    262	      SVC_I3C_MDATACTRL_FLUSHRB |
    263	      SVC_I3C_MDATACTRL_UNLOCK_TRIG |
    264	      SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL |
    265	      SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY;
    266	writel(reg, master->regs + SVC_I3C_MDATACTRL);
    267}
    268
    269static void svc_i3c_master_reset(struct svc_i3c_master *master)
    270{
    271	svc_i3c_master_clear_merrwarn(master);
    272	svc_i3c_master_reset_fifo_trigger(master);
    273	svc_i3c_master_disable_interrupts(master);
    274}
    275
    276static inline struct svc_i3c_master *
    277to_svc_i3c_master(struct i3c_master_controller *master)
    278{
    279	return container_of(master, struct svc_i3c_master, base);
    280}
    281
    282static void svc_i3c_master_hj_work(struct work_struct *work)
    283{
    284	struct svc_i3c_master *master;
    285
    286	master = container_of(work, struct svc_i3c_master, hj_work);
    287	i3c_master_do_daa(&master->base);
    288}
    289
    290static struct i3c_dev_desc *
    291svc_i3c_master_dev_from_addr(struct svc_i3c_master *master,
    292			     unsigned int ibiaddr)
    293{
    294	int i;
    295
    296	for (i = 0; i < SVC_I3C_MAX_DEVS; i++)
    297		if (master->addrs[i] == ibiaddr)
    298			break;
    299
    300	if (i == SVC_I3C_MAX_DEVS)
    301		return NULL;
    302
    303	return master->descs[i];
    304}
    305
    306static void svc_i3c_master_emit_stop(struct svc_i3c_master *master)
    307{
    308	writel(SVC_I3C_MCTRL_REQUEST_STOP, master->regs + SVC_I3C_MCTRL);
    309
    310	/*
    311	 * This delay is necessary after the emission of a stop, otherwise eg.
    312	 * repeating IBIs do not get detected. There is a note in the manual
    313	 * about it, stating that the stop condition might not be settled
    314	 * correctly if a start condition follows too rapidly.
    315	 */
    316	udelay(1);
    317}
    318
    319static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master,
    320				     struct i3c_dev_desc *dev)
    321{
    322	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
    323	struct i3c_ibi_slot *slot;
    324	unsigned int count;
    325	u32 mdatactrl;
    326	u8 *buf;
    327
    328	slot = i3c_generic_ibi_get_free_slot(data->ibi_pool);
    329	if (!slot)
    330		return -ENOSPC;
    331
    332	slot->len = 0;
    333	buf = slot->data;
    334
    335	while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))  &&
    336	       slot->len < SVC_I3C_FIFO_SIZE) {
    337		mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL);
    338		count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl);
    339		readsl(master->regs + SVC_I3C_MRDATAB, buf, count);
    340		slot->len += count;
    341		buf += count;
    342	}
    343
    344	master->ibi.tbq_slot = slot;
    345
    346	return 0;
    347}
    348
    349static void svc_i3c_master_ack_ibi(struct svc_i3c_master *master,
    350				   bool mandatory_byte)
    351{
    352	unsigned int ibi_ack_nack;
    353
    354	ibi_ack_nack = SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK;
    355	if (mandatory_byte)
    356		ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE;
    357	else
    358		ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE;
    359
    360	writel(ibi_ack_nack, master->regs + SVC_I3C_MCTRL);
    361}
    362
    363static void svc_i3c_master_nack_ibi(struct svc_i3c_master *master)
    364{
    365	writel(SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK |
    366	       SVC_I3C_MCTRL_IBIRESP_NACK,
    367	       master->regs + SVC_I3C_MCTRL);
    368}
    369
    370static void svc_i3c_master_ibi_work(struct work_struct *work)
    371{
    372	struct svc_i3c_master *master = container_of(work, struct svc_i3c_master, ibi_work);
    373	struct svc_i3c_i2c_dev_data *data;
    374	unsigned int ibitype, ibiaddr;
    375	struct i3c_dev_desc *dev;
    376	u32 status, val;
    377	int ret;
    378
    379	/* Acknowledge the incoming interrupt with the AUTOIBI mechanism */
    380	writel(SVC_I3C_MCTRL_REQUEST_AUTO_IBI |
    381	       SVC_I3C_MCTRL_IBIRESP_AUTO,
    382	       master->regs + SVC_I3C_MCTRL);
    383
    384	/* Wait for IBIWON, should take approximately 100us */
    385	ret = readl_relaxed_poll_timeout(master->regs + SVC_I3C_MSTATUS, val,
    386					 SVC_I3C_MSTATUS_IBIWON(val), 0, 1000);
    387	if (ret) {
    388		dev_err(master->dev, "Timeout when polling for IBIWON\n");
    389		goto reenable_ibis;
    390	}
    391
    392	/* Clear the interrupt status */
    393	writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS);
    394
    395	status = readl(master->regs + SVC_I3C_MSTATUS);
    396	ibitype = SVC_I3C_MSTATUS_IBITYPE(status);
    397	ibiaddr = SVC_I3C_MSTATUS_IBIADDR(status);
    398
    399	/* Handle the critical responses to IBI's */
    400	switch (ibitype) {
    401	case SVC_I3C_MSTATUS_IBITYPE_IBI:
    402		dev = svc_i3c_master_dev_from_addr(master, ibiaddr);
    403		if (!dev)
    404			svc_i3c_master_nack_ibi(master);
    405		else
    406			svc_i3c_master_handle_ibi(master, dev);
    407		break;
    408	case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
    409		svc_i3c_master_ack_ibi(master, false);
    410		break;
    411	case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
    412		svc_i3c_master_nack_ibi(master);
    413		break;
    414	default:
    415		break;
    416	}
    417
    418	/*
    419	 * If an error happened, we probably got interrupted and the exchange
    420	 * timedout. In this case we just drop everything, emit a stop and wait
    421	 * for the slave to interrupt again.
    422	 */
    423	if (svc_i3c_master_error(master)) {
    424		if (master->ibi.tbq_slot) {
    425			data = i3c_dev_get_master_data(dev);
    426			i3c_generic_ibi_recycle_slot(data->ibi_pool,
    427						     master->ibi.tbq_slot);
    428			master->ibi.tbq_slot = NULL;
    429		}
    430
    431		svc_i3c_master_emit_stop(master);
    432
    433		goto reenable_ibis;
    434	}
    435
    436	/* Handle the non critical tasks */
    437	switch (ibitype) {
    438	case SVC_I3C_MSTATUS_IBITYPE_IBI:
    439		if (dev) {
    440			i3c_master_queue_ibi(dev, master->ibi.tbq_slot);
    441			master->ibi.tbq_slot = NULL;
    442		}
    443		svc_i3c_master_emit_stop(master);
    444		break;
    445	case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
    446		queue_work(master->base.wq, &master->hj_work);
    447		break;
    448	case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
    449	default:
    450		break;
    451	}
    452
    453reenable_ibis:
    454	svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
    455}
    456
    457static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id)
    458{
    459	struct svc_i3c_master *master = (struct svc_i3c_master *)dev_id;
    460	u32 active = readl(master->regs + SVC_I3C_MINTMASKED);
    461
    462	if (!SVC_I3C_MSTATUS_SLVSTART(active))
    463		return IRQ_NONE;
    464
    465	/* Clear the interrupt status */
    466	writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS);
    467
    468	svc_i3c_master_disable_interrupts(master);
    469
    470	/* Handle the interrupt in a non atomic context */
    471	queue_work(master->base.wq, &master->ibi_work);
    472
    473	return IRQ_HANDLED;
    474}
    475
    476static int svc_i3c_master_bus_init(struct i3c_master_controller *m)
    477{
    478	struct svc_i3c_master *master = to_svc_i3c_master(m);
    479	struct i3c_bus *bus = i3c_master_get_bus(m);
    480	struct i3c_device_info info = {};
    481	unsigned long fclk_rate, fclk_period_ns;
    482	unsigned int high_period_ns, od_low_period_ns;
    483	u32 ppbaud, pplow, odhpp, odbaud, odstop, i2cbaud, reg;
    484	int ret;
    485
    486	ret = pm_runtime_resume_and_get(master->dev);
    487	if (ret < 0) {
    488		dev_err(master->dev,
    489			"<%s> cannot resume i3c bus master, err: %d\n",
    490			__func__, ret);
    491		return ret;
    492	}
    493
    494	/* Timings derivation */
    495	fclk_rate = clk_get_rate(master->fclk);
    496	if (!fclk_rate) {
    497		ret = -EINVAL;
    498		goto rpm_out;
    499	}
    500
    501	fclk_period_ns = DIV_ROUND_UP(1000000000, fclk_rate);
    502
    503	/*
    504	 * Using I3C Push-Pull mode, target is 12.5MHz/80ns period.
    505	 * Simplest configuration is using a 50% duty-cycle of 40ns.
    506	 */
    507	ppbaud = DIV_ROUND_UP(40, fclk_period_ns) - 1;
    508	pplow = 0;
    509
    510	/*
    511	 * Using I3C Open-Drain mode, target is 4.17MHz/240ns with a
    512	 * duty-cycle tuned so that high levels are filetered out by
    513	 * the 50ns filter (target being 40ns).
    514	 */
    515	odhpp = 1;
    516	high_period_ns = (ppbaud + 1) * fclk_period_ns;
    517	odbaud = DIV_ROUND_UP(240 - high_period_ns, high_period_ns) - 1;
    518	od_low_period_ns = (odbaud + 1) * high_period_ns;
    519
    520	switch (bus->mode) {
    521	case I3C_BUS_MODE_PURE:
    522		i2cbaud = 0;
    523		odstop = 0;
    524		break;
    525	case I3C_BUS_MODE_MIXED_FAST:
    526	case I3C_BUS_MODE_MIXED_LIMITED:
    527		/*
    528		 * Using I2C Fm+ mode, target is 1MHz/1000ns, the difference
    529		 * between the high and low period does not really matter.
    530		 */
    531		i2cbaud = DIV_ROUND_UP(1000, od_low_period_ns) - 2;
    532		odstop = 1;
    533		break;
    534	case I3C_BUS_MODE_MIXED_SLOW:
    535		/*
    536		 * Using I2C Fm mode, target is 0.4MHz/2500ns, with the same
    537		 * constraints as the FM+ mode.
    538		 */
    539		i2cbaud = DIV_ROUND_UP(2500, od_low_period_ns) - 2;
    540		odstop = 1;
    541		break;
    542	default:
    543		goto rpm_out;
    544	}
    545
    546	reg = SVC_I3C_MCONFIG_MASTER_EN |
    547	      SVC_I3C_MCONFIG_DISTO(0) |
    548	      SVC_I3C_MCONFIG_HKEEP(0) |
    549	      SVC_I3C_MCONFIG_ODSTOP(odstop) |
    550	      SVC_I3C_MCONFIG_PPBAUD(ppbaud) |
    551	      SVC_I3C_MCONFIG_PPLOW(pplow) |
    552	      SVC_I3C_MCONFIG_ODBAUD(odbaud) |
    553	      SVC_I3C_MCONFIG_ODHPP(odhpp) |
    554	      SVC_I3C_MCONFIG_SKEW(0) |
    555	      SVC_I3C_MCONFIG_I2CBAUD(i2cbaud);
    556	writel(reg, master->regs + SVC_I3C_MCONFIG);
    557
    558	/* Master core's registration */
    559	ret = i3c_master_get_free_addr(m, 0);
    560	if (ret < 0)
    561		goto rpm_out;
    562
    563	info.dyn_addr = ret;
    564
    565	writel(SVC_MDYNADDR_VALID | SVC_MDYNADDR_ADDR(info.dyn_addr),
    566	       master->regs + SVC_I3C_MDYNADDR);
    567
    568	ret = i3c_master_set_info(&master->base, &info);
    569	if (ret)
    570		goto rpm_out;
    571
    572rpm_out:
    573	pm_runtime_mark_last_busy(master->dev);
    574	pm_runtime_put_autosuspend(master->dev);
    575
    576	return ret;
    577}
    578
    579static void svc_i3c_master_bus_cleanup(struct i3c_master_controller *m)
    580{
    581	struct svc_i3c_master *master = to_svc_i3c_master(m);
    582	int ret;
    583
    584	ret = pm_runtime_resume_and_get(master->dev);
    585	if (ret < 0) {
    586		dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
    587		return;
    588	}
    589
    590	svc_i3c_master_disable_interrupts(master);
    591
    592	/* Disable master */
    593	writel(0, master->regs + SVC_I3C_MCONFIG);
    594
    595	pm_runtime_mark_last_busy(master->dev);
    596	pm_runtime_put_autosuspend(master->dev);
    597}
    598
    599static int svc_i3c_master_reserve_slot(struct svc_i3c_master *master)
    600{
    601	unsigned int slot;
    602
    603	if (!(master->free_slots & GENMASK(SVC_I3C_MAX_DEVS - 1, 0)))
    604		return -ENOSPC;
    605
    606	slot = ffs(master->free_slots) - 1;
    607
    608	master->free_slots &= ~BIT(slot);
    609
    610	return slot;
    611}
    612
    613static void svc_i3c_master_release_slot(struct svc_i3c_master *master,
    614					unsigned int slot)
    615{
    616	master->free_slots |= BIT(slot);
    617}
    618
    619static int svc_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev)
    620{
    621	struct i3c_master_controller *m = i3c_dev_get_master(dev);
    622	struct svc_i3c_master *master = to_svc_i3c_master(m);
    623	struct svc_i3c_i2c_dev_data *data;
    624	int slot;
    625
    626	slot = svc_i3c_master_reserve_slot(master);
    627	if (slot < 0)
    628		return slot;
    629
    630	data = kzalloc(sizeof(*data), GFP_KERNEL);
    631	if (!data) {
    632		svc_i3c_master_release_slot(master, slot);
    633		return -ENOMEM;
    634	}
    635
    636	data->ibi = -1;
    637	data->index = slot;
    638	master->addrs[slot] = dev->info.dyn_addr ? dev->info.dyn_addr :
    639						   dev->info.static_addr;
    640	master->descs[slot] = dev;
    641
    642	i3c_dev_set_master_data(dev, data);
    643
    644	return 0;
    645}
    646
    647static int svc_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev,
    648					   u8 old_dyn_addr)
    649{
    650	struct i3c_master_controller *m = i3c_dev_get_master(dev);
    651	struct svc_i3c_master *master = to_svc_i3c_master(m);
    652	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
    653
    654	master->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr :
    655							  dev->info.static_addr;
    656
    657	return 0;
    658}
    659
    660static void svc_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev)
    661{
    662	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
    663	struct i3c_master_controller *m = i3c_dev_get_master(dev);
    664	struct svc_i3c_master *master = to_svc_i3c_master(m);
    665
    666	master->addrs[data->index] = 0;
    667	svc_i3c_master_release_slot(master, data->index);
    668
    669	kfree(data);
    670}
    671
    672static int svc_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev)
    673{
    674	struct i3c_master_controller *m = i2c_dev_get_master(dev);
    675	struct svc_i3c_master *master = to_svc_i3c_master(m);
    676	struct svc_i3c_i2c_dev_data *data;
    677	int slot;
    678
    679	slot = svc_i3c_master_reserve_slot(master);
    680	if (slot < 0)
    681		return slot;
    682
    683	data = kzalloc(sizeof(*data), GFP_KERNEL);
    684	if (!data) {
    685		svc_i3c_master_release_slot(master, slot);
    686		return -ENOMEM;
    687	}
    688
    689	data->index = slot;
    690	master->addrs[slot] = dev->addr;
    691
    692	i2c_dev_set_master_data(dev, data);
    693
    694	return 0;
    695}
    696
    697static void svc_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev)
    698{
    699	struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
    700	struct i3c_master_controller *m = i2c_dev_get_master(dev);
    701	struct svc_i3c_master *master = to_svc_i3c_master(m);
    702
    703	svc_i3c_master_release_slot(master, data->index);
    704
    705	kfree(data);
    706}
    707
    708static int svc_i3c_master_readb(struct svc_i3c_master *master, u8 *dst,
    709				unsigned int len)
    710{
    711	int ret, i;
    712	u32 reg;
    713
    714	for (i = 0; i < len; i++) {
    715		ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
    716						reg,
    717						SVC_I3C_MSTATUS_RXPEND(reg),
    718						0, 1000);
    719		if (ret)
    720			return ret;
    721
    722		dst[i] = readl(master->regs + SVC_I3C_MRDATAB);
    723	}
    724
    725	return 0;
    726}
    727
    728static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master,
    729					u8 *addrs, unsigned int *count)
    730{
    731	u64 prov_id[SVC_I3C_MAX_DEVS] = {}, nacking_prov_id = 0;
    732	unsigned int dev_nb = 0, last_addr = 0;
    733	u32 reg;
    734	int ret, i;
    735
    736	while (true) {
    737		/* Enter/proceed with DAA */
    738		writel(SVC_I3C_MCTRL_REQUEST_PROC_DAA |
    739		       SVC_I3C_MCTRL_TYPE_I3C |
    740		       SVC_I3C_MCTRL_IBIRESP_NACK |
    741		       SVC_I3C_MCTRL_DIR(SVC_I3C_MCTRL_DIR_WRITE),
    742		       master->regs + SVC_I3C_MCTRL);
    743
    744		/*
    745		 * Either one slave will send its ID, or the assignment process
    746		 * is done.
    747		 */
    748		ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
    749						reg,
    750						SVC_I3C_MSTATUS_RXPEND(reg) |
    751						SVC_I3C_MSTATUS_MCTRLDONE(reg),
    752						1, 1000);
    753		if (ret)
    754			return ret;
    755
    756		if (SVC_I3C_MSTATUS_RXPEND(reg)) {
    757			u8 data[6];
    758
    759			/*
    760			 * We only care about the 48-bit provisional ID yet to
    761			 * be sure a device does not nack an address twice.
    762			 * Otherwise, we would just need to flush the RX FIFO.
    763			 */
    764			ret = svc_i3c_master_readb(master, data, 6);
    765			if (ret)
    766				return ret;
    767
    768			for (i = 0; i < 6; i++)
    769				prov_id[dev_nb] |= (u64)(data[i]) << (8 * (5 - i));
    770
    771			/* We do not care about the BCR and DCR yet */
    772			ret = svc_i3c_master_readb(master, data, 2);
    773			if (ret)
    774				return ret;
    775		} else if (SVC_I3C_MSTATUS_MCTRLDONE(reg)) {
    776			if (SVC_I3C_MSTATUS_STATE_IDLE(reg) &&
    777			    SVC_I3C_MSTATUS_COMPLETE(reg)) {
    778				/*
    779				 * All devices received and acked they dynamic
    780				 * address, this is the natural end of the DAA
    781				 * procedure.
    782				 */
    783				break;
    784			} else if (SVC_I3C_MSTATUS_NACKED(reg)) {
    785				/*
    786				 * A slave device nacked the address, this is
    787				 * allowed only once, DAA will be stopped and
    788				 * then resumed. The same device is supposed to
    789				 * answer again immediately and shall ack the
    790				 * address this time.
    791				 */
    792				if (prov_id[dev_nb] == nacking_prov_id)
    793					return -EIO;
    794
    795				dev_nb--;
    796				nacking_prov_id = prov_id[dev_nb];
    797				svc_i3c_master_emit_stop(master);
    798
    799				continue;
    800			} else {
    801				return -EIO;
    802			}
    803		}
    804
    805		/* Wait for the slave to be ready to receive its address */
    806		ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
    807						reg,
    808						SVC_I3C_MSTATUS_MCTRLDONE(reg) &&
    809						SVC_I3C_MSTATUS_STATE_DAA(reg) &&
    810						SVC_I3C_MSTATUS_BETWEEN(reg),
    811						0, 1000);
    812		if (ret)
    813			return ret;
    814
    815		/* Give the slave device a suitable dynamic address */
    816		ret = i3c_master_get_free_addr(&master->base, last_addr + 1);
    817		if (ret < 0)
    818			return ret;
    819
    820		addrs[dev_nb] = ret;
    821		dev_dbg(master->dev, "DAA: device %d assigned to 0x%02x\n",
    822			dev_nb, addrs[dev_nb]);
    823
    824		writel(addrs[dev_nb], master->regs + SVC_I3C_MWDATAB);
    825		last_addr = addrs[dev_nb++];
    826	}
    827
    828	*count = dev_nb;
    829
    830	return 0;
    831}
    832
    833static int svc_i3c_update_ibirules(struct svc_i3c_master *master)
    834{
    835	struct i3c_dev_desc *dev;
    836	u32 reg_mbyte = 0, reg_nobyte = SVC_I3C_IBIRULES_NOBYTE;
    837	unsigned int mbyte_addr_ok = 0, mbyte_addr_ko = 0, nobyte_addr_ok = 0,
    838		nobyte_addr_ko = 0;
    839	bool list_mbyte = false, list_nobyte = false;
    840
    841	/* Create the IBIRULES register for both cases */
    842	i3c_bus_for_each_i3cdev(&master->base.bus, dev) {
    843		if (I3C_BCR_DEVICE_ROLE(dev->info.bcr) == I3C_BCR_I3C_MASTER)
    844			continue;
    845
    846		if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) {
    847			reg_mbyte |= SVC_I3C_IBIRULES_ADDR(mbyte_addr_ok,
    848							   dev->info.dyn_addr);
    849
    850			/* IBI rules cannot be applied to devices with MSb=1 */
    851			if (dev->info.dyn_addr & BIT(7))
    852				mbyte_addr_ko++;
    853			else
    854				mbyte_addr_ok++;
    855		} else {
    856			reg_nobyte |= SVC_I3C_IBIRULES_ADDR(nobyte_addr_ok,
    857							    dev->info.dyn_addr);
    858
    859			/* IBI rules cannot be applied to devices with MSb=1 */
    860			if (dev->info.dyn_addr & BIT(7))
    861				nobyte_addr_ko++;
    862			else
    863				nobyte_addr_ok++;
    864		}
    865	}
    866
    867	/* Device list cannot be handled by hardware */
    868	if (!mbyte_addr_ko && mbyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS)
    869		list_mbyte = true;
    870
    871	if (!nobyte_addr_ko && nobyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS)
    872		list_nobyte = true;
    873
    874	/* No list can be properly handled, return an error */
    875	if (!list_mbyte && !list_nobyte)
    876		return -ERANGE;
    877
    878	/* Pick the first list that can be handled by hardware, randomly */
    879	if (list_mbyte)
    880		writel(reg_mbyte, master->regs + SVC_I3C_IBIRULES);
    881	else
    882		writel(reg_nobyte, master->regs + SVC_I3C_IBIRULES);
    883
    884	return 0;
    885}
    886
    887static int svc_i3c_master_do_daa(struct i3c_master_controller *m)
    888{
    889	struct svc_i3c_master *master = to_svc_i3c_master(m);
    890	u8 addrs[SVC_I3C_MAX_DEVS];
    891	unsigned long flags;
    892	unsigned int dev_nb;
    893	int ret, i;
    894
    895	ret = pm_runtime_resume_and_get(master->dev);
    896	if (ret < 0) {
    897		dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
    898		return ret;
    899	}
    900
    901	spin_lock_irqsave(&master->xferqueue.lock, flags);
    902	ret = svc_i3c_master_do_daa_locked(master, addrs, &dev_nb);
    903	spin_unlock_irqrestore(&master->xferqueue.lock, flags);
    904	if (ret) {
    905		svc_i3c_master_emit_stop(master);
    906		svc_i3c_master_clear_merrwarn(master);
    907		goto rpm_out;
    908	}
    909
    910	/* Register all devices who participated to the core */
    911	for (i = 0; i < dev_nb; i++) {
    912		ret = i3c_master_add_i3c_dev_locked(m, addrs[i]);
    913		if (ret)
    914			goto rpm_out;
    915	}
    916
    917	/* Configure IBI auto-rules */
    918	ret = svc_i3c_update_ibirules(master);
    919	if (ret)
    920		dev_err(master->dev, "Cannot handle such a list of devices");
    921
    922rpm_out:
    923	pm_runtime_mark_last_busy(master->dev);
    924	pm_runtime_put_autosuspend(master->dev);
    925
    926	return ret;
    927}
    928
    929static int svc_i3c_master_read(struct svc_i3c_master *master,
    930			       u8 *in, unsigned int len)
    931{
    932	int offset = 0, i;
    933	u32 mdctrl, mstatus;
    934	bool completed = false;
    935	unsigned int count;
    936	unsigned long start = jiffies;
    937
    938	while (!completed) {
    939		mstatus = readl(master->regs + SVC_I3C_MSTATUS);
    940		if (SVC_I3C_MSTATUS_COMPLETE(mstatus) != 0)
    941			completed = true;
    942
    943		if (time_after(jiffies, start + msecs_to_jiffies(1000))) {
    944			dev_dbg(master->dev, "I3C read timeout\n");
    945			return -ETIMEDOUT;
    946		}
    947
    948		mdctrl = readl(master->regs + SVC_I3C_MDATACTRL);
    949		count = SVC_I3C_MDATACTRL_RXCOUNT(mdctrl);
    950		if (offset + count > len) {
    951			dev_err(master->dev, "I3C receive length too long!\n");
    952			return -EINVAL;
    953		}
    954		for (i = 0; i < count; i++)
    955			in[offset + i] = readl(master->regs + SVC_I3C_MRDATAB);
    956
    957		offset += count;
    958	}
    959
    960	return offset;
    961}
    962
    963static int svc_i3c_master_write(struct svc_i3c_master *master,
    964				const u8 *out, unsigned int len)
    965{
    966	int offset = 0, ret;
    967	u32 mdctrl;
    968
    969	while (offset < len) {
    970		ret = readl_poll_timeout(master->regs + SVC_I3C_MDATACTRL,
    971					 mdctrl,
    972					 !(mdctrl & SVC_I3C_MDATACTRL_TXFULL),
    973					 0, 1000);
    974		if (ret)
    975			return ret;
    976
    977		/*
    978		 * The last byte to be sent over the bus must either have the
    979		 * "end" bit set or be written in MWDATABE.
    980		 */
    981		if (likely(offset < (len - 1)))
    982			writel(out[offset++], master->regs + SVC_I3C_MWDATAB);
    983		else
    984			writel(out[offset++], master->regs + SVC_I3C_MWDATABE);
    985	}
    986
    987	return 0;
    988}
    989
    990static int svc_i3c_master_xfer(struct svc_i3c_master *master,
    991			       bool rnw, unsigned int xfer_type, u8 addr,
    992			       u8 *in, const u8 *out, unsigned int xfer_len,
    993			       unsigned int *read_len, bool continued)
    994{
    995	u32 reg;
    996	int ret;
    997
    998	writel(SVC_I3C_MCTRL_REQUEST_START_ADDR |
    999	       xfer_type |
   1000	       SVC_I3C_MCTRL_IBIRESP_NACK |
   1001	       SVC_I3C_MCTRL_DIR(rnw) |
   1002	       SVC_I3C_MCTRL_ADDR(addr) |
   1003	       SVC_I3C_MCTRL_RDTERM(*read_len),
   1004	       master->regs + SVC_I3C_MCTRL);
   1005
   1006	ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
   1007				 SVC_I3C_MSTATUS_MCTRLDONE(reg), 0, 1000);
   1008	if (ret)
   1009		goto emit_stop;
   1010
   1011	if (rnw)
   1012		ret = svc_i3c_master_read(master, in, xfer_len);
   1013	else
   1014		ret = svc_i3c_master_write(master, out, xfer_len);
   1015	if (ret < 0)
   1016		goto emit_stop;
   1017
   1018	if (rnw)
   1019		*read_len = ret;
   1020
   1021	ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
   1022				 SVC_I3C_MSTATUS_COMPLETE(reg), 0, 1000);
   1023	if (ret)
   1024		goto emit_stop;
   1025
   1026	writel(SVC_I3C_MINT_COMPLETE, master->regs + SVC_I3C_MSTATUS);
   1027
   1028	if (!continued) {
   1029		svc_i3c_master_emit_stop(master);
   1030
   1031		/* Wait idle if stop is sent. */
   1032		readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
   1033				   SVC_I3C_MSTATUS_STATE_IDLE(reg), 0, 1000);
   1034	}
   1035
   1036	return 0;
   1037
   1038emit_stop:
   1039	svc_i3c_master_emit_stop(master);
   1040	svc_i3c_master_clear_merrwarn(master);
   1041
   1042	return ret;
   1043}
   1044
   1045static struct svc_i3c_xfer *
   1046svc_i3c_master_alloc_xfer(struct svc_i3c_master *master, unsigned int ncmds)
   1047{
   1048	struct svc_i3c_xfer *xfer;
   1049
   1050	xfer = kzalloc(struct_size(xfer, cmds, ncmds), GFP_KERNEL);
   1051	if (!xfer)
   1052		return NULL;
   1053
   1054	INIT_LIST_HEAD(&xfer->node);
   1055	xfer->ncmds = ncmds;
   1056	xfer->ret = -ETIMEDOUT;
   1057
   1058	return xfer;
   1059}
   1060
   1061static void svc_i3c_master_free_xfer(struct svc_i3c_xfer *xfer)
   1062{
   1063	kfree(xfer);
   1064}
   1065
   1066static void svc_i3c_master_dequeue_xfer_locked(struct svc_i3c_master *master,
   1067					       struct svc_i3c_xfer *xfer)
   1068{
   1069	if (master->xferqueue.cur == xfer)
   1070		master->xferqueue.cur = NULL;
   1071	else
   1072		list_del_init(&xfer->node);
   1073}
   1074
   1075static void svc_i3c_master_dequeue_xfer(struct svc_i3c_master *master,
   1076					struct svc_i3c_xfer *xfer)
   1077{
   1078	unsigned long flags;
   1079
   1080	spin_lock_irqsave(&master->xferqueue.lock, flags);
   1081	svc_i3c_master_dequeue_xfer_locked(master, xfer);
   1082	spin_unlock_irqrestore(&master->xferqueue.lock, flags);
   1083}
   1084
   1085static void svc_i3c_master_start_xfer_locked(struct svc_i3c_master *master)
   1086{
   1087	struct svc_i3c_xfer *xfer = master->xferqueue.cur;
   1088	int ret, i;
   1089
   1090	if (!xfer)
   1091		return;
   1092
   1093	ret = pm_runtime_resume_and_get(master->dev);
   1094	if (ret < 0) {
   1095		dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
   1096		return;
   1097	}
   1098
   1099	svc_i3c_master_clear_merrwarn(master);
   1100	svc_i3c_master_flush_fifo(master);
   1101
   1102	for (i = 0; i < xfer->ncmds; i++) {
   1103		struct svc_i3c_cmd *cmd = &xfer->cmds[i];
   1104
   1105		ret = svc_i3c_master_xfer(master, cmd->rnw, xfer->type,
   1106					  cmd->addr, cmd->in, cmd->out,
   1107					  cmd->len, &cmd->read_len,
   1108					  cmd->continued);
   1109		if (ret)
   1110			break;
   1111	}
   1112
   1113	pm_runtime_mark_last_busy(master->dev);
   1114	pm_runtime_put_autosuspend(master->dev);
   1115
   1116	xfer->ret = ret;
   1117	complete(&xfer->comp);
   1118
   1119	if (ret < 0)
   1120		svc_i3c_master_dequeue_xfer_locked(master, xfer);
   1121
   1122	xfer = list_first_entry_or_null(&master->xferqueue.list,
   1123					struct svc_i3c_xfer,
   1124					node);
   1125	if (xfer)
   1126		list_del_init(&xfer->node);
   1127
   1128	master->xferqueue.cur = xfer;
   1129	svc_i3c_master_start_xfer_locked(master);
   1130}
   1131
   1132static void svc_i3c_master_enqueue_xfer(struct svc_i3c_master *master,
   1133					struct svc_i3c_xfer *xfer)
   1134{
   1135	unsigned long flags;
   1136
   1137	init_completion(&xfer->comp);
   1138	spin_lock_irqsave(&master->xferqueue.lock, flags);
   1139	if (master->xferqueue.cur) {
   1140		list_add_tail(&xfer->node, &master->xferqueue.list);
   1141	} else {
   1142		master->xferqueue.cur = xfer;
   1143		svc_i3c_master_start_xfer_locked(master);
   1144	}
   1145	spin_unlock_irqrestore(&master->xferqueue.lock, flags);
   1146}
   1147
   1148static bool
   1149svc_i3c_master_supports_ccc_cmd(struct i3c_master_controller *master,
   1150				const struct i3c_ccc_cmd *cmd)
   1151{
   1152	/* No software support for CCC commands targeting more than one slave */
   1153	return (cmd->ndests == 1);
   1154}
   1155
   1156static int svc_i3c_master_send_bdcast_ccc_cmd(struct svc_i3c_master *master,
   1157					      struct i3c_ccc_cmd *ccc)
   1158{
   1159	unsigned int xfer_len = ccc->dests[0].payload.len + 1;
   1160	struct svc_i3c_xfer *xfer;
   1161	struct svc_i3c_cmd *cmd;
   1162	u8 *buf;
   1163	int ret;
   1164
   1165	xfer = svc_i3c_master_alloc_xfer(master, 1);
   1166	if (!xfer)
   1167		return -ENOMEM;
   1168
   1169	buf = kmalloc(xfer_len, GFP_KERNEL);
   1170	if (!buf) {
   1171		svc_i3c_master_free_xfer(xfer);
   1172		return -ENOMEM;
   1173	}
   1174
   1175	buf[0] = ccc->id;
   1176	memcpy(&buf[1], ccc->dests[0].payload.data, ccc->dests[0].payload.len);
   1177
   1178	xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
   1179
   1180	cmd = &xfer->cmds[0];
   1181	cmd->addr = ccc->dests[0].addr;
   1182	cmd->rnw = ccc->rnw;
   1183	cmd->in = NULL;
   1184	cmd->out = buf;
   1185	cmd->len = xfer_len;
   1186	cmd->read_len = 0;
   1187	cmd->continued = false;
   1188
   1189	svc_i3c_master_enqueue_xfer(master, xfer);
   1190	if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
   1191		svc_i3c_master_dequeue_xfer(master, xfer);
   1192
   1193	ret = xfer->ret;
   1194	kfree(buf);
   1195	svc_i3c_master_free_xfer(xfer);
   1196
   1197	return ret;
   1198}
   1199
   1200static int svc_i3c_master_send_direct_ccc_cmd(struct svc_i3c_master *master,
   1201					      struct i3c_ccc_cmd *ccc)
   1202{
   1203	unsigned int xfer_len = ccc->dests[0].payload.len;
   1204	unsigned int read_len = ccc->rnw ? xfer_len : 0;
   1205	struct svc_i3c_xfer *xfer;
   1206	struct svc_i3c_cmd *cmd;
   1207	int ret;
   1208
   1209	xfer = svc_i3c_master_alloc_xfer(master, 2);
   1210	if (!xfer)
   1211		return -ENOMEM;
   1212
   1213	xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
   1214
   1215	/* Broadcasted message */
   1216	cmd = &xfer->cmds[0];
   1217	cmd->addr = I3C_BROADCAST_ADDR;
   1218	cmd->rnw = 0;
   1219	cmd->in = NULL;
   1220	cmd->out = &ccc->id;
   1221	cmd->len = 1;
   1222	cmd->read_len = 0;
   1223	cmd->continued = true;
   1224
   1225	/* Directed message */
   1226	cmd = &xfer->cmds[1];
   1227	cmd->addr = ccc->dests[0].addr;
   1228	cmd->rnw = ccc->rnw;
   1229	cmd->in = ccc->rnw ? ccc->dests[0].payload.data : NULL;
   1230	cmd->out = ccc->rnw ? NULL : ccc->dests[0].payload.data,
   1231	cmd->len = xfer_len;
   1232	cmd->read_len = read_len;
   1233	cmd->continued = false;
   1234
   1235	svc_i3c_master_enqueue_xfer(master, xfer);
   1236	if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
   1237		svc_i3c_master_dequeue_xfer(master, xfer);
   1238
   1239	if (cmd->read_len != xfer_len)
   1240		ccc->dests[0].payload.len = cmd->read_len;
   1241
   1242	ret = xfer->ret;
   1243	svc_i3c_master_free_xfer(xfer);
   1244
   1245	return ret;
   1246}
   1247
   1248static int svc_i3c_master_send_ccc_cmd(struct i3c_master_controller *m,
   1249				       struct i3c_ccc_cmd *cmd)
   1250{
   1251	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1252	bool broadcast = cmd->id < 0x80;
   1253
   1254	if (broadcast)
   1255		return svc_i3c_master_send_bdcast_ccc_cmd(master, cmd);
   1256	else
   1257		return svc_i3c_master_send_direct_ccc_cmd(master, cmd);
   1258}
   1259
   1260static int svc_i3c_master_priv_xfers(struct i3c_dev_desc *dev,
   1261				     struct i3c_priv_xfer *xfers,
   1262				     int nxfers)
   1263{
   1264	struct i3c_master_controller *m = i3c_dev_get_master(dev);
   1265	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1266	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
   1267	struct svc_i3c_xfer *xfer;
   1268	int ret, i;
   1269
   1270	xfer = svc_i3c_master_alloc_xfer(master, nxfers);
   1271	if (!xfer)
   1272		return -ENOMEM;
   1273
   1274	xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
   1275
   1276	for (i = 0; i < nxfers; i++) {
   1277		struct svc_i3c_cmd *cmd = &xfer->cmds[i];
   1278
   1279		cmd->addr = master->addrs[data->index];
   1280		cmd->rnw = xfers[i].rnw;
   1281		cmd->in = xfers[i].rnw ? xfers[i].data.in : NULL;
   1282		cmd->out = xfers[i].rnw ? NULL : xfers[i].data.out;
   1283		cmd->len = xfers[i].len;
   1284		cmd->read_len = xfers[i].rnw ? xfers[i].len : 0;
   1285		cmd->continued = (i + 1) < nxfers;
   1286	}
   1287
   1288	svc_i3c_master_enqueue_xfer(master, xfer);
   1289	if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
   1290		svc_i3c_master_dequeue_xfer(master, xfer);
   1291
   1292	ret = xfer->ret;
   1293	svc_i3c_master_free_xfer(xfer);
   1294
   1295	return ret;
   1296}
   1297
   1298static int svc_i3c_master_i2c_xfers(struct i2c_dev_desc *dev,
   1299				    const struct i2c_msg *xfers,
   1300				    int nxfers)
   1301{
   1302	struct i3c_master_controller *m = i2c_dev_get_master(dev);
   1303	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1304	struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
   1305	struct svc_i3c_xfer *xfer;
   1306	int ret, i;
   1307
   1308	xfer = svc_i3c_master_alloc_xfer(master, nxfers);
   1309	if (!xfer)
   1310		return -ENOMEM;
   1311
   1312	xfer->type = SVC_I3C_MCTRL_TYPE_I2C;
   1313
   1314	for (i = 0; i < nxfers; i++) {
   1315		struct svc_i3c_cmd *cmd = &xfer->cmds[i];
   1316
   1317		cmd->addr = master->addrs[data->index];
   1318		cmd->rnw = xfers[i].flags & I2C_M_RD;
   1319		cmd->in = cmd->rnw ? xfers[i].buf : NULL;
   1320		cmd->out = cmd->rnw ? NULL : xfers[i].buf;
   1321		cmd->len = xfers[i].len;
   1322		cmd->read_len = cmd->rnw ? xfers[i].len : 0;
   1323		cmd->continued = (i + 1 < nxfers);
   1324	}
   1325
   1326	svc_i3c_master_enqueue_xfer(master, xfer);
   1327	if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
   1328		svc_i3c_master_dequeue_xfer(master, xfer);
   1329
   1330	ret = xfer->ret;
   1331	svc_i3c_master_free_xfer(xfer);
   1332
   1333	return ret;
   1334}
   1335
   1336static int svc_i3c_master_request_ibi(struct i3c_dev_desc *dev,
   1337				      const struct i3c_ibi_setup *req)
   1338{
   1339	struct i3c_master_controller *m = i3c_dev_get_master(dev);
   1340	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1341	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
   1342	unsigned long flags;
   1343	unsigned int i;
   1344
   1345	if (dev->ibi->max_payload_len > SVC_I3C_FIFO_SIZE) {
   1346		dev_err(master->dev, "IBI max payload %d should be < %d\n",
   1347			dev->ibi->max_payload_len, SVC_I3C_FIFO_SIZE);
   1348		return -ERANGE;
   1349	}
   1350
   1351	data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req);
   1352	if (IS_ERR(data->ibi_pool))
   1353		return PTR_ERR(data->ibi_pool);
   1354
   1355	spin_lock_irqsave(&master->ibi.lock, flags);
   1356	for (i = 0; i < master->ibi.num_slots; i++) {
   1357		if (!master->ibi.slots[i]) {
   1358			data->ibi = i;
   1359			master->ibi.slots[i] = dev;
   1360			break;
   1361		}
   1362	}
   1363	spin_unlock_irqrestore(&master->ibi.lock, flags);
   1364
   1365	if (i < master->ibi.num_slots)
   1366		return 0;
   1367
   1368	i3c_generic_ibi_free_pool(data->ibi_pool);
   1369	data->ibi_pool = NULL;
   1370
   1371	return -ENOSPC;
   1372}
   1373
   1374static void svc_i3c_master_free_ibi(struct i3c_dev_desc *dev)
   1375{
   1376	struct i3c_master_controller *m = i3c_dev_get_master(dev);
   1377	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1378	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
   1379	unsigned long flags;
   1380
   1381	spin_lock_irqsave(&master->ibi.lock, flags);
   1382	master->ibi.slots[data->ibi] = NULL;
   1383	data->ibi = -1;
   1384	spin_unlock_irqrestore(&master->ibi.lock, flags);
   1385
   1386	i3c_generic_ibi_free_pool(data->ibi_pool);
   1387}
   1388
   1389static int svc_i3c_master_enable_ibi(struct i3c_dev_desc *dev)
   1390{
   1391	struct i3c_master_controller *m = i3c_dev_get_master(dev);
   1392	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1393	int ret;
   1394
   1395	ret = pm_runtime_resume_and_get(master->dev);
   1396	if (ret < 0) {
   1397		dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
   1398		return ret;
   1399	}
   1400
   1401	svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
   1402
   1403	return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
   1404}
   1405
   1406static int svc_i3c_master_disable_ibi(struct i3c_dev_desc *dev)
   1407{
   1408	struct i3c_master_controller *m = i3c_dev_get_master(dev);
   1409	struct svc_i3c_master *master = to_svc_i3c_master(m);
   1410	int ret;
   1411
   1412	svc_i3c_master_disable_interrupts(master);
   1413
   1414	ret = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
   1415
   1416	pm_runtime_mark_last_busy(master->dev);
   1417	pm_runtime_put_autosuspend(master->dev);
   1418
   1419	return ret;
   1420}
   1421
   1422static void svc_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev,
   1423					    struct i3c_ibi_slot *slot)
   1424{
   1425	struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
   1426
   1427	i3c_generic_ibi_recycle_slot(data->ibi_pool, slot);
   1428}
   1429
   1430static const struct i3c_master_controller_ops svc_i3c_master_ops = {
   1431	.bus_init = svc_i3c_master_bus_init,
   1432	.bus_cleanup = svc_i3c_master_bus_cleanup,
   1433	.attach_i3c_dev = svc_i3c_master_attach_i3c_dev,
   1434	.detach_i3c_dev = svc_i3c_master_detach_i3c_dev,
   1435	.reattach_i3c_dev = svc_i3c_master_reattach_i3c_dev,
   1436	.attach_i2c_dev = svc_i3c_master_attach_i2c_dev,
   1437	.detach_i2c_dev = svc_i3c_master_detach_i2c_dev,
   1438	.do_daa = svc_i3c_master_do_daa,
   1439	.supports_ccc_cmd = svc_i3c_master_supports_ccc_cmd,
   1440	.send_ccc_cmd = svc_i3c_master_send_ccc_cmd,
   1441	.priv_xfers = svc_i3c_master_priv_xfers,
   1442	.i2c_xfers = svc_i3c_master_i2c_xfers,
   1443	.request_ibi = svc_i3c_master_request_ibi,
   1444	.free_ibi = svc_i3c_master_free_ibi,
   1445	.recycle_ibi_slot = svc_i3c_master_recycle_ibi_slot,
   1446	.enable_ibi = svc_i3c_master_enable_ibi,
   1447	.disable_ibi = svc_i3c_master_disable_ibi,
   1448};
   1449
   1450static int svc_i3c_master_prepare_clks(struct svc_i3c_master *master)
   1451{
   1452	int ret = 0;
   1453
   1454	ret = clk_prepare_enable(master->pclk);
   1455	if (ret)
   1456		return ret;
   1457
   1458	ret = clk_prepare_enable(master->fclk);
   1459	if (ret) {
   1460		clk_disable_unprepare(master->pclk);
   1461		return ret;
   1462	}
   1463
   1464	ret = clk_prepare_enable(master->sclk);
   1465	if (ret) {
   1466		clk_disable_unprepare(master->pclk);
   1467		clk_disable_unprepare(master->fclk);
   1468		return ret;
   1469	}
   1470
   1471	return 0;
   1472}
   1473
   1474static void svc_i3c_master_unprepare_clks(struct svc_i3c_master *master)
   1475{
   1476	clk_disable_unprepare(master->pclk);
   1477	clk_disable_unprepare(master->fclk);
   1478	clk_disable_unprepare(master->sclk);
   1479}
   1480
   1481static int svc_i3c_master_probe(struct platform_device *pdev)
   1482{
   1483	struct device *dev = &pdev->dev;
   1484	struct svc_i3c_master *master;
   1485	int ret;
   1486
   1487	master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
   1488	if (!master)
   1489		return -ENOMEM;
   1490
   1491	master->regs = devm_platform_ioremap_resource(pdev, 0);
   1492	if (IS_ERR(master->regs))
   1493		return PTR_ERR(master->regs);
   1494
   1495	master->pclk = devm_clk_get(dev, "pclk");
   1496	if (IS_ERR(master->pclk))
   1497		return PTR_ERR(master->pclk);
   1498
   1499	master->fclk = devm_clk_get(dev, "fast_clk");
   1500	if (IS_ERR(master->fclk))
   1501		return PTR_ERR(master->fclk);
   1502
   1503	master->sclk = devm_clk_get(dev, "slow_clk");
   1504	if (IS_ERR(master->sclk))
   1505		return PTR_ERR(master->sclk);
   1506
   1507	master->irq = platform_get_irq(pdev, 0);
   1508	if (master->irq <= 0)
   1509		return -ENOENT;
   1510
   1511	master->dev = dev;
   1512
   1513	ret = svc_i3c_master_prepare_clks(master);
   1514	if (ret)
   1515		return ret;
   1516
   1517	INIT_WORK(&master->hj_work, svc_i3c_master_hj_work);
   1518	INIT_WORK(&master->ibi_work, svc_i3c_master_ibi_work);
   1519	ret = devm_request_irq(dev, master->irq, svc_i3c_master_irq_handler,
   1520			       IRQF_NO_SUSPEND, "svc-i3c-irq", master);
   1521	if (ret)
   1522		goto err_disable_clks;
   1523
   1524	master->free_slots = GENMASK(SVC_I3C_MAX_DEVS - 1, 0);
   1525
   1526	spin_lock_init(&master->xferqueue.lock);
   1527	INIT_LIST_HEAD(&master->xferqueue.list);
   1528
   1529	spin_lock_init(&master->ibi.lock);
   1530	master->ibi.num_slots = SVC_I3C_MAX_DEVS;
   1531	master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots,
   1532					 sizeof(*master->ibi.slots),
   1533					 GFP_KERNEL);
   1534	if (!master->ibi.slots) {
   1535		ret = -ENOMEM;
   1536		goto err_disable_clks;
   1537	}
   1538
   1539	platform_set_drvdata(pdev, master);
   1540
   1541	pm_runtime_set_autosuspend_delay(&pdev->dev, SVC_I3C_PM_TIMEOUT_MS);
   1542	pm_runtime_use_autosuspend(&pdev->dev);
   1543	pm_runtime_get_noresume(&pdev->dev);
   1544	pm_runtime_set_active(&pdev->dev);
   1545	pm_runtime_enable(&pdev->dev);
   1546
   1547	svc_i3c_master_reset(master);
   1548
   1549	/* Register the master */
   1550	ret = i3c_master_register(&master->base, &pdev->dev,
   1551				  &svc_i3c_master_ops, false);
   1552	if (ret)
   1553		goto rpm_disable;
   1554
   1555	pm_runtime_mark_last_busy(&pdev->dev);
   1556	pm_runtime_put_autosuspend(&pdev->dev);
   1557
   1558	return 0;
   1559
   1560rpm_disable:
   1561	pm_runtime_dont_use_autosuspend(&pdev->dev);
   1562	pm_runtime_put_noidle(&pdev->dev);
   1563	pm_runtime_set_suspended(&pdev->dev);
   1564	pm_runtime_disable(&pdev->dev);
   1565
   1566err_disable_clks:
   1567	svc_i3c_master_unprepare_clks(master);
   1568
   1569	return ret;
   1570}
   1571
   1572static int svc_i3c_master_remove(struct platform_device *pdev)
   1573{
   1574	struct svc_i3c_master *master = platform_get_drvdata(pdev);
   1575	int ret;
   1576
   1577	ret = i3c_master_unregister(&master->base);
   1578	if (ret)
   1579		return ret;
   1580
   1581	pm_runtime_dont_use_autosuspend(&pdev->dev);
   1582	pm_runtime_disable(&pdev->dev);
   1583
   1584	return 0;
   1585}
   1586
   1587static int __maybe_unused svc_i3c_runtime_suspend(struct device *dev)
   1588{
   1589	struct svc_i3c_master *master = dev_get_drvdata(dev);
   1590
   1591	svc_i3c_master_unprepare_clks(master);
   1592	pinctrl_pm_select_sleep_state(dev);
   1593
   1594	return 0;
   1595}
   1596
   1597static int __maybe_unused svc_i3c_runtime_resume(struct device *dev)
   1598{
   1599	struct svc_i3c_master *master = dev_get_drvdata(dev);
   1600
   1601	pinctrl_pm_select_default_state(dev);
   1602	svc_i3c_master_prepare_clks(master);
   1603
   1604	return 0;
   1605}
   1606
   1607static const struct dev_pm_ops svc_i3c_pm_ops = {
   1608	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
   1609				      pm_runtime_force_resume)
   1610	SET_RUNTIME_PM_OPS(svc_i3c_runtime_suspend,
   1611			   svc_i3c_runtime_resume, NULL)
   1612};
   1613
   1614static const struct of_device_id svc_i3c_master_of_match_tbl[] = {
   1615	{ .compatible = "silvaco,i3c-master" },
   1616	{ /* sentinel */ },
   1617};
   1618MODULE_DEVICE_TABLE(of, svc_i3c_master_of_match_tbl);
   1619
   1620static struct platform_driver svc_i3c_master = {
   1621	.probe = svc_i3c_master_probe,
   1622	.remove = svc_i3c_master_remove,
   1623	.driver = {
   1624		.name = "silvaco-i3c-master",
   1625		.of_match_table = svc_i3c_master_of_match_tbl,
   1626		.pm = &svc_i3c_pm_ops,
   1627	},
   1628};
   1629module_platform_driver(svc_i3c_master);
   1630
   1631MODULE_AUTHOR("Conor Culhane <conor.culhane@silvaco.com>");
   1632MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
   1633MODULE_DESCRIPTION("Silvaco dual-role I3C master driver");
   1634MODULE_LICENSE("GPL v2");