cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

pcie-mediatek.c (33542B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * MediaTek PCIe host controller driver.
      4 *
      5 * Copyright (c) 2017 MediaTek Inc.
      6 * Author: Ryder Lee <ryder.lee@mediatek.com>
      7 *	   Honghui Zhang <honghui.zhang@mediatek.com>
      8 */
      9
     10#include <linux/clk.h>
     11#include <linux/delay.h>
     12#include <linux/iopoll.h>
     13#include <linux/irq.h>
     14#include <linux/irqchip/chained_irq.h>
     15#include <linux/irqdomain.h>
     16#include <linux/kernel.h>
     17#include <linux/mfd/syscon.h>
     18#include <linux/msi.h>
     19#include <linux/module.h>
     20#include <linux/of_address.h>
     21#include <linux/of_pci.h>
     22#include <linux/of_platform.h>
     23#include <linux/pci.h>
     24#include <linux/phy/phy.h>
     25#include <linux/platform_device.h>
     26#include <linux/pm_runtime.h>
     27#include <linux/regmap.h>
     28#include <linux/reset.h>
     29
     30#include "../pci.h"
     31
     32/* PCIe shared registers */
     33#define PCIE_SYS_CFG		0x00
     34#define PCIE_INT_ENABLE		0x0c
     35#define PCIE_CFG_ADDR		0x20
     36#define PCIE_CFG_DATA		0x24
     37
     38/* PCIe per port registers */
     39#define PCIE_BAR0_SETUP		0x10
     40#define PCIE_CLASS		0x34
     41#define PCIE_LINK_STATUS	0x50
     42
     43#define PCIE_PORT_INT_EN(x)	BIT(20 + (x))
     44#define PCIE_PORT_PERST(x)	BIT(1 + (x))
     45#define PCIE_PORT_LINKUP	BIT(0)
     46#define PCIE_BAR_MAP_MAX	GENMASK(31, 16)
     47
     48#define PCIE_BAR_ENABLE		BIT(0)
     49#define PCIE_REVISION_ID	BIT(0)
     50#define PCIE_CLASS_CODE		(0x60400 << 8)
     51#define PCIE_CONF_REG(regn)	(((regn) & GENMASK(7, 2)) | \
     52				((((regn) >> 8) & GENMASK(3, 0)) << 24))
     53#define PCIE_CONF_FUN(fun)	(((fun) << 8) & GENMASK(10, 8))
     54#define PCIE_CONF_DEV(dev)	(((dev) << 11) & GENMASK(15, 11))
     55#define PCIE_CONF_BUS(bus)	(((bus) << 16) & GENMASK(23, 16))
     56#define PCIE_CONF_ADDR(regn, fun, dev, bus) \
     57	(PCIE_CONF_REG(regn) | PCIE_CONF_FUN(fun) | \
     58	 PCIE_CONF_DEV(dev) | PCIE_CONF_BUS(bus))
     59
     60/* MediaTek specific configuration registers */
     61#define PCIE_FTS_NUM		0x70c
     62#define PCIE_FTS_NUM_MASK	GENMASK(15, 8)
     63#define PCIE_FTS_NUM_L0(x)	((x) & 0xff << 8)
     64
     65#define PCIE_FC_CREDIT		0x73c
     66#define PCIE_FC_CREDIT_MASK	(GENMASK(31, 31) | GENMASK(28, 16))
     67#define PCIE_FC_CREDIT_VAL(x)	((x) << 16)
     68
     69/* PCIe V2 share registers */
     70#define PCIE_SYS_CFG_V2		0x0
     71#define PCIE_CSR_LTSSM_EN(x)	BIT(0 + (x) * 8)
     72#define PCIE_CSR_ASPM_L1_EN(x)	BIT(1 + (x) * 8)
     73
     74/* PCIe V2 per-port registers */
     75#define PCIE_MSI_VECTOR		0x0c0
     76
     77#define PCIE_CONF_VEND_ID	0x100
     78#define PCIE_CONF_DEVICE_ID	0x102
     79#define PCIE_CONF_CLASS_ID	0x106
     80
     81#define PCIE_INT_MASK		0x420
     82#define INTX_MASK		GENMASK(19, 16)
     83#define INTX_SHIFT		16
     84#define PCIE_INT_STATUS		0x424
     85#define MSI_STATUS		BIT(23)
     86#define PCIE_IMSI_STATUS	0x42c
     87#define PCIE_IMSI_ADDR		0x430
     88#define MSI_MASK		BIT(23)
     89#define MTK_MSI_IRQS_NUM	32
     90
     91#define PCIE_AHB_TRANS_BASE0_L	0x438
     92#define PCIE_AHB_TRANS_BASE0_H	0x43c
     93#define AHB2PCIE_SIZE(x)	((x) & GENMASK(4, 0))
     94#define PCIE_AXI_WINDOW0	0x448
     95#define WIN_ENABLE		BIT(7)
     96/*
     97 * Define PCIe to AHB window size as 2^33 to support max 8GB address space
     98 * translate, support least 4GB DRAM size access from EP DMA(physical DRAM
     99 * start from 0x40000000).
    100 */
    101#define PCIE2AHB_SIZE	0x21
    102
    103/* PCIe V2 configuration transaction header */
    104#define PCIE_CFG_HEADER0	0x460
    105#define PCIE_CFG_HEADER1	0x464
    106#define PCIE_CFG_HEADER2	0x468
    107#define PCIE_CFG_WDATA		0x470
    108#define PCIE_APP_TLP_REQ	0x488
    109#define PCIE_CFG_RDATA		0x48c
    110#define APP_CFG_REQ		BIT(0)
    111#define APP_CPL_STATUS		GENMASK(7, 5)
    112
    113#define CFG_WRRD_TYPE_0		4
    114#define CFG_WR_FMT		2
    115#define CFG_RD_FMT		0
    116
    117#define CFG_DW0_LENGTH(length)	((length) & GENMASK(9, 0))
    118#define CFG_DW0_TYPE(type)	(((type) << 24) & GENMASK(28, 24))
    119#define CFG_DW0_FMT(fmt)	(((fmt) << 29) & GENMASK(31, 29))
    120#define CFG_DW2_REGN(regn)	((regn) & GENMASK(11, 2))
    121#define CFG_DW2_FUN(fun)	(((fun) << 16) & GENMASK(18, 16))
    122#define CFG_DW2_DEV(dev)	(((dev) << 19) & GENMASK(23, 19))
    123#define CFG_DW2_BUS(bus)	(((bus) << 24) & GENMASK(31, 24))
    124#define CFG_HEADER_DW0(type, fmt) \
    125	(CFG_DW0_LENGTH(1) | CFG_DW0_TYPE(type) | CFG_DW0_FMT(fmt))
    126#define CFG_HEADER_DW1(where, size) \
    127	(GENMASK(((size) - 1), 0) << ((where) & 0x3))
    128#define CFG_HEADER_DW2(regn, fun, dev, bus) \
    129	(CFG_DW2_REGN(regn) | CFG_DW2_FUN(fun) | \
    130	CFG_DW2_DEV(dev) | CFG_DW2_BUS(bus))
    131
    132#define PCIE_RST_CTRL		0x510
    133#define PCIE_PHY_RSTB		BIT(0)
    134#define PCIE_PIPE_SRSTB		BIT(1)
    135#define PCIE_MAC_SRSTB		BIT(2)
    136#define PCIE_CRSTB		BIT(3)
    137#define PCIE_PERSTB		BIT(8)
    138#define PCIE_LINKDOWN_RST_EN	GENMASK(15, 13)
    139#define PCIE_LINK_STATUS_V2	0x804
    140#define PCIE_PORT_LINKUP_V2	BIT(10)
    141
    142struct mtk_pcie_port;
    143
    144/**
    145 * struct mtk_pcie_soc - differentiate between host generations
    146 * @need_fix_class_id: whether this host's class ID needed to be fixed or not
    147 * @need_fix_device_id: whether this host's device ID needed to be fixed or not
    148 * @no_msi: Bridge has no MSI support, and relies on an external block
    149 * @device_id: device ID which this host need to be fixed
    150 * @ops: pointer to configuration access functions
    151 * @startup: pointer to controller setting functions
    152 * @setup_irq: pointer to initialize IRQ functions
    153 */
    154struct mtk_pcie_soc {
    155	bool need_fix_class_id;
    156	bool need_fix_device_id;
    157	bool no_msi;
    158	unsigned int device_id;
    159	struct pci_ops *ops;
    160	int (*startup)(struct mtk_pcie_port *port);
    161	int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node);
    162};
    163
    164/**
    165 * struct mtk_pcie_port - PCIe port information
    166 * @base: IO mapped register base
    167 * @list: port list
    168 * @pcie: pointer to PCIe host info
    169 * @reset: pointer to port reset control
    170 * @sys_ck: pointer to transaction/data link layer clock
    171 * @ahb_ck: pointer to AHB slave interface operating clock for CSR access
    172 *          and RC initiated MMIO access
    173 * @axi_ck: pointer to application layer MMIO channel operating clock
    174 * @aux_ck: pointer to pe2_mac_bridge and pe2_mac_core operating clock
    175 *          when pcie_mac_ck/pcie_pipe_ck is turned off
    176 * @obff_ck: pointer to OBFF functional block operating clock
    177 * @pipe_ck: pointer to LTSSM and PHY/MAC layer operating clock
    178 * @phy: pointer to PHY control block
    179 * @slot: port slot
    180 * @irq: GIC irq
    181 * @irq_domain: legacy INTx IRQ domain
    182 * @inner_domain: inner IRQ domain
    183 * @msi_domain: MSI IRQ domain
    184 * @lock: protect the msi_irq_in_use bitmap
    185 * @msi_irq_in_use: bit map for assigned MSI IRQ
    186 */
    187struct mtk_pcie_port {
    188	void __iomem *base;
    189	struct list_head list;
    190	struct mtk_pcie *pcie;
    191	struct reset_control *reset;
    192	struct clk *sys_ck;
    193	struct clk *ahb_ck;
    194	struct clk *axi_ck;
    195	struct clk *aux_ck;
    196	struct clk *obff_ck;
    197	struct clk *pipe_ck;
    198	struct phy *phy;
    199	u32 slot;
    200	int irq;
    201	struct irq_domain *irq_domain;
    202	struct irq_domain *inner_domain;
    203	struct irq_domain *msi_domain;
    204	struct mutex lock;
    205	DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM);
    206};
    207
    208/**
    209 * struct mtk_pcie - PCIe host information
    210 * @dev: pointer to PCIe device
    211 * @base: IO mapped register base
    212 * @cfg: IO mapped register map for PCIe config
    213 * @free_ck: free-run reference clock
    214 * @mem: non-prefetchable memory resource
    215 * @ports: pointer to PCIe port information
    216 * @soc: pointer to SoC-dependent operations
    217 */
    218struct mtk_pcie {
    219	struct device *dev;
    220	void __iomem *base;
    221	struct regmap *cfg;
    222	struct clk *free_ck;
    223
    224	struct list_head ports;
    225	const struct mtk_pcie_soc *soc;
    226};
    227
    228static void mtk_pcie_subsys_powerdown(struct mtk_pcie *pcie)
    229{
    230	struct device *dev = pcie->dev;
    231
    232	clk_disable_unprepare(pcie->free_ck);
    233
    234	pm_runtime_put_sync(dev);
    235	pm_runtime_disable(dev);
    236}
    237
    238static void mtk_pcie_port_free(struct mtk_pcie_port *port)
    239{
    240	struct mtk_pcie *pcie = port->pcie;
    241	struct device *dev = pcie->dev;
    242
    243	devm_iounmap(dev, port->base);
    244	list_del(&port->list);
    245	devm_kfree(dev, port);
    246}
    247
    248static void mtk_pcie_put_resources(struct mtk_pcie *pcie)
    249{
    250	struct mtk_pcie_port *port, *tmp;
    251
    252	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
    253		phy_power_off(port->phy);
    254		phy_exit(port->phy);
    255		clk_disable_unprepare(port->pipe_ck);
    256		clk_disable_unprepare(port->obff_ck);
    257		clk_disable_unprepare(port->axi_ck);
    258		clk_disable_unprepare(port->aux_ck);
    259		clk_disable_unprepare(port->ahb_ck);
    260		clk_disable_unprepare(port->sys_ck);
    261		mtk_pcie_port_free(port);
    262	}
    263
    264	mtk_pcie_subsys_powerdown(pcie);
    265}
    266
    267static int mtk_pcie_check_cfg_cpld(struct mtk_pcie_port *port)
    268{
    269	u32 val;
    270	int err;
    271
    272	err = readl_poll_timeout_atomic(port->base + PCIE_APP_TLP_REQ, val,
    273					!(val & APP_CFG_REQ), 10,
    274					100 * USEC_PER_MSEC);
    275	if (err)
    276		return PCIBIOS_SET_FAILED;
    277
    278	if (readl(port->base + PCIE_APP_TLP_REQ) & APP_CPL_STATUS)
    279		return PCIBIOS_SET_FAILED;
    280
    281	return PCIBIOS_SUCCESSFUL;
    282}
    283
    284static int mtk_pcie_hw_rd_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
    285			      int where, int size, u32 *val)
    286{
    287	u32 tmp;
    288
    289	/* Write PCIe configuration transaction header for Cfgrd */
    290	writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_RD_FMT),
    291	       port->base + PCIE_CFG_HEADER0);
    292	writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
    293	writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
    294	       port->base + PCIE_CFG_HEADER2);
    295
    296	/* Trigger h/w to transmit Cfgrd TLP */
    297	tmp = readl(port->base + PCIE_APP_TLP_REQ);
    298	tmp |= APP_CFG_REQ;
    299	writel(tmp, port->base + PCIE_APP_TLP_REQ);
    300
    301	/* Check completion status */
    302	if (mtk_pcie_check_cfg_cpld(port))
    303		return PCIBIOS_SET_FAILED;
    304
    305	/* Read cpld payload of Cfgrd */
    306	*val = readl(port->base + PCIE_CFG_RDATA);
    307
    308	if (size == 1)
    309		*val = (*val >> (8 * (where & 3))) & 0xff;
    310	else if (size == 2)
    311		*val = (*val >> (8 * (where & 3))) & 0xffff;
    312
    313	return PCIBIOS_SUCCESSFUL;
    314}
    315
    316static int mtk_pcie_hw_wr_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn,
    317			      int where, int size, u32 val)
    318{
    319	/* Write PCIe configuration transaction header for Cfgwr */
    320	writel(CFG_HEADER_DW0(CFG_WRRD_TYPE_0, CFG_WR_FMT),
    321	       port->base + PCIE_CFG_HEADER0);
    322	writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1);
    323	writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus),
    324	       port->base + PCIE_CFG_HEADER2);
    325
    326	/* Write Cfgwr data */
    327	val = val << 8 * (where & 3);
    328	writel(val, port->base + PCIE_CFG_WDATA);
    329
    330	/* Trigger h/w to transmit Cfgwr TLP */
    331	val = readl(port->base + PCIE_APP_TLP_REQ);
    332	val |= APP_CFG_REQ;
    333	writel(val, port->base + PCIE_APP_TLP_REQ);
    334
    335	/* Check completion status */
    336	return mtk_pcie_check_cfg_cpld(port);
    337}
    338
    339static struct mtk_pcie_port *mtk_pcie_find_port(struct pci_bus *bus,
    340						unsigned int devfn)
    341{
    342	struct mtk_pcie *pcie = bus->sysdata;
    343	struct mtk_pcie_port *port;
    344	struct pci_dev *dev = NULL;
    345
    346	/*
    347	 * Walk the bus hierarchy to get the devfn value
    348	 * of the port in the root bus.
    349	 */
    350	while (bus && bus->number) {
    351		dev = bus->self;
    352		bus = dev->bus;
    353		devfn = dev->devfn;
    354	}
    355
    356	list_for_each_entry(port, &pcie->ports, list)
    357		if (port->slot == PCI_SLOT(devfn))
    358			return port;
    359
    360	return NULL;
    361}
    362
    363static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
    364				int where, int size, u32 *val)
    365{
    366	struct mtk_pcie_port *port;
    367	u32 bn = bus->number;
    368
    369	port = mtk_pcie_find_port(bus, devfn);
    370	if (!port)
    371		return PCIBIOS_DEVICE_NOT_FOUND;
    372
    373	return mtk_pcie_hw_rd_cfg(port, bn, devfn, where, size, val);
    374}
    375
    376static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
    377				 int where, int size, u32 val)
    378{
    379	struct mtk_pcie_port *port;
    380	u32 bn = bus->number;
    381
    382	port = mtk_pcie_find_port(bus, devfn);
    383	if (!port)
    384		return PCIBIOS_DEVICE_NOT_FOUND;
    385
    386	return mtk_pcie_hw_wr_cfg(port, bn, devfn, where, size, val);
    387}
    388
    389static struct pci_ops mtk_pcie_ops_v2 = {
    390	.read  = mtk_pcie_config_read,
    391	.write = mtk_pcie_config_write,
    392};
    393
    394static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
    395{
    396	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
    397	phys_addr_t addr;
    398
    399	/* MT2712/MT7622 only support 32-bit MSI addresses */
    400	addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
    401	msg->address_hi = 0;
    402	msg->address_lo = lower_32_bits(addr);
    403
    404	msg->data = data->hwirq;
    405
    406	dev_dbg(port->pcie->dev, "msi#%d address_hi %#x address_lo %#x\n",
    407		(int)data->hwirq, msg->address_hi, msg->address_lo);
    408}
    409
    410static int mtk_msi_set_affinity(struct irq_data *irq_data,
    411				const struct cpumask *mask, bool force)
    412{
    413	 return -EINVAL;
    414}
    415
    416static void mtk_msi_ack_irq(struct irq_data *data)
    417{
    418	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data);
    419	u32 hwirq = data->hwirq;
    420
    421	writel(1 << hwirq, port->base + PCIE_IMSI_STATUS);
    422}
    423
    424static struct irq_chip mtk_msi_bottom_irq_chip = {
    425	.name			= "MTK MSI",
    426	.irq_compose_msi_msg	= mtk_compose_msi_msg,
    427	.irq_set_affinity	= mtk_msi_set_affinity,
    428	.irq_ack		= mtk_msi_ack_irq,
    429};
    430
    431static int mtk_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
    432				     unsigned int nr_irqs, void *args)
    433{
    434	struct mtk_pcie_port *port = domain->host_data;
    435	unsigned long bit;
    436
    437	WARN_ON(nr_irqs != 1);
    438	mutex_lock(&port->lock);
    439
    440	bit = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM);
    441	if (bit >= MTK_MSI_IRQS_NUM) {
    442		mutex_unlock(&port->lock);
    443		return -ENOSPC;
    444	}
    445
    446	__set_bit(bit, port->msi_irq_in_use);
    447
    448	mutex_unlock(&port->lock);
    449
    450	irq_domain_set_info(domain, virq, bit, &mtk_msi_bottom_irq_chip,
    451			    domain->host_data, handle_edge_irq,
    452			    NULL, NULL);
    453
    454	return 0;
    455}
    456
    457static void mtk_pcie_irq_domain_free(struct irq_domain *domain,
    458				     unsigned int virq, unsigned int nr_irqs)
    459{
    460	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
    461	struct mtk_pcie_port *port = irq_data_get_irq_chip_data(d);
    462
    463	mutex_lock(&port->lock);
    464
    465	if (!test_bit(d->hwirq, port->msi_irq_in_use))
    466		dev_err(port->pcie->dev, "trying to free unused MSI#%lu\n",
    467			d->hwirq);
    468	else
    469		__clear_bit(d->hwirq, port->msi_irq_in_use);
    470
    471	mutex_unlock(&port->lock);
    472
    473	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
    474}
    475
    476static const struct irq_domain_ops msi_domain_ops = {
    477	.alloc	= mtk_pcie_irq_domain_alloc,
    478	.free	= mtk_pcie_irq_domain_free,
    479};
    480
    481static struct irq_chip mtk_msi_irq_chip = {
    482	.name		= "MTK PCIe MSI",
    483	.irq_ack	= irq_chip_ack_parent,
    484	.irq_mask	= pci_msi_mask_irq,
    485	.irq_unmask	= pci_msi_unmask_irq,
    486};
    487
    488static struct msi_domain_info mtk_msi_domain_info = {
    489	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
    490		   MSI_FLAG_PCI_MSIX),
    491	.chip	= &mtk_msi_irq_chip,
    492};
    493
    494static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port)
    495{
    496	struct fwnode_handle *fwnode = of_node_to_fwnode(port->pcie->dev->of_node);
    497
    498	mutex_init(&port->lock);
    499
    500	port->inner_domain = irq_domain_create_linear(fwnode, MTK_MSI_IRQS_NUM,
    501						      &msi_domain_ops, port);
    502	if (!port->inner_domain) {
    503		dev_err(port->pcie->dev, "failed to create IRQ domain\n");
    504		return -ENOMEM;
    505	}
    506
    507	port->msi_domain = pci_msi_create_irq_domain(fwnode, &mtk_msi_domain_info,
    508						     port->inner_domain);
    509	if (!port->msi_domain) {
    510		dev_err(port->pcie->dev, "failed to create MSI domain\n");
    511		irq_domain_remove(port->inner_domain);
    512		return -ENOMEM;
    513	}
    514
    515	return 0;
    516}
    517
    518static void mtk_pcie_enable_msi(struct mtk_pcie_port *port)
    519{
    520	u32 val;
    521	phys_addr_t msg_addr;
    522
    523	msg_addr = virt_to_phys(port->base + PCIE_MSI_VECTOR);
    524	val = lower_32_bits(msg_addr);
    525	writel(val, port->base + PCIE_IMSI_ADDR);
    526
    527	val = readl(port->base + PCIE_INT_MASK);
    528	val &= ~MSI_MASK;
    529	writel(val, port->base + PCIE_INT_MASK);
    530}
    531
    532static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie)
    533{
    534	struct mtk_pcie_port *port, *tmp;
    535
    536	list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
    537		irq_set_chained_handler_and_data(port->irq, NULL, NULL);
    538
    539		if (port->irq_domain)
    540			irq_domain_remove(port->irq_domain);
    541
    542		if (IS_ENABLED(CONFIG_PCI_MSI)) {
    543			if (port->msi_domain)
    544				irq_domain_remove(port->msi_domain);
    545			if (port->inner_domain)
    546				irq_domain_remove(port->inner_domain);
    547		}
    548
    549		irq_dispose_mapping(port->irq);
    550	}
    551}
    552
    553static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
    554			     irq_hw_number_t hwirq)
    555{
    556	irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
    557	irq_set_chip_data(irq, domain->host_data);
    558
    559	return 0;
    560}
    561
    562static const struct irq_domain_ops intx_domain_ops = {
    563	.map = mtk_pcie_intx_map,
    564};
    565
    566static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port,
    567				    struct device_node *node)
    568{
    569	struct device *dev = port->pcie->dev;
    570	struct device_node *pcie_intc_node;
    571	int ret;
    572
    573	/* Setup INTx */
    574	pcie_intc_node = of_get_next_child(node, NULL);
    575	if (!pcie_intc_node) {
    576		dev_err(dev, "no PCIe Intc node found\n");
    577		return -ENODEV;
    578	}
    579
    580	port->irq_domain = irq_domain_add_linear(pcie_intc_node, PCI_NUM_INTX,
    581						 &intx_domain_ops, port);
    582	of_node_put(pcie_intc_node);
    583	if (!port->irq_domain) {
    584		dev_err(dev, "failed to get INTx IRQ domain\n");
    585		return -ENODEV;
    586	}
    587
    588	if (IS_ENABLED(CONFIG_PCI_MSI)) {
    589		ret = mtk_pcie_allocate_msi_domains(port);
    590		if (ret)
    591			return ret;
    592	}
    593
    594	return 0;
    595}
    596
    597static void mtk_pcie_intr_handler(struct irq_desc *desc)
    598{
    599	struct mtk_pcie_port *port = irq_desc_get_handler_data(desc);
    600	struct irq_chip *irqchip = irq_desc_get_chip(desc);
    601	unsigned long status;
    602	u32 bit = INTX_SHIFT;
    603
    604	chained_irq_enter(irqchip, desc);
    605
    606	status = readl(port->base + PCIE_INT_STATUS);
    607	if (status & INTX_MASK) {
    608		for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) {
    609			/* Clear the INTx */
    610			writel(1 << bit, port->base + PCIE_INT_STATUS);
    611			generic_handle_domain_irq(port->irq_domain,
    612						  bit - INTX_SHIFT);
    613		}
    614	}
    615
    616	if (IS_ENABLED(CONFIG_PCI_MSI)) {
    617		if (status & MSI_STATUS){
    618			unsigned long imsi_status;
    619
    620			while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) {
    621				for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM)
    622					generic_handle_domain_irq(port->inner_domain, bit);
    623			}
    624			/* Clear MSI interrupt status */
    625			writel(MSI_STATUS, port->base + PCIE_INT_STATUS);
    626		}
    627	}
    628
    629	chained_irq_exit(irqchip, desc);
    630}
    631
    632static int mtk_pcie_setup_irq(struct mtk_pcie_port *port,
    633			      struct device_node *node)
    634{
    635	struct mtk_pcie *pcie = port->pcie;
    636	struct device *dev = pcie->dev;
    637	struct platform_device *pdev = to_platform_device(dev);
    638	int err;
    639
    640	err = mtk_pcie_init_irq_domain(port, node);
    641	if (err) {
    642		dev_err(dev, "failed to init PCIe IRQ domain\n");
    643		return err;
    644	}
    645
    646	if (of_find_property(dev->of_node, "interrupt-names", NULL))
    647		port->irq = platform_get_irq_byname(pdev, "pcie_irq");
    648	else
    649		port->irq = platform_get_irq(pdev, port->slot);
    650
    651	if (port->irq < 0)
    652		return port->irq;
    653
    654	irq_set_chained_handler_and_data(port->irq,
    655					 mtk_pcie_intr_handler, port);
    656
    657	return 0;
    658}
    659
    660static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port)
    661{
    662	struct mtk_pcie *pcie = port->pcie;
    663	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
    664	struct resource *mem = NULL;
    665	struct resource_entry *entry;
    666	const struct mtk_pcie_soc *soc = port->pcie->soc;
    667	u32 val;
    668	int err;
    669
    670	entry = resource_list_first_type(&host->windows, IORESOURCE_MEM);
    671	if (entry)
    672		mem = entry->res;
    673	if (!mem)
    674		return -EINVAL;
    675
    676	/* MT7622 platforms need to enable LTSSM and ASPM from PCIe subsys */
    677	if (pcie->base) {
    678		val = readl(pcie->base + PCIE_SYS_CFG_V2);
    679		val |= PCIE_CSR_LTSSM_EN(port->slot) |
    680		       PCIE_CSR_ASPM_L1_EN(port->slot);
    681		writel(val, pcie->base + PCIE_SYS_CFG_V2);
    682	} else if (pcie->cfg) {
    683		val = PCIE_CSR_LTSSM_EN(port->slot) |
    684		      PCIE_CSR_ASPM_L1_EN(port->slot);
    685		regmap_update_bits(pcie->cfg, PCIE_SYS_CFG_V2, val, val);
    686	}
    687
    688	/* Assert all reset signals */
    689	writel(0, port->base + PCIE_RST_CTRL);
    690
    691	/*
    692	 * Enable PCIe link down reset, if link status changed from link up to
    693	 * link down, this will reset MAC control registers and configuration
    694	 * space.
    695	 */
    696	writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL);
    697
    698	/*
    699	 * Described in PCIe CEM specification sections 2.2 (PERST# Signal) and
    700	 * 2.2.1 (Initial Power-Up (G3 to S0)). The deassertion of PERST# should
    701	 * be delayed 100ms (TPVPERL) for the power and clock to become stable.
    702	 */
    703	msleep(100);
    704
    705	/* De-assert PHY, PE, PIPE, MAC and configuration reset	*/
    706	val = readl(port->base + PCIE_RST_CTRL);
    707	val |= PCIE_PHY_RSTB | PCIE_PERSTB | PCIE_PIPE_SRSTB |
    708	       PCIE_MAC_SRSTB | PCIE_CRSTB;
    709	writel(val, port->base + PCIE_RST_CTRL);
    710
    711	/* Set up vendor ID and class code */
    712	if (soc->need_fix_class_id) {
    713		val = PCI_VENDOR_ID_MEDIATEK;
    714		writew(val, port->base + PCIE_CONF_VEND_ID);
    715
    716		val = PCI_CLASS_BRIDGE_PCI;
    717		writew(val, port->base + PCIE_CONF_CLASS_ID);
    718	}
    719
    720	if (soc->need_fix_device_id)
    721		writew(soc->device_id, port->base + PCIE_CONF_DEVICE_ID);
    722
    723	/* 100ms timeout value should be enough for Gen1/2 training */
    724	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val,
    725				 !!(val & PCIE_PORT_LINKUP_V2), 20,
    726				 100 * USEC_PER_MSEC);
    727	if (err)
    728		return -ETIMEDOUT;
    729
    730	/* Set INTx mask */
    731	val = readl(port->base + PCIE_INT_MASK);
    732	val &= ~INTX_MASK;
    733	writel(val, port->base + PCIE_INT_MASK);
    734
    735	if (IS_ENABLED(CONFIG_PCI_MSI))
    736		mtk_pcie_enable_msi(port);
    737
    738	/* Set AHB to PCIe translation windows */
    739	val = lower_32_bits(mem->start) |
    740	      AHB2PCIE_SIZE(fls(resource_size(mem)));
    741	writel(val, port->base + PCIE_AHB_TRANS_BASE0_L);
    742
    743	val = upper_32_bits(mem->start);
    744	writel(val, port->base + PCIE_AHB_TRANS_BASE0_H);
    745
    746	/* Set PCIe to AXI translation memory space.*/
    747	val = PCIE2AHB_SIZE | WIN_ENABLE;
    748	writel(val, port->base + PCIE_AXI_WINDOW0);
    749
    750	return 0;
    751}
    752
    753static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus,
    754				      unsigned int devfn, int where)
    755{
    756	struct mtk_pcie *pcie = bus->sysdata;
    757
    758	writel(PCIE_CONF_ADDR(where, PCI_FUNC(devfn), PCI_SLOT(devfn),
    759			      bus->number), pcie->base + PCIE_CFG_ADDR);
    760
    761	return pcie->base + PCIE_CFG_DATA + (where & 3);
    762}
    763
    764static struct pci_ops mtk_pcie_ops = {
    765	.map_bus = mtk_pcie_map_bus,
    766	.read  = pci_generic_config_read,
    767	.write = pci_generic_config_write,
    768};
    769
    770static int mtk_pcie_startup_port(struct mtk_pcie_port *port)
    771{
    772	struct mtk_pcie *pcie = port->pcie;
    773	u32 func = PCI_FUNC(port->slot);
    774	u32 slot = PCI_SLOT(port->slot << 3);
    775	u32 val;
    776	int err;
    777
    778	/* assert port PERST_N */
    779	val = readl(pcie->base + PCIE_SYS_CFG);
    780	val |= PCIE_PORT_PERST(port->slot);
    781	writel(val, pcie->base + PCIE_SYS_CFG);
    782
    783	/* de-assert port PERST_N */
    784	val = readl(pcie->base + PCIE_SYS_CFG);
    785	val &= ~PCIE_PORT_PERST(port->slot);
    786	writel(val, pcie->base + PCIE_SYS_CFG);
    787
    788	/* 100ms timeout value should be enough for Gen1/2 training */
    789	err = readl_poll_timeout(port->base + PCIE_LINK_STATUS, val,
    790				 !!(val & PCIE_PORT_LINKUP), 20,
    791				 100 * USEC_PER_MSEC);
    792	if (err)
    793		return -ETIMEDOUT;
    794
    795	/* enable interrupt */
    796	val = readl(pcie->base + PCIE_INT_ENABLE);
    797	val |= PCIE_PORT_INT_EN(port->slot);
    798	writel(val, pcie->base + PCIE_INT_ENABLE);
    799
    800	/* map to all DDR region. We need to set it before cfg operation. */
    801	writel(PCIE_BAR_MAP_MAX | PCIE_BAR_ENABLE,
    802	       port->base + PCIE_BAR0_SETUP);
    803
    804	/* configure class code and revision ID */
    805	writel(PCIE_CLASS_CODE | PCIE_REVISION_ID, port->base + PCIE_CLASS);
    806
    807	/* configure FC credit */
    808	writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0),
    809	       pcie->base + PCIE_CFG_ADDR);
    810	val = readl(pcie->base + PCIE_CFG_DATA);
    811	val &= ~PCIE_FC_CREDIT_MASK;
    812	val |= PCIE_FC_CREDIT_VAL(0x806c);
    813	writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0),
    814	       pcie->base + PCIE_CFG_ADDR);
    815	writel(val, pcie->base + PCIE_CFG_DATA);
    816
    817	/* configure RC FTS number to 250 when it leaves L0s */
    818	writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
    819	       pcie->base + PCIE_CFG_ADDR);
    820	val = readl(pcie->base + PCIE_CFG_DATA);
    821	val &= ~PCIE_FTS_NUM_MASK;
    822	val |= PCIE_FTS_NUM_L0(0x50);
    823	writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0),
    824	       pcie->base + PCIE_CFG_ADDR);
    825	writel(val, pcie->base + PCIE_CFG_DATA);
    826
    827	return 0;
    828}
    829
    830static void mtk_pcie_enable_port(struct mtk_pcie_port *port)
    831{
    832	struct mtk_pcie *pcie = port->pcie;
    833	struct device *dev = pcie->dev;
    834	int err;
    835
    836	err = clk_prepare_enable(port->sys_ck);
    837	if (err) {
    838		dev_err(dev, "failed to enable sys_ck%d clock\n", port->slot);
    839		goto err_sys_clk;
    840	}
    841
    842	err = clk_prepare_enable(port->ahb_ck);
    843	if (err) {
    844		dev_err(dev, "failed to enable ahb_ck%d\n", port->slot);
    845		goto err_ahb_clk;
    846	}
    847
    848	err = clk_prepare_enable(port->aux_ck);
    849	if (err) {
    850		dev_err(dev, "failed to enable aux_ck%d\n", port->slot);
    851		goto err_aux_clk;
    852	}
    853
    854	err = clk_prepare_enable(port->axi_ck);
    855	if (err) {
    856		dev_err(dev, "failed to enable axi_ck%d\n", port->slot);
    857		goto err_axi_clk;
    858	}
    859
    860	err = clk_prepare_enable(port->obff_ck);
    861	if (err) {
    862		dev_err(dev, "failed to enable obff_ck%d\n", port->slot);
    863		goto err_obff_clk;
    864	}
    865
    866	err = clk_prepare_enable(port->pipe_ck);
    867	if (err) {
    868		dev_err(dev, "failed to enable pipe_ck%d\n", port->slot);
    869		goto err_pipe_clk;
    870	}
    871
    872	reset_control_assert(port->reset);
    873	reset_control_deassert(port->reset);
    874
    875	err = phy_init(port->phy);
    876	if (err) {
    877		dev_err(dev, "failed to initialize port%d phy\n", port->slot);
    878		goto err_phy_init;
    879	}
    880
    881	err = phy_power_on(port->phy);
    882	if (err) {
    883		dev_err(dev, "failed to power on port%d phy\n", port->slot);
    884		goto err_phy_on;
    885	}
    886
    887	if (!pcie->soc->startup(port))
    888		return;
    889
    890	dev_info(dev, "Port%d link down\n", port->slot);
    891
    892	phy_power_off(port->phy);
    893err_phy_on:
    894	phy_exit(port->phy);
    895err_phy_init:
    896	clk_disable_unprepare(port->pipe_ck);
    897err_pipe_clk:
    898	clk_disable_unprepare(port->obff_ck);
    899err_obff_clk:
    900	clk_disable_unprepare(port->axi_ck);
    901err_axi_clk:
    902	clk_disable_unprepare(port->aux_ck);
    903err_aux_clk:
    904	clk_disable_unprepare(port->ahb_ck);
    905err_ahb_clk:
    906	clk_disable_unprepare(port->sys_ck);
    907err_sys_clk:
    908	mtk_pcie_port_free(port);
    909}
    910
    911static int mtk_pcie_parse_port(struct mtk_pcie *pcie,
    912			       struct device_node *node,
    913			       int slot)
    914{
    915	struct mtk_pcie_port *port;
    916	struct device *dev = pcie->dev;
    917	struct platform_device *pdev = to_platform_device(dev);
    918	char name[10];
    919	int err;
    920
    921	port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
    922	if (!port)
    923		return -ENOMEM;
    924
    925	snprintf(name, sizeof(name), "port%d", slot);
    926	port->base = devm_platform_ioremap_resource_byname(pdev, name);
    927	if (IS_ERR(port->base)) {
    928		dev_err(dev, "failed to map port%d base\n", slot);
    929		return PTR_ERR(port->base);
    930	}
    931
    932	snprintf(name, sizeof(name), "sys_ck%d", slot);
    933	port->sys_ck = devm_clk_get(dev, name);
    934	if (IS_ERR(port->sys_ck)) {
    935		dev_err(dev, "failed to get sys_ck%d clock\n", slot);
    936		return PTR_ERR(port->sys_ck);
    937	}
    938
    939	/* sys_ck might be divided into the following parts in some chips */
    940	snprintf(name, sizeof(name), "ahb_ck%d", slot);
    941	port->ahb_ck = devm_clk_get_optional(dev, name);
    942	if (IS_ERR(port->ahb_ck))
    943		return PTR_ERR(port->ahb_ck);
    944
    945	snprintf(name, sizeof(name), "axi_ck%d", slot);
    946	port->axi_ck = devm_clk_get_optional(dev, name);
    947	if (IS_ERR(port->axi_ck))
    948		return PTR_ERR(port->axi_ck);
    949
    950	snprintf(name, sizeof(name), "aux_ck%d", slot);
    951	port->aux_ck = devm_clk_get_optional(dev, name);
    952	if (IS_ERR(port->aux_ck))
    953		return PTR_ERR(port->aux_ck);
    954
    955	snprintf(name, sizeof(name), "obff_ck%d", slot);
    956	port->obff_ck = devm_clk_get_optional(dev, name);
    957	if (IS_ERR(port->obff_ck))
    958		return PTR_ERR(port->obff_ck);
    959
    960	snprintf(name, sizeof(name), "pipe_ck%d", slot);
    961	port->pipe_ck = devm_clk_get_optional(dev, name);
    962	if (IS_ERR(port->pipe_ck))
    963		return PTR_ERR(port->pipe_ck);
    964
    965	snprintf(name, sizeof(name), "pcie-rst%d", slot);
    966	port->reset = devm_reset_control_get_optional_exclusive(dev, name);
    967	if (PTR_ERR(port->reset) == -EPROBE_DEFER)
    968		return PTR_ERR(port->reset);
    969
    970	/* some platforms may use default PHY setting */
    971	snprintf(name, sizeof(name), "pcie-phy%d", slot);
    972	port->phy = devm_phy_optional_get(dev, name);
    973	if (IS_ERR(port->phy))
    974		return PTR_ERR(port->phy);
    975
    976	port->slot = slot;
    977	port->pcie = pcie;
    978
    979	if (pcie->soc->setup_irq) {
    980		err = pcie->soc->setup_irq(port, node);
    981		if (err)
    982			return err;
    983	}
    984
    985	INIT_LIST_HEAD(&port->list);
    986	list_add_tail(&port->list, &pcie->ports);
    987
    988	return 0;
    989}
    990
    991static int mtk_pcie_subsys_powerup(struct mtk_pcie *pcie)
    992{
    993	struct device *dev = pcie->dev;
    994	struct platform_device *pdev = to_platform_device(dev);
    995	struct resource *regs;
    996	struct device_node *cfg_node;
    997	int err;
    998
    999	/* get shared registers, which are optional */
   1000	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "subsys");
   1001	if (regs) {
   1002		pcie->base = devm_ioremap_resource(dev, regs);
   1003		if (IS_ERR(pcie->base))
   1004			return PTR_ERR(pcie->base);
   1005	}
   1006
   1007	cfg_node = of_find_compatible_node(NULL, NULL,
   1008					   "mediatek,generic-pciecfg");
   1009	if (cfg_node) {
   1010		pcie->cfg = syscon_node_to_regmap(cfg_node);
   1011		of_node_put(cfg_node);
   1012		if (IS_ERR(pcie->cfg))
   1013			return PTR_ERR(pcie->cfg);
   1014	}
   1015
   1016	pcie->free_ck = devm_clk_get(dev, "free_ck");
   1017	if (IS_ERR(pcie->free_ck)) {
   1018		if (PTR_ERR(pcie->free_ck) == -EPROBE_DEFER)
   1019			return -EPROBE_DEFER;
   1020
   1021		pcie->free_ck = NULL;
   1022	}
   1023
   1024	pm_runtime_enable(dev);
   1025	pm_runtime_get_sync(dev);
   1026
   1027	/* enable top level clock */
   1028	err = clk_prepare_enable(pcie->free_ck);
   1029	if (err) {
   1030		dev_err(dev, "failed to enable free_ck\n");
   1031		goto err_free_ck;
   1032	}
   1033
   1034	return 0;
   1035
   1036err_free_ck:
   1037	pm_runtime_put_sync(dev);
   1038	pm_runtime_disable(dev);
   1039
   1040	return err;
   1041}
   1042
   1043static int mtk_pcie_setup(struct mtk_pcie *pcie)
   1044{
   1045	struct device *dev = pcie->dev;
   1046	struct device_node *node = dev->of_node, *child;
   1047	struct mtk_pcie_port *port, *tmp;
   1048	int err, slot;
   1049
   1050	slot = of_get_pci_domain_nr(dev->of_node);
   1051	if (slot < 0) {
   1052		for_each_available_child_of_node(node, child) {
   1053			err = of_pci_get_devfn(child);
   1054			if (err < 0) {
   1055				dev_err(dev, "failed to get devfn: %d\n", err);
   1056				goto error_put_node;
   1057			}
   1058
   1059			slot = PCI_SLOT(err);
   1060
   1061			err = mtk_pcie_parse_port(pcie, child, slot);
   1062			if (err)
   1063				goto error_put_node;
   1064		}
   1065	} else {
   1066		err = mtk_pcie_parse_port(pcie, node, slot);
   1067		if (err)
   1068			return err;
   1069	}
   1070
   1071	err = mtk_pcie_subsys_powerup(pcie);
   1072	if (err)
   1073		return err;
   1074
   1075	/* enable each port, and then check link status */
   1076	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
   1077		mtk_pcie_enable_port(port);
   1078
   1079	/* power down PCIe subsys if slots are all empty (link down) */
   1080	if (list_empty(&pcie->ports))
   1081		mtk_pcie_subsys_powerdown(pcie);
   1082
   1083	return 0;
   1084error_put_node:
   1085	of_node_put(child);
   1086	return err;
   1087}
   1088
   1089static int mtk_pcie_probe(struct platform_device *pdev)
   1090{
   1091	struct device *dev = &pdev->dev;
   1092	struct mtk_pcie *pcie;
   1093	struct pci_host_bridge *host;
   1094	int err;
   1095
   1096	host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
   1097	if (!host)
   1098		return -ENOMEM;
   1099
   1100	pcie = pci_host_bridge_priv(host);
   1101
   1102	pcie->dev = dev;
   1103	pcie->soc = of_device_get_match_data(dev);
   1104	platform_set_drvdata(pdev, pcie);
   1105	INIT_LIST_HEAD(&pcie->ports);
   1106
   1107	err = mtk_pcie_setup(pcie);
   1108	if (err)
   1109		return err;
   1110
   1111	host->ops = pcie->soc->ops;
   1112	host->sysdata = pcie;
   1113	host->msi_domain = pcie->soc->no_msi;
   1114
   1115	err = pci_host_probe(host);
   1116	if (err)
   1117		goto put_resources;
   1118
   1119	return 0;
   1120
   1121put_resources:
   1122	if (!list_empty(&pcie->ports))
   1123		mtk_pcie_put_resources(pcie);
   1124
   1125	return err;
   1126}
   1127
   1128
   1129static void mtk_pcie_free_resources(struct mtk_pcie *pcie)
   1130{
   1131	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
   1132	struct list_head *windows = &host->windows;
   1133
   1134	pci_free_resource_list(windows);
   1135}
   1136
   1137static int mtk_pcie_remove(struct platform_device *pdev)
   1138{
   1139	struct mtk_pcie *pcie = platform_get_drvdata(pdev);
   1140	struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
   1141
   1142	pci_stop_root_bus(host->bus);
   1143	pci_remove_root_bus(host->bus);
   1144	mtk_pcie_free_resources(pcie);
   1145
   1146	mtk_pcie_irq_teardown(pcie);
   1147
   1148	mtk_pcie_put_resources(pcie);
   1149
   1150	return 0;
   1151}
   1152
   1153static int __maybe_unused mtk_pcie_suspend_noirq(struct device *dev)
   1154{
   1155	struct mtk_pcie *pcie = dev_get_drvdata(dev);
   1156	struct mtk_pcie_port *port;
   1157
   1158	if (list_empty(&pcie->ports))
   1159		return 0;
   1160
   1161	list_for_each_entry(port, &pcie->ports, list) {
   1162		clk_disable_unprepare(port->pipe_ck);
   1163		clk_disable_unprepare(port->obff_ck);
   1164		clk_disable_unprepare(port->axi_ck);
   1165		clk_disable_unprepare(port->aux_ck);
   1166		clk_disable_unprepare(port->ahb_ck);
   1167		clk_disable_unprepare(port->sys_ck);
   1168		phy_power_off(port->phy);
   1169		phy_exit(port->phy);
   1170	}
   1171
   1172	clk_disable_unprepare(pcie->free_ck);
   1173
   1174	return 0;
   1175}
   1176
   1177static int __maybe_unused mtk_pcie_resume_noirq(struct device *dev)
   1178{
   1179	struct mtk_pcie *pcie = dev_get_drvdata(dev);
   1180	struct mtk_pcie_port *port, *tmp;
   1181
   1182	if (list_empty(&pcie->ports))
   1183		return 0;
   1184
   1185	clk_prepare_enable(pcie->free_ck);
   1186
   1187	list_for_each_entry_safe(port, tmp, &pcie->ports, list)
   1188		mtk_pcie_enable_port(port);
   1189
   1190	/* In case of EP was removed while system suspend. */
   1191	if (list_empty(&pcie->ports))
   1192		clk_disable_unprepare(pcie->free_ck);
   1193
   1194	return 0;
   1195}
   1196
   1197static const struct dev_pm_ops mtk_pcie_pm_ops = {
   1198	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq,
   1199				      mtk_pcie_resume_noirq)
   1200};
   1201
   1202static const struct mtk_pcie_soc mtk_pcie_soc_v1 = {
   1203	.no_msi = true,
   1204	.ops = &mtk_pcie_ops,
   1205	.startup = mtk_pcie_startup_port,
   1206};
   1207
   1208static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = {
   1209	.ops = &mtk_pcie_ops_v2,
   1210	.startup = mtk_pcie_startup_port_v2,
   1211	.setup_irq = mtk_pcie_setup_irq,
   1212};
   1213
   1214static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = {
   1215	.need_fix_class_id = true,
   1216	.ops = &mtk_pcie_ops_v2,
   1217	.startup = mtk_pcie_startup_port_v2,
   1218	.setup_irq = mtk_pcie_setup_irq,
   1219};
   1220
   1221static const struct mtk_pcie_soc mtk_pcie_soc_mt7629 = {
   1222	.need_fix_class_id = true,
   1223	.need_fix_device_id = true,
   1224	.device_id = PCI_DEVICE_ID_MEDIATEK_7629,
   1225	.ops = &mtk_pcie_ops_v2,
   1226	.startup = mtk_pcie_startup_port_v2,
   1227	.setup_irq = mtk_pcie_setup_irq,
   1228};
   1229
   1230static const struct of_device_id mtk_pcie_ids[] = {
   1231	{ .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 },
   1232	{ .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 },
   1233	{ .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_mt2712 },
   1234	{ .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_mt7622 },
   1235	{ .compatible = "mediatek,mt7629-pcie", .data = &mtk_pcie_soc_mt7629 },
   1236	{},
   1237};
   1238MODULE_DEVICE_TABLE(of, mtk_pcie_ids);
   1239
   1240static struct platform_driver mtk_pcie_driver = {
   1241	.probe = mtk_pcie_probe,
   1242	.remove = mtk_pcie_remove,
   1243	.driver = {
   1244		.name = "mtk-pcie",
   1245		.of_match_table = mtk_pcie_ids,
   1246		.suppress_bind_attrs = true,
   1247		.pm = &mtk_pcie_pm_ops,
   1248	},
   1249};
   1250module_platform_driver(mtk_pcie_driver);
   1251MODULE_LICENSE("GPL v2");