cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

octep_cn9k_pf.c (24184B)


      1// SPDX-License-Identifier: GPL-2.0
      2/* Marvell Octeon EP (EndPoint) Ethernet Driver
      3 *
      4 * Copyright (C) 2020 Marvell.
      5 *
      6 */
      7
      8#include <linux/pci.h>
      9#include <linux/netdevice.h>
     10#include <linux/etherdevice.h>
     11
     12#include "octep_config.h"
     13#include "octep_main.h"
     14#include "octep_regs_cn9k_pf.h"
     15
     16/* Names of Hardware non-queue generic interrupts */
     17static char *cn93_non_ioq_msix_names[] = {
     18	"epf_ire_rint",
     19	"epf_ore_rint",
     20	"epf_vfire_rint0",
     21	"epf_vfire_rint1",
     22	"epf_vfore_rint0",
     23	"epf_vfore_rint1",
     24	"epf_mbox_rint0",
     25	"epf_mbox_rint1",
     26	"epf_oei_rint",
     27	"epf_dma_rint",
     28	"epf_dma_vf_rint0",
     29	"epf_dma_vf_rint1",
     30	"epf_pp_vf_rint0",
     31	"epf_pp_vf_rint1",
     32	"epf_misc_rint",
     33	"epf_rsvd",
     34};
     35
     36/* Dump useful hardware CSRs for debug purpose */
     37static void cn93_dump_regs(struct octep_device *oct, int qno)
     38{
     39	struct device *dev = &oct->pdev->dev;
     40
     41	dev_info(dev, "IQ-%d register dump\n", qno);
     42	dev_info(dev, "R[%d]_IN_INSTR_DBELL[0x%llx]: 0x%016llx\n",
     43		 qno, CN93_SDP_R_IN_INSTR_DBELL(qno),
     44		 octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(qno)));
     45	dev_info(dev, "R[%d]_IN_CONTROL[0x%llx]: 0x%016llx\n",
     46		 qno, CN93_SDP_R_IN_CONTROL(qno),
     47		 octep_read_csr64(oct, CN93_SDP_R_IN_CONTROL(qno)));
     48	dev_info(dev, "R[%d]_IN_ENABLE[0x%llx]: 0x%016llx\n",
     49		 qno, CN93_SDP_R_IN_ENABLE(qno),
     50		 octep_read_csr64(oct, CN93_SDP_R_IN_ENABLE(qno)));
     51	dev_info(dev, "R[%d]_IN_INSTR_BADDR[0x%llx]: 0x%016llx\n",
     52		 qno, CN93_SDP_R_IN_INSTR_BADDR(qno),
     53		 octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_BADDR(qno)));
     54	dev_info(dev, "R[%d]_IN_INSTR_RSIZE[0x%llx]: 0x%016llx\n",
     55		 qno, CN93_SDP_R_IN_INSTR_RSIZE(qno),
     56		 octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_RSIZE(qno)));
     57	dev_info(dev, "R[%d]_IN_CNTS[0x%llx]: 0x%016llx\n",
     58		 qno, CN93_SDP_R_IN_CNTS(qno),
     59		 octep_read_csr64(oct, CN93_SDP_R_IN_CNTS(qno)));
     60	dev_info(dev, "R[%d]_IN_INT_LEVELS[0x%llx]: 0x%016llx\n",
     61		 qno, CN93_SDP_R_IN_INT_LEVELS(qno),
     62		 octep_read_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(qno)));
     63	dev_info(dev, "R[%d]_IN_PKT_CNT[0x%llx]: 0x%016llx\n",
     64		 qno, CN93_SDP_R_IN_PKT_CNT(qno),
     65		 octep_read_csr64(oct, CN93_SDP_R_IN_PKT_CNT(qno)));
     66	dev_info(dev, "R[%d]_IN_BYTE_CNT[0x%llx]: 0x%016llx\n",
     67		 qno, CN93_SDP_R_IN_BYTE_CNT(qno),
     68		 octep_read_csr64(oct, CN93_SDP_R_IN_BYTE_CNT(qno)));
     69
     70	dev_info(dev, "OQ-%d register dump\n", qno);
     71	dev_info(dev, "R[%d]_OUT_SLIST_DBELL[0x%llx]: 0x%016llx\n",
     72		 qno, CN93_SDP_R_OUT_SLIST_DBELL(qno),
     73		 octep_read_csr64(oct, CN93_SDP_R_OUT_SLIST_DBELL(qno)));
     74	dev_info(dev, "R[%d]_OUT_CONTROL[0x%llx]: 0x%016llx\n",
     75		 qno, CN93_SDP_R_OUT_CONTROL(qno),
     76		 octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(qno)));
     77	dev_info(dev, "R[%d]_OUT_ENABLE[0x%llx]: 0x%016llx\n",
     78		 qno, CN93_SDP_R_OUT_ENABLE(qno),
     79		 octep_read_csr64(oct, CN93_SDP_R_OUT_ENABLE(qno)));
     80	dev_info(dev, "R[%d]_OUT_SLIST_BADDR[0x%llx]: 0x%016llx\n",
     81		 qno, CN93_SDP_R_OUT_SLIST_BADDR(qno),
     82		 octep_read_csr64(oct, CN93_SDP_R_OUT_SLIST_BADDR(qno)));
     83	dev_info(dev, "R[%d]_OUT_SLIST_RSIZE[0x%llx]: 0x%016llx\n",
     84		 qno, CN93_SDP_R_OUT_SLIST_RSIZE(qno),
     85		 octep_read_csr64(oct, CN93_SDP_R_OUT_SLIST_RSIZE(qno)));
     86	dev_info(dev, "R[%d]_OUT_CNTS[0x%llx]: 0x%016llx\n",
     87		 qno, CN93_SDP_R_OUT_CNTS(qno),
     88		 octep_read_csr64(oct, CN93_SDP_R_OUT_CNTS(qno)));
     89	dev_info(dev, "R[%d]_OUT_INT_LEVELS[0x%llx]: 0x%016llx\n",
     90		 qno, CN93_SDP_R_OUT_INT_LEVELS(qno),
     91		 octep_read_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(qno)));
     92	dev_info(dev, "R[%d]_OUT_PKT_CNT[0x%llx]: 0x%016llx\n",
     93		 qno, CN93_SDP_R_OUT_PKT_CNT(qno),
     94		 octep_read_csr64(oct, CN93_SDP_R_OUT_PKT_CNT(qno)));
     95	dev_info(dev, "R[%d]_OUT_BYTE_CNT[0x%llx]: 0x%016llx\n",
     96		 qno, CN93_SDP_R_OUT_BYTE_CNT(qno),
     97		 octep_read_csr64(oct, CN93_SDP_R_OUT_BYTE_CNT(qno)));
     98	dev_info(dev, "R[%d]_ERR_TYPE[0x%llx]: 0x%016llx\n",
     99		 qno, CN93_SDP_R_ERR_TYPE(qno),
    100		 octep_read_csr64(oct, CN93_SDP_R_ERR_TYPE(qno)));
    101}
    102
    103/* Reset Hardware Tx queue */
    104static int cn93_reset_iq(struct octep_device *oct, int q_no)
    105{
    106	struct octep_config *conf = oct->conf;
    107	u64 val = 0ULL;
    108
    109	dev_dbg(&oct->pdev->dev, "Reset PF IQ-%d\n", q_no);
    110
    111	/* Get absolute queue number */
    112	q_no += conf->pf_ring_cfg.srn;
    113
    114	/* Disable the Tx/Instruction Ring */
    115	octep_write_csr64(oct, CN93_SDP_R_IN_ENABLE(q_no), val);
    116
    117	/* clear the Instruction Ring packet/byte counts and doorbell CSRs */
    118	octep_write_csr64(oct, CN93_SDP_R_IN_CNTS(q_no), val);
    119	octep_write_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(q_no), val);
    120	octep_write_csr64(oct, CN93_SDP_R_IN_PKT_CNT(q_no), val);
    121	octep_write_csr64(oct, CN93_SDP_R_IN_BYTE_CNT(q_no), val);
    122	octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_BADDR(q_no), val);
    123	octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_RSIZE(q_no), val);
    124
    125	val = 0xFFFFFFFF;
    126	octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(q_no), val);
    127
    128	return 0;
    129}
    130
    131/* Reset Hardware Rx queue */
    132static void cn93_reset_oq(struct octep_device *oct, int q_no)
    133{
    134	u64 val = 0ULL;
    135
    136	q_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    137
    138	/* Disable Output (Rx) Ring */
    139	octep_write_csr64(oct, CN93_SDP_R_OUT_ENABLE(q_no), val);
    140
    141	/* Clear count CSRs */
    142	val = octep_read_csr(oct, CN93_SDP_R_OUT_CNTS(q_no));
    143	octep_write_csr(oct, CN93_SDP_R_OUT_CNTS(q_no), val);
    144
    145	octep_write_csr64(oct, CN93_SDP_R_OUT_PKT_CNT(q_no), 0xFFFFFFFFFULL);
    146	octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_DBELL(q_no), 0xFFFFFFFF);
    147}
    148
    149/* Reset all hardware Tx/Rx queues */
    150static void octep_reset_io_queues_cn93_pf(struct octep_device *oct)
    151{
    152	struct pci_dev *pdev = oct->pdev;
    153	int q;
    154
    155	dev_dbg(&pdev->dev, "Reset OCTEP_CN93 PF IO Queues\n");
    156
    157	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) {
    158		cn93_reset_iq(oct, q);
    159		cn93_reset_oq(oct, q);
    160	}
    161}
    162
    163/* Initialize windowed addresses to access some hardware registers */
    164static void octep_setup_pci_window_regs_cn93_pf(struct octep_device *oct)
    165{
    166	u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr;
    167
    168	oct->pci_win_regs.pci_win_wr_addr = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_WR_ADDR64);
    169	oct->pci_win_regs.pci_win_rd_addr = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_RD_ADDR64);
    170	oct->pci_win_regs.pci_win_wr_data = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_WR_DATA64);
    171	oct->pci_win_regs.pci_win_rd_data = (u8 __iomem *)(bar0_pciaddr + CN93_SDP_WIN_RD_DATA64);
    172}
    173
    174/* Configure Hardware mapping: inform hardware which rings belong to PF. */
    175static void octep_configure_ring_mapping_cn93_pf(struct octep_device *oct)
    176{
    177	struct octep_config *conf = oct->conf;
    178	struct pci_dev *pdev = oct->pdev;
    179	u64 pf_srn = CFG_GET_PORTS_PF_SRN(oct->conf);
    180	int q;
    181
    182	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(conf); q++) {
    183		u64 regval = 0;
    184
    185		if (oct->pcie_port)
    186			regval = 8 << CN93_SDP_FUNC_SEL_EPF_BIT_POS;
    187
    188		octep_write_csr64(oct, CN93_SDP_EPVF_RING(pf_srn + q), regval);
    189
    190		regval = octep_read_csr64(oct, CN93_SDP_EPVF_RING(pf_srn + q));
    191		dev_dbg(&pdev->dev, "Write SDP_EPVF_RING[0x%llx] = 0x%llx\n",
    192			CN93_SDP_EPVF_RING(pf_srn + q), regval);
    193	}
    194}
    195
    196/* Initialize configuration limits and initial active config 93xx PF. */
    197static void octep_init_config_cn93_pf(struct octep_device *oct)
    198{
    199	struct octep_config *conf = oct->conf;
    200	struct pci_dev *pdev = oct->pdev;
    201	u64 val;
    202
    203	/* Read ring configuration:
    204	 * PF ring count, number of VFs and rings per VF supported
    205	 */
    206	val = octep_read_csr64(oct, CN93_SDP_EPF_RINFO);
    207	conf->sriov_cfg.max_rings_per_vf = CN93_SDP_EPF_RINFO_RPVF(val);
    208	conf->sriov_cfg.active_rings_per_vf = conf->sriov_cfg.max_rings_per_vf;
    209	conf->sriov_cfg.max_vfs = CN93_SDP_EPF_RINFO_NVFS(val);
    210	conf->sriov_cfg.active_vfs = conf->sriov_cfg.max_vfs;
    211	conf->sriov_cfg.vf_srn = CN93_SDP_EPF_RINFO_SRN(val);
    212
    213	val = octep_read_csr64(oct, CN93_SDP_MAC_PF_RING_CTL(oct->pcie_port));
    214	conf->pf_ring_cfg.srn =  CN93_SDP_MAC_PF_RING_CTL_SRN(val);
    215	conf->pf_ring_cfg.max_io_rings = CN93_SDP_MAC_PF_RING_CTL_RPPF(val);
    216	conf->pf_ring_cfg.active_io_rings = conf->pf_ring_cfg.max_io_rings;
    217	dev_info(&pdev->dev, "pf_srn=%u rpvf=%u nvfs=%u rppf=%u\n",
    218		 conf->pf_ring_cfg.srn, conf->sriov_cfg.active_rings_per_vf,
    219		 conf->sriov_cfg.active_vfs, conf->pf_ring_cfg.active_io_rings);
    220
    221	conf->iq.num_descs = OCTEP_IQ_MAX_DESCRIPTORS;
    222	conf->iq.instr_type = OCTEP_64BYTE_INSTR;
    223	conf->iq.pkind = 0;
    224	conf->iq.db_min = OCTEP_DB_MIN;
    225	conf->iq.intr_threshold = OCTEP_IQ_INTR_THRESHOLD;
    226
    227	conf->oq.num_descs = OCTEP_OQ_MAX_DESCRIPTORS;
    228	conf->oq.buf_size = OCTEP_OQ_BUF_SIZE;
    229	conf->oq.refill_threshold = OCTEP_OQ_REFILL_THRESHOLD;
    230	conf->oq.oq_intr_pkt = OCTEP_OQ_INTR_PKT_THRESHOLD;
    231	conf->oq.oq_intr_time = OCTEP_OQ_INTR_TIME_THRESHOLD;
    232
    233	conf->msix_cfg.non_ioq_msix = CN93_NUM_NON_IOQ_INTR;
    234	conf->msix_cfg.ioq_msix = conf->pf_ring_cfg.active_io_rings;
    235	conf->msix_cfg.non_ioq_msix_names = cn93_non_ioq_msix_names;
    236
    237	conf->ctrl_mbox_cfg.barmem_addr = (void __iomem *)oct->mmio[2].hw_addr + (0x400000ull * 7);
    238}
    239
    240/* Setup registers for a hardware Tx Queue  */
    241static void octep_setup_iq_regs_cn93_pf(struct octep_device *oct, int iq_no)
    242{
    243	struct octep_iq *iq = oct->iq[iq_no];
    244	u32 reset_instr_cnt;
    245	u64 reg_val;
    246
    247	iq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    248	reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_CONTROL(iq_no));
    249
    250	/* wait for IDLE to set to 1 */
    251	if (!(reg_val & CN93_R_IN_CTL_IDLE)) {
    252		do {
    253			reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_CONTROL(iq_no));
    254		} while (!(reg_val & CN93_R_IN_CTL_IDLE));
    255	}
    256
    257	reg_val |= CN93_R_IN_CTL_RDSIZE;
    258	reg_val |= CN93_R_IN_CTL_IS_64B;
    259	reg_val |= CN93_R_IN_CTL_ESR;
    260	octep_write_csr64(oct, CN93_SDP_R_IN_CONTROL(iq_no), reg_val);
    261
    262	/* Write the start of the input queue's ring and its size  */
    263	octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_BADDR(iq_no),
    264			  iq->desc_ring_dma);
    265	octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_RSIZE(iq_no),
    266			  iq->max_count);
    267
    268	/* Remember the doorbell & instruction count register addr
    269	 * for this queue
    270	 */
    271	iq->doorbell_reg = oct->mmio[0].hw_addr +
    272			   CN93_SDP_R_IN_INSTR_DBELL(iq_no);
    273	iq->inst_cnt_reg = oct->mmio[0].hw_addr +
    274			   CN93_SDP_R_IN_CNTS(iq_no);
    275	iq->intr_lvl_reg = oct->mmio[0].hw_addr +
    276			   CN93_SDP_R_IN_INT_LEVELS(iq_no);
    277
    278	/* Store the current instruction counter (used in flush_iq calculation) */
    279	reset_instr_cnt = readl(iq->inst_cnt_reg);
    280	writel(reset_instr_cnt, iq->inst_cnt_reg);
    281
    282	/* INTR_THRESHOLD is set to max(FFFFFFFF) to disable the INTR */
    283	reg_val = CFG_GET_IQ_INTR_THRESHOLD(oct->conf) & 0xffffffff;
    284	octep_write_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(iq_no), reg_val);
    285}
    286
    287/* Setup registers for a hardware Rx Queue  */
    288static void octep_setup_oq_regs_cn93_pf(struct octep_device *oct, int oq_no)
    289{
    290	u64 reg_val;
    291	u64 oq_ctl = 0ULL;
    292	u32 time_threshold = 0;
    293	struct octep_oq *oq = oct->oq[oq_no];
    294
    295	oq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    296	reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no));
    297
    298	/* wait for IDLE to set to 1 */
    299	if (!(reg_val & CN93_R_OUT_CTL_IDLE)) {
    300		do {
    301			reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no));
    302		} while (!(reg_val & CN93_R_OUT_CTL_IDLE));
    303	}
    304
    305	reg_val &= ~(CN93_R_OUT_CTL_IMODE);
    306	reg_val &= ~(CN93_R_OUT_CTL_ROR_P);
    307	reg_val &= ~(CN93_R_OUT_CTL_NSR_P);
    308	reg_val &= ~(CN93_R_OUT_CTL_ROR_I);
    309	reg_val &= ~(CN93_R_OUT_CTL_NSR_I);
    310	reg_val &= ~(CN93_R_OUT_CTL_ES_I);
    311	reg_val &= ~(CN93_R_OUT_CTL_ROR_D);
    312	reg_val &= ~(CN93_R_OUT_CTL_NSR_D);
    313	reg_val &= ~(CN93_R_OUT_CTL_ES_D);
    314	reg_val |= (CN93_R_OUT_CTL_ES_P);
    315
    316	octep_write_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no), reg_val);
    317	octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_BADDR(oq_no),
    318			  oq->desc_ring_dma);
    319	octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_RSIZE(oq_no),
    320			  oq->max_count);
    321
    322	oq_ctl = octep_read_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no));
    323	oq_ctl &= ~0x7fffffULL;	//clear the ISIZE and BSIZE (22-0)
    324	oq_ctl |= (oq->buffer_size & 0xffff);	//populate the BSIZE (15-0)
    325	octep_write_csr64(oct, CN93_SDP_R_OUT_CONTROL(oq_no), oq_ctl);
    326
    327	/* Get the mapped address of the pkt_sent and pkts_credit regs */
    328	oq->pkts_sent_reg = oct->mmio[0].hw_addr + CN93_SDP_R_OUT_CNTS(oq_no);
    329	oq->pkts_credit_reg = oct->mmio[0].hw_addr +
    330			      CN93_SDP_R_OUT_SLIST_DBELL(oq_no);
    331
    332	time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf);
    333	reg_val = ((u64)time_threshold << 32) |
    334		  CFG_GET_OQ_INTR_PKT(oct->conf);
    335	octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val);
    336}
    337
    338/* Setup registers for a PF mailbox */
    339static void octep_setup_mbox_regs_cn93_pf(struct octep_device *oct, int q_no)
    340{
    341	struct octep_mbox *mbox = oct->mbox[q_no];
    342
    343	mbox->q_no = q_no;
    344
    345	/* PF mbox interrupt reg */
    346	mbox->mbox_int_reg = oct->mmio[0].hw_addr + CN93_SDP_EPF_MBOX_RINT(0);
    347
    348	/* PF to VF DATA reg. PF writes into this reg */
    349	mbox->mbox_write_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_PF_VF_DATA(q_no);
    350
    351	/* VF to PF DATA reg. PF reads from this reg */
    352	mbox->mbox_read_reg = oct->mmio[0].hw_addr + CN93_SDP_R_MBOX_VF_PF_DATA(q_no);
    353}
    354
    355/* Mailbox Interrupt handler */
    356static void cn93_handle_pf_mbox_intr(struct octep_device *oct)
    357{
    358	u64 mbox_int_val = 0ULL, val = 0ULL, qno = 0ULL;
    359
    360	mbox_int_val = readq(oct->mbox[0]->mbox_int_reg);
    361	for (qno = 0; qno < OCTEP_MAX_VF; qno++) {
    362		val = readq(oct->mbox[qno]->mbox_read_reg);
    363		dev_dbg(&oct->pdev->dev,
    364			"PF MBOX READ: val:%llx from VF:%llx\n", val, qno);
    365	}
    366
    367	writeq(mbox_int_val, oct->mbox[0]->mbox_int_reg);
    368}
    369
    370/* Interrupts handler for all non-queue generic interrupts. */
    371static irqreturn_t octep_non_ioq_intr_handler_cn93_pf(void *dev)
    372{
    373	struct octep_device *oct = (struct octep_device *)dev;
    374	struct pci_dev *pdev = oct->pdev;
    375	u64 reg_val = 0;
    376	int i = 0;
    377
    378	/* Check for IRERR INTR */
    379	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_IRERR_RINT);
    380	if (reg_val) {
    381		dev_info(&pdev->dev,
    382			 "received IRERR_RINT intr: 0x%llx\n", reg_val);
    383		octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT, reg_val);
    384
    385		for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
    386			reg_val = octep_read_csr64(oct,
    387						   CN93_SDP_R_ERR_TYPE(i));
    388			if (reg_val) {
    389				dev_info(&pdev->dev,
    390					 "Received err type on IQ-%d: 0x%llx\n",
    391					 i, reg_val);
    392				octep_write_csr64(oct, CN93_SDP_R_ERR_TYPE(i),
    393						  reg_val);
    394			}
    395		}
    396		goto irq_handled;
    397	}
    398
    399	/* Check for ORERR INTR */
    400	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_ORERR_RINT);
    401	if (reg_val) {
    402		dev_info(&pdev->dev,
    403			 "Received ORERR_RINT intr: 0x%llx\n", reg_val);
    404		octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT, reg_val);
    405		for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
    406			reg_val = octep_read_csr64(oct, CN93_SDP_R_ERR_TYPE(i));
    407			if (reg_val) {
    408				dev_info(&pdev->dev,
    409					 "Received err type on OQ-%d: 0x%llx\n",
    410					 i, reg_val);
    411				octep_write_csr64(oct, CN93_SDP_R_ERR_TYPE(i),
    412						  reg_val);
    413			}
    414		}
    415
    416		goto irq_handled;
    417	}
    418
    419	/* Check for VFIRE INTR */
    420	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_VFIRE_RINT(0));
    421	if (reg_val) {
    422		dev_info(&pdev->dev,
    423			 "Received VFIRE_RINT intr: 0x%llx\n", reg_val);
    424		octep_write_csr64(oct, CN93_SDP_EPF_VFIRE_RINT(0), reg_val);
    425		goto irq_handled;
    426	}
    427
    428	/* Check for VFORE INTR */
    429	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_VFORE_RINT(0));
    430	if (reg_val) {
    431		dev_info(&pdev->dev,
    432			 "Received VFORE_RINT intr: 0x%llx\n", reg_val);
    433		octep_write_csr64(oct, CN93_SDP_EPF_VFORE_RINT(0), reg_val);
    434		goto irq_handled;
    435	}
    436
    437	/* Check for MBOX INTR */
    438	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_MBOX_RINT(0));
    439	if (reg_val) {
    440		dev_info(&pdev->dev,
    441			 "Received MBOX_RINT intr: 0x%llx\n", reg_val);
    442		cn93_handle_pf_mbox_intr(oct);
    443		goto irq_handled;
    444	}
    445
    446	/* Check for OEI INTR */
    447	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_OEI_RINT);
    448	if (reg_val) {
    449		dev_info(&pdev->dev,
    450			 "Received OEI_EINT intr: 0x%llx\n", reg_val);
    451		octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT, reg_val);
    452		queue_work(octep_wq, &oct->ctrl_mbox_task);
    453		goto irq_handled;
    454	}
    455
    456	/* Check for DMA INTR */
    457	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_DMA_RINT);
    458	if (reg_val) {
    459		octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT, reg_val);
    460		goto irq_handled;
    461	}
    462
    463	/* Check for DMA VF INTR */
    464	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_DMA_VF_RINT(0));
    465	if (reg_val) {
    466		dev_info(&pdev->dev,
    467			 "Received DMA_VF_RINT intr: 0x%llx\n", reg_val);
    468		octep_write_csr64(oct, CN93_SDP_EPF_DMA_VF_RINT(0), reg_val);
    469		goto irq_handled;
    470	}
    471
    472	/* Check for PPVF INTR */
    473	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_PP_VF_RINT(0));
    474	if (reg_val) {
    475		dev_info(&pdev->dev,
    476			 "Received PP_VF_RINT intr: 0x%llx\n", reg_val);
    477		octep_write_csr64(oct, CN93_SDP_EPF_PP_VF_RINT(0), reg_val);
    478		goto irq_handled;
    479	}
    480
    481	/* Check for MISC INTR */
    482	reg_val = octep_read_csr64(oct, CN93_SDP_EPF_MISC_RINT);
    483	if (reg_val) {
    484		dev_info(&pdev->dev,
    485			 "Received MISC_RINT intr: 0x%llx\n", reg_val);
    486		octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT, reg_val);
    487		goto irq_handled;
    488	}
    489
    490	dev_info(&pdev->dev, "Reserved interrupts raised; Ignore\n");
    491irq_handled:
    492	return IRQ_HANDLED;
    493}
    494
    495/* Tx/Rx queue interrupt handler */
    496static irqreturn_t octep_ioq_intr_handler_cn93_pf(void *data)
    497{
    498	struct octep_ioq_vector *vector = (struct octep_ioq_vector *)data;
    499	struct octep_oq *oq = vector->oq;
    500
    501	napi_schedule_irqoff(oq->napi);
    502	return IRQ_HANDLED;
    503}
    504
    505/* soft reset of 93xx */
    506static int octep_soft_reset_cn93_pf(struct octep_device *oct)
    507{
    508	dev_info(&oct->pdev->dev, "CN93XX: Doing soft reset\n");
    509
    510	octep_write_csr64(oct, CN93_SDP_WIN_WR_MASK_REG, 0xFF);
    511
    512	/* Set core domain reset bit */
    513	OCTEP_PCI_WIN_WRITE(oct, CN93_RST_CORE_DOMAIN_W1S, 1);
    514	/* Wait for 100ms as Octeon resets. */
    515	mdelay(100);
    516	/* clear core domain reset bit */
    517	OCTEP_PCI_WIN_WRITE(oct, CN93_RST_CORE_DOMAIN_W1C, 1);
    518
    519	return 0;
    520}
    521
    522/* Re-initialize Octeon hardware registers */
    523static void octep_reinit_regs_cn93_pf(struct octep_device *oct)
    524{
    525	u32 i;
    526
    527	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++)
    528		oct->hw_ops.setup_iq_regs(oct, i);
    529
    530	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++)
    531		oct->hw_ops.setup_oq_regs(oct, i);
    532
    533	oct->hw_ops.enable_interrupts(oct);
    534	oct->hw_ops.enable_io_queues(oct);
    535
    536	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++)
    537		writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
    538}
    539
    540/* Enable all interrupts */
    541static void octep_enable_interrupts_cn93_pf(struct octep_device *oct)
    542{
    543	u64 intr_mask = 0ULL;
    544	int srn, num_rings, i;
    545
    546	srn = CFG_GET_PORTS_PF_SRN(oct->conf);
    547	num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf);
    548
    549	for (i = 0; i < num_rings; i++)
    550		intr_mask |= (0x1ULL << (srn + i));
    551
    552	octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1S, intr_mask);
    553	octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1S, intr_mask);
    554	octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT_ENA_W1S, -1ULL);
    555	octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT_ENA_W1S, intr_mask);
    556	octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT_ENA_W1S, intr_mask);
    557}
    558
    559/* Disable all interrupts */
    560static void octep_disable_interrupts_cn93_pf(struct octep_device *oct)
    561{
    562	u64 intr_mask = 0ULL;
    563	int srn, num_rings, i;
    564
    565	srn = CFG_GET_PORTS_PF_SRN(oct->conf);
    566	num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf);
    567
    568	for (i = 0; i < num_rings; i++)
    569		intr_mask |= (0x1ULL << (srn + i));
    570
    571	octep_write_csr64(oct, CN93_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask);
    572	octep_write_csr64(oct, CN93_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask);
    573	octep_write_csr64(oct, CN93_SDP_EPF_OEI_RINT_ENA_W1C, -1ULL);
    574	octep_write_csr64(oct, CN93_SDP_EPF_MISC_RINT_ENA_W1C, intr_mask);
    575	octep_write_csr64(oct, CN93_SDP_EPF_DMA_RINT_ENA_W1C, intr_mask);
    576}
    577
    578/* Get new Octeon Read Index: index of descriptor that Octeon reads next. */
    579static u32 octep_update_iq_read_index_cn93_pf(struct octep_iq *iq)
    580{
    581	u32 pkt_in_done = readl(iq->inst_cnt_reg);
    582	u32 last_done, new_idx;
    583
    584	last_done = pkt_in_done - iq->pkt_in_done;
    585	iq->pkt_in_done = pkt_in_done;
    586
    587	new_idx = (iq->octep_read_index + last_done) % iq->max_count;
    588
    589	return new_idx;
    590}
    591
    592/* Enable a hardware Tx Queue */
    593static void octep_enable_iq_cn93_pf(struct octep_device *oct, int iq_no)
    594{
    595	u64 loop = HZ;
    596	u64 reg_val;
    597
    598	iq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    599
    600	octep_write_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(iq_no), 0xFFFFFFFF);
    601
    602	while (octep_read_csr64(oct, CN93_SDP_R_IN_INSTR_DBELL(iq_no)) &&
    603	       loop--) {
    604		schedule_timeout_interruptible(1);
    605	}
    606
    607	reg_val = octep_read_csr64(oct,  CN93_SDP_R_IN_INT_LEVELS(iq_no));
    608	reg_val |= (0x1ULL << 62);
    609	octep_write_csr64(oct, CN93_SDP_R_IN_INT_LEVELS(iq_no), reg_val);
    610
    611	reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no));
    612	reg_val |= 0x1ULL;
    613	octep_write_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no), reg_val);
    614}
    615
    616/* Enable a hardware Rx Queue */
    617static void octep_enable_oq_cn93_pf(struct octep_device *oct, int oq_no)
    618{
    619	u64 reg_val = 0ULL;
    620
    621	oq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    622
    623	reg_val = octep_read_csr64(oct,  CN93_SDP_R_OUT_INT_LEVELS(oq_no));
    624	reg_val |= (0x1ULL << 62);
    625	octep_write_csr64(oct, CN93_SDP_R_OUT_INT_LEVELS(oq_no), reg_val);
    626
    627	octep_write_csr64(oct, CN93_SDP_R_OUT_SLIST_DBELL(oq_no), 0xFFFFFFFF);
    628
    629	reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no));
    630	reg_val |= 0x1ULL;
    631	octep_write_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no), reg_val);
    632}
    633
    634/* Enable all hardware Tx/Rx Queues assined to PF */
    635static void octep_enable_io_queues_cn93_pf(struct octep_device *oct)
    636{
    637	u8 q;
    638
    639	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) {
    640		octep_enable_iq_cn93_pf(oct, q);
    641		octep_enable_oq_cn93_pf(oct, q);
    642	}
    643}
    644
    645/* Disable a hardware Tx Queue assined to PF */
    646static void octep_disable_iq_cn93_pf(struct octep_device *oct, int iq_no)
    647{
    648	u64 reg_val = 0ULL;
    649
    650	iq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    651
    652	reg_val = octep_read_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no));
    653	reg_val &= ~0x1ULL;
    654	octep_write_csr64(oct, CN93_SDP_R_IN_ENABLE(iq_no), reg_val);
    655}
    656
    657/* Disable a hardware Rx Queue assined to PF */
    658static void octep_disable_oq_cn93_pf(struct octep_device *oct, int oq_no)
    659{
    660	u64 reg_val = 0ULL;
    661
    662	oq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
    663	reg_val = octep_read_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no));
    664	reg_val &= ~0x1ULL;
    665	octep_write_csr64(oct, CN93_SDP_R_OUT_ENABLE(oq_no), reg_val);
    666}
    667
    668/* Disable all hardware Tx/Rx Queues assined to PF */
    669static void octep_disable_io_queues_cn93_pf(struct octep_device *oct)
    670{
    671	int q = 0;
    672
    673	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) {
    674		octep_disable_iq_cn93_pf(oct, q);
    675		octep_disable_oq_cn93_pf(oct, q);
    676	}
    677}
    678
    679/* Dump hardware registers (including Tx/Rx queues) for debugging. */
    680static void octep_dump_registers_cn93_pf(struct octep_device *oct)
    681{
    682	u8 srn, num_rings, q;
    683
    684	srn = CFG_GET_PORTS_PF_SRN(oct->conf);
    685	num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf);
    686
    687	for (q = srn; q < srn + num_rings; q++)
    688		cn93_dump_regs(oct, q);
    689}
    690
    691/**
    692 * octep_device_setup_cn93_pf() - Setup Octeon device.
    693 *
    694 * @oct: Octeon device private data structure.
    695 *
    696 * - initialize hardware operations.
    697 * - get target side pcie port number for the device.
    698 * - setup window access to hardware registers.
    699 * - set initial configuration and max limits.
    700 * - setup hardware mapping of rings to the PF device.
    701 */
    702void octep_device_setup_cn93_pf(struct octep_device *oct)
    703{
    704	oct->hw_ops.setup_iq_regs = octep_setup_iq_regs_cn93_pf;
    705	oct->hw_ops.setup_oq_regs = octep_setup_oq_regs_cn93_pf;
    706	oct->hw_ops.setup_mbox_regs = octep_setup_mbox_regs_cn93_pf;
    707
    708	oct->hw_ops.non_ioq_intr_handler = octep_non_ioq_intr_handler_cn93_pf;
    709	oct->hw_ops.ioq_intr_handler = octep_ioq_intr_handler_cn93_pf;
    710	oct->hw_ops.soft_reset = octep_soft_reset_cn93_pf;
    711	oct->hw_ops.reinit_regs = octep_reinit_regs_cn93_pf;
    712
    713	oct->hw_ops.enable_interrupts = octep_enable_interrupts_cn93_pf;
    714	oct->hw_ops.disable_interrupts = octep_disable_interrupts_cn93_pf;
    715
    716	oct->hw_ops.update_iq_read_idx = octep_update_iq_read_index_cn93_pf;
    717
    718	oct->hw_ops.enable_iq = octep_enable_iq_cn93_pf;
    719	oct->hw_ops.enable_oq = octep_enable_oq_cn93_pf;
    720	oct->hw_ops.enable_io_queues = octep_enable_io_queues_cn93_pf;
    721
    722	oct->hw_ops.disable_iq = octep_disable_iq_cn93_pf;
    723	oct->hw_ops.disable_oq = octep_disable_oq_cn93_pf;
    724	oct->hw_ops.disable_io_queues = octep_disable_io_queues_cn93_pf;
    725	oct->hw_ops.reset_io_queues = octep_reset_io_queues_cn93_pf;
    726
    727	oct->hw_ops.dump_registers = octep_dump_registers_cn93_pf;
    728
    729	octep_setup_pci_window_regs_cn93_pf(oct);
    730
    731	oct->pcie_port = octep_read_csr64(oct, CN93_SDP_MAC_NUMBER) & 0xff;
    732	dev_info(&oct->pdev->dev,
    733		 "Octeon device using PCIE Port %d\n", oct->pcie_port);
    734
    735	octep_init_config_cn93_pf(oct);
    736	octep_configure_ring_mapping_cn93_pf(oct);
    737}