cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

thunderx_edac.c (54062B)


      1/*
      2 * Cavium ThunderX memory controller kernel module
      3 *
      4 * This file is subject to the terms and conditions of the GNU General Public
      5 * License.  See the file "COPYING" in the main directory of this archive
      6 * for more details.
      7 *
      8 * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
      9 *
     10 */
     11
     12#include <linux/module.h>
     13#include <linux/pci.h>
     14#include <linux/edac.h>
     15#include <linux/interrupt.h>
     16#include <linux/string.h>
     17#include <linux/stop_machine.h>
     18#include <linux/delay.h>
     19#include <linux/sizes.h>
     20#include <linux/atomic.h>
     21#include <linux/bitfield.h>
     22#include <linux/circ_buf.h>
     23
     24#include <asm/page.h>
     25
     26#include "edac_module.h"
     27
     28#define phys_to_pfn(phys)	(PFN_DOWN(phys))
     29
     30#define THUNDERX_NODE		GENMASK(45, 44)
     31
     32enum {
     33	ERR_CORRECTED	= 1,
     34	ERR_UNCORRECTED	= 2,
     35	ERR_UNKNOWN	= 3,
     36};
     37
     38#define MAX_SYNDROME_REGS 4
     39
     40struct error_syndrome {
     41	u64 reg[MAX_SYNDROME_REGS];
     42};
     43
     44struct error_descr {
     45	int	type;
     46	u64	mask;
     47	char	*descr;
     48};
     49
     50static void decode_register(char *str, size_t size,
     51			   const struct error_descr *descr,
     52			   const uint64_t reg)
     53{
     54	int ret = 0;
     55
     56	while (descr->type && descr->mask && descr->descr) {
     57		if (reg & descr->mask) {
     58			ret = snprintf(str, size, "\n\t%s, %s",
     59				       descr->type == ERR_CORRECTED ?
     60					 "Corrected" : "Uncorrected",
     61				       descr->descr);
     62			str += ret;
     63			size -= ret;
     64		}
     65		descr++;
     66	}
     67}
     68
     69static unsigned long get_bits(unsigned long data, int pos, int width)
     70{
     71	return (data >> pos) & ((1 << width) - 1);
     72}
     73
     74#define L2C_CTL			0x87E080800000
     75#define L2C_CTL_DISIDXALIAS	BIT(0)
     76
     77#define PCI_DEVICE_ID_THUNDER_LMC 0xa022
     78
     79#define LMC_FADR		0x20
     80#define LMC_FADR_FDIMM(x)	((x >> 37) & 0x1)
     81#define LMC_FADR_FBUNK(x)	((x >> 36) & 0x1)
     82#define LMC_FADR_FBANK(x)	((x >> 32) & 0xf)
     83#define LMC_FADR_FROW(x)	((x >> 14) & 0xffff)
     84#define LMC_FADR_FCOL(x)	((x >> 0) & 0x1fff)
     85
     86#define LMC_NXM_FADR		0x28
     87#define LMC_ECC_SYND		0x38
     88
     89#define LMC_ECC_PARITY_TEST	0x108
     90
     91#define LMC_INT_W1S		0x150
     92
     93#define LMC_INT_ENA_W1C		0x158
     94#define LMC_INT_ENA_W1S		0x160
     95
     96#define LMC_CONFIG		0x188
     97
     98#define LMC_CONFIG_BG2		BIT(62)
     99#define LMC_CONFIG_RANK_ENA	BIT(42)
    100#define LMC_CONFIG_PBANK_LSB(x)	(((x) >> 5) & 0xF)
    101#define LMC_CONFIG_ROW_LSB(x)	(((x) >> 2) & 0x7)
    102
    103#define LMC_CONTROL		0x190
    104#define LMC_CONTROL_XOR_BANK	BIT(16)
    105
    106#define LMC_INT			0x1F0
    107
    108#define LMC_INT_DDR_ERR		BIT(11)
    109#define LMC_INT_DED_ERR		(0xFUL << 5)
    110#define LMC_INT_SEC_ERR         (0xFUL << 1)
    111#define LMC_INT_NXM_WR_MASK	BIT(0)
    112
    113#define LMC_DDR_PLL_CTL		0x258
    114#define LMC_DDR_PLL_CTL_DDR4	BIT(29)
    115
    116#define LMC_FADR_SCRAMBLED	0x330
    117
    118#define LMC_INT_UE              (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
    119				 LMC_INT_NXM_WR_MASK)
    120
    121#define LMC_INT_CE		(LMC_INT_SEC_ERR)
    122
    123static const struct error_descr lmc_errors[] = {
    124	{
    125		.type  = ERR_CORRECTED,
    126		.mask  = LMC_INT_SEC_ERR,
    127		.descr = "Single-bit ECC error",
    128	},
    129	{
    130		.type  = ERR_UNCORRECTED,
    131		.mask  = LMC_INT_DDR_ERR,
    132		.descr = "DDR chip error",
    133	},
    134	{
    135		.type  = ERR_UNCORRECTED,
    136		.mask  = LMC_INT_DED_ERR,
    137		.descr = "Double-bit ECC error",
    138	},
    139	{
    140		.type = ERR_UNCORRECTED,
    141		.mask = LMC_INT_NXM_WR_MASK,
    142		.descr = "Non-existent memory write",
    143	},
    144	{0, 0, NULL},
    145};
    146
    147#define LMC_INT_EN_DDR_ERROR_ALERT_ENA	BIT(5)
    148#define LMC_INT_EN_DLCRAM_DED_ERR	BIT(4)
    149#define LMC_INT_EN_DLCRAM_SEC_ERR	BIT(3)
    150#define LMC_INT_INTR_DED_ENA		BIT(2)
    151#define LMC_INT_INTR_SEC_ENA		BIT(1)
    152#define LMC_INT_INTR_NXM_WR_ENA		BIT(0)
    153
    154#define LMC_INT_ENA_ALL			GENMASK(5, 0)
    155
    156#define LMC_DDR_PLL_CTL		0x258
    157#define LMC_DDR_PLL_CTL_DDR4	BIT(29)
    158
    159#define LMC_CONTROL		0x190
    160#define LMC_CONTROL_RDIMM	BIT(0)
    161
    162#define LMC_SCRAM_FADR		0x330
    163
    164#define LMC_CHAR_MASK0		0x228
    165#define LMC_CHAR_MASK2		0x238
    166
    167#define RING_ENTRIES	8
    168
    169struct debugfs_entry {
    170	const char *name;
    171	umode_t mode;
    172	const struct file_operations fops;
    173};
    174
    175struct lmc_err_ctx {
    176	u64 reg_int;
    177	u64 reg_fadr;
    178	u64 reg_nxm_fadr;
    179	u64 reg_scram_fadr;
    180	u64 reg_ecc_synd;
    181};
    182
    183struct thunderx_lmc {
    184	void __iomem *regs;
    185	struct pci_dev *pdev;
    186	struct msix_entry msix_ent;
    187
    188	atomic_t ecc_int;
    189
    190	u64 mask0;
    191	u64 mask2;
    192	u64 parity_test;
    193	u64 node;
    194
    195	int xbits;
    196	int bank_width;
    197	int pbank_lsb;
    198	int dimm_lsb;
    199	int rank_lsb;
    200	int bank_lsb;
    201	int row_lsb;
    202	int col_hi_lsb;
    203
    204	int xor_bank;
    205	int l2c_alias;
    206
    207	struct page *mem;
    208
    209	struct lmc_err_ctx err_ctx[RING_ENTRIES];
    210	unsigned long ring_head;
    211	unsigned long ring_tail;
    212};
    213
    214#define ring_pos(pos, size) ((pos) & (size - 1))
    215
    216#define DEBUGFS_STRUCT(_name, _mode, _write, _read)			    \
    217static struct debugfs_entry debugfs_##_name = {				    \
    218	.name = __stringify(_name),					    \
    219	.mode = VERIFY_OCTAL_PERMISSIONS(_mode),			    \
    220	.fops = {							    \
    221		.open = simple_open,					    \
    222		.write = _write,					    \
    223		.read  = _read,						    \
    224		.llseek = generic_file_llseek,				    \
    225	},								    \
    226}
    227
    228#define DEBUGFS_FIELD_ATTR(_type, _field)				    \
    229static ssize_t thunderx_##_type##_##_field##_read(struct file *file,	    \
    230					    char __user *data,		    \
    231					    size_t count, loff_t *ppos)	    \
    232{									    \
    233	struct thunderx_##_type *pdata = file->private_data;		    \
    234	char buf[20];							    \
    235									    \
    236	snprintf(buf, count, "0x%016llx", pdata->_field);		    \
    237	return simple_read_from_buffer(data, count, ppos,		    \
    238				       buf, sizeof(buf));		    \
    239}									    \
    240									    \
    241static ssize_t thunderx_##_type##_##_field##_write(struct file *file,	    \
    242					     const char __user *data,	    \
    243					     size_t count, loff_t *ppos)    \
    244{									    \
    245	struct thunderx_##_type *pdata = file->private_data;		    \
    246	int res;							    \
    247									    \
    248	res = kstrtoull_from_user(data, count, 0, &pdata->_field);	    \
    249									    \
    250	return res ? res : count;					    \
    251}									    \
    252									    \
    253DEBUGFS_STRUCT(_field, 0600,						    \
    254		   thunderx_##_type##_##_field##_write,			    \
    255		   thunderx_##_type##_##_field##_read)			    \
    256
    257#define DEBUGFS_REG_ATTR(_type, _name, _reg)				    \
    258static ssize_t thunderx_##_type##_##_name##_read(struct file *file,	    \
    259					   char __user *data,		    \
    260					   size_t count, loff_t *ppos)      \
    261{									    \
    262	struct thunderx_##_type *pdata = file->private_data;		    \
    263	char buf[20];							    \
    264									    \
    265	sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));		    \
    266	return simple_read_from_buffer(data, count, ppos,		    \
    267				       buf, sizeof(buf));		    \
    268}									    \
    269									    \
    270static ssize_t thunderx_##_type##_##_name##_write(struct file *file,	    \
    271					    const char __user *data,	    \
    272					    size_t count, loff_t *ppos)     \
    273{									    \
    274	struct thunderx_##_type *pdata = file->private_data;		    \
    275	u64 val;							    \
    276	int res;							    \
    277									    \
    278	res = kstrtoull_from_user(data, count, 0, &val);		    \
    279									    \
    280	if (!res) {							    \
    281		writeq(val, pdata->regs + _reg);			    \
    282		res = count;						    \
    283	}								    \
    284									    \
    285	return res;							    \
    286}									    \
    287									    \
    288DEBUGFS_STRUCT(_name, 0600,						    \
    289	       thunderx_##_type##_##_name##_write,			    \
    290	       thunderx_##_type##_##_name##_read)
    291
    292#define LMC_DEBUGFS_ENT(_field)	DEBUGFS_FIELD_ATTR(lmc, _field)
    293
    294/*
    295 * To get an ECC error injected, the following steps are needed:
    296 * - Setup the ECC injection by writing the appropriate parameters:
    297 *	echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
    298 *	echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
    299 *	echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
    300 * - Do the actual injection:
    301 *	echo 1 > /sys/kernel/debug/<device number>/inject_ecc
    302 */
    303static ssize_t thunderx_lmc_inject_int_write(struct file *file,
    304					     const char __user *data,
    305					     size_t count, loff_t *ppos)
    306{
    307	struct thunderx_lmc *lmc = file->private_data;
    308	u64 val;
    309	int res;
    310
    311	res = kstrtoull_from_user(data, count, 0, &val);
    312
    313	if (!res) {
    314		/* Trigger the interrupt */
    315		writeq(val, lmc->regs + LMC_INT_W1S);
    316		res = count;
    317	}
    318
    319	return res;
    320}
    321
    322static ssize_t thunderx_lmc_int_read(struct file *file,
    323				     char __user *data,
    324				     size_t count, loff_t *ppos)
    325{
    326	struct thunderx_lmc *lmc = file->private_data;
    327	char buf[20];
    328	u64 lmc_int = readq(lmc->regs + LMC_INT);
    329
    330	snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
    331	return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
    332}
    333
    334#define TEST_PATTERN 0xa5
    335
    336static int inject_ecc_fn(void *arg)
    337{
    338	struct thunderx_lmc *lmc = arg;
    339	uintptr_t addr, phys;
    340	unsigned int cline_size = cache_line_size();
    341	const unsigned int lines = PAGE_SIZE / cline_size;
    342	unsigned int i, cl_idx;
    343
    344	addr = (uintptr_t)page_address(lmc->mem);
    345	phys = (uintptr_t)page_to_phys(lmc->mem);
    346
    347	cl_idx = (phys & 0x7f) >> 4;
    348	lmc->parity_test &= ~(7ULL << 8);
    349	lmc->parity_test |= (cl_idx << 8);
    350
    351	writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
    352	writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
    353	writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
    354
    355	readq(lmc->regs + LMC_CHAR_MASK0);
    356	readq(lmc->regs + LMC_CHAR_MASK2);
    357	readq(lmc->regs + LMC_ECC_PARITY_TEST);
    358
    359	for (i = 0; i < lines; i++) {
    360		memset((void *)addr, TEST_PATTERN, cline_size);
    361		barrier();
    362
    363		/*
    364		 * Flush L1 cachelines to the PoC (L2).
    365		 * This will cause cacheline eviction to the L2.
    366		 */
    367		asm volatile("dc civac, %0\n"
    368			     "dsb sy\n"
    369			     : : "r"(addr + i * cline_size));
    370	}
    371
    372	for (i = 0; i < lines; i++) {
    373		/*
    374		 * Flush L2 cachelines to the DRAM.
    375		 * This will cause cacheline eviction to the DRAM
    376		 * and ECC corruption according to the masks set.
    377		 */
    378		__asm__ volatile("sys #0,c11,C1,#2, %0\n"
    379				 : : "r"(phys + i * cline_size));
    380	}
    381
    382	for (i = 0; i < lines; i++) {
    383		/*
    384		 * Invalidate L2 cachelines.
    385		 * The subsequent load will cause cacheline fetch
    386		 * from the DRAM and an error interrupt
    387		 */
    388		__asm__ volatile("sys #0,c11,C1,#1, %0"
    389				 : : "r"(phys + i * cline_size));
    390	}
    391
    392	for (i = 0; i < lines; i++) {
    393		/*
    394		 * Invalidate L1 cachelines.
    395		 * The subsequent load will cause cacheline fetch
    396		 * from the L2 and/or DRAM
    397		 */
    398		asm volatile("dc ivac, %0\n"
    399			     "dsb sy\n"
    400			     : : "r"(addr + i * cline_size));
    401	}
    402
    403	return 0;
    404}
    405
    406static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
    407					     const char __user *data,
    408					     size_t count, loff_t *ppos)
    409{
    410	struct thunderx_lmc *lmc = file->private_data;
    411	unsigned int cline_size = cache_line_size();
    412	u8 *tmp;
    413	void __iomem *addr;
    414	unsigned int offs, timeout = 100000;
    415
    416	atomic_set(&lmc->ecc_int, 0);
    417
    418	lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
    419	if (!lmc->mem)
    420		return -ENOMEM;
    421
    422	tmp = kmalloc(cline_size, GFP_KERNEL);
    423	if (!tmp) {
    424		__free_pages(lmc->mem, 0);
    425		return -ENOMEM;
    426	}
    427
    428	addr = page_address(lmc->mem);
    429
    430	while (!atomic_read(&lmc->ecc_int) && timeout--) {
    431		stop_machine(inject_ecc_fn, lmc, NULL);
    432
    433		for (offs = 0; offs < PAGE_SIZE; offs += cline_size) {
    434			/*
    435			 * Do a load from the previously rigged location
    436			 * This should generate an error interrupt.
    437			 */
    438			memcpy(tmp, addr + offs, cline_size);
    439			asm volatile("dsb ld\n");
    440		}
    441	}
    442
    443	kfree(tmp);
    444	__free_pages(lmc->mem, 0);
    445
    446	return count;
    447}
    448
    449LMC_DEBUGFS_ENT(mask0);
    450LMC_DEBUGFS_ENT(mask2);
    451LMC_DEBUGFS_ENT(parity_test);
    452
    453DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
    454DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
    455DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
    456
    457static struct debugfs_entry *lmc_dfs_ents[] = {
    458	&debugfs_mask0,
    459	&debugfs_mask2,
    460	&debugfs_parity_test,
    461	&debugfs_inject_ecc,
    462	&debugfs_inject_int,
    463	&debugfs_int_w1c,
    464};
    465
    466static int thunderx_create_debugfs_nodes(struct dentry *parent,
    467					  struct debugfs_entry *attrs[],
    468					  void *data,
    469					  size_t num)
    470{
    471	int i;
    472	struct dentry *ent;
    473
    474	if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
    475		return 0;
    476
    477	if (!parent)
    478		return -ENOENT;
    479
    480	for (i = 0; i < num; i++) {
    481		ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
    482					       parent, data, &attrs[i]->fops);
    483
    484		if (!ent)
    485			break;
    486	}
    487
    488	return i;
    489}
    490
    491static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
    492{
    493	phys_addr_t addr = 0;
    494	int bank, xbits;
    495
    496	addr |= lmc->node << 40;
    497	addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
    498	addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
    499	addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
    500	addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
    501
    502	bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
    503
    504	if (lmc->xor_bank)
    505		bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
    506
    507	addr |= bank << lmc->bank_lsb;
    508
    509	xbits = PCI_FUNC(lmc->pdev->devfn);
    510
    511	if (lmc->l2c_alias)
    512		xbits ^= get_bits(addr, 20, lmc->xbits) ^
    513			 get_bits(addr, 12, lmc->xbits);
    514
    515	addr |= xbits << 7;
    516
    517	return addr;
    518}
    519
    520static unsigned int thunderx_get_num_lmcs(unsigned int node)
    521{
    522	unsigned int number = 0;
    523	struct pci_dev *pdev = NULL;
    524
    525	do {
    526		pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
    527				      PCI_DEVICE_ID_THUNDER_LMC,
    528				      pdev);
    529		if (pdev) {
    530#ifdef CONFIG_NUMA
    531			if (pdev->dev.numa_node == node)
    532				number++;
    533#else
    534			number++;
    535#endif
    536		}
    537	} while (pdev);
    538
    539	return number;
    540}
    541
    542#define LMC_MESSAGE_SIZE	120
    543#define LMC_OTHER_SIZE		(50 * ARRAY_SIZE(lmc_errors))
    544
    545static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
    546{
    547	struct mem_ctl_info *mci = dev_id;
    548	struct thunderx_lmc *lmc = mci->pvt_info;
    549
    550	unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
    551	struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
    552
    553	writeq(0, lmc->regs + LMC_CHAR_MASK0);
    554	writeq(0, lmc->regs + LMC_CHAR_MASK2);
    555	writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
    556
    557	ctx->reg_int = readq(lmc->regs + LMC_INT);
    558	ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
    559	ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
    560	ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
    561	ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
    562
    563	lmc->ring_head++;
    564
    565	atomic_set(&lmc->ecc_int, 1);
    566
    567	/* Clear the interrupt */
    568	writeq(ctx->reg_int, lmc->regs + LMC_INT);
    569
    570	return IRQ_WAKE_THREAD;
    571}
    572
    573static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
    574{
    575	struct mem_ctl_info *mci = dev_id;
    576	struct thunderx_lmc *lmc = mci->pvt_info;
    577	phys_addr_t phys_addr;
    578
    579	unsigned long tail;
    580	struct lmc_err_ctx *ctx;
    581
    582	irqreturn_t ret = IRQ_NONE;
    583
    584	char *msg;
    585	char *other;
    586
    587	msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
    588	other =  kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
    589
    590	if (!msg || !other)
    591		goto err_free;
    592
    593	while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
    594		ARRAY_SIZE(lmc->err_ctx))) {
    595		tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
    596
    597		ctx = &lmc->err_ctx[tail];
    598
    599		dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
    600			ctx->reg_int);
    601		dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
    602			ctx->reg_fadr);
    603		dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
    604			ctx->reg_nxm_fadr);
    605		dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
    606			ctx->reg_scram_fadr);
    607		dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
    608			ctx->reg_ecc_synd);
    609
    610		snprintf(msg, LMC_MESSAGE_SIZE,
    611			 "DIMM %lld rank %lld bank %lld row %lld col %lld",
    612			 LMC_FADR_FDIMM(ctx->reg_scram_fadr),
    613			 LMC_FADR_FBUNK(ctx->reg_scram_fadr),
    614			 LMC_FADR_FBANK(ctx->reg_scram_fadr),
    615			 LMC_FADR_FROW(ctx->reg_scram_fadr),
    616			 LMC_FADR_FCOL(ctx->reg_scram_fadr));
    617
    618		decode_register(other, LMC_OTHER_SIZE, lmc_errors,
    619				ctx->reg_int);
    620
    621		phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
    622
    623		if (ctx->reg_int & LMC_INT_UE)
    624			edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
    625					     phys_to_pfn(phys_addr),
    626					     offset_in_page(phys_addr),
    627					     0, -1, -1, -1, msg, other);
    628		else if (ctx->reg_int & LMC_INT_CE)
    629			edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
    630					     phys_to_pfn(phys_addr),
    631					     offset_in_page(phys_addr),
    632					     0, -1, -1, -1, msg, other);
    633
    634		lmc->ring_tail++;
    635	}
    636
    637	ret = IRQ_HANDLED;
    638
    639err_free:
    640	kfree(msg);
    641	kfree(other);
    642
    643	return ret;
    644}
    645
    646static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
    647	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
    648	{ 0, },
    649};
    650
    651static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
    652{
    653	int node = dev_to_node(&pdev->dev);
    654	int ret = PCI_FUNC(pdev->devfn);
    655
    656	ret += max(node, 0) << 3;
    657
    658	return ret;
    659}
    660
    661static int thunderx_lmc_probe(struct pci_dev *pdev,
    662				const struct pci_device_id *id)
    663{
    664	struct thunderx_lmc *lmc;
    665	struct edac_mc_layer layer;
    666	struct mem_ctl_info *mci;
    667	u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
    668	int ret;
    669	u64 lmc_int;
    670	void *l2c_ioaddr;
    671
    672	layer.type = EDAC_MC_LAYER_SLOT;
    673	layer.size = 2;
    674	layer.is_virt_csrow = false;
    675
    676	ret = pcim_enable_device(pdev);
    677	if (ret) {
    678		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
    679		return ret;
    680	}
    681
    682	ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
    683	if (ret) {
    684		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
    685		return ret;
    686	}
    687
    688	mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
    689			    sizeof(struct thunderx_lmc));
    690	if (!mci)
    691		return -ENOMEM;
    692
    693	mci->pdev = &pdev->dev;
    694	lmc = mci->pvt_info;
    695
    696	pci_set_drvdata(pdev, mci);
    697
    698	lmc->regs = pcim_iomap_table(pdev)[0];
    699
    700	lmc_control = readq(lmc->regs + LMC_CONTROL);
    701	lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
    702	lmc_config = readq(lmc->regs + LMC_CONFIG);
    703
    704	if (lmc_control & LMC_CONTROL_RDIMM) {
    705		mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
    706					   lmc_ddr_pll_ctl) ?
    707				MEM_RDDR4 : MEM_RDDR3;
    708	} else {
    709		mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
    710					   lmc_ddr_pll_ctl) ?
    711				MEM_DDR4 : MEM_DDR3;
    712	}
    713
    714	mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
    715	mci->edac_cap = EDAC_FLAG_SECDED;
    716
    717	mci->mod_name = "thunderx-lmc";
    718	mci->ctl_name = "thunderx-lmc";
    719	mci->dev_name = dev_name(&pdev->dev);
    720	mci->scrub_mode = SCRUB_NONE;
    721
    722	lmc->pdev = pdev;
    723	lmc->msix_ent.entry = 0;
    724
    725	lmc->ring_head = 0;
    726	lmc->ring_tail = 0;
    727
    728	ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
    729	if (ret) {
    730		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
    731		goto err_free;
    732	}
    733
    734	ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
    735					thunderx_lmc_err_isr,
    736					thunderx_lmc_threaded_isr, 0,
    737					"[EDAC] ThunderX LMC", mci);
    738	if (ret) {
    739		dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
    740		goto err_free;
    741	}
    742
    743	lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
    744
    745	lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
    746	lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
    747			   FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
    748
    749	lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
    750	lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;
    751	lmc->rank_lsb = lmc->dimm_lsb;
    752	lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
    753	lmc->bank_lsb = 7 + lmc->xbits;
    754	lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
    755
    756	lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
    757
    758	lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
    759
    760	l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
    761	if (!l2c_ioaddr) {
    762		dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
    763		ret = -ENOMEM;
    764		goto err_free;
    765	}
    766
    767	lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
    768
    769	iounmap(l2c_ioaddr);
    770
    771	ret = edac_mc_add_mc(mci);
    772	if (ret) {
    773		dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
    774		goto err_free;
    775	}
    776
    777	lmc_int = readq(lmc->regs + LMC_INT);
    778	writeq(lmc_int, lmc->regs + LMC_INT);
    779
    780	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
    781
    782	if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
    783		ret = thunderx_create_debugfs_nodes(mci->debugfs,
    784						    lmc_dfs_ents,
    785						    lmc,
    786						    ARRAY_SIZE(lmc_dfs_ents));
    787
    788		if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
    789			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
    790				 ret, ret >= 0 ? " created" : "");
    791		}
    792	}
    793
    794	return 0;
    795
    796err_free:
    797	pci_set_drvdata(pdev, NULL);
    798	edac_mc_free(mci);
    799
    800	return ret;
    801}
    802
    803static void thunderx_lmc_remove(struct pci_dev *pdev)
    804{
    805	struct mem_ctl_info *mci = pci_get_drvdata(pdev);
    806	struct thunderx_lmc *lmc = mci->pvt_info;
    807
    808	writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
    809
    810	edac_mc_del_mc(&pdev->dev);
    811	edac_mc_free(mci);
    812}
    813
    814MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
    815
    816static struct pci_driver thunderx_lmc_driver = {
    817	.name     = "thunderx_lmc_edac",
    818	.probe    = thunderx_lmc_probe,
    819	.remove   = thunderx_lmc_remove,
    820	.id_table = thunderx_lmc_pci_tbl,
    821};
    822
    823/*---------------------- OCX driver ---------------------------------*/
    824
    825#define PCI_DEVICE_ID_THUNDER_OCX 0xa013
    826
    827#define OCX_LINK_INTS		3
    828#define OCX_INTS		(OCX_LINK_INTS + 1)
    829#define OCX_RX_LANES		24
    830#define OCX_RX_LANE_STATS	15
    831
    832#define OCX_COM_INT		0x100
    833#define OCX_COM_INT_W1S		0x108
    834#define OCX_COM_INT_ENA_W1S	0x110
    835#define OCX_COM_INT_ENA_W1C	0x118
    836
    837#define OCX_COM_IO_BADID		BIT(54)
    838#define OCX_COM_MEM_BADID		BIT(53)
    839#define OCX_COM_COPR_BADID		BIT(52)
    840#define OCX_COM_WIN_REQ_BADID		BIT(51)
    841#define OCX_COM_WIN_REQ_TOUT		BIT(50)
    842#define OCX_COM_RX_LANE			GENMASK(23, 0)
    843
    844#define OCX_COM_INT_CE			(OCX_COM_IO_BADID      | \
    845					 OCX_COM_MEM_BADID     | \
    846					 OCX_COM_COPR_BADID    | \
    847					 OCX_COM_WIN_REQ_BADID | \
    848					 OCX_COM_WIN_REQ_TOUT)
    849
    850static const struct error_descr ocx_com_errors[] = {
    851	{
    852		.type  = ERR_CORRECTED,
    853		.mask  = OCX_COM_IO_BADID,
    854		.descr = "Invalid IO transaction node ID",
    855	},
    856	{
    857		.type  = ERR_CORRECTED,
    858		.mask  = OCX_COM_MEM_BADID,
    859		.descr = "Invalid memory transaction node ID",
    860	},
    861	{
    862		.type  = ERR_CORRECTED,
    863		.mask  = OCX_COM_COPR_BADID,
    864		.descr = "Invalid coprocessor transaction node ID",
    865	},
    866	{
    867		.type  = ERR_CORRECTED,
    868		.mask  = OCX_COM_WIN_REQ_BADID,
    869		.descr = "Invalid SLI transaction node ID",
    870	},
    871	{
    872		.type  = ERR_CORRECTED,
    873		.mask  = OCX_COM_WIN_REQ_TOUT,
    874		.descr = "Window/core request timeout",
    875	},
    876	{0, 0, NULL},
    877};
    878
    879#define OCX_COM_LINKX_INT(x)		(0x120 + (x) * 8)
    880#define OCX_COM_LINKX_INT_W1S(x)	(0x140 + (x) * 8)
    881#define OCX_COM_LINKX_INT_ENA_W1S(x)	(0x160 + (x) * 8)
    882#define OCX_COM_LINKX_INT_ENA_W1C(x)	(0x180 + (x) * 8)
    883
    884#define OCX_COM_LINK_BAD_WORD			BIT(13)
    885#define OCX_COM_LINK_ALIGN_FAIL			BIT(12)
    886#define OCX_COM_LINK_ALIGN_DONE			BIT(11)
    887#define OCX_COM_LINK_UP				BIT(10)
    888#define OCX_COM_LINK_STOP			BIT(9)
    889#define OCX_COM_LINK_BLK_ERR			BIT(8)
    890#define OCX_COM_LINK_REINIT			BIT(7)
    891#define OCX_COM_LINK_LNK_DATA			BIT(6)
    892#define OCX_COM_LINK_RXFIFO_DBE			BIT(5)
    893#define OCX_COM_LINK_RXFIFO_SBE			BIT(4)
    894#define OCX_COM_LINK_TXFIFO_DBE			BIT(3)
    895#define OCX_COM_LINK_TXFIFO_SBE			BIT(2)
    896#define OCX_COM_LINK_REPLAY_DBE			BIT(1)
    897#define OCX_COM_LINK_REPLAY_SBE			BIT(0)
    898
    899static const struct error_descr ocx_com_link_errors[] = {
    900	{
    901		.type  = ERR_CORRECTED,
    902		.mask  = OCX_COM_LINK_REPLAY_SBE,
    903		.descr = "Replay buffer single-bit error",
    904	},
    905	{
    906		.type  = ERR_CORRECTED,
    907		.mask  = OCX_COM_LINK_TXFIFO_SBE,
    908		.descr = "TX FIFO single-bit error",
    909	},
    910	{
    911		.type  = ERR_CORRECTED,
    912		.mask  = OCX_COM_LINK_RXFIFO_SBE,
    913		.descr = "RX FIFO single-bit error",
    914	},
    915	{
    916		.type  = ERR_CORRECTED,
    917		.mask  = OCX_COM_LINK_BLK_ERR,
    918		.descr = "Block code error",
    919	},
    920	{
    921		.type  = ERR_CORRECTED,
    922		.mask  = OCX_COM_LINK_ALIGN_FAIL,
    923		.descr = "Link alignment failure",
    924	},
    925	{
    926		.type  = ERR_CORRECTED,
    927		.mask  = OCX_COM_LINK_BAD_WORD,
    928		.descr = "Bad code word",
    929	},
    930	{
    931		.type  = ERR_UNCORRECTED,
    932		.mask  = OCX_COM_LINK_REPLAY_DBE,
    933		.descr = "Replay buffer double-bit error",
    934	},
    935	{
    936		.type  = ERR_UNCORRECTED,
    937		.mask  = OCX_COM_LINK_TXFIFO_DBE,
    938		.descr = "TX FIFO double-bit error",
    939	},
    940	{
    941		.type  = ERR_UNCORRECTED,
    942		.mask  = OCX_COM_LINK_RXFIFO_DBE,
    943		.descr = "RX FIFO double-bit error",
    944	},
    945	{
    946		.type  = ERR_UNCORRECTED,
    947		.mask  = OCX_COM_LINK_STOP,
    948		.descr = "Link stopped",
    949	},
    950	{0, 0, NULL},
    951};
    952
    953#define OCX_COM_LINK_INT_UE       (OCX_COM_LINK_REPLAY_DBE | \
    954				   OCX_COM_LINK_TXFIFO_DBE | \
    955				   OCX_COM_LINK_RXFIFO_DBE | \
    956				   OCX_COM_LINK_STOP)
    957
    958#define OCX_COM_LINK_INT_CE       (OCX_COM_LINK_REPLAY_SBE | \
    959				   OCX_COM_LINK_TXFIFO_SBE | \
    960				   OCX_COM_LINK_RXFIFO_SBE | \
    961				   OCX_COM_LINK_BLK_ERR    | \
    962				   OCX_COM_LINK_ALIGN_FAIL | \
    963				   OCX_COM_LINK_BAD_WORD)
    964
    965#define OCX_LNE_INT(x)			(0x8018 + (x) * 0x100)
    966#define OCX_LNE_INT_EN(x)		(0x8020 + (x) * 0x100)
    967#define OCX_LNE_BAD_CNT(x)		(0x8028 + (x) * 0x100)
    968#define OCX_LNE_CFG(x)			(0x8000 + (x) * 0x100)
    969#define OCX_LNE_STAT(x, y)		(0x8040 + (x) * 0x100 + (y) * 8)
    970
    971#define OCX_LNE_CFG_RX_BDRY_LOCK_DIS		BIT(8)
    972#define OCX_LNE_CFG_RX_STAT_WRAP_DIS		BIT(2)
    973#define OCX_LNE_CFG_RX_STAT_RDCLR		BIT(1)
    974#define OCX_LNE_CFG_RX_STAT_ENA			BIT(0)
    975
    976
    977#define OCX_LANE_BAD_64B67B			BIT(8)
    978#define OCX_LANE_DSKEW_FIFO_OVFL		BIT(5)
    979#define OCX_LANE_SCRM_SYNC_LOSS			BIT(4)
    980#define OCX_LANE_UKWN_CNTL_WORD			BIT(3)
    981#define OCX_LANE_CRC32_ERR			BIT(2)
    982#define OCX_LANE_BDRY_SYNC_LOSS			BIT(1)
    983#define OCX_LANE_SERDES_LOCK_LOSS		BIT(0)
    984
    985#define OCX_COM_LANE_INT_UE       (0)
    986#define OCX_COM_LANE_INT_CE       (OCX_LANE_SERDES_LOCK_LOSS | \
    987				   OCX_LANE_BDRY_SYNC_LOSS   | \
    988				   OCX_LANE_CRC32_ERR        | \
    989				   OCX_LANE_UKWN_CNTL_WORD   | \
    990				   OCX_LANE_SCRM_SYNC_LOSS   | \
    991				   OCX_LANE_DSKEW_FIFO_OVFL  | \
    992				   OCX_LANE_BAD_64B67B)
    993
    994static const struct error_descr ocx_lane_errors[] = {
    995	{
    996		.type  = ERR_CORRECTED,
    997		.mask  = OCX_LANE_SERDES_LOCK_LOSS,
    998		.descr = "RX SerDes lock lost",
    999	},
   1000	{
   1001		.type  = ERR_CORRECTED,
   1002		.mask  = OCX_LANE_BDRY_SYNC_LOSS,
   1003		.descr = "RX word boundary lost",
   1004	},
   1005	{
   1006		.type  = ERR_CORRECTED,
   1007		.mask  = OCX_LANE_CRC32_ERR,
   1008		.descr = "CRC32 error",
   1009	},
   1010	{
   1011		.type  = ERR_CORRECTED,
   1012		.mask  = OCX_LANE_UKWN_CNTL_WORD,
   1013		.descr = "Unknown control word",
   1014	},
   1015	{
   1016		.type  = ERR_CORRECTED,
   1017		.mask  = OCX_LANE_SCRM_SYNC_LOSS,
   1018		.descr = "Scrambler synchronization lost",
   1019	},
   1020	{
   1021		.type  = ERR_CORRECTED,
   1022		.mask  = OCX_LANE_DSKEW_FIFO_OVFL,
   1023		.descr = "RX deskew FIFO overflow",
   1024	},
   1025	{
   1026		.type  = ERR_CORRECTED,
   1027		.mask  = OCX_LANE_BAD_64B67B,
   1028		.descr = "Bad 64B/67B codeword",
   1029	},
   1030	{0, 0, NULL},
   1031};
   1032
   1033#define OCX_LNE_INT_ENA_ALL		(GENMASK(9, 8) | GENMASK(6, 0))
   1034#define OCX_COM_INT_ENA_ALL		(GENMASK(54, 50) | GENMASK(23, 0))
   1035#define OCX_COM_LINKX_INT_ENA_ALL	(GENMASK(13, 12) | \
   1036					 GENMASK(9, 7) | GENMASK(5, 0))
   1037
   1038#define OCX_TLKX_ECC_CTL(x)		(0x10018 + (x) * 0x2000)
   1039#define OCX_RLKX_ECC_CTL(x)		(0x18018 + (x) * 0x2000)
   1040
   1041struct ocx_com_err_ctx {
   1042	u64 reg_com_int;
   1043	u64 reg_lane_int[OCX_RX_LANES];
   1044	u64 reg_lane_stat11[OCX_RX_LANES];
   1045};
   1046
   1047struct ocx_link_err_ctx {
   1048	u64 reg_com_link_int;
   1049	int link;
   1050};
   1051
   1052struct thunderx_ocx {
   1053	void __iomem *regs;
   1054	int com_link;
   1055	struct pci_dev *pdev;
   1056	struct edac_device_ctl_info *edac_dev;
   1057
   1058	struct dentry *debugfs;
   1059	struct msix_entry msix_ent[OCX_INTS];
   1060
   1061	struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
   1062	struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
   1063
   1064	unsigned long com_ring_head;
   1065	unsigned long com_ring_tail;
   1066
   1067	unsigned long link_ring_head;
   1068	unsigned long link_ring_tail;
   1069};
   1070
   1071#define OCX_MESSAGE_SIZE	SZ_1K
   1072#define OCX_OTHER_SIZE		(50 * ARRAY_SIZE(ocx_com_link_errors))
   1073
   1074/* This handler is threaded */
   1075static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
   1076{
   1077	struct msix_entry *msix = irq_id;
   1078	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
   1079						msix_ent[msix->entry]);
   1080
   1081	int lane;
   1082	unsigned long head = ring_pos(ocx->com_ring_head,
   1083				      ARRAY_SIZE(ocx->com_err_ctx));
   1084	struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
   1085
   1086	ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
   1087
   1088	for (lane = 0; lane < OCX_RX_LANES; lane++) {
   1089		ctx->reg_lane_int[lane] =
   1090			readq(ocx->regs + OCX_LNE_INT(lane));
   1091		ctx->reg_lane_stat11[lane] =
   1092			readq(ocx->regs + OCX_LNE_STAT(lane, 11));
   1093
   1094		writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
   1095	}
   1096
   1097	writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
   1098
   1099	ocx->com_ring_head++;
   1100
   1101	return IRQ_WAKE_THREAD;
   1102}
   1103
   1104static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
   1105{
   1106	struct msix_entry *msix = irq_id;
   1107	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
   1108						msix_ent[msix->entry]);
   1109
   1110	irqreturn_t ret = IRQ_NONE;
   1111
   1112	unsigned long tail;
   1113	struct ocx_com_err_ctx *ctx;
   1114	int lane;
   1115	char *msg;
   1116	char *other;
   1117
   1118	msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
   1119	other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
   1120
   1121	if (!msg || !other)
   1122		goto err_free;
   1123
   1124	while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
   1125			ARRAY_SIZE(ocx->com_err_ctx))) {
   1126		tail = ring_pos(ocx->com_ring_tail,
   1127				ARRAY_SIZE(ocx->com_err_ctx));
   1128		ctx = &ocx->com_err_ctx[tail];
   1129
   1130		snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
   1131			ocx->edac_dev->ctl_name, ctx->reg_com_int);
   1132
   1133		decode_register(other, OCX_OTHER_SIZE,
   1134				ocx_com_errors, ctx->reg_com_int);
   1135
   1136		strncat(msg, other, OCX_MESSAGE_SIZE);
   1137
   1138		for (lane = 0; lane < OCX_RX_LANES; lane++)
   1139			if (ctx->reg_com_int & BIT(lane)) {
   1140				snprintf(other, OCX_OTHER_SIZE,
   1141					 "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
   1142					 lane, ctx->reg_lane_int[lane],
   1143					 lane, ctx->reg_lane_stat11[lane]);
   1144
   1145				strncat(msg, other, OCX_MESSAGE_SIZE);
   1146
   1147				decode_register(other, OCX_OTHER_SIZE,
   1148						ocx_lane_errors,
   1149						ctx->reg_lane_int[lane]);
   1150				strncat(msg, other, OCX_MESSAGE_SIZE);
   1151			}
   1152
   1153		if (ctx->reg_com_int & OCX_COM_INT_CE)
   1154			edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
   1155
   1156		ocx->com_ring_tail++;
   1157	}
   1158
   1159	ret = IRQ_HANDLED;
   1160
   1161err_free:
   1162	kfree(other);
   1163	kfree(msg);
   1164
   1165	return ret;
   1166}
   1167
   1168static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
   1169{
   1170	struct msix_entry *msix = irq_id;
   1171	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
   1172						msix_ent[msix->entry]);
   1173	unsigned long head = ring_pos(ocx->link_ring_head,
   1174				      ARRAY_SIZE(ocx->link_err_ctx));
   1175	struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
   1176
   1177	ctx->link = msix->entry;
   1178	ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
   1179
   1180	writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
   1181
   1182	ocx->link_ring_head++;
   1183
   1184	return IRQ_WAKE_THREAD;
   1185}
   1186
   1187static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
   1188{
   1189	struct msix_entry *msix = irq_id;
   1190	struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
   1191						msix_ent[msix->entry]);
   1192	irqreturn_t ret = IRQ_NONE;
   1193	unsigned long tail;
   1194	struct ocx_link_err_ctx *ctx;
   1195
   1196	char *msg;
   1197	char *other;
   1198
   1199	msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
   1200	other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
   1201
   1202	if (!msg || !other)
   1203		goto err_free;
   1204
   1205	while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
   1206			ARRAY_SIZE(ocx->link_err_ctx))) {
   1207		tail = ring_pos(ocx->link_ring_head,
   1208				ARRAY_SIZE(ocx->link_err_ctx));
   1209
   1210		ctx = &ocx->link_err_ctx[tail];
   1211
   1212		snprintf(msg, OCX_MESSAGE_SIZE,
   1213			 "%s: OCX_COM_LINK_INT[%d]: %016llx",
   1214			 ocx->edac_dev->ctl_name,
   1215			 ctx->link, ctx->reg_com_link_int);
   1216
   1217		decode_register(other, OCX_OTHER_SIZE,
   1218				ocx_com_link_errors, ctx->reg_com_link_int);
   1219
   1220		strncat(msg, other, OCX_MESSAGE_SIZE);
   1221
   1222		if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
   1223			edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
   1224		else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
   1225			edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
   1226
   1227		ocx->link_ring_tail++;
   1228	}
   1229
   1230	ret = IRQ_HANDLED;
   1231err_free:
   1232	kfree(other);
   1233	kfree(msg);
   1234
   1235	return ret;
   1236}
   1237
   1238#define OCX_DEBUGFS_ATTR(_name, _reg)	DEBUGFS_REG_ATTR(ocx, _name, _reg)
   1239
   1240OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
   1241OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
   1242OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
   1243
   1244OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
   1245OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
   1246OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
   1247
   1248OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
   1249OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
   1250OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
   1251
   1252OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
   1253OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
   1254OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
   1255OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
   1256OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
   1257OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
   1258OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
   1259OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
   1260
   1261OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
   1262OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
   1263OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
   1264OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
   1265OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
   1266OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
   1267OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
   1268OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
   1269
   1270OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
   1271OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
   1272OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
   1273OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
   1274OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
   1275OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
   1276OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
   1277OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
   1278
   1279OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
   1280
   1281static struct debugfs_entry *ocx_dfs_ents[] = {
   1282	&debugfs_tlk0_ecc_ctl,
   1283	&debugfs_tlk1_ecc_ctl,
   1284	&debugfs_tlk2_ecc_ctl,
   1285
   1286	&debugfs_rlk0_ecc_ctl,
   1287	&debugfs_rlk1_ecc_ctl,
   1288	&debugfs_rlk2_ecc_ctl,
   1289
   1290	&debugfs_com_link0_int,
   1291	&debugfs_com_link1_int,
   1292	&debugfs_com_link2_int,
   1293
   1294	&debugfs_lne00_badcnt,
   1295	&debugfs_lne01_badcnt,
   1296	&debugfs_lne02_badcnt,
   1297	&debugfs_lne03_badcnt,
   1298	&debugfs_lne04_badcnt,
   1299	&debugfs_lne05_badcnt,
   1300	&debugfs_lne06_badcnt,
   1301	&debugfs_lne07_badcnt,
   1302	&debugfs_lne08_badcnt,
   1303	&debugfs_lne09_badcnt,
   1304	&debugfs_lne10_badcnt,
   1305	&debugfs_lne11_badcnt,
   1306	&debugfs_lne12_badcnt,
   1307	&debugfs_lne13_badcnt,
   1308	&debugfs_lne14_badcnt,
   1309	&debugfs_lne15_badcnt,
   1310	&debugfs_lne16_badcnt,
   1311	&debugfs_lne17_badcnt,
   1312	&debugfs_lne18_badcnt,
   1313	&debugfs_lne19_badcnt,
   1314	&debugfs_lne20_badcnt,
   1315	&debugfs_lne21_badcnt,
   1316	&debugfs_lne22_badcnt,
   1317	&debugfs_lne23_badcnt,
   1318
   1319	&debugfs_com_int,
   1320};
   1321
   1322static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
   1323	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
   1324	{ 0, },
   1325};
   1326
   1327static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
   1328{
   1329	int lane, stat, cfg;
   1330
   1331	for (lane = 0; lane < OCX_RX_LANES; lane++) {
   1332		cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
   1333		cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
   1334		cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
   1335		writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
   1336
   1337		for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
   1338			readq(ocx->regs + OCX_LNE_STAT(lane, stat));
   1339	}
   1340}
   1341
   1342static int thunderx_ocx_probe(struct pci_dev *pdev,
   1343			      const struct pci_device_id *id)
   1344{
   1345	struct thunderx_ocx *ocx;
   1346	struct edac_device_ctl_info *edac_dev;
   1347	char name[32];
   1348	int idx;
   1349	int i;
   1350	int ret;
   1351	u64 reg;
   1352
   1353	ret = pcim_enable_device(pdev);
   1354	if (ret) {
   1355		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
   1356		return ret;
   1357	}
   1358
   1359	ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
   1360	if (ret) {
   1361		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
   1362		return ret;
   1363	}
   1364
   1365	idx = edac_device_alloc_index();
   1366	snprintf(name, sizeof(name), "OCX%d", idx);
   1367	edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
   1368					      name, 1, "CCPI", 1,
   1369					      0, NULL, 0, idx);
   1370	if (!edac_dev) {
   1371		dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
   1372		return -ENOMEM;
   1373	}
   1374	ocx = edac_dev->pvt_info;
   1375	ocx->edac_dev = edac_dev;
   1376	ocx->com_ring_head = 0;
   1377	ocx->com_ring_tail = 0;
   1378	ocx->link_ring_head = 0;
   1379	ocx->link_ring_tail = 0;
   1380
   1381	ocx->regs = pcim_iomap_table(pdev)[0];
   1382	if (!ocx->regs) {
   1383		dev_err(&pdev->dev, "Cannot map PCI resources\n");
   1384		ret = -ENODEV;
   1385		goto err_free;
   1386	}
   1387
   1388	ocx->pdev = pdev;
   1389
   1390	for (i = 0; i < OCX_INTS; i++) {
   1391		ocx->msix_ent[i].entry = i;
   1392		ocx->msix_ent[i].vector = 0;
   1393	}
   1394
   1395	ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
   1396	if (ret) {
   1397		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
   1398		goto err_free;
   1399	}
   1400
   1401	for (i = 0; i < OCX_INTS; i++) {
   1402		ret = devm_request_threaded_irq(&pdev->dev,
   1403						ocx->msix_ent[i].vector,
   1404						(i == 3) ?
   1405						 thunderx_ocx_com_isr :
   1406						 thunderx_ocx_lnk_isr,
   1407						(i == 3) ?
   1408						 thunderx_ocx_com_threaded_isr :
   1409						 thunderx_ocx_lnk_threaded_isr,
   1410						0, "[EDAC] ThunderX OCX",
   1411						&ocx->msix_ent[i]);
   1412		if (ret)
   1413			goto err_free;
   1414	}
   1415
   1416	edac_dev->dev = &pdev->dev;
   1417	edac_dev->dev_name = dev_name(&pdev->dev);
   1418	edac_dev->mod_name = "thunderx-ocx";
   1419	edac_dev->ctl_name = "thunderx-ocx";
   1420
   1421	ret = edac_device_add_device(edac_dev);
   1422	if (ret) {
   1423		dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
   1424		goto err_free;
   1425	}
   1426
   1427	if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
   1428		ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
   1429
   1430		ret = thunderx_create_debugfs_nodes(ocx->debugfs,
   1431						    ocx_dfs_ents,
   1432						    ocx,
   1433						    ARRAY_SIZE(ocx_dfs_ents));
   1434		if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
   1435			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
   1436				 ret, ret >= 0 ? " created" : "");
   1437		}
   1438	}
   1439
   1440	pci_set_drvdata(pdev, edac_dev);
   1441
   1442	thunderx_ocx_clearstats(ocx);
   1443
   1444	for (i = 0; i < OCX_RX_LANES; i++) {
   1445		writeq(OCX_LNE_INT_ENA_ALL,
   1446		       ocx->regs + OCX_LNE_INT_EN(i));
   1447
   1448		reg = readq(ocx->regs + OCX_LNE_INT(i));
   1449		writeq(reg, ocx->regs + OCX_LNE_INT(i));
   1450
   1451	}
   1452
   1453	for (i = 0; i < OCX_LINK_INTS; i++) {
   1454		reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
   1455		writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
   1456
   1457		writeq(OCX_COM_LINKX_INT_ENA_ALL,
   1458		       ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
   1459	}
   1460
   1461	reg = readq(ocx->regs + OCX_COM_INT);
   1462	writeq(reg, ocx->regs + OCX_COM_INT);
   1463
   1464	writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
   1465
   1466	return 0;
   1467err_free:
   1468	edac_device_free_ctl_info(edac_dev);
   1469
   1470	return ret;
   1471}
   1472
   1473static void thunderx_ocx_remove(struct pci_dev *pdev)
   1474{
   1475	struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
   1476	struct thunderx_ocx *ocx = edac_dev->pvt_info;
   1477	int i;
   1478
   1479	writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
   1480
   1481	for (i = 0; i < OCX_INTS; i++) {
   1482		writeq(OCX_COM_LINKX_INT_ENA_ALL,
   1483		       ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
   1484	}
   1485
   1486	edac_debugfs_remove_recursive(ocx->debugfs);
   1487
   1488	edac_device_del_device(&pdev->dev);
   1489	edac_device_free_ctl_info(edac_dev);
   1490}
   1491
   1492MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
   1493
   1494static struct pci_driver thunderx_ocx_driver = {
   1495	.name     = "thunderx_ocx_edac",
   1496	.probe    = thunderx_ocx_probe,
   1497	.remove   = thunderx_ocx_remove,
   1498	.id_table = thunderx_ocx_pci_tbl,
   1499};
   1500
   1501/*---------------------- L2C driver ---------------------------------*/
   1502
   1503#define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
   1504#define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
   1505#define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
   1506
   1507#define L2C_TAD_INT_W1C		0x40000
   1508#define L2C_TAD_INT_W1S		0x40008
   1509
   1510#define L2C_TAD_INT_ENA_W1C	0x40020
   1511#define L2C_TAD_INT_ENA_W1S	0x40028
   1512
   1513
   1514#define L2C_TAD_INT_L2DDBE	 BIT(1)
   1515#define L2C_TAD_INT_SBFSBE	 BIT(2)
   1516#define L2C_TAD_INT_SBFDBE	 BIT(3)
   1517#define L2C_TAD_INT_FBFSBE	 BIT(4)
   1518#define L2C_TAD_INT_FBFDBE	 BIT(5)
   1519#define L2C_TAD_INT_TAGDBE	 BIT(9)
   1520#define L2C_TAD_INT_RDDISLMC	 BIT(15)
   1521#define L2C_TAD_INT_WRDISLMC	 BIT(16)
   1522#define L2C_TAD_INT_LFBTO	 BIT(17)
   1523#define L2C_TAD_INT_GSYNCTO	 BIT(18)
   1524#define L2C_TAD_INT_RTGSBE	 BIT(32)
   1525#define L2C_TAD_INT_RTGDBE	 BIT(33)
   1526#define L2C_TAD_INT_RDDISOCI	 BIT(34)
   1527#define L2C_TAD_INT_WRDISOCI	 BIT(35)
   1528
   1529#define L2C_TAD_INT_ECC		(L2C_TAD_INT_L2DDBE | \
   1530				 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
   1531				 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
   1532
   1533#define L2C_TAD_INT_CE          (L2C_TAD_INT_SBFSBE | \
   1534				 L2C_TAD_INT_FBFSBE)
   1535
   1536#define L2C_TAD_INT_UE          (L2C_TAD_INT_L2DDBE | \
   1537				 L2C_TAD_INT_SBFDBE | \
   1538				 L2C_TAD_INT_FBFDBE | \
   1539				 L2C_TAD_INT_TAGDBE | \
   1540				 L2C_TAD_INT_RTGDBE | \
   1541				 L2C_TAD_INT_WRDISOCI | \
   1542				 L2C_TAD_INT_RDDISOCI | \
   1543				 L2C_TAD_INT_WRDISLMC | \
   1544				 L2C_TAD_INT_RDDISLMC | \
   1545				 L2C_TAD_INT_LFBTO    | \
   1546				 L2C_TAD_INT_GSYNCTO)
   1547
   1548static const struct error_descr l2_tad_errors[] = {
   1549	{
   1550		.type  = ERR_CORRECTED,
   1551		.mask  = L2C_TAD_INT_SBFSBE,
   1552		.descr = "SBF single-bit error",
   1553	},
   1554	{
   1555		.type  = ERR_CORRECTED,
   1556		.mask  = L2C_TAD_INT_FBFSBE,
   1557		.descr = "FBF single-bit error",
   1558	},
   1559	{
   1560		.type  = ERR_UNCORRECTED,
   1561		.mask  = L2C_TAD_INT_L2DDBE,
   1562		.descr = "L2D double-bit error",
   1563	},
   1564	{
   1565		.type  = ERR_UNCORRECTED,
   1566		.mask  = L2C_TAD_INT_SBFDBE,
   1567		.descr = "SBF double-bit error",
   1568	},
   1569	{
   1570		.type  = ERR_UNCORRECTED,
   1571		.mask  = L2C_TAD_INT_FBFDBE,
   1572		.descr = "FBF double-bit error",
   1573	},
   1574	{
   1575		.type  = ERR_UNCORRECTED,
   1576		.mask  = L2C_TAD_INT_TAGDBE,
   1577		.descr = "TAG double-bit error",
   1578	},
   1579	{
   1580		.type  = ERR_UNCORRECTED,
   1581		.mask  = L2C_TAD_INT_RTGDBE,
   1582		.descr = "RTG double-bit error",
   1583	},
   1584	{
   1585		.type  = ERR_UNCORRECTED,
   1586		.mask  = L2C_TAD_INT_WRDISOCI,
   1587		.descr = "Write to a disabled CCPI",
   1588	},
   1589	{
   1590		.type  = ERR_UNCORRECTED,
   1591		.mask  = L2C_TAD_INT_RDDISOCI,
   1592		.descr = "Read from a disabled CCPI",
   1593	},
   1594	{
   1595		.type  = ERR_UNCORRECTED,
   1596		.mask  = L2C_TAD_INT_WRDISLMC,
   1597		.descr = "Write to a disabled LMC",
   1598	},
   1599	{
   1600		.type  = ERR_UNCORRECTED,
   1601		.mask  = L2C_TAD_INT_RDDISLMC,
   1602		.descr = "Read from a disabled LMC",
   1603	},
   1604	{
   1605		.type  = ERR_UNCORRECTED,
   1606		.mask  = L2C_TAD_INT_LFBTO,
   1607		.descr = "LFB entry timeout",
   1608	},
   1609	{
   1610		.type  = ERR_UNCORRECTED,
   1611		.mask  = L2C_TAD_INT_GSYNCTO,
   1612		.descr = "Global sync CCPI timeout",
   1613	},
   1614	{0, 0, NULL},
   1615};
   1616
   1617#define L2C_TAD_INT_TAG		(L2C_TAD_INT_TAGDBE)
   1618
   1619#define L2C_TAD_INT_RTG		(L2C_TAD_INT_RTGDBE)
   1620
   1621#define L2C_TAD_INT_DISLMC	(L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
   1622
   1623#define L2C_TAD_INT_DISOCI	(L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
   1624
   1625#define L2C_TAD_INT_ENA_ALL	(L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
   1626				 L2C_TAD_INT_RTG | \
   1627				 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
   1628				 L2C_TAD_INT_LFBTO)
   1629
   1630#define L2C_TAD_TIMETWO		0x50000
   1631#define L2C_TAD_TIMEOUT		0x50100
   1632#define L2C_TAD_ERR		0x60000
   1633#define L2C_TAD_TQD_ERR		0x60100
   1634#define L2C_TAD_TTG_ERR		0x60200
   1635
   1636
   1637#define L2C_CBC_INT_W1C		0x60000
   1638
   1639#define L2C_CBC_INT_RSDSBE	 BIT(0)
   1640#define L2C_CBC_INT_RSDDBE	 BIT(1)
   1641
   1642#define L2C_CBC_INT_RSD		 (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
   1643
   1644#define L2C_CBC_INT_MIBSBE	 BIT(4)
   1645#define L2C_CBC_INT_MIBDBE	 BIT(5)
   1646
   1647#define L2C_CBC_INT_MIB		 (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
   1648
   1649#define L2C_CBC_INT_IORDDISOCI	 BIT(6)
   1650#define L2C_CBC_INT_IOWRDISOCI	 BIT(7)
   1651
   1652#define L2C_CBC_INT_IODISOCI	 (L2C_CBC_INT_IORDDISOCI | \
   1653				  L2C_CBC_INT_IOWRDISOCI)
   1654
   1655#define L2C_CBC_INT_CE		 (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
   1656#define L2C_CBC_INT_UE		 (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
   1657
   1658
   1659static const struct error_descr l2_cbc_errors[] = {
   1660	{
   1661		.type  = ERR_CORRECTED,
   1662		.mask  = L2C_CBC_INT_RSDSBE,
   1663		.descr = "RSD single-bit error",
   1664	},
   1665	{
   1666		.type  = ERR_CORRECTED,
   1667		.mask  = L2C_CBC_INT_MIBSBE,
   1668		.descr = "MIB single-bit error",
   1669	},
   1670	{
   1671		.type  = ERR_UNCORRECTED,
   1672		.mask  = L2C_CBC_INT_RSDDBE,
   1673		.descr = "RSD double-bit error",
   1674	},
   1675	{
   1676		.type  = ERR_UNCORRECTED,
   1677		.mask  = L2C_CBC_INT_MIBDBE,
   1678		.descr = "MIB double-bit error",
   1679	},
   1680	{
   1681		.type  = ERR_UNCORRECTED,
   1682		.mask  = L2C_CBC_INT_IORDDISOCI,
   1683		.descr = "Read from a disabled CCPI",
   1684	},
   1685	{
   1686		.type  = ERR_UNCORRECTED,
   1687		.mask  = L2C_CBC_INT_IOWRDISOCI,
   1688		.descr = "Write to a disabled CCPI",
   1689	},
   1690	{0, 0, NULL},
   1691};
   1692
   1693#define L2C_CBC_INT_W1S		0x60008
   1694#define L2C_CBC_INT_ENA_W1C	0x60020
   1695
   1696#define L2C_CBC_INT_ENA_ALL	 (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
   1697				  L2C_CBC_INT_IODISOCI)
   1698
   1699#define L2C_CBC_INT_ENA_W1S	0x60028
   1700
   1701#define L2C_CBC_IODISOCIERR	0x80008
   1702#define L2C_CBC_IOCERR		0x80010
   1703#define L2C_CBC_RSDERR		0x80018
   1704#define L2C_CBC_MIBERR		0x80020
   1705
   1706
   1707#define L2C_MCI_INT_W1C		0x0
   1708
   1709#define L2C_MCI_INT_VBFSBE	 BIT(0)
   1710#define L2C_MCI_INT_VBFDBE	 BIT(1)
   1711
   1712static const struct error_descr l2_mci_errors[] = {
   1713	{
   1714		.type  = ERR_CORRECTED,
   1715		.mask  = L2C_MCI_INT_VBFSBE,
   1716		.descr = "VBF single-bit error",
   1717	},
   1718	{
   1719		.type  = ERR_UNCORRECTED,
   1720		.mask  = L2C_MCI_INT_VBFDBE,
   1721		.descr = "VBF double-bit error",
   1722	},
   1723	{0, 0, NULL},
   1724};
   1725
   1726#define L2C_MCI_INT_W1S		0x8
   1727#define L2C_MCI_INT_ENA_W1C	0x20
   1728
   1729#define L2C_MCI_INT_ENA_ALL	 (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
   1730
   1731#define L2C_MCI_INT_ENA_W1S	0x28
   1732
   1733#define L2C_MCI_ERR		0x10000
   1734
   1735#define L2C_MESSAGE_SIZE	SZ_1K
   1736#define L2C_OTHER_SIZE		(50 * ARRAY_SIZE(l2_tad_errors))
   1737
   1738struct l2c_err_ctx {
   1739	char *reg_ext_name;
   1740	u64  reg_int;
   1741	u64  reg_ext;
   1742};
   1743
   1744struct thunderx_l2c {
   1745	void __iomem *regs;
   1746	struct pci_dev *pdev;
   1747	struct edac_device_ctl_info *edac_dev;
   1748
   1749	struct dentry *debugfs;
   1750
   1751	int index;
   1752
   1753	struct msix_entry msix_ent;
   1754
   1755	struct l2c_err_ctx err_ctx[RING_ENTRIES];
   1756	unsigned long ring_head;
   1757	unsigned long ring_tail;
   1758};
   1759
   1760static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
   1761{
   1762	struct msix_entry *msix = irq_id;
   1763	struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
   1764						msix_ent);
   1765
   1766	unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
   1767	struct l2c_err_ctx *ctx = &tad->err_ctx[head];
   1768
   1769	ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
   1770
   1771	if (ctx->reg_int & L2C_TAD_INT_ECC) {
   1772		ctx->reg_ext_name = "TQD_ERR";
   1773		ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
   1774	} else if (ctx->reg_int & L2C_TAD_INT_TAG) {
   1775		ctx->reg_ext_name = "TTG_ERR";
   1776		ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
   1777	} else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
   1778		ctx->reg_ext_name = "TIMEOUT";
   1779		ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
   1780	} else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
   1781		ctx->reg_ext_name = "ERR";
   1782		ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
   1783	}
   1784
   1785	writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
   1786
   1787	tad->ring_head++;
   1788
   1789	return IRQ_WAKE_THREAD;
   1790}
   1791
   1792static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
   1793{
   1794	struct msix_entry *msix = irq_id;
   1795	struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
   1796						msix_ent);
   1797
   1798	unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
   1799	struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
   1800
   1801	ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
   1802
   1803	if (ctx->reg_int & L2C_CBC_INT_RSD) {
   1804		ctx->reg_ext_name = "RSDERR";
   1805		ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
   1806	} else if (ctx->reg_int & L2C_CBC_INT_MIB) {
   1807		ctx->reg_ext_name = "MIBERR";
   1808		ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
   1809	} else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
   1810		ctx->reg_ext_name = "IODISOCIERR";
   1811		ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
   1812	}
   1813
   1814	writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
   1815
   1816	cbc->ring_head++;
   1817
   1818	return IRQ_WAKE_THREAD;
   1819}
   1820
   1821static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
   1822{
   1823	struct msix_entry *msix = irq_id;
   1824	struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
   1825						msix_ent);
   1826
   1827	unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
   1828	struct l2c_err_ctx *ctx = &mci->err_ctx[head];
   1829
   1830	ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
   1831	ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
   1832
   1833	writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
   1834
   1835	ctx->reg_ext_name = "ERR";
   1836
   1837	mci->ring_head++;
   1838
   1839	return IRQ_WAKE_THREAD;
   1840}
   1841
   1842static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
   1843{
   1844	struct msix_entry *msix = irq_id;
   1845	struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
   1846						msix_ent);
   1847
   1848	unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
   1849	struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
   1850	irqreturn_t ret = IRQ_NONE;
   1851
   1852	u64 mask_ue, mask_ce;
   1853	const struct error_descr *l2_errors;
   1854	char *reg_int_name;
   1855
   1856	char *msg;
   1857	char *other;
   1858
   1859	msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
   1860	other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
   1861
   1862	if (!msg || !other)
   1863		goto err_free;
   1864
   1865	switch (l2c->pdev->device) {
   1866	case PCI_DEVICE_ID_THUNDER_L2C_TAD:
   1867		reg_int_name = "L2C_TAD_INT";
   1868		mask_ue = L2C_TAD_INT_UE;
   1869		mask_ce = L2C_TAD_INT_CE;
   1870		l2_errors = l2_tad_errors;
   1871		break;
   1872	case PCI_DEVICE_ID_THUNDER_L2C_CBC:
   1873		reg_int_name = "L2C_CBC_INT";
   1874		mask_ue = L2C_CBC_INT_UE;
   1875		mask_ce = L2C_CBC_INT_CE;
   1876		l2_errors = l2_cbc_errors;
   1877		break;
   1878	case PCI_DEVICE_ID_THUNDER_L2C_MCI:
   1879		reg_int_name = "L2C_MCI_INT";
   1880		mask_ue = L2C_MCI_INT_VBFDBE;
   1881		mask_ce = L2C_MCI_INT_VBFSBE;
   1882		l2_errors = l2_mci_errors;
   1883		break;
   1884	default:
   1885		dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
   1886			l2c->pdev->device);
   1887		goto err_free;
   1888	}
   1889
   1890	while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
   1891			ARRAY_SIZE(l2c->err_ctx))) {
   1892		snprintf(msg, L2C_MESSAGE_SIZE,
   1893			 "%s: %s: %016llx, %s: %016llx",
   1894			 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
   1895			 ctx->reg_ext_name, ctx->reg_ext);
   1896
   1897		decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
   1898
   1899		strncat(msg, other, L2C_MESSAGE_SIZE);
   1900
   1901		if (ctx->reg_int & mask_ue)
   1902			edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
   1903		else if (ctx->reg_int & mask_ce)
   1904			edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
   1905
   1906		l2c->ring_tail++;
   1907	}
   1908
   1909	ret = IRQ_HANDLED;
   1910
   1911err_free:
   1912	kfree(other);
   1913	kfree(msg);
   1914
   1915	return ret;
   1916}
   1917
   1918#define L2C_DEBUGFS_ATTR(_name, _reg)	DEBUGFS_REG_ATTR(l2c, _name, _reg)
   1919
   1920L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
   1921
   1922static struct debugfs_entry *l2c_tad_dfs_ents[] = {
   1923	&debugfs_tad_int,
   1924};
   1925
   1926L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
   1927
   1928static struct debugfs_entry *l2c_cbc_dfs_ents[] = {
   1929	&debugfs_cbc_int,
   1930};
   1931
   1932L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
   1933
   1934static struct debugfs_entry *l2c_mci_dfs_ents[] = {
   1935	&debugfs_mci_int,
   1936};
   1937
   1938static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
   1939	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
   1940	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
   1941	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
   1942	{ 0, },
   1943};
   1944
   1945static int thunderx_l2c_probe(struct pci_dev *pdev,
   1946			      const struct pci_device_id *id)
   1947{
   1948	struct thunderx_l2c *l2c;
   1949	struct edac_device_ctl_info *edac_dev;
   1950	struct debugfs_entry **l2c_devattr;
   1951	size_t dfs_entries;
   1952	irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
   1953	char name[32];
   1954	const char *fmt;
   1955	u64 reg_en_offs, reg_en_mask;
   1956	int idx;
   1957	int ret;
   1958
   1959	ret = pcim_enable_device(pdev);
   1960	if (ret) {
   1961		dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
   1962		return ret;
   1963	}
   1964
   1965	ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
   1966	if (ret) {
   1967		dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
   1968		return ret;
   1969	}
   1970
   1971	switch (pdev->device) {
   1972	case PCI_DEVICE_ID_THUNDER_L2C_TAD:
   1973		thunderx_l2c_isr = thunderx_l2c_tad_isr;
   1974		l2c_devattr = l2c_tad_dfs_ents;
   1975		dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
   1976		fmt = "L2C-TAD%d";
   1977		reg_en_offs = L2C_TAD_INT_ENA_W1S;
   1978		reg_en_mask = L2C_TAD_INT_ENA_ALL;
   1979		break;
   1980	case PCI_DEVICE_ID_THUNDER_L2C_CBC:
   1981		thunderx_l2c_isr = thunderx_l2c_cbc_isr;
   1982		l2c_devattr = l2c_cbc_dfs_ents;
   1983		dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
   1984		fmt = "L2C-CBC%d";
   1985		reg_en_offs = L2C_CBC_INT_ENA_W1S;
   1986		reg_en_mask = L2C_CBC_INT_ENA_ALL;
   1987		break;
   1988	case PCI_DEVICE_ID_THUNDER_L2C_MCI:
   1989		thunderx_l2c_isr = thunderx_l2c_mci_isr;
   1990		l2c_devattr = l2c_mci_dfs_ents;
   1991		dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
   1992		fmt = "L2C-MCI%d";
   1993		reg_en_offs = L2C_MCI_INT_ENA_W1S;
   1994		reg_en_mask = L2C_MCI_INT_ENA_ALL;
   1995		break;
   1996	default:
   1997		//Should never ever get here
   1998		dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
   1999			pdev->device);
   2000		return -EINVAL;
   2001	}
   2002
   2003	idx = edac_device_alloc_index();
   2004	snprintf(name, sizeof(name), fmt, idx);
   2005
   2006	edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
   2007					      name, 1, "L2C", 1, 0,
   2008					      NULL, 0, idx);
   2009	if (!edac_dev) {
   2010		dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
   2011		return -ENOMEM;
   2012	}
   2013
   2014	l2c = edac_dev->pvt_info;
   2015	l2c->edac_dev = edac_dev;
   2016
   2017	l2c->regs = pcim_iomap_table(pdev)[0];
   2018	if (!l2c->regs) {
   2019		dev_err(&pdev->dev, "Cannot map PCI resources\n");
   2020		ret = -ENODEV;
   2021		goto err_free;
   2022	}
   2023
   2024	l2c->pdev = pdev;
   2025
   2026	l2c->ring_head = 0;
   2027	l2c->ring_tail = 0;
   2028
   2029	l2c->msix_ent.entry = 0;
   2030	l2c->msix_ent.vector = 0;
   2031
   2032	ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
   2033	if (ret) {
   2034		dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
   2035		goto err_free;
   2036	}
   2037
   2038	ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
   2039					thunderx_l2c_isr,
   2040					thunderx_l2c_threaded_isr,
   2041					0, "[EDAC] ThunderX L2C",
   2042					&l2c->msix_ent);
   2043	if (ret)
   2044		goto err_free;
   2045
   2046	edac_dev->dev = &pdev->dev;
   2047	edac_dev->dev_name = dev_name(&pdev->dev);
   2048	edac_dev->mod_name = "thunderx-l2c";
   2049	edac_dev->ctl_name = "thunderx-l2c";
   2050
   2051	ret = edac_device_add_device(edac_dev);
   2052	if (ret) {
   2053		dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
   2054		goto err_free;
   2055	}
   2056
   2057	if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
   2058		l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
   2059
   2060		ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
   2061					      l2c, dfs_entries);
   2062
   2063		if (ret != dfs_entries) {
   2064			dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
   2065				 ret, ret >= 0 ? " created" : "");
   2066		}
   2067	}
   2068
   2069	pci_set_drvdata(pdev, edac_dev);
   2070
   2071	writeq(reg_en_mask, l2c->regs + reg_en_offs);
   2072
   2073	return 0;
   2074
   2075err_free:
   2076	edac_device_free_ctl_info(edac_dev);
   2077
   2078	return ret;
   2079}
   2080
   2081static void thunderx_l2c_remove(struct pci_dev *pdev)
   2082{
   2083	struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
   2084	struct thunderx_l2c *l2c = edac_dev->pvt_info;
   2085
   2086	switch (pdev->device) {
   2087	case PCI_DEVICE_ID_THUNDER_L2C_TAD:
   2088		writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
   2089		break;
   2090	case PCI_DEVICE_ID_THUNDER_L2C_CBC:
   2091		writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
   2092		break;
   2093	case PCI_DEVICE_ID_THUNDER_L2C_MCI:
   2094		writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
   2095		break;
   2096	}
   2097
   2098	edac_debugfs_remove_recursive(l2c->debugfs);
   2099
   2100	edac_device_del_device(&pdev->dev);
   2101	edac_device_free_ctl_info(edac_dev);
   2102}
   2103
   2104MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
   2105
   2106static struct pci_driver thunderx_l2c_driver = {
   2107	.name     = "thunderx_l2c_edac",
   2108	.probe    = thunderx_l2c_probe,
   2109	.remove   = thunderx_l2c_remove,
   2110	.id_table = thunderx_l2c_pci_tbl,
   2111};
   2112
   2113static int __init thunderx_edac_init(void)
   2114{
   2115	int rc = 0;
   2116
   2117	rc = pci_register_driver(&thunderx_lmc_driver);
   2118	if (rc)
   2119		return rc;
   2120
   2121	rc = pci_register_driver(&thunderx_ocx_driver);
   2122	if (rc)
   2123		goto err_lmc;
   2124
   2125	rc = pci_register_driver(&thunderx_l2c_driver);
   2126	if (rc)
   2127		goto err_ocx;
   2128
   2129	return rc;
   2130err_ocx:
   2131	pci_unregister_driver(&thunderx_ocx_driver);
   2132err_lmc:
   2133	pci_unregister_driver(&thunderx_lmc_driver);
   2134
   2135	return rc;
   2136}
   2137
   2138static void __exit thunderx_edac_exit(void)
   2139{
   2140	pci_unregister_driver(&thunderx_l2c_driver);
   2141	pci_unregister_driver(&thunderx_ocx_driver);
   2142	pci_unregister_driver(&thunderx_lmc_driver);
   2143
   2144}
   2145
   2146module_init(thunderx_edac_init);
   2147module_exit(thunderx_edac_exit);
   2148
   2149MODULE_LICENSE("GPL v2");
   2150MODULE_AUTHOR("Cavium, Inc.");
   2151MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");