cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

traps_64.c (85644B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/* arch/sparc64/kernel/traps.c
      3 *
      4 * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net)
      5 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
      6 */
      7
      8/*
      9 * I like traps on v9, :))))
     10 */
     11
     12#include <linux/extable.h>
     13#include <linux/sched/mm.h>
     14#include <linux/sched/debug.h>
     15#include <linux/linkage.h>
     16#include <linux/kernel.h>
     17#include <linux/signal.h>
     18#include <linux/smp.h>
     19#include <linux/mm.h>
     20#include <linux/init.h>
     21#include <linux/kallsyms.h>
     22#include <linux/kdebug.h>
     23#include <linux/ftrace.h>
     24#include <linux/reboot.h>
     25#include <linux/gfp.h>
     26#include <linux/context_tracking.h>
     27
     28#include <asm/smp.h>
     29#include <asm/delay.h>
     30#include <asm/ptrace.h>
     31#include <asm/oplib.h>
     32#include <asm/page.h>
     33#include <asm/unistd.h>
     34#include <linux/uaccess.h>
     35#include <asm/fpumacro.h>
     36#include <asm/lsu.h>
     37#include <asm/dcu.h>
     38#include <asm/estate.h>
     39#include <asm/chafsr.h>
     40#include <asm/sfafsr.h>
     41#include <asm/psrcompat.h>
     42#include <asm/processor.h>
     43#include <asm/timer.h>
     44#include <asm/head.h>
     45#include <asm/prom.h>
     46#include <asm/memctrl.h>
     47#include <asm/cacheflush.h>
     48#include <asm/setup.h>
     49
     50#include "entry.h"
     51#include "kernel.h"
     52#include "kstack.h"
     53
     54/* When an irrecoverable trap occurs at tl > 0, the trap entry
     55 * code logs the trap state registers at every level in the trap
     56 * stack.  It is found at (pt_regs + sizeof(pt_regs)) and the layout
     57 * is as follows:
     58 */
     59struct tl1_traplog {
     60	struct {
     61		unsigned long tstate;
     62		unsigned long tpc;
     63		unsigned long tnpc;
     64		unsigned long tt;
     65	} trapstack[4];
     66	unsigned long tl;
     67};
     68
     69static void dump_tl1_traplog(struct tl1_traplog *p)
     70{
     71	int i, limit;
     72
     73	printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
     74	       "dumping track stack.\n", p->tl);
     75
     76	limit = (tlb_type == hypervisor) ? 2 : 4;
     77	for (i = 0; i < limit; i++) {
     78		printk(KERN_EMERG
     79		       "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
     80		       "TNPC[%016lx] TT[%lx]\n",
     81		       i + 1,
     82		       p->trapstack[i].tstate, p->trapstack[i].tpc,
     83		       p->trapstack[i].tnpc, p->trapstack[i].tt);
     84		printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
     85	}
     86}
     87
     88void bad_trap(struct pt_regs *regs, long lvl)
     89{
     90	char buffer[36];
     91
     92	if (notify_die(DIE_TRAP, "bad trap", regs,
     93		       0, lvl, SIGTRAP) == NOTIFY_STOP)
     94		return;
     95
     96	if (lvl < 0x100) {
     97		sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
     98		die_if_kernel(buffer, regs);
     99	}
    100
    101	lvl -= 0x100;
    102	if (regs->tstate & TSTATE_PRIV) {
    103		sprintf(buffer, "Kernel bad sw trap %lx", lvl);
    104		die_if_kernel(buffer, regs);
    105	}
    106	if (test_thread_flag(TIF_32BIT)) {
    107		regs->tpc &= 0xffffffff;
    108		regs->tnpc &= 0xffffffff;
    109	}
    110	force_sig_fault_trapno(SIGILL, ILL_ILLTRP,
    111			       (void __user *)regs->tpc, lvl);
    112}
    113
    114void bad_trap_tl1(struct pt_regs *regs, long lvl)
    115{
    116	char buffer[36];
    117	
    118	if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
    119		       0, lvl, SIGTRAP) == NOTIFY_STOP)
    120		return;
    121
    122	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
    123
    124	sprintf (buffer, "Bad trap %lx at tl>0", lvl);
    125	die_if_kernel (buffer, regs);
    126}
    127
    128#ifdef CONFIG_DEBUG_BUGVERBOSE
    129void do_BUG(const char *file, int line)
    130{
    131	bust_spinlocks(1);
    132	printk("kernel BUG at %s:%d!\n", file, line);
    133}
    134EXPORT_SYMBOL(do_BUG);
    135#endif
    136
    137static DEFINE_SPINLOCK(dimm_handler_lock);
    138static dimm_printer_t dimm_handler;
    139
    140static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen)
    141{
    142	unsigned long flags;
    143	int ret = -ENODEV;
    144
    145	spin_lock_irqsave(&dimm_handler_lock, flags);
    146	if (dimm_handler) {
    147		ret = dimm_handler(synd_code, paddr, buf, buflen);
    148	} else if (tlb_type == spitfire) {
    149		if (prom_getunumber(synd_code, paddr, buf, buflen) == -1)
    150			ret = -EINVAL;
    151		else
    152			ret = 0;
    153	} else
    154		ret = -ENODEV;
    155	spin_unlock_irqrestore(&dimm_handler_lock, flags);
    156
    157	return ret;
    158}
    159
    160int register_dimm_printer(dimm_printer_t func)
    161{
    162	unsigned long flags;
    163	int ret = 0;
    164
    165	spin_lock_irqsave(&dimm_handler_lock, flags);
    166	if (!dimm_handler)
    167		dimm_handler = func;
    168	else
    169		ret = -EEXIST;
    170	spin_unlock_irqrestore(&dimm_handler_lock, flags);
    171
    172	return ret;
    173}
    174EXPORT_SYMBOL_GPL(register_dimm_printer);
    175
    176void unregister_dimm_printer(dimm_printer_t func)
    177{
    178	unsigned long flags;
    179
    180	spin_lock_irqsave(&dimm_handler_lock, flags);
    181	if (dimm_handler == func)
    182		dimm_handler = NULL;
    183	spin_unlock_irqrestore(&dimm_handler_lock, flags);
    184}
    185EXPORT_SYMBOL_GPL(unregister_dimm_printer);
    186
    187void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
    188{
    189	enum ctx_state prev_state = exception_enter();
    190
    191	if (notify_die(DIE_TRAP, "instruction access exception", regs,
    192		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
    193		goto out;
    194
    195	if (regs->tstate & TSTATE_PRIV) {
    196		printk("spitfire_insn_access_exception: SFSR[%016lx] "
    197		       "SFAR[%016lx], going.\n", sfsr, sfar);
    198		die_if_kernel("Iax", regs);
    199	}
    200	if (test_thread_flag(TIF_32BIT)) {
    201		regs->tpc &= 0xffffffff;
    202		regs->tnpc &= 0xffffffff;
    203	}
    204	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)regs->tpc);
    205out:
    206	exception_exit(prev_state);
    207}
    208
    209void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
    210{
    211	if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
    212		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
    213		return;
    214
    215	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
    216	spitfire_insn_access_exception(regs, sfsr, sfar);
    217}
    218
    219void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
    220{
    221	unsigned short type = (type_ctx >> 16);
    222	unsigned short ctx  = (type_ctx & 0xffff);
    223
    224	if (notify_die(DIE_TRAP, "instruction access exception", regs,
    225		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
    226		return;
    227
    228	if (regs->tstate & TSTATE_PRIV) {
    229		printk("sun4v_insn_access_exception: ADDR[%016lx] "
    230		       "CTX[%04x] TYPE[%04x], going.\n",
    231		       addr, ctx, type);
    232		die_if_kernel("Iax", regs);
    233	}
    234
    235	if (test_thread_flag(TIF_32BIT)) {
    236		regs->tpc &= 0xffffffff;
    237		regs->tnpc &= 0xffffffff;
    238	}
    239	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *) addr);
    240}
    241
    242void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
    243{
    244	if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
    245		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
    246		return;
    247
    248	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
    249	sun4v_insn_access_exception(regs, addr, type_ctx);
    250}
    251
    252bool is_no_fault_exception(struct pt_regs *regs)
    253{
    254	unsigned char asi;
    255	u32 insn;
    256
    257	if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT)
    258		return false;
    259
    260	/*
    261	 * Must do a little instruction decoding here in order to
    262	 * decide on a course of action. The bits of interest are:
    263	 *  insn[31:30] = op, where 3 indicates the load/store group
    264	 *  insn[24:19] = op3, which identifies individual opcodes
    265	 *  insn[13] indicates an immediate offset
    266	 *  op3[4]=1 identifies alternate space instructions
    267	 *  op3[5:4]=3 identifies floating point instructions
    268	 *  op3[2]=1 identifies stores
    269	 * See "Opcode Maps" in the appendix of any Sparc V9
    270	 * architecture spec for full details.
    271	 */
    272	if ((insn & 0xc0800000) == 0xc0800000) {    /* op=3, op3[4]=1   */
    273		if (insn & 0x2000)		    /* immediate offset */
    274			asi = (regs->tstate >> 24); /* saved %asi       */
    275		else
    276			asi = (insn >> 5);	    /* immediate asi    */
    277		if ((asi & 0xf6) == ASI_PNF) {
    278			if (insn & 0x200000)        /* op3[2], stores   */
    279				return false;
    280			if (insn & 0x1000000)       /* op3[5:4]=3 (fp)  */
    281				handle_ldf_stq(insn, regs);
    282			else
    283				handle_ld_nf(insn, regs);
    284			return true;
    285		}
    286	}
    287	return false;
    288}
    289
    290void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
    291{
    292	enum ctx_state prev_state = exception_enter();
    293
    294	if (notify_die(DIE_TRAP, "data access exception", regs,
    295		       0, 0x30, SIGTRAP) == NOTIFY_STOP)
    296		goto out;
    297
    298	if (regs->tstate & TSTATE_PRIV) {
    299		/* Test if this comes from uaccess places. */
    300		const struct exception_table_entry *entry;
    301
    302		entry = search_exception_tables(regs->tpc);
    303		if (entry) {
    304			/* Ouch, somebody is trying VM hole tricks on us... */
    305#ifdef DEBUG_EXCEPTIONS
    306			printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
    307			printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
    308			       regs->tpc, entry->fixup);
    309#endif
    310			regs->tpc = entry->fixup;
    311			regs->tnpc = regs->tpc + 4;
    312			goto out;
    313		}
    314		/* Shit... */
    315		printk("spitfire_data_access_exception: SFSR[%016lx] "
    316		       "SFAR[%016lx], going.\n", sfsr, sfar);
    317		die_if_kernel("Dax", regs);
    318	}
    319
    320	if (is_no_fault_exception(regs))
    321		return;
    322
    323	force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)sfar);
    324out:
    325	exception_exit(prev_state);
    326}
    327
    328void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
    329{
    330	if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
    331		       0, 0x30, SIGTRAP) == NOTIFY_STOP)
    332		return;
    333
    334	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
    335	spitfire_data_access_exception(regs, sfsr, sfar);
    336}
    337
    338void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
    339{
    340	unsigned short type = (type_ctx >> 16);
    341	unsigned short ctx  = (type_ctx & 0xffff);
    342
    343	if (notify_die(DIE_TRAP, "data access exception", regs,
    344		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
    345		return;
    346
    347	if (regs->tstate & TSTATE_PRIV) {
    348		/* Test if this comes from uaccess places. */
    349		const struct exception_table_entry *entry;
    350
    351		entry = search_exception_tables(regs->tpc);
    352		if (entry) {
    353			/* Ouch, somebody is trying VM hole tricks on us... */
    354#ifdef DEBUG_EXCEPTIONS
    355			printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
    356			printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
    357			       regs->tpc, entry->fixup);
    358#endif
    359			regs->tpc = entry->fixup;
    360			regs->tnpc = regs->tpc + 4;
    361			return;
    362		}
    363		printk("sun4v_data_access_exception: ADDR[%016lx] "
    364		       "CTX[%04x] TYPE[%04x], going.\n",
    365		       addr, ctx, type);
    366		die_if_kernel("Dax", regs);
    367	}
    368
    369	if (test_thread_flag(TIF_32BIT)) {
    370		regs->tpc &= 0xffffffff;
    371		regs->tnpc &= 0xffffffff;
    372	}
    373	if (is_no_fault_exception(regs))
    374		return;
    375
    376	/* MCD (Memory Corruption Detection) disabled trap (TT=0x19) in HV
    377	 * is vectored thorugh data access exception trap with fault type
    378	 * set to HV_FAULT_TYPE_MCD_DIS. Check for MCD disabled trap.
    379	 * Accessing an address with invalid ASI for the address, for
    380	 * example setting an ADI tag on an address with ASI_MCD_PRIMARY
    381	 * when TTE.mcd is not set for the VA, is also vectored into
    382	 * kerbel by HV as data access exception with fault type set to
    383	 * HV_FAULT_TYPE_INV_ASI.
    384	 */
    385	switch (type) {
    386	case HV_FAULT_TYPE_INV_ASI:
    387		force_sig_fault(SIGILL, ILL_ILLADR, (void __user *)addr);
    388		break;
    389	case HV_FAULT_TYPE_MCD_DIS:
    390		force_sig_fault(SIGSEGV, SEGV_ACCADI, (void __user *)addr);
    391		break;
    392	default:
    393		force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)addr);
    394		break;
    395	}
    396}
    397
    398void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
    399{
    400	if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
    401		       0, 0x8, SIGTRAP) == NOTIFY_STOP)
    402		return;
    403
    404	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
    405	sun4v_data_access_exception(regs, addr, type_ctx);
    406}
    407
    408#ifdef CONFIG_PCI
    409#include "pci_impl.h"
    410#endif
    411
    412/* When access exceptions happen, we must do this. */
    413static void spitfire_clean_and_reenable_l1_caches(void)
    414{
    415	unsigned long va;
    416
    417	if (tlb_type != spitfire)
    418		BUG();
    419
    420	/* Clean 'em. */
    421	for (va =  0; va < (PAGE_SIZE << 1); va += 32) {
    422		spitfire_put_icache_tag(va, 0x0);
    423		spitfire_put_dcache_tag(va, 0x0);
    424	}
    425
    426	/* Re-enable in LSU. */
    427	__asm__ __volatile__("flush %%g6\n\t"
    428			     "membar #Sync\n\t"
    429			     "stxa %0, [%%g0] %1\n\t"
    430			     "membar #Sync"
    431			     : /* no outputs */
    432			     : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
    433				    LSU_CONTROL_IM | LSU_CONTROL_DM),
    434			     "i" (ASI_LSU_CONTROL)
    435			     : "memory");
    436}
    437
    438static void spitfire_enable_estate_errors(void)
    439{
    440	__asm__ __volatile__("stxa	%0, [%%g0] %1\n\t"
    441			     "membar	#Sync"
    442			     : /* no outputs */
    443			     : "r" (ESTATE_ERR_ALL),
    444			       "i" (ASI_ESTATE_ERROR_EN));
    445}
    446
    447static char ecc_syndrome_table[] = {
    448	0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
    449	0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
    450	0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
    451	0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
    452	0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
    453	0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
    454	0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
    455	0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
    456	0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
    457	0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
    458	0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
    459	0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
    460	0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
    461	0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
    462	0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
    463	0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
    464	0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
    465	0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
    466	0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
    467	0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
    468	0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
    469	0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
    470	0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
    471	0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
    472	0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
    473	0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
    474	0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
    475	0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
    476	0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
    477	0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
    478	0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
    479	0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
    480};
    481
    482static char *syndrome_unknown = "<Unknown>";
    483
    484static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
    485{
    486	unsigned short scode;
    487	char memmod_str[64], *p;
    488
    489	if (udbl & bit) {
    490		scode = ecc_syndrome_table[udbl & 0xff];
    491		if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
    492			p = syndrome_unknown;
    493		else
    494			p = memmod_str;
    495		printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
    496		       "Memory Module \"%s\"\n",
    497		       smp_processor_id(), scode, p);
    498	}
    499
    500	if (udbh & bit) {
    501		scode = ecc_syndrome_table[udbh & 0xff];
    502		if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
    503			p = syndrome_unknown;
    504		else
    505			p = memmod_str;
    506		printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
    507		       "Memory Module \"%s\"\n",
    508		       smp_processor_id(), scode, p);
    509	}
    510
    511}
    512
    513static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
    514{
    515
    516	printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
    517	       "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
    518	       smp_processor_id(), afsr, afar, udbl, udbh, tl1);
    519
    520	spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
    521
    522	/* We always log it, even if someone is listening for this
    523	 * trap.
    524	 */
    525	notify_die(DIE_TRAP, "Correctable ECC Error", regs,
    526		   0, TRAP_TYPE_CEE, SIGTRAP);
    527
    528	/* The Correctable ECC Error trap does not disable I/D caches.  So
    529	 * we only have to restore the ESTATE Error Enable register.
    530	 */
    531	spitfire_enable_estate_errors();
    532}
    533
    534static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
    535{
    536	printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
    537	       "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
    538	       smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
    539
    540	/* XXX add more human friendly logging of the error status
    541	 * XXX as is implemented for cheetah
    542	 */
    543
    544	spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
    545
    546	/* We always log it, even if someone is listening for this
    547	 * trap.
    548	 */
    549	notify_die(DIE_TRAP, "Uncorrectable Error", regs,
    550		   0, tt, SIGTRAP);
    551
    552	if (regs->tstate & TSTATE_PRIV) {
    553		if (tl1)
    554			dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
    555		die_if_kernel("UE", regs);
    556	}
    557
    558	/* XXX need more intelligent processing here, such as is implemented
    559	 * XXX for cheetah errors, in fact if the E-cache still holds the
    560	 * XXX line with bad parity this will loop
    561	 */
    562
    563	spitfire_clean_and_reenable_l1_caches();
    564	spitfire_enable_estate_errors();
    565
    566	if (test_thread_flag(TIF_32BIT)) {
    567		regs->tpc &= 0xffffffff;
    568		regs->tnpc &= 0xffffffff;
    569	}
    570	force_sig_fault(SIGBUS, BUS_OBJERR, (void *)0);
    571}
    572
    573void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
    574{
    575	unsigned long afsr, tt, udbh, udbl;
    576	int tl1;
    577
    578	afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
    579	tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
    580	tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
    581	udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
    582	udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
    583
    584#ifdef CONFIG_PCI
    585	if (tt == TRAP_TYPE_DAE &&
    586	    pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
    587		spitfire_clean_and_reenable_l1_caches();
    588		spitfire_enable_estate_errors();
    589
    590		pci_poke_faulted = 1;
    591		regs->tnpc = regs->tpc + 4;
    592		return;
    593	}
    594#endif
    595
    596	if (afsr & SFAFSR_UE)
    597		spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
    598
    599	if (tt == TRAP_TYPE_CEE) {
    600		/* Handle the case where we took a CEE trap, but ACK'd
    601		 * only the UE state in the UDB error registers.
    602		 */
    603		if (afsr & SFAFSR_UE) {
    604			if (udbh & UDBE_CE) {
    605				__asm__ __volatile__(
    606					"stxa	%0, [%1] %2\n\t"
    607					"membar	#Sync"
    608					: /* no outputs */
    609					: "r" (udbh & UDBE_CE),
    610					  "r" (0x0), "i" (ASI_UDB_ERROR_W));
    611			}
    612			if (udbl & UDBE_CE) {
    613				__asm__ __volatile__(
    614					"stxa	%0, [%1] %2\n\t"
    615					"membar	#Sync"
    616					: /* no outputs */
    617					: "r" (udbl & UDBE_CE),
    618					  "r" (0x18), "i" (ASI_UDB_ERROR_W));
    619			}
    620		}
    621
    622		spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
    623	}
    624}
    625
    626int cheetah_pcache_forced_on;
    627
    628void cheetah_enable_pcache(void)
    629{
    630	unsigned long dcr;
    631
    632	printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
    633	       smp_processor_id());
    634
    635	__asm__ __volatile__("ldxa [%%g0] %1, %0"
    636			     : "=r" (dcr)
    637			     : "i" (ASI_DCU_CONTROL_REG));
    638	dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
    639	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
    640			     "membar #Sync"
    641			     : /* no outputs */
    642			     : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
    643}
    644
    645/* Cheetah error trap handling. */
    646static unsigned long ecache_flush_physbase;
    647static unsigned long ecache_flush_linesize;
    648static unsigned long ecache_flush_size;
    649
    650/* This table is ordered in priority of errors and matches the
    651 * AFAR overwrite policy as well.
    652 */
    653
    654struct afsr_error_table {
    655	unsigned long mask;
    656	const char *name;
    657};
    658
    659static const char CHAFSR_PERR_msg[] =
    660	"System interface protocol error";
    661static const char CHAFSR_IERR_msg[] =
    662	"Internal processor error";
    663static const char CHAFSR_ISAP_msg[] =
    664	"System request parity error on incoming address";
    665static const char CHAFSR_UCU_msg[] =
    666	"Uncorrectable E-cache ECC error for ifetch/data";
    667static const char CHAFSR_UCC_msg[] =
    668	"SW Correctable E-cache ECC error for ifetch/data";
    669static const char CHAFSR_UE_msg[] =
    670	"Uncorrectable system bus data ECC error for read";
    671static const char CHAFSR_EDU_msg[] =
    672	"Uncorrectable E-cache ECC error for stmerge/blkld";
    673static const char CHAFSR_EMU_msg[] =
    674	"Uncorrectable system bus MTAG error";
    675static const char CHAFSR_WDU_msg[] =
    676	"Uncorrectable E-cache ECC error for writeback";
    677static const char CHAFSR_CPU_msg[] =
    678	"Uncorrectable ECC error for copyout";
    679static const char CHAFSR_CE_msg[] =
    680	"HW corrected system bus data ECC error for read";
    681static const char CHAFSR_EDC_msg[] =
    682	"HW corrected E-cache ECC error for stmerge/blkld";
    683static const char CHAFSR_EMC_msg[] =
    684	"HW corrected system bus MTAG ECC error";
    685static const char CHAFSR_WDC_msg[] =
    686	"HW corrected E-cache ECC error for writeback";
    687static const char CHAFSR_CPC_msg[] =
    688	"HW corrected ECC error for copyout";
    689static const char CHAFSR_TO_msg[] =
    690	"Unmapped error from system bus";
    691static const char CHAFSR_BERR_msg[] =
    692	"Bus error response from system bus";
    693static const char CHAFSR_IVC_msg[] =
    694	"HW corrected system bus data ECC error for ivec read";
    695static const char CHAFSR_IVU_msg[] =
    696	"Uncorrectable system bus data ECC error for ivec read";
    697static struct afsr_error_table __cheetah_error_table[] = {
    698	{	CHAFSR_PERR,	CHAFSR_PERR_msg		},
    699	{	CHAFSR_IERR,	CHAFSR_IERR_msg		},
    700	{	CHAFSR_ISAP,	CHAFSR_ISAP_msg		},
    701	{	CHAFSR_UCU,	CHAFSR_UCU_msg		},
    702	{	CHAFSR_UCC,	CHAFSR_UCC_msg		},
    703	{	CHAFSR_UE,	CHAFSR_UE_msg		},
    704	{	CHAFSR_EDU,	CHAFSR_EDU_msg		},
    705	{	CHAFSR_EMU,	CHAFSR_EMU_msg		},
    706	{	CHAFSR_WDU,	CHAFSR_WDU_msg		},
    707	{	CHAFSR_CPU,	CHAFSR_CPU_msg		},
    708	{	CHAFSR_CE,	CHAFSR_CE_msg		},
    709	{	CHAFSR_EDC,	CHAFSR_EDC_msg		},
    710	{	CHAFSR_EMC,	CHAFSR_EMC_msg		},
    711	{	CHAFSR_WDC,	CHAFSR_WDC_msg		},
    712	{	CHAFSR_CPC,	CHAFSR_CPC_msg		},
    713	{	CHAFSR_TO,	CHAFSR_TO_msg		},
    714	{	CHAFSR_BERR,	CHAFSR_BERR_msg		},
    715	/* These two do not update the AFAR. */
    716	{	CHAFSR_IVC,	CHAFSR_IVC_msg		},
    717	{	CHAFSR_IVU,	CHAFSR_IVU_msg		},
    718	{	0,		NULL			},
    719};
    720static const char CHPAFSR_DTO_msg[] =
    721	"System bus unmapped error for prefetch/storequeue-read";
    722static const char CHPAFSR_DBERR_msg[] =
    723	"System bus error for prefetch/storequeue-read";
    724static const char CHPAFSR_THCE_msg[] =
    725	"Hardware corrected E-cache Tag ECC error";
    726static const char CHPAFSR_TSCE_msg[] =
    727	"SW handled correctable E-cache Tag ECC error";
    728static const char CHPAFSR_TUE_msg[] =
    729	"Uncorrectable E-cache Tag ECC error";
    730static const char CHPAFSR_DUE_msg[] =
    731	"System bus uncorrectable data ECC error due to prefetch/store-fill";
    732static struct afsr_error_table __cheetah_plus_error_table[] = {
    733	{	CHAFSR_PERR,	CHAFSR_PERR_msg		},
    734	{	CHAFSR_IERR,	CHAFSR_IERR_msg		},
    735	{	CHAFSR_ISAP,	CHAFSR_ISAP_msg		},
    736	{	CHAFSR_UCU,	CHAFSR_UCU_msg		},
    737	{	CHAFSR_UCC,	CHAFSR_UCC_msg		},
    738	{	CHAFSR_UE,	CHAFSR_UE_msg		},
    739	{	CHAFSR_EDU,	CHAFSR_EDU_msg		},
    740	{	CHAFSR_EMU,	CHAFSR_EMU_msg		},
    741	{	CHAFSR_WDU,	CHAFSR_WDU_msg		},
    742	{	CHAFSR_CPU,	CHAFSR_CPU_msg		},
    743	{	CHAFSR_CE,	CHAFSR_CE_msg		},
    744	{	CHAFSR_EDC,	CHAFSR_EDC_msg		},
    745	{	CHAFSR_EMC,	CHAFSR_EMC_msg		},
    746	{	CHAFSR_WDC,	CHAFSR_WDC_msg		},
    747	{	CHAFSR_CPC,	CHAFSR_CPC_msg		},
    748	{	CHAFSR_TO,	CHAFSR_TO_msg		},
    749	{	CHAFSR_BERR,	CHAFSR_BERR_msg		},
    750	{	CHPAFSR_DTO,	CHPAFSR_DTO_msg		},
    751	{	CHPAFSR_DBERR,	CHPAFSR_DBERR_msg	},
    752	{	CHPAFSR_THCE,	CHPAFSR_THCE_msg	},
    753	{	CHPAFSR_TSCE,	CHPAFSR_TSCE_msg	},
    754	{	CHPAFSR_TUE,	CHPAFSR_TUE_msg		},
    755	{	CHPAFSR_DUE,	CHPAFSR_DUE_msg		},
    756	/* These two do not update the AFAR. */
    757	{	CHAFSR_IVC,	CHAFSR_IVC_msg		},
    758	{	CHAFSR_IVU,	CHAFSR_IVU_msg		},
    759	{	0,		NULL			},
    760};
    761static const char JPAFSR_JETO_msg[] =
    762	"System interface protocol error, hw timeout caused";
    763static const char JPAFSR_SCE_msg[] =
    764	"Parity error on system snoop results";
    765static const char JPAFSR_JEIC_msg[] =
    766	"System interface protocol error, illegal command detected";
    767static const char JPAFSR_JEIT_msg[] =
    768	"System interface protocol error, illegal ADTYPE detected";
    769static const char JPAFSR_OM_msg[] =
    770	"Out of range memory error has occurred";
    771static const char JPAFSR_ETP_msg[] =
    772	"Parity error on L2 cache tag SRAM";
    773static const char JPAFSR_UMS_msg[] =
    774	"Error due to unsupported store";
    775static const char JPAFSR_RUE_msg[] =
    776	"Uncorrectable ECC error from remote cache/memory";
    777static const char JPAFSR_RCE_msg[] =
    778	"Correctable ECC error from remote cache/memory";
    779static const char JPAFSR_BP_msg[] =
    780	"JBUS parity error on returned read data";
    781static const char JPAFSR_WBP_msg[] =
    782	"JBUS parity error on data for writeback or block store";
    783static const char JPAFSR_FRC_msg[] =
    784	"Foreign read to DRAM incurring correctable ECC error";
    785static const char JPAFSR_FRU_msg[] =
    786	"Foreign read to DRAM incurring uncorrectable ECC error";
    787static struct afsr_error_table __jalapeno_error_table[] = {
    788	{	JPAFSR_JETO,	JPAFSR_JETO_msg		},
    789	{	JPAFSR_SCE,	JPAFSR_SCE_msg		},
    790	{	JPAFSR_JEIC,	JPAFSR_JEIC_msg		},
    791	{	JPAFSR_JEIT,	JPAFSR_JEIT_msg		},
    792	{	CHAFSR_PERR,	CHAFSR_PERR_msg		},
    793	{	CHAFSR_IERR,	CHAFSR_IERR_msg		},
    794	{	CHAFSR_ISAP,	CHAFSR_ISAP_msg		},
    795	{	CHAFSR_UCU,	CHAFSR_UCU_msg		},
    796	{	CHAFSR_UCC,	CHAFSR_UCC_msg		},
    797	{	CHAFSR_UE,	CHAFSR_UE_msg		},
    798	{	CHAFSR_EDU,	CHAFSR_EDU_msg		},
    799	{	JPAFSR_OM,	JPAFSR_OM_msg		},
    800	{	CHAFSR_WDU,	CHAFSR_WDU_msg		},
    801	{	CHAFSR_CPU,	CHAFSR_CPU_msg		},
    802	{	CHAFSR_CE,	CHAFSR_CE_msg		},
    803	{	CHAFSR_EDC,	CHAFSR_EDC_msg		},
    804	{	JPAFSR_ETP,	JPAFSR_ETP_msg		},
    805	{	CHAFSR_WDC,	CHAFSR_WDC_msg		},
    806	{	CHAFSR_CPC,	CHAFSR_CPC_msg		},
    807	{	CHAFSR_TO,	CHAFSR_TO_msg		},
    808	{	CHAFSR_BERR,	CHAFSR_BERR_msg		},
    809	{	JPAFSR_UMS,	JPAFSR_UMS_msg		},
    810	{	JPAFSR_RUE,	JPAFSR_RUE_msg		},
    811	{	JPAFSR_RCE,	JPAFSR_RCE_msg		},
    812	{	JPAFSR_BP,	JPAFSR_BP_msg		},
    813	{	JPAFSR_WBP,	JPAFSR_WBP_msg		},
    814	{	JPAFSR_FRC,	JPAFSR_FRC_msg		},
    815	{	JPAFSR_FRU,	JPAFSR_FRU_msg		},
    816	/* These two do not update the AFAR. */
    817	{	CHAFSR_IVU,	CHAFSR_IVU_msg		},
    818	{	0,		NULL			},
    819};
    820static struct afsr_error_table *cheetah_error_table;
    821static unsigned long cheetah_afsr_errors;
    822
    823struct cheetah_err_info *cheetah_error_log;
    824
    825static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
    826{
    827	struct cheetah_err_info *p;
    828	int cpu = smp_processor_id();
    829
    830	if (!cheetah_error_log)
    831		return NULL;
    832
    833	p = cheetah_error_log + (cpu * 2);
    834	if ((afsr & CHAFSR_TL1) != 0UL)
    835		p++;
    836
    837	return p;
    838}
    839
    840extern unsigned int tl0_icpe[], tl1_icpe[];
    841extern unsigned int tl0_dcpe[], tl1_dcpe[];
    842extern unsigned int tl0_fecc[], tl1_fecc[];
    843extern unsigned int tl0_cee[], tl1_cee[];
    844extern unsigned int tl0_iae[], tl1_iae[];
    845extern unsigned int tl0_dae[], tl1_dae[];
    846extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
    847extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
    848extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
    849extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
    850extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
    851
    852void __init cheetah_ecache_flush_init(void)
    853{
    854	unsigned long largest_size, smallest_linesize, order, ver;
    855	int i, sz;
    856
    857	/* Scan all cpu device tree nodes, note two values:
    858	 * 1) largest E-cache size
    859	 * 2) smallest E-cache line size
    860	 */
    861	largest_size = 0UL;
    862	smallest_linesize = ~0UL;
    863
    864	for (i = 0; i < NR_CPUS; i++) {
    865		unsigned long val;
    866
    867		val = cpu_data(i).ecache_size;
    868		if (!val)
    869			continue;
    870
    871		if (val > largest_size)
    872			largest_size = val;
    873
    874		val = cpu_data(i).ecache_line_size;
    875		if (val < smallest_linesize)
    876			smallest_linesize = val;
    877
    878	}
    879
    880	if (largest_size == 0UL || smallest_linesize == ~0UL) {
    881		prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
    882			    "parameters.\n");
    883		prom_halt();
    884	}
    885
    886	ecache_flush_size = (2 * largest_size);
    887	ecache_flush_linesize = smallest_linesize;
    888
    889	ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
    890
    891	if (ecache_flush_physbase == ~0UL) {
    892		prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
    893			    "contiguous physical memory.\n",
    894			    ecache_flush_size);
    895		prom_halt();
    896	}
    897
    898	/* Now allocate error trap reporting scoreboard. */
    899	sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
    900	for (order = 0; order < MAX_ORDER; order++) {
    901		if ((PAGE_SIZE << order) >= sz)
    902			break;
    903	}
    904	cheetah_error_log = (struct cheetah_err_info *)
    905		__get_free_pages(GFP_KERNEL, order);
    906	if (!cheetah_error_log) {
    907		prom_printf("cheetah_ecache_flush_init: Failed to allocate "
    908			    "error logging scoreboard (%d bytes).\n", sz);
    909		prom_halt();
    910	}
    911	memset(cheetah_error_log, 0, PAGE_SIZE << order);
    912
    913	/* Mark all AFSRs as invalid so that the trap handler will
    914	 * log new new information there.
    915	 */
    916	for (i = 0; i < 2 * NR_CPUS; i++)
    917		cheetah_error_log[i].afsr = CHAFSR_INVALID;
    918
    919	__asm__ ("rdpr %%ver, %0" : "=r" (ver));
    920	if ((ver >> 32) == __JALAPENO_ID ||
    921	    (ver >> 32) == __SERRANO_ID) {
    922		cheetah_error_table = &__jalapeno_error_table[0];
    923		cheetah_afsr_errors = JPAFSR_ERRORS;
    924	} else if ((ver >> 32) == 0x003e0015) {
    925		cheetah_error_table = &__cheetah_plus_error_table[0];
    926		cheetah_afsr_errors = CHPAFSR_ERRORS;
    927	} else {
    928		cheetah_error_table = &__cheetah_error_table[0];
    929		cheetah_afsr_errors = CHAFSR_ERRORS;
    930	}
    931
    932	/* Now patch trap tables. */
    933	memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
    934	memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
    935	memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
    936	memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
    937	memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
    938	memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
    939	memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
    940	memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
    941	if (tlb_type == cheetah_plus) {
    942		memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
    943		memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
    944		memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
    945		memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
    946	}
    947	flushi(PAGE_OFFSET);
    948}
    949
    950static void cheetah_flush_ecache(void)
    951{
    952	unsigned long flush_base = ecache_flush_physbase;
    953	unsigned long flush_linesize = ecache_flush_linesize;
    954	unsigned long flush_size = ecache_flush_size;
    955
    956	__asm__ __volatile__("1: subcc	%0, %4, %0\n\t"
    957			     "   bne,pt	%%xcc, 1b\n\t"
    958			     "    ldxa	[%2 + %0] %3, %%g0\n\t"
    959			     : "=&r" (flush_size)
    960			     : "0" (flush_size), "r" (flush_base),
    961			       "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
    962}
    963
    964static void cheetah_flush_ecache_line(unsigned long physaddr)
    965{
    966	unsigned long alias;
    967
    968	physaddr &= ~(8UL - 1UL);
    969	physaddr = (ecache_flush_physbase +
    970		    (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
    971	alias = physaddr + (ecache_flush_size >> 1UL);
    972	__asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
    973			     "ldxa [%1] %2, %%g0\n\t"
    974			     "membar #Sync"
    975			     : /* no outputs */
    976			     : "r" (physaddr), "r" (alias),
    977			       "i" (ASI_PHYS_USE_EC));
    978}
    979
    980/* Unfortunately, the diagnostic access to the I-cache tags we need to
    981 * use to clear the thing interferes with I-cache coherency transactions.
    982 *
    983 * So we must only flush the I-cache when it is disabled.
    984 */
    985static void __cheetah_flush_icache(void)
    986{
    987	unsigned int icache_size, icache_line_size;
    988	unsigned long addr;
    989
    990	icache_size = local_cpu_data().icache_size;
    991	icache_line_size = local_cpu_data().icache_line_size;
    992
    993	/* Clear the valid bits in all the tags. */
    994	for (addr = 0; addr < icache_size; addr += icache_line_size) {
    995		__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
    996				     "membar #Sync"
    997				     : /* no outputs */
    998				     : "r" (addr | (2 << 3)),
    999				       "i" (ASI_IC_TAG));
   1000	}
   1001}
   1002
   1003static void cheetah_flush_icache(void)
   1004{
   1005	unsigned long dcu_save;
   1006
   1007	/* Save current DCU, disable I-cache. */
   1008	__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
   1009			     "or %0, %2, %%g1\n\t"
   1010			     "stxa %%g1, [%%g0] %1\n\t"
   1011			     "membar #Sync"
   1012			     : "=r" (dcu_save)
   1013			     : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
   1014			     : "g1");
   1015
   1016	__cheetah_flush_icache();
   1017
   1018	/* Restore DCU register */
   1019	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
   1020			     "membar #Sync"
   1021			     : /* no outputs */
   1022			     : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
   1023}
   1024
   1025static void cheetah_flush_dcache(void)
   1026{
   1027	unsigned int dcache_size, dcache_line_size;
   1028	unsigned long addr;
   1029
   1030	dcache_size = local_cpu_data().dcache_size;
   1031	dcache_line_size = local_cpu_data().dcache_line_size;
   1032
   1033	for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
   1034		__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
   1035				     "membar #Sync"
   1036				     : /* no outputs */
   1037				     : "r" (addr), "i" (ASI_DCACHE_TAG));
   1038	}
   1039}
   1040
   1041/* In order to make the even parity correct we must do two things.
   1042 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
   1043 * Next, we clear out all 32-bytes of data for that line.  Data of
   1044 * all-zero + tag parity value of zero == correct parity.
   1045 */
   1046static void cheetah_plus_zap_dcache_parity(void)
   1047{
   1048	unsigned int dcache_size, dcache_line_size;
   1049	unsigned long addr;
   1050
   1051	dcache_size = local_cpu_data().dcache_size;
   1052	dcache_line_size = local_cpu_data().dcache_line_size;
   1053
   1054	for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
   1055		unsigned long tag = (addr >> 14);
   1056		unsigned long line;
   1057
   1058		__asm__ __volatile__("membar	#Sync\n\t"
   1059				     "stxa	%0, [%1] %2\n\t"
   1060				     "membar	#Sync"
   1061				     : /* no outputs */
   1062				     : "r" (tag), "r" (addr),
   1063				       "i" (ASI_DCACHE_UTAG));
   1064		for (line = addr; line < addr + dcache_line_size; line += 8)
   1065			__asm__ __volatile__("membar	#Sync\n\t"
   1066					     "stxa	%%g0, [%0] %1\n\t"
   1067					     "membar	#Sync"
   1068					     : /* no outputs */
   1069					     : "r" (line),
   1070					       "i" (ASI_DCACHE_DATA));
   1071	}
   1072}
   1073
   1074/* Conversion tables used to frob Cheetah AFSR syndrome values into
   1075 * something palatable to the memory controller driver get_unumber
   1076 * routine.
   1077 */
   1078#define MT0	137
   1079#define MT1	138
   1080#define MT2	139
   1081#define NONE	254
   1082#define MTC0	140
   1083#define MTC1	141
   1084#define MTC2	142
   1085#define MTC3	143
   1086#define C0	128
   1087#define C1	129
   1088#define C2	130
   1089#define C3	131
   1090#define C4	132
   1091#define C5	133
   1092#define C6	134
   1093#define C7	135
   1094#define C8	136
   1095#define M2	144
   1096#define M3	145
   1097#define M4	146
   1098#define M	147
   1099static unsigned char cheetah_ecc_syntab[] = {
   1100/*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
   1101/*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
   1102/*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
   1103/*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
   1104/*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
   1105/*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
   1106/*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
   1107/*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
   1108/*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
   1109/*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
   1110/*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
   1111/*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
   1112/*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
   1113/*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
   1114/*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
   1115/*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
   1116/*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
   1117/*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
   1118/*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
   1119/*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
   1120/*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
   1121/*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
   1122/*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
   1123/*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
   1124/*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
   1125/*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
   1126/*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
   1127/*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
   1128/*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
   1129/*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
   1130/*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
   1131/*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
   1132};
   1133static unsigned char cheetah_mtag_syntab[] = {
   1134       NONE, MTC0,
   1135       MTC1, NONE,
   1136       MTC2, NONE,
   1137       NONE, MT0,
   1138       MTC3, NONE,
   1139       NONE, MT1,
   1140       NONE, MT2,
   1141       NONE, NONE
   1142};
   1143
   1144/* Return the highest priority error conditon mentioned. */
   1145static inline unsigned long cheetah_get_hipri(unsigned long afsr)
   1146{
   1147	unsigned long tmp = 0;
   1148	int i;
   1149
   1150	for (i = 0; cheetah_error_table[i].mask; i++) {
   1151		if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
   1152			return tmp;
   1153	}
   1154	return tmp;
   1155}
   1156
   1157static const char *cheetah_get_string(unsigned long bit)
   1158{
   1159	int i;
   1160
   1161	for (i = 0; cheetah_error_table[i].mask; i++) {
   1162		if ((bit & cheetah_error_table[i].mask) != 0UL)
   1163			return cheetah_error_table[i].name;
   1164	}
   1165	return "???";
   1166}
   1167
   1168static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
   1169			       unsigned long afsr, unsigned long afar, int recoverable)
   1170{
   1171	unsigned long hipri;
   1172	char unum[256];
   1173
   1174	printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
   1175	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1176	       afsr, afar,
   1177	       (afsr & CHAFSR_TL1) ? 1 : 0);
   1178	printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
   1179	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1180	       regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
   1181	printk("%s" "ERROR(%d): ",
   1182	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
   1183	printk("TPC<%pS>\n", (void *) regs->tpc);
   1184	printk("%s" "ERROR(%d): M_SYND(%lx),  E_SYND(%lx)%s%s\n",
   1185	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1186	       (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
   1187	       (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
   1188	       (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
   1189	       (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
   1190	hipri = cheetah_get_hipri(afsr);
   1191	printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
   1192	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1193	       hipri, cheetah_get_string(hipri));
   1194
   1195	/* Try to get unumber if relevant. */
   1196#define ESYND_ERRORS	(CHAFSR_IVC | CHAFSR_IVU | \
   1197			 CHAFSR_CPC | CHAFSR_CPU | \
   1198			 CHAFSR_UE  | CHAFSR_CE  | \
   1199			 CHAFSR_EDC | CHAFSR_EDU  | \
   1200			 CHAFSR_UCC | CHAFSR_UCU  | \
   1201			 CHAFSR_WDU | CHAFSR_WDC)
   1202#define MSYND_ERRORS	(CHAFSR_EMC | CHAFSR_EMU)
   1203	if (afsr & ESYND_ERRORS) {
   1204		int syndrome;
   1205		int ret;
   1206
   1207		syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
   1208		syndrome = cheetah_ecc_syntab[syndrome];
   1209		ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
   1210		if (ret != -1)
   1211			printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
   1212			       (recoverable ? KERN_WARNING : KERN_CRIT),
   1213			       smp_processor_id(), unum);
   1214	} else if (afsr & MSYND_ERRORS) {
   1215		int syndrome;
   1216		int ret;
   1217
   1218		syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
   1219		syndrome = cheetah_mtag_syntab[syndrome];
   1220		ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
   1221		if (ret != -1)
   1222			printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
   1223			       (recoverable ? KERN_WARNING : KERN_CRIT),
   1224			       smp_processor_id(), unum);
   1225	}
   1226
   1227	/* Now dump the cache snapshots. */
   1228	printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n",
   1229	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1230	       (int) info->dcache_index,
   1231	       info->dcache_tag,
   1232	       info->dcache_utag,
   1233	       info->dcache_stag);
   1234	printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
   1235	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1236	       info->dcache_data[0],
   1237	       info->dcache_data[1],
   1238	       info->dcache_data[2],
   1239	       info->dcache_data[3]);
   1240	printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] "
   1241	       "u[%016llx] l[%016llx]\n",
   1242	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1243	       (int) info->icache_index,
   1244	       info->icache_tag,
   1245	       info->icache_utag,
   1246	       info->icache_stag,
   1247	       info->icache_upper,
   1248	       info->icache_lower);
   1249	printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n",
   1250	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1251	       info->icache_data[0],
   1252	       info->icache_data[1],
   1253	       info->icache_data[2],
   1254	       info->icache_data[3]);
   1255	printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n",
   1256	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1257	       info->icache_data[4],
   1258	       info->icache_data[5],
   1259	       info->icache_data[6],
   1260	       info->icache_data[7]);
   1261	printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n",
   1262	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1263	       (int) info->ecache_index, info->ecache_tag);
   1264	printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
   1265	       (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
   1266	       info->ecache_data[0],
   1267	       info->ecache_data[1],
   1268	       info->ecache_data[2],
   1269	       info->ecache_data[3]);
   1270
   1271	afsr = (afsr & ~hipri) & cheetah_afsr_errors;
   1272	while (afsr != 0UL) {
   1273		unsigned long bit = cheetah_get_hipri(afsr);
   1274
   1275		printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
   1276		       (recoverable ? KERN_WARNING : KERN_CRIT),
   1277		       bit, cheetah_get_string(bit));
   1278
   1279		afsr &= ~bit;
   1280	}
   1281
   1282	if (!recoverable)
   1283		printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
   1284}
   1285
   1286static int cheetah_recheck_errors(struct cheetah_err_info *logp)
   1287{
   1288	unsigned long afsr, afar;
   1289	int ret = 0;
   1290
   1291	__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
   1292			     : "=r" (afsr)
   1293			     : "i" (ASI_AFSR));
   1294	if ((afsr & cheetah_afsr_errors) != 0) {
   1295		if (logp != NULL) {
   1296			__asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
   1297					     : "=r" (afar)
   1298					     : "i" (ASI_AFAR));
   1299			logp->afsr = afsr;
   1300			logp->afar = afar;
   1301		}
   1302		ret = 1;
   1303	}
   1304	__asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
   1305			     "membar #Sync\n\t"
   1306			     : : "r" (afsr), "i" (ASI_AFSR));
   1307
   1308	return ret;
   1309}
   1310
   1311void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
   1312{
   1313	struct cheetah_err_info local_snapshot, *p;
   1314	int recoverable;
   1315
   1316	/* Flush E-cache */
   1317	cheetah_flush_ecache();
   1318
   1319	p = cheetah_get_error_log(afsr);
   1320	if (!p) {
   1321		prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
   1322			    afsr, afar);
   1323		prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
   1324			    smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
   1325		prom_halt();
   1326	}
   1327
   1328	/* Grab snapshot of logged error. */
   1329	memcpy(&local_snapshot, p, sizeof(local_snapshot));
   1330
   1331	/* If the current trap snapshot does not match what the
   1332	 * trap handler passed along into our args, big trouble.
   1333	 * In such a case, mark the local copy as invalid.
   1334	 *
   1335	 * Else, it matches and we mark the afsr in the non-local
   1336	 * copy as invalid so we may log new error traps there.
   1337	 */
   1338	if (p->afsr != afsr || p->afar != afar)
   1339		local_snapshot.afsr = CHAFSR_INVALID;
   1340	else
   1341		p->afsr = CHAFSR_INVALID;
   1342
   1343	cheetah_flush_icache();
   1344	cheetah_flush_dcache();
   1345
   1346	/* Re-enable I-cache/D-cache */
   1347	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1348			     "or %%g1, %1, %%g1\n\t"
   1349			     "stxa %%g1, [%%g0] %0\n\t"
   1350			     "membar #Sync"
   1351			     : /* no outputs */
   1352			     : "i" (ASI_DCU_CONTROL_REG),
   1353			       "i" (DCU_DC | DCU_IC)
   1354			     : "g1");
   1355
   1356	/* Re-enable error reporting */
   1357	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1358			     "or %%g1, %1, %%g1\n\t"
   1359			     "stxa %%g1, [%%g0] %0\n\t"
   1360			     "membar #Sync"
   1361			     : /* no outputs */
   1362			     : "i" (ASI_ESTATE_ERROR_EN),
   1363			       "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
   1364			     : "g1");
   1365
   1366	/* Decide if we can continue after handling this trap and
   1367	 * logging the error.
   1368	 */
   1369	recoverable = 1;
   1370	if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
   1371		recoverable = 0;
   1372
   1373	/* Re-check AFSR/AFAR.  What we are looking for here is whether a new
   1374	 * error was logged while we had error reporting traps disabled.
   1375	 */
   1376	if (cheetah_recheck_errors(&local_snapshot)) {
   1377		unsigned long new_afsr = local_snapshot.afsr;
   1378
   1379		/* If we got a new asynchronous error, die... */
   1380		if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
   1381				CHAFSR_WDU | CHAFSR_CPU |
   1382				CHAFSR_IVU | CHAFSR_UE |
   1383				CHAFSR_BERR | CHAFSR_TO))
   1384			recoverable = 0;
   1385	}
   1386
   1387	/* Log errors. */
   1388	cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
   1389
   1390	if (!recoverable)
   1391		panic("Irrecoverable Fast-ECC error trap.\n");
   1392
   1393	/* Flush E-cache to kick the error trap handlers out. */
   1394	cheetah_flush_ecache();
   1395}
   1396
   1397/* Try to fix a correctable error by pushing the line out from
   1398 * the E-cache.  Recheck error reporting registers to see if the
   1399 * problem is intermittent.
   1400 */
   1401static int cheetah_fix_ce(unsigned long physaddr)
   1402{
   1403	unsigned long orig_estate;
   1404	unsigned long alias1, alias2;
   1405	int ret;
   1406
   1407	/* Make sure correctable error traps are disabled. */
   1408	__asm__ __volatile__("ldxa	[%%g0] %2, %0\n\t"
   1409			     "andn	%0, %1, %%g1\n\t"
   1410			     "stxa	%%g1, [%%g0] %2\n\t"
   1411			     "membar	#Sync"
   1412			     : "=&r" (orig_estate)
   1413			     : "i" (ESTATE_ERROR_CEEN),
   1414			       "i" (ASI_ESTATE_ERROR_EN)
   1415			     : "g1");
   1416
   1417	/* We calculate alias addresses that will force the
   1418	 * cache line in question out of the E-cache.  Then
   1419	 * we bring it back in with an atomic instruction so
   1420	 * that we get it in some modified/exclusive state,
   1421	 * then we displace it again to try and get proper ECC
   1422	 * pushed back into the system.
   1423	 */
   1424	physaddr &= ~(8UL - 1UL);
   1425	alias1 = (ecache_flush_physbase +
   1426		  (physaddr & ((ecache_flush_size >> 1) - 1)));
   1427	alias2 = alias1 + (ecache_flush_size >> 1);
   1428	__asm__ __volatile__("ldxa	[%0] %3, %%g0\n\t"
   1429			     "ldxa	[%1] %3, %%g0\n\t"
   1430			     "casxa	[%2] %3, %%g0, %%g0\n\t"
   1431			     "ldxa	[%0] %3, %%g0\n\t"
   1432			     "ldxa	[%1] %3, %%g0\n\t"
   1433			     "membar	#Sync"
   1434			     : /* no outputs */
   1435			     : "r" (alias1), "r" (alias2),
   1436			       "r" (physaddr), "i" (ASI_PHYS_USE_EC));
   1437
   1438	/* Did that trigger another error? */
   1439	if (cheetah_recheck_errors(NULL)) {
   1440		/* Try one more time. */
   1441		__asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
   1442				     "membar #Sync"
   1443				     : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
   1444		if (cheetah_recheck_errors(NULL))
   1445			ret = 2;
   1446		else
   1447			ret = 1;
   1448	} else {
   1449		/* No new error, intermittent problem. */
   1450		ret = 0;
   1451	}
   1452
   1453	/* Restore error enables. */
   1454	__asm__ __volatile__("stxa	%0, [%%g0] %1\n\t"
   1455			     "membar	#Sync"
   1456			     : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
   1457
   1458	return ret;
   1459}
   1460
   1461/* Return non-zero if PADDR is a valid physical memory address. */
   1462static int cheetah_check_main_memory(unsigned long paddr)
   1463{
   1464	unsigned long vaddr = PAGE_OFFSET + paddr;
   1465
   1466	if (vaddr > (unsigned long) high_memory)
   1467		return 0;
   1468
   1469	return kern_addr_valid(vaddr);
   1470}
   1471
   1472void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
   1473{
   1474	struct cheetah_err_info local_snapshot, *p;
   1475	int recoverable, is_memory;
   1476
   1477	p = cheetah_get_error_log(afsr);
   1478	if (!p) {
   1479		prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
   1480			    afsr, afar);
   1481		prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
   1482			    smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
   1483		prom_halt();
   1484	}
   1485
   1486	/* Grab snapshot of logged error. */
   1487	memcpy(&local_snapshot, p, sizeof(local_snapshot));
   1488
   1489	/* If the current trap snapshot does not match what the
   1490	 * trap handler passed along into our args, big trouble.
   1491	 * In such a case, mark the local copy as invalid.
   1492	 *
   1493	 * Else, it matches and we mark the afsr in the non-local
   1494	 * copy as invalid so we may log new error traps there.
   1495	 */
   1496	if (p->afsr != afsr || p->afar != afar)
   1497		local_snapshot.afsr = CHAFSR_INVALID;
   1498	else
   1499		p->afsr = CHAFSR_INVALID;
   1500
   1501	is_memory = cheetah_check_main_memory(afar);
   1502
   1503	if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
   1504		/* XXX Might want to log the results of this operation
   1505		 * XXX somewhere... -DaveM
   1506		 */
   1507		cheetah_fix_ce(afar);
   1508	}
   1509
   1510	{
   1511		int flush_all, flush_line;
   1512
   1513		flush_all = flush_line = 0;
   1514		if ((afsr & CHAFSR_EDC) != 0UL) {
   1515			if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
   1516				flush_line = 1;
   1517			else
   1518				flush_all = 1;
   1519		} else if ((afsr & CHAFSR_CPC) != 0UL) {
   1520			if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
   1521				flush_line = 1;
   1522			else
   1523				flush_all = 1;
   1524		}
   1525
   1526		/* Trap handler only disabled I-cache, flush it. */
   1527		cheetah_flush_icache();
   1528
   1529		/* Re-enable I-cache */
   1530		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1531				     "or %%g1, %1, %%g1\n\t"
   1532				     "stxa %%g1, [%%g0] %0\n\t"
   1533				     "membar #Sync"
   1534				     : /* no outputs */
   1535				     : "i" (ASI_DCU_CONTROL_REG),
   1536				     "i" (DCU_IC)
   1537				     : "g1");
   1538
   1539		if (flush_all)
   1540			cheetah_flush_ecache();
   1541		else if (flush_line)
   1542			cheetah_flush_ecache_line(afar);
   1543	}
   1544
   1545	/* Re-enable error reporting */
   1546	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1547			     "or %%g1, %1, %%g1\n\t"
   1548			     "stxa %%g1, [%%g0] %0\n\t"
   1549			     "membar #Sync"
   1550			     : /* no outputs */
   1551			     : "i" (ASI_ESTATE_ERROR_EN),
   1552			       "i" (ESTATE_ERROR_CEEN)
   1553			     : "g1");
   1554
   1555	/* Decide if we can continue after handling this trap and
   1556	 * logging the error.
   1557	 */
   1558	recoverable = 1;
   1559	if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
   1560		recoverable = 0;
   1561
   1562	/* Re-check AFSR/AFAR */
   1563	(void) cheetah_recheck_errors(&local_snapshot);
   1564
   1565	/* Log errors. */
   1566	cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
   1567
   1568	if (!recoverable)
   1569		panic("Irrecoverable Correctable-ECC error trap.\n");
   1570}
   1571
   1572void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
   1573{
   1574	struct cheetah_err_info local_snapshot, *p;
   1575	int recoverable, is_memory;
   1576
   1577#ifdef CONFIG_PCI
   1578	/* Check for the special PCI poke sequence. */
   1579	if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
   1580		cheetah_flush_icache();
   1581		cheetah_flush_dcache();
   1582
   1583		/* Re-enable I-cache/D-cache */
   1584		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1585				     "or %%g1, %1, %%g1\n\t"
   1586				     "stxa %%g1, [%%g0] %0\n\t"
   1587				     "membar #Sync"
   1588				     : /* no outputs */
   1589				     : "i" (ASI_DCU_CONTROL_REG),
   1590				       "i" (DCU_DC | DCU_IC)
   1591				     : "g1");
   1592
   1593		/* Re-enable error reporting */
   1594		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1595				     "or %%g1, %1, %%g1\n\t"
   1596				     "stxa %%g1, [%%g0] %0\n\t"
   1597				     "membar #Sync"
   1598				     : /* no outputs */
   1599				     : "i" (ASI_ESTATE_ERROR_EN),
   1600				       "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
   1601				     : "g1");
   1602
   1603		(void) cheetah_recheck_errors(NULL);
   1604
   1605		pci_poke_faulted = 1;
   1606		regs->tpc += 4;
   1607		regs->tnpc = regs->tpc + 4;
   1608		return;
   1609	}
   1610#endif
   1611
   1612	p = cheetah_get_error_log(afsr);
   1613	if (!p) {
   1614		prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
   1615			    afsr, afar);
   1616		prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
   1617			    smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
   1618		prom_halt();
   1619	}
   1620
   1621	/* Grab snapshot of logged error. */
   1622	memcpy(&local_snapshot, p, sizeof(local_snapshot));
   1623
   1624	/* If the current trap snapshot does not match what the
   1625	 * trap handler passed along into our args, big trouble.
   1626	 * In such a case, mark the local copy as invalid.
   1627	 *
   1628	 * Else, it matches and we mark the afsr in the non-local
   1629	 * copy as invalid so we may log new error traps there.
   1630	 */
   1631	if (p->afsr != afsr || p->afar != afar)
   1632		local_snapshot.afsr = CHAFSR_INVALID;
   1633	else
   1634		p->afsr = CHAFSR_INVALID;
   1635
   1636	is_memory = cheetah_check_main_memory(afar);
   1637
   1638	{
   1639		int flush_all, flush_line;
   1640
   1641		flush_all = flush_line = 0;
   1642		if ((afsr & CHAFSR_EDU) != 0UL) {
   1643			if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
   1644				flush_line = 1;
   1645			else
   1646				flush_all = 1;
   1647		} else if ((afsr & CHAFSR_BERR) != 0UL) {
   1648			if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
   1649				flush_line = 1;
   1650			else
   1651				flush_all = 1;
   1652		}
   1653
   1654		cheetah_flush_icache();
   1655		cheetah_flush_dcache();
   1656
   1657		/* Re-enable I/D caches */
   1658		__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1659				     "or %%g1, %1, %%g1\n\t"
   1660				     "stxa %%g1, [%%g0] %0\n\t"
   1661				     "membar #Sync"
   1662				     : /* no outputs */
   1663				     : "i" (ASI_DCU_CONTROL_REG),
   1664				     "i" (DCU_IC | DCU_DC)
   1665				     : "g1");
   1666
   1667		if (flush_all)
   1668			cheetah_flush_ecache();
   1669		else if (flush_line)
   1670			cheetah_flush_ecache_line(afar);
   1671	}
   1672
   1673	/* Re-enable error reporting */
   1674	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1675			     "or %%g1, %1, %%g1\n\t"
   1676			     "stxa %%g1, [%%g0] %0\n\t"
   1677			     "membar #Sync"
   1678			     : /* no outputs */
   1679			     : "i" (ASI_ESTATE_ERROR_EN),
   1680			     "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
   1681			     : "g1");
   1682
   1683	/* Decide if we can continue after handling this trap and
   1684	 * logging the error.
   1685	 */
   1686	recoverable = 1;
   1687	if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
   1688		recoverable = 0;
   1689
   1690	/* Re-check AFSR/AFAR.  What we are looking for here is whether a new
   1691	 * error was logged while we had error reporting traps disabled.
   1692	 */
   1693	if (cheetah_recheck_errors(&local_snapshot)) {
   1694		unsigned long new_afsr = local_snapshot.afsr;
   1695
   1696		/* If we got a new asynchronous error, die... */
   1697		if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
   1698				CHAFSR_WDU | CHAFSR_CPU |
   1699				CHAFSR_IVU | CHAFSR_UE |
   1700				CHAFSR_BERR | CHAFSR_TO))
   1701			recoverable = 0;
   1702	}
   1703
   1704	/* Log errors. */
   1705	cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
   1706
   1707	/* "Recoverable" here means we try to yank the page from ever
   1708	 * being newly used again.  This depends upon a few things:
   1709	 * 1) Must be main memory, and AFAR must be valid.
   1710	 * 2) If we trapped from user, OK.
   1711	 * 3) Else, if we trapped from kernel we must find exception
   1712	 *    table entry (ie. we have to have been accessing user
   1713	 *    space).
   1714	 *
   1715	 * If AFAR is not in main memory, or we trapped from kernel
   1716	 * and cannot find an exception table entry, it is unacceptable
   1717	 * to try and continue.
   1718	 */
   1719	if (recoverable && is_memory) {
   1720		if ((regs->tstate & TSTATE_PRIV) == 0UL) {
   1721			/* OK, usermode access. */
   1722			recoverable = 1;
   1723		} else {
   1724			const struct exception_table_entry *entry;
   1725
   1726			entry = search_exception_tables(regs->tpc);
   1727			if (entry) {
   1728				/* OK, kernel access to userspace. */
   1729				recoverable = 1;
   1730
   1731			} else {
   1732				/* BAD, privileged state is corrupted. */
   1733				recoverable = 0;
   1734			}
   1735
   1736			if (recoverable) {
   1737				if (pfn_valid(afar >> PAGE_SHIFT))
   1738					get_page(pfn_to_page(afar >> PAGE_SHIFT));
   1739				else
   1740					recoverable = 0;
   1741
   1742				/* Only perform fixup if we still have a
   1743				 * recoverable condition.
   1744				 */
   1745				if (recoverable) {
   1746					regs->tpc = entry->fixup;
   1747					regs->tnpc = regs->tpc + 4;
   1748				}
   1749			}
   1750		}
   1751	} else {
   1752		recoverable = 0;
   1753	}
   1754
   1755	if (!recoverable)
   1756		panic("Irrecoverable deferred error trap.\n");
   1757}
   1758
   1759/* Handle a D/I cache parity error trap.  TYPE is encoded as:
   1760 *
   1761 * Bit0:	0=dcache,1=icache
   1762 * Bit1:	0=recoverable,1=unrecoverable
   1763 *
   1764 * The hardware has disabled both the I-cache and D-cache in
   1765 * the %dcr register.  
   1766 */
   1767void cheetah_plus_parity_error(int type, struct pt_regs *regs)
   1768{
   1769	if (type & 0x1)
   1770		__cheetah_flush_icache();
   1771	else
   1772		cheetah_plus_zap_dcache_parity();
   1773	cheetah_flush_dcache();
   1774
   1775	/* Re-enable I-cache/D-cache */
   1776	__asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
   1777			     "or %%g1, %1, %%g1\n\t"
   1778			     "stxa %%g1, [%%g0] %0\n\t"
   1779			     "membar #Sync"
   1780			     : /* no outputs */
   1781			     : "i" (ASI_DCU_CONTROL_REG),
   1782			       "i" (DCU_DC | DCU_IC)
   1783			     : "g1");
   1784
   1785	if (type & 0x2) {
   1786		printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
   1787		       smp_processor_id(),
   1788		       (type & 0x1) ? 'I' : 'D',
   1789		       regs->tpc);
   1790		printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
   1791		panic("Irrecoverable Cheetah+ parity error.");
   1792	}
   1793
   1794	printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
   1795	       smp_processor_id(),
   1796	       (type & 0x1) ? 'I' : 'D',
   1797	       regs->tpc);
   1798	printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
   1799}
   1800
   1801struct sun4v_error_entry {
   1802	/* Unique error handle */
   1803/*0x00*/u64		err_handle;
   1804
   1805	/* %stick value at the time of the error */
   1806/*0x08*/u64		err_stick;
   1807
   1808/*0x10*/u8		reserved_1[3];
   1809
   1810	/* Error type */
   1811/*0x13*/u8		err_type;
   1812#define SUN4V_ERR_TYPE_UNDEFINED	0
   1813#define SUN4V_ERR_TYPE_UNCORRECTED_RES	1
   1814#define SUN4V_ERR_TYPE_PRECISE_NONRES	2
   1815#define SUN4V_ERR_TYPE_DEFERRED_NONRES	3
   1816#define SUN4V_ERR_TYPE_SHUTDOWN_RQST	4
   1817#define SUN4V_ERR_TYPE_DUMP_CORE	5
   1818#define SUN4V_ERR_TYPE_SP_STATE_CHANGE	6
   1819#define SUN4V_ERR_TYPE_NUM		7
   1820
   1821	/* Error attributes */
   1822/*0x14*/u32		err_attrs;
   1823#define SUN4V_ERR_ATTRS_PROCESSOR	0x00000001
   1824#define SUN4V_ERR_ATTRS_MEMORY		0x00000002
   1825#define SUN4V_ERR_ATTRS_PIO		0x00000004
   1826#define SUN4V_ERR_ATTRS_INT_REGISTERS	0x00000008
   1827#define SUN4V_ERR_ATTRS_FPU_REGISTERS	0x00000010
   1828#define SUN4V_ERR_ATTRS_SHUTDOWN_RQST	0x00000020
   1829#define SUN4V_ERR_ATTRS_ASR		0x00000040
   1830#define SUN4V_ERR_ATTRS_ASI		0x00000080
   1831#define SUN4V_ERR_ATTRS_PRIV_REG	0x00000100
   1832#define SUN4V_ERR_ATTRS_SPSTATE_MSK	0x00000600
   1833#define SUN4V_ERR_ATTRS_MCD		0x00000800
   1834#define SUN4V_ERR_ATTRS_SPSTATE_SHFT	9
   1835#define SUN4V_ERR_ATTRS_MODE_MSK	0x03000000
   1836#define SUN4V_ERR_ATTRS_MODE_SHFT	24
   1837#define SUN4V_ERR_ATTRS_RES_QUEUE_FULL	0x80000000
   1838
   1839#define SUN4V_ERR_SPSTATE_FAULTED	0
   1840#define SUN4V_ERR_SPSTATE_AVAILABLE	1
   1841#define SUN4V_ERR_SPSTATE_NOT_PRESENT	2
   1842
   1843#define SUN4V_ERR_MODE_USER		1
   1844#define SUN4V_ERR_MODE_PRIV		2
   1845
   1846	/* Real address of the memory region or PIO transaction */
   1847/*0x18*/u64		err_raddr;
   1848
   1849	/* Size of the operation triggering the error, in bytes */
   1850/*0x20*/u32		err_size;
   1851
   1852	/* ID of the CPU */
   1853/*0x24*/u16		err_cpu;
   1854
   1855	/* Grace periof for shutdown, in seconds */
   1856/*0x26*/u16		err_secs;
   1857
   1858	/* Value of the %asi register */
   1859/*0x28*/u8		err_asi;
   1860
   1861/*0x29*/u8		reserved_2;
   1862
   1863	/* Value of the ASR register number */
   1864/*0x2a*/u16		err_asr;
   1865#define SUN4V_ERR_ASR_VALID		0x8000
   1866
   1867/*0x2c*/u32		reserved_3;
   1868/*0x30*/u64		reserved_4;
   1869/*0x38*/u64		reserved_5;
   1870};
   1871
   1872static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
   1873static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
   1874
   1875static const char *sun4v_err_type_to_str(u8 type)
   1876{
   1877	static const char *types[SUN4V_ERR_TYPE_NUM] = {
   1878		"undefined",
   1879		"uncorrected resumable",
   1880		"precise nonresumable",
   1881		"deferred nonresumable",
   1882		"shutdown request",
   1883		"dump core",
   1884		"SP state change",
   1885	};
   1886
   1887	if (type < SUN4V_ERR_TYPE_NUM)
   1888		return types[type];
   1889
   1890	return "unknown";
   1891}
   1892
   1893static void sun4v_emit_err_attr_strings(u32 attrs)
   1894{
   1895	static const char *attr_names[] = {
   1896		"processor",
   1897		"memory",
   1898		"PIO",
   1899		"int-registers",
   1900		"fpu-registers",
   1901		"shutdown-request",
   1902		"ASR",
   1903		"ASI",
   1904		"priv-reg",
   1905	};
   1906	static const char *sp_states[] = {
   1907		"sp-faulted",
   1908		"sp-available",
   1909		"sp-not-present",
   1910		"sp-state-reserved",
   1911	};
   1912	static const char *modes[] = {
   1913		"mode-reserved0",
   1914		"user",
   1915		"priv",
   1916		"mode-reserved1",
   1917	};
   1918	u32 sp_state, mode;
   1919	int i;
   1920
   1921	for (i = 0; i < ARRAY_SIZE(attr_names); i++) {
   1922		if (attrs & (1U << i)) {
   1923			const char *s = attr_names[i];
   1924
   1925			pr_cont("%s ", s);
   1926		}
   1927	}
   1928
   1929	sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >>
   1930		    SUN4V_ERR_ATTRS_SPSTATE_SHFT);
   1931	pr_cont("%s ", sp_states[sp_state]);
   1932
   1933	mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >>
   1934		SUN4V_ERR_ATTRS_MODE_SHFT);
   1935	pr_cont("%s ", modes[mode]);
   1936
   1937	if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL)
   1938		pr_cont("res-queue-full ");
   1939}
   1940
   1941/* When the report contains a real-address of "-1" it means that the
   1942 * hardware did not provide the address.  So we compute the effective
   1943 * address of the load or store instruction at regs->tpc and report
   1944 * that.  Usually when this happens it's a PIO and in such a case we
   1945 * are using physical addresses with bypass ASIs anyways, so what we
   1946 * report here is exactly what we want.
   1947 */
   1948static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs)
   1949{
   1950	unsigned int insn;
   1951	u64 addr;
   1952
   1953	if (!(regs->tstate & TSTATE_PRIV))
   1954		return;
   1955
   1956	insn = *(unsigned int *) regs->tpc;
   1957
   1958	addr = compute_effective_address(regs, insn, 0);
   1959
   1960	printk("%s: insn effective address [0x%016llx]\n",
   1961	       pfx, addr);
   1962}
   1963
   1964static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
   1965			    int cpu, const char *pfx, atomic_t *ocnt)
   1966{
   1967	u64 *raw_ptr = (u64 *) ent;
   1968	u32 attrs;
   1969	int cnt;
   1970
   1971	printk("%s: Reporting on cpu %d\n", pfx, cpu);
   1972	printk("%s: TPC [0x%016lx] <%pS>\n",
   1973	       pfx, regs->tpc, (void *) regs->tpc);
   1974
   1975	printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n",
   1976	       pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]);
   1977	printk("%s:      %016llx:%016llx:%016llx:%016llx]\n",
   1978	       pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]);
   1979
   1980	printk("%s: handle [0x%016llx] stick [0x%016llx]\n",
   1981	       pfx, ent->err_handle, ent->err_stick);
   1982
   1983	printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type));
   1984
   1985	attrs = ent->err_attrs;
   1986	printk("%s: attrs [0x%08x] < ", pfx, attrs);
   1987	sun4v_emit_err_attr_strings(attrs);
   1988	pr_cont(">\n");
   1989
   1990	/* Various fields in the error report are only valid if
   1991	 * certain attribute bits are set.
   1992	 */
   1993	if (attrs & (SUN4V_ERR_ATTRS_MEMORY |
   1994		     SUN4V_ERR_ATTRS_PIO |
   1995		     SUN4V_ERR_ATTRS_ASI)) {
   1996		printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr);
   1997
   1998		if (ent->err_raddr == ~(u64)0)
   1999			sun4v_report_real_raddr(pfx, regs);
   2000	}
   2001
   2002	if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI))
   2003		printk("%s: size [0x%x]\n", pfx, ent->err_size);
   2004
   2005	if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR |
   2006		     SUN4V_ERR_ATTRS_INT_REGISTERS |
   2007		     SUN4V_ERR_ATTRS_FPU_REGISTERS |
   2008		     SUN4V_ERR_ATTRS_PRIV_REG))
   2009		printk("%s: cpu[%u]\n", pfx, ent->err_cpu);
   2010
   2011	if (attrs & SUN4V_ERR_ATTRS_ASI)
   2012		printk("%s: asi [0x%02x]\n", pfx, ent->err_asi);
   2013
   2014	if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS |
   2015		      SUN4V_ERR_ATTRS_FPU_REGISTERS |
   2016		      SUN4V_ERR_ATTRS_PRIV_REG)) &&
   2017	    (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0)
   2018		printk("%s: reg [0x%04x]\n",
   2019		       pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID);
   2020
   2021	show_regs(regs);
   2022
   2023	if ((cnt = atomic_read(ocnt)) != 0) {
   2024		atomic_set(ocnt, 0);
   2025		wmb();
   2026		printk("%s: Queue overflowed %d times.\n",
   2027		       pfx, cnt);
   2028	}
   2029}
   2030
   2031/* Handle memory corruption detected error which is vectored in
   2032 * through resumable error trap.
   2033 */
   2034void do_mcd_err(struct pt_regs *regs, struct sun4v_error_entry ent)
   2035{
   2036	if (notify_die(DIE_TRAP, "MCD error", regs, 0, 0x34,
   2037		       SIGSEGV) == NOTIFY_STOP)
   2038		return;
   2039
   2040	if (regs->tstate & TSTATE_PRIV) {
   2041		/* MCD exception could happen because the task was
   2042		 * running a system call with MCD enabled and passed a
   2043		 * non-versioned pointer or pointer with bad version
   2044		 * tag to the system call. In such cases, hypervisor
   2045		 * places the address of offending instruction in the
   2046		 * resumable error report. This is a deferred error,
   2047		 * so the read/write that caused the trap was potentially
   2048		 * retired long time back and we may have no choice
   2049		 * but to send SIGSEGV to the process.
   2050		 */
   2051		const struct exception_table_entry *entry;
   2052
   2053		entry = search_exception_tables(regs->tpc);
   2054		if (entry) {
   2055			/* Looks like a bad syscall parameter */
   2056#ifdef DEBUG_EXCEPTIONS
   2057			pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n",
   2058				 regs->tpc);
   2059			pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
   2060				 ent.err_raddr, entry->fixup);
   2061#endif
   2062			regs->tpc = entry->fixup;
   2063			regs->tnpc = regs->tpc + 4;
   2064			return;
   2065		}
   2066	}
   2067
   2068	/* Send SIGSEGV to the userspace process with the right signal
   2069	 * code
   2070	 */
   2071	force_sig_fault(SIGSEGV, SEGV_ADIDERR, (void __user *)ent.err_raddr);
   2072}
   2073
   2074/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
   2075 * Log the event and clear the first word of the entry.
   2076 */
   2077void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
   2078{
   2079	enum ctx_state prev_state = exception_enter();
   2080	struct sun4v_error_entry *ent, local_copy;
   2081	struct trap_per_cpu *tb;
   2082	unsigned long paddr;
   2083	int cpu;
   2084
   2085	cpu = get_cpu();
   2086
   2087	tb = &trap_block[cpu];
   2088	paddr = tb->resum_kernel_buf_pa + offset;
   2089	ent = __va(paddr);
   2090
   2091	memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
   2092
   2093	/* We have a local copy now, so release the entry.  */
   2094	ent->err_handle = 0;
   2095	wmb();
   2096
   2097	put_cpu();
   2098
   2099	if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) {
   2100		/* We should really take the seconds field of
   2101		 * the error report and use it for the shutdown
   2102		 * invocation, but for now do the same thing we
   2103		 * do for a DS shutdown request.
   2104		 */
   2105		pr_info("Shutdown request, %u seconds...\n",
   2106			local_copy.err_secs);
   2107		orderly_poweroff(true);
   2108		goto out;
   2109	}
   2110
   2111	/* If this is a memory corruption detected error vectored in
   2112	 * by HV through resumable error trap, call the handler
   2113	 */
   2114	if (local_copy.err_attrs & SUN4V_ERR_ATTRS_MCD) {
   2115		do_mcd_err(regs, local_copy);
   2116		return;
   2117	}
   2118
   2119	sun4v_log_error(regs, &local_copy, cpu,
   2120			KERN_ERR "RESUMABLE ERROR",
   2121			&sun4v_resum_oflow_cnt);
   2122out:
   2123	exception_exit(prev_state);
   2124}
   2125
   2126/* If we try to printk() we'll probably make matters worse, by trying
   2127 * to retake locks this cpu already holds or causing more errors. So
   2128 * just bump a counter, and we'll report these counter bumps above.
   2129 */
   2130void sun4v_resum_overflow(struct pt_regs *regs)
   2131{
   2132	atomic_inc(&sun4v_resum_oflow_cnt);
   2133}
   2134
   2135/* Given a set of registers, get the virtual addressi that was being accessed
   2136 * by the faulting instructions at tpc.
   2137 */
   2138static unsigned long sun4v_get_vaddr(struct pt_regs *regs)
   2139{
   2140	unsigned int insn;
   2141
   2142	if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) {
   2143		return compute_effective_address(regs, insn,
   2144						 (insn >> 25) & 0x1f);
   2145	}
   2146	return 0;
   2147}
   2148
   2149/* Attempt to handle non-resumable errors generated from userspace.
   2150 * Returns true if the signal was handled, false otherwise.
   2151 */
   2152bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
   2153				  struct sun4v_error_entry *ent) {
   2154
   2155	unsigned int attrs = ent->err_attrs;
   2156
   2157	if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
   2158		unsigned long addr = ent->err_raddr;
   2159
   2160		if (addr == ~(u64)0) {
   2161			/* This seems highly unlikely to ever occur */
   2162			pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n");
   2163		} else {
   2164			unsigned long page_cnt = DIV_ROUND_UP(ent->err_size,
   2165							      PAGE_SIZE);
   2166
   2167			/* Break the unfortunate news. */
   2168			pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n",
   2169				 addr);
   2170			pr_emerg("SUN4V NON-RECOVERABLE ERROR:   Claiming %lu ages.\n",
   2171				 page_cnt);
   2172
   2173			while (page_cnt-- > 0) {
   2174				if (pfn_valid(addr >> PAGE_SHIFT))
   2175					get_page(pfn_to_page(addr >> PAGE_SHIFT));
   2176				addr += PAGE_SIZE;
   2177			}
   2178		}
   2179		force_sig(SIGKILL);
   2180
   2181		return true;
   2182	}
   2183	if (attrs & SUN4V_ERR_ATTRS_PIO) {
   2184		force_sig_fault(SIGBUS, BUS_ADRERR,
   2185				(void __user *)sun4v_get_vaddr(regs));
   2186		return true;
   2187	}
   2188
   2189	/* Default to doing nothing */
   2190	return false;
   2191}
   2192
   2193/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
   2194 * Log the event, clear the first word of the entry, and die.
   2195 */
   2196void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
   2197{
   2198	struct sun4v_error_entry *ent, local_copy;
   2199	struct trap_per_cpu *tb;
   2200	unsigned long paddr;
   2201	int cpu;
   2202
   2203	cpu = get_cpu();
   2204
   2205	tb = &trap_block[cpu];
   2206	paddr = tb->nonresum_kernel_buf_pa + offset;
   2207	ent = __va(paddr);
   2208
   2209	memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
   2210
   2211	/* We have a local copy now, so release the entry.  */
   2212	ent->err_handle = 0;
   2213	wmb();
   2214
   2215	put_cpu();
   2216
   2217	if (!(regs->tstate & TSTATE_PRIV) &&
   2218	    sun4v_nonresum_error_user_handled(regs, &local_copy)) {
   2219		/* DON'T PANIC: This userspace error was handled. */
   2220		return;
   2221	}
   2222
   2223#ifdef CONFIG_PCI
   2224	/* Check for the special PCI poke sequence. */
   2225	if (pci_poke_in_progress && pci_poke_cpu == cpu) {
   2226		pci_poke_faulted = 1;
   2227		regs->tpc += 4;
   2228		regs->tnpc = regs->tpc + 4;
   2229		return;
   2230	}
   2231#endif
   2232
   2233	sun4v_log_error(regs, &local_copy, cpu,
   2234			KERN_EMERG "NON-RESUMABLE ERROR",
   2235			&sun4v_nonresum_oflow_cnt);
   2236
   2237	panic("Non-resumable error.");
   2238}
   2239
   2240/* If we try to printk() we'll probably make matters worse, by trying
   2241 * to retake locks this cpu already holds or causing more errors. So
   2242 * just bump a counter, and we'll report these counter bumps above.
   2243 */
   2244void sun4v_nonresum_overflow(struct pt_regs *regs)
   2245{
   2246	/* XXX Actually even this can make not that much sense.  Perhaps
   2247	 * XXX we should just pull the plug and panic directly from here?
   2248	 */
   2249	atomic_inc(&sun4v_nonresum_oflow_cnt);
   2250}
   2251
   2252static void sun4v_tlb_error(struct pt_regs *regs)
   2253{
   2254	die_if_kernel("TLB/TSB error", regs);
   2255}
   2256
   2257unsigned long sun4v_err_itlb_vaddr;
   2258unsigned long sun4v_err_itlb_ctx;
   2259unsigned long sun4v_err_itlb_pte;
   2260unsigned long sun4v_err_itlb_error;
   2261
   2262void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
   2263{
   2264	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2265
   2266	printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
   2267	       regs->tpc, tl);
   2268	printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
   2269	printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
   2270	printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
   2271	       (void *) regs->u_regs[UREG_I7]);
   2272	printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
   2273	       "pte[%lx] error[%lx]\n",
   2274	       sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
   2275	       sun4v_err_itlb_pte, sun4v_err_itlb_error);
   2276
   2277	sun4v_tlb_error(regs);
   2278}
   2279
   2280unsigned long sun4v_err_dtlb_vaddr;
   2281unsigned long sun4v_err_dtlb_ctx;
   2282unsigned long sun4v_err_dtlb_pte;
   2283unsigned long sun4v_err_dtlb_error;
   2284
   2285void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
   2286{
   2287	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2288
   2289	printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
   2290	       regs->tpc, tl);
   2291	printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
   2292	printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
   2293	printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
   2294	       (void *) regs->u_regs[UREG_I7]);
   2295	printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
   2296	       "pte[%lx] error[%lx]\n",
   2297	       sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
   2298	       sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
   2299
   2300	sun4v_tlb_error(regs);
   2301}
   2302
   2303void hypervisor_tlbop_error(unsigned long err, unsigned long op)
   2304{
   2305	printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
   2306	       err, op);
   2307}
   2308
   2309void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
   2310{
   2311	printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
   2312	       err, op);
   2313}
   2314
   2315static void do_fpe_common(struct pt_regs *regs)
   2316{
   2317	if (regs->tstate & TSTATE_PRIV) {
   2318		regs->tpc = regs->tnpc;
   2319		regs->tnpc += 4;
   2320	} else {
   2321		unsigned long fsr = current_thread_info()->xfsr[0];
   2322		int code;
   2323
   2324		if (test_thread_flag(TIF_32BIT)) {
   2325			regs->tpc &= 0xffffffff;
   2326			regs->tnpc &= 0xffffffff;
   2327		}
   2328		code = FPE_FLTUNK;
   2329		if ((fsr & 0x1c000) == (1 << 14)) {
   2330			if (fsr & 0x10)
   2331				code = FPE_FLTINV;
   2332			else if (fsr & 0x08)
   2333				code = FPE_FLTOVF;
   2334			else if (fsr & 0x04)
   2335				code = FPE_FLTUND;
   2336			else if (fsr & 0x02)
   2337				code = FPE_FLTDIV;
   2338			else if (fsr & 0x01)
   2339				code = FPE_FLTRES;
   2340		}
   2341		force_sig_fault(SIGFPE, code, (void __user *)regs->tpc);
   2342	}
   2343}
   2344
   2345void do_fpieee(struct pt_regs *regs)
   2346{
   2347	enum ctx_state prev_state = exception_enter();
   2348
   2349	if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
   2350		       0, 0x24, SIGFPE) == NOTIFY_STOP)
   2351		goto out;
   2352
   2353	do_fpe_common(regs);
   2354out:
   2355	exception_exit(prev_state);
   2356}
   2357
   2358void do_fpother(struct pt_regs *regs)
   2359{
   2360	enum ctx_state prev_state = exception_enter();
   2361	struct fpustate *f = FPUSTATE;
   2362	int ret = 0;
   2363
   2364	if (notify_die(DIE_TRAP, "fpu exception other", regs,
   2365		       0, 0x25, SIGFPE) == NOTIFY_STOP)
   2366		goto out;
   2367
   2368	switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
   2369	case (2 << 14): /* unfinished_FPop */
   2370	case (3 << 14): /* unimplemented_FPop */
   2371		ret = do_mathemu(regs, f, false);
   2372		break;
   2373	}
   2374	if (ret)
   2375		goto out;
   2376	do_fpe_common(regs);
   2377out:
   2378	exception_exit(prev_state);
   2379}
   2380
   2381void do_tof(struct pt_regs *regs)
   2382{
   2383	enum ctx_state prev_state = exception_enter();
   2384
   2385	if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
   2386		       0, 0x26, SIGEMT) == NOTIFY_STOP)
   2387		goto out;
   2388
   2389	if (regs->tstate & TSTATE_PRIV)
   2390		die_if_kernel("Penguin overflow trap from kernel mode", regs);
   2391	if (test_thread_flag(TIF_32BIT)) {
   2392		regs->tpc &= 0xffffffff;
   2393		regs->tnpc &= 0xffffffff;
   2394	}
   2395	force_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)regs->tpc);
   2396out:
   2397	exception_exit(prev_state);
   2398}
   2399
   2400void do_div0(struct pt_regs *regs)
   2401{
   2402	enum ctx_state prev_state = exception_enter();
   2403
   2404	if (notify_die(DIE_TRAP, "integer division by zero", regs,
   2405		       0, 0x28, SIGFPE) == NOTIFY_STOP)
   2406		goto out;
   2407
   2408	if (regs->tstate & TSTATE_PRIV)
   2409		die_if_kernel("TL0: Kernel divide by zero.", regs);
   2410	if (test_thread_flag(TIF_32BIT)) {
   2411		regs->tpc &= 0xffffffff;
   2412		regs->tnpc &= 0xffffffff;
   2413	}
   2414	force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->tpc);
   2415out:
   2416	exception_exit(prev_state);
   2417}
   2418
   2419static void instruction_dump(unsigned int *pc)
   2420{
   2421	int i;
   2422
   2423	if ((((unsigned long) pc) & 3))
   2424		return;
   2425
   2426	printk("Instruction DUMP:");
   2427	for (i = -3; i < 6; i++)
   2428		printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
   2429	printk("\n");
   2430}
   2431
   2432static void user_instruction_dump(unsigned int __user *pc)
   2433{
   2434	int i;
   2435	unsigned int buf[9];
   2436	
   2437	if ((((unsigned long) pc) & 3))
   2438		return;
   2439		
   2440	if (copy_from_user(buf, pc - 3, sizeof(buf)))
   2441		return;
   2442
   2443	printk("Instruction DUMP:");
   2444	for (i = 0; i < 9; i++)
   2445		printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
   2446	printk("\n");
   2447}
   2448
   2449void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl)
   2450{
   2451	unsigned long fp, ksp;
   2452	struct thread_info *tp;
   2453	int count = 0;
   2454#ifdef CONFIG_FUNCTION_GRAPH_TRACER
   2455	int graph = 0;
   2456#endif
   2457
   2458	ksp = (unsigned long) _ksp;
   2459	if (!tsk)
   2460		tsk = current;
   2461	tp = task_thread_info(tsk);
   2462	if (ksp == 0UL) {
   2463		if (tsk == current)
   2464			asm("mov %%fp, %0" : "=r" (ksp));
   2465		else
   2466			ksp = tp->ksp;
   2467	}
   2468	if (tp == current_thread_info())
   2469		flushw_all();
   2470
   2471	fp = ksp + STACK_BIAS;
   2472
   2473	printk("%sCall Trace:\n", loglvl);
   2474	do {
   2475		struct sparc_stackf *sf;
   2476		struct pt_regs *regs;
   2477		unsigned long pc;
   2478
   2479		if (!kstack_valid(tp, fp))
   2480			break;
   2481		sf = (struct sparc_stackf *) fp;
   2482		regs = (struct pt_regs *) (sf + 1);
   2483
   2484		if (kstack_is_trap_frame(tp, regs)) {
   2485			if (!(regs->tstate & TSTATE_PRIV))
   2486				break;
   2487			pc = regs->tpc;
   2488			fp = regs->u_regs[UREG_I6] + STACK_BIAS;
   2489		} else {
   2490			pc = sf->callers_pc;
   2491			fp = (unsigned long)sf->fp + STACK_BIAS;
   2492		}
   2493
   2494		print_ip_sym(loglvl, pc);
   2495#ifdef CONFIG_FUNCTION_GRAPH_TRACER
   2496		if ((pc + 8UL) == (unsigned long) &return_to_handler) {
   2497			struct ftrace_ret_stack *ret_stack;
   2498			ret_stack = ftrace_graph_get_ret_stack(tsk, graph);
   2499			if (ret_stack) {
   2500				pc = ret_stack->ret;
   2501				print_ip_sym(loglvl, pc);
   2502				graph++;
   2503			}
   2504		}
   2505#endif
   2506	} while (++count < 16);
   2507}
   2508
   2509static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
   2510{
   2511	unsigned long fp = rw->ins[6];
   2512
   2513	if (!fp)
   2514		return NULL;
   2515
   2516	return (struct reg_window *) (fp + STACK_BIAS);
   2517}
   2518
   2519void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
   2520{
   2521	static int die_counter;
   2522	int count = 0;
   2523	
   2524	/* Amuse the user. */
   2525	printk(
   2526"              \\|/ ____ \\|/\n"
   2527"              \"@'/ .. \\`@\"\n"
   2528"              /_| \\__/ |_\\\n"
   2529"                 \\__U_/\n");
   2530
   2531	printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter);
   2532	notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
   2533	__asm__ __volatile__("flushw");
   2534	show_regs(regs);
   2535	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
   2536	if (regs->tstate & TSTATE_PRIV) {
   2537		struct thread_info *tp = current_thread_info();
   2538		struct reg_window *rw = (struct reg_window *)
   2539			(regs->u_regs[UREG_FP] + STACK_BIAS);
   2540
   2541		/* Stop the back trace when we hit userland or we
   2542		 * find some badly aligned kernel stack.
   2543		 */
   2544		while (rw &&
   2545		       count++ < 30 &&
   2546		       kstack_valid(tp, (unsigned long) rw)) {
   2547			printk("Caller[%016lx]: %pS\n", rw->ins[7],
   2548			       (void *) rw->ins[7]);
   2549
   2550			rw = kernel_stack_up(rw);
   2551		}
   2552		instruction_dump ((unsigned int *) regs->tpc);
   2553	} else {
   2554		if (test_thread_flag(TIF_32BIT)) {
   2555			regs->tpc &= 0xffffffff;
   2556			regs->tnpc &= 0xffffffff;
   2557		}
   2558		user_instruction_dump ((unsigned int __user *) regs->tpc);
   2559	}
   2560	if (panic_on_oops)
   2561		panic("Fatal exception");
   2562	make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV);
   2563}
   2564EXPORT_SYMBOL(die_if_kernel);
   2565
   2566#define VIS_OPCODE_MASK	((0x3 << 30) | (0x3f << 19))
   2567#define VIS_OPCODE_VAL	((0x2 << 30) | (0x36 << 19))
   2568
   2569void do_illegal_instruction(struct pt_regs *regs)
   2570{
   2571	enum ctx_state prev_state = exception_enter();
   2572	unsigned long pc = regs->tpc;
   2573	unsigned long tstate = regs->tstate;
   2574	u32 insn;
   2575
   2576	if (notify_die(DIE_TRAP, "illegal instruction", regs,
   2577		       0, 0x10, SIGILL) == NOTIFY_STOP)
   2578		goto out;
   2579
   2580	if (tstate & TSTATE_PRIV)
   2581		die_if_kernel("Kernel illegal instruction", regs);
   2582	if (test_thread_flag(TIF_32BIT))
   2583		pc = (u32)pc;
   2584	if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
   2585		if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
   2586			if (handle_popc(insn, regs))
   2587				goto out;
   2588		} else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
   2589			if (handle_ldf_stq(insn, regs))
   2590				goto out;
   2591		} else if (tlb_type == hypervisor) {
   2592			if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
   2593				if (!vis_emul(regs, insn))
   2594					goto out;
   2595			} else {
   2596				struct fpustate *f = FPUSTATE;
   2597
   2598				/* On UltraSPARC T2 and later, FPU insns which
   2599				 * are not implemented in HW signal an illegal
   2600				 * instruction trap and do not set the FP Trap
   2601				 * Trap in the %fsr to unimplemented_FPop.
   2602				 */
   2603				if (do_mathemu(regs, f, true))
   2604					goto out;
   2605			}
   2606		}
   2607	}
   2608	force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc);
   2609out:
   2610	exception_exit(prev_state);
   2611}
   2612
   2613void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
   2614{
   2615	enum ctx_state prev_state = exception_enter();
   2616
   2617	if (notify_die(DIE_TRAP, "memory address unaligned", regs,
   2618		       0, 0x34, SIGSEGV) == NOTIFY_STOP)
   2619		goto out;
   2620
   2621	if (regs->tstate & TSTATE_PRIV) {
   2622		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
   2623		goto out;
   2624	}
   2625	if (is_no_fault_exception(regs))
   2626		return;
   2627
   2628	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)sfar);
   2629out:
   2630	exception_exit(prev_state);
   2631}
   2632
   2633void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
   2634{
   2635	if (notify_die(DIE_TRAP, "memory address unaligned", regs,
   2636		       0, 0x34, SIGSEGV) == NOTIFY_STOP)
   2637		return;
   2638
   2639	if (regs->tstate & TSTATE_PRIV) {
   2640		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
   2641		return;
   2642	}
   2643	if (is_no_fault_exception(regs))
   2644		return;
   2645
   2646	force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) addr);
   2647}
   2648
   2649/* sun4v_mem_corrupt_detect_precise() - Handle precise exception on an ADI
   2650 * tag mismatch.
   2651 *
   2652 * ADI version tag mismatch on a load from memory always results in a
   2653 * precise exception. Tag mismatch on a store to memory will result in
   2654 * precise exception if MCDPER or PMCDPER is set to 1.
   2655 */
   2656void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, unsigned long addr,
   2657				      unsigned long context)
   2658{
   2659	if (notify_die(DIE_TRAP, "memory corruption precise exception", regs,
   2660		       0, 0x8, SIGSEGV) == NOTIFY_STOP)
   2661		return;
   2662
   2663	if (regs->tstate & TSTATE_PRIV) {
   2664		/* MCD exception could happen because the task was running
   2665		 * a system call with MCD enabled and passed a non-versioned
   2666		 * pointer or pointer with bad version tag to  the system
   2667		 * call.
   2668		 */
   2669		const struct exception_table_entry *entry;
   2670
   2671		entry = search_exception_tables(regs->tpc);
   2672		if (entry) {
   2673			/* Looks like a bad syscall parameter */
   2674#ifdef DEBUG_EXCEPTIONS
   2675			pr_emerg("Exception: PC<%016lx> faddr<UNKNOWN>\n",
   2676				 regs->tpc);
   2677			pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
   2678				 regs->tpc, entry->fixup);
   2679#endif
   2680			regs->tpc = entry->fixup;
   2681			regs->tnpc = regs->tpc + 4;
   2682			return;
   2683		}
   2684		pr_emerg("%s: ADDR[%016lx] CTX[%lx], going.\n",
   2685			 __func__, addr, context);
   2686		die_if_kernel("MCD precise", regs);
   2687	}
   2688
   2689	if (test_thread_flag(TIF_32BIT)) {
   2690		regs->tpc &= 0xffffffff;
   2691		regs->tnpc &= 0xffffffff;
   2692	}
   2693	force_sig_fault(SIGSEGV, SEGV_ADIPERR, (void __user *)addr);
   2694}
   2695
   2696void do_privop(struct pt_regs *regs)
   2697{
   2698	enum ctx_state prev_state = exception_enter();
   2699
   2700	if (notify_die(DIE_TRAP, "privileged operation", regs,
   2701		       0, 0x11, SIGILL) == NOTIFY_STOP)
   2702		goto out;
   2703
   2704	if (test_thread_flag(TIF_32BIT)) {
   2705		regs->tpc &= 0xffffffff;
   2706		regs->tnpc &= 0xffffffff;
   2707	}
   2708	force_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)regs->tpc);
   2709out:
   2710	exception_exit(prev_state);
   2711}
   2712
   2713void do_privact(struct pt_regs *regs)
   2714{
   2715	do_privop(regs);
   2716}
   2717
   2718/* Trap level 1 stuff or other traps we should never see... */
   2719void do_cee(struct pt_regs *regs)
   2720{
   2721	exception_enter();
   2722	die_if_kernel("TL0: Cache Error Exception", regs);
   2723}
   2724
   2725void do_div0_tl1(struct pt_regs *regs)
   2726{
   2727	exception_enter();
   2728	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2729	die_if_kernel("TL1: DIV0 Exception", regs);
   2730}
   2731
   2732void do_fpieee_tl1(struct pt_regs *regs)
   2733{
   2734	exception_enter();
   2735	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2736	die_if_kernel("TL1: FPU IEEE Exception", regs);
   2737}
   2738
   2739void do_fpother_tl1(struct pt_regs *regs)
   2740{
   2741	exception_enter();
   2742	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2743	die_if_kernel("TL1: FPU Other Exception", regs);
   2744}
   2745
   2746void do_ill_tl1(struct pt_regs *regs)
   2747{
   2748	exception_enter();
   2749	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2750	die_if_kernel("TL1: Illegal Instruction Exception", regs);
   2751}
   2752
   2753void do_irq_tl1(struct pt_regs *regs)
   2754{
   2755	exception_enter();
   2756	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2757	die_if_kernel("TL1: IRQ Exception", regs);
   2758}
   2759
   2760void do_lddfmna_tl1(struct pt_regs *regs)
   2761{
   2762	exception_enter();
   2763	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2764	die_if_kernel("TL1: LDDF Exception", regs);
   2765}
   2766
   2767void do_stdfmna_tl1(struct pt_regs *regs)
   2768{
   2769	exception_enter();
   2770	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2771	die_if_kernel("TL1: STDF Exception", regs);
   2772}
   2773
   2774void do_paw(struct pt_regs *regs)
   2775{
   2776	exception_enter();
   2777	die_if_kernel("TL0: Phys Watchpoint Exception", regs);
   2778}
   2779
   2780void do_paw_tl1(struct pt_regs *regs)
   2781{
   2782	exception_enter();
   2783	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2784	die_if_kernel("TL1: Phys Watchpoint Exception", regs);
   2785}
   2786
   2787void do_vaw(struct pt_regs *regs)
   2788{
   2789	exception_enter();
   2790	die_if_kernel("TL0: Virt Watchpoint Exception", regs);
   2791}
   2792
   2793void do_vaw_tl1(struct pt_regs *regs)
   2794{
   2795	exception_enter();
   2796	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2797	die_if_kernel("TL1: Virt Watchpoint Exception", regs);
   2798}
   2799
   2800void do_tof_tl1(struct pt_regs *regs)
   2801{
   2802	exception_enter();
   2803	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
   2804	die_if_kernel("TL1: Tag Overflow Exception", regs);
   2805}
   2806
   2807void do_getpsr(struct pt_regs *regs)
   2808{
   2809	regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
   2810	regs->tpc   = regs->tnpc;
   2811	regs->tnpc += 4;
   2812	if (test_thread_flag(TIF_32BIT)) {
   2813		regs->tpc &= 0xffffffff;
   2814		regs->tnpc &= 0xffffffff;
   2815	}
   2816}
   2817
   2818u64 cpu_mondo_counter[NR_CPUS] = {0};
   2819struct trap_per_cpu trap_block[NR_CPUS];
   2820EXPORT_SYMBOL(trap_block);
   2821
   2822/* This can get invoked before sched_init() so play it super safe
   2823 * and use hard_smp_processor_id().
   2824 */
   2825void notrace init_cur_cpu_trap(struct thread_info *t)
   2826{
   2827	int cpu = hard_smp_processor_id();
   2828	struct trap_per_cpu *p = &trap_block[cpu];
   2829
   2830	p->thread = t;
   2831	p->pgd_paddr = 0;
   2832}
   2833
   2834extern void thread_info_offsets_are_bolixed_dave(void);
   2835extern void trap_per_cpu_offsets_are_bolixed_dave(void);
   2836extern void tsb_config_offsets_are_bolixed_dave(void);
   2837
   2838/* Only invoked on boot processor. */
   2839void __init trap_init(void)
   2840{
   2841	/* Compile time sanity check. */
   2842	BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) ||
   2843		     TI_FLAGS != offsetof(struct thread_info, flags) ||
   2844		     TI_CPU != offsetof(struct thread_info, cpu) ||
   2845		     TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
   2846		     TI_KSP != offsetof(struct thread_info, ksp) ||
   2847		     TI_FAULT_ADDR != offsetof(struct thread_info,
   2848					       fault_address) ||
   2849		     TI_KREGS != offsetof(struct thread_info, kregs) ||
   2850		     TI_UTRAPS != offsetof(struct thread_info, utraps) ||
   2851		     TI_REG_WINDOW != offsetof(struct thread_info,
   2852					       reg_window) ||
   2853		     TI_RWIN_SPTRS != offsetof(struct thread_info,
   2854					       rwbuf_stkptrs) ||
   2855		     TI_GSR != offsetof(struct thread_info, gsr) ||
   2856		     TI_XFSR != offsetof(struct thread_info, xfsr) ||
   2857		     TI_PRE_COUNT != offsetof(struct thread_info,
   2858					      preempt_count) ||
   2859		     TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
   2860		     TI_KUNA_REGS != offsetof(struct thread_info,
   2861					      kern_una_regs) ||
   2862		     TI_KUNA_INSN != offsetof(struct thread_info,
   2863					      kern_una_insn) ||
   2864		     TI_FPREGS != offsetof(struct thread_info, fpregs) ||
   2865		     (TI_FPREGS & (64 - 1)));
   2866
   2867	BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu,
   2868						     thread) ||
   2869		     (TRAP_PER_CPU_PGD_PADDR !=
   2870		      offsetof(struct trap_per_cpu, pgd_paddr)) ||
   2871		     (TRAP_PER_CPU_CPU_MONDO_PA !=
   2872		      offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
   2873		     (TRAP_PER_CPU_DEV_MONDO_PA !=
   2874		      offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
   2875		     (TRAP_PER_CPU_RESUM_MONDO_PA !=
   2876		      offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
   2877		     (TRAP_PER_CPU_RESUM_KBUF_PA !=
   2878		      offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
   2879		     (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
   2880		      offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
   2881		     (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
   2882		      offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
   2883		     (TRAP_PER_CPU_FAULT_INFO !=
   2884		      offsetof(struct trap_per_cpu, fault_info)) ||
   2885		     (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
   2886		      offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
   2887		     (TRAP_PER_CPU_CPU_LIST_PA !=
   2888		      offsetof(struct trap_per_cpu, cpu_list_pa)) ||
   2889		     (TRAP_PER_CPU_TSB_HUGE !=
   2890		      offsetof(struct trap_per_cpu, tsb_huge)) ||
   2891		     (TRAP_PER_CPU_TSB_HUGE_TEMP !=
   2892		      offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
   2893		     (TRAP_PER_CPU_IRQ_WORKLIST_PA !=
   2894		      offsetof(struct trap_per_cpu, irq_worklist_pa)) ||
   2895		     (TRAP_PER_CPU_CPU_MONDO_QMASK !=
   2896		      offsetof(struct trap_per_cpu, cpu_mondo_qmask)) ||
   2897		     (TRAP_PER_CPU_DEV_MONDO_QMASK !=
   2898		      offsetof(struct trap_per_cpu, dev_mondo_qmask)) ||
   2899		     (TRAP_PER_CPU_RESUM_QMASK !=
   2900		      offsetof(struct trap_per_cpu, resum_qmask)) ||
   2901		     (TRAP_PER_CPU_NONRESUM_QMASK !=
   2902		      offsetof(struct trap_per_cpu, nonresum_qmask)) ||
   2903		     (TRAP_PER_CPU_PER_CPU_BASE !=
   2904		      offsetof(struct trap_per_cpu, __per_cpu_base)));
   2905
   2906	BUILD_BUG_ON((TSB_CONFIG_TSB !=
   2907		      offsetof(struct tsb_config, tsb)) ||
   2908		     (TSB_CONFIG_RSS_LIMIT !=
   2909		      offsetof(struct tsb_config, tsb_rss_limit)) ||
   2910		     (TSB_CONFIG_NENTRIES !=
   2911		      offsetof(struct tsb_config, tsb_nentries)) ||
   2912		     (TSB_CONFIG_REG_VAL !=
   2913		      offsetof(struct tsb_config, tsb_reg_val)) ||
   2914		     (TSB_CONFIG_MAP_VADDR !=
   2915		      offsetof(struct tsb_config, tsb_map_vaddr)) ||
   2916		     (TSB_CONFIG_MAP_PTE !=
   2917		      offsetof(struct tsb_config, tsb_map_pte)));
   2918
   2919	/* Attach to the address space of init_task.  On SMP we
   2920	 * do this in smp.c:smp_callin for other cpus.
   2921	 */
   2922	mmgrab(&init_mm);
   2923	current->active_mm = &init_mm;
   2924}