cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

cherrs.S (15258B)


      1/* SPDX-License-Identifier: GPL-2.0 */
      2	/* These get patched into the trap table at boot time
      3	 * once we know we have a cheetah processor.
      4	 */
      5	.globl		cheetah_fecc_trap_vector
      6	.type		cheetah_fecc_trap_vector,#function
      7cheetah_fecc_trap_vector:
      8	membar		#Sync
      9	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1
     10	andn		%g1, DCU_DC | DCU_IC, %g1
     11	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG
     12	membar		#Sync
     13	sethi		%hi(cheetah_fast_ecc), %g2
     14	jmpl		%g2 + %lo(cheetah_fast_ecc), %g0
     15	 mov		0, %g1
     16	.size		cheetah_fecc_trap_vector,.-cheetah_fecc_trap_vector
     17
     18	.globl		cheetah_fecc_trap_vector_tl1
     19	.type		cheetah_fecc_trap_vector_tl1,#function
     20cheetah_fecc_trap_vector_tl1:
     21	membar		#Sync
     22	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1
     23	andn		%g1, DCU_DC | DCU_IC, %g1
     24	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG
     25	membar		#Sync
     26	sethi		%hi(cheetah_fast_ecc), %g2
     27	jmpl		%g2 + %lo(cheetah_fast_ecc), %g0
     28	 mov		1, %g1
     29	.size		cheetah_fecc_trap_vector_tl1,.-cheetah_fecc_trap_vector_tl1
     30
     31	.globl	cheetah_cee_trap_vector
     32	.type	cheetah_cee_trap_vector,#function
     33cheetah_cee_trap_vector:
     34	membar		#Sync
     35	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1
     36	andn		%g1, DCU_IC, %g1
     37	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG
     38	membar		#Sync
     39	sethi		%hi(cheetah_cee), %g2
     40	jmpl		%g2 + %lo(cheetah_cee), %g0
     41	 mov		0, %g1
     42	.size		cheetah_cee_trap_vector,.-cheetah_cee_trap_vector
     43
     44	.globl		cheetah_cee_trap_vector_tl1
     45	.type		cheetah_cee_trap_vector_tl1,#function
     46cheetah_cee_trap_vector_tl1:
     47	membar		#Sync
     48	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1
     49	andn		%g1, DCU_IC, %g1
     50	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG
     51	membar		#Sync
     52	sethi		%hi(cheetah_cee), %g2
     53	jmpl		%g2 + %lo(cheetah_cee), %g0
     54	 mov		1, %g1
     55	.size		cheetah_cee_trap_vector_tl1,.-cheetah_cee_trap_vector_tl1
     56
     57	.globl	cheetah_deferred_trap_vector
     58	.type	cheetah_deferred_trap_vector,#function
     59cheetah_deferred_trap_vector:
     60	membar		#Sync
     61	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1;
     62	andn		%g1, DCU_DC | DCU_IC, %g1;
     63	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG;
     64	membar		#Sync;
     65	sethi		%hi(cheetah_deferred_trap), %g2
     66	jmpl		%g2 + %lo(cheetah_deferred_trap), %g0
     67	 mov		0, %g1
     68	.size		cheetah_deferred_trap_vector,.-cheetah_deferred_trap_vector
     69
     70	.globl		cheetah_deferred_trap_vector_tl1
     71	.type		cheetah_deferred_trap_vector_tl1,#function
     72cheetah_deferred_trap_vector_tl1:
     73	membar		#Sync;
     74	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1;
     75	andn		%g1, DCU_DC | DCU_IC, %g1;
     76	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG;
     77	membar		#Sync;
     78	sethi		%hi(cheetah_deferred_trap), %g2
     79	jmpl		%g2 + %lo(cheetah_deferred_trap), %g0
     80	 mov		1, %g1
     81	.size		cheetah_deferred_trap_vector_tl1,.-cheetah_deferred_trap_vector_tl1
     82
     83	/* Cheetah+ specific traps. These are for the new I/D cache parity
     84	 * error traps.  The first argument to cheetah_plus_parity_handler
     85	 * is encoded as follows:
     86	 *
     87	 * Bit0:	0=dcache,1=icache
     88	 * Bit1:	0=recoverable,1=unrecoverable
     89	 */
     90	.globl		cheetah_plus_dcpe_trap_vector
     91	.type		cheetah_plus_dcpe_trap_vector,#function
     92cheetah_plus_dcpe_trap_vector:
     93	membar		#Sync
     94	sethi		%hi(do_cheetah_plus_data_parity), %g7
     95	jmpl		%g7 + %lo(do_cheetah_plus_data_parity), %g0
     96	 nop
     97	nop
     98	nop
     99	nop
    100	nop
    101	.size		cheetah_plus_dcpe_trap_vector,.-cheetah_plus_dcpe_trap_vector
    102
    103	.type		do_cheetah_plus_data_parity,#function
    104do_cheetah_plus_data_parity:
    105	rdpr		%pil, %g2
    106	wrpr		%g0, PIL_NORMAL_MAX, %pil
    107	ba,pt		%xcc, etrap_irq
    108	 rd		%pc, %g7
    109#ifdef CONFIG_TRACE_IRQFLAGS
    110	call		trace_hardirqs_off
    111	 nop
    112#endif
    113	mov		0x0, %o0
    114	call		cheetah_plus_parity_error
    115	 add		%sp, PTREGS_OFF, %o1
    116	ba,a,pt		%xcc, rtrap_irq
    117	.size		do_cheetah_plus_data_parity,.-do_cheetah_plus_data_parity
    118
    119	.globl		cheetah_plus_dcpe_trap_vector_tl1
    120	.type		cheetah_plus_dcpe_trap_vector_tl1,#function
    121cheetah_plus_dcpe_trap_vector_tl1:
    122	membar		#Sync
    123	wrpr		PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
    124	sethi		%hi(do_dcpe_tl1), %g3
    125	jmpl		%g3 + %lo(do_dcpe_tl1), %g0
    126	 nop
    127	nop
    128	nop
    129	nop
    130	.size		cheetah_plus_dcpe_trap_vector_tl1,.-cheetah_plus_dcpe_trap_vector_tl1
    131
    132	.globl		cheetah_plus_icpe_trap_vector
    133	.type		cheetah_plus_icpe_trap_vector,#function
    134cheetah_plus_icpe_trap_vector:
    135	membar		#Sync
    136	sethi		%hi(do_cheetah_plus_insn_parity), %g7
    137	jmpl		%g7 + %lo(do_cheetah_plus_insn_parity), %g0
    138	 nop
    139	nop
    140	nop
    141	nop
    142	nop
    143	.size		cheetah_plus_icpe_trap_vector,.-cheetah_plus_icpe_trap_vector
    144
    145	.type		do_cheetah_plus_insn_parity,#function
    146do_cheetah_plus_insn_parity:
    147	rdpr		%pil, %g2
    148	wrpr		%g0, PIL_NORMAL_MAX, %pil
    149	ba,pt		%xcc, etrap_irq
    150	 rd		%pc, %g7
    151#ifdef CONFIG_TRACE_IRQFLAGS
    152	call		trace_hardirqs_off
    153	 nop
    154#endif
    155	mov		0x1, %o0
    156	call		cheetah_plus_parity_error
    157	 add		%sp, PTREGS_OFF, %o1
    158	ba,a,pt		%xcc, rtrap_irq
    159	.size		do_cheetah_plus_insn_parity,.-do_cheetah_plus_insn_parity
    160
    161	.globl		cheetah_plus_icpe_trap_vector_tl1
    162	.type		cheetah_plus_icpe_trap_vector_tl1,#function
    163cheetah_plus_icpe_trap_vector_tl1:
    164	membar		#Sync
    165	wrpr		PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
    166	sethi		%hi(do_icpe_tl1), %g3
    167	jmpl		%g3 + %lo(do_icpe_tl1), %g0
    168	 nop
    169	nop
    170	nop
    171	nop
    172	.size		cheetah_plus_icpe_trap_vector_tl1,.-cheetah_plus_icpe_trap_vector_tl1
    173
    174	/* If we take one of these traps when tl >= 1, then we
    175	 * jump to interrupt globals.  If some trap level above us
    176	 * was also using interrupt globals, we cannot recover.
    177	 * We may use all interrupt global registers except %g6.
    178	 */
    179	.globl		do_dcpe_tl1
    180	.type		do_dcpe_tl1,#function
    181do_dcpe_tl1:
    182	rdpr		%tl, %g1		! Save original trap level
    183	mov		1, %g2			! Setup TSTATE checking loop
    184	sethi		%hi(TSTATE_IG), %g3	! TSTATE mask bit
    1851:	wrpr		%g2, %tl		! Set trap level to check
    186	rdpr		%tstate, %g4		! Read TSTATE for this level
    187	andcc		%g4, %g3, %g0		! Interrupt globals in use?
    188	bne,a,pn	%xcc, do_dcpe_tl1_fatal	! Yep, irrecoverable
    189	 wrpr		%g1, %tl		! Restore original trap level
    190	add		%g2, 1, %g2		! Next trap level
    191	cmp		%g2, %g1		! Hit them all yet?
    192	ble,pt		%icc, 1b		! Not yet
    193	 nop
    194	wrpr		%g1, %tl		! Restore original trap level
    195do_dcpe_tl1_nonfatal:	/* Ok we may use interrupt globals safely. */
    196	sethi		%hi(dcache_parity_tl1_occurred), %g2
    197	lduw		[%g2 + %lo(dcache_parity_tl1_occurred)], %g1
    198	add		%g1, 1, %g1
    199	stw		%g1, [%g2 + %lo(dcache_parity_tl1_occurred)]
    200	/* Reset D-cache parity */
    201	sethi		%hi(1 << 16), %g1	! D-cache size
    202	mov		(1 << 5), %g2		! D-cache line size
    203	sub		%g1, %g2, %g1		! Move down 1 cacheline
    2041:	srl		%g1, 14, %g3		! Compute UTAG
    205	membar		#Sync
    206	stxa		%g3, [%g1] ASI_DCACHE_UTAG
    207	membar		#Sync
    208	sub		%g2, 8, %g3		! 64-bit data word within line
    2092:	membar		#Sync
    210	stxa		%g0, [%g1 + %g3] ASI_DCACHE_DATA
    211	membar		#Sync
    212	subcc		%g3, 8, %g3		! Next 64-bit data word
    213	bge,pt		%icc, 2b
    214	 nop
    215	subcc		%g1, %g2, %g1		! Next cacheline
    216	bge,pt		%icc, 1b
    217	 nop
    218	ba,a,pt		%xcc, dcpe_icpe_tl1_common
    219
    220do_dcpe_tl1_fatal:
    221	sethi		%hi(1f), %g7
    222	ba,pt		%xcc, etraptl1
    2231:	or		%g7, %lo(1b), %g7
    224	mov		0x2, %o0
    225	call		cheetah_plus_parity_error
    226	 add		%sp, PTREGS_OFF, %o1
    227	ba,a,pt		%xcc, rtrap
    228	.size		do_dcpe_tl1,.-do_dcpe_tl1
    229
    230	.globl		do_icpe_tl1
    231	.type		do_icpe_tl1,#function
    232do_icpe_tl1:
    233	rdpr		%tl, %g1		! Save original trap level
    234	mov		1, %g2			! Setup TSTATE checking loop
    235	sethi		%hi(TSTATE_IG), %g3	! TSTATE mask bit
    2361:	wrpr		%g2, %tl		! Set trap level to check
    237	rdpr		%tstate, %g4		! Read TSTATE for this level
    238	andcc		%g4, %g3, %g0		! Interrupt globals in use?
    239	bne,a,pn	%xcc, do_icpe_tl1_fatal	! Yep, irrecoverable
    240	 wrpr		%g1, %tl		! Restore original trap level
    241	add		%g2, 1, %g2		! Next trap level
    242	cmp		%g2, %g1		! Hit them all yet?
    243	ble,pt		%icc, 1b		! Not yet
    244	 nop
    245	wrpr		%g1, %tl		! Restore original trap level
    246do_icpe_tl1_nonfatal:	/* Ok we may use interrupt globals safely. */
    247	sethi		%hi(icache_parity_tl1_occurred), %g2
    248	lduw		[%g2 + %lo(icache_parity_tl1_occurred)], %g1
    249	add		%g1, 1, %g1
    250	stw		%g1, [%g2 + %lo(icache_parity_tl1_occurred)]
    251	/* Flush I-cache */
    252	sethi		%hi(1 << 15), %g1	! I-cache size
    253	mov		(1 << 5), %g2		! I-cache line size
    254	sub		%g1, %g2, %g1
    2551:	or		%g1, (2 << 3), %g3
    256	stxa		%g0, [%g3] ASI_IC_TAG
    257	membar		#Sync
    258	subcc		%g1, %g2, %g1
    259	bge,pt		%icc, 1b
    260	 nop
    261	ba,a,pt		%xcc, dcpe_icpe_tl1_common
    262
    263do_icpe_tl1_fatal:
    264	sethi		%hi(1f), %g7
    265	ba,pt		%xcc, etraptl1
    2661:	or		%g7, %lo(1b), %g7
    267	mov		0x3, %o0
    268	call		cheetah_plus_parity_error
    269	 add		%sp, PTREGS_OFF, %o1
    270	ba,a,pt		%xcc, rtrap
    271	.size		do_icpe_tl1,.-do_icpe_tl1
    272	
    273	.type		dcpe_icpe_tl1_common,#function
    274dcpe_icpe_tl1_common:
    275	/* Flush D-cache, re-enable D/I caches in DCU and finally
    276	 * retry the trapping instruction.
    277	 */
    278	sethi		%hi(1 << 16), %g1	! D-cache size
    279	mov		(1 << 5), %g2		! D-cache line size
    280	sub		%g1, %g2, %g1
    2811:	stxa		%g0, [%g1] ASI_DCACHE_TAG
    282	membar		#Sync
    283	subcc		%g1, %g2, %g1
    284	bge,pt		%icc, 1b
    285	 nop
    286	ldxa		[%g0] ASI_DCU_CONTROL_REG, %g1
    287	or		%g1, (DCU_DC | DCU_IC), %g1
    288	stxa		%g1, [%g0] ASI_DCU_CONTROL_REG
    289	membar		#Sync
    290	retry
    291	.size		dcpe_icpe_tl1_common,.-dcpe_icpe_tl1_common
    292
    293	/* Capture I/D/E-cache state into per-cpu error scoreboard.
    294	 *
    295	 * %g1:		(TL>=0) ? 1 : 0
    296	 * %g2:		scratch
    297	 * %g3:		scratch
    298	 * %g4:		AFSR
    299	 * %g5:		AFAR
    300	 * %g6:		unused, will have current thread ptr after etrap
    301	 * %g7:		scratch
    302	 */
    303	.type		__cheetah_log_error,#function
    304__cheetah_log_error:
    305	/* Put "TL1" software bit into AFSR. */
    306	and		%g1, 0x1, %g1
    307	sllx		%g1, 63, %g2
    308	or		%g4, %g2, %g4
    309
    310	/* Get log entry pointer for this cpu at this trap level. */
    311	BRANCH_IF_JALAPENO(g2,g3,50f)
    312	ldxa		[%g0] ASI_SAFARI_CONFIG, %g2
    313	srlx		%g2, 17, %g2
    314	ba,pt		%xcc, 60f
    315	 and		%g2, 0x3ff, %g2
    316
    31750:	ldxa		[%g0] ASI_JBUS_CONFIG, %g2
    318	srlx		%g2, 17, %g2
    319	and		%g2, 0x1f, %g2
    320
    32160:	sllx		%g2, 9, %g2
    322	sethi		%hi(cheetah_error_log), %g3
    323	ldx		[%g3 + %lo(cheetah_error_log)], %g3
    324	brz,pn		%g3, 80f
    325	 nop
    326
    327	add		%g3, %g2, %g3
    328	sllx		%g1, 8, %g1
    329	add		%g3, %g1, %g1
    330
    331	/* %g1 holds pointer to the top of the logging scoreboard */
    332	ldx		[%g1 + 0x0], %g7
    333	cmp		%g7, -1
    334	bne,pn		%xcc, 80f
    335	 nop
    336
    337	stx		%g4, [%g1 + 0x0]
    338	stx		%g5, [%g1 + 0x8]
    339	add		%g1, 0x10, %g1
    340
    341	/* %g1 now points to D-cache logging area */
    342	set		0x3ff8, %g2	/* DC_addr mask		*/
    343	and		%g5, %g2, %g2	/* DC_addr bits of AFAR	*/
    344	srlx		%g5, 12, %g3
    345	or		%g3, 1, %g3	/* PHYS tag + valid	*/
    346
    34710:	ldxa		[%g2] ASI_DCACHE_TAG, %g7
    348	cmp		%g3, %g7	/* TAG match?		*/
    349	bne,pt		%xcc, 13f
    350	 nop
    351
    352	/* Yep, what we want, capture state. */
    353	stx		%g2, [%g1 + 0x20]
    354	stx		%g7, [%g1 + 0x28]
    355
    356	/* A membar Sync is required before and after utag access. */
    357	membar		#Sync
    358	ldxa		[%g2] ASI_DCACHE_UTAG, %g7
    359	membar		#Sync
    360	stx		%g7, [%g1 + 0x30]
    361	ldxa		[%g2] ASI_DCACHE_SNOOP_TAG, %g7
    362	stx		%g7, [%g1 + 0x38]
    363	clr		%g3
    364
    36512:	ldxa		[%g2 + %g3] ASI_DCACHE_DATA, %g7
    366	stx		%g7, [%g1]
    367	add		%g3, (1 << 5), %g3
    368	cmp		%g3, (4 << 5)
    369	bl,pt		%xcc, 12b
    370	 add		%g1, 0x8, %g1
    371
    372	ba,pt		%xcc, 20f
    373	 add		%g1, 0x20, %g1
    374
    37513:	sethi		%hi(1 << 14), %g7
    376	add		%g2, %g7, %g2
    377	srlx		%g2, 14, %g7
    378	cmp		%g7, 4
    379	bl,pt		%xcc, 10b
    380	 nop
    381
    382	add		%g1, 0x40, %g1
    383
    384	/* %g1 now points to I-cache logging area */
    38520:	set		0x1fe0, %g2	/* IC_addr mask		*/
    386	and		%g5, %g2, %g2	/* IC_addr bits of AFAR	*/
    387	sllx		%g2, 1, %g2	/* IC_addr[13:6]==VA[12:5] */
    388	srlx		%g5, (13 - 8), %g3 /* Make PTAG */
    389	andn		%g3, 0xff, %g3	/* Mask off undefined bits */
    390
    39121:	ldxa		[%g2] ASI_IC_TAG, %g7
    392	andn		%g7, 0xff, %g7
    393	cmp		%g3, %g7
    394	bne,pt		%xcc, 23f
    395	 nop
    396
    397	/* Yep, what we want, capture state. */
    398	stx		%g2, [%g1 + 0x40]
    399	stx		%g7, [%g1 + 0x48]
    400	add		%g2, (1 << 3), %g2
    401	ldxa		[%g2] ASI_IC_TAG, %g7
    402	add		%g2, (1 << 3), %g2
    403	stx		%g7, [%g1 + 0x50]
    404	ldxa		[%g2] ASI_IC_TAG, %g7
    405	add		%g2, (1 << 3), %g2
    406	stx		%g7, [%g1 + 0x60]
    407	ldxa		[%g2] ASI_IC_TAG, %g7
    408	stx		%g7, [%g1 + 0x68]
    409	sub		%g2, (3 << 3), %g2
    410	ldxa		[%g2] ASI_IC_STAG, %g7
    411	stx		%g7, [%g1 + 0x58]
    412	clr		%g3
    413	srlx		%g2, 2, %g2
    414
    41522:	ldxa		[%g2 + %g3] ASI_IC_INSTR, %g7
    416	stx		%g7, [%g1]
    417	add		%g3, (1 << 3), %g3
    418	cmp		%g3, (8 << 3)
    419	bl,pt		%xcc, 22b
    420	 add		%g1, 0x8, %g1
    421
    422	ba,pt		%xcc, 30f
    423	 add		%g1, 0x30, %g1
    424
    42523:	sethi		%hi(1 << 14), %g7
    426	add		%g2, %g7, %g2
    427	srlx		%g2, 14, %g7
    428	cmp		%g7, 4
    429	bl,pt		%xcc, 21b
    430	 nop
    431
    432	add		%g1, 0x70, %g1
    433
    434	/* %g1 now points to E-cache logging area */
    43530:	andn		%g5, (32 - 1), %g2
    436	stx		%g2, [%g1 + 0x20]
    437	ldxa		[%g2] ASI_EC_TAG_DATA, %g7
    438	stx		%g7, [%g1 + 0x28]
    439	ldxa		[%g2] ASI_EC_R, %g0
    440	clr		%g3
    441
    44231:	ldxa		[%g3] ASI_EC_DATA, %g7
    443	stx		%g7, [%g1 + %g3]
    444	add		%g3, 0x8, %g3
    445	cmp		%g3, 0x20
    446
    447	bl,pt		%xcc, 31b
    448	 nop
    44980:
    450	rdpr		%tt, %g2
    451	cmp		%g2, 0x70
    452	be		c_fast_ecc
    453	 cmp		%g2, 0x63
    454	be		c_cee
    455	 nop
    456	ba,a,pt		%xcc, c_deferred
    457	.size		__cheetah_log_error,.-__cheetah_log_error
    458
    459	/* Cheetah FECC trap handling, we get here from tl{0,1}_fecc
    460	 * in the trap table.  That code has done a memory barrier
    461	 * and has disabled both the I-cache and D-cache in the DCU
    462	 * control register.  The I-cache is disabled so that we may
    463	 * capture the corrupted cache line, and the D-cache is disabled
    464	 * because corrupt data may have been placed there and we don't
    465	 * want to reference it.
    466	 *
    467	 * %g1 is one if this trap occurred at %tl >= 1.
    468	 *
    469	 * Next, we turn off error reporting so that we don't recurse.
    470	 */
    471	.globl		cheetah_fast_ecc
    472	.type		cheetah_fast_ecc,#function
    473cheetah_fast_ecc:
    474	ldxa		[%g0] ASI_ESTATE_ERROR_EN, %g2
    475	andn		%g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2
    476	stxa		%g2, [%g0] ASI_ESTATE_ERROR_EN
    477	membar		#Sync
    478
    479	/* Fetch and clear AFSR/AFAR */
    480	ldxa		[%g0] ASI_AFSR, %g4
    481	ldxa		[%g0] ASI_AFAR, %g5
    482	stxa		%g4, [%g0] ASI_AFSR
    483	membar		#Sync
    484
    485	ba,pt		%xcc, __cheetah_log_error
    486	 nop
    487	.size		cheetah_fast_ecc,.-cheetah_fast_ecc
    488
    489	.type		c_fast_ecc,#function
    490c_fast_ecc:
    491	rdpr		%pil, %g2
    492	wrpr		%g0, PIL_NORMAL_MAX, %pil
    493	ba,pt		%xcc, etrap_irq
    494	 rd		%pc, %g7
    495#ifdef CONFIG_TRACE_IRQFLAGS
    496	call		trace_hardirqs_off
    497	 nop
    498#endif
    499	mov		%l4, %o1
    500	mov		%l5, %o2
    501	call		cheetah_fecc_handler
    502	 add		%sp, PTREGS_OFF, %o0
    503	ba,a,pt		%xcc, rtrap_irq
    504	.size		c_fast_ecc,.-c_fast_ecc
    505
    506	/* Our caller has disabled I-cache and performed membar Sync. */
    507	.globl		cheetah_cee
    508	.type		cheetah_cee,#function
    509cheetah_cee:
    510	ldxa		[%g0] ASI_ESTATE_ERROR_EN, %g2
    511	andn		%g2, ESTATE_ERROR_CEEN, %g2
    512	stxa		%g2, [%g0] ASI_ESTATE_ERROR_EN
    513	membar		#Sync
    514
    515	/* Fetch and clear AFSR/AFAR */
    516	ldxa		[%g0] ASI_AFSR, %g4
    517	ldxa		[%g0] ASI_AFAR, %g5
    518	stxa		%g4, [%g0] ASI_AFSR
    519	membar		#Sync
    520
    521	ba,pt		%xcc, __cheetah_log_error
    522	 nop
    523	.size		cheetah_cee,.-cheetah_cee
    524
    525	.type		c_cee,#function
    526c_cee:
    527	rdpr		%pil, %g2
    528	wrpr		%g0, PIL_NORMAL_MAX, %pil
    529	ba,pt		%xcc, etrap_irq
    530	 rd		%pc, %g7
    531#ifdef CONFIG_TRACE_IRQFLAGS
    532	call		trace_hardirqs_off
    533	 nop
    534#endif
    535	mov		%l4, %o1
    536	mov		%l5, %o2
    537	call		cheetah_cee_handler
    538	 add		%sp, PTREGS_OFF, %o0
    539	ba,a,pt		%xcc, rtrap_irq
    540	.size		c_cee,.-c_cee
    541
    542	/* Our caller has disabled I-cache+D-cache and performed membar Sync. */
    543	.globl		cheetah_deferred_trap
    544	.type		cheetah_deferred_trap,#function
    545cheetah_deferred_trap:
    546	ldxa		[%g0] ASI_ESTATE_ERROR_EN, %g2
    547	andn		%g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2
    548	stxa		%g2, [%g0] ASI_ESTATE_ERROR_EN
    549	membar		#Sync
    550
    551	/* Fetch and clear AFSR/AFAR */
    552	ldxa		[%g0] ASI_AFSR, %g4
    553	ldxa		[%g0] ASI_AFAR, %g5
    554	stxa		%g4, [%g0] ASI_AFSR
    555	membar		#Sync
    556
    557	ba,pt		%xcc, __cheetah_log_error
    558	 nop
    559	.size		cheetah_deferred_trap,.-cheetah_deferred_trap
    560
    561	.type		c_deferred,#function
    562c_deferred:
    563	rdpr		%pil, %g2
    564	wrpr		%g0, PIL_NORMAL_MAX, %pil
    565	ba,pt		%xcc, etrap_irq
    566	 rd		%pc, %g7
    567#ifdef CONFIG_TRACE_IRQFLAGS
    568	call		trace_hardirqs_off
    569	 nop
    570#endif
    571	mov		%l4, %o1
    572	mov		%l5, %o2
    573	call		cheetah_deferred_handler
    574	 add		%sp, PTREGS_OFF, %o0
    575	ba,a,pt		%xcc, rtrap_irq
    576	.size		c_deferred,.-c_deferred