cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

misc_64.S (10074B)


      1/* SPDX-License-Identifier: GPL-2.0-or-later */
      2/*
      3 * This file contains miscellaneous low-level functions.
      4 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
      5 *
      6 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
      7 * and Paul Mackerras.
      8 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
      9 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
     10 */
     11
     12#include <linux/sys.h>
     13#include <asm/unistd.h>
     14#include <asm/errno.h>
     15#include <asm/processor.h>
     16#include <asm/page.h>
     17#include <asm/cache.h>
     18#include <asm/ppc_asm.h>
     19#include <asm/asm-offsets.h>
     20#include <asm/cputable.h>
     21#include <asm/thread_info.h>
     22#include <asm/kexec.h>
     23#include <asm/ptrace.h>
     24#include <asm/mmu.h>
     25#include <asm/export.h>
     26#include <asm/feature-fixups.h>
     27
     28	.text
     29
     30_GLOBAL(__bswapdi2)
     31EXPORT_SYMBOL(__bswapdi2)
     32	srdi	r8,r3,32
     33	rlwinm	r7,r3,8,0xffffffff
     34	rlwimi	r7,r3,24,0,7
     35	rlwinm	r9,r8,8,0xffffffff
     36	rlwimi	r7,r3,24,16,23
     37	rlwimi	r9,r8,24,0,7
     38	rlwimi	r9,r8,24,16,23
     39	sldi	r7,r7,32
     40	or	r3,r7,r9
     41	blr
     42
     43
     44#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
     45_GLOBAL(rmci_on)
     46	sync
     47	isync
     48	li	r3,0x100
     49	rldicl	r3,r3,32,0
     50	mfspr	r5,SPRN_HID4
     51	or	r5,r5,r3
     52	sync
     53	mtspr	SPRN_HID4,r5
     54	isync
     55	slbia
     56	isync
     57	sync
     58	blr
     59
     60_GLOBAL(rmci_off)
     61	sync
     62	isync
     63	li	r3,0x100
     64	rldicl	r3,r3,32,0
     65	mfspr	r5,SPRN_HID4
     66	andc	r5,r5,r3
     67	sync
     68	mtspr	SPRN_HID4,r5
     69	isync
     70	slbia
     71	isync
     72	sync
     73	blr
     74#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
     75
     76#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
     77
     78/*
     79 * Do an IO access in real mode
     80 */
     81_GLOBAL(real_readb)
     82	mfmsr	r7
     83	ori	r0,r7,MSR_DR
     84	xori	r0,r0,MSR_DR
     85	sync
     86	mtmsrd	r0
     87	sync
     88	isync
     89	mfspr	r6,SPRN_HID4
     90	rldicl	r5,r6,32,0
     91	ori	r5,r5,0x100
     92	rldicl	r5,r5,32,0
     93	sync
     94	mtspr	SPRN_HID4,r5
     95	isync
     96	slbia
     97	isync
     98	lbz	r3,0(r3)
     99	sync
    100	mtspr	SPRN_HID4,r6
    101	isync
    102	slbia
    103	isync
    104	mtmsrd	r7
    105	sync
    106	isync
    107	blr
    108
    109	/*
    110 * Do an IO access in real mode
    111 */
    112_GLOBAL(real_writeb)
    113	mfmsr	r7
    114	ori	r0,r7,MSR_DR
    115	xori	r0,r0,MSR_DR
    116	sync
    117	mtmsrd	r0
    118	sync
    119	isync
    120	mfspr	r6,SPRN_HID4
    121	rldicl	r5,r6,32,0
    122	ori	r5,r5,0x100
    123	rldicl	r5,r5,32,0
    124	sync
    125	mtspr	SPRN_HID4,r5
    126	isync
    127	slbia
    128	isync
    129	stb	r3,0(r4)
    130	sync
    131	mtspr	SPRN_HID4,r6
    132	isync
    133	slbia
    134	isync
    135	mtmsrd	r7
    136	sync
    137	isync
    138	blr
    139#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
    140
    141#ifdef CONFIG_PPC_PASEMI
    142
    143_GLOBAL(real_205_readb)
    144	mfmsr	r7
    145	ori	r0,r7,MSR_DR
    146	xori	r0,r0,MSR_DR
    147	sync
    148	mtmsrd	r0
    149	sync
    150	isync
    151	LBZCIX(R3,R0,R3)
    152	isync
    153	mtmsrd	r7
    154	sync
    155	isync
    156	blr
    157
    158_GLOBAL(real_205_writeb)
    159	mfmsr	r7
    160	ori	r0,r7,MSR_DR
    161	xori	r0,r0,MSR_DR
    162	sync
    163	mtmsrd	r0
    164	sync
    165	isync
    166	STBCIX(R3,R0,R4)
    167	isync
    168	mtmsrd	r7
    169	sync
    170	isync
    171	blr
    172
    173#endif /* CONFIG_PPC_PASEMI */
    174
    175
    176#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
    177/*
    178 * SCOM access functions for 970 (FX only for now)
    179 *
    180 * unsigned long scom970_read(unsigned int address);
    181 * void scom970_write(unsigned int address, unsigned long value);
    182 *
    183 * The address passed in is the 24 bits register address. This code
    184 * is 970 specific and will not check the status bits, so you should
    185 * know what you are doing.
    186 */
    187_GLOBAL(scom970_read)
    188	/* interrupts off */
    189	mfmsr	r4
    190	ori	r0,r4,MSR_EE
    191	xori	r0,r0,MSR_EE
    192	mtmsrd	r0,1
    193
    194	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
    195	 * (including parity). On current CPUs they must be 0'd,
    196	 * and finally or in RW bit
    197	 */
    198	rlwinm	r3,r3,8,0,15
    199	ori	r3,r3,0x8000
    200
    201	/* do the actual scom read */
    202	sync
    203	mtspr	SPRN_SCOMC,r3
    204	isync
    205	mfspr	r3,SPRN_SCOMD
    206	isync
    207	mfspr	r0,SPRN_SCOMC
    208	isync
    209
    210	/* XXX:	fixup result on some buggy 970's (ouch ! we lost a bit, bah
    211	 * that's the best we can do). Not implemented yet as we don't use
    212	 * the scom on any of the bogus CPUs yet, but may have to be done
    213	 * ultimately
    214	 */
    215
    216	/* restore interrupts */
    217	mtmsrd	r4,1
    218	blr
    219
    220
    221_GLOBAL(scom970_write)
    222	/* interrupts off */
    223	mfmsr	r5
    224	ori	r0,r5,MSR_EE
    225	xori	r0,r0,MSR_EE
    226	mtmsrd	r0,1
    227
    228	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
    229	 * (including parity). On current CPUs they must be 0'd.
    230	 */
    231
    232	rlwinm	r3,r3,8,0,15
    233
    234	sync
    235	mtspr	SPRN_SCOMD,r4      /* write data */
    236	isync
    237	mtspr	SPRN_SCOMC,r3      /* write command */
    238	isync
    239	mfspr	3,SPRN_SCOMC
    240	isync
    241
    242	/* restore interrupts */
    243	mtmsrd	r5,1
    244	blr
    245#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */
    246
    247/* kexec_wait(phys_cpu)
    248 *
    249 * wait for the flag to change, indicating this kernel is going away but
    250 * the slave code for the next one is at addresses 0 to 100.
    251 *
    252 * This is used by all slaves, even those that did not find a matching
    253 * paca in the secondary startup code.
    254 *
    255 * Physical (hardware) cpu id should be in r3.
    256 */
    257_GLOBAL(kexec_wait)
    258	bcl	20,31,$+4
    2591:	mflr	r5
    260	addi	r5,r5,kexec_flag-1b
    261
    26299:	HMT_LOW
    263#ifdef CONFIG_KEXEC_CORE	/* use no memory without kexec */
    264	lwz	r4,0(r5)
    265	cmpwi	0,r4,0
    266	beq	99b
    267#ifdef CONFIG_PPC_BOOK3S_64
    268	li	r10,0x60
    269	mfmsr	r11
    270	clrrdi	r11,r11,1	/* Clear MSR_LE */
    271	mtsrr0	r10
    272	mtsrr1	r11
    273	rfid
    274#else
    275	/* Create TLB entry in book3e_secondary_core_init */
    276	li	r4,0
    277	ba	0x60
    278#endif
    279#endif
    280
    281/* this can be in text because we won't change it until we are
    282 * running in real anyways
    283 */
    284kexec_flag:
    285	.long	0
    286
    287
    288#ifdef CONFIG_KEXEC_CORE
    289#ifdef CONFIG_PPC_BOOK3E
    290/*
    291 * BOOK3E has no real MMU mode, so we have to setup the initial TLB
    292 * for a core to identity map v:0 to p:0.  This current implementation
    293 * assumes that 1G is enough for kexec.
    294 */
    295kexec_create_tlb:
    296	/*
    297	 * Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
    298	 * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
    299	 */
    300	PPC_TLBILX_ALL(0,R0)
    301	sync
    302	isync
    303
    304	mfspr	r10,SPRN_TLB1CFG
    305	andi.	r10,r10,TLBnCFG_N_ENTRY	/* Extract # entries */
    306	subi	r10,r10,1	/* Last entry: no conflict with kernel text */
    307	lis	r9,MAS0_TLBSEL(1)@h
    308	rlwimi	r9,r10,16,4,15		/* Setup MAS0 = TLBSEL | ESEL(r9) */
    309
    310/* Set up a temp identity mapping v:0 to p:0 and return to it. */
    311	mtspr	SPRN_MAS0,r9
    312
    313	lis	r9,(MAS1_VALID|MAS1_IPROT)@h
    314	ori	r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
    315	mtspr	SPRN_MAS1,r9
    316
    317	LOAD_REG_IMMEDIATE(r9, 0x0 | MAS2_M_IF_NEEDED)
    318	mtspr	SPRN_MAS2,r9
    319
    320	LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX)
    321	mtspr	SPRN_MAS3,r9
    322	li	r9,0
    323	mtspr	SPRN_MAS7,r9
    324
    325	tlbwe
    326	isync
    327	blr
    328#endif
    329
    330/* kexec_smp_wait(void)
    331 *
    332 * call with interrupts off
    333 * note: this is a terminal routine, it does not save lr
    334 *
    335 * get phys id from paca
    336 * switch to real mode
    337 * mark the paca as no longer used
    338 * join other cpus in kexec_wait(phys_id)
    339 */
    340_GLOBAL(kexec_smp_wait)
    341	lhz	r3,PACAHWCPUID(r13)
    342	bl	real_mode
    343
    344	li	r4,KEXEC_STATE_REAL_MODE
    345	stb	r4,PACAKEXECSTATE(r13)
    346
    347	b	kexec_wait
    348
    349/*
    350 * switch to real mode (turn mmu off)
    351 * we use the early kernel trick that the hardware ignores bits
    352 * 0 and 1 (big endian) of the effective address in real mode
    353 *
    354 * don't overwrite r3 here, it is live for kexec_wait above.
    355 */
    356real_mode:	/* assume normal blr return */
    357#ifdef CONFIG_PPC_BOOK3E
    358	/* Create an identity mapping. */
    359	b	kexec_create_tlb
    360#else
    3611:	li	r9,MSR_RI
    362	li	r10,MSR_DR|MSR_IR
    363	mflr	r11		/* return address to SRR0 */
    364	mfmsr	r12
    365	andc	r9,r12,r9
    366	andc	r10,r12,r10
    367
    368	mtmsrd	r9,1
    369	mtspr	SPRN_SRR1,r10
    370	mtspr	SPRN_SRR0,r11
    371	rfid
    372#endif
    373
    374/*
    375 * kexec_sequence(newstack, start, image, control, clear_all(),
    376	          copy_with_mmu_off)
    377 *
    378 * does the grungy work with stack switching and real mode switches
    379 * also does simple calls to other code
    380 */
    381
    382_GLOBAL(kexec_sequence)
    383	mflr	r0
    384	std	r0,16(r1)
    385
    386	/* switch stacks to newstack -- &kexec_stack.stack */
    387	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
    388	mr	r1,r3
    389
    390	li	r0,0
    391	std	r0,16(r1)
    392
    393	/* save regs for local vars on new stack.
    394	 * yes, we won't go back, but ...
    395	 */
    396	std	r31,-8(r1)
    397	std	r30,-16(r1)
    398	std	r29,-24(r1)
    399	std	r28,-32(r1)
    400	std	r27,-40(r1)
    401	std	r26,-48(r1)
    402	std	r25,-56(r1)
    403
    404	stdu	r1,-STACK_FRAME_OVERHEAD-64(r1)
    405
    406	/* save args into preserved regs */
    407	mr	r31,r3			/* newstack (both) */
    408	mr	r30,r4			/* start (real) */
    409	mr	r29,r5			/* image (virt) */
    410	mr	r28,r6			/* control, unused */
    411	mr	r27,r7			/* clear_all() fn desc */
    412	mr	r26,r8			/* copy_with_mmu_off */
    413	lhz	r25,PACAHWCPUID(r13)	/* get our phys cpu from paca */
    414
    415	/* disable interrupts, we are overwriting kernel data next */
    416#ifdef CONFIG_PPC_BOOK3E
    417	wrteei	0
    418#else
    419	mfmsr	r3
    420	rlwinm	r3,r3,0,17,15
    421	mtmsrd	r3,1
    422#endif
    423
    424	/* We need to turn the MMU off unless we are in hash mode
    425	 * under a hypervisor
    426	 */
    427	cmpdi	r26,0
    428	beq	1f
    429	bl	real_mode
    4301:
    431	/* copy dest pages, flush whole dest image */
    432	mr	r3,r29
    433	bl	kexec_copy_flush	/* (image) */
    434
    435	/* turn off mmu now if not done earlier */
    436	cmpdi	r26,0
    437	bne	1f
    438	bl	real_mode
    439
    440	/* copy  0x100 bytes starting at start to 0 */
    4411:	li	r3,0
    442	mr	r4,r30		/* start, aka phys mem offset */
    443	li	r5,0x100
    444	li	r6,0
    445	bl	copy_and_flush	/* (dest, src, copy limit, start offset) */
    4461:	/* assume normal blr return */
    447
    448	/* release other cpus to the new kernel secondary start at 0x60 */
    449	mflr	r5
    450	li	r6,1
    451	stw	r6,kexec_flag-1b(5)
    452
    453	cmpdi	r27,0
    454	beq	1f
    455
    456	/* clear out hardware hash page table and tlb */
    457#ifdef CONFIG_PPC64_ELF_ABI_V1
    458	ld	r12,0(r27)		/* deref function descriptor */
    459#else
    460	mr	r12,r27
    461#endif
    462	mtctr	r12
    463	bctrl				/* mmu_hash_ops.hpte_clear_all(void); */
    464
    465/*
    466 *   kexec image calling is:
    467 *      the first 0x100 bytes of the entry point are copied to 0
    468 *
    469 *      all slaves branch to slave = 0x60 (absolute)
    470 *              slave(phys_cpu_id);
    471 *
    472 *      master goes to start = entry point
    473 *              start(phys_cpu_id, start, 0);
    474 *
    475 *
    476 *   a wrapper is needed to call existing kernels, here is an approximate
    477 *   description of one method:
    478 *
    479 * v2: (2.6.10)
    480 *   start will be near the boot_block (maybe 0x100 bytes before it?)
    481 *   it will have a 0x60, which will b to boot_block, where it will wait
    482 *   and 0 will store phys into struct boot-block and load r3 from there,
    483 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
    484 *
    485 * v1: (2.6.9)
    486 *    boot block will have all cpus scanning device tree to see if they
    487 *    are the boot cpu ?????
    488 *    other device tree differences (prop sizes, va vs pa, etc)...
    489 */
    4901:	mr	r3,r25	# my phys cpu
    491	mr	r4,r30	# start, aka phys mem offset
    492	mtlr	4
    493	li	r5,0
    494	blr	/* image->start(physid, image->start, 0); */
    495#endif /* CONFIG_KEXEC_CORE */