trampoline_64.S (6265B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * 4 * Trampoline.S Derived from Setup.S by Linus Torvalds 5 * 6 * 4 Jan 1997 Michael Chastain: changed to gnu as. 7 * 15 Sept 2005 Eric Biederman: 64bit PIC support 8 * 9 * Entry: CS:IP point to the start of our code, we are 10 * in real mode with no stack, but the rest of the 11 * trampoline page to make our stack and everything else 12 * is a mystery. 13 * 14 * On entry to trampoline_start, the processor is in real mode 15 * with 16-bit addressing and 16-bit data. CS has some value 16 * and IP is zero. Thus, data addresses need to be absolute 17 * (no relocation) and are taken with regard to r_base. 18 * 19 * With the addition of trampoline_level4_pgt this code can 20 * now enter a 64bit kernel that lives at arbitrary 64bit 21 * physical addresses. 22 * 23 * If you work on this file, check the object module with objdump 24 * --full-contents --reloc to make sure there are no relocation 25 * entries. 26 */ 27 28#include <linux/linkage.h> 29#include <asm/pgtable_types.h> 30#include <asm/page_types.h> 31#include <asm/msr.h> 32#include <asm/segment.h> 33#include <asm/processor-flags.h> 34#include <asm/realmode.h> 35#include "realmode.h" 36 37 .text 38 .code16 39 40 .balign PAGE_SIZE 41SYM_CODE_START(trampoline_start) 42 cli # We should be safe anyway 43 wbinvd 44 45 LJMPW_RM(1f) 461: 47 mov %cs, %ax # Code and data in the same place 48 mov %ax, %ds 49 mov %ax, %es 50 mov %ax, %ss 51 52 # Setup stack 53 movl $rm_stack_end, %esp 54 55 call verify_cpu # Verify the cpu supports long mode 56 testl %eax, %eax # Check for return code 57 jnz no_longmode 58 59.Lswitch_to_protected: 60 /* 61 * GDT tables in non default location kernel can be beyond 16MB and 62 * lgdt will not be able to load the address as in real mode default 63 * operand size is 16bit. Use lgdtl instead to force operand size 64 * to 32 bit. 65 */ 66 67 lidtl tr_idt # load idt with 0, 0 68 lgdtl tr_gdt # load gdt with whatever is appropriate 69 70 movw $__KERNEL_DS, %dx # Data segment descriptor 71 72 # Enable protected mode 73 movl $(CR0_STATE & ~X86_CR0_PG), %eax 74 movl %eax, %cr0 # into protected mode 75 76 # flush prefetch and jump to startup_32 77 ljmpl $__KERNEL32_CS, $pa_startup_32 78 79no_longmode: 80 hlt 81 jmp no_longmode 82SYM_CODE_END(trampoline_start) 83 84#ifdef CONFIG_AMD_MEM_ENCRYPT 85/* SEV-ES supports non-zero IP for entry points - no alignment needed */ 86SYM_CODE_START(sev_es_trampoline_start) 87 cli # We should be safe anyway 88 89 LJMPW_RM(1f) 901: 91 mov %cs, %ax # Code and data in the same place 92 mov %ax, %ds 93 mov %ax, %es 94 mov %ax, %ss 95 96 # Setup stack 97 movl $rm_stack_end, %esp 98 99 jmp .Lswitch_to_protected 100SYM_CODE_END(sev_es_trampoline_start) 101#endif /* CONFIG_AMD_MEM_ENCRYPT */ 102 103#include "../kernel/verify_cpu.S" 104 105 .section ".text32","ax" 106 .code32 107 .balign 4 108SYM_CODE_START(startup_32) 109 movl %edx, %ss 110 addl $pa_real_mode_base, %esp 111 movl %edx, %ds 112 movl %edx, %es 113 movl %edx, %fs 114 movl %edx, %gs 115 116 /* 117 * Check for memory encryption support. This is a safety net in 118 * case BIOS hasn't done the necessary step of setting the bit in 119 * the MSR for this AP. If SME is active and we've gotten this far 120 * then it is safe for us to set the MSR bit and continue. If we 121 * don't we'll eventually crash trying to execute encrypted 122 * instructions. 123 */ 124 btl $TH_FLAGS_SME_ACTIVE_BIT, pa_tr_flags 125 jnc .Ldone 126 movl $MSR_AMD64_SYSCFG, %ecx 127 rdmsr 128 bts $MSR_AMD64_SYSCFG_MEM_ENCRYPT_BIT, %eax 129 jc .Ldone 130 131 /* 132 * Memory encryption is enabled but the SME enable bit for this 133 * CPU has has not been set. It is safe to set it, so do so. 134 */ 135 wrmsr 136.Ldone: 137 138 movl pa_tr_cr4, %eax 139 movl %eax, %cr4 # Enable PAE mode 140 141 # Setup trampoline 4 level pagetables 142 movl $pa_trampoline_pgd, %eax 143 movl %eax, %cr3 144 145 # Set up EFER 146 movl $MSR_EFER, %ecx 147 rdmsr 148 /* 149 * Skip writing to EFER if the register already has desired 150 * value (to avoid #VE for the TDX guest). 151 */ 152 cmp pa_tr_efer, %eax 153 jne .Lwrite_efer 154 cmp pa_tr_efer + 4, %edx 155 je .Ldone_efer 156.Lwrite_efer: 157 movl pa_tr_efer, %eax 158 movl pa_tr_efer + 4, %edx 159 wrmsr 160 161.Ldone_efer: 162 # Enable paging and in turn activate Long Mode. 163 movl $CR0_STATE, %eax 164 movl %eax, %cr0 165 166 /* 167 * At this point we're in long mode but in 32bit compatibility mode 168 * with EFER.LME = 1, CS.L = 0, CS.D = 1 (and in turn 169 * EFER.LMA = 1). Now we want to jump in 64bit mode, to do that we use 170 * the new gdt/idt that has __KERNEL_CS with CS.L = 1. 171 */ 172 ljmpl $__KERNEL_CS, $pa_startup_64 173SYM_CODE_END(startup_32) 174 175SYM_CODE_START(pa_trampoline_compat) 176 /* 177 * In compatibility mode. Prep ESP and DX for startup_32, then disable 178 * paging and complete the switch to legacy 32-bit mode. 179 */ 180 movl $rm_stack_end, %esp 181 movw $__KERNEL_DS, %dx 182 183 movl $(CR0_STATE & ~X86_CR0_PG), %eax 184 movl %eax, %cr0 185 ljmpl $__KERNEL32_CS, $pa_startup_32 186SYM_CODE_END(pa_trampoline_compat) 187 188 .section ".text64","ax" 189 .code64 190 .balign 4 191SYM_CODE_START(startup_64) 192 # Now jump into the kernel using virtual addresses 193 jmpq *tr_start(%rip) 194SYM_CODE_END(startup_64) 195 196SYM_CODE_START(trampoline_start64) 197 /* 198 * APs start here on a direct transfer from 64-bit BIOS with identity 199 * mapped page tables. Load the kernel's GDT in order to gear down to 200 * 32-bit mode (to handle 4-level vs. 5-level paging), and to (re)load 201 * segment registers. Load the zero IDT so any fault triggers a 202 * shutdown instead of jumping back into BIOS. 203 */ 204 lidt tr_idt(%rip) 205 lgdt tr_gdt64(%rip) 206 207 ljmpl *tr_compat(%rip) 208SYM_CODE_END(trampoline_start64) 209 210 .section ".rodata","a" 211 # Duplicate the global descriptor table 212 # so the kernel can live anywhere 213 .balign 16 214SYM_DATA_START(tr_gdt) 215 .short tr_gdt_end - tr_gdt - 1 # gdt limit 216 .long pa_tr_gdt 217 .short 0 218 .quad 0x00cf9b000000ffff # __KERNEL32_CS 219 .quad 0x00af9b000000ffff # __KERNEL_CS 220 .quad 0x00cf93000000ffff # __KERNEL_DS 221SYM_DATA_END_LABEL(tr_gdt, SYM_L_LOCAL, tr_gdt_end) 222 223SYM_DATA_START(tr_gdt64) 224 .short tr_gdt_end - tr_gdt - 1 # gdt limit 225 .long pa_tr_gdt 226 .long 0 227SYM_DATA_END(tr_gdt64) 228 229SYM_DATA_START(tr_compat) 230 .long pa_trampoline_compat 231 .short __KERNEL32_CS 232SYM_DATA_END(tr_compat) 233 234 .bss 235 .balign PAGE_SIZE 236SYM_DATA(trampoline_pgd, .space PAGE_SIZE) 237 238 .balign 8 239SYM_DATA_START(trampoline_header) 240 SYM_DATA_LOCAL(tr_start, .space 8) 241 SYM_DATA(tr_efer, .space 8) 242 SYM_DATA(tr_cr4, .space 4) 243 SYM_DATA(tr_flags, .space 4) 244SYM_DATA_END(trampoline_header) 245 246#include "trampoline_common.S"