cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

switch.h (13689B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (C) 2015 - ARM Ltd
      4 * Author: Marc Zyngier <marc.zyngier@arm.com>
      5 */
      6
      7#ifndef __ARM64_KVM_HYP_SWITCH_H__
      8#define __ARM64_KVM_HYP_SWITCH_H__
      9
     10#include <hyp/adjust_pc.h>
     11#include <hyp/fault.h>
     12
     13#include <linux/arm-smccc.h>
     14#include <linux/kvm_host.h>
     15#include <linux/types.h>
     16#include <linux/jump_label.h>
     17#include <uapi/linux/psci.h>
     18
     19#include <kvm/arm_psci.h>
     20
     21#include <asm/barrier.h>
     22#include <asm/cpufeature.h>
     23#include <asm/extable.h>
     24#include <asm/kprobes.h>
     25#include <asm/kvm_asm.h>
     26#include <asm/kvm_emulate.h>
     27#include <asm/kvm_hyp.h>
     28#include <asm/kvm_mmu.h>
     29#include <asm/fpsimd.h>
     30#include <asm/debug-monitors.h>
     31#include <asm/processor.h>
     32
     33struct kvm_exception_table_entry {
     34	int insn, fixup;
     35};
     36
     37extern struct kvm_exception_table_entry __start___kvm_ex_table;
     38extern struct kvm_exception_table_entry __stop___kvm_ex_table;
     39
     40/* Check whether the FP regs were dirtied while in the host-side run loop: */
     41static inline bool update_fp_enabled(struct kvm_vcpu *vcpu)
     42{
     43	/*
     44	 * When the system doesn't support FP/SIMD, we cannot rely on
     45	 * the _TIF_FOREIGN_FPSTATE flag. However, we always inject an
     46	 * abort on the very first access to FP and thus we should never
     47	 * see KVM_ARM64_FP_ENABLED. For added safety, make sure we always
     48	 * trap the accesses.
     49	 */
     50	if (!system_supports_fpsimd() ||
     51	    vcpu->arch.flags & KVM_ARM64_FP_FOREIGN_FPSTATE)
     52		vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
     53				      KVM_ARM64_FP_HOST);
     54
     55	return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED);
     56}
     57
     58/* Save the 32-bit only FPSIMD system register state */
     59static inline void __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu)
     60{
     61	if (!vcpu_el1_is_32bit(vcpu))
     62		return;
     63
     64	__vcpu_sys_reg(vcpu, FPEXC32_EL2) = read_sysreg(fpexc32_el2);
     65}
     66
     67static inline void __activate_traps_fpsimd32(struct kvm_vcpu *vcpu)
     68{
     69	/*
     70	 * We are about to set CPTR_EL2.TFP to trap all floating point
     71	 * register accesses to EL2, however, the ARM ARM clearly states that
     72	 * traps are only taken to EL2 if the operation would not otherwise
     73	 * trap to EL1.  Therefore, always make sure that for 32-bit guests,
     74	 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
     75	 * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
     76	 * it will cause an exception.
     77	 */
     78	if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {
     79		write_sysreg(1 << 30, fpexc32_el2);
     80		isb();
     81	}
     82}
     83
     84static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
     85{
     86	/* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */
     87	write_sysreg(1 << 15, hstr_el2);
     88
     89	/*
     90	 * Make sure we trap PMU access from EL0 to EL2. Also sanitize
     91	 * PMSELR_EL0 to make sure it never contains the cycle
     92	 * counter, which could make a PMXEVCNTR_EL0 access UNDEF at
     93	 * EL1 instead of being trapped to EL2.
     94	 */
     95	if (kvm_arm_support_pmu_v3()) {
     96		write_sysreg(0, pmselr_el0);
     97		write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
     98	}
     99
    100	vcpu->arch.mdcr_el2_host = read_sysreg(mdcr_el2);
    101	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
    102}
    103
    104static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu)
    105{
    106	write_sysreg(vcpu->arch.mdcr_el2_host, mdcr_el2);
    107
    108	write_sysreg(0, hstr_el2);
    109	if (kvm_arm_support_pmu_v3())
    110		write_sysreg(0, pmuserenr_el0);
    111}
    112
    113static inline void ___activate_traps(struct kvm_vcpu *vcpu)
    114{
    115	u64 hcr = vcpu->arch.hcr_el2;
    116
    117	if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
    118		hcr |= HCR_TVM;
    119
    120	write_sysreg(hcr, hcr_el2);
    121
    122	if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
    123		write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
    124}
    125
    126static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
    127{
    128	/*
    129	 * If we pended a virtual abort, preserve it until it gets
    130	 * cleared. See D1.14.3 (Virtual Interrupts) for details, but
    131	 * the crucial bit is "On taking a vSError interrupt,
    132	 * HCR_EL2.VSE is cleared to 0."
    133	 */
    134	if (vcpu->arch.hcr_el2 & HCR_VSE) {
    135		vcpu->arch.hcr_el2 &= ~HCR_VSE;
    136		vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;
    137	}
    138}
    139
    140static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
    141{
    142	return __get_fault_info(vcpu->arch.fault.esr_el2, &vcpu->arch.fault);
    143}
    144
    145static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
    146{
    147	sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
    148	__sve_restore_state(vcpu_sve_pffr(vcpu),
    149			    &vcpu->arch.ctxt.fp_regs.fpsr);
    150	write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR);
    151}
    152
    153/*
    154 * We trap the first access to the FP/SIMD to save the host context and
    155 * restore the guest context lazily.
    156 * If FP/SIMD is not implemented, handle the trap and inject an undefined
    157 * instruction exception to the guest. Similarly for trapped SVE accesses.
    158 */
    159static bool kvm_hyp_handle_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
    160{
    161	bool sve_guest;
    162	u8 esr_ec;
    163	u64 reg;
    164
    165	if (!system_supports_fpsimd())
    166		return false;
    167
    168	sve_guest = vcpu_has_sve(vcpu);
    169	esr_ec = kvm_vcpu_trap_get_class(vcpu);
    170
    171	/* Don't handle SVE traps for non-SVE vcpus here: */
    172	if (!sve_guest && esr_ec != ESR_ELx_EC_FP_ASIMD)
    173		return false;
    174
    175	/* Valid trap.  Switch the context: */
    176
    177	/* First disable enough traps to allow us to update the registers */
    178	if (has_vhe()) {
    179		reg = CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN;
    180		if (sve_guest)
    181			reg |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN;
    182
    183		sysreg_clear_set(cpacr_el1, 0, reg);
    184	} else {
    185		reg = CPTR_EL2_TFP;
    186		if (sve_guest)
    187			reg |= CPTR_EL2_TZ;
    188
    189		sysreg_clear_set(cptr_el2, reg, 0);
    190	}
    191	isb();
    192
    193	/* Write out the host state if it's in the registers */
    194	if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
    195		__fpsimd_save_state(vcpu->arch.host_fpsimd_state);
    196		vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
    197	}
    198
    199	/* Restore the guest state */
    200	if (sve_guest)
    201		__hyp_sve_restore_guest(vcpu);
    202	else
    203		__fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs);
    204
    205	/* Skip restoring fpexc32 for AArch64 guests */
    206	if (!(read_sysreg(hcr_el2) & HCR_RW))
    207		write_sysreg(__vcpu_sys_reg(vcpu, FPEXC32_EL2), fpexc32_el2);
    208
    209	vcpu->arch.flags |= KVM_ARM64_FP_ENABLED;
    210
    211	return true;
    212}
    213
    214static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu)
    215{
    216	u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_esr(vcpu));
    217	int rt = kvm_vcpu_sys_get_rt(vcpu);
    218	u64 val = vcpu_get_reg(vcpu, rt);
    219
    220	/*
    221	 * The normal sysreg handling code expects to see the traps,
    222	 * let's not do anything here.
    223	 */
    224	if (vcpu->arch.hcr_el2 & HCR_TVM)
    225		return false;
    226
    227	switch (sysreg) {
    228	case SYS_SCTLR_EL1:
    229		write_sysreg_el1(val, SYS_SCTLR);
    230		break;
    231	case SYS_TTBR0_EL1:
    232		write_sysreg_el1(val, SYS_TTBR0);
    233		break;
    234	case SYS_TTBR1_EL1:
    235		write_sysreg_el1(val, SYS_TTBR1);
    236		break;
    237	case SYS_TCR_EL1:
    238		write_sysreg_el1(val, SYS_TCR);
    239		break;
    240	case SYS_ESR_EL1:
    241		write_sysreg_el1(val, SYS_ESR);
    242		break;
    243	case SYS_FAR_EL1:
    244		write_sysreg_el1(val, SYS_FAR);
    245		break;
    246	case SYS_AFSR0_EL1:
    247		write_sysreg_el1(val, SYS_AFSR0);
    248		break;
    249	case SYS_AFSR1_EL1:
    250		write_sysreg_el1(val, SYS_AFSR1);
    251		break;
    252	case SYS_MAIR_EL1:
    253		write_sysreg_el1(val, SYS_MAIR);
    254		break;
    255	case SYS_AMAIR_EL1:
    256		write_sysreg_el1(val, SYS_AMAIR);
    257		break;
    258	case SYS_CONTEXTIDR_EL1:
    259		write_sysreg_el1(val, SYS_CONTEXTIDR);
    260		break;
    261	default:
    262		return false;
    263	}
    264
    265	__kvm_skip_instr(vcpu);
    266	return true;
    267}
    268
    269static inline bool esr_is_ptrauth_trap(u64 esr)
    270{
    271	switch (esr_sys64_to_sysreg(esr)) {
    272	case SYS_APIAKEYLO_EL1:
    273	case SYS_APIAKEYHI_EL1:
    274	case SYS_APIBKEYLO_EL1:
    275	case SYS_APIBKEYHI_EL1:
    276	case SYS_APDAKEYLO_EL1:
    277	case SYS_APDAKEYHI_EL1:
    278	case SYS_APDBKEYLO_EL1:
    279	case SYS_APDBKEYHI_EL1:
    280	case SYS_APGAKEYLO_EL1:
    281	case SYS_APGAKEYHI_EL1:
    282		return true;
    283	}
    284
    285	return false;
    286}
    287
    288#define __ptrauth_save_key(ctxt, key)					\
    289	do {								\
    290	u64 __val;                                                      \
    291	__val = read_sysreg_s(SYS_ ## key ## KEYLO_EL1);                \
    292	ctxt_sys_reg(ctxt, key ## KEYLO_EL1) = __val;                   \
    293	__val = read_sysreg_s(SYS_ ## key ## KEYHI_EL1);                \
    294	ctxt_sys_reg(ctxt, key ## KEYHI_EL1) = __val;                   \
    295} while(0)
    296
    297DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
    298
    299static bool kvm_hyp_handle_ptrauth(struct kvm_vcpu *vcpu, u64 *exit_code)
    300{
    301	struct kvm_cpu_context *ctxt;
    302	u64 val;
    303
    304	if (!vcpu_has_ptrauth(vcpu))
    305		return false;
    306
    307	ctxt = this_cpu_ptr(&kvm_hyp_ctxt);
    308	__ptrauth_save_key(ctxt, APIA);
    309	__ptrauth_save_key(ctxt, APIB);
    310	__ptrauth_save_key(ctxt, APDA);
    311	__ptrauth_save_key(ctxt, APDB);
    312	__ptrauth_save_key(ctxt, APGA);
    313
    314	vcpu_ptrauth_enable(vcpu);
    315
    316	val = read_sysreg(hcr_el2);
    317	val |= (HCR_API | HCR_APK);
    318	write_sysreg(val, hcr_el2);
    319
    320	return true;
    321}
    322
    323static bool kvm_hyp_handle_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code)
    324{
    325	if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&
    326	    handle_tx2_tvm(vcpu))
    327		return true;
    328
    329	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
    330	    __vgic_v3_perform_cpuif_access(vcpu) == 1)
    331		return true;
    332
    333	if (esr_is_ptrauth_trap(kvm_vcpu_get_esr(vcpu)))
    334		return kvm_hyp_handle_ptrauth(vcpu, exit_code);
    335
    336	return false;
    337}
    338
    339static bool kvm_hyp_handle_cp15_32(struct kvm_vcpu *vcpu, u64 *exit_code)
    340{
    341	if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
    342	    __vgic_v3_perform_cpuif_access(vcpu) == 1)
    343		return true;
    344
    345	return false;
    346}
    347
    348static bool kvm_hyp_handle_iabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
    349{
    350	if (!__populate_fault_info(vcpu))
    351		return true;
    352
    353	return false;
    354}
    355
    356static bool kvm_hyp_handle_dabt_low(struct kvm_vcpu *vcpu, u64 *exit_code)
    357{
    358	if (!__populate_fault_info(vcpu))
    359		return true;
    360
    361	if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {
    362		bool valid;
    363
    364		valid = kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
    365			kvm_vcpu_dabt_isvalid(vcpu) &&
    366			!kvm_vcpu_abt_issea(vcpu) &&
    367			!kvm_vcpu_abt_iss1tw(vcpu);
    368
    369		if (valid) {
    370			int ret = __vgic_v2_perform_cpuif_access(vcpu);
    371
    372			if (ret == 1)
    373				return true;
    374
    375			/* Promote an illegal access to an SError.*/
    376			if (ret == -1)
    377				*exit_code = ARM_EXCEPTION_EL1_SERROR;
    378		}
    379	}
    380
    381	return false;
    382}
    383
    384typedef bool (*exit_handler_fn)(struct kvm_vcpu *, u64 *);
    385
    386static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu);
    387
    388static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code);
    389
    390/*
    391 * Allow the hypervisor to handle the exit with an exit handler if it has one.
    392 *
    393 * Returns true if the hypervisor handled the exit, and control should go back
    394 * to the guest, or false if it hasn't.
    395 */
    396static inline bool kvm_hyp_handle_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
    397{
    398	const exit_handler_fn *handlers = kvm_get_exit_handler_array(vcpu);
    399	exit_handler_fn fn;
    400
    401	fn = handlers[kvm_vcpu_trap_get_class(vcpu)];
    402
    403	if (fn)
    404		return fn(vcpu, exit_code);
    405
    406	return false;
    407}
    408
    409static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu, u64 *exit_code)
    410{
    411	/*
    412	 * Check for the conditions of Cortex-A510's #2077057. When these occur
    413	 * SPSR_EL2 can't be trusted, but isn't needed either as it is
    414	 * unchanged from the value in vcpu_gp_regs(vcpu)->pstate.
    415	 * Are we single-stepping the guest, and took a PAC exception from the
    416	 * active-not-pending state?
    417	 */
    418	if (cpus_have_final_cap(ARM64_WORKAROUND_2077057)		&&
    419	    vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP			&&
    420	    *vcpu_cpsr(vcpu) & DBG_SPSR_SS				&&
    421	    ESR_ELx_EC(read_sysreg_el2(SYS_ESR)) == ESR_ELx_EC_PAC)
    422		write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
    423
    424	vcpu->arch.ctxt.regs.pstate = read_sysreg_el2(SYS_SPSR);
    425}
    426
    427/*
    428 * Return true when we were able to fixup the guest exit and should return to
    429 * the guest, false when we should restore the host state and return to the
    430 * main run loop.
    431 */
    432static inline bool fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
    433{
    434	/*
    435	 * Save PSTATE early so that we can evaluate the vcpu mode
    436	 * early on.
    437	 */
    438	synchronize_vcpu_pstate(vcpu, exit_code);
    439
    440	/*
    441	 * Check whether we want to repaint the state one way or
    442	 * another.
    443	 */
    444	early_exit_filter(vcpu, exit_code);
    445
    446	if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ)
    447		vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR);
    448
    449	if (ARM_SERROR_PENDING(*exit_code) &&
    450	    ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) {
    451		u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
    452
    453		/*
    454		 * HVC already have an adjusted PC, which we need to
    455		 * correct in order to return to after having injected
    456		 * the SError.
    457		 *
    458		 * SMC, on the other hand, is *trapped*, meaning its
    459		 * preferred return address is the SMC itself.
    460		 */
    461		if (esr_ec == ESR_ELx_EC_HVC32 || esr_ec == ESR_ELx_EC_HVC64)
    462			write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR);
    463	}
    464
    465	/*
    466	 * We're using the raw exception code in order to only process
    467	 * the trap if no SError is pending. We will come back to the
    468	 * same PC once the SError has been injected, and replay the
    469	 * trapping instruction.
    470	 */
    471	if (*exit_code != ARM_EXCEPTION_TRAP)
    472		goto exit;
    473
    474	/* Check if there's an exit handler and allow it to handle the exit. */
    475	if (kvm_hyp_handle_exit(vcpu, exit_code))
    476		goto guest;
    477exit:
    478	/* Return to the host kernel and handle the exit */
    479	return false;
    480
    481guest:
    482	/* Re-enter the guest */
    483	asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412));
    484	return true;
    485}
    486
    487static inline void __kvm_unexpected_el2_exception(void)
    488{
    489	extern char __guest_exit_panic[];
    490	unsigned long addr, fixup;
    491	struct kvm_exception_table_entry *entry, *end;
    492	unsigned long elr_el2 = read_sysreg(elr_el2);
    493
    494	entry = &__start___kvm_ex_table;
    495	end = &__stop___kvm_ex_table;
    496
    497	while (entry < end) {
    498		addr = (unsigned long)&entry->insn + entry->insn;
    499		fixup = (unsigned long)&entry->fixup + entry->fixup;
    500
    501		if (addr != elr_el2) {
    502			entry++;
    503			continue;
    504		}
    505
    506		write_sysreg(fixup, elr_el2);
    507		return;
    508	}
    509
    510	/* Trigger a panic after restoring the hyp context. */
    511	write_sysreg(__guest_exit_panic, elr_el2);
    512}
    513
    514#endif /* __ARM64_KVM_HYP_SWITCH_H__ */