cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

tlb.c (3963B)

      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (C) 2015 - ARM Ltd
      4 * Author: Marc Zyngier <marc.zyngier@arm.com>
      5 */
      6
      7#include <linux/irqflags.h>
      8
      9#include <asm/kvm_hyp.h>
     10#include <asm/kvm_mmu.h>
     11#include <asm/tlbflush.h>
     12
     13struct tlb_inv_context {
     14	unsigned long	flags;
     15	u64		tcr;
     16	u64		sctlr;
     17};
     18
     19static void __tlb_switch_to_guest(struct kvm_s2_mmu *mmu,
     20				  struct tlb_inv_context *cxt)
     21{
     22	u64 val;
     23
     24	local_irq_save(cxt->flags);
     25
     26	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
     27		/*
     28		 * For CPUs that are affected by ARM errata 1165522 or 1530923,
     29		 * we cannot trust stage-1 to be in a correct state at that
     30		 * point. Since we do not want to force a full load of the
     31		 * vcpu state, we prevent the EL1 page-table walker to
     32		 * allocate new TLBs. This is done by setting the EPD bits
     33		 * in the TCR_EL1 register. We also need to prevent it to
     34		 * allocate IPA->PA walks, so we enable the S1 MMU...
     35		 */
     36		val = cxt->tcr = read_sysreg_el1(SYS_TCR);
     37		val |= TCR_EPD1_MASK | TCR_EPD0_MASK;
     38		write_sysreg_el1(val, SYS_TCR);
     39		val = cxt->sctlr = read_sysreg_el1(SYS_SCTLR);
     40		val |= SCTLR_ELx_M;
     41		write_sysreg_el1(val, SYS_SCTLR);
     42	}
     43
     44	/*
     45	 * With VHE enabled, we have HCR_EL2.{E2H,TGE} = {1,1}, and
     46	 * most TLB operations target EL2/EL0. In order to affect the
     47	 * guest TLBs (EL1/EL0), we need to change one of these two
     48	 * bits. Changing E2H is impossible (goodbye TTBR1_EL2), so
     49	 * let's flip TGE before executing the TLB operation.
     50	 *
     51	 * ARM erratum 1165522 requires some special handling (again),
     52	 * as we need to make sure both stages of translation are in
     53	 * place before clearing TGE. __load_stage2() already
     54	 * has an ISB in order to deal with this.
     55	 */
     56	__load_stage2(mmu, mmu->arch);
     57	val = read_sysreg(hcr_el2);
     58	val &= ~HCR_TGE;
     59	write_sysreg(val, hcr_el2);
     60	isb();
     61}
     62
     63static void __tlb_switch_to_host(struct tlb_inv_context *cxt)
     64{
     65	/*
     66	 * We're done with the TLB operation, let's restore the host's
     67	 * view of HCR_EL2.
     68	 */
     69	write_sysreg(0, vttbr_el2);
     70	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
     71	isb();
     72
     73	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
     74		/* Restore the registers to what they were */
     75		write_sysreg_el1(cxt->tcr, SYS_TCR);
     76		write_sysreg_el1(cxt->sctlr, SYS_SCTLR);
     77	}
     78
     79	local_irq_restore(cxt->flags);
     80}
     81
     82void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
     83			      phys_addr_t ipa, int level)
     84{
     85	struct tlb_inv_context cxt;
     86
     87	dsb(ishst);
     88
     89	/* Switch to requested VMID */
     90	__tlb_switch_to_guest(mmu, &cxt);
     91
     92	/*
     93	 * We could do so much better if we had the VA as well.
     94	 * Instead, we invalidate Stage-2 for this IPA, and the
     95	 * whole of Stage-1. Weep...
     96	 */
     97	ipa >>= 12;
     98	__tlbi_level(ipas2e1is, ipa, level);
     99
    100	/*
    101	 * We have to ensure completion of the invalidation at Stage-2,
    102	 * since a table walk on another CPU could refill a TLB with a
    103	 * complete (S1 + S2) walk based on the old Stage-2 mapping if
    104	 * the Stage-1 invalidation happened first.
    105	 */
    106	dsb(ish);
    107	__tlbi(vmalle1is);
    108	dsb(ish);
    109	isb();
    110
    111	__tlb_switch_to_host(&cxt);
    112}
    113
    114void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
    115{
    116	struct tlb_inv_context cxt;
    117
    118	dsb(ishst);
    119
    120	/* Switch to requested VMID */
    121	__tlb_switch_to_guest(mmu, &cxt);
    122
    123	__tlbi(vmalls12e1is);
    124	dsb(ish);
    125	isb();
    126
    127	__tlb_switch_to_host(&cxt);
    128}
    129
    130void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu)
    131{
    132	struct tlb_inv_context cxt;
    133
    134	/* Switch to requested VMID */
    135	__tlb_switch_to_guest(mmu, &cxt);
    136
    137	__tlbi(vmalle1);
    138	asm volatile("ic iallu");
    139	dsb(nsh);
    140	isb();
    141
    142	__tlb_switch_to_host(&cxt);
    143}
    144
    145void __kvm_flush_vm_context(void)
    146{
    147	dsb(ishst);
    148	__tlbi(alle1is);
    149
    150	/*
    151	 * VIPT and PIPT caches are not affected by VMID, so no maintenance
    152	 * is necessary across a VMID rollover.
    153	 *
    154	 * VPIPT caches constrain lookup and maintenance to the active VMID,
    155	 * so we need to invalidate lines with a stale VMID to avoid an ABA
    156	 * race after multiple rollovers.
    157	 *
    158	 */
    159	if (icache_is_vpipt())
    160		asm volatile("ic ialluis");
    161
    162	dsb(ish);
    163}