diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index eb186bc57f6a..cefc1589e398 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -2,8 +2,14 @@ #ifndef _ASM_X86_KVM_PAGE_TRACK_H #define _ASM_X86_KVM_PAGE_TRACK_H +#include + enum kvm_page_track_mode { KVM_PAGE_TRACK_WRITE, + KVM_PAGE_TRACK_ACCESS, + KVM_PAGE_TRACK_RESET_ACCESSED, //TODO: hacky, as this is not really for page tracking + KVM_PAGE_TRACK_EXEC, + KVM_PAGE_TRACK_RESET_EXEC, KVM_PAGE_TRACK_MAX, }; diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 30f244b64523..7992f8cce838 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,8 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y += -I $(srctree)/arch/x86/kvm +ccflags-y += -I $(srctree)/arch/x86/kvm -O2 ccflags-$(CONFIG_KVM_WERROR) += -Werror +KBUILD_EXTRA_SYMBOLS := ../../../drivers/crypto/ccp/Module.symvers + ifeq ($(CONFIG_FRAME_POINTER),y) OBJECT_FILES_NON_STANDARD_vmenter.o := y endif @@ -11,8 +13,9 @@ include $(srctree)/virt/kvm/Makefile.kvm kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ - hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \ - mmu/spte.o + hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \ + svm/cachepc/cachepc.o svm/cachepc/util.o svm/cachepc/kvm.o \ + sevstep/sevstep.o sevstep/uspt.o sevstep/kvm.o ifdef CONFIG_HYPERV kvm-y += kvm_onhyperv.o @@ -25,7 +28,8 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/evmcs.o vmx/nested.o vmx/posted_intr.o kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o -kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o \ + svm/cachepc/cachepc.o svm/cachepc/util.o ifdef CONFIG_HYPERV kvm-amd-y += svm/svm_onhyperv.o diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d871b8dee7b3..32900ef5ee0b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) } } +#include "../sevstep/mmu.c" + /* * Write-protect on the specified @sptep, @pt_protect indicates whether * spte write-protection is caused by protecting shadow page table. @@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) * * Return true if tlb need be flushed. */ -static bool spte_write_protect(u64 *sptep, bool pt_protect) -{ - u64 spte = *sptep; - - if (!is_writable_pte(spte) && - !(pt_protect && is_mmu_writable_spte(spte))) - return false; - - rmap_printk("spte %p %llx\n", sptep, *sptep); - - if (pt_protect) - spte &= ~shadow_mmu_writable_mask; - spte = spte & ~PT_WRITABLE_MASK; - - return mmu_spte_update(sptep, spte); -} +// static bool spte_write_protect(u64 *sptep, bool pt_protect) +// { +// return sevstep_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE); +// } static bool rmap_write_protect(struct kvm_rmap_head *rmap_head, bool pt_protect) { - u64 *sptep; - struct rmap_iterator iter; - bool flush = false; - - for_each_rmap_spte(rmap_head, &iter, sptep) - flush |= spte_write_protect(sptep, pt_protect); - - return flush; + return sevstep_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE); } static bool spte_clear_dirty(u64 *sptep) @@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, struct kvm_memory_slot *slot, u64 gfn, int min_level) { - struct kvm_rmap_head *rmap_head; - int i; - bool write_protected = false; - - if (kvm_memslots_have_rmaps(kvm)) { - for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { - rmap_head = gfn_to_rmap(gfn, i, slot); - write_protected |= rmap_write_protect(rmap_head, true); - } - } - - if (is_tdp_mmu_enabled(kvm)) - write_protected |= - kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level); - - return write_protected; + return sevstep_kvm_mmu_slot_gfn_protect(kvm, slot, + gfn, min_level, KVM_PAGE_TRACK_WRITE); } static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn) @@ -3901,6 +3870,10 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { + int active; + + sevstep_uspt_page_fault_handle(vcpu, fault); + if (unlikely(fault->rsvd)) return false; @@ -3911,8 +3884,11 @@ static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, * guest is writing the page which is write tracked which can * not be fixed by page fault handler. */ - if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE)) - return true; + active = kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE); + active |= kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS); + if (active) return true; return false; } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 2e09d1b6249f..17b69a1f2b40 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -19,6 +19,8 @@ #include "mmu.h" #include "mmu_internal.h" +#include "../sevstep/sevstep.h" + bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) { return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || @@ -131,9 +133,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, */ kvm_mmu_gfn_disallow_lpage(slot, gfn); - if (mode == KVM_PAGE_TRACK_WRITE) - if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) - kvm_flush_remote_tlbs(kvm); + if (sevstep_kvm_mmu_slot_gfn_protect(kvm, + slot, gfn, PG_LEVEL_4K, mode)) { + kvm_flush_remote_tlbs(kvm); + } } EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); diff --git a/arch/x86/kvm/sevstep b/arch/x86/kvm/sevstep new file mode 120000 index 000000000000..642ea24bf098 --- /dev/null +++ b/arch/x86/kvm/sevstep @@ -0,0 +1 @@ +/home/louis/kvm-prime-count/sevstep \ No newline at end of file diff --git a/arch/x86/kvm/svm/cachepc b/arch/x86/kvm/svm/cachepc new file mode 120000 index 000000000000..9119e44af1f0 --- /dev/null +++ b/arch/x86/kvm/svm/cachepc @@ -0,0 +1 @@ +/home/louis/kvm-prime-count/cachepc \ No newline at end of file diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cf0bf456d520..4dbb8041541f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2,6 +2,8 @@ #include +#include "cachepc/cachepc.h" + #include "irq.h" #include "mmu.h" #include "kvm_cache_regs.h" @@ -3788,14 +3790,28 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); unsigned long vmcb_pa = svm->current_vmcb->pa; + int cpu; guest_state_enter_irqoff(); if (sev_es_guest(vcpu->kvm)) { + memset(cachepc_msrmts, 0, 64 * 2); + cpu = get_cpu(); + local_irq_disable(); + WARN_ON(cpu != 2); __svm_sev_es_vcpu_run(vmcb_pa); + cachepc_save_msrmts(cachepc_ds); + local_irq_enable(); + put_cpu(); } else { struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + memset(cachepc_msrmts, 0, 64 * 2); + cpu = get_cpu(); + local_irq_disable(); + WARN_ON(cpu != 2); + /* TODO: try closer to vcpu_run */ + /* * Use a single vmcb (vmcb01 because it's always valid) for * context switching guest state via VMLOAD/VMSAVE, that way @@ -3807,6 +3823,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) vmsave(svm->vmcb01.pa); vmload(__sme_page_pa(sd->save_area)); + + cachepc_save_msrmts(cachepc_ds); + local_irq_enable(); + put_cpu(); } guest_state_exit_irqoff(); diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index dfaeb47fcf2a..0626f3fdddfd 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -29,12 +29,59 @@ .section .noinstr.text, "ax" +.extern cachepc_msrmts +.extern cachepc_regs_tmp +.extern cachepc_regs_vm + +.macro load_tmp off reg + mov cachepc_regs_tmp+\off(%rip), \reg +.endm + +.macro save_tmp off reg + mov \reg, cachepc_regs_tmp+\off(%rip) +.endm + +.macro load_vm off reg + mov cachepc_regs_vm+\off(%rip), \reg +.endm + +.macro save_vm off reg + mov \reg, cachepc_regs_vm+\off(%rip) +.endm + +.macro apply_regs func + \func 0x00, %rax + \func 0x08, %rbx + \func 0x10, %rcx + \func 0x18, %rdx + \func 0x20, %rbp + \func 0x28, %rsp + \func 0x30, %rdi + \func 0x38, %rsi + \func 0x40, %r8 + \func 0x48, %r9 + \func 0x50, %r10 + \func 0x58, %r11 + \func 0x60, %r12 + \func 0x68, %r13 + \func 0x70, %r14 + \func 0x78, %r15 +.endm + +.macro barrier + mfence + mov $0x80000005,%eax + cpuid +.endm + /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode * @vmcb_pa: unsigned long * @regs: unsigned long * (to guest registers) */ SYM_FUNC_START(__svm_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushs +sev_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli @@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run) * @vmcb_pa: unsigned long */ SYM_FUNC_START(__svm_sev_es_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_es_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushes +sev_es_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_es_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_es_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d9adf79124f9..1809b79cb6cd 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -82,6 +82,8 @@ #include #include +#include "sevstep/kvm.h" + #define CREATE_TRACE_POINTS #include "trace.h" diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index e089fbf9017f..7899e1efe852 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -87,7 +87,7 @@ static void *sev_init_ex_buffer; static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE; static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); -static int sev_do_cmd(int cmd, void *data, int *psp_ret); +int sev_do_cmd(int cmd, void *data, int *psp_ret); static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { @@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) return ret; } -static int sev_do_cmd(int cmd, void *data, int *psp_ret) +int sev_do_cmd(int cmd, void *data, int *psp_ret) { int rc; @@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret) return rc; } +EXPORT_SYMBOL(sev_do_cmd); static int __sev_init_locked(int *error) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f2a63cb2658b..bfe4a57bcc10 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -70,6 +70,10 @@ /* Worst case buffer size needed for holding an integer. */ #define ITOA_MAX_LEN 12 +#include "../../arch/x86/kvm/svm/cachepc/kvm.h" +#include "../../arch/x86/kvm/sevstep/sevstep.h" +#include "../../arch/x86/kvm/sevstep/uspt.h" + MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -5792,6 +5796,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, r = kvm_vfio_ops_init(); WARN_ON(r); + cachepc_kvm_init(); + return 0; out_unreg: @@ -5821,6 +5827,8 @@ void kvm_exit(void) { int cpu; + cachepc_kvm_exit(); + debugfs_remove_recursive(kvm_debugfs_dir); misc_deregister(&kvm_dev); for_each_possible_cpu(cpu)