diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index eb186bc57f6a..b96e80934005 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -2,10 +2,9 @@ #ifndef _ASM_X86_KVM_PAGE_TRACK_H #define _ASM_X86_KVM_PAGE_TRACK_H -enum kvm_page_track_mode { - KVM_PAGE_TRACK_WRITE, - KVM_PAGE_TRACK_MAX, -}; +#include + +#include "../../kvm/cachepc/uapi.h" /* * The notifier represented by @kvm_page_track_notifier_node is linked into diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 30f244b64523..a1e3c5ae2f80 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y += -I $(srctree)/arch/x86/kvm +ccflags-y += -I $(srctree)/arch/x86/kvm -O2 ccflags-$(CONFIG_KVM_WERROR) += -Werror ifeq ($(CONFIG_FRAME_POINTER),y) @@ -11,8 +11,8 @@ include $(srctree)/virt/kvm/Makefile.kvm kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ - hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \ - mmu/spte.o + hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \ + cachepc/cachepc.o cachepc/kvm.o cachepc/track.o cachepc/event.o ifdef CONFIG_HYPERV kvm-y += kvm_onhyperv.o @@ -25,7 +25,8 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/evmcs.o vmx/nested.o vmx/posted_intr.o kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o -kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o \ + svm/avic.o svm/sev.o cachepc/cachepc.o cachepc/event.o ifdef CONFIG_HYPERV kvm-amd-y += svm/svm_onhyperv.o diff --git a/arch/x86/kvm/cachepc b/arch/x86/kvm/cachepc new file mode 120000 index 000000000000..9119e44af1f0 --- /dev/null +++ b/arch/x86/kvm/cachepc @@ -0,0 +1 @@ +/home/louis/kvm-prime-count/cachepc \ No newline at end of file diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index de6d44e07e34..b63672b47321 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -26,6 +26,10 @@ #include "trace.h" #include "pmu.h" +#include "cachepc/cachepc.h" +#include "cachepc/uapi.h" +#include "cachepc/event.h" + /* * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be * aligned to sizeof(unsigned long) because it's not accessed via bitops. @@ -1445,8 +1449,8 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu) if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0)) return 1; - eax = kvm_rax_read(vcpu); - ecx = kvm_rcx_read(vcpu); + eax = kvm_rax_read(vcpu); + ecx = kvm_rcx_read(vcpu); kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false); kvm_rax_write(vcpu, eax); kvm_rbx_write(vcpu, ebx); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d871b8dee7b3..c70fff62f1ab 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) } } +#include "../cachepc/mmu.c" + /* * Write-protect on the specified @sptep, @pt_protect indicates whether * spte write-protection is caused by protecting shadow page table. @@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) * * Return true if tlb need be flushed. */ -static bool spte_write_protect(u64 *sptep, bool pt_protect) -{ - u64 spte = *sptep; - - if (!is_writable_pte(spte) && - !(pt_protect && is_mmu_writable_spte(spte))) - return false; - - rmap_printk("spte %p %llx\n", sptep, *sptep); - - if (pt_protect) - spte &= ~shadow_mmu_writable_mask; - spte = spte & ~PT_WRITABLE_MASK; - - return mmu_spte_update(sptep, spte); -} +// static bool spte_write_protect(u64 *sptep, bool pt_protect) +// { +// return cachepc_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE); +// } static bool rmap_write_protect(struct kvm_rmap_head *rmap_head, bool pt_protect) { - u64 *sptep; - struct rmap_iterator iter; - bool flush = false; - - for_each_rmap_spte(rmap_head, &iter, sptep) - flush |= spte_write_protect(sptep, pt_protect); - - return flush; + return cachepc_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE); } static bool spte_clear_dirty(u64 *sptep) @@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, struct kvm_memory_slot *slot, u64 gfn, int min_level) { - struct kvm_rmap_head *rmap_head; - int i; - bool write_protected = false; - - if (kvm_memslots_have_rmaps(kvm)) { - for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { - rmap_head = gfn_to_rmap(gfn, i, slot); - write_protected |= rmap_write_protect(rmap_head, true); - } - } - - if (is_tdp_mmu_enabled(kvm)) - write_protected |= - kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level); - - return write_protected; + return cachepc_kvm_mmu_slot_gfn_protect(kvm, slot, + gfn, min_level, KVM_PAGE_TRACK_WRITE); } static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn) @@ -3901,18 +3870,25 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { - if (unlikely(fault->rsvd)) - return false; + int active; - if (!fault->present || !fault->write) - return false; + cachepc_page_fault_handle(vcpu, fault); /* * guest is writing the page which is write tracked which can * not be fixed by page fault handler. */ - if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE)) - return true; + active = kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE); + active |= kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS); + if (active) return true; + + if (unlikely(fault->rsvd)) + return false; + + if (!fault->present || !fault->write) + return false; return false; } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 2e09d1b6249f..315b2d06118c 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -19,6 +19,9 @@ #include "mmu.h" #include "mmu_internal.h" +#include "../cachepc/cachepc.h" +#include "../cachepc/track.h" + bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) { return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || @@ -115,7 +118,6 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, enum kvm_page_track_mode mode) { - if (WARN_ON(!page_track_mode_is_valid(mode))) return; @@ -123,6 +125,8 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, !kvm_page_track_write_tracking_enabled(kvm))) return; + CPC_DBG("Tracking page: %llu %i\n", gfn, mode); + update_gfn_track(slot, gfn, mode, 1); /* @@ -131,9 +135,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, */ kvm_mmu_gfn_disallow_lpage(slot, gfn); - if (mode == KVM_PAGE_TRACK_WRITE) - if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) - kvm_flush_remote_tlbs(kvm); + if (cachepc_kvm_mmu_slot_gfn_protect(kvm, + slot, gfn, PG_LEVEL_4K, mode)) { + kvm_flush_remote_tlbs(kvm); + } } EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); @@ -161,6 +166,8 @@ void kvm_slot_page_track_remove_page(struct kvm *kvm, !kvm_page_track_write_tracking_enabled(kvm))) return; + CPC_DBG("Untracking page: %llu %i\n", gfn, mode); + update_gfn_track(slot, gfn, mode, -1); /* diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 7b9265d67131..68b9134970da 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1810,13 +1810,8 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, zap_collapsible_spte_range(kvm, root, slot); } -/* - * Removes write access on the last level SPTE mapping this GFN and unsets the - * MMU-writable bit to ensure future writes continue to be intercepted. - * Returns true if an SPTE was set and a TLB flush is needed. - */ -static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t gfn, int min_level) +static bool cachepc_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, + gfn_t gfn, int min_level, int mode) { struct tdp_iter iter; u64 new_spte; @@ -1831,8 +1826,13 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, !is_last_spte(iter.old_spte, iter.level)) continue; - new_spte = iter.old_spte & - ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask); + new_spte = iter.old_spte & ~shadow_mmu_writable_mask; + new_spte &= ~PT_WRITABLE_MASK; + if (mode == KVM_PAGE_TRACK_ACCESS) { + new_spte &= ~PT_PRESENT_MASK; + new_spte &= ~PT_USER_MASK; + new_spte |= (0x1ULL << PT64_NX_SHIFT); + } if (new_spte == iter.old_spte) break; @@ -1846,6 +1846,58 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, return spte_set; } +bool cachepc_tdp_protect_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, int min_level, int mode) +{ + struct kvm_mmu_page *root; + bool spte_set = false; + + // pr_warn("Sevstep: tdp_protect_gfn\n"); + + lockdep_assert_held_write(&kvm->mmu_lock); + for_each_tdp_mmu_root(kvm, root, slot->as_id) + spte_set |= cachepc_protect_gfn(kvm, root, gfn, min_level, mode); + + return spte_set; +} +EXPORT_SYMBOL(cachepc_tdp_protect_gfn); + +/* + * Removes write access on the last level SPTE mapping this GFN and unsets the + * MMU-writable bit to ensure future writes continue to be intercepted. + * Returns true if an SPTE was set and a TLB flush is needed. + */ +// static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, +// gfn_t gfn, int min_level) +// { +// struct tdp_iter iter; +// u64 new_spte; +// bool spte_set = false; +// +// BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); +// +// rcu_read_lock(); +// +// for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) { +// if (!is_shadow_present_pte(iter.old_spte) || +// !is_last_spte(iter.old_spte, iter.level)) +// continue; +// +// new_spte = iter.old_spte & ~shadow_mmu_writable_mask; +// new_spte &= ~PT_WRITABLE_MASK; +// +// if (new_spte == iter.old_spte) +// break; +// +// tdp_mmu_set_spte(kvm, &iter, new_spte); +// spte_set = true; +// } +// +// rcu_read_unlock(); +// +// return spte_set; +// } + /* * Removes write access on the last level SPTE mapping this GFN and unsets the * MMU-writable bit to ensure future writes continue to be intercepted. @@ -1855,14 +1907,17 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, int min_level) { - struct kvm_mmu_page *root; - bool spte_set = false; + return cachepc_tdp_protect_gfn(kvm, slot, gfn, min_level, + KVM_PAGE_TRACK_WRITE); - lockdep_assert_held_write(&kvm->mmu_lock); - for_each_tdp_mmu_root(kvm, root, slot->as_id) - spte_set |= write_protect_gfn(kvm, root, gfn, min_level); + // struct kvm_mmu_page *root; + // bool spte_set = false; - return spte_set; + // lockdep_assert_held_write(&kvm->mmu_lock); + // for_each_tdp_mmu_root(kvm, root, slot->as_id) + // spte_set |= write_protect_gfn(kvm, root, gfn, min_level); + + // return spte_set; } /* diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index a4f6d10b0ef3..0c5aae1de162 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -35,6 +35,8 @@ #include "trace.h" #include "mmu.h" +#include "cachepc/cachepc.h" + #ifndef CONFIG_KVM_AMD_SEV /* * When this config is not defined, SEV feature is not supported and APIs in @@ -888,7 +890,7 @@ static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src, &data, error); } -static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, +int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, unsigned long dst_paddr, int sz, int *err) { int offset; @@ -904,12 +906,20 @@ static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr, return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false); } + +int sev_dbg_decrypt_ext(struct kvm *kvm, unsigned long src_paddr, + unsigned long dst_paddr, int sz, int *err) { + return __sev_dbg_decrypt(kvm, src_paddr, dst_paddr, sz, err); +} +EXPORT_SYMBOL(sev_dbg_decrypt_ext); + static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, void __user *dst_uaddr, unsigned long dst_paddr, int size, int *err) { struct page *tpage = NULL; + struct vcpu_svm *svm; int ret, offset; /* if inputs are not 16-byte then use intermediate buffer */ @@ -923,6 +933,11 @@ static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, dst_paddr = __sme_page_pa(tpage); } + if (dst_uaddr == CPC_VMSA_MAGIC_ADDR) { + svm = to_svm(xa_load(&kvm->vcpu_array, 0)); + paddr = __pa(svm->sev_es.vmsa); + } + ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err); if (ret) goto e_free; @@ -1024,6 +1039,7 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) struct kvm_sev_dbg debug; unsigned long n; unsigned int size; + bool vmsa_dec; int ret; if (!sev_guest(kvm)) @@ -1037,6 +1053,13 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) if (!debug.dst_uaddr) return -EINVAL; + vmsa_dec = false; + if (debug.src_uaddr == (uintptr_t) CPC_VMSA_MAGIC_ADDR) { + debug.len = PAGE_SIZE; + debug.src_uaddr = debug.dst_uaddr; + vmsa_dec = true; + } + vaddr = debug.src_uaddr; size = debug.len; vaddr_end = vaddr + size; @@ -1075,7 +1098,8 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) if (dec) ret = __sev_dbg_decrypt_user(kvm, __sme_page_pa(src_p[0]) + s_off, - (void __user *)dst_vaddr, + vmsa_dec ? CPC_VMSA_MAGIC_ADDR + : (void __user *)dst_vaddr, __sme_page_pa(dst_p[0]) + d_off, len, &argp->error); else @@ -3149,9 +3173,9 @@ static int sev_es_validate_vmgexit(struct vcpu_svm *svm, u64 *exit_code) } break; case SVM_EXIT_VMMCALL: - if (!ghcb_rax_is_valid(ghcb) || - !ghcb_cpl_is_valid(ghcb)) - goto vmgexit_err; + // if (!ghcb_rax_is_valid(ghcb) || + // !ghcb_cpl_is_valid(ghcb)) + // goto vmgexit_err; break; case SVM_EXIT_RDTSCP: break; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cf0bf456d520..ff84cedfefd0 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2,6 +2,10 @@ #include +#include "cachepc/cachepc.h" +#include "cachepc/event.h" +#include "cachepc/track.h" + #include "irq.h" #include "mmu.h" #include "kvm_cache_regs.h" @@ -1887,6 +1891,8 @@ static int npf_interception(struct kvm_vcpu *vcpu) u64 fault_address = svm->vmcb->control.exit_info_2; u64 error_code = svm->vmcb->control.exit_info_1; + cachepc_track_state_next = cachepc_track_state; + trace_kvm_page_fault(fault_address, error_code); rc = kvm_mmu_page_fault(vcpu, fault_address, error_code, static_cpu_has(X86_FEATURE_DECODEASSISTS) ? @@ -1896,6 +1902,8 @@ static int npf_interception(struct kvm_vcpu *vcpu) if (error_code & PFERR_GUEST_RMP_MASK) handle_rmp_page_fault(vcpu, fault_address, error_code); + cachepc_track_state = cachepc_track_state_next; + return rc; } @@ -2083,7 +2091,56 @@ static int smi_interception(struct kvm_vcpu *vcpu) static int intr_interception(struct kvm_vcpu *vcpu) { + struct vcpu_svm *svm; + ++vcpu->stat.irq_exits; + + if (cachepc_track_mode == CPC_TRACK_DATA_ACCESS && cachepc_single_step) { + svm = to_svm(vcpu); + + if (svm->sev_es.vmsa->rip == cachepc_prev_rip) { + cachepc_apic_timer += 1; + return 1; + } + + cachepc_single_step = false; + + switch (cachepc_track_state) { + case CPC_TRACK_AWAIT_DATA_FAULT: + CPC_INFO("Caught single step WITHOUT data!\n"); + + cachepc_track_single(vcpu, cachepc_inst_fault_gfn, + KVM_PAGE_TRACK_ACCESS); + cachepc_inst_fault_avail = false; + + cachepc_send_track_event( + cachepc_inst_fault_gfn, cachepc_inst_fault_err, + 0, 0); + + cachepc_track_state = CPC_TRACK_AWAIT_INST_FAULT; + break; + case CPC_TRACK_AWAIT_STEP_INTR: + CPC_INFO("Caught single step WITH data!\n"); + + cachepc_track_single(vcpu, cachepc_data_fault_gfn, + KVM_PAGE_TRACK_ACCESS); + cachepc_data_fault_avail = false; + + cachepc_track_single(vcpu, cachepc_inst_fault_gfn, + KVM_PAGE_TRACK_ACCESS); + cachepc_inst_fault_avail = false; + + cachepc_send_track_event( + cachepc_inst_fault_gfn, cachepc_inst_fault_err, + cachepc_data_fault_gfn, cachepc_data_fault_err); + + cachepc_track_state = CPC_TRACK_AWAIT_INST_FAULT; + break; + default: + CPC_ERR("Unexpected single step\n"); + } + } + return 1; } @@ -3269,9 +3326,25 @@ static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code) int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code) { + static const struct { + u64 code; + const char *name; + } codelut[] = { + SVM_EXIT_REASONS, + { -1, NULL } + }; + size_t i; + if (!svm_check_exit_valid(exit_code)) return svm_handle_invalid_exit(vcpu, exit_code); + if (cachepc_debug && cachepc_track_mode != CPC_TRACK_NONE) { + for (i = 0; i < sizeof(codelut) / sizeof(codelut[0]); i++) { + if (codelut[i].code == exit_code) + pr_warn("KVM EXIT (%s)\n", codelut[i].name); + } + } + #ifdef CONFIG_RETPOLINE if (exit_code == SVM_EXIT_MSR) return msr_interception(vcpu); @@ -3788,14 +3861,42 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); unsigned long vmcb_pa = svm->current_vmcb->pa; + int cpu; guest_state_enter_irqoff(); + if (cachepc_apic_timer == 0) { + cachepc_prev_rip = svm->sev_es.vmsa->rip; + cachepc_apic_timer = 100; + } + if (sev_es_guest(vcpu->kvm)) { + cpu = get_cpu(); + // local_irq_disable(); + WARN_ON(cpu != 2); + + memset(cachepc_msrmts, 0, + cachepc_msrmts_count * sizeof(cpc_msrmt_t)); + + cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC); __svm_sev_es_vcpu_run(vmcb_pa); + cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC) - cachepc_retinst; + + cachepc_save_msrmts(cachepc_ds); + if (cachepc_baseline_measure) + cachepc_update_baseline(); + // local_irq_enable(); + put_cpu(); } else { struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + cpu = get_cpu(); + // local_irq_disable(); + WARN_ON(cpu != 2); + + memset(cachepc_msrmts, 0, + cachepc_msrmts_count * sizeof(cpc_msrmt_t)); + /* * Use a single vmcb (vmcb01 because it's always valid) for * context switching guest state via VMLOAD/VMSAVE, that way @@ -3806,7 +3907,15 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs); vmsave(svm->vmcb01.pa); + cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC); vmload(__sme_page_pa(sd->save_area)); + cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC) - cachepc_retinst; + + cachepc_save_msrmts(cachepc_ds); + if (cachepc_baseline_measure) + cachepc_update_baseline(); + // local_irq_enable(); + put_cpu(); } guest_state_exit_irqoff(); diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index dfaeb47fcf2a..0626f3fdddfd 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -29,12 +29,59 @@ .section .noinstr.text, "ax" +.extern cachepc_msrmts +.extern cachepc_regs_tmp +.extern cachepc_regs_vm + +.macro load_tmp off reg + mov cachepc_regs_tmp+\off(%rip), \reg +.endm + +.macro save_tmp off reg + mov \reg, cachepc_regs_tmp+\off(%rip) +.endm + +.macro load_vm off reg + mov cachepc_regs_vm+\off(%rip), \reg +.endm + +.macro save_vm off reg + mov \reg, cachepc_regs_vm+\off(%rip) +.endm + +.macro apply_regs func + \func 0x00, %rax + \func 0x08, %rbx + \func 0x10, %rcx + \func 0x18, %rdx + \func 0x20, %rbp + \func 0x28, %rsp + \func 0x30, %rdi + \func 0x38, %rsi + \func 0x40, %r8 + \func 0x48, %r9 + \func 0x50, %r10 + \func 0x58, %r11 + \func 0x60, %r12 + \func 0x68, %r13 + \func 0x70, %r14 + \func 0x78, %r15 +.endm + +.macro barrier + mfence + mov $0x80000005,%eax + cpuid +.endm + /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode * @vmcb_pa: unsigned long * @regs: unsigned long * (to guest registers) */ SYM_FUNC_START(__svm_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushs +sev_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli @@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run) * @vmcb_pa: unsigned long */ SYM_FUNC_START(__svm_sev_es_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_es_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushes +sev_es_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_es_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_es_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d9adf79124f9..fc64f9b7614d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -82,6 +82,10 @@ #include #include +#include "cachepc/cachepc.h" +#include "cachepc/event.h" +#include "cachepc/track.h" + #define CREATE_TRACE_POINTS #include "trace.h" @@ -9267,10 +9271,10 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) a3 &= 0xFFFFFFFF; } - if (static_call(kvm_x86_get_cpl)(vcpu) != 0) { - ret = -KVM_EPERM; - goto out; - } + // if (static_call(kvm_x86_get_cpl)(vcpu) != 0) { + // ret = -KVM_EPERM; + // goto out; + // } ret = -KVM_ENOSYS; @@ -9326,11 +9330,16 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) vcpu->arch.complete_userspace_io = complete_hypercall_exit; return 0; } + case KVM_HC_CPC_VMMCALL: + CPC_WARN("Cachepc: Hypecrcall Run\n"); + cachepc_send_cpuid_event(a0, a1); + ret = 0; + break; default: ret = -KVM_ENOSYS; break; } -out: +//out: if (!op_64_bit) ret = (u32)ret; kvm_rax_write(vcpu, ret); diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index 27ab27931813..90679ec8ba79 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -1173,8 +1173,78 @@ EXPORT_SYMBOL_GPL(crypto_aes_set_key); f_rl(bo, bi, 3, k); \ } while (0) +#define L1_ASSOC 8 +#define L1_LINESIZE 64 +#define L1_SETS 64 +#define L1_SIZE (L1_SETS * L1_ASSOC * L1_LINESIZE) + +#define ACCESS_LINE(n) \ + asm volatile ("mov (%0), %%rbx" \ + : : "r"(((uint8_t*) L1) + n * L1_LINESIZE) : "rbx"); + +#define DO_ACCESS_PATTERN() \ + ACCESS_LINE(60) \ + ACCESS_LINE(13) \ + ACCESS_LINE(24) \ + ACCESS_LINE(19) \ + ACCESS_LINE(38) \ + ACCESS_LINE(17) \ + ACCESS_LINE( 2) \ + ACCESS_LINE(12) \ + ACCESS_LINE(22) \ + ACCESS_LINE(46) \ + ACCESS_LINE( 4) \ + ACCESS_LINE(61) \ + ACCESS_LINE( 5) \ + ACCESS_LINE(14) \ + ACCESS_LINE(11) \ + ACCESS_LINE(35) \ + ACCESS_LINE(45) \ + ACCESS_LINE(10) \ + ACCESS_LINE(49) \ + ACCESS_LINE(56) \ + ACCESS_LINE(27) \ + ACCESS_LINE(37) \ + ACCESS_LINE(63) \ + ACCESS_LINE(54) \ + ACCESS_LINE(55) \ + ACCESS_LINE(29) \ + ACCESS_LINE(48) \ + ACCESS_LINE( 9) \ + ACCESS_LINE(16) \ + ACCESS_LINE(39) \ + ACCESS_LINE(20) \ + ACCESS_LINE(21) \ + ACCESS_LINE(62) \ + ACCESS_LINE( 0) \ + ACCESS_LINE(34) \ + ACCESS_LINE( 8) \ + ACCESS_LINE(53) \ + ACCESS_LINE(42) \ + ACCESS_LINE(51) \ + ACCESS_LINE(50) \ + ACCESS_LINE(57) \ + ACCESS_LINE( 7) \ + ACCESS_LINE( 6) \ + ACCESS_LINE(33) \ + ACCESS_LINE(26) \ + +uint8_t *L1 = NULL; + static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { + int cpu; + + if (L1 == NULL) { + L1 = kzalloc(L1_SETS * L1_LINESIZE, GFP_KERNEL); + BUG_ON(((uintptr_t)L1) % (L1_SETS * L1_LINESIZE) != 0); + } + + pr_warn("CachePC-TEST: Running AES-Generic!"); + + cpu = get_cpu(); + DO_ACCESS_PATTERN() + const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); u32 b0[4], b1[4]; const u32 *kp = ctx->key_enc + 4; @@ -1210,6 +1280,9 @@ static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) put_unaligned_le32(b0[1], out + 4); put_unaligned_le32(b0[2], out + 8); put_unaligned_le32(b0[3], out + 12); + + DO_ACCESS_PATTERN(); + put_cpu(); } /* decrypt a block of text */ diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c old mode 100644 new mode 100755 index e089fbf9017f..7899e1efe852 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -87,7 +87,7 @@ static void *sev_init_ex_buffer; static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE; static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); -static int sev_do_cmd(int cmd, void *data, int *psp_ret); +int sev_do_cmd(int cmd, void *data, int *psp_ret); static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { @@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) return ret; } -static int sev_do_cmd(int cmd, void *data, int *psp_ret) +int sev_do_cmd(int cmd, void *data, int *psp_ret) { int rc; @@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret) return rc; } +EXPORT_SYMBOL(sev_do_cmd); static int __sev_init_locked(int *error) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f2a63cb2658b..869faf927e5d 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -13,6 +13,7 @@ * Yaniv Kamay */ +#include #include #include @@ -64,12 +65,15 @@ #define CREATE_TRACE_POINTS #include +#include "../../arch/x86/kvm/cachepc/track.h" #include /* Worst case buffer size needed for holding an integer. */ #define ITOA_MAX_LEN 12 +#include "../../arch/x86/kvm/cachepc/kvm.h" + MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -1261,6 +1265,9 @@ static void kvm_destroy_vm(struct kvm *kvm) hardware_disable_all(); mmdrop(mm); module_put(kvm_chardev_ops.owner); + + if (main_vm == kvm) + main_vm = NULL; } void kvm_get_kvm(struct kvm *kvm) @@ -1360,7 +1367,7 @@ static void kvm_insert_gfn_node(struct kvm_memslots *slots, int idx = slots->node_idx; parent = NULL; - for (node = &gfn_tree->rb_node; *node; ) { + for (node = &gfn_tree->rb_node; *node;) { struct kvm_memory_slot *tmp; tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]); @@ -4514,7 +4521,7 @@ static long kvm_vm_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; - if (kvm->mm != current->mm || kvm->vm_dead) + if ((ioctl != KVM_MEMORY_ENCRYPT_OP && kvm->mm != current->mm) || kvm->vm_dead) return -EIO; switch (ioctl) { case KVM_CREATE_VCPU: @@ -4823,6 +4830,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); fd_install(r, file); + + main_vm = kvm; + return r; put_kvm: @@ -4864,7 +4874,7 @@ static long kvm_dev_ioctl(struct file *filp, r = -EOPNOTSUPP; break; default: - return kvm_arch_dev_ioctl(filp, ioctl, arg); + return cachepc_kvm_ioctl(filp, ioctl, arg); } out: return r; @@ -5792,6 +5802,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, r = kvm_vfio_ops_init(); WARN_ON(r); + cachepc_kvm_init(); + return 0; out_unreg: @@ -5821,6 +5833,8 @@ void kvm_exit(void) { int cpu; + cachepc_kvm_exit(); + debugfs_remove_recursive(kvm_debugfs_dir); misc_deregister(&kvm_dev); for_each_possible_cpu(cpu)