diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index eb186bc57f6a..b96e80934005 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -2,10 +2,9 @@ #ifndef _ASM_X86_KVM_PAGE_TRACK_H #define _ASM_X86_KVM_PAGE_TRACK_H -enum kvm_page_track_mode { - KVM_PAGE_TRACK_WRITE, - KVM_PAGE_TRACK_MAX, -}; +#include + +#include "../../kvm/cachepc/uapi.h" /* * The notifier represented by @kvm_page_track_notifier_node is linked into diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 30f244b64523..568cc761f0e5 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y += -I $(srctree)/arch/x86/kvm +ccflags-y += -I $(srctree)/arch/x86/kvm -O2 ccflags-$(CONFIG_KVM_WERROR) += -Werror ifeq ($(CONFIG_FRAME_POINTER),y) @@ -11,8 +11,9 @@ include $(srctree)/virt/kvm/Makefile.kvm kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ - hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \ - mmu/spte.o + hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \ + cachepc/cachepc.o cachepc/kvm.o \ + cachepc/sevstep.o cachepc/uspt.o ifdef CONFIG_HYPERV kvm-y += kvm_onhyperv.o @@ -25,7 +26,8 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/evmcs.o vmx/nested.o vmx/posted_intr.o kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o -kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o \ + svm/avic.o svm/sev.o cachepc/cachepc.o cachepc/uspt.o ifdef CONFIG_HYPERV kvm-amd-y += svm/svm_onhyperv.o diff --git a/arch/x86/kvm/cachepc b/arch/x86/kvm/cachepc new file mode 120000 index 000000000000..9119e44af1f0 --- /dev/null +++ b/arch/x86/kvm/cachepc @@ -0,0 +1 @@ +/home/louis/kvm-prime-count/cachepc \ No newline at end of file diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d871b8dee7b3..3b7720aebbc6 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) } } +#include "../cachepc/mmu.c" + /* * Write-protect on the specified @sptep, @pt_protect indicates whether * spte write-protection is caused by protecting shadow page table. @@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) * * Return true if tlb need be flushed. */ -static bool spte_write_protect(u64 *sptep, bool pt_protect) -{ - u64 spte = *sptep; - - if (!is_writable_pte(spte) && - !(pt_protect && is_mmu_writable_spte(spte))) - return false; - - rmap_printk("spte %p %llx\n", sptep, *sptep); - - if (pt_protect) - spte &= ~shadow_mmu_writable_mask; - spte = spte & ~PT_WRITABLE_MASK; - - return mmu_spte_update(sptep, spte); -} +// static bool spte_write_protect(u64 *sptep, bool pt_protect) +// { +// return sevstep_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE); +// } static bool rmap_write_protect(struct kvm_rmap_head *rmap_head, bool pt_protect) { - u64 *sptep; - struct rmap_iterator iter; - bool flush = false; - - for_each_rmap_spte(rmap_head, &iter, sptep) - flush |= spte_write_protect(sptep, pt_protect); - - return flush; + return sevstep_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE); } static bool spte_clear_dirty(u64 *sptep) @@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, struct kvm_memory_slot *slot, u64 gfn, int min_level) { - struct kvm_rmap_head *rmap_head; - int i; - bool write_protected = false; - - if (kvm_memslots_have_rmaps(kvm)) { - for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { - rmap_head = gfn_to_rmap(gfn, i, slot); - write_protected |= rmap_write_protect(rmap_head, true); - } - } - - if (is_tdp_mmu_enabled(kvm)) - write_protected |= - kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level); - - return write_protected; + return sevstep_kvm_mmu_slot_gfn_protect(kvm, slot, + gfn, min_level, KVM_PAGE_TRACK_WRITE); } static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn) @@ -3901,6 +3870,10 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { + int active; + + sevstep_uspt_page_fault_handle(vcpu, fault); + if (unlikely(fault->rsvd)) return false; @@ -3911,8 +3884,11 @@ static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, * guest is writing the page which is write tracked which can * not be fixed by page fault handler. */ - if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE)) - return true; + active = kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE); + active |= kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS); + if (active) return true; return false; } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 2e09d1b6249f..9b40e71564bf 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -19,6 +19,8 @@ #include "mmu.h" #include "mmu_internal.h" +#include "../cachepc/sevstep.h" + bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) { return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || @@ -131,9 +133,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, */ kvm_mmu_gfn_disallow_lpage(slot, gfn); - if (mode == KVM_PAGE_TRACK_WRITE) - if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) - kvm_flush_remote_tlbs(kvm); + if (sevstep_kvm_mmu_slot_gfn_protect(kvm, + slot, gfn, PG_LEVEL_4K, mode)) { + kvm_flush_remote_tlbs(kvm); + } } EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cf0bf456d520..1e1667dc8f96 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2,6 +2,9 @@ #include +#include "cachepc/cachepc.h" +#include "cachepc/uspt.h" + #include "irq.h" #include "mmu.h" #include "kvm_cache_regs.h" @@ -2083,6 +2086,17 @@ static int smi_interception(struct kvm_vcpu *vcpu) static int intr_interception(struct kvm_vcpu *vcpu) { + int err; + + if (cachepc_track_single_step && cachepc_single_step) { + pr_warn("CachePC: Caught single step interrupt\n"); + cachepc_single_step = false; + + err = sevstep_uspt_send_and_block(cachepc_last_fault_gfn, + cachepc_last_fault_err); + if (err) pr_warn("Sevstep: uspt_send_and_block failed (%d)\n", err); + } + ++vcpu->stat.irq_exits; return 1; } @@ -3788,14 +3802,42 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); unsigned long vmcb_pa = svm->current_vmcb->pa; + int cpu; guest_state_enter_irqoff(); if (sev_es_guest(vcpu->kvm)) { + cpu = get_cpu(); + local_irq_disable(); + WARN_ON(cpu != 2); + + memset(cachepc_msrmts, 0, + cachepc_msrmts_count * sizeof(cpc_msrmt_t)); + cachepc_reset_pmc(CPC_L1MISS_PMC); + + cachepc_reset_pmc(CPC_RETINST_PMC); + + if (cachepc_single_step) + cachepc_apic_oneshot(10); __svm_sev_es_vcpu_run(vmcb_pa); + cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC); + + cachepc_save_msrmts(cachepc_ds); + if (cachepc_baseline_measure) + cachepc_update_baseline(); + local_irq_enable(); + put_cpu(); } else { struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + cpu = get_cpu(); + local_irq_disable(); + WARN_ON(cpu != 2); + + memset(cachepc_msrmts, 0, + cachepc_msrmts_count * sizeof(cpc_msrmt_t)); + cachepc_reset_pmc(CPC_L1MISS_PMC); + /* * Use a single vmcb (vmcb01 because it's always valid) for * context switching guest state via VMLOAD/VMSAVE, that way @@ -3803,10 +3845,20 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) * vmcb02 when switching vmcbs for nested virtualization. */ vmload(svm->vmcb01.pa); + if (cachepc_single_step) + cachepc_apic_oneshot(100); __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs); vmsave(svm->vmcb01.pa); + cachepc_reset_pmc(CPC_RETINST_PMC); vmload(__sme_page_pa(sd->save_area)); + cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC); + + cachepc_save_msrmts(cachepc_ds); + if (cachepc_baseline_measure) + cachepc_update_baseline(); + local_irq_enable(); + put_cpu(); } guest_state_exit_irqoff(); diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index dfaeb47fcf2a..0626f3fdddfd 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -29,12 +29,59 @@ .section .noinstr.text, "ax" +.extern cachepc_msrmts +.extern cachepc_regs_tmp +.extern cachepc_regs_vm + +.macro load_tmp off reg + mov cachepc_regs_tmp+\off(%rip), \reg +.endm + +.macro save_tmp off reg + mov \reg, cachepc_regs_tmp+\off(%rip) +.endm + +.macro load_vm off reg + mov cachepc_regs_vm+\off(%rip), \reg +.endm + +.macro save_vm off reg + mov \reg, cachepc_regs_vm+\off(%rip) +.endm + +.macro apply_regs func + \func 0x00, %rax + \func 0x08, %rbx + \func 0x10, %rcx + \func 0x18, %rdx + \func 0x20, %rbp + \func 0x28, %rsp + \func 0x30, %rdi + \func 0x38, %rsi + \func 0x40, %r8 + \func 0x48, %r9 + \func 0x50, %r10 + \func 0x58, %r11 + \func 0x60, %r12 + \func 0x68, %r13 + \func 0x70, %r14 + \func 0x78, %r15 +.endm + +.macro barrier + mfence + mov $0x80000005,%eax + cpuid +.endm + /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode * @vmcb_pa: unsigned long * @regs: unsigned long * (to guest registers) */ SYM_FUNC_START(__svm_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushs +sev_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli @@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run) * @vmcb_pa: unsigned long */ SYM_FUNC_START(__svm_sev_es_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_es_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushes +sev_es_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_es_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_es_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d9adf79124f9..3e5c55f9bef0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -82,6 +82,8 @@ #include #include +#include "cachepc/sevstep.h" + #define CREATE_TRACE_POINTS #include "trace.h" diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index 27ab27931813..90679ec8ba79 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -1173,8 +1173,78 @@ EXPORT_SYMBOL_GPL(crypto_aes_set_key); f_rl(bo, bi, 3, k); \ } while (0) +#define L1_ASSOC 8 +#define L1_LINESIZE 64 +#define L1_SETS 64 +#define L1_SIZE (L1_SETS * L1_ASSOC * L1_LINESIZE) + +#define ACCESS_LINE(n) \ + asm volatile ("mov (%0), %%rbx" \ + : : "r"(((uint8_t*) L1) + n * L1_LINESIZE) : "rbx"); + +#define DO_ACCESS_PATTERN() \ + ACCESS_LINE(60) \ + ACCESS_LINE(13) \ + ACCESS_LINE(24) \ + ACCESS_LINE(19) \ + ACCESS_LINE(38) \ + ACCESS_LINE(17) \ + ACCESS_LINE( 2) \ + ACCESS_LINE(12) \ + ACCESS_LINE(22) \ + ACCESS_LINE(46) \ + ACCESS_LINE( 4) \ + ACCESS_LINE(61) \ + ACCESS_LINE( 5) \ + ACCESS_LINE(14) \ + ACCESS_LINE(11) \ + ACCESS_LINE(35) \ + ACCESS_LINE(45) \ + ACCESS_LINE(10) \ + ACCESS_LINE(49) \ + ACCESS_LINE(56) \ + ACCESS_LINE(27) \ + ACCESS_LINE(37) \ + ACCESS_LINE(63) \ + ACCESS_LINE(54) \ + ACCESS_LINE(55) \ + ACCESS_LINE(29) \ + ACCESS_LINE(48) \ + ACCESS_LINE( 9) \ + ACCESS_LINE(16) \ + ACCESS_LINE(39) \ + ACCESS_LINE(20) \ + ACCESS_LINE(21) \ + ACCESS_LINE(62) \ + ACCESS_LINE( 0) \ + ACCESS_LINE(34) \ + ACCESS_LINE( 8) \ + ACCESS_LINE(53) \ + ACCESS_LINE(42) \ + ACCESS_LINE(51) \ + ACCESS_LINE(50) \ + ACCESS_LINE(57) \ + ACCESS_LINE( 7) \ + ACCESS_LINE( 6) \ + ACCESS_LINE(33) \ + ACCESS_LINE(26) \ + +uint8_t *L1 = NULL; + static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { + int cpu; + + if (L1 == NULL) { + L1 = kzalloc(L1_SETS * L1_LINESIZE, GFP_KERNEL); + BUG_ON(((uintptr_t)L1) % (L1_SETS * L1_LINESIZE) != 0); + } + + pr_warn("CachePC-TEST: Running AES-Generic!"); + + cpu = get_cpu(); + DO_ACCESS_PATTERN() + const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); u32 b0[4], b1[4]; const u32 *kp = ctx->key_enc + 4; @@ -1210,6 +1280,9 @@ static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) put_unaligned_le32(b0[1], out + 4); put_unaligned_le32(b0[2], out + 8); put_unaligned_le32(b0[3], out + 12); + + DO_ACCESS_PATTERN(); + put_cpu(); } /* decrypt a block of text */ diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c old mode 100644 new mode 100755 index e089fbf9017f..7899e1efe852 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -87,7 +87,7 @@ static void *sev_init_ex_buffer; static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE; static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); -static int sev_do_cmd(int cmd, void *data, int *psp_ret); +int sev_do_cmd(int cmd, void *data, int *psp_ret); static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { @@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) return ret; } -static int sev_do_cmd(int cmd, void *data, int *psp_ret) +int sev_do_cmd(int cmd, void *data, int *psp_ret) { int rc; @@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret) return rc; } +EXPORT_SYMBOL(sev_do_cmd); static int __sev_init_locked(int *error) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f2a63cb2658b..4c55f85fc775 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -13,6 +13,7 @@ * Yaniv Kamay */ +#include #include #include @@ -70,6 +71,10 @@ /* Worst case buffer size needed for holding an integer. */ #define ITOA_MAX_LEN 12 +#include "../../arch/x86/kvm/cachepc/kvm.h" +#include "../../arch/x86/kvm/cachepc/sevstep.h" +#include "../../arch/x86/kvm/cachepc/uspt.h" + MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -1261,6 +1266,9 @@ static void kvm_destroy_vm(struct kvm *kvm) hardware_disable_all(); mmdrop(mm); module_put(kvm_chardev_ops.owner); + + if (main_vm == kvm) + main_vm = NULL; } void kvm_get_kvm(struct kvm *kvm) @@ -1360,7 +1368,7 @@ static void kvm_insert_gfn_node(struct kvm_memslots *slots, int idx = slots->node_idx; parent = NULL; - for (node = &gfn_tree->rb_node; *node; ) { + for (node = &gfn_tree->rb_node; *node;) { struct kvm_memory_slot *tmp; tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]); @@ -4823,6 +4831,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); fd_install(r, file); + + main_vm = kvm; + return r; put_kvm: @@ -4864,7 +4875,7 @@ static long kvm_dev_ioctl(struct file *filp, r = -EOPNOTSUPP; break; default: - return kvm_arch_dev_ioctl(filp, ioctl, arg); + return cachepc_kvm_ioctl(filp, ioctl, arg); } out: return r; @@ -5792,6 +5803,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, r = kvm_vfio_ops_init(); WARN_ON(r); + cachepc_kvm_init(); + return 0; out_unreg: @@ -5821,6 +5834,8 @@ void kvm_exit(void) { int cpu; + cachepc_kvm_exit(); + debugfs_remove_recursive(kvm_debugfs_dir); misc_deregister(&kvm_dev); for_each_possible_cpu(cpu)