diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index b804444e16d4..17167ccfca22 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y += -Iarch/x86/kvm +ccflags-y += -Iarch/x86/kvm -O2 ccflags-$(CONFIG_KVM_WERROR) += -Werror ifeq ($(CONFIG_FRAME_POINTER),y) @@ -10,7 +10,9 @@ endif KVM := ../../../virt/kvm kvm-y += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o \ - $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o + $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ + svm/cachepc/cachepc.o svm/cachepc/util.o + kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ @@ -20,7 +22,8 @@ kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/evmcs.o vmx/nested.o vmx/posted_intr.o -kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o \ + svm/cachepc/cachepc.o svm/cachepc/util.o obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7b3cfbe8f7e3..75ef27b944de 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2,6 +2,8 @@ #include +#include "cachepc/cachepc.h" + #include "irq.h" #include "mmu.h" #include "kvm_cache_regs.h" @@ -3751,7 +3753,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, if (sev_es_guest(svm->vcpu.kvm)) { __svm_sev_es_vcpu_run(svm->vmcb_pa); } else { + cacheline *head = cachepc_prime(cachepc_ds); __svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs); + cachepc_probe(head); + cachepc_save_msrmts(head); #ifdef CONFIG_X86_64 native_wrmsrl(MSR_GS_BASE, svm->host.gs_base); @@ -3785,8 +3790,12 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) { - struct vcpu_svm *svm = to_svm(vcpu); + struct vcpu_svm *svm; + printk(KERN_WARNING "CachePC: svm_cpu_enter_exit()\n"); + WARN_ON(smp_processor_id() != 2); + + svm = to_svm(vcpu); svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2541a17ff1c4..7efbdfd0e3e2 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -51,6 +51,9 @@ #include #include #include +#include +#include +#include #include #include @@ -66,6 +69,8 @@ /* Worst case buffer size needed for holding an integer. */ #define ITOA_MAX_LEN 12 +#include "../../arch/x86/kvm/svm/cachepc/cachepc.h" + MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -143,6 +148,18 @@ static void hardware_disable_all(void); static void kvm_io_bus_destroy(struct kvm_io_bus *bus); +struct proc_ops cachepc_proc_ops; + +uint16_t *cachepc_msrmts; +size_t cachepc_msrmts_count; +EXPORT_SYMBOL(cachepc_msrmts); +EXPORT_SYMBOL(cachepc_msrmts_count); + +cache_ctx *cachepc_ctx; +cacheline *cachepc_ds; +EXPORT_SYMBOL(cachepc_ctx); +EXPORT_SYMBOL(cachepc_ds); + __visible bool kvm_rebooting; EXPORT_SYMBOL_GPL(kvm_rebooting); @@ -4765,12 +4782,327 @@ static void check_processor_compat(void *data) *c->ret = kvm_arch_check_processor_compat(c->opaque); } +int +kvm_cachepc_open(struct inode *inode, struct file *file) +{ + try_module_get(THIS_MODULE); + + return 0; +} + +int +kvm_cachepc_close(struct inode *inode, struct file *file) +{ + module_put(THIS_MODULE); + + return 0; +} + +ssize_t +kvm_cachepc_read(struct file *file, char *buf, size_t buflen, loff_t *off) +{ + size_t len, left; + size_t size; + + printk(KERN_WARNING "CachePC: Reading entries (%lu:%lli)\n", + buflen, off ? *off : 0); + + size = cachepc_msrmts_count * sizeof(uint16_t); + if (!off || *off >= size || *off < 0) + return 0; + + len = size - *off; + if (len > buflen) len = buflen; + + left = copy_to_user(buf, (uint8_t *) cachepc_msrmts + *off, len); + + len -= left; + *off += len; + + return len; +} + +ssize_t +kvm_cachepc_write(struct file *file, const char *buf, size_t buflen, loff_t *off) +{ + return 0; +} + +void +kvm_cachepc_prime_probe_test(void *p) +{ + cacheline *lines; + cacheline *cl, *head; + uint32_t count; + uint32_t *arg; + int i, max; + + arg = p; + + /* l2 data cache, hit or miss */ + cachepc_init_pmc(0, 0x64, 0xD8); + + lines = cachepc_aligned_alloc(PAGE_SIZE, cachepc_ctx->cache_size); + BUG_ON(lines == NULL); + + max = cachepc_ctx->nr_of_cachelines; + + cachepc_cpuid(); + cachepc_mfence(); + + for (i = 0; i < max; i++) + asm volatile ("mov (%0), %%rbx" : : "r"(lines + i) : "rbx"); + + head = cachepc_prime(cachepc_ds); + cachepc_probe(head); + + count = 0; + cl = head = cachepc_ds; + do { + count += cl->count; + cl = cl->next; + } while (cl != head); + + printk(KERN_WARNING "CachePC: Prime-probe test done (%u vs. %u => %s)\n", + count, 0, (count == 0) ? "passed" : "failed"); + + if (arg) *arg = (count == 0); + + kfree(lines); +} + +void +kvm_cachepc_stream_hwpf_test(void *p) +{ + cacheline *lines; + uint32_t count; + uint32_t *arg; + uint32_t i, max; + + arg = p; + + /* TODO: accurately detect hwpf */ + + /* l2 data cache, hit or miss */ + cachepc_init_pmc(0, 0x64, 0xD8); + + lines = cachepc_aligned_alloc(PAGE_SIZE, cachepc_ctx->cache_size); + BUG_ON(lines == NULL); + + max = cachepc_ctx->nr_of_cachelines; + + cachepc_prime(cachepc_ds); + + count -= cachepc_read_pmc(0); + for (i = 0; i < max; i++) + asm volatile ("mov (%0), %%rbx" : : "r"(lines + i) : "rbx"); + count += cachepc_read_pmc(0); + + printk(KERN_WARNING "CachePC: HWPF test done (%u vs. %u => %s)\n", + count, max, (count == max) ? "passed" : "failed"); + + if (arg) *arg = (count == max); + + kfree(lines); +} + +void +kvm_cachepc_single_access_test(void *p) +{ + cacheline *ptr; + uint64_t pre, post; + uint32_t *arg; + + /* l2 data cache, hit or miss */ + cachepc_init_pmc(0, 0x64, 0xD8); + + arg = p; + + WARN_ON(arg && *arg >= L1_SETS); + if (arg && *arg >= L1_SETS) return; + ptr = cachepc_prepare_victim(cachepc_ctx, arg ? *arg : 48); + + cachepc_prime(cachepc_ds); + + pre = cachepc_read_pmc(0); + cachepc_victim(ptr); + post = cachepc_read_pmc(0); + + printk(KERN_WARNING "CachePC: Single access test done (%llu vs %u => %s)", + post - pre, 1, (post - pre == 1) ? "passed" : "failed"); + + if (arg) *arg = post - pre; + + cachepc_release_victim(cachepc_ctx, ptr); +} + +void +kvm_cachepc_single_eviction_test(void *p) +{ + cacheline *head, *cl, *evicted; + cacheline *ptr; + uint32_t target; + uint32_t *arg; + int count; + + arg = p; + + /* l2 data cache, hit or miss */ + cachepc_init_pmc(0, 0x64, 0xD8); + + WARN_ON(arg && *arg >= L1_SETS); + if (arg && *arg >= L1_SETS) return; + target = arg ? *arg : 48; + + ptr = cachepc_prepare_victim(cachepc_ctx, target); + + head = cachepc_prime(cachepc_ds); + cachepc_victim(ptr); + cachepc_probe(head); + + count = 0; + evicted = NULL; + cl = head = cachepc_ds; + do { + if (IS_FIRST(cl->flags) && cl->count > 0) { + evicted = cl; + count += cl->count; + } + cl = cl->next; + } while (cl != head); + + printk(KERN_WARNING "CachePC: Single eviction test done (%u vs %u => %s)\n", + count, 1, (count == 1 && evicted->cache_set == target) ? "passed" : "failed"); + cachepc_save_msrmts(head); + + if (arg) *arg = count; + + cachepc_release_victim(cachepc_ctx, ptr); +} + +void +kwm_cachepc_system_setup(void) +{ + uint64_t reg_addr, val; + uint32_t lo, hi; + + /* disable streaming store */ + reg_addr = 0xc0011020; + asm volatile ("rdmsr" : "=a"(lo), "=d"(hi) : "c"(reg_addr)); + val = (uint64_t) lo | ((uint64_t) hi << 32); + val |= 1 << 13; + asm volatile ("wrmsr" : : "c"(reg_addr), "a"(val), "d"(0x00)); + printk("CachePC: Writing MSR %08llX: %016llX\n", reg_addr, val); + + /* disable speculative data cache tlb reloads */ + reg_addr = 0xc0011022; + asm volatile ("rdmsr" : "=a"(lo), "=d"(hi) : "c"(reg_addr)); + val = (uint64_t) lo | ((uint64_t) hi << 32); + val |= 1 << 4; + asm volatile ("wrmsr" : : "c"(reg_addr), "a"(val), "d"(0x00)); + printk("CachePC: Writing MSR %08llX: %016llX\n", reg_addr, val); + + /* disable data cache hardware prefetcher */ + reg_addr = 0xc0011022; + asm volatile ("rdmsr" : "=a"(lo), "=d"(hi) : "c"(reg_addr)); + val = (uint64_t) lo | ((uint64_t) hi << 32); + val |= 1 << 13; + asm volatile ("wrmsr" : : "c"(reg_addr), "a"(val), "d"(0x00)); + printk("CachePC: Writing MSR %08llX: %016llX\n", reg_addr, val); +} + +void +kvm_cachepc_init(void *p) +{ + int cpu; + + cpu = get_cpu(); + + printk(KERN_WARNING "CachePC: Running on core %i\n", cpu); + + cachepc_ctx = cachepc_get_ctx(L1); + cachepc_ds = cachepc_prepare_ds(cachepc_ctx); + + kwm_cachepc_system_setup(); + + kvm_cachepc_prime_probe_test(NULL); + kvm_cachepc_single_access_test(NULL); + kvm_cachepc_single_eviction_test(NULL); + kvm_cachepc_stream_hwpf_test(NULL); + + put_cpu(); +} + +void +kvm_cachepc_init_pmc_ioctl(void *p) +{ + uint32_t event; + uint8_t index, event_no, event_mask; + + WARN_ON(p == NULL); + if (!p) return; + + event = *(uint32_t *)p; + + index = (event & 0xFF000000) >> 24; + event_no = (event & 0x0000FF00) >> 8; + event_mask = (event & 0x000000FF) >> 0; + + cachepc_init_pmc(index, event_no, event_mask); +} + +long +kvm_cachepc_ioctl(struct file *file, unsigned int cmd, unsigned long argp) +{ + void __user *arg_user; + uint32_t u32; + int r; + + arg_user = (void __user *)argp; + switch (cmd) { + case CACHEPC_IOCTL_TEST_ACCESS: + printk(KERN_WARNING "CachePC: Called ioctl access test\n"); + if (!arg_user) return -EINVAL; + if (copy_from_user(&u32, arg_user, sizeof(uint32_t))) + return -EFAULT; + r = smp_call_function_single(2, + kvm_cachepc_single_access_test, &u32, true); + WARN_ON(r != 0); + if (copy_to_user(arg_user, &u32, sizeof(uint32_t))) + return -EFAULT; + break; + case CACHEPC_IOCTL_TEST_EVICTION: + printk(KERN_WARNING "CachePC: Called ioctl eviction test\n"); + if (!arg_user) return -EINVAL; + if (copy_from_user(&u32, arg_user, sizeof(uint32_t))) + return -EFAULT; + r = smp_call_function_single(2, + kvm_cachepc_single_eviction_test, &u32, true); + WARN_ON(r != 0); + if (copy_to_user(arg_user, &u32, sizeof(uint32_t))) + return -EFAULT; + break; + case CACHEPC_IOCTL_INIT_PMC: + printk(KERN_WARNING "CachePC: Called ioctl init counter\n"); + if (!arg_user) return -EINVAL; + if (copy_from_user(&u32, arg_user, sizeof(uint32_t))) + return -EFAULT; + r = smp_call_function_single(2, + kvm_cachepc_init_pmc_ioctl, &u32, true); + WARN_ON(r != 0); + break; + default: + return -EINVAL; + } + + return 0; +} + int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, struct module *module) { struct kvm_cpu_compat_check c; - int r; - int cpu; + int r, cpu; r = kvm_arch_init(opaque); if (r) @@ -4848,6 +5180,21 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, r = kvm_vfio_ops_init(); WARN_ON(r); + cachepc_msrmts_count = L1_SETS; + cachepc_msrmts = kzalloc(cachepc_msrmts_count * sizeof(uint16_t), GFP_KERNEL); + BUG_ON(cachepc_msrmts == NULL); + + r = smp_call_function_single(2, kvm_cachepc_init, NULL, true); + WARN_ON(r != 0); + + memset(&cachepc_proc_ops, 0, sizeof(cachepc_proc_ops)); + cachepc_proc_ops.proc_open = kvm_cachepc_open; + cachepc_proc_ops.proc_read = kvm_cachepc_read; + cachepc_proc_ops.proc_write = kvm_cachepc_write; + cachepc_proc_ops.proc_release = kvm_cachepc_close; + cachepc_proc_ops.proc_ioctl = kvm_cachepc_ioctl; + proc_create("cachepc", 0644, NULL, &cachepc_proc_ops); + return 0; out_unreg: @@ -4872,6 +5219,12 @@ EXPORT_SYMBOL_GPL(kvm_init); void kvm_exit(void) { + remove_proc_entry("cachepc", NULL); + kfree(cachepc_msrmts); + + cachepc_release_ds(cachepc_ctx, cachepc_ds); + cachepc_release_ctx(cachepc_ctx); + debugfs_remove_recursive(kvm_debugfs_dir); misc_deregister(&kvm_dev); kmem_cache_destroy(kvm_vcpu_cache);