diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index eb186bc57f6a..3f767a27045e 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -2,10 +2,9 @@ #ifndef _ASM_X86_KVM_PAGE_TRACK_H #define _ASM_X86_KVM_PAGE_TRACK_H -enum kvm_page_track_mode { - KVM_PAGE_TRACK_WRITE, - KVM_PAGE_TRACK_MAX, -}; +#include + +#include "../../kvm/sevstep/uapi.h" /* * The notifier represented by @kvm_page_track_notifier_node is linked into diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 30f244b64523..3c5f65040878 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-y += -I $(srctree)/arch/x86/kvm +ccflags-y += -I $(srctree)/arch/x86/kvm -O2 ccflags-$(CONFIG_KVM_WERROR) += -Werror ifeq ($(CONFIG_FRAME_POINTER),y) @@ -11,8 +11,9 @@ include $(srctree)/virt/kvm/Makefile.kvm kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ - hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \ - mmu/spte.o + hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \ + svm/cachepc/cachepc.o svm/cachepc/kvm.o \ + sevstep/sevstep.o sevstep/uspt.o ifdef CONFIG_HYPERV kvm-y += kvm_onhyperv.o @@ -25,7 +26,8 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \ vmx/evmcs.o vmx/nested.o vmx/posted_intr.o kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o -kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o \ + svm/cachepc/cachepc.o ifdef CONFIG_HYPERV kvm-amd-y += svm/svm_onhyperv.o diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index d871b8dee7b3..32900ef5ee0b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) } } +#include "../sevstep/mmu.c" + /* * Write-protect on the specified @sptep, @pt_protect indicates whether * spte write-protection is caused by protecting shadow page table. @@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep) * * Return true if tlb need be flushed. */ -static bool spte_write_protect(u64 *sptep, bool pt_protect) -{ - u64 spte = *sptep; - - if (!is_writable_pte(spte) && - !(pt_protect && is_mmu_writable_spte(spte))) - return false; - - rmap_printk("spte %p %llx\n", sptep, *sptep); - - if (pt_protect) - spte &= ~shadow_mmu_writable_mask; - spte = spte & ~PT_WRITABLE_MASK; - - return mmu_spte_update(sptep, spte); -} +// static bool spte_write_protect(u64 *sptep, bool pt_protect) +// { +// return sevstep_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE); +// } static bool rmap_write_protect(struct kvm_rmap_head *rmap_head, bool pt_protect) { - u64 *sptep; - struct rmap_iterator iter; - bool flush = false; - - for_each_rmap_spte(rmap_head, &iter, sptep) - flush |= spte_write_protect(sptep, pt_protect); - - return flush; + return sevstep_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE); } static bool spte_clear_dirty(u64 *sptep) @@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, struct kvm_memory_slot *slot, u64 gfn, int min_level) { - struct kvm_rmap_head *rmap_head; - int i; - bool write_protected = false; - - if (kvm_memslots_have_rmaps(kvm)) { - for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { - rmap_head = gfn_to_rmap(gfn, i, slot); - write_protected |= rmap_write_protect(rmap_head, true); - } - } - - if (is_tdp_mmu_enabled(kvm)) - write_protected |= - kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level); - - return write_protected; + return sevstep_kvm_mmu_slot_gfn_protect(kvm, slot, + gfn, min_level, KVM_PAGE_TRACK_WRITE); } static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn) @@ -3901,6 +3870,10 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct) static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { + int active; + + sevstep_uspt_page_fault_handle(vcpu, fault); + if (unlikely(fault->rsvd)) return false; @@ -3911,8 +3884,11 @@ static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu, * guest is writing the page which is write tracked which can * not be fixed by page fault handler. */ - if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE)) - return true; + active = kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE); + active |= kvm_slot_page_track_is_active(vcpu->kvm, + fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS); + if (active) return true; return false; } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 2e09d1b6249f..17b69a1f2b40 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -19,6 +19,8 @@ #include "mmu.h" #include "mmu_internal.h" +#include "../sevstep/sevstep.h" + bool kvm_page_track_write_tracking_enabled(struct kvm *kvm) { return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) || @@ -131,9 +133,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, */ kvm_mmu_gfn_disallow_lpage(slot, gfn); - if (mode == KVM_PAGE_TRACK_WRITE) - if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) - kvm_flush_remote_tlbs(kvm); + if (sevstep_kvm_mmu_slot_gfn_protect(kvm, + slot, gfn, PG_LEVEL_4K, mode)) { + kvm_flush_remote_tlbs(kvm); + } } EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); diff --git a/arch/x86/kvm/sevstep b/arch/x86/kvm/sevstep new file mode 120000 index 000000000000..642ea24bf098 --- /dev/null +++ b/arch/x86/kvm/sevstep @@ -0,0 +1 @@ +/home/louis/kvm-prime-count/sevstep \ No newline at end of file diff --git a/arch/x86/kvm/svm/cachepc b/arch/x86/kvm/svm/cachepc new file mode 120000 index 000000000000..9119e44af1f0 --- /dev/null +++ b/arch/x86/kvm/svm/cachepc @@ -0,0 +1 @@ +/home/louis/kvm-prime-count/cachepc \ No newline at end of file diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cf0bf456d520..4dbb8041541f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2,6 +2,8 @@ #include +#include "cachepc/cachepc.h" + #include "irq.h" #include "mmu.h" #include "kvm_cache_regs.h" @@ -3788,14 +3790,28 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); unsigned long vmcb_pa = svm->current_vmcb->pa; + int cpu; guest_state_enter_irqoff(); if (sev_es_guest(vcpu->kvm)) { + memset(cachepc_msrmts, 0, 64 * 2); + cpu = get_cpu(); + local_irq_disable(); + WARN_ON(cpu != 2); __svm_sev_es_vcpu_run(vmcb_pa); + cachepc_save_msrmts(cachepc_ds); + local_irq_enable(); + put_cpu(); } else { struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + memset(cachepc_msrmts, 0, 64 * 2); + cpu = get_cpu(); + local_irq_disable(); + WARN_ON(cpu != 2); + /* TODO: try closer to vcpu_run */ + /* * Use a single vmcb (vmcb01 because it's always valid) for * context switching guest state via VMLOAD/VMSAVE, that way @@ -3807,6 +3823,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) vmsave(svm->vmcb01.pa); vmload(__sme_page_pa(sd->save_area)); + + cachepc_save_msrmts(cachepc_ds); + local_irq_enable(); + put_cpu(); } guest_state_exit_irqoff(); diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index dfaeb47fcf2a..0626f3fdddfd 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -29,12 +29,59 @@ .section .noinstr.text, "ax" +.extern cachepc_msrmts +.extern cachepc_regs_tmp +.extern cachepc_regs_vm + +.macro load_tmp off reg + mov cachepc_regs_tmp+\off(%rip), \reg +.endm + +.macro save_tmp off reg + mov \reg, cachepc_regs_tmp+\off(%rip) +.endm + +.macro load_vm off reg + mov cachepc_regs_vm+\off(%rip), \reg +.endm + +.macro save_vm off reg + mov \reg, cachepc_regs_vm+\off(%rip) +.endm + +.macro apply_regs func + \func 0x00, %rax + \func 0x08, %rbx + \func 0x10, %rcx + \func 0x18, %rdx + \func 0x20, %rbp + \func 0x28, %rsp + \func 0x30, %rdi + \func 0x38, %rsi + \func 0x40, %r8 + \func 0x48, %r9 + \func 0x50, %r10 + \func 0x58, %r11 + \func 0x60, %r12 + \func 0x68, %r13 + \func 0x70, %r14 + \func 0x78, %r15 +.endm + +.macro barrier + mfence + mov $0x80000005,%eax + cpuid +.endm + /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode * @vmcb_pa: unsigned long * @regs: unsigned long * (to guest registers) */ SYM_FUNC_START(__svm_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushs +sev_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli @@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run) * @vmcb_pa: unsigned long */ SYM_FUNC_START(__svm_sev_es_vcpu_run) + apply_regs save_tmp + push %_ASM_BP #ifdef CONFIG_X86_64 push %r15 @@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX +1: + + apply_regs save_vm + apply_regs load_tmp + mov cachepc_ds, %rsi + mov 0x8(%rsi), %r15 + lea sev_es_prime_ret(%rip), %rdi + jmp cachepc_prime_vcall+5+1 // skip stack pushes +sev_es_prime_ret: + apply_regs save_tmp + apply_regs load_vm + + vmrun %_ASM_AX + + apply_regs save_vm + apply_regs load_tmp + mov %r15, %rsi + lea sev_es_probe_ret(%rip), %rdi + jmp cachepc_probe_vcall+5+8 // skip stack pushs +sev_es_probe_ret: + apply_regs save_tmp + apply_regs load_vm 2: cli diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d9adf79124f9..082dc8553566 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -82,6 +82,8 @@ #include #include +#include "sevstep/sevstep.h" + #define CREATE_TRACE_POINTS #include "trace.h" diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c old mode 100644 new mode 100755 index e089fbf9017f..7899e1efe852 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -87,7 +87,7 @@ static void *sev_init_ex_buffer; static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE; static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); -static int sev_do_cmd(int cmd, void *data, int *psp_ret); +int sev_do_cmd(int cmd, void *data, int *psp_ret); static inline bool sev_version_greater_or_equal(u8 maj, u8 min) { @@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) return ret; } -static int sev_do_cmd(int cmd, void *data, int *psp_ret) +int sev_do_cmd(int cmd, void *data, int *psp_ret) { int rc; @@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret) return rc; } +EXPORT_SYMBOL(sev_do_cmd); static int __sev_init_locked(int *error) { diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index f2a63cb2658b..bd26b7a29c9e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -70,6 +70,10 @@ /* Worst case buffer size needed for holding an integer. */ #define ITOA_MAX_LEN 12 +#include "../../arch/x86/kvm/svm/cachepc/kvm.h" +#include "../../arch/x86/kvm/sevstep/sevstep.h" +#include "../../arch/x86/kvm/sevstep/uspt.h" + MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -159,6 +163,267 @@ static unsigned long long kvm_active_vms; static DEFINE_PER_CPU(cpumask_var_t, cpu_kick_mask); +static long +get_user_pages_remote_unlocked(struct mm_struct *mm, + unsigned long start, unsigned long nr_pages, + unsigned int gup_flags, struct page **pages) +{ + struct vm_area_struct **vmas = NULL; + int locked = 1; + long ret; + + down_read(&mm->mmap_lock); + ret = get_user_pages_remote( mm, start, nr_pages, + gup_flags, pages, vmas, &locked); + if (locked) up_read(&mm->mmap_lock); + + return ret; +} + +// static int +// get_hpa_for_gpa(struct kvm *kvm, gpa_t gpa, hpa_t *hpa) +// { +// int ec; +// unsigned long hva; +// struct page *page = NULL; +// +// ec = 0; +// +// hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); +// if (kvm_is_error_hva(hva)) { +// pr_warn("in %s line %d get_hpa_for_gpa: translation to hva failed\n", +// __FILE__, __LINE__); +// ec = -100; +// goto out; +// } +// if (get_user_pages_remote_unlocked(kvm->mm, hva, 1, 0, &page) != 1) { +// pr_warn("in %s line %d get_hpa_for_gpa: failed to get page struct from mm", +// __FILE__, __LINE__); +// ec = -KVM_EINVAL; +// goto out; +// } +// +// (*hpa) = (page_to_pfn(page) << 12) + (gpa & 0xfff); +// +// out: +// put_page(page); +// +// return ec; +// } + +int +read_physical(struct kvm *kvm, u64 gpa, void *buff, u64 size, + bool decrypt_at_host) +{ + unsigned long hva; + struct page *page = NULL; + void *ptr_page = NULL; + uint64_t offset; + int ec; + + offset = (gpa & 0xFFF); + + if ((offset + size - 1) > 0xFFF) { + printk("read_phyiscal: trying to read " + "beyond page (offset+size=%016llx)\n", + offset + size); + return -EINVAL; + } + + ec = 0; + + hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); + + // TODO: test change + /* + if (kvm_is_error_hva(hva)) { + printk(KERN_CRIT "Luca: read_physical: translation to hva failed( gpa was " + "%016llx hva is %016lx\n", + gpa, hva); + ec = -100; + goto out; + } + */ + + if (get_user_pages_remote_unlocked(kvm->mm, hva, 1, 0, &page) != 1) { + pr_warn("read_physical: failed to get page struct from mm\n"); + // ec = -KVM_EINVAL; + ec = -100; + goto out; + } + + if (decrypt_at_host) { + // map with encryption bit. Content is decrypted with host key. If sev is + // disabled but sme is enable this allows to read the plaintext. + ptr_page = vmap(&page, 1, 0, PAGE_KERNEL_NOCACHE); + } else { + // map without encryption bit to read ciphertexts + ptr_page = vmap(&page, 1, 0, __pgprot(__PAGE_KERNEL_NOCACHE)); + } + + /*printk("value of buff ptr = %p\t value of ptr_page=%p\n", buff, + ptr_page + offset);*/ + memcpy(buff, ptr_page + offset, size); + +out: + if (ptr_page) + vunmap(ptr_page); + if (page) + put_page(page); + + return ec; +} + +int +print_physical(struct kvm *kvm, u64 gpa, u64 size, bool decrypt_at_host) +{ + u8 *buffer; + int i, err; + + buffer = kmalloc(size, GFP_ATOMIC); + + err = read_physical(kvm, gpa, buffer, size, decrypt_at_host); + if (err != 0) { + pr_warn("at %s line %d: read_physical " + "failed with: %d\n", __FILE__, __LINE__, err); + } + for (i = 0; i < size; i++) { + // print bytewise with line break every 16 bytes + if (i % 16 == 0) { + printk("%02x ", buffer[i]); + } else { + printk(KERN_CONT " %02x ", buffer[i]); + } + } + printk("\n"); + + kfree(buffer); + + return err; +} + +int +map_physical(struct kvm *kvm, u64 gpa, bool decrypt_at_host, + void **mapping, struct page **page) +{ + + int ec; + unsigned long hva; + uint64_t offset; + + offset = (gpa & 0xFFF); + + ec = 0; + + hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); + + if (get_user_pages_remote_unlocked(kvm->mm, hva, 1, 0, page) != 1) { + pr_warn("map_physical: failed to get page struct from mm"); + // ec = -KVM_EINVAL; + ec = -100; + return ec; + } + + if (decrypt_at_host) { + // map with encryption bit. Content is decrypted with host key. If sev is + // disabled but sme is enable this allows to read the plaintext. + (*mapping) = vmap(page, 1, 0, PAGE_KERNEL); + } else { + // map without encryption bit to read ciphertexts + (*mapping) = vmap(page, 1, 0, __pgprot(__PAGE_KERNEL)); + } + + return ec; +} + +void +unmap_physical(void **mapping, struct page **page) +{ + if (*mapping) + vunmap(*mapping); + if (*page) + put_page(*page); +} + +int +read_mapped(u64 gpa, void *buff, u64 size, void *mapping) +{ + uint64_t offset; + offset = (gpa & 0xFFF); + + if ((offset + size - 1) > 0xFFF) { + pr_warn("read_mapped: trying to read " + "beyond page (offset+size=%016llx)\n", + offset + size); + return -EINVAL; + } + memcpy(buff, mapping + offset, size); + + return 0; +} + +int +write_mapped(u64 gpa, u64 size, const void *buf, void *mapping) +{ + uint64_t offset; + + offset = (gpa & 0xFFF); + + if ((offset + size - 1) > 0xFFF) { + printk("write_physical: trying to write beyond page(offset+size=%016llx)\n", + offset + size); + return -EINVAL; + } + memcpy(mapping + offset, buf, size); + + return 0; +} + +int +write_physical(struct kvm *kvm, u64 gpa, u64 size, + const void *buf, bool write_plaintexts) +{ + int ec; + unsigned long hva; + struct page *page; + void *ptr_page; + uint64_t offset; + + offset = (gpa & 0xFFF); + + if ((offset + size - 1) > 0xFFF) { + pr_warn("write_physical: trying to write " + "beyond page(offset+size=%016llx)\n", + offset + size); + return -EINVAL; + } + + ec = 0; + hva = gfn_to_hva(kvm, gpa_to_gfn(gpa)); + + if (kvm_is_error_hva(hva)) + return -KVM_EINVAL; + + if (get_user_pages_remote_unlocked(kvm->mm, hva, 1, FOLL_WRITE, &page) != 1) + return -KVM_EINVAL; + + if (write_plaintexts) { + // map with encrytpion bit to aplly host encryption. Usefull if sev is + // disabled but sme is enabled and we want to write a certain value into a + // page + ptr_page = vmap(&page, 1, 0, PAGE_KERNEL_NOCACHE); + } else { + // map without encryption bit to write ciphertexts + ptr_page = vmap(&page, 1, 0, __pgprot(__PAGE_KERNEL_NOCACHE)); + } + + memcpy(ptr_page + offset, buf, size); + + vunmap(ptr_page); + put_page(page); + return ec; +} + __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, unsigned long start, unsigned long end) { @@ -1261,6 +1526,9 @@ static void kvm_destroy_vm(struct kvm *kvm) hardware_disable_all(); mmdrop(mm); module_put(kvm_chardev_ops.owner); + + if (main_vm == kvm) + main_vm = NULL; } void kvm_get_kvm(struct kvm *kvm) @@ -1360,7 +1628,7 @@ static void kvm_insert_gfn_node(struct kvm_memslots *slots, int idx = slots->node_idx; parent = NULL; - for (node = &gfn_tree->rb_node; *node; ) { + for (node = &gfn_tree->rb_node; *node;) { struct kvm_memory_slot *tmp; tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]); @@ -4823,6 +5091,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm); fd_install(r, file); + + main_vm = kvm; + return r; put_kvm: @@ -4836,6 +5107,315 @@ static long kvm_dev_ioctl(struct file *filp, long r = -EINVAL; switch (ioctl) { + case KVM_TRACK_PAGE: { + track_page_param_t param; + void __user* argp = (void __user *)arg; + struct kvm_vcpu *vcpu; + + if (copy_from_user(¶m, argp, sizeof(param))) { + pr_warn("KVM_TRACK_PAGE: error copying arguments, exiting\n"); + return -EFAULT; + } + + if (main_vm == NULL) { + pr_warn("KVM_TRACK_PAGE: main_vm is not initialized, aborting!\n"); + return -EFAULT; + } + + if (param.track_mode < 0 || param.track_mode >= KVM_PAGE_TRACK_MAX) { + pr_warn("KVM_TRACK_PAGE track_mode %d invalid, " + "must be in range [%d,%d]", param.track_mode, + 0, KVM_PAGE_TRACK_MAX); + return -EFAULT; + } + + vcpu = xa_load(&main_vm->vcpu_array, 0); + if (!sevstep_track_single_page(vcpu, + param.gpa >> PAGE_SHIFT, param.track_mode)) { + printk("KVM_TRACK_PAGE: sevstep_track_single_page failed"); + } + r = 0; + } + break; + case KVM_USPT_BATCH_TRACK_START: { + batch_track_config_t param; + void __user* argp = (void __user *)arg; + + if (copy_from_user(¶m, argp, sizeof(param))) { + pr_warn("KVM_USPT_BATCH_TRACK_START: " + "error copying arguments, exiting\n"); + return -EFAULT; + } + + r = sevstep_uspt_batch_tracking_start(param.tracking_type, + param.expected_events, param.perf_cpu, param.retrack); + if (r != 0) { + pr_warn("KVM_USPT_BATCH_TRACK_START: failed\n"); + return r; + } + } + break; + case KVM_USPT_BATCH_TRACK_EVENT_COUNT: { + batch_track_event_count_t result; + void __user* argp = (void __user *)arg; + + result.event_count = sevstep_uspt_batch_tracking_get_events_count(); + + if (copy_to_user(argp, &result, sizeof(result))) { + pr_warn("KVM_USPT_BATCH_TRACK_EVENT_COUNT: " + "error copying result to user, exiting\n"); + return -EFAULT; + } + + r = 0; + } + break; + case KVM_USPT_BATCH_TRACK_STOP: { + batch_track_stop_and_get_t param; + page_fault_event_t* buf; + uint64_t buf_bytes; + void __user* argp = (void __user *)arg; + void __user* inner_user_out_buf; + + if (copy_from_user(¶m, argp, sizeof(param))) { + pr_warn("KVM_USPT_BATCH_TRACK_STOP: " + "error copying arguments, exiting\n"); + return -EFAULT; + } + inner_user_out_buf = param.out_buf; + + buf_bytes = sizeof(page_fault_event_t)*param.len; + pr_warn("KVM_USPT_BATCH_TRACK_STOP: " + "allocating %llu bytes for tmp buf\n", buf_bytes); + + buf = vmalloc(buf_bytes); + if (buf == NULL) { + pr_warn("KVM_USPT_BATCH_TRACK_STOP: " + "failed to alloc tmp buf\n"); + return -EFAULT; + } + param.out_buf = buf; + + r = sevstep_uspt_batch_tracking_stop(buf, param.len, + ¶m.error_during_batch); + if (r != 0) { + pr_warn("KVM_USPT_BATCH_TRACK_STOP: failed\n"); + vfree(buf); + return -EFAULT; + } + + if (copy_to_user(argp, ¶m, sizeof(param))) { + pr_warn("KVM_USPT_BATCH_TRACK_STOP: " + "error copying result to user, exiting\n"); + vfree(buf); + return -EFAULT; + } + + if (copy_to_user(inner_user_out_buf, buf,buf_bytes)) { + pr_warn("KVM_USPT_BATCH_TRACK_STOP: " + "error copying result to user, exiting\n"); + vfree(buf); + return -EFAULT; + } + + vfree(buf); + } + break; + case KVM_USPT_TRACK_ALL: { + track_all_pages_t param; + void __user* argp = (void __user *)arg; + struct kvm_vcpu *vcpu; + long tracked_pages; + + if (copy_from_user(¶m, argp, sizeof(param))) { + pr_warn("KVM_USPT_TRACK_ALL: error copying arguments, exiting\n"); + return -EFAULT; + } + + if (main_vm == NULL) { + pr_warn("KVM_USPT_TRACK_ALL: main_vm is not initialized, aborting!\n"); + return -EFAULT; + } + + if (param.track_mode < 0 || param.track_mode >= KVM_PAGE_TRACK_MAX) { + pr_warn("KVM_USPT_TRACK_ALL: " + "track_mode %d invalid, must be in range [%d,%d]\n", + param.track_mode, 0, KVM_PAGE_TRACK_MAX); + return -EFAULT; + } + + vcpu = xa_load(&main_vm->vcpu_array, 0); + tracked_pages = sevstep_start_tracking(vcpu, param.track_mode); + r = 0; + } + break; + case KVM_USPT_UNTRACK_ALL: { + track_all_pages_t param; + void __user* argp = (void __user *)arg; + struct kvm_vcpu *vcpu; + long untrack_count; + + if (copy_from_user(¶m, argp, sizeof(param))) { + printk(KERN_CRIT + "KVM_USPT_UNTRACK_ALL: error copying arguments, exiting\n"); + return -EFAULT; + } + + if (main_vm == NULL) { + printk("KVM_USPT_UNTRACK_ALL: main_vm is not initialized, aborting!\n"); + return -EFAULT; + } + + if (param.track_mode < 0 || param.track_mode >= KVM_PAGE_TRACK_MAX) { + printk("KVM_USPT_UNTRACK_ALL: track_mode %d invalid, must be in range [%d,%d]",param.track_mode,0,KVM_PAGE_TRACK_MAX); + return -EFAULT; + } + + //printk("KVM_USPT_UNTRACK_ALL: with mode %d\n",param.track_mode); + vcpu = xa_load(&main_vm->vcpu_array, 0); + untrack_count = sevstep_stop_tracking(vcpu, param.track_mode); + //printk("KVM_USPT_UNTRACK_ALL: untracked %ld pages\n",untrack_count); + r = 0; + } + break; + case KVM_USPT_SETUP_RETINSTR_PERF: { + retired_instr_perf_config_t config; + void __user* argp = (void __user *)arg; + + printk("Received KVM_USPT_SETUP_RETINSTR_PERF ioctl!\n"); + if (copy_from_user(&config, argp, sizeof(config))) { + printk("copy from user failed\n"); + return -EACCES; + } + + cachepc_init_pmc(0, 0xc0, 0x00, PMC_GUEST, PMC_KERNEL | PMC_USER); + + r = 0; + } + break; + case KVM_USPT_READ_RETINSTR_PERF: { + retired_instr_perf_t request; + void __user* argp = (void __user *)arg; + + if (copy_from_user(&request, argp, sizeof(request))) { + printk("KVM_USPT_READ_RETINSTR_PERF: copy from user failed\n"); + return -EACCES; + } + + request.retired_instruction_count = cachepc_read_pmc(0); + if (copy_to_user(argp, &request, sizeof(request))) { + printk("KVM_USPT_READ_RETINSTR_PERF : copy to user failed\n"); + } + r = 0; + } + break; + case KVM_READ_GUEST_MEMORY: { + read_guest_memory_t param; + int res; + void * buf; + void __user* argp = (void __user *)arg; + + if (copy_from_user(¶m, argp, sizeof(read_guest_memory_t))) { + printk(KERN_CRIT + "KVM_READ_GUEST_MEMORY: error copying arguments, exiting\n"); + return -EFAULT; + } + + if (param.len > PAGE_SIZE) { + printk("KVM_READ_GUEST_MEMORY len may be at most page size"); + } + + buf = kmalloc(param.len, GFP_KERNEL); + if (buf == NULL) { + printk("KVM_READ_GUEST_MEMORY: failed to alloc memory"); + return -ENOMEM; + } + + if (param.wbinvd_cpu >= 0) { + wbinvd_on_cpu(param.wbinvd_cpu); + } + wbinvd_on_all_cpus(); + + res = read_physical(main_vm, param.gpa, buf, + param.len, param.decrypt_with_host_key); + if (res) { + printk("KVM_READ_GUEST_MEMORY: read_physical failed with %d\n", res); + return -EINVAL; + } + + if (copy_to_user(param.output_buffer, buf, param.len)) { + printk("KVM_READ_GUEST_MEMORY: failed to copy buf to userspace"); + } + + return 0; + } + break; + case KVM_USPT_RESET: { + struct kvm_vcpu *vcpu; + + printk("Received KVM_USPT_RESET ioctl!\n"); + + sevstep_uspt_clear(); + vcpu = xa_load(&main_vm->vcpu_array, 0); + sevstep_stop_tracking(vcpu, KVM_PAGE_TRACK_EXEC); + sevstep_stop_tracking(vcpu, KVM_PAGE_TRACK_ACCESS); + sevstep_stop_tracking(vcpu, KVM_PAGE_TRACK_WRITE); + r = 0; + } + break; + case KVM_USPT_REGISTER_PID: { + userspace_ctx_t ctx; + void __user* argp = (void __user *)arg; + struct kvm_vcpu *vcpu; + + printk("Received REGISTER_PID ioctl!\n"); + if (copy_from_user(&ctx, argp, sizeof(userspace_ctx_t))) { + printk("copy from user failed\n"); + return -EACCES; + } + + if (main_vm == NULL) { + printk("KVM_TRACK_PAGE: main_vm is not initialized, aborting!\n"); + return -EFAULT; + } + + sevstep_uspt_clear(); + sevstep_uspt_initialize(ctx.pid, ctx.get_rip); + + printk("Resetting page tracking\n"); + vcpu = xa_load(&main_vm->vcpu_array, 0); + sevstep_stop_tracking(vcpu, KVM_PAGE_TRACK_EXEC); + sevstep_stop_tracking(vcpu, KVM_PAGE_TRACK_ACCESS); + sevstep_stop_tracking(vcpu, KVM_PAGE_TRACK_WRITE); + + return 0; + } + break; + case KVM_USPT_POLL_EVENT: { + void __user* argp = (void __user *)arg; + if (!sevstep_uspt_is_initialiized()) { + printk("userspace context not initilaized, call REGISTER_PID"); + return -EINVAL; + } + return sevstep_uspt_handle_poll_event(argp); + } + break; + case KVM_USPT_ACK_EVENT: { + ack_event_t ack_event; + void __user* argp = (void __user *)arg; + + if (!sevstep_uspt_is_initialiized()) { + printk("userspace context not initilaized, call REGISTER_PID"); + return -EINVAL; + } + if (copy_from_user(&ack_event, argp, sizeof(ack_event_t))) { + printk("ACK_EVENT failed to copy args"); + return -EINVAL; + } + + return sevstep_uspt_handle_ack_event_ioctl(ack_event); + } + break; case KVM_GET_API_VERSION: if (arg) goto out; @@ -5792,6 +6372,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, r = kvm_vfio_ops_init(); WARN_ON(r); + cachepc_kvm_init(); + return 0; out_unreg: @@ -5821,6 +6403,8 @@ void kvm_exit(void) { int cpu; + cachepc_kvm_exit(); + debugfs_remove_recursive(kvm_debugfs_dir); misc_deregister(&kvm_dev); for_each_possible_cpu(cpu)