commit 89785aa3c8d5d4007f856b14543a9b8aef31d661
parent b8f40e776af1a82116a44ce8fac34f343194f033
Author: Louis Burda <quent.burda@gmail.com>
Date: Thu, 12 Jan 2023 15:44:57 +0100
Remove outdated kernel patch
Diffstat:
| D | patch.diff | | | 1185 | ------------------------------------------------------------------------------- |
1 file changed, 0 insertions(+), 1185 deletions(-)
diff --git a/patch.diff b/patch.diff
@@ -1,1185 +0,0 @@
-diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h
-index eb186bc57f6a..b96e80934005 100644
---- a/arch/x86/include/asm/kvm_page_track.h
-+++ b/arch/x86/include/asm/kvm_page_track.h
-@@ -2,10 +2,9 @@
- #ifndef _ASM_X86_KVM_PAGE_TRACK_H
- #define _ASM_X86_KVM_PAGE_TRACK_H
-
--enum kvm_page_track_mode {
-- KVM_PAGE_TRACK_WRITE,
-- KVM_PAGE_TRACK_MAX,
--};
-+#include<linux/srcu.h>
-+
-+#include "../../kvm/cachepc/uapi.h"
-
- /*
- * The notifier represented by @kvm_page_track_notifier_node is linked into
-diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c
-index f603a724b08e..5c2d9b07c6aa 100644
---- a/arch/x86/kernel/sev.c
-+++ b/arch/x86/kernel/sev.c
-@@ -1034,6 +1034,8 @@ static int wakeup_cpu_via_vmgexit(int apic_id, unsigned long start_ip)
- if (!vmsa)
- return -ENOMEM;
-
-+ CPC_WARN("New VMSA allocated!\n");
-+
- /* CR4 should maintain the MCE value */
- cr4 = native_read_cr4() & X86_CR4_MCE;
-
-@@ -2589,11 +2591,11 @@ static int rmpupdate(u64 pfn, struct rmpupdate *val)
- * direct map.
- */
- if (val->assigned) {
-- if (invalid_direct_map(pfn, npages)) {
-- pr_err("Failed to unmap pfn 0x%llx pages %d from direct_map\n",
-- pfn, npages);
-- return -EFAULT;
-- }
-+ // if (invalid_direct_map(pfn, npages)) {
-+ // pr_err("Failed to unmap pfn 0x%llx pages %d from direct_map\n",
-+ // pfn, npages);
-+ // return -EFAULT;
-+ // }
- }
-
- retry:
-diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
-index 30f244b64523..a1e3c5ae2f80 100644
---- a/arch/x86/kvm/Makefile
-+++ b/arch/x86/kvm/Makefile
-@@ -1,6 +1,6 @@
- # SPDX-License-Identifier: GPL-2.0
-
--ccflags-y += -I $(srctree)/arch/x86/kvm
-+ccflags-y += -I $(srctree)/arch/x86/kvm -O2
- ccflags-$(CONFIG_KVM_WERROR) += -Werror
-
- ifeq ($(CONFIG_FRAME_POINTER),y)
-@@ -11,8 +11,8 @@ include $(srctree)/virt/kvm/Makefile.kvm
-
- kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \
- i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
-- hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \
-- mmu/spte.o
-+ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \
-+ cachepc/cachepc.o cachepc/kvm.o cachepc/track.o cachepc/event.o
-
- ifdef CONFIG_HYPERV
- kvm-y += kvm_onhyperv.o
-@@ -25,7 +25,8 @@ kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
- vmx/evmcs.o vmx/nested.o vmx/posted_intr.o
- kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o
-
--kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o
-+kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o \
-+ svm/avic.o svm/sev.o cachepc/cachepc.o cachepc/event.o
-
- ifdef CONFIG_HYPERV
- kvm-amd-y += svm/svm_onhyperv.o
-diff --git a/arch/x86/kvm/cachepc b/arch/x86/kvm/cachepc
-new file mode 120000
-index 000000000000..9119e44af1f0
---- /dev/null
-+++ b/arch/x86/kvm/cachepc
-@@ -0,0 +1 @@
-+/home/louis/kvm-prime-count/cachepc
-\ No newline at end of file
-diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
-index de6d44e07e34..b63672b47321 100644
---- a/arch/x86/kvm/cpuid.c
-+++ b/arch/x86/kvm/cpuid.c
-@@ -26,6 +26,10 @@
- #include "trace.h"
- #include "pmu.h"
-
-+#include "cachepc/cachepc.h"
-+#include "cachepc/uapi.h"
-+#include "cachepc/event.h"
-+
- /*
- * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
- * aligned to sizeof(unsigned long) because it's not accessed via bitops.
-@@ -1445,8 +1449,8 @@ int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
- if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
- return 1;
-
-- eax = kvm_rax_read(vcpu);
-- ecx = kvm_rcx_read(vcpu);
-+ eax = kvm_rax_read(vcpu);
-+ ecx = kvm_rcx_read(vcpu);
- kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
- kvm_rax_write(vcpu, eax);
- kvm_rbx_write(vcpu, ebx);
-diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
-index d871b8dee7b3..c70fff62f1ab 100644
---- a/arch/x86/kvm/mmu/mmu.c
-+++ b/arch/x86/kvm/mmu/mmu.c
-@@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
- }
- }
-
-+#include "../cachepc/mmu.c"
-+
- /*
- * Write-protect on the specified @sptep, @pt_protect indicates whether
- * spte write-protection is caused by protecting shadow page table.
-@@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
- *
- * Return true if tlb need be flushed.
- */
--static bool spte_write_protect(u64 *sptep, bool pt_protect)
--{
-- u64 spte = *sptep;
--
-- if (!is_writable_pte(spte) &&
-- !(pt_protect && is_mmu_writable_spte(spte)))
-- return false;
--
-- rmap_printk("spte %p %llx\n", sptep, *sptep);
--
-- if (pt_protect)
-- spte &= ~shadow_mmu_writable_mask;
-- spte = spte & ~PT_WRITABLE_MASK;
--
-- return mmu_spte_update(sptep, spte);
--}
-+// static bool spte_write_protect(u64 *sptep, bool pt_protect)
-+// {
-+// return cachepc_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE);
-+// }
-
- static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
- bool pt_protect)
- {
-- u64 *sptep;
-- struct rmap_iterator iter;
-- bool flush = false;
--
-- for_each_rmap_spte(rmap_head, &iter, sptep)
-- flush |= spte_write_protect(sptep, pt_protect);
--
-- return flush;
-+ return cachepc_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE);
- }
-
- static bool spte_clear_dirty(u64 *sptep)
-@@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
- struct kvm_memory_slot *slot, u64 gfn,
- int min_level)
- {
-- struct kvm_rmap_head *rmap_head;
-- int i;
-- bool write_protected = false;
--
-- if (kvm_memslots_have_rmaps(kvm)) {
-- for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
-- rmap_head = gfn_to_rmap(gfn, i, slot);
-- write_protected |= rmap_write_protect(rmap_head, true);
-- }
-- }
--
-- if (is_tdp_mmu_enabled(kvm))
-- write_protected |=
-- kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
--
-- return write_protected;
-+ return cachepc_kvm_mmu_slot_gfn_protect(kvm, slot,
-+ gfn, min_level, KVM_PAGE_TRACK_WRITE);
- }
-
- static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
-@@ -3901,18 +3870,25 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
- static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
- struct kvm_page_fault *fault)
- {
-- if (unlikely(fault->rsvd))
-- return false;
-+ int active;
-
-- if (!fault->present || !fault->write)
-- return false;
-+ cachepc_page_fault_handle(vcpu, fault);
-
- /*
- * guest is writing the page which is write tracked which can
- * not be fixed by page fault handler.
- */
-- if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
-- return true;
-+ active = kvm_slot_page_track_is_active(vcpu->kvm,
-+ fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE);
-+ active |= kvm_slot_page_track_is_active(vcpu->kvm,
-+ fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS);
-+ if (active) return true;
-+
-+ if (unlikely(fault->rsvd))
-+ return false;
-+
-+ if (!fault->present || !fault->write)
-+ return false;
-
- return false;
- }
-diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c
-index 2e09d1b6249f..315b2d06118c 100644
---- a/arch/x86/kvm/mmu/page_track.c
-+++ b/arch/x86/kvm/mmu/page_track.c
-@@ -19,6 +19,9 @@
- #include "mmu.h"
- #include "mmu_internal.h"
-
-+#include "../cachepc/cachepc.h"
-+#include "../cachepc/track.h"
-+
- bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
- {
- return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) ||
-@@ -115,7 +118,6 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
- struct kvm_memory_slot *slot, gfn_t gfn,
- enum kvm_page_track_mode mode)
- {
--
- if (WARN_ON(!page_track_mode_is_valid(mode)))
- return;
-
-@@ -123,6 +125,8 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
- !kvm_page_track_write_tracking_enabled(kvm)))
- return;
-
-+ CPC_DBG("Tracking page: %llu %i\n", gfn, mode);
-+
- update_gfn_track(slot, gfn, mode, 1);
-
- /*
-@@ -131,9 +135,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
- */
- kvm_mmu_gfn_disallow_lpage(slot, gfn);
-
-- if (mode == KVM_PAGE_TRACK_WRITE)
-- if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
-- kvm_flush_remote_tlbs(kvm);
-+ if (cachepc_kvm_mmu_slot_gfn_protect(kvm,
-+ slot, gfn, PG_LEVEL_4K, mode)) {
-+ kvm_flush_remote_tlbs(kvm);
-+ }
- }
- EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
-
-@@ -161,6 +166,8 @@ void kvm_slot_page_track_remove_page(struct kvm *kvm,
- !kvm_page_track_write_tracking_enabled(kvm)))
- return;
-
-+ CPC_DBG("Untracking page: %llu %i\n", gfn, mode);
-+
- update_gfn_track(slot, gfn, mode, -1);
-
- /*
-diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
-index 7b9265d67131..68b9134970da 100644
---- a/arch/x86/kvm/mmu/tdp_mmu.c
-+++ b/arch/x86/kvm/mmu/tdp_mmu.c
-@@ -1810,13 +1810,8 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
- zap_collapsible_spte_range(kvm, root, slot);
- }
-
--/*
-- * Removes write access on the last level SPTE mapping this GFN and unsets the
-- * MMU-writable bit to ensure future writes continue to be intercepted.
-- * Returns true if an SPTE was set and a TLB flush is needed.
-- */
--static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
-- gfn_t gfn, int min_level)
-+static bool cachepc_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
-+ gfn_t gfn, int min_level, int mode)
- {
- struct tdp_iter iter;
- u64 new_spte;
-@@ -1831,8 +1826,13 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
- !is_last_spte(iter.old_spte, iter.level))
- continue;
-
-- new_spte = iter.old_spte &
-- ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask);
-+ new_spte = iter.old_spte & ~shadow_mmu_writable_mask;
-+ new_spte &= ~PT_WRITABLE_MASK;
-+ if (mode == KVM_PAGE_TRACK_ACCESS) {
-+ new_spte &= ~PT_PRESENT_MASK;
-+ new_spte &= ~PT_USER_MASK;
-+ new_spte |= (0x1ULL << PT64_NX_SHIFT);
-+ }
-
- if (new_spte == iter.old_spte)
- break;
-@@ -1846,6 +1846,58 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
- return spte_set;
- }
-
-+bool cachepc_tdp_protect_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
-+ gfn_t gfn, int min_level, int mode)
-+{
-+ struct kvm_mmu_page *root;
-+ bool spte_set = false;
-+
-+ // pr_warn("Sevstep: tdp_protect_gfn\n");
-+
-+ lockdep_assert_held_write(&kvm->mmu_lock);
-+ for_each_tdp_mmu_root(kvm, root, slot->as_id)
-+ spte_set |= cachepc_protect_gfn(kvm, root, gfn, min_level, mode);
-+
-+ return spte_set;
-+}
-+EXPORT_SYMBOL(cachepc_tdp_protect_gfn);
-+
-+/*
-+ * Removes write access on the last level SPTE mapping this GFN and unsets the
-+ * MMU-writable bit to ensure future writes continue to be intercepted.
-+ * Returns true if an SPTE was set and a TLB flush is needed.
-+ */
-+// static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
-+// gfn_t gfn, int min_level)
-+// {
-+// struct tdp_iter iter;
-+// u64 new_spte;
-+// bool spte_set = false;
-+//
-+// BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
-+//
-+// rcu_read_lock();
-+//
-+// for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) {
-+// if (!is_shadow_present_pte(iter.old_spte) ||
-+// !is_last_spte(iter.old_spte, iter.level))
-+// continue;
-+//
-+// new_spte = iter.old_spte & ~shadow_mmu_writable_mask;
-+// new_spte &= ~PT_WRITABLE_MASK;
-+//
-+// if (new_spte == iter.old_spte)
-+// break;
-+//
-+// tdp_mmu_set_spte(kvm, &iter, new_spte);
-+// spte_set = true;
-+// }
-+//
-+// rcu_read_unlock();
-+//
-+// return spte_set;
-+// }
-+
- /*
- * Removes write access on the last level SPTE mapping this GFN and unsets the
- * MMU-writable bit to ensure future writes continue to be intercepted.
-@@ -1855,14 +1907,17 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
- struct kvm_memory_slot *slot, gfn_t gfn,
- int min_level)
- {
-- struct kvm_mmu_page *root;
-- bool spte_set = false;
-+ return cachepc_tdp_protect_gfn(kvm, slot, gfn, min_level,
-+ KVM_PAGE_TRACK_WRITE);
-
-- lockdep_assert_held_write(&kvm->mmu_lock);
-- for_each_tdp_mmu_root(kvm, root, slot->as_id)
-- spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
-+ // struct kvm_mmu_page *root;
-+ // bool spte_set = false;
-
-- return spte_set;
-+ // lockdep_assert_held_write(&kvm->mmu_lock);
-+ // for_each_tdp_mmu_root(kvm, root, slot->as_id)
-+ // spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
-+
-+ // return spte_set;
- }
-
- /*
-diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
-index a4f6d10b0ef3..62c1fcd563a6 100644
---- a/arch/x86/kvm/svm/sev.c
-+++ b/arch/x86/kvm/svm/sev.c
-@@ -35,6 +35,9 @@
- #include "trace.h"
- #include "mmu.h"
-
-+#include "asm/set_memory.h"
-+#include "cachepc/cachepc.h"
-+
- #ifndef CONFIG_KVM_AMD_SEV
- /*
- * When this config is not defined, SEV feature is not supported and APIs in
-@@ -888,7 +891,7 @@ static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
- &data, error);
- }
-
--static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
-+int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
- unsigned long dst_paddr, int sz, int *err)
- {
- int offset;
-@@ -904,12 +907,20 @@ static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
- return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
- }
-
-+
-+int sev_dbg_decrypt_ext(struct kvm *kvm, unsigned long src_paddr,
-+ unsigned long dst_paddr, int sz, int *err) {
-+ return __sev_dbg_decrypt(kvm, src_paddr, dst_paddr, sz, err);
-+}
-+EXPORT_SYMBOL(sev_dbg_decrypt_ext);
-+
- static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
- void __user *dst_uaddr,
- unsigned long dst_paddr,
- int size, int *err)
- {
- struct page *tpage = NULL;
-+ struct vcpu_svm *svm;
- int ret, offset;
-
- /* if inputs are not 16-byte then use intermediate buffer */
-@@ -923,6 +934,11 @@ static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
- dst_paddr = __sme_page_pa(tpage);
- }
-
-+ if (dst_uaddr == CPC_VMSA_MAGIC_ADDR) {
-+ svm = to_svm(xa_load(&kvm->vcpu_array, 0));
-+ paddr = __pa(svm->sev_es.vmsa);
-+ }
-+
- ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
- if (ret)
- goto e_free;
-@@ -1024,6 +1040,7 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
- struct kvm_sev_dbg debug;
- unsigned long n;
- unsigned int size;
-+ bool vmsa_dec;
- int ret;
-
- if (!sev_guest(kvm))
-@@ -1037,6 +1054,13 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
- if (!debug.dst_uaddr)
- return -EINVAL;
-
-+ vmsa_dec = false;
-+ if (debug.src_uaddr == (uintptr_t) CPC_VMSA_MAGIC_ADDR) {
-+ debug.len = PAGE_SIZE;
-+ debug.src_uaddr = debug.dst_uaddr;
-+ vmsa_dec = true;
-+ }
-+
- vaddr = debug.src_uaddr;
- size = debug.len;
- vaddr_end = vaddr + size;
-@@ -1075,7 +1099,8 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
- if (dec)
- ret = __sev_dbg_decrypt_user(kvm,
- __sme_page_pa(src_p[0]) + s_off,
-- (void __user *)dst_vaddr,
-+ vmsa_dec ? CPC_VMSA_MAGIC_ADDR
-+ : (void __user *)dst_vaddr,
- __sme_page_pa(dst_p[0]) + d_off,
- len, &argp->error);
- else
-@@ -2170,6 +2195,62 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
- return ret;
- }
-
-+static int rmpupdate_noremap(u64 pfn, struct rmpupdate *val)
-+{
-+ unsigned long paddr = pfn << PAGE_SHIFT;
-+ int ret, level, npages;
-+ int retries = 0;
-+
-+ if (!pfn_valid(pfn))
-+ return -EINVAL;
-+
-+ if (!cpu_feature_enabled(X86_FEATURE_SEV_SNP))
-+ return -ENXIO;
-+
-+ level = RMP_TO_X86_PG_LEVEL(val->pagesize);
-+ npages = page_level_size(level) / PAGE_SIZE;
-+
-+
-+retry:
-+ /* Binutils version 2.36 supports the RMPUPDATE mnemonic. */
-+ asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFE"
-+ : "=a"(ret)
-+ : "a"(paddr), "c"((unsigned long)val)
-+ : "memory", "cc");
-+
-+ if (ret) {
-+ if (!retries) {
-+ pr_err("rmpupdate failed, ret: %d, pfn: %llx, npages: %d, level: %d, retrying (max: %d)...\n",
-+ ret, pfn, npages, level, 2 * num_present_cpus());
-+ dump_stack();
-+ }
-+ retries++;
-+ if (retries < 2 * num_present_cpus())
-+ goto retry;
-+ } else if (retries > 0) {
-+ pr_err("rmpupdate for pfn %llx succeeded after %d retries\n", pfn, retries);
-+ }
-+
-+ return ret;
-+}
-+
-+int rmp_make_private_noremap(u64 pfn, u64 gpa, enum pg_level level, int asid, bool immutable)
-+{
-+ struct rmpupdate val;
-+
-+ if (!pfn_valid(pfn))
-+ return -EINVAL;
-+
-+ memset(&val, 0, sizeof(val));
-+ val.assigned = 1;
-+ val.asid = asid;
-+ val.immutable = immutable;
-+ val.gpa = gpa;
-+ val.pagesize = X86_TO_RMP_PG_LEVEL(level);
-+
-+ return rmpupdate_noremap(pfn, &val);
-+}
-+
- static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
- {
- struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
-@@ -2183,16 +2264,20 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp)
- struct vcpu_svm *svm = to_svm(xa_load(&kvm->vcpu_array, i));
- u64 pfn = __pa(svm->sev_es.vmsa) >> PAGE_SHIFT;
-
-+ CPC_WARN("RIP READ PRE-PRIVATE: %llu\n", svm->sev_es.vmsa->rip);
-+
- /* Perform some pre-encryption checks against the VMSA */
- ret = sev_es_sync_vmsa(svm);
- if (ret)
- return ret;
-
- /* Transition the VMSA page to a firmware state. */
-- ret = rmp_make_private(pfn, -1, PG_LEVEL_4K, sev->asid, true);
-+ ret = rmp_make_private_noremap(pfn, -1, PG_LEVEL_4K, sev->asid, true);
- if (ret)
- return ret;
-
-+ CPC_WARN("RIP READ POST-PRIVATE: %llu\n", svm->sev_es.vmsa->rip);
-+
- /* Issue the SNP command to encrypt the VMSA */
- data.address = __sme_pa(svm->sev_es.vmsa);
- ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_SNP_LAUNCH_UPDATE,
-@@ -3149,9 +3234,9 @@ static int sev_es_validate_vmgexit(struct vcpu_svm *svm, u64 *exit_code)
- }
- break;
- case SVM_EXIT_VMMCALL:
-- if (!ghcb_rax_is_valid(ghcb) ||
-- !ghcb_cpl_is_valid(ghcb))
-- goto vmgexit_err;
-+ // if (!ghcb_rax_is_valid(ghcb) ||
-+ // !ghcb_cpl_is_valid(ghcb))
-+ // goto vmgexit_err;
- break;
- case SVM_EXIT_RDTSCP:
- break;
-@@ -3920,6 +4005,7 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm)
- goto out;
- }
-
-+ CPC_WARN("VMSA_GPA SET via VMGEXIT_AP_CREATE\n");
- target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2;
- break;
- case SVM_VMGEXIT_AP_DESTROY:
-diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
-index cf0bf456d520..4a25e306543a 100644
---- a/arch/x86/kvm/svm/svm.c
-+++ b/arch/x86/kvm/svm/svm.c
-@@ -2,6 +2,10 @@
-
- #include <linux/kvm_host.h>
-
-+#include "cachepc/cachepc.h"
-+#include "cachepc/event.h"
-+#include "cachepc/track.h"
-+
- #include "irq.h"
- #include "mmu.h"
- #include "kvm_cache_regs.h"
-@@ -2081,9 +2085,74 @@ static int smi_interception(struct kvm_vcpu *vcpu)
- return 1;
- }
-
-+// static void hexdump(uint8_t *prev, uint8_t *cur, size_t len)
-+// {
-+// size_t i;
-+//
-+// for (i = 0; i < len; i++) {
-+// //printk(KERN_CONT "%02X ", cur[i]);
-+// if (cur[i] != prev[i])
-+// printk(KERN_CONT "%02X ", cur[i]);
-+// else
-+// printk(KERN_CONT " ");
-+// if ((i+1) % 16 == 0)
-+// printk(KERN_CONT "\n");
-+// }
-+// printk(KERN_CONT "\n");
-+// }
-+
- static int intr_interception(struct kvm_vcpu *vcpu)
- {
-+ struct vmcb_control_area *control;
-+ struct vcpu_svm *svm;
-+ struct cpc_fault *fault, *next;
-+ size_t count;
-+
- ++vcpu->stat.irq_exits;
-+
-+ if (cachepc_track_mode == CPC_TRACK_DATA_ACCESS && cachepc_single_step) {
-+ svm = to_svm(vcpu);
-+ control = &svm->vmcb->control;
-+
-+ cachepc_rip = svm->sev_es.vmsa->rip;
-+ if (!cachepc_rip_prev)
-+ cachepc_rip_prev = cachepc_rip;
-+ if (cachepc_rip == cachepc_rip_prev) {
-+ cachepc_apic_timer += 1;
-+ return 1;
-+ }
-+ CPC_INFO("Detected RIP change! (%u)\n", cachepc_apic_timer);
-+
-+ // if (!cachepc_retinst_prev)
-+ // cachepc_retinst_prev = cachepc_retinst;
-+ // if (cachepc_retinst_prev == cachepc_retinst) {
-+ // cachepc_apic_timer += 1;
-+ // return 1;
-+ // }
-+ // CPC_INFO("Detected RETINST change! (%llu,%u)\n",
-+ // cachepc_retinst, cachepc_apic_timer);
-+
-+ cachepc_single_step = false;
-+
-+ count = 0;
-+ list_for_each_entry(fault, &cachepc_faults, list)
-+ count += 1;
-+
-+ CPC_INFO("Caught single step with %lu faults!\n", count);
-+ if (count == 0 || count > 2)
-+ CPC_ERR("Unexpected step fault count: %lu faults!\n", count);
-+
-+ list_for_each_entry(fault, &cachepc_faults, list)
-+ cachepc_track_single(vcpu, fault->gfn, KVM_PAGE_TRACK_ACCESS);
-+
-+ cachepc_send_track_event(&cachepc_faults);
-+
-+ list_for_each_entry_safe(fault, next, &cachepc_faults, list) {
-+ list_del(&fault->list);
-+ kfree(fault);
-+ }
-+ }
-+
- return 1;
- }
-
-@@ -3269,9 +3338,25 @@ static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
-
- int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
- {
-+ static const struct {
-+ u64 code;
-+ const char *name;
-+ } codelut[] = {
-+ SVM_EXIT_REASONS,
-+ { -1, NULL }
-+ };
-+ size_t i;
-+
- if (!svm_check_exit_valid(exit_code))
- return svm_handle_invalid_exit(vcpu, exit_code);
-
-+ if (cachepc_debug && cachepc_track_mode != CPC_TRACK_NONE) {
-+ for (i = 0; i < sizeof(codelut) / sizeof(codelut[0]); i++) {
-+ if (codelut[i].code == exit_code)
-+ pr_warn("KVM EXIT (%s)\n", codelut[i].name);
-+ }
-+ }
-+
- #ifdef CONFIG_RETPOLINE
- if (exit_code == SVM_EXIT_MSR)
- return msr_interception(vcpu);
-@@ -3787,15 +3872,48 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
- static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
- {
- struct vcpu_svm *svm = to_svm(vcpu);
-- unsigned long vmcb_pa = svm->current_vmcb->pa;
-+ unsigned long vmcb_pa = svm->current_vmcb->pa;
-+ int cpu;
-
- guest_state_enter_irqoff();
-
- if (sev_es_guest(vcpu->kvm)) {
-+ if (cachepc_single_step && cachepc_apic_timer == 0) {
-+ cachepc_apic_timer = 200;
-+ cachepc_retinst_prev = 0;
-+ cachepc_rip_prev = 0;
-+ }
-+
-+ cpu = get_cpu();
-+ WARN_ON(cpu != 2);
-+
-+ memset(cachepc_msrmts, 0,
-+ cachepc_msrmts_count * sizeof(cpc_msrmt_t));
-+
-+ cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC);
- __svm_sev_es_vcpu_run(vmcb_pa);
-+ cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC) - cachepc_retinst;
-+
-+ cachepc_save_msrmts(cachepc_ds);
-+ if (cachepc_baseline_measure)
-+ cachepc_update_baseline();
-+
-+ put_cpu();
- } else {
- struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
-
-+ if (cachepc_single_step && cachepc_apic_timer == 0) {
-+ cachepc_apic_timer = 50;
-+ cachepc_retinst_prev = 0;
-+ cachepc_rip_prev = 0;
-+ }
-+
-+ cpu = get_cpu();
-+ WARN_ON(cpu != 2);
-+
-+ memset(cachepc_msrmts, 0,
-+ cachepc_msrmts_count * sizeof(cpc_msrmt_t));
-+
- /*
- * Use a single vmcb (vmcb01 because it's always valid) for
- * context switching guest state via VMLOAD/VMSAVE, that way
-@@ -3806,7 +3924,15 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
- __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs);
- vmsave(svm->vmcb01.pa);
-
-+ cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC);
- vmload(__sme_page_pa(sd->save_area));
-+ cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC) - cachepc_retinst;
-+
-+ cachepc_save_msrmts(cachepc_ds);
-+ if (cachepc_baseline_measure)
-+ cachepc_update_baseline();
-+
-+ put_cpu();
- }
-
- guest_state_exit_irqoff();
-diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
-index dfaeb47fcf2a..0626f3fdddfd 100644
---- a/arch/x86/kvm/svm/vmenter.S
-+++ b/arch/x86/kvm/svm/vmenter.S
-@@ -29,12 +29,59 @@
-
- .section .noinstr.text, "ax"
-
-+.extern cachepc_msrmts
-+.extern cachepc_regs_tmp
-+.extern cachepc_regs_vm
-+
-+.macro load_tmp off reg
-+ mov cachepc_regs_tmp+\off(%rip), \reg
-+.endm
-+
-+.macro save_tmp off reg
-+ mov \reg, cachepc_regs_tmp+\off(%rip)
-+.endm
-+
-+.macro load_vm off reg
-+ mov cachepc_regs_vm+\off(%rip), \reg
-+.endm
-+
-+.macro save_vm off reg
-+ mov \reg, cachepc_regs_vm+\off(%rip)
-+.endm
-+
-+.macro apply_regs func
-+ \func 0x00, %rax
-+ \func 0x08, %rbx
-+ \func 0x10, %rcx
-+ \func 0x18, %rdx
-+ \func 0x20, %rbp
-+ \func 0x28, %rsp
-+ \func 0x30, %rdi
-+ \func 0x38, %rsi
-+ \func 0x40, %r8
-+ \func 0x48, %r9
-+ \func 0x50, %r10
-+ \func 0x58, %r11
-+ \func 0x60, %r12
-+ \func 0x68, %r13
-+ \func 0x70, %r14
-+ \func 0x78, %r15
-+.endm
-+
-+.macro barrier
-+ mfence
-+ mov $0x80000005,%eax
-+ cpuid
-+.endm
-+
- /**
- * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
- * @vmcb_pa: unsigned long
- * @regs: unsigned long * (to guest registers)
- */
- SYM_FUNC_START(__svm_vcpu_run)
-+ apply_regs save_tmp
-+
- push %_ASM_BP
- #ifdef CONFIG_X86_64
- push %r15
-@@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run)
- /* Enter guest mode */
- sti
-
--1: vmrun %_ASM_AX
-+1:
-+ apply_regs save_vm
-+ apply_regs load_tmp
-+ mov cachepc_ds, %rsi
-+ mov 0x8(%rsi), %r15
-+ lea sev_prime_ret(%rip), %rdi
-+ jmp cachepc_prime_vcall+5+1 // skip stack pushs
-+sev_prime_ret:
-+ apply_regs save_tmp
-+ apply_regs load_vm
-+
-+ vmrun %_ASM_AX
-+
-+ apply_regs save_vm
-+ apply_regs load_tmp
-+ mov %r15, %rsi
-+ lea sev_probe_ret(%rip), %rdi
-+ jmp cachepc_probe_vcall+5+8 // skip stack pushs
-+sev_probe_ret:
-+ apply_regs save_tmp
-+ apply_regs load_vm
-
- 2: cli
-
-@@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run)
- * @vmcb_pa: unsigned long
- */
- SYM_FUNC_START(__svm_sev_es_vcpu_run)
-+ apply_regs save_tmp
-+
- push %_ASM_BP
- #ifdef CONFIG_X86_64
- push %r15
-@@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
- /* Enter guest mode */
- sti
-
--1: vmrun %_ASM_AX
-+1:
-+
-+ apply_regs save_vm
-+ apply_regs load_tmp
-+ mov cachepc_ds, %rsi
-+ mov 0x8(%rsi), %r15
-+ lea sev_es_prime_ret(%rip), %rdi
-+ jmp cachepc_prime_vcall+5+1 // skip stack pushes
-+sev_es_prime_ret:
-+ apply_regs save_tmp
-+ apply_regs load_vm
-+
-+ vmrun %_ASM_AX
-+
-+ apply_regs save_vm
-+ apply_regs load_tmp
-+ mov %r15, %rsi
-+ lea sev_es_probe_ret(%rip), %rdi
-+ jmp cachepc_probe_vcall+5+8 // skip stack pushs
-+sev_es_probe_ret:
-+ apply_regs save_tmp
-+ apply_regs load_vm
-
- 2: cli
-
-diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
-index d9adf79124f9..dfe54e6ca5cc 100644
---- a/arch/x86/kvm/x86.c
-+++ b/arch/x86/kvm/x86.c
-@@ -82,6 +82,10 @@
- #include <asm/sgx.h>
- #include <clocksource/hyperv_timer.h>
-
-+#include "cachepc/cachepc.h"
-+#include "cachepc/event.h"
-+#include "cachepc/track.h"
-+
- #define CREATE_TRACE_POINTS
- #include "trace.h"
-
-@@ -9267,10 +9271,10 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
- a3 &= 0xFFFFFFFF;
- }
-
-- if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
-- ret = -KVM_EPERM;
-- goto out;
-- }
-+ // if (static_call(kvm_x86_get_cpl)(vcpu) != 0) {
-+ // ret = -KVM_EPERM;
-+ // goto out;
-+ // }
-
- ret = -KVM_ENOSYS;
-
-@@ -9326,11 +9330,16 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
- vcpu->arch.complete_userspace_io = complete_hypercall_exit;
- return 0;
- }
-+ case KVM_HC_CPC_VMMCALL:
-+ CPC_WARN("Cachepc: Hypecrcall Run\n");
-+ cachepc_send_guest_event(a0, a1);
-+ ret = 0;
-+ break;
- default:
- ret = -KVM_ENOSYS;
- break;
- }
--out:
-+//out:
- if (!op_64_bit)
- ret = (u32)ret;
- kvm_rax_write(vcpu, ret);
-diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c
-index 27ab27931813..90679ec8ba79 100644
---- a/crypto/aes_generic.c
-+++ b/crypto/aes_generic.c
-@@ -1173,8 +1173,78 @@ EXPORT_SYMBOL_GPL(crypto_aes_set_key);
- f_rl(bo, bi, 3, k); \
- } while (0)
-
-+#define L1_ASSOC 8
-+#define L1_LINESIZE 64
-+#define L1_SETS 64
-+#define L1_SIZE (L1_SETS * L1_ASSOC * L1_LINESIZE)
-+
-+#define ACCESS_LINE(n) \
-+ asm volatile ("mov (%0), %%rbx" \
-+ : : "r"(((uint8_t*) L1) + n * L1_LINESIZE) : "rbx");
-+
-+#define DO_ACCESS_PATTERN() \
-+ ACCESS_LINE(60) \
-+ ACCESS_LINE(13) \
-+ ACCESS_LINE(24) \
-+ ACCESS_LINE(19) \
-+ ACCESS_LINE(38) \
-+ ACCESS_LINE(17) \
-+ ACCESS_LINE( 2) \
-+ ACCESS_LINE(12) \
-+ ACCESS_LINE(22) \
-+ ACCESS_LINE(46) \
-+ ACCESS_LINE( 4) \
-+ ACCESS_LINE(61) \
-+ ACCESS_LINE( 5) \
-+ ACCESS_LINE(14) \
-+ ACCESS_LINE(11) \
-+ ACCESS_LINE(35) \
-+ ACCESS_LINE(45) \
-+ ACCESS_LINE(10) \
-+ ACCESS_LINE(49) \
-+ ACCESS_LINE(56) \
-+ ACCESS_LINE(27) \
-+ ACCESS_LINE(37) \
-+ ACCESS_LINE(63) \
-+ ACCESS_LINE(54) \
-+ ACCESS_LINE(55) \
-+ ACCESS_LINE(29) \
-+ ACCESS_LINE(48) \
-+ ACCESS_LINE( 9) \
-+ ACCESS_LINE(16) \
-+ ACCESS_LINE(39) \
-+ ACCESS_LINE(20) \
-+ ACCESS_LINE(21) \
-+ ACCESS_LINE(62) \
-+ ACCESS_LINE( 0) \
-+ ACCESS_LINE(34) \
-+ ACCESS_LINE( 8) \
-+ ACCESS_LINE(53) \
-+ ACCESS_LINE(42) \
-+ ACCESS_LINE(51) \
-+ ACCESS_LINE(50) \
-+ ACCESS_LINE(57) \
-+ ACCESS_LINE( 7) \
-+ ACCESS_LINE( 6) \
-+ ACCESS_LINE(33) \
-+ ACCESS_LINE(26) \
-+
-+uint8_t *L1 = NULL;
-+
- static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
- {
-+ int cpu;
-+
-+ if (L1 == NULL) {
-+ L1 = kzalloc(L1_SETS * L1_LINESIZE, GFP_KERNEL);
-+ BUG_ON(((uintptr_t)L1) % (L1_SETS * L1_LINESIZE) != 0);
-+ }
-+
-+ pr_warn("CachePC-TEST: Running AES-Generic!");
-+
-+ cpu = get_cpu();
-+ DO_ACCESS_PATTERN()
-+
- const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
- u32 b0[4], b1[4];
- const u32 *kp = ctx->key_enc + 4;
-@@ -1210,6 +1280,9 @@ static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
- put_unaligned_le32(b0[1], out + 4);
- put_unaligned_le32(b0[2], out + 8);
- put_unaligned_le32(b0[3], out + 12);
-+
-+ DO_ACCESS_PATTERN();
-+ put_cpu();
- }
-
- /* decrypt a block of text */
-diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
-old mode 100644
-new mode 100755
-index e089fbf9017f..7899e1efe852
---- a/drivers/crypto/ccp/sev-dev.c
-+++ b/drivers/crypto/ccp/sev-dev.c
-@@ -87,7 +87,7 @@ static void *sev_init_ex_buffer;
- static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE;
-
- static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret);
--static int sev_do_cmd(int cmd, void *data, int *psp_ret);
-+int sev_do_cmd(int cmd, void *data, int *psp_ret);
-
- static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
- {
-@@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
- return ret;
- }
-
--static int sev_do_cmd(int cmd, void *data, int *psp_ret)
-+int sev_do_cmd(int cmd, void *data, int *psp_ret)
- {
- int rc;
-
-@@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret)
-
- return rc;
- }
-+EXPORT_SYMBOL(sev_do_cmd);
-
- static int __sev_init_locked(int *error)
- {
-diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
-index 5b1019dab328..ec317e7c348a 100644
---- a/drivers/iommu/amd/amd_iommu_types.h
-+++ b/drivers/iommu/amd/amd_iommu_types.h
-@@ -275,7 +275,7 @@
- *
- * 512GB Pages are not supported due to a hardware bug
- */
--#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))
-+#define AMD_IOMMU_PGSIZES (PAGE_SIZE)
-
- /* Bit value definition for dte irq remapping fields*/
- #define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
-diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index f2a63cb2658b..869faf927e5d 100644
---- a/virt/kvm/kvm_main.c
-+++ b/virt/kvm/kvm_main.c
-@@ -13,6 +13,7 @@
- * Yaniv Kamay <yaniv@qumranet.com>
- */
-
-+#include <asm-generic/errno-base.h>
- #include <kvm/iodev.h>
-
- #include <linux/kvm_host.h>
-@@ -64,12 +65,15 @@
-
- #define CREATE_TRACE_POINTS
- #include <trace/events/kvm.h>
-+#include "../../arch/x86/kvm/cachepc/track.h"
-
- #include <linux/kvm_dirty_ring.h>
-
- /* Worst case buffer size needed for holding an integer. */
- #define ITOA_MAX_LEN 12
-
-+#include "../../arch/x86/kvm/cachepc/kvm.h"
-+
- MODULE_AUTHOR("Qumranet");
- MODULE_LICENSE("GPL");
-
-@@ -1261,6 +1265,9 @@ static void kvm_destroy_vm(struct kvm *kvm)
- hardware_disable_all();
- mmdrop(mm);
- module_put(kvm_chardev_ops.owner);
-+
-+ if (main_vm == kvm)
-+ main_vm = NULL;
- }
-
- void kvm_get_kvm(struct kvm *kvm)
-@@ -1360,7 +1367,7 @@ static void kvm_insert_gfn_node(struct kvm_memslots *slots,
- int idx = slots->node_idx;
-
- parent = NULL;
-- for (node = &gfn_tree->rb_node; *node; ) {
-+ for (node = &gfn_tree->rb_node; *node;) {
- struct kvm_memory_slot *tmp;
-
- tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
-@@ -4514,7 +4521,7 @@ static long kvm_vm_ioctl(struct file *filp,
- void __user *argp = (void __user *)arg;
- int r;
-
-- if (kvm->mm != current->mm || kvm->vm_dead)
-+ if ((ioctl != KVM_MEMORY_ENCRYPT_OP && kvm->mm != current->mm) || kvm->vm_dead)
- return -EIO;
- switch (ioctl) {
- case KVM_CREATE_VCPU:
-@@ -4823,6 +4830,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type)
- kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
-
- fd_install(r, file);
-+
-+ main_vm = kvm;
-+
- return r;
-
- put_kvm:
-@@ -4864,7 +4874,7 @@ static long kvm_dev_ioctl(struct file *filp,
- r = -EOPNOTSUPP;
- break;
- default:
-- return kvm_arch_dev_ioctl(filp, ioctl, arg);
-+ return cachepc_kvm_ioctl(filp, ioctl, arg);
- }
- out:
- return r;
-@@ -5792,6 +5802,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
- r = kvm_vfio_ops_init();
- WARN_ON(r);
-
-+ cachepc_kvm_init();
-+
- return 0;
-
- out_unreg:
-@@ -5821,6 +5833,8 @@ void kvm_exit(void)
- {
- int cpu;
-
-+ cachepc_kvm_exit();
-+
- debugfs_remove_recursive(kvm_debugfs_dir);
- misc_deregister(&kvm_dev);
- for_each_possible_cpu(cpu)
