path: root/patch.diff

diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h
index eb186bc57f6a..b96e80934005 100644
--- a/arch/x86/include/asm/kvm_page_track.h
+++ b/arch/x86/include/asm/kvm_page_track.h
@@ -2,10 +2,9 @@
 #ifndef _ASM_X86_KVM_PAGE_TRACK_H
 #define _ASM_X86_KVM_PAGE_TRACK_H
 
-enum kvm_page_track_mode {
-	KVM_PAGE_TRACK_WRITE,
-	KVM_PAGE_TRACK_MAX,
-};
+#include<linux/srcu.h>
+
+#include "../../kvm/cachepc/uapi.h"
 
 /*
  * The notifier represented by @kvm_page_track_notifier_node is linked into
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 30f244b64523..c75819a6cd77 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-ccflags-y += -I $(srctree)/arch/x86/kvm
+ccflags-y += -I $(srctree)/arch/x86/kvm -O2
 ccflags-$(CONFIG_KVM_WERROR) += -Werror
 
 ifeq ($(CONFIG_FRAME_POINTER),y)
@@ -11,8 +11,9 @@ include $(srctree)/virt/kvm/Makefile.kvm
 
 kvm-y			+= x86.o emulate.o i8259.o irq.o lapic.o \
 			   i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
-			   hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \
-			   mmu/spte.o
+			   hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \
+			   cachepc/cachepc.o cachepc/kvm.o \
+			   cachepc/tracking.o cachepc/events.o
 
 ifdef CONFIG_HYPERV
 kvm-y			+= kvm_onhyperv.o
@@ -25,7 +26,8 @@ kvm-intel-y		+= vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
 			   vmx/evmcs.o vmx/nested.o vmx/posted_intr.o
 kvm-intel-$(CONFIG_X86_SGX_KVM)	+= vmx/sgx.o
 
-kvm-amd-y		+= svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o
+kvm-amd-y		+= svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o \
+				   svm/avic.o svm/sev.o cachepc/cachepc.o cachepc/events.o
 
 ifdef CONFIG_HYPERV
 kvm-amd-y		+= svm/svm_onhyperv.o
diff --git a/arch/x86/kvm/cachepc b/arch/x86/kvm/cachepc
new file mode 120000
index 000000000000..9119e44af1f0
--- /dev/null
+++ b/arch/x86/kvm/cachepc
@@ -0,0 +1 @@
+/home/louis/kvm-prime-count/cachepc
\ No newline at end of file
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index d871b8dee7b3..317dcb165e92 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
 	}
 }
 
+#include "../cachepc/mmu.c"
+
 /*
  * Write-protect on the specified @sptep, @pt_protect indicates whether
  * spte write-protection is caused by protecting shadow page table.
@@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
  *
  * Return true if tlb need be flushed.
  */
-static bool spte_write_protect(u64 *sptep, bool pt_protect)
-{
-	u64 spte = *sptep;
-
-	if (!is_writable_pte(spte) &&
-	    !(pt_protect && is_mmu_writable_spte(spte)))
-		return false;
-
-	rmap_printk("spte %p %llx\n", sptep, *sptep);
-
-	if (pt_protect)
-		spte &= ~shadow_mmu_writable_mask;
-	spte = spte & ~PT_WRITABLE_MASK;
-
-	return mmu_spte_update(sptep, spte);
-}
+// static bool spte_write_protect(u64 *sptep, bool pt_protect)
+// {
+// 	return cachepc_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE);
+// }
 
 static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
 			       bool pt_protect)
 {
-	u64 *sptep;
-	struct rmap_iterator iter;
-	bool flush = false;
-
-	for_each_rmap_spte(rmap_head, &iter, sptep)
-		flush |= spte_write_protect(sptep, pt_protect);
-
-	return flush;
+	return cachepc_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE);
 }
 
 static bool spte_clear_dirty(u64 *sptep)
@@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
 				    struct kvm_memory_slot *slot, u64 gfn,
 				    int min_level)
 {
-	struct kvm_rmap_head *rmap_head;
-	int i;
-	bool write_protected = false;
-
-	if (kvm_memslots_have_rmaps(kvm)) {
-		for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
-			rmap_head = gfn_to_rmap(gfn, i, slot);
-			write_protected |= rmap_write_protect(rmap_head, true);
-		}
-	}
-
-	if (is_tdp_mmu_enabled(kvm))
-		write_protected |=
-			kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
-
-	return write_protected;
+	return cachepc_kvm_mmu_slot_gfn_protect(kvm, slot,
+		gfn, min_level, KVM_PAGE_TRACK_WRITE);
 }
 
 static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
@@ -3901,18 +3870,25 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
 					 struct kvm_page_fault *fault)
 {
-	if (unlikely(fault->rsvd))
-		return false;
+	int active;
 
-	if (!fault->present || !fault->write)
-		return false;
+	cachepc_uspt_page_fault_handle(vcpu, fault);
 
 	/*
 	 * guest is writing the page which is write tracked which can
 	 * not be fixed by page fault handler.
 	 */
-	if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
-		return true;
+	active = kvm_slot_page_track_is_active(vcpu->kvm,
+		fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE);
+	active |= kvm_slot_page_track_is_active(vcpu->kvm,
+		fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS);
+	if (active) return true;
+
+	if (unlikely(fault->rsvd))
+		return false;
+
+	if (!fault->present || !fault->write)
+		return false;
 
 	return false;
 }
diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c
index 2e09d1b6249f..02821df27f51 100644
--- a/arch/x86/kvm/mmu/page_track.c
+++ b/arch/x86/kvm/mmu/page_track.c
@@ -19,6 +19,8 @@
 #include "mmu.h"
 #include "mmu_internal.h"
 
+#include "../cachepc/tracking.h"
+
 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
 {
 	return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) ||
@@ -115,7 +117,6 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
 				  struct kvm_memory_slot *slot, gfn_t gfn,
 				  enum kvm_page_track_mode mode)
 {
-
 	if (WARN_ON(!page_track_mode_is_valid(mode)))
 		return;
 
@@ -123,6 +124,8 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
 		    !kvm_page_track_write_tracking_enabled(kvm)))
 		return;
 
+	pr_warn("CachePC: Tracking page: %llu\n", gfn);
+
 	update_gfn_track(slot, gfn, mode, 1);
 
 	/*
@@ -131,9 +134,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
 	 */
 	kvm_mmu_gfn_disallow_lpage(slot, gfn);
 
-	if (mode == KVM_PAGE_TRACK_WRITE)
-		if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
-			kvm_flush_remote_tlbs(kvm);
+	if (cachepc_kvm_mmu_slot_gfn_protect(kvm,
+			slot, gfn, PG_LEVEL_4K, mode)) {
+		kvm_flush_remote_tlbs(kvm);
+	}
 }
 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
 
@@ -161,6 +165,8 @@ void kvm_slot_page_track_remove_page(struct kvm *kvm,
 		    !kvm_page_track_write_tracking_enabled(kvm)))
 		return;
 
+	pr_warn("CachePC: Untracking page: %llu\n", gfn);
+
 	update_gfn_track(slot, gfn, mode, -1);
 
 	/*
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 7b9265d67131..68b9134970da 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -1810,13 +1810,8 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
 		zap_collapsible_spte_range(kvm, root, slot);
 }
 
-/*
- * Removes write access on the last level SPTE mapping this GFN and unsets the
- * MMU-writable bit to ensure future writes continue to be intercepted.
- * Returns true if an SPTE was set and a TLB flush is needed.
- */
-static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
-			      gfn_t gfn, int min_level)
+static bool cachepc_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
+	gfn_t gfn, int min_level, int mode)
 {
 	struct tdp_iter iter;
 	u64 new_spte;
@@ -1831,8 +1826,13 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
 		    !is_last_spte(iter.old_spte, iter.level))
 			continue;
 
-		new_spte = iter.old_spte &
-			~(PT_WRITABLE_MASK | shadow_mmu_writable_mask);
+		new_spte = iter.old_spte & ~shadow_mmu_writable_mask;
+		new_spte &= ~PT_WRITABLE_MASK;
+		if (mode == KVM_PAGE_TRACK_ACCESS) {
+			new_spte &= ~PT_PRESENT_MASK;
+			new_spte &= ~PT_USER_MASK;
+			new_spte |= (0x1ULL << PT64_NX_SHIFT);
+		}
 
 		if (new_spte == iter.old_spte)
 			break;
@@ -1846,6 +1846,58 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
 	return spte_set;
 }
 
+bool cachepc_tdp_protect_gfn(struct kvm *kvm, struct kvm_memory_slot *slot,
+	gfn_t gfn, int min_level, int mode)
+{
+	struct kvm_mmu_page *root;
+	bool spte_set = false;
+
+	// pr_warn("Sevstep: tdp_protect_gfn\n");
+
+	lockdep_assert_held_write(&kvm->mmu_lock);
+	for_each_tdp_mmu_root(kvm, root, slot->as_id)
+		spte_set |= cachepc_protect_gfn(kvm, root, gfn, min_level, mode);
+
+	return spte_set;
+}
+EXPORT_SYMBOL(cachepc_tdp_protect_gfn);
+
+/*
+ * Removes write access on the last level SPTE mapping this GFN and unsets the
+ * MMU-writable bit to ensure future writes continue to be intercepted.
+ * Returns true if an SPTE was set and a TLB flush is needed.
+ */
+// static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
+// 			      gfn_t gfn, int min_level)
+// {
+// 	struct tdp_iter iter;
+// 	u64 new_spte;
+// 	bool spte_set = false;
+// 
+// 	BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
+// 
+// 	rcu_read_lock();
+// 
+// 	for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) {
+// 		if (!is_shadow_present_pte(iter.old_spte) ||
+// 		    !is_last_spte(iter.old_spte, iter.level))
+// 			continue;
+// 
+// 		new_spte = iter.old_spte & ~shadow_mmu_writable_mask;
+// 		new_spte &= ~PT_WRITABLE_MASK;
+// 
+// 		if (new_spte == iter.old_spte)
+// 			break;
+// 
+// 		tdp_mmu_set_spte(kvm, &iter, new_spte);
+// 		spte_set = true;
+// 	}
+// 
+// 	rcu_read_unlock();
+// 
+// 	return spte_set;
+// }
+
 /*
  * Removes write access on the last level SPTE mapping this GFN and unsets the
  * MMU-writable bit to ensure future writes continue to be intercepted.
@@ -1855,14 +1907,17 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
 				   struct kvm_memory_slot *slot, gfn_t gfn,
 				   int min_level)
 {
-	struct kvm_mmu_page *root;
-	bool spte_set = false;
+	return cachepc_tdp_protect_gfn(kvm, slot, gfn, min_level,
+		KVM_PAGE_TRACK_WRITE);
 
-	lockdep_assert_held_write(&kvm->mmu_lock);
-	for_each_tdp_mmu_root(kvm, root, slot->as_id)
-		spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
+	// struct kvm_mmu_page *root;
+	// bool spte_set = false;
 
-	return spte_set;
+	// lockdep_assert_held_write(&kvm->mmu_lock);
+	// for_each_tdp_mmu_root(kvm, root, slot->as_id)
+	// 	spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
+
+	// return spte_set;
 }
 
 /*
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index a4f6d10b0ef3..a1ac048b35cf 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -888,7 +888,7 @@ static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
 			     &data, error);
 }
 
-static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
+int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
 			     unsigned long dst_paddr, int sz, int *err)
 {
 	int offset;
@@ -904,6 +904,13 @@ static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
 	return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
 }
 
+
+int sev_dbg_decrypt_ext(struct kvm *kvm, unsigned long src_paddr,
+			     unsigned long dst_paddr, int sz, int *err) {
+	return __sev_dbg_decrypt(kvm, src_paddr, dst_paddr, sz, err);
+}
+EXPORT_SYMBOL(sev_dbg_decrypt_ext);
+
 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
 				  void __user *dst_uaddr,
 				  unsigned long dst_paddr,
@@ -1026,6 +1033,8 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
 	unsigned int size;
 	int ret;
 
+	pr_warn("DEBUG CRYPT\n");
+
 	if (!sev_guest(kvm))
 		return -ENOTTY;
 
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index cf0bf456d520..4134049e6b08 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -2,6 +2,10 @@
 
 #include <linux/kvm_host.h>
 
+#include "cachepc/cachepc.h"
+#include "cachepc/events.h"
+#include "cachepc/tracking.h"
+
 #include "irq.h"
 #include "mmu.h"
 #include "kvm_cache_regs.h"
@@ -2083,6 +2087,38 @@ static int smi_interception(struct kvm_vcpu *vcpu)
 
 static int intr_interception(struct kvm_vcpu *vcpu)
 {
+	if (cachepc_track_mode == CPC_TRACK_DATA_ACCESS && cachepc_single_step) {
+		cachepc_single_step = false;
+
+		if (cachepc_data_fault_avail) {
+			pr_warn("CachePC: Caught single step WITH data!\n");
+
+			cachepc_send_tracking_event(
+				cachepc_inst_fault_gfn, cachepc_inst_fault_err,
+				cachepc_data_fault_gfn, cachepc_data_fault_err);
+
+			cachepc_track_single(vcpu, cachepc_data_fault_gfn,
+				KVM_PAGE_TRACK_ACCESS);
+			cachepc_data_fault_avail = false;
+
+			cachepc_track_single(vcpu, cachepc_inst_fault_gfn,
+				KVM_PAGE_TRACK_ACCESS);
+			cachepc_inst_fault_avail = false;
+		} else if (cachepc_inst_fault_avail) {
+			pr_warn("CachePC: Caught single step WITHOUT data!\n");
+
+			cachepc_send_tracking_event(
+				cachepc_inst_fault_gfn, cachepc_inst_fault_err,
+				0, 0);
+
+			cachepc_track_single(vcpu, cachepc_inst_fault_gfn,
+				KVM_PAGE_TRACK_ACCESS);
+			cachepc_inst_fault_avail = false;
+		} else {
+			pr_err("CachePC: Unexpected single step\n");
+		}
+	}
+
 	++vcpu->stat.irq_exits;
 	return 1;
 }
@@ -3269,9 +3305,23 @@ static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
 
 int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
 {
+	static const struct {
+		u64 code;
+		const char *name;
+	} codelut[] = {
+		SVM_EXIT_REASONS,
+		{ -1, NULL }
+	};
+	size_t i;
+
 	if (!svm_check_exit_valid(exit_code))
 		return svm_handle_invalid_exit(vcpu, exit_code);
 
+	for (i = 0; i < sizeof(codelut) / sizeof(codelut[0]); i++) {
+		if (codelut[i].code == exit_code)
+			pr_warn("KVM EXIT (%s)\n", codelut[i].name);
+	}
+
 #ifdef CONFIG_RETPOLINE
 	if (exit_code == SVM_EXIT_MSR)
 		return msr_interception(vcpu);
@@ -3788,14 +3838,39 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 	unsigned long vmcb_pa = svm->current_vmcb->pa;
+	int cpu;
 
 	guest_state_enter_irqoff();
 
 	if (sev_es_guest(vcpu->kvm)) {
+		cpu = get_cpu();
+		local_irq_disable();
+		WARN_ON(cpu != 2);
+
+		memset(cachepc_msrmts, 0,
+			cachepc_msrmts_count * sizeof(cpc_msrmt_t));
+		cachepc_reset_pmc(CPC_L1MISS_PMC);
+
+		cachepc_reset_pmc(CPC_RETINST_PMC);
 		__svm_sev_es_vcpu_run(vmcb_pa);
+		cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC);
+
+		cachepc_save_msrmts(cachepc_ds);
+		if (cachepc_baseline_measure)
+			cachepc_update_baseline();
+		local_irq_enable();
+		put_cpu();
 	} else {
 		struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
 
+		cpu = get_cpu();
+		local_irq_disable();
+		WARN_ON(cpu != 2);
+
+		memset(cachepc_msrmts, 0,
+			cachepc_msrmts_count * sizeof(cpc_msrmt_t));
+		cachepc_reset_pmc(CPC_L1MISS_PMC);
+
 		/*
 		 * Use a single vmcb (vmcb01 because it's always valid) for
 		 * context switching guest state via VMLOAD/VMSAVE, that way
@@ -3806,7 +3881,15 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
 		__svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs);
 		vmsave(svm->vmcb01.pa);
 
+		cachepc_reset_pmc(CPC_RETINST_PMC);
 		vmload(__sme_page_pa(sd->save_area));
+		cachepc_retinst = cachepc_read_pmc(CPC_RETINST_PMC);
+
+		cachepc_save_msrmts(cachepc_ds);
+		if (cachepc_baseline_measure)
+			cachepc_update_baseline();
+		local_irq_enable();
+		put_cpu();
 	}
 
 	guest_state_exit_irqoff();
diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
index dfaeb47fcf2a..0626f3fdddfd 100644
--- a/arch/x86/kvm/svm/vmenter.S
+++ b/arch/x86/kvm/svm/vmenter.S
@@ -29,12 +29,59 @@
 
 .section .noinstr.text, "ax"
 
+.extern cachepc_msrmts
+.extern cachepc_regs_tmp
+.extern cachepc_regs_vm
+
+.macro load_tmp off reg
+	mov cachepc_regs_tmp+\off(%rip), \reg
+.endm
+
+.macro save_tmp off reg
+	mov \reg, cachepc_regs_tmp+\off(%rip)
+.endm
+
+.macro load_vm off reg
+	mov cachepc_regs_vm+\off(%rip), \reg
+.endm
+
+.macro save_vm off reg
+	mov \reg, cachepc_regs_vm+\off(%rip)
+.endm
+
+.macro apply_regs func
+	\func 0x00, %rax
+	\func 0x08, %rbx
+	\func 0x10, %rcx
+	\func 0x18, %rdx
+	\func 0x20, %rbp
+	\func 0x28, %rsp
+	\func 0x30, %rdi
+	\func 0x38, %rsi
+	\func 0x40, %r8
+	\func 0x48, %r9
+	\func 0x50, %r10
+	\func 0x58, %r11
+	\func 0x60, %r12
+	\func 0x68, %r13
+	\func 0x70, %r14
+	\func 0x78, %r15
+.endm
+
+.macro barrier
+	mfence
+	mov $0x80000005,%eax
+	cpuid
+.endm
+
 /**
  * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
  * @vmcb_pa:	unsigned long
  * @regs:	unsigned long * (to guest registers)
  */
 SYM_FUNC_START(__svm_vcpu_run)
+	apply_regs save_tmp
+
 	push %_ASM_BP
 #ifdef CONFIG_X86_64
 	push %r15
@@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run)
 	/* Enter guest mode */
 	sti
 
-1:	vmrun %_ASM_AX
+1:	
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov cachepc_ds, %rsi
+	mov 0x8(%rsi), %r15
+	lea sev_prime_ret(%rip), %rdi
+	jmp cachepc_prime_vcall+5+1 // skip stack pushs
+sev_prime_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
+
+	vmrun %_ASM_AX
+
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov %r15, %rsi
+	lea sev_probe_ret(%rip), %rdi
+	jmp cachepc_probe_vcall+5+8 // skip stack pushs
+sev_probe_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
 
 2:	cli
 
@@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run)
  * @vmcb_pa:	unsigned long
  */
 SYM_FUNC_START(__svm_sev_es_vcpu_run)
+	apply_regs save_tmp
+
 	push %_ASM_BP
 #ifdef CONFIG_X86_64
 	push %r15
@@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
 	/* Enter guest mode */
 	sti
 
-1:	vmrun %_ASM_AX
+1:	
+
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov cachepc_ds, %rsi
+	mov 0x8(%rsi), %r15
+	lea sev_es_prime_ret(%rip), %rdi
+	jmp cachepc_prime_vcall+5+1 // skip stack pushes
+sev_es_prime_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
+
+	vmrun %_ASM_AX
+
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov %r15, %rsi
+	lea sev_es_probe_ret(%rip), %rdi
+	jmp cachepc_probe_vcall+5+8 // skip stack pushs
+sev_es_probe_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
 
 2:	cli
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index d9adf79124f9..6ca86ab417cb 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -82,6 +82,8 @@
 #include <asm/sgx.h>
 #include <clocksource/hyperv_timer.h>
 
+#include "cachepc/sevstep.h"
+
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 
@@ -6597,6 +6599,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		if (!kvm_x86_ops.mem_enc_ioctl)
 			goto out;
 
+		pr_warn("ENCRYPT_OP\n");
 		r = static_call(kvm_x86_mem_enc_ioctl)(kvm, argp);
 		break;
 	}
diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c
index 27ab27931813..90679ec8ba79 100644
--- a/crypto/aes_generic.c
+++ b/crypto/aes_generic.c
@@ -1173,8 +1173,78 @@ EXPORT_SYMBOL_GPL(crypto_aes_set_key);
 	f_rl(bo, bi, 3, k);	\
 } while (0)
 
+#define L1_ASSOC 8
+#define L1_LINESIZE 64
+#define L1_SETS 64
+#define L1_SIZE (L1_SETS * L1_ASSOC * L1_LINESIZE)
+
+#define ACCESS_LINE(n) \
+	asm volatile ("mov (%0), %%rbx" \
+		: : "r"(((uint8_t*) L1) + n * L1_LINESIZE) : "rbx");
+
+#define DO_ACCESS_PATTERN() \
+	ACCESS_LINE(60) \
+	ACCESS_LINE(13) \
+	ACCESS_LINE(24) \
+	ACCESS_LINE(19) \
+	ACCESS_LINE(38) \
+	ACCESS_LINE(17) \
+	ACCESS_LINE( 2) \
+	ACCESS_LINE(12) \
+	ACCESS_LINE(22) \
+	ACCESS_LINE(46) \
+	ACCESS_LINE( 4) \
+	ACCESS_LINE(61) \
+	ACCESS_LINE( 5) \
+	ACCESS_LINE(14) \
+	ACCESS_LINE(11) \
+	ACCESS_LINE(35) \
+	ACCESS_LINE(45) \
+	ACCESS_LINE(10) \
+	ACCESS_LINE(49) \
+	ACCESS_LINE(56) \
+	ACCESS_LINE(27) \
+	ACCESS_LINE(37) \
+	ACCESS_LINE(63) \
+	ACCESS_LINE(54) \
+	ACCESS_LINE(55) \
+	ACCESS_LINE(29) \
+	ACCESS_LINE(48) \
+	ACCESS_LINE( 9) \
+	ACCESS_LINE(16) \
+	ACCESS_LINE(39) \
+	ACCESS_LINE(20) \
+	ACCESS_LINE(21) \
+	ACCESS_LINE(62) \
+	ACCESS_LINE( 0) \
+	ACCESS_LINE(34) \
+	ACCESS_LINE( 8) \
+	ACCESS_LINE(53) \
+	ACCESS_LINE(42) \
+	ACCESS_LINE(51) \
+	ACCESS_LINE(50) \
+	ACCESS_LINE(57) \
+	ACCESS_LINE( 7) \
+	ACCESS_LINE( 6) \
+	ACCESS_LINE(33) \
+	ACCESS_LINE(26) \
+
+uint8_t *L1 = NULL;
+
 static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 {
+	int cpu;
+
+	if (L1 == NULL) {
+		L1 = kzalloc(L1_SETS * L1_LINESIZE, GFP_KERNEL);
+		BUG_ON(((uintptr_t)L1) % (L1_SETS * L1_LINESIZE) != 0);
+	}
+
+	pr_warn("CachePC-TEST: Running AES-Generic!");
+
+	cpu = get_cpu();
+	DO_ACCESS_PATTERN()
+
 	const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 	u32 b0[4], b1[4];
 	const u32 *kp = ctx->key_enc + 4;
@@ -1210,6 +1280,9 @@ static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 	put_unaligned_le32(b0[1], out + 4);
 	put_unaligned_le32(b0[2], out + 8);
 	put_unaligned_le32(b0[3], out + 12);
+
+	DO_ACCESS_PATTERN();
+	put_cpu();
 }
 
 /* decrypt a block of text */
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
old mode 100644
new mode 100755
index e089fbf9017f..7899e1efe852
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -87,7 +87,7 @@ static void *sev_init_ex_buffer;
 static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE;
 
 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret);
-static int sev_do_cmd(int cmd, void *data, int *psp_ret);
+int sev_do_cmd(int cmd, void *data, int *psp_ret);
 
 static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
 {
@@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
 	return ret;
 }
 
-static int sev_do_cmd(int cmd, void *data, int *psp_ret)
+int sev_do_cmd(int cmd, void *data, int *psp_ret)
 {
 	int rc;
 
@@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret)
 
 	return rc;
 }
+EXPORT_SYMBOL(sev_do_cmd);
 
 static int __sev_init_locked(int *error)
 {
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index f2a63cb2658b..c77a29e14771 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -13,6 +13,7 @@
  *   Yaniv Kamay  <yaniv@qumranet.com>
  */
 
+#include <asm-generic/errno-base.h>
 #include <kvm/iodev.h>
 
 #include <linux/kvm_host.h>
@@ -64,12 +65,15 @@
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/kvm.h>
+#include "../../arch/x86/kvm/cachepc/tracking.h"
 
 #include <linux/kvm_dirty_ring.h>
 
 /* Worst case buffer size needed for holding an integer. */
 #define ITOA_MAX_LEN 12
 
+#include "../../arch/x86/kvm/cachepc/kvm.h"
+
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 
@@ -1261,6 +1265,9 @@ static void kvm_destroy_vm(struct kvm *kvm)
 	hardware_disable_all();
 	mmdrop(mm);
 	module_put(kvm_chardev_ops.owner);
+
+	if (main_vm == kvm)
+		main_vm = NULL;
 }
 
 void kvm_get_kvm(struct kvm *kvm)
@@ -1360,7 +1367,7 @@ static void kvm_insert_gfn_node(struct kvm_memslots *slots,
 	int idx = slots->node_idx;
 
 	parent = NULL;
-	for (node = &gfn_tree->rb_node; *node; ) {
+	for (node = &gfn_tree->rb_node; *node;) {
 		struct kvm_memory_slot *tmp;
 
 		tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
@@ -4514,7 +4521,7 @@ static long kvm_vm_ioctl(struct file *filp,
 	void __user *argp = (void __user *)arg;
 	int r;
 
-	if (kvm->mm != current->mm || kvm->vm_dead)
+	if ((ioctl != KVM_MEMORY_ENCRYPT_OP && kvm->mm != current->mm) || kvm->vm_dead)
 		return -EIO;
 	switch (ioctl) {
 	case KVM_CREATE_VCPU:
@@ -4823,6 +4830,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type)
 	kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
 
 	fd_install(r, file);
+
+	main_vm = kvm;
+
 	return r;
 
 put_kvm:
@@ -4864,7 +4874,7 @@ static long kvm_dev_ioctl(struct file *filp,
 		r = -EOPNOTSUPP;
 		break;
 	default:
-		return kvm_arch_dev_ioctl(filp, ioctl, arg);
+		return cachepc_kvm_ioctl(filp, ioctl, arg);
 	}
 out:
 	return r;
@@ -5792,6 +5802,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
 	r = kvm_vfio_ops_init();
 	WARN_ON(r);
 
+	cachepc_kvm_init();
+
 	return 0;
 
 out_unreg:
@@ -5821,6 +5833,8 @@ void kvm_exit(void)
 {
 	int cpu;
 
+	cachepc_kvm_exit();
+
 	debugfs_remove_recursive(kvm_debugfs_dir);
 	misc_deregister(&kvm_dev);
 	for_each_possible_cpu(cpu)