path: root/patch.diff

diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h
index eb186bc57f6a..b96e80934005 100644
--- a/arch/x86/include/asm/kvm_page_track.h
+++ b/arch/x86/include/asm/kvm_page_track.h
@@ -2,10 +2,9 @@
 #ifndef _ASM_X86_KVM_PAGE_TRACK_H
 #define _ASM_X86_KVM_PAGE_TRACK_H
 
-enum kvm_page_track_mode {
-	KVM_PAGE_TRACK_WRITE,
-	KVM_PAGE_TRACK_MAX,
-};
+#include<linux/srcu.h>
+
+#include "../../kvm/cachepc/uapi.h"
 
 /*
  * The notifier represented by @kvm_page_track_notifier_node is linked into
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 30f244b64523..e0eeffd340e8 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-ccflags-y += -I $(srctree)/arch/x86/kvm
+ccflags-y += -I $(srctree)/arch/x86/kvm -O2
 ccflags-$(CONFIG_KVM_WERROR) += -Werror
 
 ifeq ($(CONFIG_FRAME_POINTER),y)
@@ -11,8 +11,9 @@ include $(srctree)/virt/kvm/Makefile.kvm
 
 kvm-y			+= x86.o emulate.o i8259.o irq.o lapic.o \
 			   i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
-			   hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \
-			   mmu/spte.o
+			   hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o \
+			   cachepc/cachepc.o cachepc/kvm.o \
+			   cachepc/sevstep.o cachepc/uspt.o
 
 ifdef CONFIG_HYPERV
 kvm-y			+= kvm_onhyperv.o
@@ -25,7 +26,8 @@ kvm-intel-y		+= vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
 			   vmx/evmcs.o vmx/nested.o vmx/posted_intr.o
 kvm-intel-$(CONFIG_X86_SGX_KVM)	+= vmx/sgx.o
 
-kvm-amd-y		+= svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o
+kvm-amd-y		+= svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o \
+				   svm/avic.o svm/sev.o cachepc/cachepc.o
 
 ifdef CONFIG_HYPERV
 kvm-amd-y		+= svm/svm_onhyperv.o
diff --git a/arch/x86/kvm/cachepc b/arch/x86/kvm/cachepc
new file mode 120000
index 000000000000..9119e44af1f0
--- /dev/null
+++ b/arch/x86/kvm/cachepc
@@ -0,0 +1 @@
+/home/louis/kvm-prime-count/cachepc
\ No newline at end of file
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index d871b8dee7b3..3b7720aebbc6 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -1152,6 +1152,8 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
 	}
 }
 
+#include "../cachepc/mmu.c"
+
 /*
  * Write-protect on the specified @sptep, @pt_protect indicates whether
  * spte write-protection is caused by protecting shadow page table.
@@ -1165,34 +1167,15 @@ static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
  *
  * Return true if tlb need be flushed.
  */
-static bool spte_write_protect(u64 *sptep, bool pt_protect)
-{
-	u64 spte = *sptep;
-
-	if (!is_writable_pte(spte) &&
-	    !(pt_protect && is_mmu_writable_spte(spte)))
-		return false;
-
-	rmap_printk("spte %p %llx\n", sptep, *sptep);
-
-	if (pt_protect)
-		spte &= ~shadow_mmu_writable_mask;
-	spte = spte & ~PT_WRITABLE_MASK;
-
-	return mmu_spte_update(sptep, spte);
-}
+// static bool spte_write_protect(u64 *sptep, bool pt_protect)
+// {
+// 	return sevstep_spte_protect(sptep, pt_protect, KVM_PAGE_TRACK_WRITE);
+// }
 
 static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
 			       bool pt_protect)
 {
-	u64 *sptep;
-	struct rmap_iterator iter;
-	bool flush = false;
-
-	for_each_rmap_spte(rmap_head, &iter, sptep)
-		flush |= spte_write_protect(sptep, pt_protect);
-
-	return flush;
+	return sevstep_rmap_protect(rmap_head, pt_protect, KVM_PAGE_TRACK_WRITE);
 }
 
 static bool spte_clear_dirty(u64 *sptep)
@@ -1358,22 +1341,8 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
 				    struct kvm_memory_slot *slot, u64 gfn,
 				    int min_level)
 {
-	struct kvm_rmap_head *rmap_head;
-	int i;
-	bool write_protected = false;
-
-	if (kvm_memslots_have_rmaps(kvm)) {
-		for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
-			rmap_head = gfn_to_rmap(gfn, i, slot);
-			write_protected |= rmap_write_protect(rmap_head, true);
-		}
-	}
-
-	if (is_tdp_mmu_enabled(kvm))
-		write_protected |=
-			kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
-
-	return write_protected;
+	return sevstep_kvm_mmu_slot_gfn_protect(kvm, slot,
+		gfn, min_level, KVM_PAGE_TRACK_WRITE);
 }
 
 static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
@@ -3901,6 +3870,10 @@ static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
 static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
 					 struct kvm_page_fault *fault)
 {
+	int active;
+
+	sevstep_uspt_page_fault_handle(vcpu, fault);
+
 	if (unlikely(fault->rsvd))
 		return false;
 
@@ -3911,8 +3884,11 @@ static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
 	 * guest is writing the page which is write tracked which can
 	 * not be fixed by page fault handler.
 	 */
-	if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
-		return true;
+	active = kvm_slot_page_track_is_active(vcpu->kvm,
+		fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE);
+	active |= kvm_slot_page_track_is_active(vcpu->kvm,
+		fault->slot, fault->gfn, KVM_PAGE_TRACK_ACCESS);
+	if (active) return true;
 
 	return false;
 }
diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c
index 2e09d1b6249f..9b40e71564bf 100644
--- a/arch/x86/kvm/mmu/page_track.c
+++ b/arch/x86/kvm/mmu/page_track.c
@@ -19,6 +19,8 @@
 #include "mmu.h"
 #include "mmu_internal.h"
 
+#include "../cachepc/sevstep.h"
+
 bool kvm_page_track_write_tracking_enabled(struct kvm *kvm)
 {
 	return IS_ENABLED(CONFIG_KVM_EXTERNAL_WRITE_TRACKING) ||
@@ -131,9 +133,10 @@ void kvm_slot_page_track_add_page(struct kvm *kvm,
 	 */
 	kvm_mmu_gfn_disallow_lpage(slot, gfn);
 
-	if (mode == KVM_PAGE_TRACK_WRITE)
-		if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K))
-			kvm_flush_remote_tlbs(kvm);
+	if (sevstep_kvm_mmu_slot_gfn_protect(kvm,
+			slot, gfn, PG_LEVEL_4K, mode)) {
+		kvm_flush_remote_tlbs(kvm);
+	}
 }
 EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page);
 
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index cf0bf456d520..d6a4002fa550 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -2,6 +2,8 @@
 
 #include <linux/kvm_host.h>
 
+#include "cachepc/cachepc.h"
+
 #include "irq.h"
 #include "mmu.h"
 #include "kvm_cache_regs.h"
@@ -3788,14 +3790,37 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 	unsigned long vmcb_pa = svm->current_vmcb->pa;
+	int cpu;
 
 	guest_state_enter_irqoff();
 
 	if (sev_es_guest(vcpu->kvm)) {
+		memset(cachepc_msrmts, 0,
+			cachepc_msrmts_count * sizeof(cpc_msrmt_t));
+		cachepc_reset_pmc(CPC_L1MISS_PMC);
+
+		cpu = get_cpu();
+		local_irq_disable();
+		WARN_ON(cpu != 2);
+
 		__svm_sev_es_vcpu_run(vmcb_pa);
+
+		cachepc_save_msrmts(cachepc_ds);
+		if (cachepc_baseline_measure)
+			cachepc_update_baseline();
+		local_irq_enable();
+		put_cpu();
 	} else {
 		struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu);
 
+		memset(cachepc_msrmts, 0,
+			cachepc_msrmts_count * sizeof(cpc_msrmt_t));
+		cachepc_reset_pmc(CPC_L1MISS_PMC);
+
+		cpu = get_cpu();
+		local_irq_disable();
+		WARN_ON(cpu != 2);
+
 		/*
 		 * Use a single vmcb (vmcb01 because it's always valid) for
 		 * context switching guest state via VMLOAD/VMSAVE, that way
@@ -3807,6 +3832,12 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
 		vmsave(svm->vmcb01.pa);
 
 		vmload(__sme_page_pa(sd->save_area));
+
+		cachepc_save_msrmts(cachepc_ds);
+		if (cachepc_baseline_measure)
+			cachepc_update_baseline();
+		local_irq_enable();
+		put_cpu();
 	}
 
 	guest_state_exit_irqoff();
diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
index dfaeb47fcf2a..0626f3fdddfd 100644
--- a/arch/x86/kvm/svm/vmenter.S
+++ b/arch/x86/kvm/svm/vmenter.S
@@ -29,12 +29,59 @@
 
 .section .noinstr.text, "ax"
 
+.extern cachepc_msrmts
+.extern cachepc_regs_tmp
+.extern cachepc_regs_vm
+
+.macro load_tmp off reg
+	mov cachepc_regs_tmp+\off(%rip), \reg
+.endm
+
+.macro save_tmp off reg
+	mov \reg, cachepc_regs_tmp+\off(%rip)
+.endm
+
+.macro load_vm off reg
+	mov cachepc_regs_vm+\off(%rip), \reg
+.endm
+
+.macro save_vm off reg
+	mov \reg, cachepc_regs_vm+\off(%rip)
+.endm
+
+.macro apply_regs func
+	\func 0x00, %rax
+	\func 0x08, %rbx
+	\func 0x10, %rcx
+	\func 0x18, %rdx
+	\func 0x20, %rbp
+	\func 0x28, %rsp
+	\func 0x30, %rdi
+	\func 0x38, %rsi
+	\func 0x40, %r8
+	\func 0x48, %r9
+	\func 0x50, %r10
+	\func 0x58, %r11
+	\func 0x60, %r12
+	\func 0x68, %r13
+	\func 0x70, %r14
+	\func 0x78, %r15
+.endm
+
+.macro barrier
+	mfence
+	mov $0x80000005,%eax
+	cpuid
+.endm
+
 /**
  * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
  * @vmcb_pa:	unsigned long
  * @regs:	unsigned long * (to guest registers)
  */
 SYM_FUNC_START(__svm_vcpu_run)
+	apply_regs save_tmp
+
 	push %_ASM_BP
 #ifdef CONFIG_X86_64
 	push %r15
@@ -80,7 +127,27 @@ SYM_FUNC_START(__svm_vcpu_run)
 	/* Enter guest mode */
 	sti
 
-1:	vmrun %_ASM_AX
+1:	
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov cachepc_ds, %rsi
+	mov 0x8(%rsi), %r15
+	lea sev_prime_ret(%rip), %rdi
+	jmp cachepc_prime_vcall+5+1 // skip stack pushs
+sev_prime_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
+
+	vmrun %_ASM_AX
+
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov %r15, %rsi
+	lea sev_probe_ret(%rip), %rdi
+	jmp cachepc_probe_vcall+5+8 // skip stack pushs
+sev_probe_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
 
 2:	cli
 
@@ -163,6 +230,8 @@ SYM_FUNC_END(__svm_vcpu_run)
  * @vmcb_pa:	unsigned long
  */
 SYM_FUNC_START(__svm_sev_es_vcpu_run)
+	apply_regs save_tmp
+
 	push %_ASM_BP
 #ifdef CONFIG_X86_64
 	push %r15
@@ -181,7 +250,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
 	/* Enter guest mode */
 	sti
 
-1:	vmrun %_ASM_AX
+1:	
+
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov cachepc_ds, %rsi
+	mov 0x8(%rsi), %r15
+	lea sev_es_prime_ret(%rip), %rdi
+	jmp cachepc_prime_vcall+5+1 // skip stack pushes
+sev_es_prime_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
+
+	vmrun %_ASM_AX
+
+	apply_regs save_vm
+	apply_regs load_tmp
+	mov %r15, %rsi
+	lea sev_es_probe_ret(%rip), %rdi
+	jmp cachepc_probe_vcall+5+8 // skip stack pushs
+sev_es_probe_ret:
+	apply_regs save_tmp
+	apply_regs load_vm
 
 2:	cli
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index d9adf79124f9..3e5c55f9bef0 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -82,6 +82,8 @@
 #include <asm/sgx.h>
 #include <clocksource/hyperv_timer.h>
 
+#include "cachepc/sevstep.h"
+
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
old mode 100644
new mode 100755
index e089fbf9017f..7899e1efe852
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -87,7 +87,7 @@ static void *sev_init_ex_buffer;
 static size_t sev_es_tmr_size = SEV_ES_TMR_SIZE;
 
 static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret);
-static int sev_do_cmd(int cmd, void *data, int *psp_ret);
+int sev_do_cmd(int cmd, void *data, int *psp_ret);
 
 static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
 {
@@ -865,7 +865,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
 	return ret;
 }
 
-static int sev_do_cmd(int cmd, void *data, int *psp_ret)
+int sev_do_cmd(int cmd, void *data, int *psp_ret)
 {
 	int rc;
 
@@ -875,6 +875,7 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret)
 
 	return rc;
 }
+EXPORT_SYMBOL(sev_do_cmd);
 
 static int __sev_init_locked(int *error)
 {
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index f2a63cb2658b..4c55f85fc775 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -13,6 +13,7 @@
  *   Yaniv Kamay  <yaniv@qumranet.com>
  */
 
+#include <asm-generic/errno-base.h>
 #include <kvm/iodev.h>
 
 #include <linux/kvm_host.h>
@@ -70,6 +71,10 @@
 /* Worst case buffer size needed for holding an integer. */
 #define ITOA_MAX_LEN 12
 
+#include "../../arch/x86/kvm/cachepc/kvm.h"
+#include "../../arch/x86/kvm/cachepc/sevstep.h"
+#include "../../arch/x86/kvm/cachepc/uspt.h"
+
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 
@@ -1261,6 +1266,9 @@ static void kvm_destroy_vm(struct kvm *kvm)
 	hardware_disable_all();
 	mmdrop(mm);
 	module_put(kvm_chardev_ops.owner);
+
+	if (main_vm == kvm)
+		main_vm = NULL;
 }
 
 void kvm_get_kvm(struct kvm *kvm)
@@ -1360,7 +1368,7 @@ static void kvm_insert_gfn_node(struct kvm_memslots *slots,
 	int idx = slots->node_idx;
 
 	parent = NULL;
-	for (node = &gfn_tree->rb_node; *node; ) {
+	for (node = &gfn_tree->rb_node; *node;) {
 		struct kvm_memory_slot *tmp;
 
 		tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
@@ -4823,6 +4831,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type)
 	kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
 
 	fd_install(r, file);
+
+	main_vm = kvm;
+
 	return r;
 
 put_kvm:
@@ -4864,7 +4875,7 @@ static long kvm_dev_ioctl(struct file *filp,
 		r = -EOPNOTSUPP;
 		break;
 	default:
-		return kvm_arch_dev_ioctl(filp, ioctl, arg);
+		return cachepc_kvm_ioctl(filp, ioctl, arg);
 	}
 out:
 	return r;
@@ -5792,6 +5803,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
 	r = kvm_vfio_ops_init();
 	WARN_ON(r);
 
+	cachepc_kvm_init();
+
 	return 0;
 
 out_unreg:
@@ -5821,6 +5834,8 @@ void kvm_exit(void)
 {
 	int cpu;
 
+	cachepc_kvm_exit();
+
 	debugfs_remove_recursive(kvm_debugfs_dir);
 	misc_deregister(&kvm_dev);
 	for_each_possible_cpu(cpu)