cachepc

commit e80d7612f93f7d8dabe3be5152d8baa15a7d29db
parent fef45f5207c99b00676c014bc02141e284c331ce
Author: Louis Burda <quent.burda@gmail.com>
Date:   Mon, 15 Aug 2022 21:52:52 +0200

Minor tweaks

Diffstat:
M
kmod/cachepc.c
 | 
4
++--
M
kmod/cachepc.h
 | 
19
++++++-------------
M
kmod/cachepc_user.h
 | 
2
+-
M
patch.diff
 | 
238
++++++++++++++++++++++++++++++++++++++++++-------------------------------------
M
test/kvm.c
 | 
16
+++++++++-------

5 files changed, 145 insertions(+), 134 deletions(-)
diff --git a/kmod/cachepc.c b/kmod/cachepc.c
@@ -140,7 +140,7 @@ cachepc_save_msrmts(cacheline *head)
 {
 	cacheline *curr_cl;
 
-	printk(KERN_WARNING "CachePC: Updating /proc/cachepc\n");
+	// printk(KERN_WARNING "CachePC: Updating /proc/cachepc\n");
 
 	curr_cl = head;
 	do {
@@ -369,7 +369,6 @@ void build_randomized_list_for_cache_set(cache_ctx *ctx, cacheline **cacheline_p
 		curr_cl = cacheline_ptr_arr[idx_map[i]];
 		curr_cl->next = cacheline_ptr_arr[idx_map[(i + 1) % len]];
 		curr_cl->prev = cacheline_ptr_arr[idx_map[(len - 1 + i) % len]];
-		curr_cl->count = 0;
 
 		if (idx_map[i] == 0) {
 			curr_cl->flags = SET_FIRST(DEFAULT_FLAGS);
@@ -405,6 +404,7 @@ allocate_cache_ds(cache_ctx *ctx)
 		cl_ptr_arr[i] = cl_arr + i;
 		cl_ptr_arr[i]->cache_set = get_virt_cache_set(ctx, cl_ptr_arr[i]);
 		cl_ptr_arr[i]->cache_line = i / ctx->sets;
+		cl_ptr_arr[i]->count = 0;
 	}
 
 	return cl_ptr_arr;
diff --git a/kmod/cachepc.h b/kmod/cachepc.h
@@ -49,21 +49,21 @@ extern cacheline *cachepc_ds;
 cacheline *
 cachepc_prime(cacheline *head)
 {
-	cacheline *curr_cl;
+	cacheline *curr_cl, *prev_cl;
 
 	cachepc_mfence();
 	cachepc_cpuid();
 	
 	curr_cl = head;
 	do {
+		prev_cl = curr_cl;
 		curr_cl = curr_cl->next;
 	} while (curr_cl != head);
-	curr_cl = curr_cl->prev;
 
 	cachepc_mfence();
 	cachepc_cpuid();
 
-	return curr_cl;
+	return prev_cl;
 }
 
 /*
@@ -114,9 +114,6 @@ cachepc_probe(cacheline *start_cl)
 	do {
 		pre = cachepc_read_pmc(0);
 
-		cachepc_mfence();
-		//cachepc_cpuid();
-		
 		asm volatile(
 			"mov 8(%[curr_cl]), %%rax \n\t"              // +8
 			"mov 8(%%rax), %%rcx \n\t"                   // +16
@@ -132,24 +129,20 @@ cachepc_probe(cacheline *start_cl)
 			: "rax", "rcx"
 		);
 
-		cachepc_mfence();
-		//cachepc_cpuid();
-
 		post = cachepc_read_pmc(0);
 
-		cachepc_mfence();
-		//cachepc_cpuid();
-
 		/* works across size boundary */
 		curr_cl->count = post - pre;
 
 		curr_cl = next_cl;
 	} while (__builtin_expect(curr_cl != start_cl, 1));
 
+	next_cl = curr_cl->next;
+
 	cachepc_mfence();
 	cachepc_cpuid();
 
-	return curr_cl->next;
+	return next_cl;
 }
 
 void
diff --git a/kmod/cachepc_user.h b/kmod/cachepc_user.h
@@ -3,6 +3,6 @@
 #include <linux/ioctl.h>
 
 #define CACHEPC_IOCTL_MAGIC 0xBF
-#define CACHEPC_IOCTL_TEST_ACCESS _IOR(CACHEPC_IOCTL_MAGIC, 0, uint32_t)
+#define CACHEPC_IOCTL_TEST_ACCESS _IOWR(CACHEPC_IOCTL_MAGIC, 0, uint32_t)
 #define CACHEPC_IOCTL_TEST_EVICTION _IOWR(CACHEPC_IOCTL_MAGIC, 1, uint32_t)
 #define CACHEPC_IOCTL_INIT_PMC _IOW(CACHEPC_IOCTL_MAGIC, 2, uint32_t)
diff --git a/patch.diff b/patch.diff
@@ -32,7 +32,7 @@ index b804444e16d4..17167ccfca22 100644
  obj-$(CONFIG_KVM)	+= kvm.o
  obj-$(CONFIG_KVM_INTEL)	+= kvm-intel.o
 diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
-index 7b3cfbe8f7e3..16dfd9b2938e 100644
+index 7b3cfbe8f7e3..75ef27b944de 100644
 --- a/arch/x86/kvm/svm/svm.c
 +++ b/arch/x86/kvm/svm/svm.c
 @@ -2,6 +2,8 @@
@@ -44,43 +44,33 @@ index 7b3cfbe8f7e3..16dfd9b2938e 100644
  #include "irq.h"
  #include "mmu.h"
  #include "kvm_cache_regs.h"
-@@ -3785,8 +3787,13 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
+@@ -3751,7 +3753,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
+ 	if (sev_es_guest(svm->vcpu.kvm)) {
+ 		__svm_sev_es_vcpu_run(svm->vmcb_pa);
+ 	} else {
++		cacheline *head = cachepc_prime(cachepc_ds);
+ 		__svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);
++		cachepc_probe(head);
++		cachepc_save_msrmts(head);
+ 
+ #ifdef CONFIG_X86_64
+ 		native_wrmsrl(MSR_GS_BASE, svm->host.gs_base);
+@@ -3785,8 +3790,12 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
  
  static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
  {
 -	struct vcpu_svm *svm = to_svm(vcpu);
-+	struct cacheline *head;
 +	struct vcpu_svm *svm;
-+	int cpu;
-+
-+	printk(KERN_WARNING "CachePC: svm_cpu_enter_exit()\n");
  
++	printk(KERN_WARNING "CachePC: svm_cpu_enter_exit()\n");
++	WARN_ON(smp_processor_id() != 2);
++
 +	svm = to_svm(vcpu);
  	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
  	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
  	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
-@@ -3835,8 +3842,19 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
- 	 */
- 	x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
- 
-+	cpu = get_cpu();
-+	WARN_ON(cpu != 2);
-+
-+	head = cachepc_prime(cachepc_ds);
-+
- 	svm_vcpu_enter_exit(vcpu, svm);
- 
-+	cachepc_probe(head);
-+	//cachepc_print_msrmts(head);
-+	cachepc_save_msrmts(head);
-+
-+	put_cpu();
-+
- 	/*
- 	 * We do not use IBRS in the kernel. If this vCPU has used the
- 	 * SPEC_CTRL MSR it may have left it on; save the value and
 diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 2541a17ff1c4..d8d78359d735 100644
+index 2541a17ff1c4..7efbdfd0e3e2 100644
 --- a/virt/kvm/kvm_main.c
 +++ b/virt/kvm/kvm_main.c
 @@ -51,6 +51,9 @@
@@ -121,7 +111,7 @@ index 2541a17ff1c4..d8d78359d735 100644
  __visible bool kvm_rebooting;
  EXPORT_SYMBOL_GPL(kvm_rebooting);
  
-@@ -4765,12 +4782,301 @@ static void check_processor_compat(void *data)
+@@ -4765,12 +4782,327 @@ static void check_processor_compat(void *data)
  	*c->ret = kvm_arch_check_processor_compat(c->opaque);
  }
  
@@ -172,9 +162,10 @@ index 2541a17ff1c4..d8d78359d735 100644
 +}
 +
 +void
-+kvm_cachepc_stream_hwpf_test(void *p)
++kvm_cachepc_prime_probe_test(void *p)
 +{
-+	cacheline *cl;
++	cacheline *lines;
++	cacheline *cl, *head;
 +	uint32_t count;
 +	uint32_t *arg;
 +	int i, max;
@@ -184,81 +175,96 @@ index 2541a17ff1c4..d8d78359d735 100644
 +	/* l2 data cache, hit or miss */
 +	cachepc_init_pmc(0, 0x64, 0xD8);
 +
-+	cl = cachepc_aligned_alloc(PAGE_SIZE, cachepc_ctx->cache_size);
-+	BUG_ON(cl == NULL);
++	lines = cachepc_aligned_alloc(PAGE_SIZE, cachepc_ctx->cache_size);
++	BUG_ON(lines == NULL);
 +
-+	cachepc_prime(cachepc_ds);
++	max = cachepc_ctx->nr_of_cachelines;
 +
-+	count = cachepc_read_pmc(0);
++	cachepc_cpuid();
++	cachepc_mfence();
 +
-+	max = cachepc_ctx->nr_of_cachelines;
 +	for (i = 0; i < max; i++)
-+		asm volatile ("mov (%0), %%rbx" : : "r"(cl + i) : "rbx");
++		asm volatile ("mov (%0), %%rbx" : : "r"(lines + i) : "rbx");
 +
-+	count = cachepc_read_pmc(0) - count;
++	head = cachepc_prime(cachepc_ds);
++	cachepc_probe(head);
 +
-+	printk(KERN_WARNING "CachePC: HWPF test done (%u vs. %u => %s)\n",
-+		count, max, (count == max) ? "passed" : "failed");
++	count = 0;
++	cl = head = cachepc_ds;
++	do {
++		count += cl->count;
++		cl = cl->next;
++	} while (cl != head);
++
++	printk(KERN_WARNING "CachePC: Prime-probe test done (%u vs. %u => %s)\n",
++		count, 0, (count == 0) ? "passed" : "failed");
 +
-+	if (arg) *arg = count != 4;
++	if (arg) *arg = (count == 0);
 +
-+	kfree(cl);
++	kfree(lines);
 +}
 +
 +void
-+kvm_cachepc_single_access_test(void *p)
++kvm_cachepc_stream_hwpf_test(void *p)
 +{
-+	cacheline *ptr;
-+	uint64_t pre, post;
-+	volatile register uint64_t i asm("r11");
-+	uint32_t *user;
++	cacheline *lines;
++	uint32_t count;
++	uint32_t *arg;
++	uint32_t i, max;
++
++	arg = p;
++
++	/* TODO: accurately detect hwpf */
 +
 +	/* l2 data cache, hit or miss */
 +	cachepc_init_pmc(0, 0x64, 0xD8);
 +
-+	user = p;
-+	
-+	WARN_ON(user && *user >= L1_SETS);
-+	if (user && *user >= L1_SETS) return;	
-+	ptr = cachepc_prepare_victim(cachepc_ctx, user ? *user : 48);
++	lines = cachepc_aligned_alloc(PAGE_SIZE, cachepc_ctx->cache_size);
++	BUG_ON(lines == NULL);
 +
-+	cachepc_mfence();
-+	cachepc_cpuid();
++	max = cachepc_ctx->nr_of_cachelines;
 +
 +	cachepc_prime(cachepc_ds);
 +
-+	cachepc_mfence();
-+	cachepc_cpuid();
++	count -= cachepc_read_pmc(0);
++	for (i = 0; i < max; i++)
++		asm volatile ("mov (%0), %%rbx" : : "r"(lines + i) : "rbx");
++	count += cachepc_read_pmc(0);
 +
-+	for (i = 0; i < 100000000LLU; i++);
++	printk(KERN_WARNING "CachePC: HWPF test done (%u vs. %u => %s)\n",
++		count, max, (count == max) ? "passed" : "failed");
 +
-+	cachepc_mfence();
-+	cachepc_cpuid();
++	if (arg) *arg = (count == max);
 +
-+	pre = cachepc_read_pmc(0);
-+
-+	cachepc_mfence();
-+	cachepc_cpuid();
++	kfree(lines);
++}
 +
-+	cachepc_victim(ptr);
++void
++kvm_cachepc_single_access_test(void *p)
++{
++	cacheline *ptr;
++	uint64_t pre, post;
++	uint32_t *arg;
 +
-+	cachepc_mfence();
-+	cachepc_cpuid();
++	/* l2 data cache, hit or miss */
++	cachepc_init_pmc(0, 0x64, 0xD8);
 +
-+	for (i = 0; i < 100000000LLU; i++);
++	arg = p;
++	
++	WARN_ON(arg && *arg >= L1_SETS);
++	if (arg && *arg >= L1_SETS) return;	
++	ptr = cachepc_prepare_victim(cachepc_ctx, arg ? *arg : 48);
 +
-+	cachepc_mfence();
-+	cachepc_cpuid();
++	cachepc_prime(cachepc_ds);
 +
++	pre = cachepc_read_pmc(0);
++	cachepc_victim(ptr);
 +	post = cachepc_read_pmc(0);
 +
-+	cachepc_mfence();
-+	cachepc_cpuid();
-+
 +	printk(KERN_WARNING "CachePC: Single access test done (%llu vs %u => %s)",
 +		post - pre, 1, (post - pre == 1) ? "passed" : "failed");
 +
-+	if (user) *user = post - pre;
++	if (arg) *arg = post - pre;
 +
 +	cachepc_release_victim(cachepc_ctx, ptr);
 +}
@@ -269,17 +275,17 @@ index 2541a17ff1c4..d8d78359d735 100644
 +	cacheline *head, *cl, *evicted;
 +        cacheline *ptr;
 +	uint32_t target;
-+	uint32_t *user;
++	uint32_t *arg;
 +	int count;
 +
-+	user = p;
++	arg = p;
 +
 +	/* l2 data cache, hit or miss */
 +	cachepc_init_pmc(0, 0x64, 0xD8);
 +
-+	WARN_ON(user && *user >= L1_SETS);
-+	if (user && *user >= L1_SETS) return;	
-+	target = user ? *user : 48;
++	WARN_ON(arg && *arg >= L1_SETS);
++	if (arg && *arg >= L1_SETS) return;	
++	target = arg ? *arg : 48;
 +
 +	ptr = cachepc_prepare_victim(cachepc_ctx, target);
 +
@@ -290,38 +296,57 @@ index 2541a17ff1c4..d8d78359d735 100644
 +	count = 0;
 +	evicted = NULL;
 +	cl = head = cachepc_ds;
-+	while (cl->next != head) {
-+		count += cl->count;
-+		if (cl->count > 0)
++	do {
++		if (IS_FIRST(cl->flags) && cl->count > 0) {
 +			evicted = cl;
++			count += cl->count;
++		}
 +		cl = cl->next;
-+	}
++	} while (cl != head);
 +
 +	printk(KERN_WARNING "CachePC: Single eviction test done (%u vs %u => %s)\n",
 +		count, 1, (count == 1 && evicted->cache_set == target) ? "passed" : "failed");
 +	cachepc_save_msrmts(head);
 +
++	if (arg) *arg = count;
++
 +	cachepc_release_victim(cachepc_ctx, ptr);
 +}
 +
 +void
-+kvm_cachepc_stream_hwpf_disable(void)
++kwm_cachepc_system_setup(void)
 +{
 +	uint64_t reg_addr, val;
 +	uint32_t lo, hi;
 +
++	/* disable streaming store */
++	reg_addr = 0xc0011020;
++	asm volatile ("rdmsr" : "=a"(lo), "=d"(hi) : "c"(reg_addr));
++	val = (uint64_t) lo | ((uint64_t) hi << 32);
++	val |= 1 << 13;
++	asm volatile ("wrmsr" : : "c"(reg_addr), "a"(val), "d"(0x00));
++	printk("CachePC: Writing MSR %08llX: %016llX\n", reg_addr, val);
++
++	/* disable speculative data cache tlb reloads */
++	reg_addr = 0xc0011022;
++	asm volatile ("rdmsr" : "=a"(lo), "=d"(hi) : "c"(reg_addr));
++	val = (uint64_t) lo | ((uint64_t) hi << 32);
++	val |= 1 << 4;
++	asm volatile ("wrmsr" : : "c"(reg_addr), "a"(val), "d"(0x00));
++	printk("CachePC: Writing MSR %08llX: %016llX\n", reg_addr, val);
++
++	/* disable data cache hardware prefetcher */
 +	reg_addr = 0xc0011022;
 +	asm volatile ("rdmsr" : "=a"(lo), "=d"(hi) : "c"(reg_addr));
 +	val = (uint64_t) lo | ((uint64_t) hi << 32);
 +	val |= 1 << 13;
 +	asm volatile ("wrmsr" : : "c"(reg_addr), "a"(val), "d"(0x00));
-+	printk("CachePC: Writing MSR %08llX to disable HWPF: %016llX\n", reg_addr, val);
++	printk("CachePC: Writing MSR %08llX: %016llX\n", reg_addr, val);
 +}
 +
 +void
 +kvm_cachepc_init(void *p)
 +{
-+	cacheline *cl, *head;
 +	int cpu;
 +
 +	cpu = get_cpu();
@@ -331,17 +356,11 @@ index 2541a17ff1c4..d8d78359d735 100644
 +	cachepc_ctx = cachepc_get_ctx(L1);
 +	cachepc_ds = cachepc_prepare_ds(cachepc_ctx);
 +
-+	printk(KERN_WARNING "CachePC: Cacheline configuration\n");
-+	cl = head = cachepc_ds;
-+	do {
-+		printk(KERN_WARNING "CachePC: %i %i\n", cl->cache_set, cl->cache_line);
-+		cl = cl->next;
-+	} while (cl != head);
-+
-+	kvm_cachepc_stream_hwpf_disable();
++	kwm_cachepc_system_setup();
 +
-+	kvm_cachepc_single_eviction_test(NULL);
++	kvm_cachepc_prime_probe_test(NULL);
 +	kvm_cachepc_single_access_test(NULL);
++	kvm_cachepc_single_eviction_test(NULL);
 +	kvm_cachepc_stream_hwpf_test(NULL);
 +
 +	put_cpu();
@@ -376,34 +395,31 @@ index 2541a17ff1c4..d8d78359d735 100644
 +	switch (cmd) {
 +	case CACHEPC_IOCTL_TEST_ACCESS:
 +		printk(KERN_WARNING "CachePC: Called ioctl access test\n");
-+		if (arg_user) {
-+			if (copy_from_user(&u32, arg_user, sizeof(uint32_t)))
-+				return -EFAULT;
-+		}
++		if (!arg_user) return -EINVAL;
++		if (copy_from_user(&u32, arg_user, sizeof(uint32_t)))
++			return -EFAULT;
 +		r = smp_call_function_single(2,
 +			kvm_cachepc_single_access_test, &u32, true);
 +		WARN_ON(r != 0);
-+		if (arg_user) {
-+			if (copy_to_user(arg_user, &u32, sizeof(uint32_t)))
-+				return -EFAULT;
-+		}
++		if (copy_to_user(arg_user, &u32, sizeof(uint32_t)))
++			return -EFAULT;
 +		break;
 +	case CACHEPC_IOCTL_TEST_EVICTION:
 +		printk(KERN_WARNING "CachePC: Called ioctl eviction test\n");
-+		if (arg_user) {
-+			if (copy_from_user(&u32, arg_user, sizeof(uint32_t)))
-+				return -EFAULT;
-+		}
++		if (!arg_user) return -EINVAL;
++		if (copy_from_user(&u32, arg_user, sizeof(uint32_t)))
++			return -EFAULT;
 +		r = smp_call_function_single(2,
 +			kvm_cachepc_single_eviction_test, &u32, true);
 +		WARN_ON(r != 0);
++		if (copy_to_user(arg_user, &u32, sizeof(uint32_t)))
++			return -EFAULT;
 +		break;
 +	case CACHEPC_IOCTL_INIT_PMC:
 +		printk(KERN_WARNING "CachePC: Called ioctl init counter\n");
-+		if (arg_user) {
-+			if (copy_from_user(&u32, arg_user, sizeof(uint32_t)))
-+				return -EFAULT;
-+		}
++		if (!arg_user) return -EINVAL;
++		if (copy_from_user(&u32, arg_user, sizeof(uint32_t)))
++			return -EFAULT;
 +		r = smp_call_function_single(2,
 +			kvm_cachepc_init_pmc_ioctl, &u32, true);
 +		WARN_ON(r != 0);
@@ -425,7 +441,7 @@ index 2541a17ff1c4..d8d78359d735 100644
  
  	r = kvm_arch_init(opaque);
  	if (r)
-@@ -4848,6 +5154,21 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
+@@ -4848,6 +5180,21 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
  	r = kvm_vfio_ops_init();
  	WARN_ON(r);
  
@@ -447,7 +463,7 @@ index 2541a17ff1c4..d8d78359d735 100644
  	return 0;
  
  out_unreg:
-@@ -4872,6 +5193,12 @@ EXPORT_SYMBOL_GPL(kvm_init);
+@@ -4872,6 +5219,12 @@ EXPORT_SYMBOL_GPL(kvm_init);
  
  void kvm_exit(void)
  {
diff --git a/test/kvm.c b/test/kvm.c
@@ -304,7 +304,7 @@ int16_t *print_accessed_sets(){
 }
 
 
-void
+int16_t *
 collect(const char *prefix, size_t code_start, size_t code_stop)
 {
 	int ret;
@@ -314,10 +314,8 @@ collect(const char *prefix, size_t code_start, size_t code_stop)
 	kvm_init(131072, code_start, code_stop);
 	printf("KVm init done\n");
 
-
 	ret = 0;
 	kvm_run->exit_reason = KVM_EXIT_IO;
-
 	
 	printf("Now calling KVM_RUN");
 	ret = ioctl(kvm.vcpufd, KVM_RUN, NULL);
@@ -328,9 +326,14 @@ collect(const char *prefix, size_t code_start, size_t code_stop)
 	if (ret < 0 || kvm_run->exit_reason != KVM_EXIT_IO)
 		errx(1, "KVM died: %i %i\n",
 			ret, kvm_run->exit_reason);
+
+	int16_t *sets = print_accessed_sets();
+
 	close(kvm.fd);
 	close(kvm.vmfd);
 	close(kvm.vcpufd);
+
+	return sets;
 }
 
 void dump_msrmt_results_to_log(char *log_file_path, int16_t msrmt_results[SAMPLE_COUNT][64])
@@ -388,13 +391,12 @@ main(int argc, const char **argv)
 	int16_t msmrt_with_access[SAMPLE_COUNT][64];
 	for(int i=0; i < SAMPLE_COUNT; ++i){
 		printf("First: Testing VM without memory access \n");
-		collect("without", __start_guest_without, __stop_guest_without);
-		int16_t *tmp_res =  print_accessed_sets();
+		int16_t *tmp_res;
+		tmp_res = collect("without", __start_guest_without, __stop_guest_without);
 		memcpy(msmrt_without_access[i], tmp_res, 64*sizeof(int16_t));
 		free(tmp_res);
 		printf("Now: Testing access with memory access \n");
-		collect( "with", __start_guest_with, __stop_guest_with);
-		tmp_res = print_accessed_sets();
+		tmp_res = collect("with", __start_guest_with, __stop_guest_with);
 		memcpy(msmrt_with_access[i], tmp_res, 64*sizeof(int16_t));
 		free(tmp_res);
 	}