Begin ioctl and test-case overhaul

14 files changed, 696 insertions, 206 deletions

diff --git a/test/.gitignore b/test/.gitignore
index b434bc4..2549182 100755
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -1,13 +1,7 @@
-access
 eviction
-kvm
-sev
-sev-es
-sev-snp
-fullstep
-execstep
-aes-detect_guest
-aes-detect_host
-access-detect_guest
-access-detect_host
-readsvme
+kvm-eviction
+kvm-step
+kvm-pagestep
+qemu-eviction
+qemu-aes
+qemu-poc
diff --git a/test/access.c b/test/access.c
deleted file mode 100644
index dc704fb..0000000
--- a/test/access.c
+++ /dev/null
@@ -1,30 +0,0 @@
-#include "cachepc/uapi.h"
-
-#include <sys/ioctl.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <fcntl.h>
-#include <stdint.h>
-#include <err.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-int
-main(int argc, const char **argv)
-{
-	uint32_t arg;
-	int fd, ret;
-	size_t i;
-
-	fd = open("/dev/kvm", O_RDONLY);
-	if (fd < 0) err(1, "open");
-
-	for (i = 0; i < 100; i++) {
-		arg = 48; /* target set */
-		ret = ioctl(fd, KVM_CPC_TEST_ACCESS, &arg);
-		if (ret == -1) err(1, "ioctl TEST_ACCESS");
-		if (arg != 1) errx(1, "access result (%i) != 1", arg);
-	}
-
-	close(fd);
-}
diff --git a/test/aes-detect.c b/test/aes-detect.c
deleted file mode 100644
index d95148b..0000000
--- a/test/aes-detect.c
+++ /dev/null
@@ -1,85 +0,0 @@
-#include "cachepc/uapi.h"
-
-#include <stdint.h>
-
-#define ACCESS_LINE(n) \
-	asm volatile ("mov (%0), %%rbx" \
-		: : "r"(((uint8_t*) L1) + n * L1_LINESIZE) : "rbx");
-
-#define DO_ACCESS_PATTERN() \
-	ACCESS_LINE(60) \
-	ACCESS_LINE(13) \
-	ACCESS_LINE(24) \
-	ACCESS_LINE(19) \
-	ACCESS_LINE(38) \
-	ACCESS_LINE(17) \
-	ACCESS_LINE( 2) \
-	ACCESS_LINE(12) \
-	ACCESS_LINE(22) \
-	ACCESS_LINE(46) \
-	ACCESS_LINE( 4) \
-	ACCESS_LINE(61) \
-	ACCESS_LINE( 5) \
-	ACCESS_LINE(14) \
-	ACCESS_LINE(11) \
-	ACCESS_LINE(35) \
-	ACCESS_LINE(45) \
-	ACCESS_LINE(10) \
-	ACCESS_LINE(49) \
-	ACCESS_LINE(56) \
-	ACCESS_LINE(27) \
-	ACCESS_LINE(37) \
-	ACCESS_LINE(63) \
-	ACCESS_LINE(54) \
-	ACCESS_LINE(55) \
-	ACCESS_LINE(29) \
-	ACCESS_LINE(48) \
-	ACCESS_LINE( 9) \
-	ACCESS_LINE(16) \
-	ACCESS_LINE(39) \
-	ACCESS_LINE(20) \
-	ACCESS_LINE(21) \
-	ACCESS_LINE(62) \
-	ACCESS_LINE( 0) \
-	ACCESS_LINE(34) \
-	ACCESS_LINE( 8) \
-	ACCESS_LINE(53) \
-	ACCESS_LINE(42) \
-	ACCESS_LINE(51) \
-	ACCESS_LINE(50) \
-	ACCESS_LINE(57) \
-	ACCESS_LINE( 7) \
-	ACCESS_LINE( 6) \
-	ACCESS_LINE(33) \
-	ACCESS_LINE(26) \
-	ACCESS_LINE(40) \
-	ACCESS_LINE(58) \
-	ACCESS_LINE( 1) \
-	ACCESS_LINE(44) \
-	ACCESS_LINE(23) \
-	ACCESS_LINE(25) \
-	ACCESS_LINE(47) \
-	ACCESS_LINE(15) \
-	ACCESS_LINE(36) \
-	ACCESS_LINE( 3) \
-	ACCESS_LINE(41) \
-	ACCESS_LINE(52) \
-	ACCESS_LINE(59) \
-	ACCESS_LINE(18) \
-	ACCESS_LINE(31) \
-	ACCESS_LINE(28) \
-	ACCESS_LINE(32) \
-	ACCESS_LINE(30) \
-	ACCESS_LINE(43)
-
-/* corresponding physical memory will also be page-aligned,
- * in our case PAGE_SIZE is the size of our L1 without associativity */
-#pragma DATA_ALIGN(L1, PAGE_SIZE)
-uint8_t L1[L1_SETS * L1_LINESIZE];
-
-int sync_access_pattern[] = {
-	60, 13, 24, 19, 38, 17,  2, 12, 22, 46,  4, 61,  5, 14, 11, 35,
-	45, 10, 49, 56, 27, 37, 63, 54, 55, 29, 48,  9, 16, 39, 20, 21,
-	62,  0, 34,  8, 53, 42, 51, 50, 57,  7,  6, 33, 26, 40, 58,  1,
-	44, 23, 25, 47, 15, 36,  3, 41, 52, 59, 18, 31, 28, 32, 30, 43
-};
diff --git a/test/aes-detect.h b/test/aes-detect.h
deleted file mode 100644
index b747860..0000000
--- a/test/aes-detect.h
+++ /dev/null
@@ -1,3 +0,0 @@
-#pragma once
-
-extern int sync_access_pattern[];
diff --git a/test/eviction.c b/test/eviction.c
index 1570205..8cb5914 100644
--- a/test/eviction.c
+++ b/test/eviction.c
@@ -2,31 +2,31 @@
 
 #include <sys/ioctl.h>
 #include <fcntl.h>
-#include <stdint.h>
 #include <assert.h>
 #include <unistd.h>
 #include <err.h>
-#include <stdlib.h>
+#include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 int
 main(int argc, const char **argv)
 {
-	cpc_msrmt_t counts[L1_SETS];
-	uint32_t arg;
+	uint8_t counts[L1_SETS];
+	uint32_t set;
 	int i, fd, ret;
 
 	fd = open("/dev/kvm", O_RDONLY);
 	if (fd < 0) err(1, "open");
 
-	arg = 48;
-	if (argc == 2) arg = atoi(argv[1]);
+	set = 48;
+	if (argc > 1) set = atoi(argv[1]);
 
-	ret = ioctl(fd, KVM_CPC_TEST_EVICTION, &arg);
-	if (ret == -1) err(1, "ioctl TEST_EVICTION");
+	ret = ioctl(fd, KVM_CPC_TEST_EVICTION, &set);
+	if (ret == -1) err(1, "ioctl KVM_CPC_TEST_EVICTION");
 
 	ret = ioctl(fd, KVM_CPC_READ_COUNTS, counts);
-	if (ret == -1) err(1, "ioctl READ_COUNTS");
+	if (ret == -1) err(1, "ioctl KVM_CPC_READ_COUNTS");
 
 	for (i = 0; i < 64; i++) {
 		if (i % 16 == 0 && i)
@@ -38,6 +38,6 @@ main(int argc, const char **argv)
 			printf("\x1b[0m");
 	}
 	printf("\n");
-	
+
 	close(fd);
 }
diff --git a/test/fulltrack b/test/fulltrack
deleted file mode 100755
index 5344b51..0000000
--- a/test/fulltrack
+++ /dev/null
diff --git a/test/kvm-eviction.c b/test/kvm-eviction.c
new file mode 100644
index 0000000..21227cb
--- /dev/null
+++ b/test/kvm-eviction.c
@@ -0,0 +1,613 @@
+#define _GNU_SOURCE
+
+#include "cachepc/uapi.h"
+
+#include <linux/psp-sev.h>
+#include <linux/kvm.h>
+#include <sys/syscall.h>
+#include <sys/ioctl.h>
+#include <sys/user.h>
+#include <sys/wait.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+#include <dirent.h>
+#include <assert.h>
+#include <errno.h>
+#include <err.h>
+#include <fcntl.h>
+#include <sched.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdarg.h>
+
+#define ARRLEN(x) (sizeof(x) / sizeof((x)[0]))
+#define MIN(a,b) ((a) > (b) ? (b) : (a))
+
+#define SAMPLE_COUNT 64
+
+#define TARGET_CORE 2
+#define SECONDARY_CORE 3
+
+#define TARGET_SET 15
+
+enum {
+	WITH,
+	WITHOUT
+};
+
+struct kvm {
+	int fd, vmfd, vcpufd;
+	void *mem;
+	size_t memsize;
+	struct kvm_run *run;
+};
+
+/* start and end for guest assembly */
+extern uint8_t __start_guest_with[];
+extern uint8_t __stop_guest_with[];
+extern uint8_t __start_guest_without[];
+extern uint8_t __stop_guest_without[];
+
+static const char *vmtype;
+
+static int kvm_dev, sev_dev;
+
+enum {
+	GSTATE_UNINIT,
+	GSTATE_LUPDATE,
+	GSTATE_LSECRET,
+	GSTATE_RUNNING,
+	GSTATE_SUPDATE,
+	GSTATE_RUPDATE,
+	GSTATE_SENT
+};
+
+const char *sev_fwerr_strs[] = {
+	"Success",
+	"Platform state is invalid",
+	"Guest state is invalid",
+	"Platform configuration is invalid",
+	"Buffer too small",
+	"Platform is already owned",
+	"Certificate is invalid",
+	"Policy is not allowed",
+	"Guest is not active",
+	"Invalid address",
+	"Bad signature",
+	"Bad measurement",
+	"Asid is already owned",
+	"Invalid ASID",
+	"WBINVD is required",
+	"DF_FLUSH is required",
+	"Guest handle is invalid",
+	"Invalid command",
+	"Guest is active",
+	"Hardware error",
+	"Hardware unsafe",
+	"Feature not supported",
+	"Invalid parameter",
+	"Out of resources",
+	"Integrity checks failed"
+};
+
+const char *sev_gstate_strs[] = {
+	"UNINIT",
+	"LUPDATE",
+	"LSECRET",
+	"RUNNING",
+	"SUPDATE",
+	"RUPDATE",
+	"SEND"
+};
+
+void
+hexdump(void *data, int len)
+{
+	int i;
+
+	for (i = 0; i < len; i++) {
+		if (i % 16 == 0 && i)
+			printf("\n");
+		printf("%02X ", *(uint8_t *)(data + i));
+	}
+	printf("\n");
+}
+
+__attribute__((section("guest_with"))) void
+vm_guest_with(void)
+{
+	while (1) {
+		asm volatile("mov (%[v]), %%bl"
+			: : [v] "r" (L1_LINESIZE * TARGET_SET));
+		CPC_DO_VMMCALL(EXIT, 0, 0);
+	}
+}
+
+__attribute__((section("guest_without"))) void
+vm_guest_without(void)
+{
+	while (1) {
+		CPC_DO_VMMCALL(EXIT, 0, 0);
+	}
+}
+
+bool
+pin_process(pid_t pid, int cpu, bool assert)
+{
+	cpu_set_t cpuset;
+	int ret;
+
+	CPU_ZERO(&cpuset);
+	CPU_SET(cpu, &cpuset);
+	ret = sched_setaffinity(pid, sizeof(cpu_set_t), &cpuset);
+	if (ret < 0) {
+		if (assert) err(1, "sched_setaffinity");
+		return false;
+	}
+
+	return true;
+}
+
+int
+read_stat_core(pid_t pid)
+{
+	char path[256];
+	char line[2048];
+	FILE *file;
+	char *p;
+	int i, cpu;
+
+	snprintf(path, sizeof(path), "/proc/%u/stat", pid);
+	file = fopen(path, "r");
+	if (!file) return -1;
+
+	if (!fgets(line, sizeof(line), file))
+		err(1, "read stat");
+
+	p = line;
+	for (i = 0; i < 38 && (p = strchr(p, ' ')); i++)
+		p += 1;
+
+	if (!p) errx(1, "stat format");
+	cpu = atoi(p);
+
+	fclose(file);
+
+	return cpu;
+}
+
+const char *
+sev_fwerr_str(int code)
+{
+	if (code < 0 || code >= ARRLEN(sev_fwerr_strs))
+		return "Unknown error";
+
+	return sev_fwerr_strs[code];
+}
+
+const char *
+sev_gstate_str(int code)
+{
+	if (code < 0 || code >= ARRLEN(sev_gstate_strs))
+		return "Unknown gstate";
+
+	return sev_gstate_strs[code];
+}
+
+int
+sev_ioctl(int vmfd, int cmd, void *data, int *error)
+{
+	struct kvm_sev_cmd input;
+	int ret;
+
+	memset(&input, 0, sizeof(input));
+	input.id = cmd;
+	input.sev_fd = sev_dev;
+	input.data = (uintptr_t) data;
+
+	ret = ioctl(vmfd, KVM_MEMORY_ENCRYPT_OP, &input);
+	if (error) *error = input.error;
+
+	return ret;
+}
+
+void
+sev_get_measure(int vmfd)
+{
+	struct kvm_sev_launch_measure msrmt;
+	int ret, fwerr;
+	uint8_t *data;
+
+	memset(&msrmt, 0, sizeof(msrmt));
+	ret = sev_ioctl(vmfd, KVM_SEV_LAUNCH_MEASURE, &msrmt, &fwerr);
+	if (ret < 0 && fwerr != SEV_RET_INVALID_LEN)
+		errx(1, "KVM_SEV_LAUNCH_MEASURE: (%s) %s",
+			strerror(errno), sev_fwerr_str(fwerr));
+
+	data = malloc(msrmt.len);
+	msrmt.uaddr = (uintptr_t) data;
+
+	ret = sev_ioctl(vmfd, KVM_SEV_LAUNCH_MEASURE, &msrmt, &fwerr);
+	if (ret < 0)
+		errx(1, "KVM_SEV_LAUNCH_MEASURE: (%s) %s",
+			strerror(errno), sev_fwerr_str(fwerr));
+
+	free(data);
+}
+
+uint8_t
+sev_guest_state(int vmfd, uint32_t handle)
+{
+	struct kvm_sev_guest_status status;
+	int ret, fwerr;
+
+	status.handle = handle;
+	ret = sev_ioctl(vmfd, KVM_SEV_GUEST_STATUS, &status, &fwerr);
+	if (ret < 0) {
+		errx(1, "KVM_SEV_GUEST_STATUS: (%s) %s",
+			strerror(errno), sev_fwerr_str(fwerr));
+	}
+
+	return status.state;
+}
+
+void
+sev_debug_encrypt(int vmfd, void *src, void *dst, size_t size)
+{
+	struct kvm_sev_dbg enc;
+	int ret, fwerr;
+
+	enc.src_uaddr = (uintptr_t) src;
+	enc.dst_uaddr = (uintptr_t) dst;
+	enc.len = size;
+	ret = sev_ioctl(vmfd, KVM_SEV_DBG_ENCRYPT, &enc, &fwerr);
+	if (ret < 0) errx(1, "KVM_SEV_DBG_ENCRYPT: (%s) %s",
+		strerror(errno), sev_fwerr_str(fwerr));
+}
+
+void
+sev_debug_decrypt(int vmfd, void *src, void *dst, size_t size)
+{
+	struct kvm_sev_dbg enc;
+	int ret, fwerr;
+
+	enc.src_uaddr = (uintptr_t) src;
+	enc.dst_uaddr = (uintptr_t) dst;
+	enc.len = size;
+	ret = sev_ioctl(vmfd, KVM_SEV_DBG_DECRYPT, &enc, &fwerr);
+	if (ret < 0) errx(1, "KVM_SEV_DBG_DECRYPT: (%s) %s",
+		strerror(errno), sev_fwerr_str(fwerr));
+}
+
+void
+kvm_init(struct kvm *kvm, size_t ramsize,
+	void *code_start, void *code_stop)
+{
+	struct kvm_userspace_memory_region region;
+	struct kvm_regs regs;
+	struct kvm_sregs sregs;
+	int ret;
+
+	/* Create a kvm instance */
+	kvm->vmfd = ioctl(kvm_dev, KVM_CREATE_VM, 0);
+	if (kvm->vmfd < 0) err(1, "KVM_CREATE_VM");
+
+	/* Allocate guest memory */
+	kvm->memsize = ramsize;
+	kvm->mem = mmap(NULL, kvm->memsize, PROT_READ | PROT_WRITE,
+		MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	if (!kvm->mem) err(1, "Allocating guest memory");
+	assert(code_stop - code_start <= kvm->memsize);
+	memcpy(kvm->mem, code_start, code_stop - code_start);
+
+	/* Map it into the vm */
+	memset(&region, 0, sizeof(region));
+	region.slot = 0;
+	region.memory_size = kvm->memsize;
+	region.guest_phys_addr = 0x0000;
+	region.userspace_addr = (uintptr_t) kvm->mem;
+	ret = ioctl(kvm->vmfd, KVM_SET_USER_MEMORY_REGION, &region);
+	if (ret < 0) err(1, "KVM_SET_USER_MEMORY_REGION");
+
+	/* Create virtual cpu core */
+	kvm->vcpufd = ioctl(kvm->vmfd, KVM_CREATE_VCPU, 0);
+	if (kvm->vcpufd < 0) err(1, "KVM_CREATE_VCPU");
+
+	/* Initialize segment regs */
+	memset(&sregs, 0, sizeof(sregs));
+	ret = ioctl(kvm->vcpufd, KVM_GET_SREGS, &sregs);
+	if (ret < 0) err(1, "KVM_GET_SREGS");
+	sregs.cs.base = 0;
+	sregs.cs.selector = 0;
+	ret = ioctl(kvm->vcpufd, KVM_SET_SREGS, &sregs);
+	if (ret < 0) err(1, "KVM_SET_SREGS");
+
+	/* Initialize rest of registers */
+	memset(&regs, 0, sizeof(regs));
+	regs.rip = 0x0;
+	regs.rsp = kvm->memsize - 8;
+	regs.rbp = kvm->memsize - 8;
+	regs.rax = 0;
+	regs.rdx = 0;
+	regs.rflags = 0x2;
+	ret = ioctl(kvm->vcpufd, KVM_SET_REGS, &regs);
+	if (ret < 0) err(1, "KVM_SET_REGS");
+}
+
+void
+sev_kvm_init(struct kvm *kvm, size_t ramsize,
+	void *code_start, void *code_stop)
+{
+	struct kvm_userspace_memory_region region;
+	struct kvm_sev_launch_update_data update;
+	struct kvm_sev_launch_start start;
+	struct kvm_regs regs;
+	struct kvm_sregs sregs;
+	int ret, fwerr;
+
+	/* Create a kvm instance */
+	kvm->vmfd = ioctl(kvm_dev, KVM_CREATE_VM, 0);
+	if (kvm->vmfd < 0) err(1, "KVM_CREATE_VM");
+
+	/* Allocate guest memory */
+	kvm->memsize = ramsize;
+	kvm->mem = mmap(NULL, kvm->memsize, PROT_READ | PROT_WRITE,
+		MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+	if (!kvm->mem) err(1, "Allocating guest memory");
+	assert(code_stop - code_start <= kvm->memsize);
+	memcpy(kvm->mem, code_start, code_stop - code_start);
+
+	/* Map it into the vm */
+	memset(&region, 0, sizeof(region));
+	region.slot = 0;
+	region.memory_size = kvm->memsize;
+	region.guest_phys_addr = 0x0000;
+	region.userspace_addr = (uintptr_t) kvm->mem;
+	ret = ioctl(kvm->vmfd, KVM_SET_USER_MEMORY_REGION, &region);
+	if (ret < 0) err(1, "KVM_SET_USER_MEMORY_REGION");
+
+	/* Enable SEV for vm */
+	ret = sev_ioctl(kvm->vmfd, KVM_SEV_INIT, NULL, &fwerr);
+	if (ret < 0) errx(1, "KVM_SEV_INIT: (%s) %s",
+		strerror(errno), sev_fwerr_str(fwerr));
+
+	/* Generate encryption keys and set policy */
+	memset(&start, 0, sizeof(start));
+	start.handle = 0;
+	start.policy = 0;
+	ret = sev_ioctl(kvm->vmfd, KVM_SEV_LAUNCH_START, &start, &fwerr);
+	if (ret < 0) errx(1, "KVM_SEV_LAUNCH_START: (%s) %s",
+		strerror(errno), sev_fwerr_str(fwerr));
+
+	/* Prepare the vm memory (by encrypting it) */
+	memset(&update, 0, sizeof(update));
+	update.uaddr = (uintptr_t) kvm->mem;
+	update.len = ramsize;
+	ret = sev_ioctl(kvm->vmfd, KVM_SEV_LAUNCH_UPDATE_DATA, &update, &fwerr);
+	if (ret < 0) errx(1, "KVM_SEV_LAUNCH_UPDATE_DATA: (%s) %s",
+		strerror(errno), sev_fwerr_str(fwerr));
+
+	/* Collect a measurement (necessary) */
+	sev_get_measure(kvm->vmfd);
+
+	/* Finalize launch process */
+	ret = sev_ioctl(kvm->vmfd, KVM_SEV_LAUNCH_FINISH, 0, &fwerr);
+	if (ret < 0) errx(1, "KVM_SEV_LAUNCH_FINISH: (%s) %s",
+		strerror(errno), sev_fwerr_str(fwerr));	
+	ret = sev_guest_state(kvm->vmfd, start.handle);
+	if (ret != GSTATE_RUNNING)
+		errx(1, "Bad guest state: %s", sev_gstate_str(fwerr));
+
+	/* Create virtual cpu core */
+	kvm->vcpufd = ioctl(kvm->vmfd, KVM_CREATE_VCPU, 0);
+	if (kvm->vcpufd < 0) err(1, "KVM_CREATE_VCPU");
+
+	/* Map the shared kvm_run structure and following data */
+	ret = ioctl(kvm_dev, KVM_GET_VCPU_MMAP_SIZE, NULL);
+	if (ret < 0) err(1, "KVM_GET_VCPU_MMAP_SIZE");
+	if (ret < sizeof(struct kvm_run))
+		errx(1, "KVM_GET_VCPU_MMAP_SIZE too small");
+	kvm->run = mmap(NULL, ret, PROT_READ | PROT_WRITE,
+		MAP_SHARED, kvm->vcpufd, 0);
+	if (!kvm->run) err(1, "mmap vcpu");
+
+	/* Initialize segment regs */
+	memset(&sregs, 0, sizeof(sregs));
+	ret = ioctl(kvm->vcpufd, KVM_GET_SREGS, &sregs);
+	if (ret < 0) err(1, "KVM_GET_SREGS");
+	sregs.cs.base = 0;
+	sregs.cs.selector = 0;
+	ret = ioctl(kvm->vcpufd, KVM_SET_SREGS, &sregs);
+	if (ret < 0) err(1, "KVM_SET_SREGS");
+
+	/* Initialize rest of registers */
+	memset(&regs, 0, sizeof(regs));
+	regs.rip = 0x0;
+	regs.rsp = kvm->memsize - 8;
+	regs.rbp = kvm->memsize - 8;
+	regs.rax = 0;
+	regs.rdx = 0;
+	regs.rflags = 0x2;
+	ret = ioctl(kvm->vcpufd, KVM_SET_REGS, &regs);
+	if (ret < 0) err(1, "KVM_SET_REGS");
+}
+
+void
+sev_es_kvm_init(struct kvm *kvm, size_t ramsize,
+	void *code_start, void *code_stop)
+{
+	errx(1, "Not implemented");
+}
+
+void
+sev_snp_kvm_init(struct kvm *kvm, size_t ramsize,
+	void *code_start, void *code_stop)
+{
+	errx(1, "Not implemented");
+}
+
+void
+vm_init(struct kvm *kvm, size_t ramsize, void *code_start, void *code_end)
+{
+	if (!strcmp(vmtype, "kvm")) {
+		kvm_init(kvm, ramsize, code_start, code_end);
+	} else if (!strcmp(vmtype, "sev")) {
+		sev_kvm_init(kvm, ramsize, code_start, code_end);
+	} else if (!strcmp(vmtype, "sev-es")) {
+		sev_es_kvm_init(kvm, ramsize, code_start, code_end);
+	} else if (!strcmp(vmtype, "sev-snp")) {
+		sev_snp_kvm_init(kvm, ramsize, code_start, code_end);
+	} else {
+		errx(1, "invalid version");
+	}
+}
+
+void
+vm_deinit(struct kvm *kvm)
+{
+	close(kvm->vmfd);
+	close(kvm->vcpufd);
+	munmap(kvm->mem, kvm->memsize);
+}
+
+void
+print_counts(uint8_t *counts)
+{
+	int i;
+
+	for (i = 0; i < 64; i++) {
+		if (i % 16 == 0 && i)
+			printf("\n");
+		if (counts[i] == 1)
+			printf("\x1b[38;5;88m");
+		else if (counts[i] > 1)
+			printf("\x1b[38;5;196m");
+		printf("%2i ", i);
+		if (counts[i] > 0)
+			printf("\x1b[0m");
+	}
+
+	printf("\nTarget Set %i Count: %u\n\n",
+		TARGET_SET, counts[TARGET_SET]);
+}
+
+void
+collect(struct kvm *kvm, uint8_t *counts)
+{
+	struct kvm_regs regs;
+	int ret;
+
+	/* run vm twice, use count without initial stack setup */
+	ret = ioctl(kvm->vcpufd, KVM_RUN, NULL);
+	if (ret < 0) err(1, "KVM_RUN");
+
+	if (kvm->run->exit_reason == KVM_EXIT_MMIO) {
+		memset(&regs, 0, sizeof(regs));
+		ret = ioctl(kvm->vcpufd, KVM_GET_REGS, &regs);
+		if (ret < 0) err(1, "KVM_GET_REGS");
+		errx(1, "Victim access OOB: %llu %08llx => %02X\n",
+			kvm->run->mmio.phys_addr, regs.rip,
+			((uint8_t *)kvm->mem)[regs.rip]);
+	} else if (kvm->run->exit_reason != KVM_EXIT_IO) {
+		errx(1, "KVM died: %i\n", kvm->run->exit_reason);
+	}
+
+	ret = ioctl(kvm_dev, KVM_CPC_READ_COUNTS, counts);
+	if (ret == -1) err(1, "ioctl KVM_CPC_READ_COUNTS");
+}
+
+int
+main(int argc, const char **argv)
+{
+	struct kvm vms[2];
+	uint8_t counts[2][SAMPLE_COUNT][64];
+	uint8_t baseline[64];
+	uint32_t calc;
+	int i, k, ret;
+
+	vmtype = "kvm";
+	if (argc > 1) vmtype = argv[1];
+	if (strcmp(vmtype, "kvm") && strcmp(vmtype, "sev")
+			&& strcmp(vmtype, "sev-es")
+			&& strcmp(vmtype, "sev-snp"))
+		errx(1, "invalid version: %s", vmtype);
+
+	setvbuf(stdout, NULL, _IONBF, 0);
+
+	pin_process(0, TARGET_CORE, true);
+
+	kvm_dev = open("/dev/kvm", O_RDWR | O_CLOEXEC);
+	if (kvm_dev < 0) err(1, "open /dev/kvm");
+
+	sev_dev = open("/dev/sev", O_RDWR | O_CLOEXEC);
+	if (sev_dev < 0) err(1, "open /dev/sev");
+
+	/* Make sure we have the stable version of the API */
+	ret = ioctl(kvm_dev, KVM_GET_API_VERSION, NULL);
+	if (ret < 0) err(1, "KVM_GET_API_VERSION");
+	if (ret != 12) errx(1, "KVM_GET_API_VERSION %d, expected 12", ret);
+
+	/* Reset kernel module state */
+	ret = ioctl(kvm_dev, KVM_CPC_RESET);
+	if (ret < 0) err(1, "ioctl KVM_CPC_RESET");
+
+	vm_init(&vms[WITH], 64 * 64 * 8 * 2,
+		__start_guest_with, __stop_guest_with);
+	vm_init(&vms[WITHOUT], 64 * 64 * 8 * 2,
+		__start_guest_without, __stop_guest_without);
+
+	calc = true;
+	ret = ioctl(kvm_dev, KVM_CPC_CALC_BASELINE, &calc);
+	if (ret == -1) err(1, "ioctl KVM_CPC_CALC_BASELINE");
+
+	for (i = 0; i < SAMPLE_COUNT; i++) {
+		collect(&vms[WITH], counts[WITH][i]);
+		collect(&vms[WITHOUT], counts[WITHOUT][i]);
+	}
+
+	calc = false;
+	ret = ioctl(kvm_dev, KVM_CPC_CALC_BASELINE, &calc);
+	if (ret == -1) err(1, "ioctl KVM_CPC_CALC_BASELINE");
+
+	ret = ioctl(kvm_dev, KVM_CPC_READ_BASELINE, baseline);
+	if (ret == -1) err(1, "ioctl KVM_CPC_READ_BASELINE");
+
+	for (i = 0; i < SAMPLE_COUNT; i++) {
+		printf("Evictions with access:\n");
+		print_counts(counts[WITH][i]);
+
+		printf("Evictions without access:\n");
+		print_counts(counts[WITHOUT][i]);
+
+		for (k = 0; k < 64; k++) {
+			if (counts[WITH][i][k] >= 8)
+				warnx("with: Count OOB (%i, %i)", i, k);
+			if (baseline[k] > counts[WITH][i][k])
+				warnx("with: Baseline oob (%i, %i)", i, k);
+			counts[WITH][i][k] -= baseline[k];
+
+			if (counts[WITHOUT][i][k] >= 8)
+				warnx("without: Count OOB (%i, %i)", i, k);
+			if (baseline[k] > counts[WITHOUT][i][k])
+				warnx("without: Baseline OOB (%i, %i)", i, k);
+			counts[WITHOUT][i][k] -= baseline[k];
+		}
+
+		if (!counts[WITH][i][TARGET_SET])
+			warnx("with: Missing eviction in target set");
+	}
+
+	vm_deinit(&vms[WITH]);
+	vm_deinit(&vms[WITHOUT]);
+
+	close(kvm_dev);
+	close(sev_dev);
+}
+
diff --git a/test/execstep.c b/test/kvm-pagestep.c
index 14f8d42..14f8d42 100644
--- a/test/execstep.c
+++ b/test/kvm-pagestep.c
diff --git a/test/fullstep.c b/test/kvm-step.c
index 52e745a..14f8d42 100644
--- a/test/fullstep.c
+++ b/test/kvm-step.c
@@ -81,7 +81,9 @@ const char *sev_fwerr_strs[] = {
 	[0x15] = "Feature not supported",
 	[0x16] = "Invalid parameter",
 	[0x17] = "Out of resources",
-	[0x18] = "Integrity checks failed"
+	[0x18] = "Integrity checks failed",
+	[0x19] = "RMP page size is incorrect",
+	[0x1A] = "RMP page state is incorrect",
 };
 
 const char *sev_gstate_strs[] = {
@@ -106,13 +108,9 @@ hexdump(void *data, int len)
 __attribute__((section("guest_with"))) void
 vm_guest_with(void)
 {
-	while (1) {
-		asm volatile("mov (%0), %%eax" : :
-			"r" (L1_LINESIZE * (L1_SETS * 3 + TARGET_SET)) : "rax");
-		asm volatile("nop");
-		asm volatile("mov (%0), %%eax" : :
-			"r" (L1_LINESIZE * (L1_SETS * 3 + TARGET_SET)) : "rax");
-	}
+	asm volatile ("mov %rbp, %rsp; pop %rbp; \
+			movq $4096, %rcx; movq $0, %rdx; cmp %rcx, %rdx; \
+			cmovne %rdx, %rcx; jmp *%rcx");
 }
 
 bool
@@ -163,8 +161,10 @@ read_stat_core(pid_t pid)
 const char *
 sev_fwerr_str(int code)
 {
-	if (code < 0 || code >= ARRLEN(sev_fwerr_strs))
+	if (code < 0 || code >= ARRLEN(sev_fwerr_strs)) {
+		warnx("Unknown firmware error %i", code);
 		return "Unknown error";
+	}
 
 	return sev_fwerr_strs[code];
 }
@@ -172,8 +172,10 @@ sev_fwerr_str(int code)
 const char *
 sev_gstate_str(int code)
 {
-	if (code < 0 || code >= ARRLEN(sev_gstate_strs))
+	if (code < 0 || code >= ARRLEN(sev_gstate_strs)) {
+		warnx("Unknown guest state %i", code);
 		return "Unknown gstate";
+	}
 
 	return sev_gstate_strs[code];
 }
@@ -210,6 +212,7 @@ snp_guest_state(int vmfd)
 	return status.state;
 }
 
+
 void
 snp_dbg_encrypt(int vmfd, void *dst, void *src, size_t size)
 {
@@ -247,7 +250,7 @@ snp_dbg_decrypt(int vmfd, void *dst, void *src, size_t size)
 }
 
 uint64_t
-snp_dbg_rip(int vmfd)
+snp_dbg_decrypt_rip(int vmfd)
 {
 	uint8_t vmsa[PAGE_SIZE];
 	uint64_t rip;
@@ -284,7 +287,12 @@ snp_kvm_init(struct kvm *kvm, size_t ramsize, void *code_start, void *code_stop)
 		MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 	if (!kvm->mem) err(1, "Allocating guest memory");
 	assert(code_stop - code_start <= kvm->memsize);
-	memcpy(kvm->mem, code_start, code_stop - code_start);
+
+	/* Fill memory with nops and put jump code a bit from start
+	 * such that we access multiple different pages while running */
+	memset(kvm->mem, 0x90, kvm->memsize);
+	memcpy(kvm->mem + L1_SIZE, // - (code_stop - code_start),
+		code_start, code_stop - code_start);
 
 	/* Map it into the vm */
 	memset(&region, 0, sizeof(region));
@@ -382,7 +390,7 @@ read_counts()
 	if (!counts) err(1, "malloc");
 
 	ret = ioctl(kvm_dev, KVM_CPC_READ_COUNTS, counts);
-	if (ret == -1) err(1, "ioctl READ_COUNTS");
+	if (ret) err(1, "ioctl READ_COUNTS");
 
 	for (i = 0; i < L1_SETS; i++) {
 		if (counts[i] > 8)
@@ -444,7 +452,6 @@ monitor(struct kvm *kvm, bool baseline)
 {
 	struct cpc_event event;
 	cpc_msrmt_t counts[64];
-	uint64_t rip;
 	int ret, i;
 
 	/* Get page fault info */
@@ -452,36 +459,43 @@ monitor(struct kvm *kvm, bool baseline)
 	if (ret) {
 		if (errno == EAGAIN)
 			return 0;
-		perror("ioctl POLL_EVENT");
+		warn("ioctl POLL_EVENT");
 		return 1;
 	}
 
 	if (event.type == CPC_EVENT_TRACK_STEP) {
 		ret = ioctl(kvm_dev, KVM_CPC_READ_COUNTS, counts);
-		if (ret == -1) err(1, "ioctl READ_COUNTS");
+		if (ret) err(1, "ioctl READ_COUNTS");
 
 		if (!baseline) {
-			rip = snp_dbg_rip(kvm->vmfd);
-			printf("Event: cnt:%llu inst:%llu data:%llu retired:%llu rip:%lu\n",
+			printf("Event: cnt:%llu rip:%lu, inst:%llu data:%llu retired:%llu\n",
 				event.step.fault_count,
+				0, // snp_dbg_decrypt_rip(kvm->vmfd),
 				event.step.fault_gfns[0],
 				event.step.fault_gfns[1],
-				event.step.retinst, rip);
+				event.step.retinst);
 			print_counts(counts);
 			printf("\n");
 		}
-		faultcnt++;
 
 		for (i = 0; i < 64; i++) {
 			if (counts[i] > 8) {
-				errx(1, "Invalid count for set %i (%llu)",
+				warnx("Invalid count for set %i (%llu)",
 					i, counts[i]);
 			}
 		}
+
+		if (baseline) faultcnt++;
+	} else if (event.type == CPC_EVENT_TRACK_PAGE) {
+		printf("Event: inst page from:%llu to:%llu rip:%lu\n\n",
+			event.page.inst_gfn_prev, event.page.inst_gfn,
+			0); //snp_dbg_decrypt_rip(kvm->vmfd));
+
+		if (!baseline) faultcnt++;
 	}
 
 	ret = ioctl(kvm_dev, KVM_CPC_ACK_EVENT, &event.id);
-	if (ret == -1) err(1, "ioctl ACK_EVENT");
+	if (ret) err(1, "ioctl ACK_EVENT");
 
 	return 0;
 }
@@ -522,27 +536,29 @@ main(int argc, const char **argv)
 	/* Page tracking init needs to happen after kvm
 	 * init so main_kvm is set.. */
 
+	/* Reset previous tracking */
 	ret = ioctl(kvm_dev, KVM_CPC_RESET_TRACKING, NULL);
-	if (ret == -1) err(1, "ioctl RESET_TRACKING");
+	if (ret) err(1, "ioctl RESET_TRACKING");
 
-	arg = CPC_TRACK_FULL;
+	/* Do data access stepping */
+	arg = CPC_TRACK_STUB;
 	ret = ioctl(kvm_dev, KVM_CPC_TRACK_MODE, &arg);
-	if (ret == -1) err(1, "ioctl TRACK_MODE");
+	if (ret) err(1, "ioctl TRACK_MODE");
 
 	/* Init page tracking */
-	track_mode = KVM_PAGE_TRACK_ACCESS;
+	track_mode = KVM_PAGE_TRACK_EXEC;
 	ret = ioctl(kvm_dev, KVM_CPC_TRACK_ALL, &track_mode);
-	if (ret == -1) err(1, "ioctl TRACK_ALL");
+	if (ret) err(1, "ioctl TRACK_ALL");
 
 	arg = true;
 	ret = ioctl(kvm_dev, KVM_CPC_MEASURE_BASELINE, &arg);
-	if (ret == -1) err(1, "ioctl MEASURE_BASELINE");
+	if (ret) err(1, "ioctl MEASURE_BASELINE");
 
 	ppid = getpid();
 	if ((pid = fork())) {
 		if (pid < 0) err(1, "fork");
 
-		sleep(1); /* wait for child to pin other core */
+		sleep(1); /* give time for child to pin other core */
 
 		printf("VMRUN\n");
 		runonce(&kvm_with_access);
@@ -558,16 +574,16 @@ main(int argc, const char **argv)
 
 		do {
 			ret = ioctl(kvm_dev, KVM_CPC_POLL_EVENT, &event);
-			if (ret == -1 && errno != EAGAIN)
+			if (ret && errno != EAGAIN)
 				err(1, "ioctl POLL_EVENT");
-		} while (ret == -1 && errno == EAGAIN);
+		} while (ret && errno == EAGAIN);
 
 		arg = false;
 		ret = ioctl(kvm_dev, KVM_CPC_MEASURE_BASELINE, &arg);
-		if (ret == -1) err(1, "ioctl MEASURE_BASELINE");
+		if (ret) err(1, "ioctl MEASURE_BASELINE");
 
 		ret = ioctl(kvm_dev, KVM_CPC_READ_BASELINE, baseline);
-		if (ret == -1) err(1, "ioctl READ_BASELINE");
+		if (ret) err(1, "ioctl READ_BASELINE");
 
 		printf("\n>>> BASELINE:\n");
 		print_counts(baseline);
@@ -578,15 +594,26 @@ main(int argc, const char **argv)
 		/* Check baseline for saturated sets */
 		for (i = 0; i < 64; i++) {
 			if (baseline[i] >= 8)
-				errx(1, "!!! Baseline set %i full\n", i);
+				warnx("!!! Baseline set %i full\n", i);
 		}
 
 		arg = true;
 		ret = ioctl(kvm_dev, KVM_CPC_SUB_BASELINE, &arg);
-		if (ret == -1) err(1, "ioctl SUB_BASELINE");
+		if (ret) err(1, "ioctl SUB_BASELINE");
+
+		ret = ioctl(kvm_dev, KVM_CPC_RESET_TRACKING, NULL);
+		if (ret) err(1, "ioctl RESET_TRACKING");
+
+		arg = CPC_TRACK_EXEC;
+		ret = ioctl(kvm_dev, KVM_CPC_TRACK_MODE, &arg);
+		if (ret) err(1, "ioctl TRACK_MODE");
+
+		track_mode = KVM_PAGE_TRACK_EXEC;
+		ret = ioctl(kvm_dev, KVM_CPC_TRACK_ALL, &track_mode);
+		if (ret) err(1, "ioctl TRACK_ALL");
 
 		ret = ioctl(kvm_dev, KVM_CPC_ACK_EVENT, &event.id);
-		if (ret == -1) err(1, "ioctl ACK_EVENT");
+		if (ret) err(1, "ioctl ACK_EVENT");
 
 		faultcnt = 0;
 		while (faultcnt < 20) {
diff --git a/test/aes-detect_guest.c b/test/qemu-aes_guest.c
index 2b0f2ad..949c661 100644
--- a/test/aes-detect_guest.c
+++ b/test/qemu-aes_guest.c
@@ -11,8 +11,6 @@
 #include <stdint.h>
 #include <stdlib.h>
 
-// #include "aes-detect.c"
-
 static uint8_t key[16];
 
 void
@@ -62,8 +60,6 @@ main(int argc, const char **argv)
 		memset(block, 0, sizeof(block));
 		strncpy((char *) block, "Hello world", sizeof(block));
 
-		// DO_ACCESS_PATTERN();
-
 		printhex(block, sizeof(block));
 		n = kcapi_cipher_encrypt(kcapi, block, sizeof(block), NULL,
 			block, sizeof(block), KCAPI_ACCESS_HEURISTIC);
diff --git a/test/aes-detect_host.c b/test/qemu-aes_host.c
index 0e6dff6..0e6dff6 100644
--- a/test/aes-detect_host.c
+++ b/test/qemu-aes_host.c
diff --git a/test/access-detect_guest.c b/test/qemu-eviction_guest.c
index a54e753..0cb5cab 100644
--- a/test/access-detect_guest.c
+++ b/test/qemu-eviction_guest.c
@@ -1,5 +1,7 @@
 #include "cachepc/uapi.h"
 
+#include <sys/time.h>
+#include <sys/resource.h>
 #include <err.h>
 #include <unistd.h>
 #include <stdint.h>
@@ -17,6 +19,8 @@ main(int argc, const char **argv)
 		err(1, "memalign");
 	memset(buf, 0, L1_LINESIZE * L1_SETS);
 
+	setpriority(PRIO_PROCESS, 0, -20);
+
 	while (1) {
 		printf("LOOP\n");
 		CPC_DO_VMMCALL(CPC_GUEST_START_TRACK, 0);
diff --git a/test/access-detect_host.c b/test/qemu-eviction_host.c
index 87a5edd..87a5edd 100644
--- a/test/access-detect_host.c
+++ b/test/qemu-eviction_host.c
diff --git a/test/readsvme.c b/test/readsvme.c
deleted file mode 100644
index 4386e89..0000000
--- a/test/readsvme.c
+++ /dev/null
@@ -1,26 +0,0 @@
-#include "cachepc/uapi.h"
-
-#include <sys/ioctl.h>
-#include <err.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-int
-main(int argc, const char **argv)
-{
-	uint32_t svme;
-	int kvm_fd;
-
-	kvm_fd = open("/dev/kvm", O_RDWR);
-	if (kvm_fd < 0) err(1, "open /dev/kvm");
-
-	if (ioctl(kvm_fd, KVM_CPC_SVME_READ, &svme))
-		err(1, "ioctl SVME_READ");
-
-	printf("%u\n", svme);
-
-	close(kvm_fd);
-}