commit 1f418a1c4480cef90b8596ae17bdca9cc7ca1b88
parent 5e21196a9c7ee8eee921d74f6b5eef2f1980ec97
Author: Louis Burda <quent.burda@gmail.com>
Date: Fri, 27 Jan 2023 00:01:09 +0100
Added initial qemu-eviction and qemu-pagestep
Qemu-eviction seems to get stuck somewhere, potentially in the VC-handler(?) since we use the active gfn after resuming execution. Added qemu-pagestep to show viability of page-stepping for later use.
Diffstat:
15 files changed, 162 insertions(+), 48 deletions(-)
diff --git a/Makefile b/Makefile
@@ -9,6 +9,7 @@ BINS = test/eviction test/kvm-eviction
BINS += test/kvm-eviction-with_guest test/kvm-eviction-without_guest
BINS += test/kvm-step test/kvm-step_guest
BINS += test/kvm-pagestep test/kvm-pagestep_guest
+BINS += test/qemu-pagestep
BINS += test/qemu-eviction test/qemu-eviction_guest
# BINS += test/qemu-aes_guest test/qemu-aes
BINS += util/debug util/reset
diff --git a/README b/README
@@ -34,6 +34,9 @@ test/kvm-pagestep:
which can be used to infer what the guest is doing and to begin
fine-grained single-stepping.
+test/qemu-pagestep:
+ Replicate result from kvm-pagestep on a qemu-based vm running debian.
+
test/qemu-eviction:
Replicate result from kvm-eviction on a qemu-based vm running debian
using a specially crafted guest program to signal when measurement
@@ -48,6 +51,41 @@ test/qemu-poc:
unmodified qemu-based linux guest.
+modes
+-----
+
+The kernel module employs a few different modes of tracking described
+in more detail below:
+
+CPC_TRACK_FAULT_NO_RUN:
+ Tracks access to all guest pages and lets the guest page fault over and over
+ without untracking / handling any page faults. This results in a decent
+ baseline measurement when we dont want to step the vm.
+
+CPC_TRACK_EXIT_EVICTION:
+ Set apic timer such that for any reasonably short KVM_RUN no local apic
+ interrupts will occur to cause exits. Good for collecting PRIME+COUNT
+ measurements over a clean run to a "natural" exit such as KVM_EXIT_HLT.
+
+CPC_TRACK_PAGES:
+ Track execution of all guest pages. While the guest is running untrack
+ a single executable page at a time based on page-faults. Allows tracking
+ which guest pages are executed and how long using retired instructions.
+
+CPC_TRACK_STEPS_AND_FAULTS:
+ Track access to all guest pages and single-step guest exection. For each
+ step, collect all page-faults that needed to be handled to enable that
+ step. Allows tracking not only which sets were evicted but what gfns
+ were involved in the access.
+
+CPC_TRACK_STEPS_SIGNALLED:
+ Track execution of a specific set of guest pages and single-step guest
+ execution for only this set. A guest program makes a specific vmmcall
+ to signal when to start and stop tracking. When a page-fault signals
+ that the target pages were reached, single-stepping begins and the
+ target pages are retracked to detect when the pages are left.
+
+
setup
-----
diff --git a/cachepc/cachepc.h b/cachepc/cachepc.h
@@ -36,10 +36,17 @@ struct cpc_fault {
struct list_head list;
};
-struct cpc_track_exec {
+struct cpc_track_pages {
bool cur_avail;
uint64_t cur_gfn;
uint64_t retinst;
+ bool step;
+};
+
+struct cpc_track_steps_signalled {
+ bool enabled;
+ bool target_avail;
+ uint64_t target_gfn;
};
static_assert(sizeof(struct cacheline) == L1_LINESIZE, "Bad cacheline struct");
@@ -98,8 +105,8 @@ extern uint64_t cachepc_rip;
extern uint64_t cachepc_rip_prev;
extern bool cachepc_rip_prev_set;
-extern struct cpc_track_exec cachepc_track_exec;
-extern bool cachepc_track_signalled_enable;
+extern struct cpc_track_pages cpc_track_pages;
+extern struct cpc_track_steps_signalled cpc_track_steps_signalled;
extern struct list_head cachepc_faults;
diff --git a/cachepc/event.c b/cachepc/event.c
@@ -59,12 +59,12 @@ cachepc_send_event(struct cpc_event event)
cachepc_event_avail = true;
cachepc_event = event;
- CPC_DBG("Sent Event: id %llu\n", event.id);
+ //CPC_DBG("Sent Event: id %llu\n", event.id);
write_unlock(&cachepc_event_lock);
/* wait for ack with timeout */
deadline = ktime_get_ns() + 10000000000ULL; /* 10s in ns */
- while (!cachepc_event_is_done(cachepc_event.id)) {
+ while (!cachepc_event_is_done()) {
if (ktime_get_ns() > deadline) {
CPC_WARN("Timeout waiting for ack of event %llu\n",
cachepc_event.id);
@@ -155,14 +155,14 @@ cachepc_send_track_step_event_single(uint64_t gfn, uint32_t err, uint64_t retins
EXPORT_SYMBOL(cachepc_send_track_step_event_single);
bool
-cachepc_event_is_done(uint64_t id)
+cachepc_event_is_done(void)
{
bool done;
read_lock(&cachepc_event_lock);
- CPC_DBG("Event Send: Event not done %llu %llu\n",
- cachepc_last_event_acked, id);
- done = cachepc_last_event_acked == id;
+ //CPC_DBG("Event Send: Event not done %llu %llu\n",
+ // cachepc_last_event_acked, id);
+ done = cachepc_last_event_acked == cachepc_last_event_sent;
read_unlock(&cachepc_event_lock);
return done;
@@ -175,8 +175,8 @@ cachepc_poll_event_ioctl(void __user *arg_user)
read_lock(&cachepc_event_lock);
if (!cachepc_event_avail) {
- CPC_DBG("Event Poll: No event avail %llu %llu\n",
- cachepc_last_event_sent, cachepc_last_event_acked);
+ //CPC_DBG("Event Poll: No event avail %llu %llu\n",
+ // cachepc_last_event_sent, cachepc_last_event_acked);
read_unlock(&cachepc_event_lock);
return -EAGAIN;
}
@@ -185,16 +185,13 @@ cachepc_poll_event_ioctl(void __user *arg_user)
err = 0;
write_lock(&cachepc_event_lock);
if (cachepc_event_avail) {
- CPC_DBG("Event Poll: Event is avail %px %llu %llu\n", arg_user,
- cachepc_last_event_sent, cachepc_last_event_acked);
- err = copy_to_user(arg_user, &cachepc_event, sizeof(cachepc_event));
- if (err != 0) {
- CPC_ERR("copy_to_user %i %lu\n", err, sizeof(cachepc_event));
+ //CPC_DBG("Event Poll: Event is avail %px %llu %llu\n", arg_user,
+ // cachepc_last_event_sent, cachepc_last_event_acked);
+ if (copy_to_user(arg_user, &cachepc_event, sizeof(cachepc_event)))
err = -EFAULT;
- }
} else {
- CPC_DBG("Event Poll: Event was avail %llu %llu\n",
- cachepc_last_event_sent, cachepc_last_event_acked);
+ //CPC_DBG("Event Poll: Event was avail %llu %llu\n",
+ // cachepc_last_event_sent, cachepc_last_event_acked);
err = -EAGAIN;
}
if (!err) cachepc_event_avail = false;
diff --git a/cachepc/event.h b/cachepc/event.h
@@ -15,7 +15,7 @@ int cachepc_send_track_step_event(struct list_head *list);
int cachepc_send_track_step_event_single(uint64_t gfn, uint32_t err, uint64_t retinst);
int cachepc_send_track_page_event(uint64_t gfn_prev, uint64_t gfn, uint64_t retinst);
-bool cachepc_event_is_done(uint64_t id);
+bool cachepc_event_is_done(void);
int cachepc_poll_event_ioctl(void __user *arg_user);
int cachepc_ack_event_ioctl(void __user *arg_user);
diff --git a/cachepc/kvm.c b/cachepc/kvm.c
@@ -71,10 +71,10 @@ EXPORT_SYMBOL(cachepc_track_end_gfn);
LIST_HEAD(cachepc_faults);
EXPORT_SYMBOL(cachepc_faults);
-struct cpc_track_exec cachepc_track_exec;
-bool cachepc_track_signalled_enable;
-EXPORT_SYMBOL(cachepc_track_exec);
-EXPORT_SYMBOL(cachepc_track_signalled_enable);
+struct cpc_track_pages cpc_track_pages;
+struct cpc_track_steps_signalled cpc_track_steps_signalled;
+EXPORT_SYMBOL(cpc_track_pages);
+EXPORT_SYMBOL(cpc_track_steps_signalled);
struct cacheline *cachepc_ds_ul = NULL;
struct cacheline *cachepc_ds = NULL;
@@ -286,6 +286,8 @@ cachepc_kvm_reset_ioctl(void __user *arg_user)
cachepc_apic_oneshot = false;
cachepc_apic_timer = 0;
+ cachepc_prime_probe = false;
+
cachepc_retinst = 0;
cachepc_rip_prev_set = false;
@@ -430,6 +432,8 @@ cachepc_kvm_reset_tracking_ioctl(void __user *arg_user)
cachepc_singlestep = false;
cachepc_singlestep_reset = false;
+ cachepc_long_step = false;
+
cachepc_track_mode = CPC_TRACK_NONE;
list_for_each_entry_safe(fault, next, &cachepc_faults, list) {
@@ -460,6 +464,7 @@ cachepc_kvm_track_mode_ioctl(void __user *arg_user)
cachepc_untrack_all(vcpu, KVM_PAGE_TRACK_ACCESS);
cachepc_untrack_all(vcpu, KVM_PAGE_TRACK_WRITE);
+ cachepc_apic_timer = 0;
cachepc_apic_oneshot = false;
cachepc_prime_probe = false;
cachepc_singlestep = false;
@@ -476,17 +481,18 @@ cachepc_kvm_track_mode_ioctl(void __user *arg_user)
cachepc_long_step = true;
break;
case CPC_TRACK_PAGES:
- memset(&cachepc_track_exec, 0, sizeof(cachepc_track_exec));
+ case CPC_TRACK_PAGES_RESOLVE:
+ memset(&cpc_track_pages, 0, sizeof(cpc_track_pages));
cachepc_track_all(vcpu, KVM_PAGE_TRACK_EXEC);
- cachepc_singlestep_reset = true;
break;
- case CPC_TRACK_STEPS:
+ case CPC_TRACK_STEPS_AND_FAULTS:
cachepc_prime_probe = true;
cachepc_track_all(vcpu, KVM_PAGE_TRACK_ACCESS);
cachepc_singlestep_reset = true;
break;
case CPC_TRACK_STEPS_SIGNALLED:
- cachepc_track_signalled_enable = false;
+ memset(&cpc_track_steps_signalled, 0,
+ sizeof(cpc_track_steps_signalled));
break;
case CPC_TRACK_NONE:
break;
diff --git a/cachepc/uapi.h b/cachepc/uapi.h
@@ -53,7 +53,9 @@ enum {
CPC_TRACK_FAULT_NO_RUN,
CPC_TRACK_EXIT_EVICTIONS,
CPC_TRACK_PAGES,
+ CPC_TRACK_PAGES_RESOLVE,
CPC_TRACK_STEPS,
+ CPC_TRACK_STEPS_AND_FAULTS,
CPC_TRACK_STEPS_SIGNALLED,
};
diff --git a/test/kvm-eviction.c b/test/kvm-eviction.c
@@ -15,9 +15,6 @@
#define SAMPLE_COUNT 64
-#define TARGET_CORE 2
-#define SECONDARY_CORE 3
-
void
collect(struct kvm *kvm, uint8_t *counts)
{
diff --git a/test/kvm-pagestep.c b/test/kvm-pagestep.c
@@ -15,9 +15,6 @@
#include <stdio.h>
#include <stdlib.h>
-#define TARGET_CORE 2
-#define SECONDARY_CORE 3
-
static int child;
uint64_t
@@ -26,7 +23,6 @@ monitor(struct kvm *kvm, bool baseline)
struct cpc_event event;
int ret;
- /* Get page fault info */
ret = ioctl(kvm_dev, KVM_CPC_POLL_EVENT, &event);
if (ret && errno == EAGAIN) return 0;
if (ret) err(1, "KVM_CPC_POLL_EVENT");
diff --git a/test/kvm-step.c b/test/kvm-step.c
@@ -15,9 +15,6 @@
#include <stdio.h>
#include <stdlib.h>
-#define TARGET_CORE 2
-#define SECONDARY_CORE 3
-
static int child;
static struct cpc_event event;
@@ -123,7 +120,7 @@ main(int argc, const char **argv)
printf("Monitor start\n");
/* single step and log all accessed pages */
- arg = CPC_TRACK_STEPS;
+ arg = CPC_TRACK_STEPS_AND_FAULTS;
ret = ioctl(kvm_dev, KVM_CPC_TRACK_MODE, &arg);
if (ret) err(1, "KVM_CPC_TRACK_MODE");
diff --git a/test/qemu-eviction.c b/test/qemu-eviction.c
@@ -15,9 +15,6 @@
#include <stdio.h>
#include <stdlib.h>
-#define TARGET_CORE 2
-#define SECONDARY_CORE 3
-
static struct cpc_event event;
int
@@ -32,6 +29,7 @@ monitor(bool baseline)
switch (event.type) {
case CPC_EVENT_GUEST:
+ printf("Guest event: %i\n", event.guest.type);
if (event.guest.type == CPC_GUEST_STOP_TRACK)
return 2;
break;
@@ -40,7 +38,7 @@ monitor(bool baseline)
if (ret) err(1, "KVM_CPC_READ_COUNTS");
printf("Event: rip:%016llx cnt:%llu "
- "inst:%08llu data:%08llx ret:%llu\n",
+ "inst:%08llx data:%08llx ret:%llu\n",
vm_get_rip(), event.step.fault_count,
event.step.fault_gfns[0], event.step.fault_gfns[1],
event.step.retinst);
@@ -74,12 +72,8 @@ main(int argc, const char **argv)
uint8_t baseline[L1_SETS];
uint32_t eventcnt;
uint32_t arg;
- pid_t qemu;
int ret;
- qemu = pgrep("qemu-system-x86_64");
- if (!qemu) errx(1, "pgrep failed");
-
pin_process(0, SECONDARY_CORE, true);
setvbuf(stdout, NULL, _IONBF, 0);
@@ -93,7 +87,7 @@ main(int argc, const char **argv)
ret = ioctl(kvm_dev, KVM_CPC_CALC_BASELINE, &arg);
if (ret) err(1, "KVM_CPC_CALC_BASELINE");
- arg = CPC_TRACK_STEPS;
+ arg = CPC_TRACK_STEPS_AND_FAULTS;
ret = ioctl(kvm_dev, KVM_CPC_TRACK_MODE, &arg);
if (ret) err(1, "KVM_CPC_RESET");
diff --git a/test/qemu-eviction_guest.c b/test/qemu-eviction_guest.c
@@ -25,6 +25,19 @@ main(int argc, const char **argv)
printf("LOOP\n");
CPC_DO_VMMCALL(KVM_HC_CPC_VMMCALL_SIGNAL,
CPC_GUEST_START_TRACK, 0);
+ *(uint8_t *)(buf + L1_LINESIZE * 9) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 10) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 11) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 12) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 13) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 14) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 15) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 9) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 10) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 11) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 12) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 13) = 1;
+ *(uint8_t *)(buf + L1_LINESIZE * 14) = 1;
*(uint8_t *)(buf + L1_LINESIZE * 15) = 1;
CPC_DO_VMMCALL(KVM_HC_CPC_VMMCALL_SIGNAL,
CPC_GUEST_STOP_TRACK, 0);
diff --git a/test/qemu-pagestep b/test/qemu-pagestep
Binary files differ.
diff --git a/test/qemu-pagestep.c b/test/qemu-pagestep.c
@@ -0,0 +1,63 @@
+#include "test/kvm-eviction.h"
+#include "test/kvm.h"
+#include "test/util.h"
+#include "cachepc/uapi.h"
+
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <signal.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <err.h>
+#include <string.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+void
+monitor(void)
+{
+ struct cpc_event event;
+ int ret;
+
+ ret = ioctl(kvm_dev, KVM_CPC_POLL_EVENT, &event);
+ if (ret && errno == EAGAIN) return;
+ if (ret) err(1, "KVM_CPC_POLL_EVENT");
+
+ if (event.type != CPC_EVENT_TRACK_PAGE)
+ errx(1, "unexpected event type %i", event.type);
+
+ printf("Event: rip:%016llx prev:%08llx next:%08llx ret:%llu\n",
+ vm_get_rip(), event.page.inst_gfn_prev,
+ event.page.inst_gfn, event.page.retinst);
+ printf("\n");
+
+ ret = ioctl(kvm_dev, KVM_CPC_ACK_EVENT, &event.id);
+ if (ret) err(1, "KVM_CPC_ACK_EVENT");
+}
+
+int
+main(int argc, const char **argv)
+{
+ uint32_t arg;
+ int ret;
+
+ setvbuf(stdout, NULL, _IONBF, 0);
+
+ kvm_setup_init();
+
+ pin_process(0, SECONDARY_CORE, true);
+
+ ret = ioctl(kvm_dev, KVM_CPC_RESET);
+ if (ret) err(1, "KVM_CPC_RESET");
+
+ arg = CPC_TRACK_PAGES_RESOLVE;
+ ret = ioctl(kvm_dev, KVM_CPC_TRACK_MODE, &arg);
+ if (ret) err(1, "KVM_CPC_TRACK_MODE");
+
+ while (1) monitor();
+
+ kvm_setup_deinit();
+}
+
diff --git a/test/util.h b/test/util.h
@@ -7,6 +7,9 @@
#define ARRLEN(x) (sizeof(x) / sizeof((x)[0]))
#define MIN(a,b) ((a) > (b) ? (b) : (a))
+#define TARGET_CORE 2
+#define SECONDARY_CORE 3
+
struct ipc {
pthread_mutex_t lock;
pthread_cond_t sig_parent;
