commit 9f603b90b8de3028791f8e487a3981ac8843c436
parent 27ac7a95b69d70622f281c1b8d0300d38e5c541d
Author: Louis Burda <quent.burda@gmail.com>
Date: Mon, 30 Jan 2023 13:01:00 +0100
Event batching
Diffstat:
4 files changed, 154 insertions(+), 83 deletions(-)
diff --git a/cachepc/event.c b/cachepc/event.c
@@ -13,6 +13,10 @@
#define ARRLEN(x) (sizeof(x)/sizeof((x)[0]))
+struct cpc_event *cpc_eventbuf;
+size_t cpc_eventbuf_len;
+bool cpc_event_batching;
+
uint64_t cpc_last_event_sent;
uint64_t cpc_last_event_acked;
rwlock_t cpc_event_lock;
@@ -20,13 +24,32 @@ rwlock_t cpc_event_lock;
struct cpc_event cpc_event;
bool cpc_event_avail;
-bool cpc_events_init;
+EXPORT_SYMBOL(cpc_send_guest_event);
+EXPORT_SYMBOL(cpc_send_pause_event);
+EXPORT_SYMBOL(cpc_send_track_step_event);
+EXPORT_SYMBOL(cpc_send_track_step_event_single);
+EXPORT_SYMBOL(cpc_send_track_page_event);
+
+void
+cpc_events_init(void)
+{
+ cpc_eventbuf = NULL;
+ cpc_eventbuf_len = 0;
+ cpc_event_batching = false;
+ cpc_events_reset();
+}
+
+void
+cpc_events_deinit(void)
+{
+ kfree(cpc_eventbuf);
+ cpc_eventbuf = NULL;
+}
void
cpc_events_reset(void)
{
write_lock(&cpc_event_lock);
- cpc_events_init = true;
cpc_last_event_sent = 1;
cpc_last_event_acked = 1;
cpc_event_avail = false;
@@ -38,28 +61,31 @@ cpc_send_event(struct cpc_event event)
{
ktime_t deadline;
- read_lock(&cpc_event_lock);
- if (!cpc_events_init) {
- CPC_WARN("events ctx not initialized!\n");
- read_unlock(&cpc_event_lock);
- return 1;
- }
- read_unlock(&cpc_event_lock);
-
write_lock(&cpc_event_lock);
if (cpc_last_event_sent != cpc_last_event_acked) {
CPC_WARN("event IDs out of sync\n");
write_unlock(&cpc_event_lock);
return 1;
+ }
+
+ if (cpc_event_batching) {
+ if (cpc_eventbuf_len < CPC_EVENTBUF_CAP) {
+ memcpy(&cpc_eventbuf[cpc_eventbuf_len], &event,
+ sizeof(struct cpc_event));
+ cpc_eventbuf_len++;
+ write_unlock(&cpc_event_lock);
+ return 0;
+ } else {
+ cpc_event.type = CPC_EVENT_BATCH;
+ }
} else {
- cpc_last_event_sent++;
+ cpc_event = event;
}
+ cpc_last_event_sent++;
event.id = cpc_last_event_sent;
cpc_event_avail = true;
- cpc_event = event;
- //CPC_DBG("Sent Event: id %llu\n", event.id);
write_unlock(&cpc_event_lock);
/* wait for ack with timeout */
@@ -86,7 +112,6 @@ cpc_send_guest_event(uint64_t type, uint64_t val)
return cpc_send_event(event);
}
-EXPORT_SYMBOL(cpc_send_guest_event);
int
cpc_send_pause_event(void)
@@ -97,7 +122,6 @@ cpc_send_pause_event(void)
return cpc_send_event(event);
}
-EXPORT_SYMBOL(cpc_send_pause_event);
int
cpc_send_track_step_event(struct list_head *list)
@@ -122,7 +146,6 @@ cpc_send_track_step_event(struct list_head *list)
return cpc_send_event(event);
}
-EXPORT_SYMBOL(cpc_send_track_step_event);
int
cpc_send_track_page_event(uint64_t gfn_prev, uint64_t gfn, uint64_t retinst)
@@ -136,7 +159,6 @@ cpc_send_track_page_event(uint64_t gfn_prev, uint64_t gfn, uint64_t retinst)
return cpc_send_event(event);
}
-EXPORT_SYMBOL(cpc_send_track_page_event);
int
cpc_send_track_step_event_single(uint64_t gfn, uint32_t err, uint64_t retinst)
@@ -152,7 +174,6 @@ cpc_send_track_step_event_single(uint64_t gfn, uint32_t err, uint64_t retinst)
return cpc_send_event(event);
}
-EXPORT_SYMBOL(cpc_send_track_step_event_single);
bool
cpc_event_is_done(void)
@@ -220,3 +241,34 @@ cpc_ack_event_ioctl(void __user *arg_user)
return err;
}
+int
+cpc_read_events_ioctl(void __user *arg_user)
+{
+ struct cpc_event_batch batch;
+ size_t n;
+
+ if (!arg_user) return -EINVAL;
+
+ if (copy_from_user(&batch, arg_user, sizeof(batch)))
+ return -EFAULT;
+
+ if (!batch.buf || !batch.maxcnt) return -EINVAL;
+
+ write_lock(&cpc_event_lock);
+ if (!cpc_eventbuf_len) {
+ write_unlock(&cpc_event_lock);
+ return -EAGAIN;
+ }
+
+ n = cpc_eventbuf_len;
+ if (batch.maxcnt < n)
+ n = batch.maxcnt;
+
+ if (copy_to_user(batch.buf, cpc_eventbuf, sizeof(struct cpc_event) * n)) {
+ write_unlock(&cpc_event_lock);
+ return -EFAULT;
+ }
+ write_unlock(&cpc_event_lock);
+
+ return 0;
+}
diff --git a/cachepc/event.h b/cachepc/event.h
@@ -7,6 +7,14 @@
#include <linux/kvm_host.h>
#include <linux/types.h>
+#define CPC_EVENTBUF_CAP 5000
+
+extern struct cpc_event *cpc_eventbuf;
+extern size_t cpc_eventbuf_len;
+extern bool cpc_event_batching;
+
+void cpc_event_init(void);
+void cpc_event_deinit(void);
void cpc_events_reset(void);
int cpc_send_guest_event(uint64_t type, uint64_t val);
@@ -19,3 +27,4 @@ bool cpc_event_is_done(void);
int cpc_poll_event_ioctl(void __user *arg_user);
int cpc_ack_event_ioctl(void __user *arg_user);
+int cpc_read_events_ioctl(void __user *arg_user);
diff --git a/cachepc/kvm.c b/cachepc/kvm.c
@@ -99,31 +99,31 @@ static noinline void cpc_stream_hwpf_test(void);
void cpc_single_eviction_test_asm(void *ptr);
static noinline void cpc_single_eviction_test(void *p);
-static void cpc_kvm_pmc_setup(void *p);
-static void cpc_kvm_system_setup(void);
+static void cpc_pmc_setup(void *p);
+static void cpc_system_setup(void);
-static int cpc_kvm_reset_ioctl(void __user *arg_user);
-static int cpc_kvm_debug_ioctl(void __user *arg_user);
+static int cpc_reset_ioctl(void __user *arg_user);
+static int cpc_debug_ioctl(void __user *arg_user);
-static int cpc_kvm_memory_encrypt_op_ioctl(void __user *arg_user);
+static int cpc_memory_encrypt_op_ioctl(void __user *arg_user);
-static int cpc_kvm_test_eviction_ioctl(void __user *arg_user);
+static int cpc_test_eviction_ioctl(void __user *arg_user);
-static int cpc_kvm_read_counts_ioctl(void __user *arg_user);
+static int cpc_read_counts_ioctl(void __user *arg_user);
-static int cpc_kvm_reset_baseline_ioctl(void __user *arg_user);
-static int cpc_kvm_calc_baseline_ioctl(void __user *arg_user);
-static int cpc_kvm_read_baseline_ioctl(void __user *arg_user);
-static int cpc_kvm_apply_baseline_ioctl(void __user *arg_user);
+static int cpc_reset_baseline_ioctl(void __user *arg_user);
+static int cpc_calc_baseline_ioctl(void __user *arg_user);
+static int cpc_read_baseline_ioctl(void __user *arg_user);
+static int cpc_apply_baseline_ioctl(void __user *arg_user);
-static int cpc_kvm_reset_tracking_ioctl(void __user *arg_user);
-static int cpc_kvm_track_mode_ioctl(void __user *arg_user);
-// static int cpc_kvm_track_page_ioctl(void __user *arg_user);
-// static int cpc_kvm_track_range_start_ioctl(void __user *arg_user);
-// static int cpc_kvm_track_range_end_ioctl(void __user *arg_user);
-// static int cpc_kvm_track_exec_cur_ioctl(void __user *arg_user);
+static int cpc_reset_tracking_ioctl(void __user *arg_user);
+static int cpc_track_mode_ioctl(void __user *arg_user);
+// static int cpc_track_page_ioctl(void __user *arg_user);
+// static int cpc_track_range_start_ioctl(void __user *arg_user);
+// static int cpc_track_range_end_ioctl(void __user *arg_user);
+// static int cpc_track_exec_cur_ioctl(void __user *arg_user);
-static int cpc_kvm_req_pause_ioctl(void __user *arg_user);
+static int cpc_req_pause_ioctl(void __user *arg_user);
void
cpc_prime_probe_test(void)
@@ -228,7 +228,7 @@ cpc_single_eviction_test(void *p)
}
void
-cpc_kvm_pmc_setup(void *p)
+cpc_pmc_setup(void *p)
{
/* L1 misses in host kernel */
cpc_init_pmc(CPC_L1MISS_PMC, 0x64, 0xD8,
@@ -240,7 +240,7 @@ cpc_kvm_pmc_setup(void *p)
}
void
-cpc_kvm_system_setup(void)
+cpc_system_setup(void)
{
/* NOTE: since most of these MSRs are poorly documented and some
* guessing work was involved, it is likely that one or more of
@@ -270,18 +270,18 @@ cpc_kvm_system_setup(void)
}
int
-cpc_kvm_reset_ioctl(void __user *arg_user)
+cpc_reset_ioctl(void __user *arg_user)
{
int ret;
ret = smp_call_function_single(CPC_ISOLCPU,
- cpc_kvm_pmc_setup, NULL, true);
+ cpc_pmc_setup, NULL, true);
if (ret) return -EFAULT;
cpc_events_reset();
- cpc_kvm_reset_tracking_ioctl(NULL);
- cpc_kvm_reset_baseline_ioctl(NULL);
+ cpc_reset_tracking_ioctl(NULL);
+ cpc_reset_baseline_ioctl(NULL);
cpc_pause_vm = false;
@@ -300,7 +300,7 @@ cpc_kvm_reset_ioctl(void __user *arg_user)
}
int
-cpc_kvm_debug_ioctl(void __user *arg_user)
+cpc_debug_ioctl(void __user *arg_user)
{
uint32_t debug;
@@ -315,7 +315,7 @@ cpc_kvm_debug_ioctl(void __user *arg_user)
}
int
-cpc_kvm_memory_encrypt_op_ioctl(void __user *arg_user)
+cpc_memory_encrypt_op_ioctl(void __user *arg_user)
{
if (!arg_user || !main_vm) return -EFAULT;
@@ -323,7 +323,7 @@ cpc_kvm_memory_encrypt_op_ioctl(void __user *arg_user)
}
int
-cpc_kvm_test_eviction_ioctl(void __user *arg_user)
+cpc_test_eviction_ioctl(void __user *arg_user)
{
uint32_t u32;
int ret;
@@ -344,7 +344,7 @@ cpc_kvm_test_eviction_ioctl(void __user *arg_user)
}
int
-cpc_kvm_read_counts_ioctl(void __user *arg_user)
+cpc_read_counts_ioctl(void __user *arg_user)
{
if (!arg_user) return -EINVAL;
@@ -355,7 +355,7 @@ cpc_kvm_read_counts_ioctl(void __user *arg_user)
}
int
-cpc_kvm_reset_baseline_ioctl(void __user *arg_user)
+cpc_reset_baseline_ioctl(void __user *arg_user)
{
if (arg_user) return -EINVAL;
@@ -367,7 +367,7 @@ cpc_kvm_reset_baseline_ioctl(void __user *arg_user)
}
int
-cpc_kvm_calc_baseline_ioctl(void __user *arg_user)
+cpc_calc_baseline_ioctl(void __user *arg_user)
{
uint32_t state;
@@ -382,7 +382,7 @@ cpc_kvm_calc_baseline_ioctl(void __user *arg_user)
}
int
-cpc_kvm_read_baseline_ioctl(void __user *arg_user)
+cpc_read_baseline_ioctl(void __user *arg_user)
{
if (!arg_user) return -EINVAL;
@@ -393,7 +393,7 @@ cpc_kvm_read_baseline_ioctl(void __user *arg_user)
}
int
-cpc_kvm_apply_baseline_ioctl(void __user *arg_user)
+cpc_apply_baseline_ioctl(void __user *arg_user)
{
uint32_t state;
@@ -408,7 +408,7 @@ cpc_kvm_apply_baseline_ioctl(void __user *arg_user)
}
int
-cpc_kvm_long_step_ioctl(void __user *arg_user)
+cpc_long_step_ioctl(void __user *arg_user)
{
if (arg_user) return -EINVAL;
@@ -418,7 +418,7 @@ cpc_kvm_long_step_ioctl(void __user *arg_user)
}
int
-cpc_kvm_reset_tracking_ioctl(void __user *arg_user)
+cpc_reset_tracking_ioctl(void __user *arg_user)
{
struct kvm_vcpu *vcpu;
struct cpc_fault *fault, *next;
@@ -450,7 +450,7 @@ cpc_kvm_reset_tracking_ioctl(void __user *arg_user)
}
int
-cpc_kvm_track_mode_ioctl(void __user *arg_user)
+cpc_track_mode_ioctl(void __user *arg_user)
{
struct kvm_vcpu *vcpu;
uint32_t mode;
@@ -513,7 +513,7 @@ cpc_kvm_track_mode_ioctl(void __user *arg_user)
}
// int
-// cpc_kvm_track_page_ioctl(void __user *arg_user)
+// cpc_track_page_ioctl(void __user *arg_user)
// {
// struct cpc_track_config cfg;
// struct kvm_vcpu *vcpu;
@@ -535,7 +535,7 @@ cpc_kvm_track_mode_ioctl(void __user *arg_user)
// }
//
// int
-// cpc_kvm_track_range_start_ioctl(void __user *arg_user)
+// cpc_track_range_start_ioctl(void __user *arg_user)
// {
// if (!arg_user) return -EINVAL;
//
@@ -546,7 +546,7 @@ cpc_kvm_track_mode_ioctl(void __user *arg_user)
// }
//
// int
-// cpc_kvm_track_range_end_ioctl(void __user *arg_user)
+// cpc_track_range_end_ioctl(void __user *arg_user)
// {
// if (!arg_user) return -EINVAL;
//
@@ -557,7 +557,7 @@ cpc_kvm_track_mode_ioctl(void __user *arg_user)
// }
//
// int
-// cpc_kvm_track_exec_cur_ioctl(void __user *arg_user)
+// cpc_track_exec_cur_ioctl(void __user *arg_user)
// {
// struct cpc_fault *fault;
//
@@ -573,7 +573,7 @@ cpc_kvm_track_mode_ioctl(void __user *arg_user)
// }
int
-cpc_kvm_req_pause_ioctl(void __user *arg_user)
+cpc_req_pause_ioctl(void __user *arg_user)
{
if (arg_user) return -EINVAL;
@@ -583,57 +583,59 @@ cpc_kvm_req_pause_ioctl(void __user *arg_user)
}
long
-cpc_kvm_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
+cpc_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
{
void __user *arg_user;
arg_user = (void __user *)arg;
switch (ioctl) {
case KVM_CPC_RESET:
- return cpc_kvm_reset_ioctl(arg_user);
+ return cpc_reset_ioctl(arg_user);
case KVM_CPC_DEBUG:
- return cpc_kvm_debug_ioctl(arg_user);
+ return cpc_debug_ioctl(arg_user);
case KVM_CPC_MEMORY_ENCRYPT_OP:
- return cpc_kvm_memory_encrypt_op_ioctl(arg_user);
+ return cpc_memory_encrypt_op_ioctl(arg_user);
case KVM_CPC_TEST_EVICTION:
- return cpc_kvm_test_eviction_ioctl(arg_user);
+ return cpc_test_eviction_ioctl(arg_user);
case KVM_CPC_READ_COUNTS:
- return cpc_kvm_read_counts_ioctl(arg_user);
+ return cpc_read_counts_ioctl(arg_user);
case KVM_CPC_RESET_BASELINE:
- return cpc_kvm_reset_baseline_ioctl(arg_user);
+ return cpc_reset_baseline_ioctl(arg_user);
case KVM_CPC_READ_BASELINE:
- return cpc_kvm_read_baseline_ioctl(arg_user);
+ return cpc_read_baseline_ioctl(arg_user);
case KVM_CPC_CALC_BASELINE:
- return cpc_kvm_calc_baseline_ioctl(arg_user);
+ return cpc_calc_baseline_ioctl(arg_user);
case KVM_CPC_APPLY_BASELINE:
- return cpc_kvm_apply_baseline_ioctl(arg_user);
+ return cpc_apply_baseline_ioctl(arg_user);
case KVM_CPC_LONG_STEP:
- return cpc_kvm_long_step_ioctl(arg_user);
+ return cpc_long_step_ioctl(arg_user);
case KVM_CPC_RESET_TRACKING:
- return cpc_kvm_reset_tracking_ioctl(arg_user);
+ return cpc_reset_tracking_ioctl(arg_user);
case KVM_CPC_TRACK_MODE:
- return cpc_kvm_track_mode_ioctl(arg_user);
+ return cpc_track_mode_ioctl(arg_user);
case KVM_CPC_POLL_EVENT:
return cpc_poll_event_ioctl(arg_user);
case KVM_CPC_ACK_EVENT:
return cpc_ack_event_ioctl(arg_user);
+ case KVM_CPC_READ_EVENTS:
+ return cpc_read_events_ioctl(arg_user);
// case KVM_CPC_TRACK_PAGE:
- // return cpc_kvm_track_page_ioctl(arg_user);
+ // return cpc_track_page_ioctl(arg_user);
// case KVM_CPC_TRACK_RANGE_START:
- // return cpc_kvm_track_range_start_ioctl(arg_user);
+ // return cpc_track_range_start_ioctl(arg_user);
// case KVM_CPC_TRACK_RANGE_END:
- // return cpc_kvm_track_range_end_ioctl(arg_user);
+ // return cpc_track_range_end_ioctl(arg_user);
// case KVM_CPC_TRACK_EXEC_CUR:
- // return cpc_kvm_track_exec_cur_ioctl(arg_user);
+ // return cpc_track_exec_cur_ioctl(arg_user);
case KVM_CPC_VM_REQ_PAUSE:
- return cpc_kvm_req_pause_ioctl(arg_user);
+ return cpc_req_pause_ioctl(arg_user);
default:
return kvm_arch_dev_ioctl(file, ioctl, arg);
}
}
void
-cpc_kvm_setup_test(void *p)
+cpc_setup_test(void *p)
{
spinlock_t lock;
int cpu;
@@ -647,7 +649,7 @@ cpc_kvm_setup_test(void *p)
cpc_ds = cpc_ds_alloc(&cpc_ds_ul);
- cpc_kvm_system_setup();
+ cpc_system_setup();
spin_lock_irq(&lock);
cpc_prime_probe_test();
@@ -692,7 +694,7 @@ cpc_kvm_init(void)
cpc_events_reset();
ret = smp_call_function_single(CPC_ISOLCPU,
- cpc_kvm_setup_test, NULL, true);
+ cpc_setup_test, NULL, true);
WARN_ON(ret != 0);
}
diff --git a/cachepc/uapi.h b/cachepc/uapi.h
@@ -25,11 +25,12 @@
#define KVM_CPC_LONG_STEP _IO(KVMIO, 0x2A)
-#define KVM_CPC_TRACK_MODE _IOWR(KVMIO, 0x40, __u32)
-#define KVM_CPC_RESET_TRACKING _IO(KVMIO, 0x44)
+#define KVM_CPC_TRACK_MODE _IOWR(KVMIO, 0x30, __u32)
+#define KVM_CPC_RESET_TRACKING _IO(KVMIO, 0x31)
-#define KVM_CPC_POLL_EVENT _IOWR(KVMIO, 0x48, struct cpc_event)
-#define KVM_CPC_ACK_EVENT _IOWR(KVMIO, 0x49, __u64)
+#define KVM_CPC_POLL_EVENT _IOWR(KVMIO, 0x40, struct cpc_event)
+#define KVM_CPC_ACK_EVENT _IOWR(KVMIO, 0x41, __u64)
+#define KVM_CPC_READ_EVENTS _IOWR(KVMIO, 0x42, struct cpc_event_batch)
#define KVM_CPC_VM_REQ_PAUSE _IO(KVMIO, 0x50)
@@ -41,6 +42,7 @@ enum {
CPC_EVENT_TRACK_PAGE,
CPC_EVENT_PAUSE,
CPC_EVENT_GUEST,
+ CPC_EVENT_BATCH
};
enum {
@@ -96,6 +98,12 @@ struct cpc_event {
};
};
+struct cpc_event_batch {
+ __u64 cnt;
+ __u64 maxcnt;
+ struct cpc_event *buf;
+};
+
struct cpc_sev_cmd {
__u32 id;
__u64 data;
