commit d4ca1bb9a12e64e8b0a09f7a4316eda628f4edf1
parent 24d7c448e49e3dc2abbf3bc804247fb30410775a
Author: Louis Burda <quent.burda@gmail.com>
Date: Tue, 6 Sep 2022 14:30:25 +0200
Improve register saving and restoring for prime & probe around vmrun
Diffstat:
10 files changed, 91 insertions(+), 99 deletions(-)
diff --git a/kmod/asm.h b/kmod/asm.h
diff --git a/kmod/cache_types.h b/kmod/cache_types.h
diff --git a/kmod/cachepc.c b/kmod/cachepc.c
@@ -16,8 +16,6 @@ static void build_randomized_list_for_cache_set(cache_ctx *ctx, cacheline **cach
static cacheline **allocate_cache_ds(cache_ctx *ctx);
static uint16_t get_virt_cache_set(cache_ctx *ctx, void *ptr);
-cacheline *cachepc_prime_cl = NULL;
-
void
cachepc_prime_vcall(uintptr_t ret, cacheline *cl)
{
diff --git a/kmod/cachepc.h b/kmod/cachepc.h
@@ -45,6 +45,9 @@ extern size_t cachepc_msrmts_count;
extern cache_ctx *cachepc_ctx;
extern cacheline *cachepc_ds;
+extern uint64_t cachepc_regs_tmp[16];
+extern uint64_t cachepc_regs_vm[16];
+
/*
* Prime phase: fill the target cache (encoded in the size of the data structure)
* with the prepared data structure, i.e. with attacker data.
diff --git a/kmod/cachepc_user.h b/kmod/cachepc_user.h
diff --git a/kmod/device_conf.h b/kmod/device_conf.h
diff --git a/kmod/kvm.c b/kmod/kvm.c
@@ -18,6 +18,11 @@ cacheline *cachepc_ds;
EXPORT_SYMBOL(cachepc_ctx);
EXPORT_SYMBOL(cachepc_ds);
+uint64_t cachepc_regs_tmp[16];
+uint64_t cachepc_regs_vm[16];
+EXPORT_SYMBOL(cachepc_regs_tmp);
+EXPORT_SYMBOL(cachepc_regs_vm);
+
int
cachepc_kvm_proc_open(struct inode *inode, struct file *file)
{
diff --git a/kmod/util.c b/kmod/util.c
diff --git a/kmod/util.h b/kmod/util.h
diff --git a/patch.diff b/patch.diff
@@ -1,5 +1,5 @@
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
-index b804444e16d4..66a4d56e331a 100644
+index b804444e16d4..e94fa8c02a1d 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -1,6 +1,6 @@
@@ -32,7 +32,7 @@ index b804444e16d4..66a4d56e331a 100644
obj-$(CONFIG_KVM) += kvm.o
obj-$(CONFIG_KVM_INTEL) += kvm-intel.o
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
-index 7b3cfbe8f7e3..c7952eab7c6d 100644
+index 7b3cfbe8f7e3..241ce70885dc 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -2,6 +2,8 @@
@@ -44,24 +44,30 @@ index 7b3cfbe8f7e3..c7952eab7c6d 100644
#include "irq.h"
#include "mmu.h"
#include "kvm_cache_regs.h"
-@@ -3749,9 +3751,26 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
+@@ -3728,6 +3730,8 @@ void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs);
+ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
+ struct vcpu_svm *svm)
+ {
++ int cpu;
++
+ /*
+ * VMENTER enables interrupts (host state), but the kernel state is
+ * interrupts disabled when this is invoked. Also tell RCU about
+@@ -3749,9 +3753,23 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
lockdep_hardirqs_on(CALLER_ADDR0);
if (sev_es_guest(svm->vcpu.kvm)) {
+ memset(cachepc_msrmts, 0, 64 * 2);
-+ int cpu = get_cpu();
++ cpu = get_cpu();
+ local_irq_disable();
+ WARN_ON(cpu != 2);
-+ cacheline *next;
-+ next = cachepc_prime(cachepc_ds);
__svm_sev_es_vcpu_run(svm->vmcb_pa);
-+ cachepc_probe(next);
+ cachepc_save_msrmts(cachepc_ds);
+ local_irq_enable();
+ put_cpu();
} else {
+ memset(cachepc_msrmts, 0, 64 * 2);
-+ int cpu = get_cpu();
++ cpu = get_cpu();
+ local_irq_disable();
+ WARN_ON(cpu != 2);
__svm_vcpu_run(svm->vmcb_pa, (unsigned long *)&svm->vcpu.arch.regs);
@@ -71,21 +77,21 @@ index 7b3cfbe8f7e3..c7952eab7c6d 100644
#ifdef CONFIG_X86_64
native_wrmsrl(MSR_GS_BASE, svm->host.gs_base);
-@@ -3785,8 +3804,12 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
+@@ -3785,8 +3803,12 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu,
static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
{
- struct vcpu_svm *svm = to_svm(vcpu);
+ struct vcpu_svm *svm;
-
++
+ printk(KERN_WARNING "CachePC: svm_cpu_enter_exit()\n");
+ WARN_ON(smp_processor_id() != 2);
-+
+
+ svm = to_svm(vcpu);
svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
-@@ -3888,7 +3911,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
+@@ -3888,7 +3910,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu)
svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
vmcb_mark_all_clean(svm->vmcb);
@@ -95,10 +101,10 @@ index 7b3cfbe8f7e3..c7952eab7c6d 100644
if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
svm->vcpu.arch.apf.host_apf_flags =
diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
-index 6feb8c08f45a..eb0ea02ef187 100644
+index 6feb8c08f45a..60da3cff6c49 100644
--- a/arch/x86/kvm/svm/vmenter.S
+++ b/arch/x86/kvm/svm/vmenter.S
-@@ -27,14 +27,74 @@
+@@ -27,14 +27,59 @@
#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE
#endif
@@ -106,54 +112,39 @@ index 6feb8c08f45a..eb0ea02ef187 100644
+
.section .noinstr.text, "ax"
-+.macro push_xmm gpr xmm
-+ vmovq \gpr, \xmm
++.macro load_tmp off reg
++ mov cachepc_regs_tmp(off), reg
+.endm
+
-+.macro pop_xmm gpr xmm
-+ vmovq \xmm, \gpr
++.macro save_tmp off reg
++ mov reg, cachepc_regs_tmp(off)
+.endm
+
-+.macro swap_xmm grp xmm
-+ vmovq \grp, %xmm15
-+ vmovq \xmm, \grp
-+ vmovq %xmm15, \xmm
++.macro load_vm off reg
++ mov cachepc_regs_vm(off), reg
+.endm
+
-+.macro push_all
-+ push_xmm %rax, %xmm0
-+ push_xmm %rbx, %xmm1
-+ push_xmm %rcx, %xmm2
-+ push_xmm %rdx, %xmm3
-+ push_xmm %rbp, %xmm4
-+ push_xmm %rsp, %xmm5
-+ push_xmm %rdi, %xmm6
-+ push_xmm %rsi, %xmm7
-+ push_xmm %r8, %xmm8
-+ push_xmm %r9, %xmm9
-+ push_xmm %r10, %xmm10
-+ push_xmm %r11, %xmm11
-+ push_xmm %r12, %xmm12
-+ push_xmm %r13, %xmm13
-+ push_xmm %r14, %xmm14
++.macro save_vm off reg
++ mov reg, cachepc_regs_vm(off)
+.endm
+
-+.macro swap_all
-+ swap_xmm %rax, %xmm0
-+ swap_xmm %rbx, %xmm1
-+ swap_xmm %rcx, %xmm2
-+ swap_xmm %rdx, %xmm3
-+ swap_xmm %rbp, %xmm4
-+ swap_xmm %rsp, %xmm5
-+ swap_xmm %rdi, %xmm6
-+ swap_xmm %rsi, %xmm7
-+ swap_xmm %r8, %xmm8
-+ swap_xmm %r9, %xmm9
-+ swap_xmm %r10, %xmm10
-+ swap_xmm %r11, %xmm11
-+ swap_xmm %r12, %xmm12
-+ swap_xmm %r13, %xmm13
-+ swap_xmm %r14, %xmm14
++.macro apply_regs func
++ func 0x00, %rax
++ func 0x08, %rbx
++ func 0x10, %rcx
++ func 0x18, %rdx
++ func 0x20, %rbp
++ func 0x28, %rsp
++ func 0x30, %rdi
++ func 0x38, %rsi
++ func 0x40, %r8
++ func 0x48, %r9
++ func 0x50, %r10
++ func 0x58, %r11
++ func 0x60, %r12
++ func 0x68, %r13
++ func 0x70, %r14
++ func 0x78, %r15
+.endm
+
+.macro barrier
@@ -168,12 +159,12 @@ index 6feb8c08f45a..eb0ea02ef187 100644
* @regs: unsigned long * (to guest registers)
*/
SYM_FUNC_START(__svm_vcpu_run)
-+ push_all
++ apply_regs save_tmp
+
push %_ASM_BP
#ifdef CONFIG_X86_64
push %r15
-@@ -45,6 +105,7 @@ SYM_FUNC_START(__svm_vcpu_run)
+@@ -45,6 +90,7 @@ SYM_FUNC_START(__svm_vcpu_run)
push %edi
push %esi
#endif
@@ -181,43 +172,37 @@ index 6feb8c08f45a..eb0ea02ef187 100644
push %_ASM_BX
/* Save @regs. */
-@@ -85,8 +146,25 @@ SYM_FUNC_START(__svm_vcpu_run)
- jne 3f
+@@ -86,7 +132,28 @@ SYM_FUNC_START(__svm_vcpu_run)
ud2
_ASM_EXTABLE(1b, 2b)
-+
+
+-3: vmrun %_ASM_AX
+3:
-+ swap_all
++ apply_regs save_vm
++ apply_regs load_tmp
+ mov cachepc_ds, %rsi
+ mov 0x8(%rsi), %r15
+ lea sev_prime_ret(%rip), %rdi
+ jmp cachepc_prime_vcall+1 // skip stack pushes
+sev_prime_ret:
-+ swap_all
++ apply_regs save_tmp
++ apply_regs load_vm
+
+ vmrun %_ASM_AX
+
-+ swap_all
++ apply_regs save_vm
++ apply_regs load_tmp
+ mov %r15, %rsi
+ lea sev_probe_ret(%rip), %rdi
+ jmp cachepc_probe_vcall+6 // skip stack pushs
+sev_probe_ret:
-+ swap_all
-
--3: vmrun %_ASM_AX
++ apply_regs save_tmp
++ apply_regs load_vm
++
jmp 5f
4: cmpb $0, kvm_rebooting
jne 5f
-@@ -100,7 +178,7 @@ SYM_FUNC_START(__svm_vcpu_run)
- ud2
- _ASM_EXTABLE(5b, 6b)
- 7:
-- cli
-+ cli
-
- #ifdef CONFIG_RETPOLINE
- /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
-@@ -166,6 +244,11 @@ SYM_FUNC_START(__svm_vcpu_run)
+@@ -166,6 +233,11 @@ SYM_FUNC_START(__svm_vcpu_run)
pop %edi
#endif
pop %_ASM_BP
@@ -229,47 +214,48 @@ index 6feb8c08f45a..eb0ea02ef187 100644
ret
SYM_FUNC_END(__svm_vcpu_run)
-@@ -174,6 +257,8 @@ SYM_FUNC_END(__svm_vcpu_run)
+@@ -174,6 +246,8 @@ SYM_FUNC_END(__svm_vcpu_run)
* @vmcb_pa: unsigned long
*/
SYM_FUNC_START(__svm_sev_es_vcpu_run)
-+ push_all
++ apply_regs save_tmp
+
push %_ASM_BP
#ifdef CONFIG_X86_64
push %r15
-@@ -190,7 +275,28 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
+@@ -190,7 +264,29 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
mov %_ASM_ARG1, %_ASM_AX
sti
-1: vmrun %_ASM_AX
+1:
+
-+// swap_all
-+// mov cachepc_ds, %rsi
-+// mov 0x8(%rsi), %r15
-+// lea sev_es_prime_ret(%rip), %rdi
-+// jmp cachepc_prime_vcall+1 // skip stack pushes
-+//sev_es_prime_ret:
-+// swap_all
-+
-+// // TEST r15 dependance
-+// movq $0x41414141, %r15
++ apply_regs save_vm
++ apply_regs load_tmp
++ mov cachepc_ds, %rsi
++ mov 0x8(%rsi), %r15
++ lea sev_es_prime_ret(%rip), %rdi
++ jmp cachepc_prime_vcall+1 // skip stack pushes
++sev_es_prime_ret:
++ apply_regs save_tmp
++ apply_regs load_vm
+
+ vmrun %_ASM_AX
+
-+// swap_all
-+// mov %r15, %rsi
-+// lea sev_es_probe_ret(%rip), %rdi
-+// jmp cachepc_probe_vcall+6 // skip stack pushs
-+//sev_es_probe_ret:
-+// swap_all
++ apply_regs save_vm
++ apply_regs load_tmp
++ mov %r15, %rsi
++ lea sev_es_probe_ret(%rip), %rdi
++ jmp cachepc_probe_vcall+6 // skip stack pushs
++sev_es_probe_ret:
++ apply_regs save_tmp
++ apply_regs load_vm
+
jmp 3f
2: cmpb $0, kvm_rebooting
jne 3f
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
-index 2541a17ff1c4..8796ad5e9b73 100644
+index 2541a17ff1c4..1345938d1d2b 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -66,6 +66,8 @@
@@ -281,7 +267,7 @@ index 2541a17ff1c4..8796ad5e9b73 100644
MODULE_AUTHOR("Qumranet");
MODULE_LICENSE("GPL");
-@@ -4848,6 +4849,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
+@@ -4848,6 +4850,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
r = kvm_vfio_ops_init();
WARN_ON(r);
@@ -290,7 +276,7 @@ index 2541a17ff1c4..8796ad5e9b73 100644
return 0;
out_unreg:
-@@ -4872,6 +4875,8 @@ EXPORT_SYMBOL_GPL(kvm_init);
+@@ -4872,6 +4876,8 @@ EXPORT_SYMBOL_GPL(kvm_init);
void kvm_exit(void)
{
