cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

sync_core.h (3288B)

      1/* SPDX-License-Identifier: GPL-2.0 */
      2#ifndef _ASM_X86_SYNC_CORE_H
      3#define _ASM_X86_SYNC_CORE_H
      4
      5#include <linux/preempt.h>
      6#include <asm/processor.h>
      7#include <asm/cpufeature.h>
      8#include <asm/special_insns.h>
      9
     10#ifdef CONFIG_X86_32
     11static inline void iret_to_self(void)
     12{
     13	asm volatile (
     14		"pushfl\n\t"
     15		"pushl %%cs\n\t"
     16		"pushl $1f\n\t"
     17		"iret\n\t"
     18		"1:"
     19		: ASM_CALL_CONSTRAINT : : "memory");
     20}
     21#else
     22static inline void iret_to_self(void)
     23{
     24	unsigned int tmp;
     25
     26	asm volatile (
     27		"mov %%ss, %0\n\t"
     28		"pushq %q0\n\t"
     29		"pushq %%rsp\n\t"
     30		"addq $8, (%%rsp)\n\t"
     31		"pushfq\n\t"
     32		"mov %%cs, %0\n\t"
     33		"pushq %q0\n\t"
     34		"pushq $1f\n\t"
     35		"iretq\n\t"
     36		"1:"
     37		: "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
     38}
     39#endif /* CONFIG_X86_32 */
     40
     41/*
     42 * This function forces the icache and prefetched instruction stream to
     43 * catch up with reality in two very specific cases:
     44 *
     45 *  a) Text was modified using one virtual address and is about to be executed
     46 *     from the same physical page at a different virtual address.
     47 *
     48 *  b) Text was modified on a different CPU, may subsequently be
     49 *     executed on this CPU, and you want to make sure the new version
     50 *     gets executed.  This generally means you're calling this in an IPI.
     51 *
     52 * If you're calling this for a different reason, you're probably doing
     53 * it wrong.
     54 *
     55 * Like all of Linux's memory ordering operations, this is a
     56 * compiler barrier as well.
     57 */
     58static inline void sync_core(void)
     59{
     60	/*
     61	 * The SERIALIZE instruction is the most straightforward way to
     62	 * do this, but it is not universally available.
     63	 */
     64	if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
     65		serialize();
     66		return;
     67	}
     68
     69	/*
     70	 * For all other processors, there are quite a few ways to do this.
     71	 * IRET-to-self is nice because it works on every CPU, at any CPL
     72	 * (so it's compatible with paravirtualization), and it never exits
     73	 * to a hypervisor.  The only downsides are that it's a bit slow
     74	 * (it seems to be a bit more than 2x slower than the fastest
     75	 * options) and that it unmasks NMIs.  The "push %cs" is needed,
     76	 * because in paravirtual environments __KERNEL_CS may not be a
     77	 * valid CS value when we do IRET directly.
     78	 *
     79	 * In case NMI unmasking or performance ever becomes a problem,
     80	 * the next best option appears to be MOV-to-CR2 and an
     81	 * unconditional jump.  That sequence also works on all CPUs,
     82	 * but it will fault at CPL3 (i.e. Xen PV).
     83	 *
     84	 * CPUID is the conventional way, but it's nasty: it doesn't
     85	 * exist on some 486-like CPUs, and it usually exits to a
     86	 * hypervisor.
     87	 */
     88	iret_to_self();
     89}
     90
     91/*
     92 * Ensure that a core serializing instruction is issued before returning
     93 * to user-mode. x86 implements return to user-space through sysexit,
     94 * sysrel, and sysretq, which are not core serializing.
     95 */
     96static inline void sync_core_before_usermode(void)
     97{
     98	/* With PTI, we unconditionally serialize before running user code. */
     99	if (static_cpu_has(X86_FEATURE_PTI))
    100		return;
    101
    102	/*
    103	 * Even if we're in an interrupt, we might reschedule before returning,
    104	 * in which case we could switch to a different thread in the same mm
    105	 * and return using SYSRET or SYSEXIT.  Instead of trying to keep
    106	 * track of our need to sync the core, just sync right away.
    107	 */
    108	sync_core();
    109}
    110
    111#endif /* _ASM_X86_SYNC_CORE_H */