cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

patching.c (3366B)

      1// SPDX-License-Identifier: GPL-2.0-only
      2#include <linux/kernel.h>
      3#include <linux/mm.h>
      4#include <linux/smp.h>
      5#include <linux/spinlock.h>
      6#include <linux/stop_machine.h>
      7#include <linux/uaccess.h>
      8
      9#include <asm/cacheflush.h>
     10#include <asm/fixmap.h>
     11#include <asm/insn.h>
     12#include <asm/kprobes.h>
     13#include <asm/patching.h>
     14#include <asm/sections.h>
     15
     16static DEFINE_RAW_SPINLOCK(patch_lock);
     17
     18static bool is_exit_text(unsigned long addr)
     19{
     20	/* discarded with init text/data */
     21	return system_state < SYSTEM_RUNNING &&
     22		addr >= (unsigned long)__exittext_begin &&
     23		addr < (unsigned long)__exittext_end;
     24}
     25
     26static bool is_image_text(unsigned long addr)
     27{
     28	return core_kernel_text(addr) || is_exit_text(addr);
     29}
     30
     31static void __kprobes *patch_map(void *addr, int fixmap)
     32{
     33	unsigned long uintaddr = (uintptr_t) addr;
     34	bool image = is_image_text(uintaddr);
     35	struct page *page;
     36
     37	if (image)
     38		page = phys_to_page(__pa_symbol(addr));
     39	else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
     40		page = vmalloc_to_page(addr);
     41	else
     42		return addr;
     43
     44	BUG_ON(!page);
     45	return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
     46			(uintaddr & ~PAGE_MASK));
     47}
     48
     49static void __kprobes patch_unmap(int fixmap)
     50{
     51	clear_fixmap(fixmap);
     52}
     53/*
     54 * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
     55 * little-endian.
     56 */
     57int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
     58{
     59	int ret;
     60	__le32 val;
     61
     62	ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE);
     63	if (!ret)
     64		*insnp = le32_to_cpu(val);
     65
     66	return ret;
     67}
     68
     69static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
     70{
     71	void *waddr = addr;
     72	unsigned long flags = 0;
     73	int ret;
     74
     75	raw_spin_lock_irqsave(&patch_lock, flags);
     76	waddr = patch_map(addr, FIX_TEXT_POKE0);
     77
     78	ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
     79
     80	patch_unmap(FIX_TEXT_POKE0);
     81	raw_spin_unlock_irqrestore(&patch_lock, flags);
     82
     83	return ret;
     84}
     85
     86int __kprobes aarch64_insn_write(void *addr, u32 insn)
     87{
     88	return __aarch64_insn_write(addr, cpu_to_le32(insn));
     89}
     90
     91int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
     92{
     93	u32 *tp = addr;
     94	int ret;
     95
     96	/* A64 instructions must be word aligned */
     97	if ((uintptr_t)tp & 0x3)
     98		return -EINVAL;
     99
    100	ret = aarch64_insn_write(tp, insn);
    101	if (ret == 0)
    102		caches_clean_inval_pou((uintptr_t)tp,
    103				     (uintptr_t)tp + AARCH64_INSN_SIZE);
    104
    105	return ret;
    106}
    107
    108struct aarch64_insn_patch {
    109	void		**text_addrs;
    110	u32		*new_insns;
    111	int		insn_cnt;
    112	atomic_t	cpu_count;
    113};
    114
    115static int __kprobes aarch64_insn_patch_text_cb(void *arg)
    116{
    117	int i, ret = 0;
    118	struct aarch64_insn_patch *pp = arg;
    119
    120	/* The last CPU becomes master */
    121	if (atomic_inc_return(&pp->cpu_count) == num_online_cpus()) {
    122		for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
    123			ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
    124							     pp->new_insns[i]);
    125		/* Notify other processors with an additional increment. */
    126		atomic_inc(&pp->cpu_count);
    127	} else {
    128		while (atomic_read(&pp->cpu_count) <= num_online_cpus())
    129			cpu_relax();
    130		isb();
    131	}
    132
    133	return ret;
    134}
    135
    136int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
    137{
    138	struct aarch64_insn_patch patch = {
    139		.text_addrs = addrs,
    140		.new_insns = insns,
    141		.insn_cnt = cnt,
    142		.cpu_count = ATOMIC_INIT(0),
    143	};
    144
    145	if (cnt <= 0)
    146		return -EINVAL;
    147
    148	return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch,
    149				       cpu_online_mask);
    150}