cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

jump_label.c (3962B)

      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * jump label x86 support
      4 *
      5 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
      6 *
      7 */
      8#include <linux/jump_label.h>
      9#include <linux/memory.h>
     10#include <linux/uaccess.h>
     11#include <linux/module.h>
     12#include <linux/list.h>
     13#include <linux/jhash.h>
     14#include <linux/cpu.h>
     15#include <asm/kprobes.h>
     16#include <asm/alternative.h>
     17#include <asm/text-patching.h>
     18#include <asm/insn.h>
     19
     20int arch_jump_entry_size(struct jump_entry *entry)
     21{
     22	struct insn insn = {};
     23
     24	insn_decode_kernel(&insn, (void *)jump_entry_code(entry));
     25	BUG_ON(insn.length != 2 && insn.length != 5);
     26
     27	return insn.length;
     28}
     29
     30struct jump_label_patch {
     31	const void *code;
     32	int size;
     33};
     34
     35static struct jump_label_patch
     36__jump_label_patch(struct jump_entry *entry, enum jump_label_type type)
     37{
     38	const void *expect, *code, *nop;
     39	const void *addr, *dest;
     40	int size;
     41
     42	addr = (void *)jump_entry_code(entry);
     43	dest = (void *)jump_entry_target(entry);
     44
     45	size = arch_jump_entry_size(entry);
     46	switch (size) {
     47	case JMP8_INSN_SIZE:
     48		code = text_gen_insn(JMP8_INSN_OPCODE, addr, dest);
     49		nop = x86_nops[size];
     50		break;
     51
     52	case JMP32_INSN_SIZE:
     53		code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
     54		nop = x86_nops[size];
     55		break;
     56
     57	default: BUG();
     58	}
     59
     60	if (type == JUMP_LABEL_JMP)
     61		expect = nop;
     62	else
     63		expect = code;
     64
     65	if (memcmp(addr, expect, size)) {
     66		/*
     67		 * The location is not an op that we were expecting.
     68		 * Something went wrong. Crash the box, as something could be
     69		 * corrupting the kernel.
     70		 */
     71		pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph != %5ph)) size:%d type:%d\n",
     72				addr, addr, addr, expect, size, type);
     73		BUG();
     74	}
     75
     76	if (type == JUMP_LABEL_NOP)
     77		code = nop;
     78
     79	return (struct jump_label_patch){.code = code, .size = size};
     80}
     81
     82static __always_inline void
     83__jump_label_transform(struct jump_entry *entry,
     84		       enum jump_label_type type,
     85		       int init)
     86{
     87	const struct jump_label_patch jlp = __jump_label_patch(entry, type);
     88
     89	/*
     90	 * As long as only a single processor is running and the code is still
     91	 * not marked as RO, text_poke_early() can be used; Checking that
     92	 * system_state is SYSTEM_BOOTING guarantees it. It will be set to
     93	 * SYSTEM_SCHEDULING before other cores are awaken and before the
     94	 * code is write-protected.
     95	 *
     96	 * At the time the change is being done, just ignore whether we
     97	 * are doing nop -> jump or jump -> nop transition, and assume
     98	 * always nop being the 'currently valid' instruction
     99	 */
    100	if (init || system_state == SYSTEM_BOOTING) {
    101		text_poke_early((void *)jump_entry_code(entry), jlp.code, jlp.size);
    102		return;
    103	}
    104
    105	text_poke_bp((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
    106}
    107
    108static void __ref jump_label_transform(struct jump_entry *entry,
    109				       enum jump_label_type type,
    110				       int init)
    111{
    112	mutex_lock(&text_mutex);
    113	__jump_label_transform(entry, type, init);
    114	mutex_unlock(&text_mutex);
    115}
    116
    117void arch_jump_label_transform(struct jump_entry *entry,
    118			       enum jump_label_type type)
    119{
    120	jump_label_transform(entry, type, 0);
    121}
    122
    123bool arch_jump_label_transform_queue(struct jump_entry *entry,
    124				     enum jump_label_type type)
    125{
    126	struct jump_label_patch jlp;
    127
    128	if (system_state == SYSTEM_BOOTING) {
    129		/*
    130		 * Fallback to the non-batching mode.
    131		 */
    132		arch_jump_label_transform(entry, type);
    133		return true;
    134	}
    135
    136	mutex_lock(&text_mutex);
    137	jlp = __jump_label_patch(entry, type);
    138	text_poke_queue((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
    139	mutex_unlock(&text_mutex);
    140	return true;
    141}
    142
    143void arch_jump_label_transform_apply(void)
    144{
    145	mutex_lock(&text_mutex);
    146	text_poke_finish();
    147	mutex_unlock(&text_mutex);
    148}
    149
    150static enum {
    151	JL_STATE_START,
    152	JL_STATE_NO_UPDATE,
    153	JL_STATE_UPDATE,
    154} jlstate __initdata_or_module = JL_STATE_START;
    155
    156__init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
    157				      enum jump_label_type type)
    158{
    159	if (jlstate == JL_STATE_UPDATE)
    160		jump_label_transform(entry, type, 1);
    161}