cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

mmu.c (6408B)


      1#define pr_fmt(fmt)  "Hyper-V: " fmt
      2
      3#include <linux/hyperv.h>
      4#include <linux/log2.h>
      5#include <linux/slab.h>
      6#include <linux/types.h>
      7
      8#include <asm/fpu/api.h>
      9#include <asm/mshyperv.h>
     10#include <asm/msr.h>
     11#include <asm/tlbflush.h>
     12#include <asm/tlb.h>
     13
     14#define CREATE_TRACE_POINTS
     15#include <asm/trace/hyperv.h>
     16
     17/* Each gva in gva_list encodes up to 4096 pages to flush */
     18#define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
     19
     20static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
     21				      const struct flush_tlb_info *info);
     22
     23/*
     24 * Fills in gva_list starting from offset. Returns the number of items added.
     25 */
     26static inline int fill_gva_list(u64 gva_list[], int offset,
     27				unsigned long start, unsigned long end)
     28{
     29	int gva_n = offset;
     30	unsigned long cur = start, diff;
     31
     32	do {
     33		diff = end > cur ? end - cur : 0;
     34
     35		gva_list[gva_n] = cur & PAGE_MASK;
     36		/*
     37		 * Lower 12 bits encode the number of additional
     38		 * pages to flush (in addition to the 'cur' page).
     39		 */
     40		if (diff >= HV_TLB_FLUSH_UNIT) {
     41			gva_list[gva_n] |= ~PAGE_MASK;
     42			cur += HV_TLB_FLUSH_UNIT;
     43		}  else if (diff) {
     44			gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
     45			cur = end;
     46		}
     47
     48		gva_n++;
     49
     50	} while (cur < end);
     51
     52	return gva_n - offset;
     53}
     54
     55static void hyperv_flush_tlb_multi(const struct cpumask *cpus,
     56				   const struct flush_tlb_info *info)
     57{
     58	int cpu, vcpu, gva_n, max_gvas;
     59	struct hv_tlb_flush **flush_pcpu;
     60	struct hv_tlb_flush *flush;
     61	u64 status;
     62	unsigned long flags;
     63
     64	trace_hyperv_mmu_flush_tlb_multi(cpus, info);
     65
     66	if (!hv_hypercall_pg)
     67		goto do_native;
     68
     69	local_irq_save(flags);
     70
     71	flush_pcpu = (struct hv_tlb_flush **)
     72		     this_cpu_ptr(hyperv_pcpu_input_arg);
     73
     74	flush = *flush_pcpu;
     75
     76	if (unlikely(!flush)) {
     77		local_irq_restore(flags);
     78		goto do_native;
     79	}
     80
     81	if (info->mm) {
     82		/*
     83		 * AddressSpace argument must match the CR3 with PCID bits
     84		 * stripped out.
     85		 */
     86		flush->address_space = virt_to_phys(info->mm->pgd);
     87		flush->address_space &= CR3_ADDR_MASK;
     88		flush->flags = 0;
     89	} else {
     90		flush->address_space = 0;
     91		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
     92	}
     93
     94	flush->processor_mask = 0;
     95	if (cpumask_equal(cpus, cpu_present_mask)) {
     96		flush->flags |= HV_FLUSH_ALL_PROCESSORS;
     97	} else {
     98		/*
     99		 * From the supplied CPU set we need to figure out if we can get
    100		 * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
    101		 * hypercalls. This is possible when the highest VP number in
    102		 * the set is < 64. As VP numbers are usually in ascending order
    103		 * and match Linux CPU ids, here is an optimization: we check
    104		 * the VP number for the highest bit in the supplied set first
    105		 * so we can quickly find out if using *_EX hypercalls is a
    106		 * must. We will also check all VP numbers when walking the
    107		 * supplied CPU set to remain correct in all cases.
    108		 */
    109		cpu = cpumask_last(cpus);
    110
    111		if (cpu < nr_cpumask_bits && hv_cpu_number_to_vp_number(cpu) >= 64)
    112			goto do_ex_hypercall;
    113
    114		for_each_cpu(cpu, cpus) {
    115			vcpu = hv_cpu_number_to_vp_number(cpu);
    116			if (vcpu == VP_INVAL) {
    117				local_irq_restore(flags);
    118				goto do_native;
    119			}
    120
    121			if (vcpu >= 64)
    122				goto do_ex_hypercall;
    123
    124			__set_bit(vcpu, (unsigned long *)
    125				  &flush->processor_mask);
    126		}
    127
    128		/* nothing to flush if 'processor_mask' ends up being empty */
    129		if (!flush->processor_mask) {
    130			local_irq_restore(flags);
    131			return;
    132		}
    133	}
    134
    135	/*
    136	 * We can flush not more than max_gvas with one hypercall. Flush the
    137	 * whole address space if we were asked to do more.
    138	 */
    139	max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
    140
    141	if (info->end == TLB_FLUSH_ALL) {
    142		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
    143		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
    144					 flush, NULL);
    145	} else if (info->end &&
    146		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
    147		status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
    148					 flush, NULL);
    149	} else {
    150		gva_n = fill_gva_list(flush->gva_list, 0,
    151				      info->start, info->end);
    152		status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
    153					     gva_n, 0, flush, NULL);
    154	}
    155	goto check_status;
    156
    157do_ex_hypercall:
    158	status = hyperv_flush_tlb_others_ex(cpus, info);
    159
    160check_status:
    161	local_irq_restore(flags);
    162
    163	if (hv_result_success(status))
    164		return;
    165do_native:
    166	native_flush_tlb_multi(cpus, info);
    167}
    168
    169static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
    170				      const struct flush_tlb_info *info)
    171{
    172	int nr_bank = 0, max_gvas, gva_n;
    173	struct hv_tlb_flush_ex **flush_pcpu;
    174	struct hv_tlb_flush_ex *flush;
    175	u64 status;
    176
    177	if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
    178		return HV_STATUS_INVALID_PARAMETER;
    179
    180	flush_pcpu = (struct hv_tlb_flush_ex **)
    181		     this_cpu_ptr(hyperv_pcpu_input_arg);
    182
    183	flush = *flush_pcpu;
    184
    185	if (info->mm) {
    186		/*
    187		 * AddressSpace argument must match the CR3 with PCID bits
    188		 * stripped out.
    189		 */
    190		flush->address_space = virt_to_phys(info->mm->pgd);
    191		flush->address_space &= CR3_ADDR_MASK;
    192		flush->flags = 0;
    193	} else {
    194		flush->address_space = 0;
    195		flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
    196	}
    197
    198	flush->hv_vp_set.valid_bank_mask = 0;
    199
    200	flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
    201	nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
    202	if (nr_bank < 0)
    203		return HV_STATUS_INVALID_PARAMETER;
    204
    205	/*
    206	 * We can flush not more than max_gvas with one hypercall. Flush the
    207	 * whole address space if we were asked to do more.
    208	 */
    209	max_gvas =
    210		(PAGE_SIZE - sizeof(*flush) - nr_bank *
    211		 sizeof(flush->hv_vp_set.bank_contents[0])) /
    212		sizeof(flush->gva_list[0]);
    213
    214	if (info->end == TLB_FLUSH_ALL) {
    215		flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
    216		status = hv_do_rep_hypercall(
    217			HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
    218			0, nr_bank, flush, NULL);
    219	} else if (info->end &&
    220		   ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
    221		status = hv_do_rep_hypercall(
    222			HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
    223			0, nr_bank, flush, NULL);
    224	} else {
    225		gva_n = fill_gva_list(flush->gva_list, nr_bank,
    226				      info->start, info->end);
    227		status = hv_do_rep_hypercall(
    228			HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
    229			gva_n, nr_bank, flush, NULL);
    230	}
    231
    232	return status;
    233}
    234
    235void hyperv_setup_mmu_ops(void)
    236{
    237	if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
    238		return;
    239
    240	pr_info("Using hypercall for remote TLB flush\n");
    241	pv_ops.mmu.flush_tlb_multi = hyperv_flush_tlb_multi;
    242	pv_ops.mmu.tlb_remove_table = tlb_remove_table;
    243}