cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

copy_mc.c (2453B)

      1// SPDX-License-Identifier: GPL-2.0
      2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
      3
      4#include <linux/jump_label.h>
      5#include <linux/uaccess.h>
      6#include <linux/export.h>
      7#include <linux/string.h>
      8#include <linux/types.h>
      9
     10#include <asm/mce.h>
     11
     12#ifdef CONFIG_X86_MCE
     13static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
     14
     15void enable_copy_mc_fragile(void)
     16{
     17	static_branch_inc(&copy_mc_fragile_key);
     18}
     19#define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
     20
     21/*
     22 * Similar to copy_user_handle_tail, probe for the write fault point, or
     23 * source exception point.
     24 */
     25__visible notrace unsigned long
     26copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
     27{
     28	for (; len; --len, to++, from++)
     29		if (copy_mc_fragile(to, from, 1))
     30			break;
     31	return len;
     32}
     33#else
     34/*
     35 * No point in doing careful copying, or consulting a static key when
     36 * there is no #MC handler in the CONFIG_X86_MCE=n case.
     37 */
     38void enable_copy_mc_fragile(void)
     39{
     40}
     41#define copy_mc_fragile_enabled (0)
     42#endif
     43
     44unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
     45
     46/**
     47 * copy_mc_to_kernel - memory copy that handles source exceptions
     48 *
     49 * @dst:	destination address
     50 * @src:	source address
     51 * @len:	number of bytes to copy
     52 *
     53 * Call into the 'fragile' version on systems that benefit from avoiding
     54 * corner case poison consumption scenarios, For example, accessing
     55 * poison across 2 cachelines with a single instruction. Almost all
     56 * other uses case can use copy_mc_enhanced_fast_string() for a fast
     57 * recoverable copy, or fallback to plain memcpy.
     58 *
     59 * Return 0 for success, or number of bytes not copied if there was an
     60 * exception.
     61 */
     62unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
     63{
     64	if (copy_mc_fragile_enabled)
     65		return copy_mc_fragile(dst, src, len);
     66	if (static_cpu_has(X86_FEATURE_ERMS))
     67		return copy_mc_enhanced_fast_string(dst, src, len);
     68	memcpy(dst, src, len);
     69	return 0;
     70}
     71EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
     72
     73unsigned long __must_check copy_mc_to_user(void *dst, const void *src, unsigned len)
     74{
     75	unsigned long ret;
     76
     77	if (copy_mc_fragile_enabled) {
     78		__uaccess_begin();
     79		ret = copy_mc_fragile(dst, src, len);
     80		__uaccess_end();
     81		return ret;
     82	}
     83
     84	if (static_cpu_has(X86_FEATURE_ERMS)) {
     85		__uaccess_begin();
     86		ret = copy_mc_enhanced_fast_string(dst, src, len);
     87		__uaccess_end();
     88		return ret;
     89	}
     90
     91	return copy_user_generic(dst, src, len);
     92}