cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

iomem.c (2763B)

      1#include <linux/string.h>
      2#include <linux/module.h>
      3#include <linux/io.h>
      4
      5#define movs(type,to,from) \
      6	asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory")
      7
      8/* Originally from i386/string.h */
      9static __always_inline void rep_movs(void *to, const void *from, size_t n)
     10{
     11	unsigned long d0, d1, d2;
     12	asm volatile("rep ; movsl\n\t"
     13		     "testb $2,%b4\n\t"
     14		     "je 1f\n\t"
     15		     "movsw\n"
     16		     "1:\ttestb $1,%b4\n\t"
     17		     "je 2f\n\t"
     18		     "movsb\n"
     19		     "2:"
     20		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
     21		     : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from)
     22		     : "memory");
     23}
     24
     25static void string_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
     26{
     27	if (unlikely(!n))
     28		return;
     29
     30	/* Align any unaligned source IO */
     31	if (unlikely(1 & (unsigned long)from)) {
     32		movs("b", to, from);
     33		n--;
     34	}
     35	if (n > 1 && unlikely(2 & (unsigned long)from)) {
     36		movs("w", to, from);
     37		n-=2;
     38	}
     39	rep_movs(to, (const void *)from, n);
     40}
     41
     42static void string_memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
     43{
     44	if (unlikely(!n))
     45		return;
     46
     47	/* Align any unaligned destination IO */
     48	if (unlikely(1 & (unsigned long)to)) {
     49		movs("b", to, from);
     50		n--;
     51	}
     52	if (n > 1 && unlikely(2 & (unsigned long)to)) {
     53		movs("w", to, from);
     54		n-=2;
     55	}
     56	rep_movs((void *)to, (const void *) from, n);
     57}
     58
     59static void unrolled_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
     60{
     61	const volatile char __iomem *in = from;
     62	char *out = to;
     63	int i;
     64
     65	for (i = 0; i < n; ++i)
     66		out[i] = readb(&in[i]);
     67}
     68
     69static void unrolled_memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
     70{
     71	volatile char __iomem *out = to;
     72	const char *in = from;
     73	int i;
     74
     75	for (i = 0; i < n; ++i)
     76		writeb(in[i], &out[i]);
     77}
     78
     79static void unrolled_memset_io(volatile void __iomem *a, int b, size_t c)
     80{
     81	volatile char __iomem *mem = a;
     82	int i;
     83
     84	for (i = 0; i < c; ++i)
     85		writeb(b, &mem[i]);
     86}
     87
     88void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
     89{
     90	if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO))
     91		unrolled_memcpy_fromio(to, from, n);
     92	else
     93		string_memcpy_fromio(to, from, n);
     94}
     95EXPORT_SYMBOL(memcpy_fromio);
     96
     97void memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
     98{
     99	if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO))
    100		unrolled_memcpy_toio(to, from, n);
    101	else
    102		string_memcpy_toio(to, from, n);
    103}
    104EXPORT_SYMBOL(memcpy_toio);
    105
    106void memset_io(volatile void __iomem *a, int b, size_t c)
    107{
    108	if (cc_platform_has(CC_ATTR_GUEST_UNROLL_STRING_IO)) {
    109		unrolled_memset_io(a, b, c);
    110	} else {
    111		/*
    112		 * TODO: memset can mangle the IO patterns quite a bit.
    113		 * perhaps it would be better to use a dumb one:
    114		 */
    115		memset((void *)a, b, c);
    116	}
    117}
    118EXPORT_SYMBOL(memset_io);