cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

barrier.h (4306B)

      1/* SPDX-License-Identifier: GPL-2.0 */
      2/*
      3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
      4 */
      5#ifndef __ASM_BARRIER_H
      6#define __ASM_BARRIER_H
      7
      8#define __sync()	__asm__ __volatile__("dbar 0" : : : "memory")
      9
     10#define fast_wmb()	__sync()
     11#define fast_rmb()	__sync()
     12#define fast_mb()	__sync()
     13#define fast_iob()	__sync()
     14#define wbflush()	__sync()
     15
     16#define wmb()		fast_wmb()
     17#define rmb()		fast_rmb()
     18#define mb()		fast_mb()
     19#define iob()		fast_iob()
     20
     21#define __smp_mb()	__asm__ __volatile__("dbar 0" : : : "memory")
     22#define __smp_rmb()	__asm__ __volatile__("dbar 0" : : : "memory")
     23#define __smp_wmb()	__asm__ __volatile__("dbar 0" : : : "memory")
     24
     25#ifdef CONFIG_SMP
     26#define __WEAK_LLSC_MB		"	dbar 0  \n"
     27#else
     28#define __WEAK_LLSC_MB		"		\n"
     29#endif
     30
     31#define __smp_mb__before_atomic()	barrier()
     32#define __smp_mb__after_atomic()	barrier()
     33
     34/**
     35 * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
     36 * @index: array element index
     37 * @size: number of elements in array
     38 *
     39 * Returns:
     40 *     0 - (@index < @size)
     41 */
     42#define array_index_mask_nospec array_index_mask_nospec
     43static inline unsigned long array_index_mask_nospec(unsigned long index,
     44						    unsigned long size)
     45{
     46	unsigned long mask;
     47
     48	__asm__ __volatile__(
     49		"sltu	%0, %1, %2\n\t"
     50#if (__SIZEOF_LONG__ == 4)
     51		"sub.w	%0, $r0, %0\n\t"
     52#elif (__SIZEOF_LONG__ == 8)
     53		"sub.d	%0, $r0, %0\n\t"
     54#endif
     55		: "=r" (mask)
     56		: "r" (index), "r" (size)
     57		:);
     58
     59	return mask;
     60}
     61
     62#define __smp_load_acquire(p)							\
     63({										\
     64	union { typeof(*p) __val; char __c[1]; } __u;				\
     65	unsigned long __tmp = 0;							\
     66	compiletime_assert_atomic_type(*p);					\
     67	switch (sizeof(*p)) {							\
     68	case 1:									\
     69		*(__u8 *)__u.__c = *(volatile __u8 *)p;				\
     70		__smp_mb();							\
     71		break;								\
     72	case 2:									\
     73		*(__u16 *)__u.__c = *(volatile __u16 *)p;			\
     74		__smp_mb();							\
     75		break;								\
     76	case 4:									\
     77		__asm__ __volatile__(						\
     78		"amor_db.w %[val], %[tmp], %[mem]	\n"				\
     79		: [val] "=&r" (*(__u32 *)__u.__c)				\
     80		: [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp)			\
     81		: "memory");							\
     82		break;								\
     83	case 8:									\
     84		__asm__ __volatile__(						\
     85		"amor_db.d %[val], %[tmp], %[mem]	\n"				\
     86		: [val] "=&r" (*(__u64 *)__u.__c)				\
     87		: [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp)			\
     88		: "memory");							\
     89		break;								\
     90	}									\
     91	(typeof(*p))__u.__val;								\
     92})
     93
     94#define __smp_store_release(p, v)						\
     95do {										\
     96	union { typeof(*p) __val; char __c[1]; } __u =				\
     97		{ .__val = (__force typeof(*p)) (v) };				\
     98	unsigned long __tmp;							\
     99	compiletime_assert_atomic_type(*p);					\
    100	switch (sizeof(*p)) {							\
    101	case 1:									\
    102		__smp_mb();							\
    103		*(volatile __u8 *)p = *(__u8 *)__u.__c;				\
    104		break;								\
    105	case 2:									\
    106		__smp_mb();							\
    107		*(volatile __u16 *)p = *(__u16 *)__u.__c;			\
    108		break;								\
    109	case 4:									\
    110		__asm__ __volatile__(						\
    111		"amswap_db.w %[tmp], %[val], %[mem]	\n"			\
    112		: [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp)			\
    113		: [val] "r" (*(__u32 *)__u.__c)					\
    114		: );								\
    115		break;								\
    116	case 8:									\
    117		__asm__ __volatile__(						\
    118		"amswap_db.d %[tmp], %[val], %[mem]	\n"			\
    119		: [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp)			\
    120		: [val] "r" (*(__u64 *)__u.__c)					\
    121		: );								\
    122		break;								\
    123	}									\
    124} while (0)
    125
    126#define __smp_store_mb(p, v)							\
    127do {										\
    128	union { typeof(p) __val; char __c[1]; } __u =				\
    129		{ .__val = (__force typeof(p)) (v) };				\
    130	unsigned long __tmp;							\
    131	switch (sizeof(p)) {							\
    132	case 1:									\
    133		*(volatile __u8 *)&p = *(__u8 *)__u.__c;			\
    134		__smp_mb();							\
    135		break;								\
    136	case 2:									\
    137		*(volatile __u16 *)&p = *(__u16 *)__u.__c;			\
    138		__smp_mb();							\
    139		break;								\
    140	case 4:									\
    141		__asm__ __volatile__(						\
    142		"amswap_db.w %[tmp], %[val], %[mem]	\n"			\
    143		: [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp)			\
    144		: [val] "r" (*(__u32 *)__u.__c)					\
    145		: );								\
    146		break;								\
    147	case 8:									\
    148		__asm__ __volatile__(						\
    149		"amswap_db.d %[tmp], %[val], %[mem]	\n"			\
    150		: [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp)			\
    151		: [val] "r" (*(__u64 *)__u.__c)					\
    152		: );								\
    153		break;								\
    154	}									\
    155} while (0)
    156
    157#include <asm-generic/barrier.h>
    158
    159#endif /* __ASM_BARRIER_H */