rwonce.h (2009B)
1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020 Google LLC. 4 */ 5#ifndef __ASM_RWONCE_H 6#define __ASM_RWONCE_H 7 8#if defined(CONFIG_LTO) && !defined(__ASSEMBLY__) 9 10#include <linux/compiler_types.h> 11#include <asm/alternative-macros.h> 12 13#ifndef BUILD_VDSO 14 15#ifdef CONFIG_AS_HAS_LDAPR 16#define __LOAD_RCPC(sfx, regs...) \ 17 ALTERNATIVE( \ 18 "ldar" #sfx "\t" #regs, \ 19 ".arch_extension rcpc\n" \ 20 "ldapr" #sfx "\t" #regs, \ 21 ARM64_HAS_LDAPR) 22#else 23#define __LOAD_RCPC(sfx, regs...) "ldar" #sfx "\t" #regs 24#endif /* CONFIG_AS_HAS_LDAPR */ 25 26/* 27 * When building with LTO, there is an increased risk of the compiler 28 * converting an address dependency headed by a READ_ONCE() invocation 29 * into a control dependency and consequently allowing for harmful 30 * reordering by the CPU. 31 * 32 * Ensure that such transformations are harmless by overriding the generic 33 * READ_ONCE() definition with one that provides RCpc acquire semantics 34 * when building with LTO. 35 */ 36#define __READ_ONCE(x) \ 37({ \ 38 typeof(&(x)) __x = &(x); \ 39 int atomic = 1; \ 40 union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \ 41 switch (sizeof(x)) { \ 42 case 1: \ 43 asm volatile(__LOAD_RCPC(b, %w0, %1) \ 44 : "=r" (*(__u8 *)__u.__c) \ 45 : "Q" (*__x) : "memory"); \ 46 break; \ 47 case 2: \ 48 asm volatile(__LOAD_RCPC(h, %w0, %1) \ 49 : "=r" (*(__u16 *)__u.__c) \ 50 : "Q" (*__x) : "memory"); \ 51 break; \ 52 case 4: \ 53 asm volatile(__LOAD_RCPC(, %w0, %1) \ 54 : "=r" (*(__u32 *)__u.__c) \ 55 : "Q" (*__x) : "memory"); \ 56 break; \ 57 case 8: \ 58 asm volatile(__LOAD_RCPC(, %0, %1) \ 59 : "=r" (*(__u64 *)__u.__c) \ 60 : "Q" (*__x) : "memory"); \ 61 break; \ 62 default: \ 63 atomic = 0; \ 64 } \ 65 atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ 66}) 67 68#endif /* !BUILD_VDSO */ 69#endif /* CONFIG_LTO && !__ASSEMBLY__ */ 70 71#include <asm-generic/rwonce.h> 72 73#endif /* __ASM_RWONCE_H */