cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

instrumented-non-atomic.h (4425B)

      1/* SPDX-License-Identifier: GPL-2.0 */
      2
      3/*
      4 * This file provides wrappers with sanitizer instrumentation for non-atomic
      5 * bit operations.
      6 *
      7 * To use this functionality, an arch's bitops.h file needs to define each of
      8 * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
      9 * arch___set_bit(), etc.).
     10 */
     11#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
     12#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
     13
     14#include <linux/instrumented.h>
     15
     16/**
     17 * __set_bit - Set a bit in memory
     18 * @nr: the bit to set
     19 * @addr: the address to start counting from
     20 *
     21 * Unlike set_bit(), this function is non-atomic. If it is called on the same
     22 * region of memory concurrently, the effect may be that only one operation
     23 * succeeds.
     24 */
     25static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
     26{
     27	instrument_write(addr + BIT_WORD(nr), sizeof(long));
     28	arch___set_bit(nr, addr);
     29}
     30
     31/**
     32 * __clear_bit - Clears a bit in memory
     33 * @nr: the bit to clear
     34 * @addr: the address to start counting from
     35 *
     36 * Unlike clear_bit(), this function is non-atomic. If it is called on the same
     37 * region of memory concurrently, the effect may be that only one operation
     38 * succeeds.
     39 */
     40static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
     41{
     42	instrument_write(addr + BIT_WORD(nr), sizeof(long));
     43	arch___clear_bit(nr, addr);
     44}
     45
     46/**
     47 * __change_bit - Toggle a bit in memory
     48 * @nr: the bit to change
     49 * @addr: the address to start counting from
     50 *
     51 * Unlike change_bit(), this function is non-atomic. If it is called on the same
     52 * region of memory concurrently, the effect may be that only one operation
     53 * succeeds.
     54 */
     55static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
     56{
     57	instrument_write(addr + BIT_WORD(nr), sizeof(long));
     58	arch___change_bit(nr, addr);
     59}
     60
     61static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
     62{
     63	if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
     64		/*
     65		 * We treat non-atomic read-write bitops a little more special.
     66		 * Given the operations here only modify a single bit, assuming
     67		 * non-atomicity of the writer is sufficient may be reasonable
     68		 * for certain usage (and follows the permissible nature of the
     69		 * assume-plain-writes-atomic rule):
     70		 * 1. report read-modify-write races -> check read;
     71		 * 2. do not report races with marked readers, but do report
     72		 *    races with unmarked readers -> check "atomic" write.
     73		 */
     74		kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
     75		/*
     76		 * Use generic write instrumentation, in case other sanitizers
     77		 * or tools are enabled alongside KCSAN.
     78		 */
     79		instrument_write(addr + BIT_WORD(nr), sizeof(long));
     80	} else {
     81		instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
     82	}
     83}
     84
     85/**
     86 * __test_and_set_bit - Set a bit and return its old value
     87 * @nr: Bit to set
     88 * @addr: Address to count from
     89 *
     90 * This operation is non-atomic. If two instances of this operation race, one
     91 * can appear to succeed but actually fail.
     92 */
     93static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
     94{
     95	__instrument_read_write_bitop(nr, addr);
     96	return arch___test_and_set_bit(nr, addr);
     97}
     98
     99/**
    100 * __test_and_clear_bit - Clear a bit and return its old value
    101 * @nr: Bit to clear
    102 * @addr: Address to count from
    103 *
    104 * This operation is non-atomic. If two instances of this operation race, one
    105 * can appear to succeed but actually fail.
    106 */
    107static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
    108{
    109	__instrument_read_write_bitop(nr, addr);
    110	return arch___test_and_clear_bit(nr, addr);
    111}
    112
    113/**
    114 * __test_and_change_bit - Change a bit and return its old value
    115 * @nr: Bit to change
    116 * @addr: Address to count from
    117 *
    118 * This operation is non-atomic. If two instances of this operation race, one
    119 * can appear to succeed but actually fail.
    120 */
    121static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
    122{
    123	__instrument_read_write_bitop(nr, addr);
    124	return arch___test_and_change_bit(nr, addr);
    125}
    126
    127/**
    128 * test_bit - Determine whether a bit is set
    129 * @nr: bit number to test
    130 * @addr: Address to start counting from
    131 */
    132static __always_inline bool test_bit(long nr, const volatile unsigned long *addr)
    133{
    134	instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
    135	return arch_test_bit(nr, addr);
    136}
    137
    138#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */