cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

freelist.h (3890B)

      1/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
      2#ifndef FREELIST_H
      3#define FREELIST_H
      4
      5#include <linux/atomic.h>
      6
      7/*
      8 * Copyright: cameron@moodycamel.com
      9 *
     10 * A simple CAS-based lock-free free list. Not the fastest thing in the world
     11 * under heavy contention, but simple and correct (assuming nodes are never
     12 * freed until after the free list is destroyed), and fairly speedy under low
     13 * contention.
     14 *
     15 * Adapted from: https://moodycamel.com/blog/2014/solving-the-aba-problem-for-lock-free-free-lists
     16 */
     17
     18struct freelist_node {
     19	atomic_t		refs;
     20	struct freelist_node	*next;
     21};
     22
     23struct freelist_head {
     24	struct freelist_node	*head;
     25};
     26
     27#define REFS_ON_FREELIST 0x80000000
     28#define REFS_MASK	 0x7FFFFFFF
     29
     30static inline void __freelist_add(struct freelist_node *node, struct freelist_head *list)
     31{
     32	/*
     33	 * Since the refcount is zero, and nobody can increase it once it's
     34	 * zero (except us, and we run only one copy of this method per node at
     35	 * a time, i.e. the single thread case), then we know we can safely
     36	 * change the next pointer of the node; however, once the refcount is
     37	 * back above zero, then other threads could increase it (happens under
     38	 * heavy contention, when the refcount goes to zero in between a load
     39	 * and a refcount increment of a node in try_get, then back up to
     40	 * something non-zero, then the refcount increment is done by the other
     41	 * thread) -- so if the CAS to add the node to the actual list fails,
     42	 * decrese the refcount and leave the add operation to the next thread
     43	 * who puts the refcount back to zero (which could be us, hence the
     44	 * loop).
     45	 */
     46	struct freelist_node *head = READ_ONCE(list->head);
     47
     48	for (;;) {
     49		WRITE_ONCE(node->next, head);
     50		atomic_set_release(&node->refs, 1);
     51
     52		if (!try_cmpxchg_release(&list->head, &head, node)) {
     53			/*
     54			 * Hmm, the add failed, but we can only try again when
     55			 * the refcount goes back to zero.
     56			 */
     57			if (atomic_fetch_add_release(REFS_ON_FREELIST - 1, &node->refs) == 1)
     58				continue;
     59		}
     60		return;
     61	}
     62}
     63
     64static inline void freelist_add(struct freelist_node *node, struct freelist_head *list)
     65{
     66	/*
     67	 * We know that the should-be-on-freelist bit is 0 at this point, so
     68	 * it's safe to set it using a fetch_add.
     69	 */
     70	if (!atomic_fetch_add_release(REFS_ON_FREELIST, &node->refs)) {
     71		/*
     72		 * Oh look! We were the last ones referencing this node, and we
     73		 * know we want to add it to the free list, so let's do it!
     74		 */
     75		__freelist_add(node, list);
     76	}
     77}
     78
     79static inline struct freelist_node *freelist_try_get(struct freelist_head *list)
     80{
     81	struct freelist_node *prev, *next, *head = smp_load_acquire(&list->head);
     82	unsigned int refs;
     83
     84	while (head) {
     85		prev = head;
     86		refs = atomic_read(&head->refs);
     87		if ((refs & REFS_MASK) == 0 ||
     88		    !atomic_try_cmpxchg_acquire(&head->refs, &refs, refs+1)) {
     89			head = smp_load_acquire(&list->head);
     90			continue;
     91		}
     92
     93		/*
     94		 * Good, reference count has been incremented (it wasn't at
     95		 * zero), which means we can read the next and not worry about
     96		 * it changing between now and the time we do the CAS.
     97		 */
     98		next = READ_ONCE(head->next);
     99		if (try_cmpxchg_acquire(&list->head, &head, next)) {
    100			/*
    101			 * Yay, got the node. This means it was on the list,
    102			 * which means should-be-on-freelist must be false no
    103			 * matter the refcount (because nobody else knows it's
    104			 * been taken off yet, it can't have been put back on).
    105			 */
    106			WARN_ON_ONCE(atomic_read(&head->refs) & REFS_ON_FREELIST);
    107
    108			/*
    109			 * Decrease refcount twice, once for our ref, and once
    110			 * for the list's ref.
    111			 */
    112			atomic_fetch_add(-2, &head->refs);
    113
    114			return head;
    115		}
    116
    117		/*
    118		 * OK, the head must have changed on us, but we still need to decrement
    119		 * the refcount we increased.
    120		 */
    121		refs = atomic_fetch_add(-1, &prev->refs);
    122		if (refs == REFS_ON_FREELIST + 1)
    123			__freelist_add(prev, list);
    124	}
    125
    126	return NULL;
    127}
    128
    129#endif /* FREELIST_H */