cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

queue.c (3620B)

      1#include <sys/types.h>
      2#include <sys/stat.h>
      3#include <sys/mman.h>
      4#include <unistd.h>
      5#include <errno.h>
      6#include <string.h>
      7
      8#include "liburing.h"
      9#include "barrier.h"
     10
     11static int __io_uring_get_cqe(struct io_uring *ring,
     12			      struct io_uring_cqe **cqe_ptr, int wait)
     13{
     14	struct io_uring_cq *cq = &ring->cq;
     15	const unsigned mask = *cq->kring_mask;
     16	unsigned head;
     17	int ret;
     18
     19	*cqe_ptr = NULL;
     20	head = *cq->khead;
     21	do {
     22		/*
     23		 * It's necessary to use a read_barrier() before reading
     24		 * the CQ tail, since the kernel updates it locklessly. The
     25		 * kernel has the matching store barrier for the update. The
     26		 * kernel also ensures that previous stores to CQEs are ordered
     27		 * with the tail update.
     28		 */
     29		read_barrier();
     30		if (head != *cq->ktail) {
     31			*cqe_ptr = &cq->cqes[head & mask];
     32			break;
     33		}
     34		if (!wait)
     35			break;
     36		ret = io_uring_enter(ring->ring_fd, 0, 1,
     37					IORING_ENTER_GETEVENTS, NULL);
     38		if (ret < 0)
     39			return -errno;
     40	} while (1);
     41
     42	return 0;
     43}
     44
     45/*
     46 * Return an IO completion, if one is readily available. Returns 0 with
     47 * cqe_ptr filled in on success, -errno on failure.
     48 */
     49int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
     50{
     51	return __io_uring_get_cqe(ring, cqe_ptr, 0);
     52}
     53
     54/*
     55 * Return an IO completion, waiting for it if necessary. Returns 0 with
     56 * cqe_ptr filled in on success, -errno on failure.
     57 */
     58int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
     59{
     60	return __io_uring_get_cqe(ring, cqe_ptr, 1);
     61}
     62
     63/*
     64 * Submit sqes acquired from io_uring_get_sqe() to the kernel.
     65 *
     66 * Returns number of sqes submitted
     67 */
     68int io_uring_submit(struct io_uring *ring)
     69{
     70	struct io_uring_sq *sq = &ring->sq;
     71	const unsigned mask = *sq->kring_mask;
     72	unsigned ktail, ktail_next, submitted, to_submit;
     73	int ret;
     74
     75	/*
     76	 * If we have pending IO in the kring, submit it first. We need a
     77	 * read barrier here to match the kernels store barrier when updating
     78	 * the SQ head.
     79	 */
     80	read_barrier();
     81	if (*sq->khead != *sq->ktail) {
     82		submitted = *sq->kring_entries;
     83		goto submit;
     84	}
     85
     86	if (sq->sqe_head == sq->sqe_tail)
     87		return 0;
     88
     89	/*
     90	 * Fill in sqes that we have queued up, adding them to the kernel ring
     91	 */
     92	submitted = 0;
     93	ktail = ktail_next = *sq->ktail;
     94	to_submit = sq->sqe_tail - sq->sqe_head;
     95	while (to_submit--) {
     96		ktail_next++;
     97		read_barrier();
     98
     99		sq->array[ktail & mask] = sq->sqe_head & mask;
    100		ktail = ktail_next;
    101
    102		sq->sqe_head++;
    103		submitted++;
    104	}
    105
    106	if (!submitted)
    107		return 0;
    108
    109	if (*sq->ktail != ktail) {
    110		/*
    111		 * First write barrier ensures that the SQE stores are updated
    112		 * with the tail update. This is needed so that the kernel
    113		 * will never see a tail update without the preceeding sQE
    114		 * stores being done.
    115		 */
    116		write_barrier();
    117		*sq->ktail = ktail;
    118		/*
    119		 * The kernel has the matching read barrier for reading the
    120		 * SQ tail.
    121		 */
    122		write_barrier();
    123	}
    124
    125submit:
    126	ret = io_uring_enter(ring->ring_fd, submitted, 0,
    127				IORING_ENTER_GETEVENTS, NULL);
    128	if (ret < 0)
    129		return -errno;
    130
    131	return ret;
    132}
    133
    134/*
    135 * Return an sqe to fill. Application must later call io_uring_submit()
    136 * when it's ready to tell the kernel about it. The caller may call this
    137 * function multiple times before calling io_uring_submit().
    138 *
    139 * Returns a vacant sqe, or NULL if we're full.
    140 */
    141struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
    142{
    143	struct io_uring_sq *sq = &ring->sq;
    144	unsigned next = sq->sqe_tail + 1;
    145	struct io_uring_sqe *sqe;
    146
    147	/*
    148	 * All sqes are used
    149	 */
    150	if (next - sq->sqe_head > *sq->kring_entries)
    151		return NULL;
    152
    153	sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
    154	sq->sqe_tail = next;
    155	return sqe;
    156}