cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

siw_cq.c (2711B)

      1// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
      2
      3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
      4/* Copyright (c) 2008-2019, IBM Corporation */
      5
      6#include <linux/errno.h>
      7#include <linux/types.h>
      8
      9#include <rdma/ib_verbs.h>
     10
     11#include "siw.h"
     12
     13static int map_wc_opcode[SIW_NUM_OPCODES] = {
     14	[SIW_OP_WRITE] = IB_WC_RDMA_WRITE,
     15	[SIW_OP_SEND] = IB_WC_SEND,
     16	[SIW_OP_SEND_WITH_IMM] = IB_WC_SEND,
     17	[SIW_OP_READ] = IB_WC_RDMA_READ,
     18	[SIW_OP_READ_LOCAL_INV] = IB_WC_RDMA_READ,
     19	[SIW_OP_COMP_AND_SWAP] = IB_WC_COMP_SWAP,
     20	[SIW_OP_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
     21	[SIW_OP_INVAL_STAG] = IB_WC_LOCAL_INV,
     22	[SIW_OP_REG_MR] = IB_WC_REG_MR,
     23	[SIW_OP_RECEIVE] = IB_WC_RECV,
     24	[SIW_OP_READ_RESPONSE] = -1 /* not used */
     25};
     26
     27static struct {
     28	enum siw_wc_status siw;
     29	enum ib_wc_status ib;
     30} map_cqe_status[SIW_NUM_WC_STATUS] = {
     31	{ SIW_WC_SUCCESS, IB_WC_SUCCESS },
     32	{ SIW_WC_LOC_LEN_ERR, IB_WC_LOC_LEN_ERR },
     33	{ SIW_WC_LOC_PROT_ERR, IB_WC_LOC_PROT_ERR },
     34	{ SIW_WC_LOC_QP_OP_ERR, IB_WC_LOC_QP_OP_ERR },
     35	{ SIW_WC_WR_FLUSH_ERR, IB_WC_WR_FLUSH_ERR },
     36	{ SIW_WC_BAD_RESP_ERR, IB_WC_BAD_RESP_ERR },
     37	{ SIW_WC_LOC_ACCESS_ERR, IB_WC_LOC_ACCESS_ERR },
     38	{ SIW_WC_REM_ACCESS_ERR, IB_WC_REM_ACCESS_ERR },
     39	{ SIW_WC_REM_INV_REQ_ERR, IB_WC_REM_INV_REQ_ERR },
     40	{ SIW_WC_GENERAL_ERR, IB_WC_GENERAL_ERR }
     41};
     42
     43/*
     44 * Reap one CQE from the CQ. Only used by kernel clients
     45 * during CQ normal operation. Might be called during CQ
     46 * flush for user mapped CQE array as well.
     47 */
     48int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc)
     49{
     50	struct siw_cqe *cqe;
     51	unsigned long flags;
     52
     53	spin_lock_irqsave(&cq->lock, flags);
     54
     55	cqe = &cq->queue[cq->cq_get % cq->num_cqe];
     56	if (READ_ONCE(cqe->flags) & SIW_WQE_VALID) {
     57		memset(wc, 0, sizeof(*wc));
     58		wc->wr_id = cqe->id;
     59		wc->status = map_cqe_status[cqe->status].ib;
     60		wc->opcode = map_wc_opcode[cqe->opcode];
     61		wc->byte_len = cqe->bytes;
     62
     63		/*
     64		 * During CQ flush, also user land CQE's may get
     65		 * reaped here, which do not hold a QP reference
     66		 * and do not qualify for memory extension verbs.
     67		 */
     68		if (likely(rdma_is_kernel_res(&cq->base_cq.res))) {
     69			if (cqe->flags & SIW_WQE_REM_INVAL) {
     70				wc->ex.invalidate_rkey = cqe->inval_stag;
     71				wc->wc_flags = IB_WC_WITH_INVALIDATE;
     72			}
     73			wc->qp = cqe->base_qp;
     74			siw_dbg_cq(cq,
     75				   "idx %u, type %d, flags %2x, id 0x%pK\n",
     76				   cq->cq_get % cq->num_cqe, cqe->opcode,
     77				   cqe->flags, (void *)(uintptr_t)cqe->id);
     78		}
     79		WRITE_ONCE(cqe->flags, 0);
     80		cq->cq_get++;
     81
     82		spin_unlock_irqrestore(&cq->lock, flags);
     83
     84		return 1;
     85	}
     86	spin_unlock_irqrestore(&cq->lock, flags);
     87
     88	return 0;
     89}
     90
     91/*
     92 * siw_cq_flush()
     93 *
     94 * Flush all CQ elements.
     95 */
     96void siw_cq_flush(struct siw_cq *cq)
     97{
     98	struct ib_wc wc;
     99
    100	while (siw_reap_cqe(cq, &wc))
    101		;
    102}