cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

rc.c (4595B)

      1// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
      2/*
      3 * Copyright(c) 2016 Intel Corporation.
      4 */
      5
      6#include <rdma/rdmavt_qp.h>
      7#include <rdma/ib_hdrs.h>
      8
      9/*
     10 * Convert the AETH credit code into the number of credits.
     11 */
     12static const u16 credit_table[31] = {
     13	0,                      /* 0 */
     14	1,                      /* 1 */
     15	2,                      /* 2 */
     16	3,                      /* 3 */
     17	4,                      /* 4 */
     18	6,                      /* 5 */
     19	8,                      /* 6 */
     20	12,                     /* 7 */
     21	16,                     /* 8 */
     22	24,                     /* 9 */
     23	32,                     /* A */
     24	48,                     /* B */
     25	64,                     /* C */
     26	96,                     /* D */
     27	128,                    /* E */
     28	192,                    /* F */
     29	256,                    /* 10 */
     30	384,                    /* 11 */
     31	512,                    /* 12 */
     32	768,                    /* 13 */
     33	1024,                   /* 14 */
     34	1536,                   /* 15 */
     35	2048,                   /* 16 */
     36	3072,                   /* 17 */
     37	4096,                   /* 18 */
     38	6144,                   /* 19 */
     39	8192,                   /* 1A */
     40	12288,                  /* 1B */
     41	16384,                  /* 1C */
     42	24576,                  /* 1D */
     43	32768                   /* 1E */
     44};
     45
     46/**
     47 * rvt_compute_aeth - compute the AETH (syndrome + MSN)
     48 * @qp: the queue pair to compute the AETH for
     49 *
     50 * Returns the AETH.
     51 */
     52__be32 rvt_compute_aeth(struct rvt_qp *qp)
     53{
     54	u32 aeth = qp->r_msn & IB_MSN_MASK;
     55
     56	if (qp->ibqp.srq) {
     57		/*
     58		 * Shared receive queues don't generate credits.
     59		 * Set the credit field to the invalid value.
     60		 */
     61		aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
     62	} else {
     63		u32 min, max, x;
     64		u32 credits;
     65		u32 head;
     66		u32 tail;
     67
     68		credits = READ_ONCE(qp->r_rq.kwq->count);
     69		if (credits == 0) {
     70			/* sanity check pointers before trusting them */
     71			if (qp->ip) {
     72				head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
     73				tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
     74			} else {
     75				head = READ_ONCE(qp->r_rq.kwq->head);
     76				tail = READ_ONCE(qp->r_rq.kwq->tail);
     77			}
     78			if (head >= qp->r_rq.size)
     79				head = 0;
     80			if (tail >= qp->r_rq.size)
     81				tail = 0;
     82			/*
     83			 * Compute the number of credits available (RWQEs).
     84			 * There is a small chance that the pair of reads are
     85			 * not atomic, which is OK, since the fuzziness is
     86			 * resolved as further ACKs go out.
     87			 */
     88			credits = rvt_get_rq_count(&qp->r_rq, head, tail);
     89		}
     90		/*
     91		 * Binary search the credit table to find the code to
     92		 * use.
     93		 */
     94		min = 0;
     95		max = 31;
     96		for (;;) {
     97			x = (min + max) / 2;
     98			if (credit_table[x] == credits)
     99				break;
    100			if (credit_table[x] > credits) {
    101				max = x;
    102			} else {
    103				if (min == x)
    104					break;
    105				min = x;
    106			}
    107		}
    108		aeth |= x << IB_AETH_CREDIT_SHIFT;
    109	}
    110	return cpu_to_be32(aeth);
    111}
    112EXPORT_SYMBOL(rvt_compute_aeth);
    113
    114/**
    115 * rvt_get_credit - flush the send work queue of a QP
    116 * @qp: the qp who's send work queue to flush
    117 * @aeth: the Acknowledge Extended Transport Header
    118 *
    119 * The QP s_lock should be held.
    120 */
    121void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
    122{
    123	struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
    124	u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
    125
    126	lockdep_assert_held(&qp->s_lock);
    127	/*
    128	 * If the credit is invalid, we can send
    129	 * as many packets as we like.  Otherwise, we have to
    130	 * honor the credit field.
    131	 */
    132	if (credit == IB_AETH_CREDIT_INVAL) {
    133		if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
    134			qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
    135			if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
    136				qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
    137				rdi->driver_f.schedule_send(qp);
    138			}
    139		}
    140	} else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
    141		/* Compute new LSN (i.e., MSN + credit) */
    142		credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
    143		if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
    144			qp->s_lsn = credit;
    145			if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
    146				qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
    147				rdi->driver_f.schedule_send(qp);
    148			}
    149		}
    150	}
    151}
    152EXPORT_SYMBOL(rvt_get_credit);
    153
    154/**
    155 * rvt_restart_sge - rewind the sge state for a wqe
    156 * @ss: the sge state pointer
    157 * @wqe: the wqe to rewind
    158 * @len: the data length from the start of the wqe in bytes
    159 *
    160 * Returns the remaining data length.
    161 */
    162u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
    163{
    164	ss->sge = wqe->sg_list[0];
    165	ss->sg_list = wqe->sg_list + 1;
    166	ss->num_sge = wqe->wr.num_sge;
    167	ss->total_len = wqe->length;
    168	rvt_skip_sge(ss, len, false);
    169	return wqe->length - len;
    170}
    171EXPORT_SYMBOL(rvt_restart_sge);
    172