cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

rxe_queue.c (4587B)


      1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
      2/*
      3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
      4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
      5 */
      6
      7#include <linux/vmalloc.h>
      8#include "rxe.h"
      9#include "rxe_loc.h"
     10#include "rxe_queue.h"
     11
     12int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
     13		 struct ib_udata *udata, struct rxe_queue_buf *buf,
     14		 size_t buf_size, struct rxe_mmap_info **ip_p)
     15{
     16	int err;
     17	struct rxe_mmap_info *ip = NULL;
     18
     19	if (outbuf) {
     20		ip = rxe_create_mmap_info(rxe, buf_size, udata, buf);
     21		if (IS_ERR(ip)) {
     22			err = PTR_ERR(ip);
     23			goto err1;
     24		}
     25
     26		if (copy_to_user(outbuf, &ip->info, sizeof(ip->info))) {
     27			err = -EFAULT;
     28			goto err2;
     29		}
     30
     31		spin_lock_bh(&rxe->pending_lock);
     32		list_add(&ip->pending_mmaps, &rxe->pending_mmaps);
     33		spin_unlock_bh(&rxe->pending_lock);
     34	}
     35
     36	*ip_p = ip;
     37
     38	return 0;
     39
     40err2:
     41	kfree(ip);
     42err1:
     43	return err;
     44}
     45
     46inline void rxe_queue_reset(struct rxe_queue *q)
     47{
     48	/* queue is comprised from header and the memory
     49	 * of the actual queue. See "struct rxe_queue_buf" in rxe_queue.h
     50	 * reset only the queue itself and not the management header
     51	 */
     52	memset(q->buf->data, 0, q->buf_size - sizeof(struct rxe_queue_buf));
     53}
     54
     55struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
     56			unsigned int elem_size, enum queue_type type)
     57{
     58	struct rxe_queue *q;
     59	size_t buf_size;
     60	unsigned int num_slots;
     61
     62	/* num_elem == 0 is allowed, but uninteresting */
     63	if (*num_elem < 0)
     64		goto err1;
     65
     66	q = kzalloc(sizeof(*q), GFP_KERNEL);
     67	if (!q)
     68		goto err1;
     69
     70	q->rxe = rxe;
     71	q->type = type;
     72
     73	/* used in resize, only need to copy used part of queue */
     74	q->elem_size = elem_size;
     75
     76	/* pad element up to at least a cacheline and always a power of 2 */
     77	if (elem_size < cache_line_size())
     78		elem_size = cache_line_size();
     79	elem_size = roundup_pow_of_two(elem_size);
     80
     81	q->log2_elem_size = order_base_2(elem_size);
     82
     83	num_slots = *num_elem + 1;
     84	num_slots = roundup_pow_of_two(num_slots);
     85	q->index_mask = num_slots - 1;
     86
     87	buf_size = sizeof(struct rxe_queue_buf) + num_slots * elem_size;
     88
     89	q->buf = vmalloc_user(buf_size);
     90	if (!q->buf)
     91		goto err2;
     92
     93	q->buf->log2_elem_size = q->log2_elem_size;
     94	q->buf->index_mask = q->index_mask;
     95
     96	q->buf_size = buf_size;
     97
     98	*num_elem = num_slots - 1;
     99	return q;
    100
    101err2:
    102	kfree(q);
    103err1:
    104	return NULL;
    105}
    106
    107/* copies elements from original q to new q and then swaps the contents of the
    108 * two q headers. This is so that if anyone is holding a pointer to q it will
    109 * still work
    110 */
    111static int resize_finish(struct rxe_queue *q, struct rxe_queue *new_q,
    112			 unsigned int num_elem)
    113{
    114	enum queue_type type = q->type;
    115	u32 prod;
    116	u32 cons;
    117
    118	if (!queue_empty(q, q->type) && (num_elem < queue_count(q, type)))
    119		return -EINVAL;
    120
    121	prod = queue_get_producer(new_q, type);
    122	cons = queue_get_consumer(q, type);
    123
    124	while (!queue_empty(q, type)) {
    125		memcpy(queue_addr_from_index(new_q, prod),
    126		       queue_addr_from_index(q, cons), new_q->elem_size);
    127		prod = queue_next_index(new_q, prod);
    128		cons = queue_next_index(q, cons);
    129	}
    130
    131	new_q->buf->producer_index = prod;
    132	q->buf->consumer_index = cons;
    133
    134	/* update private index copies */
    135	if (type == QUEUE_TYPE_TO_CLIENT)
    136		new_q->index = new_q->buf->producer_index;
    137	else
    138		q->index = q->buf->consumer_index;
    139
    140	/* exchange rxe_queue headers */
    141	swap(*q, *new_q);
    142
    143	return 0;
    144}
    145
    146int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
    147		     unsigned int elem_size, struct ib_udata *udata,
    148		     struct mminfo __user *outbuf, spinlock_t *producer_lock,
    149		     spinlock_t *consumer_lock)
    150{
    151	struct rxe_queue *new_q;
    152	unsigned int num_elem = *num_elem_p;
    153	int err;
    154	unsigned long producer_flags;
    155	unsigned long consumer_flags;
    156
    157	new_q = rxe_queue_init(q->rxe, &num_elem, elem_size, q->type);
    158	if (!new_q)
    159		return -ENOMEM;
    160
    161	err = do_mmap_info(new_q->rxe, outbuf, udata, new_q->buf,
    162			   new_q->buf_size, &new_q->ip);
    163	if (err) {
    164		vfree(new_q->buf);
    165		kfree(new_q);
    166		goto err1;
    167	}
    168
    169	spin_lock_irqsave(consumer_lock, consumer_flags);
    170
    171	if (producer_lock) {
    172		spin_lock_irqsave(producer_lock, producer_flags);
    173		err = resize_finish(q, new_q, num_elem);
    174		spin_unlock_irqrestore(producer_lock, producer_flags);
    175	} else {
    176		err = resize_finish(q, new_q, num_elem);
    177	}
    178
    179	spin_unlock_irqrestore(consumer_lock, consumer_flags);
    180
    181	rxe_queue_cleanup(new_q);	/* new/old dep on err */
    182	if (err)
    183		goto err1;
    184
    185	*num_elem_p = num_elem;
    186	return 0;
    187
    188err1:
    189	return err;
    190}
    191
    192void rxe_queue_cleanup(struct rxe_queue *q)
    193{
    194	if (q->ip)
    195		kref_put(&q->ip->ref, rxe_mmap_release);
    196	else
    197		vfree(q->buf);
    198
    199	kfree(q);
    200}