cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

hns_roce_db.c (4022B)

      1/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
      2/*
      3 * Copyright (c) 2017 Hisilicon Limited.
      4 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
      5 */
      6
      7#include <rdma/ib_umem.h>
      8#include "hns_roce_device.h"
      9
     10int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
     11			 struct hns_roce_db *db)
     12{
     13	unsigned long page_addr = virt & PAGE_MASK;
     14	struct hns_roce_user_db_page *page;
     15	unsigned int offset;
     16	int ret = 0;
     17
     18	mutex_lock(&context->page_mutex);
     19
     20	list_for_each_entry(page, &context->page_list, list)
     21		if (page->user_virt == page_addr)
     22			goto found;
     23
     24	page = kmalloc(sizeof(*page), GFP_KERNEL);
     25	if (!page) {
     26		ret = -ENOMEM;
     27		goto out;
     28	}
     29
     30	refcount_set(&page->refcount, 1);
     31	page->user_virt = page_addr;
     32	page->umem = ib_umem_get(context->ibucontext.device, page_addr,
     33				 PAGE_SIZE, 0);
     34	if (IS_ERR(page->umem)) {
     35		ret = PTR_ERR(page->umem);
     36		kfree(page);
     37		goto out;
     38	}
     39
     40	list_add(&page->list, &context->page_list);
     41
     42found:
     43	offset = virt - page_addr;
     44	db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset;
     45	db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset;
     46	db->u.user_page = page;
     47	refcount_inc(&page->refcount);
     48
     49out:
     50	mutex_unlock(&context->page_mutex);
     51
     52	return ret;
     53}
     54
     55void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
     56			    struct hns_roce_db *db)
     57{
     58	mutex_lock(&context->page_mutex);
     59
     60	refcount_dec(&db->u.user_page->refcount);
     61	if (refcount_dec_if_one(&db->u.user_page->refcount)) {
     62		list_del(&db->u.user_page->list);
     63		ib_umem_release(db->u.user_page->umem);
     64		kfree(db->u.user_page);
     65	}
     66
     67	mutex_unlock(&context->page_mutex);
     68}
     69
     70static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
     71					struct device *dma_device)
     72{
     73	struct hns_roce_db_pgdir *pgdir;
     74
     75	pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
     76	if (!pgdir)
     77		return NULL;
     78
     79	bitmap_fill(pgdir->order1,
     80		    HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
     81	pgdir->bits[0] = pgdir->order0;
     82	pgdir->bits[1] = pgdir->order1;
     83	pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,
     84					 &pgdir->db_dma, GFP_KERNEL);
     85	if (!pgdir->page) {
     86		kfree(pgdir);
     87		return NULL;
     88	}
     89
     90	return pgdir;
     91}
     92
     93static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,
     94					struct hns_roce_db *db, int order)
     95{
     96	unsigned long o;
     97	unsigned long i;
     98
     99	for (o = order; o <= 1; ++o) {
    100		i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);
    101		if (i < HNS_ROCE_DB_PER_PAGE >> o)
    102			goto found;
    103	}
    104
    105	return -ENOMEM;
    106
    107found:
    108	clear_bit(i, pgdir->bits[o]);
    109
    110	i <<= o;
    111
    112	if (o > order)
    113		set_bit(i ^ 1, pgdir->bits[order]);
    114
    115	db->u.pgdir	= pgdir;
    116	db->index	= i;
    117	db->db_record	= pgdir->page + db->index;
    118	db->dma		= pgdir->db_dma  + db->index * HNS_ROCE_DB_UNIT_SIZE;
    119	db->order	= order;
    120
    121	return 0;
    122}
    123
    124int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
    125		      int order)
    126{
    127	struct hns_roce_db_pgdir *pgdir;
    128	int ret = 0;
    129
    130	mutex_lock(&hr_dev->pgdir_mutex);
    131
    132	list_for_each_entry(pgdir, &hr_dev->pgdir_list, list)
    133		if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order))
    134			goto out;
    135
    136	pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev);
    137	if (!pgdir) {
    138		ret = -ENOMEM;
    139		goto out;
    140	}
    141
    142	list_add(&pgdir->list, &hr_dev->pgdir_list);
    143
    144	/* This should never fail -- we just allocated an empty page: */
    145	WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order));
    146
    147out:
    148	mutex_unlock(&hr_dev->pgdir_mutex);
    149
    150	return ret;
    151}
    152
    153void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
    154{
    155	unsigned long o;
    156	unsigned long i;
    157
    158	mutex_lock(&hr_dev->pgdir_mutex);
    159
    160	o = db->order;
    161	i = db->index;
    162
    163	if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
    164		clear_bit(i ^ 1, db->u.pgdir->order0);
    165		++o;
    166	}
    167
    168	i >>= o;
    169	set_bit(i, db->u.pgdir->bits[o]);
    170
    171	if (bitmap_full(db->u.pgdir->order1,
    172			HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {
    173		dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,
    174				  db->u.pgdir->db_dma);
    175		list_del(&db->u.pgdir->list);
    176		kfree(db->u.pgdir);
    177	}
    178
    179	mutex_unlock(&hr_dev->pgdir_mutex);
    180}