cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

i40e_hmc.c (9779B)


      1// SPDX-License-Identifier: GPL-2.0
      2/* Copyright(c) 2013 - 2018 Intel Corporation. */
      3
      4#include "i40e.h"
      5#include "i40e_osdep.h"
      6#include "i40e_register.h"
      7#include "i40e_status.h"
      8#include "i40e_alloc.h"
      9#include "i40e_hmc.h"
     10#include "i40e_type.h"
     11
     12/**
     13 * i40e_add_sd_table_entry - Adds a segment descriptor to the table
     14 * @hw: pointer to our hw struct
     15 * @hmc_info: pointer to the HMC configuration information struct
     16 * @sd_index: segment descriptor index to manipulate
     17 * @type: what type of segment descriptor we're manipulating
     18 * @direct_mode_sz: size to alloc in direct mode
     19 **/
     20i40e_status i40e_add_sd_table_entry(struct i40e_hw *hw,
     21					      struct i40e_hmc_info *hmc_info,
     22					      u32 sd_index,
     23					      enum i40e_sd_entry_type type,
     24					      u64 direct_mode_sz)
     25{
     26	enum i40e_memory_type mem_type __attribute__((unused));
     27	struct i40e_hmc_sd_entry *sd_entry;
     28	bool dma_mem_alloc_done = false;
     29	struct i40e_dma_mem mem;
     30	i40e_status ret_code = I40E_SUCCESS;
     31	u64 alloc_len;
     32
     33	if (NULL == hmc_info->sd_table.sd_entry) {
     34		ret_code = I40E_ERR_BAD_PTR;
     35		hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_entry\n");
     36		goto exit;
     37	}
     38
     39	if (sd_index >= hmc_info->sd_table.sd_cnt) {
     40		ret_code = I40E_ERR_INVALID_SD_INDEX;
     41		hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_index\n");
     42		goto exit;
     43	}
     44
     45	sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
     46	if (!sd_entry->valid) {
     47		if (I40E_SD_TYPE_PAGED == type) {
     48			mem_type = i40e_mem_pd;
     49			alloc_len = I40E_HMC_PAGED_BP_SIZE;
     50		} else {
     51			mem_type = i40e_mem_bp_jumbo;
     52			alloc_len = direct_mode_sz;
     53		}
     54
     55		/* allocate a 4K pd page or 2M backing page */
     56		ret_code = i40e_allocate_dma_mem(hw, &mem, mem_type, alloc_len,
     57						 I40E_HMC_PD_BP_BUF_ALIGNMENT);
     58		if (ret_code)
     59			goto exit;
     60		dma_mem_alloc_done = true;
     61		if (I40E_SD_TYPE_PAGED == type) {
     62			ret_code = i40e_allocate_virt_mem(hw,
     63					&sd_entry->u.pd_table.pd_entry_virt_mem,
     64					sizeof(struct i40e_hmc_pd_entry) * 512);
     65			if (ret_code)
     66				goto exit;
     67			sd_entry->u.pd_table.pd_entry =
     68				(struct i40e_hmc_pd_entry *)
     69				sd_entry->u.pd_table.pd_entry_virt_mem.va;
     70			sd_entry->u.pd_table.pd_page_addr = mem;
     71		} else {
     72			sd_entry->u.bp.addr = mem;
     73			sd_entry->u.bp.sd_pd_index = sd_index;
     74		}
     75		/* initialize the sd entry */
     76		hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
     77
     78		/* increment the ref count */
     79		I40E_INC_SD_REFCNT(&hmc_info->sd_table);
     80	}
     81	/* Increment backing page reference count */
     82	if (I40E_SD_TYPE_DIRECT == sd_entry->entry_type)
     83		I40E_INC_BP_REFCNT(&sd_entry->u.bp);
     84exit:
     85	if (ret_code)
     86		if (dma_mem_alloc_done)
     87			i40e_free_dma_mem(hw, &mem);
     88
     89	return ret_code;
     90}
     91
     92/**
     93 * i40e_add_pd_table_entry - Adds page descriptor to the specified table
     94 * @hw: pointer to our HW structure
     95 * @hmc_info: pointer to the HMC configuration information structure
     96 * @pd_index: which page descriptor index to manipulate
     97 * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
     98 *
     99 * This function:
    100 *	1. Initializes the pd entry
    101 *	2. Adds pd_entry in the pd_table
    102 *	3. Mark the entry valid in i40e_hmc_pd_entry structure
    103 *	4. Initializes the pd_entry's ref count to 1
    104 * assumptions:
    105 *	1. The memory for pd should be pinned down, physically contiguous and
    106 *	   aligned on 4K boundary and zeroed memory.
    107 *	2. It should be 4K in size.
    108 **/
    109i40e_status i40e_add_pd_table_entry(struct i40e_hw *hw,
    110					      struct i40e_hmc_info *hmc_info,
    111					      u32 pd_index,
    112					      struct i40e_dma_mem *rsrc_pg)
    113{
    114	i40e_status ret_code = 0;
    115	struct i40e_hmc_pd_table *pd_table;
    116	struct i40e_hmc_pd_entry *pd_entry;
    117	struct i40e_dma_mem mem;
    118	struct i40e_dma_mem *page = &mem;
    119	u32 sd_idx, rel_pd_idx;
    120	u64 *pd_addr;
    121	u64 page_desc;
    122
    123	if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) {
    124		ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX;
    125		hw_dbg(hw, "i40e_add_pd_table_entry: bad pd_index\n");
    126		goto exit;
    127	}
    128
    129	/* find corresponding sd */
    130	sd_idx = (pd_index / I40E_HMC_PD_CNT_IN_SD);
    131	if (I40E_SD_TYPE_PAGED !=
    132	    hmc_info->sd_table.sd_entry[sd_idx].entry_type)
    133		goto exit;
    134
    135	rel_pd_idx = (pd_index % I40E_HMC_PD_CNT_IN_SD);
    136	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
    137	pd_entry = &pd_table->pd_entry[rel_pd_idx];
    138	if (!pd_entry->valid) {
    139		if (rsrc_pg) {
    140			pd_entry->rsrc_pg = true;
    141			page = rsrc_pg;
    142		} else {
    143			/* allocate a 4K backing page */
    144			ret_code = i40e_allocate_dma_mem(hw, page, i40e_mem_bp,
    145						I40E_HMC_PAGED_BP_SIZE,
    146						I40E_HMC_PD_BP_BUF_ALIGNMENT);
    147			if (ret_code)
    148				goto exit;
    149			pd_entry->rsrc_pg = false;
    150		}
    151
    152		pd_entry->bp.addr = *page;
    153		pd_entry->bp.sd_pd_index = pd_index;
    154		pd_entry->bp.entry_type = I40E_SD_TYPE_PAGED;
    155		/* Set page address and valid bit */
    156		page_desc = page->pa | 0x1;
    157
    158		pd_addr = (u64 *)pd_table->pd_page_addr.va;
    159		pd_addr += rel_pd_idx;
    160
    161		/* Add the backing page physical address in the pd entry */
    162		memcpy(pd_addr, &page_desc, sizeof(u64));
    163
    164		pd_entry->sd_index = sd_idx;
    165		pd_entry->valid = true;
    166		I40E_INC_PD_REFCNT(pd_table);
    167	}
    168	I40E_INC_BP_REFCNT(&pd_entry->bp);
    169exit:
    170	return ret_code;
    171}
    172
    173/**
    174 * i40e_remove_pd_bp - remove a backing page from a page descriptor
    175 * @hw: pointer to our HW structure
    176 * @hmc_info: pointer to the HMC configuration information structure
    177 * @idx: the page index
    178 *
    179 * This function:
    180 *	1. Marks the entry in pd tabe (for paged address mode) or in sd table
    181 *	   (for direct address mode) invalid.
    182 *	2. Write to register PMPDINV to invalidate the backing page in FV cache
    183 *	3. Decrement the ref count for the pd _entry
    184 * assumptions:
    185 *	1. Caller can deallocate the memory used by backing storage after this
    186 *	   function returns.
    187 **/
    188i40e_status i40e_remove_pd_bp(struct i40e_hw *hw,
    189					struct i40e_hmc_info *hmc_info,
    190					u32 idx)
    191{
    192	i40e_status ret_code = 0;
    193	struct i40e_hmc_pd_entry *pd_entry;
    194	struct i40e_hmc_pd_table *pd_table;
    195	struct i40e_hmc_sd_entry *sd_entry;
    196	u32 sd_idx, rel_pd_idx;
    197	u64 *pd_addr;
    198
    199	/* calculate index */
    200	sd_idx = idx / I40E_HMC_PD_CNT_IN_SD;
    201	rel_pd_idx = idx % I40E_HMC_PD_CNT_IN_SD;
    202	if (sd_idx >= hmc_info->sd_table.sd_cnt) {
    203		ret_code = I40E_ERR_INVALID_PAGE_DESC_INDEX;
    204		hw_dbg(hw, "i40e_remove_pd_bp: bad idx\n");
    205		goto exit;
    206	}
    207	sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
    208	if (I40E_SD_TYPE_PAGED != sd_entry->entry_type) {
    209		ret_code = I40E_ERR_INVALID_SD_TYPE;
    210		hw_dbg(hw, "i40e_remove_pd_bp: wrong sd_entry type\n");
    211		goto exit;
    212	}
    213	/* get the entry and decrease its ref counter */
    214	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
    215	pd_entry = &pd_table->pd_entry[rel_pd_idx];
    216	I40E_DEC_BP_REFCNT(&pd_entry->bp);
    217	if (pd_entry->bp.ref_cnt)
    218		goto exit;
    219
    220	/* mark the entry invalid */
    221	pd_entry->valid = false;
    222	I40E_DEC_PD_REFCNT(pd_table);
    223	pd_addr = (u64 *)pd_table->pd_page_addr.va;
    224	pd_addr += rel_pd_idx;
    225	memset(pd_addr, 0, sizeof(u64));
    226	I40E_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx);
    227
    228	/* free memory here */
    229	if (!pd_entry->rsrc_pg)
    230		ret_code = i40e_free_dma_mem(hw, &pd_entry->bp.addr);
    231	if (ret_code)
    232		goto exit;
    233	if (!pd_table->ref_cnt)
    234		i40e_free_virt_mem(hw, &pd_table->pd_entry_virt_mem);
    235exit:
    236	return ret_code;
    237}
    238
    239/**
    240 * i40e_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
    241 * @hmc_info: pointer to the HMC configuration information structure
    242 * @idx: the page index
    243 **/
    244i40e_status i40e_prep_remove_sd_bp(struct i40e_hmc_info *hmc_info,
    245					     u32 idx)
    246{
    247	i40e_status ret_code = 0;
    248	struct i40e_hmc_sd_entry *sd_entry;
    249
    250	/* get the entry and decrease its ref counter */
    251	sd_entry = &hmc_info->sd_table.sd_entry[idx];
    252	I40E_DEC_BP_REFCNT(&sd_entry->u.bp);
    253	if (sd_entry->u.bp.ref_cnt) {
    254		ret_code = I40E_ERR_NOT_READY;
    255		goto exit;
    256	}
    257	I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
    258
    259	/* mark the entry invalid */
    260	sd_entry->valid = false;
    261exit:
    262	return ret_code;
    263}
    264
    265/**
    266 * i40e_remove_sd_bp_new - Removes a backing page from a segment descriptor
    267 * @hw: pointer to our hw struct
    268 * @hmc_info: pointer to the HMC configuration information structure
    269 * @idx: the page index
    270 * @is_pf: used to distinguish between VF and PF
    271 **/
    272i40e_status i40e_remove_sd_bp_new(struct i40e_hw *hw,
    273					    struct i40e_hmc_info *hmc_info,
    274					    u32 idx, bool is_pf)
    275{
    276	struct i40e_hmc_sd_entry *sd_entry;
    277
    278	if (!is_pf)
    279		return I40E_NOT_SUPPORTED;
    280
    281	/* get the entry and decrease its ref counter */
    282	sd_entry = &hmc_info->sd_table.sd_entry[idx];
    283	I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_DIRECT);
    284
    285	return i40e_free_dma_mem(hw, &sd_entry->u.bp.addr);
    286}
    287
    288/**
    289 * i40e_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
    290 * @hmc_info: pointer to the HMC configuration information structure
    291 * @idx: segment descriptor index to find the relevant page descriptor
    292 **/
    293i40e_status i40e_prep_remove_pd_page(struct i40e_hmc_info *hmc_info,
    294					       u32 idx)
    295{
    296	i40e_status ret_code = 0;
    297	struct i40e_hmc_sd_entry *sd_entry;
    298
    299	sd_entry = &hmc_info->sd_table.sd_entry[idx];
    300
    301	if (sd_entry->u.pd_table.ref_cnt) {
    302		ret_code = I40E_ERR_NOT_READY;
    303		goto exit;
    304	}
    305
    306	/* mark the entry invalid */
    307	sd_entry->valid = false;
    308
    309	I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
    310exit:
    311	return ret_code;
    312}
    313
    314/**
    315 * i40e_remove_pd_page_new - Removes a PD page from sd entry.
    316 * @hw: pointer to our hw struct
    317 * @hmc_info: pointer to the HMC configuration information structure
    318 * @idx: segment descriptor index to find the relevant page descriptor
    319 * @is_pf: used to distinguish between VF and PF
    320 **/
    321i40e_status i40e_remove_pd_page_new(struct i40e_hw *hw,
    322					      struct i40e_hmc_info *hmc_info,
    323					      u32 idx, bool is_pf)
    324{
    325	struct i40e_hmc_sd_entry *sd_entry;
    326
    327	if (!is_pf)
    328		return I40E_NOT_SUPPORTED;
    329
    330	sd_entry = &hmc_info->sd_table.sd_entry[idx];
    331	I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_PAGED);
    332
    333	return  i40e_free_dma_mem(hw, &sd_entry->u.pd_table.pd_page_addr);
    334}