cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

tir.c (5718B)


      1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
      2/* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
      3
      4#include "tir.h"
      5#include "params.h"
      6#include <linux/mlx5/transobj.h>
      7
      8#define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
      9
     10/* max() doesn't work inside square brackets. */
     11#define MLX5E_TIR_CMD_IN_SZ_DW ( \
     12	MLX5_ST_SZ_DW(create_tir_in) > MLX5_ST_SZ_DW(modify_tir_in) ? \
     13	MLX5_ST_SZ_DW(create_tir_in) : MLX5_ST_SZ_DW(modify_tir_in) \
     14)
     15
     16struct mlx5e_tir_builder {
     17	u32 in[MLX5E_TIR_CMD_IN_SZ_DW];
     18	bool modify;
     19};
     20
     21struct mlx5e_tir_builder *mlx5e_tir_builder_alloc(bool modify)
     22{
     23	struct mlx5e_tir_builder *builder;
     24
     25	builder = kvzalloc(sizeof(*builder), GFP_KERNEL);
     26	builder->modify = modify;
     27
     28	return builder;
     29}
     30
     31void mlx5e_tir_builder_free(struct mlx5e_tir_builder *builder)
     32{
     33	kvfree(builder);
     34}
     35
     36void mlx5e_tir_builder_clear(struct mlx5e_tir_builder *builder)
     37{
     38	memset(builder->in, 0, sizeof(builder->in));
     39}
     40
     41static void *mlx5e_tir_builder_get_tirc(struct mlx5e_tir_builder *builder)
     42{
     43	if (builder->modify)
     44		return MLX5_ADDR_OF(modify_tir_in, builder->in, ctx);
     45	return MLX5_ADDR_OF(create_tir_in, builder->in, ctx);
     46}
     47
     48void mlx5e_tir_builder_build_inline(struct mlx5e_tir_builder *builder, u32 tdn, u32 rqn)
     49{
     50	void *tirc = mlx5e_tir_builder_get_tirc(builder);
     51
     52	WARN_ON(builder->modify);
     53
     54	MLX5_SET(tirc, tirc, transport_domain, tdn);
     55	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
     56	MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_NONE);
     57	MLX5_SET(tirc, tirc, inline_rqn, rqn);
     58}
     59
     60void mlx5e_tir_builder_build_rqt(struct mlx5e_tir_builder *builder, u32 tdn,
     61				 u32 rqtn, bool inner_ft_support)
     62{
     63	void *tirc = mlx5e_tir_builder_get_tirc(builder);
     64
     65	WARN_ON(builder->modify);
     66
     67	MLX5_SET(tirc, tirc, transport_domain, tdn);
     68	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
     69	MLX5_SET(tirc, tirc, indirect_table, rqtn);
     70	MLX5_SET(tirc, tirc, tunneled_offload_en, inner_ft_support);
     71}
     72
     73void mlx5e_tir_builder_build_packet_merge(struct mlx5e_tir_builder *builder,
     74					  const struct mlx5e_packet_merge_param *pkt_merge_param)
     75{
     76	void *tirc = mlx5e_tir_builder_get_tirc(builder);
     77	const unsigned int rough_max_l2_l3_hdr_sz = 256;
     78
     79	if (builder->modify)
     80		MLX5_SET(modify_tir_in, builder->in, bitmask.packet_merge, 1);
     81
     82	switch (pkt_merge_param->type) {
     83	case MLX5E_PACKET_MERGE_LRO:
     84		MLX5_SET(tirc, tirc, packet_merge_mask,
     85			 MLX5_TIRC_PACKET_MERGE_MASK_IPV4_LRO |
     86			 MLX5_TIRC_PACKET_MERGE_MASK_IPV6_LRO);
     87		MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
     88			 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - rough_max_l2_l3_hdr_sz) >> 8);
     89		MLX5_SET(tirc, tirc, lro_timeout_period_usecs, pkt_merge_param->timeout);
     90		break;
     91	default:
     92		break;
     93	}
     94}
     95
     96static int mlx5e_hfunc_to_hw(u8 hfunc)
     97{
     98	switch (hfunc) {
     99	case ETH_RSS_HASH_TOP:
    100		return MLX5_RX_HASH_FN_TOEPLITZ;
    101	case ETH_RSS_HASH_XOR:
    102		return MLX5_RX_HASH_FN_INVERTED_XOR8;
    103	default:
    104		return MLX5_RX_HASH_FN_NONE;
    105	}
    106}
    107
    108void mlx5e_tir_builder_build_rss(struct mlx5e_tir_builder *builder,
    109				 const struct mlx5e_rss_params_hash *rss_hash,
    110				 const struct mlx5e_rss_params_traffic_type *rss_tt,
    111				 bool inner)
    112{
    113	void *tirc = mlx5e_tir_builder_get_tirc(builder);
    114	void *hfso;
    115
    116	if (builder->modify)
    117		MLX5_SET(modify_tir_in, builder->in, bitmask.hash, 1);
    118
    119	MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_hfunc_to_hw(rss_hash->hfunc));
    120	if (rss_hash->hfunc == ETH_RSS_HASH_TOP) {
    121		const size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
    122		void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
    123
    124		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
    125		memcpy(rss_key, rss_hash->toeplitz_hash_key, len);
    126	}
    127
    128	if (inner)
    129		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
    130	else
    131		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
    132	MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, rss_tt->l3_prot_type);
    133	MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, rss_tt->l4_prot_type);
    134	MLX5_SET(rx_hash_field_select, hfso, selected_fields, rss_tt->rx_hash_fields);
    135}
    136
    137void mlx5e_tir_builder_build_direct(struct mlx5e_tir_builder *builder)
    138{
    139	void *tirc = mlx5e_tir_builder_get_tirc(builder);
    140
    141	WARN_ON(builder->modify);
    142
    143	MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
    144}
    145
    146void mlx5e_tir_builder_build_tls(struct mlx5e_tir_builder *builder)
    147{
    148	void *tirc = mlx5e_tir_builder_get_tirc(builder);
    149
    150	WARN_ON(builder->modify);
    151
    152	MLX5_SET(tirc, tirc, tls_en, 1);
    153	MLX5_SET(tirc, tirc, self_lb_block,
    154		 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST |
    155		 MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST);
    156}
    157
    158int mlx5e_tir_init(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder,
    159		   struct mlx5_core_dev *mdev, bool reg)
    160{
    161	int err;
    162
    163	tir->mdev = mdev;
    164
    165	err = mlx5_core_create_tir(tir->mdev, builder->in, &tir->tirn);
    166	if (err)
    167		return err;
    168
    169	if (reg) {
    170		struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs;
    171
    172		mutex_lock(&res->td.list_lock);
    173		list_add(&tir->list, &res->td.tirs_list);
    174		mutex_unlock(&res->td.list_lock);
    175	} else {
    176		INIT_LIST_HEAD(&tir->list);
    177	}
    178
    179	return 0;
    180}
    181
    182void mlx5e_tir_destroy(struct mlx5e_tir *tir)
    183{
    184	struct mlx5e_hw_objs *res = &tir->mdev->mlx5e_res.hw_objs;
    185
    186	/* Skip mutex if list_del is no-op (the TIR wasn't registered in the
    187	 * list). list_empty will never return true for an item of tirs_list,
    188	 * and READ_ONCE/WRITE_ONCE in list_empty/list_del guarantee consistency
    189	 * of the list->next value.
    190	 */
    191	if (!list_empty(&tir->list)) {
    192		mutex_lock(&res->td.list_lock);
    193		list_del(&tir->list);
    194		mutex_unlock(&res->td.list_lock);
    195	}
    196
    197	mlx5_core_destroy_tir(tir->mdev, tir->tirn);
    198}
    199
    200int mlx5e_tir_modify(struct mlx5e_tir *tir, struct mlx5e_tir_builder *builder)
    201{
    202	return mlx5_core_modify_tir(tir->mdev, tir->tirn, builder->in);
    203}