cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

gdma.h (14052B)


      1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
      2/* Copyright (c) 2021, Microsoft Corporation. */
      3
      4#ifndef _GDMA_H
      5#define _GDMA_H
      6
      7#include <linux/dma-mapping.h>
      8#include <linux/netdevice.h>
      9
     10#include "shm_channel.h"
     11
     12/* Structures labeled with "HW DATA" are exchanged with the hardware. All of
     13 * them are naturally aligned and hence don't need __packed.
     14 */
     15
     16enum gdma_request_type {
     17	GDMA_VERIFY_VF_DRIVER_VERSION	= 1,
     18	GDMA_QUERY_MAX_RESOURCES	= 2,
     19	GDMA_LIST_DEVICES		= 3,
     20	GDMA_REGISTER_DEVICE		= 4,
     21	GDMA_DEREGISTER_DEVICE		= 5,
     22	GDMA_GENERATE_TEST_EQE		= 10,
     23	GDMA_CREATE_QUEUE		= 12,
     24	GDMA_DISABLE_QUEUE		= 13,
     25	GDMA_CREATE_DMA_REGION		= 25,
     26	GDMA_DMA_REGION_ADD_PAGES	= 26,
     27	GDMA_DESTROY_DMA_REGION		= 27,
     28};
     29
     30enum gdma_queue_type {
     31	GDMA_INVALID_QUEUE,
     32	GDMA_SQ,
     33	GDMA_RQ,
     34	GDMA_CQ,
     35	GDMA_EQ,
     36};
     37
     38enum gdma_work_request_flags {
     39	GDMA_WR_NONE			= 0,
     40	GDMA_WR_OOB_IN_SGL		= BIT(0),
     41	GDMA_WR_PAD_BY_SGE0		= BIT(1),
     42};
     43
     44enum gdma_eqe_type {
     45	GDMA_EQE_COMPLETION		= 3,
     46	GDMA_EQE_TEST_EVENT		= 64,
     47	GDMA_EQE_HWC_INIT_EQ_ID_DB	= 129,
     48	GDMA_EQE_HWC_INIT_DATA		= 130,
     49	GDMA_EQE_HWC_INIT_DONE		= 131,
     50};
     51
     52enum {
     53	GDMA_DEVICE_NONE	= 0,
     54	GDMA_DEVICE_HWC		= 1,
     55	GDMA_DEVICE_MANA	= 2,
     56};
     57
     58struct gdma_resource {
     59	/* Protect the bitmap */
     60	spinlock_t lock;
     61
     62	/* The bitmap size in bits. */
     63	u32 size;
     64
     65	/* The bitmap tracks the resources. */
     66	unsigned long *map;
     67};
     68
     69union gdma_doorbell_entry {
     70	u64	as_uint64;
     71
     72	struct {
     73		u64 id		: 24;
     74		u64 reserved	: 8;
     75		u64 tail_ptr	: 31;
     76		u64 arm		: 1;
     77	} cq;
     78
     79	struct {
     80		u64 id		: 24;
     81		u64 wqe_cnt	: 8;
     82		u64 tail_ptr	: 32;
     83	} rq;
     84
     85	struct {
     86		u64 id		: 24;
     87		u64 reserved	: 8;
     88		u64 tail_ptr	: 32;
     89	} sq;
     90
     91	struct {
     92		u64 id		: 16;
     93		u64 reserved	: 16;
     94		u64 tail_ptr	: 31;
     95		u64 arm		: 1;
     96	} eq;
     97}; /* HW DATA */
     98
     99struct gdma_msg_hdr {
    100	u32 hdr_type;
    101	u32 msg_type;
    102	u16 msg_version;
    103	u16 hwc_msg_id;
    104	u32 msg_size;
    105}; /* HW DATA */
    106
    107struct gdma_dev_id {
    108	union {
    109		struct {
    110			u16 type;
    111			u16 instance;
    112		};
    113
    114		u32 as_uint32;
    115	};
    116}; /* HW DATA */
    117
    118struct gdma_req_hdr {
    119	struct gdma_msg_hdr req;
    120	struct gdma_msg_hdr resp; /* The expected response */
    121	struct gdma_dev_id dev_id;
    122	u32 activity_id;
    123}; /* HW DATA */
    124
    125struct gdma_resp_hdr {
    126	struct gdma_msg_hdr response;
    127	struct gdma_dev_id dev_id;
    128	u32 activity_id;
    129	u32 status;
    130	u32 reserved;
    131}; /* HW DATA */
    132
    133struct gdma_general_req {
    134	struct gdma_req_hdr hdr;
    135}; /* HW DATA */
    136
    137#define GDMA_MESSAGE_V1 1
    138
    139struct gdma_general_resp {
    140	struct gdma_resp_hdr hdr;
    141}; /* HW DATA */
    142
    143#define GDMA_STANDARD_HEADER_TYPE 0
    144
    145static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
    146					u32 req_size, u32 resp_size)
    147{
    148	hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
    149	hdr->req.msg_type = code;
    150	hdr->req.msg_version = GDMA_MESSAGE_V1;
    151	hdr->req.msg_size = req_size;
    152
    153	hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
    154	hdr->resp.msg_type = code;
    155	hdr->resp.msg_version = GDMA_MESSAGE_V1;
    156	hdr->resp.msg_size = resp_size;
    157}
    158
    159/* The 16-byte struct is part of the GDMA work queue entry (WQE). */
    160struct gdma_sge {
    161	u64 address;
    162	u32 mem_key;
    163	u32 size;
    164}; /* HW DATA */
    165
    166struct gdma_wqe_request {
    167	struct gdma_sge *sgl;
    168	u32 num_sge;
    169
    170	u32 inline_oob_size;
    171	const void *inline_oob_data;
    172
    173	u32 flags;
    174	u32 client_data_unit;
    175};
    176
    177enum gdma_page_type {
    178	GDMA_PAGE_TYPE_4K,
    179};
    180
    181#define GDMA_INVALID_DMA_REGION 0
    182
    183struct gdma_mem_info {
    184	struct device *dev;
    185
    186	dma_addr_t dma_handle;
    187	void *virt_addr;
    188	u64 length;
    189
    190	/* Allocated by the PF driver */
    191	u64 gdma_region;
    192};
    193
    194#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
    195
    196struct gdma_dev {
    197	struct gdma_context *gdma_context;
    198
    199	struct gdma_dev_id dev_id;
    200
    201	u32 pdid;
    202	u32 doorbell;
    203	u32 gpa_mkey;
    204
    205	/* GDMA driver specific pointer */
    206	void *driver_data;
    207};
    208
    209#define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
    210
    211#define GDMA_CQE_SIZE 64
    212#define GDMA_EQE_SIZE 16
    213#define GDMA_MAX_SQE_SIZE 512
    214#define GDMA_MAX_RQE_SIZE 256
    215
    216#define GDMA_COMP_DATA_SIZE 0x3C
    217
    218#define GDMA_EVENT_DATA_SIZE 0xC
    219
    220/* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
    221#define GDMA_WQE_BU_SIZE 32
    222
    223#define INVALID_PDID		UINT_MAX
    224#define INVALID_DOORBELL	UINT_MAX
    225#define INVALID_MEM_KEY		UINT_MAX
    226#define INVALID_QUEUE_ID	UINT_MAX
    227#define INVALID_PCI_MSIX_INDEX  UINT_MAX
    228
    229struct gdma_comp {
    230	u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
    231	u32 wq_num;
    232	bool is_sq;
    233};
    234
    235struct gdma_event {
    236	u32 details[GDMA_EVENT_DATA_SIZE / 4];
    237	u8  type;
    238};
    239
    240struct gdma_queue;
    241
    242struct mana_eq {
    243	struct gdma_queue *eq;
    244};
    245
    246typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
    247			      struct gdma_event *e);
    248
    249typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
    250
    251/* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
    252 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
    253 * driver increases the 'head' in BUs rather than in bytes, and notifies
    254 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
    255 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
    256 *
    257 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
    258 * processed, the driver increases the 'tail' to indicate that WQEs have
    259 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
    260 *
    261 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
    262 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
    263 * the owner bits mechanism to detect if the queue has become empty.
    264 */
    265struct gdma_queue {
    266	struct gdma_dev *gdma_dev;
    267
    268	enum gdma_queue_type type;
    269	u32 id;
    270
    271	struct gdma_mem_info mem_info;
    272
    273	void *queue_mem_ptr;
    274	u32 queue_size;
    275
    276	bool monitor_avl_buf;
    277
    278	u32 head;
    279	u32 tail;
    280
    281	/* Extra fields specific to EQ/CQ. */
    282	union {
    283		struct {
    284			bool disable_needed;
    285
    286			gdma_eq_callback *callback;
    287			void *context;
    288
    289			unsigned int msix_index;
    290
    291			u32 log2_throttle_limit;
    292		} eq;
    293
    294		struct {
    295			gdma_cq_callback *callback;
    296			void *context;
    297
    298			struct gdma_queue *parent; /* For CQ/EQ relationship */
    299		} cq;
    300	};
    301};
    302
    303struct gdma_queue_spec {
    304	enum gdma_queue_type type;
    305	bool monitor_avl_buf;
    306	unsigned int queue_size;
    307
    308	/* Extra fields specific to EQ/CQ. */
    309	union {
    310		struct {
    311			gdma_eq_callback *callback;
    312			void *context;
    313
    314			unsigned long log2_throttle_limit;
    315		} eq;
    316
    317		struct {
    318			gdma_cq_callback *callback;
    319			void *context;
    320
    321			struct gdma_queue *parent_eq;
    322
    323		} cq;
    324	};
    325};
    326
    327struct gdma_irq_context {
    328	void (*handler)(void *arg);
    329	void *arg;
    330};
    331
    332struct gdma_context {
    333	struct device		*dev;
    334
    335	/* Per-vPort max number of queues */
    336	unsigned int		max_num_queues;
    337	unsigned int		max_num_msix;
    338	unsigned int		num_msix_usable;
    339	struct gdma_resource	msix_resource;
    340	struct gdma_irq_context	*irq_contexts;
    341
    342	/* This maps a CQ index to the queue structure. */
    343	unsigned int		max_num_cqs;
    344	struct gdma_queue	**cq_table;
    345
    346	/* Protect eq_test_event and test_event_eq_id  */
    347	struct mutex		eq_test_event_mutex;
    348	struct completion	eq_test_event;
    349	u32			test_event_eq_id;
    350
    351	void __iomem		*bar0_va;
    352	void __iomem		*shm_base;
    353	void __iomem		*db_page_base;
    354	u32 db_page_size;
    355
    356	/* Shared memory chanenl (used to bootstrap HWC) */
    357	struct shm_channel	shm_channel;
    358
    359	/* Hardware communication channel (HWC) */
    360	struct gdma_dev		hwc;
    361
    362	/* Azure network adapter */
    363	struct gdma_dev		mana;
    364};
    365
    366#define MAX_NUM_GDMA_DEVICES	4
    367
    368static inline bool mana_gd_is_mana(struct gdma_dev *gd)
    369{
    370	return gd->dev_id.type == GDMA_DEVICE_MANA;
    371}
    372
    373static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
    374{
    375	return gd->dev_id.type == GDMA_DEVICE_HWC;
    376}
    377
    378u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
    379u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
    380
    381int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
    382
    383int mana_gd_create_hwc_queue(struct gdma_dev *gd,
    384			     const struct gdma_queue_spec *spec,
    385			     struct gdma_queue **queue_ptr);
    386
    387int mana_gd_create_mana_eq(struct gdma_dev *gd,
    388			   const struct gdma_queue_spec *spec,
    389			   struct gdma_queue **queue_ptr);
    390
    391int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
    392			      const struct gdma_queue_spec *spec,
    393			      struct gdma_queue **queue_ptr);
    394
    395void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
    396
    397int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
    398
    399void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
    400
    401struct gdma_wqe {
    402	u32 reserved	:24;
    403	u32 last_vbytes	:8;
    404
    405	union {
    406		u32 flags;
    407
    408		struct {
    409			u32 num_sge		:8;
    410			u32 inline_oob_size_div4:3;
    411			u32 client_oob_in_sgl	:1;
    412			u32 reserved1		:4;
    413			u32 client_data_unit	:14;
    414			u32 reserved2		:2;
    415		};
    416	};
    417}; /* HW DATA */
    418
    419#define INLINE_OOB_SMALL_SIZE 8
    420#define INLINE_OOB_LARGE_SIZE 24
    421
    422#define MAX_TX_WQE_SIZE 512
    423#define MAX_RX_WQE_SIZE 256
    424
    425struct gdma_cqe {
    426	u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
    427
    428	union {
    429		u32 as_uint32;
    430
    431		struct {
    432			u32 wq_num	: 24;
    433			u32 is_sq	: 1;
    434			u32 reserved	: 4;
    435			u32 owner_bits	: 3;
    436		};
    437	} cqe_info;
    438}; /* HW DATA */
    439
    440#define GDMA_CQE_OWNER_BITS 3
    441
    442#define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
    443
    444#define SET_ARM_BIT 1
    445
    446#define GDMA_EQE_OWNER_BITS 3
    447
    448union gdma_eqe_info {
    449	u32 as_uint32;
    450
    451	struct {
    452		u32 type	: 8;
    453		u32 reserved1	: 8;
    454		u32 client_id	: 2;
    455		u32 reserved2	: 11;
    456		u32 owner_bits	: 3;
    457	};
    458}; /* HW DATA */
    459
    460#define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
    461#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
    462
    463struct gdma_eqe {
    464	u32 details[GDMA_EVENT_DATA_SIZE / 4];
    465	u32 eqe_info;
    466}; /* HW DATA */
    467
    468#define GDMA_REG_DB_PAGE_OFFSET	8
    469#define GDMA_REG_DB_PAGE_SIZE	0x10
    470#define GDMA_REG_SHM_OFFSET	0x18
    471
    472struct gdma_posted_wqe_info {
    473	u32 wqe_size_in_bu;
    474};
    475
    476/* GDMA_GENERATE_TEST_EQE */
    477struct gdma_generate_test_event_req {
    478	struct gdma_req_hdr hdr;
    479	u32 queue_index;
    480}; /* HW DATA */
    481
    482/* GDMA_VERIFY_VF_DRIVER_VERSION */
    483enum {
    484	GDMA_PROTOCOL_V1	= 1,
    485	GDMA_PROTOCOL_FIRST	= GDMA_PROTOCOL_V1,
    486	GDMA_PROTOCOL_LAST	= GDMA_PROTOCOL_V1,
    487};
    488
    489#define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
    490
    491#define GDMA_DRV_CAP_FLAGS1 GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT
    492
    493#define GDMA_DRV_CAP_FLAGS2 0
    494
    495#define GDMA_DRV_CAP_FLAGS3 0
    496
    497#define GDMA_DRV_CAP_FLAGS4 0
    498
    499struct gdma_verify_ver_req {
    500	struct gdma_req_hdr hdr;
    501
    502	/* Mandatory fields required for protocol establishment */
    503	u64 protocol_ver_min;
    504	u64 protocol_ver_max;
    505
    506	/* Gdma Driver Capability Flags */
    507	u64 gd_drv_cap_flags1;
    508	u64 gd_drv_cap_flags2;
    509	u64 gd_drv_cap_flags3;
    510	u64 gd_drv_cap_flags4;
    511
    512	/* Advisory fields */
    513	u64 drv_ver;
    514	u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
    515	u32 reserved;
    516	u32 os_ver_major;
    517	u32 os_ver_minor;
    518	u32 os_ver_build;
    519	u32 os_ver_platform;
    520	u64 reserved_2;
    521	u8 os_ver_str1[128];
    522	u8 os_ver_str2[128];
    523	u8 os_ver_str3[128];
    524	u8 os_ver_str4[128];
    525}; /* HW DATA */
    526
    527struct gdma_verify_ver_resp {
    528	struct gdma_resp_hdr hdr;
    529	u64 gdma_protocol_ver;
    530	u64 pf_cap_flags1;
    531	u64 pf_cap_flags2;
    532	u64 pf_cap_flags3;
    533	u64 pf_cap_flags4;
    534}; /* HW DATA */
    535
    536/* GDMA_QUERY_MAX_RESOURCES */
    537struct gdma_query_max_resources_resp {
    538	struct gdma_resp_hdr hdr;
    539	u32 status;
    540	u32 max_sq;
    541	u32 max_rq;
    542	u32 max_cq;
    543	u32 max_eq;
    544	u32 max_db;
    545	u32 max_mst;
    546	u32 max_cq_mod_ctx;
    547	u32 max_mod_cq;
    548	u32 max_msix;
    549}; /* HW DATA */
    550
    551/* GDMA_LIST_DEVICES */
    552struct gdma_list_devices_resp {
    553	struct gdma_resp_hdr hdr;
    554	u32 num_of_devs;
    555	u32 reserved;
    556	struct gdma_dev_id devs[64];
    557}; /* HW DATA */
    558
    559/* GDMA_REGISTER_DEVICE */
    560struct gdma_register_device_resp {
    561	struct gdma_resp_hdr hdr;
    562	u32 pdid;
    563	u32 gpa_mkey;
    564	u32 db_id;
    565}; /* HW DATA */
    566
    567/* GDMA_CREATE_QUEUE */
    568struct gdma_create_queue_req {
    569	struct gdma_req_hdr hdr;
    570	u32 type;
    571	u32 reserved1;
    572	u32 pdid;
    573	u32 doolbell_id;
    574	u64 gdma_region;
    575	u32 reserved2;
    576	u32 queue_size;
    577	u32 log2_throttle_limit;
    578	u32 eq_pci_msix_index;
    579	u32 cq_mod_ctx_id;
    580	u32 cq_parent_eq_id;
    581	u8  rq_drop_on_overrun;
    582	u8  rq_err_on_wqe_overflow;
    583	u8  rq_chain_rec_wqes;
    584	u8  sq_hw_db;
    585	u32 reserved3;
    586}; /* HW DATA */
    587
    588struct gdma_create_queue_resp {
    589	struct gdma_resp_hdr hdr;
    590	u32 queue_index;
    591}; /* HW DATA */
    592
    593/* GDMA_DISABLE_QUEUE */
    594struct gdma_disable_queue_req {
    595	struct gdma_req_hdr hdr;
    596	u32 type;
    597	u32 queue_index;
    598	u32 alloc_res_id_on_creation;
    599}; /* HW DATA */
    600
    601/* GDMA_CREATE_DMA_REGION */
    602struct gdma_create_dma_region_req {
    603	struct gdma_req_hdr hdr;
    604
    605	/* The total size of the DMA region */
    606	u64 length;
    607
    608	/* The offset in the first page */
    609	u32 offset_in_page;
    610
    611	/* enum gdma_page_type */
    612	u32 gdma_page_type;
    613
    614	/* The total number of pages */
    615	u32 page_count;
    616
    617	/* If page_addr_list_len is smaller than page_count,
    618	 * the remaining page addresses will be added via the
    619	 * message GDMA_DMA_REGION_ADD_PAGES.
    620	 */
    621	u32 page_addr_list_len;
    622	u64 page_addr_list[];
    623}; /* HW DATA */
    624
    625struct gdma_create_dma_region_resp {
    626	struct gdma_resp_hdr hdr;
    627	u64 gdma_region;
    628}; /* HW DATA */
    629
    630/* GDMA_DMA_REGION_ADD_PAGES */
    631struct gdma_dma_region_add_pages_req {
    632	struct gdma_req_hdr hdr;
    633
    634	u64 gdma_region;
    635
    636	u32 page_addr_list_len;
    637	u32 reserved3;
    638
    639	u64 page_addr_list[];
    640}; /* HW DATA */
    641
    642/* GDMA_DESTROY_DMA_REGION */
    643struct gdma_destroy_dma_region_req {
    644	struct gdma_req_hdr hdr;
    645
    646	u64 gdma_region;
    647}; /* HW DATA */
    648
    649int mana_gd_verify_vf_version(struct pci_dev *pdev);
    650
    651int mana_gd_register_device(struct gdma_dev *gd);
    652int mana_gd_deregister_device(struct gdma_dev *gd);
    653
    654int mana_gd_post_work_request(struct gdma_queue *wq,
    655			      const struct gdma_wqe_request *wqe_req,
    656			      struct gdma_posted_wqe_info *wqe_info);
    657
    658int mana_gd_post_and_ring(struct gdma_queue *queue,
    659			  const struct gdma_wqe_request *wqe,
    660			  struct gdma_posted_wqe_info *wqe_info);
    661
    662int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
    663void mana_gd_free_res_map(struct gdma_resource *r);
    664
    665void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
    666			      struct gdma_queue *queue);
    667
    668int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
    669			 struct gdma_mem_info *gmi);
    670
    671void mana_gd_free_memory(struct gdma_mem_info *gmi);
    672
    673int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
    674			 u32 resp_len, void *resp);
    675#endif /* _GDMA_H */