gdma.h (14052B)
1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ 2/* Copyright (c) 2021, Microsoft Corporation. */ 3 4#ifndef _GDMA_H 5#define _GDMA_H 6 7#include <linux/dma-mapping.h> 8#include <linux/netdevice.h> 9 10#include "shm_channel.h" 11 12/* Structures labeled with "HW DATA" are exchanged with the hardware. All of 13 * them are naturally aligned and hence don't need __packed. 14 */ 15 16enum gdma_request_type { 17 GDMA_VERIFY_VF_DRIVER_VERSION = 1, 18 GDMA_QUERY_MAX_RESOURCES = 2, 19 GDMA_LIST_DEVICES = 3, 20 GDMA_REGISTER_DEVICE = 4, 21 GDMA_DEREGISTER_DEVICE = 5, 22 GDMA_GENERATE_TEST_EQE = 10, 23 GDMA_CREATE_QUEUE = 12, 24 GDMA_DISABLE_QUEUE = 13, 25 GDMA_CREATE_DMA_REGION = 25, 26 GDMA_DMA_REGION_ADD_PAGES = 26, 27 GDMA_DESTROY_DMA_REGION = 27, 28}; 29 30enum gdma_queue_type { 31 GDMA_INVALID_QUEUE, 32 GDMA_SQ, 33 GDMA_RQ, 34 GDMA_CQ, 35 GDMA_EQ, 36}; 37 38enum gdma_work_request_flags { 39 GDMA_WR_NONE = 0, 40 GDMA_WR_OOB_IN_SGL = BIT(0), 41 GDMA_WR_PAD_BY_SGE0 = BIT(1), 42}; 43 44enum gdma_eqe_type { 45 GDMA_EQE_COMPLETION = 3, 46 GDMA_EQE_TEST_EVENT = 64, 47 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129, 48 GDMA_EQE_HWC_INIT_DATA = 130, 49 GDMA_EQE_HWC_INIT_DONE = 131, 50}; 51 52enum { 53 GDMA_DEVICE_NONE = 0, 54 GDMA_DEVICE_HWC = 1, 55 GDMA_DEVICE_MANA = 2, 56}; 57 58struct gdma_resource { 59 /* Protect the bitmap */ 60 spinlock_t lock; 61 62 /* The bitmap size in bits. */ 63 u32 size; 64 65 /* The bitmap tracks the resources. */ 66 unsigned long *map; 67}; 68 69union gdma_doorbell_entry { 70 u64 as_uint64; 71 72 struct { 73 u64 id : 24; 74 u64 reserved : 8; 75 u64 tail_ptr : 31; 76 u64 arm : 1; 77 } cq; 78 79 struct { 80 u64 id : 24; 81 u64 wqe_cnt : 8; 82 u64 tail_ptr : 32; 83 } rq; 84 85 struct { 86 u64 id : 24; 87 u64 reserved : 8; 88 u64 tail_ptr : 32; 89 } sq; 90 91 struct { 92 u64 id : 16; 93 u64 reserved : 16; 94 u64 tail_ptr : 31; 95 u64 arm : 1; 96 } eq; 97}; /* HW DATA */ 98 99struct gdma_msg_hdr { 100 u32 hdr_type; 101 u32 msg_type; 102 u16 msg_version; 103 u16 hwc_msg_id; 104 u32 msg_size; 105}; /* HW DATA */ 106 107struct gdma_dev_id { 108 union { 109 struct { 110 u16 type; 111 u16 instance; 112 }; 113 114 u32 as_uint32; 115 }; 116}; /* HW DATA */ 117 118struct gdma_req_hdr { 119 struct gdma_msg_hdr req; 120 struct gdma_msg_hdr resp; /* The expected response */ 121 struct gdma_dev_id dev_id; 122 u32 activity_id; 123}; /* HW DATA */ 124 125struct gdma_resp_hdr { 126 struct gdma_msg_hdr response; 127 struct gdma_dev_id dev_id; 128 u32 activity_id; 129 u32 status; 130 u32 reserved; 131}; /* HW DATA */ 132 133struct gdma_general_req { 134 struct gdma_req_hdr hdr; 135}; /* HW DATA */ 136 137#define GDMA_MESSAGE_V1 1 138 139struct gdma_general_resp { 140 struct gdma_resp_hdr hdr; 141}; /* HW DATA */ 142 143#define GDMA_STANDARD_HEADER_TYPE 0 144 145static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code, 146 u32 req_size, u32 resp_size) 147{ 148 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE; 149 hdr->req.msg_type = code; 150 hdr->req.msg_version = GDMA_MESSAGE_V1; 151 hdr->req.msg_size = req_size; 152 153 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE; 154 hdr->resp.msg_type = code; 155 hdr->resp.msg_version = GDMA_MESSAGE_V1; 156 hdr->resp.msg_size = resp_size; 157} 158 159/* The 16-byte struct is part of the GDMA work queue entry (WQE). */ 160struct gdma_sge { 161 u64 address; 162 u32 mem_key; 163 u32 size; 164}; /* HW DATA */ 165 166struct gdma_wqe_request { 167 struct gdma_sge *sgl; 168 u32 num_sge; 169 170 u32 inline_oob_size; 171 const void *inline_oob_data; 172 173 u32 flags; 174 u32 client_data_unit; 175}; 176 177enum gdma_page_type { 178 GDMA_PAGE_TYPE_4K, 179}; 180 181#define GDMA_INVALID_DMA_REGION 0 182 183struct gdma_mem_info { 184 struct device *dev; 185 186 dma_addr_t dma_handle; 187 void *virt_addr; 188 u64 length; 189 190 /* Allocated by the PF driver */ 191 u64 gdma_region; 192}; 193 194#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8 195 196struct gdma_dev { 197 struct gdma_context *gdma_context; 198 199 struct gdma_dev_id dev_id; 200 201 u32 pdid; 202 u32 doorbell; 203 u32 gpa_mkey; 204 205 /* GDMA driver specific pointer */ 206 void *driver_data; 207}; 208 209#define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE 210 211#define GDMA_CQE_SIZE 64 212#define GDMA_EQE_SIZE 16 213#define GDMA_MAX_SQE_SIZE 512 214#define GDMA_MAX_RQE_SIZE 256 215 216#define GDMA_COMP_DATA_SIZE 0x3C 217 218#define GDMA_EVENT_DATA_SIZE 0xC 219 220/* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */ 221#define GDMA_WQE_BU_SIZE 32 222 223#define INVALID_PDID UINT_MAX 224#define INVALID_DOORBELL UINT_MAX 225#define INVALID_MEM_KEY UINT_MAX 226#define INVALID_QUEUE_ID UINT_MAX 227#define INVALID_PCI_MSIX_INDEX UINT_MAX 228 229struct gdma_comp { 230 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4]; 231 u32 wq_num; 232 bool is_sq; 233}; 234 235struct gdma_event { 236 u32 details[GDMA_EVENT_DATA_SIZE / 4]; 237 u8 type; 238}; 239 240struct gdma_queue; 241 242struct mana_eq { 243 struct gdma_queue *eq; 244}; 245 246typedef void gdma_eq_callback(void *context, struct gdma_queue *q, 247 struct gdma_event *e); 248 249typedef void gdma_cq_callback(void *context, struct gdma_queue *q); 250 251/* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE 252 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the 253 * driver increases the 'head' in BUs rather than in bytes, and notifies 254 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track 255 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE. 256 * 257 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is 258 * processed, the driver increases the 'tail' to indicate that WQEs have 259 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ. 260 * 261 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures 262 * that the EQ/CQ is big enough so they can't overflow, and the driver uses 263 * the owner bits mechanism to detect if the queue has become empty. 264 */ 265struct gdma_queue { 266 struct gdma_dev *gdma_dev; 267 268 enum gdma_queue_type type; 269 u32 id; 270 271 struct gdma_mem_info mem_info; 272 273 void *queue_mem_ptr; 274 u32 queue_size; 275 276 bool monitor_avl_buf; 277 278 u32 head; 279 u32 tail; 280 281 /* Extra fields specific to EQ/CQ. */ 282 union { 283 struct { 284 bool disable_needed; 285 286 gdma_eq_callback *callback; 287 void *context; 288 289 unsigned int msix_index; 290 291 u32 log2_throttle_limit; 292 } eq; 293 294 struct { 295 gdma_cq_callback *callback; 296 void *context; 297 298 struct gdma_queue *parent; /* For CQ/EQ relationship */ 299 } cq; 300 }; 301}; 302 303struct gdma_queue_spec { 304 enum gdma_queue_type type; 305 bool monitor_avl_buf; 306 unsigned int queue_size; 307 308 /* Extra fields specific to EQ/CQ. */ 309 union { 310 struct { 311 gdma_eq_callback *callback; 312 void *context; 313 314 unsigned long log2_throttle_limit; 315 } eq; 316 317 struct { 318 gdma_cq_callback *callback; 319 void *context; 320 321 struct gdma_queue *parent_eq; 322 323 } cq; 324 }; 325}; 326 327struct gdma_irq_context { 328 void (*handler)(void *arg); 329 void *arg; 330}; 331 332struct gdma_context { 333 struct device *dev; 334 335 /* Per-vPort max number of queues */ 336 unsigned int max_num_queues; 337 unsigned int max_num_msix; 338 unsigned int num_msix_usable; 339 struct gdma_resource msix_resource; 340 struct gdma_irq_context *irq_contexts; 341 342 /* This maps a CQ index to the queue structure. */ 343 unsigned int max_num_cqs; 344 struct gdma_queue **cq_table; 345 346 /* Protect eq_test_event and test_event_eq_id */ 347 struct mutex eq_test_event_mutex; 348 struct completion eq_test_event; 349 u32 test_event_eq_id; 350 351 void __iomem *bar0_va; 352 void __iomem *shm_base; 353 void __iomem *db_page_base; 354 u32 db_page_size; 355 356 /* Shared memory chanenl (used to bootstrap HWC) */ 357 struct shm_channel shm_channel; 358 359 /* Hardware communication channel (HWC) */ 360 struct gdma_dev hwc; 361 362 /* Azure network adapter */ 363 struct gdma_dev mana; 364}; 365 366#define MAX_NUM_GDMA_DEVICES 4 367 368static inline bool mana_gd_is_mana(struct gdma_dev *gd) 369{ 370 return gd->dev_id.type == GDMA_DEVICE_MANA; 371} 372 373static inline bool mana_gd_is_hwc(struct gdma_dev *gd) 374{ 375 return gd->dev_id.type == GDMA_DEVICE_HWC; 376} 377 378u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset); 379u32 mana_gd_wq_avail_space(struct gdma_queue *wq); 380 381int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq); 382 383int mana_gd_create_hwc_queue(struct gdma_dev *gd, 384 const struct gdma_queue_spec *spec, 385 struct gdma_queue **queue_ptr); 386 387int mana_gd_create_mana_eq(struct gdma_dev *gd, 388 const struct gdma_queue_spec *spec, 389 struct gdma_queue **queue_ptr); 390 391int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, 392 const struct gdma_queue_spec *spec, 393 struct gdma_queue **queue_ptr); 394 395void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue); 396 397int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe); 398 399void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit); 400 401struct gdma_wqe { 402 u32 reserved :24; 403 u32 last_vbytes :8; 404 405 union { 406 u32 flags; 407 408 struct { 409 u32 num_sge :8; 410 u32 inline_oob_size_div4:3; 411 u32 client_oob_in_sgl :1; 412 u32 reserved1 :4; 413 u32 client_data_unit :14; 414 u32 reserved2 :2; 415 }; 416 }; 417}; /* HW DATA */ 418 419#define INLINE_OOB_SMALL_SIZE 8 420#define INLINE_OOB_LARGE_SIZE 24 421 422#define MAX_TX_WQE_SIZE 512 423#define MAX_RX_WQE_SIZE 256 424 425struct gdma_cqe { 426 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4]; 427 428 union { 429 u32 as_uint32; 430 431 struct { 432 u32 wq_num : 24; 433 u32 is_sq : 1; 434 u32 reserved : 4; 435 u32 owner_bits : 3; 436 }; 437 } cqe_info; 438}; /* HW DATA */ 439 440#define GDMA_CQE_OWNER_BITS 3 441 442#define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1) 443 444#define SET_ARM_BIT 1 445 446#define GDMA_EQE_OWNER_BITS 3 447 448union gdma_eqe_info { 449 u32 as_uint32; 450 451 struct { 452 u32 type : 8; 453 u32 reserved1 : 8; 454 u32 client_id : 2; 455 u32 reserved2 : 11; 456 u32 owner_bits : 3; 457 }; 458}; /* HW DATA */ 459 460#define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1) 461#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries)) 462 463struct gdma_eqe { 464 u32 details[GDMA_EVENT_DATA_SIZE / 4]; 465 u32 eqe_info; 466}; /* HW DATA */ 467 468#define GDMA_REG_DB_PAGE_OFFSET 8 469#define GDMA_REG_DB_PAGE_SIZE 0x10 470#define GDMA_REG_SHM_OFFSET 0x18 471 472struct gdma_posted_wqe_info { 473 u32 wqe_size_in_bu; 474}; 475 476/* GDMA_GENERATE_TEST_EQE */ 477struct gdma_generate_test_event_req { 478 struct gdma_req_hdr hdr; 479 u32 queue_index; 480}; /* HW DATA */ 481 482/* GDMA_VERIFY_VF_DRIVER_VERSION */ 483enum { 484 GDMA_PROTOCOL_V1 = 1, 485 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1, 486 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1, 487}; 488 489#define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0) 490 491#define GDMA_DRV_CAP_FLAGS1 GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT 492 493#define GDMA_DRV_CAP_FLAGS2 0 494 495#define GDMA_DRV_CAP_FLAGS3 0 496 497#define GDMA_DRV_CAP_FLAGS4 0 498 499struct gdma_verify_ver_req { 500 struct gdma_req_hdr hdr; 501 502 /* Mandatory fields required for protocol establishment */ 503 u64 protocol_ver_min; 504 u64 protocol_ver_max; 505 506 /* Gdma Driver Capability Flags */ 507 u64 gd_drv_cap_flags1; 508 u64 gd_drv_cap_flags2; 509 u64 gd_drv_cap_flags3; 510 u64 gd_drv_cap_flags4; 511 512 /* Advisory fields */ 513 u64 drv_ver; 514 u32 os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */ 515 u32 reserved; 516 u32 os_ver_major; 517 u32 os_ver_minor; 518 u32 os_ver_build; 519 u32 os_ver_platform; 520 u64 reserved_2; 521 u8 os_ver_str1[128]; 522 u8 os_ver_str2[128]; 523 u8 os_ver_str3[128]; 524 u8 os_ver_str4[128]; 525}; /* HW DATA */ 526 527struct gdma_verify_ver_resp { 528 struct gdma_resp_hdr hdr; 529 u64 gdma_protocol_ver; 530 u64 pf_cap_flags1; 531 u64 pf_cap_flags2; 532 u64 pf_cap_flags3; 533 u64 pf_cap_flags4; 534}; /* HW DATA */ 535 536/* GDMA_QUERY_MAX_RESOURCES */ 537struct gdma_query_max_resources_resp { 538 struct gdma_resp_hdr hdr; 539 u32 status; 540 u32 max_sq; 541 u32 max_rq; 542 u32 max_cq; 543 u32 max_eq; 544 u32 max_db; 545 u32 max_mst; 546 u32 max_cq_mod_ctx; 547 u32 max_mod_cq; 548 u32 max_msix; 549}; /* HW DATA */ 550 551/* GDMA_LIST_DEVICES */ 552struct gdma_list_devices_resp { 553 struct gdma_resp_hdr hdr; 554 u32 num_of_devs; 555 u32 reserved; 556 struct gdma_dev_id devs[64]; 557}; /* HW DATA */ 558 559/* GDMA_REGISTER_DEVICE */ 560struct gdma_register_device_resp { 561 struct gdma_resp_hdr hdr; 562 u32 pdid; 563 u32 gpa_mkey; 564 u32 db_id; 565}; /* HW DATA */ 566 567/* GDMA_CREATE_QUEUE */ 568struct gdma_create_queue_req { 569 struct gdma_req_hdr hdr; 570 u32 type; 571 u32 reserved1; 572 u32 pdid; 573 u32 doolbell_id; 574 u64 gdma_region; 575 u32 reserved2; 576 u32 queue_size; 577 u32 log2_throttle_limit; 578 u32 eq_pci_msix_index; 579 u32 cq_mod_ctx_id; 580 u32 cq_parent_eq_id; 581 u8 rq_drop_on_overrun; 582 u8 rq_err_on_wqe_overflow; 583 u8 rq_chain_rec_wqes; 584 u8 sq_hw_db; 585 u32 reserved3; 586}; /* HW DATA */ 587 588struct gdma_create_queue_resp { 589 struct gdma_resp_hdr hdr; 590 u32 queue_index; 591}; /* HW DATA */ 592 593/* GDMA_DISABLE_QUEUE */ 594struct gdma_disable_queue_req { 595 struct gdma_req_hdr hdr; 596 u32 type; 597 u32 queue_index; 598 u32 alloc_res_id_on_creation; 599}; /* HW DATA */ 600 601/* GDMA_CREATE_DMA_REGION */ 602struct gdma_create_dma_region_req { 603 struct gdma_req_hdr hdr; 604 605 /* The total size of the DMA region */ 606 u64 length; 607 608 /* The offset in the first page */ 609 u32 offset_in_page; 610 611 /* enum gdma_page_type */ 612 u32 gdma_page_type; 613 614 /* The total number of pages */ 615 u32 page_count; 616 617 /* If page_addr_list_len is smaller than page_count, 618 * the remaining page addresses will be added via the 619 * message GDMA_DMA_REGION_ADD_PAGES. 620 */ 621 u32 page_addr_list_len; 622 u64 page_addr_list[]; 623}; /* HW DATA */ 624 625struct gdma_create_dma_region_resp { 626 struct gdma_resp_hdr hdr; 627 u64 gdma_region; 628}; /* HW DATA */ 629 630/* GDMA_DMA_REGION_ADD_PAGES */ 631struct gdma_dma_region_add_pages_req { 632 struct gdma_req_hdr hdr; 633 634 u64 gdma_region; 635 636 u32 page_addr_list_len; 637 u32 reserved3; 638 639 u64 page_addr_list[]; 640}; /* HW DATA */ 641 642/* GDMA_DESTROY_DMA_REGION */ 643struct gdma_destroy_dma_region_req { 644 struct gdma_req_hdr hdr; 645 646 u64 gdma_region; 647}; /* HW DATA */ 648 649int mana_gd_verify_vf_version(struct pci_dev *pdev); 650 651int mana_gd_register_device(struct gdma_dev *gd); 652int mana_gd_deregister_device(struct gdma_dev *gd); 653 654int mana_gd_post_work_request(struct gdma_queue *wq, 655 const struct gdma_wqe_request *wqe_req, 656 struct gdma_posted_wqe_info *wqe_info); 657 658int mana_gd_post_and_ring(struct gdma_queue *queue, 659 const struct gdma_wqe_request *wqe, 660 struct gdma_posted_wqe_info *wqe_info); 661 662int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r); 663void mana_gd_free_res_map(struct gdma_resource *r); 664 665void mana_gd_wq_ring_doorbell(struct gdma_context *gc, 666 struct gdma_queue *queue); 667 668int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, 669 struct gdma_mem_info *gmi); 670 671void mana_gd_free_memory(struct gdma_mem_info *gmi); 672 673int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, 674 u32 resp_len, void *resp); 675#endif /* _GDMA_H */