cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

hal.c (42253B)


      1// SPDX-License-Identifier: BSD-3-Clause-Clear
      2/*
      3 * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
      4 * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
      5 */
      6#include <linux/dma-mapping.h>
      7#include "hal_tx.h"
      8#include "debug.h"
      9#include "hal_desc.h"
     10#include "hif.h"
     11
     12static const struct hal_srng_config hw_srng_config_template[] = {
     13	/* TODO: max_rings can populated by querying HW capabilities */
     14	{ /* REO_DST */
     15		.start_ring_id = HAL_SRNG_RING_ID_REO2SW1,
     16		.max_rings = 4,
     17		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
     18		.lmac_ring = false,
     19		.ring_dir = HAL_SRNG_DIR_DST,
     20		.max_size = HAL_REO_REO2SW1_RING_BASE_MSB_RING_SIZE,
     21	},
     22	{ /* REO_EXCEPTION */
     23		/* Designating REO2TCL ring as exception ring. This ring is
     24		 * similar to other REO2SW rings though it is named as REO2TCL.
     25		 * Any of theREO2SW rings can be used as exception ring.
     26		 */
     27		.start_ring_id = HAL_SRNG_RING_ID_REO2TCL,
     28		.max_rings = 1,
     29		.entry_size = sizeof(struct hal_reo_dest_ring) >> 2,
     30		.lmac_ring = false,
     31		.ring_dir = HAL_SRNG_DIR_DST,
     32		.max_size = HAL_REO_REO2TCL_RING_BASE_MSB_RING_SIZE,
     33	},
     34	{ /* REO_REINJECT */
     35		.start_ring_id = HAL_SRNG_RING_ID_SW2REO,
     36		.max_rings = 1,
     37		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
     38		.lmac_ring = false,
     39		.ring_dir = HAL_SRNG_DIR_SRC,
     40		.max_size = HAL_REO_SW2REO_RING_BASE_MSB_RING_SIZE,
     41	},
     42	{ /* REO_CMD */
     43		.start_ring_id = HAL_SRNG_RING_ID_REO_CMD,
     44		.max_rings = 1,
     45		.entry_size = (sizeof(struct hal_tlv_hdr) +
     46			sizeof(struct hal_reo_get_queue_stats)) >> 2,
     47		.lmac_ring = false,
     48		.ring_dir = HAL_SRNG_DIR_SRC,
     49		.max_size = HAL_REO_CMD_RING_BASE_MSB_RING_SIZE,
     50	},
     51	{ /* REO_STATUS */
     52		.start_ring_id = HAL_SRNG_RING_ID_REO_STATUS,
     53		.max_rings = 1,
     54		.entry_size = (sizeof(struct hal_tlv_hdr) +
     55			sizeof(struct hal_reo_get_queue_stats_status)) >> 2,
     56		.lmac_ring = false,
     57		.ring_dir = HAL_SRNG_DIR_DST,
     58		.max_size = HAL_REO_STATUS_RING_BASE_MSB_RING_SIZE,
     59	},
     60	{ /* TCL_DATA */
     61		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL1,
     62		.max_rings = 3,
     63		.entry_size = (sizeof(struct hal_tlv_hdr) +
     64			     sizeof(struct hal_tcl_data_cmd)) >> 2,
     65		.lmac_ring = false,
     66		.ring_dir = HAL_SRNG_DIR_SRC,
     67		.max_size = HAL_SW2TCL1_RING_BASE_MSB_RING_SIZE,
     68	},
     69	{ /* TCL_CMD */
     70		.start_ring_id = HAL_SRNG_RING_ID_SW2TCL_CMD,
     71		.max_rings = 1,
     72		.entry_size = (sizeof(struct hal_tlv_hdr) +
     73			     sizeof(struct hal_tcl_gse_cmd)) >> 2,
     74		.lmac_ring =  false,
     75		.ring_dir = HAL_SRNG_DIR_SRC,
     76		.max_size = HAL_SW2TCL1_CMD_RING_BASE_MSB_RING_SIZE,
     77	},
     78	{ /* TCL_STATUS */
     79		.start_ring_id = HAL_SRNG_RING_ID_TCL_STATUS,
     80		.max_rings = 1,
     81		.entry_size = (sizeof(struct hal_tlv_hdr) +
     82			     sizeof(struct hal_tcl_status_ring)) >> 2,
     83		.lmac_ring = false,
     84		.ring_dir = HAL_SRNG_DIR_DST,
     85		.max_size = HAL_TCL_STATUS_RING_BASE_MSB_RING_SIZE,
     86	},
     87	{ /* CE_SRC */
     88		.start_ring_id = HAL_SRNG_RING_ID_CE0_SRC,
     89		.max_rings = 12,
     90		.entry_size = sizeof(struct hal_ce_srng_src_desc) >> 2,
     91		.lmac_ring = false,
     92		.ring_dir = HAL_SRNG_DIR_SRC,
     93		.max_size = HAL_CE_SRC_RING_BASE_MSB_RING_SIZE,
     94	},
     95	{ /* CE_DST */
     96		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST,
     97		.max_rings = 12,
     98		.entry_size = sizeof(struct hal_ce_srng_dest_desc) >> 2,
     99		.lmac_ring = false,
    100		.ring_dir = HAL_SRNG_DIR_SRC,
    101		.max_size = HAL_CE_DST_RING_BASE_MSB_RING_SIZE,
    102	},
    103	{ /* CE_DST_STATUS */
    104		.start_ring_id = HAL_SRNG_RING_ID_CE0_DST_STATUS,
    105		.max_rings = 12,
    106		.entry_size = sizeof(struct hal_ce_srng_dst_status_desc) >> 2,
    107		.lmac_ring = false,
    108		.ring_dir = HAL_SRNG_DIR_DST,
    109		.max_size = HAL_CE_DST_STATUS_RING_BASE_MSB_RING_SIZE,
    110	},
    111	{ /* WBM_IDLE_LINK */
    112		.start_ring_id = HAL_SRNG_RING_ID_WBM_IDLE_LINK,
    113		.max_rings = 1,
    114		.entry_size = sizeof(struct hal_wbm_link_desc) >> 2,
    115		.lmac_ring = false,
    116		.ring_dir = HAL_SRNG_DIR_SRC,
    117		.max_size = HAL_WBM_IDLE_LINK_RING_BASE_MSB_RING_SIZE,
    118	},
    119	{ /* SW2WBM_RELEASE */
    120		.start_ring_id = HAL_SRNG_RING_ID_WBM_SW_RELEASE,
    121		.max_rings = 1,
    122		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
    123		.lmac_ring = false,
    124		.ring_dir = HAL_SRNG_DIR_SRC,
    125		.max_size = HAL_SW2WBM_RELEASE_RING_BASE_MSB_RING_SIZE,
    126	},
    127	{ /* WBM2SW_RELEASE */
    128		.start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,
    129		.max_rings = 4,
    130		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
    131		.lmac_ring = false,
    132		.ring_dir = HAL_SRNG_DIR_DST,
    133		.max_size = HAL_WBM2SW_RELEASE_RING_BASE_MSB_RING_SIZE,
    134	},
    135	{ /* RXDMA_BUF */
    136		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA0_BUF,
    137		.max_rings = 2,
    138		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
    139		.lmac_ring = true,
    140		.ring_dir = HAL_SRNG_DIR_SRC,
    141		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    142	},
    143	{ /* RXDMA_DST */
    144		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW0,
    145		.max_rings = 1,
    146		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
    147		.lmac_ring = true,
    148		.ring_dir = HAL_SRNG_DIR_DST,
    149		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    150	},
    151	{ /* RXDMA_MONITOR_BUF */
    152		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA2_BUF,
    153		.max_rings = 1,
    154		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
    155		.lmac_ring = true,
    156		.ring_dir = HAL_SRNG_DIR_SRC,
    157		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    158	},
    159	{ /* RXDMA_MONITOR_STATUS */
    160		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_STATBUF,
    161		.max_rings = 1,
    162		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
    163		.lmac_ring = true,
    164		.ring_dir = HAL_SRNG_DIR_SRC,
    165		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    166	},
    167	{ /* RXDMA_MONITOR_DST */
    168		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_RXDMA2SW1,
    169		.max_rings = 1,
    170		.entry_size = sizeof(struct hal_reo_entrance_ring) >> 2,
    171		.lmac_ring = true,
    172		.ring_dir = HAL_SRNG_DIR_DST,
    173		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    174	},
    175	{ /* RXDMA_MONITOR_DESC */
    176		.start_ring_id = HAL_SRNG_RING_ID_WMAC1_SW2RXDMA1_DESC,
    177		.max_rings = 1,
    178		.entry_size = sizeof(struct hal_wbm_buffer_ring) >> 2,
    179		.lmac_ring = true,
    180		.ring_dir = HAL_SRNG_DIR_SRC,
    181		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    182	},
    183	{ /* RXDMA DIR BUF */
    184		.start_ring_id = HAL_SRNG_RING_ID_RXDMA_DIR_BUF,
    185		.max_rings = 1,
    186		.entry_size = 8 >> 2, /* TODO: Define the struct */
    187		.lmac_ring = true,
    188		.ring_dir = HAL_SRNG_DIR_SRC,
    189		.max_size = HAL_RXDMA_RING_MAX_SIZE,
    190	},
    191};
    192
    193static int ath11k_hal_alloc_cont_rdp(struct ath11k_base *ab)
    194{
    195	struct ath11k_hal *hal = &ab->hal;
    196	size_t size;
    197
    198	size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
    199	hal->rdp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->rdp.paddr,
    200					    GFP_KERNEL);
    201	if (!hal->rdp.vaddr)
    202		return -ENOMEM;
    203
    204	return 0;
    205}
    206
    207static void ath11k_hal_free_cont_rdp(struct ath11k_base *ab)
    208{
    209	struct ath11k_hal *hal = &ab->hal;
    210	size_t size;
    211
    212	if (!hal->rdp.vaddr)
    213		return;
    214
    215	size = sizeof(u32) * HAL_SRNG_RING_ID_MAX;
    216	dma_free_coherent(ab->dev, size,
    217			  hal->rdp.vaddr, hal->rdp.paddr);
    218	hal->rdp.vaddr = NULL;
    219}
    220
    221static int ath11k_hal_alloc_cont_wrp(struct ath11k_base *ab)
    222{
    223	struct ath11k_hal *hal = &ab->hal;
    224	size_t size;
    225
    226	size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS;
    227	hal->wrp.vaddr = dma_alloc_coherent(ab->dev, size, &hal->wrp.paddr,
    228					    GFP_KERNEL);
    229	if (!hal->wrp.vaddr)
    230		return -ENOMEM;
    231
    232	return 0;
    233}
    234
    235static void ath11k_hal_free_cont_wrp(struct ath11k_base *ab)
    236{
    237	struct ath11k_hal *hal = &ab->hal;
    238	size_t size;
    239
    240	if (!hal->wrp.vaddr)
    241		return;
    242
    243	size = sizeof(u32) * HAL_SRNG_NUM_LMAC_RINGS;
    244	dma_free_coherent(ab->dev, size,
    245			  hal->wrp.vaddr, hal->wrp.paddr);
    246	hal->wrp.vaddr = NULL;
    247}
    248
    249static void ath11k_hal_ce_dst_setup(struct ath11k_base *ab,
    250				    struct hal_srng *srng, int ring_num)
    251{
    252	struct hal_srng_config *srng_config = &ab->hal.srng_config[HAL_CE_DST];
    253	u32 addr;
    254	u32 val;
    255
    256	addr = HAL_CE_DST_RING_CTRL +
    257	       srng_config->reg_start[HAL_SRNG_REG_GRP_R0] +
    258	       ring_num * srng_config->reg_size[HAL_SRNG_REG_GRP_R0];
    259
    260	val = ath11k_hif_read32(ab, addr);
    261	val &= ~HAL_CE_DST_R0_DEST_CTRL_MAX_LEN;
    262	val |= FIELD_PREP(HAL_CE_DST_R0_DEST_CTRL_MAX_LEN,
    263			  srng->u.dst_ring.max_buffer_length);
    264	ath11k_hif_write32(ab, addr, val);
    265}
    266
    267static void ath11k_hal_srng_dst_hw_init(struct ath11k_base *ab,
    268					struct hal_srng *srng)
    269{
    270	struct ath11k_hal *hal = &ab->hal;
    271	u32 val;
    272	u64 hp_addr;
    273	u32 reg_base;
    274
    275	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
    276
    277	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
    278		ath11k_hif_write32(ab, reg_base +
    279				   HAL_REO1_RING_MSI1_BASE_LSB_OFFSET(ab),
    280				   srng->msi_addr);
    281
    282		val = FIELD_PREP(HAL_REO1_RING_MSI1_BASE_MSB_ADDR,
    283				 ((u64)srng->msi_addr >>
    284				  HAL_ADDR_MSB_REG_SHIFT)) |
    285		      HAL_REO1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
    286		ath11k_hif_write32(ab, reg_base +
    287				       HAL_REO1_RING_MSI1_BASE_MSB_OFFSET(ab), val);
    288
    289		ath11k_hif_write32(ab,
    290				   reg_base + HAL_REO1_RING_MSI1_DATA_OFFSET(ab),
    291				   srng->msi_data);
    292	}
    293
    294	ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr);
    295
    296	val = FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB,
    297			 ((u64)srng->ring_base_paddr >>
    298			  HAL_ADDR_MSB_REG_SHIFT)) |
    299	      FIELD_PREP(HAL_REO1_RING_BASE_MSB_RING_SIZE,
    300			 (srng->entry_size * srng->num_entries));
    301	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_BASE_MSB_OFFSET(ab), val);
    302
    303	val = FIELD_PREP(HAL_REO1_RING_ID_RING_ID, srng->ring_id) |
    304	      FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size);
    305	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_ID_OFFSET(ab), val);
    306
    307	/* interrupt setup */
    308	val = FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_INTR_TMR_THOLD,
    309			 (srng->intr_timer_thres_us >> 3));
    310
    311	val |= FIELD_PREP(HAL_REO1_RING_PRDR_INT_SETUP_BATCH_COUNTER_THOLD,
    312			  (srng->intr_batch_cntr_thres_entries *
    313			   srng->entry_size));
    314
    315	ath11k_hif_write32(ab,
    316			   reg_base + HAL_REO1_RING_PRODUCER_INT_SETUP_OFFSET(ab),
    317			   val);
    318
    319	hp_addr = hal->rdp.paddr +
    320		  ((unsigned long)srng->u.dst_ring.hp_addr -
    321		   (unsigned long)hal->rdp.vaddr);
    322	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_LSB_OFFSET(ab),
    323			   hp_addr & HAL_ADDR_LSB_REG_MASK);
    324	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_HP_ADDR_MSB_OFFSET(ab),
    325			   hp_addr >> HAL_ADDR_MSB_REG_SHIFT);
    326
    327	/* Initialize head and tail pointers to indicate ring is empty */
    328	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
    329	ath11k_hif_write32(ab, reg_base, 0);
    330	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_TP_OFFSET(ab), 0);
    331	*srng->u.dst_ring.hp_addr = 0;
    332
    333	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
    334	val = 0;
    335	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
    336		val |= HAL_REO1_RING_MISC_DATA_TLV_SWAP;
    337	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
    338		val |= HAL_REO1_RING_MISC_HOST_FW_SWAP;
    339	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
    340		val |= HAL_REO1_RING_MISC_MSI_SWAP;
    341	val |= HAL_REO1_RING_MISC_SRNG_ENABLE;
    342
    343	ath11k_hif_write32(ab, reg_base + HAL_REO1_RING_MISC_OFFSET(ab), val);
    344}
    345
    346static void ath11k_hal_srng_src_hw_init(struct ath11k_base *ab,
    347					struct hal_srng *srng)
    348{
    349	struct ath11k_hal *hal = &ab->hal;
    350	u32 val;
    351	u64 tp_addr;
    352	u32 reg_base;
    353
    354	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
    355
    356	if (srng->flags & HAL_SRNG_FLAGS_MSI_INTR) {
    357		ath11k_hif_write32(ab, reg_base +
    358				   HAL_TCL1_RING_MSI1_BASE_LSB_OFFSET(ab),
    359				   srng->msi_addr);
    360
    361		val = FIELD_PREP(HAL_TCL1_RING_MSI1_BASE_MSB_ADDR,
    362				 ((u64)srng->msi_addr >>
    363				  HAL_ADDR_MSB_REG_SHIFT)) |
    364		      HAL_TCL1_RING_MSI1_BASE_MSB_MSI1_ENABLE;
    365		ath11k_hif_write32(ab, reg_base +
    366				       HAL_TCL1_RING_MSI1_BASE_MSB_OFFSET(ab),
    367				   val);
    368
    369		ath11k_hif_write32(ab, reg_base +
    370				       HAL_TCL1_RING_MSI1_DATA_OFFSET(ab),
    371				   srng->msi_data);
    372	}
    373
    374	ath11k_hif_write32(ab, reg_base, srng->ring_base_paddr);
    375
    376	val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB,
    377			 ((u64)srng->ring_base_paddr >>
    378			  HAL_ADDR_MSB_REG_SHIFT)) |
    379	      FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE,
    380			 (srng->entry_size * srng->num_entries));
    381	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);
    382
    383	val = FIELD_PREP(HAL_REO1_RING_ID_ENTRY_SIZE, srng->entry_size);
    384	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_ID_OFFSET(ab), val);
    385
    386	if (srng->ring_id == HAL_SRNG_RING_ID_WBM_IDLE_LINK) {
    387		ath11k_hif_write32(ab, reg_base, (u32)srng->ring_base_paddr);
    388		val = FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB,
    389				 ((u64)srng->ring_base_paddr >>
    390				 HAL_ADDR_MSB_REG_SHIFT)) |
    391			FIELD_PREP(HAL_TCL1_RING_BASE_MSB_RING_SIZE,
    392				   (srng->entry_size * srng->num_entries));
    393		ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_BASE_MSB_OFFSET(ab), val);
    394	}
    395
    396	/* interrupt setup */
    397	/* NOTE: IPQ8074 v2 requires the interrupt timer threshold in the
    398	 * unit of 8 usecs instead of 1 usec (as required by v1).
    399	 */
    400	val = FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_INTR_TMR_THOLD,
    401			 srng->intr_timer_thres_us);
    402
    403	val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX0_BATCH_COUNTER_THOLD,
    404			  (srng->intr_batch_cntr_thres_entries *
    405			   srng->entry_size));
    406
    407	ath11k_hif_write32(ab,
    408			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX0_OFFSET(ab),
    409			   val);
    410
    411	val = 0;
    412	if (srng->flags & HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN) {
    413		val |= FIELD_PREP(HAL_TCL1_RING_CONSR_INT_SETUP_IX1_LOW_THOLD,
    414				  srng->u.src_ring.low_threshold);
    415	}
    416	ath11k_hif_write32(ab,
    417			   reg_base + HAL_TCL1_RING_CONSR_INT_SETUP_IX1_OFFSET(ab),
    418			   val);
    419
    420	if (srng->ring_id != HAL_SRNG_RING_ID_WBM_IDLE_LINK) {
    421		tp_addr = hal->rdp.paddr +
    422			  ((unsigned long)srng->u.src_ring.tp_addr -
    423			   (unsigned long)hal->rdp.vaddr);
    424		ath11k_hif_write32(ab,
    425				   reg_base + HAL_TCL1_RING_TP_ADDR_LSB_OFFSET(ab),
    426				   tp_addr & HAL_ADDR_LSB_REG_MASK);
    427		ath11k_hif_write32(ab,
    428				   reg_base + HAL_TCL1_RING_TP_ADDR_MSB_OFFSET(ab),
    429				   tp_addr >> HAL_ADDR_MSB_REG_SHIFT);
    430	}
    431
    432	/* Initialize head and tail pointers to indicate ring is empty */
    433	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
    434	ath11k_hif_write32(ab, reg_base, 0);
    435	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_TP_OFFSET, 0);
    436	*srng->u.src_ring.tp_addr = 0;
    437
    438	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R0];
    439	val = 0;
    440	if (srng->flags & HAL_SRNG_FLAGS_DATA_TLV_SWAP)
    441		val |= HAL_TCL1_RING_MISC_DATA_TLV_SWAP;
    442	if (srng->flags & HAL_SRNG_FLAGS_RING_PTR_SWAP)
    443		val |= HAL_TCL1_RING_MISC_HOST_FW_SWAP;
    444	if (srng->flags & HAL_SRNG_FLAGS_MSI_SWAP)
    445		val |= HAL_TCL1_RING_MISC_MSI_SWAP;
    446
    447	/* Loop count is not used for SRC rings */
    448	val |= HAL_TCL1_RING_MISC_MSI_LOOPCNT_DISABLE;
    449
    450	val |= HAL_TCL1_RING_MISC_SRNG_ENABLE;
    451
    452	ath11k_hif_write32(ab, reg_base + HAL_TCL1_RING_MISC_OFFSET(ab), val);
    453}
    454
    455static void ath11k_hal_srng_hw_init(struct ath11k_base *ab,
    456				    struct hal_srng *srng)
    457{
    458	if (srng->ring_dir == HAL_SRNG_DIR_SRC)
    459		ath11k_hal_srng_src_hw_init(ab, srng);
    460	else
    461		ath11k_hal_srng_dst_hw_init(ab, srng);
    462}
    463
    464static int ath11k_hal_srng_get_ring_id(struct ath11k_base *ab,
    465				       enum hal_ring_type type,
    466				       int ring_num, int mac_id)
    467{
    468	struct hal_srng_config *srng_config = &ab->hal.srng_config[type];
    469	int ring_id;
    470
    471	if (ring_num >= srng_config->max_rings) {
    472		ath11k_warn(ab, "invalid ring number :%d\n", ring_num);
    473		return -EINVAL;
    474	}
    475
    476	ring_id = srng_config->start_ring_id + ring_num;
    477	if (srng_config->lmac_ring)
    478		ring_id += mac_id * HAL_SRNG_RINGS_PER_LMAC;
    479
    480	if (WARN_ON(ring_id >= HAL_SRNG_RING_ID_MAX))
    481		return -EINVAL;
    482
    483	return ring_id;
    484}
    485
    486int ath11k_hal_srng_get_entrysize(struct ath11k_base *ab, u32 ring_type)
    487{
    488	struct hal_srng_config *srng_config;
    489
    490	if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))
    491		return -EINVAL;
    492
    493	srng_config = &ab->hal.srng_config[ring_type];
    494
    495	return (srng_config->entry_size << 2);
    496}
    497
    498int ath11k_hal_srng_get_max_entries(struct ath11k_base *ab, u32 ring_type)
    499{
    500	struct hal_srng_config *srng_config;
    501
    502	if (WARN_ON(ring_type >= HAL_MAX_RING_TYPES))
    503		return -EINVAL;
    504
    505	srng_config = &ab->hal.srng_config[ring_type];
    506
    507	return (srng_config->max_size / srng_config->entry_size);
    508}
    509
    510void ath11k_hal_srng_get_params(struct ath11k_base *ab, struct hal_srng *srng,
    511				struct hal_srng_params *params)
    512{
    513	params->ring_base_paddr = srng->ring_base_paddr;
    514	params->ring_base_vaddr = srng->ring_base_vaddr;
    515	params->num_entries = srng->num_entries;
    516	params->intr_timer_thres_us = srng->intr_timer_thres_us;
    517	params->intr_batch_cntr_thres_entries =
    518		srng->intr_batch_cntr_thres_entries;
    519	params->low_threshold = srng->u.src_ring.low_threshold;
    520	params->msi_addr = srng->msi_addr;
    521	params->msi_data = srng->msi_data;
    522	params->flags = srng->flags;
    523}
    524
    525dma_addr_t ath11k_hal_srng_get_hp_addr(struct ath11k_base *ab,
    526				       struct hal_srng *srng)
    527{
    528	if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))
    529		return 0;
    530
    531	if (srng->ring_dir == HAL_SRNG_DIR_SRC)
    532		return ab->hal.wrp.paddr +
    533		       ((unsigned long)srng->u.src_ring.hp_addr -
    534			(unsigned long)ab->hal.wrp.vaddr);
    535	else
    536		return ab->hal.rdp.paddr +
    537		       ((unsigned long)srng->u.dst_ring.hp_addr -
    538			 (unsigned long)ab->hal.rdp.vaddr);
    539}
    540
    541dma_addr_t ath11k_hal_srng_get_tp_addr(struct ath11k_base *ab,
    542				       struct hal_srng *srng)
    543{
    544	if (!(srng->flags & HAL_SRNG_FLAGS_LMAC_RING))
    545		return 0;
    546
    547	if (srng->ring_dir == HAL_SRNG_DIR_SRC)
    548		return ab->hal.rdp.paddr +
    549		       ((unsigned long)srng->u.src_ring.tp_addr -
    550			(unsigned long)ab->hal.rdp.vaddr);
    551	else
    552		return ab->hal.wrp.paddr +
    553		       ((unsigned long)srng->u.dst_ring.tp_addr -
    554			(unsigned long)ab->hal.wrp.vaddr);
    555}
    556
    557u32 ath11k_hal_ce_get_desc_size(enum hal_ce_desc type)
    558{
    559	switch (type) {
    560	case HAL_CE_DESC_SRC:
    561		return sizeof(struct hal_ce_srng_src_desc);
    562	case HAL_CE_DESC_DST:
    563		return sizeof(struct hal_ce_srng_dest_desc);
    564	case HAL_CE_DESC_DST_STATUS:
    565		return sizeof(struct hal_ce_srng_dst_status_desc);
    566	}
    567
    568	return 0;
    569}
    570
    571void ath11k_hal_ce_src_set_desc(void *buf, dma_addr_t paddr, u32 len, u32 id,
    572				u8 byte_swap_data)
    573{
    574	struct hal_ce_srng_src_desc *desc = (struct hal_ce_srng_src_desc *)buf;
    575
    576	desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK;
    577	desc->buffer_addr_info =
    578		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_ADDR_HI,
    579			   ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) |
    580		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_BYTE_SWAP,
    581			   byte_swap_data) |
    582		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_GATHER, 0) |
    583		FIELD_PREP(HAL_CE_SRC_DESC_ADDR_INFO_LEN, len);
    584	desc->meta_info = FIELD_PREP(HAL_CE_SRC_DESC_META_INFO_DATA, id);
    585}
    586
    587void ath11k_hal_ce_dst_set_desc(void *buf, dma_addr_t paddr)
    588{
    589	struct hal_ce_srng_dest_desc *desc =
    590		(struct hal_ce_srng_dest_desc *)buf;
    591
    592	desc->buffer_addr_low = paddr & HAL_ADDR_LSB_REG_MASK;
    593	desc->buffer_addr_info =
    594		FIELD_PREP(HAL_CE_DEST_DESC_ADDR_INFO_ADDR_HI,
    595			   ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT));
    596}
    597
    598u32 ath11k_hal_ce_dst_status_get_length(void *buf)
    599{
    600	struct hal_ce_srng_dst_status_desc *desc =
    601		(struct hal_ce_srng_dst_status_desc *)buf;
    602	u32 len;
    603
    604	len = FIELD_GET(HAL_CE_DST_STATUS_DESC_FLAGS_LEN, desc->flags);
    605	desc->flags &= ~HAL_CE_DST_STATUS_DESC_FLAGS_LEN;
    606
    607	return len;
    608}
    609
    610void ath11k_hal_set_link_desc_addr(struct hal_wbm_link_desc *desc, u32 cookie,
    611				   dma_addr_t paddr)
    612{
    613	desc->buf_addr_info.info0 = FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
    614					       (paddr & HAL_ADDR_LSB_REG_MASK));
    615	desc->buf_addr_info.info1 = FIELD_PREP(BUFFER_ADDR_INFO1_ADDR,
    616					       ((u64)paddr >> HAL_ADDR_MSB_REG_SHIFT)) |
    617				    FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, 1) |
    618				    FIELD_PREP(BUFFER_ADDR_INFO1_SW_COOKIE, cookie);
    619}
    620
    621u32 *ath11k_hal_srng_dst_peek(struct ath11k_base *ab, struct hal_srng *srng)
    622{
    623	lockdep_assert_held(&srng->lock);
    624
    625	if (srng->u.dst_ring.tp != srng->u.dst_ring.cached_hp)
    626		return (srng->ring_base_vaddr + srng->u.dst_ring.tp);
    627
    628	return NULL;
    629}
    630
    631static void ath11k_hal_srng_prefetch_desc(struct ath11k_base *ab,
    632					  struct hal_srng *srng)
    633{
    634	u32 *desc;
    635
    636	/* prefetch only if desc is available */
    637	desc = ath11k_hal_srng_dst_peek(ab, srng);
    638	if (likely(desc)) {
    639		dma_sync_single_for_cpu(ab->dev, virt_to_phys(desc),
    640					(srng->entry_size * sizeof(u32)),
    641					DMA_FROM_DEVICE);
    642		prefetch(desc);
    643	}
    644}
    645
    646u32 *ath11k_hal_srng_dst_get_next_entry(struct ath11k_base *ab,
    647					struct hal_srng *srng)
    648{
    649	u32 *desc;
    650
    651	lockdep_assert_held(&srng->lock);
    652
    653	if (srng->u.dst_ring.tp == srng->u.dst_ring.cached_hp)
    654		return NULL;
    655
    656	desc = srng->ring_base_vaddr + srng->u.dst_ring.tp;
    657
    658	srng->u.dst_ring.tp += srng->entry_size;
    659
    660	/* wrap around to start of ring*/
    661	if (srng->u.dst_ring.tp == srng->ring_size)
    662		srng->u.dst_ring.tp = 0;
    663
    664	/* Try to prefetch the next descriptor in the ring */
    665	if (srng->flags & HAL_SRNG_FLAGS_CACHED)
    666		ath11k_hal_srng_prefetch_desc(ab, srng);
    667
    668	return desc;
    669}
    670
    671int ath11k_hal_srng_dst_num_free(struct ath11k_base *ab, struct hal_srng *srng,
    672				 bool sync_hw_ptr)
    673{
    674	u32 tp, hp;
    675
    676	lockdep_assert_held(&srng->lock);
    677
    678	tp = srng->u.dst_ring.tp;
    679
    680	if (sync_hw_ptr) {
    681		hp = *srng->u.dst_ring.hp_addr;
    682		srng->u.dst_ring.cached_hp = hp;
    683	} else {
    684		hp = srng->u.dst_ring.cached_hp;
    685	}
    686
    687	if (hp >= tp)
    688		return (hp - tp) / srng->entry_size;
    689	else
    690		return (srng->ring_size - tp + hp) / srng->entry_size;
    691}
    692
    693/* Returns number of available entries in src ring */
    694int ath11k_hal_srng_src_num_free(struct ath11k_base *ab, struct hal_srng *srng,
    695				 bool sync_hw_ptr)
    696{
    697	u32 tp, hp;
    698
    699	lockdep_assert_held(&srng->lock);
    700
    701	hp = srng->u.src_ring.hp;
    702
    703	if (sync_hw_ptr) {
    704		tp = *srng->u.src_ring.tp_addr;
    705		srng->u.src_ring.cached_tp = tp;
    706	} else {
    707		tp = srng->u.src_ring.cached_tp;
    708	}
    709
    710	if (tp > hp)
    711		return ((tp - hp) / srng->entry_size) - 1;
    712	else
    713		return ((srng->ring_size - hp + tp) / srng->entry_size) - 1;
    714}
    715
    716u32 *ath11k_hal_srng_src_get_next_entry(struct ath11k_base *ab,
    717					struct hal_srng *srng)
    718{
    719	u32 *desc;
    720	u32 next_hp;
    721
    722	lockdep_assert_held(&srng->lock);
    723
    724	/* TODO: Using % is expensive, but we have to do this since size of some
    725	 * SRNG rings is not power of 2 (due to descriptor sizes). Need to see
    726	 * if separate function is defined for rings having power of 2 ring size
    727	 * (TCL2SW, REO2SW, SW2RXDMA and CE rings) so that we can avoid the
    728	 * overhead of % by using mask (with &).
    729	 */
    730	next_hp = (srng->u.src_ring.hp + srng->entry_size) % srng->ring_size;
    731
    732	if (next_hp == srng->u.src_ring.cached_tp)
    733		return NULL;
    734
    735	desc = srng->ring_base_vaddr + srng->u.src_ring.hp;
    736	srng->u.src_ring.hp = next_hp;
    737
    738	/* TODO: Reap functionality is not used by all rings. If particular
    739	 * ring does not use reap functionality, we need not update reap_hp
    740	 * with next_hp pointer. Need to make sure a separate function is used
    741	 * before doing any optimization by removing below code updating
    742	 * reap_hp.
    743	 */
    744	srng->u.src_ring.reap_hp = next_hp;
    745
    746	return desc;
    747}
    748
    749u32 *ath11k_hal_srng_src_reap_next(struct ath11k_base *ab,
    750				   struct hal_srng *srng)
    751{
    752	u32 *desc;
    753	u32 next_reap_hp;
    754
    755	lockdep_assert_held(&srng->lock);
    756
    757	next_reap_hp = (srng->u.src_ring.reap_hp + srng->entry_size) %
    758		       srng->ring_size;
    759
    760	if (next_reap_hp == srng->u.src_ring.cached_tp)
    761		return NULL;
    762
    763	desc = srng->ring_base_vaddr + next_reap_hp;
    764	srng->u.src_ring.reap_hp = next_reap_hp;
    765
    766	return desc;
    767}
    768
    769u32 *ath11k_hal_srng_src_get_next_reaped(struct ath11k_base *ab,
    770					 struct hal_srng *srng)
    771{
    772	u32 *desc;
    773
    774	lockdep_assert_held(&srng->lock);
    775
    776	if (srng->u.src_ring.hp == srng->u.src_ring.reap_hp)
    777		return NULL;
    778
    779	desc = srng->ring_base_vaddr + srng->u.src_ring.hp;
    780	srng->u.src_ring.hp = (srng->u.src_ring.hp + srng->entry_size) %
    781			      srng->ring_size;
    782
    783	return desc;
    784}
    785
    786u32 *ath11k_hal_srng_src_peek(struct ath11k_base *ab, struct hal_srng *srng)
    787{
    788	lockdep_assert_held(&srng->lock);
    789
    790	if (((srng->u.src_ring.hp + srng->entry_size) % srng->ring_size) ==
    791	    srng->u.src_ring.cached_tp)
    792		return NULL;
    793
    794	return srng->ring_base_vaddr + srng->u.src_ring.hp;
    795}
    796
    797void ath11k_hal_srng_access_begin(struct ath11k_base *ab, struct hal_srng *srng)
    798{
    799	lockdep_assert_held(&srng->lock);
    800
    801	if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
    802		srng->u.src_ring.cached_tp =
    803			*(volatile u32 *)srng->u.src_ring.tp_addr;
    804	} else {
    805		srng->u.dst_ring.cached_hp = *srng->u.dst_ring.hp_addr;
    806
    807		/* Try to prefetch the next descriptor in the ring */
    808		if (srng->flags & HAL_SRNG_FLAGS_CACHED)
    809			ath11k_hal_srng_prefetch_desc(ab, srng);
    810	}
    811}
    812
    813/* Update cached ring head/tail pointers to HW. ath11k_hal_srng_access_begin()
    814 * should have been called before this.
    815 */
    816void ath11k_hal_srng_access_end(struct ath11k_base *ab, struct hal_srng *srng)
    817{
    818	lockdep_assert_held(&srng->lock);
    819
    820	/* TODO: See if we need a write memory barrier here */
    821	if (srng->flags & HAL_SRNG_FLAGS_LMAC_RING) {
    822		/* For LMAC rings, ring pointer updates are done through FW and
    823		 * hence written to a shared memory location that is read by FW
    824		 */
    825		if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
    826			srng->u.src_ring.last_tp =
    827				*(volatile u32 *)srng->u.src_ring.tp_addr;
    828			*srng->u.src_ring.hp_addr = srng->u.src_ring.hp;
    829		} else {
    830			srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;
    831			*srng->u.dst_ring.tp_addr = srng->u.dst_ring.tp;
    832		}
    833	} else {
    834		if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
    835			srng->u.src_ring.last_tp =
    836				*(volatile u32 *)srng->u.src_ring.tp_addr;
    837			ath11k_hif_write32(ab,
    838					   (unsigned long)srng->u.src_ring.hp_addr -
    839					   (unsigned long)ab->mem,
    840					   srng->u.src_ring.hp);
    841		} else {
    842			srng->u.dst_ring.last_hp = *srng->u.dst_ring.hp_addr;
    843			ath11k_hif_write32(ab,
    844					   (unsigned long)srng->u.dst_ring.tp_addr -
    845					   (unsigned long)ab->mem,
    846					   srng->u.dst_ring.tp);
    847		}
    848	}
    849
    850	srng->timestamp = jiffies;
    851}
    852
    853void ath11k_hal_setup_link_idle_list(struct ath11k_base *ab,
    854				     struct hal_wbm_idle_scatter_list *sbuf,
    855				     u32 nsbufs, u32 tot_link_desc,
    856				     u32 end_offset)
    857{
    858	struct ath11k_buffer_addr *link_addr;
    859	int i;
    860	u32 reg_scatter_buf_sz = HAL_WBM_IDLE_SCATTER_BUF_SIZE / 64;
    861
    862	link_addr = (void *)sbuf[0].vaddr + HAL_WBM_IDLE_SCATTER_BUF_SIZE;
    863
    864	for (i = 1; i < nsbufs; i++) {
    865		link_addr->info0 = sbuf[i].paddr & HAL_ADDR_LSB_REG_MASK;
    866		link_addr->info1 = FIELD_PREP(
    867				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
    868				(u64)sbuf[i].paddr >> HAL_ADDR_MSB_REG_SHIFT) |
    869				FIELD_PREP(
    870				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG,
    871				BASE_ADDR_MATCH_TAG_VAL);
    872
    873		link_addr = (void *)sbuf[i].vaddr +
    874			     HAL_WBM_IDLE_SCATTER_BUF_SIZE;
    875	}
    876
    877	ath11k_hif_write32(ab,
    878			   HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_CONTROL_ADDR,
    879			   FIELD_PREP(HAL_WBM_SCATTER_BUFFER_SIZE, reg_scatter_buf_sz) |
    880			   FIELD_PREP(HAL_WBM_LINK_DESC_IDLE_LIST_MODE, 0x1));
    881	ath11k_hif_write32(ab,
    882			   HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_R0_IDLE_LIST_SIZE_ADDR,
    883			   FIELD_PREP(HAL_WBM_SCATTER_RING_SIZE_OF_IDLE_LINK_DESC_LIST,
    884				      reg_scatter_buf_sz * nsbufs));
    885	ath11k_hif_write32(ab,
    886			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    887			   HAL_WBM_SCATTERED_RING_BASE_LSB,
    888			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
    889				      sbuf[0].paddr & HAL_ADDR_LSB_REG_MASK));
    890	ath11k_hif_write32(ab,
    891			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    892			   HAL_WBM_SCATTERED_RING_BASE_MSB,
    893			   FIELD_PREP(
    894				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
    895				(u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT) |
    896				FIELD_PREP(
    897				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_MATCH_TAG,
    898				BASE_ADDR_MATCH_TAG_VAL));
    899
    900	/* Setup head and tail pointers for the idle list */
    901	ath11k_hif_write32(ab,
    902			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    903			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0,
    904			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
    905				      sbuf[nsbufs - 1].paddr));
    906	ath11k_hif_write32(ab,
    907			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    908			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX1,
    909			   FIELD_PREP(
    910				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
    911				((u64)sbuf[nsbufs - 1].paddr >>
    912				 HAL_ADDR_MSB_REG_SHIFT)) |
    913			   FIELD_PREP(HAL_WBM_SCATTERED_DESC_HEAD_P_OFFSET_IX1,
    914				      (end_offset >> 2)));
    915	ath11k_hif_write32(ab,
    916			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    917			   HAL_WBM_SCATTERED_DESC_PTR_HEAD_INFO_IX0,
    918			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
    919				      sbuf[0].paddr));
    920
    921	ath11k_hif_write32(ab,
    922			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    923			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX0,
    924			   FIELD_PREP(BUFFER_ADDR_INFO0_ADDR,
    925				      sbuf[0].paddr));
    926	ath11k_hif_write32(ab,
    927			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    928			   HAL_WBM_SCATTERED_DESC_PTR_TAIL_INFO_IX1,
    929			   FIELD_PREP(
    930				HAL_WBM_SCATTERED_DESC_MSB_BASE_ADDR_39_32,
    931				((u64)sbuf[0].paddr >> HAL_ADDR_MSB_REG_SHIFT)) |
    932			   FIELD_PREP(HAL_WBM_SCATTERED_DESC_TAIL_P_OFFSET_IX1,
    933				      0));
    934	ath11k_hif_write32(ab,
    935			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    936			   HAL_WBM_SCATTERED_DESC_PTR_HP_ADDR,
    937			   2 * tot_link_desc);
    938
    939	/* Enable the SRNG */
    940	ath11k_hif_write32(ab,
    941			   HAL_SEQ_WCSS_UMAC_WBM_REG +
    942			   HAL_WBM_IDLE_LINK_RING_MISC_ADDR(ab), 0x40);
    943}
    944
    945int ath11k_hal_srng_setup(struct ath11k_base *ab, enum hal_ring_type type,
    946			  int ring_num, int mac_id,
    947			  struct hal_srng_params *params)
    948{
    949	struct ath11k_hal *hal = &ab->hal;
    950	struct hal_srng_config *srng_config = &ab->hal.srng_config[type];
    951	struct hal_srng *srng;
    952	int ring_id;
    953	u32 lmac_idx;
    954	int i;
    955	u32 reg_base;
    956
    957	ring_id = ath11k_hal_srng_get_ring_id(ab, type, ring_num, mac_id);
    958	if (ring_id < 0)
    959		return ring_id;
    960
    961	srng = &hal->srng_list[ring_id];
    962
    963	srng->ring_id = ring_id;
    964	srng->ring_dir = srng_config->ring_dir;
    965	srng->ring_base_paddr = params->ring_base_paddr;
    966	srng->ring_base_vaddr = params->ring_base_vaddr;
    967	srng->entry_size = srng_config->entry_size;
    968	srng->num_entries = params->num_entries;
    969	srng->ring_size = srng->entry_size * srng->num_entries;
    970	srng->intr_batch_cntr_thres_entries =
    971				params->intr_batch_cntr_thres_entries;
    972	srng->intr_timer_thres_us = params->intr_timer_thres_us;
    973	srng->flags = params->flags;
    974	srng->msi_addr = params->msi_addr;
    975	srng->msi_data = params->msi_data;
    976	srng->initialized = 1;
    977	spin_lock_init(&srng->lock);
    978	lockdep_set_class(&srng->lock, hal->srng_key + ring_id);
    979
    980	for (i = 0; i < HAL_SRNG_NUM_REG_GRP; i++) {
    981		srng->hwreg_base[i] = srng_config->reg_start[i] +
    982				      (ring_num * srng_config->reg_size[i]);
    983	}
    984
    985	memset(srng->ring_base_vaddr, 0,
    986	       (srng->entry_size * srng->num_entries) << 2);
    987
    988	/* TODO: Add comments on these swap configurations */
    989	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
    990		srng->flags |= HAL_SRNG_FLAGS_MSI_SWAP | HAL_SRNG_FLAGS_DATA_TLV_SWAP |
    991			       HAL_SRNG_FLAGS_RING_PTR_SWAP;
    992
    993	reg_base = srng->hwreg_base[HAL_SRNG_REG_GRP_R2];
    994
    995	if (srng->ring_dir == HAL_SRNG_DIR_SRC) {
    996		srng->u.src_ring.hp = 0;
    997		srng->u.src_ring.cached_tp = 0;
    998		srng->u.src_ring.reap_hp = srng->ring_size - srng->entry_size;
    999		srng->u.src_ring.tp_addr = (void *)(hal->rdp.vaddr + ring_id);
   1000		srng->u.src_ring.low_threshold = params->low_threshold *
   1001						 srng->entry_size;
   1002		if (srng_config->lmac_ring) {
   1003			lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START;
   1004			srng->u.src_ring.hp_addr = (void *)(hal->wrp.vaddr +
   1005						   lmac_idx);
   1006			srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;
   1007		} else {
   1008			if (!ab->hw_params.supports_shadow_regs)
   1009				srng->u.src_ring.hp_addr =
   1010				(u32 *)((unsigned long)ab->mem + reg_base);
   1011			else
   1012				ath11k_dbg(ab, ATH11k_DBG_HAL,
   1013					   "hal type %d ring_num %d reg_base 0x%x shadow 0x%lx\n",
   1014					   type, ring_num,
   1015					   reg_base,
   1016					   (unsigned long)srng->u.src_ring.hp_addr -
   1017					   (unsigned long)ab->mem);
   1018		}
   1019	} else {
   1020		/* During initialization loop count in all the descriptors
   1021		 * will be set to zero, and HW will set it to 1 on completing
   1022		 * descriptor update in first loop, and increments it by 1 on
   1023		 * subsequent loops (loop count wraps around after reaching
   1024		 * 0xffff). The 'loop_cnt' in SW ring state is the expected
   1025		 * loop count in descriptors updated by HW (to be processed
   1026		 * by SW).
   1027		 */
   1028		srng->u.dst_ring.loop_cnt = 1;
   1029		srng->u.dst_ring.tp = 0;
   1030		srng->u.dst_ring.cached_hp = 0;
   1031		srng->u.dst_ring.hp_addr = (void *)(hal->rdp.vaddr + ring_id);
   1032		if (srng_config->lmac_ring) {
   1033			/* For LMAC rings, tail pointer updates will be done
   1034			 * through FW by writing to a shared memory location
   1035			 */
   1036			lmac_idx = ring_id - HAL_SRNG_RING_ID_LMAC1_ID_START;
   1037			srng->u.dst_ring.tp_addr = (void *)(hal->wrp.vaddr +
   1038						   lmac_idx);
   1039			srng->flags |= HAL_SRNG_FLAGS_LMAC_RING;
   1040		} else {
   1041			if (!ab->hw_params.supports_shadow_regs)
   1042				srng->u.dst_ring.tp_addr =
   1043				(u32 *)((unsigned long)ab->mem + reg_base +
   1044					(HAL_REO1_RING_TP(ab) - HAL_REO1_RING_HP(ab)));
   1045			else
   1046				ath11k_dbg(ab, ATH11k_DBG_HAL,
   1047					   "type %d ring_num %d target_reg 0x%x shadow 0x%lx\n",
   1048					   type, ring_num,
   1049					   reg_base + (HAL_REO1_RING_TP(ab) -
   1050						       HAL_REO1_RING_HP(ab)),
   1051					   (unsigned long)srng->u.dst_ring.tp_addr -
   1052					   (unsigned long)ab->mem);
   1053		}
   1054	}
   1055
   1056	if (srng_config->lmac_ring)
   1057		return ring_id;
   1058
   1059	ath11k_hal_srng_hw_init(ab, srng);
   1060
   1061	if (type == HAL_CE_DST) {
   1062		srng->u.dst_ring.max_buffer_length = params->max_buffer_len;
   1063		ath11k_hal_ce_dst_setup(ab, srng, ring_num);
   1064	}
   1065
   1066	return ring_id;
   1067}
   1068
   1069static void ath11k_hal_srng_update_hp_tp_addr(struct ath11k_base *ab,
   1070					      int shadow_cfg_idx,
   1071					  enum hal_ring_type ring_type,
   1072					  int ring_num)
   1073{
   1074	struct hal_srng *srng;
   1075	struct ath11k_hal *hal = &ab->hal;
   1076	int ring_id;
   1077	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
   1078
   1079	ring_id = ath11k_hal_srng_get_ring_id(ab, ring_type, ring_num, 0);
   1080	if (ring_id < 0)
   1081		return;
   1082
   1083	srng = &hal->srng_list[ring_id];
   1084
   1085	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
   1086		srng->u.dst_ring.tp_addr = (u32 *)(HAL_SHADOW_REG(ab, shadow_cfg_idx) +
   1087						   (unsigned long)ab->mem);
   1088	else
   1089		srng->u.src_ring.hp_addr = (u32 *)(HAL_SHADOW_REG(ab, shadow_cfg_idx) +
   1090						   (unsigned long)ab->mem);
   1091}
   1092
   1093int ath11k_hal_srng_update_shadow_config(struct ath11k_base *ab,
   1094					 enum hal_ring_type ring_type,
   1095					 int ring_num)
   1096{
   1097	struct ath11k_hal *hal = &ab->hal;
   1098	struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
   1099	int shadow_cfg_idx = hal->num_shadow_reg_configured;
   1100	u32 target_reg;
   1101
   1102	if (shadow_cfg_idx >= HAL_SHADOW_NUM_REGS)
   1103		return -EINVAL;
   1104
   1105	hal->num_shadow_reg_configured++;
   1106
   1107	target_reg = srng_config->reg_start[HAL_HP_OFFSET_IN_REG_START];
   1108	target_reg += srng_config->reg_size[HAL_HP_OFFSET_IN_REG_START] *
   1109		ring_num;
   1110
   1111	/* For destination ring, shadow the TP */
   1112	if (srng_config->ring_dir == HAL_SRNG_DIR_DST)
   1113		target_reg += HAL_OFFSET_FROM_HP_TO_TP;
   1114
   1115	hal->shadow_reg_addr[shadow_cfg_idx] = target_reg;
   1116
   1117	/* update hp/tp addr to hal structure*/
   1118	ath11k_hal_srng_update_hp_tp_addr(ab, shadow_cfg_idx, ring_type,
   1119					  ring_num);
   1120
   1121	ath11k_dbg(ab, ATH11k_DBG_HAL,
   1122		   "target_reg %x, shadow reg 0x%x shadow_idx 0x%x, ring_type %d, ring num %d",
   1123		  target_reg,
   1124		  HAL_SHADOW_REG(ab, shadow_cfg_idx),
   1125		  shadow_cfg_idx,
   1126		  ring_type, ring_num);
   1127
   1128	return 0;
   1129}
   1130
   1131void ath11k_hal_srng_shadow_config(struct ath11k_base *ab)
   1132{
   1133	struct ath11k_hal *hal = &ab->hal;
   1134	int ring_type, ring_num;
   1135
   1136	/* update all the non-CE srngs. */
   1137	for (ring_type = 0; ring_type < HAL_MAX_RING_TYPES; ring_type++) {
   1138		struct hal_srng_config *srng_config = &hal->srng_config[ring_type];
   1139
   1140		if (ring_type == HAL_CE_SRC ||
   1141		    ring_type == HAL_CE_DST ||
   1142			ring_type == HAL_CE_DST_STATUS)
   1143			continue;
   1144
   1145		if (srng_config->lmac_ring)
   1146			continue;
   1147
   1148		for (ring_num = 0; ring_num < srng_config->max_rings; ring_num++)
   1149			ath11k_hal_srng_update_shadow_config(ab, ring_type, ring_num);
   1150	}
   1151}
   1152
   1153void ath11k_hal_srng_get_shadow_config(struct ath11k_base *ab,
   1154				       u32 **cfg, u32 *len)
   1155{
   1156	struct ath11k_hal *hal = &ab->hal;
   1157
   1158	*len = hal->num_shadow_reg_configured;
   1159	*cfg = hal->shadow_reg_addr;
   1160}
   1161
   1162void ath11k_hal_srng_shadow_update_hp_tp(struct ath11k_base *ab,
   1163					 struct hal_srng *srng)
   1164{
   1165	lockdep_assert_held(&srng->lock);
   1166
   1167	/* check whether the ring is emptry. Update the shadow
   1168	 * HP only when then ring isn't' empty.
   1169	 */
   1170	if (srng->ring_dir == HAL_SRNG_DIR_SRC &&
   1171	    *srng->u.src_ring.tp_addr != srng->u.src_ring.hp)
   1172		ath11k_hal_srng_access_end(ab, srng);
   1173}
   1174
   1175static int ath11k_hal_srng_create_config(struct ath11k_base *ab)
   1176{
   1177	struct ath11k_hal *hal = &ab->hal;
   1178	struct hal_srng_config *s;
   1179
   1180	hal->srng_config = kmemdup(hw_srng_config_template,
   1181				   sizeof(hw_srng_config_template),
   1182				   GFP_KERNEL);
   1183	if (!hal->srng_config)
   1184		return -ENOMEM;
   1185
   1186	s = &hal->srng_config[HAL_REO_DST];
   1187	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_BASE_LSB(ab);
   1188	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO1_RING_HP(ab);
   1189	s->reg_size[0] = HAL_REO2_RING_BASE_LSB(ab) - HAL_REO1_RING_BASE_LSB(ab);
   1190	s->reg_size[1] = HAL_REO2_RING_HP(ab) - HAL_REO1_RING_HP(ab);
   1191
   1192	s = &hal->srng_config[HAL_REO_EXCEPTION];
   1193	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_BASE_LSB(ab);
   1194	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_TCL_RING_HP(ab);
   1195
   1196	s = &hal->srng_config[HAL_REO_REINJECT];
   1197	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_BASE_LSB(ab);
   1198	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_SW2REO_RING_HP(ab);
   1199
   1200	s = &hal->srng_config[HAL_REO_CMD];
   1201	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_RING_BASE_LSB(ab);
   1202	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_CMD_HP(ab);
   1203
   1204	s = &hal->srng_config[HAL_REO_STATUS];
   1205	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_RING_BASE_LSB(ab);
   1206	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_REO_REG + HAL_REO_STATUS_HP(ab);
   1207
   1208	s = &hal->srng_config[HAL_TCL_DATA];
   1209	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_BASE_LSB(ab);
   1210	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL1_RING_HP;
   1211	s->reg_size[0] = HAL_TCL2_RING_BASE_LSB(ab) - HAL_TCL1_RING_BASE_LSB(ab);
   1212	s->reg_size[1] = HAL_TCL2_RING_HP - HAL_TCL1_RING_HP;
   1213
   1214	s = &hal->srng_config[HAL_TCL_CMD];
   1215	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_BASE_LSB(ab);
   1216	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_RING_HP;
   1217
   1218	s = &hal->srng_config[HAL_TCL_STATUS];
   1219	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_BASE_LSB(ab);
   1220	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_TCL_REG + HAL_TCL_STATUS_RING_HP;
   1221
   1222	s = &hal->srng_config[HAL_CE_SRC];
   1223	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_BASE_LSB;
   1224	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab) + HAL_CE_DST_RING_HP;
   1225	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
   1226		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
   1227	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_SRC_REG(ab) -
   1228		HAL_SEQ_WCSS_UMAC_CE0_SRC_REG(ab);
   1229
   1230	s = &hal->srng_config[HAL_CE_DST];
   1231	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_BASE_LSB;
   1232	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_RING_HP;
   1233	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
   1234		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
   1235	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
   1236		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
   1237
   1238	s = &hal->srng_config[HAL_CE_DST_STATUS];
   1239	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) +
   1240		HAL_CE_DST_STATUS_RING_BASE_LSB;
   1241	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab) + HAL_CE_DST_STATUS_RING_HP;
   1242	s->reg_size[0] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
   1243		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
   1244	s->reg_size[1] = HAL_SEQ_WCSS_UMAC_CE1_DST_REG(ab) -
   1245		HAL_SEQ_WCSS_UMAC_CE0_DST_REG(ab);
   1246
   1247	s = &hal->srng_config[HAL_WBM_IDLE_LINK];
   1248	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_BASE_LSB(ab);
   1249	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_IDLE_LINK_RING_HP;
   1250
   1251	s = &hal->srng_config[HAL_SW2WBM_RELEASE];
   1252	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_BASE_LSB(ab);
   1253	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM_RELEASE_RING_HP;
   1254
   1255	s = &hal->srng_config[HAL_WBM2SW_RELEASE];
   1256	s->reg_start[0] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
   1257	s->reg_start[1] = HAL_SEQ_WCSS_UMAC_WBM_REG + HAL_WBM0_RELEASE_RING_HP;
   1258	s->reg_size[0] = HAL_WBM1_RELEASE_RING_BASE_LSB(ab) -
   1259		HAL_WBM0_RELEASE_RING_BASE_LSB(ab);
   1260	s->reg_size[1] = HAL_WBM1_RELEASE_RING_HP - HAL_WBM0_RELEASE_RING_HP;
   1261
   1262	return 0;
   1263}
   1264
   1265static void ath11k_hal_register_srng_key(struct ath11k_base *ab)
   1266{
   1267	struct ath11k_hal *hal = &ab->hal;
   1268	u32 ring_id;
   1269
   1270	for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
   1271		lockdep_register_key(hal->srng_key + ring_id);
   1272}
   1273
   1274static void ath11k_hal_unregister_srng_key(struct ath11k_base *ab)
   1275{
   1276	struct ath11k_hal *hal = &ab->hal;
   1277	u32 ring_id;
   1278
   1279	for (ring_id = 0; ring_id < HAL_SRNG_RING_ID_MAX; ring_id++)
   1280		lockdep_unregister_key(hal->srng_key + ring_id);
   1281}
   1282
   1283int ath11k_hal_srng_init(struct ath11k_base *ab)
   1284{
   1285	struct ath11k_hal *hal = &ab->hal;
   1286	int ret;
   1287
   1288	memset(hal, 0, sizeof(*hal));
   1289
   1290	ret = ath11k_hal_srng_create_config(ab);
   1291	if (ret)
   1292		goto err_hal;
   1293
   1294	ret = ath11k_hal_alloc_cont_rdp(ab);
   1295	if (ret)
   1296		goto err_hal;
   1297
   1298	ret = ath11k_hal_alloc_cont_wrp(ab);
   1299	if (ret)
   1300		goto err_free_cont_rdp;
   1301
   1302	ath11k_hal_register_srng_key(ab);
   1303
   1304	return 0;
   1305
   1306err_free_cont_rdp:
   1307	ath11k_hal_free_cont_rdp(ab);
   1308
   1309err_hal:
   1310	return ret;
   1311}
   1312EXPORT_SYMBOL(ath11k_hal_srng_init);
   1313
   1314void ath11k_hal_srng_deinit(struct ath11k_base *ab)
   1315{
   1316	struct ath11k_hal *hal = &ab->hal;
   1317
   1318	ath11k_hal_unregister_srng_key(ab);
   1319	ath11k_hal_free_cont_rdp(ab);
   1320	ath11k_hal_free_cont_wrp(ab);
   1321	kfree(hal->srng_config);
   1322}
   1323EXPORT_SYMBOL(ath11k_hal_srng_deinit);
   1324
   1325void ath11k_hal_dump_srng_stats(struct ath11k_base *ab)
   1326{
   1327	struct hal_srng *srng;
   1328	struct ath11k_ext_irq_grp *irq_grp;
   1329	struct ath11k_ce_pipe *ce_pipe;
   1330	int i;
   1331
   1332	ath11k_err(ab, "Last interrupt received for each CE:\n");
   1333	for (i = 0; i < ab->hw_params.ce_count; i++) {
   1334		ce_pipe = &ab->ce.ce_pipe[i];
   1335
   1336		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR)
   1337			continue;
   1338
   1339		ath11k_err(ab, "CE_id %d pipe_num %d %ums before\n",
   1340			   i, ce_pipe->pipe_num,
   1341			   jiffies_to_msecs(jiffies - ce_pipe->timestamp));
   1342	}
   1343
   1344	ath11k_err(ab, "\nLast interrupt received for each group:\n");
   1345	for (i = 0; i < ATH11K_EXT_IRQ_GRP_NUM_MAX; i++) {
   1346		irq_grp = &ab->ext_irq_grp[i];
   1347		ath11k_err(ab, "group_id %d %ums before\n",
   1348			   irq_grp->grp_id,
   1349			   jiffies_to_msecs(jiffies - irq_grp->timestamp));
   1350	}
   1351
   1352	for (i = 0; i < HAL_SRNG_RING_ID_MAX; i++) {
   1353		srng = &ab->hal.srng_list[i];
   1354
   1355		if (!srng->initialized)
   1356			continue;
   1357
   1358		if (srng->ring_dir == HAL_SRNG_DIR_SRC)
   1359			ath11k_err(ab,
   1360				   "src srng id %u hp %u, reap_hp %u, cur tp %u, cached tp %u last tp %u napi processed before %ums\n",
   1361				   srng->ring_id, srng->u.src_ring.hp,
   1362				   srng->u.src_ring.reap_hp,
   1363				   *srng->u.src_ring.tp_addr, srng->u.src_ring.cached_tp,
   1364				   srng->u.src_ring.last_tp,
   1365				   jiffies_to_msecs(jiffies - srng->timestamp));
   1366		else if (srng->ring_dir == HAL_SRNG_DIR_DST)
   1367			ath11k_err(ab,
   1368				   "dst srng id %u tp %u, cur hp %u, cached hp %u last hp %u napi processed before %ums\n",
   1369				   srng->ring_id, srng->u.dst_ring.tp,
   1370				   *srng->u.dst_ring.hp_addr,
   1371				   srng->u.dst_ring.cached_hp,
   1372				   srng->u.dst_ring.last_hp,
   1373				   jiffies_to_msecs(jiffies - srng->timestamp));
   1374	}
   1375}