cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

dbg.c (95674B)


      1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
      2/*
      3 * Copyright (C) 2005-2014, 2018-2021 Intel Corporation
      4 * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
      5 * Copyright (C) 2015-2017 Intel Deutschland GmbH
      6 */
      7#include <linux/devcoredump.h>
      8#include "iwl-drv.h"
      9#include "runtime.h"
     10#include "dbg.h"
     11#include "debugfs.h"
     12#include "iwl-io.h"
     13#include "iwl-prph.h"
     14#include "iwl-csr.h"
     15#include "iwl-fh.h"
     16/**
     17 * struct iwl_fw_dump_ptrs - set of pointers needed for the fw-error-dump
     18 *
     19 * @fwrt_ptr: pointer to the buffer coming from fwrt
     20 * @trans_ptr: pointer to struct %iwl_trans_dump_data which contains the
     21 *	transport's data.
     22 * @trans_len: length of the valid data in trans_ptr
     23 * @fwrt_len: length of the valid data in fwrt_ptr
     24 */
     25struct iwl_fw_dump_ptrs {
     26	struct iwl_trans_dump_data *trans_ptr;
     27	void *fwrt_ptr;
     28	u32 fwrt_len;
     29};
     30
     31#define RADIO_REG_MAX_READ 0x2ad
     32static void iwl_read_radio_regs(struct iwl_fw_runtime *fwrt,
     33				struct iwl_fw_error_dump_data **dump_data)
     34{
     35	u8 *pos = (void *)(*dump_data)->data;
     36	int i;
     37
     38	IWL_DEBUG_INFO(fwrt, "WRT radio registers dump\n");
     39
     40	if (!iwl_trans_grab_nic_access(fwrt->trans))
     41		return;
     42
     43	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RADIO_REG);
     44	(*dump_data)->len = cpu_to_le32(RADIO_REG_MAX_READ);
     45
     46	for (i = 0; i < RADIO_REG_MAX_READ; i++) {
     47		u32 rd_cmd = RADIO_RSP_RD_CMD;
     48
     49		rd_cmd |= i << RADIO_RSP_ADDR_POS;
     50		iwl_write_prph_no_grab(fwrt->trans, RSP_RADIO_CMD, rd_cmd);
     51		*pos = (u8)iwl_read_prph_no_grab(fwrt->trans, RSP_RADIO_RDDAT);
     52
     53		pos++;
     54	}
     55
     56	*dump_data = iwl_fw_error_next_data(*dump_data);
     57
     58	iwl_trans_release_nic_access(fwrt->trans);
     59}
     60
     61static void iwl_fwrt_dump_rxf(struct iwl_fw_runtime *fwrt,
     62			      struct iwl_fw_error_dump_data **dump_data,
     63			      int size, u32 offset, int fifo_num)
     64{
     65	struct iwl_fw_error_dump_fifo *fifo_hdr;
     66	u32 *fifo_data;
     67	u32 fifo_len;
     68	int i;
     69
     70	fifo_hdr = (void *)(*dump_data)->data;
     71	fifo_data = (void *)fifo_hdr->data;
     72	fifo_len = size;
     73
     74	/* No need to try to read the data if the length is 0 */
     75	if (fifo_len == 0)
     76		return;
     77
     78	/* Add a TLV for the RXF */
     79	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RXF);
     80	(*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
     81
     82	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
     83	fifo_hdr->available_bytes =
     84		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
     85						RXF_RD_D_SPACE + offset));
     86	fifo_hdr->wr_ptr =
     87		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
     88						RXF_RD_WR_PTR + offset));
     89	fifo_hdr->rd_ptr =
     90		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
     91						RXF_RD_RD_PTR + offset));
     92	fifo_hdr->fence_ptr =
     93		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
     94						RXF_RD_FENCE_PTR + offset));
     95	fifo_hdr->fence_mode =
     96		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
     97						RXF_SET_FENCE_MODE + offset));
     98
     99	/* Lock fence */
    100	iwl_trans_write_prph(fwrt->trans, RXF_SET_FENCE_MODE + offset, 0x1);
    101	/* Set fence pointer to the same place like WR pointer */
    102	iwl_trans_write_prph(fwrt->trans, RXF_LD_WR2FENCE + offset, 0x1);
    103	/* Set fence offset */
    104	iwl_trans_write_prph(fwrt->trans,
    105			     RXF_LD_FENCE_OFFSET_ADDR + offset, 0x0);
    106
    107	/* Read FIFO */
    108	fifo_len /= sizeof(u32); /* Size in DWORDS */
    109	for (i = 0; i < fifo_len; i++)
    110		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
    111						 RXF_FIFO_RD_FENCE_INC +
    112						 offset);
    113	*dump_data = iwl_fw_error_next_data(*dump_data);
    114}
    115
    116static void iwl_fwrt_dump_txf(struct iwl_fw_runtime *fwrt,
    117			      struct iwl_fw_error_dump_data **dump_data,
    118			      int size, u32 offset, int fifo_num)
    119{
    120	struct iwl_fw_error_dump_fifo *fifo_hdr;
    121	u32 *fifo_data;
    122	u32 fifo_len;
    123	int i;
    124
    125	fifo_hdr = (void *)(*dump_data)->data;
    126	fifo_data = (void *)fifo_hdr->data;
    127	fifo_len = size;
    128
    129	/* No need to try to read the data if the length is 0 */
    130	if (fifo_len == 0)
    131		return;
    132
    133	/* Add a TLV for the FIFO */
    134	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXF);
    135	(*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
    136
    137	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
    138	fifo_hdr->available_bytes =
    139		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    140						TXF_FIFO_ITEM_CNT + offset));
    141	fifo_hdr->wr_ptr =
    142		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    143						TXF_WR_PTR + offset));
    144	fifo_hdr->rd_ptr =
    145		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    146						TXF_RD_PTR + offset));
    147	fifo_hdr->fence_ptr =
    148		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    149						TXF_FENCE_PTR + offset));
    150	fifo_hdr->fence_mode =
    151		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    152						TXF_LOCK_FENCE + offset));
    153
    154	/* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
    155	iwl_trans_write_prph(fwrt->trans, TXF_READ_MODIFY_ADDR + offset,
    156			     TXF_WR_PTR + offset);
    157
    158	/* Dummy-read to advance the read pointer to the head */
    159	iwl_trans_read_prph(fwrt->trans, TXF_READ_MODIFY_DATA + offset);
    160
    161	/* Read FIFO */
    162	for (i = 0; i < fifo_len / sizeof(u32); i++)
    163		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
    164						  TXF_READ_MODIFY_DATA +
    165						  offset);
    166
    167	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
    168		fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
    169					     fifo_data, fifo_len);
    170
    171	*dump_data = iwl_fw_error_next_data(*dump_data);
    172}
    173
    174static void iwl_fw_dump_rxf(struct iwl_fw_runtime *fwrt,
    175			    struct iwl_fw_error_dump_data **dump_data)
    176{
    177	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
    178
    179	IWL_DEBUG_INFO(fwrt, "WRT RX FIFO dump\n");
    180
    181	if (!iwl_trans_grab_nic_access(fwrt->trans))
    182		return;
    183
    184	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF)) {
    185		/* Pull RXF1 */
    186		iwl_fwrt_dump_rxf(fwrt, dump_data,
    187				  cfg->lmac[0].rxfifo1_size, 0, 0);
    188		/* Pull RXF2 */
    189		iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
    190				  RXF_DIFF_FROM_PREV +
    191				  fwrt->trans->trans_cfg->umac_prph_offset, 1);
    192		/* Pull LMAC2 RXF1 */
    193		if (fwrt->smem_cfg.num_lmacs > 1)
    194			iwl_fwrt_dump_rxf(fwrt, dump_data,
    195					  cfg->lmac[1].rxfifo1_size,
    196					  LMAC2_PRPH_OFFSET, 2);
    197	}
    198
    199	iwl_trans_release_nic_access(fwrt->trans);
    200}
    201
    202static void iwl_fw_dump_txf(struct iwl_fw_runtime *fwrt,
    203			    struct iwl_fw_error_dump_data **dump_data)
    204{
    205	struct iwl_fw_error_dump_fifo *fifo_hdr;
    206	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
    207	u32 *fifo_data;
    208	u32 fifo_len;
    209	int i, j;
    210
    211	IWL_DEBUG_INFO(fwrt, "WRT TX FIFO dump\n");
    212
    213	if (!iwl_trans_grab_nic_access(fwrt->trans))
    214		return;
    215
    216	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF)) {
    217		/* Pull TXF data from LMAC1 */
    218		for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
    219			/* Mark the number of TXF we're pulling now */
    220			iwl_trans_write_prph(fwrt->trans, TXF_LARC_NUM, i);
    221			iwl_fwrt_dump_txf(fwrt, dump_data,
    222					  cfg->lmac[0].txfifo_size[i], 0, i);
    223		}
    224
    225		/* Pull TXF data from LMAC2 */
    226		if (fwrt->smem_cfg.num_lmacs > 1) {
    227			for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries;
    228			     i++) {
    229				/* Mark the number of TXF we're pulling now */
    230				iwl_trans_write_prph(fwrt->trans,
    231						     TXF_LARC_NUM +
    232						     LMAC2_PRPH_OFFSET, i);
    233				iwl_fwrt_dump_txf(fwrt, dump_data,
    234						  cfg->lmac[1].txfifo_size[i],
    235						  LMAC2_PRPH_OFFSET,
    236						  i + cfg->num_txfifo_entries);
    237			}
    238		}
    239	}
    240
    241	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
    242	    fw_has_capa(&fwrt->fw->ucode_capa,
    243			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
    244		/* Pull UMAC internal TXF data from all TXFs */
    245		for (i = 0;
    246		     i < ARRAY_SIZE(fwrt->smem_cfg.internal_txfifo_size);
    247		     i++) {
    248			fifo_hdr = (void *)(*dump_data)->data;
    249			fifo_data = (void *)fifo_hdr->data;
    250			fifo_len = fwrt->smem_cfg.internal_txfifo_size[i];
    251
    252			/* No need to try to read the data if the length is 0 */
    253			if (fifo_len == 0)
    254				continue;
    255
    256			/* Add a TLV for the internal FIFOs */
    257			(*dump_data)->type =
    258				cpu_to_le32(IWL_FW_ERROR_DUMP_INTERNAL_TXF);
    259			(*dump_data)->len =
    260				cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
    261
    262			fifo_hdr->fifo_num = cpu_to_le32(i);
    263
    264			/* Mark the number of TXF we're pulling now */
    265			iwl_trans_write_prph(fwrt->trans, TXF_CPU2_NUM, i +
    266				fwrt->smem_cfg.num_txfifo_entries);
    267
    268			fifo_hdr->available_bytes =
    269				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    270								TXF_CPU2_FIFO_ITEM_CNT));
    271			fifo_hdr->wr_ptr =
    272				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    273								TXF_CPU2_WR_PTR));
    274			fifo_hdr->rd_ptr =
    275				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    276								TXF_CPU2_RD_PTR));
    277			fifo_hdr->fence_ptr =
    278				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    279								TXF_CPU2_FENCE_PTR));
    280			fifo_hdr->fence_mode =
    281				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
    282								TXF_CPU2_LOCK_FENCE));
    283
    284			/* Set TXF_CPU2_READ_MODIFY_ADDR to TXF_CPU2_WR_PTR */
    285			iwl_trans_write_prph(fwrt->trans,
    286					     TXF_CPU2_READ_MODIFY_ADDR,
    287					     TXF_CPU2_WR_PTR);
    288
    289			/* Dummy-read to advance the read pointer to head */
    290			iwl_trans_read_prph(fwrt->trans,
    291					    TXF_CPU2_READ_MODIFY_DATA);
    292
    293			/* Read FIFO */
    294			fifo_len /= sizeof(u32); /* Size in DWORDS */
    295			for (j = 0; j < fifo_len; j++)
    296				fifo_data[j] =
    297					iwl_trans_read_prph(fwrt->trans,
    298							    TXF_CPU2_READ_MODIFY_DATA);
    299			*dump_data = iwl_fw_error_next_data(*dump_data);
    300		}
    301	}
    302
    303	iwl_trans_release_nic_access(fwrt->trans);
    304}
    305
    306struct iwl_prph_range {
    307	u32 start, end;
    308};
    309
    310static const struct iwl_prph_range iwl_prph_dump_addr_comm[] = {
    311	{ .start = 0x00a00000, .end = 0x00a00000 },
    312	{ .start = 0x00a0000c, .end = 0x00a00024 },
    313	{ .start = 0x00a0002c, .end = 0x00a0003c },
    314	{ .start = 0x00a00410, .end = 0x00a00418 },
    315	{ .start = 0x00a00420, .end = 0x00a00420 },
    316	{ .start = 0x00a00428, .end = 0x00a00428 },
    317	{ .start = 0x00a00430, .end = 0x00a0043c },
    318	{ .start = 0x00a00444, .end = 0x00a00444 },
    319	{ .start = 0x00a004c0, .end = 0x00a004cc },
    320	{ .start = 0x00a004d8, .end = 0x00a004d8 },
    321	{ .start = 0x00a004e0, .end = 0x00a004f0 },
    322	{ .start = 0x00a00840, .end = 0x00a00840 },
    323	{ .start = 0x00a00850, .end = 0x00a00858 },
    324	{ .start = 0x00a01004, .end = 0x00a01008 },
    325	{ .start = 0x00a01010, .end = 0x00a01010 },
    326	{ .start = 0x00a01018, .end = 0x00a01018 },
    327	{ .start = 0x00a01024, .end = 0x00a01024 },
    328	{ .start = 0x00a0102c, .end = 0x00a01034 },
    329	{ .start = 0x00a0103c, .end = 0x00a01040 },
    330	{ .start = 0x00a01048, .end = 0x00a01094 },
    331	{ .start = 0x00a01c00, .end = 0x00a01c20 },
    332	{ .start = 0x00a01c58, .end = 0x00a01c58 },
    333	{ .start = 0x00a01c7c, .end = 0x00a01c7c },
    334	{ .start = 0x00a01c28, .end = 0x00a01c54 },
    335	{ .start = 0x00a01c5c, .end = 0x00a01c5c },
    336	{ .start = 0x00a01c60, .end = 0x00a01cdc },
    337	{ .start = 0x00a01ce0, .end = 0x00a01d0c },
    338	{ .start = 0x00a01d18, .end = 0x00a01d20 },
    339	{ .start = 0x00a01d2c, .end = 0x00a01d30 },
    340	{ .start = 0x00a01d40, .end = 0x00a01d5c },
    341	{ .start = 0x00a01d80, .end = 0x00a01d80 },
    342	{ .start = 0x00a01d98, .end = 0x00a01d9c },
    343	{ .start = 0x00a01da8, .end = 0x00a01da8 },
    344	{ .start = 0x00a01db8, .end = 0x00a01df4 },
    345	{ .start = 0x00a01dc0, .end = 0x00a01dfc },
    346	{ .start = 0x00a01e00, .end = 0x00a01e2c },
    347	{ .start = 0x00a01e40, .end = 0x00a01e60 },
    348	{ .start = 0x00a01e68, .end = 0x00a01e6c },
    349	{ .start = 0x00a01e74, .end = 0x00a01e74 },
    350	{ .start = 0x00a01e84, .end = 0x00a01e90 },
    351	{ .start = 0x00a01e9c, .end = 0x00a01ec4 },
    352	{ .start = 0x00a01ed0, .end = 0x00a01ee0 },
    353	{ .start = 0x00a01f00, .end = 0x00a01f1c },
    354	{ .start = 0x00a01f44, .end = 0x00a01ffc },
    355	{ .start = 0x00a02000, .end = 0x00a02048 },
    356	{ .start = 0x00a02068, .end = 0x00a020f0 },
    357	{ .start = 0x00a02100, .end = 0x00a02118 },
    358	{ .start = 0x00a02140, .end = 0x00a0214c },
    359	{ .start = 0x00a02168, .end = 0x00a0218c },
    360	{ .start = 0x00a021c0, .end = 0x00a021c0 },
    361	{ .start = 0x00a02400, .end = 0x00a02410 },
    362	{ .start = 0x00a02418, .end = 0x00a02420 },
    363	{ .start = 0x00a02428, .end = 0x00a0242c },
    364	{ .start = 0x00a02434, .end = 0x00a02434 },
    365	{ .start = 0x00a02440, .end = 0x00a02460 },
    366	{ .start = 0x00a02468, .end = 0x00a024b0 },
    367	{ .start = 0x00a024c8, .end = 0x00a024cc },
    368	{ .start = 0x00a02500, .end = 0x00a02504 },
    369	{ .start = 0x00a0250c, .end = 0x00a02510 },
    370	{ .start = 0x00a02540, .end = 0x00a02554 },
    371	{ .start = 0x00a02580, .end = 0x00a025f4 },
    372	{ .start = 0x00a02600, .end = 0x00a0260c },
    373	{ .start = 0x00a02648, .end = 0x00a02650 },
    374	{ .start = 0x00a02680, .end = 0x00a02680 },
    375	{ .start = 0x00a026c0, .end = 0x00a026d0 },
    376	{ .start = 0x00a02700, .end = 0x00a0270c },
    377	{ .start = 0x00a02804, .end = 0x00a02804 },
    378	{ .start = 0x00a02818, .end = 0x00a0281c },
    379	{ .start = 0x00a02c00, .end = 0x00a02db4 },
    380	{ .start = 0x00a02df4, .end = 0x00a02fb0 },
    381	{ .start = 0x00a03000, .end = 0x00a03014 },
    382	{ .start = 0x00a0301c, .end = 0x00a0302c },
    383	{ .start = 0x00a03034, .end = 0x00a03038 },
    384	{ .start = 0x00a03040, .end = 0x00a03048 },
    385	{ .start = 0x00a03060, .end = 0x00a03068 },
    386	{ .start = 0x00a03070, .end = 0x00a03074 },
    387	{ .start = 0x00a0307c, .end = 0x00a0307c },
    388	{ .start = 0x00a03080, .end = 0x00a03084 },
    389	{ .start = 0x00a0308c, .end = 0x00a03090 },
    390	{ .start = 0x00a03098, .end = 0x00a03098 },
    391	{ .start = 0x00a030a0, .end = 0x00a030a0 },
    392	{ .start = 0x00a030a8, .end = 0x00a030b4 },
    393	{ .start = 0x00a030bc, .end = 0x00a030bc },
    394	{ .start = 0x00a030c0, .end = 0x00a0312c },
    395	{ .start = 0x00a03c00, .end = 0x00a03c5c },
    396	{ .start = 0x00a04400, .end = 0x00a04454 },
    397	{ .start = 0x00a04460, .end = 0x00a04474 },
    398	{ .start = 0x00a044c0, .end = 0x00a044ec },
    399	{ .start = 0x00a04500, .end = 0x00a04504 },
    400	{ .start = 0x00a04510, .end = 0x00a04538 },
    401	{ .start = 0x00a04540, .end = 0x00a04548 },
    402	{ .start = 0x00a04560, .end = 0x00a0457c },
    403	{ .start = 0x00a04590, .end = 0x00a04598 },
    404	{ .start = 0x00a045c0, .end = 0x00a045f4 },
    405};
    406
    407static const struct iwl_prph_range iwl_prph_dump_addr_9000[] = {
    408	{ .start = 0x00a05c00, .end = 0x00a05c18 },
    409	{ .start = 0x00a05400, .end = 0x00a056e8 },
    410	{ .start = 0x00a08000, .end = 0x00a098bc },
    411	{ .start = 0x00a02400, .end = 0x00a02758 },
    412	{ .start = 0x00a04764, .end = 0x00a0476c },
    413	{ .start = 0x00a04770, .end = 0x00a04774 },
    414	{ .start = 0x00a04620, .end = 0x00a04624 },
    415};
    416
    417static const struct iwl_prph_range iwl_prph_dump_addr_22000[] = {
    418	{ .start = 0x00a00000, .end = 0x00a00000 },
    419	{ .start = 0x00a0000c, .end = 0x00a00024 },
    420	{ .start = 0x00a0002c, .end = 0x00a00034 },
    421	{ .start = 0x00a0003c, .end = 0x00a0003c },
    422	{ .start = 0x00a00410, .end = 0x00a00418 },
    423	{ .start = 0x00a00420, .end = 0x00a00420 },
    424	{ .start = 0x00a00428, .end = 0x00a00428 },
    425	{ .start = 0x00a00430, .end = 0x00a0043c },
    426	{ .start = 0x00a00444, .end = 0x00a00444 },
    427	{ .start = 0x00a00840, .end = 0x00a00840 },
    428	{ .start = 0x00a00850, .end = 0x00a00858 },
    429	{ .start = 0x00a01004, .end = 0x00a01008 },
    430	{ .start = 0x00a01010, .end = 0x00a01010 },
    431	{ .start = 0x00a01018, .end = 0x00a01018 },
    432	{ .start = 0x00a01024, .end = 0x00a01024 },
    433	{ .start = 0x00a0102c, .end = 0x00a01034 },
    434	{ .start = 0x00a0103c, .end = 0x00a01040 },
    435	{ .start = 0x00a01048, .end = 0x00a01050 },
    436	{ .start = 0x00a01058, .end = 0x00a01058 },
    437	{ .start = 0x00a01060, .end = 0x00a01070 },
    438	{ .start = 0x00a0108c, .end = 0x00a0108c },
    439	{ .start = 0x00a01c20, .end = 0x00a01c28 },
    440	{ .start = 0x00a01d10, .end = 0x00a01d10 },
    441	{ .start = 0x00a01e28, .end = 0x00a01e2c },
    442	{ .start = 0x00a01e60, .end = 0x00a01e60 },
    443	{ .start = 0x00a01e80, .end = 0x00a01e80 },
    444	{ .start = 0x00a01ea0, .end = 0x00a01ea0 },
    445	{ .start = 0x00a02000, .end = 0x00a0201c },
    446	{ .start = 0x00a02024, .end = 0x00a02024 },
    447	{ .start = 0x00a02040, .end = 0x00a02048 },
    448	{ .start = 0x00a020c0, .end = 0x00a020e0 },
    449	{ .start = 0x00a02400, .end = 0x00a02404 },
    450	{ .start = 0x00a0240c, .end = 0x00a02414 },
    451	{ .start = 0x00a0241c, .end = 0x00a0243c },
    452	{ .start = 0x00a02448, .end = 0x00a024bc },
    453	{ .start = 0x00a024c4, .end = 0x00a024cc },
    454	{ .start = 0x00a02508, .end = 0x00a02508 },
    455	{ .start = 0x00a02510, .end = 0x00a02514 },
    456	{ .start = 0x00a0251c, .end = 0x00a0251c },
    457	{ .start = 0x00a0252c, .end = 0x00a0255c },
    458	{ .start = 0x00a02564, .end = 0x00a025a0 },
    459	{ .start = 0x00a025a8, .end = 0x00a025b4 },
    460	{ .start = 0x00a025c0, .end = 0x00a025c0 },
    461	{ .start = 0x00a025e8, .end = 0x00a025f4 },
    462	{ .start = 0x00a02c08, .end = 0x00a02c18 },
    463	{ .start = 0x00a02c2c, .end = 0x00a02c38 },
    464	{ .start = 0x00a02c68, .end = 0x00a02c78 },
    465	{ .start = 0x00a03000, .end = 0x00a03000 },
    466	{ .start = 0x00a03010, .end = 0x00a03014 },
    467	{ .start = 0x00a0301c, .end = 0x00a0302c },
    468	{ .start = 0x00a03034, .end = 0x00a03038 },
    469	{ .start = 0x00a03040, .end = 0x00a03044 },
    470	{ .start = 0x00a03060, .end = 0x00a03068 },
    471	{ .start = 0x00a03070, .end = 0x00a03070 },
    472	{ .start = 0x00a0307c, .end = 0x00a03084 },
    473	{ .start = 0x00a0308c, .end = 0x00a03090 },
    474	{ .start = 0x00a03098, .end = 0x00a03098 },
    475	{ .start = 0x00a030a0, .end = 0x00a030a0 },
    476	{ .start = 0x00a030a8, .end = 0x00a030b4 },
    477	{ .start = 0x00a030bc, .end = 0x00a030c0 },
    478	{ .start = 0x00a030c8, .end = 0x00a030f4 },
    479	{ .start = 0x00a03100, .end = 0x00a0312c },
    480	{ .start = 0x00a03c00, .end = 0x00a03c5c },
    481	{ .start = 0x00a04400, .end = 0x00a04454 },
    482	{ .start = 0x00a04460, .end = 0x00a04474 },
    483	{ .start = 0x00a044c0, .end = 0x00a044ec },
    484	{ .start = 0x00a04500, .end = 0x00a04504 },
    485	{ .start = 0x00a04510, .end = 0x00a04538 },
    486	{ .start = 0x00a04540, .end = 0x00a04548 },
    487	{ .start = 0x00a04560, .end = 0x00a04560 },
    488	{ .start = 0x00a04570, .end = 0x00a0457c },
    489	{ .start = 0x00a04590, .end = 0x00a04590 },
    490	{ .start = 0x00a04598, .end = 0x00a04598 },
    491	{ .start = 0x00a045c0, .end = 0x00a045f4 },
    492	{ .start = 0x00a05c18, .end = 0x00a05c1c },
    493	{ .start = 0x00a0c000, .end = 0x00a0c018 },
    494	{ .start = 0x00a0c020, .end = 0x00a0c028 },
    495	{ .start = 0x00a0c038, .end = 0x00a0c094 },
    496	{ .start = 0x00a0c0c0, .end = 0x00a0c104 },
    497	{ .start = 0x00a0c10c, .end = 0x00a0c118 },
    498	{ .start = 0x00a0c150, .end = 0x00a0c174 },
    499	{ .start = 0x00a0c17c, .end = 0x00a0c188 },
    500	{ .start = 0x00a0c190, .end = 0x00a0c198 },
    501	{ .start = 0x00a0c1a0, .end = 0x00a0c1a8 },
    502	{ .start = 0x00a0c1b0, .end = 0x00a0c1b8 },
    503};
    504
    505static const struct iwl_prph_range iwl_prph_dump_addr_ax210[] = {
    506	{ .start = 0x00d03c00, .end = 0x00d03c64 },
    507	{ .start = 0x00d05c18, .end = 0x00d05c1c },
    508	{ .start = 0x00d0c000, .end = 0x00d0c174 },
    509};
    510
    511static void iwl_read_prph_block(struct iwl_trans *trans, u32 start,
    512				u32 len_bytes, __le32 *data)
    513{
    514	u32 i;
    515
    516	for (i = 0; i < len_bytes; i += 4)
    517		*data++ = cpu_to_le32(iwl_read_prph_no_grab(trans, start + i));
    518}
    519
    520static void iwl_dump_prph(struct iwl_fw_runtime *fwrt,
    521			  const struct iwl_prph_range *iwl_prph_dump_addr,
    522			  u32 range_len, void *ptr)
    523{
    524	struct iwl_fw_error_dump_prph *prph;
    525	struct iwl_trans *trans = fwrt->trans;
    526	struct iwl_fw_error_dump_data **data =
    527		(struct iwl_fw_error_dump_data **)ptr;
    528	u32 i;
    529
    530	if (!data)
    531		return;
    532
    533	IWL_DEBUG_INFO(trans, "WRT PRPH dump\n");
    534
    535	if (!iwl_trans_grab_nic_access(trans))
    536		return;
    537
    538	for (i = 0; i < range_len; i++) {
    539		/* The range includes both boundaries */
    540		int num_bytes_in_chunk = iwl_prph_dump_addr[i].end -
    541			 iwl_prph_dump_addr[i].start + 4;
    542
    543		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PRPH);
    544		(*data)->len = cpu_to_le32(sizeof(*prph) +
    545					num_bytes_in_chunk);
    546		prph = (void *)(*data)->data;
    547		prph->prph_start = cpu_to_le32(iwl_prph_dump_addr[i].start);
    548
    549		iwl_read_prph_block(trans, iwl_prph_dump_addr[i].start,
    550				    /* our range is inclusive, hence + 4 */
    551				    iwl_prph_dump_addr[i].end -
    552				    iwl_prph_dump_addr[i].start + 4,
    553				    (void *)prph->data);
    554
    555		*data = iwl_fw_error_next_data(*data);
    556	}
    557
    558	iwl_trans_release_nic_access(trans);
    559}
    560
    561/*
    562 * alloc_sgtable - allocates scallerlist table in the given size,
    563 * fills it with pages and returns it
    564 * @size: the size (in bytes) of the table
    565*/
    566static struct scatterlist *alloc_sgtable(int size)
    567{
    568	int alloc_size, nents, i;
    569	struct page *new_page;
    570	struct scatterlist *iter;
    571	struct scatterlist *table;
    572
    573	nents = DIV_ROUND_UP(size, PAGE_SIZE);
    574	table = kcalloc(nents, sizeof(*table), GFP_KERNEL);
    575	if (!table)
    576		return NULL;
    577	sg_init_table(table, nents);
    578	iter = table;
    579	for_each_sg(table, iter, sg_nents(table), i) {
    580		new_page = alloc_page(GFP_KERNEL);
    581		if (!new_page) {
    582			/* release all previous allocated pages in the table */
    583			iter = table;
    584			for_each_sg(table, iter, sg_nents(table), i) {
    585				new_page = sg_page(iter);
    586				if (new_page)
    587					__free_page(new_page);
    588			}
    589			kfree(table);
    590			return NULL;
    591		}
    592		alloc_size = min_t(int, size, PAGE_SIZE);
    593		size -= PAGE_SIZE;
    594		sg_set_page(iter, new_page, alloc_size, 0);
    595	}
    596	return table;
    597}
    598
    599static void iwl_fw_get_prph_len(struct iwl_fw_runtime *fwrt,
    600				const struct iwl_prph_range *iwl_prph_dump_addr,
    601				u32 range_len, void *ptr)
    602{
    603	u32 *prph_len = (u32 *)ptr;
    604	int i, num_bytes_in_chunk;
    605
    606	if (!prph_len)
    607		return;
    608
    609	for (i = 0; i < range_len; i++) {
    610		/* The range includes both boundaries */
    611		num_bytes_in_chunk =
    612			iwl_prph_dump_addr[i].end -
    613			iwl_prph_dump_addr[i].start + 4;
    614
    615		*prph_len += sizeof(struct iwl_fw_error_dump_data) +
    616			sizeof(struct iwl_fw_error_dump_prph) +
    617			num_bytes_in_chunk;
    618	}
    619}
    620
    621static void iwl_fw_prph_handler(struct iwl_fw_runtime *fwrt, void *ptr,
    622				void (*handler)(struct iwl_fw_runtime *,
    623						const struct iwl_prph_range *,
    624						u32, void *))
    625{
    626	u32 range_len;
    627
    628	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
    629		range_len = ARRAY_SIZE(iwl_prph_dump_addr_ax210);
    630		handler(fwrt, iwl_prph_dump_addr_ax210, range_len, ptr);
    631	} else if (fwrt->trans->trans_cfg->device_family >=
    632		   IWL_DEVICE_FAMILY_22000) {
    633		range_len = ARRAY_SIZE(iwl_prph_dump_addr_22000);
    634		handler(fwrt, iwl_prph_dump_addr_22000, range_len, ptr);
    635	} else {
    636		range_len = ARRAY_SIZE(iwl_prph_dump_addr_comm);
    637		handler(fwrt, iwl_prph_dump_addr_comm, range_len, ptr);
    638
    639		if (fwrt->trans->trans_cfg->mq_rx_supported) {
    640			range_len = ARRAY_SIZE(iwl_prph_dump_addr_9000);
    641			handler(fwrt, iwl_prph_dump_addr_9000, range_len, ptr);
    642		}
    643	}
    644}
    645
    646static void iwl_fw_dump_mem(struct iwl_fw_runtime *fwrt,
    647			    struct iwl_fw_error_dump_data **dump_data,
    648			    u32 len, u32 ofs, u32 type)
    649{
    650	struct iwl_fw_error_dump_mem *dump_mem;
    651
    652	if (!len)
    653		return;
    654
    655	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
    656	(*dump_data)->len = cpu_to_le32(len + sizeof(*dump_mem));
    657	dump_mem = (void *)(*dump_data)->data;
    658	dump_mem->type = cpu_to_le32(type);
    659	dump_mem->offset = cpu_to_le32(ofs);
    660	iwl_trans_read_mem_bytes(fwrt->trans, ofs, dump_mem->data, len);
    661	*dump_data = iwl_fw_error_next_data(*dump_data);
    662
    663	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
    664		fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, ofs,
    665					     dump_mem->data, len);
    666
    667	IWL_DEBUG_INFO(fwrt, "WRT memory dump. Type=%u\n", dump_mem->type);
    668}
    669
    670#define ADD_LEN(len, item_len, const_len) \
    671	do {size_t item = item_len; len += (!!item) * const_len + item; } \
    672	while (0)
    673
    674static int iwl_fw_rxf_len(struct iwl_fw_runtime *fwrt,
    675			  struct iwl_fwrt_shared_mem_cfg *mem_cfg)
    676{
    677	size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
    678			 sizeof(struct iwl_fw_error_dump_fifo);
    679	u32 fifo_len = 0;
    680	int i;
    681
    682	if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF))
    683		return 0;
    684
    685	/* Count RXF2 size */
    686	ADD_LEN(fifo_len, mem_cfg->rxfifo2_size, hdr_len);
    687
    688	/* Count RXF1 sizes */
    689	if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
    690		mem_cfg->num_lmacs = MAX_NUM_LMAC;
    691
    692	for (i = 0; i < mem_cfg->num_lmacs; i++)
    693		ADD_LEN(fifo_len, mem_cfg->lmac[i].rxfifo1_size, hdr_len);
    694
    695	return fifo_len;
    696}
    697
    698static int iwl_fw_txf_len(struct iwl_fw_runtime *fwrt,
    699			  struct iwl_fwrt_shared_mem_cfg *mem_cfg)
    700{
    701	size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
    702			 sizeof(struct iwl_fw_error_dump_fifo);
    703	u32 fifo_len = 0;
    704	int i;
    705
    706	if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF))
    707		goto dump_internal_txf;
    708
    709	/* Count TXF sizes */
    710	if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
    711		mem_cfg->num_lmacs = MAX_NUM_LMAC;
    712
    713	for (i = 0; i < mem_cfg->num_lmacs; i++) {
    714		int j;
    715
    716		for (j = 0; j < mem_cfg->num_txfifo_entries; j++)
    717			ADD_LEN(fifo_len, mem_cfg->lmac[i].txfifo_size[j],
    718				hdr_len);
    719	}
    720
    721dump_internal_txf:
    722	if (!(iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
    723	      fw_has_capa(&fwrt->fw->ucode_capa,
    724			  IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)))
    725		goto out;
    726
    727	for (i = 0; i < ARRAY_SIZE(mem_cfg->internal_txfifo_size); i++)
    728		ADD_LEN(fifo_len, mem_cfg->internal_txfifo_size[i], hdr_len);
    729
    730out:
    731	return fifo_len;
    732}
    733
    734static void iwl_dump_paging(struct iwl_fw_runtime *fwrt,
    735			    struct iwl_fw_error_dump_data **data)
    736{
    737	int i;
    738
    739	IWL_DEBUG_INFO(fwrt, "WRT paging dump\n");
    740	for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
    741		struct iwl_fw_error_dump_paging *paging;
    742		struct page *pages =
    743			fwrt->fw_paging_db[i].fw_paging_block;
    744		dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
    745
    746		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
    747		(*data)->len = cpu_to_le32(sizeof(*paging) +
    748					     PAGING_BLOCK_SIZE);
    749		paging =  (void *)(*data)->data;
    750		paging->index = cpu_to_le32(i);
    751		dma_sync_single_for_cpu(fwrt->trans->dev, addr,
    752					PAGING_BLOCK_SIZE,
    753					DMA_BIDIRECTIONAL);
    754		memcpy(paging->data, page_address(pages),
    755		       PAGING_BLOCK_SIZE);
    756		dma_sync_single_for_device(fwrt->trans->dev, addr,
    757					   PAGING_BLOCK_SIZE,
    758					   DMA_BIDIRECTIONAL);
    759		(*data) = iwl_fw_error_next_data(*data);
    760
    761		if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
    762			fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
    763						     fwrt->fw_paging_db[i].fw_offs,
    764						     paging->data,
    765						     PAGING_BLOCK_SIZE);
    766	}
    767}
    768
    769static struct iwl_fw_error_dump_file *
    770iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
    771		       struct iwl_fw_dump_ptrs *fw_error_dump,
    772		       struct iwl_fwrt_dump_data *data)
    773{
    774	struct iwl_fw_error_dump_file *dump_file;
    775	struct iwl_fw_error_dump_data *dump_data;
    776	struct iwl_fw_error_dump_info *dump_info;
    777	struct iwl_fw_error_dump_smem_cfg *dump_smem_cfg;
    778	struct iwl_fw_error_dump_trigger_desc *dump_trig;
    779	u32 sram_len, sram_ofs;
    780	const struct iwl_fw_dbg_mem_seg_tlv *fw_mem = fwrt->fw->dbg.mem_tlv;
    781	struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
    782	u32 file_len, fifo_len = 0, prph_len = 0, radio_len = 0;
    783	u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->cfg->smem_len;
    784	u32 sram2_len = fwrt->fw->dbg.n_mem_tlv ?
    785				0 : fwrt->trans->cfg->dccm2_len;
    786	int i;
    787
    788	/* SRAM - include stack CCM if driver knows the values for it */
    789	if (!fwrt->trans->cfg->dccm_offset || !fwrt->trans->cfg->dccm_len) {
    790		const struct fw_img *img;
    791
    792		if (fwrt->cur_fw_img >= IWL_UCODE_TYPE_MAX)
    793			return NULL;
    794		img = &fwrt->fw->img[fwrt->cur_fw_img];
    795		sram_ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
    796		sram_len = img->sec[IWL_UCODE_SECTION_DATA].len;
    797	} else {
    798		sram_ofs = fwrt->trans->cfg->dccm_offset;
    799		sram_len = fwrt->trans->cfg->dccm_len;
    800	}
    801
    802	/* reading RXF/TXF sizes */
    803	if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status)) {
    804		fifo_len = iwl_fw_rxf_len(fwrt, mem_cfg);
    805		fifo_len += iwl_fw_txf_len(fwrt, mem_cfg);
    806
    807		/* Make room for PRPH registers */
    808		if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_PRPH))
    809			iwl_fw_prph_handler(fwrt, &prph_len,
    810					    iwl_fw_get_prph_len);
    811
    812		if (fwrt->trans->trans_cfg->device_family ==
    813		    IWL_DEVICE_FAMILY_7000 &&
    814		    iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RADIO_REG))
    815			radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
    816	}
    817
    818	file_len = sizeof(*dump_file) + fifo_len + prph_len + radio_len;
    819
    820	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO))
    821		file_len += sizeof(*dump_data) + sizeof(*dump_info);
    822	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG))
    823		file_len += sizeof(*dump_data) + sizeof(*dump_smem_cfg);
    824
    825	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
    826		size_t hdr_len = sizeof(*dump_data) +
    827				 sizeof(struct iwl_fw_error_dump_mem);
    828
    829		/* Dump SRAM only if no mem_tlvs */
    830		if (!fwrt->fw->dbg.n_mem_tlv)
    831			ADD_LEN(file_len, sram_len, hdr_len);
    832
    833		/* Make room for all mem types that exist */
    834		ADD_LEN(file_len, smem_len, hdr_len);
    835		ADD_LEN(file_len, sram2_len, hdr_len);
    836
    837		for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++)
    838			ADD_LEN(file_len, le32_to_cpu(fw_mem[i].len), hdr_len);
    839	}
    840
    841	/* Make room for fw's virtual image pages, if it exists */
    842	if (iwl_fw_dbg_is_paging_enabled(fwrt))
    843		file_len += fwrt->num_of_paging_blk *
    844			(sizeof(*dump_data) +
    845			 sizeof(struct iwl_fw_error_dump_paging) +
    846			 PAGING_BLOCK_SIZE);
    847
    848	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
    849		file_len += sizeof(*dump_data) +
    850			fwrt->trans->cfg->d3_debug_data_length * 2;
    851	}
    852
    853	/* If we only want a monitor dump, reset the file length */
    854	if (data->monitor_only) {
    855		file_len = sizeof(*dump_file) + sizeof(*dump_data) * 2 +
    856			   sizeof(*dump_info) + sizeof(*dump_smem_cfg);
    857	}
    858
    859	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
    860	    data->desc)
    861		file_len += sizeof(*dump_data) + sizeof(*dump_trig) +
    862			data->desc->len;
    863
    864	dump_file = vzalloc(file_len);
    865	if (!dump_file)
    866		return NULL;
    867
    868	fw_error_dump->fwrt_ptr = dump_file;
    869
    870	dump_file->barker = cpu_to_le32(IWL_FW_ERROR_DUMP_BARKER);
    871	dump_data = (void *)dump_file->data;
    872
    873	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO)) {
    874		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_DEV_FW_INFO);
    875		dump_data->len = cpu_to_le32(sizeof(*dump_info));
    876		dump_info = (void *)dump_data->data;
    877		dump_info->hw_type =
    878			cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->hw_rev));
    879		dump_info->hw_step =
    880			cpu_to_le32(fwrt->trans->hw_rev_step);
    881		memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
    882		       sizeof(dump_info->fw_human_readable));
    883		strncpy(dump_info->dev_human_readable, fwrt->trans->name,
    884			sizeof(dump_info->dev_human_readable) - 1);
    885		strncpy(dump_info->bus_human_readable, fwrt->dev->bus->name,
    886			sizeof(dump_info->bus_human_readable) - 1);
    887		dump_info->num_of_lmacs = fwrt->smem_cfg.num_lmacs;
    888		dump_info->lmac_err_id[0] =
    889			cpu_to_le32(fwrt->dump.lmac_err_id[0]);
    890		if (fwrt->smem_cfg.num_lmacs > 1)
    891			dump_info->lmac_err_id[1] =
    892				cpu_to_le32(fwrt->dump.lmac_err_id[1]);
    893		dump_info->umac_err_id = cpu_to_le32(fwrt->dump.umac_err_id);
    894
    895		dump_data = iwl_fw_error_next_data(dump_data);
    896	}
    897
    898	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG)) {
    899		/* Dump shared memory configuration */
    900		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_CFG);
    901		dump_data->len = cpu_to_le32(sizeof(*dump_smem_cfg));
    902		dump_smem_cfg = (void *)dump_data->data;
    903		dump_smem_cfg->num_lmacs = cpu_to_le32(mem_cfg->num_lmacs);
    904		dump_smem_cfg->num_txfifo_entries =
    905			cpu_to_le32(mem_cfg->num_txfifo_entries);
    906		for (i = 0; i < MAX_NUM_LMAC; i++) {
    907			int j;
    908			u32 *txf_size = mem_cfg->lmac[i].txfifo_size;
    909
    910			for (j = 0; j < TX_FIFO_MAX_NUM; j++)
    911				dump_smem_cfg->lmac[i].txfifo_size[j] =
    912					cpu_to_le32(txf_size[j]);
    913			dump_smem_cfg->lmac[i].rxfifo1_size =
    914				cpu_to_le32(mem_cfg->lmac[i].rxfifo1_size);
    915		}
    916		dump_smem_cfg->rxfifo2_size =
    917			cpu_to_le32(mem_cfg->rxfifo2_size);
    918		dump_smem_cfg->internal_txfifo_addr =
    919			cpu_to_le32(mem_cfg->internal_txfifo_addr);
    920		for (i = 0; i < TX_FIFO_INTERNAL_MAX_NUM; i++) {
    921			dump_smem_cfg->internal_txfifo_size[i] =
    922				cpu_to_le32(mem_cfg->internal_txfifo_size[i]);
    923		}
    924
    925		dump_data = iwl_fw_error_next_data(dump_data);
    926	}
    927
    928	/* We only dump the FIFOs if the FW is in error state */
    929	if (fifo_len) {
    930		iwl_fw_dump_rxf(fwrt, &dump_data);
    931		iwl_fw_dump_txf(fwrt, &dump_data);
    932	}
    933
    934	if (radio_len)
    935		iwl_read_radio_regs(fwrt, &dump_data);
    936
    937	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
    938	    data->desc) {
    939		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_ERROR_INFO);
    940		dump_data->len = cpu_to_le32(sizeof(*dump_trig) +
    941					     data->desc->len);
    942		dump_trig = (void *)dump_data->data;
    943		memcpy(dump_trig, &data->desc->trig_desc,
    944		       sizeof(*dump_trig) + data->desc->len);
    945
    946		dump_data = iwl_fw_error_next_data(dump_data);
    947	}
    948
    949	/* In case we only want monitor dump, skip to dump trasport data */
    950	if (data->monitor_only)
    951		goto out;
    952
    953	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
    954		const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem =
    955			fwrt->fw->dbg.mem_tlv;
    956
    957		if (!fwrt->fw->dbg.n_mem_tlv)
    958			iwl_fw_dump_mem(fwrt, &dump_data, sram_len, sram_ofs,
    959					IWL_FW_ERROR_DUMP_MEM_SRAM);
    960
    961		for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++) {
    962			u32 len = le32_to_cpu(fw_dbg_mem[i].len);
    963			u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
    964
    965			iwl_fw_dump_mem(fwrt, &dump_data, len, ofs,
    966					le32_to_cpu(fw_dbg_mem[i].data_type));
    967		}
    968
    969		iwl_fw_dump_mem(fwrt, &dump_data, smem_len,
    970				fwrt->trans->cfg->smem_offset,
    971				IWL_FW_ERROR_DUMP_MEM_SMEM);
    972
    973		iwl_fw_dump_mem(fwrt, &dump_data, sram2_len,
    974				fwrt->trans->cfg->dccm2_offset,
    975				IWL_FW_ERROR_DUMP_MEM_SRAM);
    976	}
    977
    978	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
    979		u32 addr = fwrt->trans->cfg->d3_debug_data_base_addr;
    980		size_t data_size = fwrt->trans->cfg->d3_debug_data_length;
    981
    982		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
    983		dump_data->len = cpu_to_le32(data_size * 2);
    984
    985		memcpy(dump_data->data, fwrt->dump.d3_debug_data, data_size);
    986
    987		kfree(fwrt->dump.d3_debug_data);
    988		fwrt->dump.d3_debug_data = NULL;
    989
    990		iwl_trans_read_mem_bytes(fwrt->trans, addr,
    991					 dump_data->data + data_size,
    992					 data_size);
    993
    994		if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
    995			fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, addr,
    996						     dump_data->data + data_size,
    997						     data_size);
    998
    999		dump_data = iwl_fw_error_next_data(dump_data);
   1000	}
   1001
   1002	/* Dump fw's virtual image */
   1003	if (iwl_fw_dbg_is_paging_enabled(fwrt))
   1004		iwl_dump_paging(fwrt, &dump_data);
   1005
   1006	if (prph_len)
   1007		iwl_fw_prph_handler(fwrt, &dump_data, iwl_dump_prph);
   1008
   1009out:
   1010	dump_file->file_len = cpu_to_le32(file_len);
   1011	return dump_file;
   1012}
   1013
   1014/**
   1015 * struct iwl_dump_ini_region_data - region data
   1016 * @reg_tlv: region TLV
   1017 * @dump_data: dump data
   1018 */
   1019struct iwl_dump_ini_region_data {
   1020	struct iwl_ucode_tlv *reg_tlv;
   1021	struct iwl_fwrt_dump_data *dump_data;
   1022};
   1023
   1024static int
   1025iwl_dump_ini_prph_mac_iter(struct iwl_fw_runtime *fwrt,
   1026			   struct iwl_dump_ini_region_data *reg_data,
   1027			   void *range_ptr, u32 range_len, int idx)
   1028{
   1029	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1030	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1031	__le32 *val = range->data;
   1032	u32 prph_val;
   1033	u32 addr = le32_to_cpu(reg->addrs[idx]) +
   1034		   le32_to_cpu(reg->dev_addr.offset);
   1035	int i;
   1036
   1037	range->internal_base_addr = cpu_to_le32(addr);
   1038	range->range_data_size = reg->dev_addr.size;
   1039	for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
   1040		prph_val = iwl_read_prph(fwrt->trans, addr + i);
   1041		if (prph_val == 0x5a5a5a5a)
   1042			return -EBUSY;
   1043		*val++ = cpu_to_le32(prph_val);
   1044	}
   1045
   1046	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1047}
   1048
   1049static int
   1050iwl_dump_ini_prph_phy_iter(struct iwl_fw_runtime *fwrt,
   1051			   struct iwl_dump_ini_region_data *reg_data,
   1052			   void *range_ptr, u32 range_len, int idx)
   1053{
   1054	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1055	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1056	__le32 *val = range->data;
   1057	u32 indirect_wr_addr = WMAL_INDRCT_RD_CMD1;
   1058	u32 indirect_rd_addr = WMAL_MRSPF_1;
   1059	u32 prph_val;
   1060	u32 addr = le32_to_cpu(reg->addrs[idx]);
   1061	u32 dphy_state;
   1062	u32 dphy_addr;
   1063	int i;
   1064
   1065	range->internal_base_addr = cpu_to_le32(addr);
   1066	range->range_data_size = reg->dev_addr.size;
   1067
   1068	if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
   1069		indirect_wr_addr = WMAL_INDRCT_CMD1;
   1070
   1071	indirect_wr_addr += le32_to_cpu(reg->dev_addr.offset);
   1072	indirect_rd_addr += le32_to_cpu(reg->dev_addr.offset);
   1073
   1074	if (!iwl_trans_grab_nic_access(fwrt->trans))
   1075		return -EBUSY;
   1076
   1077	dphy_addr = (reg->dev_addr.offset) ? WFPM_LMAC2_PS_CTL_RW :
   1078					     WFPM_LMAC1_PS_CTL_RW;
   1079	dphy_state = iwl_read_umac_prph_no_grab(fwrt->trans, dphy_addr);
   1080
   1081	for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
   1082		if (dphy_state == HBUS_TIMEOUT ||
   1083		    (dphy_state & WFPM_PS_CTL_RW_PHYRF_PD_FSM_CURSTATE_MSK) !=
   1084		    WFPM_PHYRF_STATE_ON) {
   1085			*val++ = cpu_to_le32(WFPM_DPHY_OFF);
   1086			continue;
   1087		}
   1088
   1089		iwl_write_prph_no_grab(fwrt->trans, indirect_wr_addr,
   1090				       WMAL_INDRCT_CMD(addr + i));
   1091		prph_val = iwl_read_prph_no_grab(fwrt->trans,
   1092						 indirect_rd_addr);
   1093		*val++ = cpu_to_le32(prph_val);
   1094	}
   1095
   1096	iwl_trans_release_nic_access(fwrt->trans);
   1097	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1098}
   1099
   1100static int iwl_dump_ini_csr_iter(struct iwl_fw_runtime *fwrt,
   1101				 struct iwl_dump_ini_region_data *reg_data,
   1102				 void *range_ptr, u32 range_len, int idx)
   1103{
   1104	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1105	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1106	__le32 *val = range->data;
   1107	u32 addr = le32_to_cpu(reg->addrs[idx]) +
   1108		   le32_to_cpu(reg->dev_addr.offset);
   1109	int i;
   1110
   1111	range->internal_base_addr = cpu_to_le32(addr);
   1112	range->range_data_size = reg->dev_addr.size;
   1113	for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4)
   1114		*val++ = cpu_to_le32(iwl_trans_read32(fwrt->trans, addr + i));
   1115
   1116	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1117}
   1118
   1119static int iwl_dump_ini_config_iter(struct iwl_fw_runtime *fwrt,
   1120				    struct iwl_dump_ini_region_data *reg_data,
   1121				    void *range_ptr, u32 range_len, int idx)
   1122{
   1123	struct iwl_trans *trans = fwrt->trans;
   1124	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1125	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1126	__le32 *val = range->data;
   1127	u32 addr = le32_to_cpu(reg->addrs[idx]) +
   1128		   le32_to_cpu(reg->dev_addr.offset);
   1129	int i;
   1130
   1131	/* we shouldn't get here if the trans doesn't have read_config32 */
   1132	if (WARN_ON_ONCE(!trans->ops->read_config32))
   1133		return -EOPNOTSUPP;
   1134
   1135	range->internal_base_addr = cpu_to_le32(addr);
   1136	range->range_data_size = reg->dev_addr.size;
   1137	for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
   1138		int ret;
   1139		u32 tmp;
   1140
   1141		ret = trans->ops->read_config32(trans, addr + i, &tmp);
   1142		if (ret < 0)
   1143			return ret;
   1144
   1145		*val++ = cpu_to_le32(tmp);
   1146	}
   1147
   1148	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1149}
   1150
   1151static int iwl_dump_ini_dev_mem_iter(struct iwl_fw_runtime *fwrt,
   1152				     struct iwl_dump_ini_region_data *reg_data,
   1153				     void *range_ptr, u32 range_len, int idx)
   1154{
   1155	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1156	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1157	u32 addr = le32_to_cpu(reg->addrs[idx]) +
   1158		   le32_to_cpu(reg->dev_addr.offset);
   1159
   1160	range->internal_base_addr = cpu_to_le32(addr);
   1161	range->range_data_size = reg->dev_addr.size;
   1162	iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
   1163				 le32_to_cpu(reg->dev_addr.size));
   1164
   1165	if (reg->sub_type == IWL_FW_INI_REGION_DEVICE_MEMORY_SUBTYPE_HW_SMEM &&
   1166	    fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
   1167		fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
   1168					     range->data,
   1169					     le32_to_cpu(reg->dev_addr.size));
   1170
   1171	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1172}
   1173
   1174static int _iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
   1175				     void *range_ptr, u32 range_len, int idx)
   1176{
   1177	struct page *page = fwrt->fw_paging_db[idx].fw_paging_block;
   1178	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1179	dma_addr_t addr = fwrt->fw_paging_db[idx].fw_paging_phys;
   1180	u32 page_size = fwrt->fw_paging_db[idx].fw_paging_size;
   1181
   1182	range->page_num = cpu_to_le32(idx);
   1183	range->range_data_size = cpu_to_le32(page_size);
   1184	dma_sync_single_for_cpu(fwrt->trans->dev, addr,	page_size,
   1185				DMA_BIDIRECTIONAL);
   1186	memcpy(range->data, page_address(page), page_size);
   1187	dma_sync_single_for_device(fwrt->trans->dev, addr, page_size,
   1188				   DMA_BIDIRECTIONAL);
   1189
   1190	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1191}
   1192
   1193static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
   1194				    struct iwl_dump_ini_region_data *reg_data,
   1195				    void *range_ptr, u32 range_len, int idx)
   1196{
   1197	struct iwl_fw_ini_error_dump_range *range;
   1198	u32 page_size;
   1199
   1200	/* all paged index start from 1 to skip CSS section */
   1201	idx++;
   1202
   1203	if (!fwrt->trans->trans_cfg->gen2)
   1204		return _iwl_dump_ini_paging_iter(fwrt, range_ptr, range_len, idx);
   1205
   1206	range = range_ptr;
   1207	page_size = fwrt->trans->init_dram.paging[idx].size;
   1208
   1209	range->page_num = cpu_to_le32(idx);
   1210	range->range_data_size = cpu_to_le32(page_size);
   1211	memcpy(range->data, fwrt->trans->init_dram.paging[idx].block,
   1212	       page_size);
   1213
   1214	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1215}
   1216
   1217static int
   1218iwl_dump_ini_mon_dram_iter(struct iwl_fw_runtime *fwrt,
   1219			   struct iwl_dump_ini_region_data *reg_data,
   1220			   void *range_ptr, u32 range_len, int idx)
   1221{
   1222	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1223	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1224	struct iwl_dram_data *frag;
   1225	u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
   1226
   1227	frag = &fwrt->trans->dbg.fw_mon_ini[alloc_id].frags[idx];
   1228
   1229	range->dram_base_addr = cpu_to_le64(frag->physical);
   1230	range->range_data_size = cpu_to_le32(frag->size);
   1231
   1232	memcpy(range->data, frag->block, frag->size);
   1233
   1234	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1235}
   1236
   1237static int iwl_dump_ini_mon_smem_iter(struct iwl_fw_runtime *fwrt,
   1238				      struct iwl_dump_ini_region_data *reg_data,
   1239				      void *range_ptr, u32 range_len, int idx)
   1240{
   1241	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1242	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1243	u32 addr = le32_to_cpu(reg->internal_buffer.base_addr);
   1244
   1245	range->internal_base_addr = cpu_to_le32(addr);
   1246	range->range_data_size = reg->internal_buffer.size;
   1247	iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
   1248				 le32_to_cpu(reg->internal_buffer.size));
   1249
   1250	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1251}
   1252
   1253static bool iwl_ini_txf_iter(struct iwl_fw_runtime *fwrt,
   1254			     struct iwl_dump_ini_region_data *reg_data, int idx)
   1255{
   1256	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1257	struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
   1258	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
   1259	int txf_num = cfg->num_txfifo_entries;
   1260	int int_txf_num = ARRAY_SIZE(cfg->internal_txfifo_size);
   1261	u32 lmac_bitmap = le32_to_cpu(reg->fifos.fid[0]);
   1262
   1263	if (!idx) {
   1264		if (le32_to_cpu(reg->fifos.offset) && cfg->num_lmacs == 1) {
   1265			IWL_ERR(fwrt, "WRT: Invalid lmac offset 0x%x\n",
   1266				le32_to_cpu(reg->fifos.offset));
   1267			return false;
   1268		}
   1269
   1270		iter->internal_txf = 0;
   1271		iter->fifo_size = 0;
   1272		iter->fifo = -1;
   1273		if (le32_to_cpu(reg->fifos.offset))
   1274			iter->lmac = 1;
   1275		else
   1276			iter->lmac = 0;
   1277	}
   1278
   1279	if (!iter->internal_txf) {
   1280		for (iter->fifo++; iter->fifo < txf_num; iter->fifo++) {
   1281			iter->fifo_size =
   1282				cfg->lmac[iter->lmac].txfifo_size[iter->fifo];
   1283			if (iter->fifo_size && (lmac_bitmap & BIT(iter->fifo)))
   1284				return true;
   1285		}
   1286		iter->fifo--;
   1287	}
   1288
   1289	iter->internal_txf = 1;
   1290
   1291	if (!fw_has_capa(&fwrt->fw->ucode_capa,
   1292			 IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
   1293		return false;
   1294
   1295	for (iter->fifo++; iter->fifo < int_txf_num + txf_num; iter->fifo++) {
   1296		iter->fifo_size =
   1297			cfg->internal_txfifo_size[iter->fifo - txf_num];
   1298		if (iter->fifo_size && (lmac_bitmap & BIT(iter->fifo)))
   1299			return true;
   1300	}
   1301
   1302	return false;
   1303}
   1304
   1305static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
   1306				 struct iwl_dump_ini_region_data *reg_data,
   1307				 void *range_ptr, u32 range_len, int idx)
   1308{
   1309	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1310	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1311	struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
   1312	struct iwl_fw_ini_error_dump_register *reg_dump = (void *)range->data;
   1313	u32 offs = le32_to_cpu(reg->fifos.offset), addr;
   1314	u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
   1315	u32 registers_size = registers_num * sizeof(*reg_dump);
   1316	__le32 *data;
   1317	int i;
   1318
   1319	if (!iwl_ini_txf_iter(fwrt, reg_data, idx))
   1320		return -EIO;
   1321
   1322	if (!iwl_trans_grab_nic_access(fwrt->trans))
   1323		return -EBUSY;
   1324
   1325	range->fifo_hdr.fifo_num = cpu_to_le32(iter->fifo);
   1326	range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
   1327	range->range_data_size = cpu_to_le32(iter->fifo_size + registers_size);
   1328
   1329	iwl_write_prph_no_grab(fwrt->trans, TXF_LARC_NUM + offs, iter->fifo);
   1330
   1331	/*
   1332	 * read txf registers. for each register, write to the dump the
   1333	 * register address and its value
   1334	 */
   1335	for (i = 0; i < registers_num; i++) {
   1336		addr = le32_to_cpu(reg->addrs[i]) + offs;
   1337
   1338		reg_dump->addr = cpu_to_le32(addr);
   1339		reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
   1340								   addr));
   1341
   1342		reg_dump++;
   1343	}
   1344
   1345	if (reg->fifos.hdr_only) {
   1346		range->range_data_size = cpu_to_le32(registers_size);
   1347		goto out;
   1348	}
   1349
   1350	/* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
   1351	iwl_write_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_ADDR + offs,
   1352			       TXF_WR_PTR + offs);
   1353
   1354	/* Dummy-read to advance the read pointer to the head */
   1355	iwl_read_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_DATA + offs);
   1356
   1357	/* Read FIFO */
   1358	addr = TXF_READ_MODIFY_DATA + offs;
   1359	data = (void *)reg_dump;
   1360	for (i = 0; i < iter->fifo_size; i += sizeof(*data))
   1361		*data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
   1362
   1363	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
   1364		fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
   1365					     reg_dump, iter->fifo_size);
   1366
   1367out:
   1368	iwl_trans_release_nic_access(fwrt->trans);
   1369
   1370	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1371}
   1372
   1373struct iwl_ini_rxf_data {
   1374	u32 fifo_num;
   1375	u32 size;
   1376	u32 offset;
   1377};
   1378
   1379static void iwl_ini_get_rxf_data(struct iwl_fw_runtime *fwrt,
   1380				 struct iwl_dump_ini_region_data *reg_data,
   1381				 struct iwl_ini_rxf_data *data)
   1382{
   1383	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1384	u32 fid1 = le32_to_cpu(reg->fifos.fid[0]);
   1385	u32 fid2 = le32_to_cpu(reg->fifos.fid[1]);
   1386	u8 fifo_idx;
   1387
   1388	if (!data)
   1389		return;
   1390
   1391	/* make sure only one bit is set in only one fid */
   1392	if (WARN_ONCE(hweight_long(fid1) + hweight_long(fid2) != 1,
   1393		      "fid1=%x, fid2=%x\n", fid1, fid2))
   1394		return;
   1395
   1396	memset(data, 0, sizeof(*data));
   1397
   1398	if (fid1) {
   1399		fifo_idx = ffs(fid1) - 1;
   1400		if (WARN_ONCE(fifo_idx >= MAX_NUM_LMAC, "fifo_idx=%d\n",
   1401			      fifo_idx))
   1402			return;
   1403
   1404		data->size = fwrt->smem_cfg.lmac[fifo_idx].rxfifo1_size;
   1405		data->fifo_num = fifo_idx;
   1406	} else {
   1407		u8 max_idx;
   1408
   1409		fifo_idx = ffs(fid2) - 1;
   1410		if (iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP,
   1411					    SHARED_MEM_CFG_CMD, 0) <= 3)
   1412			max_idx = 0;
   1413		else
   1414			max_idx = 1;
   1415
   1416		if (WARN_ONCE(fifo_idx > max_idx,
   1417			      "invalid umac fifo idx %d", fifo_idx))
   1418			return;
   1419
   1420		/* use bit 31 to distinguish between umac and lmac rxf while
   1421		 * parsing the dump
   1422		 */
   1423		data->fifo_num = fifo_idx | IWL_RXF_UMAC_BIT;
   1424
   1425		switch (fifo_idx) {
   1426		case 0:
   1427			data->size = fwrt->smem_cfg.rxfifo2_size;
   1428			data->offset = iwl_umac_prph(fwrt->trans,
   1429						     RXF_DIFF_FROM_PREV);
   1430			break;
   1431		case 1:
   1432			data->size = fwrt->smem_cfg.rxfifo2_control_size;
   1433			data->offset = iwl_umac_prph(fwrt->trans,
   1434						     RXF2C_DIFF_FROM_PREV);
   1435			break;
   1436		}
   1437	}
   1438}
   1439
   1440static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
   1441				 struct iwl_dump_ini_region_data *reg_data,
   1442				 void *range_ptr, u32 range_len, int idx)
   1443{
   1444	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1445	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1446	struct iwl_ini_rxf_data rxf_data;
   1447	struct iwl_fw_ini_error_dump_register *reg_dump = (void *)range->data;
   1448	u32 offs = le32_to_cpu(reg->fifos.offset), addr;
   1449	u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
   1450	u32 registers_size = registers_num * sizeof(*reg_dump);
   1451	__le32 *data;
   1452	int i;
   1453
   1454	iwl_ini_get_rxf_data(fwrt, reg_data, &rxf_data);
   1455	if (!rxf_data.size)
   1456		return -EIO;
   1457
   1458	if (!iwl_trans_grab_nic_access(fwrt->trans))
   1459		return -EBUSY;
   1460
   1461	range->fifo_hdr.fifo_num = cpu_to_le32(rxf_data.fifo_num);
   1462	range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
   1463	range->range_data_size = cpu_to_le32(rxf_data.size + registers_size);
   1464
   1465	/*
   1466	 * read rxf registers. for each register, write to the dump the
   1467	 * register address and its value
   1468	 */
   1469	for (i = 0; i < registers_num; i++) {
   1470		addr = le32_to_cpu(reg->addrs[i]) + offs;
   1471
   1472		reg_dump->addr = cpu_to_le32(addr);
   1473		reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
   1474								   addr));
   1475
   1476		reg_dump++;
   1477	}
   1478
   1479	if (reg->fifos.hdr_only) {
   1480		range->range_data_size = cpu_to_le32(registers_size);
   1481		goto out;
   1482	}
   1483
   1484	offs = rxf_data.offset;
   1485
   1486	/* Lock fence */
   1487	iwl_write_prph_no_grab(fwrt->trans, RXF_SET_FENCE_MODE + offs, 0x1);
   1488	/* Set fence pointer to the same place like WR pointer */
   1489	iwl_write_prph_no_grab(fwrt->trans, RXF_LD_WR2FENCE + offs, 0x1);
   1490	/* Set fence offset */
   1491	iwl_write_prph_no_grab(fwrt->trans, RXF_LD_FENCE_OFFSET_ADDR + offs,
   1492			       0x0);
   1493
   1494	/* Read FIFO */
   1495	addr =  RXF_FIFO_RD_FENCE_INC + offs;
   1496	data = (void *)reg_dump;
   1497	for (i = 0; i < rxf_data.size; i += sizeof(*data))
   1498		*data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
   1499
   1500out:
   1501	iwl_trans_release_nic_access(fwrt->trans);
   1502
   1503	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1504}
   1505
   1506static int
   1507iwl_dump_ini_err_table_iter(struct iwl_fw_runtime *fwrt,
   1508			    struct iwl_dump_ini_region_data *reg_data,
   1509			    void *range_ptr, u32 range_len, int idx)
   1510{
   1511	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1512	struct iwl_fw_ini_region_err_table *err_table = &reg->err_table;
   1513	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1514	u32 addr = le32_to_cpu(err_table->base_addr) +
   1515		   le32_to_cpu(err_table->offset);
   1516
   1517	range->internal_base_addr = cpu_to_le32(addr);
   1518	range->range_data_size = err_table->size;
   1519	iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
   1520				 le32_to_cpu(err_table->size));
   1521
   1522	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1523}
   1524
   1525static int
   1526iwl_dump_ini_special_mem_iter(struct iwl_fw_runtime *fwrt,
   1527			      struct iwl_dump_ini_region_data *reg_data,
   1528			      void *range_ptr, u32 range_len, int idx)
   1529{
   1530	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1531	struct iwl_fw_ini_region_special_device_memory *special_mem =
   1532		&reg->special_mem;
   1533
   1534	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1535	u32 addr = le32_to_cpu(special_mem->base_addr) +
   1536		   le32_to_cpu(special_mem->offset);
   1537
   1538	range->internal_base_addr = cpu_to_le32(addr);
   1539	range->range_data_size = special_mem->size;
   1540	iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
   1541				 le32_to_cpu(special_mem->size));
   1542
   1543	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1544}
   1545
   1546static int
   1547iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
   1548			    struct iwl_dump_ini_region_data *reg_data,
   1549			    void *range_ptr, u32 range_len, int idx)
   1550{
   1551	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1552	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1553	__le32 *val = range->data;
   1554	u32 prph_data;
   1555	int i;
   1556
   1557	if (!iwl_trans_grab_nic_access(fwrt->trans))
   1558		return -EBUSY;
   1559
   1560	range->range_data_size = reg->dev_addr.size;
   1561	for (i = 0; i < (le32_to_cpu(reg->dev_addr.size) / 4); i++) {
   1562		prph_data = iwl_read_prph_no_grab(fwrt->trans, (i % 2) ?
   1563					  DBGI_SRAM_TARGET_ACCESS_RDATA_MSB :
   1564					  DBGI_SRAM_TARGET_ACCESS_RDATA_LSB);
   1565		if (prph_data == 0x5a5a5a5a) {
   1566			iwl_trans_release_nic_access(fwrt->trans);
   1567			return -EBUSY;
   1568		}
   1569		*val++ = cpu_to_le32(prph_data);
   1570	}
   1571	iwl_trans_release_nic_access(fwrt->trans);
   1572	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1573}
   1574
   1575static int iwl_dump_ini_fw_pkt_iter(struct iwl_fw_runtime *fwrt,
   1576				    struct iwl_dump_ini_region_data *reg_data,
   1577				    void *range_ptr, u32 range_len, int idx)
   1578{
   1579	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1580	struct iwl_rx_packet *pkt = reg_data->dump_data->fw_pkt;
   1581	u32 pkt_len;
   1582
   1583	if (!pkt)
   1584		return -EIO;
   1585
   1586	pkt_len = iwl_rx_packet_payload_len(pkt);
   1587
   1588	memcpy(&range->fw_pkt_hdr, &pkt->hdr, sizeof(range->fw_pkt_hdr));
   1589	range->range_data_size = cpu_to_le32(pkt_len);
   1590
   1591	memcpy(range->data, pkt->data, pkt_len);
   1592
   1593	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1594}
   1595
   1596static int iwl_dump_ini_imr_iter(struct iwl_fw_runtime *fwrt,
   1597				 struct iwl_dump_ini_region_data *reg_data,
   1598				 void *range_ptr, u32 range_len, int idx)
   1599{
   1600	/* read the IMR memory and DMA it to SRAM */
   1601	struct iwl_fw_ini_error_dump_range *range = range_ptr;
   1602	u64 imr_curr_addr = fwrt->trans->dbg.imr_data.imr_curr_addr;
   1603	u32 imr_rem_bytes = fwrt->trans->dbg.imr_data.imr2sram_remainbyte;
   1604	u32 sram_addr = fwrt->trans->dbg.imr_data.sram_addr;
   1605	u32 sram_size = fwrt->trans->dbg.imr_data.sram_size;
   1606	u32 size_to_dump = (imr_rem_bytes > sram_size) ? sram_size : imr_rem_bytes;
   1607
   1608	range->range_data_size = cpu_to_le32(size_to_dump);
   1609	if (iwl_trans_write_imr_mem(fwrt->trans, sram_addr,
   1610				    imr_curr_addr, size_to_dump)) {
   1611		IWL_ERR(fwrt, "WRT_DEBUG: IMR Memory transfer failed\n");
   1612		return -1;
   1613	}
   1614
   1615	fwrt->trans->dbg.imr_data.imr_curr_addr = imr_curr_addr + size_to_dump;
   1616	fwrt->trans->dbg.imr_data.imr2sram_remainbyte -= size_to_dump;
   1617
   1618	iwl_trans_read_mem_bytes(fwrt->trans, sram_addr, range->data,
   1619				 size_to_dump);
   1620	return sizeof(*range) + le32_to_cpu(range->range_data_size);
   1621}
   1622
   1623static void *
   1624iwl_dump_ini_mem_fill_header(struct iwl_fw_runtime *fwrt,
   1625			     struct iwl_dump_ini_region_data *reg_data,
   1626			     void *data, u32 data_len)
   1627{
   1628	struct iwl_fw_ini_error_dump *dump = data;
   1629
   1630	dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
   1631
   1632	return dump->data;
   1633}
   1634
   1635/**
   1636 * mask_apply_and_normalize - applies mask on val and normalize the result
   1637 *
   1638 * The normalization is based on the first set bit in the mask
   1639 *
   1640 * @val: value
   1641 * @mask: mask to apply and to normalize with
   1642 */
   1643static u32 mask_apply_and_normalize(u32 val, u32 mask)
   1644{
   1645	return (val & mask) >> (ffs(mask) - 1);
   1646}
   1647
   1648static __le32 iwl_get_mon_reg(struct iwl_fw_runtime *fwrt, u32 alloc_id,
   1649			      const struct iwl_fw_mon_reg *reg_info)
   1650{
   1651	u32 val, offs;
   1652
   1653	/* The header addresses of DBGCi is calculate as follows:
   1654	 * DBGC1 address + (0x100 * i)
   1655	 */
   1656	offs = (alloc_id - IWL_FW_INI_ALLOCATION_ID_DBGC1) * 0x100;
   1657
   1658	if (!reg_info || !reg_info->addr || !reg_info->mask)
   1659		return 0;
   1660
   1661	val = iwl_read_prph_no_grab(fwrt->trans, reg_info->addr + offs);
   1662
   1663	return cpu_to_le32(mask_apply_and_normalize(val, reg_info->mask));
   1664}
   1665
   1666static void *
   1667iwl_dump_ini_mon_fill_header(struct iwl_fw_runtime *fwrt,
   1668			     struct iwl_dump_ini_region_data *reg_data,
   1669			     struct iwl_fw_ini_monitor_dump *data,
   1670			     const struct iwl_fw_mon_regs *addrs)
   1671{
   1672	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1673	u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
   1674
   1675	if (!iwl_trans_grab_nic_access(fwrt->trans)) {
   1676		IWL_ERR(fwrt, "Failed to get monitor header\n");
   1677		return NULL;
   1678	}
   1679
   1680	data->write_ptr = iwl_get_mon_reg(fwrt, alloc_id,
   1681					  &addrs->write_ptr);
   1682	if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
   1683		u32 wrt_ptr = le32_to_cpu(data->write_ptr);
   1684
   1685		data->write_ptr = cpu_to_le32(wrt_ptr >> 2);
   1686	}
   1687	data->cycle_cnt = iwl_get_mon_reg(fwrt, alloc_id,
   1688					  &addrs->cycle_cnt);
   1689	data->cur_frag = iwl_get_mon_reg(fwrt, alloc_id,
   1690					 &addrs->cur_frag);
   1691
   1692	iwl_trans_release_nic_access(fwrt->trans);
   1693
   1694	data->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
   1695
   1696	return data->data;
   1697}
   1698
   1699static void *
   1700iwl_dump_ini_mon_dram_fill_header(struct iwl_fw_runtime *fwrt,
   1701				  struct iwl_dump_ini_region_data *reg_data,
   1702				  void *data, u32 data_len)
   1703{
   1704	struct iwl_fw_ini_monitor_dump *mon_dump = (void *)data;
   1705
   1706	return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
   1707					    &fwrt->trans->cfg->mon_dram_regs);
   1708}
   1709
   1710static void *
   1711iwl_dump_ini_mon_smem_fill_header(struct iwl_fw_runtime *fwrt,
   1712				  struct iwl_dump_ini_region_data *reg_data,
   1713				  void *data, u32 data_len)
   1714{
   1715	struct iwl_fw_ini_monitor_dump *mon_dump = (void *)data;
   1716
   1717	return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
   1718					    &fwrt->trans->cfg->mon_smem_regs);
   1719}
   1720
   1721static void *
   1722iwl_dump_ini_mon_dbgi_fill_header(struct iwl_fw_runtime *fwrt,
   1723				  struct iwl_dump_ini_region_data *reg_data,
   1724				  void *data, u32 data_len)
   1725{
   1726	struct iwl_fw_ini_monitor_dump *mon_dump = (void *)data;
   1727
   1728	return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
   1729					    &fwrt->trans->cfg->mon_dbgi_regs);
   1730}
   1731
   1732static void *
   1733iwl_dump_ini_err_table_fill_header(struct iwl_fw_runtime *fwrt,
   1734				   struct iwl_dump_ini_region_data *reg_data,
   1735				   void *data, u32 data_len)
   1736{
   1737	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1738	struct iwl_fw_ini_err_table_dump *dump = data;
   1739
   1740	dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
   1741	dump->version = reg->err_table.version;
   1742
   1743	return dump->data;
   1744}
   1745
   1746static void *
   1747iwl_dump_ini_special_mem_fill_header(struct iwl_fw_runtime *fwrt,
   1748				     struct iwl_dump_ini_region_data *reg_data,
   1749				     void *data, u32 data_len)
   1750{
   1751	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1752	struct iwl_fw_ini_special_device_memory *dump = data;
   1753
   1754	dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
   1755	dump->type = reg->special_mem.type;
   1756	dump->version = reg->special_mem.version;
   1757
   1758	return dump->data;
   1759}
   1760
   1761static void *
   1762iwl_dump_ini_imr_fill_header(struct iwl_fw_runtime *fwrt,
   1763			     struct iwl_dump_ini_region_data *reg_data,
   1764			     void *data, u32 data_len)
   1765{
   1766	struct iwl_fw_ini_error_dump *dump = data;
   1767
   1768	dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
   1769
   1770	return dump->data;
   1771}
   1772
   1773static u32 iwl_dump_ini_mem_ranges(struct iwl_fw_runtime *fwrt,
   1774				   struct iwl_dump_ini_region_data *reg_data)
   1775{
   1776	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1777
   1778	return iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
   1779}
   1780
   1781static u32 iwl_dump_ini_paging_ranges(struct iwl_fw_runtime *fwrt,
   1782				      struct iwl_dump_ini_region_data *reg_data)
   1783{
   1784	if (fwrt->trans->trans_cfg->gen2) {
   1785		if (fwrt->trans->init_dram.paging_cnt)
   1786			return fwrt->trans->init_dram.paging_cnt - 1;
   1787		else
   1788			return 0;
   1789	}
   1790
   1791	return fwrt->num_of_paging_blk;
   1792}
   1793
   1794static u32
   1795iwl_dump_ini_mon_dram_ranges(struct iwl_fw_runtime *fwrt,
   1796			     struct iwl_dump_ini_region_data *reg_data)
   1797{
   1798	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1799	struct iwl_fw_mon *fw_mon;
   1800	u32 ranges = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
   1801	int i;
   1802
   1803	fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
   1804
   1805	for (i = 0; i < fw_mon->num_frags; i++) {
   1806		if (!fw_mon->frags[i].size)
   1807			break;
   1808
   1809		ranges++;
   1810	}
   1811
   1812	return ranges;
   1813}
   1814
   1815static u32 iwl_dump_ini_txf_ranges(struct iwl_fw_runtime *fwrt,
   1816				   struct iwl_dump_ini_region_data *reg_data)
   1817{
   1818	u32 num_of_fifos = 0;
   1819
   1820	while (iwl_ini_txf_iter(fwrt, reg_data, num_of_fifos))
   1821		num_of_fifos++;
   1822
   1823	return num_of_fifos;
   1824}
   1825
   1826static u32 iwl_dump_ini_single_range(struct iwl_fw_runtime *fwrt,
   1827				     struct iwl_dump_ini_region_data *reg_data)
   1828{
   1829	return 1;
   1830}
   1831
   1832static u32 iwl_dump_ini_imr_ranges(struct iwl_fw_runtime *fwrt,
   1833				   struct iwl_dump_ini_region_data *reg_data)
   1834{
   1835	/* range is total number of pages need to copied from
   1836	 *IMR memory to SRAM and later from SRAM to DRAM
   1837	 */
   1838	u32 imr_enable = fwrt->trans->dbg.imr_data.imr_enable;
   1839	u32 imr_size = fwrt->trans->dbg.imr_data.imr_size;
   1840	u32 sram_size = fwrt->trans->dbg.imr_data.sram_size;
   1841
   1842	if (imr_enable == 0 || imr_size == 0 || sram_size == 0) {
   1843		IWL_DEBUG_INFO(fwrt,
   1844			       "WRT: Invalid imr data enable: %d, imr_size: %d, sram_size: %d\n",
   1845			       imr_enable, imr_size, sram_size);
   1846		return 0;
   1847	}
   1848
   1849	return((imr_size % sram_size) ? (imr_size / sram_size + 1) : (imr_size / sram_size));
   1850}
   1851
   1852static u32 iwl_dump_ini_mem_get_size(struct iwl_fw_runtime *fwrt,
   1853				     struct iwl_dump_ini_region_data *reg_data)
   1854{
   1855	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1856	u32 size = le32_to_cpu(reg->dev_addr.size);
   1857	u32 ranges = iwl_dump_ini_mem_ranges(fwrt, reg_data);
   1858
   1859	if (!size || !ranges)
   1860		return 0;
   1861
   1862	return sizeof(struct iwl_fw_ini_error_dump) + ranges *
   1863		(size + sizeof(struct iwl_fw_ini_error_dump_range));
   1864}
   1865
   1866static u32
   1867iwl_dump_ini_paging_get_size(struct iwl_fw_runtime *fwrt,
   1868			     struct iwl_dump_ini_region_data *reg_data)
   1869{
   1870	int i;
   1871	u32 range_header_len = sizeof(struct iwl_fw_ini_error_dump_range);
   1872	u32 size = sizeof(struct iwl_fw_ini_error_dump);
   1873
   1874	/* start from 1 to skip CSS section */
   1875	for (i = 1; i <= iwl_dump_ini_paging_ranges(fwrt, reg_data); i++) {
   1876		size += range_header_len;
   1877		if (fwrt->trans->trans_cfg->gen2)
   1878			size += fwrt->trans->init_dram.paging[i].size;
   1879		else
   1880			size += fwrt->fw_paging_db[i].fw_paging_size;
   1881	}
   1882
   1883	return size;
   1884}
   1885
   1886static u32
   1887iwl_dump_ini_mon_dram_get_size(struct iwl_fw_runtime *fwrt,
   1888			       struct iwl_dump_ini_region_data *reg_data)
   1889{
   1890	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1891	struct iwl_fw_mon *fw_mon;
   1892	u32 size = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
   1893	int i;
   1894
   1895	fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
   1896
   1897	for (i = 0; i < fw_mon->num_frags; i++) {
   1898		struct iwl_dram_data *frag = &fw_mon->frags[i];
   1899
   1900		if (!frag->size)
   1901			break;
   1902
   1903		size += sizeof(struct iwl_fw_ini_error_dump_range) + frag->size;
   1904	}
   1905
   1906	if (size)
   1907		size += sizeof(struct iwl_fw_ini_monitor_dump);
   1908
   1909	return size;
   1910}
   1911
   1912static u32
   1913iwl_dump_ini_mon_smem_get_size(struct iwl_fw_runtime *fwrt,
   1914			       struct iwl_dump_ini_region_data *reg_data)
   1915{
   1916	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1917	u32 size;
   1918
   1919	size = le32_to_cpu(reg->internal_buffer.size);
   1920	if (!size)
   1921		return 0;
   1922
   1923	size += sizeof(struct iwl_fw_ini_monitor_dump) +
   1924		sizeof(struct iwl_fw_ini_error_dump_range);
   1925
   1926	return size;
   1927}
   1928
   1929static u32 iwl_dump_ini_mon_dbgi_get_size(struct iwl_fw_runtime *fwrt,
   1930					  struct iwl_dump_ini_region_data *reg_data)
   1931{
   1932	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1933	u32 size = le32_to_cpu(reg->dev_addr.size);
   1934	u32 ranges = iwl_dump_ini_mem_ranges(fwrt, reg_data);
   1935
   1936	if (!size || !ranges)
   1937		return 0;
   1938
   1939	return sizeof(struct iwl_fw_ini_monitor_dump) + ranges *
   1940		(size + sizeof(struct iwl_fw_ini_error_dump_range));
   1941}
   1942
   1943static u32 iwl_dump_ini_txf_get_size(struct iwl_fw_runtime *fwrt,
   1944				     struct iwl_dump_ini_region_data *reg_data)
   1945{
   1946	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1947	struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
   1948	u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
   1949	u32 size = 0;
   1950	u32 fifo_hdr = sizeof(struct iwl_fw_ini_error_dump_range) +
   1951		       registers_num *
   1952		       sizeof(struct iwl_fw_ini_error_dump_register);
   1953
   1954	while (iwl_ini_txf_iter(fwrt, reg_data, size)) {
   1955		size += fifo_hdr;
   1956		if (!reg->fifos.hdr_only)
   1957			size += iter->fifo_size;
   1958	}
   1959
   1960	if (!size)
   1961		return 0;
   1962
   1963	return size + sizeof(struct iwl_fw_ini_error_dump);
   1964}
   1965
   1966static u32 iwl_dump_ini_rxf_get_size(struct iwl_fw_runtime *fwrt,
   1967				     struct iwl_dump_ini_region_data *reg_data)
   1968{
   1969	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1970	struct iwl_ini_rxf_data rx_data;
   1971	u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
   1972	u32 size = sizeof(struct iwl_fw_ini_error_dump) +
   1973		sizeof(struct iwl_fw_ini_error_dump_range) +
   1974		registers_num * sizeof(struct iwl_fw_ini_error_dump_register);
   1975
   1976	if (reg->fifos.hdr_only)
   1977		return size;
   1978
   1979	iwl_ini_get_rxf_data(fwrt, reg_data, &rx_data);
   1980	size += rx_data.size;
   1981
   1982	return size;
   1983}
   1984
   1985static u32
   1986iwl_dump_ini_err_table_get_size(struct iwl_fw_runtime *fwrt,
   1987				struct iwl_dump_ini_region_data *reg_data)
   1988{
   1989	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   1990	u32 size = le32_to_cpu(reg->err_table.size);
   1991
   1992	if (size)
   1993		size += sizeof(struct iwl_fw_ini_err_table_dump) +
   1994			sizeof(struct iwl_fw_ini_error_dump_range);
   1995
   1996	return size;
   1997}
   1998
   1999static u32
   2000iwl_dump_ini_special_mem_get_size(struct iwl_fw_runtime *fwrt,
   2001				  struct iwl_dump_ini_region_data *reg_data)
   2002{
   2003	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   2004	u32 size = le32_to_cpu(reg->special_mem.size);
   2005
   2006	if (size)
   2007		size += sizeof(struct iwl_fw_ini_special_device_memory) +
   2008			sizeof(struct iwl_fw_ini_error_dump_range);
   2009
   2010	return size;
   2011}
   2012
   2013static u32
   2014iwl_dump_ini_fw_pkt_get_size(struct iwl_fw_runtime *fwrt,
   2015			     struct iwl_dump_ini_region_data *reg_data)
   2016{
   2017	u32 size = 0;
   2018
   2019	if (!reg_data->dump_data->fw_pkt)
   2020		return 0;
   2021
   2022	size += iwl_rx_packet_payload_len(reg_data->dump_data->fw_pkt);
   2023	if (size)
   2024		size += sizeof(struct iwl_fw_ini_error_dump) +
   2025			sizeof(struct iwl_fw_ini_error_dump_range);
   2026
   2027	return size;
   2028}
   2029
   2030static u32
   2031iwl_dump_ini_imr_get_size(struct iwl_fw_runtime *fwrt,
   2032			  struct iwl_dump_ini_region_data *reg_data)
   2033{
   2034	u32 size = 0;
   2035	u32 ranges = 0;
   2036	u32 imr_enable = fwrt->trans->dbg.imr_data.imr_enable;
   2037	u32 imr_size = fwrt->trans->dbg.imr_data.imr_size;
   2038	u32 sram_size = fwrt->trans->dbg.imr_data.sram_size;
   2039
   2040	if (imr_enable == 0 || imr_size == 0 || sram_size == 0) {
   2041		IWL_DEBUG_INFO(fwrt,
   2042			       "WRT: Invalid imr data enable: %d, imr_size: %d, sram_size: %d\n",
   2043			       imr_enable, imr_size, sram_size);
   2044		return size;
   2045	}
   2046	size = imr_size;
   2047	ranges = iwl_dump_ini_imr_ranges(fwrt, reg_data);
   2048	if (!size && !ranges) {
   2049		IWL_ERR(fwrt, "WRT: imr_size :=%d, ranges :=%d\n", size, ranges);
   2050		return 0;
   2051	}
   2052	size += sizeof(struct iwl_fw_ini_error_dump) +
   2053		ranges * sizeof(struct iwl_fw_ini_error_dump_range);
   2054	return size;
   2055}
   2056
   2057/**
   2058 * struct iwl_dump_ini_mem_ops - ini memory dump operations
   2059 * @get_num_of_ranges: returns the number of memory ranges in the region.
   2060 * @get_size: returns the total size of the region.
   2061 * @fill_mem_hdr: fills region type specific headers and returns pointer to
   2062 *	the first range or NULL if failed to fill headers.
   2063 * @fill_range: copies a given memory range into the dump.
   2064 *	Returns the size of the range or negative error value otherwise.
   2065 */
   2066struct iwl_dump_ini_mem_ops {
   2067	u32 (*get_num_of_ranges)(struct iwl_fw_runtime *fwrt,
   2068				 struct iwl_dump_ini_region_data *reg_data);
   2069	u32 (*get_size)(struct iwl_fw_runtime *fwrt,
   2070			struct iwl_dump_ini_region_data *reg_data);
   2071	void *(*fill_mem_hdr)(struct iwl_fw_runtime *fwrt,
   2072			      struct iwl_dump_ini_region_data *reg_data,
   2073			      void *data, u32 data_len);
   2074	int (*fill_range)(struct iwl_fw_runtime *fwrt,
   2075			  struct iwl_dump_ini_region_data *reg_data,
   2076			  void *range, u32 range_len, int idx);
   2077};
   2078
   2079/**
   2080 * iwl_dump_ini_mem
   2081 *
   2082 * Creates a dump tlv and copy a memory region into it.
   2083 * Returns the size of the current dump tlv or 0 if failed
   2084 *
   2085 * @fwrt: fw runtime struct
   2086 * @list: list to add the dump tlv to
   2087 * @reg_data: memory region
   2088 * @ops: memory dump operations
   2089 */
   2090static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
   2091			    struct iwl_dump_ini_region_data *reg_data,
   2092			    const struct iwl_dump_ini_mem_ops *ops)
   2093{
   2094	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
   2095	struct iwl_fw_ini_dump_entry *entry;
   2096	struct iwl_fw_ini_error_dump_data *tlv;
   2097	struct iwl_fw_ini_error_dump_header *header;
   2098	u32 type = reg->type;
   2099	u32 id = le32_get_bits(reg->id, IWL_FW_INI_REGION_ID_MASK);
   2100	u32 num_of_ranges, i, size;
   2101	u8 *range;
   2102	u32 free_size;
   2103	u64 header_size;
   2104	u32 dump_policy = IWL_FW_INI_DUMP_VERBOSE;
   2105
   2106	IWL_DEBUG_FW(fwrt, "WRT: Collecting region: dump type=%d, id=%d, type=%d\n",
   2107		     dump_policy, id, type);
   2108
   2109	if (le32_to_cpu(reg->hdr.version) >= 2) {
   2110		u32 dp = le32_get_bits(reg->id,
   2111				       IWL_FW_INI_REGION_DUMP_POLICY_MASK);
   2112
   2113		if (dump_policy == IWL_FW_INI_DUMP_VERBOSE &&
   2114		    !(dp & IWL_FW_INI_DEBUG_DUMP_POLICY_NO_LIMIT)) {
   2115			IWL_DEBUG_FW(fwrt,
   2116				     "WRT: no dump - type %d and policy mismatch=%d\n",
   2117				     dump_policy, dp);
   2118			return 0;
   2119		} else if (dump_policy == IWL_FW_INI_DUMP_MEDIUM &&
   2120			   !(dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_MAX_LIMIT_5MB)) {
   2121			IWL_DEBUG_FW(fwrt,
   2122				     "WRT: no dump - type %d and policy mismatch=%d\n",
   2123				     dump_policy, dp);
   2124			return 0;
   2125		} else if (dump_policy == IWL_FW_INI_DUMP_BRIEF &&
   2126			   !(dp & IWL_FW_INI_DEBUG_DUMP_POLICY_MAX_LIMIT_600KB)) {
   2127			IWL_DEBUG_FW(fwrt,
   2128				     "WRT: no dump - type %d and policy mismatch=%d\n",
   2129				     dump_policy, dp);
   2130			return 0;
   2131		}
   2132	}
   2133
   2134	if (!ops->get_num_of_ranges || !ops->get_size || !ops->fill_mem_hdr ||
   2135	    !ops->fill_range) {
   2136		IWL_DEBUG_FW(fwrt, "WRT: no ops for collecting data\n");
   2137		return 0;
   2138	}
   2139
   2140	size = ops->get_size(fwrt, reg_data);
   2141
   2142	if (size < sizeof(*header)) {
   2143		IWL_DEBUG_FW(fwrt, "WRT: size didn't include space for header\n");
   2144		return 0;
   2145	}
   2146
   2147	entry = vzalloc(sizeof(*entry) + sizeof(*tlv) + size);
   2148	if (!entry)
   2149		return 0;
   2150
   2151	entry->size = sizeof(*tlv) + size;
   2152
   2153	tlv = (void *)entry->data;
   2154	tlv->type = reg->type;
   2155	tlv->sub_type = reg->sub_type;
   2156	tlv->sub_type_ver = reg->sub_type_ver;
   2157	tlv->reserved = reg->reserved;
   2158	tlv->len = cpu_to_le32(size);
   2159
   2160	num_of_ranges = ops->get_num_of_ranges(fwrt, reg_data);
   2161
   2162	header = (void *)tlv->data;
   2163	header->region_id = cpu_to_le32(id);
   2164	header->num_of_ranges = cpu_to_le32(num_of_ranges);
   2165	header->name_len = cpu_to_le32(IWL_FW_INI_MAX_NAME);
   2166	memcpy(header->name, reg->name, IWL_FW_INI_MAX_NAME);
   2167
   2168	free_size = size;
   2169	range = ops->fill_mem_hdr(fwrt, reg_data, header, free_size);
   2170	if (!range) {
   2171		IWL_ERR(fwrt,
   2172			"WRT: Failed to fill region header: id=%d, type=%d\n",
   2173			id, type);
   2174		goto out_err;
   2175	}
   2176
   2177	header_size = range - (u8 *)header;
   2178
   2179	if (WARN(header_size > free_size,
   2180		 "header size %llu > free_size %d",
   2181		 header_size, free_size)) {
   2182		IWL_ERR(fwrt,
   2183			"WRT: fill_mem_hdr used more than given free_size\n");
   2184		goto out_err;
   2185	}
   2186
   2187	free_size -= header_size;
   2188
   2189	for (i = 0; i < num_of_ranges; i++) {
   2190		int range_size = ops->fill_range(fwrt, reg_data, range,
   2191						 free_size, i);
   2192
   2193		if (range_size < 0) {
   2194			IWL_ERR(fwrt,
   2195				"WRT: Failed to dump region: id=%d, type=%d\n",
   2196				id, type);
   2197			goto out_err;
   2198		}
   2199
   2200		if (WARN(range_size > free_size, "range_size %d > free_size %d",
   2201			 range_size, free_size)) {
   2202			IWL_ERR(fwrt,
   2203				"WRT: fill_raged used more than given free_size\n");
   2204			goto out_err;
   2205		}
   2206
   2207		free_size -= range_size;
   2208		range = range + range_size;
   2209	}
   2210
   2211	list_add_tail(&entry->list, list);
   2212
   2213	return entry->size;
   2214
   2215out_err:
   2216	vfree(entry);
   2217
   2218	return 0;
   2219}
   2220
   2221static u32 iwl_dump_ini_info(struct iwl_fw_runtime *fwrt,
   2222			     struct iwl_fw_ini_trigger_tlv *trigger,
   2223			     struct list_head *list)
   2224{
   2225	struct iwl_fw_ini_dump_entry *entry;
   2226	struct iwl_fw_error_dump_data *tlv;
   2227	struct iwl_fw_ini_dump_info *dump;
   2228	struct iwl_dbg_tlv_node *node;
   2229	struct iwl_fw_ini_dump_cfg_name *cfg_name;
   2230	u32 size = sizeof(*tlv) + sizeof(*dump);
   2231	u32 num_of_cfg_names = 0;
   2232	u32 hw_type;
   2233
   2234	list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
   2235		size += sizeof(*cfg_name);
   2236		num_of_cfg_names++;
   2237	}
   2238
   2239	entry = vzalloc(sizeof(*entry) + size);
   2240	if (!entry)
   2241		return 0;
   2242
   2243	entry->size = size;
   2244
   2245	tlv = (void *)entry->data;
   2246	tlv->type = cpu_to_le32(IWL_INI_DUMP_INFO_TYPE);
   2247	tlv->len = cpu_to_le32(size - sizeof(*tlv));
   2248
   2249	dump = (void *)tlv->data;
   2250
   2251	dump->version = cpu_to_le32(IWL_INI_DUMP_VER);
   2252	dump->time_point = trigger->time_point;
   2253	dump->trigger_reason = trigger->trigger_reason;
   2254	dump->external_cfg_state =
   2255		cpu_to_le32(fwrt->trans->dbg.external_ini_cfg);
   2256
   2257	dump->ver_type = cpu_to_le32(fwrt->dump.fw_ver.type);
   2258	dump->ver_subtype = cpu_to_le32(fwrt->dump.fw_ver.subtype);
   2259
   2260	dump->hw_step = cpu_to_le32(fwrt->trans->hw_rev_step);
   2261
   2262	/*
   2263	 * Several HWs all have type == 0x42, so we'll override this value
   2264	 * according to the detected HW
   2265	 */
   2266	hw_type = CSR_HW_REV_TYPE(fwrt->trans->hw_rev);
   2267	if (hw_type == IWL_AX210_HW_TYPE) {
   2268		u32 prph_val = iwl_read_umac_prph(fwrt->trans, WFPM_OTP_CFG1_ADDR);
   2269		u32 is_jacket = !!(prph_val & WFPM_OTP_CFG1_IS_JACKET_BIT);
   2270		u32 is_cdb = !!(prph_val & WFPM_OTP_CFG1_IS_CDB_BIT);
   2271		u32 masked_bits = is_jacket | (is_cdb << 1);
   2272
   2273		/*
   2274		 * The HW type depends on certain bits in this case, so add
   2275		 * these bits to the HW type. We won't have collisions since we
   2276		 * add these bits after the highest possible bit in the mask.
   2277		 */
   2278		hw_type |= masked_bits << IWL_AX210_HW_TYPE_ADDITION_SHIFT;
   2279	}
   2280	dump->hw_type = cpu_to_le32(hw_type);
   2281
   2282	dump->rf_id_flavor =
   2283		cpu_to_le32(CSR_HW_RFID_FLAVOR(fwrt->trans->hw_rf_id));
   2284	dump->rf_id_dash = cpu_to_le32(CSR_HW_RFID_DASH(fwrt->trans->hw_rf_id));
   2285	dump->rf_id_step = cpu_to_le32(CSR_HW_RFID_STEP(fwrt->trans->hw_rf_id));
   2286	dump->rf_id_type = cpu_to_le32(CSR_HW_RFID_TYPE(fwrt->trans->hw_rf_id));
   2287
   2288	dump->lmac_major = cpu_to_le32(fwrt->dump.fw_ver.lmac_major);
   2289	dump->lmac_minor = cpu_to_le32(fwrt->dump.fw_ver.lmac_minor);
   2290	dump->umac_major = cpu_to_le32(fwrt->dump.fw_ver.umac_major);
   2291	dump->umac_minor = cpu_to_le32(fwrt->dump.fw_ver.umac_minor);
   2292
   2293	dump->fw_mon_mode = cpu_to_le32(fwrt->trans->dbg.ini_dest);
   2294	dump->regions_mask = trigger->regions_mask &
   2295			     ~cpu_to_le64(fwrt->trans->dbg.unsupported_region_msk);
   2296
   2297	dump->build_tag_len = cpu_to_le32(sizeof(dump->build_tag));
   2298	memcpy(dump->build_tag, fwrt->fw->human_readable,
   2299	       sizeof(dump->build_tag));
   2300
   2301	cfg_name = dump->cfg_names;
   2302	dump->num_of_cfg_names = cpu_to_le32(num_of_cfg_names);
   2303	list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
   2304		struct iwl_fw_ini_debug_info_tlv *debug_info =
   2305			(void *)node->tlv.data;
   2306
   2307		cfg_name->image_type = debug_info->image_type;
   2308		cfg_name->cfg_name_len =
   2309			cpu_to_le32(IWL_FW_INI_MAX_CFG_NAME);
   2310		memcpy(cfg_name->cfg_name, debug_info->debug_cfg_name,
   2311		       sizeof(cfg_name->cfg_name));
   2312		cfg_name++;
   2313	}
   2314
   2315	/* add dump info TLV to the beginning of the list since it needs to be
   2316	 * the first TLV in the dump
   2317	 */
   2318	list_add(&entry->list, list);
   2319
   2320	return entry->size;
   2321}
   2322
   2323static const struct iwl_dump_ini_mem_ops iwl_dump_ini_region_ops[] = {
   2324	[IWL_FW_INI_REGION_INVALID] = {},
   2325	[IWL_FW_INI_REGION_INTERNAL_BUFFER] = {
   2326		.get_num_of_ranges = iwl_dump_ini_single_range,
   2327		.get_size = iwl_dump_ini_mon_smem_get_size,
   2328		.fill_mem_hdr = iwl_dump_ini_mon_smem_fill_header,
   2329		.fill_range = iwl_dump_ini_mon_smem_iter,
   2330	},
   2331	[IWL_FW_INI_REGION_DRAM_BUFFER] = {
   2332		.get_num_of_ranges = iwl_dump_ini_mon_dram_ranges,
   2333		.get_size = iwl_dump_ini_mon_dram_get_size,
   2334		.fill_mem_hdr = iwl_dump_ini_mon_dram_fill_header,
   2335		.fill_range = iwl_dump_ini_mon_dram_iter,
   2336	},
   2337	[IWL_FW_INI_REGION_TXF] = {
   2338		.get_num_of_ranges = iwl_dump_ini_txf_ranges,
   2339		.get_size = iwl_dump_ini_txf_get_size,
   2340		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2341		.fill_range = iwl_dump_ini_txf_iter,
   2342	},
   2343	[IWL_FW_INI_REGION_RXF] = {
   2344		.get_num_of_ranges = iwl_dump_ini_single_range,
   2345		.get_size = iwl_dump_ini_rxf_get_size,
   2346		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2347		.fill_range = iwl_dump_ini_rxf_iter,
   2348	},
   2349	[IWL_FW_INI_REGION_LMAC_ERROR_TABLE] = {
   2350		.get_num_of_ranges = iwl_dump_ini_single_range,
   2351		.get_size = iwl_dump_ini_err_table_get_size,
   2352		.fill_mem_hdr = iwl_dump_ini_err_table_fill_header,
   2353		.fill_range = iwl_dump_ini_err_table_iter,
   2354	},
   2355	[IWL_FW_INI_REGION_UMAC_ERROR_TABLE] = {
   2356		.get_num_of_ranges = iwl_dump_ini_single_range,
   2357		.get_size = iwl_dump_ini_err_table_get_size,
   2358		.fill_mem_hdr = iwl_dump_ini_err_table_fill_header,
   2359		.fill_range = iwl_dump_ini_err_table_iter,
   2360	},
   2361	[IWL_FW_INI_REGION_RSP_OR_NOTIF] = {
   2362		.get_num_of_ranges = iwl_dump_ini_single_range,
   2363		.get_size = iwl_dump_ini_fw_pkt_get_size,
   2364		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2365		.fill_range = iwl_dump_ini_fw_pkt_iter,
   2366	},
   2367	[IWL_FW_INI_REGION_DEVICE_MEMORY] = {
   2368		.get_num_of_ranges = iwl_dump_ini_mem_ranges,
   2369		.get_size = iwl_dump_ini_mem_get_size,
   2370		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2371		.fill_range = iwl_dump_ini_dev_mem_iter,
   2372	},
   2373	[IWL_FW_INI_REGION_PERIPHERY_MAC] = {
   2374		.get_num_of_ranges = iwl_dump_ini_mem_ranges,
   2375		.get_size = iwl_dump_ini_mem_get_size,
   2376		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2377		.fill_range = iwl_dump_ini_prph_mac_iter,
   2378	},
   2379	[IWL_FW_INI_REGION_PERIPHERY_PHY] = {
   2380		.get_num_of_ranges = iwl_dump_ini_mem_ranges,
   2381		.get_size = iwl_dump_ini_mem_get_size,
   2382		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2383		.fill_range = iwl_dump_ini_prph_phy_iter,
   2384	},
   2385	[IWL_FW_INI_REGION_PERIPHERY_AUX] = {},
   2386	[IWL_FW_INI_REGION_PAGING] = {
   2387		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2388		.get_num_of_ranges = iwl_dump_ini_paging_ranges,
   2389		.get_size = iwl_dump_ini_paging_get_size,
   2390		.fill_range = iwl_dump_ini_paging_iter,
   2391	},
   2392	[IWL_FW_INI_REGION_CSR] = {
   2393		.get_num_of_ranges = iwl_dump_ini_mem_ranges,
   2394		.get_size = iwl_dump_ini_mem_get_size,
   2395		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2396		.fill_range = iwl_dump_ini_csr_iter,
   2397	},
   2398	[IWL_FW_INI_REGION_DRAM_IMR] = {
   2399		.get_num_of_ranges = iwl_dump_ini_imr_ranges,
   2400		.get_size = iwl_dump_ini_imr_get_size,
   2401		.fill_mem_hdr = iwl_dump_ini_imr_fill_header,
   2402		.fill_range = iwl_dump_ini_imr_iter,
   2403	},
   2404	[IWL_FW_INI_REGION_PCI_IOSF_CONFIG] = {
   2405		.get_num_of_ranges = iwl_dump_ini_mem_ranges,
   2406		.get_size = iwl_dump_ini_mem_get_size,
   2407		.fill_mem_hdr = iwl_dump_ini_mem_fill_header,
   2408		.fill_range = iwl_dump_ini_config_iter,
   2409	},
   2410	[IWL_FW_INI_REGION_SPECIAL_DEVICE_MEMORY] = {
   2411		.get_num_of_ranges = iwl_dump_ini_single_range,
   2412		.get_size = iwl_dump_ini_special_mem_get_size,
   2413		.fill_mem_hdr = iwl_dump_ini_special_mem_fill_header,
   2414		.fill_range = iwl_dump_ini_special_mem_iter,
   2415	},
   2416	[IWL_FW_INI_REGION_DBGI_SRAM] = {
   2417		.get_num_of_ranges = iwl_dump_ini_mem_ranges,
   2418		.get_size = iwl_dump_ini_mon_dbgi_get_size,
   2419		.fill_mem_hdr = iwl_dump_ini_mon_dbgi_fill_header,
   2420		.fill_range = iwl_dump_ini_dbgi_sram_iter,
   2421	},
   2422};
   2423
   2424static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
   2425				struct iwl_fwrt_dump_data *dump_data,
   2426				struct list_head *list)
   2427{
   2428	struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
   2429	enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trigger->time_point);
   2430	struct iwl_dump_ini_region_data reg_data = {
   2431		.dump_data = dump_data,
   2432	};
   2433	struct iwl_dump_ini_region_data imr_reg_data = {
   2434		.dump_data = dump_data,
   2435	};
   2436	int i;
   2437	u32 size = 0;
   2438	u64 regions_mask = le64_to_cpu(trigger->regions_mask) &
   2439			   ~(fwrt->trans->dbg.unsupported_region_msk);
   2440
   2441	BUILD_BUG_ON(sizeof(trigger->regions_mask) != sizeof(regions_mask));
   2442	BUILD_BUG_ON((sizeof(trigger->regions_mask) * BITS_PER_BYTE) <
   2443		     ARRAY_SIZE(fwrt->trans->dbg.active_regions));
   2444
   2445	for (i = 0; i < ARRAY_SIZE(fwrt->trans->dbg.active_regions); i++) {
   2446		u32 reg_type;
   2447		struct iwl_fw_ini_region_tlv *reg;
   2448
   2449		if (!(BIT_ULL(i) & regions_mask))
   2450			continue;
   2451
   2452		reg_data.reg_tlv = fwrt->trans->dbg.active_regions[i];
   2453		if (!reg_data.reg_tlv) {
   2454			IWL_WARN(fwrt,
   2455				 "WRT: Unassigned region id %d, skipping\n", i);
   2456			continue;
   2457		}
   2458
   2459		reg = (void *)reg_data.reg_tlv->data;
   2460		reg_type = reg->type;
   2461		if (reg_type >= ARRAY_SIZE(iwl_dump_ini_region_ops))
   2462			continue;
   2463
   2464		if (reg_type == IWL_FW_INI_REGION_PERIPHERY_PHY &&
   2465		    tp_id != IWL_FW_INI_TIME_POINT_FW_ASSERT) {
   2466			IWL_WARN(fwrt,
   2467				 "WRT: trying to collect phy prph at time point: %d, skipping\n",
   2468				 tp_id);
   2469			continue;
   2470		}
   2471		/*
   2472		 * DRAM_IMR can be collected only for FW/HW error timepoint
   2473		 * when fw is not alive. In addition, it must be collected
   2474		 * lastly as it overwrites SRAM that can possibly contain
   2475		 * debug data which also need to be collected.
   2476		 */
   2477		if (reg_type == IWL_FW_INI_REGION_DRAM_IMR) {
   2478			if (tp_id == IWL_FW_INI_TIME_POINT_FW_ASSERT ||
   2479			    tp_id == IWL_FW_INI_TIME_POINT_FW_HW_ERROR)
   2480				imr_reg_data.reg_tlv = fwrt->trans->dbg.active_regions[i];
   2481			else
   2482				IWL_INFO(fwrt,
   2483					 "WRT: trying to collect DRAM_IMR at time point: %d, skipping\n",
   2484					 tp_id);
   2485		/* continue to next region */
   2486			continue;
   2487		}
   2488
   2489
   2490		size += iwl_dump_ini_mem(fwrt, list, &reg_data,
   2491					 &iwl_dump_ini_region_ops[reg_type]);
   2492	}
   2493	/* collect DRAM_IMR region in the last */
   2494	if (imr_reg_data.reg_tlv)
   2495		size += iwl_dump_ini_mem(fwrt, list, &reg_data,
   2496					 &iwl_dump_ini_region_ops[IWL_FW_INI_REGION_DRAM_IMR]);
   2497
   2498	if (size)
   2499		size += iwl_dump_ini_info(fwrt, trigger, list);
   2500
   2501	return size;
   2502}
   2503
   2504static bool iwl_fw_ini_trigger_on(struct iwl_fw_runtime *fwrt,
   2505				  struct iwl_fw_ini_trigger_tlv *trig)
   2506{
   2507	enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trig->time_point);
   2508	u32 usec = le32_to_cpu(trig->ignore_consec);
   2509
   2510	if (!iwl_trans_dbg_ini_valid(fwrt->trans) ||
   2511	    tp_id == IWL_FW_INI_TIME_POINT_INVALID ||
   2512	    tp_id >= IWL_FW_INI_TIME_POINT_NUM ||
   2513	    iwl_fw_dbg_no_trig_window(fwrt, tp_id, usec))
   2514		return false;
   2515
   2516	return true;
   2517}
   2518
   2519static u32 iwl_dump_ini_file_gen(struct iwl_fw_runtime *fwrt,
   2520				 struct iwl_fwrt_dump_data *dump_data,
   2521				 struct list_head *list)
   2522{
   2523	struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
   2524	struct iwl_fw_ini_dump_entry *entry;
   2525	struct iwl_fw_ini_dump_file_hdr *hdr;
   2526	u32 size;
   2527
   2528	if (!trigger || !iwl_fw_ini_trigger_on(fwrt, trigger) ||
   2529	    !le64_to_cpu(trigger->regions_mask))
   2530		return 0;
   2531
   2532	entry = vzalloc(sizeof(*entry) + sizeof(*hdr));
   2533	if (!entry)
   2534		return 0;
   2535
   2536	entry->size = sizeof(*hdr);
   2537
   2538	size = iwl_dump_ini_trigger(fwrt, dump_data, list);
   2539	if (!size) {
   2540		vfree(entry);
   2541		return 0;
   2542	}
   2543
   2544	hdr = (void *)entry->data;
   2545	hdr->barker = cpu_to_le32(IWL_FW_INI_ERROR_DUMP_BARKER);
   2546	hdr->file_len = cpu_to_le32(size + entry->size);
   2547
   2548	list_add(&entry->list, list);
   2549
   2550	return le32_to_cpu(hdr->file_len);
   2551}
   2552
   2553static inline void iwl_fw_free_dump_desc(struct iwl_fw_runtime *fwrt,
   2554					 const struct iwl_fw_dump_desc *desc)
   2555{
   2556	if (desc && desc != &iwl_dump_desc_assert)
   2557		kfree(desc);
   2558
   2559	fwrt->dump.lmac_err_id[0] = 0;
   2560	if (fwrt->smem_cfg.num_lmacs > 1)
   2561		fwrt->dump.lmac_err_id[1] = 0;
   2562	fwrt->dump.umac_err_id = 0;
   2563}
   2564
   2565static void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt,
   2566			      struct iwl_fwrt_dump_data *dump_data)
   2567{
   2568	struct iwl_fw_dump_ptrs fw_error_dump = {};
   2569	struct iwl_fw_error_dump_file *dump_file;
   2570	struct scatterlist *sg_dump_data;
   2571	u32 file_len;
   2572	u32 dump_mask = fwrt->fw->dbg.dump_mask;
   2573
   2574	dump_file = iwl_fw_error_dump_file(fwrt, &fw_error_dump, dump_data);
   2575	if (!dump_file)
   2576		return;
   2577
   2578	if (dump_data->monitor_only)
   2579		dump_mask &= BIT(IWL_FW_ERROR_DUMP_FW_MONITOR);
   2580
   2581	fw_error_dump.trans_ptr = iwl_trans_dump_data(fwrt->trans, dump_mask,
   2582						      fwrt->sanitize_ops,
   2583						      fwrt->sanitize_ctx);
   2584	file_len = le32_to_cpu(dump_file->file_len);
   2585	fw_error_dump.fwrt_len = file_len;
   2586
   2587	if (fw_error_dump.trans_ptr) {
   2588		file_len += fw_error_dump.trans_ptr->len;
   2589		dump_file->file_len = cpu_to_le32(file_len);
   2590	}
   2591
   2592	sg_dump_data = alloc_sgtable(file_len);
   2593	if (sg_dump_data) {
   2594		sg_pcopy_from_buffer(sg_dump_data,
   2595				     sg_nents(sg_dump_data),
   2596				     fw_error_dump.fwrt_ptr,
   2597				     fw_error_dump.fwrt_len, 0);
   2598		if (fw_error_dump.trans_ptr)
   2599			sg_pcopy_from_buffer(sg_dump_data,
   2600					     sg_nents(sg_dump_data),
   2601					     fw_error_dump.trans_ptr->data,
   2602					     fw_error_dump.trans_ptr->len,
   2603					     fw_error_dump.fwrt_len);
   2604		dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
   2605			       GFP_KERNEL);
   2606	}
   2607	vfree(fw_error_dump.fwrt_ptr);
   2608	vfree(fw_error_dump.trans_ptr);
   2609}
   2610
   2611static void iwl_dump_ini_list_free(struct list_head *list)
   2612{
   2613	while (!list_empty(list)) {
   2614		struct iwl_fw_ini_dump_entry *entry =
   2615			list_entry(list->next, typeof(*entry), list);
   2616
   2617		list_del(&entry->list);
   2618		vfree(entry);
   2619	}
   2620}
   2621
   2622static void iwl_fw_error_dump_data_free(struct iwl_fwrt_dump_data *dump_data)
   2623{
   2624	dump_data->trig = NULL;
   2625	kfree(dump_data->fw_pkt);
   2626	dump_data->fw_pkt = NULL;
   2627}
   2628
   2629static void iwl_fw_error_ini_dump(struct iwl_fw_runtime *fwrt,
   2630				  struct iwl_fwrt_dump_data *dump_data)
   2631{
   2632	LIST_HEAD(dump_list);
   2633	struct scatterlist *sg_dump_data;
   2634	u32 file_len = iwl_dump_ini_file_gen(fwrt, dump_data, &dump_list);
   2635
   2636	if (!file_len)
   2637		return;
   2638
   2639	sg_dump_data = alloc_sgtable(file_len);
   2640	if (sg_dump_data) {
   2641		struct iwl_fw_ini_dump_entry *entry;
   2642		int sg_entries = sg_nents(sg_dump_data);
   2643		u32 offs = 0;
   2644
   2645		list_for_each_entry(entry, &dump_list, list) {
   2646			sg_pcopy_from_buffer(sg_dump_data, sg_entries,
   2647					     entry->data, entry->size, offs);
   2648			offs += entry->size;
   2649		}
   2650		dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
   2651			       GFP_KERNEL);
   2652	}
   2653	iwl_dump_ini_list_free(&dump_list);
   2654}
   2655
   2656const struct iwl_fw_dump_desc iwl_dump_desc_assert = {
   2657	.trig_desc = {
   2658		.type = cpu_to_le32(FW_DBG_TRIGGER_FW_ASSERT),
   2659	},
   2660};
   2661IWL_EXPORT_SYMBOL(iwl_dump_desc_assert);
   2662
   2663int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
   2664			    const struct iwl_fw_dump_desc *desc,
   2665			    bool monitor_only,
   2666			    unsigned int delay)
   2667{
   2668	struct iwl_fwrt_wk_data *wk_data;
   2669	unsigned long idx;
   2670
   2671	if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
   2672		iwl_fw_free_dump_desc(fwrt, desc);
   2673		return 0;
   2674	}
   2675
   2676	/*
   2677	 * Check there is an available worker.
   2678	 * ffz return value is undefined if no zero exists,
   2679	 * so check against ~0UL first.
   2680	 */
   2681	if (fwrt->dump.active_wks == ~0UL)
   2682		return -EBUSY;
   2683
   2684	idx = ffz(fwrt->dump.active_wks);
   2685
   2686	if (idx >= IWL_FW_RUNTIME_DUMP_WK_NUM ||
   2687	    test_and_set_bit(fwrt->dump.wks[idx].idx, &fwrt->dump.active_wks))
   2688		return -EBUSY;
   2689
   2690	wk_data = &fwrt->dump.wks[idx];
   2691
   2692	if (WARN_ON(wk_data->dump_data.desc))
   2693		iwl_fw_free_dump_desc(fwrt, wk_data->dump_data.desc);
   2694
   2695	wk_data->dump_data.desc = desc;
   2696	wk_data->dump_data.monitor_only = monitor_only;
   2697
   2698	IWL_WARN(fwrt, "Collecting data: trigger %d fired.\n",
   2699		 le32_to_cpu(desc->trig_desc.type));
   2700
   2701	schedule_delayed_work(&wk_data->wk, usecs_to_jiffies(delay));
   2702
   2703	return 0;
   2704}
   2705IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_desc);
   2706
   2707int iwl_fw_dbg_error_collect(struct iwl_fw_runtime *fwrt,
   2708			     enum iwl_fw_dbg_trigger trig_type)
   2709{
   2710	if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status))
   2711		return -EIO;
   2712
   2713	if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
   2714		if (trig_type != FW_DBG_TRIGGER_ALIVE_TIMEOUT &&
   2715		    trig_type != FW_DBG_TRIGGER_DRIVER)
   2716			return -EIO;
   2717
   2718		iwl_dbg_tlv_time_point(fwrt,
   2719				       IWL_FW_INI_TIME_POINT_HOST_ALIVE_TIMEOUT,
   2720				       NULL);
   2721	} else {
   2722		struct iwl_fw_dump_desc *iwl_dump_error_desc;
   2723		int ret;
   2724
   2725		iwl_dump_error_desc =
   2726			kmalloc(sizeof(*iwl_dump_error_desc), GFP_KERNEL);
   2727
   2728		if (!iwl_dump_error_desc)
   2729			return -ENOMEM;
   2730
   2731		iwl_dump_error_desc->trig_desc.type = cpu_to_le32(trig_type);
   2732		iwl_dump_error_desc->len = 0;
   2733
   2734		ret = iwl_fw_dbg_collect_desc(fwrt, iwl_dump_error_desc,
   2735					      false, 0);
   2736		if (ret) {
   2737			kfree(iwl_dump_error_desc);
   2738			return ret;
   2739		}
   2740	}
   2741
   2742	iwl_trans_sync_nmi(fwrt->trans);
   2743
   2744	return 0;
   2745}
   2746IWL_EXPORT_SYMBOL(iwl_fw_dbg_error_collect);
   2747
   2748int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
   2749		       enum iwl_fw_dbg_trigger trig,
   2750		       const char *str, size_t len,
   2751		       struct iwl_fw_dbg_trigger_tlv *trigger)
   2752{
   2753	struct iwl_fw_dump_desc *desc;
   2754	unsigned int delay = 0;
   2755	bool monitor_only = false;
   2756
   2757	if (trigger) {
   2758		u16 occurrences = le16_to_cpu(trigger->occurrences) - 1;
   2759
   2760		if (!le16_to_cpu(trigger->occurrences))
   2761			return 0;
   2762
   2763		if (trigger->flags & IWL_FW_DBG_FORCE_RESTART) {
   2764			IWL_WARN(fwrt, "Force restart: trigger %d fired.\n",
   2765				 trig);
   2766			iwl_force_nmi(fwrt->trans);
   2767			return 0;
   2768		}
   2769
   2770		trigger->occurrences = cpu_to_le16(occurrences);
   2771		monitor_only = trigger->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY;
   2772
   2773		/* convert msec to usec */
   2774		delay = le32_to_cpu(trigger->stop_delay) * USEC_PER_MSEC;
   2775	}
   2776
   2777	desc = kzalloc(struct_size(desc, trig_desc.data, len), GFP_ATOMIC);
   2778	if (!desc)
   2779		return -ENOMEM;
   2780
   2781
   2782	desc->len = len;
   2783	desc->trig_desc.type = cpu_to_le32(trig);
   2784	memcpy(desc->trig_desc.data, str, len);
   2785
   2786	return iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay);
   2787}
   2788IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect);
   2789
   2790int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
   2791			    struct iwl_fw_dbg_trigger_tlv *trigger,
   2792			    const char *fmt, ...)
   2793{
   2794	int ret, len = 0;
   2795	char buf[64];
   2796
   2797	if (iwl_trans_dbg_ini_valid(fwrt->trans))
   2798		return 0;
   2799
   2800	if (fmt) {
   2801		va_list ap;
   2802
   2803		buf[sizeof(buf) - 1] = '\0';
   2804
   2805		va_start(ap, fmt);
   2806		vsnprintf(buf, sizeof(buf), fmt, ap);
   2807		va_end(ap);
   2808
   2809		/* check for truncation */
   2810		if (WARN_ON_ONCE(buf[sizeof(buf) - 1]))
   2811			buf[sizeof(buf) - 1] = '\0';
   2812
   2813		len = strlen(buf) + 1;
   2814	}
   2815
   2816	ret = iwl_fw_dbg_collect(fwrt, le32_to_cpu(trigger->id), buf, len,
   2817				 trigger);
   2818
   2819	if (ret)
   2820		return ret;
   2821
   2822	return 0;
   2823}
   2824IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_trig);
   2825
   2826int iwl_fw_start_dbg_conf(struct iwl_fw_runtime *fwrt, u8 conf_id)
   2827{
   2828	u8 *ptr;
   2829	int ret;
   2830	int i;
   2831
   2832	if (WARN_ONCE(conf_id >= ARRAY_SIZE(fwrt->fw->dbg.conf_tlv),
   2833		      "Invalid configuration %d\n", conf_id))
   2834		return -EINVAL;
   2835
   2836	/* EARLY START - firmware's configuration is hard coded */
   2837	if ((!fwrt->fw->dbg.conf_tlv[conf_id] ||
   2838	     !fwrt->fw->dbg.conf_tlv[conf_id]->num_of_hcmds) &&
   2839	    conf_id == FW_DBG_START_FROM_ALIVE)
   2840		return 0;
   2841
   2842	if (!fwrt->fw->dbg.conf_tlv[conf_id])
   2843		return -EINVAL;
   2844
   2845	if (fwrt->dump.conf != FW_DBG_INVALID)
   2846		IWL_INFO(fwrt, "FW already configured (%d) - re-configuring\n",
   2847			 fwrt->dump.conf);
   2848
   2849	/* Send all HCMDs for configuring the FW debug */
   2850	ptr = (void *)&fwrt->fw->dbg.conf_tlv[conf_id]->hcmd;
   2851	for (i = 0; i < fwrt->fw->dbg.conf_tlv[conf_id]->num_of_hcmds; i++) {
   2852		struct iwl_fw_dbg_conf_hcmd *cmd = (void *)ptr;
   2853		struct iwl_host_cmd hcmd = {
   2854			.id = cmd->id,
   2855			.len = { le16_to_cpu(cmd->len), },
   2856			.data = { cmd->data, },
   2857		};
   2858
   2859		ret = iwl_trans_send_cmd(fwrt->trans, &hcmd);
   2860		if (ret)
   2861			return ret;
   2862
   2863		ptr += sizeof(*cmd);
   2864		ptr += le16_to_cpu(cmd->len);
   2865	}
   2866
   2867	fwrt->dump.conf = conf_id;
   2868
   2869	return 0;
   2870}
   2871IWL_EXPORT_SYMBOL(iwl_fw_start_dbg_conf);
   2872
   2873void iwl_send_dbg_dump_complete_cmd(struct iwl_fw_runtime *fwrt,
   2874				    u32 timepoint,
   2875				    u32 timepoint_data)
   2876{
   2877	struct iwl_dbg_dump_complete_cmd hcmd_data;
   2878	struct iwl_host_cmd hcmd = {
   2879		.id = WIDE_ID(DEBUG_GROUP, FW_DUMP_COMPLETE_CMD),
   2880		.data[0] = &hcmd_data,
   2881		.len[0] = sizeof(hcmd_data),
   2882	};
   2883
   2884	if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status))
   2885		return;
   2886
   2887	if (fw_has_capa(&fwrt->fw->ucode_capa,
   2888			IWL_UCODE_TLV_CAPA_DUMP_COMPLETE_SUPPORT)) {
   2889		hcmd_data.tp = cpu_to_le32(timepoint);
   2890		hcmd_data.tp_data = cpu_to_le32(timepoint_data);
   2891		iwl_trans_send_cmd(fwrt->trans, &hcmd);
   2892	}
   2893}
   2894
   2895/* this function assumes dump_start was called beforehand and dump_end will be
   2896 * called afterwards
   2897 */
   2898static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
   2899{
   2900	struct iwl_fw_dbg_params params = {0};
   2901	struct iwl_fwrt_dump_data *dump_data =
   2902		&fwrt->dump.wks[wk_idx].dump_data;
   2903	u32 policy;
   2904	u32 time_point;
   2905	if (!test_bit(wk_idx, &fwrt->dump.active_wks))
   2906		return;
   2907
   2908	if (!dump_data->trig) {
   2909		IWL_ERR(fwrt, "dump trigger data is not set\n");
   2910		goto out;
   2911	}
   2912
   2913	if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status)) {
   2914		IWL_ERR(fwrt, "Device is not enabled - cannot dump error\n");
   2915		goto out;
   2916	}
   2917
   2918	/* there's no point in fw dump if the bus is dead */
   2919	if (test_bit(STATUS_TRANS_DEAD, &fwrt->trans->status)) {
   2920		IWL_ERR(fwrt, "Skip fw error dump since bus is dead\n");
   2921		goto out;
   2922	}
   2923
   2924	iwl_fw_dbg_stop_restart_recording(fwrt, &params, true);
   2925
   2926	IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection start\n");
   2927	if (iwl_trans_dbg_ini_valid(fwrt->trans))
   2928		iwl_fw_error_ini_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
   2929	else
   2930		iwl_fw_error_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
   2931	IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection done\n");
   2932
   2933	iwl_fw_dbg_stop_restart_recording(fwrt, &params, false);
   2934
   2935	policy = le32_to_cpu(dump_data->trig->apply_policy);
   2936	time_point = le32_to_cpu(dump_data->trig->time_point);
   2937
   2938	if (policy & IWL_FW_INI_APPLY_POLICY_DUMP_COMPLETE_CMD) {
   2939		IWL_DEBUG_FW_INFO(fwrt, "WRT: sending dump complete\n");
   2940		iwl_send_dbg_dump_complete_cmd(fwrt, time_point, 0);
   2941	}
   2942	if (fwrt->trans->dbg.last_tp_resetfw == IWL_FW_INI_RESET_FW_MODE_STOP_FW_ONLY)
   2943		iwl_force_nmi(fwrt->trans);
   2944
   2945out:
   2946	if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
   2947		iwl_fw_error_dump_data_free(dump_data);
   2948	} else {
   2949		iwl_fw_free_dump_desc(fwrt, dump_data->desc);
   2950		dump_data->desc = NULL;
   2951	}
   2952
   2953	clear_bit(wk_idx, &fwrt->dump.active_wks);
   2954}
   2955
   2956int iwl_fw_dbg_ini_collect(struct iwl_fw_runtime *fwrt,
   2957			   struct iwl_fwrt_dump_data *dump_data,
   2958			   bool sync)
   2959{
   2960	struct iwl_fw_ini_trigger_tlv *trig = dump_data->trig;
   2961	enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trig->time_point);
   2962	u32 occur, delay;
   2963	unsigned long idx;
   2964
   2965	if (!iwl_fw_ini_trigger_on(fwrt, trig)) {
   2966		IWL_WARN(fwrt, "WRT: Trigger %d is not active, aborting dump\n",
   2967			 tp_id);
   2968		return -EINVAL;
   2969	}
   2970
   2971	delay = le32_to_cpu(trig->dump_delay);
   2972	occur = le32_to_cpu(trig->occurrences);
   2973	if (!occur)
   2974		return 0;
   2975
   2976	trig->occurrences = cpu_to_le32(--occur);
   2977
   2978	/* Check there is an available worker.
   2979	 * ffz return value is undefined if no zero exists,
   2980	 * so check against ~0UL first.
   2981	 */
   2982	if (fwrt->dump.active_wks == ~0UL)
   2983		return -EBUSY;
   2984
   2985	idx = ffz(fwrt->dump.active_wks);
   2986
   2987	if (idx >= IWL_FW_RUNTIME_DUMP_WK_NUM ||
   2988	    test_and_set_bit(fwrt->dump.wks[idx].idx, &fwrt->dump.active_wks))
   2989		return -EBUSY;
   2990
   2991	fwrt->dump.wks[idx].dump_data = *dump_data;
   2992
   2993	if (sync)
   2994		delay = 0;
   2995
   2996	IWL_WARN(fwrt,
   2997		 "WRT: Collecting data: ini trigger %d fired (delay=%dms).\n",
   2998		 tp_id, (u32)(delay / USEC_PER_MSEC));
   2999
   3000	if (sync)
   3001		iwl_fw_dbg_collect_sync(fwrt, idx);
   3002	else
   3003		schedule_delayed_work(&fwrt->dump.wks[idx].wk, usecs_to_jiffies(delay));
   3004
   3005	return 0;
   3006}
   3007
   3008void iwl_fw_error_dump_wk(struct work_struct *work)
   3009{
   3010	struct iwl_fwrt_wk_data *wks =
   3011		container_of(work, typeof(*wks), wk.work);
   3012	struct iwl_fw_runtime *fwrt =
   3013		container_of(wks, typeof(*fwrt), dump.wks[wks->idx]);
   3014
   3015	/* assumes the op mode mutex is locked in dump_start since
   3016	 * iwl_fw_dbg_collect_sync can't run in parallel
   3017	 */
   3018	if (fwrt->ops && fwrt->ops->dump_start)
   3019		fwrt->ops->dump_start(fwrt->ops_ctx);
   3020
   3021	iwl_fw_dbg_collect_sync(fwrt, wks->idx);
   3022
   3023	if (fwrt->ops && fwrt->ops->dump_end)
   3024		fwrt->ops->dump_end(fwrt->ops_ctx);
   3025}
   3026
   3027void iwl_fw_dbg_read_d3_debug_data(struct iwl_fw_runtime *fwrt)
   3028{
   3029	const struct iwl_cfg *cfg = fwrt->trans->cfg;
   3030
   3031	if (!iwl_fw_dbg_is_d3_debug_enabled(fwrt))
   3032		return;
   3033
   3034	if (!fwrt->dump.d3_debug_data) {
   3035		fwrt->dump.d3_debug_data = kmalloc(cfg->d3_debug_data_length,
   3036						   GFP_KERNEL);
   3037		if (!fwrt->dump.d3_debug_data) {
   3038			IWL_ERR(fwrt,
   3039				"failed to allocate memory for D3 debug data\n");
   3040			return;
   3041		}
   3042	}
   3043
   3044	/* if the buffer holds previous debug data it is overwritten */
   3045	iwl_trans_read_mem_bytes(fwrt->trans, cfg->d3_debug_data_base_addr,
   3046				 fwrt->dump.d3_debug_data,
   3047				 cfg->d3_debug_data_length);
   3048
   3049	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
   3050		fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
   3051					     cfg->d3_debug_data_base_addr,
   3052					     fwrt->dump.d3_debug_data,
   3053					     cfg->d3_debug_data_length);
   3054}
   3055IWL_EXPORT_SYMBOL(iwl_fw_dbg_read_d3_debug_data);
   3056
   3057void iwl_fw_dbg_stop_sync(struct iwl_fw_runtime *fwrt)
   3058{
   3059	int i;
   3060
   3061	iwl_dbg_tlv_del_timers(fwrt->trans);
   3062	for (i = 0; i < IWL_FW_RUNTIME_DUMP_WK_NUM; i++)
   3063		iwl_fw_dbg_collect_sync(fwrt, i);
   3064
   3065	iwl_fw_dbg_stop_restart_recording(fwrt, NULL, true);
   3066}
   3067IWL_EXPORT_SYMBOL(iwl_fw_dbg_stop_sync);
   3068
   3069static int iwl_fw_dbg_suspend_resume_hcmd(struct iwl_trans *trans, bool suspend)
   3070{
   3071	struct iwl_dbg_suspend_resume_cmd cmd = {
   3072		.operation = suspend ?
   3073			cpu_to_le32(DBGC_SUSPEND_CMD) :
   3074			cpu_to_le32(DBGC_RESUME_CMD),
   3075	};
   3076	struct iwl_host_cmd hcmd = {
   3077		.id = WIDE_ID(DEBUG_GROUP, DBGC_SUSPEND_RESUME),
   3078		.data[0] = &cmd,
   3079		.len[0] = sizeof(cmd),
   3080	};
   3081
   3082	return iwl_trans_send_cmd(trans, &hcmd);
   3083}
   3084
   3085static void iwl_fw_dbg_stop_recording(struct iwl_trans *trans,
   3086				      struct iwl_fw_dbg_params *params)
   3087{
   3088	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
   3089		iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
   3090		return;
   3091	}
   3092
   3093	if (params) {
   3094		params->in_sample = iwl_read_umac_prph(trans, DBGC_IN_SAMPLE);
   3095		params->out_ctrl = iwl_read_umac_prph(trans, DBGC_OUT_CTRL);
   3096	}
   3097
   3098	iwl_write_umac_prph(trans, DBGC_IN_SAMPLE, 0);
   3099	/* wait for the DBGC to finish writing the internal buffer to DRAM to
   3100	 * avoid halting the HW while writing
   3101	 */
   3102	usleep_range(700, 1000);
   3103	iwl_write_umac_prph(trans, DBGC_OUT_CTRL, 0);
   3104}
   3105
   3106static int iwl_fw_dbg_restart_recording(struct iwl_trans *trans,
   3107					struct iwl_fw_dbg_params *params)
   3108{
   3109	if (!params)
   3110		return -EIO;
   3111
   3112	if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
   3113		iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
   3114		iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
   3115		iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
   3116	} else {
   3117		iwl_write_umac_prph(trans, DBGC_IN_SAMPLE, params->in_sample);
   3118		iwl_write_umac_prph(trans, DBGC_OUT_CTRL, params->out_ctrl);
   3119	}
   3120
   3121	return 0;
   3122}
   3123
   3124void iwl_fw_dbg_stop_restart_recording(struct iwl_fw_runtime *fwrt,
   3125				       struct iwl_fw_dbg_params *params,
   3126				       bool stop)
   3127{
   3128	int ret __maybe_unused = 0;
   3129
   3130	if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status))
   3131		return;
   3132
   3133	if (fw_has_capa(&fwrt->fw->ucode_capa,
   3134			IWL_UCODE_TLV_CAPA_DBG_SUSPEND_RESUME_CMD_SUPP))
   3135		ret = iwl_fw_dbg_suspend_resume_hcmd(fwrt->trans, stop);
   3136	else if (stop)
   3137		iwl_fw_dbg_stop_recording(fwrt->trans, params);
   3138	else
   3139		ret = iwl_fw_dbg_restart_recording(fwrt->trans, params);
   3140#ifdef CONFIG_IWLWIFI_DEBUGFS
   3141	if (!ret) {
   3142		if (stop)
   3143			fwrt->trans->dbg.rec_on = false;
   3144		else
   3145			iwl_fw_set_dbg_rec_on(fwrt);
   3146	}
   3147#endif
   3148}
   3149IWL_EXPORT_SYMBOL(iwl_fw_dbg_stop_restart_recording);