cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

hfi_venus.c (40578B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
      4 * Copyright (C) 2017 Linaro Ltd.
      5 */
      6
      7#include <linux/delay.h>
      8#include <linux/device.h>
      9#include <linux/dma-mapping.h>
     10#include <linux/interrupt.h>
     11#include <linux/iopoll.h>
     12#include <linux/kernel.h>
     13#include <linux/slab.h>
     14
     15#include "core.h"
     16#include "hfi_cmds.h"
     17#include "hfi_msgs.h"
     18#include "hfi_venus.h"
     19#include "hfi_venus_io.h"
     20#include "firmware.h"
     21
     22#define HFI_MASK_QHDR_TX_TYPE		0xff000000
     23#define HFI_MASK_QHDR_RX_TYPE		0x00ff0000
     24#define HFI_MASK_QHDR_PRI_TYPE		0x0000ff00
     25#define HFI_MASK_QHDR_ID_TYPE		0x000000ff
     26
     27#define HFI_HOST_TO_CTRL_CMD_Q		0
     28#define HFI_CTRL_TO_HOST_MSG_Q		1
     29#define HFI_CTRL_TO_HOST_DBG_Q		2
     30#define HFI_MASK_QHDR_STATUS		0x000000ff
     31
     32#define IFACEQ_NUM			3
     33#define IFACEQ_CMD_IDX			0
     34#define IFACEQ_MSG_IDX			1
     35#define IFACEQ_DBG_IDX			2
     36#define IFACEQ_MAX_BUF_COUNT		50
     37#define IFACEQ_MAX_PARALLEL_CLNTS	16
     38#define IFACEQ_DFLT_QHDR		0x01010000
     39
     40#define POLL_INTERVAL_US		50
     41
     42#define IFACEQ_MAX_PKT_SIZE		1024
     43#define IFACEQ_MED_PKT_SIZE		768
     44#define IFACEQ_MIN_PKT_SIZE		8
     45#define IFACEQ_VAR_SMALL_PKT_SIZE	100
     46#define IFACEQ_VAR_LARGE_PKT_SIZE	512
     47#define IFACEQ_VAR_HUGE_PKT_SIZE	(1024 * 12)
     48
     49struct hfi_queue_table_header {
     50	u32 version;
     51	u32 size;
     52	u32 qhdr0_offset;
     53	u32 qhdr_size;
     54	u32 num_q;
     55	u32 num_active_q;
     56};
     57
     58struct hfi_queue_header {
     59	u32 status;
     60	u32 start_addr;
     61	u32 type;
     62	u32 q_size;
     63	u32 pkt_size;
     64	u32 pkt_drop_cnt;
     65	u32 rx_wm;
     66	u32 tx_wm;
     67	u32 rx_req;
     68	u32 tx_req;
     69	u32 rx_irq_status;
     70	u32 tx_irq_status;
     71	u32 read_idx;
     72	u32 write_idx;
     73};
     74
     75#define IFACEQ_TABLE_SIZE	\
     76	(sizeof(struct hfi_queue_table_header) +	\
     77	 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
     78
     79#define IFACEQ_QUEUE_SIZE	(IFACEQ_MAX_PKT_SIZE *	\
     80	IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
     81
     82#define IFACEQ_GET_QHDR_START_ADDR(ptr, i)	\
     83	(void *)(((ptr) + sizeof(struct hfi_queue_table_header)) +	\
     84		((i) * sizeof(struct hfi_queue_header)))
     85
     86#define QDSS_SIZE		SZ_4K
     87#define SFR_SIZE		SZ_4K
     88#define QUEUE_SIZE		\
     89	(IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
     90
     91#define ALIGNED_QDSS_SIZE	ALIGN(QDSS_SIZE, SZ_4K)
     92#define ALIGNED_SFR_SIZE	ALIGN(SFR_SIZE, SZ_4K)
     93#define ALIGNED_QUEUE_SIZE	ALIGN(QUEUE_SIZE, SZ_4K)
     94#define SHARED_QSIZE		ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
     95				      ALIGNED_QDSS_SIZE, SZ_1M)
     96
     97struct mem_desc {
     98	dma_addr_t da;	/* device address */
     99	void *kva;	/* kernel virtual address */
    100	u32 size;
    101	unsigned long attrs;
    102};
    103
    104struct iface_queue {
    105	struct hfi_queue_header *qhdr;
    106	struct mem_desc qmem;
    107};
    108
    109enum venus_state {
    110	VENUS_STATE_DEINIT = 1,
    111	VENUS_STATE_INIT,
    112};
    113
    114struct venus_hfi_device {
    115	struct venus_core *core;
    116	u32 irq_status;
    117	u32 last_packet_type;
    118	bool power_enabled;
    119	bool suspended;
    120	enum venus_state state;
    121	/* serialize read / write to the shared memory */
    122	struct mutex lock;
    123	struct completion pwr_collapse_prep;
    124	struct completion release_resource;
    125	struct mem_desc ifaceq_table;
    126	struct mem_desc sfr;
    127	struct iface_queue queues[IFACEQ_NUM];
    128	u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
    129	u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
    130};
    131
    132static bool venus_pkt_debug;
    133int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
    134static bool venus_sys_idle_indicator;
    135static bool venus_fw_low_power_mode = true;
    136static int venus_hw_rsp_timeout = 1000;
    137static bool venus_fw_coverage;
    138
    139static void venus_set_state(struct venus_hfi_device *hdev,
    140			    enum venus_state state)
    141{
    142	mutex_lock(&hdev->lock);
    143	hdev->state = state;
    144	mutex_unlock(&hdev->lock);
    145}
    146
    147static bool venus_is_valid_state(struct venus_hfi_device *hdev)
    148{
    149	return hdev->state != VENUS_STATE_DEINIT;
    150}
    151
    152static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
    153{
    154	size_t pkt_size = *(u32 *)packet;
    155
    156	if (!venus_pkt_debug)
    157		return;
    158
    159	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
    160		       pkt_size, true);
    161}
    162
    163static int venus_write_queue(struct venus_hfi_device *hdev,
    164			     struct iface_queue *queue,
    165			     void *packet, u32 *rx_req)
    166{
    167	struct hfi_queue_header *qhdr;
    168	u32 dwords, new_wr_idx;
    169	u32 empty_space, rd_idx, wr_idx, qsize;
    170	u32 *wr_ptr;
    171
    172	if (!queue->qmem.kva)
    173		return -EINVAL;
    174
    175	qhdr = queue->qhdr;
    176	if (!qhdr)
    177		return -EINVAL;
    178
    179	venus_dump_packet(hdev, packet);
    180
    181	dwords = (*(u32 *)packet) >> 2;
    182	if (!dwords)
    183		return -EINVAL;
    184
    185	rd_idx = qhdr->read_idx;
    186	wr_idx = qhdr->write_idx;
    187	qsize = qhdr->q_size;
    188	/* ensure rd/wr indices's are read from memory */
    189	rmb();
    190
    191	if (wr_idx >= rd_idx)
    192		empty_space = qsize - (wr_idx - rd_idx);
    193	else
    194		empty_space = rd_idx - wr_idx;
    195
    196	if (empty_space <= dwords) {
    197		qhdr->tx_req = 1;
    198		/* ensure tx_req is updated in memory */
    199		wmb();
    200		return -ENOSPC;
    201	}
    202
    203	qhdr->tx_req = 0;
    204	/* ensure tx_req is updated in memory */
    205	wmb();
    206
    207	new_wr_idx = wr_idx + dwords;
    208	wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
    209	if (new_wr_idx < qsize) {
    210		memcpy(wr_ptr, packet, dwords << 2);
    211	} else {
    212		size_t len;
    213
    214		new_wr_idx -= qsize;
    215		len = (dwords - new_wr_idx) << 2;
    216		memcpy(wr_ptr, packet, len);
    217		memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
    218	}
    219
    220	/* make sure packet is written before updating the write index */
    221	wmb();
    222
    223	qhdr->write_idx = new_wr_idx;
    224	*rx_req = qhdr->rx_req ? 1 : 0;
    225
    226	/* make sure write index is updated before an interrupt is raised */
    227	mb();
    228
    229	return 0;
    230}
    231
    232static int venus_read_queue(struct venus_hfi_device *hdev,
    233			    struct iface_queue *queue, void *pkt, u32 *tx_req)
    234{
    235	struct hfi_queue_header *qhdr;
    236	u32 dwords, new_rd_idx;
    237	u32 rd_idx, wr_idx, type, qsize;
    238	u32 *rd_ptr;
    239	u32 recv_request = 0;
    240	int ret = 0;
    241
    242	if (!queue->qmem.kva)
    243		return -EINVAL;
    244
    245	qhdr = queue->qhdr;
    246	if (!qhdr)
    247		return -EINVAL;
    248
    249	type = qhdr->type;
    250	rd_idx = qhdr->read_idx;
    251	wr_idx = qhdr->write_idx;
    252	qsize = qhdr->q_size;
    253
    254	/* make sure data is valid before using it */
    255	rmb();
    256
    257	/*
    258	 * Do not set receive request for debug queue, if set, Venus generates
    259	 * interrupt for debug messages even when there is no response message
    260	 * available. In general debug queue will not become full as it is being
    261	 * emptied out for every interrupt from Venus. Venus will anyway
    262	 * generates interrupt if it is full.
    263	 */
    264	if (type & HFI_CTRL_TO_HOST_MSG_Q)
    265		recv_request = 1;
    266
    267	if (rd_idx == wr_idx) {
    268		qhdr->rx_req = recv_request;
    269		*tx_req = 0;
    270		/* update rx_req field in memory */
    271		wmb();
    272		return -ENODATA;
    273	}
    274
    275	rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
    276	dwords = *rd_ptr >> 2;
    277	if (!dwords)
    278		return -EINVAL;
    279
    280	new_rd_idx = rd_idx + dwords;
    281	if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
    282		if (new_rd_idx < qsize) {
    283			memcpy(pkt, rd_ptr, dwords << 2);
    284		} else {
    285			size_t len;
    286
    287			new_rd_idx -= qsize;
    288			len = (dwords - new_rd_idx) << 2;
    289			memcpy(pkt, rd_ptr, len);
    290			memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
    291		}
    292	} else {
    293		/* bad packet received, dropping */
    294		new_rd_idx = qhdr->write_idx;
    295		ret = -EBADMSG;
    296	}
    297
    298	/* ensure the packet is read before updating read index */
    299	rmb();
    300
    301	qhdr->read_idx = new_rd_idx;
    302	/* ensure updating read index */
    303	wmb();
    304
    305	rd_idx = qhdr->read_idx;
    306	wr_idx = qhdr->write_idx;
    307	/* ensure rd/wr indices are read from memory */
    308	rmb();
    309
    310	if (rd_idx != wr_idx)
    311		qhdr->rx_req = 0;
    312	else
    313		qhdr->rx_req = recv_request;
    314
    315	*tx_req = qhdr->tx_req ? 1 : 0;
    316
    317	/* ensure rx_req is stored to memory and tx_req is loaded from memory */
    318	mb();
    319
    320	venus_dump_packet(hdev, pkt);
    321
    322	return ret;
    323}
    324
    325static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
    326		       u32 size)
    327{
    328	struct device *dev = hdev->core->dev;
    329
    330	desc->attrs = DMA_ATTR_WRITE_COMBINE;
    331	desc->size = ALIGN(size, SZ_4K);
    332
    333	desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
    334				    desc->attrs);
    335	if (!desc->kva)
    336		return -ENOMEM;
    337
    338	return 0;
    339}
    340
    341static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
    342{
    343	struct device *dev = hdev->core->dev;
    344
    345	dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
    346}
    347
    348static void venus_set_registers(struct venus_hfi_device *hdev)
    349{
    350	const struct venus_resources *res = hdev->core->res;
    351	const struct reg_val *tbl = res->reg_tbl;
    352	unsigned int count = res->reg_tbl_size;
    353	unsigned int i;
    354
    355	for (i = 0; i < count; i++)
    356		writel(tbl[i].value, hdev->core->base + tbl[i].reg);
    357}
    358
    359static void venus_soft_int(struct venus_hfi_device *hdev)
    360{
    361	void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
    362	u32 clear_bit;
    363
    364	if (IS_V6(hdev->core))
    365		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
    366	else
    367		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
    368
    369	writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
    370}
    371
    372static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
    373					 void *pkt, bool sync)
    374{
    375	struct device *dev = hdev->core->dev;
    376	struct hfi_pkt_hdr *cmd_packet;
    377	struct iface_queue *queue;
    378	u32 rx_req;
    379	int ret;
    380
    381	if (!venus_is_valid_state(hdev))
    382		return -EINVAL;
    383
    384	cmd_packet = (struct hfi_pkt_hdr *)pkt;
    385	hdev->last_packet_type = cmd_packet->pkt_type;
    386
    387	queue = &hdev->queues[IFACEQ_CMD_IDX];
    388
    389	ret = venus_write_queue(hdev, queue, pkt, &rx_req);
    390	if (ret) {
    391		dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
    392		return ret;
    393	}
    394
    395	if (sync) {
    396		/*
    397		 * Inform video hardware to raise interrupt for synchronous
    398		 * commands
    399		 */
    400		queue = &hdev->queues[IFACEQ_MSG_IDX];
    401		queue->qhdr->rx_req = 1;
    402		/* ensure rx_req is updated in memory */
    403		wmb();
    404	}
    405
    406	if (rx_req)
    407		venus_soft_int(hdev);
    408
    409	return 0;
    410}
    411
    412static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
    413{
    414	int ret;
    415
    416	mutex_lock(&hdev->lock);
    417	ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
    418	mutex_unlock(&hdev->lock);
    419
    420	return ret;
    421}
    422
    423static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
    424				       u32 size, u32 addr, void *cookie)
    425{
    426	struct venus_hfi_device *hdev = to_hfi_priv(core);
    427	struct hfi_sys_set_resource_pkt *pkt;
    428	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
    429	int ret;
    430
    431	if (id == VIDC_RESOURCE_NONE)
    432		return 0;
    433
    434	pkt = (struct hfi_sys_set_resource_pkt *)packet;
    435
    436	ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
    437	if (ret)
    438		return ret;
    439
    440	ret = venus_iface_cmdq_write(hdev, pkt, false);
    441	if (ret)
    442		return ret;
    443
    444	return 0;
    445}
    446
    447static int venus_boot_core(struct venus_hfi_device *hdev)
    448{
    449	struct device *dev = hdev->core->dev;
    450	static const unsigned int max_tries = 100;
    451	u32 ctrl_status = 0, mask_val;
    452	unsigned int count = 0;
    453	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
    454	void __iomem *wrapper_base = hdev->core->wrapper_base;
    455	int ret = 0;
    456
    457	writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
    458	if (IS_V6(hdev->core)) {
    459		mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
    460		mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
    461			      WRAPPER_INTR_MASK_A2HCPU_MASK);
    462	} else {
    463		mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
    464	}
    465	writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
    466	writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
    467
    468	while (!ctrl_status && count < max_tries) {
    469		ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
    470		if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
    471			dev_err(dev, "invalid setting for UC_REGION\n");
    472			ret = -EINVAL;
    473			break;
    474		}
    475
    476		usleep_range(500, 1000);
    477		count++;
    478	}
    479
    480	if (count >= max_tries)
    481		ret = -ETIMEDOUT;
    482
    483	if (IS_V6(hdev->core)) {
    484		writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
    485		writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
    486	}
    487
    488	return ret;
    489}
    490
    491static u32 venus_hwversion(struct venus_hfi_device *hdev)
    492{
    493	struct device *dev = hdev->core->dev;
    494	void __iomem *wrapper_base = hdev->core->wrapper_base;
    495	u32 ver;
    496	u32 major, minor, step;
    497
    498	ver = readl(wrapper_base + WRAPPER_HW_VERSION);
    499	major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
    500	major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
    501	minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
    502	minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
    503	step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
    504
    505	dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
    506
    507	return major;
    508}
    509
    510static int venus_run(struct venus_hfi_device *hdev)
    511{
    512	struct device *dev = hdev->core->dev;
    513	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
    514	int ret;
    515
    516	/*
    517	 * Re-program all of the registers that get reset as a result of
    518	 * regulator_disable() and _enable()
    519	 */
    520	venus_set_registers(hdev);
    521
    522	writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
    523	writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
    524	writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
    525	writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
    526	if (hdev->sfr.da)
    527		writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
    528
    529	ret = venus_boot_core(hdev);
    530	if (ret) {
    531		dev_err(dev, "failed to reset venus core\n");
    532		return ret;
    533	}
    534
    535	venus_hwversion(hdev);
    536
    537	return 0;
    538}
    539
    540static int venus_halt_axi(struct venus_hfi_device *hdev)
    541{
    542	void __iomem *wrapper_base = hdev->core->wrapper_base;
    543	void __iomem *vbif_base = hdev->core->vbif_base;
    544	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
    545	void __iomem *aon_base = hdev->core->aon_base;
    546	struct device *dev = hdev->core->dev;
    547	u32 val;
    548	u32 mask_val;
    549	int ret;
    550
    551	if (IS_V6(hdev->core)) {
    552		writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
    553
    554		if (hdev->core->res->num_vpp_pipes == 1)
    555			goto skip_aon_mvp_noc;
    556
    557		writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
    558		ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
    559					 val,
    560					 val & BIT(0),
    561					 POLL_INTERVAL_US,
    562					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
    563		if (ret)
    564			return -ETIMEDOUT;
    565
    566skip_aon_mvp_noc:
    567		mask_val = (BIT(2) | BIT(1) | BIT(0));
    568		writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
    569
    570		writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
    571		ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
    572					 val,
    573					 val == 0,
    574					 POLL_INTERVAL_US,
    575					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
    576
    577		if (ret) {
    578			dev_err(dev, "DBLP Release: lpi_status %x\n", val);
    579			return -ETIMEDOUT;
    580		}
    581		return 0;
    582	}
    583
    584	if (IS_V4(hdev->core)) {
    585		val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
    586		val |= WRAPPER_CPU_AXI_HALT_HALT;
    587		writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
    588
    589		ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
    590					 val,
    591					 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
    592					 POLL_INTERVAL_US,
    593					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
    594		if (ret) {
    595			dev_err(dev, "AXI bus port halt timeout\n");
    596			return ret;
    597		}
    598
    599		return 0;
    600	}
    601
    602	/* Halt AXI and AXI IMEM VBIF Access */
    603	val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
    604	val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
    605	writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
    606
    607	/* Request for AXI bus port halt */
    608	ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
    609				 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
    610				 POLL_INTERVAL_US,
    611				 VBIF_AXI_HALT_ACK_TIMEOUT_US);
    612	if (ret) {
    613		dev_err(dev, "AXI bus port halt timeout\n");
    614		return ret;
    615	}
    616
    617	return 0;
    618}
    619
    620static int venus_power_off(struct venus_hfi_device *hdev)
    621{
    622	int ret;
    623
    624	if (!hdev->power_enabled)
    625		return 0;
    626
    627	ret = venus_set_hw_state_suspend(hdev->core);
    628	if (ret)
    629		return ret;
    630
    631	ret = venus_halt_axi(hdev);
    632	if (ret)
    633		return ret;
    634
    635	hdev->power_enabled = false;
    636
    637	return 0;
    638}
    639
    640static int venus_power_on(struct venus_hfi_device *hdev)
    641{
    642	int ret;
    643
    644	if (hdev->power_enabled)
    645		return 0;
    646
    647	ret = venus_set_hw_state_resume(hdev->core);
    648	if (ret)
    649		goto err;
    650
    651	ret = venus_run(hdev);
    652	if (ret)
    653		goto err_suspend;
    654
    655	hdev->power_enabled = true;
    656
    657	return 0;
    658
    659err_suspend:
    660	venus_set_hw_state_suspend(hdev->core);
    661err:
    662	hdev->power_enabled = false;
    663	return ret;
    664}
    665
    666static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
    667					void *pkt)
    668{
    669	struct iface_queue *queue;
    670	u32 tx_req;
    671	int ret;
    672
    673	if (!venus_is_valid_state(hdev))
    674		return -EINVAL;
    675
    676	queue = &hdev->queues[IFACEQ_MSG_IDX];
    677
    678	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
    679	if (ret)
    680		return ret;
    681
    682	if (tx_req)
    683		venus_soft_int(hdev);
    684
    685	return 0;
    686}
    687
    688static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
    689{
    690	int ret;
    691
    692	mutex_lock(&hdev->lock);
    693	ret = venus_iface_msgq_read_nolock(hdev, pkt);
    694	mutex_unlock(&hdev->lock);
    695
    696	return ret;
    697}
    698
    699static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
    700					void *pkt)
    701{
    702	struct iface_queue *queue;
    703	u32 tx_req;
    704	int ret;
    705
    706	ret = venus_is_valid_state(hdev);
    707	if (!ret)
    708		return -EINVAL;
    709
    710	queue = &hdev->queues[IFACEQ_DBG_IDX];
    711
    712	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
    713	if (ret)
    714		return ret;
    715
    716	if (tx_req)
    717		venus_soft_int(hdev);
    718
    719	return 0;
    720}
    721
    722static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
    723{
    724	int ret;
    725
    726	if (!pkt)
    727		return -EINVAL;
    728
    729	mutex_lock(&hdev->lock);
    730	ret = venus_iface_dbgq_read_nolock(hdev, pkt);
    731	mutex_unlock(&hdev->lock);
    732
    733	return ret;
    734}
    735
    736static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
    737{
    738	qhdr->status = 1;
    739	qhdr->type = IFACEQ_DFLT_QHDR;
    740	qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
    741	qhdr->pkt_size = 0;
    742	qhdr->rx_wm = 1;
    743	qhdr->tx_wm = 1;
    744	qhdr->rx_req = 1;
    745	qhdr->tx_req = 0;
    746	qhdr->rx_irq_status = 0;
    747	qhdr->tx_irq_status = 0;
    748	qhdr->read_idx = 0;
    749	qhdr->write_idx = 0;
    750}
    751
    752static void venus_interface_queues_release(struct venus_hfi_device *hdev)
    753{
    754	mutex_lock(&hdev->lock);
    755
    756	venus_free(hdev, &hdev->ifaceq_table);
    757	venus_free(hdev, &hdev->sfr);
    758
    759	memset(hdev->queues, 0, sizeof(hdev->queues));
    760	memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
    761	memset(&hdev->sfr, 0, sizeof(hdev->sfr));
    762
    763	mutex_unlock(&hdev->lock);
    764}
    765
    766static int venus_interface_queues_init(struct venus_hfi_device *hdev)
    767{
    768	struct hfi_queue_table_header *tbl_hdr;
    769	struct iface_queue *queue;
    770	struct hfi_sfr *sfr;
    771	struct mem_desc desc = {0};
    772	unsigned int offset;
    773	unsigned int i;
    774	int ret;
    775
    776	ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
    777	if (ret)
    778		return ret;
    779
    780	hdev->ifaceq_table = desc;
    781	offset = IFACEQ_TABLE_SIZE;
    782
    783	for (i = 0; i < IFACEQ_NUM; i++) {
    784		queue = &hdev->queues[i];
    785		queue->qmem.da = desc.da + offset;
    786		queue->qmem.kva = desc.kva + offset;
    787		queue->qmem.size = IFACEQ_QUEUE_SIZE;
    788		offset += queue->qmem.size;
    789		queue->qhdr =
    790			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
    791
    792		venus_set_qhdr_defaults(queue->qhdr);
    793
    794		queue->qhdr->start_addr = queue->qmem.da;
    795
    796		if (i == IFACEQ_CMD_IDX)
    797			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
    798		else if (i == IFACEQ_MSG_IDX)
    799			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
    800		else if (i == IFACEQ_DBG_IDX)
    801			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
    802	}
    803
    804	tbl_hdr = hdev->ifaceq_table.kva;
    805	tbl_hdr->version = 0;
    806	tbl_hdr->size = IFACEQ_TABLE_SIZE;
    807	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
    808	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
    809	tbl_hdr->num_q = IFACEQ_NUM;
    810	tbl_hdr->num_active_q = IFACEQ_NUM;
    811
    812	/*
    813	 * Set receive request to zero on debug queue as there is no
    814	 * need of interrupt from video hardware for debug messages
    815	 */
    816	queue = &hdev->queues[IFACEQ_DBG_IDX];
    817	queue->qhdr->rx_req = 0;
    818
    819	ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
    820	if (ret) {
    821		hdev->sfr.da = 0;
    822	} else {
    823		hdev->sfr = desc;
    824		sfr = hdev->sfr.kva;
    825		sfr->buf_size = ALIGNED_SFR_SIZE;
    826	}
    827
    828	/* ensure table and queue header structs are settled in memory */
    829	wmb();
    830
    831	return 0;
    832}
    833
    834static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
    835{
    836	struct hfi_sys_set_property_pkt *pkt;
    837	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
    838	int ret;
    839
    840	pkt = (struct hfi_sys_set_property_pkt *)packet;
    841
    842	pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
    843
    844	ret = venus_iface_cmdq_write(hdev, pkt, false);
    845	if (ret)
    846		return ret;
    847
    848	return 0;
    849}
    850
    851static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
    852{
    853	struct hfi_sys_set_property_pkt *pkt;
    854	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
    855	int ret;
    856
    857	pkt = (struct hfi_sys_set_property_pkt *)packet;
    858
    859	pkt_sys_coverage_config(pkt, mode);
    860
    861	ret = venus_iface_cmdq_write(hdev, pkt, false);
    862	if (ret)
    863		return ret;
    864
    865	return 0;
    866}
    867
    868static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
    869				      bool enable)
    870{
    871	struct hfi_sys_set_property_pkt *pkt;
    872	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
    873	int ret;
    874
    875	if (!enable)
    876		return 0;
    877
    878	pkt = (struct hfi_sys_set_property_pkt *)packet;
    879
    880	pkt_sys_idle_indicator(pkt, enable);
    881
    882	ret = venus_iface_cmdq_write(hdev, pkt, false);
    883	if (ret)
    884		return ret;
    885
    886	return 0;
    887}
    888
    889static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
    890				       bool enable)
    891{
    892	struct hfi_sys_set_property_pkt *pkt;
    893	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
    894	int ret;
    895
    896	pkt = (struct hfi_sys_set_property_pkt *)packet;
    897
    898	pkt_sys_power_control(pkt, enable);
    899
    900	ret = venus_iface_cmdq_write(hdev, pkt, false);
    901	if (ret)
    902		return ret;
    903
    904	return 0;
    905}
    906
    907static int venus_get_queue_size(struct venus_hfi_device *hdev,
    908				unsigned int index)
    909{
    910	struct hfi_queue_header *qhdr;
    911
    912	if (index >= IFACEQ_NUM)
    913		return -EINVAL;
    914
    915	qhdr = hdev->queues[index].qhdr;
    916	if (!qhdr)
    917		return -EINVAL;
    918
    919	return abs(qhdr->read_idx - qhdr->write_idx);
    920}
    921
    922static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
    923{
    924	struct device *dev = hdev->core->dev;
    925	int ret;
    926
    927	ret = venus_sys_set_debug(hdev, venus_fw_debug);
    928	if (ret)
    929		dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
    930
    931	/*
    932	 * Idle indicator is disabled by default on some 4xx firmware versions,
    933	 * enable it explicitly in order to make suspend functional by checking
    934	 * WFI (wait-for-interrupt) bit.
    935	 */
    936	if (IS_V4(hdev->core) || IS_V6(hdev->core))
    937		venus_sys_idle_indicator = true;
    938
    939	ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator);
    940	if (ret)
    941		dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
    942
    943	ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
    944	if (ret)
    945		dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
    946			 ret);
    947
    948	return ret;
    949}
    950
    951static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
    952{
    953	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
    954	struct hfi_session_pkt pkt;
    955
    956	pkt_session_cmd(&pkt, pkt_type, inst);
    957
    958	return venus_iface_cmdq_write(hdev, &pkt, sync);
    959}
    960
    961static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
    962{
    963	struct device *dev = hdev->core->dev;
    964	void *packet = hdev->dbg_buf;
    965
    966	while (!venus_iface_dbgq_read(hdev, packet)) {
    967		struct hfi_msg_sys_coverage_pkt *pkt = packet;
    968
    969		if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
    970			struct hfi_msg_sys_debug_pkt *pkt = packet;
    971
    972			dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
    973		}
    974	}
    975}
    976
    977static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
    978					bool wait)
    979{
    980	unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
    981	struct hfi_sys_pc_prep_pkt pkt;
    982	int ret;
    983
    984	init_completion(&hdev->pwr_collapse_prep);
    985
    986	pkt_sys_pc_prep(&pkt);
    987
    988	ret = venus_iface_cmdq_write(hdev, &pkt, false);
    989	if (ret)
    990		return ret;
    991
    992	if (!wait)
    993		return 0;
    994
    995	ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
    996	if (!ret) {
    997		venus_flush_debug_queue(hdev);
    998		return -ETIMEDOUT;
    999	}
   1000
   1001	return 0;
   1002}
   1003
   1004static int venus_are_queues_empty(struct venus_hfi_device *hdev)
   1005{
   1006	int ret1, ret2;
   1007
   1008	ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
   1009	if (ret1 < 0)
   1010		return ret1;
   1011
   1012	ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
   1013	if (ret2 < 0)
   1014		return ret2;
   1015
   1016	if (!ret1 && !ret2)
   1017		return 1;
   1018
   1019	return 0;
   1020}
   1021
   1022static void venus_sfr_print(struct venus_hfi_device *hdev)
   1023{
   1024	struct device *dev = hdev->core->dev;
   1025	struct hfi_sfr *sfr = hdev->sfr.kva;
   1026	void *p;
   1027
   1028	if (!sfr)
   1029		return;
   1030
   1031	p = memchr(sfr->data, '\0', sfr->buf_size);
   1032	/*
   1033	 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
   1034	 * that Venus is in the process of crashing.
   1035	 */
   1036	if (!p)
   1037		sfr->data[sfr->buf_size - 1] = '\0';
   1038
   1039	dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
   1040}
   1041
   1042static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
   1043					void *packet)
   1044{
   1045	struct hfi_msg_event_notify_pkt *event_pkt = packet;
   1046
   1047	if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
   1048		return;
   1049
   1050	venus_set_state(hdev, VENUS_STATE_DEINIT);
   1051
   1052	venus_sfr_print(hdev);
   1053}
   1054
   1055static irqreturn_t venus_isr_thread(struct venus_core *core)
   1056{
   1057	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1058	const struct venus_resources *res;
   1059	void *pkt;
   1060	u32 msg_ret;
   1061
   1062	if (!hdev)
   1063		return IRQ_NONE;
   1064
   1065	res = hdev->core->res;
   1066	pkt = hdev->pkt_buf;
   1067
   1068
   1069	while (!venus_iface_msgq_read(hdev, pkt)) {
   1070		msg_ret = hfi_process_msg_packet(core, pkt);
   1071		switch (msg_ret) {
   1072		case HFI_MSG_EVENT_NOTIFY:
   1073			venus_process_msg_sys_error(hdev, pkt);
   1074			break;
   1075		case HFI_MSG_SYS_INIT:
   1076			venus_hfi_core_set_resource(core, res->vmem_id,
   1077						    res->vmem_size,
   1078						    res->vmem_addr,
   1079						    hdev);
   1080			break;
   1081		case HFI_MSG_SYS_RELEASE_RESOURCE:
   1082			complete(&hdev->release_resource);
   1083			break;
   1084		case HFI_MSG_SYS_PC_PREP:
   1085			complete(&hdev->pwr_collapse_prep);
   1086			break;
   1087		default:
   1088			break;
   1089		}
   1090	}
   1091
   1092	venus_flush_debug_queue(hdev);
   1093
   1094	return IRQ_HANDLED;
   1095}
   1096
   1097static irqreturn_t venus_isr(struct venus_core *core)
   1098{
   1099	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1100	u32 status;
   1101	void __iomem *cpu_cs_base;
   1102	void __iomem *wrapper_base;
   1103
   1104	if (!hdev)
   1105		return IRQ_NONE;
   1106
   1107	cpu_cs_base = hdev->core->cpu_cs_base;
   1108	wrapper_base = hdev->core->wrapper_base;
   1109
   1110	status = readl(wrapper_base + WRAPPER_INTR_STATUS);
   1111	if (IS_V6(core)) {
   1112		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
   1113		    status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
   1114		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
   1115			hdev->irq_status = status;
   1116	} else {
   1117		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
   1118		    status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
   1119		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
   1120			hdev->irq_status = status;
   1121	}
   1122	writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
   1123	if (!IS_V6(core))
   1124		writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
   1125
   1126	return IRQ_WAKE_THREAD;
   1127}
   1128
   1129static int venus_core_init(struct venus_core *core)
   1130{
   1131	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1132	struct device *dev = core->dev;
   1133	struct hfi_sys_get_property_pkt version_pkt;
   1134	struct hfi_sys_init_pkt pkt;
   1135	int ret;
   1136
   1137	pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
   1138
   1139	venus_set_state(hdev, VENUS_STATE_INIT);
   1140
   1141	ret = venus_iface_cmdq_write(hdev, &pkt, false);
   1142	if (ret)
   1143		return ret;
   1144
   1145	pkt_sys_image_version(&version_pkt);
   1146
   1147	ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
   1148	if (ret)
   1149		dev_warn(dev, "failed to send image version pkt to fw\n");
   1150
   1151	ret = venus_sys_set_default_properties(hdev);
   1152	if (ret)
   1153		return ret;
   1154
   1155	return 0;
   1156}
   1157
   1158static int venus_core_deinit(struct venus_core *core)
   1159{
   1160	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1161
   1162	venus_set_state(hdev, VENUS_STATE_DEINIT);
   1163	hdev->suspended = true;
   1164	hdev->power_enabled = false;
   1165
   1166	return 0;
   1167}
   1168
   1169static int venus_core_ping(struct venus_core *core, u32 cookie)
   1170{
   1171	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1172	struct hfi_sys_ping_pkt pkt;
   1173
   1174	pkt_sys_ping(&pkt, cookie);
   1175
   1176	return venus_iface_cmdq_write(hdev, &pkt, false);
   1177}
   1178
   1179static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
   1180{
   1181	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1182	struct hfi_sys_test_ssr_pkt pkt;
   1183	int ret;
   1184
   1185	ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
   1186	if (ret)
   1187		return ret;
   1188
   1189	return venus_iface_cmdq_write(hdev, &pkt, false);
   1190}
   1191
   1192static int venus_session_init(struct venus_inst *inst, u32 session_type,
   1193			      u32 codec)
   1194{
   1195	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1196	struct hfi_session_init_pkt pkt;
   1197	int ret;
   1198
   1199	ret = venus_sys_set_debug(hdev, venus_fw_debug);
   1200	if (ret)
   1201		goto err;
   1202
   1203	ret = pkt_session_init(&pkt, inst, session_type, codec);
   1204	if (ret)
   1205		goto err;
   1206
   1207	ret = venus_iface_cmdq_write(hdev, &pkt, true);
   1208	if (ret)
   1209		goto err;
   1210
   1211	return 0;
   1212
   1213err:
   1214	venus_flush_debug_queue(hdev);
   1215	return ret;
   1216}
   1217
   1218static int venus_session_end(struct venus_inst *inst)
   1219{
   1220	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1221	struct device *dev = hdev->core->dev;
   1222
   1223	if (venus_fw_coverage) {
   1224		if (venus_sys_set_coverage(hdev, venus_fw_coverage))
   1225			dev_warn(dev, "fw coverage msg ON failed\n");
   1226	}
   1227
   1228	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
   1229}
   1230
   1231static int venus_session_abort(struct venus_inst *inst)
   1232{
   1233	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1234
   1235	venus_flush_debug_queue(hdev);
   1236
   1237	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
   1238}
   1239
   1240static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
   1241{
   1242	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1243	struct hfi_session_flush_pkt pkt;
   1244	int ret;
   1245
   1246	ret = pkt_session_flush(&pkt, inst, flush_mode);
   1247	if (ret)
   1248		return ret;
   1249
   1250	return venus_iface_cmdq_write(hdev, &pkt, true);
   1251}
   1252
   1253static int venus_session_start(struct venus_inst *inst)
   1254{
   1255	return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
   1256}
   1257
   1258static int venus_session_stop(struct venus_inst *inst)
   1259{
   1260	return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
   1261}
   1262
   1263static int venus_session_continue(struct venus_inst *inst)
   1264{
   1265	return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
   1266}
   1267
   1268static int venus_session_etb(struct venus_inst *inst,
   1269			     struct hfi_frame_data *in_frame)
   1270{
   1271	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1272	u32 session_type = inst->session_type;
   1273	int ret;
   1274
   1275	if (session_type == VIDC_SESSION_TYPE_DEC) {
   1276		struct hfi_session_empty_buffer_compressed_pkt pkt;
   1277
   1278		ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
   1279		if (ret)
   1280			return ret;
   1281
   1282		ret = venus_iface_cmdq_write(hdev, &pkt, false);
   1283	} else if (session_type == VIDC_SESSION_TYPE_ENC) {
   1284		struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
   1285
   1286		ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
   1287		if (ret)
   1288			return ret;
   1289
   1290		ret = venus_iface_cmdq_write(hdev, &pkt, false);
   1291	} else {
   1292		ret = -EINVAL;
   1293	}
   1294
   1295	return ret;
   1296}
   1297
   1298static int venus_session_ftb(struct venus_inst *inst,
   1299			     struct hfi_frame_data *out_frame)
   1300{
   1301	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1302	struct hfi_session_fill_buffer_pkt pkt;
   1303	int ret;
   1304
   1305	ret = pkt_session_ftb(&pkt, inst, out_frame);
   1306	if (ret)
   1307		return ret;
   1308
   1309	return venus_iface_cmdq_write(hdev, &pkt, false);
   1310}
   1311
   1312static int venus_session_set_buffers(struct venus_inst *inst,
   1313				     struct hfi_buffer_desc *bd)
   1314{
   1315	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1316	struct hfi_session_set_buffers_pkt *pkt;
   1317	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
   1318	int ret;
   1319
   1320	if (bd->buffer_type == HFI_BUFFER_INPUT)
   1321		return 0;
   1322
   1323	pkt = (struct hfi_session_set_buffers_pkt *)packet;
   1324
   1325	ret = pkt_session_set_buffers(pkt, inst, bd);
   1326	if (ret)
   1327		return ret;
   1328
   1329	return venus_iface_cmdq_write(hdev, pkt, false);
   1330}
   1331
   1332static int venus_session_unset_buffers(struct venus_inst *inst,
   1333				       struct hfi_buffer_desc *bd)
   1334{
   1335	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1336	struct hfi_session_release_buffer_pkt *pkt;
   1337	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
   1338	int ret;
   1339
   1340	if (bd->buffer_type == HFI_BUFFER_INPUT)
   1341		return 0;
   1342
   1343	pkt = (struct hfi_session_release_buffer_pkt *)packet;
   1344
   1345	ret = pkt_session_unset_buffers(pkt, inst, bd);
   1346	if (ret)
   1347		return ret;
   1348
   1349	return venus_iface_cmdq_write(hdev, pkt, true);
   1350}
   1351
   1352static int venus_session_load_res(struct venus_inst *inst)
   1353{
   1354	return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
   1355}
   1356
   1357static int venus_session_release_res(struct venus_inst *inst)
   1358{
   1359	return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
   1360}
   1361
   1362static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
   1363				       u32 seq_hdr_len)
   1364{
   1365	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1366	struct hfi_session_parse_sequence_header_pkt *pkt;
   1367	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
   1368	int ret;
   1369
   1370	pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
   1371
   1372	ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
   1373	if (ret)
   1374		return ret;
   1375
   1376	ret = venus_iface_cmdq_write(hdev, pkt, false);
   1377	if (ret)
   1378		return ret;
   1379
   1380	return 0;
   1381}
   1382
   1383static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
   1384				     u32 seq_hdr_len)
   1385{
   1386	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1387	struct hfi_session_get_sequence_header_pkt *pkt;
   1388	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
   1389	int ret;
   1390
   1391	pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
   1392
   1393	ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
   1394	if (ret)
   1395		return ret;
   1396
   1397	return venus_iface_cmdq_write(hdev, pkt, false);
   1398}
   1399
   1400static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
   1401				      void *pdata)
   1402{
   1403	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1404	struct hfi_session_set_property_pkt *pkt;
   1405	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
   1406	int ret;
   1407
   1408	pkt = (struct hfi_session_set_property_pkt *)packet;
   1409
   1410	ret = pkt_session_set_property(pkt, inst, ptype, pdata);
   1411	if (ret == -ENOTSUPP)
   1412		return 0;
   1413	if (ret)
   1414		return ret;
   1415
   1416	return venus_iface_cmdq_write(hdev, pkt, false);
   1417}
   1418
   1419static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
   1420{
   1421	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
   1422	struct hfi_session_get_property_pkt pkt;
   1423	int ret;
   1424
   1425	ret = pkt_session_get_property(&pkt, inst, ptype);
   1426	if (ret)
   1427		return ret;
   1428
   1429	return venus_iface_cmdq_write(hdev, &pkt, true);
   1430}
   1431
   1432static int venus_resume(struct venus_core *core)
   1433{
   1434	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1435	int ret = 0;
   1436
   1437	mutex_lock(&hdev->lock);
   1438
   1439	if (!hdev->suspended)
   1440		goto unlock;
   1441
   1442	ret = venus_power_on(hdev);
   1443
   1444unlock:
   1445	if (!ret)
   1446		hdev->suspended = false;
   1447
   1448	mutex_unlock(&hdev->lock);
   1449
   1450	return ret;
   1451}
   1452
   1453static int venus_suspend_1xx(struct venus_core *core)
   1454{
   1455	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1456	struct device *dev = core->dev;
   1457	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
   1458	u32 ctrl_status;
   1459	int ret;
   1460
   1461	if (!hdev->power_enabled || hdev->suspended)
   1462		return 0;
   1463
   1464	mutex_lock(&hdev->lock);
   1465	ret = venus_is_valid_state(hdev);
   1466	mutex_unlock(&hdev->lock);
   1467
   1468	if (!ret) {
   1469		dev_err(dev, "bad state, cannot suspend\n");
   1470		return -EINVAL;
   1471	}
   1472
   1473	ret = venus_prepare_power_collapse(hdev, true);
   1474	if (ret) {
   1475		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
   1476		return ret;
   1477	}
   1478
   1479	mutex_lock(&hdev->lock);
   1480
   1481	if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
   1482		mutex_unlock(&hdev->lock);
   1483		return -EINVAL;
   1484	}
   1485
   1486	ret = venus_are_queues_empty(hdev);
   1487	if (ret < 0 || !ret) {
   1488		mutex_unlock(&hdev->lock);
   1489		return -EINVAL;
   1490	}
   1491
   1492	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
   1493	if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
   1494		mutex_unlock(&hdev->lock);
   1495		return -EINVAL;
   1496	}
   1497
   1498	ret = venus_power_off(hdev);
   1499	if (ret) {
   1500		mutex_unlock(&hdev->lock);
   1501		return ret;
   1502	}
   1503
   1504	hdev->suspended = true;
   1505
   1506	mutex_unlock(&hdev->lock);
   1507
   1508	return 0;
   1509}
   1510
   1511static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
   1512{
   1513	void __iomem *wrapper_base = hdev->core->wrapper_base;
   1514	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
   1515	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
   1516	u32 ctrl_status, cpu_status;
   1517
   1518	if (IS_V6(hdev->core))
   1519		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
   1520	else
   1521		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
   1522	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
   1523
   1524	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
   1525	    ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
   1526		return true;
   1527
   1528	return false;
   1529}
   1530
   1531static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
   1532{
   1533	void __iomem *wrapper_base = hdev->core->wrapper_base;
   1534	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
   1535	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
   1536	u32 ctrl_status, cpu_status;
   1537
   1538	if (IS_V6(hdev->core))
   1539		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
   1540	else
   1541		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
   1542	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
   1543
   1544	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
   1545	    ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
   1546		return true;
   1547
   1548	return false;
   1549}
   1550
   1551static int venus_suspend_3xx(struct venus_core *core)
   1552{
   1553	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1554	struct device *dev = core->dev;
   1555	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
   1556	u32 ctrl_status;
   1557	bool val;
   1558	int ret;
   1559
   1560	if (!hdev->power_enabled || hdev->suspended)
   1561		return 0;
   1562
   1563	mutex_lock(&hdev->lock);
   1564	ret = venus_is_valid_state(hdev);
   1565	mutex_unlock(&hdev->lock);
   1566
   1567	if (!ret) {
   1568		dev_err(dev, "bad state, cannot suspend\n");
   1569		return -EINVAL;
   1570	}
   1571
   1572	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
   1573	if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
   1574		goto power_off;
   1575
   1576	/*
   1577	 * Power collapse sequence for Venus 3xx and 4xx versions:
   1578	 * 1. Check for ARM9 and video core to be idle by checking WFI bit
   1579	 *    (bit 0) in CPU status register and by checking Idle (bit 30) in
   1580	 *    Control status register for video core.
   1581	 * 2. Send a command to prepare for power collapse.
   1582	 * 3. Check for WFI and PC_READY bits.
   1583	 */
   1584	ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
   1585				 1500, 100 * 1500);
   1586	if (ret) {
   1587		dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
   1588		return ret;
   1589	}
   1590
   1591	ret = venus_prepare_power_collapse(hdev, false);
   1592	if (ret) {
   1593		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
   1594		return ret;
   1595	}
   1596
   1597	ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
   1598				 1500, 100 * 1500);
   1599	if (ret)
   1600		return ret;
   1601
   1602power_off:
   1603	mutex_lock(&hdev->lock);
   1604
   1605	ret = venus_power_off(hdev);
   1606	if (ret) {
   1607		dev_err(dev, "venus_power_off (%d)\n", ret);
   1608		mutex_unlock(&hdev->lock);
   1609		return ret;
   1610	}
   1611
   1612	hdev->suspended = true;
   1613
   1614	mutex_unlock(&hdev->lock);
   1615
   1616	return 0;
   1617}
   1618
   1619static int venus_suspend(struct venus_core *core)
   1620{
   1621	if (IS_V3(core) || IS_V4(core) || IS_V6(core))
   1622		return venus_suspend_3xx(core);
   1623
   1624	return venus_suspend_1xx(core);
   1625}
   1626
   1627static const struct hfi_ops venus_hfi_ops = {
   1628	.core_init			= venus_core_init,
   1629	.core_deinit			= venus_core_deinit,
   1630	.core_ping			= venus_core_ping,
   1631	.core_trigger_ssr		= venus_core_trigger_ssr,
   1632
   1633	.session_init			= venus_session_init,
   1634	.session_end			= venus_session_end,
   1635	.session_abort			= venus_session_abort,
   1636	.session_flush			= venus_session_flush,
   1637	.session_start			= venus_session_start,
   1638	.session_stop			= venus_session_stop,
   1639	.session_continue		= venus_session_continue,
   1640	.session_etb			= venus_session_etb,
   1641	.session_ftb			= venus_session_ftb,
   1642	.session_set_buffers		= venus_session_set_buffers,
   1643	.session_unset_buffers		= venus_session_unset_buffers,
   1644	.session_load_res		= venus_session_load_res,
   1645	.session_release_res		= venus_session_release_res,
   1646	.session_parse_seq_hdr		= venus_session_parse_seq_hdr,
   1647	.session_get_seq_hdr		= venus_session_get_seq_hdr,
   1648	.session_set_property		= venus_session_set_property,
   1649	.session_get_property		= venus_session_get_property,
   1650
   1651	.resume				= venus_resume,
   1652	.suspend			= venus_suspend,
   1653
   1654	.isr				= venus_isr,
   1655	.isr_thread			= venus_isr_thread,
   1656};
   1657
   1658void venus_hfi_destroy(struct venus_core *core)
   1659{
   1660	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1661
   1662	core->priv = NULL;
   1663	venus_interface_queues_release(hdev);
   1664	mutex_destroy(&hdev->lock);
   1665	kfree(hdev);
   1666	core->ops = NULL;
   1667}
   1668
   1669int venus_hfi_create(struct venus_core *core)
   1670{
   1671	struct venus_hfi_device *hdev;
   1672	int ret;
   1673
   1674	hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
   1675	if (!hdev)
   1676		return -ENOMEM;
   1677
   1678	mutex_init(&hdev->lock);
   1679
   1680	hdev->core = core;
   1681	hdev->suspended = true;
   1682	core->priv = hdev;
   1683	core->ops = &venus_hfi_ops;
   1684
   1685	ret = venus_interface_queues_init(hdev);
   1686	if (ret)
   1687		goto err_kfree;
   1688
   1689	return 0;
   1690
   1691err_kfree:
   1692	kfree(hdev);
   1693	core->priv = NULL;
   1694	core->ops = NULL;
   1695	return ret;
   1696}
   1697
   1698void venus_hfi_queues_reinit(struct venus_core *core)
   1699{
   1700	struct venus_hfi_device *hdev = to_hfi_priv(core);
   1701	struct hfi_queue_table_header *tbl_hdr;
   1702	struct iface_queue *queue;
   1703	struct hfi_sfr *sfr;
   1704	unsigned int i;
   1705
   1706	mutex_lock(&hdev->lock);
   1707
   1708	for (i = 0; i < IFACEQ_NUM; i++) {
   1709		queue = &hdev->queues[i];
   1710		queue->qhdr =
   1711			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
   1712
   1713		venus_set_qhdr_defaults(queue->qhdr);
   1714
   1715		queue->qhdr->start_addr = queue->qmem.da;
   1716
   1717		if (i == IFACEQ_CMD_IDX)
   1718			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
   1719		else if (i == IFACEQ_MSG_IDX)
   1720			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
   1721		else if (i == IFACEQ_DBG_IDX)
   1722			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
   1723	}
   1724
   1725	tbl_hdr = hdev->ifaceq_table.kva;
   1726	tbl_hdr->version = 0;
   1727	tbl_hdr->size = IFACEQ_TABLE_SIZE;
   1728	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
   1729	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
   1730	tbl_hdr->num_q = IFACEQ_NUM;
   1731	tbl_hdr->num_active_q = IFACEQ_NUM;
   1732
   1733	/*
   1734	 * Set receive request to zero on debug queue as there is no
   1735	 * need of interrupt from video hardware for debug messages
   1736	 */
   1737	queue = &hdev->queues[IFACEQ_DBG_IDX];
   1738	queue->qhdr->rx_req = 0;
   1739
   1740	sfr = hdev->sfr.kva;
   1741	sfr->buf_size = ALIGNED_SFR_SIZE;
   1742
   1743	/* ensure table and queue header structs are settled in memory */
   1744	wmb();
   1745
   1746	mutex_unlock(&hdev->lock);
   1747}