cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

vega20_ih.c (19576B)


      1/*
      2 * Copyright 2020 Advanced Micro Devices, Inc.
      3 *
      4 * Permission is hereby granted, free of charge, to any person obtaining a
      5 * copy of this software and associated documentation files (the "Software"),
      6 * to deal in the Software without restriction, including without limitation
      7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      8 * and/or sell copies of the Software, and to permit persons to whom the
      9 * Software is furnished to do so, subject to the following conditions:
     10 *
     11 * The above copyright notice and this permission notice shall be included in
     12 * all copies or substantial portions of the Software.
     13 *
     14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     20 * OTHER DEALINGS IN THE SOFTWARE.
     21 *
     22 */
     23
     24#include <linux/pci.h>
     25
     26#include "amdgpu.h"
     27#include "amdgpu_ih.h"
     28#include "soc15.h"
     29
     30#include "oss/osssys_4_2_0_offset.h"
     31#include "oss/osssys_4_2_0_sh_mask.h"
     32
     33#include "soc15_common.h"
     34#include "vega20_ih.h"
     35
     36#define MAX_REARM_RETRY 10
     37
     38#define mmIH_CHICKEN_ALDEBARAN			0x18d
     39#define mmIH_CHICKEN_ALDEBARAN_BASE_IDX		0
     40
     41static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev);
     42
     43/**
     44 * vega20_ih_init_register_offset - Initialize register offset for ih rings
     45 *
     46 * @adev: amdgpu_device pointer
     47 *
     48 * Initialize register offset ih rings (VEGA20).
     49 */
     50static void vega20_ih_init_register_offset(struct amdgpu_device *adev)
     51{
     52	struct amdgpu_ih_regs *ih_regs;
     53
     54	if (adev->irq.ih.ring_size) {
     55		ih_regs = &adev->irq.ih.ih_regs;
     56		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE);
     57		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI);
     58		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
     59		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
     60		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR);
     61		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR);
     62		ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO);
     63		ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI);
     64		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL;
     65	}
     66
     67	if (adev->irq.ih1.ring_size) {
     68		ih_regs = &adev->irq.ih1.ih_regs;
     69		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1);
     70		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1);
     71		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1);
     72		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1);
     73		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1);
     74		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1);
     75		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1;
     76	}
     77
     78	if (adev->irq.ih2.ring_size) {
     79		ih_regs = &adev->irq.ih2.ih_regs;
     80		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2);
     81		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2);
     82		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2);
     83		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2);
     84		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2);
     85		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2);
     86		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2;
     87	}
     88}
     89
     90/**
     91 * vega20_ih_toggle_ring_interrupts - toggle the interrupt ring buffer
     92 *
     93 * @adev: amdgpu_device pointer
     94 * @ih: amdgpu_ih_ring pointer
     95 * @enable: true - enable the interrupts, false - disable the interrupts
     96 *
     97 * Toggle the interrupt ring buffer (VEGA20)
     98 */
     99static int vega20_ih_toggle_ring_interrupts(struct amdgpu_device *adev,
    100					    struct amdgpu_ih_ring *ih,
    101					    bool enable)
    102{
    103	struct amdgpu_ih_regs *ih_regs;
    104	uint32_t tmp;
    105
    106	ih_regs = &ih->ih_regs;
    107
    108	tmp = RREG32(ih_regs->ih_rb_cntl);
    109	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));
    110	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_GPU_TS_ENABLE, 1);
    111
    112	/* enable_intr field is only valid in ring0 */
    113	if (ih == &adev->irq.ih)
    114		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));
    115	if (amdgpu_sriov_vf(adev)) {
    116		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) {
    117			dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n");
    118			return -ETIMEDOUT;
    119		}
    120	} else {
    121		WREG32(ih_regs->ih_rb_cntl, tmp);
    122	}
    123
    124	if (enable) {
    125		ih->enabled = true;
    126	} else {
    127		/* set rptr, wptr to 0 */
    128		WREG32(ih_regs->ih_rb_rptr, 0);
    129		WREG32(ih_regs->ih_rb_wptr, 0);
    130		ih->enabled = false;
    131		ih->rptr = 0;
    132	}
    133
    134	return 0;
    135}
    136
    137/**
    138 * vega20_ih_toggle_interrupts - Toggle all the available interrupt ring buffers
    139 *
    140 * @adev: amdgpu_device pointer
    141 * @enable: enable or disable interrupt ring buffers
    142 *
    143 * Toggle all the available interrupt ring buffers (VEGA20).
    144 */
    145static int vega20_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable)
    146{
    147	struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};
    148	int i;
    149	int r;
    150
    151	for (i = 0; i < ARRAY_SIZE(ih); i++) {
    152		if (ih[i]->ring_size) {
    153			r = vega20_ih_toggle_ring_interrupts(adev, ih[i], enable);
    154			if (r)
    155				return r;
    156		}
    157	}
    158
    159	return 0;
    160}
    161
    162static uint32_t vega20_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
    163{
    164	int rb_bufsz = order_base_2(ih->ring_size / 4);
    165
    166	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
    167				   MC_SPACE, ih->use_bus_addr ? 1 : 4);
    168	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
    169				   WPTR_OVERFLOW_CLEAR, 1);
    170	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
    171				   WPTR_OVERFLOW_ENABLE, 1);
    172	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
    173	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
    174	 * value is written to memory
    175	 */
    176	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
    177				   WPTR_WRITEBACK_ENABLE, 1);
    178	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
    179	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
    180	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
    181
    182	return ih_rb_cntl;
    183}
    184
    185static uint32_t vega20_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)
    186{
    187	u32 ih_doorbell_rtpr = 0;
    188
    189	if (ih->use_doorbell) {
    190		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
    191						 IH_DOORBELL_RPTR, OFFSET,
    192						 ih->doorbell_index);
    193		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
    194						 IH_DOORBELL_RPTR,
    195						 ENABLE, 1);
    196	} else {
    197		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
    198						 IH_DOORBELL_RPTR,
    199						 ENABLE, 0);
    200	}
    201	return ih_doorbell_rtpr;
    202}
    203
    204/**
    205 * vega20_ih_enable_ring - enable an ih ring buffer
    206 *
    207 * @adev: amdgpu_device pointer
    208 * @ih: amdgpu_ih_ring pointer
    209 *
    210 * Enable an ih ring buffer (VEGA20)
    211 */
    212static int vega20_ih_enable_ring(struct amdgpu_device *adev,
    213				 struct amdgpu_ih_ring *ih)
    214{
    215	struct amdgpu_ih_regs *ih_regs;
    216	uint32_t tmp;
    217
    218	ih_regs = &ih->ih_regs;
    219
    220	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
    221	WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8);
    222	WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff);
    223
    224	tmp = RREG32(ih_regs->ih_rb_cntl);
    225	tmp = vega20_ih_rb_cntl(ih, tmp);
    226	if (ih == &adev->irq.ih)
    227		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled);
    228	if (ih == &adev->irq.ih1)
    229		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1);
    230	if (amdgpu_sriov_vf(adev)) {
    231		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) {
    232			dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n");
    233			return -ETIMEDOUT;
    234		}
    235	} else {
    236		WREG32(ih_regs->ih_rb_cntl, tmp);
    237	}
    238
    239	if (ih == &adev->irq.ih) {
    240		/* set the ih ring 0 writeback address whether it's enabled or not */
    241		WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr));
    242		WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF);
    243	}
    244
    245	/* set rptr, wptr to 0 */
    246	WREG32(ih_regs->ih_rb_wptr, 0);
    247	WREG32(ih_regs->ih_rb_rptr, 0);
    248
    249	WREG32(ih_regs->ih_doorbell_rptr, vega20_ih_doorbell_rptr(ih));
    250
    251	return 0;
    252}
    253
    254/**
    255 * vega20_ih_reroute_ih - reroute VMC/UTCL2 ih to an ih ring
    256 *
    257 * @adev: amdgpu_device pointer
    258 *
    259 * Reroute VMC and UMC interrupts on primary ih ring to
    260 * ih ring 1 so they won't lose when bunches of page faults
    261 * interrupts overwhelms the interrupt handler(VEGA20)
    262 */
    263static void vega20_ih_reroute_ih(struct amdgpu_device *adev)
    264{
    265	uint32_t tmp;
    266
    267	/* vega20 ih reroute will go through psp this
    268	 * function is used for newer asics starting arcturus
    269	 */
    270	if (adev->asic_type >= CHIP_ARCTURUS) {
    271		/* Reroute to IH ring 1 for VMC */
    272		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12);
    273		tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA);
    274		tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1);
    275		tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
    276		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp);
    277
    278		/* Reroute IH ring 1 for UTCL2 */
    279		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B);
    280		tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA);
    281		tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
    282		WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp);
    283	}
    284}
    285
    286/**
    287 * vega20_ih_irq_init - init and enable the interrupt ring
    288 *
    289 * @adev: amdgpu_device pointer
    290 *
    291 * Allocate a ring buffer for the interrupt controller,
    292 * enable the RLC, disable interrupts, enable the IH
    293 * ring buffer and enable it (VI).
    294 * Called at device load and reume.
    295 * Returns 0 for success, errors for failure.
    296 */
    297static int vega20_ih_irq_init(struct amdgpu_device *adev)
    298{
    299	struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};
    300	u32 ih_chicken;
    301	int ret;
    302	int i;
    303
    304	/* disable irqs */
    305	ret = vega20_ih_toggle_interrupts(adev, false);
    306	if (ret)
    307		return ret;
    308
    309	adev->nbio.funcs->ih_control(adev);
    310
    311	if (adev->asic_type == CHIP_ARCTURUS &&
    312	    adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
    313		ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
    314		if (adev->irq.ih.use_bus_addr) {
    315			ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
    316						   MC_SPACE_GPA_ENABLE, 1);
    317		}
    318		WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
    319	}
    320
    321	/* psp firmware won't program IH_CHICKEN for aldebaran
    322	 * driver needs to program it properly according to
    323	 * MC_SPACE type in IH_RB_CNTL */
    324	if (adev->asic_type == CHIP_ALDEBARAN) {
    325		ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN_ALDEBARAN);
    326		if (adev->irq.ih.use_bus_addr) {
    327			ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
    328						   MC_SPACE_GPA_ENABLE, 1);
    329		}
    330		WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN_ALDEBARAN, ih_chicken);
    331	}
    332
    333	for (i = 0; i < ARRAY_SIZE(ih); i++) {
    334		if (ih[i]->ring_size) {
    335			if (i == 1)
    336				vega20_ih_reroute_ih(adev);
    337			ret = vega20_ih_enable_ring(adev, ih[i]);
    338			if (ret)
    339				return ret;
    340		}
    341	}
    342
    343	pci_set_master(adev->pdev);
    344
    345	/* enable interrupts */
    346	ret = vega20_ih_toggle_interrupts(adev, true);
    347	if (ret)
    348		return ret;
    349
    350	if (adev->irq.ih_soft.ring_size)
    351		adev->irq.ih_soft.enabled = true;
    352
    353	return 0;
    354}
    355
    356/**
    357 * vega20_ih_irq_disable - disable interrupts
    358 *
    359 * @adev: amdgpu_device pointer
    360 *
    361 * Disable interrupts on the hw (VEGA20).
    362 */
    363static void vega20_ih_irq_disable(struct amdgpu_device *adev)
    364{
    365	vega20_ih_toggle_interrupts(adev, false);
    366
    367	/* Wait and acknowledge irq */
    368	mdelay(1);
    369}
    370
    371/**
    372 * vega20_ih_get_wptr - get the IH ring buffer wptr
    373 *
    374 * @adev: amdgpu_device pointer
    375 * @ih: amdgpu_ih_ring pointer
    376 *
    377 * Get the IH ring buffer wptr from either the register
    378 * or the writeback memory buffer (VEGA20).  Also check for
    379 * ring buffer overflow and deal with it.
    380 * Returns the value of the wptr.
    381 */
    382static u32 vega20_ih_get_wptr(struct amdgpu_device *adev,
    383			      struct amdgpu_ih_ring *ih)
    384{
    385	u32 wptr, tmp;
    386	struct amdgpu_ih_regs *ih_regs;
    387
    388	if (ih == &adev->irq.ih) {
    389		/* Only ring0 supports writeback. On other rings fall back
    390		 * to register-based code with overflow checking below.
    391		 */
    392		wptr = le32_to_cpu(*ih->wptr_cpu);
    393
    394		if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
    395			goto out;
    396	}
    397
    398	ih_regs = &ih->ih_regs;
    399
    400	/* Double check that the overflow wasn't already cleared. */
    401	wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr);
    402	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
    403		goto out;
    404
    405	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
    406
    407	/* When a ring buffer overflow happen start parsing interrupt
    408	 * from the last not overwritten vector (wptr + 32). Hopefully
    409	 * this should allow us to catchup.
    410	 */
    411	tmp = (wptr + 32) & ih->ptr_mask;
    412	dev_warn(adev->dev, "IH ring buffer overflow "
    413		 "(0x%08X, 0x%08X, 0x%08X)\n",
    414		 wptr, ih->rptr, tmp);
    415	ih->rptr = tmp;
    416
    417	tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl);
    418	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
    419	WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp);
    420
    421out:
    422	return (wptr & ih->ptr_mask);
    423}
    424
    425/**
    426 * vega20_ih_irq_rearm - rearm IRQ if lost
    427 *
    428 * @adev: amdgpu_device pointer
    429 * @ih: amdgpu_ih_ring pointer
    430 *
    431 */
    432static void vega20_ih_irq_rearm(struct amdgpu_device *adev,
    433			       struct amdgpu_ih_ring *ih)
    434{
    435	uint32_t v = 0;
    436	uint32_t i = 0;
    437	struct amdgpu_ih_regs *ih_regs;
    438
    439	ih_regs = &ih->ih_regs;
    440
    441	/* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */
    442	for (i = 0; i < MAX_REARM_RETRY; i++) {
    443		v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr);
    444		if ((v < ih->ring_size) && (v != ih->rptr))
    445			WDOORBELL32(ih->doorbell_index, ih->rptr);
    446		else
    447			break;
    448	}
    449}
    450
    451/**
    452 * vega20_ih_set_rptr - set the IH ring buffer rptr
    453 *
    454 * @adev: amdgpu_device pointer
    455 * @ih: amdgpu_ih_ring pointer
    456 *
    457 * Set the IH ring buffer rptr.
    458 */
    459static void vega20_ih_set_rptr(struct amdgpu_device *adev,
    460			       struct amdgpu_ih_ring *ih)
    461{
    462	struct amdgpu_ih_regs *ih_regs;
    463
    464	if (ih->use_doorbell) {
    465		/* XXX check if swapping is necessary on BE */
    466		*ih->rptr_cpu = ih->rptr;
    467		WDOORBELL32(ih->doorbell_index, ih->rptr);
    468
    469		if (amdgpu_sriov_vf(adev))
    470			vega20_ih_irq_rearm(adev, ih);
    471	} else {
    472		ih_regs = &ih->ih_regs;
    473		WREG32(ih_regs->ih_rb_rptr, ih->rptr);
    474	}
    475}
    476
    477/**
    478 * vega20_ih_self_irq - dispatch work for ring 1 and 2
    479 *
    480 * @adev: amdgpu_device pointer
    481 * @source: irq source
    482 * @entry: IV with WPTR update
    483 *
    484 * Update the WPTR from the IV and schedule work to handle the entries.
    485 */
    486static int vega20_ih_self_irq(struct amdgpu_device *adev,
    487			      struct amdgpu_irq_src *source,
    488			      struct amdgpu_iv_entry *entry)
    489{
    490	switch (entry->ring_id) {
    491	case 1:
    492		schedule_work(&adev->irq.ih1_work);
    493		break;
    494	case 2:
    495		schedule_work(&adev->irq.ih2_work);
    496		break;
    497	default: break;
    498	}
    499	return 0;
    500}
    501
    502static const struct amdgpu_irq_src_funcs vega20_ih_self_irq_funcs = {
    503	.process = vega20_ih_self_irq,
    504};
    505
    506static void vega20_ih_set_self_irq_funcs(struct amdgpu_device *adev)
    507{
    508	adev->irq.self_irq.num_types = 0;
    509	adev->irq.self_irq.funcs = &vega20_ih_self_irq_funcs;
    510}
    511
    512static int vega20_ih_early_init(void *handle)
    513{
    514	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    515
    516	vega20_ih_set_interrupt_funcs(adev);
    517	vega20_ih_set_self_irq_funcs(adev);
    518	return 0;
    519}
    520
    521static int vega20_ih_sw_init(void *handle)
    522{
    523	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    524	int r;
    525
    526	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0,
    527			      &adev->irq.self_irq);
    528	if (r)
    529		return r;
    530
    531	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
    532	if (r)
    533		return r;
    534
    535	adev->irq.ih.use_doorbell = true;
    536	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
    537
    538	r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
    539	if (r)
    540		return r;
    541
    542	adev->irq.ih1.use_doorbell = true;
    543	adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
    544
    545	r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
    546	if (r)
    547		return r;
    548
    549	adev->irq.ih2.use_doorbell = true;
    550	adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
    551
    552	/* initialize ih control registers offset */
    553	vega20_ih_init_register_offset(adev);
    554
    555	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true);
    556	if (r)
    557		return r;
    558
    559	r = amdgpu_irq_init(adev);
    560
    561	return r;
    562}
    563
    564static int vega20_ih_sw_fini(void *handle)
    565{
    566	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    567
    568	amdgpu_irq_fini_sw(adev);
    569
    570	return 0;
    571}
    572
    573static int vega20_ih_hw_init(void *handle)
    574{
    575	int r;
    576	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    577
    578	r = vega20_ih_irq_init(adev);
    579	if (r)
    580		return r;
    581
    582	return 0;
    583}
    584
    585static int vega20_ih_hw_fini(void *handle)
    586{
    587	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    588
    589	vega20_ih_irq_disable(adev);
    590
    591	return 0;
    592}
    593
    594static int vega20_ih_suspend(void *handle)
    595{
    596	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    597
    598	return vega20_ih_hw_fini(adev);
    599}
    600
    601static int vega20_ih_resume(void *handle)
    602{
    603	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    604
    605	return vega20_ih_hw_init(adev);
    606}
    607
    608static bool vega20_ih_is_idle(void *handle)
    609{
    610	/* todo */
    611	return true;
    612}
    613
    614static int vega20_ih_wait_for_idle(void *handle)
    615{
    616	/* todo */
    617	return -ETIMEDOUT;
    618}
    619
    620static int vega20_ih_soft_reset(void *handle)
    621{
    622	/* todo */
    623
    624	return 0;
    625}
    626
    627static void vega20_ih_update_clockgating_state(struct amdgpu_device *adev,
    628					       bool enable)
    629{
    630	uint32_t data, def, field_val;
    631
    632	if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
    633		def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
    634		field_val = enable ? 0 : 1;
    635		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    636				     IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val);
    637		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    638				     IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val);
    639		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    640				     DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
    641		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    642				     OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
    643		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    644				     LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
    645		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    646				     DYN_CLK_SOFT_OVERRIDE, field_val);
    647		data = REG_SET_FIELD(data, IH_CLK_CTRL,
    648				     REG_CLK_SOFT_OVERRIDE, field_val);
    649		if (def != data)
    650			WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
    651	}
    652}
    653
    654static int vega20_ih_set_clockgating_state(void *handle,
    655					  enum amd_clockgating_state state)
    656{
    657	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    658
    659	vega20_ih_update_clockgating_state(adev,
    660				state == AMD_CG_STATE_GATE);
    661	return 0;
    662
    663}
    664
    665static int vega20_ih_set_powergating_state(void *handle,
    666					  enum amd_powergating_state state)
    667{
    668	return 0;
    669}
    670
    671const struct amd_ip_funcs vega20_ih_ip_funcs = {
    672	.name = "vega20_ih",
    673	.early_init = vega20_ih_early_init,
    674	.late_init = NULL,
    675	.sw_init = vega20_ih_sw_init,
    676	.sw_fini = vega20_ih_sw_fini,
    677	.hw_init = vega20_ih_hw_init,
    678	.hw_fini = vega20_ih_hw_fini,
    679	.suspend = vega20_ih_suspend,
    680	.resume = vega20_ih_resume,
    681	.is_idle = vega20_ih_is_idle,
    682	.wait_for_idle = vega20_ih_wait_for_idle,
    683	.soft_reset = vega20_ih_soft_reset,
    684	.set_clockgating_state = vega20_ih_set_clockgating_state,
    685	.set_powergating_state = vega20_ih_set_powergating_state,
    686};
    687
    688static const struct amdgpu_ih_funcs vega20_ih_funcs = {
    689	.get_wptr = vega20_ih_get_wptr,
    690	.decode_iv = amdgpu_ih_decode_iv_helper,
    691	.decode_iv_ts = amdgpu_ih_decode_iv_ts_helper,
    692	.set_rptr = vega20_ih_set_rptr
    693};
    694
    695static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev)
    696{
    697	adev->irq.ih_funcs = &vega20_ih_funcs;
    698}
    699
    700const struct amdgpu_ip_block_version vega20_ih_ip_block =
    701{
    702	.type = AMD_IP_BLOCK_TYPE_IH,
    703	.major = 4,
    704	.minor = 2,
    705	.rev = 0,
    706	.funcs = &vega20_ih_ip_funcs,
    707};