cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

cz_ih.c (12089B)


      1/*
      2 * Copyright 2014 Advanced Micro Devices, Inc.
      3 *
      4 * Permission is hereby granted, free of charge, to any person obtaining a
      5 * copy of this software and associated documentation files (the "Software"),
      6 * to deal in the Software without restriction, including without limitation
      7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      8 * and/or sell copies of the Software, and to permit persons to whom the
      9 * Software is furnished to do so, subject to the following conditions:
     10 *
     11 * The above copyright notice and this permission notice shall be included in
     12 * all copies or substantial portions of the Software.
     13 *
     14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     20 * OTHER DEALINGS IN THE SOFTWARE.
     21 *
     22 */
     23
     24#include <linux/pci.h>
     25
     26#include "amdgpu.h"
     27#include "amdgpu_ih.h"
     28#include "vid.h"
     29
     30#include "oss/oss_3_0_1_d.h"
     31#include "oss/oss_3_0_1_sh_mask.h"
     32
     33#include "bif/bif_5_1_d.h"
     34#include "bif/bif_5_1_sh_mask.h"
     35
     36/*
     37 * Interrupts
     38 * Starting with r6xx, interrupts are handled via a ring buffer.
     39 * Ring buffers are areas of GPU accessible memory that the GPU
     40 * writes interrupt vectors into and the host reads vectors out of.
     41 * There is a rptr (read pointer) that determines where the
     42 * host is currently reading, and a wptr (write pointer)
     43 * which determines where the GPU has written.  When the
     44 * pointers are equal, the ring is idle.  When the GPU
     45 * writes vectors to the ring buffer, it increments the
     46 * wptr.  When there is an interrupt, the host then starts
     47 * fetching commands and processing them until the pointers are
     48 * equal again at which point it updates the rptr.
     49 */
     50
     51static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev);
     52
     53/**
     54 * cz_ih_enable_interrupts - Enable the interrupt ring buffer
     55 *
     56 * @adev: amdgpu_device pointer
     57 *
     58 * Enable the interrupt ring buffer (VI).
     59 */
     60static void cz_ih_enable_interrupts(struct amdgpu_device *adev)
     61{
     62	u32 ih_cntl = RREG32(mmIH_CNTL);
     63	u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
     64
     65	ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 1);
     66	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
     67	WREG32(mmIH_CNTL, ih_cntl);
     68	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
     69	adev->irq.ih.enabled = true;
     70}
     71
     72/**
     73 * cz_ih_disable_interrupts - Disable the interrupt ring buffer
     74 *
     75 * @adev: amdgpu_device pointer
     76 *
     77 * Disable the interrupt ring buffer (VI).
     78 */
     79static void cz_ih_disable_interrupts(struct amdgpu_device *adev)
     80{
     81	u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
     82	u32 ih_cntl = RREG32(mmIH_CNTL);
     83
     84	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
     85	ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, ENABLE_INTR, 0);
     86	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
     87	WREG32(mmIH_CNTL, ih_cntl);
     88	/* set rptr, wptr to 0 */
     89	WREG32(mmIH_RB_RPTR, 0);
     90	WREG32(mmIH_RB_WPTR, 0);
     91	adev->irq.ih.enabled = false;
     92	adev->irq.ih.rptr = 0;
     93}
     94
     95/**
     96 * cz_ih_irq_init - init and enable the interrupt ring
     97 *
     98 * @adev: amdgpu_device pointer
     99 *
    100 * Allocate a ring buffer for the interrupt controller,
    101 * enable the RLC, disable interrupts, enable the IH
    102 * ring buffer and enable it (VI).
    103 * Called at device load and reume.
    104 * Returns 0 for success, errors for failure.
    105 */
    106static int cz_ih_irq_init(struct amdgpu_device *adev)
    107{
    108	struct amdgpu_ih_ring *ih = &adev->irq.ih;
    109	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
    110	int rb_bufsz;
    111
    112	/* disable irqs */
    113	cz_ih_disable_interrupts(adev);
    114
    115	/* setup interrupt control */
    116	WREG32(mmINTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
    117	interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
    118	/* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
    119	 * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
    120	 */
    121	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_DUMMY_RD_OVERRIDE, 0);
    122	/* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
    123	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, INTERRUPT_CNTL, IH_REQ_NONSNOOP_EN, 0);
    124	WREG32(mmINTERRUPT_CNTL, interrupt_cntl);
    125
    126	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
    127	WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
    128
    129	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
    130	ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
    131	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
    132	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
    133
    134	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
    135	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
    136
    137	/* set the writeback address whether it's enabled or not */
    138	WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr));
    139	WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF);
    140
    141	WREG32(mmIH_RB_CNTL, ih_rb_cntl);
    142
    143	/* set rptr, wptr to 0 */
    144	WREG32(mmIH_RB_RPTR, 0);
    145	WREG32(mmIH_RB_WPTR, 0);
    146
    147	/* Default settings for IH_CNTL (disabled at first) */
    148	ih_cntl = RREG32(mmIH_CNTL);
    149	ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, MC_VMID, 0);
    150
    151	if (adev->irq.msi_enabled)
    152		ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL, RPTR_REARM, 1);
    153	WREG32(mmIH_CNTL, ih_cntl);
    154
    155	pci_set_master(adev->pdev);
    156
    157	/* enable interrupts */
    158	cz_ih_enable_interrupts(adev);
    159
    160	return 0;
    161}
    162
    163/**
    164 * cz_ih_irq_disable - disable interrupts
    165 *
    166 * @adev: amdgpu_device pointer
    167 *
    168 * Disable interrupts on the hw (VI).
    169 */
    170static void cz_ih_irq_disable(struct amdgpu_device *adev)
    171{
    172	cz_ih_disable_interrupts(adev);
    173
    174	/* Wait and acknowledge irq */
    175	mdelay(1);
    176}
    177
    178/**
    179 * cz_ih_get_wptr - get the IH ring buffer wptr
    180 *
    181 * @adev: amdgpu_device pointer
    182 * @ih: IH ring buffer to fetch wptr
    183 *
    184 * Get the IH ring buffer wptr from either the register
    185 * or the writeback memory buffer (VI).  Also check for
    186 * ring buffer overflow and deal with it.
    187 * Used by cz_irq_process(VI).
    188 * Returns the value of the wptr.
    189 */
    190static u32 cz_ih_get_wptr(struct amdgpu_device *adev,
    191			  struct amdgpu_ih_ring *ih)
    192{
    193	u32 wptr, tmp;
    194
    195	wptr = le32_to_cpu(*ih->wptr_cpu);
    196
    197	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
    198		goto out;
    199
    200	/* Double check that the overflow wasn't already cleared. */
    201	wptr = RREG32(mmIH_RB_WPTR);
    202
    203	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
    204		goto out;
    205
    206	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
    207
    208	/* When a ring buffer overflow happen start parsing interrupt
    209	 * from the last not overwritten vector (wptr + 16). Hopefully
    210	 * this should allow us to catchup.
    211	 */
    212	dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
    213		wptr, ih->rptr, (wptr + 16) & ih->ptr_mask);
    214	ih->rptr = (wptr + 16) & ih->ptr_mask;
    215	tmp = RREG32(mmIH_RB_CNTL);
    216	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
    217	WREG32(mmIH_RB_CNTL, tmp);
    218
    219
    220out:
    221	return (wptr & ih->ptr_mask);
    222}
    223
    224/**
    225 * cz_ih_decode_iv - decode an interrupt vector
    226 *
    227 * @adev: amdgpu_device pointer
    228 * @ih: IH ring buffer to decode
    229 * @entry: IV entry to place decoded information into
    230 *
    231 * Decodes the interrupt vector at the current rptr
    232 * position and also advance the position.
    233 */
    234static void cz_ih_decode_iv(struct amdgpu_device *adev,
    235			    struct amdgpu_ih_ring *ih,
    236			    struct amdgpu_iv_entry *entry)
    237{
    238	/* wptr/rptr are in bytes! */
    239	u32 ring_index = ih->rptr >> 2;
    240	uint32_t dw[4];
    241
    242	dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
    243	dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
    244	dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
    245	dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
    246
    247	entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY;
    248	entry->src_id = dw[0] & 0xff;
    249	entry->src_data[0] = dw[1] & 0xfffffff;
    250	entry->ring_id = dw[2] & 0xff;
    251	entry->vmid = (dw[2] >> 8) & 0xff;
    252	entry->pasid = (dw[2] >> 16) & 0xffff;
    253
    254	/* wptr/rptr are in bytes! */
    255	ih->rptr += 16;
    256}
    257
    258/**
    259 * cz_ih_set_rptr - set the IH ring buffer rptr
    260 *
    261 * @adev: amdgpu_device pointer
    262 * @ih: IH ring buffer to set rptr
    263 *
    264 * Set the IH ring buffer rptr.
    265 */
    266static void cz_ih_set_rptr(struct amdgpu_device *adev,
    267			   struct amdgpu_ih_ring *ih)
    268{
    269	WREG32(mmIH_RB_RPTR, ih->rptr);
    270}
    271
    272static int cz_ih_early_init(void *handle)
    273{
    274	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    275	int ret;
    276
    277	ret = amdgpu_irq_add_domain(adev);
    278	if (ret)
    279		return ret;
    280
    281	cz_ih_set_interrupt_funcs(adev);
    282
    283	return 0;
    284}
    285
    286static int cz_ih_sw_init(void *handle)
    287{
    288	int r;
    289	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    290
    291	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, false);
    292	if (r)
    293		return r;
    294
    295	r = amdgpu_irq_init(adev);
    296
    297	return r;
    298}
    299
    300static int cz_ih_sw_fini(void *handle)
    301{
    302	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    303
    304	amdgpu_irq_fini_sw(adev);
    305	amdgpu_irq_remove_domain(adev);
    306
    307	return 0;
    308}
    309
    310static int cz_ih_hw_init(void *handle)
    311{
    312	int r;
    313	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    314
    315	r = cz_ih_irq_init(adev);
    316	if (r)
    317		return r;
    318
    319	return 0;
    320}
    321
    322static int cz_ih_hw_fini(void *handle)
    323{
    324	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    325
    326	cz_ih_irq_disable(adev);
    327
    328	return 0;
    329}
    330
    331static int cz_ih_suspend(void *handle)
    332{
    333	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    334
    335	return cz_ih_hw_fini(adev);
    336}
    337
    338static int cz_ih_resume(void *handle)
    339{
    340	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    341
    342	return cz_ih_hw_init(adev);
    343}
    344
    345static bool cz_ih_is_idle(void *handle)
    346{
    347	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    348	u32 tmp = RREG32(mmSRBM_STATUS);
    349
    350	if (REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
    351		return false;
    352
    353	return true;
    354}
    355
    356static int cz_ih_wait_for_idle(void *handle)
    357{
    358	unsigned i;
    359	u32 tmp;
    360	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    361
    362	for (i = 0; i < adev->usec_timeout; i++) {
    363		/* read MC_STATUS */
    364		tmp = RREG32(mmSRBM_STATUS);
    365		if (!REG_GET_FIELD(tmp, SRBM_STATUS, IH_BUSY))
    366			return 0;
    367		udelay(1);
    368	}
    369	return -ETIMEDOUT;
    370}
    371
    372static int cz_ih_soft_reset(void *handle)
    373{
    374	u32 srbm_soft_reset = 0;
    375	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    376	u32 tmp = RREG32(mmSRBM_STATUS);
    377
    378	if (tmp & SRBM_STATUS__IH_BUSY_MASK)
    379		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
    380						SOFT_RESET_IH, 1);
    381
    382	if (srbm_soft_reset) {
    383		tmp = RREG32(mmSRBM_SOFT_RESET);
    384		tmp |= srbm_soft_reset;
    385		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
    386		WREG32(mmSRBM_SOFT_RESET, tmp);
    387		tmp = RREG32(mmSRBM_SOFT_RESET);
    388
    389		udelay(50);
    390
    391		tmp &= ~srbm_soft_reset;
    392		WREG32(mmSRBM_SOFT_RESET, tmp);
    393		tmp = RREG32(mmSRBM_SOFT_RESET);
    394
    395		/* Wait a little for things to settle down */
    396		udelay(50);
    397	}
    398
    399	return 0;
    400}
    401
    402static int cz_ih_set_clockgating_state(void *handle,
    403					  enum amd_clockgating_state state)
    404{
    405	// TODO
    406	return 0;
    407}
    408
    409static int cz_ih_set_powergating_state(void *handle,
    410					  enum amd_powergating_state state)
    411{
    412	// TODO
    413	return 0;
    414}
    415
    416static const struct amd_ip_funcs cz_ih_ip_funcs = {
    417	.name = "cz_ih",
    418	.early_init = cz_ih_early_init,
    419	.late_init = NULL,
    420	.sw_init = cz_ih_sw_init,
    421	.sw_fini = cz_ih_sw_fini,
    422	.hw_init = cz_ih_hw_init,
    423	.hw_fini = cz_ih_hw_fini,
    424	.suspend = cz_ih_suspend,
    425	.resume = cz_ih_resume,
    426	.is_idle = cz_ih_is_idle,
    427	.wait_for_idle = cz_ih_wait_for_idle,
    428	.soft_reset = cz_ih_soft_reset,
    429	.set_clockgating_state = cz_ih_set_clockgating_state,
    430	.set_powergating_state = cz_ih_set_powergating_state,
    431};
    432
    433static const struct amdgpu_ih_funcs cz_ih_funcs = {
    434	.get_wptr = cz_ih_get_wptr,
    435	.decode_iv = cz_ih_decode_iv,
    436	.set_rptr = cz_ih_set_rptr
    437};
    438
    439static void cz_ih_set_interrupt_funcs(struct amdgpu_device *adev)
    440{
    441	adev->irq.ih_funcs = &cz_ih_funcs;
    442}
    443
    444const struct amdgpu_ip_block_version cz_ih_ip_block =
    445{
    446	.type = AMD_IP_BLOCK_TYPE_IH,
    447	.major = 3,
    448	.minor = 0,
    449	.rev = 0,
    450	.funcs = &cz_ih_ip_funcs,
    451};