cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

interrupt.c (24210B)


      1/*
      2 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
      3 *
      4 * Permission is hereby granted, free of charge, to any person obtaining a
      5 * copy of this software and associated documentation files (the "Software"),
      6 * to deal in the Software without restriction, including without limitation
      7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      8 * and/or sell copies of the Software, and to permit persons to whom the
      9 * Software is furnished to do so, subject to the following conditions:
     10 *
     11 * The above copyright notice and this permission notice (including the next
     12 * paragraph) shall be included in all copies or substantial portions of the
     13 * Software.
     14 *
     15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
     21 * SOFTWARE.
     22 *
     23 * Authors:
     24 *    Kevin Tian <kevin.tian@intel.com>
     25 *    Zhi Wang <zhi.a.wang@intel.com>
     26 *
     27 * Contributors:
     28 *    Min he <min.he@intel.com>
     29 *
     30 */
     31
     32#include <linux/eventfd.h>
     33
     34#include "i915_drv.h"
     35#include "i915_reg.h"
     36#include "gvt.h"
     37#include "trace.h"
     38
     39/* common offset among interrupt control registers */
     40#define regbase_to_isr(base)	(base)
     41#define regbase_to_imr(base)	(base + 0x4)
     42#define regbase_to_iir(base)	(base + 0x8)
     43#define regbase_to_ier(base)	(base + 0xC)
     44
     45#define iir_to_regbase(iir)    (iir - 0x8)
     46#define ier_to_regbase(ier)    (ier - 0xC)
     47
     48#define get_event_virt_handler(irq, e)	(irq->events[e].v_handler)
     49#define get_irq_info(irq, e)		(irq->events[e].info)
     50
     51#define irq_to_gvt(irq) \
     52	container_of(irq, struct intel_gvt, irq)
     53
     54static void update_upstream_irq(struct intel_vgpu *vgpu,
     55		struct intel_gvt_irq_info *info);
     56
     57static const char * const irq_name[INTEL_GVT_EVENT_MAX] = {
     58	[RCS_MI_USER_INTERRUPT] = "Render CS MI USER INTERRUPT",
     59	[RCS_DEBUG] = "Render EU debug from SVG",
     60	[RCS_MMIO_SYNC_FLUSH] = "Render MMIO sync flush status",
     61	[RCS_CMD_STREAMER_ERR] = "Render CS error interrupt",
     62	[RCS_PIPE_CONTROL] = "Render PIPE CONTROL notify",
     63	[RCS_WATCHDOG_EXCEEDED] = "Render CS Watchdog counter exceeded",
     64	[RCS_PAGE_DIRECTORY_FAULT] = "Render page directory faults",
     65	[RCS_AS_CONTEXT_SWITCH] = "Render AS Context Switch Interrupt",
     66
     67	[VCS_MI_USER_INTERRUPT] = "Video CS MI USER INTERRUPT",
     68	[VCS_MMIO_SYNC_FLUSH] = "Video MMIO sync flush status",
     69	[VCS_CMD_STREAMER_ERR] = "Video CS error interrupt",
     70	[VCS_MI_FLUSH_DW] = "Video MI FLUSH DW notify",
     71	[VCS_WATCHDOG_EXCEEDED] = "Video CS Watchdog counter exceeded",
     72	[VCS_PAGE_DIRECTORY_FAULT] = "Video page directory faults",
     73	[VCS_AS_CONTEXT_SWITCH] = "Video AS Context Switch Interrupt",
     74	[VCS2_MI_USER_INTERRUPT] = "VCS2 Video CS MI USER INTERRUPT",
     75	[VCS2_MI_FLUSH_DW] = "VCS2 Video MI FLUSH DW notify",
     76	[VCS2_AS_CONTEXT_SWITCH] = "VCS2 Context Switch Interrupt",
     77
     78	[BCS_MI_USER_INTERRUPT] = "Blitter CS MI USER INTERRUPT",
     79	[BCS_MMIO_SYNC_FLUSH] = "Billter MMIO sync flush status",
     80	[BCS_CMD_STREAMER_ERR] = "Blitter CS error interrupt",
     81	[BCS_MI_FLUSH_DW] = "Blitter MI FLUSH DW notify",
     82	[BCS_PAGE_DIRECTORY_FAULT] = "Blitter page directory faults",
     83	[BCS_AS_CONTEXT_SWITCH] = "Blitter AS Context Switch Interrupt",
     84
     85	[VECS_MI_FLUSH_DW] = "Video Enhanced Streamer MI FLUSH DW notify",
     86	[VECS_AS_CONTEXT_SWITCH] = "VECS Context Switch Interrupt",
     87
     88	[PIPE_A_FIFO_UNDERRUN] = "Pipe A FIFO underrun",
     89	[PIPE_A_CRC_ERR] = "Pipe A CRC error",
     90	[PIPE_A_CRC_DONE] = "Pipe A CRC done",
     91	[PIPE_A_VSYNC] = "Pipe A vsync",
     92	[PIPE_A_LINE_COMPARE] = "Pipe A line compare",
     93	[PIPE_A_ODD_FIELD] = "Pipe A odd field",
     94	[PIPE_A_EVEN_FIELD] = "Pipe A even field",
     95	[PIPE_A_VBLANK] = "Pipe A vblank",
     96	[PIPE_B_FIFO_UNDERRUN] = "Pipe B FIFO underrun",
     97	[PIPE_B_CRC_ERR] = "Pipe B CRC error",
     98	[PIPE_B_CRC_DONE] = "Pipe B CRC done",
     99	[PIPE_B_VSYNC] = "Pipe B vsync",
    100	[PIPE_B_LINE_COMPARE] = "Pipe B line compare",
    101	[PIPE_B_ODD_FIELD] = "Pipe B odd field",
    102	[PIPE_B_EVEN_FIELD] = "Pipe B even field",
    103	[PIPE_B_VBLANK] = "Pipe B vblank",
    104	[PIPE_C_VBLANK] = "Pipe C vblank",
    105	[DPST_PHASE_IN] = "DPST phase in event",
    106	[DPST_HISTOGRAM] = "DPST histogram event",
    107	[GSE] = "GSE",
    108	[DP_A_HOTPLUG] = "DP A Hotplug",
    109	[AUX_CHANNEL_A] = "AUX Channel A",
    110	[PERF_COUNTER] = "Performance counter",
    111	[POISON] = "Poison",
    112	[GTT_FAULT] = "GTT fault",
    113	[PRIMARY_A_FLIP_DONE] = "Primary Plane A flip done",
    114	[PRIMARY_B_FLIP_DONE] = "Primary Plane B flip done",
    115	[PRIMARY_C_FLIP_DONE] = "Primary Plane C flip done",
    116	[SPRITE_A_FLIP_DONE] = "Sprite Plane A flip done",
    117	[SPRITE_B_FLIP_DONE] = "Sprite Plane B flip done",
    118	[SPRITE_C_FLIP_DONE] = "Sprite Plane C flip done",
    119
    120	[PCU_THERMAL] = "PCU Thermal Event",
    121	[PCU_PCODE2DRIVER_MAILBOX] = "PCU pcode2driver mailbox event",
    122
    123	[FDI_RX_INTERRUPTS_TRANSCODER_A] = "FDI RX Interrupts Combined A",
    124	[AUDIO_CP_CHANGE_TRANSCODER_A] = "Audio CP Change Transcoder A",
    125	[AUDIO_CP_REQUEST_TRANSCODER_A] = "Audio CP Request Transcoder A",
    126	[FDI_RX_INTERRUPTS_TRANSCODER_B] = "FDI RX Interrupts Combined B",
    127	[AUDIO_CP_CHANGE_TRANSCODER_B] = "Audio CP Change Transcoder B",
    128	[AUDIO_CP_REQUEST_TRANSCODER_B] = "Audio CP Request Transcoder B",
    129	[FDI_RX_INTERRUPTS_TRANSCODER_C] = "FDI RX Interrupts Combined C",
    130	[AUDIO_CP_CHANGE_TRANSCODER_C] = "Audio CP Change Transcoder C",
    131	[AUDIO_CP_REQUEST_TRANSCODER_C] = "Audio CP Request Transcoder C",
    132	[ERR_AND_DBG] = "South Error and Debug Interrupts Combined",
    133	[GMBUS] = "Gmbus",
    134	[SDVO_B_HOTPLUG] = "SDVO B hotplug",
    135	[CRT_HOTPLUG] = "CRT Hotplug",
    136	[DP_B_HOTPLUG] = "DisplayPort/HDMI/DVI B Hotplug",
    137	[DP_C_HOTPLUG] = "DisplayPort/HDMI/DVI C Hotplug",
    138	[DP_D_HOTPLUG] = "DisplayPort/HDMI/DVI D Hotplug",
    139	[AUX_CHANNEL_B] = "AUX Channel B",
    140	[AUX_CHANNEL_C] = "AUX Channel C",
    141	[AUX_CHANNEL_D] = "AUX Channel D",
    142	[AUDIO_POWER_STATE_CHANGE_B] = "Audio Power State change Port B",
    143	[AUDIO_POWER_STATE_CHANGE_C] = "Audio Power State change Port C",
    144	[AUDIO_POWER_STATE_CHANGE_D] = "Audio Power State change Port D",
    145
    146	[INTEL_GVT_EVENT_RESERVED] = "RESERVED EVENTS!!!",
    147};
    148
    149static inline struct intel_gvt_irq_info *regbase_to_irq_info(
    150		struct intel_gvt *gvt,
    151		unsigned int reg)
    152{
    153	struct intel_gvt_irq *irq = &gvt->irq;
    154	int i;
    155
    156	for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) {
    157		if (i915_mmio_reg_offset(irq->info[i]->reg_base) == reg)
    158			return irq->info[i];
    159	}
    160
    161	return NULL;
    162}
    163
    164/**
    165 * intel_vgpu_reg_imr_handler - Generic IMR register emulation write handler
    166 * @vgpu: a vGPU
    167 * @reg: register offset written by guest
    168 * @p_data: register data written by guest
    169 * @bytes: register data length
    170 *
    171 * This function is used to emulate the generic IMR register bit change
    172 * behavior.
    173 *
    174 * Returns:
    175 * Zero on success, negative error code if failed.
    176 *
    177 */
    178int intel_vgpu_reg_imr_handler(struct intel_vgpu *vgpu,
    179	unsigned int reg, void *p_data, unsigned int bytes)
    180{
    181	struct intel_gvt *gvt = vgpu->gvt;
    182	const struct intel_gvt_irq_ops *ops = gvt->irq.ops;
    183	u32 imr = *(u32 *)p_data;
    184
    185	trace_write_ir(vgpu->id, "IMR", reg, imr, vgpu_vreg(vgpu, reg),
    186		       (vgpu_vreg(vgpu, reg) ^ imr));
    187
    188	vgpu_vreg(vgpu, reg) = imr;
    189
    190	ops->check_pending_irq(vgpu);
    191
    192	return 0;
    193}
    194
    195/**
    196 * intel_vgpu_reg_master_irq_handler - master IRQ write emulation handler
    197 * @vgpu: a vGPU
    198 * @reg: register offset written by guest
    199 * @p_data: register data written by guest
    200 * @bytes: register data length
    201 *
    202 * This function is used to emulate the master IRQ register on gen8+.
    203 *
    204 * Returns:
    205 * Zero on success, negative error code if failed.
    206 *
    207 */
    208int intel_vgpu_reg_master_irq_handler(struct intel_vgpu *vgpu,
    209	unsigned int reg, void *p_data, unsigned int bytes)
    210{
    211	struct intel_gvt *gvt = vgpu->gvt;
    212	const struct intel_gvt_irq_ops *ops = gvt->irq.ops;
    213	u32 ier = *(u32 *)p_data;
    214	u32 virtual_ier = vgpu_vreg(vgpu, reg);
    215
    216	trace_write_ir(vgpu->id, "MASTER_IRQ", reg, ier, virtual_ier,
    217		       (virtual_ier ^ ier));
    218
    219	/*
    220	 * GEN8_MASTER_IRQ is a special irq register,
    221	 * only bit 31 is allowed to be modified
    222	 * and treated as an IER bit.
    223	 */
    224	ier &= GEN8_MASTER_IRQ_CONTROL;
    225	virtual_ier &= GEN8_MASTER_IRQ_CONTROL;
    226	vgpu_vreg(vgpu, reg) &= ~GEN8_MASTER_IRQ_CONTROL;
    227	vgpu_vreg(vgpu, reg) |= ier;
    228
    229	ops->check_pending_irq(vgpu);
    230
    231	return 0;
    232}
    233
    234/**
    235 * intel_vgpu_reg_ier_handler - Generic IER write emulation handler
    236 * @vgpu: a vGPU
    237 * @reg: register offset written by guest
    238 * @p_data: register data written by guest
    239 * @bytes: register data length
    240 *
    241 * This function is used to emulate the generic IER register behavior.
    242 *
    243 * Returns:
    244 * Zero on success, negative error code if failed.
    245 *
    246 */
    247int intel_vgpu_reg_ier_handler(struct intel_vgpu *vgpu,
    248	unsigned int reg, void *p_data, unsigned int bytes)
    249{
    250	struct intel_gvt *gvt = vgpu->gvt;
    251	struct drm_i915_private *i915 = gvt->gt->i915;
    252	const struct intel_gvt_irq_ops *ops = gvt->irq.ops;
    253	struct intel_gvt_irq_info *info;
    254	u32 ier = *(u32 *)p_data;
    255
    256	trace_write_ir(vgpu->id, "IER", reg, ier, vgpu_vreg(vgpu, reg),
    257		       (vgpu_vreg(vgpu, reg) ^ ier));
    258
    259	vgpu_vreg(vgpu, reg) = ier;
    260
    261	info = regbase_to_irq_info(gvt, ier_to_regbase(reg));
    262	if (drm_WARN_ON(&i915->drm, !info))
    263		return -EINVAL;
    264
    265	if (info->has_upstream_irq)
    266		update_upstream_irq(vgpu, info);
    267
    268	ops->check_pending_irq(vgpu);
    269
    270	return 0;
    271}
    272
    273/**
    274 * intel_vgpu_reg_iir_handler - Generic IIR write emulation handler
    275 * @vgpu: a vGPU
    276 * @reg: register offset written by guest
    277 * @p_data: register data written by guest
    278 * @bytes: register data length
    279 *
    280 * This function is used to emulate the generic IIR register behavior.
    281 *
    282 * Returns:
    283 * Zero on success, negative error code if failed.
    284 *
    285 */
    286int intel_vgpu_reg_iir_handler(struct intel_vgpu *vgpu, unsigned int reg,
    287	void *p_data, unsigned int bytes)
    288{
    289	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
    290	struct intel_gvt_irq_info *info = regbase_to_irq_info(vgpu->gvt,
    291		iir_to_regbase(reg));
    292	u32 iir = *(u32 *)p_data;
    293
    294	trace_write_ir(vgpu->id, "IIR", reg, iir, vgpu_vreg(vgpu, reg),
    295		       (vgpu_vreg(vgpu, reg) ^ iir));
    296
    297	if (drm_WARN_ON(&i915->drm, !info))
    298		return -EINVAL;
    299
    300	vgpu_vreg(vgpu, reg) &= ~iir;
    301
    302	if (info->has_upstream_irq)
    303		update_upstream_irq(vgpu, info);
    304	return 0;
    305}
    306
    307static struct intel_gvt_irq_map gen8_irq_map[] = {
    308	{ INTEL_GVT_IRQ_INFO_MASTER, 0, INTEL_GVT_IRQ_INFO_GT0, 0xffff },
    309	{ INTEL_GVT_IRQ_INFO_MASTER, 1, INTEL_GVT_IRQ_INFO_GT0, 0xffff0000 },
    310	{ INTEL_GVT_IRQ_INFO_MASTER, 2, INTEL_GVT_IRQ_INFO_GT1, 0xffff },
    311	{ INTEL_GVT_IRQ_INFO_MASTER, 3, INTEL_GVT_IRQ_INFO_GT1, 0xffff0000 },
    312	{ INTEL_GVT_IRQ_INFO_MASTER, 4, INTEL_GVT_IRQ_INFO_GT2, 0xffff },
    313	{ INTEL_GVT_IRQ_INFO_MASTER, 6, INTEL_GVT_IRQ_INFO_GT3, 0xffff },
    314	{ INTEL_GVT_IRQ_INFO_MASTER, 16, INTEL_GVT_IRQ_INFO_DE_PIPE_A, ~0 },
    315	{ INTEL_GVT_IRQ_INFO_MASTER, 17, INTEL_GVT_IRQ_INFO_DE_PIPE_B, ~0 },
    316	{ INTEL_GVT_IRQ_INFO_MASTER, 18, INTEL_GVT_IRQ_INFO_DE_PIPE_C, ~0 },
    317	{ INTEL_GVT_IRQ_INFO_MASTER, 20, INTEL_GVT_IRQ_INFO_DE_PORT, ~0 },
    318	{ INTEL_GVT_IRQ_INFO_MASTER, 22, INTEL_GVT_IRQ_INFO_DE_MISC, ~0 },
    319	{ INTEL_GVT_IRQ_INFO_MASTER, 23, INTEL_GVT_IRQ_INFO_PCH, ~0 },
    320	{ INTEL_GVT_IRQ_INFO_MASTER, 30, INTEL_GVT_IRQ_INFO_PCU, ~0 },
    321	{ -1, -1, ~0 },
    322};
    323
    324static void update_upstream_irq(struct intel_vgpu *vgpu,
    325		struct intel_gvt_irq_info *info)
    326{
    327	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
    328	struct intel_gvt_irq *irq = &vgpu->gvt->irq;
    329	struct intel_gvt_irq_map *map = irq->irq_map;
    330	struct intel_gvt_irq_info *up_irq_info = NULL;
    331	u32 set_bits = 0;
    332	u32 clear_bits = 0;
    333	int bit;
    334	u32 val = vgpu_vreg(vgpu,
    335			regbase_to_iir(i915_mmio_reg_offset(info->reg_base)))
    336		& vgpu_vreg(vgpu,
    337			regbase_to_ier(i915_mmio_reg_offset(info->reg_base)));
    338
    339	if (!info->has_upstream_irq)
    340		return;
    341
    342	for (map = irq->irq_map; map->up_irq_bit != -1; map++) {
    343		if (info->group != map->down_irq_group)
    344			continue;
    345
    346		if (!up_irq_info)
    347			up_irq_info = irq->info[map->up_irq_group];
    348		else
    349			drm_WARN_ON(&i915->drm, up_irq_info !=
    350				    irq->info[map->up_irq_group]);
    351
    352		bit = map->up_irq_bit;
    353
    354		if (val & map->down_irq_bitmask)
    355			set_bits |= (1 << bit);
    356		else
    357			clear_bits |= (1 << bit);
    358	}
    359
    360	if (drm_WARN_ON(&i915->drm, !up_irq_info))
    361		return;
    362
    363	if (up_irq_info->group == INTEL_GVT_IRQ_INFO_MASTER) {
    364		u32 isr = i915_mmio_reg_offset(up_irq_info->reg_base);
    365
    366		vgpu_vreg(vgpu, isr) &= ~clear_bits;
    367		vgpu_vreg(vgpu, isr) |= set_bits;
    368	} else {
    369		u32 iir = regbase_to_iir(
    370			i915_mmio_reg_offset(up_irq_info->reg_base));
    371		u32 imr = regbase_to_imr(
    372			i915_mmio_reg_offset(up_irq_info->reg_base));
    373
    374		vgpu_vreg(vgpu, iir) |= (set_bits & ~vgpu_vreg(vgpu, imr));
    375	}
    376
    377	if (up_irq_info->has_upstream_irq)
    378		update_upstream_irq(vgpu, up_irq_info);
    379}
    380
    381static void init_irq_map(struct intel_gvt_irq *irq)
    382{
    383	struct intel_gvt_irq_map *map;
    384	struct intel_gvt_irq_info *up_info, *down_info;
    385	int up_bit;
    386
    387	for (map = irq->irq_map; map->up_irq_bit != -1; map++) {
    388		up_info = irq->info[map->up_irq_group];
    389		up_bit = map->up_irq_bit;
    390		down_info = irq->info[map->down_irq_group];
    391
    392		set_bit(up_bit, up_info->downstream_irq_bitmap);
    393		down_info->has_upstream_irq = true;
    394
    395		gvt_dbg_irq("[up] grp %d bit %d -> [down] grp %d bitmask %x\n",
    396			up_info->group, up_bit,
    397			down_info->group, map->down_irq_bitmask);
    398	}
    399}
    400
    401/* =======================vEvent injection===================== */
    402
    403#define MSI_CAP_CONTROL(offset) (offset + 2)
    404#define MSI_CAP_ADDRESS(offset) (offset + 4)
    405#define MSI_CAP_DATA(offset) (offset + 8)
    406#define MSI_CAP_EN 0x1
    407
    408static int inject_virtual_interrupt(struct intel_vgpu *vgpu)
    409{
    410	unsigned long offset = vgpu->gvt->device_info.msi_cap_offset;
    411	u16 control, data;
    412	u32 addr;
    413
    414	control = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_CONTROL(offset));
    415	addr = *(u32 *)(vgpu_cfg_space(vgpu) + MSI_CAP_ADDRESS(offset));
    416	data = *(u16 *)(vgpu_cfg_space(vgpu) + MSI_CAP_DATA(offset));
    417
    418	/* Do not generate MSI if MSIEN is disabled */
    419	if (!(control & MSI_CAP_EN))
    420		return 0;
    421
    422	if (WARN(control & GENMASK(15, 1), "only support one MSI format\n"))
    423		return -EINVAL;
    424
    425	trace_inject_msi(vgpu->id, addr, data);
    426
    427	/*
    428	 * When guest is powered off, msi_trigger is set to NULL, but vgpu's
    429	 * config and mmio register isn't restored to default during guest
    430	 * poweroff. If this vgpu is still used in next vm, this vgpu's pipe
    431	 * may be enabled, then once this vgpu is active, it will get inject
    432	 * vblank interrupt request. But msi_trigger is null until msi is
    433	 * enabled by guest. so if msi_trigger is null, success is still
    434	 * returned and don't inject interrupt into guest.
    435	 */
    436	if (!vgpu->attached)
    437		return -ESRCH;
    438	if (vgpu->msi_trigger && eventfd_signal(vgpu->msi_trigger, 1) != 1)
    439		return -EFAULT;
    440	return 0;
    441}
    442
    443static void propagate_event(struct intel_gvt_irq *irq,
    444	enum intel_gvt_event_type event, struct intel_vgpu *vgpu)
    445{
    446	struct intel_gvt_irq_info *info;
    447	unsigned int reg_base;
    448	int bit;
    449
    450	info = get_irq_info(irq, event);
    451	if (WARN_ON(!info))
    452		return;
    453
    454	reg_base = i915_mmio_reg_offset(info->reg_base);
    455	bit = irq->events[event].bit;
    456
    457	if (!test_bit(bit, (void *)&vgpu_vreg(vgpu,
    458					regbase_to_imr(reg_base)))) {
    459		trace_propagate_event(vgpu->id, irq_name[event], bit);
    460		set_bit(bit, (void *)&vgpu_vreg(vgpu,
    461					regbase_to_iir(reg_base)));
    462	}
    463}
    464
    465/* =======================vEvent Handlers===================== */
    466static void handle_default_event_virt(struct intel_gvt_irq *irq,
    467	enum intel_gvt_event_type event, struct intel_vgpu *vgpu)
    468{
    469	if (!vgpu->irq.irq_warn_once[event]) {
    470		gvt_dbg_core("vgpu%d: IRQ receive event %d (%s)\n",
    471			vgpu->id, event, irq_name[event]);
    472		vgpu->irq.irq_warn_once[event] = true;
    473	}
    474	propagate_event(irq, event, vgpu);
    475}
    476
    477/* =====================GEN specific logic======================= */
    478/* GEN8 interrupt routines. */
    479
    480#define DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(regname, regbase) \
    481static struct intel_gvt_irq_info gen8_##regname##_info = { \
    482	.name = #regname"-IRQ", \
    483	.reg_base = (regbase), \
    484	.bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] = \
    485		INTEL_GVT_EVENT_RESERVED}, \
    486}
    487
    488DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt0, GEN8_GT_ISR(0));
    489DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt1, GEN8_GT_ISR(1));
    490DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt2, GEN8_GT_ISR(2));
    491DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(gt3, GEN8_GT_ISR(3));
    492DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_a, GEN8_DE_PIPE_ISR(PIPE_A));
    493DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_b, GEN8_DE_PIPE_ISR(PIPE_B));
    494DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_pipe_c, GEN8_DE_PIPE_ISR(PIPE_C));
    495DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_port, GEN8_DE_PORT_ISR);
    496DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(de_misc, GEN8_DE_MISC_ISR);
    497DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(pcu, GEN8_PCU_ISR);
    498DEFINE_GVT_GEN8_INTEL_GVT_IRQ_INFO(master, GEN8_MASTER_IRQ);
    499
    500static struct intel_gvt_irq_info gvt_base_pch_info = {
    501	.name = "PCH-IRQ",
    502	.reg_base = SDEISR,
    503	.bit_to_event = {[0 ... INTEL_GVT_IRQ_BITWIDTH-1] =
    504		INTEL_GVT_EVENT_RESERVED},
    505};
    506
    507static void gen8_check_pending_irq(struct intel_vgpu *vgpu)
    508{
    509	struct intel_gvt_irq *irq = &vgpu->gvt->irq;
    510	int i;
    511
    512	if (!(vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ)) &
    513				GEN8_MASTER_IRQ_CONTROL))
    514		return;
    515
    516	for_each_set_bit(i, irq->irq_info_bitmap, INTEL_GVT_IRQ_INFO_MAX) {
    517		struct intel_gvt_irq_info *info = irq->info[i];
    518		u32 reg_base;
    519
    520		if (!info->has_upstream_irq)
    521			continue;
    522
    523		reg_base = i915_mmio_reg_offset(info->reg_base);
    524		if ((vgpu_vreg(vgpu, regbase_to_iir(reg_base))
    525				& vgpu_vreg(vgpu, regbase_to_ier(reg_base))))
    526			update_upstream_irq(vgpu, info);
    527	}
    528
    529	if (vgpu_vreg(vgpu, i915_mmio_reg_offset(GEN8_MASTER_IRQ))
    530			& ~GEN8_MASTER_IRQ_CONTROL)
    531		inject_virtual_interrupt(vgpu);
    532}
    533
    534static void gen8_init_irq(
    535		struct intel_gvt_irq *irq)
    536{
    537	struct intel_gvt *gvt = irq_to_gvt(irq);
    538
    539#define SET_BIT_INFO(s, b, e, i)		\
    540	do {					\
    541		s->events[e].bit = b;		\
    542		s->events[e].info = s->info[i];	\
    543		s->info[i]->bit_to_event[b] = e;\
    544	} while (0)
    545
    546#define SET_IRQ_GROUP(s, g, i) \
    547	do { \
    548		s->info[g] = i; \
    549		(i)->group = g; \
    550		set_bit(g, s->irq_info_bitmap); \
    551	} while (0)
    552
    553	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_MASTER, &gen8_master_info);
    554	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT0, &gen8_gt0_info);
    555	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT1, &gen8_gt1_info);
    556	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT2, &gen8_gt2_info);
    557	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_GT3, &gen8_gt3_info);
    558	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_A, &gen8_de_pipe_a_info);
    559	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_B, &gen8_de_pipe_b_info);
    560	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PIPE_C, &gen8_de_pipe_c_info);
    561	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_PORT, &gen8_de_port_info);
    562	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_DE_MISC, &gen8_de_misc_info);
    563	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCU, &gen8_pcu_info);
    564	SET_IRQ_GROUP(irq, INTEL_GVT_IRQ_INFO_PCH, &gvt_base_pch_info);
    565
    566	/* GEN8 level 2 interrupts. */
    567
    568	/* GEN8 interrupt GT0 events */
    569	SET_BIT_INFO(irq, 0, RCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0);
    570	SET_BIT_INFO(irq, 4, RCS_PIPE_CONTROL, INTEL_GVT_IRQ_INFO_GT0);
    571	SET_BIT_INFO(irq, 8, RCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0);
    572
    573	SET_BIT_INFO(irq, 16, BCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT0);
    574	SET_BIT_INFO(irq, 20, BCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT0);
    575	SET_BIT_INFO(irq, 24, BCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT0);
    576
    577	/* GEN8 interrupt GT1 events */
    578	SET_BIT_INFO(irq, 0, VCS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT1);
    579	SET_BIT_INFO(irq, 4, VCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT1);
    580	SET_BIT_INFO(irq, 8, VCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT1);
    581
    582	if (HAS_ENGINE(gvt->gt, VCS1)) {
    583		SET_BIT_INFO(irq, 16, VCS2_MI_USER_INTERRUPT,
    584			INTEL_GVT_IRQ_INFO_GT1);
    585		SET_BIT_INFO(irq, 20, VCS2_MI_FLUSH_DW,
    586			INTEL_GVT_IRQ_INFO_GT1);
    587		SET_BIT_INFO(irq, 24, VCS2_AS_CONTEXT_SWITCH,
    588			INTEL_GVT_IRQ_INFO_GT1);
    589	}
    590
    591	/* GEN8 interrupt GT3 events */
    592	SET_BIT_INFO(irq, 0, VECS_MI_USER_INTERRUPT, INTEL_GVT_IRQ_INFO_GT3);
    593	SET_BIT_INFO(irq, 4, VECS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT3);
    594	SET_BIT_INFO(irq, 8, VECS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT3);
    595
    596	SET_BIT_INFO(irq, 0, PIPE_A_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
    597	SET_BIT_INFO(irq, 0, PIPE_B_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
    598	SET_BIT_INFO(irq, 0, PIPE_C_VBLANK, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
    599
    600	/* GEN8 interrupt DE PORT events */
    601	SET_BIT_INFO(irq, 0, AUX_CHANNEL_A, INTEL_GVT_IRQ_INFO_DE_PORT);
    602	SET_BIT_INFO(irq, 3, DP_A_HOTPLUG, INTEL_GVT_IRQ_INFO_DE_PORT);
    603
    604	/* GEN8 interrupt DE MISC events */
    605	SET_BIT_INFO(irq, 0, GSE, INTEL_GVT_IRQ_INFO_DE_MISC);
    606
    607	/* PCH events */
    608	SET_BIT_INFO(irq, 17, GMBUS, INTEL_GVT_IRQ_INFO_PCH);
    609	SET_BIT_INFO(irq, 19, CRT_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
    610	SET_BIT_INFO(irq, 21, DP_B_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
    611	SET_BIT_INFO(irq, 22, DP_C_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
    612	SET_BIT_INFO(irq, 23, DP_D_HOTPLUG, INTEL_GVT_IRQ_INFO_PCH);
    613
    614	if (IS_BROADWELL(gvt->gt->i915)) {
    615		SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_PCH);
    616		SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_PCH);
    617		SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_PCH);
    618
    619		SET_BIT_INFO(irq, 4, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
    620		SET_BIT_INFO(irq, 5, SPRITE_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
    621
    622		SET_BIT_INFO(irq, 4, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
    623		SET_BIT_INFO(irq, 5, SPRITE_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
    624
    625		SET_BIT_INFO(irq, 4, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
    626		SET_BIT_INFO(irq, 5, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
    627	} else if (GRAPHICS_VER(gvt->gt->i915) >= 9) {
    628		SET_BIT_INFO(irq, 25, AUX_CHANNEL_B, INTEL_GVT_IRQ_INFO_DE_PORT);
    629		SET_BIT_INFO(irq, 26, AUX_CHANNEL_C, INTEL_GVT_IRQ_INFO_DE_PORT);
    630		SET_BIT_INFO(irq, 27, AUX_CHANNEL_D, INTEL_GVT_IRQ_INFO_DE_PORT);
    631
    632		SET_BIT_INFO(irq, 3, PRIMARY_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
    633		SET_BIT_INFO(irq, 3, PRIMARY_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
    634		SET_BIT_INFO(irq, 3, PRIMARY_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
    635
    636		SET_BIT_INFO(irq, 4, SPRITE_A_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_A);
    637		SET_BIT_INFO(irq, 4, SPRITE_B_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_B);
    638		SET_BIT_INFO(irq, 4, SPRITE_C_FLIP_DONE, INTEL_GVT_IRQ_INFO_DE_PIPE_C);
    639	}
    640
    641	/* GEN8 interrupt PCU events */
    642	SET_BIT_INFO(irq, 24, PCU_THERMAL, INTEL_GVT_IRQ_INFO_PCU);
    643	SET_BIT_INFO(irq, 25, PCU_PCODE2DRIVER_MAILBOX, INTEL_GVT_IRQ_INFO_PCU);
    644}
    645
    646static const struct intel_gvt_irq_ops gen8_irq_ops = {
    647	.init_irq = gen8_init_irq,
    648	.check_pending_irq = gen8_check_pending_irq,
    649};
    650
    651/**
    652 * intel_vgpu_trigger_virtual_event - Trigger a virtual event for a vGPU
    653 * @vgpu: a vGPU
    654 * @event: interrupt event
    655 *
    656 * This function is used to trigger a virtual interrupt event for vGPU.
    657 * The caller provides the event to be triggered, the framework itself
    658 * will emulate the IRQ register bit change.
    659 *
    660 */
    661void intel_vgpu_trigger_virtual_event(struct intel_vgpu *vgpu,
    662	enum intel_gvt_event_type event)
    663{
    664	struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
    665	struct intel_gvt *gvt = vgpu->gvt;
    666	struct intel_gvt_irq *irq = &gvt->irq;
    667	gvt_event_virt_handler_t handler;
    668	const struct intel_gvt_irq_ops *ops = gvt->irq.ops;
    669
    670	handler = get_event_virt_handler(irq, event);
    671	drm_WARN_ON(&i915->drm, !handler);
    672
    673	handler(irq, event, vgpu);
    674
    675	ops->check_pending_irq(vgpu);
    676}
    677
    678static void init_events(
    679	struct intel_gvt_irq *irq)
    680{
    681	int i;
    682
    683	for (i = 0; i < INTEL_GVT_EVENT_MAX; i++) {
    684		irq->events[i].info = NULL;
    685		irq->events[i].v_handler = handle_default_event_virt;
    686	}
    687}
    688
    689/**
    690 * intel_gvt_init_irq - initialize GVT-g IRQ emulation subsystem
    691 * @gvt: a GVT device
    692 *
    693 * This function is called at driver loading stage, to initialize the GVT-g IRQ
    694 * emulation subsystem.
    695 *
    696 * Returns:
    697 * Zero on success, negative error code if failed.
    698 */
    699int intel_gvt_init_irq(struct intel_gvt *gvt)
    700{
    701	struct intel_gvt_irq *irq = &gvt->irq;
    702
    703	gvt_dbg_core("init irq framework\n");
    704
    705	irq->ops = &gen8_irq_ops;
    706	irq->irq_map = gen8_irq_map;
    707
    708	/* common event initialization */
    709	init_events(irq);
    710
    711	/* gen specific initialization */
    712	irq->ops->init_irq(irq);
    713
    714	init_irq_map(irq);
    715
    716	return 0;
    717}