cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

interrupt.c (92170B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * handling kvm guest interrupts
      4 *
      5 * Copyright IBM Corp. 2008, 2020
      6 *
      7 *    Author(s): Carsten Otte <cotte@de.ibm.com>
      8 */
      9
     10#define KMSG_COMPONENT "kvm-s390"
     11#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
     12
     13#include <linux/interrupt.h>
     14#include <linux/kvm_host.h>
     15#include <linux/hrtimer.h>
     16#include <linux/mmu_context.h>
     17#include <linux/nospec.h>
     18#include <linux/signal.h>
     19#include <linux/slab.h>
     20#include <linux/bitmap.h>
     21#include <linux/vmalloc.h>
     22#include <asm/asm-offsets.h>
     23#include <asm/dis.h>
     24#include <linux/uaccess.h>
     25#include <asm/sclp.h>
     26#include <asm/isc.h>
     27#include <asm/gmap.h>
     28#include <asm/switch_to.h>
     29#include <asm/nmi.h>
     30#include <asm/airq.h>
     31#include "kvm-s390.h"
     32#include "gaccess.h"
     33#include "trace-s390.h"
     34
     35#define PFAULT_INIT 0x0600
     36#define PFAULT_DONE 0x0680
     37#define VIRTIO_PARAM 0x0d00
     38
     39static struct kvm_s390_gib *gib;
     40
     41/* handle external calls via sigp interpretation facility */
     42static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
     43{
     44	int c, scn;
     45
     46	if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
     47		return 0;
     48
     49	BUG_ON(!kvm_s390_use_sca_entries());
     50	read_lock(&vcpu->kvm->arch.sca_lock);
     51	if (vcpu->kvm->arch.use_esca) {
     52		struct esca_block *sca = vcpu->kvm->arch.sca;
     53		union esca_sigp_ctrl sigp_ctrl =
     54			sca->cpu[vcpu->vcpu_id].sigp_ctrl;
     55
     56		c = sigp_ctrl.c;
     57		scn = sigp_ctrl.scn;
     58	} else {
     59		struct bsca_block *sca = vcpu->kvm->arch.sca;
     60		union bsca_sigp_ctrl sigp_ctrl =
     61			sca->cpu[vcpu->vcpu_id].sigp_ctrl;
     62
     63		c = sigp_ctrl.c;
     64		scn = sigp_ctrl.scn;
     65	}
     66	read_unlock(&vcpu->kvm->arch.sca_lock);
     67
     68	if (src_id)
     69		*src_id = scn;
     70
     71	return c;
     72}
     73
     74static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
     75{
     76	int expect, rc;
     77
     78	BUG_ON(!kvm_s390_use_sca_entries());
     79	read_lock(&vcpu->kvm->arch.sca_lock);
     80	if (vcpu->kvm->arch.use_esca) {
     81		struct esca_block *sca = vcpu->kvm->arch.sca;
     82		union esca_sigp_ctrl *sigp_ctrl =
     83			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
     84		union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
     85
     86		new_val.scn = src_id;
     87		new_val.c = 1;
     88		old_val.c = 0;
     89
     90		expect = old_val.value;
     91		rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
     92	} else {
     93		struct bsca_block *sca = vcpu->kvm->arch.sca;
     94		union bsca_sigp_ctrl *sigp_ctrl =
     95			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
     96		union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl;
     97
     98		new_val.scn = src_id;
     99		new_val.c = 1;
    100		old_val.c = 0;
    101
    102		expect = old_val.value;
    103		rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value);
    104	}
    105	read_unlock(&vcpu->kvm->arch.sca_lock);
    106
    107	if (rc != expect) {
    108		/* another external call is pending */
    109		return -EBUSY;
    110	}
    111	kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
    112	return 0;
    113}
    114
    115static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
    116{
    117	int rc, expect;
    118
    119	if (!kvm_s390_use_sca_entries())
    120		return;
    121	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
    122	read_lock(&vcpu->kvm->arch.sca_lock);
    123	if (vcpu->kvm->arch.use_esca) {
    124		struct esca_block *sca = vcpu->kvm->arch.sca;
    125		union esca_sigp_ctrl *sigp_ctrl =
    126			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
    127		union esca_sigp_ctrl old = *sigp_ctrl;
    128
    129		expect = old.value;
    130		rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
    131	} else {
    132		struct bsca_block *sca = vcpu->kvm->arch.sca;
    133		union bsca_sigp_ctrl *sigp_ctrl =
    134			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
    135		union bsca_sigp_ctrl old = *sigp_ctrl;
    136
    137		expect = old.value;
    138		rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
    139	}
    140	read_unlock(&vcpu->kvm->arch.sca_lock);
    141	WARN_ON(rc != expect); /* cannot clear? */
    142}
    143
    144int psw_extint_disabled(struct kvm_vcpu *vcpu)
    145{
    146	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
    147}
    148
    149static int psw_ioint_disabled(struct kvm_vcpu *vcpu)
    150{
    151	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO);
    152}
    153
    154static int psw_mchk_disabled(struct kvm_vcpu *vcpu)
    155{
    156	return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK);
    157}
    158
    159static int psw_interrupts_disabled(struct kvm_vcpu *vcpu)
    160{
    161	return psw_extint_disabled(vcpu) &&
    162	       psw_ioint_disabled(vcpu) &&
    163	       psw_mchk_disabled(vcpu);
    164}
    165
    166static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
    167{
    168	if (psw_extint_disabled(vcpu) ||
    169	    !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
    170		return 0;
    171	if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu))
    172		/* No timer interrupts when single stepping */
    173		return 0;
    174	return 1;
    175}
    176
    177static int ckc_irq_pending(struct kvm_vcpu *vcpu)
    178{
    179	const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
    180	const u64 ckc = vcpu->arch.sie_block->ckc;
    181
    182	if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
    183		if ((s64)ckc >= (s64)now)
    184			return 0;
    185	} else if (ckc >= now) {
    186		return 0;
    187	}
    188	return ckc_interrupts_enabled(vcpu);
    189}
    190
    191static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
    192{
    193	return !psw_extint_disabled(vcpu) &&
    194	       (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK);
    195}
    196
    197static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
    198{
    199	if (!cpu_timer_interrupts_enabled(vcpu))
    200		return 0;
    201	return kvm_s390_get_cpu_timer(vcpu) >> 63;
    202}
    203
    204static uint64_t isc_to_isc_bits(int isc)
    205{
    206	return (0x80 >> isc) << 24;
    207}
    208
    209static inline u32 isc_to_int_word(u8 isc)
    210{
    211	return ((u32)isc << 27) | 0x80000000;
    212}
    213
    214static inline u8 int_word_to_isc(u32 int_word)
    215{
    216	return (int_word & 0x38000000) >> 27;
    217}
    218
    219/*
    220 * To use atomic bitmap functions, we have to provide a bitmap address
    221 * that is u64 aligned. However, the ipm might be u32 aligned.
    222 * Therefore, we logically start the bitmap at the very beginning of the
    223 * struct and fixup the bit number.
    224 */
    225#define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
    226
    227/**
    228 * gisa_set_iam - change the GISA interruption alert mask
    229 *
    230 * @gisa: gisa to operate on
    231 * @iam: new IAM value to use
    232 *
    233 * Change the IAM atomically with the next alert address and the IPM
    234 * of the GISA if the GISA is not part of the GIB alert list. All three
    235 * fields are located in the first long word of the GISA.
    236 *
    237 * Returns: 0 on success
    238 *          -EBUSY in case the gisa is part of the alert list
    239 */
    240static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
    241{
    242	u64 word, _word;
    243
    244	do {
    245		word = READ_ONCE(gisa->u64.word[0]);
    246		if ((u64)gisa != word >> 32)
    247			return -EBUSY;
    248		_word = (word & ~0xffUL) | iam;
    249	} while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
    250
    251	return 0;
    252}
    253
    254/**
    255 * gisa_clear_ipm - clear the GISA interruption pending mask
    256 *
    257 * @gisa: gisa to operate on
    258 *
    259 * Clear the IPM atomically with the next alert address and the IAM
    260 * of the GISA unconditionally. All three fields are located in the
    261 * first long word of the GISA.
    262 */
    263static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
    264{
    265	u64 word, _word;
    266
    267	do {
    268		word = READ_ONCE(gisa->u64.word[0]);
    269		_word = word & ~(0xffUL << 24);
    270	} while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
    271}
    272
    273/**
    274 * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM
    275 *
    276 * @gi: gisa interrupt struct to work on
    277 *
    278 * Atomically restores the interruption alert mask if none of the
    279 * relevant ISCs are pending and return the IPM.
    280 *
    281 * Returns: the relevant pending ISCs
    282 */
    283static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
    284{
    285	u8 pending_mask, alert_mask;
    286	u64 word, _word;
    287
    288	do {
    289		word = READ_ONCE(gi->origin->u64.word[0]);
    290		alert_mask = READ_ONCE(gi->alert.mask);
    291		pending_mask = (u8)(word >> 24) & alert_mask;
    292		if (pending_mask)
    293			return pending_mask;
    294		_word = (word & ~0xffUL) | alert_mask;
    295	} while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
    296
    297	return 0;
    298}
    299
    300static inline int gisa_in_alert_list(struct kvm_s390_gisa *gisa)
    301{
    302	return READ_ONCE(gisa->next_alert) != (u32)(u64)gisa;
    303}
    304
    305static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
    306{
    307	set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
    308}
    309
    310static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa)
    311{
    312	return READ_ONCE(gisa->ipm);
    313}
    314
    315static inline void gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
    316{
    317	clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
    318}
    319
    320static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
    321{
    322	return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
    323}
    324
    325static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
    326{
    327	unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs |
    328				vcpu->arch.local_int.pending_irqs;
    329
    330	pending &= ~vcpu->kvm->arch.float_int.masked_irqs;
    331	return pending;
    332}
    333
    334static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
    335{
    336	struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
    337	unsigned long pending_mask;
    338
    339	pending_mask = pending_irqs_no_gisa(vcpu);
    340	if (gi->origin)
    341		pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7;
    342	return pending_mask;
    343}
    344
    345static inline int isc_to_irq_type(unsigned long isc)
    346{
    347	return IRQ_PEND_IO_ISC_0 - isc;
    348}
    349
    350static inline int irq_type_to_isc(unsigned long irq_type)
    351{
    352	return IRQ_PEND_IO_ISC_0 - irq_type;
    353}
    354
    355static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
    356				   unsigned long active_mask)
    357{
    358	int i;
    359
    360	for (i = 0; i <= MAX_ISC; i++)
    361		if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
    362			active_mask &= ~(1UL << (isc_to_irq_type(i)));
    363
    364	return active_mask;
    365}
    366
    367static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
    368{
    369	unsigned long active_mask;
    370
    371	active_mask = pending_irqs(vcpu);
    372	if (!active_mask)
    373		return 0;
    374
    375	if (psw_extint_disabled(vcpu))
    376		active_mask &= ~IRQ_PEND_EXT_MASK;
    377	if (psw_ioint_disabled(vcpu))
    378		active_mask &= ~IRQ_PEND_IO_MASK;
    379	else
    380		active_mask = disable_iscs(vcpu, active_mask);
    381	if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
    382		__clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
    383	if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK))
    384		__clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask);
    385	if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK))
    386		__clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
    387	if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK))
    388		__clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
    389	if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) {
    390		__clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
    391		__clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask);
    392	}
    393	if (psw_mchk_disabled(vcpu))
    394		active_mask &= ~IRQ_PEND_MCHK_MASK;
    395	/* PV guest cpus can have a single interruption injected at a time. */
    396	if (kvm_s390_pv_cpu_get_handle(vcpu) &&
    397	    vcpu->arch.sie_block->iictl != IICTL_CODE_NONE)
    398		active_mask &= ~(IRQ_PEND_EXT_II_MASK |
    399				 IRQ_PEND_IO_MASK |
    400				 IRQ_PEND_MCHK_MASK);
    401	/*
    402	 * Check both floating and local interrupt's cr14 because
    403	 * bit IRQ_PEND_MCHK_REP could be set in both cases.
    404	 */
    405	if (!(vcpu->arch.sie_block->gcr[14] &
    406	   (vcpu->kvm->arch.float_int.mchk.cr14 |
    407	   vcpu->arch.local_int.irq.mchk.cr14)))
    408		__clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
    409
    410	/*
    411	 * STOP irqs will never be actively delivered. They are triggered via
    412	 * intercept requests and cleared when the stop intercept is performed.
    413	 */
    414	__clear_bit(IRQ_PEND_SIGP_STOP, &active_mask);
    415
    416	return active_mask;
    417}
    418
    419static void __set_cpu_idle(struct kvm_vcpu *vcpu)
    420{
    421	kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
    422	set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
    423}
    424
    425static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
    426{
    427	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
    428	clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask);
    429}
    430
    431static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
    432{
    433	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
    434				      CPUSTAT_STOP_INT);
    435	vcpu->arch.sie_block->lctl = 0x0000;
    436	vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
    437
    438	if (guestdbg_enabled(vcpu)) {
    439		vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 |
    440					       LCTL_CR10 | LCTL_CR11);
    441		vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT);
    442	}
    443}
    444
    445static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
    446{
    447	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
    448		return;
    449	if (psw_ioint_disabled(vcpu))
    450		kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
    451	else
    452		vcpu->arch.sie_block->lctl |= LCTL_CR6;
    453}
    454
    455static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
    456{
    457	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK))
    458		return;
    459	if (psw_extint_disabled(vcpu))
    460		kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
    461	else
    462		vcpu->arch.sie_block->lctl |= LCTL_CR0;
    463}
    464
    465static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
    466{
    467	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK))
    468		return;
    469	if (psw_mchk_disabled(vcpu))
    470		vcpu->arch.sie_block->ictl |= ICTL_LPSW;
    471	else
    472		vcpu->arch.sie_block->lctl |= LCTL_CR14;
    473}
    474
    475static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
    476{
    477	if (kvm_s390_is_stop_irq_pending(vcpu))
    478		kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
    479}
    480
    481/* Set interception request for non-deliverable interrupts */
    482static void set_intercept_indicators(struct kvm_vcpu *vcpu)
    483{
    484	set_intercept_indicators_io(vcpu);
    485	set_intercept_indicators_ext(vcpu);
    486	set_intercept_indicators_mchk(vcpu);
    487	set_intercept_indicators_stop(vcpu);
    488}
    489
    490static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu)
    491{
    492	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    493	int rc = 0;
    494
    495	vcpu->stat.deliver_cputm++;
    496	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
    497					 0, 0);
    498	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
    499		vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
    500		vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER;
    501	} else {
    502		rc  = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER,
    503				   (u16 *)__LC_EXT_INT_CODE);
    504		rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
    505		rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
    506				     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    507		rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
    508				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    509	}
    510	clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
    511	return rc ? -EFAULT : 0;
    512}
    513
    514static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu)
    515{
    516	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    517	int rc = 0;
    518
    519	vcpu->stat.deliver_ckc++;
    520	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
    521					 0, 0);
    522	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
    523		vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
    524		vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP;
    525	} else {
    526		rc  = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP,
    527				   (u16 __user *)__LC_EXT_INT_CODE);
    528		rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
    529		rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
    530				     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    531		rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
    532				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    533	}
    534	clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
    535	return rc ? -EFAULT : 0;
    536}
    537
    538static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
    539{
    540	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    541	struct kvm_s390_ext_info ext;
    542	int rc;
    543
    544	spin_lock(&li->lock);
    545	ext = li->irq.ext;
    546	clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
    547	li->irq.ext.ext_params2 = 0;
    548	spin_unlock(&li->lock);
    549
    550	VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx",
    551		   ext.ext_params2);
    552	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
    553					 KVM_S390_INT_PFAULT_INIT,
    554					 0, ext.ext_params2);
    555
    556	rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE);
    557	rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR);
    558	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
    559			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    560	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
    561			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    562	rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2);
    563	return rc ? -EFAULT : 0;
    564}
    565
    566static int __write_machine_check(struct kvm_vcpu *vcpu,
    567				 struct kvm_s390_mchk_info *mchk)
    568{
    569	unsigned long ext_sa_addr;
    570	unsigned long lc;
    571	freg_t fprs[NUM_FPRS];
    572	union mci mci;
    573	int rc;
    574
    575	/*
    576	 * All other possible payload for a machine check (e.g. the register
    577	 * contents in the save area) will be handled by the ultravisor, as
    578	 * the hypervisor does not not have the needed information for
    579	 * protected guests.
    580	 */
    581	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
    582		vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK;
    583		vcpu->arch.sie_block->mcic = mchk->mcic;
    584		vcpu->arch.sie_block->faddr = mchk->failing_storage_address;
    585		vcpu->arch.sie_block->edc = mchk->ext_damage_code;
    586		return 0;
    587	}
    588
    589	mci.val = mchk->mcic;
    590	/* take care of lazy register loading */
    591	save_fpu_regs();
    592	save_access_regs(vcpu->run->s.regs.acrs);
    593	if (MACHINE_HAS_GS && vcpu->arch.gs_enabled)
    594		save_gs_cb(current->thread.gs_cb);
    595
    596	/* Extended save area */
    597	rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr,
    598			   sizeof(unsigned long));
    599	/* Only bits 0 through 63-LC are used for address formation */
    600	lc = ext_sa_addr & MCESA_LC_MASK;
    601	if (test_kvm_facility(vcpu->kvm, 133)) {
    602		switch (lc) {
    603		case 0:
    604		case 10:
    605			ext_sa_addr &= ~0x3ffUL;
    606			break;
    607		case 11:
    608			ext_sa_addr &= ~0x7ffUL;
    609			break;
    610		case 12:
    611			ext_sa_addr &= ~0xfffUL;
    612			break;
    613		default:
    614			ext_sa_addr = 0;
    615			break;
    616		}
    617	} else {
    618		ext_sa_addr &= ~0x3ffUL;
    619	}
    620
    621	if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) {
    622		if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs,
    623				    512))
    624			mci.vr = 0;
    625	} else {
    626		mci.vr = 0;
    627	}
    628	if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133)
    629	    && (lc == 11 || lc == 12)) {
    630		if (write_guest_abs(vcpu, ext_sa_addr + 1024,
    631				    &vcpu->run->s.regs.gscb, 32))
    632			mci.gs = 0;
    633	} else {
    634		mci.gs = 0;
    635	}
    636
    637	/* General interruption information */
    638	rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID);
    639	rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
    640			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    641	rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
    642			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    643	rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE);
    644
    645	/* Register-save areas */
    646	if (MACHINE_HAS_VX) {
    647		convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
    648		rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128);
    649	} else {
    650		rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA,
    651				     vcpu->run->s.regs.fprs, 128);
    652	}
    653	rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA,
    654			     vcpu->run->s.regs.gprs, 128);
    655	rc |= put_guest_lc(vcpu, current->thread.fpu.fpc,
    656			   (u32 __user *) __LC_FP_CREG_SAVE_AREA);
    657	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr,
    658			   (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA);
    659	rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu),
    660			   (u64 __user *) __LC_CPU_TIMER_SAVE_AREA);
    661	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8,
    662			   (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA);
    663	rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA,
    664			     &vcpu->run->s.regs.acrs, 64);
    665	rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA,
    666			     &vcpu->arch.sie_block->gcr, 128);
    667
    668	/* Extended interruption information */
    669	rc |= put_guest_lc(vcpu, mchk->ext_damage_code,
    670			   (u32 __user *) __LC_EXT_DAMAGE_CODE);
    671	rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
    672			   (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
    673	rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout,
    674			     sizeof(mchk->fixed_logout));
    675	return rc ? -EFAULT : 0;
    676}
    677
    678static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
    679{
    680	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
    681	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    682	struct kvm_s390_mchk_info mchk = {};
    683	int deliver = 0;
    684	int rc = 0;
    685
    686	spin_lock(&fi->lock);
    687	spin_lock(&li->lock);
    688	if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
    689	    test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
    690		/*
    691		 * If there was an exigent machine check pending, then any
    692		 * repressible machine checks that might have been pending
    693		 * are indicated along with it, so always clear bits for
    694		 * repressible and exigent interrupts
    695		 */
    696		mchk = li->irq.mchk;
    697		clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
    698		clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
    699		memset(&li->irq.mchk, 0, sizeof(mchk));
    700		deliver = 1;
    701	}
    702	/*
    703	 * We indicate floating repressible conditions along with
    704	 * other pending conditions. Channel Report Pending and Channel
    705	 * Subsystem damage are the only two and and are indicated by
    706	 * bits in mcic and masked in cr14.
    707	 */
    708	if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
    709		mchk.mcic |= fi->mchk.mcic;
    710		mchk.cr14 |= fi->mchk.cr14;
    711		memset(&fi->mchk, 0, sizeof(mchk));
    712		deliver = 1;
    713	}
    714	spin_unlock(&li->lock);
    715	spin_unlock(&fi->lock);
    716
    717	if (deliver) {
    718		VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx",
    719			   mchk.mcic);
    720		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
    721						 KVM_S390_MCHK,
    722						 mchk.cr14, mchk.mcic);
    723		vcpu->stat.deliver_machine_check++;
    724		rc = __write_machine_check(vcpu, &mchk);
    725	}
    726	return rc;
    727}
    728
    729static int __must_check __deliver_restart(struct kvm_vcpu *vcpu)
    730{
    731	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    732	int rc = 0;
    733
    734	VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart");
    735	vcpu->stat.deliver_restart_signal++;
    736	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
    737
    738	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
    739		vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART;
    740	} else {
    741		rc  = write_guest_lc(vcpu,
    742				     offsetof(struct lowcore, restart_old_psw),
    743				     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    744		rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw),
    745				    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    746	}
    747	clear_bit(IRQ_PEND_RESTART, &li->pending_irqs);
    748	return rc ? -EFAULT : 0;
    749}
    750
    751static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu)
    752{
    753	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    754	struct kvm_s390_prefix_info prefix;
    755
    756	spin_lock(&li->lock);
    757	prefix = li->irq.prefix;
    758	li->irq.prefix.address = 0;
    759	clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
    760	spin_unlock(&li->lock);
    761
    762	vcpu->stat.deliver_prefix_signal++;
    763	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
    764					 KVM_S390_SIGP_SET_PREFIX,
    765					 prefix.address, 0);
    766
    767	kvm_s390_set_prefix(vcpu, prefix.address);
    768	return 0;
    769}
    770
    771static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu)
    772{
    773	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    774	int rc;
    775	int cpu_addr;
    776
    777	spin_lock(&li->lock);
    778	cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS);
    779	clear_bit(cpu_addr, li->sigp_emerg_pending);
    780	if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS))
    781		clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
    782	spin_unlock(&li->lock);
    783
    784	VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg");
    785	vcpu->stat.deliver_emergency_signal++;
    786	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
    787					 cpu_addr, 0);
    788	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
    789		vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
    790		vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG;
    791		vcpu->arch.sie_block->extcpuaddr = cpu_addr;
    792		return 0;
    793	}
    794
    795	rc  = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG,
    796			   (u16 *)__LC_EXT_INT_CODE);
    797	rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR);
    798	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
    799			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    800	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
    801			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    802	return rc ? -EFAULT : 0;
    803}
    804
    805static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu)
    806{
    807	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    808	struct kvm_s390_extcall_info extcall;
    809	int rc;
    810
    811	spin_lock(&li->lock);
    812	extcall = li->irq.extcall;
    813	li->irq.extcall.code = 0;
    814	clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
    815	spin_unlock(&li->lock);
    816
    817	VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call");
    818	vcpu->stat.deliver_external_call++;
    819	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
    820					 KVM_S390_INT_EXTERNAL_CALL,
    821					 extcall.code, 0);
    822	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
    823		vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
    824		vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL;
    825		vcpu->arch.sie_block->extcpuaddr = extcall.code;
    826		return 0;
    827	}
    828
    829	rc  = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL,
    830			   (u16 *)__LC_EXT_INT_CODE);
    831	rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR);
    832	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
    833			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    834	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw,
    835			    sizeof(psw_t));
    836	return rc ? -EFAULT : 0;
    837}
    838
    839static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code)
    840{
    841	switch (code) {
    842	case PGM_SPECIFICATION:
    843		vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION;
    844		break;
    845	case PGM_OPERAND:
    846		vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND;
    847		break;
    848	default:
    849		return -EINVAL;
    850	}
    851	return 0;
    852}
    853
    854static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
    855{
    856	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
    857	struct kvm_s390_pgm_info pgm_info;
    858	int rc = 0, nullifying = false;
    859	u16 ilen;
    860
    861	spin_lock(&li->lock);
    862	pgm_info = li->irq.pgm;
    863	clear_bit(IRQ_PEND_PROG, &li->pending_irqs);
    864	memset(&li->irq.pgm, 0, sizeof(pgm_info));
    865	spin_unlock(&li->lock);
    866
    867	ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK;
    868	VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d",
    869		   pgm_info.code, ilen);
    870	vcpu->stat.deliver_program++;
    871	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
    872					 pgm_info.code, 0);
    873
    874	/* PER is handled by the ultravisor */
    875	if (kvm_s390_pv_cpu_is_protected(vcpu))
    876		return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER);
    877
    878	switch (pgm_info.code & ~PGM_PER) {
    879	case PGM_AFX_TRANSLATION:
    880	case PGM_ASX_TRANSLATION:
    881	case PGM_EX_TRANSLATION:
    882	case PGM_LFX_TRANSLATION:
    883	case PGM_LSTE_SEQUENCE:
    884	case PGM_LSX_TRANSLATION:
    885	case PGM_LX_TRANSLATION:
    886	case PGM_PRIMARY_AUTHORITY:
    887	case PGM_SECONDARY_AUTHORITY:
    888		nullifying = true;
    889		fallthrough;
    890	case PGM_SPACE_SWITCH:
    891		rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
    892				  (u64 *)__LC_TRANS_EXC_CODE);
    893		break;
    894	case PGM_ALEN_TRANSLATION:
    895	case PGM_ALE_SEQUENCE:
    896	case PGM_ASTE_INSTANCE:
    897	case PGM_ASTE_SEQUENCE:
    898	case PGM_ASTE_VALIDITY:
    899	case PGM_EXTENDED_AUTHORITY:
    900		rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
    901				  (u8 *)__LC_EXC_ACCESS_ID);
    902		nullifying = true;
    903		break;
    904	case PGM_ASCE_TYPE:
    905	case PGM_PAGE_TRANSLATION:
    906	case PGM_REGION_FIRST_TRANS:
    907	case PGM_REGION_SECOND_TRANS:
    908	case PGM_REGION_THIRD_TRANS:
    909	case PGM_SEGMENT_TRANSLATION:
    910		rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
    911				  (u64 *)__LC_TRANS_EXC_CODE);
    912		rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
    913				   (u8 *)__LC_EXC_ACCESS_ID);
    914		rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
    915				   (u8 *)__LC_OP_ACCESS_ID);
    916		nullifying = true;
    917		break;
    918	case PGM_MONITOR:
    919		rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
    920				  (u16 *)__LC_MON_CLASS_NR);
    921		rc |= put_guest_lc(vcpu, pgm_info.mon_code,
    922				   (u64 *)__LC_MON_CODE);
    923		break;
    924	case PGM_VECTOR_PROCESSING:
    925	case PGM_DATA:
    926		rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
    927				  (u32 *)__LC_DATA_EXC_CODE);
    928		break;
    929	case PGM_PROTECTION:
    930		rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
    931				  (u64 *)__LC_TRANS_EXC_CODE);
    932		rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
    933				   (u8 *)__LC_EXC_ACCESS_ID);
    934		break;
    935	case PGM_STACK_FULL:
    936	case PGM_STACK_EMPTY:
    937	case PGM_STACK_SPECIFICATION:
    938	case PGM_STACK_TYPE:
    939	case PGM_STACK_OPERATION:
    940	case PGM_TRACE_TABEL:
    941	case PGM_CRYPTO_OPERATION:
    942		nullifying = true;
    943		break;
    944	}
    945
    946	if (pgm_info.code & PGM_PER) {
    947		rc |= put_guest_lc(vcpu, pgm_info.per_code,
    948				   (u8 *) __LC_PER_CODE);
    949		rc |= put_guest_lc(vcpu, pgm_info.per_atmid,
    950				   (u8 *)__LC_PER_ATMID);
    951		rc |= put_guest_lc(vcpu, pgm_info.per_address,
    952				   (u64 *) __LC_PER_ADDRESS);
    953		rc |= put_guest_lc(vcpu, pgm_info.per_access_id,
    954				   (u8 *) __LC_PER_ACCESS_ID);
    955	}
    956
    957	if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND))
    958		kvm_s390_rewind_psw(vcpu, ilen);
    959
    960	/* bit 1+2 of the target are the ilc, so we can directly use ilen */
    961	rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC);
    962	rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
    963				 (u64 *) __LC_PGM_LAST_BREAK);
    964	rc |= put_guest_lc(vcpu, pgm_info.code,
    965			   (u16 *)__LC_PGM_INT_CODE);
    966	rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
    967			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    968	rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW,
    969			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    970	return rc ? -EFAULT : 0;
    971}
    972
    973#define SCCB_MASK 0xFFFFFFF8
    974#define SCCB_EVENT_PENDING 0x3
    975
    976static int write_sclp(struct kvm_vcpu *vcpu, u32 parm)
    977{
    978	int rc;
    979
    980	if (kvm_s390_pv_cpu_get_handle(vcpu)) {
    981		vcpu->arch.sie_block->iictl = IICTL_CODE_EXT;
    982		vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG;
    983		vcpu->arch.sie_block->eiparams = parm;
    984		return 0;
    985	}
    986
    987	rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
    988	rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
    989	rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
    990			     &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    991	rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
    992			    &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
    993	rc |= put_guest_lc(vcpu, parm,
    994			   (u32 *)__LC_EXT_PARAMS);
    995
    996	return rc ? -EFAULT : 0;
    997}
    998
    999static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
   1000{
   1001	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
   1002	struct kvm_s390_ext_info ext;
   1003
   1004	spin_lock(&fi->lock);
   1005	if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) ||
   1006	    !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
   1007		spin_unlock(&fi->lock);
   1008		return 0;
   1009	}
   1010	ext = fi->srv_signal;
   1011	memset(&fi->srv_signal, 0, sizeof(ext));
   1012	clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
   1013	clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
   1014	if (kvm_s390_pv_cpu_is_protected(vcpu))
   1015		set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
   1016	spin_unlock(&fi->lock);
   1017
   1018	VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x",
   1019		   ext.ext_params);
   1020	vcpu->stat.deliver_service_signal++;
   1021	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
   1022					 ext.ext_params, 0);
   1023
   1024	return write_sclp(vcpu, ext.ext_params);
   1025}
   1026
   1027static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu)
   1028{
   1029	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
   1030	struct kvm_s390_ext_info ext;
   1031
   1032	spin_lock(&fi->lock);
   1033	if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) {
   1034		spin_unlock(&fi->lock);
   1035		return 0;
   1036	}
   1037	ext = fi->srv_signal;
   1038	/* only clear the event bit */
   1039	fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING;
   1040	clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
   1041	spin_unlock(&fi->lock);
   1042
   1043	VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event");
   1044	vcpu->stat.deliver_service_signal++;
   1045	trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
   1046					 ext.ext_params, 0);
   1047
   1048	return write_sclp(vcpu, SCCB_EVENT_PENDING);
   1049}
   1050
   1051static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
   1052{
   1053	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
   1054	struct kvm_s390_interrupt_info *inti;
   1055	int rc = 0;
   1056
   1057	spin_lock(&fi->lock);
   1058	inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
   1059					struct kvm_s390_interrupt_info,
   1060					list);
   1061	if (inti) {
   1062		list_del(&inti->list);
   1063		fi->counters[FIRQ_CNTR_PFAULT] -= 1;
   1064	}
   1065	if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
   1066		clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
   1067	spin_unlock(&fi->lock);
   1068
   1069	if (inti) {
   1070		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
   1071						 KVM_S390_INT_PFAULT_DONE, 0,
   1072						 inti->ext.ext_params2);
   1073		VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx",
   1074			   inti->ext.ext_params2);
   1075
   1076		rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
   1077				(u16 *)__LC_EXT_INT_CODE);
   1078		rc |= put_guest_lc(vcpu, PFAULT_DONE,
   1079				(u16 *)__LC_EXT_CPU_ADDR);
   1080		rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
   1081				&vcpu->arch.sie_block->gpsw,
   1082				sizeof(psw_t));
   1083		rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
   1084				&vcpu->arch.sie_block->gpsw,
   1085				sizeof(psw_t));
   1086		rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
   1087				(u64 *)__LC_EXT_PARAMS2);
   1088		kfree(inti);
   1089	}
   1090	return rc ? -EFAULT : 0;
   1091}
   1092
   1093static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
   1094{
   1095	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
   1096	struct kvm_s390_interrupt_info *inti;
   1097	int rc = 0;
   1098
   1099	spin_lock(&fi->lock);
   1100	inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
   1101					struct kvm_s390_interrupt_info,
   1102					list);
   1103	if (inti) {
   1104		VCPU_EVENT(vcpu, 4,
   1105			   "deliver: virtio parm: 0x%x,parm64: 0x%llx",
   1106			   inti->ext.ext_params, inti->ext.ext_params2);
   1107		vcpu->stat.deliver_virtio++;
   1108		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
   1109				inti->type,
   1110				inti->ext.ext_params,
   1111				inti->ext.ext_params2);
   1112		list_del(&inti->list);
   1113		fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
   1114	}
   1115	if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
   1116		clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
   1117	spin_unlock(&fi->lock);
   1118
   1119	if (inti) {
   1120		rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
   1121				(u16 *)__LC_EXT_INT_CODE);
   1122		rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
   1123				(u16 *)__LC_EXT_CPU_ADDR);
   1124		rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
   1125				&vcpu->arch.sie_block->gpsw,
   1126				sizeof(psw_t));
   1127		rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
   1128				&vcpu->arch.sie_block->gpsw,
   1129				sizeof(psw_t));
   1130		rc |= put_guest_lc(vcpu, inti->ext.ext_params,
   1131				(u32 *)__LC_EXT_PARAMS);
   1132		rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
   1133				(u64 *)__LC_EXT_PARAMS2);
   1134		kfree(inti);
   1135	}
   1136	return rc ? -EFAULT : 0;
   1137}
   1138
   1139static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
   1140{
   1141	int rc;
   1142
   1143	if (kvm_s390_pv_cpu_is_protected(vcpu)) {
   1144		vcpu->arch.sie_block->iictl = IICTL_CODE_IO;
   1145		vcpu->arch.sie_block->subchannel_id = io->subchannel_id;
   1146		vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr;
   1147		vcpu->arch.sie_block->io_int_parm = io->io_int_parm;
   1148		vcpu->arch.sie_block->io_int_word = io->io_int_word;
   1149		return 0;
   1150	}
   1151
   1152	rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
   1153	rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
   1154	rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
   1155	rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
   1156	rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
   1157			     &vcpu->arch.sie_block->gpsw,
   1158			     sizeof(psw_t));
   1159	rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
   1160			    &vcpu->arch.sie_block->gpsw,
   1161			    sizeof(psw_t));
   1162	return rc ? -EFAULT : 0;
   1163}
   1164
   1165static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
   1166				     unsigned long irq_type)
   1167{
   1168	struct list_head *isc_list;
   1169	struct kvm_s390_float_interrupt *fi;
   1170	struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
   1171	struct kvm_s390_interrupt_info *inti = NULL;
   1172	struct kvm_s390_io_info io;
   1173	u32 isc;
   1174	int rc = 0;
   1175
   1176	fi = &vcpu->kvm->arch.float_int;
   1177
   1178	spin_lock(&fi->lock);
   1179	isc = irq_type_to_isc(irq_type);
   1180	isc_list = &fi->lists[isc];
   1181	inti = list_first_entry_or_null(isc_list,
   1182					struct kvm_s390_interrupt_info,
   1183					list);
   1184	if (inti) {
   1185		if (inti->type & KVM_S390_INT_IO_AI_MASK)
   1186			VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)");
   1187		else
   1188			VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x",
   1189			inti->io.subchannel_id >> 8,
   1190			inti->io.subchannel_id >> 1 & 0x3,
   1191			inti->io.subchannel_nr);
   1192
   1193		vcpu->stat.deliver_io++;
   1194		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
   1195				inti->type,
   1196				((__u32)inti->io.subchannel_id << 16) |
   1197				inti->io.subchannel_nr,
   1198				((__u64)inti->io.io_int_parm << 32) |
   1199				inti->io.io_int_word);
   1200		list_del(&inti->list);
   1201		fi->counters[FIRQ_CNTR_IO] -= 1;
   1202	}
   1203	if (list_empty(isc_list))
   1204		clear_bit(irq_type, &fi->pending_irqs);
   1205	spin_unlock(&fi->lock);
   1206
   1207	if (inti) {
   1208		rc = __do_deliver_io(vcpu, &(inti->io));
   1209		kfree(inti);
   1210		goto out;
   1211	}
   1212
   1213	if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) {
   1214		/*
   1215		 * in case an adapter interrupt was not delivered
   1216		 * in SIE context KVM will handle the delivery
   1217		 */
   1218		VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
   1219		memset(&io, 0, sizeof(io));
   1220		io.io_int_word = isc_to_int_word(isc);
   1221		vcpu->stat.deliver_io++;
   1222		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
   1223			KVM_S390_INT_IO(1, 0, 0, 0),
   1224			((__u32)io.subchannel_id << 16) |
   1225			io.subchannel_nr,
   1226			((__u64)io.io_int_parm << 32) |
   1227			io.io_int_word);
   1228		rc = __do_deliver_io(vcpu, &io);
   1229	}
   1230out:
   1231	return rc;
   1232}
   1233
   1234/* Check whether an external call is pending (deliverable or not) */
   1235int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
   1236{
   1237	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1238
   1239	if (!sclp.has_sigpif)
   1240		return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs);
   1241
   1242	return sca_ext_call_pending(vcpu, NULL);
   1243}
   1244
   1245int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
   1246{
   1247	if (deliverable_irqs(vcpu))
   1248		return 1;
   1249
   1250	if (kvm_cpu_has_pending_timer(vcpu))
   1251		return 1;
   1252
   1253	/* external call pending and deliverable */
   1254	if (kvm_s390_ext_call_pending(vcpu) &&
   1255	    !psw_extint_disabled(vcpu) &&
   1256	    (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK))
   1257		return 1;
   1258
   1259	if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu))
   1260		return 1;
   1261	return 0;
   1262}
   1263
   1264int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
   1265{
   1266	return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
   1267}
   1268
   1269static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
   1270{
   1271	const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
   1272	const u64 ckc = vcpu->arch.sie_block->ckc;
   1273	u64 cputm, sltime = 0;
   1274
   1275	if (ckc_interrupts_enabled(vcpu)) {
   1276		if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) {
   1277			if ((s64)now < (s64)ckc)
   1278				sltime = tod_to_ns((s64)ckc - (s64)now);
   1279		} else if (now < ckc) {
   1280			sltime = tod_to_ns(ckc - now);
   1281		}
   1282		/* already expired */
   1283		if (!sltime)
   1284			return 0;
   1285		if (cpu_timer_interrupts_enabled(vcpu)) {
   1286			cputm = kvm_s390_get_cpu_timer(vcpu);
   1287			/* already expired? */
   1288			if (cputm >> 63)
   1289				return 0;
   1290			return min_t(u64, sltime, tod_to_ns(cputm));
   1291		}
   1292	} else if (cpu_timer_interrupts_enabled(vcpu)) {
   1293		sltime = kvm_s390_get_cpu_timer(vcpu);
   1294		/* already expired? */
   1295		if (sltime >> 63)
   1296			return 0;
   1297	}
   1298	return sltime;
   1299}
   1300
   1301int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
   1302{
   1303	struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int;
   1304	u64 sltime;
   1305
   1306	vcpu->stat.exit_wait_state++;
   1307
   1308	/* fast path */
   1309	if (kvm_arch_vcpu_runnable(vcpu))
   1310		return 0;
   1311
   1312	if (psw_interrupts_disabled(vcpu)) {
   1313		VCPU_EVENT(vcpu, 3, "%s", "disabled wait");
   1314		return -EOPNOTSUPP; /* disabled wait */
   1315	}
   1316
   1317	if (gi->origin &&
   1318	    (gisa_get_ipm_or_restore_iam(gi) &
   1319	     vcpu->arch.sie_block->gcr[6] >> 24))
   1320		return 0;
   1321
   1322	if (!ckc_interrupts_enabled(vcpu) &&
   1323	    !cpu_timer_interrupts_enabled(vcpu)) {
   1324		VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer");
   1325		__set_cpu_idle(vcpu);
   1326		goto no_timer;
   1327	}
   1328
   1329	sltime = __calculate_sltime(vcpu);
   1330	if (!sltime)
   1331		return 0;
   1332
   1333	__set_cpu_idle(vcpu);
   1334	hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL);
   1335	VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime);
   1336no_timer:
   1337	kvm_vcpu_srcu_read_unlock(vcpu);
   1338	kvm_vcpu_halt(vcpu);
   1339	vcpu->valid_wakeup = false;
   1340	__unset_cpu_idle(vcpu);
   1341	kvm_vcpu_srcu_read_lock(vcpu);
   1342
   1343	hrtimer_cancel(&vcpu->arch.ckc_timer);
   1344	return 0;
   1345}
   1346
   1347void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu)
   1348{
   1349	vcpu->valid_wakeup = true;
   1350	kvm_vcpu_wake_up(vcpu);
   1351
   1352	/*
   1353	 * The VCPU might not be sleeping but rather executing VSIE. Let's
   1354	 * kick it, so it leaves the SIE to process the request.
   1355	 */
   1356	kvm_s390_vsie_kick(vcpu);
   1357}
   1358
   1359enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer)
   1360{
   1361	struct kvm_vcpu *vcpu;
   1362	u64 sltime;
   1363
   1364	vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer);
   1365	sltime = __calculate_sltime(vcpu);
   1366
   1367	/*
   1368	 * If the monotonic clock runs faster than the tod clock we might be
   1369	 * woken up too early and have to go back to sleep to avoid deadlocks.
   1370	 */
   1371	if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime)))
   1372		return HRTIMER_RESTART;
   1373	kvm_s390_vcpu_wakeup(vcpu);
   1374	return HRTIMER_NORESTART;
   1375}
   1376
   1377void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
   1378{
   1379	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1380
   1381	spin_lock(&li->lock);
   1382	li->pending_irqs = 0;
   1383	bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS);
   1384	memset(&li->irq, 0, sizeof(li->irq));
   1385	spin_unlock(&li->lock);
   1386
   1387	sca_clear_ext_call(vcpu);
   1388}
   1389
   1390int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
   1391{
   1392	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1393	int rc = 0;
   1394	unsigned long irq_type;
   1395	unsigned long irqs;
   1396
   1397	__reset_intercept_indicators(vcpu);
   1398
   1399	/* pending ckc conditions might have been invalidated */
   1400	clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
   1401	if (ckc_irq_pending(vcpu))
   1402		set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
   1403
   1404	/* pending cpu timer conditions might have been invalidated */
   1405	clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
   1406	if (cpu_timer_irq_pending(vcpu))
   1407		set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
   1408
   1409	while ((irqs = deliverable_irqs(vcpu)) && !rc) {
   1410		/* bits are in the reverse order of interrupt priority */
   1411		irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
   1412		switch (irq_type) {
   1413		case IRQ_PEND_IO_ISC_0:
   1414		case IRQ_PEND_IO_ISC_1:
   1415		case IRQ_PEND_IO_ISC_2:
   1416		case IRQ_PEND_IO_ISC_3:
   1417		case IRQ_PEND_IO_ISC_4:
   1418		case IRQ_PEND_IO_ISC_5:
   1419		case IRQ_PEND_IO_ISC_6:
   1420		case IRQ_PEND_IO_ISC_7:
   1421			rc = __deliver_io(vcpu, irq_type);
   1422			break;
   1423		case IRQ_PEND_MCHK_EX:
   1424		case IRQ_PEND_MCHK_REP:
   1425			rc = __deliver_machine_check(vcpu);
   1426			break;
   1427		case IRQ_PEND_PROG:
   1428			rc = __deliver_prog(vcpu);
   1429			break;
   1430		case IRQ_PEND_EXT_EMERGENCY:
   1431			rc = __deliver_emergency_signal(vcpu);
   1432			break;
   1433		case IRQ_PEND_EXT_EXTERNAL:
   1434			rc = __deliver_external_call(vcpu);
   1435			break;
   1436		case IRQ_PEND_EXT_CLOCK_COMP:
   1437			rc = __deliver_ckc(vcpu);
   1438			break;
   1439		case IRQ_PEND_EXT_CPU_TIMER:
   1440			rc = __deliver_cpu_timer(vcpu);
   1441			break;
   1442		case IRQ_PEND_RESTART:
   1443			rc = __deliver_restart(vcpu);
   1444			break;
   1445		case IRQ_PEND_SET_PREFIX:
   1446			rc = __deliver_set_prefix(vcpu);
   1447			break;
   1448		case IRQ_PEND_PFAULT_INIT:
   1449			rc = __deliver_pfault_init(vcpu);
   1450			break;
   1451		case IRQ_PEND_EXT_SERVICE:
   1452			rc = __deliver_service(vcpu);
   1453			break;
   1454		case IRQ_PEND_EXT_SERVICE_EV:
   1455			rc = __deliver_service_ev(vcpu);
   1456			break;
   1457		case IRQ_PEND_PFAULT_DONE:
   1458			rc = __deliver_pfault_done(vcpu);
   1459			break;
   1460		case IRQ_PEND_VIRTIO:
   1461			rc = __deliver_virtio(vcpu);
   1462			break;
   1463		default:
   1464			WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
   1465			clear_bit(irq_type, &li->pending_irqs);
   1466		}
   1467	}
   1468
   1469	set_intercept_indicators(vcpu);
   1470
   1471	return rc;
   1472}
   1473
   1474static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   1475{
   1476	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1477
   1478	vcpu->stat.inject_program++;
   1479	VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code);
   1480	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT,
   1481				   irq->u.pgm.code, 0);
   1482
   1483	if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) {
   1484		/* auto detection if no valid ILC was given */
   1485		irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK;
   1486		irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu);
   1487		irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID;
   1488	}
   1489
   1490	if (irq->u.pgm.code == PGM_PER) {
   1491		li->irq.pgm.code |= PGM_PER;
   1492		li->irq.pgm.flags = irq->u.pgm.flags;
   1493		/* only modify PER related information */
   1494		li->irq.pgm.per_address = irq->u.pgm.per_address;
   1495		li->irq.pgm.per_code = irq->u.pgm.per_code;
   1496		li->irq.pgm.per_atmid = irq->u.pgm.per_atmid;
   1497		li->irq.pgm.per_access_id = irq->u.pgm.per_access_id;
   1498	} else if (!(irq->u.pgm.code & PGM_PER)) {
   1499		li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) |
   1500				   irq->u.pgm.code;
   1501		li->irq.pgm.flags = irq->u.pgm.flags;
   1502		/* only modify non-PER information */
   1503		li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code;
   1504		li->irq.pgm.mon_code = irq->u.pgm.mon_code;
   1505		li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code;
   1506		li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr;
   1507		li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id;
   1508		li->irq.pgm.op_access_id = irq->u.pgm.op_access_id;
   1509	} else {
   1510		li->irq.pgm = irq->u.pgm;
   1511	}
   1512	set_bit(IRQ_PEND_PROG, &li->pending_irqs);
   1513	return 0;
   1514}
   1515
   1516static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   1517{
   1518	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1519
   1520	vcpu->stat.inject_pfault_init++;
   1521	VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx",
   1522		   irq->u.ext.ext_params2);
   1523	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT,
   1524				   irq->u.ext.ext_params,
   1525				   irq->u.ext.ext_params2);
   1526
   1527	li->irq.ext = irq->u.ext;
   1528	set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
   1529	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
   1530	return 0;
   1531}
   1532
   1533static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   1534{
   1535	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1536	struct kvm_s390_extcall_info *extcall = &li->irq.extcall;
   1537	uint16_t src_id = irq->u.extcall.code;
   1538
   1539	vcpu->stat.inject_external_call++;
   1540	VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u",
   1541		   src_id);
   1542	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL,
   1543				   src_id, 0);
   1544
   1545	/* sending vcpu invalid */
   1546	if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL)
   1547		return -EINVAL;
   1548
   1549	if (sclp.has_sigpif && !kvm_s390_pv_cpu_get_handle(vcpu))
   1550		return sca_inject_ext_call(vcpu, src_id);
   1551
   1552	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
   1553		return -EBUSY;
   1554	*extcall = irq->u.extcall;
   1555	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
   1556	return 0;
   1557}
   1558
   1559static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   1560{
   1561	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1562	struct kvm_s390_prefix_info *prefix = &li->irq.prefix;
   1563
   1564	vcpu->stat.inject_set_prefix++;
   1565	VCPU_EVENT(vcpu, 3, "inject: set prefix to %x",
   1566		   irq->u.prefix.address);
   1567	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX,
   1568				   irq->u.prefix.address, 0);
   1569
   1570	if (!is_vcpu_stopped(vcpu))
   1571		return -EBUSY;
   1572
   1573	*prefix = irq->u.prefix;
   1574	set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs);
   1575	return 0;
   1576}
   1577
   1578#define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS)
   1579static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   1580{
   1581	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1582	struct kvm_s390_stop_info *stop = &li->irq.stop;
   1583	int rc = 0;
   1584
   1585	vcpu->stat.inject_stop_signal++;
   1586	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0);
   1587
   1588	if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS)
   1589		return -EINVAL;
   1590
   1591	if (is_vcpu_stopped(vcpu)) {
   1592		if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS)
   1593			rc = kvm_s390_store_status_unloaded(vcpu,
   1594						KVM_S390_STORE_STATUS_NOADDR);
   1595		return rc;
   1596	}
   1597
   1598	if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
   1599		return -EBUSY;
   1600	stop->flags = irq->u.stop.flags;
   1601	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
   1602	return 0;
   1603}
   1604
   1605static int __inject_sigp_restart(struct kvm_vcpu *vcpu)
   1606{
   1607	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1608
   1609	vcpu->stat.inject_restart++;
   1610	VCPU_EVENT(vcpu, 3, "%s", "inject: restart int");
   1611	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0);
   1612
   1613	set_bit(IRQ_PEND_RESTART, &li->pending_irqs);
   1614	return 0;
   1615}
   1616
   1617static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
   1618				   struct kvm_s390_irq *irq)
   1619{
   1620	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1621
   1622	vcpu->stat.inject_emergency_signal++;
   1623	VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u",
   1624		   irq->u.emerg.code);
   1625	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY,
   1626				   irq->u.emerg.code, 0);
   1627
   1628	/* sending vcpu invalid */
   1629	if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL)
   1630		return -EINVAL;
   1631
   1632	set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
   1633	set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
   1634	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
   1635	return 0;
   1636}
   1637
   1638static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   1639{
   1640	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1641	struct kvm_s390_mchk_info *mchk = &li->irq.mchk;
   1642
   1643	vcpu->stat.inject_mchk++;
   1644	VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx",
   1645		   irq->u.mchk.mcic);
   1646	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0,
   1647				   irq->u.mchk.mcic);
   1648
   1649	/*
   1650	 * Because repressible machine checks can be indicated along with
   1651	 * exigent machine checks (PoP, Chapter 11, Interruption action)
   1652	 * we need to combine cr14, mcic and external damage code.
   1653	 * Failing storage address and the logout area should not be or'ed
   1654	 * together, we just indicate the last occurrence of the corresponding
   1655	 * machine check
   1656	 */
   1657	mchk->cr14 |= irq->u.mchk.cr14;
   1658	mchk->mcic |= irq->u.mchk.mcic;
   1659	mchk->ext_damage_code |= irq->u.mchk.ext_damage_code;
   1660	mchk->failing_storage_address = irq->u.mchk.failing_storage_address;
   1661	memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout,
   1662	       sizeof(mchk->fixed_logout));
   1663	if (mchk->mcic & MCHK_EX_MASK)
   1664		set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
   1665	else if (mchk->mcic & MCHK_REP_MASK)
   1666		set_bit(IRQ_PEND_MCHK_REP,  &li->pending_irqs);
   1667	return 0;
   1668}
   1669
   1670static int __inject_ckc(struct kvm_vcpu *vcpu)
   1671{
   1672	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1673
   1674	vcpu->stat.inject_ckc++;
   1675	VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external");
   1676	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP,
   1677				   0, 0);
   1678
   1679	set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
   1680	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
   1681	return 0;
   1682}
   1683
   1684static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
   1685{
   1686	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   1687
   1688	vcpu->stat.inject_cputm++;
   1689	VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external");
   1690	trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER,
   1691				   0, 0);
   1692
   1693	set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
   1694	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
   1695	return 0;
   1696}
   1697
   1698static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
   1699						  int isc, u32 schid)
   1700{
   1701	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   1702	struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
   1703	struct kvm_s390_interrupt_info *iter;
   1704	u16 id = (schid & 0xffff0000U) >> 16;
   1705	u16 nr = schid & 0x0000ffffU;
   1706
   1707	spin_lock(&fi->lock);
   1708	list_for_each_entry(iter, isc_list, list) {
   1709		if (schid && (id != iter->io.subchannel_id ||
   1710			      nr != iter->io.subchannel_nr))
   1711			continue;
   1712		/* found an appropriate entry */
   1713		list_del_init(&iter->list);
   1714		fi->counters[FIRQ_CNTR_IO] -= 1;
   1715		if (list_empty(isc_list))
   1716			clear_bit(isc_to_irq_type(isc), &fi->pending_irqs);
   1717		spin_unlock(&fi->lock);
   1718		return iter;
   1719	}
   1720	spin_unlock(&fi->lock);
   1721	return NULL;
   1722}
   1723
   1724static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
   1725						      u64 isc_mask, u32 schid)
   1726{
   1727	struct kvm_s390_interrupt_info *inti = NULL;
   1728	int isc;
   1729
   1730	for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
   1731		if (isc_mask & isc_to_isc_bits(isc))
   1732			inti = get_io_int(kvm, isc, schid);
   1733	}
   1734	return inti;
   1735}
   1736
   1737static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
   1738{
   1739	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   1740	unsigned long active_mask;
   1741	int isc;
   1742
   1743	if (schid)
   1744		goto out;
   1745	if (!gi->origin)
   1746		goto out;
   1747
   1748	active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32;
   1749	while (active_mask) {
   1750		isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
   1751		if (gisa_tac_ipm_gisc(gi->origin, isc))
   1752			return isc;
   1753		clear_bit_inv(isc, &active_mask);
   1754	}
   1755out:
   1756	return -EINVAL;
   1757}
   1758
   1759/*
   1760 * Dequeue and return an I/O interrupt matching any of the interruption
   1761 * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
   1762 * Take into account the interrupts pending in the interrupt list and in GISA.
   1763 *
   1764 * Note that for a guest that does not enable I/O interrupts
   1765 * but relies on TPI, a flood of classic interrupts may starve
   1766 * out adapter interrupts on the same isc. Linux does not do
   1767 * that, and it is possible to work around the issue by configuring
   1768 * different iscs for classic and adapter interrupts in the guest,
   1769 * but we may want to revisit this in the future.
   1770 */
   1771struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
   1772						    u64 isc_mask, u32 schid)
   1773{
   1774	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   1775	struct kvm_s390_interrupt_info *inti, *tmp_inti;
   1776	int isc;
   1777
   1778	inti = get_top_io_int(kvm, isc_mask, schid);
   1779
   1780	isc = get_top_gisa_isc(kvm, isc_mask, schid);
   1781	if (isc < 0)
   1782		/* no AI in GISA */
   1783		goto out;
   1784
   1785	if (!inti)
   1786		/* AI in GISA but no classical IO int */
   1787		goto gisa_out;
   1788
   1789	/* both types of interrupts present */
   1790	if (int_word_to_isc(inti->io.io_int_word) <= isc) {
   1791		/* classical IO int with higher priority */
   1792		gisa_set_ipm_gisc(gi->origin, isc);
   1793		goto out;
   1794	}
   1795gisa_out:
   1796	tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
   1797	if (tmp_inti) {
   1798		tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
   1799		tmp_inti->io.io_int_word = isc_to_int_word(isc);
   1800		if (inti)
   1801			kvm_s390_reinject_io_int(kvm, inti);
   1802		inti = tmp_inti;
   1803	} else
   1804		gisa_set_ipm_gisc(gi->origin, isc);
   1805out:
   1806	return inti;
   1807}
   1808
   1809static int __inject_service(struct kvm *kvm,
   1810			     struct kvm_s390_interrupt_info *inti)
   1811{
   1812	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   1813
   1814	kvm->stat.inject_service_signal++;
   1815	spin_lock(&fi->lock);
   1816	fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
   1817
   1818	/* We always allow events, track them separately from the sccb ints */
   1819	if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING)
   1820		set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs);
   1821
   1822	/*
   1823	 * Early versions of the QEMU s390 bios will inject several
   1824	 * service interrupts after another without handling a
   1825	 * condition code indicating busy.
   1826	 * We will silently ignore those superfluous sccb values.
   1827	 * A future version of QEMU will take care of serialization
   1828	 * of servc requests
   1829	 */
   1830	if (fi->srv_signal.ext_params & SCCB_MASK)
   1831		goto out;
   1832	fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
   1833	set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
   1834out:
   1835	spin_unlock(&fi->lock);
   1836	kfree(inti);
   1837	return 0;
   1838}
   1839
   1840static int __inject_virtio(struct kvm *kvm,
   1841			    struct kvm_s390_interrupt_info *inti)
   1842{
   1843	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   1844
   1845	kvm->stat.inject_virtio++;
   1846	spin_lock(&fi->lock);
   1847	if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
   1848		spin_unlock(&fi->lock);
   1849		return -EBUSY;
   1850	}
   1851	fi->counters[FIRQ_CNTR_VIRTIO] += 1;
   1852	list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
   1853	set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
   1854	spin_unlock(&fi->lock);
   1855	return 0;
   1856}
   1857
   1858static int __inject_pfault_done(struct kvm *kvm,
   1859				 struct kvm_s390_interrupt_info *inti)
   1860{
   1861	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   1862
   1863	kvm->stat.inject_pfault_done++;
   1864	spin_lock(&fi->lock);
   1865	if (fi->counters[FIRQ_CNTR_PFAULT] >=
   1866		(ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
   1867		spin_unlock(&fi->lock);
   1868		return -EBUSY;
   1869	}
   1870	fi->counters[FIRQ_CNTR_PFAULT] += 1;
   1871	list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
   1872	set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
   1873	spin_unlock(&fi->lock);
   1874	return 0;
   1875}
   1876
   1877#define CR_PENDING_SUBCLASS 28
   1878static int __inject_float_mchk(struct kvm *kvm,
   1879				struct kvm_s390_interrupt_info *inti)
   1880{
   1881	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   1882
   1883	kvm->stat.inject_float_mchk++;
   1884	spin_lock(&fi->lock);
   1885	fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
   1886	fi->mchk.mcic |= inti->mchk.mcic;
   1887	set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
   1888	spin_unlock(&fi->lock);
   1889	kfree(inti);
   1890	return 0;
   1891}
   1892
   1893static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
   1894{
   1895	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   1896	struct kvm_s390_float_interrupt *fi;
   1897	struct list_head *list;
   1898	int isc;
   1899
   1900	kvm->stat.inject_io++;
   1901	isc = int_word_to_isc(inti->io.io_int_word);
   1902
   1903	/*
   1904	 * We do not use the lock checking variant as this is just a
   1905	 * performance optimization and we do not hold the lock here.
   1906	 * This is ok as the code will pick interrupts from both "lists"
   1907	 * for delivery.
   1908	 */
   1909	if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) {
   1910		VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
   1911		gisa_set_ipm_gisc(gi->origin, isc);
   1912		kfree(inti);
   1913		return 0;
   1914	}
   1915
   1916	fi = &kvm->arch.float_int;
   1917	spin_lock(&fi->lock);
   1918	if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
   1919		spin_unlock(&fi->lock);
   1920		return -EBUSY;
   1921	}
   1922	fi->counters[FIRQ_CNTR_IO] += 1;
   1923
   1924	if (inti->type & KVM_S390_INT_IO_AI_MASK)
   1925		VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)");
   1926	else
   1927		VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x",
   1928			inti->io.subchannel_id >> 8,
   1929			inti->io.subchannel_id >> 1 & 0x3,
   1930			inti->io.subchannel_nr);
   1931	list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
   1932	list_add_tail(&inti->list, list);
   1933	set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
   1934	spin_unlock(&fi->lock);
   1935	return 0;
   1936}
   1937
   1938/*
   1939 * Find a destination VCPU for a floating irq and kick it.
   1940 */
   1941static void __floating_irq_kick(struct kvm *kvm, u64 type)
   1942{
   1943	struct kvm_vcpu *dst_vcpu;
   1944	int sigcpu, online_vcpus, nr_tries = 0;
   1945
   1946	online_vcpus = atomic_read(&kvm->online_vcpus);
   1947	if (!online_vcpus)
   1948		return;
   1949
   1950	/* find idle VCPUs first, then round robin */
   1951	sigcpu = find_first_bit(kvm->arch.idle_mask, online_vcpus);
   1952	if (sigcpu == online_vcpus) {
   1953		do {
   1954			sigcpu = kvm->arch.float_int.next_rr_cpu++;
   1955			kvm->arch.float_int.next_rr_cpu %= online_vcpus;
   1956			/* avoid endless loops if all vcpus are stopped */
   1957			if (nr_tries++ >= online_vcpus)
   1958				return;
   1959		} while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu)));
   1960	}
   1961	dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
   1962
   1963	/* make the VCPU drop out of the SIE, or wake it up if sleeping */
   1964	switch (type) {
   1965	case KVM_S390_MCHK:
   1966		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
   1967		break;
   1968	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
   1969		if (!(type & KVM_S390_INT_IO_AI_MASK &&
   1970		      kvm->arch.gisa_int.origin) ||
   1971		      kvm_s390_pv_cpu_get_handle(dst_vcpu))
   1972			kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
   1973		break;
   1974	default:
   1975		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
   1976		break;
   1977	}
   1978	kvm_s390_vcpu_wakeup(dst_vcpu);
   1979}
   1980
   1981static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
   1982{
   1983	u64 type = READ_ONCE(inti->type);
   1984	int rc;
   1985
   1986	switch (type) {
   1987	case KVM_S390_MCHK:
   1988		rc = __inject_float_mchk(kvm, inti);
   1989		break;
   1990	case KVM_S390_INT_VIRTIO:
   1991		rc = __inject_virtio(kvm, inti);
   1992		break;
   1993	case KVM_S390_INT_SERVICE:
   1994		rc = __inject_service(kvm, inti);
   1995		break;
   1996	case KVM_S390_INT_PFAULT_DONE:
   1997		rc = __inject_pfault_done(kvm, inti);
   1998		break;
   1999	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
   2000		rc = __inject_io(kvm, inti);
   2001		break;
   2002	default:
   2003		rc = -EINVAL;
   2004	}
   2005	if (rc)
   2006		return rc;
   2007
   2008	__floating_irq_kick(kvm, type);
   2009	return 0;
   2010}
   2011
   2012int kvm_s390_inject_vm(struct kvm *kvm,
   2013		       struct kvm_s390_interrupt *s390int)
   2014{
   2015	struct kvm_s390_interrupt_info *inti;
   2016	int rc;
   2017
   2018	inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
   2019	if (!inti)
   2020		return -ENOMEM;
   2021
   2022	inti->type = s390int->type;
   2023	switch (inti->type) {
   2024	case KVM_S390_INT_VIRTIO:
   2025		VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx",
   2026			 s390int->parm, s390int->parm64);
   2027		inti->ext.ext_params = s390int->parm;
   2028		inti->ext.ext_params2 = s390int->parm64;
   2029		break;
   2030	case KVM_S390_INT_SERVICE:
   2031		VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm);
   2032		inti->ext.ext_params = s390int->parm;
   2033		break;
   2034	case KVM_S390_INT_PFAULT_DONE:
   2035		inti->ext.ext_params2 = s390int->parm64;
   2036		break;
   2037	case KVM_S390_MCHK:
   2038		VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx",
   2039			 s390int->parm64);
   2040		inti->mchk.cr14 = s390int->parm; /* upper bits are not used */
   2041		inti->mchk.mcic = s390int->parm64;
   2042		break;
   2043	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
   2044		inti->io.subchannel_id = s390int->parm >> 16;
   2045		inti->io.subchannel_nr = s390int->parm & 0x0000ffffu;
   2046		inti->io.io_int_parm = s390int->parm64 >> 32;
   2047		inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull;
   2048		break;
   2049	default:
   2050		kfree(inti);
   2051		return -EINVAL;
   2052	}
   2053	trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64,
   2054				 2);
   2055
   2056	rc = __inject_vm(kvm, inti);
   2057	if (rc)
   2058		kfree(inti);
   2059	return rc;
   2060}
   2061
   2062int kvm_s390_reinject_io_int(struct kvm *kvm,
   2063			      struct kvm_s390_interrupt_info *inti)
   2064{
   2065	return __inject_vm(kvm, inti);
   2066}
   2067
   2068int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
   2069		       struct kvm_s390_irq *irq)
   2070{
   2071	irq->type = s390int->type;
   2072	switch (irq->type) {
   2073	case KVM_S390_PROGRAM_INT:
   2074		if (s390int->parm & 0xffff0000)
   2075			return -EINVAL;
   2076		irq->u.pgm.code = s390int->parm;
   2077		break;
   2078	case KVM_S390_SIGP_SET_PREFIX:
   2079		irq->u.prefix.address = s390int->parm;
   2080		break;
   2081	case KVM_S390_SIGP_STOP:
   2082		irq->u.stop.flags = s390int->parm;
   2083		break;
   2084	case KVM_S390_INT_EXTERNAL_CALL:
   2085		if (s390int->parm & 0xffff0000)
   2086			return -EINVAL;
   2087		irq->u.extcall.code = s390int->parm;
   2088		break;
   2089	case KVM_S390_INT_EMERGENCY:
   2090		if (s390int->parm & 0xffff0000)
   2091			return -EINVAL;
   2092		irq->u.emerg.code = s390int->parm;
   2093		break;
   2094	case KVM_S390_MCHK:
   2095		irq->u.mchk.mcic = s390int->parm64;
   2096		break;
   2097	case KVM_S390_INT_PFAULT_INIT:
   2098		irq->u.ext.ext_params = s390int->parm;
   2099		irq->u.ext.ext_params2 = s390int->parm64;
   2100		break;
   2101	case KVM_S390_RESTART:
   2102	case KVM_S390_INT_CLOCK_COMP:
   2103	case KVM_S390_INT_CPU_TIMER:
   2104		break;
   2105	default:
   2106		return -EINVAL;
   2107	}
   2108	return 0;
   2109}
   2110
   2111int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu)
   2112{
   2113	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   2114
   2115	return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
   2116}
   2117
   2118int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu)
   2119{
   2120	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   2121
   2122	return test_bit(IRQ_PEND_RESTART, &li->pending_irqs);
   2123}
   2124
   2125void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
   2126{
   2127	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   2128
   2129	spin_lock(&li->lock);
   2130	li->irq.stop.flags = 0;
   2131	clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs);
   2132	spin_unlock(&li->lock);
   2133}
   2134
   2135static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   2136{
   2137	int rc;
   2138
   2139	switch (irq->type) {
   2140	case KVM_S390_PROGRAM_INT:
   2141		rc = __inject_prog(vcpu, irq);
   2142		break;
   2143	case KVM_S390_SIGP_SET_PREFIX:
   2144		rc = __inject_set_prefix(vcpu, irq);
   2145		break;
   2146	case KVM_S390_SIGP_STOP:
   2147		rc = __inject_sigp_stop(vcpu, irq);
   2148		break;
   2149	case KVM_S390_RESTART:
   2150		rc = __inject_sigp_restart(vcpu);
   2151		break;
   2152	case KVM_S390_INT_CLOCK_COMP:
   2153		rc = __inject_ckc(vcpu);
   2154		break;
   2155	case KVM_S390_INT_CPU_TIMER:
   2156		rc = __inject_cpu_timer(vcpu);
   2157		break;
   2158	case KVM_S390_INT_EXTERNAL_CALL:
   2159		rc = __inject_extcall(vcpu, irq);
   2160		break;
   2161	case KVM_S390_INT_EMERGENCY:
   2162		rc = __inject_sigp_emergency(vcpu, irq);
   2163		break;
   2164	case KVM_S390_MCHK:
   2165		rc = __inject_mchk(vcpu, irq);
   2166		break;
   2167	case KVM_S390_INT_PFAULT_INIT:
   2168		rc = __inject_pfault_init(vcpu, irq);
   2169		break;
   2170	case KVM_S390_INT_VIRTIO:
   2171	case KVM_S390_INT_SERVICE:
   2172	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
   2173	default:
   2174		rc = -EINVAL;
   2175	}
   2176
   2177	return rc;
   2178}
   2179
   2180int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
   2181{
   2182	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   2183	int rc;
   2184
   2185	spin_lock(&li->lock);
   2186	rc = do_inject_vcpu(vcpu, irq);
   2187	spin_unlock(&li->lock);
   2188	if (!rc)
   2189		kvm_s390_vcpu_wakeup(vcpu);
   2190	return rc;
   2191}
   2192
   2193static inline void clear_irq_list(struct list_head *_list)
   2194{
   2195	struct kvm_s390_interrupt_info *inti, *n;
   2196
   2197	list_for_each_entry_safe(inti, n, _list, list) {
   2198		list_del(&inti->list);
   2199		kfree(inti);
   2200	}
   2201}
   2202
   2203static void inti_to_irq(struct kvm_s390_interrupt_info *inti,
   2204		       struct kvm_s390_irq *irq)
   2205{
   2206	irq->type = inti->type;
   2207	switch (inti->type) {
   2208	case KVM_S390_INT_PFAULT_INIT:
   2209	case KVM_S390_INT_PFAULT_DONE:
   2210	case KVM_S390_INT_VIRTIO:
   2211		irq->u.ext = inti->ext;
   2212		break;
   2213	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
   2214		irq->u.io = inti->io;
   2215		break;
   2216	}
   2217}
   2218
   2219void kvm_s390_clear_float_irqs(struct kvm *kvm)
   2220{
   2221	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   2222	int i;
   2223
   2224	mutex_lock(&kvm->lock);
   2225	if (!kvm_s390_pv_is_protected(kvm))
   2226		fi->masked_irqs = 0;
   2227	mutex_unlock(&kvm->lock);
   2228	spin_lock(&fi->lock);
   2229	fi->pending_irqs = 0;
   2230	memset(&fi->srv_signal, 0, sizeof(fi->srv_signal));
   2231	memset(&fi->mchk, 0, sizeof(fi->mchk));
   2232	for (i = 0; i < FIRQ_LIST_COUNT; i++)
   2233		clear_irq_list(&fi->lists[i]);
   2234	for (i = 0; i < FIRQ_MAX_COUNT; i++)
   2235		fi->counters[i] = 0;
   2236	spin_unlock(&fi->lock);
   2237	kvm_s390_gisa_clear(kvm);
   2238};
   2239
   2240static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
   2241{
   2242	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   2243	struct kvm_s390_interrupt_info *inti;
   2244	struct kvm_s390_float_interrupt *fi;
   2245	struct kvm_s390_irq *buf;
   2246	struct kvm_s390_irq *irq;
   2247	int max_irqs;
   2248	int ret = 0;
   2249	int n = 0;
   2250	int i;
   2251
   2252	if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0)
   2253		return -EINVAL;
   2254
   2255	/*
   2256	 * We are already using -ENOMEM to signal
   2257	 * userspace it may retry with a bigger buffer,
   2258	 * so we need to use something else for this case
   2259	 */
   2260	buf = vzalloc(len);
   2261	if (!buf)
   2262		return -ENOBUFS;
   2263
   2264	max_irqs = len / sizeof(struct kvm_s390_irq);
   2265
   2266	if (gi->origin && gisa_get_ipm(gi->origin)) {
   2267		for (i = 0; i <= MAX_ISC; i++) {
   2268			if (n == max_irqs) {
   2269				/* signal userspace to try again */
   2270				ret = -ENOMEM;
   2271				goto out_nolock;
   2272			}
   2273			if (gisa_tac_ipm_gisc(gi->origin, i)) {
   2274				irq = (struct kvm_s390_irq *) &buf[n];
   2275				irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
   2276				irq->u.io.io_int_word = isc_to_int_word(i);
   2277				n++;
   2278			}
   2279		}
   2280	}
   2281	fi = &kvm->arch.float_int;
   2282	spin_lock(&fi->lock);
   2283	for (i = 0; i < FIRQ_LIST_COUNT; i++) {
   2284		list_for_each_entry(inti, &fi->lists[i], list) {
   2285			if (n == max_irqs) {
   2286				/* signal userspace to try again */
   2287				ret = -ENOMEM;
   2288				goto out;
   2289			}
   2290			inti_to_irq(inti, &buf[n]);
   2291			n++;
   2292		}
   2293	}
   2294	if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) ||
   2295	    test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) {
   2296		if (n == max_irqs) {
   2297			/* signal userspace to try again */
   2298			ret = -ENOMEM;
   2299			goto out;
   2300		}
   2301		irq = (struct kvm_s390_irq *) &buf[n];
   2302		irq->type = KVM_S390_INT_SERVICE;
   2303		irq->u.ext = fi->srv_signal;
   2304		n++;
   2305	}
   2306	if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
   2307		if (n == max_irqs) {
   2308				/* signal userspace to try again */
   2309				ret = -ENOMEM;
   2310				goto out;
   2311		}
   2312		irq = (struct kvm_s390_irq *) &buf[n];
   2313		irq->type = KVM_S390_MCHK;
   2314		irq->u.mchk = fi->mchk;
   2315		n++;
   2316}
   2317
   2318out:
   2319	spin_unlock(&fi->lock);
   2320out_nolock:
   2321	if (!ret && n > 0) {
   2322		if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
   2323			ret = -EFAULT;
   2324	}
   2325	vfree(buf);
   2326
   2327	return ret < 0 ? ret : n;
   2328}
   2329
   2330static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr)
   2331{
   2332	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   2333	struct kvm_s390_ais_all ais;
   2334
   2335	if (attr->attr < sizeof(ais))
   2336		return -EINVAL;
   2337
   2338	if (!test_kvm_facility(kvm, 72))
   2339		return -EOPNOTSUPP;
   2340
   2341	mutex_lock(&fi->ais_lock);
   2342	ais.simm = fi->simm;
   2343	ais.nimm = fi->nimm;
   2344	mutex_unlock(&fi->ais_lock);
   2345
   2346	if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais)))
   2347		return -EFAULT;
   2348
   2349	return 0;
   2350}
   2351
   2352static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
   2353{
   2354	int r;
   2355
   2356	switch (attr->group) {
   2357	case KVM_DEV_FLIC_GET_ALL_IRQS:
   2358		r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr,
   2359					  attr->attr);
   2360		break;
   2361	case KVM_DEV_FLIC_AISM_ALL:
   2362		r = flic_ais_mode_get_all(dev->kvm, attr);
   2363		break;
   2364	default:
   2365		r = -EINVAL;
   2366	}
   2367
   2368	return r;
   2369}
   2370
   2371static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti,
   2372				     u64 addr)
   2373{
   2374	struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr;
   2375	void *target = NULL;
   2376	void __user *source;
   2377	u64 size;
   2378
   2379	if (get_user(inti->type, (u64 __user *)addr))
   2380		return -EFAULT;
   2381
   2382	switch (inti->type) {
   2383	case KVM_S390_INT_PFAULT_INIT:
   2384	case KVM_S390_INT_PFAULT_DONE:
   2385	case KVM_S390_INT_VIRTIO:
   2386	case KVM_S390_INT_SERVICE:
   2387		target = (void *) &inti->ext;
   2388		source = &uptr->u.ext;
   2389		size = sizeof(inti->ext);
   2390		break;
   2391	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
   2392		target = (void *) &inti->io;
   2393		source = &uptr->u.io;
   2394		size = sizeof(inti->io);
   2395		break;
   2396	case KVM_S390_MCHK:
   2397		target = (void *) &inti->mchk;
   2398		source = &uptr->u.mchk;
   2399		size = sizeof(inti->mchk);
   2400		break;
   2401	default:
   2402		return -EINVAL;
   2403	}
   2404
   2405	if (copy_from_user(target, source, size))
   2406		return -EFAULT;
   2407
   2408	return 0;
   2409}
   2410
   2411static int enqueue_floating_irq(struct kvm_device *dev,
   2412				struct kvm_device_attr *attr)
   2413{
   2414	struct kvm_s390_interrupt_info *inti = NULL;
   2415	int r = 0;
   2416	int len = attr->attr;
   2417
   2418	if (len % sizeof(struct kvm_s390_irq) != 0)
   2419		return -EINVAL;
   2420	else if (len > KVM_S390_FLIC_MAX_BUFFER)
   2421		return -EINVAL;
   2422
   2423	while (len >= sizeof(struct kvm_s390_irq)) {
   2424		inti = kzalloc(sizeof(*inti), GFP_KERNEL_ACCOUNT);
   2425		if (!inti)
   2426			return -ENOMEM;
   2427
   2428		r = copy_irq_from_user(inti, attr->addr);
   2429		if (r) {
   2430			kfree(inti);
   2431			return r;
   2432		}
   2433		r = __inject_vm(dev->kvm, inti);
   2434		if (r) {
   2435			kfree(inti);
   2436			return r;
   2437		}
   2438		len -= sizeof(struct kvm_s390_irq);
   2439		attr->addr += sizeof(struct kvm_s390_irq);
   2440	}
   2441
   2442	return r;
   2443}
   2444
   2445static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id)
   2446{
   2447	if (id >= MAX_S390_IO_ADAPTERS)
   2448		return NULL;
   2449	id = array_index_nospec(id, MAX_S390_IO_ADAPTERS);
   2450	return kvm->arch.adapters[id];
   2451}
   2452
   2453static int register_io_adapter(struct kvm_device *dev,
   2454			       struct kvm_device_attr *attr)
   2455{
   2456	struct s390_io_adapter *adapter;
   2457	struct kvm_s390_io_adapter adapter_info;
   2458
   2459	if (copy_from_user(&adapter_info,
   2460			   (void __user *)attr->addr, sizeof(adapter_info)))
   2461		return -EFAULT;
   2462
   2463	if (adapter_info.id >= MAX_S390_IO_ADAPTERS)
   2464		return -EINVAL;
   2465
   2466	adapter_info.id = array_index_nospec(adapter_info.id,
   2467					     MAX_S390_IO_ADAPTERS);
   2468
   2469	if (dev->kvm->arch.adapters[adapter_info.id] != NULL)
   2470		return -EINVAL;
   2471
   2472	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL_ACCOUNT);
   2473	if (!adapter)
   2474		return -ENOMEM;
   2475
   2476	adapter->id = adapter_info.id;
   2477	adapter->isc = adapter_info.isc;
   2478	adapter->maskable = adapter_info.maskable;
   2479	adapter->masked = false;
   2480	adapter->swap = adapter_info.swap;
   2481	adapter->suppressible = (adapter_info.flags) &
   2482				KVM_S390_ADAPTER_SUPPRESSIBLE;
   2483	dev->kvm->arch.adapters[adapter->id] = adapter;
   2484
   2485	return 0;
   2486}
   2487
   2488int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked)
   2489{
   2490	int ret;
   2491	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
   2492
   2493	if (!adapter || !adapter->maskable)
   2494		return -EINVAL;
   2495	ret = adapter->masked;
   2496	adapter->masked = masked;
   2497	return ret;
   2498}
   2499
   2500void kvm_s390_destroy_adapters(struct kvm *kvm)
   2501{
   2502	int i;
   2503
   2504	for (i = 0; i < MAX_S390_IO_ADAPTERS; i++)
   2505		kfree(kvm->arch.adapters[i]);
   2506}
   2507
   2508static int modify_io_adapter(struct kvm_device *dev,
   2509			     struct kvm_device_attr *attr)
   2510{
   2511	struct kvm_s390_io_adapter_req req;
   2512	struct s390_io_adapter *adapter;
   2513	int ret;
   2514
   2515	if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
   2516		return -EFAULT;
   2517
   2518	adapter = get_io_adapter(dev->kvm, req.id);
   2519	if (!adapter)
   2520		return -EINVAL;
   2521	switch (req.type) {
   2522	case KVM_S390_IO_ADAPTER_MASK:
   2523		ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask);
   2524		if (ret > 0)
   2525			ret = 0;
   2526		break;
   2527	/*
   2528	 * The following operations are no longer needed and therefore no-ops.
   2529	 * The gpa to hva translation is done when an IRQ route is set up. The
   2530	 * set_irq code uses get_user_pages_remote() to do the actual write.
   2531	 */
   2532	case KVM_S390_IO_ADAPTER_MAP:
   2533	case KVM_S390_IO_ADAPTER_UNMAP:
   2534		ret = 0;
   2535		break;
   2536	default:
   2537		ret = -EINVAL;
   2538	}
   2539
   2540	return ret;
   2541}
   2542
   2543static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr)
   2544
   2545{
   2546	const u64 isc_mask = 0xffUL << 24; /* all iscs set */
   2547	u32 schid;
   2548
   2549	if (attr->flags)
   2550		return -EINVAL;
   2551	if (attr->attr != sizeof(schid))
   2552		return -EINVAL;
   2553	if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid)))
   2554		return -EFAULT;
   2555	if (!schid)
   2556		return -EINVAL;
   2557	kfree(kvm_s390_get_io_int(kvm, isc_mask, schid));
   2558	/*
   2559	 * If userspace is conforming to the architecture, we can have at most
   2560	 * one pending I/O interrupt per subchannel, so this is effectively a
   2561	 * clear all.
   2562	 */
   2563	return 0;
   2564}
   2565
   2566static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr)
   2567{
   2568	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   2569	struct kvm_s390_ais_req req;
   2570	int ret = 0;
   2571
   2572	if (!test_kvm_facility(kvm, 72))
   2573		return -EOPNOTSUPP;
   2574
   2575	if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req)))
   2576		return -EFAULT;
   2577
   2578	if (req.isc > MAX_ISC)
   2579		return -EINVAL;
   2580
   2581	trace_kvm_s390_modify_ais_mode(req.isc,
   2582				       (fi->simm & AIS_MODE_MASK(req.isc)) ?
   2583				       (fi->nimm & AIS_MODE_MASK(req.isc)) ?
   2584				       2 : KVM_S390_AIS_MODE_SINGLE :
   2585				       KVM_S390_AIS_MODE_ALL, req.mode);
   2586
   2587	mutex_lock(&fi->ais_lock);
   2588	switch (req.mode) {
   2589	case KVM_S390_AIS_MODE_ALL:
   2590		fi->simm &= ~AIS_MODE_MASK(req.isc);
   2591		fi->nimm &= ~AIS_MODE_MASK(req.isc);
   2592		break;
   2593	case KVM_S390_AIS_MODE_SINGLE:
   2594		fi->simm |= AIS_MODE_MASK(req.isc);
   2595		fi->nimm &= ~AIS_MODE_MASK(req.isc);
   2596		break;
   2597	default:
   2598		ret = -EINVAL;
   2599	}
   2600	mutex_unlock(&fi->ais_lock);
   2601
   2602	return ret;
   2603}
   2604
   2605static int kvm_s390_inject_airq(struct kvm *kvm,
   2606				struct s390_io_adapter *adapter)
   2607{
   2608	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   2609	struct kvm_s390_interrupt s390int = {
   2610		.type = KVM_S390_INT_IO(1, 0, 0, 0),
   2611		.parm = 0,
   2612		.parm64 = isc_to_int_word(adapter->isc),
   2613	};
   2614	int ret = 0;
   2615
   2616	if (!test_kvm_facility(kvm, 72) || !adapter->suppressible)
   2617		return kvm_s390_inject_vm(kvm, &s390int);
   2618
   2619	mutex_lock(&fi->ais_lock);
   2620	if (fi->nimm & AIS_MODE_MASK(adapter->isc)) {
   2621		trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc);
   2622		goto out;
   2623	}
   2624
   2625	ret = kvm_s390_inject_vm(kvm, &s390int);
   2626	if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) {
   2627		fi->nimm |= AIS_MODE_MASK(adapter->isc);
   2628		trace_kvm_s390_modify_ais_mode(adapter->isc,
   2629					       KVM_S390_AIS_MODE_SINGLE, 2);
   2630	}
   2631out:
   2632	mutex_unlock(&fi->ais_lock);
   2633	return ret;
   2634}
   2635
   2636static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr)
   2637{
   2638	unsigned int id = attr->attr;
   2639	struct s390_io_adapter *adapter = get_io_adapter(kvm, id);
   2640
   2641	if (!adapter)
   2642		return -EINVAL;
   2643
   2644	return kvm_s390_inject_airq(kvm, adapter);
   2645}
   2646
   2647static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr)
   2648{
   2649	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
   2650	struct kvm_s390_ais_all ais;
   2651
   2652	if (!test_kvm_facility(kvm, 72))
   2653		return -EOPNOTSUPP;
   2654
   2655	if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais)))
   2656		return -EFAULT;
   2657
   2658	mutex_lock(&fi->ais_lock);
   2659	fi->simm = ais.simm;
   2660	fi->nimm = ais.nimm;
   2661	mutex_unlock(&fi->ais_lock);
   2662
   2663	return 0;
   2664}
   2665
   2666static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
   2667{
   2668	int r = 0;
   2669	unsigned long i;
   2670	struct kvm_vcpu *vcpu;
   2671
   2672	switch (attr->group) {
   2673	case KVM_DEV_FLIC_ENQUEUE:
   2674		r = enqueue_floating_irq(dev, attr);
   2675		break;
   2676	case KVM_DEV_FLIC_CLEAR_IRQS:
   2677		kvm_s390_clear_float_irqs(dev->kvm);
   2678		break;
   2679	case KVM_DEV_FLIC_APF_ENABLE:
   2680		dev->kvm->arch.gmap->pfault_enabled = 1;
   2681		break;
   2682	case KVM_DEV_FLIC_APF_DISABLE_WAIT:
   2683		dev->kvm->arch.gmap->pfault_enabled = 0;
   2684		/*
   2685		 * Make sure no async faults are in transition when
   2686		 * clearing the queues. So we don't need to worry
   2687		 * about late coming workers.
   2688		 */
   2689		synchronize_srcu(&dev->kvm->srcu);
   2690		kvm_for_each_vcpu(i, vcpu, dev->kvm)
   2691			kvm_clear_async_pf_completion_queue(vcpu);
   2692		break;
   2693	case KVM_DEV_FLIC_ADAPTER_REGISTER:
   2694		r = register_io_adapter(dev, attr);
   2695		break;
   2696	case KVM_DEV_FLIC_ADAPTER_MODIFY:
   2697		r = modify_io_adapter(dev, attr);
   2698		break;
   2699	case KVM_DEV_FLIC_CLEAR_IO_IRQ:
   2700		r = clear_io_irq(dev->kvm, attr);
   2701		break;
   2702	case KVM_DEV_FLIC_AISM:
   2703		r = modify_ais_mode(dev->kvm, attr);
   2704		break;
   2705	case KVM_DEV_FLIC_AIRQ_INJECT:
   2706		r = flic_inject_airq(dev->kvm, attr);
   2707		break;
   2708	case KVM_DEV_FLIC_AISM_ALL:
   2709		r = flic_ais_mode_set_all(dev->kvm, attr);
   2710		break;
   2711	default:
   2712		r = -EINVAL;
   2713	}
   2714
   2715	return r;
   2716}
   2717
   2718static int flic_has_attr(struct kvm_device *dev,
   2719			     struct kvm_device_attr *attr)
   2720{
   2721	switch (attr->group) {
   2722	case KVM_DEV_FLIC_GET_ALL_IRQS:
   2723	case KVM_DEV_FLIC_ENQUEUE:
   2724	case KVM_DEV_FLIC_CLEAR_IRQS:
   2725	case KVM_DEV_FLIC_APF_ENABLE:
   2726	case KVM_DEV_FLIC_APF_DISABLE_WAIT:
   2727	case KVM_DEV_FLIC_ADAPTER_REGISTER:
   2728	case KVM_DEV_FLIC_ADAPTER_MODIFY:
   2729	case KVM_DEV_FLIC_CLEAR_IO_IRQ:
   2730	case KVM_DEV_FLIC_AISM:
   2731	case KVM_DEV_FLIC_AIRQ_INJECT:
   2732	case KVM_DEV_FLIC_AISM_ALL:
   2733		return 0;
   2734	}
   2735	return -ENXIO;
   2736}
   2737
   2738static int flic_create(struct kvm_device *dev, u32 type)
   2739{
   2740	if (!dev)
   2741		return -EINVAL;
   2742	if (dev->kvm->arch.flic)
   2743		return -EINVAL;
   2744	dev->kvm->arch.flic = dev;
   2745	return 0;
   2746}
   2747
   2748static void flic_destroy(struct kvm_device *dev)
   2749{
   2750	dev->kvm->arch.flic = NULL;
   2751	kfree(dev);
   2752}
   2753
   2754/* s390 floating irq controller (flic) */
   2755struct kvm_device_ops kvm_flic_ops = {
   2756	.name = "kvm-flic",
   2757	.get_attr = flic_get_attr,
   2758	.set_attr = flic_set_attr,
   2759	.has_attr = flic_has_attr,
   2760	.create = flic_create,
   2761	.destroy = flic_destroy,
   2762};
   2763
   2764static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
   2765{
   2766	unsigned long bit;
   2767
   2768	bit = bit_nr + (addr % PAGE_SIZE) * 8;
   2769
   2770	return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
   2771}
   2772
   2773static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
   2774{
   2775	struct page *page = NULL;
   2776
   2777	mmap_read_lock(kvm->mm);
   2778	get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
   2779			      &page, NULL, NULL);
   2780	mmap_read_unlock(kvm->mm);
   2781	return page;
   2782}
   2783
   2784static int adapter_indicators_set(struct kvm *kvm,
   2785				  struct s390_io_adapter *adapter,
   2786				  struct kvm_s390_adapter_int *adapter_int)
   2787{
   2788	unsigned long bit;
   2789	int summary_set, idx;
   2790	struct page *ind_page, *summary_page;
   2791	void *map;
   2792
   2793	ind_page = get_map_page(kvm, adapter_int->ind_addr);
   2794	if (!ind_page)
   2795		return -1;
   2796	summary_page = get_map_page(kvm, adapter_int->summary_addr);
   2797	if (!summary_page) {
   2798		put_page(ind_page);
   2799		return -1;
   2800	}
   2801
   2802	idx = srcu_read_lock(&kvm->srcu);
   2803	map = page_address(ind_page);
   2804	bit = get_ind_bit(adapter_int->ind_addr,
   2805			  adapter_int->ind_offset, adapter->swap);
   2806	set_bit(bit, map);
   2807	mark_page_dirty(kvm, adapter_int->ind_addr >> PAGE_SHIFT);
   2808	set_page_dirty_lock(ind_page);
   2809	map = page_address(summary_page);
   2810	bit = get_ind_bit(adapter_int->summary_addr,
   2811			  adapter_int->summary_offset, adapter->swap);
   2812	summary_set = test_and_set_bit(bit, map);
   2813	mark_page_dirty(kvm, adapter_int->summary_addr >> PAGE_SHIFT);
   2814	set_page_dirty_lock(summary_page);
   2815	srcu_read_unlock(&kvm->srcu, idx);
   2816
   2817	put_page(ind_page);
   2818	put_page(summary_page);
   2819	return summary_set ? 0 : 1;
   2820}
   2821
   2822/*
   2823 * < 0 - not injected due to error
   2824 * = 0 - coalesced, summary indicator already active
   2825 * > 0 - injected interrupt
   2826 */
   2827static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e,
   2828			   struct kvm *kvm, int irq_source_id, int level,
   2829			   bool line_status)
   2830{
   2831	int ret;
   2832	struct s390_io_adapter *adapter;
   2833
   2834	/* We're only interested in the 0->1 transition. */
   2835	if (!level)
   2836		return 0;
   2837	adapter = get_io_adapter(kvm, e->adapter.adapter_id);
   2838	if (!adapter)
   2839		return -1;
   2840	ret = adapter_indicators_set(kvm, adapter, &e->adapter);
   2841	if ((ret > 0) && !adapter->masked) {
   2842		ret = kvm_s390_inject_airq(kvm, adapter);
   2843		if (ret == 0)
   2844			ret = 1;
   2845	}
   2846	return ret;
   2847}
   2848
   2849/*
   2850 * Inject the machine check to the guest.
   2851 */
   2852void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu,
   2853				     struct mcck_volatile_info *mcck_info)
   2854{
   2855	struct kvm_s390_interrupt_info inti;
   2856	struct kvm_s390_irq irq;
   2857	struct kvm_s390_mchk_info *mchk;
   2858	union mci mci;
   2859	__u64 cr14 = 0;         /* upper bits are not used */
   2860	int rc;
   2861
   2862	mci.val = mcck_info->mcic;
   2863	if (mci.sr)
   2864		cr14 |= CR14_RECOVERY_SUBMASK;
   2865	if (mci.dg)
   2866		cr14 |= CR14_DEGRADATION_SUBMASK;
   2867	if (mci.w)
   2868		cr14 |= CR14_WARNING_SUBMASK;
   2869
   2870	mchk = mci.ck ? &inti.mchk : &irq.u.mchk;
   2871	mchk->cr14 = cr14;
   2872	mchk->mcic = mcck_info->mcic;
   2873	mchk->ext_damage_code = mcck_info->ext_damage_code;
   2874	mchk->failing_storage_address = mcck_info->failing_storage_address;
   2875	if (mci.ck) {
   2876		/* Inject the floating machine check */
   2877		inti.type = KVM_S390_MCHK;
   2878		rc = __inject_vm(vcpu->kvm, &inti);
   2879	} else {
   2880		/* Inject the machine check to specified vcpu */
   2881		irq.type = KVM_S390_MCHK;
   2882		rc = kvm_s390_inject_vcpu(vcpu, &irq);
   2883	}
   2884	WARN_ON_ONCE(rc);
   2885}
   2886
   2887int kvm_set_routing_entry(struct kvm *kvm,
   2888			  struct kvm_kernel_irq_routing_entry *e,
   2889			  const struct kvm_irq_routing_entry *ue)
   2890{
   2891	u64 uaddr;
   2892
   2893	switch (ue->type) {
   2894	/* we store the userspace addresses instead of the guest addresses */
   2895	case KVM_IRQ_ROUTING_S390_ADAPTER:
   2896		e->set = set_adapter_int;
   2897		uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.summary_addr);
   2898		if (uaddr == -EFAULT)
   2899			return -EFAULT;
   2900		e->adapter.summary_addr = uaddr;
   2901		uaddr =  gmap_translate(kvm->arch.gmap, ue->u.adapter.ind_addr);
   2902		if (uaddr == -EFAULT)
   2903			return -EFAULT;
   2904		e->adapter.ind_addr = uaddr;
   2905		e->adapter.summary_offset = ue->u.adapter.summary_offset;
   2906		e->adapter.ind_offset = ue->u.adapter.ind_offset;
   2907		e->adapter.adapter_id = ue->u.adapter.adapter_id;
   2908		return 0;
   2909	default:
   2910		return -EINVAL;
   2911	}
   2912}
   2913
   2914int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm,
   2915		int irq_source_id, int level, bool line_status)
   2916{
   2917	return -EINVAL;
   2918}
   2919
   2920int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len)
   2921{
   2922	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   2923	struct kvm_s390_irq *buf;
   2924	int r = 0;
   2925	int n;
   2926
   2927	buf = vmalloc(len);
   2928	if (!buf)
   2929		return -ENOMEM;
   2930
   2931	if (copy_from_user((void *) buf, irqstate, len)) {
   2932		r = -EFAULT;
   2933		goto out_free;
   2934	}
   2935
   2936	/*
   2937	 * Don't allow setting the interrupt state
   2938	 * when there are already interrupts pending
   2939	 */
   2940	spin_lock(&li->lock);
   2941	if (li->pending_irqs) {
   2942		r = -EBUSY;
   2943		goto out_unlock;
   2944	}
   2945
   2946	for (n = 0; n < len / sizeof(*buf); n++) {
   2947		r = do_inject_vcpu(vcpu, &buf[n]);
   2948		if (r)
   2949			break;
   2950	}
   2951
   2952out_unlock:
   2953	spin_unlock(&li->lock);
   2954out_free:
   2955	vfree(buf);
   2956
   2957	return r;
   2958}
   2959
   2960static void store_local_irq(struct kvm_s390_local_interrupt *li,
   2961			    struct kvm_s390_irq *irq,
   2962			    unsigned long irq_type)
   2963{
   2964	switch (irq_type) {
   2965	case IRQ_PEND_MCHK_EX:
   2966	case IRQ_PEND_MCHK_REP:
   2967		irq->type = KVM_S390_MCHK;
   2968		irq->u.mchk = li->irq.mchk;
   2969		break;
   2970	case IRQ_PEND_PROG:
   2971		irq->type = KVM_S390_PROGRAM_INT;
   2972		irq->u.pgm = li->irq.pgm;
   2973		break;
   2974	case IRQ_PEND_PFAULT_INIT:
   2975		irq->type = KVM_S390_INT_PFAULT_INIT;
   2976		irq->u.ext = li->irq.ext;
   2977		break;
   2978	case IRQ_PEND_EXT_EXTERNAL:
   2979		irq->type = KVM_S390_INT_EXTERNAL_CALL;
   2980		irq->u.extcall = li->irq.extcall;
   2981		break;
   2982	case IRQ_PEND_EXT_CLOCK_COMP:
   2983		irq->type = KVM_S390_INT_CLOCK_COMP;
   2984		break;
   2985	case IRQ_PEND_EXT_CPU_TIMER:
   2986		irq->type = KVM_S390_INT_CPU_TIMER;
   2987		break;
   2988	case IRQ_PEND_SIGP_STOP:
   2989		irq->type = KVM_S390_SIGP_STOP;
   2990		irq->u.stop = li->irq.stop;
   2991		break;
   2992	case IRQ_PEND_RESTART:
   2993		irq->type = KVM_S390_RESTART;
   2994		break;
   2995	case IRQ_PEND_SET_PREFIX:
   2996		irq->type = KVM_S390_SIGP_SET_PREFIX;
   2997		irq->u.prefix = li->irq.prefix;
   2998		break;
   2999	}
   3000}
   3001
   3002int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
   3003{
   3004	int scn;
   3005	DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
   3006	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
   3007	unsigned long pending_irqs;
   3008	struct kvm_s390_irq irq;
   3009	unsigned long irq_type;
   3010	int cpuaddr;
   3011	int n = 0;
   3012
   3013	spin_lock(&li->lock);
   3014	pending_irqs = li->pending_irqs;
   3015	memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending,
   3016	       sizeof(sigp_emerg_pending));
   3017	spin_unlock(&li->lock);
   3018
   3019	for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) {
   3020		memset(&irq, 0, sizeof(irq));
   3021		if (irq_type == IRQ_PEND_EXT_EMERGENCY)
   3022			continue;
   3023		if (n + sizeof(irq) > len)
   3024			return -ENOBUFS;
   3025		store_local_irq(&vcpu->arch.local_int, &irq, irq_type);
   3026		if (copy_to_user(&buf[n], &irq, sizeof(irq)))
   3027			return -EFAULT;
   3028		n += sizeof(irq);
   3029	}
   3030
   3031	if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) {
   3032		for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) {
   3033			memset(&irq, 0, sizeof(irq));
   3034			if (n + sizeof(irq) > len)
   3035				return -ENOBUFS;
   3036			irq.type = KVM_S390_INT_EMERGENCY;
   3037			irq.u.emerg.code = cpuaddr;
   3038			if (copy_to_user(&buf[n], &irq, sizeof(irq)))
   3039				return -EFAULT;
   3040			n += sizeof(irq);
   3041		}
   3042	}
   3043
   3044	if (sca_ext_call_pending(vcpu, &scn)) {
   3045		if (n + sizeof(irq) > len)
   3046			return -ENOBUFS;
   3047		memset(&irq, 0, sizeof(irq));
   3048		irq.type = KVM_S390_INT_EXTERNAL_CALL;
   3049		irq.u.extcall.code = scn;
   3050		if (copy_to_user(&buf[n], &irq, sizeof(irq)))
   3051			return -EFAULT;
   3052		n += sizeof(irq);
   3053	}
   3054
   3055	return n;
   3056}
   3057
   3058static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask)
   3059{
   3060	int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus);
   3061	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3062	struct kvm_vcpu *vcpu;
   3063	u8 vcpu_isc_mask;
   3064
   3065	for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) {
   3066		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
   3067		if (psw_ioint_disabled(vcpu))
   3068			continue;
   3069		vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24);
   3070		if (deliverable_mask & vcpu_isc_mask) {
   3071			/* lately kicked but not yet running */
   3072			if (test_and_set_bit(vcpu_idx, gi->kicked_mask))
   3073				return;
   3074			kvm_s390_vcpu_wakeup(vcpu);
   3075			return;
   3076		}
   3077	}
   3078}
   3079
   3080static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer)
   3081{
   3082	struct kvm_s390_gisa_interrupt *gi =
   3083		container_of(timer, struct kvm_s390_gisa_interrupt, timer);
   3084	struct kvm *kvm =
   3085		container_of(gi->origin, struct sie_page2, gisa)->kvm;
   3086	u8 pending_mask;
   3087
   3088	pending_mask = gisa_get_ipm_or_restore_iam(gi);
   3089	if (pending_mask) {
   3090		__airqs_kick_single_vcpu(kvm, pending_mask);
   3091		hrtimer_forward_now(timer, ns_to_ktime(gi->expires));
   3092		return HRTIMER_RESTART;
   3093	}
   3094
   3095	return HRTIMER_NORESTART;
   3096}
   3097
   3098#define NULL_GISA_ADDR 0x00000000UL
   3099#define NONE_GISA_ADDR 0x00000001UL
   3100#define GISA_ADDR_MASK 0xfffff000UL
   3101
   3102static void process_gib_alert_list(void)
   3103{
   3104	struct kvm_s390_gisa_interrupt *gi;
   3105	struct kvm_s390_gisa *gisa;
   3106	struct kvm *kvm;
   3107	u32 final, origin = 0UL;
   3108
   3109	do {
   3110		/*
   3111		 * If the NONE_GISA_ADDR is still stored in the alert list
   3112		 * origin, we will leave the outer loop. No further GISA has
   3113		 * been added to the alert list by millicode while processing
   3114		 * the current alert list.
   3115		 */
   3116		final = (origin & NONE_GISA_ADDR);
   3117		/*
   3118		 * Cut off the alert list and store the NONE_GISA_ADDR in the
   3119		 * alert list origin to avoid further GAL interruptions.
   3120		 * A new alert list can be build up by millicode in parallel
   3121		 * for guests not in the yet cut-off alert list. When in the
   3122		 * final loop, store the NULL_GISA_ADDR instead. This will re-
   3123		 * enable GAL interruptions on the host again.
   3124		 */
   3125		origin = xchg(&gib->alert_list_origin,
   3126			      (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR);
   3127		/*
   3128		 * Loop through the just cut-off alert list and start the
   3129		 * gisa timers to kick idle vcpus to consume the pending
   3130		 * interruptions asap.
   3131		 */
   3132		while (origin & GISA_ADDR_MASK) {
   3133			gisa = (struct kvm_s390_gisa *)(u64)origin;
   3134			origin = gisa->next_alert;
   3135			gisa->next_alert = (u32)(u64)gisa;
   3136			kvm = container_of(gisa, struct sie_page2, gisa)->kvm;
   3137			gi = &kvm->arch.gisa_int;
   3138			if (hrtimer_active(&gi->timer))
   3139				hrtimer_cancel(&gi->timer);
   3140			hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL);
   3141		}
   3142	} while (!final);
   3143
   3144}
   3145
   3146void kvm_s390_gisa_clear(struct kvm *kvm)
   3147{
   3148	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3149
   3150	if (!gi->origin)
   3151		return;
   3152	gisa_clear_ipm(gi->origin);
   3153	VM_EVENT(kvm, 3, "gisa 0x%pK cleared", gi->origin);
   3154}
   3155
   3156void kvm_s390_gisa_init(struct kvm *kvm)
   3157{
   3158	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3159
   3160	if (!css_general_characteristics.aiv)
   3161		return;
   3162	gi->origin = &kvm->arch.sie_page2->gisa;
   3163	gi->alert.mask = 0;
   3164	spin_lock_init(&gi->alert.ref_lock);
   3165	gi->expires = 50 * 1000; /* 50 usec */
   3166	hrtimer_init(&gi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
   3167	gi->timer.function = gisa_vcpu_kicker;
   3168	memset(gi->origin, 0, sizeof(struct kvm_s390_gisa));
   3169	gi->origin->next_alert = (u32)(u64)gi->origin;
   3170	VM_EVENT(kvm, 3, "gisa 0x%pK initialized", gi->origin);
   3171}
   3172
   3173void kvm_s390_gisa_enable(struct kvm *kvm)
   3174{
   3175	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3176	struct kvm_vcpu *vcpu;
   3177	unsigned long i;
   3178	u32 gisa_desc;
   3179
   3180	if (gi->origin)
   3181		return;
   3182	kvm_s390_gisa_init(kvm);
   3183	gisa_desc = kvm_s390_get_gisa_desc(kvm);
   3184	if (!gisa_desc)
   3185		return;
   3186	kvm_for_each_vcpu(i, vcpu, kvm) {
   3187		mutex_lock(&vcpu->mutex);
   3188		vcpu->arch.sie_block->gd = gisa_desc;
   3189		vcpu->arch.sie_block->eca |= ECA_AIV;
   3190		VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
   3191			   vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
   3192		mutex_unlock(&vcpu->mutex);
   3193	}
   3194}
   3195
   3196void kvm_s390_gisa_destroy(struct kvm *kvm)
   3197{
   3198	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3199	struct kvm_s390_gisa *gisa = gi->origin;
   3200
   3201	if (!gi->origin)
   3202		return;
   3203	if (gi->alert.mask)
   3204		KVM_EVENT(3, "vm 0x%pK has unexpected iam 0x%02x",
   3205			  kvm, gi->alert.mask);
   3206	while (gisa_in_alert_list(gi->origin))
   3207		cpu_relax();
   3208	hrtimer_cancel(&gi->timer);
   3209	gi->origin = NULL;
   3210	VM_EVENT(kvm, 3, "gisa 0x%pK destroyed", gisa);
   3211}
   3212
   3213void kvm_s390_gisa_disable(struct kvm *kvm)
   3214{
   3215	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3216	struct kvm_vcpu *vcpu;
   3217	unsigned long i;
   3218
   3219	if (!gi->origin)
   3220		return;
   3221	kvm_for_each_vcpu(i, vcpu, kvm) {
   3222		mutex_lock(&vcpu->mutex);
   3223		vcpu->arch.sie_block->eca &= ~ECA_AIV;
   3224		vcpu->arch.sie_block->gd = 0U;
   3225		mutex_unlock(&vcpu->mutex);
   3226		VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id);
   3227	}
   3228	kvm_s390_gisa_destroy(kvm);
   3229}
   3230
   3231/**
   3232 * kvm_s390_gisc_register - register a guest ISC
   3233 *
   3234 * @kvm:  the kernel vm to work with
   3235 * @gisc: the guest interruption sub class to register
   3236 *
   3237 * The function extends the vm specific alert mask to use.
   3238 * The effective IAM mask in the GISA is updated as well
   3239 * in case the GISA is not part of the GIB alert list.
   3240 * It will be updated latest when the IAM gets restored
   3241 * by gisa_get_ipm_or_restore_iam().
   3242 *
   3243 * Returns: the nonspecific ISC (NISC) the gib alert mechanism
   3244 *          has registered with the channel subsystem.
   3245 *          -ENODEV in case the vm uses no GISA
   3246 *          -ERANGE in case the guest ISC is invalid
   3247 */
   3248int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc)
   3249{
   3250	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3251
   3252	if (!gi->origin)
   3253		return -ENODEV;
   3254	if (gisc > MAX_ISC)
   3255		return -ERANGE;
   3256
   3257	spin_lock(&gi->alert.ref_lock);
   3258	gi->alert.ref_count[gisc]++;
   3259	if (gi->alert.ref_count[gisc] == 1) {
   3260		gi->alert.mask |= 0x80 >> gisc;
   3261		gisa_set_iam(gi->origin, gi->alert.mask);
   3262	}
   3263	spin_unlock(&gi->alert.ref_lock);
   3264
   3265	return gib->nisc;
   3266}
   3267EXPORT_SYMBOL_GPL(kvm_s390_gisc_register);
   3268
   3269/**
   3270 * kvm_s390_gisc_unregister - unregister a guest ISC
   3271 *
   3272 * @kvm:  the kernel vm to work with
   3273 * @gisc: the guest interruption sub class to register
   3274 *
   3275 * The function reduces the vm specific alert mask to use.
   3276 * The effective IAM mask in the GISA is updated as well
   3277 * in case the GISA is not part of the GIB alert list.
   3278 * It will be updated latest when the IAM gets restored
   3279 * by gisa_get_ipm_or_restore_iam().
   3280 *
   3281 * Returns: the nonspecific ISC (NISC) the gib alert mechanism
   3282 *          has registered with the channel subsystem.
   3283 *          -ENODEV in case the vm uses no GISA
   3284 *          -ERANGE in case the guest ISC is invalid
   3285 *          -EINVAL in case the guest ISC is not registered
   3286 */
   3287int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc)
   3288{
   3289	struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int;
   3290	int rc = 0;
   3291
   3292	if (!gi->origin)
   3293		return -ENODEV;
   3294	if (gisc > MAX_ISC)
   3295		return -ERANGE;
   3296
   3297	spin_lock(&gi->alert.ref_lock);
   3298	if (gi->alert.ref_count[gisc] == 0) {
   3299		rc = -EINVAL;
   3300		goto out;
   3301	}
   3302	gi->alert.ref_count[gisc]--;
   3303	if (gi->alert.ref_count[gisc] == 0) {
   3304		gi->alert.mask &= ~(0x80 >> gisc);
   3305		gisa_set_iam(gi->origin, gi->alert.mask);
   3306	}
   3307out:
   3308	spin_unlock(&gi->alert.ref_lock);
   3309
   3310	return rc;
   3311}
   3312EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister);
   3313
   3314static void gib_alert_irq_handler(struct airq_struct *airq, bool floating)
   3315{
   3316	inc_irq_stat(IRQIO_GAL);
   3317	process_gib_alert_list();
   3318}
   3319
   3320static struct airq_struct gib_alert_irq = {
   3321	.handler = gib_alert_irq_handler,
   3322	.lsi_ptr = &gib_alert_irq.lsi_mask,
   3323};
   3324
   3325void kvm_s390_gib_destroy(void)
   3326{
   3327	if (!gib)
   3328		return;
   3329	chsc_sgib(0);
   3330	unregister_adapter_interrupt(&gib_alert_irq);
   3331	free_page((unsigned long)gib);
   3332	gib = NULL;
   3333}
   3334
   3335int kvm_s390_gib_init(u8 nisc)
   3336{
   3337	int rc = 0;
   3338
   3339	if (!css_general_characteristics.aiv) {
   3340		KVM_EVENT(3, "%s", "gib not initialized, no AIV facility");
   3341		goto out;
   3342	}
   3343
   3344	gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
   3345	if (!gib) {
   3346		rc = -ENOMEM;
   3347		goto out;
   3348	}
   3349
   3350	gib_alert_irq.isc = nisc;
   3351	if (register_adapter_interrupt(&gib_alert_irq)) {
   3352		pr_err("Registering the GIB alert interruption handler failed\n");
   3353		rc = -EIO;
   3354		goto out_free_gib;
   3355	}
   3356
   3357	gib->nisc = nisc;
   3358	if (chsc_sgib((u32)(u64)gib)) {
   3359		pr_err("Associating the GIB with the AIV facility failed\n");
   3360		free_page((unsigned long)gib);
   3361		gib = NULL;
   3362		rc = -EIO;
   3363		goto out_unreg_gal;
   3364	}
   3365
   3366	KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc);
   3367	goto out;
   3368
   3369out_unreg_gal:
   3370	unregister_adapter_interrupt(&gib_alert_irq);
   3371out_free_gib:
   3372	free_page((unsigned long)gib);
   3373	gib = NULL;
   3374out:
   3375	return rc;
   3376}