cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

card_ddcb.c (39130B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * IBM Accelerator Family 'GenWQE'
      4 *
      5 * (C) Copyright IBM Corp. 2013
      6 *
      7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
      8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
      9 * Author: Michael Jung <mijung@gmx.net>
     10 * Author: Michael Ruettger <michael@ibmra.de>
     11 */
     12
     13/*
     14 * Device Driver Control Block (DDCB) queue support. Definition of
     15 * interrupt handlers for queue support as well as triggering the
     16 * health monitor code in case of problems. The current hardware uses
     17 * an MSI interrupt which is shared between error handling and
     18 * functional code.
     19 */
     20
     21#include <linux/types.h>
     22#include <linux/sched.h>
     23#include <linux/wait.h>
     24#include <linux/pci.h>
     25#include <linux/string.h>
     26#include <linux/dma-mapping.h>
     27#include <linux/delay.h>
     28#include <linux/module.h>
     29#include <linux/interrupt.h>
     30#include <linux/crc-itu-t.h>
     31
     32#include "card_base.h"
     33#include "card_ddcb.h"
     34
     35/*
     36 * N: next DDCB, this is where the next DDCB will be put.
     37 * A: active DDCB, this is where the code will look for the next completion.
     38 * x: DDCB is enqueued, we are waiting for its completion.
     39
     40 * Situation (1): Empty queue
     41 *  +---+---+---+---+---+---+---+---+
     42 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
     43 *  |   |   |   |   |   |   |   |   |
     44 *  +---+---+---+---+---+---+---+---+
     45 *           A/N
     46 *  enqueued_ddcbs = A - N = 2 - 2 = 0
     47 *
     48 * Situation (2): Wrapped, N > A
     49 *  +---+---+---+---+---+---+---+---+
     50 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
     51 *  |   |   | x | x |   |   |   |   |
     52 *  +---+---+---+---+---+---+---+---+
     53 *            A       N
     54 *  enqueued_ddcbs = N - A = 4 - 2 = 2
     55 *
     56 * Situation (3): Queue wrapped, A > N
     57 *  +---+---+---+---+---+---+---+---+
     58 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
     59 *  | x | x |   |   | x | x | x | x |
     60 *  +---+---+---+---+---+---+---+---+
     61 *            N       A
     62 *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
     63 *
     64 * Situation (4a): Queue full N > A
     65 *  +---+---+---+---+---+---+---+---+
     66 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
     67 *  | x | x | x | x | x | x | x |   |
     68 *  +---+---+---+---+---+---+---+---+
     69 *    A                           N
     70 *
     71 *  enqueued_ddcbs = N - A = 7 - 0 = 7
     72 *
     73 * Situation (4a): Queue full A > N
     74 *  +---+---+---+---+---+---+---+---+
     75 *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
     76 *  | x | x | x |   | x | x | x | x |
     77 *  +---+---+---+---+---+---+---+---+
     78 *                N   A
     79 *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
     80 */
     81
     82static int queue_empty(struct ddcb_queue *queue)
     83{
     84	return queue->ddcb_next == queue->ddcb_act;
     85}
     86
     87static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
     88{
     89	if (queue->ddcb_next >= queue->ddcb_act)
     90		return queue->ddcb_next - queue->ddcb_act;
     91
     92	return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
     93}
     94
     95static int queue_free_ddcbs(struct ddcb_queue *queue)
     96{
     97	int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
     98
     99	if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
    100		return 0;
    101	}
    102	return free_ddcbs;
    103}
    104
    105/*
    106 * Use of the PRIV field in the DDCB for queue debugging:
    107 *
    108 * (1) Trying to get rid of a DDCB which saw a timeout:
    109 *     pddcb->priv[6] = 0xcc;   # cleared
    110 *
    111 * (2) Append a DDCB via NEXT bit:
    112 *     pddcb->priv[7] = 0xaa;	# appended
    113 *
    114 * (3) DDCB needed tapping:
    115 *     pddcb->priv[7] = 0xbb;   # tapped
    116 *
    117 * (4) DDCB marked as correctly finished:
    118 *     pddcb->priv[6] = 0xff;	# finished
    119 */
    120
    121static inline void ddcb_mark_tapped(struct ddcb *pddcb)
    122{
    123	pddcb->priv[7] = 0xbb;  /* tapped */
    124}
    125
    126static inline void ddcb_mark_appended(struct ddcb *pddcb)
    127{
    128	pddcb->priv[7] = 0xaa;	/* appended */
    129}
    130
    131static inline void ddcb_mark_cleared(struct ddcb *pddcb)
    132{
    133	pddcb->priv[6] = 0xcc; /* cleared */
    134}
    135
    136static inline void ddcb_mark_finished(struct ddcb *pddcb)
    137{
    138	pddcb->priv[6] = 0xff;	/* finished */
    139}
    140
    141static inline void ddcb_mark_unused(struct ddcb *pddcb)
    142{
    143	pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
    144}
    145
    146/**
    147 * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
    148 * @buff:       pointer to data buffer
    149 * @len:        length of data for calculation
    150 * @init:       initial crc (0xffff at start)
    151 *
    152 * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
    153 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
    154 *          should result in a crc16 of 0x89c3
    155 *
    156 * Return: crc16 checksum in big endian format !
    157 */
    158static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
    159{
    160	return crc_itu_t(init, buff, len);
    161}
    162
    163static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
    164{
    165	int i;
    166	struct ddcb *pddcb;
    167	unsigned long flags;
    168	struct pci_dev *pci_dev = cd->pci_dev;
    169
    170	spin_lock_irqsave(&cd->print_lock, flags);
    171
    172	dev_info(&pci_dev->dev,
    173		 "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
    174		 cd->card_idx, queue->ddcb_act, queue->ddcb_next);
    175
    176	pddcb = queue->ddcb_vaddr;
    177	for (i = 0; i < queue->ddcb_max; i++) {
    178		dev_err(&pci_dev->dev,
    179			"  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
    180			i == queue->ddcb_act ? '>' : ' ',
    181			i,
    182			be16_to_cpu(pddcb->retc_16),
    183			be16_to_cpu(pddcb->seqnum_16),
    184			pddcb->hsi,
    185			pddcb->shi,
    186			be64_to_cpu(pddcb->priv_64),
    187			pddcb->cmd);
    188		pddcb++;
    189	}
    190	spin_unlock_irqrestore(&cd->print_lock, flags);
    191}
    192
    193struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
    194{
    195	struct ddcb_requ *req;
    196
    197	req = kzalloc(sizeof(*req), GFP_KERNEL);
    198	if (!req)
    199		return NULL;
    200
    201	return &req->cmd;
    202}
    203
    204void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
    205{
    206	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
    207
    208	kfree(req);
    209}
    210
    211static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
    212{
    213	return req->req_state;
    214}
    215
    216static inline void ddcb_requ_set_state(struct ddcb_requ *req,
    217				       enum genwqe_requ_state new_state)
    218{
    219	req->req_state = new_state;
    220}
    221
    222static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
    223{
    224	return req->cmd.ddata_addr != 0x0;
    225}
    226
    227/**
    228 * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
    229 * @cd:          pointer to genwqe device descriptor
    230 * @req:         DDCB work request
    231 *
    232 * Status of ddcb_requ mirrors this hardware state, but is copied in
    233 * the ddcb_requ on interrupt/polling function. The lowlevel code
    234 * should check the hardware state directly, the higher level code
    235 * should check the copy.
    236 *
    237 * This function will also return true if the state of the queue is
    238 * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
    239 * shutdown case.
    240 */
    241static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
    242{
    243	return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
    244		(cd->card_state != GENWQE_CARD_USED);
    245}
    246
    247#define RET_DDCB_APPENDED 1
    248#define RET_DDCB_TAPPED   2
    249/**
    250 * enqueue_ddcb() - Enqueue a DDCB
    251 * @cd:         pointer to genwqe device descriptor
    252 * @queue:	queue this operation should be done on
    253 * @pddcb:      pointer to ddcb structure
    254 * @ddcb_no:    pointer to ddcb number being tapped
    255 *
    256 * Start execution of DDCB by tapping or append to queue via NEXT
    257 * bit. This is done by an atomic 'compare and swap' instruction and
    258 * checking SHI and HSI of the previous DDCB.
    259 *
    260 * This function must only be called with ddcb_lock held.
    261 *
    262 * Return: 1 if new DDCB is appended to previous
    263 *         2 if DDCB queue is tapped via register/simulation
    264 */
    265static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
    266			struct ddcb *pddcb, int ddcb_no)
    267{
    268	unsigned int try;
    269	int prev_no;
    270	struct ddcb *prev_ddcb;
    271	__be32 old, new, icrc_hsi_shi;
    272	u64 num;
    273
    274	/*
    275	 * For performance checks a Dispatch Timestamp can be put into
    276	 * DDCB It is supposed to use the SLU's free running counter,
    277	 * but this requires PCIe cycles.
    278	 */
    279	ddcb_mark_unused(pddcb);
    280
    281	/* check previous DDCB if already fetched */
    282	prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
    283	prev_ddcb = &queue->ddcb_vaddr[prev_no];
    284
    285	/*
    286	 * It might have happened that the HSI.FETCHED bit is
    287	 * set. Retry in this case. Therefore I expect maximum 2 times
    288	 * trying.
    289	 */
    290	ddcb_mark_appended(pddcb);
    291	for (try = 0; try < 2; try++) {
    292		old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
    293
    294		/* try to append via NEXT bit if prev DDCB is not completed */
    295		if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
    296			break;
    297
    298		new = (old | DDCB_NEXT_BE32);
    299
    300		wmb();		/* need to ensure write ordering */
    301		icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
    302
    303		if (icrc_hsi_shi == old)
    304			return RET_DDCB_APPENDED; /* appended to queue */
    305	}
    306
    307	/* Queue must be re-started by updating QUEUE_OFFSET */
    308	ddcb_mark_tapped(pddcb);
    309	num = (u64)ddcb_no << 8;
    310
    311	wmb();			/* need to ensure write ordering */
    312	__genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
    313
    314	return RET_DDCB_TAPPED;
    315}
    316
    317/**
    318 * copy_ddcb_results() - Copy output state from real DDCB to request
    319 * @req:        pointer to requested DDCB parameters
    320 * @ddcb_no:    pointer to ddcb number being tapped
    321 *
    322 * Copy DDCB ASV to request struct. There is no endian
    323 * conversion made, since data structure in ASV is still
    324 * unknown here.
    325 *
    326 * This is needed by:
    327 *   - genwqe_purge_ddcb()
    328 *   - genwqe_check_ddcb_queue()
    329 */
    330static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
    331{
    332	struct ddcb_queue *queue = req->queue;
    333	struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
    334
    335	memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
    336
    337	/* copy status flags of the variant part */
    338	req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
    339	req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
    340	req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
    341
    342	req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
    343	req->cmd.progress = be32_to_cpu(pddcb->progress_32);
    344	req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
    345
    346	if (ddcb_requ_collect_debug_data(req)) {
    347		int prev_no = (ddcb_no == 0) ?
    348			queue->ddcb_max - 1 : ddcb_no - 1;
    349		struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
    350
    351		memcpy(&req->debug_data.ddcb_finished, pddcb,
    352		       sizeof(req->debug_data.ddcb_finished));
    353		memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
    354		       sizeof(req->debug_data.ddcb_prev));
    355	}
    356}
    357
    358/**
    359 * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work requests.
    360 * @cd:         pointer to genwqe device descriptor
    361 * @queue:	queue to be checked
    362 *
    363 * Return: Number of DDCBs which were finished
    364 */
    365static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
    366				   struct ddcb_queue *queue)
    367{
    368	unsigned long flags;
    369	int ddcbs_finished = 0;
    370	struct pci_dev *pci_dev = cd->pci_dev;
    371
    372	spin_lock_irqsave(&queue->ddcb_lock, flags);
    373
    374	/* FIXME avoid soft locking CPU */
    375	while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
    376
    377		struct ddcb *pddcb;
    378		struct ddcb_requ *req;
    379		u16 vcrc, vcrc_16, retc_16;
    380
    381		pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
    382
    383		if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
    384		    0x00000000)
    385			goto go_home; /* not completed, continue waiting */
    386
    387		wmb();  /*  Add sync to decouple prev. read operations */
    388
    389		/* Note: DDCB could be purged */
    390		req = queue->ddcb_req[queue->ddcb_act];
    391		if (req == NULL) {
    392			/* this occurs if DDCB is purged, not an error */
    393			/* Move active DDCB further; Nothing to do anymore. */
    394			goto pick_next_one;
    395		}
    396
    397		/*
    398		 * HSI=0x44 (fetched and completed), but RETC is
    399		 * 0x101, or even worse 0x000.
    400		 *
    401		 * In case of seeing the queue in inconsistent state
    402		 * we read the errcnts and the queue status to provide
    403		 * a trigger for our PCIe analyzer stop capturing.
    404		 */
    405		retc_16 = be16_to_cpu(pddcb->retc_16);
    406		if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
    407			u64 errcnts, status;
    408			u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
    409
    410			errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
    411			status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
    412
    413			dev_err(&pci_dev->dev,
    414				"[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
    415				__func__, be16_to_cpu(pddcb->seqnum_16),
    416				pddcb->hsi, retc_16, errcnts, status,
    417				queue->ddcb_daddr + ddcb_offs);
    418		}
    419
    420		copy_ddcb_results(req, queue->ddcb_act);
    421		queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
    422
    423		dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
    424		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
    425
    426		ddcb_mark_finished(pddcb);
    427
    428		/* calculate CRC_16 to see if VCRC is correct */
    429		vcrc = genwqe_crc16(pddcb->asv,
    430				   VCRC_LENGTH(req->cmd.asv_length),
    431				   0xffff);
    432		vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
    433		if (vcrc != vcrc_16) {
    434			printk_ratelimited(KERN_ERR
    435				"%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
    436				GENWQE_DEVNAME, dev_name(&pci_dev->dev),
    437				pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
    438				vcrc, vcrc_16);
    439		}
    440
    441		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
    442		queue->ddcbs_completed++;
    443		queue->ddcbs_in_flight--;
    444
    445		/* wake up process waiting for this DDCB, and
    446                   processes on the busy queue */
    447		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
    448		wake_up_interruptible(&queue->busy_waitq);
    449
    450pick_next_one:
    451		queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
    452		ddcbs_finished++;
    453	}
    454
    455 go_home:
    456	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    457	return ddcbs_finished;
    458}
    459
    460/**
    461 * __genwqe_wait_ddcb(): Waits until DDCB is completed
    462 * @cd:         pointer to genwqe device descriptor
    463 * @req:        pointer to requsted DDCB parameters
    464 *
    465 * The Service Layer will update the RETC in DDCB when processing is
    466 * pending or done.
    467 *
    468 * Return: > 0 remaining jiffies, DDCB completed
    469 *           -ETIMEDOUT	when timeout
    470 *           -ERESTARTSYS when ^C
    471 *           -EINVAL when unknown error condition
    472 *
    473 * When an error is returned the called needs to ensure that
    474 * purge_ddcb() is being called to get the &req removed from the
    475 * queue.
    476 */
    477int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
    478{
    479	int rc;
    480	unsigned int ddcb_no;
    481	struct ddcb_queue *queue;
    482	struct pci_dev *pci_dev = cd->pci_dev;
    483
    484	if (req == NULL)
    485		return -EINVAL;
    486
    487	queue = req->queue;
    488	if (queue == NULL)
    489		return -EINVAL;
    490
    491	ddcb_no = req->num;
    492	if (ddcb_no >= queue->ddcb_max)
    493		return -EINVAL;
    494
    495	rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
    496				ddcb_requ_finished(cd, req),
    497				GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
    498
    499	/*
    500	 * We need to distinguish 3 cases here:
    501	 *   1. rc == 0              timeout occurred
    502	 *   2. rc == -ERESTARTSYS   signal received
    503	 *   3. rc > 0               remaining jiffies condition is true
    504	 */
    505	if (rc == 0) {
    506		struct ddcb_queue *queue = req->queue;
    507		struct ddcb *pddcb;
    508
    509		/*
    510		 * Timeout may be caused by long task switching time.
    511		 * When timeout happens, check if the request has
    512		 * meanwhile completed.
    513		 */
    514		genwqe_check_ddcb_queue(cd, req->queue);
    515		if (ddcb_requ_finished(cd, req))
    516			return rc;
    517
    518		dev_err(&pci_dev->dev,
    519			"[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
    520			__func__, req->num, rc,	ddcb_requ_get_state(req),
    521			req);
    522		dev_err(&pci_dev->dev,
    523			"[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
    524			__genwqe_readq(cd, queue->IO_QUEUE_STATUS));
    525
    526		pddcb = &queue->ddcb_vaddr[req->num];
    527		genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
    528
    529		print_ddcb_info(cd, req->queue);
    530		return -ETIMEDOUT;
    531
    532	} else if (rc == -ERESTARTSYS) {
    533		return rc;
    534		/*
    535		 * EINTR:       Stops the application
    536		 * ERESTARTSYS: Restartable systemcall; called again
    537		 */
    538
    539	} else if (rc < 0) {
    540		dev_err(&pci_dev->dev,
    541			"[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
    542			__func__, req->num, rc, ddcb_requ_get_state(req));
    543		return -EINVAL;
    544	}
    545
    546	/* Severe error occured. Driver is forced to stop operation */
    547	if (cd->card_state != GENWQE_CARD_USED) {
    548		dev_err(&pci_dev->dev,
    549			"[%s] err: DDCB#%d forced to stop (rc=%d)\n",
    550			__func__, req->num, rc);
    551		return -EIO;
    552	}
    553	return rc;
    554}
    555
    556/**
    557 * get_next_ddcb() - Get next available DDCB
    558 * @cd:         pointer to genwqe device descriptor
    559 * @queue:      DDCB queue
    560 * @num:        internal DDCB number
    561 *
    562 * DDCB's content is completely cleared but presets for PRE and
    563 * SEQNUM. This function must only be called when ddcb_lock is held.
    564 *
    565 * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
    566 */
    567static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
    568				  struct ddcb_queue *queue,
    569				  int *num)
    570{
    571	u64 *pu64;
    572	struct ddcb *pddcb;
    573
    574	if (queue_free_ddcbs(queue) == 0) /* queue is  full */
    575		return NULL;
    576
    577	/* find new ddcb */
    578	pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
    579
    580	/* if it is not completed, we are not allowed to use it */
    581	/* barrier(); */
    582	if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
    583		return NULL;
    584
    585	*num = queue->ddcb_next;	/* internal DDCB number */
    586	queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
    587
    588	/* clear important DDCB fields */
    589	pu64 = (u64 *)pddcb;
    590	pu64[0] = 0ULL;		/* offs 0x00 (ICRC,HSI,SHI,...) */
    591	pu64[1] = 0ULL;		/* offs 0x01 (ACFUNC,CMD...) */
    592
    593	/* destroy previous results in ASV */
    594	pu64[0x80/8] = 0ULL;	/* offs 0x80 (ASV + 0) */
    595	pu64[0x88/8] = 0ULL;	/* offs 0x88 (ASV + 0x08) */
    596	pu64[0x90/8] = 0ULL;	/* offs 0x90 (ASV + 0x10) */
    597	pu64[0x98/8] = 0ULL;	/* offs 0x98 (ASV + 0x18) */
    598	pu64[0xd0/8] = 0ULL;	/* offs 0xd0 (RETC,ATTN...) */
    599
    600	pddcb->pre = DDCB_PRESET_PRE; /* 128 */
    601	pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
    602	return pddcb;
    603}
    604
    605/**
    606 * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
    607 * @cd:         genwqe device descriptor
    608 * @req:        DDCB request
    609 *
    610 * This will fail when the request was already FETCHED. In this case
    611 * we need to wait until it is finished. Else the DDCB can be
    612 * reused. This function also ensures that the request data structure
    613 * is removed from ddcb_req[].
    614 *
    615 * Do not forget to call this function when genwqe_wait_ddcb() fails,
    616 * such that the request gets really removed from ddcb_req[].
    617 *
    618 * Return: 0 success
    619 */
    620int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
    621{
    622	struct ddcb *pddcb = NULL;
    623	unsigned int t;
    624	unsigned long flags;
    625	struct ddcb_queue *queue = req->queue;
    626	struct pci_dev *pci_dev = cd->pci_dev;
    627	u64 queue_status;
    628	__be32 icrc_hsi_shi = 0x0000;
    629	__be32 old, new;
    630
    631	/* unsigned long flags; */
    632	if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
    633		dev_err(&pci_dev->dev,
    634			"[%s] err: software timeout is not set!\n", __func__);
    635		return -EFAULT;
    636	}
    637
    638	pddcb = &queue->ddcb_vaddr[req->num];
    639
    640	for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
    641
    642		spin_lock_irqsave(&queue->ddcb_lock, flags);
    643
    644		/* Check if req was meanwhile finished */
    645		if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
    646			goto go_home;
    647
    648		/* try to set PURGE bit if FETCHED/COMPLETED are not set */
    649		old = pddcb->icrc_hsi_shi_32;	/* read SHI/HSI in BE32 */
    650		if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
    651
    652			new = (old | DDCB_PURGE_BE32);
    653			icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
    654					       old, new);
    655			if (icrc_hsi_shi == old)
    656				goto finish_ddcb;
    657		}
    658
    659		/* normal finish with HSI bit */
    660		barrier();
    661		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
    662		if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
    663			goto finish_ddcb;
    664
    665		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    666
    667		/*
    668		 * Here the check_ddcb() function will most likely
    669		 * discover this DDCB to be finished some point in
    670		 * time. It will mark the req finished and free it up
    671		 * in the list.
    672		 */
    673
    674		copy_ddcb_results(req, req->num); /* for the failing case */
    675		msleep(100); /* sleep for 1/10 second and try again */
    676		continue;
    677
    678finish_ddcb:
    679		copy_ddcb_results(req, req->num);
    680		ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
    681		queue->ddcbs_in_flight--;
    682		queue->ddcb_req[req->num] = NULL; /* delete from array */
    683		ddcb_mark_cleared(pddcb);
    684
    685		/* Move active DDCB further; Nothing to do here anymore. */
    686
    687		/*
    688		 * We need to ensure that there is at least one free
    689		 * DDCB in the queue. To do that, we must update
    690		 * ddcb_act only if the COMPLETED bit is set for the
    691		 * DDCB we are working on else we treat that DDCB even
    692		 * if we PURGED it as occupied (hardware is supposed
    693		 * to set the COMPLETED bit yet!).
    694		 */
    695		icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
    696		if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
    697		    (queue->ddcb_act == req->num)) {
    698			queue->ddcb_act = ((queue->ddcb_act + 1) %
    699					   queue->ddcb_max);
    700		}
    701go_home:
    702		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    703		return 0;
    704	}
    705
    706	/*
    707	 * If the card is dead and the queue is forced to stop, we
    708	 * might see this in the queue status register.
    709	 */
    710	queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
    711
    712	dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
    713	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
    714
    715	dev_err(&pci_dev->dev,
    716		"[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
    717		__func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
    718		queue_status);
    719
    720	print_ddcb_info(cd, req->queue);
    721
    722	return -EFAULT;
    723}
    724
    725int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
    726{
    727	int len;
    728	struct pci_dev *pci_dev = cd->pci_dev;
    729
    730	if (d == NULL) {
    731		dev_err(&pci_dev->dev,
    732			"[%s] err: invalid memory for debug data!\n",
    733			__func__);
    734		return -EFAULT;
    735	}
    736
    737	len  = sizeof(d->driver_version);
    738	snprintf(d->driver_version, len, "%s", DRV_VERSION);
    739	d->slu_unitcfg = cd->slu_unitcfg;
    740	d->app_unitcfg = cd->app_unitcfg;
    741	return 0;
    742}
    743
    744/**
    745 * __genwqe_enqueue_ddcb() - Enqueue a DDCB
    746 * @cd:         pointer to genwqe device descriptor
    747 * @req:        pointer to DDCB execution request
    748 * @f_flags:    file mode: blocking, non-blocking
    749 *
    750 * Return: 0 if enqueuing succeeded
    751 *         -EIO if card is unusable/PCIe problems
    752 *         -EBUSY if enqueuing failed
    753 */
    754int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
    755			  unsigned int f_flags)
    756{
    757	struct ddcb *pddcb;
    758	unsigned long flags;
    759	struct ddcb_queue *queue;
    760	struct pci_dev *pci_dev = cd->pci_dev;
    761	u16 icrc;
    762
    763 retry:
    764	if (cd->card_state != GENWQE_CARD_USED) {
    765		printk_ratelimited(KERN_ERR
    766			"%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
    767			GENWQE_DEVNAME, dev_name(&pci_dev->dev),
    768			__func__, req->num);
    769		return -EIO;
    770	}
    771
    772	queue = req->queue = &cd->queue;
    773
    774	/* FIXME circumvention to improve performance when no irq is
    775	 * there.
    776	 */
    777	if (GENWQE_POLLING_ENABLED)
    778		genwqe_check_ddcb_queue(cd, queue);
    779
    780	/*
    781	 * It must be ensured to process all DDCBs in successive
    782	 * order. Use a lock here in order to prevent nested DDCB
    783	 * enqueuing.
    784	 */
    785	spin_lock_irqsave(&queue->ddcb_lock, flags);
    786
    787	pddcb = get_next_ddcb(cd, queue, &req->num);	/* get ptr and num */
    788	if (pddcb == NULL) {
    789		int rc;
    790
    791		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    792
    793		if (f_flags & O_NONBLOCK) {
    794			queue->return_on_busy++;
    795			return -EBUSY;
    796		}
    797
    798		queue->wait_on_busy++;
    799		rc = wait_event_interruptible(queue->busy_waitq,
    800					      queue_free_ddcbs(queue) != 0);
    801		dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
    802			__func__, rc);
    803		if (rc == -ERESTARTSYS)
    804			return rc;  /* interrupted by a signal */
    805
    806		goto retry;
    807	}
    808
    809	if (queue->ddcb_req[req->num] != NULL) {
    810		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    811
    812		dev_err(&pci_dev->dev,
    813			"[%s] picked DDCB %d with req=%p still in use!!\n",
    814			__func__, req->num, req);
    815		return -EFAULT;
    816	}
    817	ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
    818	queue->ddcb_req[req->num] = req;
    819
    820	pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
    821	pddcb->cmd = req->cmd.cmd;
    822	pddcb->acfunc = req->cmd.acfunc;	/* functional unit */
    823
    824	/*
    825	 * We know that we can get retc 0x104 with CRC error, do not
    826	 * stop the queue in those cases for this command. XDIR = 1
    827	 * does not work for old SLU versions.
    828	 *
    829	 * Last bitstream with the old XDIR behavior had SLU_ID
    830	 * 0x34199.
    831	 */
    832	if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
    833		pddcb->xdir = 0x1;
    834	else
    835		pddcb->xdir = 0x0;
    836
    837
    838	pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
    839		      ((req->cmd.asv_length  / 8)));
    840	pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
    841
    842	/*
    843	 * If copying the whole DDCB_ASIV_LENGTH is impacting
    844	 * performance we need to change it to
    845	 * req->cmd.asiv_length. But simulation benefits from some
    846	 * non-architectured bits behind the architectured content.
    847	 *
    848	 * How much data is copied depends on the availability of the
    849	 * ATS field, which was introduced late. If the ATS field is
    850	 * supported ASIV is 8 bytes shorter than it used to be. Since
    851	 * the ATS field is copied too, the code should do exactly
    852	 * what it did before, but I wanted to make copying of the ATS
    853	 * field very explicit.
    854	 */
    855	if (genwqe_get_slu_id(cd) <= 0x2) {
    856		memcpy(&pddcb->__asiv[0],	/* destination */
    857		       &req->cmd.__asiv[0],	/* source */
    858		       DDCB_ASIV_LENGTH);	/* req->cmd.asiv_length */
    859	} else {
    860		pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
    861		memcpy(&pddcb->n.asiv[0],	/* destination */
    862			&req->cmd.asiv[0],	/* source */
    863			DDCB_ASIV_LENGTH_ATS);	/* req->cmd.asiv_length */
    864	}
    865
    866	pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
    867
    868	/*
    869	 * Calculate CRC_16 for corresponding range PSP(7:4). Include
    870	 * empty 4 bytes prior to the data.
    871	 */
    872	icrc = genwqe_crc16((const u8 *)pddcb,
    873			   ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
    874	pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
    875
    876	/* enable DDCB completion irq */
    877	if (!GENWQE_POLLING_ENABLED)
    878		pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
    879
    880	dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
    881	genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
    882
    883	if (ddcb_requ_collect_debug_data(req)) {
    884		/* use the kernel copy of debug data. copying back to
    885		   user buffer happens later */
    886
    887		genwqe_init_debug_data(cd, &req->debug_data);
    888		memcpy(&req->debug_data.ddcb_before, pddcb,
    889		       sizeof(req->debug_data.ddcb_before));
    890	}
    891
    892	enqueue_ddcb(cd, queue, pddcb, req->num);
    893	queue->ddcbs_in_flight++;
    894
    895	if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
    896		queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
    897
    898	ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
    899	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    900	wake_up_interruptible(&cd->queue_waitq);
    901
    902	return 0;
    903}
    904
    905/**
    906 * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
    907 * @cd:         pointer to genwqe device descriptor
    908 * @cmd:        user provided DDCB command
    909 * @f_flags:    file mode: blocking, non-blocking
    910 */
    911int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
    912			      struct genwqe_ddcb_cmd *cmd,
    913			      unsigned int f_flags)
    914{
    915	int rc = 0;
    916	struct pci_dev *pci_dev = cd->pci_dev;
    917	struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
    918
    919	if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
    920		dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
    921			__func__, cmd->asiv_length);
    922		return -EINVAL;
    923	}
    924	if (cmd->asv_length > DDCB_ASV_LENGTH) {
    925		dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
    926			__func__, cmd->asiv_length);
    927		return -EINVAL;
    928	}
    929	rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
    930	if (rc != 0)
    931		return rc;
    932
    933	rc = __genwqe_wait_ddcb(cd, req);
    934	if (rc < 0)		/* error or signal interrupt */
    935		goto err_exit;
    936
    937	if (ddcb_requ_collect_debug_data(req)) {
    938		if (copy_to_user((struct genwqe_debug_data __user *)
    939				 (unsigned long)cmd->ddata_addr,
    940				 &req->debug_data,
    941				 sizeof(struct genwqe_debug_data)))
    942			return -EFAULT;
    943	}
    944
    945	/*
    946	 * Higher values than 0x102 indicate completion with faults,
    947	 * lower values than 0x102 indicate processing faults. Note
    948	 * that DDCB might have been purged. E.g. Cntl+C.
    949	 */
    950	if (cmd->retc != DDCB_RETC_COMPLETE) {
    951		/* This might happen e.g. flash read, and needs to be
    952		   handled by the upper layer code. */
    953		rc = -EBADMSG;	/* not processed/error retc */
    954	}
    955
    956	return rc;
    957
    958 err_exit:
    959	__genwqe_purge_ddcb(cd, req);
    960
    961	if (ddcb_requ_collect_debug_data(req)) {
    962		if (copy_to_user((struct genwqe_debug_data __user *)
    963				 (unsigned long)cmd->ddata_addr,
    964				 &req->debug_data,
    965				 sizeof(struct genwqe_debug_data)))
    966			return -EFAULT;
    967	}
    968	return rc;
    969}
    970
    971/**
    972 * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
    973 * @cd:         pointer to genwqe device descriptor
    974 *
    975 * We use this as condition for our wait-queue code.
    976 */
    977static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
    978{
    979	unsigned long flags;
    980	struct ddcb *pddcb;
    981	struct ddcb_queue *queue = &cd->queue;
    982
    983	spin_lock_irqsave(&queue->ddcb_lock, flags);
    984
    985	if (queue_empty(queue)) { /* empty queue */
    986		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    987		return 0;
    988	}
    989
    990	pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
    991	if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
    992		spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    993		return 1;
    994	}
    995
    996	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
    997	return 0;
    998}
    999
   1000/**
   1001 * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
   1002 * @cd:         pointer to genwqe device descriptor
   1003 *
   1004 * Keep track on the number of DDCBs which ware currently in the
   1005 * queue. This is needed for statistics as well as condition if we want
   1006 * to wait or better do polling in case of no interrupts available.
   1007 */
   1008int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
   1009{
   1010	unsigned long flags;
   1011	int ddcbs_in_flight = 0;
   1012	struct ddcb_queue *queue = &cd->queue;
   1013
   1014	spin_lock_irqsave(&queue->ddcb_lock, flags);
   1015	ddcbs_in_flight += queue->ddcbs_in_flight;
   1016	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
   1017
   1018	return ddcbs_in_flight;
   1019}
   1020
   1021static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
   1022{
   1023	int rc, i;
   1024	struct ddcb *pddcb;
   1025	u64 val64;
   1026	unsigned int queue_size;
   1027	struct pci_dev *pci_dev = cd->pci_dev;
   1028
   1029	if (GENWQE_DDCB_MAX < 2)
   1030		return -EINVAL;
   1031
   1032	queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
   1033
   1034	queue->ddcbs_in_flight = 0;  /* statistics */
   1035	queue->ddcbs_max_in_flight = 0;
   1036	queue->ddcbs_completed = 0;
   1037	queue->return_on_busy = 0;
   1038	queue->wait_on_busy = 0;
   1039
   1040	queue->ddcb_seq	  = 0x100; /* start sequence number */
   1041	queue->ddcb_max	  = GENWQE_DDCB_MAX;
   1042	queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
   1043						&queue->ddcb_daddr);
   1044	if (queue->ddcb_vaddr == NULL) {
   1045		dev_err(&pci_dev->dev,
   1046			"[%s] **err: could not allocate DDCB **\n", __func__);
   1047		return -ENOMEM;
   1048	}
   1049	queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
   1050				  GFP_KERNEL);
   1051	if (!queue->ddcb_req) {
   1052		rc = -ENOMEM;
   1053		goto free_ddcbs;
   1054	}
   1055
   1056	queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
   1057				     sizeof(wait_queue_head_t),
   1058				     GFP_KERNEL);
   1059	if (!queue->ddcb_waitqs) {
   1060		rc = -ENOMEM;
   1061		goto free_requs;
   1062	}
   1063
   1064	for (i = 0; i < queue->ddcb_max; i++) {
   1065		pddcb = &queue->ddcb_vaddr[i];		     /* DDCBs */
   1066		pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
   1067		pddcb->retc_16 = cpu_to_be16(0xfff);
   1068
   1069		queue->ddcb_req[i] = NULL;		     /* requests */
   1070		init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
   1071	}
   1072
   1073	queue->ddcb_act  = 0;
   1074	queue->ddcb_next = 0;	/* queue is empty */
   1075
   1076	spin_lock_init(&queue->ddcb_lock);
   1077	init_waitqueue_head(&queue->busy_waitq);
   1078
   1079	val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
   1080	__genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
   1081	__genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
   1082	__genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
   1083	__genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
   1084	return 0;
   1085
   1086 free_requs:
   1087	kfree(queue->ddcb_req);
   1088	queue->ddcb_req = NULL;
   1089 free_ddcbs:
   1090	__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
   1091				queue->ddcb_daddr);
   1092	queue->ddcb_vaddr = NULL;
   1093	queue->ddcb_daddr = 0ull;
   1094	return rc;
   1095
   1096}
   1097
   1098static int ddcb_queue_initialized(struct ddcb_queue *queue)
   1099{
   1100	return queue->ddcb_vaddr != NULL;
   1101}
   1102
   1103static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
   1104{
   1105	unsigned int queue_size;
   1106
   1107	queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
   1108
   1109	kfree(queue->ddcb_req);
   1110	queue->ddcb_req = NULL;
   1111
   1112	if (queue->ddcb_vaddr) {
   1113		__genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
   1114					queue->ddcb_daddr);
   1115		queue->ddcb_vaddr = NULL;
   1116		queue->ddcb_daddr = 0ull;
   1117	}
   1118}
   1119
   1120static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
   1121{
   1122	u64 gfir;
   1123	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
   1124	struct pci_dev *pci_dev = cd->pci_dev;
   1125
   1126	/*
   1127	 * In case of fatal FIR error the queue is stopped, such that
   1128	 * we can safely check it without risking anything.
   1129	 */
   1130	cd->irqs_processed++;
   1131	wake_up_interruptible(&cd->queue_waitq);
   1132
   1133	/*
   1134	 * Checking for errors before kicking the queue might be
   1135	 * safer, but slower for the good-case ... See above.
   1136	 */
   1137	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
   1138	if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
   1139	    !pci_channel_offline(pci_dev)) {
   1140
   1141		if (cd->use_platform_recovery) {
   1142			/*
   1143			 * Since we use raw accessors, EEH errors won't be
   1144			 * detected by the platform until we do a non-raw
   1145			 * MMIO or config space read
   1146			 */
   1147			readq(cd->mmio + IO_SLC_CFGREG_GFIR);
   1148
   1149			/* Don't do anything if the PCI channel is frozen */
   1150			if (pci_channel_offline(pci_dev))
   1151				goto exit;
   1152		}
   1153
   1154		wake_up_interruptible(&cd->health_waitq);
   1155
   1156		/*
   1157		 * By default GFIRs causes recovery actions. This
   1158		 * count is just for debug when recovery is masked.
   1159		 */
   1160		dev_err_ratelimited(&pci_dev->dev,
   1161				    "[%s] GFIR=%016llx\n",
   1162				    __func__, gfir);
   1163	}
   1164
   1165 exit:
   1166	return IRQ_HANDLED;
   1167}
   1168
   1169static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
   1170{
   1171	struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
   1172
   1173	cd->irqs_processed++;
   1174	wake_up_interruptible(&cd->queue_waitq);
   1175
   1176	return IRQ_HANDLED;
   1177}
   1178
   1179/**
   1180 * genwqe_card_thread() - Work thread for the DDCB queue
   1181 * @data:         pointer to genwqe device descriptor
   1182 *
   1183 * The idea is to check if there are DDCBs in processing. If there are
   1184 * some finished DDCBs, we process them and wakeup the
   1185 * requestors. Otherwise we give other processes time using
   1186 * cond_resched().
   1187 */
   1188static int genwqe_card_thread(void *data)
   1189{
   1190	int should_stop = 0;
   1191	struct genwqe_dev *cd = (struct genwqe_dev *)data;
   1192
   1193	while (!kthread_should_stop()) {
   1194
   1195		genwqe_check_ddcb_queue(cd, &cd->queue);
   1196
   1197		if (GENWQE_POLLING_ENABLED) {
   1198			wait_event_interruptible_timeout(
   1199				cd->queue_waitq,
   1200				genwqe_ddcbs_in_flight(cd) ||
   1201				(should_stop = kthread_should_stop()), 1);
   1202		} else {
   1203			wait_event_interruptible_timeout(
   1204				cd->queue_waitq,
   1205				genwqe_next_ddcb_ready(cd) ||
   1206				(should_stop = kthread_should_stop()), HZ);
   1207		}
   1208		if (should_stop)
   1209			break;
   1210
   1211		/*
   1212		 * Avoid soft lockups on heavy loads; we do not want
   1213		 * to disable our interrupts.
   1214		 */
   1215		cond_resched();
   1216	}
   1217	return 0;
   1218}
   1219
   1220/**
   1221 * genwqe_setup_service_layer() - Setup DDCB queue
   1222 * @cd:         pointer to genwqe device descriptor
   1223 *
   1224 * Allocate DDCBs. Configure Service Layer Controller (SLC).
   1225 *
   1226 * Return: 0 success
   1227 */
   1228int genwqe_setup_service_layer(struct genwqe_dev *cd)
   1229{
   1230	int rc;
   1231	struct ddcb_queue *queue;
   1232	struct pci_dev *pci_dev = cd->pci_dev;
   1233
   1234	if (genwqe_is_privileged(cd)) {
   1235		rc = genwqe_card_reset(cd);
   1236		if (rc < 0) {
   1237			dev_err(&pci_dev->dev,
   1238				"[%s] err: reset failed.\n", __func__);
   1239			return rc;
   1240		}
   1241		genwqe_read_softreset(cd);
   1242	}
   1243
   1244	queue = &cd->queue;
   1245	queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
   1246	queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
   1247	queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
   1248	queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
   1249	queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
   1250	queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
   1251	queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
   1252	queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
   1253	queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
   1254
   1255	rc = setup_ddcb_queue(cd, queue);
   1256	if (rc != 0) {
   1257		rc = -ENODEV;
   1258		goto err_out;
   1259	}
   1260
   1261	init_waitqueue_head(&cd->queue_waitq);
   1262	cd->card_thread = kthread_run(genwqe_card_thread, cd,
   1263				      GENWQE_DEVNAME "%d_thread",
   1264				      cd->card_idx);
   1265	if (IS_ERR(cd->card_thread)) {
   1266		rc = PTR_ERR(cd->card_thread);
   1267		cd->card_thread = NULL;
   1268		goto stop_free_queue;
   1269	}
   1270
   1271	rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
   1272	if (rc)
   1273		goto stop_kthread;
   1274
   1275	/*
   1276	 * We must have all wait-queues initialized when we enable the
   1277	 * interrupts. Otherwise we might crash if we get an early
   1278	 * irq.
   1279	 */
   1280	init_waitqueue_head(&cd->health_waitq);
   1281
   1282	if (genwqe_is_privileged(cd)) {
   1283		rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
   1284				 GENWQE_DEVNAME, cd);
   1285	} else {
   1286		rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
   1287				 GENWQE_DEVNAME, cd);
   1288	}
   1289	if (rc < 0) {
   1290		dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
   1291		goto stop_irq_cap;
   1292	}
   1293
   1294	cd->card_state = GENWQE_CARD_USED;
   1295	return 0;
   1296
   1297 stop_irq_cap:
   1298	genwqe_reset_interrupt_capability(cd);
   1299 stop_kthread:
   1300	kthread_stop(cd->card_thread);
   1301	cd->card_thread = NULL;
   1302 stop_free_queue:
   1303	free_ddcb_queue(cd, queue);
   1304 err_out:
   1305	return rc;
   1306}
   1307
   1308/**
   1309 * queue_wake_up_all() - Handles fatal error case
   1310 * @cd:         pointer to genwqe device descriptor
   1311 *
   1312 * The PCI device got unusable and we have to stop all pending
   1313 * requests as fast as we can. The code after this must purge the
   1314 * DDCBs in question and ensure that all mappings are freed.
   1315 */
   1316static int queue_wake_up_all(struct genwqe_dev *cd)
   1317{
   1318	unsigned int i;
   1319	unsigned long flags;
   1320	struct ddcb_queue *queue = &cd->queue;
   1321
   1322	spin_lock_irqsave(&queue->ddcb_lock, flags);
   1323
   1324	for (i = 0; i < queue->ddcb_max; i++)
   1325		wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
   1326
   1327	wake_up_interruptible(&queue->busy_waitq);
   1328	spin_unlock_irqrestore(&queue->ddcb_lock, flags);
   1329
   1330	return 0;
   1331}
   1332
   1333/**
   1334 * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
   1335 * @cd:         pointer to genwqe device descriptor
   1336 *
   1337 * Relies on the pre-condition that there are no users of the card
   1338 * device anymore e.g. with open file-descriptors.
   1339 *
   1340 * This function must be robust enough to be called twice.
   1341 */
   1342int genwqe_finish_queue(struct genwqe_dev *cd)
   1343{
   1344	int i, rc = 0, in_flight;
   1345	int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
   1346	struct pci_dev *pci_dev = cd->pci_dev;
   1347	struct ddcb_queue *queue = &cd->queue;
   1348
   1349	if (!ddcb_queue_initialized(queue))
   1350		return 0;
   1351
   1352	/* Do not wipe out the error state. */
   1353	if (cd->card_state == GENWQE_CARD_USED)
   1354		cd->card_state = GENWQE_CARD_UNUSED;
   1355
   1356	/* Wake up all requests in the DDCB queue such that they
   1357	   should be removed nicely. */
   1358	queue_wake_up_all(cd);
   1359
   1360	/* We must wait to get rid of the DDCBs in flight */
   1361	for (i = 0; i < waitmax; i++) {
   1362		in_flight = genwqe_ddcbs_in_flight(cd);
   1363
   1364		if (in_flight == 0)
   1365			break;
   1366
   1367		dev_dbg(&pci_dev->dev,
   1368			"  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
   1369			i, waitmax, in_flight);
   1370
   1371		/*
   1372		 * Severe severe error situation: The card itself has
   1373		 * 16 DDCB queues, each queue has e.g. 32 entries,
   1374		 * each DDBC has a hardware timeout of currently 250
   1375		 * msec but the PFs have a hardware timeout of 8 sec
   1376		 * ... so I take something large.
   1377		 */
   1378		msleep(1000);
   1379	}
   1380	if (i == waitmax) {
   1381		dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
   1382			__func__);
   1383		rc = -EIO;
   1384	}
   1385	return rc;
   1386}
   1387
   1388/**
   1389 * genwqe_release_service_layer() - Shutdown DDCB queue
   1390 * @cd:       genwqe device descriptor
   1391 *
   1392 * This function must be robust enough to be called twice.
   1393 */
   1394int genwqe_release_service_layer(struct genwqe_dev *cd)
   1395{
   1396	struct pci_dev *pci_dev = cd->pci_dev;
   1397
   1398	if (!ddcb_queue_initialized(&cd->queue))
   1399		return 1;
   1400
   1401	free_irq(pci_dev->irq, cd);
   1402	genwqe_reset_interrupt_capability(cd);
   1403
   1404	if (cd->card_thread != NULL) {
   1405		kthread_stop(cd->card_thread);
   1406		cd->card_thread = NULL;
   1407	}
   1408
   1409	free_ddcb_queue(cd, &cd->queue);
   1410	return 0;
   1411}