cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

fun_queue.c (15690B)


      1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
      2
      3#include <linux/dma-mapping.h>
      4#include <linux/interrupt.h>
      5#include <linux/log2.h>
      6#include <linux/mm.h>
      7#include <linux/netdevice.h>
      8#include <linux/pci.h>
      9#include <linux/slab.h>
     10
     11#include "fun_dev.h"
     12#include "fun_queue.h"
     13
     14/* Allocate memory for a queue. This includes the memory for the HW descriptor
     15 * ring, an optional 64b HW write-back area, and an optional SW state ring.
     16 * Returns the virtual and DMA addresses of the HW ring, the VA of the SW ring,
     17 * and the VA of the write-back area.
     18 */
     19void *fun_alloc_ring_mem(struct device *dma_dev, size_t depth,
     20			 size_t hw_desc_sz, size_t sw_desc_sz, bool wb,
     21			 int numa_node, dma_addr_t *dma_addr, void **sw_va,
     22			 volatile __be64 **wb_va)
     23{
     24	int dev_node = dev_to_node(dma_dev);
     25	size_t dma_sz;
     26	void *va;
     27
     28	if (numa_node == NUMA_NO_NODE)
     29		numa_node = dev_node;
     30
     31	/* Place optional write-back area at end of descriptor ring. */
     32	dma_sz = hw_desc_sz * depth;
     33	if (wb)
     34		dma_sz += sizeof(u64);
     35
     36	set_dev_node(dma_dev, numa_node);
     37	va = dma_alloc_coherent(dma_dev, dma_sz, dma_addr, GFP_KERNEL);
     38	set_dev_node(dma_dev, dev_node);
     39	if (!va)
     40		return NULL;
     41
     42	if (sw_desc_sz) {
     43		*sw_va = kvzalloc_node(sw_desc_sz * depth, GFP_KERNEL,
     44				       numa_node);
     45		if (!*sw_va) {
     46			dma_free_coherent(dma_dev, dma_sz, va, *dma_addr);
     47			return NULL;
     48		}
     49	}
     50
     51	if (wb)
     52		*wb_va = va + dma_sz - sizeof(u64);
     53	return va;
     54}
     55EXPORT_SYMBOL_GPL(fun_alloc_ring_mem);
     56
     57void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,
     58		       bool wb, void *hw_va, dma_addr_t dma_addr, void *sw_va)
     59{
     60	if (hw_va) {
     61		size_t sz = depth * hw_desc_sz;
     62
     63		if (wb)
     64			sz += sizeof(u64);
     65		dma_free_coherent(dma_dev, sz, hw_va, dma_addr);
     66	}
     67	kvfree(sw_va);
     68}
     69EXPORT_SYMBOL_GPL(fun_free_ring_mem);
     70
     71/* Prepare and issue an admin command to create an SQ on the device with the
     72 * provided parameters. If the queue ID is auto-allocated by the device it is
     73 * returned in *sqidp.
     74 */
     75int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,
     76		  u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,
     77		  u8 coal_nentries, u8 coal_usec, u32 irq_num,
     78		  u32 scan_start_id, u32 scan_end_id,
     79		  u32 rq_buf_size_log2, u32 *sqidp, u32 __iomem **dbp)
     80{
     81	union {
     82		struct fun_admin_epsq_req req;
     83		struct fun_admin_generic_create_rsp rsp;
     84	} cmd;
     85	dma_addr_t wb_addr;
     86	u32 hw_qid;
     87	int rc;
     88
     89	if (sq_depth > fdev->q_depth)
     90		return -EINVAL;
     91	if (flags & FUN_ADMIN_EPSQ_CREATE_FLAG_RQ)
     92		sqe_size_log2 = ilog2(sizeof(struct fun_eprq_rqbuf));
     93
     94	wb_addr = dma_addr + (sq_depth << sqe_size_log2);
     95
     96	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPSQ,
     97						    sizeof(cmd.req));
     98	cmd.req.u.create =
     99		FUN_ADMIN_EPSQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
    100					       sqid, cqid, sqe_size_log2,
    101					       sq_depth - 1, dma_addr, 0,
    102					       coal_nentries, coal_usec,
    103					       irq_num, scan_start_id,
    104					       scan_end_id, 0,
    105					       rq_buf_size_log2,
    106					       ilog2(sizeof(u64)), wb_addr);
    107
    108	rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
    109				       &cmd.rsp, sizeof(cmd.rsp), 0);
    110	if (rc)
    111		return rc;
    112
    113	hw_qid = be32_to_cpu(cmd.rsp.id);
    114	*dbp = fun_sq_db_addr(fdev, hw_qid);
    115	if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
    116		*sqidp = hw_qid;
    117	return rc;
    118}
    119EXPORT_SYMBOL_GPL(fun_sq_create);
    120
    121/* Prepare and issue an admin command to create a CQ on the device with the
    122 * provided parameters. If the queue ID is auto-allocated by the device it is
    123 * returned in *cqidp.
    124 */
    125int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,
    126		  u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,
    127		  u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,
    128		  u32 irq_num, u32 scan_start_id, u32 scan_end_id, u32 *cqidp,
    129		  u32 __iomem **dbp)
    130{
    131	union {
    132		struct fun_admin_epcq_req req;
    133		struct fun_admin_generic_create_rsp rsp;
    134	} cmd;
    135	u32 hw_qid;
    136	int rc;
    137
    138	if (cq_depth > fdev->q_depth)
    139		return -EINVAL;
    140
    141	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,
    142						    sizeof(cmd.req));
    143	cmd.req.u.create =
    144		FUN_ADMIN_EPCQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
    145					       cqid, rqid, cqe_size_log2,
    146					       cq_depth - 1, dma_addr, tailroom,
    147					       headroom / 2, 0, coal_nentries,
    148					       coal_usec, irq_num,
    149					       scan_start_id, scan_end_id, 0);
    150
    151	rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
    152				       &cmd.rsp, sizeof(cmd.rsp), 0);
    153	if (rc)
    154		return rc;
    155
    156	hw_qid = be32_to_cpu(cmd.rsp.id);
    157	*dbp = fun_cq_db_addr(fdev, hw_qid);
    158	if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
    159		*cqidp = hw_qid;
    160	return rc;
    161}
    162EXPORT_SYMBOL_GPL(fun_cq_create);
    163
    164static bool fun_sq_is_head_wb(const struct fun_queue *funq)
    165{
    166	return funq->sq_flags & FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS;
    167}
    168
    169static void fun_clean_rq(struct fun_queue *funq)
    170{
    171	struct fun_dev *fdev = funq->fdev;
    172	struct fun_rq_info *rqinfo;
    173	unsigned int i;
    174
    175	for (i = 0; i < funq->rq_depth; i++) {
    176		rqinfo = &funq->rq_info[i];
    177		if (rqinfo->page) {
    178			dma_unmap_page(fdev->dev, rqinfo->dma, PAGE_SIZE,
    179				       DMA_FROM_DEVICE);
    180			put_page(rqinfo->page);
    181			rqinfo->page = NULL;
    182		}
    183	}
    184}
    185
    186static int fun_fill_rq(struct fun_queue *funq)
    187{
    188	struct device *dev = funq->fdev->dev;
    189	int i, node = dev_to_node(dev);
    190	struct fun_rq_info *rqinfo;
    191
    192	for (i = 0; i < funq->rq_depth; i++) {
    193		rqinfo = &funq->rq_info[i];
    194		rqinfo->page = alloc_pages_node(node, GFP_KERNEL, 0);
    195		if (unlikely(!rqinfo->page))
    196			return -ENOMEM;
    197
    198		rqinfo->dma = dma_map_page(dev, rqinfo->page, 0,
    199					   PAGE_SIZE, DMA_FROM_DEVICE);
    200		if (unlikely(dma_mapping_error(dev, rqinfo->dma))) {
    201			put_page(rqinfo->page);
    202			rqinfo->page = NULL;
    203			return -ENOMEM;
    204		}
    205
    206		funq->rqes[i] = FUN_EPRQ_RQBUF_INIT(rqinfo->dma);
    207	}
    208
    209	funq->rq_tail = funq->rq_depth - 1;
    210	return 0;
    211}
    212
    213static void fun_rq_update_pos(struct fun_queue *funq, int buf_offset)
    214{
    215	if (buf_offset <= funq->rq_buf_offset) {
    216		struct fun_rq_info *rqinfo = &funq->rq_info[funq->rq_buf_idx];
    217		struct device *dev = funq->fdev->dev;
    218
    219		dma_sync_single_for_device(dev, rqinfo->dma, PAGE_SIZE,
    220					   DMA_FROM_DEVICE);
    221		funq->num_rqe_to_fill++;
    222		if (++funq->rq_buf_idx == funq->rq_depth)
    223			funq->rq_buf_idx = 0;
    224	}
    225	funq->rq_buf_offset = buf_offset;
    226}
    227
    228/* Given a command response with data scattered across >= 1 RQ buffers return
    229 * a pointer to a contiguous buffer containing all the data. If the data is in
    230 * one RQ buffer the start address within that buffer is returned, otherwise a
    231 * new buffer is allocated and the data is gathered into it.
    232 */
    233static void *fun_data_from_rq(struct fun_queue *funq,
    234			      const struct fun_rsp_common *rsp, bool *need_free)
    235{
    236	u32 bufoff, total_len, remaining, fragsize, dataoff;
    237	struct device *dma_dev = funq->fdev->dev;
    238	const struct fun_dataop_rqbuf *databuf;
    239	const struct fun_dataop_hdr *dataop;
    240	const struct fun_rq_info *rqinfo;
    241	void *data;
    242
    243	dataop = (void *)rsp + rsp->suboff8 * 8;
    244	total_len = be32_to_cpu(dataop->total_len);
    245
    246	if (likely(dataop->nsgl == 1)) {
    247		databuf = (struct fun_dataop_rqbuf *)dataop->imm;
    248		bufoff = be32_to_cpu(databuf->bufoff);
    249		fun_rq_update_pos(funq, bufoff);
    250		rqinfo = &funq->rq_info[funq->rq_buf_idx];
    251		dma_sync_single_for_cpu(dma_dev, rqinfo->dma + bufoff,
    252					total_len, DMA_FROM_DEVICE);
    253		*need_free = false;
    254		return page_address(rqinfo->page) + bufoff;
    255	}
    256
    257	/* For scattered completions gather the fragments into one buffer. */
    258
    259	data = kmalloc(total_len, GFP_ATOMIC);
    260	/* NULL is OK here. In case of failure we still need to consume the data
    261	 * for proper buffer accounting but indicate an error in the response.
    262	 */
    263	if (likely(data))
    264		*need_free = true;
    265
    266	dataoff = 0;
    267	for (remaining = total_len; remaining; remaining -= fragsize) {
    268		fun_rq_update_pos(funq, 0);
    269		fragsize = min_t(unsigned int, PAGE_SIZE, remaining);
    270		if (data) {
    271			rqinfo = &funq->rq_info[funq->rq_buf_idx];
    272			dma_sync_single_for_cpu(dma_dev, rqinfo->dma, fragsize,
    273						DMA_FROM_DEVICE);
    274			memcpy(data + dataoff, page_address(rqinfo->page),
    275			       fragsize);
    276			dataoff += fragsize;
    277		}
    278	}
    279	return data;
    280}
    281
    282unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max)
    283{
    284	const struct fun_cqe_info *info;
    285	struct fun_rsp_common *rsp;
    286	unsigned int new_cqes;
    287	u16 sf_p, flags;
    288	bool need_free;
    289	void *cqe;
    290
    291	if (!max)
    292		max = funq->cq_depth - 1;
    293
    294	for (new_cqes = 0; new_cqes < max; new_cqes++) {
    295		cqe = funq->cqes + (funq->cq_head << funq->cqe_size_log2);
    296		info = funq_cqe_info(funq, cqe);
    297		sf_p = be16_to_cpu(info->sf_p);
    298
    299		if ((sf_p & 1) != funq->cq_phase)
    300			break;
    301
    302		/* ensure the phase tag is read before other CQE fields */
    303		dma_rmb();
    304
    305		if (++funq->cq_head == funq->cq_depth) {
    306			funq->cq_head = 0;
    307			funq->cq_phase = !funq->cq_phase;
    308		}
    309
    310		rsp = cqe;
    311		flags = be16_to_cpu(rsp->flags);
    312
    313		need_free = false;
    314		if (unlikely(flags & FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF)) {
    315			rsp = fun_data_from_rq(funq, rsp, &need_free);
    316			if (!rsp) {
    317				rsp = cqe;
    318				rsp->len8 = 1;
    319				if (rsp->ret == 0)
    320					rsp->ret = ENOMEM;
    321			}
    322		}
    323
    324		if (funq->cq_cb)
    325			funq->cq_cb(funq, funq->cb_data, rsp, info);
    326		if (need_free)
    327			kfree(rsp);
    328	}
    329
    330	dev_dbg(funq->fdev->dev, "CQ %u, new CQEs %u/%u, head %u, phase %u\n",
    331		funq->cqid, new_cqes, max, funq->cq_head, funq->cq_phase);
    332	return new_cqes;
    333}
    334
    335unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max)
    336{
    337	unsigned int processed;
    338	u32 db;
    339
    340	processed = __fun_process_cq(funq, max);
    341
    342	if (funq->num_rqe_to_fill) {
    343		funq->rq_tail = (funq->rq_tail + funq->num_rqe_to_fill) %
    344				funq->rq_depth;
    345		funq->num_rqe_to_fill = 0;
    346		writel(funq->rq_tail, funq->rq_db);
    347	}
    348
    349	db = funq->cq_head | FUN_DB_IRQ_ARM_F;
    350	writel(db, funq->cq_db);
    351	return processed;
    352}
    353
    354static int fun_alloc_sqes(struct fun_queue *funq)
    355{
    356	funq->sq_cmds = fun_alloc_ring_mem(funq->fdev->dev, funq->sq_depth,
    357					   1 << funq->sqe_size_log2, 0,
    358					   fun_sq_is_head_wb(funq),
    359					   NUMA_NO_NODE, &funq->sq_dma_addr,
    360					   NULL, &funq->sq_head);
    361	return funq->sq_cmds ? 0 : -ENOMEM;
    362}
    363
    364static int fun_alloc_cqes(struct fun_queue *funq)
    365{
    366	funq->cqes = fun_alloc_ring_mem(funq->fdev->dev, funq->cq_depth,
    367					1 << funq->cqe_size_log2, 0, false,
    368					NUMA_NO_NODE, &funq->cq_dma_addr, NULL,
    369					NULL);
    370	return funq->cqes ? 0 : -ENOMEM;
    371}
    372
    373static int fun_alloc_rqes(struct fun_queue *funq)
    374{
    375	funq->rqes = fun_alloc_ring_mem(funq->fdev->dev, funq->rq_depth,
    376					sizeof(*funq->rqes),
    377					sizeof(*funq->rq_info), false,
    378					NUMA_NO_NODE, &funq->rq_dma_addr,
    379					(void **)&funq->rq_info, NULL);
    380	return funq->rqes ? 0 : -ENOMEM;
    381}
    382
    383/* Free a queue's structures. */
    384void fun_free_queue(struct fun_queue *funq)
    385{
    386	struct device *dev = funq->fdev->dev;
    387
    388	fun_free_ring_mem(dev, funq->cq_depth, 1 << funq->cqe_size_log2, false,
    389			  funq->cqes, funq->cq_dma_addr, NULL);
    390	fun_free_ring_mem(dev, funq->sq_depth, 1 << funq->sqe_size_log2,
    391			  fun_sq_is_head_wb(funq), funq->sq_cmds,
    392			  funq->sq_dma_addr, NULL);
    393
    394	if (funq->rqes) {
    395		fun_clean_rq(funq);
    396		fun_free_ring_mem(dev, funq->rq_depth, sizeof(*funq->rqes),
    397				  false, funq->rqes, funq->rq_dma_addr,
    398				  funq->rq_info);
    399	}
    400
    401	kfree(funq);
    402}
    403
    404/* Allocate and initialize a funq's structures. */
    405struct fun_queue *fun_alloc_queue(struct fun_dev *fdev, int qid,
    406				  const struct fun_queue_alloc_req *req)
    407{
    408	struct fun_queue *funq = kzalloc(sizeof(*funq), GFP_KERNEL);
    409
    410	if (!funq)
    411		return NULL;
    412
    413	funq->fdev = fdev;
    414	spin_lock_init(&funq->sq_lock);
    415
    416	funq->qid = qid;
    417
    418	/* Initial CQ/SQ/RQ ids */
    419	if (req->rq_depth) {
    420		funq->cqid = 2 * qid;
    421		if (funq->qid) {
    422			/* I/O Q: use rqid = cqid, sqid = +1 */
    423			funq->rqid = funq->cqid;
    424			funq->sqid = funq->rqid + 1;
    425		} else {
    426			/* Admin Q: sqid is always 0, use ID 1 for RQ */
    427			funq->sqid = 0;
    428			funq->rqid = 1;
    429		}
    430	} else {
    431		funq->cqid = qid;
    432		funq->sqid = qid;
    433	}
    434
    435	funq->cq_flags = req->cq_flags;
    436	funq->sq_flags = req->sq_flags;
    437
    438	funq->cqe_size_log2 = req->cqe_size_log2;
    439	funq->sqe_size_log2 = req->sqe_size_log2;
    440
    441	funq->cq_depth = req->cq_depth;
    442	funq->sq_depth = req->sq_depth;
    443
    444	funq->cq_intcoal_nentries = req->cq_intcoal_nentries;
    445	funq->cq_intcoal_usec = req->cq_intcoal_usec;
    446
    447	funq->sq_intcoal_nentries = req->sq_intcoal_nentries;
    448	funq->sq_intcoal_usec = req->sq_intcoal_usec;
    449
    450	if (fun_alloc_cqes(funq))
    451		goto free_funq;
    452
    453	funq->cq_phase = 1;
    454
    455	if (fun_alloc_sqes(funq))
    456		goto free_funq;
    457
    458	if (req->rq_depth) {
    459		funq->rq_flags = req->rq_flags | FUN_ADMIN_EPSQ_CREATE_FLAG_RQ;
    460		funq->rq_depth = req->rq_depth;
    461		funq->rq_buf_offset = -1;
    462
    463		if (fun_alloc_rqes(funq) || fun_fill_rq(funq))
    464			goto free_funq;
    465	}
    466
    467	funq->cq_vector = -1;
    468	funq->cqe_info_offset = (1 << funq->cqe_size_log2) - sizeof(struct fun_cqe_info);
    469
    470	/* SQ/CQ 0 are implicitly created, assign their doorbells now.
    471	 * Other queues are assigned doorbells at their explicit creation.
    472	 */
    473	if (funq->sqid == 0)
    474		funq->sq_db = fun_sq_db_addr(fdev, 0);
    475	if (funq->cqid == 0)
    476		funq->cq_db = fun_cq_db_addr(fdev, 0);
    477
    478	return funq;
    479
    480free_funq:
    481	fun_free_queue(funq);
    482	return NULL;
    483}
    484
    485/* Create a funq's CQ on the device. */
    486static int fun_create_cq(struct fun_queue *funq)
    487{
    488	struct fun_dev *fdev = funq->fdev;
    489	unsigned int rqid;
    490	int rc;
    491
    492	rqid = funq->cq_flags & FUN_ADMIN_EPCQ_CREATE_FLAG_RQ ?
    493		funq->rqid : FUN_HCI_ID_INVALID;
    494	rc = fun_cq_create(fdev, funq->cq_flags, funq->cqid, rqid,
    495			   funq->cqe_size_log2, funq->cq_depth,
    496			   funq->cq_dma_addr, 0, 0, funq->cq_intcoal_nentries,
    497			   funq->cq_intcoal_usec, funq->cq_vector, 0, 0,
    498			   &funq->cqid, &funq->cq_db);
    499	if (!rc)
    500		dev_dbg(fdev->dev, "created CQ %u\n", funq->cqid);
    501
    502	return rc;
    503}
    504
    505/* Create a funq's SQ on the device. */
    506static int fun_create_sq(struct fun_queue *funq)
    507{
    508	struct fun_dev *fdev = funq->fdev;
    509	int rc;
    510
    511	rc = fun_sq_create(fdev, funq->sq_flags, funq->sqid, funq->cqid,
    512			   funq->sqe_size_log2, funq->sq_depth,
    513			   funq->sq_dma_addr, funq->sq_intcoal_nentries,
    514			   funq->sq_intcoal_usec, funq->cq_vector, 0, 0,
    515			   0, &funq->sqid, &funq->sq_db);
    516	if (!rc)
    517		dev_dbg(fdev->dev, "created SQ %u\n", funq->sqid);
    518
    519	return rc;
    520}
    521
    522/* Create a funq's RQ on the device. */
    523int fun_create_rq(struct fun_queue *funq)
    524{
    525	struct fun_dev *fdev = funq->fdev;
    526	int rc;
    527
    528	rc = fun_sq_create(fdev, funq->rq_flags, funq->rqid, funq->cqid, 0,
    529			   funq->rq_depth, funq->rq_dma_addr, 0, 0,
    530			   funq->cq_vector, 0, 0, PAGE_SHIFT, &funq->rqid,
    531			   &funq->rq_db);
    532	if (!rc)
    533		dev_dbg(fdev->dev, "created RQ %u\n", funq->rqid);
    534
    535	return rc;
    536}
    537
    538static unsigned int funq_irq(struct fun_queue *funq)
    539{
    540	return pci_irq_vector(to_pci_dev(funq->fdev->dev), funq->cq_vector);
    541}
    542
    543int fun_request_irq(struct fun_queue *funq, const char *devname,
    544		    irq_handler_t handler, void *data)
    545{
    546	int rc;
    547
    548	if (funq->cq_vector < 0)
    549		return -EINVAL;
    550
    551	funq->irq_handler = handler;
    552	funq->irq_data = data;
    553
    554	snprintf(funq->irqname, sizeof(funq->irqname),
    555		 funq->qid ? "%s-q[%d]" : "%s-adminq", devname, funq->qid);
    556
    557	rc = request_irq(funq_irq(funq), handler, 0, funq->irqname, data);
    558	if (rc)
    559		funq->irq_handler = NULL;
    560
    561	return rc;
    562}
    563
    564/* Create all component queues of a funq  on the device. */
    565int fun_create_queue(struct fun_queue *funq)
    566{
    567	int rc;
    568
    569	rc = fun_create_cq(funq);
    570	if (rc)
    571		return rc;
    572
    573	if (funq->rq_depth) {
    574		rc = fun_create_rq(funq);
    575		if (rc)
    576			goto release_cq;
    577	}
    578
    579	rc = fun_create_sq(funq);
    580	if (rc)
    581		goto release_rq;
    582
    583	return 0;
    584
    585release_rq:
    586	fun_destroy_sq(funq->fdev, funq->rqid);
    587release_cq:
    588	fun_destroy_cq(funq->fdev, funq->cqid);
    589	return rc;
    590}
    591
    592void fun_free_irq(struct fun_queue *funq)
    593{
    594	if (funq->irq_handler) {
    595		unsigned int vector = funq_irq(funq);
    596
    597		free_irq(vector, funq->irq_data);
    598		funq->irq_handler = NULL;
    599		funq->irq_data = NULL;
    600	}
    601}