cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

hinic_rx.c (15271B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Huawei HiNIC PCI Express Linux driver
      4 * Copyright(c) 2017 Huawei Technologies Co., Ltd
      5 */
      6
      7#include <linux/kernel.h>
      8#include <linux/types.h>
      9#include <linux/errno.h>
     10#include <linux/pci.h>
     11#include <linux/device.h>
     12#include <linux/netdevice.h>
     13#include <linux/etherdevice.h>
     14#include <linux/u64_stats_sync.h>
     15#include <linux/slab.h>
     16#include <linux/interrupt.h>
     17#include <linux/skbuff.h>
     18#include <linux/dma-mapping.h>
     19#include <linux/prefetch.h>
     20#include <linux/cpumask.h>
     21#include <linux/if_vlan.h>
     22#include <asm/barrier.h>
     23
     24#include "hinic_common.h"
     25#include "hinic_hw_if.h"
     26#include "hinic_hw_wqe.h"
     27#include "hinic_hw_wq.h"
     28#include "hinic_hw_qp.h"
     29#include "hinic_hw_dev.h"
     30#include "hinic_rx.h"
     31#include "hinic_dev.h"
     32
     33#define RX_IRQ_NO_PENDING               0
     34#define RX_IRQ_NO_COALESC               0
     35#define RX_IRQ_NO_LLI_TIMER             0
     36#define RX_IRQ_NO_CREDIT                0
     37#define RX_IRQ_NO_RESEND_TIMER          0
     38#define HINIC_RX_BUFFER_WRITE           16
     39
     40#define HINIC_RX_IPV6_PKT		7
     41#define LRO_PKT_HDR_LEN_IPV4		66
     42#define LRO_PKT_HDR_LEN_IPV6		86
     43#define LRO_REPLENISH_THLD		256
     44
     45#define LRO_PKT_HDR_LEN(cqe)		\
     46	(HINIC_GET_RX_PKT_TYPE(be32_to_cpu((cqe)->offload_type)) == \
     47	 HINIC_RX_IPV6_PKT ? LRO_PKT_HDR_LEN_IPV6 : LRO_PKT_HDR_LEN_IPV4)
     48
     49/**
     50 * hinic_rxq_clean_stats - Clean the statistics of specific queue
     51 * @rxq: Logical Rx Queue
     52 **/
     53void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
     54{
     55	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
     56
     57	u64_stats_update_begin(&rxq_stats->syncp);
     58	rxq_stats->pkts  = 0;
     59	rxq_stats->bytes = 0;
     60	rxq_stats->errors = 0;
     61	rxq_stats->csum_errors = 0;
     62	rxq_stats->other_errors = 0;
     63	u64_stats_update_end(&rxq_stats->syncp);
     64}
     65
     66/**
     67 * hinic_rxq_get_stats - get statistics of Rx Queue
     68 * @rxq: Logical Rx Queue
     69 * @stats: return updated stats here
     70 **/
     71void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
     72{
     73	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
     74	unsigned int start;
     75
     76	u64_stats_update_begin(&stats->syncp);
     77	do {
     78		start = u64_stats_fetch_begin(&rxq_stats->syncp);
     79		stats->pkts = rxq_stats->pkts;
     80		stats->bytes = rxq_stats->bytes;
     81		stats->errors = rxq_stats->csum_errors +
     82				rxq_stats->other_errors;
     83		stats->csum_errors = rxq_stats->csum_errors;
     84		stats->other_errors = rxq_stats->other_errors;
     85	} while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
     86	u64_stats_update_end(&stats->syncp);
     87}
     88
     89/**
     90 * rxq_stats_init - Initialize the statistics of specific queue
     91 * @rxq: Logical Rx Queue
     92 **/
     93static void rxq_stats_init(struct hinic_rxq *rxq)
     94{
     95	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
     96
     97	u64_stats_init(&rxq_stats->syncp);
     98	hinic_rxq_clean_stats(rxq);
     99}
    100
    101static void rx_csum(struct hinic_rxq *rxq, u32 status,
    102		    struct sk_buff *skb)
    103{
    104	struct net_device *netdev = rxq->netdev;
    105	u32 csum_err;
    106
    107	csum_err = HINIC_RQ_CQE_STATUS_GET(status, CSUM_ERR);
    108
    109	if (!(netdev->features & NETIF_F_RXCSUM))
    110		return;
    111
    112	if (!csum_err) {
    113		skb->ip_summed = CHECKSUM_UNNECESSARY;
    114	} else {
    115		if (!(csum_err & (HINIC_RX_CSUM_HW_CHECK_NONE |
    116			HINIC_RX_CSUM_IPSU_OTHER_ERR)))
    117			rxq->rxq_stats.csum_errors++;
    118		skb->ip_summed = CHECKSUM_NONE;
    119	}
    120}
    121
    122/**
    123 * rx_alloc_skb - allocate skb and map it to dma address
    124 * @rxq: rx queue
    125 * @dma_addr: returned dma address for the skb
    126 *
    127 * Return skb
    128 **/
    129static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,
    130				    dma_addr_t *dma_addr)
    131{
    132	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
    133	struct hinic_hwdev *hwdev = nic_dev->hwdev;
    134	struct hinic_hwif *hwif = hwdev->hwif;
    135	struct pci_dev *pdev = hwif->pdev;
    136	struct sk_buff *skb;
    137	dma_addr_t addr;
    138	int err;
    139
    140	skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);
    141	if (!skb)
    142		return NULL;
    143
    144	addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,
    145			      DMA_FROM_DEVICE);
    146	err = dma_mapping_error(&pdev->dev, addr);
    147	if (err) {
    148		dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);
    149		goto err_rx_map;
    150	}
    151
    152	*dma_addr = addr;
    153	return skb;
    154
    155err_rx_map:
    156	dev_kfree_skb_any(skb);
    157	return NULL;
    158}
    159
    160/**
    161 * rx_unmap_skb - unmap the dma address of the skb
    162 * @rxq: rx queue
    163 * @dma_addr: dma address of the skb
    164 **/
    165static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)
    166{
    167	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
    168	struct hinic_hwdev *hwdev = nic_dev->hwdev;
    169	struct hinic_hwif *hwif = hwdev->hwif;
    170	struct pci_dev *pdev = hwif->pdev;
    171
    172	dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,
    173			 DMA_FROM_DEVICE);
    174}
    175
    176/**
    177 * rx_free_skb - unmap and free skb
    178 * @rxq: rx queue
    179 * @skb: skb to free
    180 * @dma_addr: dma address of the skb
    181 **/
    182static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,
    183			dma_addr_t dma_addr)
    184{
    185	rx_unmap_skb(rxq, dma_addr);
    186	dev_kfree_skb_any(skb);
    187}
    188
    189/**
    190 * rx_alloc_pkts - allocate pkts in rx queue
    191 * @rxq: rx queue
    192 *
    193 * Return number of skbs allocated
    194 **/
    195static int rx_alloc_pkts(struct hinic_rxq *rxq)
    196{
    197	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
    198	struct hinic_rq_wqe *rq_wqe;
    199	unsigned int free_wqebbs;
    200	struct hinic_sge sge;
    201	dma_addr_t dma_addr;
    202	struct sk_buff *skb;
    203	u16 prod_idx;
    204	int i;
    205
    206	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
    207
    208	/* Limit the allocation chunks */
    209	if (free_wqebbs > nic_dev->rx_weight)
    210		free_wqebbs = nic_dev->rx_weight;
    211
    212	for (i = 0; i < free_wqebbs; i++) {
    213		skb = rx_alloc_skb(rxq, &dma_addr);
    214		if (!skb)
    215			goto skb_out;
    216
    217		hinic_set_sge(&sge, dma_addr, skb->len);
    218
    219		rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
    220					  &prod_idx);
    221		if (!rq_wqe) {
    222			rx_free_skb(rxq, skb, dma_addr);
    223			goto skb_out;
    224		}
    225
    226		hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);
    227
    228		hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);
    229	}
    230
    231skb_out:
    232	if (i) {
    233		wmb();  /* write all the wqes before update PI */
    234
    235		hinic_rq_update(rxq->rq, prod_idx);
    236	}
    237
    238	return i;
    239}
    240
    241/**
    242 * free_all_rx_skbs - free all skbs in rx queue
    243 * @rxq: rx queue
    244 **/
    245static void free_all_rx_skbs(struct hinic_rxq *rxq)
    246{
    247	struct hinic_rq *rq = rxq->rq;
    248	struct hinic_hw_wqe *hw_wqe;
    249	struct hinic_sge sge;
    250	u16 ci;
    251
    252	while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {
    253		if (IS_ERR(hw_wqe))
    254			break;
    255
    256		hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);
    257
    258		hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);
    259
    260		rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
    261	}
    262}
    263
    264/**
    265 * rx_recv_jumbo_pkt - Rx handler for jumbo pkt
    266 * @rxq: rx queue
    267 * @head_skb: the first skb in the list
    268 * @left_pkt_len: left size of the pkt exclude head skb
    269 * @ci: consumer index
    270 *
    271 * Return number of wqes that used for the left of the pkt
    272 **/
    273static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,
    274			     unsigned int left_pkt_len, u16 ci)
    275{
    276	struct sk_buff *skb, *curr_skb = head_skb;
    277	struct hinic_rq_wqe *rq_wqe;
    278	unsigned int curr_len;
    279	struct hinic_sge sge;
    280	int num_wqes = 0;
    281
    282	while (left_pkt_len > 0) {
    283		rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
    284						&skb, &ci);
    285
    286		num_wqes++;
    287
    288		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
    289
    290		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
    291
    292		prefetch(skb->data);
    293
    294		curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ :
    295			    left_pkt_len;
    296
    297		left_pkt_len -= curr_len;
    298
    299		__skb_put(skb, curr_len);
    300
    301		if (curr_skb == head_skb)
    302			skb_shinfo(head_skb)->frag_list = skb;
    303		else
    304			curr_skb->next = skb;
    305
    306		head_skb->len += skb->len;
    307		head_skb->data_len += skb->len;
    308		head_skb->truesize += skb->truesize;
    309
    310		curr_skb = skb;
    311	}
    312
    313	return num_wqes;
    314}
    315
    316static void hinic_copy_lp_data(struct hinic_dev *nic_dev,
    317			       struct sk_buff *skb)
    318{
    319	struct net_device *netdev = nic_dev->netdev;
    320	u8 *lb_buf = nic_dev->lb_test_rx_buf;
    321	int lb_len = nic_dev->lb_pkt_len;
    322	int pkt_offset, frag_len, i;
    323	void *frag_data = NULL;
    324
    325	if (nic_dev->lb_test_rx_idx == LP_PKT_CNT) {
    326		nic_dev->lb_test_rx_idx = 0;
    327		netif_warn(nic_dev, drv, netdev, "Loopback test warning, receive too more test pkts\n");
    328	}
    329
    330	if (skb->len != nic_dev->lb_pkt_len) {
    331		netif_warn(nic_dev, drv, netdev, "Wrong packet length\n");
    332		nic_dev->lb_test_rx_idx++;
    333		return;
    334	}
    335
    336	pkt_offset = nic_dev->lb_test_rx_idx * lb_len;
    337	frag_len = (int)skb_headlen(skb);
    338	memcpy(lb_buf + pkt_offset, skb->data, frag_len);
    339	pkt_offset += frag_len;
    340	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
    341		frag_data = skb_frag_address(&skb_shinfo(skb)->frags[i]);
    342		frag_len = (int)skb_frag_size(&skb_shinfo(skb)->frags[i]);
    343		memcpy((lb_buf + pkt_offset), frag_data, frag_len);
    344		pkt_offset += frag_len;
    345	}
    346	nic_dev->lb_test_rx_idx++;
    347}
    348
    349/**
    350 * rxq_recv - Rx handler
    351 * @rxq: rx queue
    352 * @budget: maximum pkts to process
    353 *
    354 * Return number of pkts received
    355 **/
    356static int rxq_recv(struct hinic_rxq *rxq, int budget)
    357{
    358	struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
    359	struct net_device *netdev = rxq->netdev;
    360	u64 pkt_len = 0, rx_bytes = 0;
    361	struct hinic_rq *rq = rxq->rq;
    362	struct hinic_rq_wqe *rq_wqe;
    363	struct hinic_dev *nic_dev;
    364	unsigned int free_wqebbs;
    365	struct hinic_rq_cqe *cqe;
    366	int num_wqes, pkts = 0;
    367	struct hinic_sge sge;
    368	unsigned int status;
    369	struct sk_buff *skb;
    370	u32 offload_type;
    371	u16 ci, num_lro;
    372	u16 num_wqe = 0;
    373	u32 vlan_len;
    374	u16 vid;
    375
    376	nic_dev = netdev_priv(netdev);
    377
    378	while (pkts < budget) {
    379		num_wqes = 0;
    380
    381		rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,
    382					   &ci);
    383		if (!rq_wqe)
    384			break;
    385
    386		/* make sure we read rx_done before packet length */
    387		dma_rmb();
    388
    389		cqe = rq->cqe[ci];
    390		status =  be32_to_cpu(cqe->status);
    391		hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
    392
    393		rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
    394
    395		rx_csum(rxq, status, skb);
    396
    397		prefetch(skb->data);
    398
    399		pkt_len = sge.len;
    400
    401		if (pkt_len <= HINIC_RX_BUF_SZ) {
    402			__skb_put(skb, pkt_len);
    403		} else {
    404			__skb_put(skb, HINIC_RX_BUF_SZ);
    405			num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -
    406						     HINIC_RX_BUF_SZ, ci);
    407		}
    408
    409		hinic_rq_put_wqe(rq, ci,
    410				 (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
    411
    412		offload_type = be32_to_cpu(cqe->offload_type);
    413		vlan_len = be32_to_cpu(cqe->len);
    414		if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
    415		    HINIC_GET_RX_VLAN_OFFLOAD_EN(offload_type)) {
    416			vid = HINIC_GET_RX_VLAN_TAG(vlan_len);
    417			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
    418		}
    419
    420		if (unlikely(nic_dev->flags & HINIC_LP_TEST))
    421			hinic_copy_lp_data(nic_dev, skb);
    422
    423		skb_record_rx_queue(skb, qp->q_id);
    424		skb->protocol = eth_type_trans(skb, rxq->netdev);
    425
    426		napi_gro_receive(&rxq->napi, skb);
    427
    428		pkts++;
    429		rx_bytes += pkt_len;
    430
    431		num_lro = HINIC_GET_RX_NUM_LRO(status);
    432		if (num_lro) {
    433			rx_bytes += ((num_lro - 1) *
    434				     LRO_PKT_HDR_LEN(cqe));
    435
    436			num_wqe +=
    437			(u16)(pkt_len >> rxq->rx_buff_shift) +
    438			((pkt_len & (rxq->buf_len - 1)) ? 1 : 0);
    439		}
    440
    441		cqe->status = 0;
    442
    443		if (num_wqe >= LRO_REPLENISH_THLD)
    444			break;
    445	}
    446
    447	free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
    448	if (free_wqebbs > HINIC_RX_BUFFER_WRITE)
    449		rx_alloc_pkts(rxq);
    450
    451	u64_stats_update_begin(&rxq->rxq_stats.syncp);
    452	rxq->rxq_stats.pkts += pkts;
    453	rxq->rxq_stats.bytes += rx_bytes;
    454	u64_stats_update_end(&rxq->rxq_stats.syncp);
    455
    456	return pkts;
    457}
    458
    459static int rx_poll(struct napi_struct *napi, int budget)
    460{
    461	struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
    462	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
    463	struct hinic_rq *rq = rxq->rq;
    464	int pkts;
    465
    466	pkts = rxq_recv(rxq, budget);
    467	if (pkts >= budget)
    468		return budget;
    469
    470	napi_complete(napi);
    471
    472	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))
    473		hinic_hwdev_set_msix_state(nic_dev->hwdev,
    474					   rq->msix_entry,
    475					   HINIC_MSIX_ENABLE);
    476
    477	return pkts;
    478}
    479
    480static void rx_add_napi(struct hinic_rxq *rxq)
    481{
    482	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
    483
    484	netif_napi_add_weight(rxq->netdev, &rxq->napi, rx_poll,
    485			      nic_dev->rx_weight);
    486	napi_enable(&rxq->napi);
    487}
    488
    489static void rx_del_napi(struct hinic_rxq *rxq)
    490{
    491	napi_disable(&rxq->napi);
    492	netif_napi_del(&rxq->napi);
    493}
    494
    495static irqreturn_t rx_irq(int irq, void *data)
    496{
    497	struct hinic_rxq *rxq = (struct hinic_rxq *)data;
    498	struct hinic_rq *rq = rxq->rq;
    499	struct hinic_dev *nic_dev;
    500
    501	/* Disable the interrupt until napi will be completed */
    502	nic_dev = netdev_priv(rxq->netdev);
    503	if (!HINIC_IS_VF(nic_dev->hwdev->hwif))
    504		hinic_hwdev_set_msix_state(nic_dev->hwdev,
    505					   rq->msix_entry,
    506					   HINIC_MSIX_DISABLE);
    507
    508	nic_dev = netdev_priv(rxq->netdev);
    509	hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
    510
    511	napi_schedule(&rxq->napi);
    512	return IRQ_HANDLED;
    513}
    514
    515static int rx_request_irq(struct hinic_rxq *rxq)
    516{
    517	struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
    518	struct hinic_msix_config interrupt_info = {0};
    519	struct hinic_intr_coal_info *intr_coal = NULL;
    520	struct hinic_hwdev *hwdev = nic_dev->hwdev;
    521	struct hinic_rq *rq = rxq->rq;
    522	struct hinic_qp *qp;
    523	int err;
    524
    525	qp = container_of(rq, struct hinic_qp, rq);
    526
    527	rx_add_napi(rxq);
    528
    529	hinic_hwdev_msix_set(hwdev, rq->msix_entry,
    530			     RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC,
    531			     RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
    532			     RX_IRQ_NO_RESEND_TIMER);
    533
    534	intr_coal = &nic_dev->rx_intr_coalesce[qp->q_id];
    535	interrupt_info.msix_index = rq->msix_entry;
    536	interrupt_info.coalesce_timer_cnt = intr_coal->coalesce_timer_cfg;
    537	interrupt_info.pending_cnt = intr_coal->pending_limt;
    538	interrupt_info.resend_timer_cnt = intr_coal->resend_timer_cfg;
    539
    540	err = hinic_set_interrupt_cfg(hwdev, &interrupt_info);
    541	if (err) {
    542		netif_err(nic_dev, drv, rxq->netdev,
    543			  "Failed to set RX interrupt coalescing attribute\n");
    544		goto err_req_irq;
    545	}
    546
    547	err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
    548	if (err)
    549		goto err_req_irq;
    550
    551	cpumask_set_cpu(qp->q_id % num_online_cpus(), &rq->affinity_mask);
    552	err = irq_set_affinity_and_hint(rq->irq, &rq->affinity_mask);
    553	if (err)
    554		goto err_irq_affinity;
    555
    556	return 0;
    557
    558err_irq_affinity:
    559	free_irq(rq->irq, rxq);
    560err_req_irq:
    561	rx_del_napi(rxq);
    562	return err;
    563}
    564
    565static void rx_free_irq(struct hinic_rxq *rxq)
    566{
    567	struct hinic_rq *rq = rxq->rq;
    568
    569	irq_update_affinity_hint(rq->irq, NULL);
    570	free_irq(rq->irq, rxq);
    571	rx_del_napi(rxq);
    572}
    573
    574/**
    575 * hinic_init_rxq - Initialize the Rx Queue
    576 * @rxq: Logical Rx Queue
    577 * @rq: Hardware Rx Queue to connect the Logical queue with
    578 * @netdev: network device to connect the Logical queue with
    579 *
    580 * Return 0 - Success, negative - Failure
    581 **/
    582int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
    583		   struct net_device *netdev)
    584{
    585	struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
    586	int err, pkts;
    587
    588	rxq->netdev = netdev;
    589	rxq->rq = rq;
    590	rxq->buf_len = HINIC_RX_BUF_SZ;
    591	rxq->rx_buff_shift = ilog2(HINIC_RX_BUF_SZ);
    592
    593	rxq_stats_init(rxq);
    594
    595	rxq->irq_name = devm_kasprintf(&netdev->dev, GFP_KERNEL,
    596				       "%s_rxq%d", netdev->name, qp->q_id);
    597	if (!rxq->irq_name)
    598		return -ENOMEM;
    599
    600	pkts = rx_alloc_pkts(rxq);
    601	if (!pkts) {
    602		err = -ENOMEM;
    603		goto err_rx_pkts;
    604	}
    605
    606	err = rx_request_irq(rxq);
    607	if (err) {
    608		netdev_err(netdev, "Failed to request Rx irq\n");
    609		goto err_req_rx_irq;
    610	}
    611
    612	return 0;
    613
    614err_req_rx_irq:
    615err_rx_pkts:
    616	free_all_rx_skbs(rxq);
    617	devm_kfree(&netdev->dev, rxq->irq_name);
    618	return err;
    619}
    620
    621/**
    622 * hinic_clean_rxq - Clean the Rx Queue
    623 * @rxq: Logical Rx Queue
    624 **/
    625void hinic_clean_rxq(struct hinic_rxq *rxq)
    626{
    627	struct net_device *netdev = rxq->netdev;
    628
    629	rx_free_irq(rxq);
    630
    631	free_all_rx_skbs(rxq);
    632	devm_kfree(&netdev->dev, rxq->irq_name);
    633}