cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

qlge_main.c (134502B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * QLogic qlge NIC HBA Driver
      4 * Copyright (c)  2003-2008 QLogic Corporation
      5 * Author:     Linux qlge network device driver by
      6 *                      Ron Mercer <ron.mercer@qlogic.com>
      7 */
      8#include <linux/kernel.h>
      9#include <linux/bitops.h>
     10#include <linux/types.h>
     11#include <linux/module.h>
     12#include <linux/list.h>
     13#include <linux/pci.h>
     14#include <linux/dma-mapping.h>
     15#include <linux/pagemap.h>
     16#include <linux/sched.h>
     17#include <linux/slab.h>
     18#include <linux/dmapool.h>
     19#include <linux/mempool.h>
     20#include <linux/spinlock.h>
     21#include <linux/kthread.h>
     22#include <linux/interrupt.h>
     23#include <linux/errno.h>
     24#include <linux/ioport.h>
     25#include <linux/in.h>
     26#include <linux/ip.h>
     27#include <linux/ipv6.h>
     28#include <net/ipv6.h>
     29#include <linux/tcp.h>
     30#include <linux/udp.h>
     31#include <linux/if_arp.h>
     32#include <linux/if_ether.h>
     33#include <linux/netdevice.h>
     34#include <linux/etherdevice.h>
     35#include <linux/ethtool.h>
     36#include <linux/if_vlan.h>
     37#include <linux/skbuff.h>
     38#include <linux/delay.h>
     39#include <linux/mm.h>
     40#include <linux/vmalloc.h>
     41#include <linux/prefetch.h>
     42#include <net/ip6_checksum.h>
     43
     44#include "qlge.h"
     45#include "qlge_devlink.h"
     46
     47char qlge_driver_name[] = DRV_NAME;
     48const char qlge_driver_version[] = DRV_VERSION;
     49
     50MODULE_AUTHOR("Ron Mercer <ron.mercer@qlogic.com>");
     51MODULE_DESCRIPTION(DRV_STRING " ");
     52MODULE_LICENSE("GPL");
     53MODULE_VERSION(DRV_VERSION);
     54
     55static const u32 default_msg =
     56	NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK |
     57	NETIF_MSG_IFDOWN |
     58	NETIF_MSG_IFUP |
     59	NETIF_MSG_RX_ERR |
     60	NETIF_MSG_TX_ERR |
     61	NETIF_MSG_HW | NETIF_MSG_WOL | 0;
     62
     63static int debug = -1;	/* defaults above */
     64module_param(debug, int, 0664);
     65MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
     66
     67#define MSIX_IRQ 0
     68#define MSI_IRQ 1
     69#define LEG_IRQ 2
     70static int qlge_irq_type = MSIX_IRQ;
     71module_param(qlge_irq_type, int, 0664);
     72MODULE_PARM_DESC(qlge_irq_type, "0 = MSI-X, 1 = MSI, 2 = Legacy.");
     73
     74static int qlge_mpi_coredump;
     75module_param(qlge_mpi_coredump, int, 0);
     76MODULE_PARM_DESC(qlge_mpi_coredump,
     77		 "Option to enable MPI firmware dump. Default is OFF - Do Not allocate memory. ");
     78
     79static int qlge_force_coredump;
     80module_param(qlge_force_coredump, int, 0);
     81MODULE_PARM_DESC(qlge_force_coredump,
     82		 "Option to allow force of firmware core dump. Default is OFF - Do not allow.");
     83
     84static const struct pci_device_id qlge_pci_tbl[] = {
     85	{PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, QLGE_DEVICE_ID_8012)},
     86	{PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, QLGE_DEVICE_ID_8000)},
     87	/* required last entry */
     88	{0,}
     89};
     90
     91MODULE_DEVICE_TABLE(pci, qlge_pci_tbl);
     92
     93static int qlge_wol(struct qlge_adapter *);
     94static void qlge_set_multicast_list(struct net_device *);
     95static int qlge_adapter_down(struct qlge_adapter *);
     96static int qlge_adapter_up(struct qlge_adapter *);
     97
     98/* This hardware semaphore causes exclusive access to
     99 * resources shared between the NIC driver, MPI firmware,
    100 * FCOE firmware and the FC driver.
    101 */
    102static int qlge_sem_trylock(struct qlge_adapter *qdev, u32 sem_mask)
    103{
    104	u32 sem_bits = 0;
    105
    106	switch (sem_mask) {
    107	case SEM_XGMAC0_MASK:
    108		sem_bits = SEM_SET << SEM_XGMAC0_SHIFT;
    109		break;
    110	case SEM_XGMAC1_MASK:
    111		sem_bits = SEM_SET << SEM_XGMAC1_SHIFT;
    112		break;
    113	case SEM_ICB_MASK:
    114		sem_bits = SEM_SET << SEM_ICB_SHIFT;
    115		break;
    116	case SEM_MAC_ADDR_MASK:
    117		sem_bits = SEM_SET << SEM_MAC_ADDR_SHIFT;
    118		break;
    119	case SEM_FLASH_MASK:
    120		sem_bits = SEM_SET << SEM_FLASH_SHIFT;
    121		break;
    122	case SEM_PROBE_MASK:
    123		sem_bits = SEM_SET << SEM_PROBE_SHIFT;
    124		break;
    125	case SEM_RT_IDX_MASK:
    126		sem_bits = SEM_SET << SEM_RT_IDX_SHIFT;
    127		break;
    128	case SEM_PROC_REG_MASK:
    129		sem_bits = SEM_SET << SEM_PROC_REG_SHIFT;
    130		break;
    131	default:
    132		netif_alert(qdev, probe, qdev->ndev, "bad Semaphore mask!.\n");
    133		return -EINVAL;
    134	}
    135
    136	qlge_write32(qdev, SEM, sem_bits | sem_mask);
    137	return !(qlge_read32(qdev, SEM) & sem_bits);
    138}
    139
    140int qlge_sem_spinlock(struct qlge_adapter *qdev, u32 sem_mask)
    141{
    142	unsigned int wait_count = 30;
    143
    144	do {
    145		if (!qlge_sem_trylock(qdev, sem_mask))
    146			return 0;
    147		udelay(100);
    148	} while (--wait_count);
    149	return -ETIMEDOUT;
    150}
    151
    152void qlge_sem_unlock(struct qlge_adapter *qdev, u32 sem_mask)
    153{
    154	qlge_write32(qdev, SEM, sem_mask);
    155	qlge_read32(qdev, SEM);	/* flush */
    156}
    157
    158/* This function waits for a specific bit to come ready
    159 * in a given register.  It is used mostly by the initialize
    160 * process, but is also used in kernel thread API such as
    161 * netdev->set_multi, netdev->set_mac_address, netdev->vlan_rx_add_vid.
    162 */
    163int qlge_wait_reg_rdy(struct qlge_adapter *qdev, u32 reg, u32 bit, u32 err_bit)
    164{
    165	u32 temp;
    166	int count;
    167
    168	for (count = 0; count < UDELAY_COUNT; count++) {
    169		temp = qlge_read32(qdev, reg);
    170
    171		/* check for errors */
    172		if (temp & err_bit) {
    173			netif_alert(qdev, probe, qdev->ndev,
    174				    "register 0x%.08x access error, value = 0x%.08x!.\n",
    175				    reg, temp);
    176			return -EIO;
    177		} else if (temp & bit) {
    178			return 0;
    179		}
    180		udelay(UDELAY_DELAY);
    181	}
    182	netif_alert(qdev, probe, qdev->ndev,
    183		    "Timed out waiting for reg %x to come ready.\n", reg);
    184	return -ETIMEDOUT;
    185}
    186
    187/* The CFG register is used to download TX and RX control blocks
    188 * to the chip. This function waits for an operation to complete.
    189 */
    190static int qlge_wait_cfg(struct qlge_adapter *qdev, u32 bit)
    191{
    192	int count;
    193	u32 temp;
    194
    195	for (count = 0; count < UDELAY_COUNT; count++) {
    196		temp = qlge_read32(qdev, CFG);
    197		if (temp & CFG_LE)
    198			return -EIO;
    199		if (!(temp & bit))
    200			return 0;
    201		udelay(UDELAY_DELAY);
    202	}
    203	return -ETIMEDOUT;
    204}
    205
    206/* Used to issue init control blocks to hw. Maps control block,
    207 * sets address, triggers download, waits for completion.
    208 */
    209int qlge_write_cfg(struct qlge_adapter *qdev, void *ptr, int size, u32 bit,
    210		   u16 q_id)
    211{
    212	u64 map;
    213	int status = 0;
    214	int direction;
    215	u32 mask;
    216	u32 value;
    217
    218	if (bit & (CFG_LRQ | CFG_LR | CFG_LCQ))
    219		direction = DMA_TO_DEVICE;
    220	else
    221		direction = DMA_FROM_DEVICE;
    222
    223	map = dma_map_single(&qdev->pdev->dev, ptr, size, direction);
    224	if (dma_mapping_error(&qdev->pdev->dev, map)) {
    225		netif_err(qdev, ifup, qdev->ndev, "Couldn't map DMA area.\n");
    226		return -ENOMEM;
    227	}
    228
    229	status = qlge_sem_spinlock(qdev, SEM_ICB_MASK);
    230	if (status)
    231		goto lock_failed;
    232
    233	status = qlge_wait_cfg(qdev, bit);
    234	if (status) {
    235		netif_err(qdev, ifup, qdev->ndev,
    236			  "Timed out waiting for CFG to come ready.\n");
    237		goto exit;
    238	}
    239
    240	qlge_write32(qdev, ICB_L, (u32)map);
    241	qlge_write32(qdev, ICB_H, (u32)(map >> 32));
    242
    243	mask = CFG_Q_MASK | (bit << 16);
    244	value = bit | (q_id << CFG_Q_SHIFT);
    245	qlge_write32(qdev, CFG, (mask | value));
    246
    247	/*
    248	 * Wait for the bit to clear after signaling hw.
    249	 */
    250	status = qlge_wait_cfg(qdev, bit);
    251exit:
    252	qlge_sem_unlock(qdev, SEM_ICB_MASK);	/* does flush too */
    253lock_failed:
    254	dma_unmap_single(&qdev->pdev->dev, map, size, direction);
    255	return status;
    256}
    257
    258/* Get a specific MAC address from the CAM.  Used for debug and reg dump. */
    259int qlge_get_mac_addr_reg(struct qlge_adapter *qdev, u32 type, u16 index,
    260			  u32 *value)
    261{
    262	u32 offset = 0;
    263	int status;
    264
    265	switch (type) {
    266	case MAC_ADDR_TYPE_MULTI_MAC:
    267	case MAC_ADDR_TYPE_CAM_MAC: {
    268		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    269		if (status)
    270			break;
    271		qlge_write32(qdev, MAC_ADDR_IDX,
    272			     (offset++) | /* offset */
    273				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
    274				   MAC_ADDR_ADR | MAC_ADDR_RS |
    275				   type); /* type */
    276		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MR, 0);
    277		if (status)
    278			break;
    279		*value++ = qlge_read32(qdev, MAC_ADDR_DATA);
    280		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    281		if (status)
    282			break;
    283		qlge_write32(qdev, MAC_ADDR_IDX,
    284			     (offset++) | /* offset */
    285				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
    286				   MAC_ADDR_ADR | MAC_ADDR_RS |
    287				   type); /* type */
    288		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MR, 0);
    289		if (status)
    290			break;
    291		*value++ = qlge_read32(qdev, MAC_ADDR_DATA);
    292		if (type == MAC_ADDR_TYPE_CAM_MAC) {
    293			status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX,
    294						   MAC_ADDR_MW, 0);
    295			if (status)
    296				break;
    297			qlge_write32(qdev, MAC_ADDR_IDX,
    298				     (offset++) | /* offset */
    299					   (index
    300					    << MAC_ADDR_IDX_SHIFT) | /* index */
    301					   MAC_ADDR_ADR |
    302					   MAC_ADDR_RS | type); /* type */
    303			status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX,
    304						   MAC_ADDR_MR, 0);
    305			if (status)
    306				break;
    307			*value++ = qlge_read32(qdev, MAC_ADDR_DATA);
    308		}
    309		break;
    310	}
    311	case MAC_ADDR_TYPE_VLAN:
    312	case MAC_ADDR_TYPE_MULTI_FLTR:
    313	default:
    314		netif_crit(qdev, ifup, qdev->ndev,
    315			   "Address type %d not yet supported.\n", type);
    316		status = -EPERM;
    317	}
    318	return status;
    319}
    320
    321/* Set up a MAC, multicast or VLAN address for the
    322 * inbound frame matching.
    323 */
    324static int qlge_set_mac_addr_reg(struct qlge_adapter *qdev, const u8 *addr,
    325				 u32 type, u16 index)
    326{
    327	u32 offset = 0;
    328	int status = 0;
    329
    330	switch (type) {
    331	case MAC_ADDR_TYPE_MULTI_MAC: {
    332		u32 upper = (addr[0] << 8) | addr[1];
    333		u32 lower = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
    334			    (addr[5]);
    335
    336		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    337		if (status)
    338			break;
    339		qlge_write32(qdev, MAC_ADDR_IDX,
    340			     (offset++) | (index << MAC_ADDR_IDX_SHIFT) | type |
    341				   MAC_ADDR_E);
    342		qlge_write32(qdev, MAC_ADDR_DATA, lower);
    343		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    344		if (status)
    345			break;
    346		qlge_write32(qdev, MAC_ADDR_IDX,
    347			     (offset++) | (index << MAC_ADDR_IDX_SHIFT) | type |
    348				   MAC_ADDR_E);
    349
    350		qlge_write32(qdev, MAC_ADDR_DATA, upper);
    351		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    352		break;
    353	}
    354	case MAC_ADDR_TYPE_CAM_MAC: {
    355		u32 cam_output;
    356		u32 upper = (addr[0] << 8) | addr[1];
    357		u32 lower = (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
    358			    (addr[5]);
    359		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    360		if (status)
    361			break;
    362		qlge_write32(qdev, MAC_ADDR_IDX,
    363			     (offset++) | /* offset */
    364				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
    365				   type); /* type */
    366		qlge_write32(qdev, MAC_ADDR_DATA, lower);
    367		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    368		if (status)
    369			break;
    370		qlge_write32(qdev, MAC_ADDR_IDX,
    371			     (offset++) | /* offset */
    372				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
    373				   type); /* type */
    374		qlge_write32(qdev, MAC_ADDR_DATA, upper);
    375		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    376		if (status)
    377			break;
    378		qlge_write32(qdev, MAC_ADDR_IDX,
    379			     (offset) | /* offset */
    380				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
    381				   type); /* type */
    382		/* This field should also include the queue id
    383		 * and possibly the function id.  Right now we hardcode
    384		 * the route field to NIC core.
    385		 */
    386		cam_output = (CAM_OUT_ROUTE_NIC |
    387			      (qdev->func << CAM_OUT_FUNC_SHIFT) |
    388			      (0 << CAM_OUT_CQ_ID_SHIFT));
    389		if (qdev->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
    390			cam_output |= CAM_OUT_RV;
    391		/* route to NIC core */
    392		qlge_write32(qdev, MAC_ADDR_DATA, cam_output);
    393		break;
    394	}
    395	case MAC_ADDR_TYPE_VLAN: {
    396		u32 enable_bit = *((u32 *)&addr[0]);
    397		/* For VLAN, the addr actually holds a bit that
    398		 * either enables or disables the vlan id we are
    399		 * addressing. It's either MAC_ADDR_E on or off.
    400		 * That's bit-27 we're talking about.
    401		 */
    402		status = qlge_wait_reg_rdy(qdev, MAC_ADDR_IDX, MAC_ADDR_MW, 0);
    403		if (status)
    404			break;
    405		qlge_write32(qdev, MAC_ADDR_IDX,
    406			     offset | /* offset */
    407				   (index << MAC_ADDR_IDX_SHIFT) | /* index */
    408				   type | /* type */
    409				   enable_bit); /* enable/disable */
    410		break;
    411	}
    412	case MAC_ADDR_TYPE_MULTI_FLTR:
    413	default:
    414		netif_crit(qdev, ifup, qdev->ndev,
    415			   "Address type %d not yet supported.\n", type);
    416		status = -EPERM;
    417	}
    418	return status;
    419}
    420
    421/* Set or clear MAC address in hardware. We sometimes
    422 * have to clear it to prevent wrong frame routing
    423 * especially in a bonding environment.
    424 */
    425static int qlge_set_mac_addr(struct qlge_adapter *qdev, int set)
    426{
    427	int status;
    428	char zero_mac_addr[ETH_ALEN];
    429	char *addr;
    430
    431	if (set) {
    432		addr = &qdev->current_mac_addr[0];
    433		netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
    434			     "Set Mac addr %pM\n", addr);
    435	} else {
    436		eth_zero_addr(zero_mac_addr);
    437		addr = &zero_mac_addr[0];
    438		netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
    439			     "Clearing MAC address\n");
    440	}
    441	status = qlge_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
    442	if (status)
    443		return status;
    444	status = qlge_set_mac_addr_reg(qdev, (const u8 *)addr,
    445				       MAC_ADDR_TYPE_CAM_MAC,
    446				       qdev->func * MAX_CQ);
    447	qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
    448	if (status)
    449		netif_err(qdev, ifup, qdev->ndev,
    450			  "Failed to init mac address.\n");
    451	return status;
    452}
    453
    454void qlge_link_on(struct qlge_adapter *qdev)
    455{
    456	netif_err(qdev, link, qdev->ndev, "Link is up.\n");
    457	netif_carrier_on(qdev->ndev);
    458	qlge_set_mac_addr(qdev, 1);
    459}
    460
    461void qlge_link_off(struct qlge_adapter *qdev)
    462{
    463	netif_err(qdev, link, qdev->ndev, "Link is down.\n");
    464	netif_carrier_off(qdev->ndev);
    465	qlge_set_mac_addr(qdev, 0);
    466}
    467
    468/* Get a specific frame routing value from the CAM.
    469 * Used for debug and reg dump.
    470 */
    471int qlge_get_routing_reg(struct qlge_adapter *qdev, u32 index, u32 *value)
    472{
    473	int status = 0;
    474
    475	status = qlge_wait_reg_rdy(qdev, RT_IDX, RT_IDX_MW, 0);
    476	if (status)
    477		goto exit;
    478
    479	qlge_write32(qdev, RT_IDX,
    480		     RT_IDX_TYPE_NICQ | RT_IDX_RS | (index << RT_IDX_IDX_SHIFT));
    481	status = qlge_wait_reg_rdy(qdev, RT_IDX, RT_IDX_MR, 0);
    482	if (status)
    483		goto exit;
    484	*value = qlge_read32(qdev, RT_DATA);
    485exit:
    486	return status;
    487}
    488
    489/* The NIC function for this chip has 16 routing indexes.  Each one can be used
    490 * to route different frame types to various inbound queues.  We send broadcast/
    491 * multicast/error frames to the default queue for slow handling,
    492 * and CAM hit/RSS frames to the fast handling queues.
    493 */
    494static int qlge_set_routing_reg(struct qlge_adapter *qdev, u32 index, u32 mask,
    495				int enable)
    496{
    497	int status = -EINVAL; /* Return error if no mask match. */
    498	u32 value = 0;
    499
    500	switch (mask) {
    501	case RT_IDX_CAM_HIT:
    502		{
    503			value = RT_IDX_DST_CAM_Q |	/* dest */
    504			    RT_IDX_TYPE_NICQ |	/* type */
    505			    (RT_IDX_CAM_HIT_SLOT << RT_IDX_IDX_SHIFT);/* index */
    506			break;
    507		}
    508	case RT_IDX_VALID:	/* Promiscuous Mode frames. */
    509		{
    510			value = RT_IDX_DST_DFLT_Q |	/* dest */
    511			    RT_IDX_TYPE_NICQ |	/* type */
    512			    (RT_IDX_PROMISCUOUS_SLOT << RT_IDX_IDX_SHIFT);/* index */
    513			break;
    514		}
    515	case RT_IDX_ERR:	/* Pass up MAC,IP,TCP/UDP error frames. */
    516		{
    517			value = RT_IDX_DST_DFLT_Q |	/* dest */
    518			    RT_IDX_TYPE_NICQ |	/* type */
    519			    (RT_IDX_ALL_ERR_SLOT << RT_IDX_IDX_SHIFT);/* index */
    520			break;
    521		}
    522	case RT_IDX_IP_CSUM_ERR: /* Pass up IP CSUM error frames. */
    523		{
    524			value = RT_IDX_DST_DFLT_Q | /* dest */
    525				RT_IDX_TYPE_NICQ | /* type */
    526				(RT_IDX_IP_CSUM_ERR_SLOT <<
    527				RT_IDX_IDX_SHIFT); /* index */
    528			break;
    529		}
    530	case RT_IDX_TU_CSUM_ERR: /* Pass up TCP/UDP CSUM error frames. */
    531		{
    532			value = RT_IDX_DST_DFLT_Q | /* dest */
    533				RT_IDX_TYPE_NICQ | /* type */
    534				(RT_IDX_TCP_UDP_CSUM_ERR_SLOT <<
    535				RT_IDX_IDX_SHIFT); /* index */
    536			break;
    537		}
    538	case RT_IDX_BCAST:	/* Pass up Broadcast frames to default Q. */
    539		{
    540			value = RT_IDX_DST_DFLT_Q |	/* dest */
    541			    RT_IDX_TYPE_NICQ |	/* type */
    542			    (RT_IDX_BCAST_SLOT << RT_IDX_IDX_SHIFT);/* index */
    543			break;
    544		}
    545	case RT_IDX_MCAST:	/* Pass up All Multicast frames. */
    546		{
    547			value = RT_IDX_DST_DFLT_Q |	/* dest */
    548			    RT_IDX_TYPE_NICQ |	/* type */
    549			    (RT_IDX_ALLMULTI_SLOT << RT_IDX_IDX_SHIFT);/* index */
    550			break;
    551		}
    552	case RT_IDX_MCAST_MATCH:	/* Pass up matched Multicast frames. */
    553		{
    554			value = RT_IDX_DST_DFLT_Q |	/* dest */
    555			    RT_IDX_TYPE_NICQ |	/* type */
    556			    (RT_IDX_MCAST_MATCH_SLOT << RT_IDX_IDX_SHIFT);/* index */
    557			break;
    558		}
    559	case RT_IDX_RSS_MATCH:	/* Pass up matched RSS frames. */
    560		{
    561			value = RT_IDX_DST_RSS |	/* dest */
    562			    RT_IDX_TYPE_NICQ |	/* type */
    563			    (RT_IDX_RSS_MATCH_SLOT << RT_IDX_IDX_SHIFT);/* index */
    564			break;
    565		}
    566	case 0:		/* Clear the E-bit on an entry. */
    567		{
    568			value = RT_IDX_DST_DFLT_Q |	/* dest */
    569			    RT_IDX_TYPE_NICQ |	/* type */
    570			    (index << RT_IDX_IDX_SHIFT);/* index */
    571			break;
    572		}
    573	default:
    574		netif_err(qdev, ifup, qdev->ndev,
    575			  "Mask type %d not yet supported.\n", mask);
    576		status = -EPERM;
    577		goto exit;
    578	}
    579
    580	if (value) {
    581		status = qlge_wait_reg_rdy(qdev, RT_IDX, RT_IDX_MW, 0);
    582		if (status)
    583			goto exit;
    584		value |= (enable ? RT_IDX_E : 0);
    585		qlge_write32(qdev, RT_IDX, value);
    586		qlge_write32(qdev, RT_DATA, enable ? mask : 0);
    587	}
    588exit:
    589	return status;
    590}
    591
    592static void qlge_enable_interrupts(struct qlge_adapter *qdev)
    593{
    594	qlge_write32(qdev, INTR_EN, (INTR_EN_EI << 16) | INTR_EN_EI);
    595}
    596
    597static void qlge_disable_interrupts(struct qlge_adapter *qdev)
    598{
    599	qlge_write32(qdev, INTR_EN, (INTR_EN_EI << 16));
    600}
    601
    602static void qlge_enable_completion_interrupt(struct qlge_adapter *qdev, u32 intr)
    603{
    604	struct intr_context *ctx = &qdev->intr_context[intr];
    605
    606	qlge_write32(qdev, INTR_EN, ctx->intr_en_mask);
    607}
    608
    609static void qlge_disable_completion_interrupt(struct qlge_adapter *qdev, u32 intr)
    610{
    611	struct intr_context *ctx = &qdev->intr_context[intr];
    612
    613	qlge_write32(qdev, INTR_EN, ctx->intr_dis_mask);
    614}
    615
    616static void qlge_enable_all_completion_interrupts(struct qlge_adapter *qdev)
    617{
    618	int i;
    619
    620	for (i = 0; i < qdev->intr_count; i++)
    621		qlge_enable_completion_interrupt(qdev, i);
    622}
    623
    624static int qlge_validate_flash(struct qlge_adapter *qdev, u32 size, const char *str)
    625{
    626	int status, i;
    627	u16 csum = 0;
    628	__le16 *flash = (__le16 *)&qdev->flash;
    629
    630	status = strncmp((char *)&qdev->flash, str, 4);
    631	if (status) {
    632		netif_err(qdev, ifup, qdev->ndev, "Invalid flash signature.\n");
    633		return	status;
    634	}
    635
    636	for (i = 0; i < size; i++)
    637		csum += le16_to_cpu(*flash++);
    638
    639	if (csum)
    640		netif_err(qdev, ifup, qdev->ndev,
    641			  "Invalid flash checksum, csum = 0x%.04x.\n", csum);
    642
    643	return csum;
    644}
    645
    646static int qlge_read_flash_word(struct qlge_adapter *qdev, int offset, __le32 *data)
    647{
    648	int status = 0;
    649	/* wait for reg to come ready */
    650	status = qlge_wait_reg_rdy(qdev,
    651				   FLASH_ADDR, FLASH_ADDR_RDY, FLASH_ADDR_ERR);
    652	if (status)
    653		goto exit;
    654	/* set up for reg read */
    655	qlge_write32(qdev, FLASH_ADDR, FLASH_ADDR_R | offset);
    656	/* wait for reg to come ready */
    657	status = qlge_wait_reg_rdy(qdev,
    658				   FLASH_ADDR, FLASH_ADDR_RDY, FLASH_ADDR_ERR);
    659	if (status)
    660		goto exit;
    661	/* This data is stored on flash as an array of
    662	 * __le32.  Since qlge_read32() returns cpu endian
    663	 * we need to swap it back.
    664	 */
    665	*data = cpu_to_le32(qlge_read32(qdev, FLASH_DATA));
    666exit:
    667	return status;
    668}
    669
    670static int qlge_get_8000_flash_params(struct qlge_adapter *qdev)
    671{
    672	u32 i, size;
    673	int status;
    674	__le32 *p = (__le32 *)&qdev->flash;
    675	u32 offset;
    676	u8 mac_addr[6];
    677
    678	/* Get flash offset for function and adjust
    679	 * for dword access.
    680	 */
    681	if (!qdev->port)
    682		offset = FUNC0_FLASH_OFFSET / sizeof(u32);
    683	else
    684		offset = FUNC1_FLASH_OFFSET / sizeof(u32);
    685
    686	if (qlge_sem_spinlock(qdev, SEM_FLASH_MASK))
    687		return -ETIMEDOUT;
    688
    689	size = sizeof(struct flash_params_8000) / sizeof(u32);
    690	for (i = 0; i < size; i++, p++) {
    691		status = qlge_read_flash_word(qdev, i + offset, p);
    692		if (status) {
    693			netif_err(qdev, ifup, qdev->ndev,
    694				  "Error reading flash.\n");
    695			goto exit;
    696		}
    697	}
    698
    699	status = qlge_validate_flash(qdev,
    700				     sizeof(struct flash_params_8000) /
    701				   sizeof(u16),
    702				   "8000");
    703	if (status) {
    704		netif_err(qdev, ifup, qdev->ndev, "Invalid flash.\n");
    705		status = -EINVAL;
    706		goto exit;
    707	}
    708
    709	/* Extract either manufacturer or BOFM modified
    710	 * MAC address.
    711	 */
    712	if (qdev->flash.flash_params_8000.data_type1 == 2)
    713		memcpy(mac_addr,
    714		       qdev->flash.flash_params_8000.mac_addr1,
    715		       qdev->ndev->addr_len);
    716	else
    717		memcpy(mac_addr,
    718		       qdev->flash.flash_params_8000.mac_addr,
    719		       qdev->ndev->addr_len);
    720
    721	if (!is_valid_ether_addr(mac_addr)) {
    722		netif_err(qdev, ifup, qdev->ndev, "Invalid MAC address.\n");
    723		status = -EINVAL;
    724		goto exit;
    725	}
    726
    727	eth_hw_addr_set(qdev->ndev, mac_addr);
    728
    729exit:
    730	qlge_sem_unlock(qdev, SEM_FLASH_MASK);
    731	return status;
    732}
    733
    734static int qlge_get_8012_flash_params(struct qlge_adapter *qdev)
    735{
    736	int i;
    737	int status;
    738	__le32 *p = (__le32 *)&qdev->flash;
    739	u32 offset = 0;
    740	u32 size = sizeof(struct flash_params_8012) / sizeof(u32);
    741
    742	/* Second function's parameters follow the first
    743	 * function's.
    744	 */
    745	if (qdev->port)
    746		offset = size;
    747
    748	if (qlge_sem_spinlock(qdev, SEM_FLASH_MASK))
    749		return -ETIMEDOUT;
    750
    751	for (i = 0; i < size; i++, p++) {
    752		status = qlge_read_flash_word(qdev, i + offset, p);
    753		if (status) {
    754			netif_err(qdev, ifup, qdev->ndev,
    755				  "Error reading flash.\n");
    756			goto exit;
    757		}
    758	}
    759
    760	status = qlge_validate_flash(qdev,
    761				     sizeof(struct flash_params_8012) /
    762				       sizeof(u16),
    763				     "8012");
    764	if (status) {
    765		netif_err(qdev, ifup, qdev->ndev, "Invalid flash.\n");
    766		status = -EINVAL;
    767		goto exit;
    768	}
    769
    770	if (!is_valid_ether_addr(qdev->flash.flash_params_8012.mac_addr)) {
    771		status = -EINVAL;
    772		goto exit;
    773	}
    774
    775	eth_hw_addr_set(qdev->ndev, qdev->flash.flash_params_8012.mac_addr);
    776
    777exit:
    778	qlge_sem_unlock(qdev, SEM_FLASH_MASK);
    779	return status;
    780}
    781
    782/* xgmac register are located behind the xgmac_addr and xgmac_data
    783 * register pair.  Each read/write requires us to wait for the ready
    784 * bit before reading/writing the data.
    785 */
    786static int qlge_write_xgmac_reg(struct qlge_adapter *qdev, u32 reg, u32 data)
    787{
    788	int status;
    789	/* wait for reg to come ready */
    790	status = qlge_wait_reg_rdy(qdev,
    791				   XGMAC_ADDR, XGMAC_ADDR_RDY, XGMAC_ADDR_XME);
    792	if (status)
    793		return status;
    794	/* write the data to the data reg */
    795	qlge_write32(qdev, XGMAC_DATA, data);
    796	/* trigger the write */
    797	qlge_write32(qdev, XGMAC_ADDR, reg);
    798	return status;
    799}
    800
    801/* xgmac register are located behind the xgmac_addr and xgmac_data
    802 * register pair.  Each read/write requires us to wait for the ready
    803 * bit before reading/writing the data.
    804 */
    805int qlge_read_xgmac_reg(struct qlge_adapter *qdev, u32 reg, u32 *data)
    806{
    807	int status = 0;
    808	/* wait for reg to come ready */
    809	status = qlge_wait_reg_rdy(qdev,
    810				   XGMAC_ADDR, XGMAC_ADDR_RDY, XGMAC_ADDR_XME);
    811	if (status)
    812		goto exit;
    813	/* set up for reg read */
    814	qlge_write32(qdev, XGMAC_ADDR, reg | XGMAC_ADDR_R);
    815	/* wait for reg to come ready */
    816	status = qlge_wait_reg_rdy(qdev,
    817				   XGMAC_ADDR, XGMAC_ADDR_RDY, XGMAC_ADDR_XME);
    818	if (status)
    819		goto exit;
    820	/* get the data */
    821	*data = qlge_read32(qdev, XGMAC_DATA);
    822exit:
    823	return status;
    824}
    825
    826/* This is used for reading the 64-bit statistics regs. */
    827int qlge_read_xgmac_reg64(struct qlge_adapter *qdev, u32 reg, u64 *data)
    828{
    829	int status = 0;
    830	u32 hi = 0;
    831	u32 lo = 0;
    832
    833	status = qlge_read_xgmac_reg(qdev, reg, &lo);
    834	if (status)
    835		goto exit;
    836
    837	status = qlge_read_xgmac_reg(qdev, reg + 4, &hi);
    838	if (status)
    839		goto exit;
    840
    841	*data = (u64)lo | ((u64)hi << 32);
    842
    843exit:
    844	return status;
    845}
    846
    847static int qlge_8000_port_initialize(struct qlge_adapter *qdev)
    848{
    849	int status;
    850	/*
    851	 * Get MPI firmware version for driver banner
    852	 * and ethool info.
    853	 */
    854	status = qlge_mb_about_fw(qdev);
    855	if (status)
    856		goto exit;
    857	status = qlge_mb_get_fw_state(qdev);
    858	if (status)
    859		goto exit;
    860	/* Wake up a worker to get/set the TX/RX frame sizes. */
    861	queue_delayed_work(qdev->workqueue, &qdev->mpi_port_cfg_work, 0);
    862exit:
    863	return status;
    864}
    865
    866/* Take the MAC Core out of reset.
    867 * Enable statistics counting.
    868 * Take the transmitter/receiver out of reset.
    869 * This functionality may be done in the MPI firmware at a
    870 * later date.
    871 */
    872static int qlge_8012_port_initialize(struct qlge_adapter *qdev)
    873{
    874	int status = 0;
    875	u32 data;
    876
    877	if (qlge_sem_trylock(qdev, qdev->xg_sem_mask)) {
    878		/* Another function has the semaphore, so
    879		 * wait for the port init bit to come ready.
    880		 */
    881		netif_info(qdev, link, qdev->ndev,
    882			   "Another function has the semaphore, so wait for the port init bit to come ready.\n");
    883		status = qlge_wait_reg_rdy(qdev, STS, qdev->port_init, 0);
    884		if (status) {
    885			netif_crit(qdev, link, qdev->ndev,
    886				   "Port initialize timed out.\n");
    887		}
    888		return status;
    889	}
    890
    891	netif_info(qdev, link, qdev->ndev, "Got xgmac semaphore!.\n");
    892	/* Set the core reset. */
    893	status = qlge_read_xgmac_reg(qdev, GLOBAL_CFG, &data);
    894	if (status)
    895		goto end;
    896	data |= GLOBAL_CFG_RESET;
    897	status = qlge_write_xgmac_reg(qdev, GLOBAL_CFG, data);
    898	if (status)
    899		goto end;
    900
    901	/* Clear the core reset and turn on jumbo for receiver. */
    902	data &= ~GLOBAL_CFG_RESET;	/* Clear core reset. */
    903	data |= GLOBAL_CFG_JUMBO;	/* Turn on jumbo. */
    904	data |= GLOBAL_CFG_TX_STAT_EN;
    905	data |= GLOBAL_CFG_RX_STAT_EN;
    906	status = qlge_write_xgmac_reg(qdev, GLOBAL_CFG, data);
    907	if (status)
    908		goto end;
    909
    910	/* Enable transmitter, and clear it's reset. */
    911	status = qlge_read_xgmac_reg(qdev, TX_CFG, &data);
    912	if (status)
    913		goto end;
    914	data &= ~TX_CFG_RESET;	/* Clear the TX MAC reset. */
    915	data |= TX_CFG_EN;	/* Enable the transmitter. */
    916	status = qlge_write_xgmac_reg(qdev, TX_CFG, data);
    917	if (status)
    918		goto end;
    919
    920	/* Enable receiver and clear it's reset. */
    921	status = qlge_read_xgmac_reg(qdev, RX_CFG, &data);
    922	if (status)
    923		goto end;
    924	data &= ~RX_CFG_RESET;	/* Clear the RX MAC reset. */
    925	data |= RX_CFG_EN;	/* Enable the receiver. */
    926	status = qlge_write_xgmac_reg(qdev, RX_CFG, data);
    927	if (status)
    928		goto end;
    929
    930	/* Turn on jumbo. */
    931	status =
    932	    qlge_write_xgmac_reg(qdev, MAC_TX_PARAMS, MAC_TX_PARAMS_JUMBO | (0x2580 << 16));
    933	if (status)
    934		goto end;
    935	status =
    936	    qlge_write_xgmac_reg(qdev, MAC_RX_PARAMS, 0x2580);
    937	if (status)
    938		goto end;
    939
    940	/* Signal to the world that the port is enabled.        */
    941	qlge_write32(qdev, STS, ((qdev->port_init << 16) | qdev->port_init));
    942end:
    943	qlge_sem_unlock(qdev, qdev->xg_sem_mask);
    944	return status;
    945}
    946
    947static inline unsigned int qlge_lbq_block_size(struct qlge_adapter *qdev)
    948{
    949	return PAGE_SIZE << qdev->lbq_buf_order;
    950}
    951
    952static struct qlge_bq_desc *qlge_get_curr_buf(struct qlge_bq *bq)
    953{
    954	struct qlge_bq_desc *bq_desc;
    955
    956	bq_desc = &bq->queue[bq->next_to_clean];
    957	bq->next_to_clean = QLGE_BQ_WRAP(bq->next_to_clean + 1);
    958
    959	return bq_desc;
    960}
    961
    962static struct qlge_bq_desc *qlge_get_curr_lchunk(struct qlge_adapter *qdev,
    963						 struct rx_ring *rx_ring)
    964{
    965	struct qlge_bq_desc *lbq_desc = qlge_get_curr_buf(&rx_ring->lbq);
    966
    967	dma_sync_single_for_cpu(&qdev->pdev->dev, lbq_desc->dma_addr,
    968				qdev->lbq_buf_size, DMA_FROM_DEVICE);
    969
    970	if ((lbq_desc->p.pg_chunk.offset + qdev->lbq_buf_size) ==
    971	    qlge_lbq_block_size(qdev)) {
    972		/* last chunk of the master page */
    973		dma_unmap_page(&qdev->pdev->dev, lbq_desc->dma_addr,
    974			       qlge_lbq_block_size(qdev), DMA_FROM_DEVICE);
    975	}
    976
    977	return lbq_desc;
    978}
    979
    980/* Update an rx ring index. */
    981static void qlge_update_cq(struct rx_ring *rx_ring)
    982{
    983	rx_ring->cnsmr_idx++;
    984	rx_ring->curr_entry++;
    985	if (unlikely(rx_ring->cnsmr_idx == rx_ring->cq_len)) {
    986		rx_ring->cnsmr_idx = 0;
    987		rx_ring->curr_entry = rx_ring->cq_base;
    988	}
    989}
    990
    991static void qlge_write_cq_idx(struct rx_ring *rx_ring)
    992{
    993	qlge_write_db_reg(rx_ring->cnsmr_idx, rx_ring->cnsmr_idx_db_reg);
    994}
    995
    996static const char * const bq_type_name[] = {
    997	[QLGE_SB] = "sbq",
    998	[QLGE_LB] = "lbq",
    999};
   1000
   1001/* return 0 or negative error */
   1002static int qlge_refill_sb(struct rx_ring *rx_ring,
   1003			  struct qlge_bq_desc *sbq_desc, gfp_t gfp)
   1004{
   1005	struct qlge_adapter *qdev = rx_ring->qdev;
   1006	struct sk_buff *skb;
   1007
   1008	if (sbq_desc->p.skb)
   1009		return 0;
   1010
   1011	netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1012		     "ring %u sbq: getting new skb for index %d.\n",
   1013		     rx_ring->cq_id, sbq_desc->index);
   1014
   1015	skb = __netdev_alloc_skb(qdev->ndev, SMALL_BUFFER_SIZE, gfp);
   1016	if (!skb)
   1017		return -ENOMEM;
   1018	skb_reserve(skb, QLGE_SB_PAD);
   1019
   1020	sbq_desc->dma_addr = dma_map_single(&qdev->pdev->dev, skb->data,
   1021					    SMALL_BUF_MAP_SIZE,
   1022					    DMA_FROM_DEVICE);
   1023	if (dma_mapping_error(&qdev->pdev->dev, sbq_desc->dma_addr)) {
   1024		netif_err(qdev, ifup, qdev->ndev, "PCI mapping failed.\n");
   1025		dev_kfree_skb_any(skb);
   1026		return -EIO;
   1027	}
   1028	*sbq_desc->buf_ptr = cpu_to_le64(sbq_desc->dma_addr);
   1029
   1030	sbq_desc->p.skb = skb;
   1031	return 0;
   1032}
   1033
   1034/* return 0 or negative error */
   1035static int qlge_refill_lb(struct rx_ring *rx_ring,
   1036			  struct qlge_bq_desc *lbq_desc, gfp_t gfp)
   1037{
   1038	struct qlge_adapter *qdev = rx_ring->qdev;
   1039	struct qlge_page_chunk *master_chunk = &rx_ring->master_chunk;
   1040
   1041	if (!master_chunk->page) {
   1042		struct page *page;
   1043		dma_addr_t dma_addr;
   1044
   1045		page = alloc_pages(gfp | __GFP_COMP, qdev->lbq_buf_order);
   1046		if (unlikely(!page))
   1047			return -ENOMEM;
   1048		dma_addr = dma_map_page(&qdev->pdev->dev, page, 0,
   1049					qlge_lbq_block_size(qdev),
   1050					DMA_FROM_DEVICE);
   1051		if (dma_mapping_error(&qdev->pdev->dev, dma_addr)) {
   1052			__free_pages(page, qdev->lbq_buf_order);
   1053			netif_err(qdev, drv, qdev->ndev,
   1054				  "PCI mapping failed.\n");
   1055			return -EIO;
   1056		}
   1057		master_chunk->page = page;
   1058		master_chunk->va = page_address(page);
   1059		master_chunk->offset = 0;
   1060		rx_ring->chunk_dma_addr = dma_addr;
   1061	}
   1062
   1063	lbq_desc->p.pg_chunk = *master_chunk;
   1064	lbq_desc->dma_addr = rx_ring->chunk_dma_addr;
   1065	*lbq_desc->buf_ptr = cpu_to_le64(lbq_desc->dma_addr +
   1066					 lbq_desc->p.pg_chunk.offset);
   1067
   1068	/* Adjust the master page chunk for next
   1069	 * buffer get.
   1070	 */
   1071	master_chunk->offset += qdev->lbq_buf_size;
   1072	if (master_chunk->offset == qlge_lbq_block_size(qdev)) {
   1073		master_chunk->page = NULL;
   1074	} else {
   1075		master_chunk->va += qdev->lbq_buf_size;
   1076		get_page(master_chunk->page);
   1077	}
   1078
   1079	return 0;
   1080}
   1081
   1082/* return 0 or negative error */
   1083static int qlge_refill_bq(struct qlge_bq *bq, gfp_t gfp)
   1084{
   1085	struct rx_ring *rx_ring = QLGE_BQ_CONTAINER(bq);
   1086	struct qlge_adapter *qdev = rx_ring->qdev;
   1087	struct qlge_bq_desc *bq_desc;
   1088	int refill_count;
   1089	int retval;
   1090	int i;
   1091
   1092	refill_count = QLGE_BQ_WRAP(QLGE_BQ_ALIGN(bq->next_to_clean - 1) -
   1093				    bq->next_to_use);
   1094	if (!refill_count)
   1095		return 0;
   1096
   1097	i = bq->next_to_use;
   1098	bq_desc = &bq->queue[i];
   1099	i -= QLGE_BQ_LEN;
   1100	do {
   1101		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1102			     "ring %u %s: try cleaning idx %d\n",
   1103			     rx_ring->cq_id, bq_type_name[bq->type], i);
   1104
   1105		if (bq->type == QLGE_SB)
   1106			retval = qlge_refill_sb(rx_ring, bq_desc, gfp);
   1107		else
   1108			retval = qlge_refill_lb(rx_ring, bq_desc, gfp);
   1109		if (retval < 0) {
   1110			netif_err(qdev, ifup, qdev->ndev,
   1111				  "ring %u %s: Could not get a page chunk, idx %d\n",
   1112				  rx_ring->cq_id, bq_type_name[bq->type], i);
   1113			break;
   1114		}
   1115
   1116		bq_desc++;
   1117		i++;
   1118		if (unlikely(!i)) {
   1119			bq_desc = &bq->queue[0];
   1120			i -= QLGE_BQ_LEN;
   1121		}
   1122		refill_count--;
   1123	} while (refill_count);
   1124	i += QLGE_BQ_LEN;
   1125
   1126	if (bq->next_to_use != i) {
   1127		if (QLGE_BQ_ALIGN(bq->next_to_use) != QLGE_BQ_ALIGN(i)) {
   1128			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1129				     "ring %u %s: updating prod idx = %d.\n",
   1130				     rx_ring->cq_id, bq_type_name[bq->type],
   1131				     i);
   1132			qlge_write_db_reg(i, bq->prod_idx_db_reg);
   1133		}
   1134		bq->next_to_use = i;
   1135	}
   1136
   1137	return retval;
   1138}
   1139
   1140static void qlge_update_buffer_queues(struct rx_ring *rx_ring, gfp_t gfp,
   1141				      unsigned long delay)
   1142{
   1143	bool sbq_fail, lbq_fail;
   1144
   1145	sbq_fail = !!qlge_refill_bq(&rx_ring->sbq, gfp);
   1146	lbq_fail = !!qlge_refill_bq(&rx_ring->lbq, gfp);
   1147
   1148	/* Minimum number of buffers needed to be able to receive at least one
   1149	 * frame of any format:
   1150	 * sbq: 1 for header + 1 for data
   1151	 * lbq: mtu 9000 / lb size
   1152	 * Below this, the queue might stall.
   1153	 */
   1154	if ((sbq_fail && QLGE_BQ_HW_OWNED(&rx_ring->sbq) < 2) ||
   1155	    (lbq_fail && QLGE_BQ_HW_OWNED(&rx_ring->lbq) <
   1156	     DIV_ROUND_UP(9000, LARGE_BUFFER_MAX_SIZE)))
   1157		/* Allocations can take a long time in certain cases (ex.
   1158		 * reclaim). Therefore, use a workqueue for long-running
   1159		 * work items.
   1160		 */
   1161		queue_delayed_work_on(smp_processor_id(), system_long_wq,
   1162				      &rx_ring->refill_work, delay);
   1163}
   1164
   1165static void qlge_slow_refill(struct work_struct *work)
   1166{
   1167	struct rx_ring *rx_ring = container_of(work, struct rx_ring,
   1168					       refill_work.work);
   1169	struct napi_struct *napi = &rx_ring->napi;
   1170
   1171	napi_disable(napi);
   1172	qlge_update_buffer_queues(rx_ring, GFP_KERNEL, HZ / 2);
   1173	napi_enable(napi);
   1174
   1175	local_bh_disable();
   1176	/* napi_disable() might have prevented incomplete napi work from being
   1177	 * rescheduled.
   1178	 */
   1179	napi_schedule(napi);
   1180	/* trigger softirq processing */
   1181	local_bh_enable();
   1182}
   1183
   1184/* Unmaps tx buffers.  Can be called from send() if a pci mapping
   1185 * fails at some stage, or from the interrupt when a tx completes.
   1186 */
   1187static void qlge_unmap_send(struct qlge_adapter *qdev,
   1188			    struct tx_ring_desc *tx_ring_desc, int mapped)
   1189{
   1190	int i;
   1191
   1192	for (i = 0; i < mapped; i++) {
   1193		if (i == 0 || (i == 7 && mapped > 7)) {
   1194			/*
   1195			 * Unmap the skb->data area, or the
   1196			 * external sglist (AKA the Outbound
   1197			 * Address List (OAL)).
   1198			 * If its the zeroeth element, then it's
   1199			 * the skb->data area.  If it's the 7th
   1200			 * element and there is more than 6 frags,
   1201			 * then its an OAL.
   1202			 */
   1203			if (i == 7) {
   1204				netif_printk(qdev, tx_done, KERN_DEBUG,
   1205					     qdev->ndev,
   1206					     "unmapping OAL area.\n");
   1207			}
   1208			dma_unmap_single(&qdev->pdev->dev,
   1209					 dma_unmap_addr(&tx_ring_desc->map[i],
   1210							mapaddr),
   1211					 dma_unmap_len(&tx_ring_desc->map[i],
   1212						       maplen),
   1213					 DMA_TO_DEVICE);
   1214		} else {
   1215			netif_printk(qdev, tx_done, KERN_DEBUG, qdev->ndev,
   1216				     "unmapping frag %d.\n", i);
   1217			dma_unmap_page(&qdev->pdev->dev,
   1218				       dma_unmap_addr(&tx_ring_desc->map[i],
   1219						      mapaddr),
   1220				       dma_unmap_len(&tx_ring_desc->map[i],
   1221						     maplen), DMA_TO_DEVICE);
   1222		}
   1223	}
   1224}
   1225
   1226/* Map the buffers for this transmit.  This will return
   1227 * NETDEV_TX_BUSY or NETDEV_TX_OK based on success.
   1228 */
   1229static int qlge_map_send(struct qlge_adapter *qdev,
   1230			 struct qlge_ob_mac_iocb_req *mac_iocb_ptr,
   1231			 struct sk_buff *skb, struct tx_ring_desc *tx_ring_desc)
   1232{
   1233	int len = skb_headlen(skb);
   1234	dma_addr_t map;
   1235	int frag_idx, err, map_idx = 0;
   1236	struct tx_buf_desc *tbd = mac_iocb_ptr->tbd;
   1237	int frag_cnt = skb_shinfo(skb)->nr_frags;
   1238
   1239	if (frag_cnt) {
   1240		netif_printk(qdev, tx_queued, KERN_DEBUG, qdev->ndev,
   1241			     "frag_cnt = %d.\n", frag_cnt);
   1242	}
   1243	/*
   1244	 * Map the skb buffer first.
   1245	 */
   1246	map = dma_map_single(&qdev->pdev->dev, skb->data, len, DMA_TO_DEVICE);
   1247
   1248	err = dma_mapping_error(&qdev->pdev->dev, map);
   1249	if (err) {
   1250		netif_err(qdev, tx_queued, qdev->ndev,
   1251			  "PCI mapping failed with error: %d\n", err);
   1252
   1253		return NETDEV_TX_BUSY;
   1254	}
   1255
   1256	tbd->len = cpu_to_le32(len);
   1257	tbd->addr = cpu_to_le64(map);
   1258	dma_unmap_addr_set(&tx_ring_desc->map[map_idx], mapaddr, map);
   1259	dma_unmap_len_set(&tx_ring_desc->map[map_idx], maplen, len);
   1260	map_idx++;
   1261
   1262	/*
   1263	 * This loop fills the remainder of the 8 address descriptors
   1264	 * in the IOCB.  If there are more than 7 fragments, then the
   1265	 * eighth address desc will point to an external list (OAL).
   1266	 * When this happens, the remainder of the frags will be stored
   1267	 * in this list.
   1268	 */
   1269	for (frag_idx = 0; frag_idx < frag_cnt; frag_idx++, map_idx++) {
   1270		skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_idx];
   1271
   1272		tbd++;
   1273		if (frag_idx == 6 && frag_cnt > 7) {
   1274			/* Let's tack on an sglist.
   1275			 * Our control block will now
   1276			 * look like this:
   1277			 * iocb->seg[0] = skb->data
   1278			 * iocb->seg[1] = frag[0]
   1279			 * iocb->seg[2] = frag[1]
   1280			 * iocb->seg[3] = frag[2]
   1281			 * iocb->seg[4] = frag[3]
   1282			 * iocb->seg[5] = frag[4]
   1283			 * iocb->seg[6] = frag[5]
   1284			 * iocb->seg[7] = ptr to OAL (external sglist)
   1285			 * oal->seg[0] = frag[6]
   1286			 * oal->seg[1] = frag[7]
   1287			 * oal->seg[2] = frag[8]
   1288			 * oal->seg[3] = frag[9]
   1289			 * oal->seg[4] = frag[10]
   1290			 *      etc...
   1291			 */
   1292			/* Tack on the OAL in the eighth segment of IOCB. */
   1293			map = dma_map_single(&qdev->pdev->dev, &tx_ring_desc->oal,
   1294					     sizeof(struct qlge_oal),
   1295					     DMA_TO_DEVICE);
   1296			err = dma_mapping_error(&qdev->pdev->dev, map);
   1297			if (err) {
   1298				netif_err(qdev, tx_queued, qdev->ndev,
   1299					  "PCI mapping outbound address list with error: %d\n",
   1300					  err);
   1301				goto map_error;
   1302			}
   1303
   1304			tbd->addr = cpu_to_le64(map);
   1305			/*
   1306			 * The length is the number of fragments
   1307			 * that remain to be mapped times the length
   1308			 * of our sglist (OAL).
   1309			 */
   1310			tbd->len =
   1311			    cpu_to_le32((sizeof(struct tx_buf_desc) *
   1312					 (frag_cnt - frag_idx)) | TX_DESC_C);
   1313			dma_unmap_addr_set(&tx_ring_desc->map[map_idx], mapaddr,
   1314					   map);
   1315			dma_unmap_len_set(&tx_ring_desc->map[map_idx], maplen,
   1316					  sizeof(struct qlge_oal));
   1317			tbd = (struct tx_buf_desc *)&tx_ring_desc->oal;
   1318			map_idx++;
   1319		}
   1320
   1321		map = skb_frag_dma_map(&qdev->pdev->dev, frag, 0, skb_frag_size(frag),
   1322				       DMA_TO_DEVICE);
   1323
   1324		err = dma_mapping_error(&qdev->pdev->dev, map);
   1325		if (err) {
   1326			netif_err(qdev, tx_queued, qdev->ndev,
   1327				  "PCI mapping frags failed with error: %d.\n",
   1328				  err);
   1329			goto map_error;
   1330		}
   1331
   1332		tbd->addr = cpu_to_le64(map);
   1333		tbd->len = cpu_to_le32(skb_frag_size(frag));
   1334		dma_unmap_addr_set(&tx_ring_desc->map[map_idx], mapaddr, map);
   1335		dma_unmap_len_set(&tx_ring_desc->map[map_idx], maplen,
   1336				  skb_frag_size(frag));
   1337	}
   1338	/* Save the number of segments we've mapped. */
   1339	tx_ring_desc->map_cnt = map_idx;
   1340	/* Terminate the last segment. */
   1341	tbd->len = cpu_to_le32(le32_to_cpu(tbd->len) | TX_DESC_E);
   1342	return NETDEV_TX_OK;
   1343
   1344map_error:
   1345	/*
   1346	 * If the first frag mapping failed, then i will be zero.
   1347	 * This causes the unmap of the skb->data area.  Otherwise
   1348	 * we pass in the number of frags that mapped successfully
   1349	 * so they can be umapped.
   1350	 */
   1351	qlge_unmap_send(qdev, tx_ring_desc, map_idx);
   1352	return NETDEV_TX_BUSY;
   1353}
   1354
   1355/* Categorizing receive firmware frame errors */
   1356static void qlge_categorize_rx_err(struct qlge_adapter *qdev, u8 rx_err,
   1357				   struct rx_ring *rx_ring)
   1358{
   1359	struct nic_stats *stats = &qdev->nic_stats;
   1360
   1361	stats->rx_err_count++;
   1362	rx_ring->rx_errors++;
   1363
   1364	switch (rx_err & IB_MAC_IOCB_RSP_ERR_MASK) {
   1365	case IB_MAC_IOCB_RSP_ERR_CODE_ERR:
   1366		stats->rx_code_err++;
   1367		break;
   1368	case IB_MAC_IOCB_RSP_ERR_OVERSIZE:
   1369		stats->rx_oversize_err++;
   1370		break;
   1371	case IB_MAC_IOCB_RSP_ERR_UNDERSIZE:
   1372		stats->rx_undersize_err++;
   1373		break;
   1374	case IB_MAC_IOCB_RSP_ERR_PREAMBLE:
   1375		stats->rx_preamble_err++;
   1376		break;
   1377	case IB_MAC_IOCB_RSP_ERR_FRAME_LEN:
   1378		stats->rx_frame_len_err++;
   1379		break;
   1380	case IB_MAC_IOCB_RSP_ERR_CRC:
   1381		stats->rx_crc_err++;
   1382		break;
   1383	default:
   1384		break;
   1385	}
   1386}
   1387
   1388/*
   1389 * qlge_update_mac_hdr_len - helper routine to update the mac header length
   1390 * based on vlan tags if present
   1391 */
   1392static void qlge_update_mac_hdr_len(struct qlge_adapter *qdev,
   1393				    struct qlge_ib_mac_iocb_rsp *ib_mac_rsp,
   1394				    void *page, size_t *len)
   1395{
   1396	u16 *tags;
   1397
   1398	if (qdev->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
   1399		return;
   1400	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V) {
   1401		tags = (u16 *)page;
   1402		/* Look for stacked vlan tags in ethertype field */
   1403		if (tags[6] == ETH_P_8021Q &&
   1404		    tags[8] == ETH_P_8021Q)
   1405			*len += 2 * VLAN_HLEN;
   1406		else
   1407			*len += VLAN_HLEN;
   1408	}
   1409}
   1410
   1411/* Process an inbound completion from an rx ring. */
   1412static void qlge_process_mac_rx_gro_page(struct qlge_adapter *qdev,
   1413					 struct rx_ring *rx_ring,
   1414					 struct qlge_ib_mac_iocb_rsp *ib_mac_rsp,
   1415					 u32 length, u16 vlan_id)
   1416{
   1417	struct sk_buff *skb;
   1418	struct qlge_bq_desc *lbq_desc = qlge_get_curr_lchunk(qdev, rx_ring);
   1419	struct napi_struct *napi = &rx_ring->napi;
   1420
   1421	/* Frame error, so drop the packet. */
   1422	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK) {
   1423		qlge_categorize_rx_err(qdev, ib_mac_rsp->flags2, rx_ring);
   1424		put_page(lbq_desc->p.pg_chunk.page);
   1425		return;
   1426	}
   1427	napi->dev = qdev->ndev;
   1428
   1429	skb = napi_get_frags(napi);
   1430	if (!skb) {
   1431		netif_err(qdev, drv, qdev->ndev,
   1432			  "Couldn't get an skb, exiting.\n");
   1433		rx_ring->rx_dropped++;
   1434		put_page(lbq_desc->p.pg_chunk.page);
   1435		return;
   1436	}
   1437	prefetch(lbq_desc->p.pg_chunk.va);
   1438	__skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
   1439			     lbq_desc->p.pg_chunk.page,
   1440			     lbq_desc->p.pg_chunk.offset,
   1441			     length);
   1442
   1443	skb->len += length;
   1444	skb->data_len += length;
   1445	skb->truesize += length;
   1446	skb_shinfo(skb)->nr_frags++;
   1447
   1448	rx_ring->rx_packets++;
   1449	rx_ring->rx_bytes += length;
   1450	skb->ip_summed = CHECKSUM_UNNECESSARY;
   1451	skb_record_rx_queue(skb, rx_ring->cq_id);
   1452	if (vlan_id != 0xffff)
   1453		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_id);
   1454	napi_gro_frags(napi);
   1455}
   1456
   1457/* Process an inbound completion from an rx ring. */
   1458static void qlge_process_mac_rx_page(struct qlge_adapter *qdev,
   1459				     struct rx_ring *rx_ring,
   1460				     struct qlge_ib_mac_iocb_rsp *ib_mac_rsp,
   1461				     u32 length, u16 vlan_id)
   1462{
   1463	struct net_device *ndev = qdev->ndev;
   1464	struct sk_buff *skb = NULL;
   1465	void *addr;
   1466	struct qlge_bq_desc *lbq_desc = qlge_get_curr_lchunk(qdev, rx_ring);
   1467	struct napi_struct *napi = &rx_ring->napi;
   1468	size_t hlen = ETH_HLEN;
   1469
   1470	skb = netdev_alloc_skb(ndev, length);
   1471	if (!skb) {
   1472		rx_ring->rx_dropped++;
   1473		put_page(lbq_desc->p.pg_chunk.page);
   1474		return;
   1475	}
   1476
   1477	addr = lbq_desc->p.pg_chunk.va;
   1478	prefetch(addr);
   1479
   1480	/* Frame error, so drop the packet. */
   1481	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK) {
   1482		qlge_categorize_rx_err(qdev, ib_mac_rsp->flags2, rx_ring);
   1483		goto err_out;
   1484	}
   1485
   1486	/* Update the MAC header length*/
   1487	qlge_update_mac_hdr_len(qdev, ib_mac_rsp, addr, &hlen);
   1488
   1489	/* The max framesize filter on this chip is set higher than
   1490	 * MTU since FCoE uses 2k frames.
   1491	 */
   1492	if (skb->len > ndev->mtu + hlen) {
   1493		netif_err(qdev, drv, qdev->ndev,
   1494			  "Segment too small, dropping.\n");
   1495		rx_ring->rx_dropped++;
   1496		goto err_out;
   1497	}
   1498	skb_put_data(skb, addr, hlen);
   1499	netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1500		     "%d bytes of headers and data in large. Chain page to new skb and pull tail.\n",
   1501		     length);
   1502	skb_fill_page_desc(skb, 0, lbq_desc->p.pg_chunk.page,
   1503			   lbq_desc->p.pg_chunk.offset + hlen, length - hlen);
   1504	skb->len += length - hlen;
   1505	skb->data_len += length - hlen;
   1506	skb->truesize += length - hlen;
   1507
   1508	rx_ring->rx_packets++;
   1509	rx_ring->rx_bytes += skb->len;
   1510	skb->protocol = eth_type_trans(skb, ndev);
   1511	skb_checksum_none_assert(skb);
   1512
   1513	if ((ndev->features & NETIF_F_RXCSUM) &&
   1514	    !(ib_mac_rsp->flags1 & IB_MAC_CSUM_ERR_MASK)) {
   1515		/* TCP frame. */
   1516		if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_T) {
   1517			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1518				     "TCP checksum done!\n");
   1519			skb->ip_summed = CHECKSUM_UNNECESSARY;
   1520		} else if ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_U) &&
   1521			   (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_V4)) {
   1522			/* Unfragmented ipv4 UDP frame. */
   1523			struct iphdr *iph =
   1524				(struct iphdr *)((u8 *)addr + hlen);
   1525			if (!(iph->frag_off &
   1526			      htons(IP_MF | IP_OFFSET))) {
   1527				skb->ip_summed = CHECKSUM_UNNECESSARY;
   1528				netif_printk(qdev, rx_status, KERN_DEBUG,
   1529					     qdev->ndev,
   1530					     "UDP checksum done!\n");
   1531			}
   1532		}
   1533	}
   1534
   1535	skb_record_rx_queue(skb, rx_ring->cq_id);
   1536	if (vlan_id != 0xffff)
   1537		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_id);
   1538	if (skb->ip_summed == CHECKSUM_UNNECESSARY)
   1539		napi_gro_receive(napi, skb);
   1540	else
   1541		netif_receive_skb(skb);
   1542	return;
   1543err_out:
   1544	dev_kfree_skb_any(skb);
   1545	put_page(lbq_desc->p.pg_chunk.page);
   1546}
   1547
   1548/* Process an inbound completion from an rx ring. */
   1549static void qlge_process_mac_rx_skb(struct qlge_adapter *qdev,
   1550				    struct rx_ring *rx_ring,
   1551				    struct qlge_ib_mac_iocb_rsp *ib_mac_rsp,
   1552				    u32 length, u16 vlan_id)
   1553{
   1554	struct qlge_bq_desc *sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
   1555	struct net_device *ndev = qdev->ndev;
   1556	struct sk_buff *skb, *new_skb;
   1557
   1558	skb = sbq_desc->p.skb;
   1559	/* Allocate new_skb and copy */
   1560	new_skb = netdev_alloc_skb(qdev->ndev, length + NET_IP_ALIGN);
   1561	if (!new_skb) {
   1562		rx_ring->rx_dropped++;
   1563		return;
   1564	}
   1565	skb_reserve(new_skb, NET_IP_ALIGN);
   1566
   1567	dma_sync_single_for_cpu(&qdev->pdev->dev, sbq_desc->dma_addr,
   1568				SMALL_BUF_MAP_SIZE, DMA_FROM_DEVICE);
   1569
   1570	skb_put_data(new_skb, skb->data, length);
   1571
   1572	skb = new_skb;
   1573
   1574	/* Frame error, so drop the packet. */
   1575	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK) {
   1576		qlge_categorize_rx_err(qdev, ib_mac_rsp->flags2, rx_ring);
   1577		dev_kfree_skb_any(skb);
   1578		return;
   1579	}
   1580
   1581	/* loopback self test for ethtool */
   1582	if (test_bit(QL_SELFTEST, &qdev->flags)) {
   1583		qlge_check_lb_frame(qdev, skb);
   1584		dev_kfree_skb_any(skb);
   1585		return;
   1586	}
   1587
   1588	/* The max framesize filter on this chip is set higher than
   1589	 * MTU since FCoE uses 2k frames.
   1590	 */
   1591	if (skb->len > ndev->mtu + ETH_HLEN) {
   1592		dev_kfree_skb_any(skb);
   1593		rx_ring->rx_dropped++;
   1594		return;
   1595	}
   1596
   1597	prefetch(skb->data);
   1598	if (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) {
   1599		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1600			     "%s Multicast.\n",
   1601			     (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
   1602			     IB_MAC_IOCB_RSP_M_HASH ? "Hash" :
   1603			     (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
   1604			     IB_MAC_IOCB_RSP_M_REG ? "Registered" :
   1605			     (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
   1606			     IB_MAC_IOCB_RSP_M_PROM ? "Promiscuous" : "");
   1607	}
   1608	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_P)
   1609		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1610			     "Promiscuous Packet.\n");
   1611
   1612	rx_ring->rx_packets++;
   1613	rx_ring->rx_bytes += skb->len;
   1614	skb->protocol = eth_type_trans(skb, ndev);
   1615	skb_checksum_none_assert(skb);
   1616
   1617	/* If rx checksum is on, and there are no
   1618	 * csum or frame errors.
   1619	 */
   1620	if ((ndev->features & NETIF_F_RXCSUM) &&
   1621	    !(ib_mac_rsp->flags1 & IB_MAC_CSUM_ERR_MASK)) {
   1622		/* TCP frame. */
   1623		if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_T) {
   1624			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1625				     "TCP checksum done!\n");
   1626			skb->ip_summed = CHECKSUM_UNNECESSARY;
   1627		} else if ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_U) &&
   1628			   (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_V4)) {
   1629			/* Unfragmented ipv4 UDP frame. */
   1630			struct iphdr *iph = (struct iphdr *)skb->data;
   1631
   1632			if (!(iph->frag_off &
   1633			      htons(IP_MF | IP_OFFSET))) {
   1634				skb->ip_summed = CHECKSUM_UNNECESSARY;
   1635				netif_printk(qdev, rx_status, KERN_DEBUG,
   1636					     qdev->ndev,
   1637					     "UDP checksum done!\n");
   1638			}
   1639		}
   1640	}
   1641
   1642	skb_record_rx_queue(skb, rx_ring->cq_id);
   1643	if (vlan_id != 0xffff)
   1644		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_id);
   1645	if (skb->ip_summed == CHECKSUM_UNNECESSARY)
   1646		napi_gro_receive(&rx_ring->napi, skb);
   1647	else
   1648		netif_receive_skb(skb);
   1649}
   1650
   1651static void qlge_realign_skb(struct sk_buff *skb, int len)
   1652{
   1653	void *temp_addr = skb->data;
   1654
   1655	/* Undo the skb_reserve(skb,32) we did before
   1656	 * giving to hardware, and realign data on
   1657	 * a 2-byte boundary.
   1658	 */
   1659	skb->data -= QLGE_SB_PAD - NET_IP_ALIGN;
   1660	skb->tail -= QLGE_SB_PAD - NET_IP_ALIGN;
   1661	memmove(skb->data, temp_addr, len);
   1662}
   1663
   1664/*
   1665 * This function builds an skb for the given inbound
   1666 * completion.  It will be rewritten for readability in the near
   1667 * future, but for not it works well.
   1668 */
   1669static struct sk_buff *qlge_build_rx_skb(struct qlge_adapter *qdev,
   1670					 struct rx_ring *rx_ring,
   1671					 struct qlge_ib_mac_iocb_rsp *ib_mac_rsp)
   1672{
   1673	u32 length = le32_to_cpu(ib_mac_rsp->data_len);
   1674	u32 hdr_len = le32_to_cpu(ib_mac_rsp->hdr_len);
   1675	struct qlge_bq_desc *lbq_desc, *sbq_desc;
   1676	struct sk_buff *skb = NULL;
   1677	size_t hlen = ETH_HLEN;
   1678
   1679	/*
   1680	 * Handle the header buffer if present.
   1681	 */
   1682	if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HV &&
   1683	    ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS) {
   1684		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1685			     "Header of %d bytes in small buffer.\n", hdr_len);
   1686		/*
   1687		 * Headers fit nicely into a small buffer.
   1688		 */
   1689		sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
   1690		dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
   1691				 SMALL_BUF_MAP_SIZE, DMA_FROM_DEVICE);
   1692		skb = sbq_desc->p.skb;
   1693		qlge_realign_skb(skb, hdr_len);
   1694		skb_put(skb, hdr_len);
   1695		sbq_desc->p.skb = NULL;
   1696	}
   1697
   1698	/*
   1699	 * Handle the data buffer(s).
   1700	 */
   1701	if (unlikely(!length)) {	/* Is there data too? */
   1702		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1703			     "No Data buffer in this packet.\n");
   1704		return skb;
   1705	}
   1706
   1707	if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DS) {
   1708		if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS) {
   1709			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1710				     "Headers in small, data of %d bytes in small, combine them.\n",
   1711				     length);
   1712			/*
   1713			 * Data is less than small buffer size so it's
   1714			 * stuffed in a small buffer.
   1715			 * For this case we append the data
   1716			 * from the "data" small buffer to the "header" small
   1717			 * buffer.
   1718			 */
   1719			sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
   1720			dma_sync_single_for_cpu(&qdev->pdev->dev,
   1721						sbq_desc->dma_addr,
   1722						SMALL_BUF_MAP_SIZE,
   1723						DMA_FROM_DEVICE);
   1724			skb_put_data(skb, sbq_desc->p.skb->data, length);
   1725		} else {
   1726			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1727				     "%d bytes in a single small buffer.\n",
   1728				     length);
   1729			sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
   1730			skb = sbq_desc->p.skb;
   1731			qlge_realign_skb(skb, length);
   1732			skb_put(skb, length);
   1733			dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
   1734					 SMALL_BUF_MAP_SIZE,
   1735					 DMA_FROM_DEVICE);
   1736			sbq_desc->p.skb = NULL;
   1737		}
   1738	} else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) {
   1739		if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS) {
   1740			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1741				     "Header in small, %d bytes in large. Chain large to small!\n",
   1742				     length);
   1743			/*
   1744			 * The data is in a single large buffer.  We
   1745			 * chain it to the header buffer's skb and let
   1746			 * it rip.
   1747			 */
   1748			lbq_desc = qlge_get_curr_lchunk(qdev, rx_ring);
   1749			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1750				     "Chaining page at offset = %d, for %d bytes  to skb.\n",
   1751				     lbq_desc->p.pg_chunk.offset, length);
   1752			skb_fill_page_desc(skb, 0, lbq_desc->p.pg_chunk.page,
   1753					   lbq_desc->p.pg_chunk.offset, length);
   1754			skb->len += length;
   1755			skb->data_len += length;
   1756			skb->truesize += length;
   1757		} else {
   1758			/*
   1759			 * The headers and data are in a single large buffer. We
   1760			 * copy it to a new skb and let it go. This can happen with
   1761			 * jumbo mtu on a non-TCP/UDP frame.
   1762			 */
   1763			lbq_desc = qlge_get_curr_lchunk(qdev, rx_ring);
   1764			skb = netdev_alloc_skb(qdev->ndev, length);
   1765			if (!skb) {
   1766				netif_printk(qdev, probe, KERN_DEBUG, qdev->ndev,
   1767					     "No skb available, drop the packet.\n");
   1768				return NULL;
   1769			}
   1770			dma_unmap_page(&qdev->pdev->dev, lbq_desc->dma_addr,
   1771				       qdev->lbq_buf_size,
   1772				       DMA_FROM_DEVICE);
   1773			skb_reserve(skb, NET_IP_ALIGN);
   1774			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1775				     "%d bytes of headers and data in large. Chain page to new skb and pull tail.\n",
   1776				     length);
   1777			skb_fill_page_desc(skb, 0, lbq_desc->p.pg_chunk.page,
   1778					   lbq_desc->p.pg_chunk.offset,
   1779					   length);
   1780			skb->len += length;
   1781			skb->data_len += length;
   1782			skb->truesize += length;
   1783			qlge_update_mac_hdr_len(qdev, ib_mac_rsp,
   1784						lbq_desc->p.pg_chunk.va,
   1785						&hlen);
   1786			__pskb_pull_tail(skb, hlen);
   1787		}
   1788	} else {
   1789		/*
   1790		 * The data is in a chain of large buffers
   1791		 * pointed to by a small buffer.  We loop
   1792		 * thru and chain them to the our small header
   1793		 * buffer's skb.
   1794		 * frags:  There are 18 max frags and our small
   1795		 *         buffer will hold 32 of them. The thing is,
   1796		 *         we'll use 3 max for our 9000 byte jumbo
   1797		 *         frames.  If the MTU goes up we could
   1798		 *          eventually be in trouble.
   1799		 */
   1800		int size, i = 0;
   1801
   1802		sbq_desc = qlge_get_curr_buf(&rx_ring->sbq);
   1803		dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
   1804				 SMALL_BUF_MAP_SIZE, DMA_FROM_DEVICE);
   1805		if (!(ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS)) {
   1806			/*
   1807			 * This is an non TCP/UDP IP frame, so
   1808			 * the headers aren't split into a small
   1809			 * buffer.  We have to use the small buffer
   1810			 * that contains our sg list as our skb to
   1811			 * send upstairs. Copy the sg list here to
   1812			 * a local buffer and use it to find the
   1813			 * pages to chain.
   1814			 */
   1815			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1816				     "%d bytes of headers & data in chain of large.\n",
   1817				     length);
   1818			skb = sbq_desc->p.skb;
   1819			sbq_desc->p.skb = NULL;
   1820			skb_reserve(skb, NET_IP_ALIGN);
   1821		}
   1822		do {
   1823			lbq_desc = qlge_get_curr_lchunk(qdev, rx_ring);
   1824			size = min(length, qdev->lbq_buf_size);
   1825
   1826			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1827				     "Adding page %d to skb for %d bytes.\n",
   1828				     i, size);
   1829			skb_fill_page_desc(skb, i,
   1830					   lbq_desc->p.pg_chunk.page,
   1831					   lbq_desc->p.pg_chunk.offset, size);
   1832			skb->len += size;
   1833			skb->data_len += size;
   1834			skb->truesize += size;
   1835			length -= size;
   1836			i++;
   1837		} while (length > 0);
   1838		qlge_update_mac_hdr_len(qdev, ib_mac_rsp, lbq_desc->p.pg_chunk.va,
   1839					&hlen);
   1840		__pskb_pull_tail(skb, hlen);
   1841	}
   1842	return skb;
   1843}
   1844
   1845/* Process an inbound completion from an rx ring. */
   1846static void qlge_process_mac_split_rx_intr(struct qlge_adapter *qdev,
   1847					   struct rx_ring *rx_ring,
   1848					   struct qlge_ib_mac_iocb_rsp *ib_mac_rsp,
   1849					   u16 vlan_id)
   1850{
   1851	struct net_device *ndev = qdev->ndev;
   1852	struct sk_buff *skb = NULL;
   1853
   1854	skb = qlge_build_rx_skb(qdev, rx_ring, ib_mac_rsp);
   1855	if (unlikely(!skb)) {
   1856		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1857			     "No skb available, drop packet.\n");
   1858		rx_ring->rx_dropped++;
   1859		return;
   1860	}
   1861
   1862	/* Frame error, so drop the packet. */
   1863	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK) {
   1864		qlge_categorize_rx_err(qdev, ib_mac_rsp->flags2, rx_ring);
   1865		dev_kfree_skb_any(skb);
   1866		return;
   1867	}
   1868
   1869	/* The max framesize filter on this chip is set higher than
   1870	 * MTU since FCoE uses 2k frames.
   1871	 */
   1872	if (skb->len > ndev->mtu + ETH_HLEN) {
   1873		dev_kfree_skb_any(skb);
   1874		rx_ring->rx_dropped++;
   1875		return;
   1876	}
   1877
   1878	/* loopback self test for ethtool */
   1879	if (test_bit(QL_SELFTEST, &qdev->flags)) {
   1880		qlge_check_lb_frame(qdev, skb);
   1881		dev_kfree_skb_any(skb);
   1882		return;
   1883	}
   1884
   1885	prefetch(skb->data);
   1886	if (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) {
   1887		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev, "%s Multicast.\n",
   1888			     (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
   1889			     IB_MAC_IOCB_RSP_M_HASH ? "Hash" :
   1890			     (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
   1891			     IB_MAC_IOCB_RSP_M_REG ? "Registered" :
   1892			     (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
   1893			     IB_MAC_IOCB_RSP_M_PROM ? "Promiscuous" : "");
   1894		rx_ring->rx_multicast++;
   1895	}
   1896	if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_P) {
   1897		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1898			     "Promiscuous Packet.\n");
   1899	}
   1900
   1901	skb->protocol = eth_type_trans(skb, ndev);
   1902	skb_checksum_none_assert(skb);
   1903
   1904	/* If rx checksum is on, and there are no
   1905	 * csum or frame errors.
   1906	 */
   1907	if ((ndev->features & NETIF_F_RXCSUM) &&
   1908	    !(ib_mac_rsp->flags1 & IB_MAC_CSUM_ERR_MASK)) {
   1909		/* TCP frame. */
   1910		if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_T) {
   1911			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1912				     "TCP checksum done!\n");
   1913			skb->ip_summed = CHECKSUM_UNNECESSARY;
   1914		} else if ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_U) &&
   1915			   (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_V4)) {
   1916			/* Unfragmented ipv4 UDP frame. */
   1917			struct iphdr *iph = (struct iphdr *)skb->data;
   1918
   1919			if (!(iph->frag_off &
   1920			      htons(IP_MF | IP_OFFSET))) {
   1921				skb->ip_summed = CHECKSUM_UNNECESSARY;
   1922				netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   1923					     "TCP checksum done!\n");
   1924			}
   1925		}
   1926	}
   1927
   1928	rx_ring->rx_packets++;
   1929	rx_ring->rx_bytes += skb->len;
   1930	skb_record_rx_queue(skb, rx_ring->cq_id);
   1931	if (vlan_id != 0xffff)
   1932		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_id);
   1933	if (skb->ip_summed == CHECKSUM_UNNECESSARY)
   1934		napi_gro_receive(&rx_ring->napi, skb);
   1935	else
   1936		netif_receive_skb(skb);
   1937}
   1938
   1939/* Process an inbound completion from an rx ring. */
   1940static unsigned long qlge_process_mac_rx_intr(struct qlge_adapter *qdev,
   1941					      struct rx_ring *rx_ring,
   1942					      struct qlge_ib_mac_iocb_rsp *ib_mac_rsp)
   1943{
   1944	u32 length = le32_to_cpu(ib_mac_rsp->data_len);
   1945	u16 vlan_id = ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V) &&
   1946		       (qdev->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)) ?
   1947		((le16_to_cpu(ib_mac_rsp->vlan_id) &
   1948		  IB_MAC_IOCB_RSP_VLAN_MASK)) : 0xffff;
   1949
   1950	if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HV) {
   1951		/* The data and headers are split into
   1952		 * separate buffers.
   1953		 */
   1954		qlge_process_mac_split_rx_intr(qdev, rx_ring, ib_mac_rsp,
   1955					       vlan_id);
   1956	} else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DS) {
   1957		/* The data fit in a single small buffer.
   1958		 * Allocate a new skb, copy the data and
   1959		 * return the buffer to the free pool.
   1960		 */
   1961		qlge_process_mac_rx_skb(qdev, rx_ring, ib_mac_rsp, length,
   1962					vlan_id);
   1963	} else if ((ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) &&
   1964		   !(ib_mac_rsp->flags1 & IB_MAC_CSUM_ERR_MASK) &&
   1965		   (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_T)) {
   1966		/* TCP packet in a page chunk that's been checksummed.
   1967		 * Tack it on to our GRO skb and let it go.
   1968		 */
   1969		qlge_process_mac_rx_gro_page(qdev, rx_ring, ib_mac_rsp, length,
   1970					     vlan_id);
   1971	} else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) {
   1972		/* Non-TCP packet in a page chunk. Allocate an
   1973		 * skb, tack it on frags, and send it up.
   1974		 */
   1975		qlge_process_mac_rx_page(qdev, rx_ring, ib_mac_rsp, length,
   1976					 vlan_id);
   1977	} else {
   1978		/* Non-TCP/UDP large frames that span multiple buffers
   1979		 * can be processed corrrectly by the split frame logic.
   1980		 */
   1981		qlge_process_mac_split_rx_intr(qdev, rx_ring, ib_mac_rsp,
   1982					       vlan_id);
   1983	}
   1984
   1985	return (unsigned long)length;
   1986}
   1987
   1988/* Process an outbound completion from an rx ring. */
   1989static void qlge_process_mac_tx_intr(struct qlge_adapter *qdev,
   1990				     struct qlge_ob_mac_iocb_rsp *mac_rsp)
   1991{
   1992	struct tx_ring *tx_ring;
   1993	struct tx_ring_desc *tx_ring_desc;
   1994
   1995	tx_ring = &qdev->tx_ring[mac_rsp->txq_idx];
   1996	tx_ring_desc = &tx_ring->q[mac_rsp->tid];
   1997	qlge_unmap_send(qdev, tx_ring_desc, tx_ring_desc->map_cnt);
   1998	tx_ring->tx_bytes += (tx_ring_desc->skb)->len;
   1999	tx_ring->tx_packets++;
   2000	dev_kfree_skb(tx_ring_desc->skb);
   2001	tx_ring_desc->skb = NULL;
   2002
   2003	if (unlikely(mac_rsp->flags1 & (OB_MAC_IOCB_RSP_E |
   2004					OB_MAC_IOCB_RSP_S |
   2005					OB_MAC_IOCB_RSP_L |
   2006					OB_MAC_IOCB_RSP_P | OB_MAC_IOCB_RSP_B))) {
   2007		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_E) {
   2008			netif_warn(qdev, tx_done, qdev->ndev,
   2009				   "Total descriptor length did not match transfer length.\n");
   2010		}
   2011		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_S) {
   2012			netif_warn(qdev, tx_done, qdev->ndev,
   2013				   "Frame too short to be valid, not sent.\n");
   2014		}
   2015		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_L) {
   2016			netif_warn(qdev, tx_done, qdev->ndev,
   2017				   "Frame too long, but sent anyway.\n");
   2018		}
   2019		if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_B) {
   2020			netif_warn(qdev, tx_done, qdev->ndev,
   2021				   "PCI backplane error. Frame not sent.\n");
   2022		}
   2023	}
   2024	atomic_inc(&tx_ring->tx_count);
   2025}
   2026
   2027/* Fire up a handler to reset the MPI processor. */
   2028void qlge_queue_fw_error(struct qlge_adapter *qdev)
   2029{
   2030	qlge_link_off(qdev);
   2031	queue_delayed_work(qdev->workqueue, &qdev->mpi_reset_work, 0);
   2032}
   2033
   2034void qlge_queue_asic_error(struct qlge_adapter *qdev)
   2035{
   2036	qlge_link_off(qdev);
   2037	qlge_disable_interrupts(qdev);
   2038	/* Clear adapter up bit to signal the recovery
   2039	 * process that it shouldn't kill the reset worker
   2040	 * thread
   2041	 */
   2042	clear_bit(QL_ADAPTER_UP, &qdev->flags);
   2043	/* Set asic recovery bit to indicate reset process that we are
   2044	 * in fatal error recovery process rather than normal close
   2045	 */
   2046	set_bit(QL_ASIC_RECOVERY, &qdev->flags);
   2047	queue_delayed_work(qdev->workqueue, &qdev->asic_reset_work, 0);
   2048}
   2049
   2050static void qlge_process_chip_ae_intr(struct qlge_adapter *qdev,
   2051				      struct qlge_ib_ae_iocb_rsp *ib_ae_rsp)
   2052{
   2053	switch (ib_ae_rsp->event) {
   2054	case MGMT_ERR_EVENT:
   2055		netif_err(qdev, rx_err, qdev->ndev,
   2056			  "Management Processor Fatal Error.\n");
   2057		qlge_queue_fw_error(qdev);
   2058		return;
   2059
   2060	case CAM_LOOKUP_ERR_EVENT:
   2061		netdev_err(qdev->ndev, "Multiple CAM hits lookup occurred.\n");
   2062		netdev_err(qdev->ndev, "This event shouldn't occur.\n");
   2063		qlge_queue_asic_error(qdev);
   2064		return;
   2065
   2066	case SOFT_ECC_ERROR_EVENT:
   2067		netdev_err(qdev->ndev, "Soft ECC error detected.\n");
   2068		qlge_queue_asic_error(qdev);
   2069		break;
   2070
   2071	case PCI_ERR_ANON_BUF_RD:
   2072		netdev_err(qdev->ndev,
   2073			   "PCI error occurred when reading anonymous buffers from rx_ring %d.\n",
   2074			   ib_ae_rsp->q_id);
   2075		qlge_queue_asic_error(qdev);
   2076		break;
   2077
   2078	default:
   2079		netif_err(qdev, drv, qdev->ndev, "Unexpected event %d.\n",
   2080			  ib_ae_rsp->event);
   2081		qlge_queue_asic_error(qdev);
   2082		break;
   2083	}
   2084}
   2085
   2086static int qlge_clean_outbound_rx_ring(struct rx_ring *rx_ring)
   2087{
   2088	struct qlge_adapter *qdev = rx_ring->qdev;
   2089	u32 prod = qlge_read_sh_reg(rx_ring->prod_idx_sh_reg);
   2090	struct qlge_ob_mac_iocb_rsp *net_rsp = NULL;
   2091	int count = 0;
   2092
   2093	struct tx_ring *tx_ring;
   2094	/* While there are entries in the completion queue. */
   2095	while (prod != rx_ring->cnsmr_idx) {
   2096		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   2097			     "cq_id = %d, prod = %d, cnsmr = %d\n",
   2098			     rx_ring->cq_id, prod, rx_ring->cnsmr_idx);
   2099
   2100		net_rsp = (struct qlge_ob_mac_iocb_rsp *)rx_ring->curr_entry;
   2101		rmb();
   2102		switch (net_rsp->opcode) {
   2103		case OPCODE_OB_MAC_TSO_IOCB:
   2104		case OPCODE_OB_MAC_IOCB:
   2105			qlge_process_mac_tx_intr(qdev, net_rsp);
   2106			break;
   2107		default:
   2108			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   2109				     "Hit default case, not handled! dropping the packet, opcode = %x.\n",
   2110				     net_rsp->opcode);
   2111		}
   2112		count++;
   2113		qlge_update_cq(rx_ring);
   2114		prod = qlge_read_sh_reg(rx_ring->prod_idx_sh_reg);
   2115	}
   2116	if (!net_rsp)
   2117		return 0;
   2118	qlge_write_cq_idx(rx_ring);
   2119	tx_ring = &qdev->tx_ring[net_rsp->txq_idx];
   2120	if (__netif_subqueue_stopped(qdev->ndev, tx_ring->wq_id)) {
   2121		if ((atomic_read(&tx_ring->tx_count) > (tx_ring->wq_len / 4)))
   2122			/*
   2123			 * The queue got stopped because the tx_ring was full.
   2124			 * Wake it up, because it's now at least 25% empty.
   2125			 */
   2126			netif_wake_subqueue(qdev->ndev, tx_ring->wq_id);
   2127	}
   2128
   2129	return count;
   2130}
   2131
   2132static int qlge_clean_inbound_rx_ring(struct rx_ring *rx_ring, int budget)
   2133{
   2134	struct qlge_adapter *qdev = rx_ring->qdev;
   2135	u32 prod = qlge_read_sh_reg(rx_ring->prod_idx_sh_reg);
   2136	struct qlge_net_rsp_iocb *net_rsp;
   2137	int count = 0;
   2138
   2139	/* While there are entries in the completion queue. */
   2140	while (prod != rx_ring->cnsmr_idx) {
   2141		netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   2142			     "cq_id = %d, prod = %d, cnsmr = %d\n",
   2143			     rx_ring->cq_id, prod, rx_ring->cnsmr_idx);
   2144
   2145		net_rsp = rx_ring->curr_entry;
   2146		rmb();
   2147		switch (net_rsp->opcode) {
   2148		case OPCODE_IB_MAC_IOCB:
   2149			qlge_process_mac_rx_intr(qdev, rx_ring,
   2150						 (struct qlge_ib_mac_iocb_rsp *)
   2151						 net_rsp);
   2152			break;
   2153
   2154		case OPCODE_IB_AE_IOCB:
   2155			qlge_process_chip_ae_intr(qdev, (struct qlge_ib_ae_iocb_rsp *)
   2156						  net_rsp);
   2157			break;
   2158		default:
   2159			netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   2160				     "Hit default case, not handled! dropping the packet, opcode = %x.\n",
   2161				     net_rsp->opcode);
   2162			break;
   2163		}
   2164		count++;
   2165		qlge_update_cq(rx_ring);
   2166		prod = qlge_read_sh_reg(rx_ring->prod_idx_sh_reg);
   2167		if (count == budget)
   2168			break;
   2169	}
   2170	qlge_update_buffer_queues(rx_ring, GFP_ATOMIC, 0);
   2171	qlge_write_cq_idx(rx_ring);
   2172	return count;
   2173}
   2174
   2175static int qlge_napi_poll_msix(struct napi_struct *napi, int budget)
   2176{
   2177	struct rx_ring *rx_ring = container_of(napi, struct rx_ring, napi);
   2178	struct qlge_adapter *qdev = rx_ring->qdev;
   2179	struct rx_ring *trx_ring;
   2180	int i, work_done = 0;
   2181	struct intr_context *ctx = &qdev->intr_context[rx_ring->cq_id];
   2182
   2183	netif_printk(qdev, rx_status, KERN_DEBUG, qdev->ndev,
   2184		     "Enter, NAPI POLL cq_id = %d.\n", rx_ring->cq_id);
   2185
   2186	/* Service the TX rings first.  They start
   2187	 * right after the RSS rings.
   2188	 */
   2189	for (i = qdev->rss_ring_count; i < qdev->rx_ring_count; i++) {
   2190		trx_ring = &qdev->rx_ring[i];
   2191		/* If this TX completion ring belongs to this vector and
   2192		 * it's not empty then service it.
   2193		 */
   2194		if ((ctx->irq_mask & (1 << trx_ring->cq_id)) &&
   2195		    (qlge_read_sh_reg(trx_ring->prod_idx_sh_reg) !=
   2196		     trx_ring->cnsmr_idx)) {
   2197			netif_printk(qdev, intr, KERN_DEBUG, qdev->ndev,
   2198				     "%s: Servicing TX completion ring %d.\n",
   2199				     __func__, trx_ring->cq_id);
   2200			qlge_clean_outbound_rx_ring(trx_ring);
   2201		}
   2202	}
   2203
   2204	/*
   2205	 * Now service the RSS ring if it's active.
   2206	 */
   2207	if (qlge_read_sh_reg(rx_ring->prod_idx_sh_reg) !=
   2208	    rx_ring->cnsmr_idx) {
   2209		netif_printk(qdev, intr, KERN_DEBUG, qdev->ndev,
   2210			     "%s: Servicing RX completion ring %d.\n",
   2211			     __func__, rx_ring->cq_id);
   2212		work_done = qlge_clean_inbound_rx_ring(rx_ring, budget);
   2213	}
   2214
   2215	if (work_done < budget) {
   2216		napi_complete_done(napi, work_done);
   2217		qlge_enable_completion_interrupt(qdev, rx_ring->irq);
   2218	}
   2219	return work_done;
   2220}
   2221
   2222static void qlge_vlan_mode(struct net_device *ndev, netdev_features_t features)
   2223{
   2224	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   2225
   2226	if (features & NETIF_F_HW_VLAN_CTAG_RX) {
   2227		qlge_write32(qdev, NIC_RCV_CFG, NIC_RCV_CFG_VLAN_MASK |
   2228			     NIC_RCV_CFG_VLAN_MATCH_AND_NON);
   2229	} else {
   2230		qlge_write32(qdev, NIC_RCV_CFG, NIC_RCV_CFG_VLAN_MASK);
   2231	}
   2232}
   2233
   2234/*
   2235 * qlge_update_hw_vlan_features - helper routine to reinitialize the adapter
   2236 * based on the features to enable/disable hardware vlan accel
   2237 */
   2238static int qlge_update_hw_vlan_features(struct net_device *ndev,
   2239					netdev_features_t features)
   2240{
   2241	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   2242	bool need_restart = netif_running(ndev);
   2243	int status = 0;
   2244
   2245	if (need_restart) {
   2246		status = qlge_adapter_down(qdev);
   2247		if (status) {
   2248			netif_err(qdev, link, qdev->ndev,
   2249				  "Failed to bring down the adapter\n");
   2250			return status;
   2251		}
   2252	}
   2253
   2254	/* update the features with resent change */
   2255	ndev->features = features;
   2256
   2257	if (need_restart) {
   2258		status = qlge_adapter_up(qdev);
   2259		if (status) {
   2260			netif_err(qdev, link, qdev->ndev,
   2261				  "Failed to bring up the adapter\n");
   2262			return status;
   2263		}
   2264	}
   2265
   2266	return status;
   2267}
   2268
   2269static int qlge_set_features(struct net_device *ndev,
   2270			     netdev_features_t features)
   2271{
   2272	netdev_features_t changed = ndev->features ^ features;
   2273	int err;
   2274
   2275	if (changed & NETIF_F_HW_VLAN_CTAG_RX) {
   2276		/* Update the behavior of vlan accel in the adapter */
   2277		err = qlge_update_hw_vlan_features(ndev, features);
   2278		if (err)
   2279			return err;
   2280
   2281		qlge_vlan_mode(ndev, features);
   2282	}
   2283
   2284	return 0;
   2285}
   2286
   2287static int __qlge_vlan_rx_add_vid(struct qlge_adapter *qdev, u16 vid)
   2288{
   2289	u32 enable_bit = MAC_ADDR_E;
   2290	int err;
   2291
   2292	err = qlge_set_mac_addr_reg(qdev, (u8 *)&enable_bit,
   2293				    MAC_ADDR_TYPE_VLAN, vid);
   2294	if (err)
   2295		netif_err(qdev, ifup, qdev->ndev,
   2296			  "Failed to init vlan address.\n");
   2297	return err;
   2298}
   2299
   2300static int qlge_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
   2301{
   2302	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   2303	int status;
   2304	int err;
   2305
   2306	status = qlge_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
   2307	if (status)
   2308		return status;
   2309
   2310	err = __qlge_vlan_rx_add_vid(qdev, vid);
   2311	set_bit(vid, qdev->active_vlans);
   2312
   2313	qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
   2314
   2315	return err;
   2316}
   2317
   2318static int __qlge_vlan_rx_kill_vid(struct qlge_adapter *qdev, u16 vid)
   2319{
   2320	u32 enable_bit = 0;
   2321	int err;
   2322
   2323	err = qlge_set_mac_addr_reg(qdev, (u8 *)&enable_bit,
   2324				    MAC_ADDR_TYPE_VLAN, vid);
   2325	if (err)
   2326		netif_err(qdev, ifup, qdev->ndev,
   2327			  "Failed to clear vlan address.\n");
   2328	return err;
   2329}
   2330
   2331static int qlge_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
   2332{
   2333	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   2334	int status;
   2335	int err;
   2336
   2337	status = qlge_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
   2338	if (status)
   2339		return status;
   2340
   2341	err = __qlge_vlan_rx_kill_vid(qdev, vid);
   2342	clear_bit(vid, qdev->active_vlans);
   2343
   2344	qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
   2345
   2346	return err;
   2347}
   2348
   2349static void qlge_restore_vlan(struct qlge_adapter *qdev)
   2350{
   2351	int status;
   2352	u16 vid;
   2353
   2354	status = qlge_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
   2355	if (status)
   2356		return;
   2357
   2358	for_each_set_bit(vid, qdev->active_vlans, VLAN_N_VID)
   2359		__qlge_vlan_rx_add_vid(qdev, vid);
   2360
   2361	qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
   2362}
   2363
   2364/* MSI-X Multiple Vector Interrupt Handler for inbound completions. */
   2365static irqreturn_t qlge_msix_rx_isr(int irq, void *dev_id)
   2366{
   2367	struct rx_ring *rx_ring = dev_id;
   2368
   2369	napi_schedule(&rx_ring->napi);
   2370	return IRQ_HANDLED;
   2371}
   2372
   2373/* This handles a fatal error, MPI activity, and the default
   2374 * rx_ring in an MSI-X multiple vector environment.
   2375 * In MSI/Legacy environment it also process the rest of
   2376 * the rx_rings.
   2377 */
   2378static irqreturn_t qlge_isr(int irq, void *dev_id)
   2379{
   2380	struct rx_ring *rx_ring = dev_id;
   2381	struct qlge_adapter *qdev = rx_ring->qdev;
   2382	struct intr_context *intr_context = &qdev->intr_context[0];
   2383	u32 var;
   2384	int work_done = 0;
   2385
   2386	/* Experience shows that when using INTx interrupts, interrupts must
   2387	 * be masked manually.
   2388	 * When using MSI mode, INTR_EN_EN must be explicitly disabled
   2389	 * (even though it is auto-masked), otherwise a later command to
   2390	 * enable it is not effective.
   2391	 */
   2392	if (!test_bit(QL_MSIX_ENABLED, &qdev->flags))
   2393		qlge_disable_completion_interrupt(qdev, 0);
   2394
   2395	var = qlge_read32(qdev, STS);
   2396
   2397	/*
   2398	 * Check for fatal error.
   2399	 */
   2400	if (var & STS_FE) {
   2401		qlge_disable_completion_interrupt(qdev, 0);
   2402		qlge_queue_asic_error(qdev);
   2403		netdev_err(qdev->ndev, "Got fatal error, STS = %x.\n", var);
   2404		var = qlge_read32(qdev, ERR_STS);
   2405		netdev_err(qdev->ndev, "Resetting chip. Error Status Register = 0x%x\n", var);
   2406		return IRQ_HANDLED;
   2407	}
   2408
   2409	/*
   2410	 * Check MPI processor activity.
   2411	 */
   2412	if ((var & STS_PI) &&
   2413	    (qlge_read32(qdev, INTR_MASK) & INTR_MASK_PI)) {
   2414		/*
   2415		 * We've got an async event or mailbox completion.
   2416		 * Handle it and clear the source of the interrupt.
   2417		 */
   2418		netif_err(qdev, intr, qdev->ndev,
   2419			  "Got MPI processor interrupt.\n");
   2420		qlge_write32(qdev, INTR_MASK, (INTR_MASK_PI << 16));
   2421		queue_delayed_work_on(smp_processor_id(),
   2422				      qdev->workqueue, &qdev->mpi_work, 0);
   2423		work_done++;
   2424	}
   2425
   2426	/*
   2427	 * Get the bit-mask that shows the active queues for this
   2428	 * pass.  Compare it to the queues that this irq services
   2429	 * and call napi if there's a match.
   2430	 */
   2431	var = qlge_read32(qdev, ISR1);
   2432	if (var & intr_context->irq_mask) {
   2433		netif_info(qdev, intr, qdev->ndev,
   2434			   "Waking handler for rx_ring[0].\n");
   2435		napi_schedule(&rx_ring->napi);
   2436		work_done++;
   2437	} else {
   2438		/* Experience shows that the device sometimes signals an
   2439		 * interrupt but no work is scheduled from this function.
   2440		 * Nevertheless, the interrupt is auto-masked. Therefore, we
   2441		 * systematically re-enable the interrupt if we didn't
   2442		 * schedule napi.
   2443		 */
   2444		qlge_enable_completion_interrupt(qdev, 0);
   2445	}
   2446
   2447	return work_done ? IRQ_HANDLED : IRQ_NONE;
   2448}
   2449
   2450static int qlge_tso(struct sk_buff *skb, struct qlge_ob_mac_tso_iocb_req *mac_iocb_ptr)
   2451{
   2452	if (skb_is_gso(skb)) {
   2453		int err;
   2454		__be16 l3_proto = vlan_get_protocol(skb);
   2455
   2456		err = skb_cow_head(skb, 0);
   2457		if (err < 0)
   2458			return err;
   2459
   2460		mac_iocb_ptr->opcode = OPCODE_OB_MAC_TSO_IOCB;
   2461		mac_iocb_ptr->flags3 |= OB_MAC_TSO_IOCB_IC;
   2462		mac_iocb_ptr->frame_len = cpu_to_le32((u32)skb->len);
   2463		mac_iocb_ptr->total_hdrs_len =
   2464			cpu_to_le16(skb_transport_offset(skb) + tcp_hdrlen(skb));
   2465		mac_iocb_ptr->net_trans_offset =
   2466			cpu_to_le16(skb_network_offset(skb) |
   2467				    skb_transport_offset(skb)
   2468				    << OB_MAC_TRANSPORT_HDR_SHIFT);
   2469		mac_iocb_ptr->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
   2470		mac_iocb_ptr->flags2 |= OB_MAC_TSO_IOCB_LSO;
   2471		if (likely(l3_proto == htons(ETH_P_IP))) {
   2472			struct iphdr *iph = ip_hdr(skb);
   2473
   2474			iph->check = 0;
   2475			mac_iocb_ptr->flags1 |= OB_MAC_TSO_IOCB_IP4;
   2476			tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
   2477								 iph->daddr, 0,
   2478								 IPPROTO_TCP,
   2479								 0);
   2480		} else if (l3_proto == htons(ETH_P_IPV6)) {
   2481			mac_iocb_ptr->flags1 |= OB_MAC_TSO_IOCB_IP6;
   2482			tcp_hdr(skb)->check =
   2483				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
   2484						 &ipv6_hdr(skb)->daddr,
   2485						 0, IPPROTO_TCP, 0);
   2486		}
   2487		return 1;
   2488	}
   2489	return 0;
   2490}
   2491
   2492static void qlge_hw_csum_setup(struct sk_buff *skb,
   2493			       struct qlge_ob_mac_tso_iocb_req *mac_iocb_ptr)
   2494{
   2495	int len;
   2496	struct iphdr *iph = ip_hdr(skb);
   2497	__sum16 *check;
   2498
   2499	mac_iocb_ptr->opcode = OPCODE_OB_MAC_TSO_IOCB;
   2500	mac_iocb_ptr->frame_len = cpu_to_le32((u32)skb->len);
   2501	mac_iocb_ptr->net_trans_offset =
   2502		cpu_to_le16(skb_network_offset(skb) |
   2503			    skb_transport_offset(skb) << OB_MAC_TRANSPORT_HDR_SHIFT);
   2504
   2505	mac_iocb_ptr->flags1 |= OB_MAC_TSO_IOCB_IP4;
   2506	len = (ntohs(iph->tot_len) - (iph->ihl << 2));
   2507	if (likely(iph->protocol == IPPROTO_TCP)) {
   2508		check = &(tcp_hdr(skb)->check);
   2509		mac_iocb_ptr->flags2 |= OB_MAC_TSO_IOCB_TC;
   2510		mac_iocb_ptr->total_hdrs_len =
   2511			cpu_to_le16(skb_transport_offset(skb) +
   2512				    (tcp_hdr(skb)->doff << 2));
   2513	} else {
   2514		check = &(udp_hdr(skb)->check);
   2515		mac_iocb_ptr->flags2 |= OB_MAC_TSO_IOCB_UC;
   2516		mac_iocb_ptr->total_hdrs_len =
   2517			cpu_to_le16(skb_transport_offset(skb) +
   2518				    sizeof(struct udphdr));
   2519	}
   2520	*check = ~csum_tcpudp_magic(iph->saddr,
   2521				    iph->daddr, len, iph->protocol, 0);
   2522}
   2523
   2524static netdev_tx_t qlge_send(struct sk_buff *skb, struct net_device *ndev)
   2525{
   2526	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   2527	struct qlge_ob_mac_iocb_req *mac_iocb_ptr;
   2528	struct tx_ring_desc *tx_ring_desc;
   2529	int tso;
   2530	struct tx_ring *tx_ring;
   2531	u32 tx_ring_idx = (u32)skb->queue_mapping;
   2532
   2533	tx_ring = &qdev->tx_ring[tx_ring_idx];
   2534
   2535	if (skb_padto(skb, ETH_ZLEN))
   2536		return NETDEV_TX_OK;
   2537
   2538	if (unlikely(atomic_read(&tx_ring->tx_count) < 2)) {
   2539		netif_info(qdev, tx_queued, qdev->ndev,
   2540			   "%s: BUG! shutting down tx queue %d due to lack of resources.\n",
   2541			   __func__, tx_ring_idx);
   2542		netif_stop_subqueue(ndev, tx_ring->wq_id);
   2543		tx_ring->tx_errors++;
   2544		return NETDEV_TX_BUSY;
   2545	}
   2546	tx_ring_desc = &tx_ring->q[tx_ring->prod_idx];
   2547	mac_iocb_ptr = tx_ring_desc->queue_entry;
   2548	memset((void *)mac_iocb_ptr, 0, sizeof(*mac_iocb_ptr));
   2549
   2550	mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB;
   2551	mac_iocb_ptr->tid = tx_ring_desc->index;
   2552	/* We use the upper 32-bits to store the tx queue for this IO.
   2553	 * When we get the completion we can use it to establish the context.
   2554	 */
   2555	mac_iocb_ptr->txq_idx = tx_ring_idx;
   2556	tx_ring_desc->skb = skb;
   2557
   2558	mac_iocb_ptr->frame_len = cpu_to_le16((u16)skb->len);
   2559
   2560	if (skb_vlan_tag_present(skb)) {
   2561		netif_printk(qdev, tx_queued, KERN_DEBUG, qdev->ndev,
   2562			     "Adding a vlan tag %d.\n", skb_vlan_tag_get(skb));
   2563		mac_iocb_ptr->flags3 |= OB_MAC_IOCB_V;
   2564		mac_iocb_ptr->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
   2565	}
   2566	tso = qlge_tso(skb, (struct qlge_ob_mac_tso_iocb_req *)mac_iocb_ptr);
   2567	if (tso < 0) {
   2568		dev_kfree_skb_any(skb);
   2569		return NETDEV_TX_OK;
   2570	} else if (unlikely(!tso) && (skb->ip_summed == CHECKSUM_PARTIAL)) {
   2571		qlge_hw_csum_setup(skb,
   2572				   (struct qlge_ob_mac_tso_iocb_req *)mac_iocb_ptr);
   2573	}
   2574	if (qlge_map_send(qdev, mac_iocb_ptr, skb, tx_ring_desc) !=
   2575	    NETDEV_TX_OK) {
   2576		netif_err(qdev, tx_queued, qdev->ndev,
   2577			  "Could not map the segments.\n");
   2578		tx_ring->tx_errors++;
   2579		return NETDEV_TX_BUSY;
   2580	}
   2581
   2582	tx_ring->prod_idx++;
   2583	if (tx_ring->prod_idx == tx_ring->wq_len)
   2584		tx_ring->prod_idx = 0;
   2585	wmb();
   2586
   2587	qlge_write_db_reg_relaxed(tx_ring->prod_idx, tx_ring->prod_idx_db_reg);
   2588	netif_printk(qdev, tx_queued, KERN_DEBUG, qdev->ndev,
   2589		     "tx queued, slot %d, len %d\n",
   2590		     tx_ring->prod_idx, skb->len);
   2591
   2592	atomic_dec(&tx_ring->tx_count);
   2593
   2594	if (unlikely(atomic_read(&tx_ring->tx_count) < 2)) {
   2595		netif_stop_subqueue(ndev, tx_ring->wq_id);
   2596		if ((atomic_read(&tx_ring->tx_count) > (tx_ring->wq_len / 4)))
   2597			/*
   2598			 * The queue got stopped because the tx_ring was full.
   2599			 * Wake it up, because it's now at least 25% empty.
   2600			 */
   2601			netif_wake_subqueue(qdev->ndev, tx_ring->wq_id);
   2602	}
   2603	return NETDEV_TX_OK;
   2604}
   2605
   2606static void qlge_free_shadow_space(struct qlge_adapter *qdev)
   2607{
   2608	if (qdev->rx_ring_shadow_reg_area) {
   2609		dma_free_coherent(&qdev->pdev->dev,
   2610				  PAGE_SIZE,
   2611				  qdev->rx_ring_shadow_reg_area,
   2612				  qdev->rx_ring_shadow_reg_dma);
   2613		qdev->rx_ring_shadow_reg_area = NULL;
   2614	}
   2615	if (qdev->tx_ring_shadow_reg_area) {
   2616		dma_free_coherent(&qdev->pdev->dev,
   2617				  PAGE_SIZE,
   2618				  qdev->tx_ring_shadow_reg_area,
   2619				  qdev->tx_ring_shadow_reg_dma);
   2620		qdev->tx_ring_shadow_reg_area = NULL;
   2621	}
   2622}
   2623
   2624static int qlge_alloc_shadow_space(struct qlge_adapter *qdev)
   2625{
   2626	qdev->rx_ring_shadow_reg_area =
   2627		dma_alloc_coherent(&qdev->pdev->dev, PAGE_SIZE,
   2628				   &qdev->rx_ring_shadow_reg_dma, GFP_ATOMIC);
   2629	if (!qdev->rx_ring_shadow_reg_area) {
   2630		netif_err(qdev, ifup, qdev->ndev,
   2631			  "Allocation of RX shadow space failed.\n");
   2632		return -ENOMEM;
   2633	}
   2634
   2635	qdev->tx_ring_shadow_reg_area =
   2636		dma_alloc_coherent(&qdev->pdev->dev, PAGE_SIZE,
   2637				   &qdev->tx_ring_shadow_reg_dma, GFP_ATOMIC);
   2638	if (!qdev->tx_ring_shadow_reg_area) {
   2639		netif_err(qdev, ifup, qdev->ndev,
   2640			  "Allocation of TX shadow space failed.\n");
   2641		goto err_wqp_sh_area;
   2642	}
   2643	return 0;
   2644
   2645err_wqp_sh_area:
   2646	dma_free_coherent(&qdev->pdev->dev,
   2647			  PAGE_SIZE,
   2648			  qdev->rx_ring_shadow_reg_area,
   2649			  qdev->rx_ring_shadow_reg_dma);
   2650	return -ENOMEM;
   2651}
   2652
   2653static void qlge_init_tx_ring(struct qlge_adapter *qdev, struct tx_ring *tx_ring)
   2654{
   2655	struct tx_ring_desc *tx_ring_desc;
   2656	int i;
   2657	struct qlge_ob_mac_iocb_req *mac_iocb_ptr;
   2658
   2659	mac_iocb_ptr = tx_ring->wq_base;
   2660	tx_ring_desc = tx_ring->q;
   2661	for (i = 0; i < tx_ring->wq_len; i++) {
   2662		tx_ring_desc->index = i;
   2663		tx_ring_desc->skb = NULL;
   2664		tx_ring_desc->queue_entry = mac_iocb_ptr;
   2665		mac_iocb_ptr++;
   2666		tx_ring_desc++;
   2667	}
   2668	atomic_set(&tx_ring->tx_count, tx_ring->wq_len);
   2669}
   2670
   2671static void qlge_free_tx_resources(struct qlge_adapter *qdev,
   2672				   struct tx_ring *tx_ring)
   2673{
   2674	if (tx_ring->wq_base) {
   2675		dma_free_coherent(&qdev->pdev->dev, tx_ring->wq_size,
   2676				  tx_ring->wq_base, tx_ring->wq_base_dma);
   2677		tx_ring->wq_base = NULL;
   2678	}
   2679	kfree(tx_ring->q);
   2680	tx_ring->q = NULL;
   2681}
   2682
   2683static int qlge_alloc_tx_resources(struct qlge_adapter *qdev,
   2684				   struct tx_ring *tx_ring)
   2685{
   2686	tx_ring->wq_base =
   2687		dma_alloc_coherent(&qdev->pdev->dev, tx_ring->wq_size,
   2688				   &tx_ring->wq_base_dma, GFP_ATOMIC);
   2689
   2690	if (!tx_ring->wq_base ||
   2691	    tx_ring->wq_base_dma & WQ_ADDR_ALIGN)
   2692		goto pci_alloc_err;
   2693
   2694	tx_ring->q =
   2695		kmalloc_array(tx_ring->wq_len, sizeof(struct tx_ring_desc),
   2696			      GFP_KERNEL);
   2697	if (!tx_ring->q)
   2698		goto err;
   2699
   2700	return 0;
   2701err:
   2702	dma_free_coherent(&qdev->pdev->dev, tx_ring->wq_size,
   2703			  tx_ring->wq_base, tx_ring->wq_base_dma);
   2704	tx_ring->wq_base = NULL;
   2705pci_alloc_err:
   2706	netif_err(qdev, ifup, qdev->ndev, "tx_ring alloc failed.\n");
   2707	return -ENOMEM;
   2708}
   2709
   2710static void qlge_free_lbq_buffers(struct qlge_adapter *qdev, struct rx_ring *rx_ring)
   2711{
   2712	struct qlge_bq *lbq = &rx_ring->lbq;
   2713	unsigned int last_offset;
   2714
   2715	last_offset = qlge_lbq_block_size(qdev) - qdev->lbq_buf_size;
   2716	while (lbq->next_to_clean != lbq->next_to_use) {
   2717		struct qlge_bq_desc *lbq_desc =
   2718			&lbq->queue[lbq->next_to_clean];
   2719
   2720		if (lbq_desc->p.pg_chunk.offset == last_offset)
   2721			dma_unmap_page(&qdev->pdev->dev, lbq_desc->dma_addr,
   2722				       qlge_lbq_block_size(qdev),
   2723				       DMA_FROM_DEVICE);
   2724		put_page(lbq_desc->p.pg_chunk.page);
   2725
   2726		lbq->next_to_clean = QLGE_BQ_WRAP(lbq->next_to_clean + 1);
   2727	}
   2728
   2729	if (rx_ring->master_chunk.page) {
   2730		dma_unmap_page(&qdev->pdev->dev, rx_ring->chunk_dma_addr,
   2731			       qlge_lbq_block_size(qdev), DMA_FROM_DEVICE);
   2732		put_page(rx_ring->master_chunk.page);
   2733		rx_ring->master_chunk.page = NULL;
   2734	}
   2735}
   2736
   2737static void qlge_free_sbq_buffers(struct qlge_adapter *qdev, struct rx_ring *rx_ring)
   2738{
   2739	int i;
   2740
   2741	for (i = 0; i < QLGE_BQ_LEN; i++) {
   2742		struct qlge_bq_desc *sbq_desc = &rx_ring->sbq.queue[i];
   2743
   2744		if (!sbq_desc) {
   2745			netif_err(qdev, ifup, qdev->ndev,
   2746				  "sbq_desc %d is NULL.\n", i);
   2747			return;
   2748		}
   2749		if (sbq_desc->p.skb) {
   2750			dma_unmap_single(&qdev->pdev->dev, sbq_desc->dma_addr,
   2751					 SMALL_BUF_MAP_SIZE,
   2752					 DMA_FROM_DEVICE);
   2753			dev_kfree_skb(sbq_desc->p.skb);
   2754			sbq_desc->p.skb = NULL;
   2755		}
   2756	}
   2757}
   2758
   2759/* Free all large and small rx buffers associated
   2760 * with the completion queues for this device.
   2761 */
   2762static void qlge_free_rx_buffers(struct qlge_adapter *qdev)
   2763{
   2764	int i;
   2765
   2766	for (i = 0; i < qdev->rx_ring_count; i++) {
   2767		struct rx_ring *rx_ring = &qdev->rx_ring[i];
   2768
   2769		if (rx_ring->lbq.queue)
   2770			qlge_free_lbq_buffers(qdev, rx_ring);
   2771		if (rx_ring->sbq.queue)
   2772			qlge_free_sbq_buffers(qdev, rx_ring);
   2773	}
   2774}
   2775
   2776static void qlge_alloc_rx_buffers(struct qlge_adapter *qdev)
   2777{
   2778	int i;
   2779
   2780	for (i = 0; i < qdev->rss_ring_count; i++)
   2781		qlge_update_buffer_queues(&qdev->rx_ring[i], GFP_KERNEL,
   2782					  HZ / 2);
   2783}
   2784
   2785static int qlge_init_bq(struct qlge_bq *bq)
   2786{
   2787	struct rx_ring *rx_ring = QLGE_BQ_CONTAINER(bq);
   2788	struct qlge_adapter *qdev = rx_ring->qdev;
   2789	struct qlge_bq_desc *bq_desc;
   2790	__le64 *buf_ptr;
   2791	int i;
   2792
   2793	bq->base = dma_alloc_coherent(&qdev->pdev->dev, QLGE_BQ_SIZE,
   2794				      &bq->base_dma, GFP_ATOMIC);
   2795	if (!bq->base)
   2796		return -ENOMEM;
   2797
   2798	bq->queue = kmalloc_array(QLGE_BQ_LEN, sizeof(struct qlge_bq_desc),
   2799				  GFP_KERNEL);
   2800	if (!bq->queue)
   2801		return -ENOMEM;
   2802
   2803	buf_ptr = bq->base;
   2804	bq_desc = &bq->queue[0];
   2805	for (i = 0; i < QLGE_BQ_LEN; i++, buf_ptr++, bq_desc++) {
   2806		bq_desc->p.skb = NULL;
   2807		bq_desc->index = i;
   2808		bq_desc->buf_ptr = buf_ptr;
   2809	}
   2810
   2811	return 0;
   2812}
   2813
   2814static void qlge_free_rx_resources(struct qlge_adapter *qdev,
   2815				   struct rx_ring *rx_ring)
   2816{
   2817	/* Free the small buffer queue. */
   2818	if (rx_ring->sbq.base) {
   2819		dma_free_coherent(&qdev->pdev->dev, QLGE_BQ_SIZE,
   2820				  rx_ring->sbq.base, rx_ring->sbq.base_dma);
   2821		rx_ring->sbq.base = NULL;
   2822	}
   2823
   2824	/* Free the small buffer queue control blocks. */
   2825	kfree(rx_ring->sbq.queue);
   2826	rx_ring->sbq.queue = NULL;
   2827
   2828	/* Free the large buffer queue. */
   2829	if (rx_ring->lbq.base) {
   2830		dma_free_coherent(&qdev->pdev->dev, QLGE_BQ_SIZE,
   2831				  rx_ring->lbq.base, rx_ring->lbq.base_dma);
   2832		rx_ring->lbq.base = NULL;
   2833	}
   2834
   2835	/* Free the large buffer queue control blocks. */
   2836	kfree(rx_ring->lbq.queue);
   2837	rx_ring->lbq.queue = NULL;
   2838
   2839	/* Free the rx queue. */
   2840	if (rx_ring->cq_base) {
   2841		dma_free_coherent(&qdev->pdev->dev,
   2842				  rx_ring->cq_size,
   2843				  rx_ring->cq_base, rx_ring->cq_base_dma);
   2844		rx_ring->cq_base = NULL;
   2845	}
   2846}
   2847
   2848/* Allocate queues and buffers for this completions queue based
   2849 * on the values in the parameter structure.
   2850 */
   2851static int qlge_alloc_rx_resources(struct qlge_adapter *qdev,
   2852				   struct rx_ring *rx_ring)
   2853{
   2854	/*
   2855	 * Allocate the completion queue for this rx_ring.
   2856	 */
   2857	rx_ring->cq_base =
   2858		dma_alloc_coherent(&qdev->pdev->dev, rx_ring->cq_size,
   2859				   &rx_ring->cq_base_dma, GFP_ATOMIC);
   2860
   2861	if (!rx_ring->cq_base) {
   2862		netif_err(qdev, ifup, qdev->ndev, "rx_ring alloc failed.\n");
   2863		return -ENOMEM;
   2864	}
   2865
   2866	if (rx_ring->cq_id < qdev->rss_ring_count &&
   2867	    (qlge_init_bq(&rx_ring->sbq) || qlge_init_bq(&rx_ring->lbq))) {
   2868		qlge_free_rx_resources(qdev, rx_ring);
   2869		return -ENOMEM;
   2870	}
   2871
   2872	return 0;
   2873}
   2874
   2875static void qlge_tx_ring_clean(struct qlge_adapter *qdev)
   2876{
   2877	struct tx_ring *tx_ring;
   2878	struct tx_ring_desc *tx_ring_desc;
   2879	int i, j;
   2880
   2881	/*
   2882	 * Loop through all queues and free
   2883	 * any resources.
   2884	 */
   2885	for (j = 0; j < qdev->tx_ring_count; j++) {
   2886		tx_ring = &qdev->tx_ring[j];
   2887		for (i = 0; i < tx_ring->wq_len; i++) {
   2888			tx_ring_desc = &tx_ring->q[i];
   2889			if (tx_ring_desc && tx_ring_desc->skb) {
   2890				netif_err(qdev, ifdown, qdev->ndev,
   2891					  "Freeing lost SKB %p, from queue %d, index %d.\n",
   2892					  tx_ring_desc->skb, j,
   2893					  tx_ring_desc->index);
   2894				qlge_unmap_send(qdev, tx_ring_desc,
   2895						tx_ring_desc->map_cnt);
   2896				dev_kfree_skb(tx_ring_desc->skb);
   2897				tx_ring_desc->skb = NULL;
   2898			}
   2899		}
   2900	}
   2901}
   2902
   2903static void qlge_free_mem_resources(struct qlge_adapter *qdev)
   2904{
   2905	int i;
   2906
   2907	for (i = 0; i < qdev->tx_ring_count; i++)
   2908		qlge_free_tx_resources(qdev, &qdev->tx_ring[i]);
   2909	for (i = 0; i < qdev->rx_ring_count; i++)
   2910		qlge_free_rx_resources(qdev, &qdev->rx_ring[i]);
   2911	qlge_free_shadow_space(qdev);
   2912}
   2913
   2914static int qlge_alloc_mem_resources(struct qlge_adapter *qdev)
   2915{
   2916	int i;
   2917
   2918	/* Allocate space for our shadow registers and such. */
   2919	if (qlge_alloc_shadow_space(qdev))
   2920		return -ENOMEM;
   2921
   2922	for (i = 0; i < qdev->rx_ring_count; i++) {
   2923		if (qlge_alloc_rx_resources(qdev, &qdev->rx_ring[i]) != 0) {
   2924			netif_err(qdev, ifup, qdev->ndev,
   2925				  "RX resource allocation failed.\n");
   2926			goto err_mem;
   2927		}
   2928	}
   2929	/* Allocate tx queue resources */
   2930	for (i = 0; i < qdev->tx_ring_count; i++) {
   2931		if (qlge_alloc_tx_resources(qdev, &qdev->tx_ring[i]) != 0) {
   2932			netif_err(qdev, ifup, qdev->ndev,
   2933				  "TX resource allocation failed.\n");
   2934			goto err_mem;
   2935		}
   2936	}
   2937	return 0;
   2938
   2939err_mem:
   2940	qlge_free_mem_resources(qdev);
   2941	return -ENOMEM;
   2942}
   2943
   2944/* Set up the rx ring control block and pass it to the chip.
   2945 * The control block is defined as
   2946 * "Completion Queue Initialization Control Block", or cqicb.
   2947 */
   2948static int qlge_start_rx_ring(struct qlge_adapter *qdev, struct rx_ring *rx_ring)
   2949{
   2950	struct cqicb *cqicb = &rx_ring->cqicb;
   2951	void *shadow_reg = qdev->rx_ring_shadow_reg_area +
   2952		(rx_ring->cq_id * RX_RING_SHADOW_SPACE);
   2953	u64 shadow_reg_dma = qdev->rx_ring_shadow_reg_dma +
   2954		(rx_ring->cq_id * RX_RING_SHADOW_SPACE);
   2955	void __iomem *doorbell_area =
   2956		qdev->doorbell_area + (DB_PAGE_SIZE * (128 + rx_ring->cq_id));
   2957	int err = 0;
   2958	u64 tmp;
   2959	__le64 *base_indirect_ptr;
   2960	int page_entries;
   2961
   2962	/* Set up the shadow registers for this ring. */
   2963	rx_ring->prod_idx_sh_reg = shadow_reg;
   2964	rx_ring->prod_idx_sh_reg_dma = shadow_reg_dma;
   2965	*rx_ring->prod_idx_sh_reg = 0;
   2966	shadow_reg += sizeof(u64);
   2967	shadow_reg_dma += sizeof(u64);
   2968	rx_ring->lbq.base_indirect = shadow_reg;
   2969	rx_ring->lbq.base_indirect_dma = shadow_reg_dma;
   2970	shadow_reg += (sizeof(u64) * MAX_DB_PAGES_PER_BQ(QLGE_BQ_LEN));
   2971	shadow_reg_dma += (sizeof(u64) * MAX_DB_PAGES_PER_BQ(QLGE_BQ_LEN));
   2972	rx_ring->sbq.base_indirect = shadow_reg;
   2973	rx_ring->sbq.base_indirect_dma = shadow_reg_dma;
   2974
   2975	/* PCI doorbell mem area + 0x00 for consumer index register */
   2976	rx_ring->cnsmr_idx_db_reg = (u32 __iomem *)doorbell_area;
   2977	rx_ring->cnsmr_idx = 0;
   2978	rx_ring->curr_entry = rx_ring->cq_base;
   2979
   2980	/* PCI doorbell mem area + 0x04 for valid register */
   2981	rx_ring->valid_db_reg = doorbell_area + 0x04;
   2982
   2983	/* PCI doorbell mem area + 0x18 for large buffer consumer */
   2984	rx_ring->lbq.prod_idx_db_reg = (u32 __iomem *)(doorbell_area + 0x18);
   2985
   2986	/* PCI doorbell mem area + 0x1c */
   2987	rx_ring->sbq.prod_idx_db_reg = (u32 __iomem *)(doorbell_area + 0x1c);
   2988
   2989	memset((void *)cqicb, 0, sizeof(struct cqicb));
   2990	cqicb->msix_vect = rx_ring->irq;
   2991
   2992	cqicb->len = cpu_to_le16(QLGE_FIT16(rx_ring->cq_len) | LEN_V |
   2993				 LEN_CPP_CONT);
   2994
   2995	cqicb->addr = cpu_to_le64(rx_ring->cq_base_dma);
   2996
   2997	cqicb->prod_idx_addr = cpu_to_le64(rx_ring->prod_idx_sh_reg_dma);
   2998
   2999	/*
   3000	 * Set up the control block load flags.
   3001	 */
   3002	cqicb->flags = FLAGS_LC |	/* Load queue base address */
   3003		FLAGS_LV |		/* Load MSI-X vector */
   3004		FLAGS_LI;		/* Load irq delay values */
   3005	if (rx_ring->cq_id < qdev->rss_ring_count) {
   3006		cqicb->flags |= FLAGS_LL;	/* Load lbq values */
   3007		tmp = (u64)rx_ring->lbq.base_dma;
   3008		base_indirect_ptr = rx_ring->lbq.base_indirect;
   3009		page_entries = 0;
   3010		do {
   3011			*base_indirect_ptr = cpu_to_le64(tmp);
   3012			tmp += DB_PAGE_SIZE;
   3013			base_indirect_ptr++;
   3014			page_entries++;
   3015		} while (page_entries < MAX_DB_PAGES_PER_BQ(QLGE_BQ_LEN));
   3016		cqicb->lbq_addr = cpu_to_le64(rx_ring->lbq.base_indirect_dma);
   3017		cqicb->lbq_buf_size =
   3018			cpu_to_le16(QLGE_FIT16(qdev->lbq_buf_size));
   3019		cqicb->lbq_len = cpu_to_le16(QLGE_FIT16(QLGE_BQ_LEN));
   3020		rx_ring->lbq.next_to_use = 0;
   3021		rx_ring->lbq.next_to_clean = 0;
   3022
   3023		cqicb->flags |= FLAGS_LS;	/* Load sbq values */
   3024		tmp = (u64)rx_ring->sbq.base_dma;
   3025		base_indirect_ptr = rx_ring->sbq.base_indirect;
   3026		page_entries = 0;
   3027		do {
   3028			*base_indirect_ptr = cpu_to_le64(tmp);
   3029			tmp += DB_PAGE_SIZE;
   3030			base_indirect_ptr++;
   3031			page_entries++;
   3032		} while (page_entries < MAX_DB_PAGES_PER_BQ(QLGE_BQ_LEN));
   3033		cqicb->sbq_addr =
   3034			cpu_to_le64(rx_ring->sbq.base_indirect_dma);
   3035		cqicb->sbq_buf_size = cpu_to_le16(SMALL_BUFFER_SIZE);
   3036		cqicb->sbq_len = cpu_to_le16(QLGE_FIT16(QLGE_BQ_LEN));
   3037		rx_ring->sbq.next_to_use = 0;
   3038		rx_ring->sbq.next_to_clean = 0;
   3039	}
   3040	if (rx_ring->cq_id < qdev->rss_ring_count) {
   3041		/* Inbound completion handling rx_rings run in
   3042		 * separate NAPI contexts.
   3043		 */
   3044		netif_napi_add(qdev->ndev, &rx_ring->napi, qlge_napi_poll_msix,
   3045			       64);
   3046		cqicb->irq_delay = cpu_to_le16(qdev->rx_coalesce_usecs);
   3047		cqicb->pkt_delay = cpu_to_le16(qdev->rx_max_coalesced_frames);
   3048	} else {
   3049		cqicb->irq_delay = cpu_to_le16(qdev->tx_coalesce_usecs);
   3050		cqicb->pkt_delay = cpu_to_le16(qdev->tx_max_coalesced_frames);
   3051	}
   3052	err = qlge_write_cfg(qdev, cqicb, sizeof(struct cqicb),
   3053			     CFG_LCQ, rx_ring->cq_id);
   3054	if (err) {
   3055		netif_err(qdev, ifup, qdev->ndev, "Failed to load CQICB.\n");
   3056		return err;
   3057	}
   3058	return err;
   3059}
   3060
   3061static int qlge_start_tx_ring(struct qlge_adapter *qdev, struct tx_ring *tx_ring)
   3062{
   3063	struct wqicb *wqicb = (struct wqicb *)tx_ring;
   3064	void __iomem *doorbell_area =
   3065		qdev->doorbell_area + (DB_PAGE_SIZE * tx_ring->wq_id);
   3066	void *shadow_reg = qdev->tx_ring_shadow_reg_area +
   3067		(tx_ring->wq_id * sizeof(u64));
   3068	u64 shadow_reg_dma = qdev->tx_ring_shadow_reg_dma +
   3069		(tx_ring->wq_id * sizeof(u64));
   3070	int err = 0;
   3071
   3072	/*
   3073	 * Assign doorbell registers for this tx_ring.
   3074	 */
   3075	/* TX PCI doorbell mem area for tx producer index */
   3076	tx_ring->prod_idx_db_reg = (u32 __iomem *)doorbell_area;
   3077	tx_ring->prod_idx = 0;
   3078	/* TX PCI doorbell mem area + 0x04 */
   3079	tx_ring->valid_db_reg = doorbell_area + 0x04;
   3080
   3081	/*
   3082	 * Assign shadow registers for this tx_ring.
   3083	 */
   3084	tx_ring->cnsmr_idx_sh_reg = shadow_reg;
   3085	tx_ring->cnsmr_idx_sh_reg_dma = shadow_reg_dma;
   3086
   3087	wqicb->len = cpu_to_le16(tx_ring->wq_len | Q_LEN_V | Q_LEN_CPP_CONT);
   3088	wqicb->flags = cpu_to_le16(Q_FLAGS_LC |
   3089				   Q_FLAGS_LB | Q_FLAGS_LI | Q_FLAGS_LO);
   3090	wqicb->cq_id_rss = cpu_to_le16(tx_ring->cq_id);
   3091	wqicb->rid = 0;
   3092	wqicb->addr = cpu_to_le64(tx_ring->wq_base_dma);
   3093
   3094	wqicb->cnsmr_idx_addr = cpu_to_le64(tx_ring->cnsmr_idx_sh_reg_dma);
   3095
   3096	qlge_init_tx_ring(qdev, tx_ring);
   3097
   3098	err = qlge_write_cfg(qdev, wqicb, sizeof(*wqicb), CFG_LRQ,
   3099			     (u16)tx_ring->wq_id);
   3100	if (err) {
   3101		netif_err(qdev, ifup, qdev->ndev, "Failed to load tx_ring.\n");
   3102		return err;
   3103	}
   3104	return err;
   3105}
   3106
   3107static void qlge_disable_msix(struct qlge_adapter *qdev)
   3108{
   3109	if (test_bit(QL_MSIX_ENABLED, &qdev->flags)) {
   3110		pci_disable_msix(qdev->pdev);
   3111		clear_bit(QL_MSIX_ENABLED, &qdev->flags);
   3112		kfree(qdev->msi_x_entry);
   3113		qdev->msi_x_entry = NULL;
   3114	} else if (test_bit(QL_MSI_ENABLED, &qdev->flags)) {
   3115		pci_disable_msi(qdev->pdev);
   3116		clear_bit(QL_MSI_ENABLED, &qdev->flags);
   3117	}
   3118}
   3119
   3120/* We start by trying to get the number of vectors
   3121 * stored in qdev->intr_count. If we don't get that
   3122 * many then we reduce the count and try again.
   3123 */
   3124static void qlge_enable_msix(struct qlge_adapter *qdev)
   3125{
   3126	int i, err;
   3127
   3128	/* Get the MSIX vectors. */
   3129	if (qlge_irq_type == MSIX_IRQ) {
   3130		/* Try to alloc space for the msix struct,
   3131		 * if it fails then go to MSI/legacy.
   3132		 */
   3133		qdev->msi_x_entry = kcalloc(qdev->intr_count,
   3134					    sizeof(struct msix_entry),
   3135					    GFP_KERNEL);
   3136		if (!qdev->msi_x_entry) {
   3137			qlge_irq_type = MSI_IRQ;
   3138			goto msi;
   3139		}
   3140
   3141		for (i = 0; i < qdev->intr_count; i++)
   3142			qdev->msi_x_entry[i].entry = i;
   3143
   3144		err = pci_enable_msix_range(qdev->pdev, qdev->msi_x_entry,
   3145					    1, qdev->intr_count);
   3146		if (err < 0) {
   3147			kfree(qdev->msi_x_entry);
   3148			qdev->msi_x_entry = NULL;
   3149			netif_warn(qdev, ifup, qdev->ndev,
   3150				   "MSI-X Enable failed, trying MSI.\n");
   3151			qlge_irq_type = MSI_IRQ;
   3152		} else {
   3153			qdev->intr_count = err;
   3154			set_bit(QL_MSIX_ENABLED, &qdev->flags);
   3155			netif_info(qdev, ifup, qdev->ndev,
   3156				   "MSI-X Enabled, got %d vectors.\n",
   3157				   qdev->intr_count);
   3158			return;
   3159		}
   3160	}
   3161msi:
   3162	qdev->intr_count = 1;
   3163	if (qlge_irq_type == MSI_IRQ) {
   3164		if (pci_alloc_irq_vectors(qdev->pdev, 1, 1, PCI_IRQ_MSI) >= 0) {
   3165			set_bit(QL_MSI_ENABLED, &qdev->flags);
   3166			netif_info(qdev, ifup, qdev->ndev,
   3167				   "Running with MSI interrupts.\n");
   3168			return;
   3169		}
   3170	}
   3171	qlge_irq_type = LEG_IRQ;
   3172	set_bit(QL_LEGACY_ENABLED, &qdev->flags);
   3173	netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
   3174		     "Running with legacy interrupts.\n");
   3175}
   3176
   3177/* Each vector services 1 RSS ring and 1 or more
   3178 * TX completion rings.  This function loops through
   3179 * the TX completion rings and assigns the vector that
   3180 * will service it.  An example would be if there are
   3181 * 2 vectors (so 2 RSS rings) and 8 TX completion rings.
   3182 * This would mean that vector 0 would service RSS ring 0
   3183 * and TX completion rings 0,1,2 and 3.  Vector 1 would
   3184 * service RSS ring 1 and TX completion rings 4,5,6 and 7.
   3185 */
   3186static void qlge_set_tx_vect(struct qlge_adapter *qdev)
   3187{
   3188	int i, j, vect;
   3189	u32 tx_rings_per_vector = qdev->tx_ring_count / qdev->intr_count;
   3190
   3191	if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags))) {
   3192		/* Assign irq vectors to TX rx_rings.*/
   3193		for (vect = 0, j = 0, i = qdev->rss_ring_count;
   3194		     i < qdev->rx_ring_count; i++) {
   3195			if (j == tx_rings_per_vector) {
   3196				vect++;
   3197				j = 0;
   3198			}
   3199			qdev->rx_ring[i].irq = vect;
   3200			j++;
   3201		}
   3202	} else {
   3203		/* For single vector all rings have an irq
   3204		 * of zero.
   3205		 */
   3206		for (i = 0; i < qdev->rx_ring_count; i++)
   3207			qdev->rx_ring[i].irq = 0;
   3208	}
   3209}
   3210
   3211/* Set the interrupt mask for this vector.  Each vector
   3212 * will service 1 RSS ring and 1 or more TX completion
   3213 * rings.  This function sets up a bit mask per vector
   3214 * that indicates which rings it services.
   3215 */
   3216static void qlge_set_irq_mask(struct qlge_adapter *qdev, struct intr_context *ctx)
   3217{
   3218	int j, vect = ctx->intr;
   3219	u32 tx_rings_per_vector = qdev->tx_ring_count / qdev->intr_count;
   3220
   3221	if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags))) {
   3222		/* Add the RSS ring serviced by this vector
   3223		 * to the mask.
   3224		 */
   3225		ctx->irq_mask = (1 << qdev->rx_ring[vect].cq_id);
   3226		/* Add the TX ring(s) serviced by this vector
   3227		 * to the mask.
   3228		 */
   3229		for (j = 0; j < tx_rings_per_vector; j++) {
   3230			ctx->irq_mask |=
   3231				(1 << qdev->rx_ring[qdev->rss_ring_count +
   3232				 (vect * tx_rings_per_vector) + j].cq_id);
   3233		}
   3234	} else {
   3235		/* For single vector we just shift each queue's
   3236		 * ID into the mask.
   3237		 */
   3238		for (j = 0; j < qdev->rx_ring_count; j++)
   3239			ctx->irq_mask |= (1 << qdev->rx_ring[j].cq_id);
   3240	}
   3241}
   3242
   3243/*
   3244 * Here we build the intr_context structures based on
   3245 * our rx_ring count and intr vector count.
   3246 * The intr_context structure is used to hook each vector
   3247 * to possibly different handlers.
   3248 */
   3249static void qlge_resolve_queues_to_irqs(struct qlge_adapter *qdev)
   3250{
   3251	int i = 0;
   3252	struct intr_context *intr_context = &qdev->intr_context[0];
   3253
   3254	if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags))) {
   3255		/* Each rx_ring has it's
   3256		 * own intr_context since we have separate
   3257		 * vectors for each queue.
   3258		 */
   3259		for (i = 0; i < qdev->intr_count; i++, intr_context++) {
   3260			qdev->rx_ring[i].irq = i;
   3261			intr_context->intr = i;
   3262			intr_context->qdev = qdev;
   3263			/* Set up this vector's bit-mask that indicates
   3264			 * which queues it services.
   3265			 */
   3266			qlge_set_irq_mask(qdev, intr_context);
   3267			/*
   3268			 * We set up each vectors enable/disable/read bits so
   3269			 * there's no bit/mask calculations in the critical path.
   3270			 */
   3271			intr_context->intr_en_mask =
   3272				INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
   3273				INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK | INTR_EN_IHD
   3274				| i;
   3275			intr_context->intr_dis_mask =
   3276				INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
   3277				INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK |
   3278				INTR_EN_IHD | i;
   3279			intr_context->intr_read_mask =
   3280				INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
   3281				INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD |
   3282				i;
   3283			if (i == 0) {
   3284				/* The first vector/queue handles
   3285				 * broadcast/multicast, fatal errors,
   3286				 * and firmware events.  This in addition
   3287				 * to normal inbound NAPI processing.
   3288				 */
   3289				intr_context->handler = qlge_isr;
   3290				sprintf(intr_context->name, "%s-rx-%d",
   3291					qdev->ndev->name, i);
   3292			} else {
   3293				/*
   3294				 * Inbound queues handle unicast frames only.
   3295				 */
   3296				intr_context->handler = qlge_msix_rx_isr;
   3297				sprintf(intr_context->name, "%s-rx-%d",
   3298					qdev->ndev->name, i);
   3299			}
   3300		}
   3301	} else {
   3302		/*
   3303		 * All rx_rings use the same intr_context since
   3304		 * there is only one vector.
   3305		 */
   3306		intr_context->intr = 0;
   3307		intr_context->qdev = qdev;
   3308		/*
   3309		 * We set up each vectors enable/disable/read bits so
   3310		 * there's no bit/mask calculations in the critical path.
   3311		 */
   3312		intr_context->intr_en_mask =
   3313			INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_ENABLE;
   3314		intr_context->intr_dis_mask =
   3315			INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
   3316			INTR_EN_TYPE_DISABLE;
   3317		if (test_bit(QL_LEGACY_ENABLED, &qdev->flags)) {
   3318			/* Experience shows that when using INTx interrupts,
   3319			 * the device does not always auto-mask INTR_EN_EN.
   3320			 * Moreover, masking INTR_EN_EN manually does not
   3321			 * immediately prevent interrupt generation.
   3322			 */
   3323			intr_context->intr_en_mask |= INTR_EN_EI << 16 |
   3324				INTR_EN_EI;
   3325			intr_context->intr_dis_mask |= INTR_EN_EI << 16;
   3326		}
   3327		intr_context->intr_read_mask =
   3328			INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_READ;
   3329		/*
   3330		 * Single interrupt means one handler for all rings.
   3331		 */
   3332		intr_context->handler = qlge_isr;
   3333		sprintf(intr_context->name, "%s-single_irq", qdev->ndev->name);
   3334		/* Set up this vector's bit-mask that indicates
   3335		 * which queues it services. In this case there is
   3336		 * a single vector so it will service all RSS and
   3337		 * TX completion rings.
   3338		 */
   3339		qlge_set_irq_mask(qdev, intr_context);
   3340	}
   3341	/* Tell the TX completion rings which MSIx vector
   3342	 * they will be using.
   3343	 */
   3344	qlge_set_tx_vect(qdev);
   3345}
   3346
   3347static void qlge_free_irq(struct qlge_adapter *qdev)
   3348{
   3349	int i;
   3350	struct intr_context *intr_context = &qdev->intr_context[0];
   3351
   3352	for (i = 0; i < qdev->intr_count; i++, intr_context++) {
   3353		if (intr_context->hooked) {
   3354			if (test_bit(QL_MSIX_ENABLED, &qdev->flags)) {
   3355				free_irq(qdev->msi_x_entry[i].vector,
   3356					 &qdev->rx_ring[i]);
   3357			} else {
   3358				free_irq(qdev->pdev->irq, &qdev->rx_ring[0]);
   3359			}
   3360		}
   3361	}
   3362	qlge_disable_msix(qdev);
   3363}
   3364
   3365static int qlge_request_irq(struct qlge_adapter *qdev)
   3366{
   3367	int i;
   3368	int status = 0;
   3369	struct pci_dev *pdev = qdev->pdev;
   3370	struct intr_context *intr_context = &qdev->intr_context[0];
   3371
   3372	qlge_resolve_queues_to_irqs(qdev);
   3373
   3374	for (i = 0; i < qdev->intr_count; i++, intr_context++) {
   3375		if (test_bit(QL_MSIX_ENABLED, &qdev->flags)) {
   3376			status = request_irq(qdev->msi_x_entry[i].vector,
   3377					     intr_context->handler,
   3378					     0,
   3379					     intr_context->name,
   3380					     &qdev->rx_ring[i]);
   3381			if (status) {
   3382				netif_err(qdev, ifup, qdev->ndev,
   3383					  "Failed request for MSIX interrupt %d.\n",
   3384					  i);
   3385				goto err_irq;
   3386			}
   3387		} else {
   3388			netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
   3389				     "trying msi or legacy interrupts.\n");
   3390			netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
   3391				     "%s: irq = %d.\n", __func__, pdev->irq);
   3392			netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
   3393				     "%s: context->name = %s.\n", __func__,
   3394				     intr_context->name);
   3395			netif_printk(qdev, ifup, KERN_DEBUG, qdev->ndev,
   3396				     "%s: dev_id = 0x%p.\n", __func__,
   3397				     &qdev->rx_ring[0]);
   3398			status =
   3399				request_irq(pdev->irq, qlge_isr,
   3400					    test_bit(QL_MSI_ENABLED, &qdev->flags)
   3401					    ? 0
   3402					    : IRQF_SHARED,
   3403					    intr_context->name, &qdev->rx_ring[0]);
   3404			if (status)
   3405				goto err_irq;
   3406
   3407			netif_err(qdev, ifup, qdev->ndev,
   3408				  "Hooked intr 0, queue type RX_Q, with name %s.\n",
   3409				  intr_context->name);
   3410		}
   3411		intr_context->hooked = 1;
   3412	}
   3413	return status;
   3414err_irq:
   3415	netif_err(qdev, ifup, qdev->ndev, "Failed to get the interrupts!!!\n");
   3416	qlge_free_irq(qdev);
   3417	return status;
   3418}
   3419
   3420static int qlge_start_rss(struct qlge_adapter *qdev)
   3421{
   3422	static const u8 init_hash_seed[] = {
   3423		0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
   3424		0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
   3425		0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
   3426		0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
   3427		0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa
   3428	};
   3429	struct ricb *ricb = &qdev->ricb;
   3430	int status = 0;
   3431	int i;
   3432	u8 *hash_id = (u8 *)ricb->hash_cq_id;
   3433
   3434	memset((void *)ricb, 0, sizeof(*ricb));
   3435
   3436	ricb->base_cq = RSS_L4K;
   3437	ricb->flags =
   3438		(RSS_L6K | RSS_LI | RSS_LB | RSS_LM | RSS_RT4 | RSS_RT6);
   3439	ricb->mask = cpu_to_le16((u16)(0x3ff));
   3440
   3441	/*
   3442	 * Fill out the Indirection Table.
   3443	 */
   3444	for (i = 0; i < 1024; i++)
   3445		hash_id[i] = (i & (qdev->rss_ring_count - 1));
   3446
   3447	memcpy((void *)&ricb->ipv6_hash_key[0], init_hash_seed, 40);
   3448	memcpy((void *)&ricb->ipv4_hash_key[0], init_hash_seed, 16);
   3449
   3450	status = qlge_write_cfg(qdev, ricb, sizeof(*ricb), CFG_LR, 0);
   3451	if (status) {
   3452		netif_err(qdev, ifup, qdev->ndev, "Failed to load RICB.\n");
   3453		return status;
   3454	}
   3455	return status;
   3456}
   3457
   3458static int qlge_clear_routing_entries(struct qlge_adapter *qdev)
   3459{
   3460	int i, status = 0;
   3461
   3462	status = qlge_sem_spinlock(qdev, SEM_RT_IDX_MASK);
   3463	if (status)
   3464		return status;
   3465	/* Clear all the entries in the routing table. */
   3466	for (i = 0; i < 16; i++) {
   3467		status = qlge_set_routing_reg(qdev, i, 0, 0);
   3468		if (status) {
   3469			netif_err(qdev, ifup, qdev->ndev,
   3470				  "Failed to init routing register for CAM packets.\n");
   3471			break;
   3472		}
   3473	}
   3474	qlge_sem_unlock(qdev, SEM_RT_IDX_MASK);
   3475	return status;
   3476}
   3477
   3478/* Initialize the frame-to-queue routing. */
   3479static int qlge_route_initialize(struct qlge_adapter *qdev)
   3480{
   3481	int status = 0;
   3482
   3483	/* Clear all the entries in the routing table. */
   3484	status = qlge_clear_routing_entries(qdev);
   3485	if (status)
   3486		return status;
   3487
   3488	status = qlge_sem_spinlock(qdev, SEM_RT_IDX_MASK);
   3489	if (status)
   3490		return status;
   3491
   3492	status = qlge_set_routing_reg(qdev, RT_IDX_IP_CSUM_ERR_SLOT,
   3493				      RT_IDX_IP_CSUM_ERR, 1);
   3494	if (status) {
   3495		netif_err(qdev, ifup, qdev->ndev,
   3496			  "Failed to init routing register for IP CSUM error packets.\n");
   3497		goto exit;
   3498	}
   3499	status = qlge_set_routing_reg(qdev, RT_IDX_TCP_UDP_CSUM_ERR_SLOT,
   3500				      RT_IDX_TU_CSUM_ERR, 1);
   3501	if (status) {
   3502		netif_err(qdev, ifup, qdev->ndev,
   3503			  "Failed to init routing register for TCP/UDP CSUM error packets.\n");
   3504		goto exit;
   3505	}
   3506	status = qlge_set_routing_reg(qdev, RT_IDX_BCAST_SLOT, RT_IDX_BCAST, 1);
   3507	if (status) {
   3508		netif_err(qdev, ifup, qdev->ndev,
   3509			  "Failed to init routing register for broadcast packets.\n");
   3510		goto exit;
   3511	}
   3512	/* If we have more than one inbound queue, then turn on RSS in the
   3513	 * routing block.
   3514	 */
   3515	if (qdev->rss_ring_count > 1) {
   3516		status = qlge_set_routing_reg(qdev, RT_IDX_RSS_MATCH_SLOT,
   3517					      RT_IDX_RSS_MATCH, 1);
   3518		if (status) {
   3519			netif_err(qdev, ifup, qdev->ndev,
   3520				  "Failed to init routing register for MATCH RSS packets.\n");
   3521			goto exit;
   3522		}
   3523	}
   3524
   3525	status = qlge_set_routing_reg(qdev, RT_IDX_CAM_HIT_SLOT,
   3526				      RT_IDX_CAM_HIT, 1);
   3527	if (status)
   3528		netif_err(qdev, ifup, qdev->ndev,
   3529			  "Failed to init routing register for CAM packets.\n");
   3530exit:
   3531	qlge_sem_unlock(qdev, SEM_RT_IDX_MASK);
   3532	return status;
   3533}
   3534
   3535int qlge_cam_route_initialize(struct qlge_adapter *qdev)
   3536{
   3537	int status, set;
   3538
   3539	/* If check if the link is up and use to
   3540	 * determine if we are setting or clearing
   3541	 * the MAC address in the CAM.
   3542	 */
   3543	set = qlge_read32(qdev, STS);
   3544	set &= qdev->port_link_up;
   3545	status = qlge_set_mac_addr(qdev, set);
   3546	if (status) {
   3547		netif_err(qdev, ifup, qdev->ndev, "Failed to init mac address.\n");
   3548		return status;
   3549	}
   3550
   3551	status = qlge_route_initialize(qdev);
   3552	if (status)
   3553		netif_err(qdev, ifup, qdev->ndev, "Failed to init routing table.\n");
   3554
   3555	return status;
   3556}
   3557
   3558static int qlge_adapter_initialize(struct qlge_adapter *qdev)
   3559{
   3560	u32 value, mask;
   3561	int i;
   3562	int status = 0;
   3563
   3564	/*
   3565	 * Set up the System register to halt on errors.
   3566	 */
   3567	value = SYS_EFE | SYS_FAE;
   3568	mask = value << 16;
   3569	qlge_write32(qdev, SYS, mask | value);
   3570
   3571	/* Set the default queue, and VLAN behavior. */
   3572	value = NIC_RCV_CFG_DFQ;
   3573	mask = NIC_RCV_CFG_DFQ_MASK;
   3574	if (qdev->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) {
   3575		value |= NIC_RCV_CFG_RV;
   3576		mask |= (NIC_RCV_CFG_RV << 16);
   3577	}
   3578	qlge_write32(qdev, NIC_RCV_CFG, (mask | value));
   3579
   3580	/* Set the MPI interrupt to enabled. */
   3581	qlge_write32(qdev, INTR_MASK, (INTR_MASK_PI << 16) | INTR_MASK_PI);
   3582
   3583	/* Enable the function, set pagesize, enable error checking. */
   3584	value = FSC_FE | FSC_EPC_INBOUND | FSC_EPC_OUTBOUND |
   3585		FSC_EC | FSC_VM_PAGE_4K;
   3586	value |= SPLT_SETTING;
   3587
   3588	/* Set/clear header splitting. */
   3589	mask = FSC_VM_PAGESIZE_MASK |
   3590		FSC_DBL_MASK | FSC_DBRST_MASK | (value << 16);
   3591	qlge_write32(qdev, FSC, mask | value);
   3592
   3593	qlge_write32(qdev, SPLT_HDR, SPLT_LEN);
   3594
   3595	/* Set RX packet routing to use port/pci function on which the
   3596	 * packet arrived on in addition to usual frame routing.
   3597	 * This is helpful on bonding where both interfaces can have
   3598	 * the same MAC address.
   3599	 */
   3600	qlge_write32(qdev, RST_FO, RST_FO_RR_MASK | RST_FO_RR_RCV_FUNC_CQ);
   3601	/* Reroute all packets to our Interface.
   3602	 * They may have been routed to MPI firmware
   3603	 * due to WOL.
   3604	 */
   3605	value = qlge_read32(qdev, MGMT_RCV_CFG);
   3606	value &= ~MGMT_RCV_CFG_RM;
   3607	mask = 0xffff0000;
   3608
   3609	/* Sticky reg needs clearing due to WOL. */
   3610	qlge_write32(qdev, MGMT_RCV_CFG, mask);
   3611	qlge_write32(qdev, MGMT_RCV_CFG, mask | value);
   3612
   3613	/* Default WOL is enable on Mezz cards */
   3614	if (qdev->pdev->subsystem_device == 0x0068 ||
   3615	    qdev->pdev->subsystem_device == 0x0180)
   3616		qdev->wol = WAKE_MAGIC;
   3617
   3618	/* Start up the rx queues. */
   3619	for (i = 0; i < qdev->rx_ring_count; i++) {
   3620		status = qlge_start_rx_ring(qdev, &qdev->rx_ring[i]);
   3621		if (status) {
   3622			netif_err(qdev, ifup, qdev->ndev,
   3623				  "Failed to start rx ring[%d].\n", i);
   3624			return status;
   3625		}
   3626	}
   3627
   3628	/* If there is more than one inbound completion queue
   3629	 * then download a RICB to configure RSS.
   3630	 */
   3631	if (qdev->rss_ring_count > 1) {
   3632		status = qlge_start_rss(qdev);
   3633		if (status) {
   3634			netif_err(qdev, ifup, qdev->ndev, "Failed to start RSS.\n");
   3635			return status;
   3636		}
   3637	}
   3638
   3639	/* Start up the tx queues. */
   3640	for (i = 0; i < qdev->tx_ring_count; i++) {
   3641		status = qlge_start_tx_ring(qdev, &qdev->tx_ring[i]);
   3642		if (status) {
   3643			netif_err(qdev, ifup, qdev->ndev,
   3644				  "Failed to start tx ring[%d].\n", i);
   3645			return status;
   3646		}
   3647	}
   3648
   3649	/* Initialize the port and set the max framesize. */
   3650	status = qdev->nic_ops->port_initialize(qdev);
   3651	if (status)
   3652		netif_err(qdev, ifup, qdev->ndev, "Failed to start port.\n");
   3653
   3654	/* Set up the MAC address and frame routing filter. */
   3655	status = qlge_cam_route_initialize(qdev);
   3656	if (status) {
   3657		netif_err(qdev, ifup, qdev->ndev,
   3658			  "Failed to init CAM/Routing tables.\n");
   3659		return status;
   3660	}
   3661
   3662	/* Start NAPI for the RSS queues. */
   3663	for (i = 0; i < qdev->rss_ring_count; i++)
   3664		napi_enable(&qdev->rx_ring[i].napi);
   3665
   3666	return status;
   3667}
   3668
   3669/* Issue soft reset to chip. */
   3670static int qlge_adapter_reset(struct qlge_adapter *qdev)
   3671{
   3672	u32 value;
   3673	int status = 0;
   3674	unsigned long end_jiffies;
   3675
   3676	/* Clear all the entries in the routing table. */
   3677	status = qlge_clear_routing_entries(qdev);
   3678	if (status) {
   3679		netif_err(qdev, ifup, qdev->ndev, "Failed to clear routing bits.\n");
   3680		return status;
   3681	}
   3682
   3683	/* Check if bit is set then skip the mailbox command and
   3684	 * clear the bit, else we are in normal reset process.
   3685	 */
   3686	if (!test_bit(QL_ASIC_RECOVERY, &qdev->flags)) {
   3687		/* Stop management traffic. */
   3688		qlge_mb_set_mgmnt_traffic_ctl(qdev, MB_SET_MPI_TFK_STOP);
   3689
   3690		/* Wait for the NIC and MGMNT FIFOs to empty. */
   3691		qlge_wait_fifo_empty(qdev);
   3692	} else {
   3693		clear_bit(QL_ASIC_RECOVERY, &qdev->flags);
   3694	}
   3695
   3696	qlge_write32(qdev, RST_FO, (RST_FO_FR << 16) | RST_FO_FR);
   3697
   3698	end_jiffies = jiffies + usecs_to_jiffies(30);
   3699	do {
   3700		value = qlge_read32(qdev, RST_FO);
   3701		if ((value & RST_FO_FR) == 0)
   3702			break;
   3703		cpu_relax();
   3704	} while (time_before(jiffies, end_jiffies));
   3705
   3706	if (value & RST_FO_FR) {
   3707		netif_err(qdev, ifdown, qdev->ndev,
   3708			  "ETIMEDOUT!!! errored out of resetting the chip!\n");
   3709		status = -ETIMEDOUT;
   3710	}
   3711
   3712	/* Resume management traffic. */
   3713	qlge_mb_set_mgmnt_traffic_ctl(qdev, MB_SET_MPI_TFK_RESUME);
   3714	return status;
   3715}
   3716
   3717static void qlge_display_dev_info(struct net_device *ndev)
   3718{
   3719	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   3720
   3721	netif_info(qdev, probe, qdev->ndev,
   3722		   "Function #%d, Port %d, NIC Roll %d, NIC Rev = %d, XG Roll = %d, XG Rev = %d.\n",
   3723		   qdev->func,
   3724		   qdev->port,
   3725		   qdev->chip_rev_id & 0x0000000f,
   3726		   qdev->chip_rev_id >> 4 & 0x0000000f,
   3727		   qdev->chip_rev_id >> 8 & 0x0000000f,
   3728		   qdev->chip_rev_id >> 12 & 0x0000000f);
   3729	netif_info(qdev, probe, qdev->ndev,
   3730		   "MAC address %pM\n", ndev->dev_addr);
   3731}
   3732
   3733static int qlge_wol(struct qlge_adapter *qdev)
   3734{
   3735	int status = 0;
   3736	u32 wol = MB_WOL_DISABLE;
   3737
   3738	/* The CAM is still intact after a reset, but if we
   3739	 * are doing WOL, then we may need to program the
   3740	 * routing regs. We would also need to issue the mailbox
   3741	 * commands to instruct the MPI what to do per the ethtool
   3742	 * settings.
   3743	 */
   3744
   3745	if (qdev->wol & (WAKE_ARP | WAKE_MAGICSECURE | WAKE_PHY | WAKE_UCAST |
   3746			 WAKE_MCAST | WAKE_BCAST)) {
   3747		netif_err(qdev, ifdown, qdev->ndev,
   3748			  "Unsupported WOL parameter. qdev->wol = 0x%x.\n",
   3749			  qdev->wol);
   3750		return -EINVAL;
   3751	}
   3752
   3753	if (qdev->wol & WAKE_MAGIC) {
   3754		status = qlge_mb_wol_set_magic(qdev, 1);
   3755		if (status) {
   3756			netif_err(qdev, ifdown, qdev->ndev,
   3757				  "Failed to set magic packet on %s.\n",
   3758				  qdev->ndev->name);
   3759			return status;
   3760		}
   3761		netif_info(qdev, drv, qdev->ndev,
   3762			   "Enabled magic packet successfully on %s.\n",
   3763			   qdev->ndev->name);
   3764
   3765		wol |= MB_WOL_MAGIC_PKT;
   3766	}
   3767
   3768	if (qdev->wol) {
   3769		wol |= MB_WOL_MODE_ON;
   3770		status = qlge_mb_wol_mode(qdev, wol);
   3771		netif_err(qdev, drv, qdev->ndev,
   3772			  "WOL %s (wol code 0x%x) on %s\n",
   3773			  (status == 0) ? "Successfully set" : "Failed",
   3774			  wol, qdev->ndev->name);
   3775	}
   3776
   3777	return status;
   3778}
   3779
   3780static void qlge_cancel_all_work_sync(struct qlge_adapter *qdev)
   3781{
   3782	/* Don't kill the reset worker thread if we
   3783	 * are in the process of recovery.
   3784	 */
   3785	if (test_bit(QL_ADAPTER_UP, &qdev->flags))
   3786		cancel_delayed_work_sync(&qdev->asic_reset_work);
   3787	cancel_delayed_work_sync(&qdev->mpi_reset_work);
   3788	cancel_delayed_work_sync(&qdev->mpi_work);
   3789	cancel_delayed_work_sync(&qdev->mpi_idc_work);
   3790	cancel_delayed_work_sync(&qdev->mpi_port_cfg_work);
   3791}
   3792
   3793static int qlge_adapter_down(struct qlge_adapter *qdev)
   3794{
   3795	int i, status = 0;
   3796
   3797	qlge_link_off(qdev);
   3798
   3799	qlge_cancel_all_work_sync(qdev);
   3800
   3801	for (i = 0; i < qdev->rss_ring_count; i++)
   3802		napi_disable(&qdev->rx_ring[i].napi);
   3803
   3804	clear_bit(QL_ADAPTER_UP, &qdev->flags);
   3805
   3806	qlge_disable_interrupts(qdev);
   3807
   3808	qlge_tx_ring_clean(qdev);
   3809
   3810	/* Call netif_napi_del() from common point. */
   3811	for (i = 0; i < qdev->rss_ring_count; i++)
   3812		netif_napi_del(&qdev->rx_ring[i].napi);
   3813
   3814	status = qlge_adapter_reset(qdev);
   3815	if (status)
   3816		netif_err(qdev, ifdown, qdev->ndev, "reset(func #%d) FAILED!\n",
   3817			  qdev->func);
   3818	qlge_free_rx_buffers(qdev);
   3819
   3820	return status;
   3821}
   3822
   3823static int qlge_adapter_up(struct qlge_adapter *qdev)
   3824{
   3825	int err = 0;
   3826
   3827	err = qlge_adapter_initialize(qdev);
   3828	if (err) {
   3829		netif_info(qdev, ifup, qdev->ndev, "Unable to initialize adapter.\n");
   3830		goto err_init;
   3831	}
   3832	set_bit(QL_ADAPTER_UP, &qdev->flags);
   3833	qlge_alloc_rx_buffers(qdev);
   3834	/* If the port is initialized and the
   3835	 * link is up the turn on the carrier.
   3836	 */
   3837	if ((qlge_read32(qdev, STS) & qdev->port_init) &&
   3838	    (qlge_read32(qdev, STS) & qdev->port_link_up))
   3839		qlge_link_on(qdev);
   3840	/* Restore rx mode. */
   3841	clear_bit(QL_ALLMULTI, &qdev->flags);
   3842	clear_bit(QL_PROMISCUOUS, &qdev->flags);
   3843	qlge_set_multicast_list(qdev->ndev);
   3844
   3845	/* Restore vlan setting. */
   3846	qlge_restore_vlan(qdev);
   3847
   3848	qlge_enable_interrupts(qdev);
   3849	qlge_enable_all_completion_interrupts(qdev);
   3850	netif_tx_start_all_queues(qdev->ndev);
   3851
   3852	return 0;
   3853err_init:
   3854	qlge_adapter_reset(qdev);
   3855	return err;
   3856}
   3857
   3858static void qlge_release_adapter_resources(struct qlge_adapter *qdev)
   3859{
   3860	qlge_free_mem_resources(qdev);
   3861	qlge_free_irq(qdev);
   3862}
   3863
   3864static int qlge_get_adapter_resources(struct qlge_adapter *qdev)
   3865{
   3866	if (qlge_alloc_mem_resources(qdev)) {
   3867		netif_err(qdev, ifup, qdev->ndev, "Unable to  allocate memory.\n");
   3868		return -ENOMEM;
   3869	}
   3870	return qlge_request_irq(qdev);
   3871}
   3872
   3873static int qlge_close(struct net_device *ndev)
   3874{
   3875	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   3876	int i;
   3877
   3878	/* If we hit pci_channel_io_perm_failure
   3879	 * failure condition, then we already
   3880	 * brought the adapter down.
   3881	 */
   3882	if (test_bit(QL_EEH_FATAL, &qdev->flags)) {
   3883		netif_err(qdev, drv, qdev->ndev, "EEH fatal did unload.\n");
   3884		clear_bit(QL_EEH_FATAL, &qdev->flags);
   3885		return 0;
   3886	}
   3887
   3888	/*
   3889	 * Wait for device to recover from a reset.
   3890	 * (Rarely happens, but possible.)
   3891	 */
   3892	while (!test_bit(QL_ADAPTER_UP, &qdev->flags))
   3893		msleep(1);
   3894
   3895	/* Make sure refill_work doesn't re-enable napi */
   3896	for (i = 0; i < qdev->rss_ring_count; i++)
   3897		cancel_delayed_work_sync(&qdev->rx_ring[i].refill_work);
   3898
   3899	qlge_adapter_down(qdev);
   3900	qlge_release_adapter_resources(qdev);
   3901	return 0;
   3902}
   3903
   3904static void qlge_set_lb_size(struct qlge_adapter *qdev)
   3905{
   3906	if (qdev->ndev->mtu <= 1500)
   3907		qdev->lbq_buf_size = LARGE_BUFFER_MIN_SIZE;
   3908	else
   3909		qdev->lbq_buf_size = LARGE_BUFFER_MAX_SIZE;
   3910	qdev->lbq_buf_order = get_order(qdev->lbq_buf_size);
   3911}
   3912
   3913static int qlge_configure_rings(struct qlge_adapter *qdev)
   3914{
   3915	int i;
   3916	struct rx_ring *rx_ring;
   3917	struct tx_ring *tx_ring;
   3918	int cpu_cnt = min_t(int, MAX_CPUS, num_online_cpus());
   3919
   3920	/* In a perfect world we have one RSS ring for each CPU
   3921	 * and each has it's own vector.  To do that we ask for
   3922	 * cpu_cnt vectors.  qlge_enable_msix() will adjust the
   3923	 * vector count to what we actually get.  We then
   3924	 * allocate an RSS ring for each.
   3925	 * Essentially, we are doing min(cpu_count, msix_vector_count).
   3926	 */
   3927	qdev->intr_count = cpu_cnt;
   3928	qlge_enable_msix(qdev);
   3929	/* Adjust the RSS ring count to the actual vector count. */
   3930	qdev->rss_ring_count = qdev->intr_count;
   3931	qdev->tx_ring_count = cpu_cnt;
   3932	qdev->rx_ring_count = qdev->tx_ring_count + qdev->rss_ring_count;
   3933
   3934	for (i = 0; i < qdev->tx_ring_count; i++) {
   3935		tx_ring = &qdev->tx_ring[i];
   3936		memset((void *)tx_ring, 0, sizeof(*tx_ring));
   3937		tx_ring->qdev = qdev;
   3938		tx_ring->wq_id = i;
   3939		tx_ring->wq_len = qdev->tx_ring_size;
   3940		tx_ring->wq_size =
   3941			tx_ring->wq_len * sizeof(struct qlge_ob_mac_iocb_req);
   3942
   3943		/*
   3944		 * The completion queue ID for the tx rings start
   3945		 * immediately after the rss rings.
   3946		 */
   3947		tx_ring->cq_id = qdev->rss_ring_count + i;
   3948	}
   3949
   3950	for (i = 0; i < qdev->rx_ring_count; i++) {
   3951		rx_ring = &qdev->rx_ring[i];
   3952		memset((void *)rx_ring, 0, sizeof(*rx_ring));
   3953		rx_ring->qdev = qdev;
   3954		rx_ring->cq_id = i;
   3955		rx_ring->cpu = i % cpu_cnt;	/* CPU to run handler on. */
   3956		if (i < qdev->rss_ring_count) {
   3957			/*
   3958			 * Inbound (RSS) queues.
   3959			 */
   3960			rx_ring->cq_len = qdev->rx_ring_size;
   3961			rx_ring->cq_size =
   3962				rx_ring->cq_len * sizeof(struct qlge_net_rsp_iocb);
   3963			rx_ring->lbq.type = QLGE_LB;
   3964			rx_ring->sbq.type = QLGE_SB;
   3965			INIT_DELAYED_WORK(&rx_ring->refill_work,
   3966					  &qlge_slow_refill);
   3967		} else {
   3968			/*
   3969			 * Outbound queue handles outbound completions only.
   3970			 */
   3971			/* outbound cq is same size as tx_ring it services. */
   3972			rx_ring->cq_len = qdev->tx_ring_size;
   3973			rx_ring->cq_size =
   3974				rx_ring->cq_len * sizeof(struct qlge_net_rsp_iocb);
   3975		}
   3976	}
   3977	return 0;
   3978}
   3979
   3980static int qlge_open(struct net_device *ndev)
   3981{
   3982	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   3983	int err = 0;
   3984
   3985	err = qlge_adapter_reset(qdev);
   3986	if (err)
   3987		return err;
   3988
   3989	qlge_set_lb_size(qdev);
   3990	err = qlge_configure_rings(qdev);
   3991	if (err)
   3992		return err;
   3993
   3994	err = qlge_get_adapter_resources(qdev);
   3995	if (err)
   3996		goto error_up;
   3997
   3998	err = qlge_adapter_up(qdev);
   3999	if (err)
   4000		goto error_up;
   4001
   4002	return err;
   4003
   4004error_up:
   4005	qlge_release_adapter_resources(qdev);
   4006	return err;
   4007}
   4008
   4009static int qlge_change_rx_buffers(struct qlge_adapter *qdev)
   4010{
   4011	int status;
   4012
   4013	/* Wait for an outstanding reset to complete. */
   4014	if (!test_bit(QL_ADAPTER_UP, &qdev->flags)) {
   4015		int i = 4;
   4016
   4017		while (--i && !test_bit(QL_ADAPTER_UP, &qdev->flags)) {
   4018			netif_err(qdev, ifup, qdev->ndev,
   4019				  "Waiting for adapter UP...\n");
   4020			ssleep(1);
   4021		}
   4022
   4023		if (!i) {
   4024			netif_err(qdev, ifup, qdev->ndev,
   4025				  "Timed out waiting for adapter UP\n");
   4026			return -ETIMEDOUT;
   4027		}
   4028	}
   4029
   4030	status = qlge_adapter_down(qdev);
   4031	if (status)
   4032		goto error;
   4033
   4034	qlge_set_lb_size(qdev);
   4035
   4036	status = qlge_adapter_up(qdev);
   4037	if (status)
   4038		goto error;
   4039
   4040	return status;
   4041error:
   4042	netif_alert(qdev, ifup, qdev->ndev,
   4043		    "Driver up/down cycle failed, closing device.\n");
   4044	set_bit(QL_ADAPTER_UP, &qdev->flags);
   4045	dev_close(qdev->ndev);
   4046	return status;
   4047}
   4048
   4049static int qlge_change_mtu(struct net_device *ndev, int new_mtu)
   4050{
   4051	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   4052	int status;
   4053
   4054	if (ndev->mtu == 1500 && new_mtu == 9000)
   4055		netif_err(qdev, ifup, qdev->ndev, "Changing to jumbo MTU.\n");
   4056	else if (ndev->mtu == 9000 && new_mtu == 1500)
   4057		netif_err(qdev, ifup, qdev->ndev, "Changing to normal MTU.\n");
   4058	else
   4059		return -EINVAL;
   4060
   4061	queue_delayed_work(qdev->workqueue,
   4062			   &qdev->mpi_port_cfg_work, 3 * HZ);
   4063
   4064	ndev->mtu = new_mtu;
   4065
   4066	if (!netif_running(qdev->ndev))
   4067		return 0;
   4068
   4069	status = qlge_change_rx_buffers(qdev);
   4070	if (status) {
   4071		netif_err(qdev, ifup, qdev->ndev,
   4072			  "Changing MTU failed.\n");
   4073	}
   4074
   4075	return status;
   4076}
   4077
   4078static struct net_device_stats *qlge_get_stats(struct net_device
   4079					       *ndev)
   4080{
   4081	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   4082	struct rx_ring *rx_ring = &qdev->rx_ring[0];
   4083	struct tx_ring *tx_ring = &qdev->tx_ring[0];
   4084	unsigned long pkts, mcast, dropped, errors, bytes;
   4085	int i;
   4086
   4087	/* Get RX stats. */
   4088	pkts = mcast = dropped = errors = bytes = 0;
   4089	for (i = 0; i < qdev->rss_ring_count; i++, rx_ring++) {
   4090		pkts += rx_ring->rx_packets;
   4091		bytes += rx_ring->rx_bytes;
   4092		dropped += rx_ring->rx_dropped;
   4093		errors += rx_ring->rx_errors;
   4094		mcast += rx_ring->rx_multicast;
   4095	}
   4096	ndev->stats.rx_packets = pkts;
   4097	ndev->stats.rx_bytes = bytes;
   4098	ndev->stats.rx_dropped = dropped;
   4099	ndev->stats.rx_errors = errors;
   4100	ndev->stats.multicast = mcast;
   4101
   4102	/* Get TX stats. */
   4103	pkts = errors = bytes = 0;
   4104	for (i = 0; i < qdev->tx_ring_count; i++, tx_ring++) {
   4105		pkts += tx_ring->tx_packets;
   4106		bytes += tx_ring->tx_bytes;
   4107		errors += tx_ring->tx_errors;
   4108	}
   4109	ndev->stats.tx_packets = pkts;
   4110	ndev->stats.tx_bytes = bytes;
   4111	ndev->stats.tx_errors = errors;
   4112	return &ndev->stats;
   4113}
   4114
   4115static void qlge_set_multicast_list(struct net_device *ndev)
   4116{
   4117	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   4118	struct netdev_hw_addr *ha;
   4119	int i, status;
   4120
   4121	status = qlge_sem_spinlock(qdev, SEM_RT_IDX_MASK);
   4122	if (status)
   4123		return;
   4124	/*
   4125	 * Set or clear promiscuous mode if a
   4126	 * transition is taking place.
   4127	 */
   4128	if (ndev->flags & IFF_PROMISC) {
   4129		if (!test_bit(QL_PROMISCUOUS, &qdev->flags)) {
   4130			if (qlge_set_routing_reg
   4131			    (qdev, RT_IDX_PROMISCUOUS_SLOT, RT_IDX_VALID, 1)) {
   4132				netif_err(qdev, hw, qdev->ndev,
   4133					  "Failed to set promiscuous mode.\n");
   4134			} else {
   4135				set_bit(QL_PROMISCUOUS, &qdev->flags);
   4136			}
   4137		}
   4138	} else {
   4139		if (test_bit(QL_PROMISCUOUS, &qdev->flags)) {
   4140			if (qlge_set_routing_reg
   4141			    (qdev, RT_IDX_PROMISCUOUS_SLOT, RT_IDX_VALID, 0)) {
   4142				netif_err(qdev, hw, qdev->ndev,
   4143					  "Failed to clear promiscuous mode.\n");
   4144			} else {
   4145				clear_bit(QL_PROMISCUOUS, &qdev->flags);
   4146			}
   4147		}
   4148	}
   4149
   4150	/*
   4151	 * Set or clear all multicast mode if a
   4152	 * transition is taking place.
   4153	 */
   4154	if ((ndev->flags & IFF_ALLMULTI) ||
   4155	    (netdev_mc_count(ndev) > MAX_MULTICAST_ENTRIES)) {
   4156		if (!test_bit(QL_ALLMULTI, &qdev->flags)) {
   4157			if (qlge_set_routing_reg
   4158			    (qdev, RT_IDX_ALLMULTI_SLOT, RT_IDX_MCAST, 1)) {
   4159				netif_err(qdev, hw, qdev->ndev,
   4160					  "Failed to set all-multi mode.\n");
   4161			} else {
   4162				set_bit(QL_ALLMULTI, &qdev->flags);
   4163			}
   4164		}
   4165	} else {
   4166		if (test_bit(QL_ALLMULTI, &qdev->flags)) {
   4167			if (qlge_set_routing_reg
   4168			    (qdev, RT_IDX_ALLMULTI_SLOT, RT_IDX_MCAST, 0)) {
   4169				netif_err(qdev, hw, qdev->ndev,
   4170					  "Failed to clear all-multi mode.\n");
   4171			} else {
   4172				clear_bit(QL_ALLMULTI, &qdev->flags);
   4173			}
   4174		}
   4175	}
   4176
   4177	if (!netdev_mc_empty(ndev)) {
   4178		status = qlge_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
   4179		if (status)
   4180			goto exit;
   4181		i = 0;
   4182		netdev_for_each_mc_addr(ha, ndev) {
   4183			if (qlge_set_mac_addr_reg(qdev, (u8 *)ha->addr,
   4184						  MAC_ADDR_TYPE_MULTI_MAC, i)) {
   4185				netif_err(qdev, hw, qdev->ndev,
   4186					  "Failed to loadmulticast address.\n");
   4187				qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
   4188				goto exit;
   4189			}
   4190			i++;
   4191		}
   4192		qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
   4193		if (qlge_set_routing_reg
   4194		    (qdev, RT_IDX_MCAST_MATCH_SLOT, RT_IDX_MCAST_MATCH, 1)) {
   4195			netif_err(qdev, hw, qdev->ndev,
   4196				  "Failed to set multicast match mode.\n");
   4197		} else {
   4198			set_bit(QL_ALLMULTI, &qdev->flags);
   4199		}
   4200	}
   4201exit:
   4202	qlge_sem_unlock(qdev, SEM_RT_IDX_MASK);
   4203}
   4204
   4205static int qlge_set_mac_address(struct net_device *ndev, void *p)
   4206{
   4207	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   4208	struct sockaddr *addr = p;
   4209	int status;
   4210
   4211	if (!is_valid_ether_addr(addr->sa_data))
   4212		return -EADDRNOTAVAIL;
   4213	eth_hw_addr_set(ndev, addr->sa_data);
   4214	/* Update local copy of current mac address. */
   4215	memcpy(qdev->current_mac_addr, ndev->dev_addr, ndev->addr_len);
   4216
   4217	status = qlge_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
   4218	if (status)
   4219		return status;
   4220	status = qlge_set_mac_addr_reg(qdev, (const u8 *)ndev->dev_addr,
   4221				       MAC_ADDR_TYPE_CAM_MAC,
   4222				       qdev->func * MAX_CQ);
   4223	if (status)
   4224		netif_err(qdev, hw, qdev->ndev, "Failed to load MAC address.\n");
   4225	qlge_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
   4226	return status;
   4227}
   4228
   4229static void qlge_tx_timeout(struct net_device *ndev, unsigned int txqueue)
   4230{
   4231	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   4232
   4233	qlge_queue_asic_error(qdev);
   4234}
   4235
   4236static void qlge_asic_reset_work(struct work_struct *work)
   4237{
   4238	struct qlge_adapter *qdev =
   4239		container_of(work, struct qlge_adapter, asic_reset_work.work);
   4240	int status;
   4241
   4242	rtnl_lock();
   4243	status = qlge_adapter_down(qdev);
   4244	if (status)
   4245		goto error;
   4246
   4247	status = qlge_adapter_up(qdev);
   4248	if (status)
   4249		goto error;
   4250
   4251	/* Restore rx mode. */
   4252	clear_bit(QL_ALLMULTI, &qdev->flags);
   4253	clear_bit(QL_PROMISCUOUS, &qdev->flags);
   4254	qlge_set_multicast_list(qdev->ndev);
   4255
   4256	rtnl_unlock();
   4257	return;
   4258error:
   4259	netif_alert(qdev, ifup, qdev->ndev,
   4260		    "Driver up/down cycle failed, closing device\n");
   4261
   4262	set_bit(QL_ADAPTER_UP, &qdev->flags);
   4263	dev_close(qdev->ndev);
   4264	rtnl_unlock();
   4265}
   4266
   4267static const struct nic_operations qla8012_nic_ops = {
   4268	.get_flash		= qlge_get_8012_flash_params,
   4269	.port_initialize	= qlge_8012_port_initialize,
   4270};
   4271
   4272static const struct nic_operations qla8000_nic_ops = {
   4273	.get_flash		= qlge_get_8000_flash_params,
   4274	.port_initialize	= qlge_8000_port_initialize,
   4275};
   4276
   4277/* Find the pcie function number for the other NIC
   4278 * on this chip.  Since both NIC functions share a
   4279 * common firmware we have the lowest enabled function
   4280 * do any common work.  Examples would be resetting
   4281 * after a fatal firmware error, or doing a firmware
   4282 * coredump.
   4283 */
   4284static int qlge_get_alt_pcie_func(struct qlge_adapter *qdev)
   4285{
   4286	int status = 0;
   4287	u32 temp;
   4288	u32 nic_func1, nic_func2;
   4289
   4290	status = qlge_read_mpi_reg(qdev, MPI_TEST_FUNC_PORT_CFG,
   4291				   &temp);
   4292	if (status)
   4293		return status;
   4294
   4295	nic_func1 = ((temp >> MPI_TEST_NIC1_FUNC_SHIFT) &
   4296		     MPI_TEST_NIC_FUNC_MASK);
   4297	nic_func2 = ((temp >> MPI_TEST_NIC2_FUNC_SHIFT) &
   4298		     MPI_TEST_NIC_FUNC_MASK);
   4299
   4300	if (qdev->func == nic_func1)
   4301		qdev->alt_func = nic_func2;
   4302	else if (qdev->func == nic_func2)
   4303		qdev->alt_func = nic_func1;
   4304	else
   4305		status = -EIO;
   4306
   4307	return status;
   4308}
   4309
   4310static int qlge_get_board_info(struct qlge_adapter *qdev)
   4311{
   4312	int status;
   4313
   4314	qdev->func =
   4315		(qlge_read32(qdev, STS) & STS_FUNC_ID_MASK) >> STS_FUNC_ID_SHIFT;
   4316	if (qdev->func > 3)
   4317		return -EIO;
   4318
   4319	status = qlge_get_alt_pcie_func(qdev);
   4320	if (status)
   4321		return status;
   4322
   4323	qdev->port = (qdev->func < qdev->alt_func) ? 0 : 1;
   4324	if (qdev->port) {
   4325		qdev->xg_sem_mask = SEM_XGMAC1_MASK;
   4326		qdev->port_link_up = STS_PL1;
   4327		qdev->port_init = STS_PI1;
   4328		qdev->mailbox_in = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC2_MBI;
   4329		qdev->mailbox_out = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC2_MBO;
   4330	} else {
   4331		qdev->xg_sem_mask = SEM_XGMAC0_MASK;
   4332		qdev->port_link_up = STS_PL0;
   4333		qdev->port_init = STS_PI0;
   4334		qdev->mailbox_in = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC0_MBI;
   4335		qdev->mailbox_out = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC0_MBO;
   4336	}
   4337	qdev->chip_rev_id = qlge_read32(qdev, REV_ID);
   4338	qdev->device_id = qdev->pdev->device;
   4339	if (qdev->device_id == QLGE_DEVICE_ID_8012)
   4340		qdev->nic_ops = &qla8012_nic_ops;
   4341	else if (qdev->device_id == QLGE_DEVICE_ID_8000)
   4342		qdev->nic_ops = &qla8000_nic_ops;
   4343	return status;
   4344}
   4345
   4346static void qlge_release_all(struct pci_dev *pdev)
   4347{
   4348	struct qlge_adapter *qdev = pci_get_drvdata(pdev);
   4349
   4350	if (qdev->workqueue) {
   4351		destroy_workqueue(qdev->workqueue);
   4352		qdev->workqueue = NULL;
   4353	}
   4354
   4355	if (qdev->reg_base)
   4356		iounmap(qdev->reg_base);
   4357	if (qdev->doorbell_area)
   4358		iounmap(qdev->doorbell_area);
   4359	vfree(qdev->mpi_coredump);
   4360	pci_release_regions(pdev);
   4361}
   4362
   4363static int qlge_init_device(struct pci_dev *pdev, struct qlge_adapter *qdev,
   4364			    int cards_found)
   4365{
   4366	struct net_device *ndev = qdev->ndev;
   4367	int err = 0;
   4368
   4369	err = pci_enable_device(pdev);
   4370	if (err) {
   4371		dev_err(&pdev->dev, "PCI device enable failed.\n");
   4372		return err;
   4373	}
   4374
   4375	qdev->pdev = pdev;
   4376	pci_set_drvdata(pdev, qdev);
   4377
   4378	/* Set PCIe read request size */
   4379	err = pcie_set_readrq(pdev, 4096);
   4380	if (err) {
   4381		dev_err(&pdev->dev, "Set readrq failed.\n");
   4382		goto err_disable_pci;
   4383	}
   4384
   4385	err = pci_request_regions(pdev, DRV_NAME);
   4386	if (err) {
   4387		dev_err(&pdev->dev, "PCI region request failed.\n");
   4388		goto err_disable_pci;
   4389	}
   4390
   4391	pci_set_master(pdev);
   4392	if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) {
   4393		set_bit(QL_DMA64, &qdev->flags);
   4394		err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
   4395	} else {
   4396		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
   4397		if (!err)
   4398			err = dma_set_coherent_mask(&pdev->dev,
   4399						    DMA_BIT_MASK(32));
   4400	}
   4401
   4402	if (err) {
   4403		dev_err(&pdev->dev, "No usable DMA configuration.\n");
   4404		goto err_release_pci;
   4405	}
   4406
   4407	/* Set PCIe reset type for EEH to fundamental. */
   4408	pdev->needs_freset = 1;
   4409	pci_save_state(pdev);
   4410	qdev->reg_base =
   4411		ioremap(pci_resource_start(pdev, 1), pci_resource_len(pdev, 1));
   4412	if (!qdev->reg_base) {
   4413		dev_err(&pdev->dev, "Register mapping failed.\n");
   4414		err = -ENOMEM;
   4415		goto err_release_pci;
   4416	}
   4417
   4418	qdev->doorbell_area_size = pci_resource_len(pdev, 3);
   4419	qdev->doorbell_area =
   4420		ioremap(pci_resource_start(pdev, 3), pci_resource_len(pdev, 3));
   4421	if (!qdev->doorbell_area) {
   4422		dev_err(&pdev->dev, "Doorbell register mapping failed.\n");
   4423		err = -ENOMEM;
   4424		goto err_iounmap_base;
   4425	}
   4426
   4427	err = qlge_get_board_info(qdev);
   4428	if (err) {
   4429		dev_err(&pdev->dev, "Register access failed.\n");
   4430		err = -EIO;
   4431		goto err_iounmap_doorbell;
   4432	}
   4433	qdev->msg_enable = netif_msg_init(debug, default_msg);
   4434	spin_lock_init(&qdev->stats_lock);
   4435
   4436	if (qlge_mpi_coredump) {
   4437		qdev->mpi_coredump =
   4438			vmalloc(sizeof(struct qlge_mpi_coredump));
   4439		if (!qdev->mpi_coredump) {
   4440			err = -ENOMEM;
   4441			goto err_iounmap_doorbell;
   4442		}
   4443		if (qlge_force_coredump)
   4444			set_bit(QL_FRC_COREDUMP, &qdev->flags);
   4445	}
   4446	/* make sure the EEPROM is good */
   4447	err = qdev->nic_ops->get_flash(qdev);
   4448	if (err) {
   4449		dev_err(&pdev->dev, "Invalid FLASH.\n");
   4450		goto err_free_mpi_coredump;
   4451	}
   4452
   4453	/* Keep local copy of current mac address. */
   4454	memcpy(qdev->current_mac_addr, ndev->dev_addr, ndev->addr_len);
   4455
   4456	/* Set up the default ring sizes. */
   4457	qdev->tx_ring_size = NUM_TX_RING_ENTRIES;
   4458	qdev->rx_ring_size = NUM_RX_RING_ENTRIES;
   4459
   4460	/* Set up the coalescing parameters. */
   4461	qdev->rx_coalesce_usecs = DFLT_COALESCE_WAIT;
   4462	qdev->tx_coalesce_usecs = DFLT_COALESCE_WAIT;
   4463	qdev->rx_max_coalesced_frames = DFLT_INTER_FRAME_WAIT;
   4464	qdev->tx_max_coalesced_frames = DFLT_INTER_FRAME_WAIT;
   4465
   4466	/*
   4467	 * Set up the operating parameters.
   4468	 */
   4469	qdev->workqueue = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,
   4470						  ndev->name);
   4471	if (!qdev->workqueue) {
   4472		err = -ENOMEM;
   4473		goto err_free_mpi_coredump;
   4474	}
   4475
   4476	INIT_DELAYED_WORK(&qdev->asic_reset_work, qlge_asic_reset_work);
   4477	INIT_DELAYED_WORK(&qdev->mpi_reset_work, qlge_mpi_reset_work);
   4478	INIT_DELAYED_WORK(&qdev->mpi_work, qlge_mpi_work);
   4479	INIT_DELAYED_WORK(&qdev->mpi_port_cfg_work, qlge_mpi_port_cfg_work);
   4480	INIT_DELAYED_WORK(&qdev->mpi_idc_work, qlge_mpi_idc_work);
   4481	init_completion(&qdev->ide_completion);
   4482	mutex_init(&qdev->mpi_mutex);
   4483
   4484	if (!cards_found) {
   4485		dev_info(&pdev->dev, "%s\n", DRV_STRING);
   4486		dev_info(&pdev->dev, "Driver name: %s, Version: %s.\n",
   4487			 DRV_NAME, DRV_VERSION);
   4488	}
   4489	return 0;
   4490
   4491err_free_mpi_coredump:
   4492	vfree(qdev->mpi_coredump);
   4493err_iounmap_doorbell:
   4494	iounmap(qdev->doorbell_area);
   4495err_iounmap_base:
   4496	iounmap(qdev->reg_base);
   4497err_release_pci:
   4498	pci_release_regions(pdev);
   4499err_disable_pci:
   4500	pci_disable_device(pdev);
   4501
   4502	return err;
   4503}
   4504
   4505static const struct net_device_ops qlge_netdev_ops = {
   4506	.ndo_open		= qlge_open,
   4507	.ndo_stop		= qlge_close,
   4508	.ndo_start_xmit		= qlge_send,
   4509	.ndo_change_mtu		= qlge_change_mtu,
   4510	.ndo_get_stats		= qlge_get_stats,
   4511	.ndo_set_rx_mode	= qlge_set_multicast_list,
   4512	.ndo_set_mac_address	= qlge_set_mac_address,
   4513	.ndo_validate_addr	= eth_validate_addr,
   4514	.ndo_tx_timeout		= qlge_tx_timeout,
   4515	.ndo_set_features	= qlge_set_features,
   4516	.ndo_vlan_rx_add_vid	= qlge_vlan_rx_add_vid,
   4517	.ndo_vlan_rx_kill_vid	= qlge_vlan_rx_kill_vid,
   4518};
   4519
   4520static void qlge_timer(struct timer_list *t)
   4521{
   4522	struct qlge_adapter *qdev = from_timer(qdev, t, timer);
   4523	u32 var = 0;
   4524
   4525	var = qlge_read32(qdev, STS);
   4526	if (pci_channel_offline(qdev->pdev)) {
   4527		netif_err(qdev, ifup, qdev->ndev, "EEH STS = 0x%.08x.\n", var);
   4528		return;
   4529	}
   4530
   4531	mod_timer(&qdev->timer, jiffies + (5 * HZ));
   4532}
   4533
   4534static const struct devlink_ops qlge_devlink_ops;
   4535
   4536static int qlge_probe(struct pci_dev *pdev,
   4537		      const struct pci_device_id *pci_entry)
   4538{
   4539	struct qlge_netdev_priv *ndev_priv;
   4540	struct qlge_adapter *qdev = NULL;
   4541	struct net_device *ndev = NULL;
   4542	struct devlink *devlink;
   4543	static int cards_found;
   4544	int err;
   4545
   4546	devlink = devlink_alloc(&qlge_devlink_ops, sizeof(struct qlge_adapter),
   4547				&pdev->dev);
   4548	if (!devlink)
   4549		return -ENOMEM;
   4550
   4551	qdev = devlink_priv(devlink);
   4552
   4553	ndev = alloc_etherdev_mq(sizeof(struct qlge_netdev_priv),
   4554				 min(MAX_CPUS,
   4555				     netif_get_num_default_rss_queues()));
   4556	if (!ndev) {
   4557		err = -ENOMEM;
   4558		goto devlink_free;
   4559	}
   4560
   4561	ndev_priv = netdev_priv(ndev);
   4562	ndev_priv->qdev = qdev;
   4563	ndev_priv->ndev = ndev;
   4564	qdev->ndev = ndev;
   4565	err = qlge_init_device(pdev, qdev, cards_found);
   4566	if (err < 0)
   4567		goto netdev_free;
   4568
   4569	SET_NETDEV_DEV(ndev, &pdev->dev);
   4570	ndev->hw_features = NETIF_F_SG |
   4571		NETIF_F_IP_CSUM |
   4572		NETIF_F_TSO |
   4573		NETIF_F_TSO_ECN |
   4574		NETIF_F_HW_VLAN_CTAG_TX |
   4575		NETIF_F_HW_VLAN_CTAG_RX |
   4576		NETIF_F_HW_VLAN_CTAG_FILTER |
   4577		NETIF_F_RXCSUM;
   4578	ndev->features = ndev->hw_features;
   4579	ndev->vlan_features = ndev->hw_features;
   4580	/* vlan gets same features (except vlan filter) */
   4581	ndev->vlan_features &= ~(NETIF_F_HW_VLAN_CTAG_FILTER |
   4582				 NETIF_F_HW_VLAN_CTAG_TX |
   4583				 NETIF_F_HW_VLAN_CTAG_RX);
   4584
   4585	if (test_bit(QL_DMA64, &qdev->flags))
   4586		ndev->features |= NETIF_F_HIGHDMA;
   4587
   4588	/*
   4589	 * Set up net_device structure.
   4590	 */
   4591	ndev->tx_queue_len = qdev->tx_ring_size;
   4592	ndev->irq = pdev->irq;
   4593
   4594	ndev->netdev_ops = &qlge_netdev_ops;
   4595	ndev->ethtool_ops = &qlge_ethtool_ops;
   4596	ndev->watchdog_timeo = 10 * HZ;
   4597
   4598	/* MTU range: this driver only supports 1500 or 9000, so this only
   4599	 * filters out values above or below, and we'll rely on
   4600	 * qlge_change_mtu to make sure only 1500 or 9000 are allowed
   4601	 */
   4602	ndev->min_mtu = ETH_DATA_LEN;
   4603	ndev->max_mtu = 9000;
   4604
   4605	err = register_netdev(ndev);
   4606	if (err) {
   4607		dev_err(&pdev->dev, "net device registration failed.\n");
   4608		goto cleanup_pdev;
   4609	}
   4610
   4611	err = qlge_health_create_reporters(qdev);
   4612	if (err)
   4613		goto unregister_netdev;
   4614
   4615	/* Start up the timer to trigger EEH if
   4616	 * the bus goes dead
   4617	 */
   4618	timer_setup(&qdev->timer, qlge_timer, TIMER_DEFERRABLE);
   4619	mod_timer(&qdev->timer, jiffies + (5 * HZ));
   4620	qlge_link_off(qdev);
   4621	qlge_display_dev_info(ndev);
   4622	atomic_set(&qdev->lb_count, 0);
   4623	cards_found++;
   4624	devlink_register(devlink);
   4625	return 0;
   4626
   4627unregister_netdev:
   4628	unregister_netdev(ndev);
   4629cleanup_pdev:
   4630	qlge_release_all(pdev);
   4631	pci_disable_device(pdev);
   4632netdev_free:
   4633	free_netdev(ndev);
   4634devlink_free:
   4635	devlink_free(devlink);
   4636
   4637	return err;
   4638}
   4639
   4640netdev_tx_t qlge_lb_send(struct sk_buff *skb, struct net_device *ndev)
   4641{
   4642	return qlge_send(skb, ndev);
   4643}
   4644
   4645int qlge_clean_lb_rx_ring(struct rx_ring *rx_ring, int budget)
   4646{
   4647	return qlge_clean_inbound_rx_ring(rx_ring, budget);
   4648}
   4649
   4650static void qlge_remove(struct pci_dev *pdev)
   4651{
   4652	struct qlge_adapter *qdev = pci_get_drvdata(pdev);
   4653	struct net_device *ndev = qdev->ndev;
   4654	struct devlink *devlink = priv_to_devlink(qdev);
   4655
   4656	devlink_unregister(devlink);
   4657	del_timer_sync(&qdev->timer);
   4658	qlge_cancel_all_work_sync(qdev);
   4659	unregister_netdev(ndev);
   4660	qlge_release_all(pdev);
   4661	pci_disable_device(pdev);
   4662	devlink_health_reporter_destroy(qdev->reporter);
   4663	devlink_free(devlink);
   4664	free_netdev(ndev);
   4665}
   4666
   4667/* Clean up resources without touching hardware. */
   4668static void qlge_eeh_close(struct net_device *ndev)
   4669{
   4670	struct qlge_adapter *qdev = netdev_to_qdev(ndev);
   4671	int i;
   4672
   4673	if (netif_carrier_ok(ndev)) {
   4674		netif_carrier_off(ndev);
   4675		netif_stop_queue(ndev);
   4676	}
   4677
   4678	/* Disabling the timer */
   4679	qlge_cancel_all_work_sync(qdev);
   4680
   4681	for (i = 0; i < qdev->rss_ring_count; i++)
   4682		netif_napi_del(&qdev->rx_ring[i].napi);
   4683
   4684	clear_bit(QL_ADAPTER_UP, &qdev->flags);
   4685	qlge_tx_ring_clean(qdev);
   4686	qlge_free_rx_buffers(qdev);
   4687	qlge_release_adapter_resources(qdev);
   4688}
   4689
   4690/*
   4691 * This callback is called by the PCI subsystem whenever
   4692 * a PCI bus error is detected.
   4693 */
   4694static pci_ers_result_t qlge_io_error_detected(struct pci_dev *pdev,
   4695					       pci_channel_state_t state)
   4696{
   4697	struct qlge_adapter *qdev = pci_get_drvdata(pdev);
   4698	struct net_device *ndev = qdev->ndev;
   4699
   4700	switch (state) {
   4701	case pci_channel_io_normal:
   4702		return PCI_ERS_RESULT_CAN_RECOVER;
   4703	case pci_channel_io_frozen:
   4704		netif_device_detach(ndev);
   4705		del_timer_sync(&qdev->timer);
   4706		if (netif_running(ndev))
   4707			qlge_eeh_close(ndev);
   4708		pci_disable_device(pdev);
   4709		return PCI_ERS_RESULT_NEED_RESET;
   4710	case pci_channel_io_perm_failure:
   4711		dev_err(&pdev->dev,
   4712			"%s: pci_channel_io_perm_failure.\n", __func__);
   4713		del_timer_sync(&qdev->timer);
   4714		qlge_eeh_close(ndev);
   4715		set_bit(QL_EEH_FATAL, &qdev->flags);
   4716		return PCI_ERS_RESULT_DISCONNECT;
   4717	}
   4718
   4719	/* Request a slot reset. */
   4720	return PCI_ERS_RESULT_NEED_RESET;
   4721}
   4722
   4723/*
   4724 * This callback is called after the PCI buss has been reset.
   4725 * Basically, this tries to restart the card from scratch.
   4726 * This is a shortened version of the device probe/discovery code,
   4727 * it resembles the first-half of the () routine.
   4728 */
   4729static pci_ers_result_t qlge_io_slot_reset(struct pci_dev *pdev)
   4730{
   4731	struct qlge_adapter *qdev = pci_get_drvdata(pdev);
   4732
   4733	pdev->error_state = pci_channel_io_normal;
   4734
   4735	pci_restore_state(pdev);
   4736	if (pci_enable_device(pdev)) {
   4737		netif_err(qdev, ifup, qdev->ndev,
   4738			  "Cannot re-enable PCI device after reset.\n");
   4739		return PCI_ERS_RESULT_DISCONNECT;
   4740	}
   4741	pci_set_master(pdev);
   4742
   4743	if (qlge_adapter_reset(qdev)) {
   4744		netif_err(qdev, drv, qdev->ndev, "reset FAILED!\n");
   4745		set_bit(QL_EEH_FATAL, &qdev->flags);
   4746		return PCI_ERS_RESULT_DISCONNECT;
   4747	}
   4748
   4749	return PCI_ERS_RESULT_RECOVERED;
   4750}
   4751
   4752static void qlge_io_resume(struct pci_dev *pdev)
   4753{
   4754	struct qlge_adapter *qdev = pci_get_drvdata(pdev);
   4755	struct net_device *ndev = qdev->ndev;
   4756	int err = 0;
   4757
   4758	if (netif_running(ndev)) {
   4759		err = qlge_open(ndev);
   4760		if (err) {
   4761			netif_err(qdev, ifup, qdev->ndev,
   4762				  "Device initialization failed after reset.\n");
   4763			return;
   4764		}
   4765	} else {
   4766		netif_err(qdev, ifup, qdev->ndev,
   4767			  "Device was not running prior to EEH.\n");
   4768	}
   4769	mod_timer(&qdev->timer, jiffies + (5 * HZ));
   4770	netif_device_attach(ndev);
   4771}
   4772
   4773static const struct pci_error_handlers qlge_err_handler = {
   4774	.error_detected = qlge_io_error_detected,
   4775	.slot_reset = qlge_io_slot_reset,
   4776	.resume = qlge_io_resume,
   4777};
   4778
   4779static int __maybe_unused qlge_suspend(struct device *dev_d)
   4780{
   4781	struct pci_dev *pdev = to_pci_dev(dev_d);
   4782	struct qlge_adapter *qdev;
   4783	struct net_device *ndev;
   4784	int err;
   4785
   4786	qdev = pci_get_drvdata(pdev);
   4787	ndev = qdev->ndev;
   4788	netif_device_detach(ndev);
   4789	del_timer_sync(&qdev->timer);
   4790
   4791	if (netif_running(ndev)) {
   4792		err = qlge_adapter_down(qdev);
   4793		if (!err)
   4794			return err;
   4795	}
   4796
   4797	qlge_wol(qdev);
   4798
   4799	return 0;
   4800}
   4801
   4802static int __maybe_unused qlge_resume(struct device *dev_d)
   4803{
   4804	struct pci_dev *pdev = to_pci_dev(dev_d);
   4805	struct qlge_adapter *qdev;
   4806	struct net_device *ndev;
   4807	int err;
   4808
   4809	qdev = pci_get_drvdata(pdev);
   4810	ndev = qdev->ndev;
   4811
   4812	pci_set_master(pdev);
   4813
   4814	device_wakeup_disable(dev_d);
   4815
   4816	if (netif_running(ndev)) {
   4817		err = qlge_adapter_up(qdev);
   4818		if (err)
   4819			return err;
   4820	}
   4821
   4822	mod_timer(&qdev->timer, jiffies + (5 * HZ));
   4823	netif_device_attach(ndev);
   4824
   4825	return 0;
   4826}
   4827
   4828static void qlge_shutdown(struct pci_dev *pdev)
   4829{
   4830	qlge_suspend(&pdev->dev);
   4831}
   4832
   4833static SIMPLE_DEV_PM_OPS(qlge_pm_ops, qlge_suspend, qlge_resume);
   4834
   4835static struct pci_driver qlge_driver = {
   4836	.name = DRV_NAME,
   4837	.id_table = qlge_pci_tbl,
   4838	.probe = qlge_probe,
   4839	.remove = qlge_remove,
   4840	.driver.pm = &qlge_pm_ops,
   4841	.shutdown = qlge_shutdown,
   4842	.err_handler = &qlge_err_handler
   4843};
   4844
   4845module_pci_driver(qlge_driver);