cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

efx.c (33321B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/****************************************************************************
      3 * Driver for Solarflare network controllers and boards
      4 * Copyright 2005-2006 Fen Systems Ltd.
      5 * Copyright 2005-2013 Solarflare Communications Inc.
      6 */
      7
      8#include <linux/filter.h>
      9#include <linux/module.h>
     10#include <linux/pci.h>
     11#include <linux/netdevice.h>
     12#include <linux/etherdevice.h>
     13#include <linux/delay.h>
     14#include <linux/notifier.h>
     15#include <linux/ip.h>
     16#include <linux/tcp.h>
     17#include <linux/in.h>
     18#include <linux/ethtool.h>
     19#include <linux/topology.h>
     20#include <linux/gfp.h>
     21#include <linux/aer.h>
     22#include <linux/interrupt.h>
     23#include "net_driver.h"
     24#include <net/gre.h>
     25#include <net/udp_tunnel.h>
     26#include "efx.h"
     27#include "efx_common.h"
     28#include "efx_channels.h"
     29#include "rx_common.h"
     30#include "tx_common.h"
     31#include "nic.h"
     32#include "io.h"
     33#include "selftest.h"
     34#include "sriov.h"
     35#ifdef CONFIG_SFC_SIENA_SRIOV
     36#include "siena_sriov.h"
     37#endif
     38
     39#include "mcdi_port_common.h"
     40#include "mcdi_pcol.h"
     41#include "workarounds.h"
     42
     43/**************************************************************************
     44 *
     45 * Configurable values
     46 *
     47 *************************************************************************/
     48
     49module_param_named(interrupt_mode, efx_siena_interrupt_mode, uint, 0444);
     50MODULE_PARM_DESC(interrupt_mode,
     51		 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
     52
     53module_param_named(rss_cpus, efx_siena_rss_cpus, uint, 0444);
     54MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
     55
     56/*
     57 * Use separate channels for TX and RX events
     58 *
     59 * Set this to 1 to use separate channels for TX and RX. It allows us
     60 * to control interrupt affinity separately for TX and RX.
     61 *
     62 * This is only used in MSI-X interrupt mode
     63 */
     64bool efx_siena_separate_tx_channels;
     65module_param_named(efx_separate_tx_channels, efx_siena_separate_tx_channels,
     66		   bool, 0444);
     67MODULE_PARM_DESC(efx_separate_tx_channels,
     68		 "Use separate channels for TX and RX");
     69
     70/* Initial interrupt moderation settings.  They can be modified after
     71 * module load with ethtool.
     72 *
     73 * The default for RX should strike a balance between increasing the
     74 * round-trip latency and reducing overhead.
     75 */
     76static unsigned int rx_irq_mod_usec = 60;
     77
     78/* Initial interrupt moderation settings.  They can be modified after
     79 * module load with ethtool.
     80 *
     81 * This default is chosen to ensure that a 10G link does not go idle
     82 * while a TX queue is stopped after it has become full.  A queue is
     83 * restarted when it drops below half full.  The time this takes (assuming
     84 * worst case 3 descriptors per packet and 1024 descriptors) is
     85 *   512 / 3 * 1.2 = 205 usec.
     86 */
     87static unsigned int tx_irq_mod_usec = 150;
     88
     89static bool phy_flash_cfg;
     90module_param(phy_flash_cfg, bool, 0644);
     91MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
     92
     93static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
     94			 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
     95			 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
     96			 NETIF_MSG_TX_ERR | NETIF_MSG_HW);
     97module_param(debug, uint, 0);
     98MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
     99
    100/**************************************************************************
    101 *
    102 * Utility functions and prototypes
    103 *
    104 *************************************************************************/
    105
    106static void efx_remove_port(struct efx_nic *efx);
    107static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog);
    108static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp);
    109static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs,
    110			u32 flags);
    111
    112#define EFX_ASSERT_RESET_SERIALISED(efx)		\
    113	do {						\
    114		if ((efx->state == STATE_READY) ||	\
    115		    (efx->state == STATE_RECOVERY) ||	\
    116		    (efx->state == STATE_DISABLED))	\
    117			ASSERT_RTNL();			\
    118	} while (0)
    119
    120/**************************************************************************
    121 *
    122 * Port handling
    123 *
    124 **************************************************************************/
    125
    126static void efx_fini_port(struct efx_nic *efx);
    127
    128static int efx_probe_port(struct efx_nic *efx)
    129{
    130	int rc;
    131
    132	netif_dbg(efx, probe, efx->net_dev, "create port\n");
    133
    134	if (phy_flash_cfg)
    135		efx->phy_mode = PHY_MODE_SPECIAL;
    136
    137	/* Connect up MAC/PHY operations table */
    138	rc = efx->type->probe_port(efx);
    139	if (rc)
    140		return rc;
    141
    142	/* Initialise MAC address to permanent address */
    143	eth_hw_addr_set(efx->net_dev, efx->net_dev->perm_addr);
    144
    145	return 0;
    146}
    147
    148static int efx_init_port(struct efx_nic *efx)
    149{
    150	int rc;
    151
    152	netif_dbg(efx, drv, efx->net_dev, "init port\n");
    153
    154	mutex_lock(&efx->mac_lock);
    155
    156	efx->port_initialized = true;
    157
    158	/* Ensure the PHY advertises the correct flow control settings */
    159	rc = efx_siena_mcdi_port_reconfigure(efx);
    160	if (rc && rc != -EPERM)
    161		goto fail;
    162
    163	mutex_unlock(&efx->mac_lock);
    164	return 0;
    165
    166fail:
    167	mutex_unlock(&efx->mac_lock);
    168	return rc;
    169}
    170
    171static void efx_fini_port(struct efx_nic *efx)
    172{
    173	netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
    174
    175	if (!efx->port_initialized)
    176		return;
    177
    178	efx->port_initialized = false;
    179
    180	efx->link_state.up = false;
    181	efx_siena_link_status_changed(efx);
    182}
    183
    184static void efx_remove_port(struct efx_nic *efx)
    185{
    186	netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
    187
    188	efx->type->remove_port(efx);
    189}
    190
    191/**************************************************************************
    192 *
    193 * NIC handling
    194 *
    195 **************************************************************************/
    196
    197static LIST_HEAD(efx_primary_list);
    198static LIST_HEAD(efx_unassociated_list);
    199
    200static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)
    201{
    202	return left->type == right->type &&
    203		left->vpd_sn && right->vpd_sn &&
    204		!strcmp(left->vpd_sn, right->vpd_sn);
    205}
    206
    207static void efx_associate(struct efx_nic *efx)
    208{
    209	struct efx_nic *other, *next;
    210
    211	if (efx->primary == efx) {
    212		/* Adding primary function; look for secondaries */
    213
    214		netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
    215		list_add_tail(&efx->node, &efx_primary_list);
    216
    217		list_for_each_entry_safe(other, next, &efx_unassociated_list,
    218					 node) {
    219			if (efx_same_controller(efx, other)) {
    220				list_del(&other->node);
    221				netif_dbg(other, probe, other->net_dev,
    222					  "moving to secondary list of %s %s\n",
    223					  pci_name(efx->pci_dev),
    224					  efx->net_dev->name);
    225				list_add_tail(&other->node,
    226					      &efx->secondary_list);
    227				other->primary = efx;
    228			}
    229		}
    230	} else {
    231		/* Adding secondary function; look for primary */
    232
    233		list_for_each_entry(other, &efx_primary_list, node) {
    234			if (efx_same_controller(efx, other)) {
    235				netif_dbg(efx, probe, efx->net_dev,
    236					  "adding to secondary list of %s %s\n",
    237					  pci_name(other->pci_dev),
    238					  other->net_dev->name);
    239				list_add_tail(&efx->node,
    240					      &other->secondary_list);
    241				efx->primary = other;
    242				return;
    243			}
    244		}
    245
    246		netif_dbg(efx, probe, efx->net_dev,
    247			  "adding to unassociated list\n");
    248		list_add_tail(&efx->node, &efx_unassociated_list);
    249	}
    250}
    251
    252static void efx_dissociate(struct efx_nic *efx)
    253{
    254	struct efx_nic *other, *next;
    255
    256	list_del(&efx->node);
    257	efx->primary = NULL;
    258
    259	list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
    260		list_del(&other->node);
    261		netif_dbg(other, probe, other->net_dev,
    262			  "moving to unassociated list\n");
    263		list_add_tail(&other->node, &efx_unassociated_list);
    264		other->primary = NULL;
    265	}
    266}
    267
    268static int efx_probe_nic(struct efx_nic *efx)
    269{
    270	int rc;
    271
    272	netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
    273
    274	/* Carry out hardware-type specific initialisation */
    275	rc = efx->type->probe(efx);
    276	if (rc)
    277		return rc;
    278
    279	do {
    280		if (!efx->max_channels || !efx->max_tx_channels) {
    281			netif_err(efx, drv, efx->net_dev,
    282				  "Insufficient resources to allocate"
    283				  " any channels\n");
    284			rc = -ENOSPC;
    285			goto fail1;
    286		}
    287
    288		/* Determine the number of channels and queues by trying
    289		 * to hook in MSI-X interrupts.
    290		 */
    291		rc = efx_siena_probe_interrupts(efx);
    292		if (rc)
    293			goto fail1;
    294
    295		rc = efx_siena_set_channels(efx);
    296		if (rc)
    297			goto fail1;
    298
    299		/* dimension_resources can fail with EAGAIN */
    300		rc = efx->type->dimension_resources(efx);
    301		if (rc != 0 && rc != -EAGAIN)
    302			goto fail2;
    303
    304		if (rc == -EAGAIN)
    305			/* try again with new max_channels */
    306			efx_siena_remove_interrupts(efx);
    307
    308	} while (rc == -EAGAIN);
    309
    310	if (efx->n_channels > 1)
    311		netdev_rss_key_fill(efx->rss_context.rx_hash_key,
    312				    sizeof(efx->rss_context.rx_hash_key));
    313	efx_siena_set_default_rx_indir_table(efx, &efx->rss_context);
    314
    315	/* Initialise the interrupt moderation settings */
    316	efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
    317	efx_siena_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec,
    318				      true, true);
    319
    320	return 0;
    321
    322fail2:
    323	efx_siena_remove_interrupts(efx);
    324fail1:
    325	efx->type->remove(efx);
    326	return rc;
    327}
    328
    329static void efx_remove_nic(struct efx_nic *efx)
    330{
    331	netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
    332
    333	efx_siena_remove_interrupts(efx);
    334	efx->type->remove(efx);
    335}
    336
    337/**************************************************************************
    338 *
    339 * NIC startup/shutdown
    340 *
    341 *************************************************************************/
    342
    343static int efx_probe_all(struct efx_nic *efx)
    344{
    345	int rc;
    346
    347	rc = efx_probe_nic(efx);
    348	if (rc) {
    349		netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
    350		goto fail1;
    351	}
    352
    353	rc = efx_probe_port(efx);
    354	if (rc) {
    355		netif_err(efx, probe, efx->net_dev, "failed to create port\n");
    356		goto fail2;
    357	}
    358
    359	BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
    360	if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
    361		rc = -EINVAL;
    362		goto fail3;
    363	}
    364
    365#ifdef CONFIG_SFC_SIENA_SRIOV
    366	rc = efx->type->vswitching_probe(efx);
    367	if (rc) /* not fatal; the PF will still work fine */
    368		netif_warn(efx, probe, efx->net_dev,
    369			   "failed to setup vswitching rc=%d;"
    370			   " VFs may not function\n", rc);
    371#endif
    372
    373	rc = efx_siena_probe_filters(efx);
    374	if (rc) {
    375		netif_err(efx, probe, efx->net_dev,
    376			  "failed to create filter tables\n");
    377		goto fail4;
    378	}
    379
    380	rc = efx_siena_probe_channels(efx);
    381	if (rc)
    382		goto fail5;
    383
    384	return 0;
    385
    386 fail5:
    387	efx_siena_remove_filters(efx);
    388 fail4:
    389#ifdef CONFIG_SFC_SIENA_SRIOV
    390	efx->type->vswitching_remove(efx);
    391#endif
    392 fail3:
    393	efx_remove_port(efx);
    394 fail2:
    395	efx_remove_nic(efx);
    396 fail1:
    397	return rc;
    398}
    399
    400static void efx_remove_all(struct efx_nic *efx)
    401{
    402	rtnl_lock();
    403	efx_xdp_setup_prog(efx, NULL);
    404	rtnl_unlock();
    405
    406	efx_siena_remove_channels(efx);
    407	efx_siena_remove_filters(efx);
    408#ifdef CONFIG_SFC_SIENA_SRIOV
    409	efx->type->vswitching_remove(efx);
    410#endif
    411	efx_remove_port(efx);
    412	efx_remove_nic(efx);
    413}
    414
    415/**************************************************************************
    416 *
    417 * Interrupt moderation
    418 *
    419 **************************************************************************/
    420unsigned int efx_siena_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)
    421{
    422	if (usecs == 0)
    423		return 0;
    424	if (usecs * 1000 < efx->timer_quantum_ns)
    425		return 1; /* never round down to 0 */
    426	return usecs * 1000 / efx->timer_quantum_ns;
    427}
    428
    429/* Set interrupt moderation parameters */
    430int efx_siena_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
    431				  unsigned int rx_usecs, bool rx_adaptive,
    432				  bool rx_may_override_tx)
    433{
    434	struct efx_channel *channel;
    435	unsigned int timer_max_us;
    436
    437	EFX_ASSERT_RESET_SERIALISED(efx);
    438
    439	timer_max_us = efx->timer_max_ns / 1000;
    440
    441	if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)
    442		return -EINVAL;
    443
    444	if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
    445	    !rx_may_override_tx) {
    446		netif_err(efx, drv, efx->net_dev, "Channels are shared. "
    447			  "RX and TX IRQ moderation must be equal\n");
    448		return -EINVAL;
    449	}
    450
    451	efx->irq_rx_adaptive = rx_adaptive;
    452	efx->irq_rx_moderation_us = rx_usecs;
    453	efx_for_each_channel(channel, efx) {
    454		if (efx_channel_has_rx_queue(channel))
    455			channel->irq_moderation_us = rx_usecs;
    456		else if (efx_channel_has_tx_queues(channel))
    457			channel->irq_moderation_us = tx_usecs;
    458		else if (efx_channel_is_xdp_tx(channel))
    459			channel->irq_moderation_us = tx_usecs;
    460	}
    461
    462	return 0;
    463}
    464
    465void efx_siena_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
    466				  unsigned int *rx_usecs, bool *rx_adaptive)
    467{
    468	*rx_adaptive = efx->irq_rx_adaptive;
    469	*rx_usecs = efx->irq_rx_moderation_us;
    470
    471	/* If channels are shared between RX and TX, so is IRQ
    472	 * moderation.  Otherwise, IRQ moderation is the same for all
    473	 * TX channels and is not adaptive.
    474	 */
    475	if (efx->tx_channel_offset == 0) {
    476		*tx_usecs = *rx_usecs;
    477	} else {
    478		struct efx_channel *tx_channel;
    479
    480		tx_channel = efx->channel[efx->tx_channel_offset];
    481		*tx_usecs = tx_channel->irq_moderation_us;
    482	}
    483}
    484
    485/**************************************************************************
    486 *
    487 * ioctls
    488 *
    489 *************************************************************************/
    490
    491/* Net device ioctl
    492 * Context: process, rtnl_lock() held.
    493 */
    494static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
    495{
    496	struct efx_nic *efx = netdev_priv(net_dev);
    497	struct mii_ioctl_data *data = if_mii(ifr);
    498
    499	if (cmd == SIOCSHWTSTAMP)
    500		return efx_siena_ptp_set_ts_config(efx, ifr);
    501	if (cmd == SIOCGHWTSTAMP)
    502		return efx_siena_ptp_get_ts_config(efx, ifr);
    503
    504	/* Convert phy_id from older PRTAD/DEVAD format */
    505	if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
    506	    (data->phy_id & 0xfc00) == 0x0400)
    507		data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
    508
    509	return mdio_mii_ioctl(&efx->mdio, data, cmd);
    510}
    511
    512/**************************************************************************
    513 *
    514 * Kernel net device interface
    515 *
    516 *************************************************************************/
    517
    518/* Context: process, rtnl_lock() held. */
    519static int efx_net_open(struct net_device *net_dev)
    520{
    521	struct efx_nic *efx = netdev_priv(net_dev);
    522	int rc;
    523
    524	netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
    525		  raw_smp_processor_id());
    526
    527	rc = efx_check_disabled(efx);
    528	if (rc)
    529		return rc;
    530	if (efx->phy_mode & PHY_MODE_SPECIAL)
    531		return -EBUSY;
    532	if (efx_siena_mcdi_poll_reboot(efx) && efx_siena_reset(efx, RESET_TYPE_ALL))
    533		return -EIO;
    534
    535	/* Notify the kernel of the link state polled during driver load,
    536	 * before the monitor starts running */
    537	efx_siena_link_status_changed(efx);
    538
    539	efx_siena_start_all(efx);
    540	if (efx->state == STATE_DISABLED || efx->reset_pending)
    541		netif_device_detach(efx->net_dev);
    542	efx_siena_selftest_async_start(efx);
    543	return 0;
    544}
    545
    546/* Context: process, rtnl_lock() held.
    547 * Note that the kernel will ignore our return code; this method
    548 * should really be a void.
    549 */
    550static int efx_net_stop(struct net_device *net_dev)
    551{
    552	struct efx_nic *efx = netdev_priv(net_dev);
    553
    554	netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
    555		  raw_smp_processor_id());
    556
    557	/* Stop the device and flush all the channels */
    558	efx_siena_stop_all(efx);
    559
    560	return 0;
    561}
    562
    563static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)
    564{
    565	struct efx_nic *efx = netdev_priv(net_dev);
    566
    567	if (efx->type->vlan_rx_add_vid)
    568		return efx->type->vlan_rx_add_vid(efx, proto, vid);
    569	else
    570		return -EOPNOTSUPP;
    571}
    572
    573static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)
    574{
    575	struct efx_nic *efx = netdev_priv(net_dev);
    576
    577	if (efx->type->vlan_rx_kill_vid)
    578		return efx->type->vlan_rx_kill_vid(efx, proto, vid);
    579	else
    580		return -EOPNOTSUPP;
    581}
    582
    583static const struct net_device_ops efx_netdev_ops = {
    584	.ndo_open		= efx_net_open,
    585	.ndo_stop		= efx_net_stop,
    586	.ndo_get_stats64	= efx_siena_net_stats,
    587	.ndo_tx_timeout		= efx_siena_watchdog,
    588	.ndo_start_xmit		= efx_siena_hard_start_xmit,
    589	.ndo_validate_addr	= eth_validate_addr,
    590	.ndo_eth_ioctl		= efx_ioctl,
    591	.ndo_change_mtu		= efx_siena_change_mtu,
    592	.ndo_set_mac_address	= efx_siena_set_mac_address,
    593	.ndo_set_rx_mode	= efx_siena_set_rx_mode,
    594	.ndo_set_features	= efx_siena_set_features,
    595	.ndo_features_check	= efx_siena_features_check,
    596	.ndo_vlan_rx_add_vid	= efx_vlan_rx_add_vid,
    597	.ndo_vlan_rx_kill_vid	= efx_vlan_rx_kill_vid,
    598#ifdef CONFIG_SFC_SIENA_SRIOV
    599	.ndo_set_vf_mac		= efx_sriov_set_vf_mac,
    600	.ndo_set_vf_vlan	= efx_sriov_set_vf_vlan,
    601	.ndo_set_vf_spoofchk	= efx_sriov_set_vf_spoofchk,
    602	.ndo_get_vf_config	= efx_sriov_get_vf_config,
    603	.ndo_set_vf_link_state  = efx_sriov_set_vf_link_state,
    604#endif
    605	.ndo_get_phys_port_id   = efx_siena_get_phys_port_id,
    606	.ndo_get_phys_port_name	= efx_siena_get_phys_port_name,
    607	.ndo_setup_tc		= efx_siena_setup_tc,
    608#ifdef CONFIG_RFS_ACCEL
    609	.ndo_rx_flow_steer	= efx_siena_filter_rfs,
    610#endif
    611	.ndo_xdp_xmit		= efx_xdp_xmit,
    612	.ndo_bpf		= efx_xdp
    613};
    614
    615static int efx_xdp_setup_prog(struct efx_nic *efx, struct bpf_prog *prog)
    616{
    617	struct bpf_prog *old_prog;
    618
    619	if (efx->xdp_rxq_info_failed) {
    620		netif_err(efx, drv, efx->net_dev,
    621			  "Unable to bind XDP program due to previous failure of rxq_info\n");
    622		return -EINVAL;
    623	}
    624
    625	if (prog && efx->net_dev->mtu > efx_siena_xdp_max_mtu(efx)) {
    626		netif_err(efx, drv, efx->net_dev,
    627			  "Unable to configure XDP with MTU of %d (max: %d)\n",
    628			  efx->net_dev->mtu, efx_siena_xdp_max_mtu(efx));
    629		return -EINVAL;
    630	}
    631
    632	old_prog = rtnl_dereference(efx->xdp_prog);
    633	rcu_assign_pointer(efx->xdp_prog, prog);
    634	/* Release the reference that was originally passed by the caller. */
    635	if (old_prog)
    636		bpf_prog_put(old_prog);
    637
    638	return 0;
    639}
    640
    641/* Context: process, rtnl_lock() held. */
    642static int efx_xdp(struct net_device *dev, struct netdev_bpf *xdp)
    643{
    644	struct efx_nic *efx = netdev_priv(dev);
    645
    646	switch (xdp->command) {
    647	case XDP_SETUP_PROG:
    648		return efx_xdp_setup_prog(efx, xdp->prog);
    649	default:
    650		return -EINVAL;
    651	}
    652}
    653
    654static int efx_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **xdpfs,
    655			u32 flags)
    656{
    657	struct efx_nic *efx = netdev_priv(dev);
    658
    659	if (!netif_running(dev))
    660		return -EINVAL;
    661
    662	return efx_siena_xdp_tx_buffers(efx, n, xdpfs, flags & XDP_XMIT_FLUSH);
    663}
    664
    665static void efx_update_name(struct efx_nic *efx)
    666{
    667	strcpy(efx->name, efx->net_dev->name);
    668	efx_siena_mtd_rename(efx);
    669	efx_siena_set_channel_names(efx);
    670}
    671
    672static int efx_netdev_event(struct notifier_block *this,
    673			    unsigned long event, void *ptr)
    674{
    675	struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
    676
    677	if ((net_dev->netdev_ops == &efx_netdev_ops) &&
    678	    event == NETDEV_CHANGENAME)
    679		efx_update_name(netdev_priv(net_dev));
    680
    681	return NOTIFY_DONE;
    682}
    683
    684static struct notifier_block efx_netdev_notifier = {
    685	.notifier_call = efx_netdev_event,
    686};
    687
    688static ssize_t phy_type_show(struct device *dev,
    689			     struct device_attribute *attr, char *buf)
    690{
    691	struct efx_nic *efx = dev_get_drvdata(dev);
    692	return sprintf(buf, "%d\n", efx->phy_type);
    693}
    694static DEVICE_ATTR_RO(phy_type);
    695
    696static int efx_register_netdev(struct efx_nic *efx)
    697{
    698	struct net_device *net_dev = efx->net_dev;
    699	struct efx_channel *channel;
    700	int rc;
    701
    702	net_dev->watchdog_timeo = 5 * HZ;
    703	net_dev->irq = efx->pci_dev->irq;
    704	net_dev->netdev_ops = &efx_netdev_ops;
    705	if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
    706		net_dev->priv_flags |= IFF_UNICAST_FLT;
    707	net_dev->ethtool_ops = &efx_siena_ethtool_ops;
    708	netif_set_tso_max_segs(net_dev, EFX_TSO_MAX_SEGS);
    709	net_dev->min_mtu = EFX_MIN_MTU;
    710	net_dev->max_mtu = EFX_MAX_MTU;
    711
    712	rtnl_lock();
    713
    714	/* Enable resets to be scheduled and check whether any were
    715	 * already requested.  If so, the NIC is probably hosed so we
    716	 * abort.
    717	 */
    718	efx->state = STATE_READY;
    719	smp_mb(); /* ensure we change state before checking reset_pending */
    720	if (efx->reset_pending) {
    721		pci_err(efx->pci_dev, "aborting probe due to scheduled reset\n");
    722		rc = -EIO;
    723		goto fail_locked;
    724	}
    725
    726	rc = dev_alloc_name(net_dev, net_dev->name);
    727	if (rc < 0)
    728		goto fail_locked;
    729	efx_update_name(efx);
    730
    731	/* Always start with carrier off; PHY events will detect the link */
    732	netif_carrier_off(net_dev);
    733
    734	rc = register_netdevice(net_dev);
    735	if (rc)
    736		goto fail_locked;
    737
    738	efx_for_each_channel(channel, efx) {
    739		struct efx_tx_queue *tx_queue;
    740		efx_for_each_channel_tx_queue(tx_queue, channel)
    741			efx_siena_init_tx_queue_core_txq(tx_queue);
    742	}
    743
    744	efx_associate(efx);
    745
    746	rtnl_unlock();
    747
    748	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
    749	if (rc) {
    750		netif_err(efx, drv, efx->net_dev,
    751			  "failed to init net dev attributes\n");
    752		goto fail_registered;
    753	}
    754
    755	efx_siena_init_mcdi_logging(efx);
    756
    757	return 0;
    758
    759fail_registered:
    760	rtnl_lock();
    761	efx_dissociate(efx);
    762	unregister_netdevice(net_dev);
    763fail_locked:
    764	efx->state = STATE_UNINIT;
    765	rtnl_unlock();
    766	netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
    767	return rc;
    768}
    769
    770static void efx_unregister_netdev(struct efx_nic *efx)
    771{
    772	if (!efx->net_dev)
    773		return;
    774
    775	BUG_ON(netdev_priv(efx->net_dev) != efx);
    776
    777	if (efx_dev_registered(efx)) {
    778		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
    779		efx_siena_fini_mcdi_logging(efx);
    780		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
    781		unregister_netdev(efx->net_dev);
    782	}
    783}
    784
    785/**************************************************************************
    786 *
    787 * List of NICs we support
    788 *
    789 **************************************************************************/
    790
    791/* PCI device ID table */
    792static const struct pci_device_id efx_pci_table[] = {
    793	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803),	/* SFC9020 */
    794	 .driver_data = (unsigned long)&siena_a0_nic_type},
    795	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813),	/* SFL9021 */
    796	 .driver_data = (unsigned long)&siena_a0_nic_type},
    797	{0}			/* end of list */
    798};
    799
    800/**************************************************************************
    801 *
    802 * Data housekeeping
    803 *
    804 **************************************************************************/
    805
    806void efx_siena_update_sw_stats(struct efx_nic *efx, u64 *stats)
    807{
    808	u64 n_rx_nodesc_trunc = 0;
    809	struct efx_channel *channel;
    810
    811	efx_for_each_channel(channel, efx)
    812		n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
    813	stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
    814	stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
    815}
    816
    817/**************************************************************************
    818 *
    819 * PCI interface
    820 *
    821 **************************************************************************/
    822
    823/* Main body of final NIC shutdown code
    824 * This is called only at module unload (or hotplug removal).
    825 */
    826static void efx_pci_remove_main(struct efx_nic *efx)
    827{
    828	/* Flush reset_work. It can no longer be scheduled since we
    829	 * are not READY.
    830	 */
    831	BUG_ON(efx->state == STATE_READY);
    832	efx_siena_flush_reset_workqueue(efx);
    833
    834	efx_siena_disable_interrupts(efx);
    835	efx_siena_clear_interrupt_affinity(efx);
    836	efx_siena_fini_interrupt(efx);
    837	efx_fini_port(efx);
    838	efx->type->fini(efx);
    839	efx_siena_fini_napi(efx);
    840	efx_remove_all(efx);
    841}
    842
    843/* Final NIC shutdown
    844 * This is called only at module unload (or hotplug removal).  A PF can call
    845 * this on its VFs to ensure they are unbound first.
    846 */
    847static void efx_pci_remove(struct pci_dev *pci_dev)
    848{
    849	struct efx_nic *efx;
    850
    851	efx = pci_get_drvdata(pci_dev);
    852	if (!efx)
    853		return;
    854
    855	/* Mark the NIC as fini, then stop the interface */
    856	rtnl_lock();
    857	efx_dissociate(efx);
    858	dev_close(efx->net_dev);
    859	efx_siena_disable_interrupts(efx);
    860	efx->state = STATE_UNINIT;
    861	rtnl_unlock();
    862
    863	if (efx->type->sriov_fini)
    864		efx->type->sriov_fini(efx);
    865
    866	efx_unregister_netdev(efx);
    867
    868	efx_siena_mtd_remove(efx);
    869
    870	efx_pci_remove_main(efx);
    871
    872	efx_siena_fini_io(efx);
    873	netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
    874
    875	efx_siena_fini_struct(efx);
    876	free_netdev(efx->net_dev);
    877
    878	pci_disable_pcie_error_reporting(pci_dev);
    879};
    880
    881/* NIC VPD information
    882 * Called during probe to display the part number of the
    883 * installed NIC.
    884 */
    885static void efx_probe_vpd_strings(struct efx_nic *efx)
    886{
    887	struct pci_dev *dev = efx->pci_dev;
    888	unsigned int vpd_size, kw_len;
    889	u8 *vpd_data;
    890	int start;
    891
    892	vpd_data = pci_vpd_alloc(dev, &vpd_size);
    893	if (IS_ERR(vpd_data)) {
    894		pci_warn(dev, "Unable to read VPD\n");
    895		return;
    896	}
    897
    898	start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
    899					     PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
    900	if (start < 0)
    901		pci_err(dev, "Part number not found or incomplete\n");
    902	else
    903		pci_info(dev, "Part Number : %.*s\n", kw_len, vpd_data + start);
    904
    905	start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
    906					     PCI_VPD_RO_KEYWORD_SERIALNO, &kw_len);
    907	if (start < 0)
    908		pci_err(dev, "Serial number not found or incomplete\n");
    909	else
    910		efx->vpd_sn = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL);
    911
    912	kfree(vpd_data);
    913}
    914
    915
    916/* Main body of NIC initialisation
    917 * This is called at module load (or hotplug insertion, theoretically).
    918 */
    919static int efx_pci_probe_main(struct efx_nic *efx)
    920{
    921	int rc;
    922
    923	/* Do start-of-day initialisation */
    924	rc = efx_probe_all(efx);
    925	if (rc)
    926		goto fail1;
    927
    928	efx_siena_init_napi(efx);
    929
    930	down_write(&efx->filter_sem);
    931	rc = efx->type->init(efx);
    932	up_write(&efx->filter_sem);
    933	if (rc) {
    934		pci_err(efx->pci_dev, "failed to initialise NIC\n");
    935		goto fail3;
    936	}
    937
    938	rc = efx_init_port(efx);
    939	if (rc) {
    940		netif_err(efx, probe, efx->net_dev,
    941			  "failed to initialise port\n");
    942		goto fail4;
    943	}
    944
    945	rc = efx_siena_init_interrupt(efx);
    946	if (rc)
    947		goto fail5;
    948
    949	efx_siena_set_interrupt_affinity(efx);
    950	rc = efx_siena_enable_interrupts(efx);
    951	if (rc)
    952		goto fail6;
    953
    954	return 0;
    955
    956 fail6:
    957	efx_siena_clear_interrupt_affinity(efx);
    958	efx_siena_fini_interrupt(efx);
    959 fail5:
    960	efx_fini_port(efx);
    961 fail4:
    962	efx->type->fini(efx);
    963 fail3:
    964	efx_siena_fini_napi(efx);
    965	efx_remove_all(efx);
    966 fail1:
    967	return rc;
    968}
    969
    970static int efx_pci_probe_post_io(struct efx_nic *efx)
    971{
    972	struct net_device *net_dev = efx->net_dev;
    973	int rc = efx_pci_probe_main(efx);
    974
    975	if (rc)
    976		return rc;
    977
    978	if (efx->type->sriov_init) {
    979		rc = efx->type->sriov_init(efx);
    980		if (rc)
    981			pci_err(efx->pci_dev, "SR-IOV can't be enabled rc %d\n",
    982				rc);
    983	}
    984
    985	/* Determine netdevice features */
    986	net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
    987			      NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL);
    988	if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
    989		net_dev->features |= NETIF_F_TSO6;
    990	/* Check whether device supports TSO */
    991	if (!efx->type->tso_versions || !efx->type->tso_versions(efx))
    992		net_dev->features &= ~NETIF_F_ALL_TSO;
    993	/* Mask for features that also apply to VLAN devices */
    994	net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |
    995				   NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
    996				   NETIF_F_RXCSUM);
    997
    998	net_dev->hw_features |= net_dev->features & ~efx->fixed_features;
    999
   1000	/* Disable receiving frames with bad FCS, by default. */
   1001	net_dev->features &= ~NETIF_F_RXALL;
   1002
   1003	/* Disable VLAN filtering by default.  It may be enforced if
   1004	 * the feature is fixed (i.e. VLAN filters are required to
   1005	 * receive VLAN tagged packets due to vPort restrictions).
   1006	 */
   1007	net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
   1008	net_dev->features |= efx->fixed_features;
   1009
   1010	rc = efx_register_netdev(efx);
   1011	if (!rc)
   1012		return 0;
   1013
   1014	efx_pci_remove_main(efx);
   1015	return rc;
   1016}
   1017
   1018/* NIC initialisation
   1019 *
   1020 * This is called at module load (or hotplug insertion,
   1021 * theoretically).  It sets up PCI mappings, resets the NIC,
   1022 * sets up and registers the network devices with the kernel and hooks
   1023 * the interrupt service routine.  It does not prepare the device for
   1024 * transmission; this is left to the first time one of the network
   1025 * interfaces is brought up (i.e. efx_net_open).
   1026 */
   1027static int efx_pci_probe(struct pci_dev *pci_dev,
   1028			 const struct pci_device_id *entry)
   1029{
   1030	struct net_device *net_dev;
   1031	struct efx_nic *efx;
   1032	int rc;
   1033
   1034	/* Allocate and initialise a struct net_device and struct efx_nic */
   1035	net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
   1036				     EFX_MAX_RX_QUEUES);
   1037	if (!net_dev)
   1038		return -ENOMEM;
   1039	efx = netdev_priv(net_dev);
   1040	efx->type = (const struct efx_nic_type *) entry->driver_data;
   1041	efx->fixed_features |= NETIF_F_HIGHDMA;
   1042
   1043	pci_set_drvdata(pci_dev, efx);
   1044	SET_NETDEV_DEV(net_dev, &pci_dev->dev);
   1045	rc = efx_siena_init_struct(efx, pci_dev, net_dev);
   1046	if (rc)
   1047		goto fail1;
   1048
   1049	pci_info(pci_dev, "Solarflare NIC detected\n");
   1050
   1051	if (!efx->type->is_vf)
   1052		efx_probe_vpd_strings(efx);
   1053
   1054	/* Set up basic I/O (BAR mappings etc) */
   1055	rc = efx_siena_init_io(efx, efx->type->mem_bar(efx),
   1056			       efx->type->max_dma_mask,
   1057			       efx->type->mem_map_size(efx));
   1058	if (rc)
   1059		goto fail2;
   1060
   1061	rc = efx_pci_probe_post_io(efx);
   1062	if (rc) {
   1063		/* On failure, retry once immediately.
   1064		 * If we aborted probe due to a scheduled reset, dismiss it.
   1065		 */
   1066		efx->reset_pending = 0;
   1067		rc = efx_pci_probe_post_io(efx);
   1068		if (rc) {
   1069			/* On another failure, retry once more
   1070			 * after a 50-305ms delay.
   1071			 */
   1072			unsigned char r;
   1073
   1074			get_random_bytes(&r, 1);
   1075			msleep((unsigned int)r + 50);
   1076			efx->reset_pending = 0;
   1077			rc = efx_pci_probe_post_io(efx);
   1078		}
   1079	}
   1080	if (rc)
   1081		goto fail3;
   1082
   1083	netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
   1084
   1085	/* Try to create MTDs, but allow this to fail */
   1086	rtnl_lock();
   1087	rc = efx_mtd_probe(efx);
   1088	rtnl_unlock();
   1089	if (rc && rc != -EPERM)
   1090		netif_warn(efx, probe, efx->net_dev,
   1091			   "failed to create MTDs (%d)\n", rc);
   1092
   1093	(void)pci_enable_pcie_error_reporting(pci_dev);
   1094
   1095	if (efx->type->udp_tnl_push_ports)
   1096		efx->type->udp_tnl_push_ports(efx);
   1097
   1098	return 0;
   1099
   1100 fail3:
   1101	efx_siena_fini_io(efx);
   1102 fail2:
   1103	efx_siena_fini_struct(efx);
   1104 fail1:
   1105	WARN_ON(rc > 0);
   1106	netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
   1107	free_netdev(net_dev);
   1108	return rc;
   1109}
   1110
   1111/* efx_pci_sriov_configure returns the actual number of Virtual Functions
   1112 * enabled on success
   1113 */
   1114#ifdef CONFIG_SFC_SIENA_SRIOV
   1115static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
   1116{
   1117	int rc;
   1118	struct efx_nic *efx = pci_get_drvdata(dev);
   1119
   1120	if (efx->type->sriov_configure) {
   1121		rc = efx->type->sriov_configure(efx, num_vfs);
   1122		if (rc)
   1123			return rc;
   1124		else
   1125			return num_vfs;
   1126	} else
   1127		return -EOPNOTSUPP;
   1128}
   1129#endif
   1130
   1131static int efx_pm_freeze(struct device *dev)
   1132{
   1133	struct efx_nic *efx = dev_get_drvdata(dev);
   1134
   1135	rtnl_lock();
   1136
   1137	if (efx->state != STATE_DISABLED) {
   1138		efx->state = STATE_UNINIT;
   1139
   1140		efx_device_detach_sync(efx);
   1141
   1142		efx_siena_stop_all(efx);
   1143		efx_siena_disable_interrupts(efx);
   1144	}
   1145
   1146	rtnl_unlock();
   1147
   1148	return 0;
   1149}
   1150
   1151static int efx_pm_thaw(struct device *dev)
   1152{
   1153	int rc;
   1154	struct efx_nic *efx = dev_get_drvdata(dev);
   1155
   1156	rtnl_lock();
   1157
   1158	if (efx->state != STATE_DISABLED) {
   1159		rc = efx_siena_enable_interrupts(efx);
   1160		if (rc)
   1161			goto fail;
   1162
   1163		mutex_lock(&efx->mac_lock);
   1164		efx_siena_mcdi_port_reconfigure(efx);
   1165		mutex_unlock(&efx->mac_lock);
   1166
   1167		efx_siena_start_all(efx);
   1168
   1169		efx_device_attach_if_not_resetting(efx);
   1170
   1171		efx->state = STATE_READY;
   1172
   1173		efx->type->resume_wol(efx);
   1174	}
   1175
   1176	rtnl_unlock();
   1177
   1178	/* Reschedule any quenched resets scheduled during efx_pm_freeze() */
   1179	efx_siena_queue_reset_work(efx);
   1180
   1181	return 0;
   1182
   1183fail:
   1184	rtnl_unlock();
   1185
   1186	return rc;
   1187}
   1188
   1189static int efx_pm_poweroff(struct device *dev)
   1190{
   1191	struct pci_dev *pci_dev = to_pci_dev(dev);
   1192	struct efx_nic *efx = pci_get_drvdata(pci_dev);
   1193
   1194	efx->type->fini(efx);
   1195
   1196	efx->reset_pending = 0;
   1197
   1198	pci_save_state(pci_dev);
   1199	return pci_set_power_state(pci_dev, PCI_D3hot);
   1200}
   1201
   1202/* Used for both resume and restore */
   1203static int efx_pm_resume(struct device *dev)
   1204{
   1205	struct pci_dev *pci_dev = to_pci_dev(dev);
   1206	struct efx_nic *efx = pci_get_drvdata(pci_dev);
   1207	int rc;
   1208
   1209	rc = pci_set_power_state(pci_dev, PCI_D0);
   1210	if (rc)
   1211		return rc;
   1212	pci_restore_state(pci_dev);
   1213	rc = pci_enable_device(pci_dev);
   1214	if (rc)
   1215		return rc;
   1216	pci_set_master(efx->pci_dev);
   1217	rc = efx->type->reset(efx, RESET_TYPE_ALL);
   1218	if (rc)
   1219		return rc;
   1220	down_write(&efx->filter_sem);
   1221	rc = efx->type->init(efx);
   1222	up_write(&efx->filter_sem);
   1223	if (rc)
   1224		return rc;
   1225	rc = efx_pm_thaw(dev);
   1226	return rc;
   1227}
   1228
   1229static int efx_pm_suspend(struct device *dev)
   1230{
   1231	int rc;
   1232
   1233	efx_pm_freeze(dev);
   1234	rc = efx_pm_poweroff(dev);
   1235	if (rc)
   1236		efx_pm_resume(dev);
   1237	return rc;
   1238}
   1239
   1240static const struct dev_pm_ops efx_pm_ops = {
   1241	.suspend	= efx_pm_suspend,
   1242	.resume		= efx_pm_resume,
   1243	.freeze		= efx_pm_freeze,
   1244	.thaw		= efx_pm_thaw,
   1245	.poweroff	= efx_pm_poweroff,
   1246	.restore	= efx_pm_resume,
   1247};
   1248
   1249static struct pci_driver efx_pci_driver = {
   1250	.name		= KBUILD_MODNAME,
   1251	.id_table	= efx_pci_table,
   1252	.probe		= efx_pci_probe,
   1253	.remove		= efx_pci_remove,
   1254	.driver.pm	= &efx_pm_ops,
   1255	.err_handler	= &efx_siena_err_handlers,
   1256#ifdef CONFIG_SFC_SIENA_SRIOV
   1257	.sriov_configure = efx_pci_sriov_configure,
   1258#endif
   1259};
   1260
   1261/**************************************************************************
   1262 *
   1263 * Kernel module interface
   1264 *
   1265 *************************************************************************/
   1266
   1267static int __init efx_init_module(void)
   1268{
   1269	int rc;
   1270
   1271	pr_info("Solarflare Siena driver\n");
   1272
   1273	rc = register_netdevice_notifier(&efx_netdev_notifier);
   1274	if (rc)
   1275		goto err_notifier;
   1276
   1277#ifdef CONFIG_SFC_SIENA_SRIOV
   1278	rc = efx_init_sriov();
   1279	if (rc)
   1280		goto err_sriov;
   1281#endif
   1282
   1283	rc = efx_siena_create_reset_workqueue();
   1284	if (rc)
   1285		goto err_reset;
   1286
   1287	rc = pci_register_driver(&efx_pci_driver);
   1288	if (rc < 0)
   1289		goto err_pci;
   1290
   1291	return 0;
   1292
   1293 err_pci:
   1294	efx_siena_destroy_reset_workqueue();
   1295 err_reset:
   1296#ifdef CONFIG_SFC_SIENA_SRIOV
   1297	efx_fini_sriov();
   1298 err_sriov:
   1299#endif
   1300	unregister_netdevice_notifier(&efx_netdev_notifier);
   1301 err_notifier:
   1302	return rc;
   1303}
   1304
   1305static void __exit efx_exit_module(void)
   1306{
   1307	pr_info("Solarflare Siena driver unloading\n");
   1308
   1309	pci_unregister_driver(&efx_pci_driver);
   1310	efx_siena_destroy_reset_workqueue();
   1311#ifdef CONFIG_SFC_SIENA_SRIOV
   1312	efx_fini_sriov();
   1313#endif
   1314	unregister_netdevice_notifier(&efx_netdev_notifier);
   1315
   1316}
   1317
   1318module_init(efx_init_module);
   1319module_exit(efx_exit_module);
   1320
   1321MODULE_AUTHOR("Solarflare Communications and "
   1322	      "Michael Brown <mbrown@fensystems.co.uk>");
   1323MODULE_DESCRIPTION("Solarflare Siena network driver");
   1324MODULE_LICENSE("GPL");
   1325MODULE_DEVICE_TABLE(pci, efx_pci_table);