cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

ionic_lif.c (96197B)


      1// SPDX-License-Identifier: GPL-2.0
      2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
      3
      4#include <linux/ethtool.h>
      5#include <linux/printk.h>
      6#include <linux/dynamic_debug.h>
      7#include <linux/netdevice.h>
      8#include <linux/etherdevice.h>
      9#include <linux/if_vlan.h>
     10#include <linux/rtnetlink.h>
     11#include <linux/interrupt.h>
     12#include <linux/pci.h>
     13#include <linux/cpumask.h>
     14#include <linux/crash_dump.h>
     15#include <linux/vmalloc.h>
     16
     17#include "ionic.h"
     18#include "ionic_bus.h"
     19#include "ionic_lif.h"
     20#include "ionic_txrx.h"
     21#include "ionic_ethtool.h"
     22#include "ionic_debugfs.h"
     23
     24/* queuetype support level */
     25static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
     26	[IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
     27	[IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
     28	[IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
     29	[IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
     30				      * 1 =   ... with Tx SG version 1
     31				      */
     32};
     33
     34static void ionic_link_status_check(struct ionic_lif *lif);
     35static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
     36static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
     37static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
     38
     39static void ionic_txrx_deinit(struct ionic_lif *lif);
     40static int ionic_txrx_init(struct ionic_lif *lif);
     41static int ionic_start_queues(struct ionic_lif *lif);
     42static void ionic_stop_queues(struct ionic_lif *lif);
     43static void ionic_lif_queue_identify(struct ionic_lif *lif);
     44
     45static void ionic_dim_work(struct work_struct *work)
     46{
     47	struct dim *dim = container_of(work, struct dim, work);
     48	struct dim_cq_moder cur_moder;
     49	struct ionic_qcq *qcq;
     50	u32 new_coal;
     51
     52	cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
     53	qcq = container_of(dim, struct ionic_qcq, dim);
     54	new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
     55	new_coal = new_coal ? new_coal : 1;
     56
     57	if (qcq->intr.dim_coal_hw != new_coal) {
     58		unsigned int qi = qcq->cq.bound_q->index;
     59		struct ionic_lif *lif = qcq->q.lif;
     60
     61		qcq->intr.dim_coal_hw = new_coal;
     62
     63		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
     64				     lif->rxqcqs[qi]->intr.index,
     65				     qcq->intr.dim_coal_hw);
     66	}
     67
     68	dim->state = DIM_START_MEASURE;
     69}
     70
     71static void ionic_lif_deferred_work(struct work_struct *work)
     72{
     73	struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
     74	struct ionic_deferred *def = &lif->deferred;
     75	struct ionic_deferred_work *w = NULL;
     76
     77	do {
     78		spin_lock_bh(&def->lock);
     79		if (!list_empty(&def->list)) {
     80			w = list_first_entry(&def->list,
     81					     struct ionic_deferred_work, list);
     82			list_del(&w->list);
     83		}
     84		spin_unlock_bh(&def->lock);
     85
     86		if (!w)
     87			break;
     88
     89		switch (w->type) {
     90		case IONIC_DW_TYPE_RX_MODE:
     91			ionic_lif_rx_mode(lif);
     92			break;
     93		case IONIC_DW_TYPE_LINK_STATUS:
     94			ionic_link_status_check(lif);
     95			break;
     96		case IONIC_DW_TYPE_LIF_RESET:
     97			if (w->fw_status) {
     98				ionic_lif_handle_fw_up(lif);
     99			} else {
    100				ionic_lif_handle_fw_down(lif);
    101
    102				/* Fire off another watchdog to see
    103				 * if the FW is already back rather than
    104				 * waiting another whole cycle
    105				 */
    106				mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
    107			}
    108			break;
    109		default:
    110			break;
    111		}
    112		kfree(w);
    113		w = NULL;
    114	} while (true);
    115}
    116
    117void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
    118				struct ionic_deferred_work *work)
    119{
    120	spin_lock_bh(&def->lock);
    121	list_add_tail(&work->list, &def->list);
    122	spin_unlock_bh(&def->lock);
    123	schedule_work(&def->work);
    124}
    125
    126static void ionic_link_status_check(struct ionic_lif *lif)
    127{
    128	struct net_device *netdev = lif->netdev;
    129	u16 link_status;
    130	bool link_up;
    131
    132	if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
    133		return;
    134
    135	/* Don't put carrier back up if we're in a broken state */
    136	if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
    137		clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
    138		return;
    139	}
    140
    141	link_status = le16_to_cpu(lif->info->status.link_status);
    142	link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
    143
    144	if (link_up) {
    145		int err = 0;
    146
    147		if (netdev->flags & IFF_UP && netif_running(netdev)) {
    148			mutex_lock(&lif->queue_lock);
    149			err = ionic_start_queues(lif);
    150			if (err && err != -EBUSY) {
    151				netdev_err(lif->netdev,
    152					   "Failed to start queues: %d\n", err);
    153				set_bit(IONIC_LIF_F_BROKEN, lif->state);
    154				netif_carrier_off(lif->netdev);
    155			}
    156			mutex_unlock(&lif->queue_lock);
    157		}
    158
    159		if (!err && !netif_carrier_ok(netdev)) {
    160			ionic_port_identify(lif->ionic);
    161			netdev_info(netdev, "Link up - %d Gbps\n",
    162				    le32_to_cpu(lif->info->status.link_speed) / 1000);
    163			netif_carrier_on(netdev);
    164		}
    165	} else {
    166		if (netif_carrier_ok(netdev)) {
    167			netdev_info(netdev, "Link down\n");
    168			netif_carrier_off(netdev);
    169		}
    170
    171		if (netdev->flags & IFF_UP && netif_running(netdev)) {
    172			mutex_lock(&lif->queue_lock);
    173			ionic_stop_queues(lif);
    174			mutex_unlock(&lif->queue_lock);
    175		}
    176	}
    177
    178	clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
    179}
    180
    181void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
    182{
    183	struct ionic_deferred_work *work;
    184
    185	/* we only need one request outstanding at a time */
    186	if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
    187		return;
    188
    189	if (!can_sleep) {
    190		work = kzalloc(sizeof(*work), GFP_ATOMIC);
    191		if (!work) {
    192			clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
    193			return;
    194		}
    195
    196		work->type = IONIC_DW_TYPE_LINK_STATUS;
    197		ionic_lif_deferred_enqueue(&lif->deferred, work);
    198	} else {
    199		ionic_link_status_check(lif);
    200	}
    201}
    202
    203static irqreturn_t ionic_isr(int irq, void *data)
    204{
    205	struct napi_struct *napi = data;
    206
    207	napi_schedule_irqoff(napi);
    208
    209	return IRQ_HANDLED;
    210}
    211
    212static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
    213{
    214	struct ionic_intr_info *intr = &qcq->intr;
    215	struct device *dev = lif->ionic->dev;
    216	struct ionic_queue *q = &qcq->q;
    217	const char *name;
    218
    219	if (lif->registered)
    220		name = lif->netdev->name;
    221	else
    222		name = dev_name(dev);
    223
    224	snprintf(intr->name, sizeof(intr->name),
    225		 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
    226
    227	return devm_request_irq(dev, intr->vector, ionic_isr,
    228				0, intr->name, &qcq->napi);
    229}
    230
    231static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
    232{
    233	struct ionic *ionic = lif->ionic;
    234	int index;
    235
    236	index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
    237	if (index == ionic->nintrs) {
    238		netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
    239			    __func__, index, ionic->nintrs);
    240		return -ENOSPC;
    241	}
    242
    243	set_bit(index, ionic->intrs);
    244	ionic_intr_init(&ionic->idev, intr, index);
    245
    246	return 0;
    247}
    248
    249static void ionic_intr_free(struct ionic *ionic, int index)
    250{
    251	if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
    252		clear_bit(index, ionic->intrs);
    253}
    254
    255static int ionic_qcq_enable(struct ionic_qcq *qcq)
    256{
    257	struct ionic_queue *q = &qcq->q;
    258	struct ionic_lif *lif = q->lif;
    259	struct ionic_dev *idev;
    260	struct device *dev;
    261
    262	struct ionic_admin_ctx ctx = {
    263		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
    264		.cmd.q_control = {
    265			.opcode = IONIC_CMD_Q_CONTROL,
    266			.lif_index = cpu_to_le16(lif->index),
    267			.type = q->type,
    268			.index = cpu_to_le32(q->index),
    269			.oper = IONIC_Q_ENABLE,
    270		},
    271	};
    272
    273	idev = &lif->ionic->idev;
    274	dev = lif->ionic->dev;
    275
    276	dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
    277		ctx.cmd.q_control.index, ctx.cmd.q_control.type);
    278
    279	if (qcq->flags & IONIC_QCQ_F_INTR) {
    280		irq_set_affinity_hint(qcq->intr.vector,
    281				      &qcq->intr.affinity_mask);
    282		napi_enable(&qcq->napi);
    283		ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
    284		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
    285				IONIC_INTR_MASK_CLEAR);
    286	}
    287
    288	return ionic_adminq_post_wait(lif, &ctx);
    289}
    290
    291static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
    292{
    293	struct ionic_queue *q;
    294
    295	struct ionic_admin_ctx ctx = {
    296		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
    297		.cmd.q_control = {
    298			.opcode = IONIC_CMD_Q_CONTROL,
    299			.oper = IONIC_Q_DISABLE,
    300		},
    301	};
    302
    303	if (!qcq) {
    304		netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
    305		return -ENXIO;
    306	}
    307
    308	q = &qcq->q;
    309
    310	if (qcq->flags & IONIC_QCQ_F_INTR) {
    311		struct ionic_dev *idev = &lif->ionic->idev;
    312
    313		cancel_work_sync(&qcq->dim.work);
    314		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
    315				IONIC_INTR_MASK_SET);
    316		synchronize_irq(qcq->intr.vector);
    317		irq_set_affinity_hint(qcq->intr.vector, NULL);
    318		napi_disable(&qcq->napi);
    319	}
    320
    321	/* If there was a previous fw communcation error, don't bother with
    322	 * sending the adminq command and just return the same error value.
    323	 */
    324	if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
    325		return fw_err;
    326
    327	ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
    328	ctx.cmd.q_control.type = q->type;
    329	ctx.cmd.q_control.index = cpu_to_le32(q->index);
    330	dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
    331		ctx.cmd.q_control.index, ctx.cmd.q_control.type);
    332
    333	return ionic_adminq_post_wait(lif, &ctx);
    334}
    335
    336static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
    337{
    338	struct ionic_dev *idev = &lif->ionic->idev;
    339
    340	if (!qcq)
    341		return;
    342
    343	if (!(qcq->flags & IONIC_QCQ_F_INITED))
    344		return;
    345
    346	if (qcq->flags & IONIC_QCQ_F_INTR) {
    347		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
    348				IONIC_INTR_MASK_SET);
    349		netif_napi_del(&qcq->napi);
    350	}
    351
    352	qcq->flags &= ~IONIC_QCQ_F_INITED;
    353}
    354
    355static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
    356{
    357	if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
    358		return;
    359
    360	irq_set_affinity_hint(qcq->intr.vector, NULL);
    361	devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
    362	qcq->intr.vector = 0;
    363	ionic_intr_free(lif->ionic, qcq->intr.index);
    364	qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
    365}
    366
    367static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
    368{
    369	struct device *dev = lif->ionic->dev;
    370
    371	if (!qcq)
    372		return;
    373
    374	ionic_debugfs_del_qcq(qcq);
    375
    376	if (qcq->q_base) {
    377		dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
    378		qcq->q_base = NULL;
    379		qcq->q_base_pa = 0;
    380	}
    381
    382	if (qcq->cq_base) {
    383		dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
    384		qcq->cq_base = NULL;
    385		qcq->cq_base_pa = 0;
    386	}
    387
    388	if (qcq->sg_base) {
    389		dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
    390		qcq->sg_base = NULL;
    391		qcq->sg_base_pa = 0;
    392	}
    393
    394	ionic_qcq_intr_free(lif, qcq);
    395
    396	if (qcq->cq.info) {
    397		vfree(qcq->cq.info);
    398		qcq->cq.info = NULL;
    399	}
    400	if (qcq->q.info) {
    401		vfree(qcq->q.info);
    402		qcq->q.info = NULL;
    403	}
    404}
    405
    406static void ionic_qcqs_free(struct ionic_lif *lif)
    407{
    408	struct device *dev = lif->ionic->dev;
    409	struct ionic_qcq *adminqcq;
    410	unsigned long irqflags;
    411
    412	if (lif->notifyqcq) {
    413		ionic_qcq_free(lif, lif->notifyqcq);
    414		devm_kfree(dev, lif->notifyqcq);
    415		lif->notifyqcq = NULL;
    416	}
    417
    418	if (lif->adminqcq) {
    419		spin_lock_irqsave(&lif->adminq_lock, irqflags);
    420		adminqcq = READ_ONCE(lif->adminqcq);
    421		lif->adminqcq = NULL;
    422		spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
    423		if (adminqcq) {
    424			ionic_qcq_free(lif, adminqcq);
    425			devm_kfree(dev, adminqcq);
    426		}
    427	}
    428
    429	if (lif->rxqcqs) {
    430		devm_kfree(dev, lif->rxqstats);
    431		lif->rxqstats = NULL;
    432		devm_kfree(dev, lif->rxqcqs);
    433		lif->rxqcqs = NULL;
    434	}
    435
    436	if (lif->txqcqs) {
    437		devm_kfree(dev, lif->txqstats);
    438		lif->txqstats = NULL;
    439		devm_kfree(dev, lif->txqcqs);
    440		lif->txqcqs = NULL;
    441	}
    442}
    443
    444static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
    445				      struct ionic_qcq *n_qcq)
    446{
    447	if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
    448		ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
    449		n_qcq->flags &= ~IONIC_QCQ_F_INTR;
    450	}
    451
    452	n_qcq->intr.vector = src_qcq->intr.vector;
    453	n_qcq->intr.index = src_qcq->intr.index;
    454}
    455
    456static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
    457{
    458	int err;
    459
    460	if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
    461		qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
    462		return 0;
    463	}
    464
    465	err = ionic_intr_alloc(lif, &qcq->intr);
    466	if (err) {
    467		netdev_warn(lif->netdev, "no intr for %s: %d\n",
    468			    qcq->q.name, err);
    469		goto err_out;
    470	}
    471
    472	err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
    473	if (err < 0) {
    474		netdev_warn(lif->netdev, "no vector for %s: %d\n",
    475			    qcq->q.name, err);
    476		goto err_out_free_intr;
    477	}
    478	qcq->intr.vector = err;
    479	ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
    480			       IONIC_INTR_MASK_SET);
    481
    482	err = ionic_request_irq(lif, qcq);
    483	if (err) {
    484		netdev_warn(lif->netdev, "irq request failed %d\n", err);
    485		goto err_out_free_intr;
    486	}
    487
    488	/* try to get the irq on the local numa node first */
    489	qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
    490					     dev_to_node(lif->ionic->dev));
    491	if (qcq->intr.cpu != -1)
    492		cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
    493
    494	netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
    495	return 0;
    496
    497err_out_free_intr:
    498	ionic_intr_free(lif->ionic, qcq->intr.index);
    499err_out:
    500	return err;
    501}
    502
    503static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
    504			   unsigned int index,
    505			   const char *name, unsigned int flags,
    506			   unsigned int num_descs, unsigned int desc_size,
    507			   unsigned int cq_desc_size,
    508			   unsigned int sg_desc_size,
    509			   unsigned int pid, struct ionic_qcq **qcq)
    510{
    511	struct ionic_dev *idev = &lif->ionic->idev;
    512	struct device *dev = lif->ionic->dev;
    513	void *q_base, *cq_base, *sg_base;
    514	dma_addr_t cq_base_pa = 0;
    515	dma_addr_t sg_base_pa = 0;
    516	dma_addr_t q_base_pa = 0;
    517	struct ionic_qcq *new;
    518	int err;
    519
    520	*qcq = NULL;
    521
    522	new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
    523	if (!new) {
    524		netdev_err(lif->netdev, "Cannot allocate queue structure\n");
    525		err = -ENOMEM;
    526		goto err_out;
    527	}
    528
    529	new->q.dev = dev;
    530	new->flags = flags;
    531
    532	new->q.info = vzalloc(num_descs * sizeof(*new->q.info));
    533	if (!new->q.info) {
    534		netdev_err(lif->netdev, "Cannot allocate queue info\n");
    535		err = -ENOMEM;
    536		goto err_out_free_qcq;
    537	}
    538
    539	new->q.type = type;
    540	new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
    541
    542	err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
    543			   desc_size, sg_desc_size, pid);
    544	if (err) {
    545		netdev_err(lif->netdev, "Cannot initialize queue\n");
    546		goto err_out_free_q_info;
    547	}
    548
    549	err = ionic_alloc_qcq_interrupt(lif, new);
    550	if (err)
    551		goto err_out;
    552
    553	new->cq.info = vzalloc(num_descs * sizeof(*new->cq.info));
    554	if (!new->cq.info) {
    555		netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
    556		err = -ENOMEM;
    557		goto err_out_free_irq;
    558	}
    559
    560	err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
    561	if (err) {
    562		netdev_err(lif->netdev, "Cannot initialize completion queue\n");
    563		goto err_out_free_cq_info;
    564	}
    565
    566	if (flags & IONIC_QCQ_F_NOTIFYQ) {
    567		int q_size, cq_size;
    568
    569		/* q & cq need to be contiguous in case of notifyq */
    570		q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
    571		cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
    572
    573		new->q_size = PAGE_SIZE + q_size + cq_size;
    574		new->q_base = dma_alloc_coherent(dev, new->q_size,
    575						 &new->q_base_pa, GFP_KERNEL);
    576		if (!new->q_base) {
    577			netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
    578			err = -ENOMEM;
    579			goto err_out_free_cq_info;
    580		}
    581		q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
    582		q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
    583		ionic_q_map(&new->q, q_base, q_base_pa);
    584
    585		cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
    586		cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
    587		ionic_cq_map(&new->cq, cq_base, cq_base_pa);
    588		ionic_cq_bind(&new->cq, &new->q);
    589	} else {
    590		new->q_size = PAGE_SIZE + (num_descs * desc_size);
    591		new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
    592						 GFP_KERNEL);
    593		if (!new->q_base) {
    594			netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
    595			err = -ENOMEM;
    596			goto err_out_free_cq_info;
    597		}
    598		q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
    599		q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
    600		ionic_q_map(&new->q, q_base, q_base_pa);
    601
    602		new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
    603		new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
    604						  GFP_KERNEL);
    605		if (!new->cq_base) {
    606			netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
    607			err = -ENOMEM;
    608			goto err_out_free_q;
    609		}
    610		cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
    611		cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
    612		ionic_cq_map(&new->cq, cq_base, cq_base_pa);
    613		ionic_cq_bind(&new->cq, &new->q);
    614	}
    615
    616	if (flags & IONIC_QCQ_F_SG) {
    617		new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
    618		new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
    619						  GFP_KERNEL);
    620		if (!new->sg_base) {
    621			netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
    622			err = -ENOMEM;
    623			goto err_out_free_cq;
    624		}
    625		sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
    626		sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
    627		ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
    628	}
    629
    630	INIT_WORK(&new->dim.work, ionic_dim_work);
    631	new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
    632
    633	*qcq = new;
    634
    635	return 0;
    636
    637err_out_free_cq:
    638	dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
    639err_out_free_q:
    640	dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
    641err_out_free_cq_info:
    642	vfree(new->cq.info);
    643err_out_free_irq:
    644	if (flags & IONIC_QCQ_F_INTR) {
    645		devm_free_irq(dev, new->intr.vector, &new->napi);
    646		ionic_intr_free(lif->ionic, new->intr.index);
    647	}
    648err_out_free_q_info:
    649	vfree(new->q.info);
    650err_out_free_qcq:
    651	devm_kfree(dev, new);
    652err_out:
    653	dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
    654	return err;
    655}
    656
    657static int ionic_qcqs_alloc(struct ionic_lif *lif)
    658{
    659	struct device *dev = lif->ionic->dev;
    660	unsigned int flags;
    661	int err;
    662
    663	flags = IONIC_QCQ_F_INTR;
    664	err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
    665			      IONIC_ADMINQ_LENGTH,
    666			      sizeof(struct ionic_admin_cmd),
    667			      sizeof(struct ionic_admin_comp),
    668			      0, lif->kern_pid, &lif->adminqcq);
    669	if (err)
    670		return err;
    671	ionic_debugfs_add_qcq(lif, lif->adminqcq);
    672
    673	if (lif->ionic->nnqs_per_lif) {
    674		flags = IONIC_QCQ_F_NOTIFYQ;
    675		err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
    676				      flags, IONIC_NOTIFYQ_LENGTH,
    677				      sizeof(struct ionic_notifyq_cmd),
    678				      sizeof(union ionic_notifyq_comp),
    679				      0, lif->kern_pid, &lif->notifyqcq);
    680		if (err)
    681			goto err_out;
    682		ionic_debugfs_add_qcq(lif, lif->notifyqcq);
    683
    684		/* Let the notifyq ride on the adminq interrupt */
    685		ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
    686	}
    687
    688	err = -ENOMEM;
    689	lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
    690				   sizeof(*lif->txqcqs), GFP_KERNEL);
    691	if (!lif->txqcqs)
    692		goto err_out;
    693	lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
    694				   sizeof(*lif->rxqcqs), GFP_KERNEL);
    695	if (!lif->rxqcqs)
    696		goto err_out;
    697
    698	lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
    699				     sizeof(*lif->txqstats), GFP_KERNEL);
    700	if (!lif->txqstats)
    701		goto err_out;
    702	lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
    703				     sizeof(*lif->rxqstats), GFP_KERNEL);
    704	if (!lif->rxqstats)
    705		goto err_out;
    706
    707	return 0;
    708
    709err_out:
    710	ionic_qcqs_free(lif);
    711	return err;
    712}
    713
    714static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
    715{
    716	qcq->q.tail_idx = 0;
    717	qcq->q.head_idx = 0;
    718	qcq->cq.tail_idx = 0;
    719	qcq->cq.done_color = 1;
    720	memset(qcq->q_base, 0, qcq->q_size);
    721	memset(qcq->cq_base, 0, qcq->cq_size);
    722	memset(qcq->sg_base, 0, qcq->sg_size);
    723}
    724
    725static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
    726{
    727	struct device *dev = lif->ionic->dev;
    728	struct ionic_queue *q = &qcq->q;
    729	struct ionic_cq *cq = &qcq->cq;
    730	struct ionic_admin_ctx ctx = {
    731		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
    732		.cmd.q_init = {
    733			.opcode = IONIC_CMD_Q_INIT,
    734			.lif_index = cpu_to_le16(lif->index),
    735			.type = q->type,
    736			.ver = lif->qtype_info[q->type].version,
    737			.index = cpu_to_le32(q->index),
    738			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
    739					     IONIC_QINIT_F_SG),
    740			.pid = cpu_to_le16(q->pid),
    741			.ring_size = ilog2(q->num_descs),
    742			.ring_base = cpu_to_le64(q->base_pa),
    743			.cq_ring_base = cpu_to_le64(cq->base_pa),
    744			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
    745			.features = cpu_to_le64(q->features),
    746		},
    747	};
    748	unsigned int intr_index;
    749	int err;
    750
    751	intr_index = qcq->intr.index;
    752
    753	ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
    754
    755	dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
    756	dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
    757	dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
    758	dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
    759	dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
    760	dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
    761	dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
    762
    763	ionic_qcq_sanitize(qcq);
    764
    765	err = ionic_adminq_post_wait(lif, &ctx);
    766	if (err)
    767		return err;
    768
    769	q->hw_type = ctx.comp.q_init.hw_type;
    770	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
    771	q->dbval = IONIC_DBELL_QID(q->hw_index);
    772
    773	dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
    774	dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
    775
    776	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
    777		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
    778			       NAPI_POLL_WEIGHT);
    779
    780	qcq->flags |= IONIC_QCQ_F_INITED;
    781
    782	return 0;
    783}
    784
    785static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
    786{
    787	struct device *dev = lif->ionic->dev;
    788	struct ionic_queue *q = &qcq->q;
    789	struct ionic_cq *cq = &qcq->cq;
    790	struct ionic_admin_ctx ctx = {
    791		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
    792		.cmd.q_init = {
    793			.opcode = IONIC_CMD_Q_INIT,
    794			.lif_index = cpu_to_le16(lif->index),
    795			.type = q->type,
    796			.ver = lif->qtype_info[q->type].version,
    797			.index = cpu_to_le32(q->index),
    798			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
    799					     IONIC_QINIT_F_SG),
    800			.intr_index = cpu_to_le16(cq->bound_intr->index),
    801			.pid = cpu_to_le16(q->pid),
    802			.ring_size = ilog2(q->num_descs),
    803			.ring_base = cpu_to_le64(q->base_pa),
    804			.cq_ring_base = cpu_to_le64(cq->base_pa),
    805			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
    806			.features = cpu_to_le64(q->features),
    807		},
    808	};
    809	int err;
    810
    811	dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
    812	dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
    813	dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
    814	dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
    815	dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
    816	dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
    817	dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
    818
    819	ionic_qcq_sanitize(qcq);
    820
    821	err = ionic_adminq_post_wait(lif, &ctx);
    822	if (err)
    823		return err;
    824
    825	q->hw_type = ctx.comp.q_init.hw_type;
    826	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
    827	q->dbval = IONIC_DBELL_QID(q->hw_index);
    828
    829	dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
    830	dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
    831
    832	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
    833		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
    834			       NAPI_POLL_WEIGHT);
    835	else
    836		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
    837			       NAPI_POLL_WEIGHT);
    838
    839	qcq->flags |= IONIC_QCQ_F_INITED;
    840
    841	return 0;
    842}
    843
    844int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
    845{
    846	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
    847	unsigned int txq_i, flags;
    848	struct ionic_qcq *txq;
    849	u64 features;
    850	int err;
    851
    852	if (lif->hwstamp_txq)
    853		return 0;
    854
    855	features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
    856
    857	num_desc = IONIC_MIN_TXRX_DESC;
    858	desc_sz = sizeof(struct ionic_txq_desc);
    859	comp_sz = 2 * sizeof(struct ionic_txq_comp);
    860
    861	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
    862	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
    863		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
    864	else
    865		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
    866
    867	txq_i = lif->ionic->ntxqs_per_lif;
    868	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
    869
    870	err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
    871			      num_desc, desc_sz, comp_sz, sg_desc_sz,
    872			      lif->kern_pid, &txq);
    873	if (err)
    874		goto err_qcq_alloc;
    875
    876	txq->q.features = features;
    877
    878	ionic_link_qcq_interrupts(lif->adminqcq, txq);
    879	ionic_debugfs_add_qcq(lif, txq);
    880
    881	lif->hwstamp_txq = txq;
    882
    883	if (netif_running(lif->netdev)) {
    884		err = ionic_lif_txq_init(lif, txq);
    885		if (err)
    886			goto err_qcq_init;
    887
    888		if (test_bit(IONIC_LIF_F_UP, lif->state)) {
    889			err = ionic_qcq_enable(txq);
    890			if (err)
    891				goto err_qcq_enable;
    892		}
    893	}
    894
    895	return 0;
    896
    897err_qcq_enable:
    898	ionic_lif_qcq_deinit(lif, txq);
    899err_qcq_init:
    900	lif->hwstamp_txq = NULL;
    901	ionic_debugfs_del_qcq(txq);
    902	ionic_qcq_free(lif, txq);
    903	devm_kfree(lif->ionic->dev, txq);
    904err_qcq_alloc:
    905	return err;
    906}
    907
    908int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
    909{
    910	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
    911	unsigned int rxq_i, flags;
    912	struct ionic_qcq *rxq;
    913	u64 features;
    914	int err;
    915
    916	if (lif->hwstamp_rxq)
    917		return 0;
    918
    919	features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
    920
    921	num_desc = IONIC_MIN_TXRX_DESC;
    922	desc_sz = sizeof(struct ionic_rxq_desc);
    923	comp_sz = 2 * sizeof(struct ionic_rxq_comp);
    924	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
    925
    926	rxq_i = lif->ionic->nrxqs_per_lif;
    927	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
    928
    929	err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
    930			      num_desc, desc_sz, comp_sz, sg_desc_sz,
    931			      lif->kern_pid, &rxq);
    932	if (err)
    933		goto err_qcq_alloc;
    934
    935	rxq->q.features = features;
    936
    937	ionic_link_qcq_interrupts(lif->adminqcq, rxq);
    938	ionic_debugfs_add_qcq(lif, rxq);
    939
    940	lif->hwstamp_rxq = rxq;
    941
    942	if (netif_running(lif->netdev)) {
    943		err = ionic_lif_rxq_init(lif, rxq);
    944		if (err)
    945			goto err_qcq_init;
    946
    947		if (test_bit(IONIC_LIF_F_UP, lif->state)) {
    948			ionic_rx_fill(&rxq->q);
    949			err = ionic_qcq_enable(rxq);
    950			if (err)
    951				goto err_qcq_enable;
    952		}
    953	}
    954
    955	return 0;
    956
    957err_qcq_enable:
    958	ionic_lif_qcq_deinit(lif, rxq);
    959err_qcq_init:
    960	lif->hwstamp_rxq = NULL;
    961	ionic_debugfs_del_qcq(rxq);
    962	ionic_qcq_free(lif, rxq);
    963	devm_kfree(lif->ionic->dev, rxq);
    964err_qcq_alloc:
    965	return err;
    966}
    967
    968int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
    969{
    970	struct ionic_queue_params qparam;
    971
    972	ionic_init_queue_params(lif, &qparam);
    973
    974	if (rx_all)
    975		qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
    976	else
    977		qparam.rxq_features = 0;
    978
    979	/* if we're not running, just set the values and return */
    980	if (!netif_running(lif->netdev)) {
    981		lif->rxq_features = qparam.rxq_features;
    982		return 0;
    983	}
    984
    985	return ionic_reconfigure_queues(lif, &qparam);
    986}
    987
    988int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
    989{
    990	struct ionic_admin_ctx ctx = {
    991		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
    992		.cmd.lif_setattr = {
    993			.opcode = IONIC_CMD_LIF_SETATTR,
    994			.index = cpu_to_le16(lif->index),
    995			.attr = IONIC_LIF_ATTR_TXSTAMP,
    996			.txstamp_mode = cpu_to_le16(txstamp_mode),
    997		},
    998	};
    999
   1000	return ionic_adminq_post_wait(lif, &ctx);
   1001}
   1002
   1003static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
   1004{
   1005	struct ionic_admin_ctx ctx = {
   1006		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1007		.cmd.rx_filter_del = {
   1008			.opcode = IONIC_CMD_RX_FILTER_DEL,
   1009			.lif_index = cpu_to_le16(lif->index),
   1010		},
   1011	};
   1012	struct ionic_rx_filter *f;
   1013	u32 filter_id;
   1014	int err;
   1015
   1016	spin_lock_bh(&lif->rx_filters.lock);
   1017
   1018	f = ionic_rx_filter_rxsteer(lif);
   1019	if (!f) {
   1020		spin_unlock_bh(&lif->rx_filters.lock);
   1021		return;
   1022	}
   1023
   1024	filter_id = f->filter_id;
   1025	ionic_rx_filter_free(lif, f);
   1026
   1027	spin_unlock_bh(&lif->rx_filters.lock);
   1028
   1029	netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
   1030
   1031	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
   1032
   1033	err = ionic_adminq_post_wait(lif, &ctx);
   1034	if (err && err != -EEXIST)
   1035		netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
   1036}
   1037
   1038static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
   1039{
   1040	struct ionic_admin_ctx ctx = {
   1041		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1042		.cmd.rx_filter_add = {
   1043			.opcode = IONIC_CMD_RX_FILTER_ADD,
   1044			.lif_index = cpu_to_le16(lif->index),
   1045			.match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
   1046			.pkt_class = cpu_to_le64(pkt_class),
   1047		},
   1048	};
   1049	u8 qtype;
   1050	u32 qid;
   1051	int err;
   1052
   1053	if (!lif->hwstamp_rxq)
   1054		return -EINVAL;
   1055
   1056	qtype = lif->hwstamp_rxq->q.type;
   1057	ctx.cmd.rx_filter_add.qtype = qtype;
   1058
   1059	qid = lif->hwstamp_rxq->q.index;
   1060	ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
   1061
   1062	netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
   1063	err = ionic_adminq_post_wait(lif, &ctx);
   1064	if (err && err != -EEXIST)
   1065		return err;
   1066
   1067	spin_lock_bh(&lif->rx_filters.lock);
   1068	err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
   1069	spin_unlock_bh(&lif->rx_filters.lock);
   1070
   1071	return err;
   1072}
   1073
   1074int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
   1075{
   1076	ionic_lif_del_hwstamp_rxfilt(lif);
   1077
   1078	if (!pkt_class)
   1079		return 0;
   1080
   1081	return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
   1082}
   1083
   1084static bool ionic_notifyq_service(struct ionic_cq *cq,
   1085				  struct ionic_cq_info *cq_info)
   1086{
   1087	union ionic_notifyq_comp *comp = cq_info->cq_desc;
   1088	struct ionic_deferred_work *work;
   1089	struct net_device *netdev;
   1090	struct ionic_queue *q;
   1091	struct ionic_lif *lif;
   1092	u64 eid;
   1093
   1094	q = cq->bound_q;
   1095	lif = q->info[0].cb_arg;
   1096	netdev = lif->netdev;
   1097	eid = le64_to_cpu(comp->event.eid);
   1098
   1099	/* Have we run out of new completions to process? */
   1100	if ((s64)(eid - lif->last_eid) <= 0)
   1101		return false;
   1102
   1103	lif->last_eid = eid;
   1104
   1105	dev_dbg(lif->ionic->dev, "notifyq event:\n");
   1106	dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
   1107			 comp, sizeof(*comp), true);
   1108
   1109	switch (le16_to_cpu(comp->event.ecode)) {
   1110	case IONIC_EVENT_LINK_CHANGE:
   1111		ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
   1112		break;
   1113	case IONIC_EVENT_RESET:
   1114		if (lif->ionic->idev.fw_status_ready &&
   1115		    !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
   1116		    !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
   1117			work = kzalloc(sizeof(*work), GFP_ATOMIC);
   1118			if (!work) {
   1119				netdev_err(lif->netdev, "Reset event dropped\n");
   1120				clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
   1121			} else {
   1122				work->type = IONIC_DW_TYPE_LIF_RESET;
   1123				ionic_lif_deferred_enqueue(&lif->deferred, work);
   1124			}
   1125		}
   1126		break;
   1127	default:
   1128		netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
   1129			    comp->event.ecode, eid);
   1130		break;
   1131	}
   1132
   1133	return true;
   1134}
   1135
   1136static bool ionic_adminq_service(struct ionic_cq *cq,
   1137				 struct ionic_cq_info *cq_info)
   1138{
   1139	struct ionic_admin_comp *comp = cq_info->cq_desc;
   1140
   1141	if (!color_match(comp->color, cq->done_color))
   1142		return false;
   1143
   1144	ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
   1145
   1146	return true;
   1147}
   1148
   1149static int ionic_adminq_napi(struct napi_struct *napi, int budget)
   1150{
   1151	struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
   1152	struct ionic_lif *lif = napi_to_cq(napi)->lif;
   1153	struct ionic_dev *idev = &lif->ionic->idev;
   1154	unsigned long irqflags;
   1155	unsigned int flags = 0;
   1156	int rx_work = 0;
   1157	int tx_work = 0;
   1158	int n_work = 0;
   1159	int a_work = 0;
   1160	int work_done;
   1161	int credits;
   1162
   1163	if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
   1164		n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
   1165					  ionic_notifyq_service, NULL, NULL);
   1166
   1167	spin_lock_irqsave(&lif->adminq_lock, irqflags);
   1168	if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
   1169		a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
   1170					  ionic_adminq_service, NULL, NULL);
   1171	spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
   1172
   1173	if (lif->hwstamp_rxq)
   1174		rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
   1175					   ionic_rx_service, NULL, NULL);
   1176
   1177	if (lif->hwstamp_txq)
   1178		tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
   1179					   ionic_tx_service, NULL, NULL);
   1180
   1181	work_done = max(max(n_work, a_work), max(rx_work, tx_work));
   1182	if (work_done < budget && napi_complete_done(napi, work_done)) {
   1183		flags |= IONIC_INTR_CRED_UNMASK;
   1184		intr->rearm_count++;
   1185	}
   1186
   1187	if (work_done || flags) {
   1188		flags |= IONIC_INTR_CRED_RESET_COALESCE;
   1189		credits = n_work + a_work + rx_work + tx_work;
   1190		ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
   1191	}
   1192
   1193	return work_done;
   1194}
   1195
   1196void ionic_get_stats64(struct net_device *netdev,
   1197		       struct rtnl_link_stats64 *ns)
   1198{
   1199	struct ionic_lif *lif = netdev_priv(netdev);
   1200	struct ionic_lif_stats *ls;
   1201
   1202	memset(ns, 0, sizeof(*ns));
   1203	ls = &lif->info->stats;
   1204
   1205	ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
   1206			 le64_to_cpu(ls->rx_mcast_packets) +
   1207			 le64_to_cpu(ls->rx_bcast_packets);
   1208
   1209	ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
   1210			 le64_to_cpu(ls->tx_mcast_packets) +
   1211			 le64_to_cpu(ls->tx_bcast_packets);
   1212
   1213	ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
   1214		       le64_to_cpu(ls->rx_mcast_bytes) +
   1215		       le64_to_cpu(ls->rx_bcast_bytes);
   1216
   1217	ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
   1218		       le64_to_cpu(ls->tx_mcast_bytes) +
   1219		       le64_to_cpu(ls->tx_bcast_bytes);
   1220
   1221	ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
   1222			 le64_to_cpu(ls->rx_mcast_drop_packets) +
   1223			 le64_to_cpu(ls->rx_bcast_drop_packets);
   1224
   1225	ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
   1226			 le64_to_cpu(ls->tx_mcast_drop_packets) +
   1227			 le64_to_cpu(ls->tx_bcast_drop_packets);
   1228
   1229	ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
   1230
   1231	ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
   1232
   1233	ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
   1234			       le64_to_cpu(ls->rx_queue_disabled) +
   1235			       le64_to_cpu(ls->rx_desc_fetch_error) +
   1236			       le64_to_cpu(ls->rx_desc_data_error);
   1237
   1238	ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
   1239				le64_to_cpu(ls->tx_queue_disabled) +
   1240				le64_to_cpu(ls->tx_desc_fetch_error) +
   1241				le64_to_cpu(ls->tx_desc_data_error);
   1242
   1243	ns->rx_errors = ns->rx_over_errors +
   1244			ns->rx_missed_errors;
   1245
   1246	ns->tx_errors = ns->tx_aborted_errors;
   1247}
   1248
   1249static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
   1250{
   1251	return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
   1252}
   1253
   1254static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
   1255{
   1256	/* Don't delete our own address from the uc list */
   1257	if (ether_addr_equal(addr, netdev->dev_addr))
   1258		return 0;
   1259
   1260	return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
   1261}
   1262
   1263void ionic_lif_rx_mode(struct ionic_lif *lif)
   1264{
   1265	struct net_device *netdev = lif->netdev;
   1266	unsigned int nfilters;
   1267	unsigned int nd_flags;
   1268	char buf[128];
   1269	u16 rx_mode;
   1270	int i;
   1271#define REMAIN(__x) (sizeof(buf) - (__x))
   1272
   1273	mutex_lock(&lif->config_lock);
   1274
   1275	/* grab the flags once for local use */
   1276	nd_flags = netdev->flags;
   1277
   1278	rx_mode = IONIC_RX_MODE_F_UNICAST;
   1279	rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
   1280	rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
   1281	rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
   1282	rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
   1283
   1284	/* sync the filters */
   1285	ionic_rx_filter_sync(lif);
   1286
   1287	/* check for overflow state
   1288	 *    if so, we track that we overflowed and enable NIC PROMISC
   1289	 *    else if the overflow is set and not needed
   1290	 *       we remove our overflow flag and check the netdev flags
   1291	 *       to see if we can disable NIC PROMISC
   1292	 */
   1293	nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
   1294
   1295	if (((lif->nucast + lif->nmcast) >= nfilters) ||
   1296	    (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
   1297		rx_mode |= IONIC_RX_MODE_F_PROMISC;
   1298		rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
   1299	} else {
   1300		if (!(nd_flags & IFF_PROMISC))
   1301			rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
   1302		if (!(nd_flags & IFF_ALLMULTI))
   1303			rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
   1304	}
   1305
   1306	i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
   1307		      lif->rx_mode, rx_mode);
   1308	if (rx_mode & IONIC_RX_MODE_F_UNICAST)
   1309		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
   1310	if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
   1311		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
   1312	if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
   1313		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
   1314	if (rx_mode & IONIC_RX_MODE_F_PROMISC)
   1315		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
   1316	if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
   1317		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
   1318	if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
   1319		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
   1320	netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
   1321
   1322	if (lif->rx_mode != rx_mode) {
   1323		struct ionic_admin_ctx ctx = {
   1324			.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1325			.cmd.rx_mode_set = {
   1326				.opcode = IONIC_CMD_RX_MODE_SET,
   1327				.lif_index = cpu_to_le16(lif->index),
   1328			},
   1329		};
   1330		int err;
   1331
   1332		ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
   1333		err = ionic_adminq_post_wait(lif, &ctx);
   1334		if (err)
   1335			netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
   1336				    rx_mode, err);
   1337		else
   1338			lif->rx_mode = rx_mode;
   1339	}
   1340
   1341	mutex_unlock(&lif->config_lock);
   1342}
   1343
   1344static void ionic_ndo_set_rx_mode(struct net_device *netdev)
   1345{
   1346	struct ionic_lif *lif = netdev_priv(netdev);
   1347	struct ionic_deferred_work *work;
   1348
   1349	/* Sync the kernel filter list with the driver filter list */
   1350	__dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
   1351	__dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
   1352
   1353	/* Shove off the rest of the rxmode work to the work task
   1354	 * which will include syncing the filters to the firmware.
   1355	 */
   1356	work = kzalloc(sizeof(*work), GFP_ATOMIC);
   1357	if (!work) {
   1358		netdev_err(lif->netdev, "rxmode change dropped\n");
   1359		return;
   1360	}
   1361	work->type = IONIC_DW_TYPE_RX_MODE;
   1362	netdev_dbg(lif->netdev, "deferred: rx_mode\n");
   1363	ionic_lif_deferred_enqueue(&lif->deferred, work);
   1364}
   1365
   1366static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
   1367{
   1368	u64 wanted = 0;
   1369
   1370	if (features & NETIF_F_HW_VLAN_CTAG_TX)
   1371		wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
   1372	if (features & NETIF_F_HW_VLAN_CTAG_RX)
   1373		wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
   1374	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
   1375		wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
   1376	if (features & NETIF_F_RXHASH)
   1377		wanted |= IONIC_ETH_HW_RX_HASH;
   1378	if (features & NETIF_F_RXCSUM)
   1379		wanted |= IONIC_ETH_HW_RX_CSUM;
   1380	if (features & NETIF_F_SG)
   1381		wanted |= IONIC_ETH_HW_TX_SG;
   1382	if (features & NETIF_F_HW_CSUM)
   1383		wanted |= IONIC_ETH_HW_TX_CSUM;
   1384	if (features & NETIF_F_TSO)
   1385		wanted |= IONIC_ETH_HW_TSO;
   1386	if (features & NETIF_F_TSO6)
   1387		wanted |= IONIC_ETH_HW_TSO_IPV6;
   1388	if (features & NETIF_F_TSO_ECN)
   1389		wanted |= IONIC_ETH_HW_TSO_ECN;
   1390	if (features & NETIF_F_GSO_GRE)
   1391		wanted |= IONIC_ETH_HW_TSO_GRE;
   1392	if (features & NETIF_F_GSO_GRE_CSUM)
   1393		wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
   1394	if (features & NETIF_F_GSO_IPXIP4)
   1395		wanted |= IONIC_ETH_HW_TSO_IPXIP4;
   1396	if (features & NETIF_F_GSO_IPXIP6)
   1397		wanted |= IONIC_ETH_HW_TSO_IPXIP6;
   1398	if (features & NETIF_F_GSO_UDP_TUNNEL)
   1399		wanted |= IONIC_ETH_HW_TSO_UDP;
   1400	if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
   1401		wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
   1402
   1403	return cpu_to_le64(wanted);
   1404}
   1405
   1406static int ionic_set_nic_features(struct ionic_lif *lif,
   1407				  netdev_features_t features)
   1408{
   1409	struct device *dev = lif->ionic->dev;
   1410	struct ionic_admin_ctx ctx = {
   1411		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1412		.cmd.lif_setattr = {
   1413			.opcode = IONIC_CMD_LIF_SETATTR,
   1414			.index = cpu_to_le16(lif->index),
   1415			.attr = IONIC_LIF_ATTR_FEATURES,
   1416		},
   1417	};
   1418	u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
   1419			 IONIC_ETH_HW_VLAN_RX_STRIP |
   1420			 IONIC_ETH_HW_VLAN_RX_FILTER;
   1421	u64 old_hw_features;
   1422	int err;
   1423
   1424	ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
   1425
   1426	if (lif->phc)
   1427		ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
   1428
   1429	err = ionic_adminq_post_wait(lif, &ctx);
   1430	if (err)
   1431		return err;
   1432
   1433	old_hw_features = lif->hw_features;
   1434	lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
   1435				       ctx.comp.lif_setattr.features);
   1436
   1437	if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
   1438		ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
   1439
   1440	if ((vlan_flags & features) &&
   1441	    !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
   1442		dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
   1443
   1444	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
   1445		dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
   1446	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
   1447		dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
   1448	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
   1449		dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
   1450	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
   1451		dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
   1452	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
   1453		dev_dbg(dev, "feature ETH_HW_TX_SG\n");
   1454	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
   1455		dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
   1456	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
   1457		dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
   1458	if (lif->hw_features & IONIC_ETH_HW_TSO)
   1459		dev_dbg(dev, "feature ETH_HW_TSO\n");
   1460	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
   1461		dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
   1462	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
   1463		dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
   1464	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
   1465		dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
   1466	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
   1467		dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
   1468	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
   1469		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
   1470	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
   1471		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
   1472	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
   1473		dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
   1474	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
   1475		dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
   1476	if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
   1477		dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
   1478
   1479	return 0;
   1480}
   1481
   1482static int ionic_init_nic_features(struct ionic_lif *lif)
   1483{
   1484	struct net_device *netdev = lif->netdev;
   1485	netdev_features_t features;
   1486	int err;
   1487
   1488	/* set up what we expect to support by default */
   1489	features = NETIF_F_HW_VLAN_CTAG_TX |
   1490		   NETIF_F_HW_VLAN_CTAG_RX |
   1491		   NETIF_F_HW_VLAN_CTAG_FILTER |
   1492		   NETIF_F_SG |
   1493		   NETIF_F_HW_CSUM |
   1494		   NETIF_F_RXCSUM |
   1495		   NETIF_F_TSO |
   1496		   NETIF_F_TSO6 |
   1497		   NETIF_F_TSO_ECN;
   1498
   1499	if (lif->nxqs > 1)
   1500		features |= NETIF_F_RXHASH;
   1501
   1502	err = ionic_set_nic_features(lif, features);
   1503	if (err)
   1504		return err;
   1505
   1506	/* tell the netdev what we actually can support */
   1507	netdev->features |= NETIF_F_HIGHDMA;
   1508
   1509	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
   1510		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
   1511	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
   1512		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
   1513	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
   1514		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
   1515	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
   1516		netdev->hw_features |= NETIF_F_RXHASH;
   1517	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
   1518		netdev->hw_features |= NETIF_F_SG;
   1519
   1520	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
   1521		netdev->hw_enc_features |= NETIF_F_HW_CSUM;
   1522	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
   1523		netdev->hw_enc_features |= NETIF_F_RXCSUM;
   1524	if (lif->hw_features & IONIC_ETH_HW_TSO)
   1525		netdev->hw_enc_features |= NETIF_F_TSO;
   1526	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
   1527		netdev->hw_enc_features |= NETIF_F_TSO6;
   1528	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
   1529		netdev->hw_enc_features |= NETIF_F_TSO_ECN;
   1530	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
   1531		netdev->hw_enc_features |= NETIF_F_GSO_GRE;
   1532	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
   1533		netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
   1534	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
   1535		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
   1536	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
   1537		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
   1538	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
   1539		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
   1540	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
   1541		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
   1542
   1543	netdev->hw_features |= netdev->hw_enc_features;
   1544	netdev->features |= netdev->hw_features;
   1545	netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
   1546
   1547	netdev->priv_flags |= IFF_UNICAST_FLT |
   1548			      IFF_LIVE_ADDR_CHANGE;
   1549
   1550	return 0;
   1551}
   1552
   1553static int ionic_set_features(struct net_device *netdev,
   1554			      netdev_features_t features)
   1555{
   1556	struct ionic_lif *lif = netdev_priv(netdev);
   1557	int err;
   1558
   1559	netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
   1560		   __func__, (u64)lif->netdev->features, (u64)features);
   1561
   1562	err = ionic_set_nic_features(lif, features);
   1563
   1564	return err;
   1565}
   1566
   1567static int ionic_set_mac_address(struct net_device *netdev, void *sa)
   1568{
   1569	struct sockaddr *addr = sa;
   1570	u8 *mac;
   1571	int err;
   1572
   1573	mac = (u8 *)addr->sa_data;
   1574	if (ether_addr_equal(netdev->dev_addr, mac))
   1575		return 0;
   1576
   1577	err = eth_prepare_mac_addr_change(netdev, addr);
   1578	if (err)
   1579		return err;
   1580
   1581	if (!is_zero_ether_addr(netdev->dev_addr)) {
   1582		netdev_info(netdev, "deleting mac addr %pM\n",
   1583			    netdev->dev_addr);
   1584		ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
   1585	}
   1586
   1587	eth_commit_mac_addr_change(netdev, addr);
   1588	netdev_info(netdev, "updating mac addr %pM\n", mac);
   1589
   1590	return ionic_lif_addr_add(netdev_priv(netdev), mac);
   1591}
   1592
   1593static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
   1594{
   1595	/* Stop and clean the queues before reconfiguration */
   1596	netif_device_detach(lif->netdev);
   1597	ionic_stop_queues(lif);
   1598	ionic_txrx_deinit(lif);
   1599}
   1600
   1601static int ionic_start_queues_reconfig(struct ionic_lif *lif)
   1602{
   1603	int err;
   1604
   1605	/* Re-init the queues after reconfiguration */
   1606
   1607	/* The only way txrx_init can fail here is if communication
   1608	 * with FW is suddenly broken.  There's not much we can do
   1609	 * at this point - error messages have already been printed,
   1610	 * so we can continue on and the user can eventually do a
   1611	 * DOWN and UP to try to reset and clear the issue.
   1612	 */
   1613	err = ionic_txrx_init(lif);
   1614	ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
   1615	netif_device_attach(lif->netdev);
   1616
   1617	return err;
   1618}
   1619
   1620static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
   1621{
   1622	struct ionic_lif *lif = netdev_priv(netdev);
   1623	struct ionic_admin_ctx ctx = {
   1624		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1625		.cmd.lif_setattr = {
   1626			.opcode = IONIC_CMD_LIF_SETATTR,
   1627			.index = cpu_to_le16(lif->index),
   1628			.attr = IONIC_LIF_ATTR_MTU,
   1629			.mtu = cpu_to_le32(new_mtu),
   1630		},
   1631	};
   1632	int err;
   1633
   1634	err = ionic_adminq_post_wait(lif, &ctx);
   1635	if (err)
   1636		return err;
   1637
   1638	/* if we're not running, nothing more to do */
   1639	if (!netif_running(netdev)) {
   1640		netdev->mtu = new_mtu;
   1641		return 0;
   1642	}
   1643
   1644	mutex_lock(&lif->queue_lock);
   1645	ionic_stop_queues_reconfig(lif);
   1646	netdev->mtu = new_mtu;
   1647	err = ionic_start_queues_reconfig(lif);
   1648	mutex_unlock(&lif->queue_lock);
   1649
   1650	return err;
   1651}
   1652
   1653static void ionic_tx_timeout_work(struct work_struct *ws)
   1654{
   1655	struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
   1656
   1657	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
   1658		return;
   1659
   1660	/* if we were stopped before this scheduled job was launched,
   1661	 * don't bother the queues as they are already stopped.
   1662	 */
   1663	if (!netif_running(lif->netdev))
   1664		return;
   1665
   1666	mutex_lock(&lif->queue_lock);
   1667	ionic_stop_queues_reconfig(lif);
   1668	ionic_start_queues_reconfig(lif);
   1669	mutex_unlock(&lif->queue_lock);
   1670}
   1671
   1672static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
   1673{
   1674	struct ionic_lif *lif = netdev_priv(netdev);
   1675
   1676	netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
   1677	schedule_work(&lif->tx_timeout_work);
   1678}
   1679
   1680static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
   1681				 u16 vid)
   1682{
   1683	struct ionic_lif *lif = netdev_priv(netdev);
   1684	int err;
   1685
   1686	err = ionic_lif_vlan_add(lif, vid);
   1687	if (err)
   1688		return err;
   1689
   1690	ionic_lif_rx_mode(lif);
   1691
   1692	return 0;
   1693}
   1694
   1695static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
   1696				  u16 vid)
   1697{
   1698	struct ionic_lif *lif = netdev_priv(netdev);
   1699	int err;
   1700
   1701	err = ionic_lif_vlan_del(lif, vid);
   1702	if (err)
   1703		return err;
   1704
   1705	ionic_lif_rx_mode(lif);
   1706
   1707	return 0;
   1708}
   1709
   1710int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
   1711			 const u8 *key, const u32 *indir)
   1712{
   1713	struct ionic_admin_ctx ctx = {
   1714		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1715		.cmd.lif_setattr = {
   1716			.opcode = IONIC_CMD_LIF_SETATTR,
   1717			.attr = IONIC_LIF_ATTR_RSS,
   1718			.rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
   1719		},
   1720	};
   1721	unsigned int i, tbl_sz;
   1722
   1723	if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
   1724		lif->rss_types = types;
   1725		ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
   1726	}
   1727
   1728	if (key)
   1729		memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
   1730
   1731	if (indir) {
   1732		tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
   1733		for (i = 0; i < tbl_sz; i++)
   1734			lif->rss_ind_tbl[i] = indir[i];
   1735	}
   1736
   1737	memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
   1738	       IONIC_RSS_HASH_KEY_SIZE);
   1739
   1740	return ionic_adminq_post_wait(lif, &ctx);
   1741}
   1742
   1743static int ionic_lif_rss_init(struct ionic_lif *lif)
   1744{
   1745	unsigned int tbl_sz;
   1746	unsigned int i;
   1747
   1748	lif->rss_types = IONIC_RSS_TYPE_IPV4     |
   1749			 IONIC_RSS_TYPE_IPV4_TCP |
   1750			 IONIC_RSS_TYPE_IPV4_UDP |
   1751			 IONIC_RSS_TYPE_IPV6     |
   1752			 IONIC_RSS_TYPE_IPV6_TCP |
   1753			 IONIC_RSS_TYPE_IPV6_UDP;
   1754
   1755	/* Fill indirection table with 'default' values */
   1756	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
   1757	for (i = 0; i < tbl_sz; i++)
   1758		lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
   1759
   1760	return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
   1761}
   1762
   1763static void ionic_lif_rss_deinit(struct ionic_lif *lif)
   1764{
   1765	int tbl_sz;
   1766
   1767	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
   1768	memset(lif->rss_ind_tbl, 0, tbl_sz);
   1769	memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
   1770
   1771	ionic_lif_rss_config(lif, 0x0, NULL, NULL);
   1772}
   1773
   1774static void ionic_lif_quiesce(struct ionic_lif *lif)
   1775{
   1776	struct ionic_admin_ctx ctx = {
   1777		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   1778		.cmd.lif_setattr = {
   1779			.opcode = IONIC_CMD_LIF_SETATTR,
   1780			.index = cpu_to_le16(lif->index),
   1781			.attr = IONIC_LIF_ATTR_STATE,
   1782			.state = IONIC_LIF_QUIESCE,
   1783		},
   1784	};
   1785	int err;
   1786
   1787	err = ionic_adminq_post_wait(lif, &ctx);
   1788	if (err)
   1789		netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
   1790}
   1791
   1792static void ionic_txrx_disable(struct ionic_lif *lif)
   1793{
   1794	unsigned int i;
   1795	int err = 0;
   1796
   1797	if (lif->txqcqs) {
   1798		for (i = 0; i < lif->nxqs; i++)
   1799			err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
   1800	}
   1801
   1802	if (lif->hwstamp_txq)
   1803		err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
   1804
   1805	if (lif->rxqcqs) {
   1806		for (i = 0; i < lif->nxqs; i++)
   1807			err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
   1808	}
   1809
   1810	if (lif->hwstamp_rxq)
   1811		err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
   1812
   1813	ionic_lif_quiesce(lif);
   1814}
   1815
   1816static void ionic_txrx_deinit(struct ionic_lif *lif)
   1817{
   1818	unsigned int i;
   1819
   1820	if (lif->txqcqs) {
   1821		for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
   1822			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
   1823			ionic_tx_flush(&lif->txqcqs[i]->cq);
   1824			ionic_tx_empty(&lif->txqcqs[i]->q);
   1825		}
   1826	}
   1827
   1828	if (lif->rxqcqs) {
   1829		for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
   1830			ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
   1831			ionic_rx_empty(&lif->rxqcqs[i]->q);
   1832		}
   1833	}
   1834	lif->rx_mode = 0;
   1835
   1836	if (lif->hwstamp_txq) {
   1837		ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
   1838		ionic_tx_flush(&lif->hwstamp_txq->cq);
   1839		ionic_tx_empty(&lif->hwstamp_txq->q);
   1840	}
   1841
   1842	if (lif->hwstamp_rxq) {
   1843		ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
   1844		ionic_rx_empty(&lif->hwstamp_rxq->q);
   1845	}
   1846}
   1847
   1848static void ionic_txrx_free(struct ionic_lif *lif)
   1849{
   1850	unsigned int i;
   1851
   1852	if (lif->txqcqs) {
   1853		for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
   1854			ionic_qcq_free(lif, lif->txqcqs[i]);
   1855			devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
   1856			lif->txqcqs[i] = NULL;
   1857		}
   1858	}
   1859
   1860	if (lif->rxqcqs) {
   1861		for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
   1862			ionic_qcq_free(lif, lif->rxqcqs[i]);
   1863			devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
   1864			lif->rxqcqs[i] = NULL;
   1865		}
   1866	}
   1867
   1868	if (lif->hwstamp_txq) {
   1869		ionic_qcq_free(lif, lif->hwstamp_txq);
   1870		devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
   1871		lif->hwstamp_txq = NULL;
   1872	}
   1873
   1874	if (lif->hwstamp_rxq) {
   1875		ionic_qcq_free(lif, lif->hwstamp_rxq);
   1876		devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
   1877		lif->hwstamp_rxq = NULL;
   1878	}
   1879}
   1880
   1881static int ionic_txrx_alloc(struct ionic_lif *lif)
   1882{
   1883	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
   1884	unsigned int flags, i;
   1885	int err = 0;
   1886
   1887	num_desc = lif->ntxq_descs;
   1888	desc_sz = sizeof(struct ionic_txq_desc);
   1889	comp_sz = sizeof(struct ionic_txq_comp);
   1890
   1891	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
   1892	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
   1893					  sizeof(struct ionic_txq_sg_desc_v1))
   1894		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
   1895	else
   1896		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
   1897
   1898	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
   1899	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
   1900		flags |= IONIC_QCQ_F_INTR;
   1901	for (i = 0; i < lif->nxqs; i++) {
   1902		err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
   1903				      num_desc, desc_sz, comp_sz, sg_desc_sz,
   1904				      lif->kern_pid, &lif->txqcqs[i]);
   1905		if (err)
   1906			goto err_out;
   1907
   1908		if (flags & IONIC_QCQ_F_INTR) {
   1909			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
   1910					     lif->txqcqs[i]->intr.index,
   1911					     lif->tx_coalesce_hw);
   1912			if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
   1913				lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
   1914		}
   1915
   1916		ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
   1917	}
   1918
   1919	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
   1920
   1921	num_desc = lif->nrxq_descs;
   1922	desc_sz = sizeof(struct ionic_rxq_desc);
   1923	comp_sz = sizeof(struct ionic_rxq_comp);
   1924	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
   1925
   1926	if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
   1927		comp_sz *= 2;
   1928
   1929	for (i = 0; i < lif->nxqs; i++) {
   1930		err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
   1931				      num_desc, desc_sz, comp_sz, sg_desc_sz,
   1932				      lif->kern_pid, &lif->rxqcqs[i]);
   1933		if (err)
   1934			goto err_out;
   1935
   1936		lif->rxqcqs[i]->q.features = lif->rxq_features;
   1937
   1938		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
   1939				     lif->rxqcqs[i]->intr.index,
   1940				     lif->rx_coalesce_hw);
   1941		if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
   1942			lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
   1943
   1944		if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
   1945			ionic_link_qcq_interrupts(lif->rxqcqs[i],
   1946						  lif->txqcqs[i]);
   1947
   1948		ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
   1949	}
   1950
   1951	return 0;
   1952
   1953err_out:
   1954	ionic_txrx_free(lif);
   1955
   1956	return err;
   1957}
   1958
   1959static int ionic_txrx_init(struct ionic_lif *lif)
   1960{
   1961	unsigned int i;
   1962	int err;
   1963
   1964	for (i = 0; i < lif->nxqs; i++) {
   1965		err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
   1966		if (err)
   1967			goto err_out;
   1968
   1969		err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
   1970		if (err) {
   1971			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
   1972			goto err_out;
   1973		}
   1974	}
   1975
   1976	if (lif->netdev->features & NETIF_F_RXHASH)
   1977		ionic_lif_rss_init(lif);
   1978
   1979	ionic_lif_rx_mode(lif);
   1980
   1981	return 0;
   1982
   1983err_out:
   1984	while (i--) {
   1985		ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
   1986		ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
   1987	}
   1988
   1989	return err;
   1990}
   1991
   1992static int ionic_txrx_enable(struct ionic_lif *lif)
   1993{
   1994	int derr = 0;
   1995	int i, err;
   1996
   1997	for (i = 0; i < lif->nxqs; i++) {
   1998		if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
   1999			dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
   2000			err = -ENXIO;
   2001			goto err_out;
   2002		}
   2003
   2004		ionic_rx_fill(&lif->rxqcqs[i]->q);
   2005		err = ionic_qcq_enable(lif->rxqcqs[i]);
   2006		if (err)
   2007			goto err_out;
   2008
   2009		err = ionic_qcq_enable(lif->txqcqs[i]);
   2010		if (err) {
   2011			derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
   2012			goto err_out;
   2013		}
   2014	}
   2015
   2016	if (lif->hwstamp_rxq) {
   2017		ionic_rx_fill(&lif->hwstamp_rxq->q);
   2018		err = ionic_qcq_enable(lif->hwstamp_rxq);
   2019		if (err)
   2020			goto err_out_hwstamp_rx;
   2021	}
   2022
   2023	if (lif->hwstamp_txq) {
   2024		err = ionic_qcq_enable(lif->hwstamp_txq);
   2025		if (err)
   2026			goto err_out_hwstamp_tx;
   2027	}
   2028
   2029	return 0;
   2030
   2031err_out_hwstamp_tx:
   2032	if (lif->hwstamp_rxq)
   2033		derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
   2034err_out_hwstamp_rx:
   2035	i = lif->nxqs;
   2036err_out:
   2037	while (i--) {
   2038		derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
   2039		derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
   2040	}
   2041
   2042	return err;
   2043}
   2044
   2045static int ionic_start_queues(struct ionic_lif *lif)
   2046{
   2047	int err;
   2048
   2049	if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
   2050		return -EIO;
   2051
   2052	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
   2053		return -EBUSY;
   2054
   2055	if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
   2056		return 0;
   2057
   2058	err = ionic_txrx_enable(lif);
   2059	if (err) {
   2060		clear_bit(IONIC_LIF_F_UP, lif->state);
   2061		return err;
   2062	}
   2063	netif_tx_wake_all_queues(lif->netdev);
   2064
   2065	return 0;
   2066}
   2067
   2068static int ionic_open(struct net_device *netdev)
   2069{
   2070	struct ionic_lif *lif = netdev_priv(netdev);
   2071	int err;
   2072
   2073	/* If recovering from a broken state, clear the bit and we'll try again */
   2074	if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
   2075		netdev_info(netdev, "clearing broken state\n");
   2076
   2077	mutex_lock(&lif->queue_lock);
   2078
   2079	err = ionic_txrx_alloc(lif);
   2080	if (err)
   2081		goto err_unlock;
   2082
   2083	err = ionic_txrx_init(lif);
   2084	if (err)
   2085		goto err_txrx_free;
   2086
   2087	err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
   2088	if (err)
   2089		goto err_txrx_deinit;
   2090
   2091	err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
   2092	if (err)
   2093		goto err_txrx_deinit;
   2094
   2095	/* don't start the queues until we have link */
   2096	if (netif_carrier_ok(netdev)) {
   2097		err = ionic_start_queues(lif);
   2098		if (err)
   2099			goto err_txrx_deinit;
   2100	}
   2101
   2102	/* If hardware timestamping is enabled, but the queues were freed by
   2103	 * ionic_stop, those need to be reallocated and initialized, too.
   2104	 */
   2105	ionic_lif_hwstamp_recreate_queues(lif);
   2106
   2107	mutex_unlock(&lif->queue_lock);
   2108
   2109	return 0;
   2110
   2111err_txrx_deinit:
   2112	ionic_txrx_deinit(lif);
   2113err_txrx_free:
   2114	ionic_txrx_free(lif);
   2115err_unlock:
   2116	mutex_unlock(&lif->queue_lock);
   2117	return err;
   2118}
   2119
   2120static void ionic_stop_queues(struct ionic_lif *lif)
   2121{
   2122	if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
   2123		return;
   2124
   2125	netif_tx_disable(lif->netdev);
   2126	ionic_txrx_disable(lif);
   2127}
   2128
   2129static int ionic_stop(struct net_device *netdev)
   2130{
   2131	struct ionic_lif *lif = netdev_priv(netdev);
   2132
   2133	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
   2134		return 0;
   2135
   2136	mutex_lock(&lif->queue_lock);
   2137	ionic_stop_queues(lif);
   2138	ionic_txrx_deinit(lif);
   2139	ionic_txrx_free(lif);
   2140	mutex_unlock(&lif->queue_lock);
   2141
   2142	return 0;
   2143}
   2144
   2145static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
   2146{
   2147	struct ionic_lif *lif = netdev_priv(netdev);
   2148
   2149	switch (cmd) {
   2150	case SIOCSHWTSTAMP:
   2151		return ionic_lif_hwstamp_set(lif, ifr);
   2152	case SIOCGHWTSTAMP:
   2153		return ionic_lif_hwstamp_get(lif, ifr);
   2154	default:
   2155		return -EOPNOTSUPP;
   2156	}
   2157}
   2158
   2159static int ionic_update_cached_vf_config(struct ionic *ionic, int vf)
   2160{
   2161	struct ionic_vf_getattr_comp comp = { 0 };
   2162	int err;
   2163	u8 attr;
   2164
   2165	attr = IONIC_VF_ATTR_VLAN;
   2166	err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
   2167	if (err && comp.status != IONIC_RC_ENOSUPP)
   2168		goto err_out;
   2169	if (!err)
   2170		ionic->vfs[vf].vlanid = comp.vlanid;
   2171
   2172	attr = IONIC_VF_ATTR_SPOOFCHK;
   2173	err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
   2174	if (err && comp.status != IONIC_RC_ENOSUPP)
   2175		goto err_out;
   2176	if (!err)
   2177		ionic->vfs[vf].spoofchk = comp.spoofchk;
   2178
   2179	attr = IONIC_VF_ATTR_LINKSTATE;
   2180	err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
   2181	if (err && comp.status != IONIC_RC_ENOSUPP)
   2182		goto err_out;
   2183	if (!err) {
   2184		switch (comp.linkstate) {
   2185		case IONIC_VF_LINK_STATUS_UP:
   2186			ionic->vfs[vf].linkstate = IFLA_VF_LINK_STATE_ENABLE;
   2187			break;
   2188		case IONIC_VF_LINK_STATUS_DOWN:
   2189			ionic->vfs[vf].linkstate = IFLA_VF_LINK_STATE_DISABLE;
   2190			break;
   2191		case IONIC_VF_LINK_STATUS_AUTO:
   2192			ionic->vfs[vf].linkstate = IFLA_VF_LINK_STATE_AUTO;
   2193			break;
   2194		default:
   2195			dev_warn(ionic->dev, "Unexpected link state %u\n", comp.linkstate);
   2196			break;
   2197		}
   2198	}
   2199
   2200	attr = IONIC_VF_ATTR_RATE;
   2201	err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
   2202	if (err && comp.status != IONIC_RC_ENOSUPP)
   2203		goto err_out;
   2204	if (!err)
   2205		ionic->vfs[vf].maxrate = comp.maxrate;
   2206
   2207	attr = IONIC_VF_ATTR_TRUST;
   2208	err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
   2209	if (err && comp.status != IONIC_RC_ENOSUPP)
   2210		goto err_out;
   2211	if (!err)
   2212		ionic->vfs[vf].trusted = comp.trust;
   2213
   2214	attr = IONIC_VF_ATTR_MAC;
   2215	err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
   2216	if (err && comp.status != IONIC_RC_ENOSUPP)
   2217		goto err_out;
   2218	if (!err)
   2219		ether_addr_copy(ionic->vfs[vf].macaddr, comp.macaddr);
   2220
   2221err_out:
   2222	if (err)
   2223		dev_err(ionic->dev, "Failed to get %s for VF %d\n",
   2224			ionic_vf_attr_to_str(attr), vf);
   2225
   2226	return err;
   2227}
   2228
   2229static int ionic_get_vf_config(struct net_device *netdev,
   2230			       int vf, struct ifla_vf_info *ivf)
   2231{
   2232	struct ionic_lif *lif = netdev_priv(netdev);
   2233	struct ionic *ionic = lif->ionic;
   2234	int ret = 0;
   2235
   2236	if (!netif_device_present(netdev))
   2237		return -EBUSY;
   2238
   2239	down_read(&ionic->vf_op_lock);
   2240
   2241	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2242		ret = -EINVAL;
   2243	} else {
   2244		ivf->vf = vf;
   2245		ivf->qos = 0;
   2246
   2247		ret = ionic_update_cached_vf_config(ionic, vf);
   2248		if (!ret) {
   2249			ivf->vlan         = le16_to_cpu(ionic->vfs[vf].vlanid);
   2250			ivf->spoofchk     = ionic->vfs[vf].spoofchk;
   2251			ivf->linkstate    = ionic->vfs[vf].linkstate;
   2252			ivf->max_tx_rate  = le32_to_cpu(ionic->vfs[vf].maxrate);
   2253			ivf->trusted      = ionic->vfs[vf].trusted;
   2254			ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
   2255		}
   2256	}
   2257
   2258	up_read(&ionic->vf_op_lock);
   2259	return ret;
   2260}
   2261
   2262static int ionic_get_vf_stats(struct net_device *netdev, int vf,
   2263			      struct ifla_vf_stats *vf_stats)
   2264{
   2265	struct ionic_lif *lif = netdev_priv(netdev);
   2266	struct ionic *ionic = lif->ionic;
   2267	struct ionic_lif_stats *vs;
   2268	int ret = 0;
   2269
   2270	if (!netif_device_present(netdev))
   2271		return -EBUSY;
   2272
   2273	down_read(&ionic->vf_op_lock);
   2274
   2275	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2276		ret = -EINVAL;
   2277	} else {
   2278		memset(vf_stats, 0, sizeof(*vf_stats));
   2279		vs = &ionic->vfs[vf].stats;
   2280
   2281		vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
   2282		vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
   2283		vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
   2284		vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
   2285		vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
   2286		vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
   2287		vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
   2288				       le64_to_cpu(vs->rx_mcast_drop_packets) +
   2289				       le64_to_cpu(vs->rx_bcast_drop_packets);
   2290		vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
   2291				       le64_to_cpu(vs->tx_mcast_drop_packets) +
   2292				       le64_to_cpu(vs->tx_bcast_drop_packets);
   2293	}
   2294
   2295	up_read(&ionic->vf_op_lock);
   2296	return ret;
   2297}
   2298
   2299static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
   2300{
   2301	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
   2302	struct ionic_lif *lif = netdev_priv(netdev);
   2303	struct ionic *ionic = lif->ionic;
   2304	int ret;
   2305
   2306	if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
   2307		return -EINVAL;
   2308
   2309	if (!netif_device_present(netdev))
   2310		return -EBUSY;
   2311
   2312	down_write(&ionic->vf_op_lock);
   2313
   2314	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2315		ret = -EINVAL;
   2316	} else {
   2317		ether_addr_copy(vfc.macaddr, mac);
   2318		dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
   2319			__func__, vf, vfc.macaddr);
   2320
   2321		ret = ionic_set_vf_config(ionic, vf, &vfc);
   2322		if (!ret)
   2323			ether_addr_copy(ionic->vfs[vf].macaddr, mac);
   2324	}
   2325
   2326	up_write(&ionic->vf_op_lock);
   2327	return ret;
   2328}
   2329
   2330static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
   2331			     u8 qos, __be16 proto)
   2332{
   2333	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
   2334	struct ionic_lif *lif = netdev_priv(netdev);
   2335	struct ionic *ionic = lif->ionic;
   2336	int ret;
   2337
   2338	/* until someday when we support qos */
   2339	if (qos)
   2340		return -EINVAL;
   2341
   2342	if (vlan > 4095)
   2343		return -EINVAL;
   2344
   2345	if (proto != htons(ETH_P_8021Q))
   2346		return -EPROTONOSUPPORT;
   2347
   2348	if (!netif_device_present(netdev))
   2349		return -EBUSY;
   2350
   2351	down_write(&ionic->vf_op_lock);
   2352
   2353	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2354		ret = -EINVAL;
   2355	} else {
   2356		vfc.vlanid = cpu_to_le16(vlan);
   2357		dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
   2358			__func__, vf, le16_to_cpu(vfc.vlanid));
   2359
   2360		ret = ionic_set_vf_config(ionic, vf, &vfc);
   2361		if (!ret)
   2362			ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
   2363	}
   2364
   2365	up_write(&ionic->vf_op_lock);
   2366	return ret;
   2367}
   2368
   2369static int ionic_set_vf_rate(struct net_device *netdev, int vf,
   2370			     int tx_min, int tx_max)
   2371{
   2372	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
   2373	struct ionic_lif *lif = netdev_priv(netdev);
   2374	struct ionic *ionic = lif->ionic;
   2375	int ret;
   2376
   2377	/* setting the min just seems silly */
   2378	if (tx_min)
   2379		return -EINVAL;
   2380
   2381	if (!netif_device_present(netdev))
   2382		return -EBUSY;
   2383
   2384	down_write(&ionic->vf_op_lock);
   2385
   2386	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2387		ret = -EINVAL;
   2388	} else {
   2389		vfc.maxrate = cpu_to_le32(tx_max);
   2390		dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
   2391			__func__, vf, le32_to_cpu(vfc.maxrate));
   2392
   2393		ret = ionic_set_vf_config(ionic, vf, &vfc);
   2394		if (!ret)
   2395			lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
   2396	}
   2397
   2398	up_write(&ionic->vf_op_lock);
   2399	return ret;
   2400}
   2401
   2402static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
   2403{
   2404	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
   2405	struct ionic_lif *lif = netdev_priv(netdev);
   2406	struct ionic *ionic = lif->ionic;
   2407	int ret;
   2408
   2409	if (!netif_device_present(netdev))
   2410		return -EBUSY;
   2411
   2412	down_write(&ionic->vf_op_lock);
   2413
   2414	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2415		ret = -EINVAL;
   2416	} else {
   2417		vfc.spoofchk = set;
   2418		dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
   2419			__func__, vf, vfc.spoofchk);
   2420
   2421		ret = ionic_set_vf_config(ionic, vf, &vfc);
   2422		if (!ret)
   2423			ionic->vfs[vf].spoofchk = set;
   2424	}
   2425
   2426	up_write(&ionic->vf_op_lock);
   2427	return ret;
   2428}
   2429
   2430static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
   2431{
   2432	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
   2433	struct ionic_lif *lif = netdev_priv(netdev);
   2434	struct ionic *ionic = lif->ionic;
   2435	int ret;
   2436
   2437	if (!netif_device_present(netdev))
   2438		return -EBUSY;
   2439
   2440	down_write(&ionic->vf_op_lock);
   2441
   2442	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2443		ret = -EINVAL;
   2444	} else {
   2445		vfc.trust = set;
   2446		dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
   2447			__func__, vf, vfc.trust);
   2448
   2449		ret = ionic_set_vf_config(ionic, vf, &vfc);
   2450		if (!ret)
   2451			ionic->vfs[vf].trusted = set;
   2452	}
   2453
   2454	up_write(&ionic->vf_op_lock);
   2455	return ret;
   2456}
   2457
   2458static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
   2459{
   2460	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
   2461	struct ionic_lif *lif = netdev_priv(netdev);
   2462	struct ionic *ionic = lif->ionic;
   2463	u8 vfls;
   2464	int ret;
   2465
   2466	switch (set) {
   2467	case IFLA_VF_LINK_STATE_ENABLE:
   2468		vfls = IONIC_VF_LINK_STATUS_UP;
   2469		break;
   2470	case IFLA_VF_LINK_STATE_DISABLE:
   2471		vfls = IONIC_VF_LINK_STATUS_DOWN;
   2472		break;
   2473	case IFLA_VF_LINK_STATE_AUTO:
   2474		vfls = IONIC_VF_LINK_STATUS_AUTO;
   2475		break;
   2476	default:
   2477		return -EINVAL;
   2478	}
   2479
   2480	if (!netif_device_present(netdev))
   2481		return -EBUSY;
   2482
   2483	down_write(&ionic->vf_op_lock);
   2484
   2485	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
   2486		ret = -EINVAL;
   2487	} else {
   2488		vfc.linkstate = vfls;
   2489		dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
   2490			__func__, vf, vfc.linkstate);
   2491
   2492		ret = ionic_set_vf_config(ionic, vf, &vfc);
   2493		if (!ret)
   2494			ionic->vfs[vf].linkstate = set;
   2495	}
   2496
   2497	up_write(&ionic->vf_op_lock);
   2498	return ret;
   2499}
   2500
   2501static const struct net_device_ops ionic_netdev_ops = {
   2502	.ndo_open               = ionic_open,
   2503	.ndo_stop               = ionic_stop,
   2504	.ndo_eth_ioctl		= ionic_eth_ioctl,
   2505	.ndo_start_xmit		= ionic_start_xmit,
   2506	.ndo_get_stats64	= ionic_get_stats64,
   2507	.ndo_set_rx_mode	= ionic_ndo_set_rx_mode,
   2508	.ndo_set_features	= ionic_set_features,
   2509	.ndo_set_mac_address	= ionic_set_mac_address,
   2510	.ndo_validate_addr	= eth_validate_addr,
   2511	.ndo_tx_timeout         = ionic_tx_timeout,
   2512	.ndo_change_mtu         = ionic_change_mtu,
   2513	.ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
   2514	.ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
   2515	.ndo_set_vf_vlan	= ionic_set_vf_vlan,
   2516	.ndo_set_vf_trust	= ionic_set_vf_trust,
   2517	.ndo_set_vf_mac		= ionic_set_vf_mac,
   2518	.ndo_set_vf_rate	= ionic_set_vf_rate,
   2519	.ndo_set_vf_spoofchk	= ionic_set_vf_spoofchk,
   2520	.ndo_get_vf_config	= ionic_get_vf_config,
   2521	.ndo_set_vf_link_state	= ionic_set_vf_link_state,
   2522	.ndo_get_vf_stats       = ionic_get_vf_stats,
   2523};
   2524
   2525static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
   2526{
   2527	/* only swapping the queues, not the napi, flags, or other stuff */
   2528	swap(a->q.features,   b->q.features);
   2529	swap(a->q.num_descs,  b->q.num_descs);
   2530	swap(a->q.desc_size,  b->q.desc_size);
   2531	swap(a->q.base,       b->q.base);
   2532	swap(a->q.base_pa,    b->q.base_pa);
   2533	swap(a->q.info,       b->q.info);
   2534	swap(a->q_base,       b->q_base);
   2535	swap(a->q_base_pa,    b->q_base_pa);
   2536	swap(a->q_size,       b->q_size);
   2537
   2538	swap(a->q.sg_desc_size, b->q.sg_desc_size);
   2539	swap(a->q.sg_base,    b->q.sg_base);
   2540	swap(a->q.sg_base_pa, b->q.sg_base_pa);
   2541	swap(a->sg_base,      b->sg_base);
   2542	swap(a->sg_base_pa,   b->sg_base_pa);
   2543	swap(a->sg_size,      b->sg_size);
   2544
   2545	swap(a->cq.num_descs, b->cq.num_descs);
   2546	swap(a->cq.desc_size, b->cq.desc_size);
   2547	swap(a->cq.base,      b->cq.base);
   2548	swap(a->cq.base_pa,   b->cq.base_pa);
   2549	swap(a->cq.info,      b->cq.info);
   2550	swap(a->cq_base,      b->cq_base);
   2551	swap(a->cq_base_pa,   b->cq_base_pa);
   2552	swap(a->cq_size,      b->cq_size);
   2553
   2554	ionic_debugfs_del_qcq(a);
   2555	ionic_debugfs_add_qcq(a->q.lif, a);
   2556}
   2557
   2558int ionic_reconfigure_queues(struct ionic_lif *lif,
   2559			     struct ionic_queue_params *qparam)
   2560{
   2561	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
   2562	struct ionic_qcq **tx_qcqs = NULL;
   2563	struct ionic_qcq **rx_qcqs = NULL;
   2564	unsigned int flags, i;
   2565	int err = 0;
   2566
   2567	/* allocate temporary qcq arrays to hold new queue structs */
   2568	if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
   2569		tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
   2570				       sizeof(struct ionic_qcq *), GFP_KERNEL);
   2571		if (!tx_qcqs) {
   2572			err = -ENOMEM;
   2573			goto err_out;
   2574		}
   2575	}
   2576	if (qparam->nxqs != lif->nxqs ||
   2577	    qparam->nrxq_descs != lif->nrxq_descs ||
   2578	    qparam->rxq_features != lif->rxq_features) {
   2579		rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
   2580				       sizeof(struct ionic_qcq *), GFP_KERNEL);
   2581		if (!rx_qcqs) {
   2582			err = -ENOMEM;
   2583			goto err_out;
   2584		}
   2585	}
   2586
   2587	/* allocate new desc_info and rings, but leave the interrupt setup
   2588	 * until later so as to not mess with the still-running queues
   2589	 */
   2590	if (tx_qcqs) {
   2591		num_desc = qparam->ntxq_descs;
   2592		desc_sz = sizeof(struct ionic_txq_desc);
   2593		comp_sz = sizeof(struct ionic_txq_comp);
   2594
   2595		if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
   2596		    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
   2597		    sizeof(struct ionic_txq_sg_desc_v1))
   2598			sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
   2599		else
   2600			sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
   2601
   2602		for (i = 0; i < qparam->nxqs; i++) {
   2603			flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
   2604			err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
   2605					      num_desc, desc_sz, comp_sz, sg_desc_sz,
   2606					      lif->kern_pid, &tx_qcqs[i]);
   2607			if (err)
   2608				goto err_out;
   2609		}
   2610	}
   2611
   2612	if (rx_qcqs) {
   2613		num_desc = qparam->nrxq_descs;
   2614		desc_sz = sizeof(struct ionic_rxq_desc);
   2615		comp_sz = sizeof(struct ionic_rxq_comp);
   2616		sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
   2617
   2618		if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
   2619			comp_sz *= 2;
   2620
   2621		for (i = 0; i < qparam->nxqs; i++) {
   2622			flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
   2623			err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
   2624					      num_desc, desc_sz, comp_sz, sg_desc_sz,
   2625					      lif->kern_pid, &rx_qcqs[i]);
   2626			if (err)
   2627				goto err_out;
   2628
   2629			rx_qcqs[i]->q.features = qparam->rxq_features;
   2630		}
   2631	}
   2632
   2633	/* stop and clean the queues */
   2634	ionic_stop_queues_reconfig(lif);
   2635
   2636	if (qparam->nxqs != lif->nxqs) {
   2637		err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
   2638		if (err)
   2639			goto err_out_reinit_unlock;
   2640		err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
   2641		if (err) {
   2642			netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
   2643			goto err_out_reinit_unlock;
   2644		}
   2645	}
   2646
   2647	/* swap new desc_info and rings, keeping existing interrupt config */
   2648	if (tx_qcqs) {
   2649		lif->ntxq_descs = qparam->ntxq_descs;
   2650		for (i = 0; i < qparam->nxqs; i++)
   2651			ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
   2652	}
   2653
   2654	if (rx_qcqs) {
   2655		lif->nrxq_descs = qparam->nrxq_descs;
   2656		for (i = 0; i < qparam->nxqs; i++)
   2657			ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
   2658	}
   2659
   2660	/* if we need to change the interrupt layout, this is the time */
   2661	if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
   2662	    qparam->nxqs != lif->nxqs) {
   2663		if (qparam->intr_split) {
   2664			set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
   2665		} else {
   2666			clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
   2667			lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
   2668			lif->tx_coalesce_hw = lif->rx_coalesce_hw;
   2669		}
   2670
   2671		/* clear existing interrupt assignments */
   2672		for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
   2673			ionic_qcq_intr_free(lif, lif->txqcqs[i]);
   2674			ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
   2675		}
   2676
   2677		/* re-assign the interrupts */
   2678		for (i = 0; i < qparam->nxqs; i++) {
   2679			lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
   2680			err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
   2681			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
   2682					     lif->rxqcqs[i]->intr.index,
   2683					     lif->rx_coalesce_hw);
   2684
   2685			if (qparam->intr_split) {
   2686				lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
   2687				err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
   2688				ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
   2689						     lif->txqcqs[i]->intr.index,
   2690						     lif->tx_coalesce_hw);
   2691				if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
   2692					lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
   2693			} else {
   2694				lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
   2695				ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
   2696			}
   2697		}
   2698	}
   2699
   2700	/* now we can rework the debugfs mappings */
   2701	if (tx_qcqs) {
   2702		for (i = 0; i < qparam->nxqs; i++) {
   2703			ionic_debugfs_del_qcq(lif->txqcqs[i]);
   2704			ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
   2705		}
   2706	}
   2707
   2708	if (rx_qcqs) {
   2709		for (i = 0; i < qparam->nxqs; i++) {
   2710			ionic_debugfs_del_qcq(lif->rxqcqs[i]);
   2711			ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
   2712		}
   2713	}
   2714
   2715	swap(lif->nxqs, qparam->nxqs);
   2716	swap(lif->rxq_features, qparam->rxq_features);
   2717
   2718err_out_reinit_unlock:
   2719	/* re-init the queues, but don't lose an error code */
   2720	if (err)
   2721		ionic_start_queues_reconfig(lif);
   2722	else
   2723		err = ionic_start_queues_reconfig(lif);
   2724
   2725err_out:
   2726	/* free old allocs without cleaning intr */
   2727	for (i = 0; i < qparam->nxqs; i++) {
   2728		if (tx_qcqs && tx_qcqs[i]) {
   2729			tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
   2730			ionic_qcq_free(lif, tx_qcqs[i]);
   2731			devm_kfree(lif->ionic->dev, tx_qcqs[i]);
   2732			tx_qcqs[i] = NULL;
   2733		}
   2734		if (rx_qcqs && rx_qcqs[i]) {
   2735			rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
   2736			ionic_qcq_free(lif, rx_qcqs[i]);
   2737			devm_kfree(lif->ionic->dev, rx_qcqs[i]);
   2738			rx_qcqs[i] = NULL;
   2739		}
   2740	}
   2741
   2742	/* free q array */
   2743	if (rx_qcqs) {
   2744		devm_kfree(lif->ionic->dev, rx_qcqs);
   2745		rx_qcqs = NULL;
   2746	}
   2747	if (tx_qcqs) {
   2748		devm_kfree(lif->ionic->dev, tx_qcqs);
   2749		tx_qcqs = NULL;
   2750	}
   2751
   2752	/* clean the unused dma and info allocations when new set is smaller
   2753	 * than the full array, but leave the qcq shells in place
   2754	 */
   2755	for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
   2756		lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
   2757		ionic_qcq_free(lif, lif->txqcqs[i]);
   2758
   2759		lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
   2760		ionic_qcq_free(lif, lif->rxqcqs[i]);
   2761	}
   2762
   2763	if (err)
   2764		netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
   2765
   2766	return err;
   2767}
   2768
   2769int ionic_lif_alloc(struct ionic *ionic)
   2770{
   2771	struct device *dev = ionic->dev;
   2772	union ionic_lif_identity *lid;
   2773	struct net_device *netdev;
   2774	struct ionic_lif *lif;
   2775	int tbl_sz;
   2776	int err;
   2777
   2778	lid = kzalloc(sizeof(*lid), GFP_KERNEL);
   2779	if (!lid)
   2780		return -ENOMEM;
   2781
   2782	netdev = alloc_etherdev_mqs(sizeof(*lif),
   2783				    ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
   2784	if (!netdev) {
   2785		dev_err(dev, "Cannot allocate netdev, aborting\n");
   2786		err = -ENOMEM;
   2787		goto err_out_free_lid;
   2788	}
   2789
   2790	SET_NETDEV_DEV(netdev, dev);
   2791
   2792	lif = netdev_priv(netdev);
   2793	lif->netdev = netdev;
   2794	ionic->lif = lif;
   2795	netdev->netdev_ops = &ionic_netdev_ops;
   2796	ionic_ethtool_set_ops(netdev);
   2797
   2798	netdev->watchdog_timeo = 2 * HZ;
   2799	netif_carrier_off(netdev);
   2800
   2801	lif->identity = lid;
   2802	lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
   2803	err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
   2804	if (err) {
   2805		dev_err(ionic->dev, "Cannot identify type %d: %d\n",
   2806			lif->lif_type, err);
   2807		goto err_out_free_netdev;
   2808	}
   2809	lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
   2810				     le32_to_cpu(lif->identity->eth.min_frame_size));
   2811	lif->netdev->max_mtu =
   2812		le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
   2813
   2814	lif->neqs = ionic->neqs_per_lif;
   2815	lif->nxqs = ionic->ntxqs_per_lif;
   2816
   2817	lif->ionic = ionic;
   2818	lif->index = 0;
   2819
   2820	if (is_kdump_kernel()) {
   2821		lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
   2822		lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
   2823	} else {
   2824		lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
   2825		lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
   2826	}
   2827
   2828	/* Convert the default coalesce value to actual hw resolution */
   2829	lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
   2830	lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
   2831						    lif->rx_coalesce_usecs);
   2832	lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
   2833	lif->tx_coalesce_hw = lif->rx_coalesce_hw;
   2834	set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
   2835	set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
   2836
   2837	snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
   2838
   2839	mutex_init(&lif->queue_lock);
   2840	mutex_init(&lif->config_lock);
   2841
   2842	spin_lock_init(&lif->adminq_lock);
   2843
   2844	spin_lock_init(&lif->deferred.lock);
   2845	INIT_LIST_HEAD(&lif->deferred.list);
   2846	INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
   2847
   2848	/* allocate lif info */
   2849	lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
   2850	lif->info = dma_alloc_coherent(dev, lif->info_sz,
   2851				       &lif->info_pa, GFP_KERNEL);
   2852	if (!lif->info) {
   2853		dev_err(dev, "Failed to allocate lif info, aborting\n");
   2854		err = -ENOMEM;
   2855		goto err_out_free_mutex;
   2856	}
   2857
   2858	ionic_debugfs_add_lif(lif);
   2859
   2860	/* allocate control queues and txrx queue arrays */
   2861	ionic_lif_queue_identify(lif);
   2862	err = ionic_qcqs_alloc(lif);
   2863	if (err)
   2864		goto err_out_free_lif_info;
   2865
   2866	/* allocate rss indirection table */
   2867	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
   2868	lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
   2869	lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
   2870					      &lif->rss_ind_tbl_pa,
   2871					      GFP_KERNEL);
   2872
   2873	if (!lif->rss_ind_tbl) {
   2874		err = -ENOMEM;
   2875		dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
   2876		goto err_out_free_qcqs;
   2877	}
   2878	netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
   2879
   2880	ionic_lif_alloc_phc(lif);
   2881
   2882	return 0;
   2883
   2884err_out_free_qcqs:
   2885	ionic_qcqs_free(lif);
   2886err_out_free_lif_info:
   2887	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
   2888	lif->info = NULL;
   2889	lif->info_pa = 0;
   2890err_out_free_mutex:
   2891	mutex_destroy(&lif->config_lock);
   2892	mutex_destroy(&lif->queue_lock);
   2893err_out_free_netdev:
   2894	free_netdev(lif->netdev);
   2895	lif = NULL;
   2896err_out_free_lid:
   2897	kfree(lid);
   2898
   2899	return err;
   2900}
   2901
   2902static void ionic_lif_reset(struct ionic_lif *lif)
   2903{
   2904	struct ionic_dev *idev = &lif->ionic->idev;
   2905
   2906	mutex_lock(&lif->ionic->dev_cmd_lock);
   2907	ionic_dev_cmd_lif_reset(idev, lif->index);
   2908	ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
   2909	mutex_unlock(&lif->ionic->dev_cmd_lock);
   2910}
   2911
   2912static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
   2913{
   2914	struct ionic *ionic = lif->ionic;
   2915
   2916	if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
   2917		return;
   2918
   2919	dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
   2920
   2921	netif_device_detach(lif->netdev);
   2922
   2923	mutex_lock(&lif->queue_lock);
   2924	if (test_bit(IONIC_LIF_F_UP, lif->state)) {
   2925		dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
   2926		ionic_stop_queues(lif);
   2927	}
   2928
   2929	if (netif_running(lif->netdev)) {
   2930		ionic_txrx_deinit(lif);
   2931		ionic_txrx_free(lif);
   2932	}
   2933	ionic_lif_deinit(lif);
   2934	ionic_reset(ionic);
   2935	ionic_qcqs_free(lif);
   2936
   2937	mutex_unlock(&lif->queue_lock);
   2938
   2939	clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
   2940	dev_info(ionic->dev, "FW Down: LIFs stopped\n");
   2941}
   2942
   2943static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
   2944{
   2945	struct ionic *ionic = lif->ionic;
   2946	int err;
   2947
   2948	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
   2949		return;
   2950
   2951	dev_info(ionic->dev, "FW Up: restarting LIFs\n");
   2952
   2953	ionic_init_devinfo(ionic);
   2954	err = ionic_identify(ionic);
   2955	if (err)
   2956		goto err_out;
   2957	err = ionic_port_identify(ionic);
   2958	if (err)
   2959		goto err_out;
   2960	err = ionic_port_init(ionic);
   2961	if (err)
   2962		goto err_out;
   2963
   2964	mutex_lock(&lif->queue_lock);
   2965
   2966	err = ionic_qcqs_alloc(lif);
   2967	if (err)
   2968		goto err_unlock;
   2969
   2970	err = ionic_lif_init(lif);
   2971	if (err)
   2972		goto err_qcqs_free;
   2973
   2974	if (lif->registered)
   2975		ionic_lif_set_netdev_info(lif);
   2976
   2977	ionic_rx_filter_replay(lif);
   2978
   2979	if (netif_running(lif->netdev)) {
   2980		err = ionic_txrx_alloc(lif);
   2981		if (err)
   2982			goto err_lifs_deinit;
   2983
   2984		err = ionic_txrx_init(lif);
   2985		if (err)
   2986			goto err_txrx_free;
   2987	}
   2988
   2989	mutex_unlock(&lif->queue_lock);
   2990
   2991	clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
   2992	ionic_link_status_check_request(lif, CAN_SLEEP);
   2993	netif_device_attach(lif->netdev);
   2994	dev_info(ionic->dev, "FW Up: LIFs restarted\n");
   2995
   2996	/* restore the hardware timestamping queues */
   2997	ionic_lif_hwstamp_replay(lif);
   2998
   2999	return;
   3000
   3001err_txrx_free:
   3002	ionic_txrx_free(lif);
   3003err_lifs_deinit:
   3004	ionic_lif_deinit(lif);
   3005err_qcqs_free:
   3006	ionic_qcqs_free(lif);
   3007err_unlock:
   3008	mutex_unlock(&lif->queue_lock);
   3009err_out:
   3010	dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
   3011}
   3012
   3013void ionic_lif_free(struct ionic_lif *lif)
   3014{
   3015	struct device *dev = lif->ionic->dev;
   3016
   3017	ionic_lif_free_phc(lif);
   3018
   3019	/* free rss indirection table */
   3020	dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
   3021			  lif->rss_ind_tbl_pa);
   3022	lif->rss_ind_tbl = NULL;
   3023	lif->rss_ind_tbl_pa = 0;
   3024
   3025	/* free queues */
   3026	ionic_qcqs_free(lif);
   3027	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
   3028		ionic_lif_reset(lif);
   3029
   3030	/* free lif info */
   3031	kfree(lif->identity);
   3032	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
   3033	lif->info = NULL;
   3034	lif->info_pa = 0;
   3035
   3036	/* unmap doorbell page */
   3037	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
   3038	lif->kern_dbpage = NULL;
   3039
   3040	mutex_destroy(&lif->config_lock);
   3041	mutex_destroy(&lif->queue_lock);
   3042
   3043	/* free netdev & lif */
   3044	ionic_debugfs_del_lif(lif);
   3045	free_netdev(lif->netdev);
   3046}
   3047
   3048void ionic_lif_deinit(struct ionic_lif *lif)
   3049{
   3050	if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
   3051		return;
   3052
   3053	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
   3054		cancel_work_sync(&lif->deferred.work);
   3055		cancel_work_sync(&lif->tx_timeout_work);
   3056		ionic_rx_filters_deinit(lif);
   3057		if (lif->netdev->features & NETIF_F_RXHASH)
   3058			ionic_lif_rss_deinit(lif);
   3059	}
   3060
   3061	napi_disable(&lif->adminqcq->napi);
   3062	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
   3063	ionic_lif_qcq_deinit(lif, lif->adminqcq);
   3064
   3065	ionic_lif_reset(lif);
   3066}
   3067
   3068static int ionic_lif_adminq_init(struct ionic_lif *lif)
   3069{
   3070	struct device *dev = lif->ionic->dev;
   3071	struct ionic_q_init_comp comp;
   3072	struct ionic_dev *idev;
   3073	struct ionic_qcq *qcq;
   3074	struct ionic_queue *q;
   3075	int err;
   3076
   3077	idev = &lif->ionic->idev;
   3078	qcq = lif->adminqcq;
   3079	q = &qcq->q;
   3080
   3081	mutex_lock(&lif->ionic->dev_cmd_lock);
   3082	ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
   3083	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
   3084	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
   3085	mutex_unlock(&lif->ionic->dev_cmd_lock);
   3086	if (err) {
   3087		netdev_err(lif->netdev, "adminq init failed %d\n", err);
   3088		return err;
   3089	}
   3090
   3091	q->hw_type = comp.hw_type;
   3092	q->hw_index = le32_to_cpu(comp.hw_index);
   3093	q->dbval = IONIC_DBELL_QID(q->hw_index);
   3094
   3095	dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
   3096	dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
   3097
   3098	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
   3099		       NAPI_POLL_WEIGHT);
   3100
   3101	napi_enable(&qcq->napi);
   3102
   3103	if (qcq->flags & IONIC_QCQ_F_INTR)
   3104		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
   3105				IONIC_INTR_MASK_CLEAR);
   3106
   3107	qcq->flags |= IONIC_QCQ_F_INITED;
   3108
   3109	return 0;
   3110}
   3111
   3112static int ionic_lif_notifyq_init(struct ionic_lif *lif)
   3113{
   3114	struct ionic_qcq *qcq = lif->notifyqcq;
   3115	struct device *dev = lif->ionic->dev;
   3116	struct ionic_queue *q = &qcq->q;
   3117	int err;
   3118
   3119	struct ionic_admin_ctx ctx = {
   3120		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   3121		.cmd.q_init = {
   3122			.opcode = IONIC_CMD_Q_INIT,
   3123			.lif_index = cpu_to_le16(lif->index),
   3124			.type = q->type,
   3125			.ver = lif->qtype_info[q->type].version,
   3126			.index = cpu_to_le32(q->index),
   3127			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
   3128					     IONIC_QINIT_F_ENA),
   3129			.intr_index = cpu_to_le16(lif->adminqcq->intr.index),
   3130			.pid = cpu_to_le16(q->pid),
   3131			.ring_size = ilog2(q->num_descs),
   3132			.ring_base = cpu_to_le64(q->base_pa),
   3133		}
   3134	};
   3135
   3136	dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
   3137	dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
   3138	dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
   3139	dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
   3140
   3141	err = ionic_adminq_post_wait(lif, &ctx);
   3142	if (err)
   3143		return err;
   3144
   3145	lif->last_eid = 0;
   3146	q->hw_type = ctx.comp.q_init.hw_type;
   3147	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
   3148	q->dbval = IONIC_DBELL_QID(q->hw_index);
   3149
   3150	dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
   3151	dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
   3152
   3153	/* preset the callback info */
   3154	q->info[0].cb_arg = lif;
   3155
   3156	qcq->flags |= IONIC_QCQ_F_INITED;
   3157
   3158	return 0;
   3159}
   3160
   3161static int ionic_station_set(struct ionic_lif *lif)
   3162{
   3163	struct net_device *netdev = lif->netdev;
   3164	struct ionic_admin_ctx ctx = {
   3165		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   3166		.cmd.lif_getattr = {
   3167			.opcode = IONIC_CMD_LIF_GETATTR,
   3168			.index = cpu_to_le16(lif->index),
   3169			.attr = IONIC_LIF_ATTR_MAC,
   3170		},
   3171	};
   3172	struct sockaddr addr;
   3173	int err;
   3174
   3175	err = ionic_adminq_post_wait(lif, &ctx);
   3176	if (err)
   3177		return err;
   3178	netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
   3179		   ctx.comp.lif_getattr.mac);
   3180	if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
   3181		return 0;
   3182
   3183	if (!is_zero_ether_addr(netdev->dev_addr)) {
   3184		/* If the netdev mac is non-zero and doesn't match the default
   3185		 * device address, it was set by something earlier and we're
   3186		 * likely here again after a fw-upgrade reset.  We need to be
   3187		 * sure the netdev mac is in our filter list.
   3188		 */
   3189		if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
   3190				      netdev->dev_addr))
   3191			ionic_lif_addr_add(lif, netdev->dev_addr);
   3192	} else {
   3193		/* Update the netdev mac with the device's mac */
   3194		memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
   3195		addr.sa_family = AF_INET;
   3196		err = eth_prepare_mac_addr_change(netdev, &addr);
   3197		if (err) {
   3198			netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
   3199				    addr.sa_data, err);
   3200			return 0;
   3201		}
   3202
   3203		eth_commit_mac_addr_change(netdev, &addr);
   3204	}
   3205
   3206	netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
   3207		   netdev->dev_addr);
   3208	ionic_lif_addr_add(lif, netdev->dev_addr);
   3209
   3210	return 0;
   3211}
   3212
   3213int ionic_lif_init(struct ionic_lif *lif)
   3214{
   3215	struct ionic_dev *idev = &lif->ionic->idev;
   3216	struct device *dev = lif->ionic->dev;
   3217	struct ionic_lif_init_comp comp;
   3218	int dbpage_num;
   3219	int err;
   3220
   3221	mutex_lock(&lif->ionic->dev_cmd_lock);
   3222	ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
   3223	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
   3224	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
   3225	mutex_unlock(&lif->ionic->dev_cmd_lock);
   3226	if (err)
   3227		return err;
   3228
   3229	lif->hw_index = le16_to_cpu(comp.hw_index);
   3230
   3231	/* now that we have the hw_index we can figure out our doorbell page */
   3232	lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
   3233	if (!lif->dbid_count) {
   3234		dev_err(dev, "No doorbell pages, aborting\n");
   3235		return -EINVAL;
   3236	}
   3237
   3238	lif->kern_pid = 0;
   3239	dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
   3240	lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
   3241	if (!lif->kern_dbpage) {
   3242		dev_err(dev, "Cannot map dbpage, aborting\n");
   3243		return -ENOMEM;
   3244	}
   3245
   3246	err = ionic_lif_adminq_init(lif);
   3247	if (err)
   3248		goto err_out_adminq_deinit;
   3249
   3250	if (lif->ionic->nnqs_per_lif) {
   3251		err = ionic_lif_notifyq_init(lif);
   3252		if (err)
   3253			goto err_out_notifyq_deinit;
   3254	}
   3255
   3256	err = ionic_init_nic_features(lif);
   3257	if (err)
   3258		goto err_out_notifyq_deinit;
   3259
   3260	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
   3261		err = ionic_rx_filters_init(lif);
   3262		if (err)
   3263			goto err_out_notifyq_deinit;
   3264	}
   3265
   3266	err = ionic_station_set(lif);
   3267	if (err)
   3268		goto err_out_notifyq_deinit;
   3269
   3270	lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
   3271
   3272	set_bit(IONIC_LIF_F_INITED, lif->state);
   3273
   3274	INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
   3275
   3276	return 0;
   3277
   3278err_out_notifyq_deinit:
   3279	napi_disable(&lif->adminqcq->napi);
   3280	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
   3281err_out_adminq_deinit:
   3282	ionic_lif_qcq_deinit(lif, lif->adminqcq);
   3283	ionic_lif_reset(lif);
   3284	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
   3285	lif->kern_dbpage = NULL;
   3286
   3287	return err;
   3288}
   3289
   3290static void ionic_lif_notify_work(struct work_struct *ws)
   3291{
   3292}
   3293
   3294static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
   3295{
   3296	struct ionic_admin_ctx ctx = {
   3297		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
   3298		.cmd.lif_setattr = {
   3299			.opcode = IONIC_CMD_LIF_SETATTR,
   3300			.index = cpu_to_le16(lif->index),
   3301			.attr = IONIC_LIF_ATTR_NAME,
   3302		},
   3303	};
   3304
   3305	strscpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
   3306		sizeof(ctx.cmd.lif_setattr.name));
   3307
   3308	ionic_adminq_post_wait(lif, &ctx);
   3309}
   3310
   3311static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
   3312{
   3313	if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
   3314		return NULL;
   3315
   3316	return netdev_priv(netdev);
   3317}
   3318
   3319static int ionic_lif_notify(struct notifier_block *nb,
   3320			    unsigned long event, void *info)
   3321{
   3322	struct net_device *ndev = netdev_notifier_info_to_dev(info);
   3323	struct ionic *ionic = container_of(nb, struct ionic, nb);
   3324	struct ionic_lif *lif = ionic_netdev_lif(ndev);
   3325
   3326	if (!lif || lif->ionic != ionic)
   3327		return NOTIFY_DONE;
   3328
   3329	switch (event) {
   3330	case NETDEV_CHANGENAME:
   3331		ionic_lif_set_netdev_info(lif);
   3332		break;
   3333	}
   3334
   3335	return NOTIFY_DONE;
   3336}
   3337
   3338int ionic_lif_register(struct ionic_lif *lif)
   3339{
   3340	int err;
   3341
   3342	ionic_lif_register_phc(lif);
   3343
   3344	INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
   3345
   3346	lif->ionic->nb.notifier_call = ionic_lif_notify;
   3347
   3348	err = register_netdevice_notifier(&lif->ionic->nb);
   3349	if (err)
   3350		lif->ionic->nb.notifier_call = NULL;
   3351
   3352	/* only register LIF0 for now */
   3353	err = register_netdev(lif->netdev);
   3354	if (err) {
   3355		dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
   3356		ionic_lif_unregister_phc(lif);
   3357		return err;
   3358	}
   3359
   3360	ionic_link_status_check_request(lif, CAN_SLEEP);
   3361	lif->registered = true;
   3362	ionic_lif_set_netdev_info(lif);
   3363
   3364	return 0;
   3365}
   3366
   3367void ionic_lif_unregister(struct ionic_lif *lif)
   3368{
   3369	if (lif->ionic->nb.notifier_call) {
   3370		unregister_netdevice_notifier(&lif->ionic->nb);
   3371		cancel_work_sync(&lif->ionic->nb_work);
   3372		lif->ionic->nb.notifier_call = NULL;
   3373	}
   3374
   3375	if (lif->netdev->reg_state == NETREG_REGISTERED)
   3376		unregister_netdev(lif->netdev);
   3377
   3378	ionic_lif_unregister_phc(lif);
   3379
   3380	lif->registered = false;
   3381}
   3382
   3383static void ionic_lif_queue_identify(struct ionic_lif *lif)
   3384{
   3385	union ionic_q_identity __iomem *q_ident;
   3386	struct ionic *ionic = lif->ionic;
   3387	struct ionic_dev *idev;
   3388	int qtype;
   3389	int err;
   3390
   3391	idev = &lif->ionic->idev;
   3392	q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
   3393
   3394	for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
   3395		struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
   3396
   3397		/* filter out the ones we know about */
   3398		switch (qtype) {
   3399		case IONIC_QTYPE_ADMINQ:
   3400		case IONIC_QTYPE_NOTIFYQ:
   3401		case IONIC_QTYPE_RXQ:
   3402		case IONIC_QTYPE_TXQ:
   3403			break;
   3404		default:
   3405			continue;
   3406		}
   3407
   3408		memset(qti, 0, sizeof(*qti));
   3409
   3410		mutex_lock(&ionic->dev_cmd_lock);
   3411		ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
   3412					     ionic_qtype_versions[qtype]);
   3413		err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
   3414		if (!err) {
   3415			qti->version   = readb(&q_ident->version);
   3416			qti->supported = readb(&q_ident->supported);
   3417			qti->features  = readq(&q_ident->features);
   3418			qti->desc_sz   = readw(&q_ident->desc_sz);
   3419			qti->comp_sz   = readw(&q_ident->comp_sz);
   3420			qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
   3421			qti->max_sg_elems = readw(&q_ident->max_sg_elems);
   3422			qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
   3423		}
   3424		mutex_unlock(&ionic->dev_cmd_lock);
   3425
   3426		if (err == -EINVAL) {
   3427			dev_err(ionic->dev, "qtype %d not supported\n", qtype);
   3428			continue;
   3429		} else if (err == -EIO) {
   3430			dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
   3431			return;
   3432		} else if (err) {
   3433			dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
   3434				qtype, err);
   3435			return;
   3436		}
   3437
   3438		dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
   3439			qtype, qti->version);
   3440		dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
   3441			qtype, qti->supported);
   3442		dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
   3443			qtype, qti->features);
   3444		dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
   3445			qtype, qti->desc_sz);
   3446		dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
   3447			qtype, qti->comp_sz);
   3448		dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
   3449			qtype, qti->sg_desc_sz);
   3450		dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
   3451			qtype, qti->max_sg_elems);
   3452		dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
   3453			qtype, qti->sg_desc_stride);
   3454	}
   3455}
   3456
   3457int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
   3458		       union ionic_lif_identity *lid)
   3459{
   3460	struct ionic_dev *idev = &ionic->idev;
   3461	size_t sz;
   3462	int err;
   3463
   3464	sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
   3465
   3466	mutex_lock(&ionic->dev_cmd_lock);
   3467	ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
   3468	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
   3469	memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
   3470	mutex_unlock(&ionic->dev_cmd_lock);
   3471	if (err)
   3472		return (err);
   3473
   3474	dev_dbg(ionic->dev, "capabilities 0x%llx\n",
   3475		le64_to_cpu(lid->capabilities));
   3476
   3477	dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
   3478		le32_to_cpu(lid->eth.max_ucast_filters));
   3479	dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
   3480		le32_to_cpu(lid->eth.max_mcast_filters));
   3481	dev_dbg(ionic->dev, "eth.features 0x%llx\n",
   3482		le64_to_cpu(lid->eth.config.features));
   3483	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
   3484		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
   3485	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
   3486		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
   3487	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
   3488		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
   3489	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
   3490		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
   3491	dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
   3492	dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
   3493	dev_dbg(ionic->dev, "eth.config.mtu %d\n",
   3494		le32_to_cpu(lid->eth.config.mtu));
   3495
   3496	return 0;
   3497}
   3498
   3499int ionic_lif_size(struct ionic *ionic)
   3500{
   3501	struct ionic_identity *ident = &ionic->ident;
   3502	unsigned int nintrs, dev_nintrs;
   3503	union ionic_lif_config *lc;
   3504	unsigned int ntxqs_per_lif;
   3505	unsigned int nrxqs_per_lif;
   3506	unsigned int neqs_per_lif;
   3507	unsigned int nnqs_per_lif;
   3508	unsigned int nxqs, neqs;
   3509	unsigned int min_intrs;
   3510	int err;
   3511
   3512	/* retrieve basic values from FW */
   3513	lc = &ident->lif.eth.config;
   3514	dev_nintrs = le32_to_cpu(ident->dev.nintrs);
   3515	neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
   3516	nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
   3517	ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
   3518	nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
   3519
   3520	/* limit values to play nice with kdump */
   3521	if (is_kdump_kernel()) {
   3522		dev_nintrs = 2;
   3523		neqs_per_lif = 0;
   3524		nnqs_per_lif = 0;
   3525		ntxqs_per_lif = 1;
   3526		nrxqs_per_lif = 1;
   3527	}
   3528
   3529	/* reserve last queue id for hardware timestamping */
   3530	if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
   3531		if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
   3532			lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
   3533		} else {
   3534			ntxqs_per_lif -= 1;
   3535			nrxqs_per_lif -= 1;
   3536		}
   3537	}
   3538
   3539	nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
   3540	nxqs = min(nxqs, num_online_cpus());
   3541	neqs = min(neqs_per_lif, num_online_cpus());
   3542
   3543try_again:
   3544	/* interrupt usage:
   3545	 *    1 for master lif adminq/notifyq
   3546	 *    1 for each CPU for master lif TxRx queue pairs
   3547	 *    whatever's left is for RDMA queues
   3548	 */
   3549	nintrs = 1 + nxqs + neqs;
   3550	min_intrs = 2;  /* adminq + 1 TxRx queue pair */
   3551
   3552	if (nintrs > dev_nintrs)
   3553		goto try_fewer;
   3554
   3555	err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
   3556	if (err < 0 && err != -ENOSPC) {
   3557		dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
   3558		return err;
   3559	}
   3560	if (err == -ENOSPC)
   3561		goto try_fewer;
   3562
   3563	if (err != nintrs) {
   3564		ionic_bus_free_irq_vectors(ionic);
   3565		goto try_fewer;
   3566	}
   3567
   3568	ionic->nnqs_per_lif = nnqs_per_lif;
   3569	ionic->neqs_per_lif = neqs;
   3570	ionic->ntxqs_per_lif = nxqs;
   3571	ionic->nrxqs_per_lif = nxqs;
   3572	ionic->nintrs = nintrs;
   3573
   3574	ionic_debugfs_add_sizes(ionic);
   3575
   3576	return 0;
   3577
   3578try_fewer:
   3579	if (nnqs_per_lif > 1) {
   3580		nnqs_per_lif >>= 1;
   3581		goto try_again;
   3582	}
   3583	if (neqs > 1) {
   3584		neqs >>= 1;
   3585		goto try_again;
   3586	}
   3587	if (nxqs > 1) {
   3588		nxqs >>= 1;
   3589		goto try_again;
   3590	}
   3591	dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
   3592	return -ENOSPC;
   3593}