cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

tcp.c (70187B)


      1// SPDX-License-Identifier: GPL-2.0
      2/*
      3 * NVMe over Fabrics TCP host.
      4 * Copyright (c) 2018 Lightbits Labs. All rights reserved.
      5 */
      6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
      7#include <linux/module.h>
      8#include <linux/init.h>
      9#include <linux/slab.h>
     10#include <linux/err.h>
     11#include <linux/nvme-tcp.h>
     12#include <net/sock.h>
     13#include <net/tcp.h>
     14#include <linux/blk-mq.h>
     15#include <crypto/hash.h>
     16#include <net/busy_poll.h>
     17
     18#include "nvme.h"
     19#include "fabrics.h"
     20
     21struct nvme_tcp_queue;
     22
     23/* Define the socket priority to use for connections were it is desirable
     24 * that the NIC consider performing optimized packet processing or filtering.
     25 * A non-zero value being sufficient to indicate general consideration of any
     26 * possible optimization.  Making it a module param allows for alternative
     27 * values that may be unique for some NIC implementations.
     28 */
     29static int so_priority;
     30module_param(so_priority, int, 0644);
     31MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
     32
     33#ifdef CONFIG_DEBUG_LOCK_ALLOC
     34/* lockdep can detect a circular dependency of the form
     35 *   sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
     36 * because dependencies are tracked for both nvme-tcp and user contexts. Using
     37 * a separate class prevents lockdep from conflating nvme-tcp socket use with
     38 * user-space socket API use.
     39 */
     40static struct lock_class_key nvme_tcp_sk_key[2];
     41static struct lock_class_key nvme_tcp_slock_key[2];
     42
     43static void nvme_tcp_reclassify_socket(struct socket *sock)
     44{
     45	struct sock *sk = sock->sk;
     46
     47	if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
     48		return;
     49
     50	switch (sk->sk_family) {
     51	case AF_INET:
     52		sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
     53					      &nvme_tcp_slock_key[0],
     54					      "sk_lock-AF_INET-NVME",
     55					      &nvme_tcp_sk_key[0]);
     56		break;
     57	case AF_INET6:
     58		sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
     59					      &nvme_tcp_slock_key[1],
     60					      "sk_lock-AF_INET6-NVME",
     61					      &nvme_tcp_sk_key[1]);
     62		break;
     63	default:
     64		WARN_ON_ONCE(1);
     65	}
     66}
     67#else
     68static void nvme_tcp_reclassify_socket(struct socket *sock) { }
     69#endif
     70
     71enum nvme_tcp_send_state {
     72	NVME_TCP_SEND_CMD_PDU = 0,
     73	NVME_TCP_SEND_H2C_PDU,
     74	NVME_TCP_SEND_DATA,
     75	NVME_TCP_SEND_DDGST,
     76};
     77
     78struct nvme_tcp_request {
     79	struct nvme_request	req;
     80	void			*pdu;
     81	struct nvme_tcp_queue	*queue;
     82	u32			data_len;
     83	u32			pdu_len;
     84	u32			pdu_sent;
     85	u32			h2cdata_left;
     86	u32			h2cdata_offset;
     87	u16			ttag;
     88	__le16			status;
     89	struct list_head	entry;
     90	struct llist_node	lentry;
     91	__le32			ddgst;
     92
     93	struct bio		*curr_bio;
     94	struct iov_iter		iter;
     95
     96	/* send state */
     97	size_t			offset;
     98	size_t			data_sent;
     99	enum nvme_tcp_send_state state;
    100};
    101
    102enum nvme_tcp_queue_flags {
    103	NVME_TCP_Q_ALLOCATED	= 0,
    104	NVME_TCP_Q_LIVE		= 1,
    105	NVME_TCP_Q_POLLING	= 2,
    106};
    107
    108enum nvme_tcp_recv_state {
    109	NVME_TCP_RECV_PDU = 0,
    110	NVME_TCP_RECV_DATA,
    111	NVME_TCP_RECV_DDGST,
    112};
    113
    114struct nvme_tcp_ctrl;
    115struct nvme_tcp_queue {
    116	struct socket		*sock;
    117	struct work_struct	io_work;
    118	int			io_cpu;
    119
    120	struct mutex		queue_lock;
    121	struct mutex		send_mutex;
    122	struct llist_head	req_list;
    123	struct list_head	send_list;
    124	bool			more_requests;
    125
    126	/* recv state */
    127	void			*pdu;
    128	int			pdu_remaining;
    129	int			pdu_offset;
    130	size_t			data_remaining;
    131	size_t			ddgst_remaining;
    132	unsigned int		nr_cqe;
    133
    134	/* send state */
    135	struct nvme_tcp_request *request;
    136
    137	int			queue_size;
    138	u32			maxh2cdata;
    139	size_t			cmnd_capsule_len;
    140	struct nvme_tcp_ctrl	*ctrl;
    141	unsigned long		flags;
    142	bool			rd_enabled;
    143
    144	bool			hdr_digest;
    145	bool			data_digest;
    146	struct ahash_request	*rcv_hash;
    147	struct ahash_request	*snd_hash;
    148	__le32			exp_ddgst;
    149	__le32			recv_ddgst;
    150
    151	struct page_frag_cache	pf_cache;
    152
    153	void (*state_change)(struct sock *);
    154	void (*data_ready)(struct sock *);
    155	void (*write_space)(struct sock *);
    156};
    157
    158struct nvme_tcp_ctrl {
    159	/* read only in the hot path */
    160	struct nvme_tcp_queue	*queues;
    161	struct blk_mq_tag_set	tag_set;
    162
    163	/* other member variables */
    164	struct list_head	list;
    165	struct blk_mq_tag_set	admin_tag_set;
    166	struct sockaddr_storage addr;
    167	struct sockaddr_storage src_addr;
    168	struct nvme_ctrl	ctrl;
    169
    170	struct work_struct	err_work;
    171	struct delayed_work	connect_work;
    172	struct nvme_tcp_request async_req;
    173	u32			io_queues[HCTX_MAX_TYPES];
    174};
    175
    176static LIST_HEAD(nvme_tcp_ctrl_list);
    177static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
    178static struct workqueue_struct *nvme_tcp_wq;
    179static const struct blk_mq_ops nvme_tcp_mq_ops;
    180static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
    181static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
    182
    183static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
    184{
    185	return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
    186}
    187
    188static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
    189{
    190	return queue - queue->ctrl->queues;
    191}
    192
    193static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
    194{
    195	u32 queue_idx = nvme_tcp_queue_id(queue);
    196
    197	if (queue_idx == 0)
    198		return queue->ctrl->admin_tag_set.tags[queue_idx];
    199	return queue->ctrl->tag_set.tags[queue_idx - 1];
    200}
    201
    202static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
    203{
    204	return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
    205}
    206
    207static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
    208{
    209	return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
    210}
    211
    212static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
    213{
    214	return queue->cmnd_capsule_len - sizeof(struct nvme_command);
    215}
    216
    217static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
    218{
    219	return req == &req->queue->ctrl->async_req;
    220}
    221
    222static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
    223{
    224	struct request *rq;
    225
    226	if (unlikely(nvme_tcp_async_req(req)))
    227		return false; /* async events don't have a request */
    228
    229	rq = blk_mq_rq_from_pdu(req);
    230
    231	return rq_data_dir(rq) == WRITE && req->data_len &&
    232		req->data_len <= nvme_tcp_inline_data_size(req->queue);
    233}
    234
    235static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
    236{
    237	return req->iter.bvec->bv_page;
    238}
    239
    240static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
    241{
    242	return req->iter.bvec->bv_offset + req->iter.iov_offset;
    243}
    244
    245static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
    246{
    247	return min_t(size_t, iov_iter_single_seg_count(&req->iter),
    248			req->pdu_len - req->pdu_sent);
    249}
    250
    251static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
    252{
    253	return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
    254			req->pdu_len - req->pdu_sent : 0;
    255}
    256
    257static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
    258		int len)
    259{
    260	return nvme_tcp_pdu_data_left(req) <= len;
    261}
    262
    263static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
    264		unsigned int dir)
    265{
    266	struct request *rq = blk_mq_rq_from_pdu(req);
    267	struct bio_vec *vec;
    268	unsigned int size;
    269	int nr_bvec;
    270	size_t offset;
    271
    272	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
    273		vec = &rq->special_vec;
    274		nr_bvec = 1;
    275		size = blk_rq_payload_bytes(rq);
    276		offset = 0;
    277	} else {
    278		struct bio *bio = req->curr_bio;
    279		struct bvec_iter bi;
    280		struct bio_vec bv;
    281
    282		vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
    283		nr_bvec = 0;
    284		bio_for_each_bvec(bv, bio, bi) {
    285			nr_bvec++;
    286		}
    287		size = bio->bi_iter.bi_size;
    288		offset = bio->bi_iter.bi_bvec_done;
    289	}
    290
    291	iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size);
    292	req->iter.iov_offset = offset;
    293}
    294
    295static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
    296		int len)
    297{
    298	req->data_sent += len;
    299	req->pdu_sent += len;
    300	iov_iter_advance(&req->iter, len);
    301	if (!iov_iter_count(&req->iter) &&
    302	    req->data_sent < req->data_len) {
    303		req->curr_bio = req->curr_bio->bi_next;
    304		nvme_tcp_init_iter(req, WRITE);
    305	}
    306}
    307
    308static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
    309{
    310	int ret;
    311
    312	/* drain the send queue as much as we can... */
    313	do {
    314		ret = nvme_tcp_try_send(queue);
    315	} while (ret > 0);
    316}
    317
    318static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
    319{
    320	return !list_empty(&queue->send_list) ||
    321		!llist_empty(&queue->req_list) || queue->more_requests;
    322}
    323
    324static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
    325		bool sync, bool last)
    326{
    327	struct nvme_tcp_queue *queue = req->queue;
    328	bool empty;
    329
    330	empty = llist_add(&req->lentry, &queue->req_list) &&
    331		list_empty(&queue->send_list) && !queue->request;
    332
    333	/*
    334	 * if we're the first on the send_list and we can try to send
    335	 * directly, otherwise queue io_work. Also, only do that if we
    336	 * are on the same cpu, so we don't introduce contention.
    337	 */
    338	if (queue->io_cpu == raw_smp_processor_id() &&
    339	    sync && empty && mutex_trylock(&queue->send_mutex)) {
    340		queue->more_requests = !last;
    341		nvme_tcp_send_all(queue);
    342		queue->more_requests = false;
    343		mutex_unlock(&queue->send_mutex);
    344	}
    345
    346	if (last && nvme_tcp_queue_more(queue))
    347		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
    348}
    349
    350static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
    351{
    352	struct nvme_tcp_request *req;
    353	struct llist_node *node;
    354
    355	for (node = llist_del_all(&queue->req_list); node; node = node->next) {
    356		req = llist_entry(node, struct nvme_tcp_request, lentry);
    357		list_add(&req->entry, &queue->send_list);
    358	}
    359}
    360
    361static inline struct nvme_tcp_request *
    362nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
    363{
    364	struct nvme_tcp_request *req;
    365
    366	req = list_first_entry_or_null(&queue->send_list,
    367			struct nvme_tcp_request, entry);
    368	if (!req) {
    369		nvme_tcp_process_req_list(queue);
    370		req = list_first_entry_or_null(&queue->send_list,
    371				struct nvme_tcp_request, entry);
    372		if (unlikely(!req))
    373			return NULL;
    374	}
    375
    376	list_del(&req->entry);
    377	return req;
    378}
    379
    380static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
    381		__le32 *dgst)
    382{
    383	ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
    384	crypto_ahash_final(hash);
    385}
    386
    387static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
    388		struct page *page, off_t off, size_t len)
    389{
    390	struct scatterlist sg;
    391
    392	sg_init_marker(&sg, 1);
    393	sg_set_page(&sg, page, len, off);
    394	ahash_request_set_crypt(hash, &sg, NULL, len);
    395	crypto_ahash_update(hash);
    396}
    397
    398static inline void nvme_tcp_hdgst(struct ahash_request *hash,
    399		void *pdu, size_t len)
    400{
    401	struct scatterlist sg;
    402
    403	sg_init_one(&sg, pdu, len);
    404	ahash_request_set_crypt(hash, &sg, pdu + len, len);
    405	crypto_ahash_digest(hash);
    406}
    407
    408static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
    409		void *pdu, size_t pdu_len)
    410{
    411	struct nvme_tcp_hdr *hdr = pdu;
    412	__le32 recv_digest;
    413	__le32 exp_digest;
    414
    415	if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
    416		dev_err(queue->ctrl->ctrl.device,
    417			"queue %d: header digest flag is cleared\n",
    418			nvme_tcp_queue_id(queue));
    419		return -EPROTO;
    420	}
    421
    422	recv_digest = *(__le32 *)(pdu + hdr->hlen);
    423	nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
    424	exp_digest = *(__le32 *)(pdu + hdr->hlen);
    425	if (recv_digest != exp_digest) {
    426		dev_err(queue->ctrl->ctrl.device,
    427			"header digest error: recv %#x expected %#x\n",
    428			le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
    429		return -EIO;
    430	}
    431
    432	return 0;
    433}
    434
    435static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
    436{
    437	struct nvme_tcp_hdr *hdr = pdu;
    438	u8 digest_len = nvme_tcp_hdgst_len(queue);
    439	u32 len;
    440
    441	len = le32_to_cpu(hdr->plen) - hdr->hlen -
    442		((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
    443
    444	if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
    445		dev_err(queue->ctrl->ctrl.device,
    446			"queue %d: data digest flag is cleared\n",
    447		nvme_tcp_queue_id(queue));
    448		return -EPROTO;
    449	}
    450	crypto_ahash_init(queue->rcv_hash);
    451
    452	return 0;
    453}
    454
    455static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
    456		struct request *rq, unsigned int hctx_idx)
    457{
    458	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
    459
    460	page_frag_free(req->pdu);
    461}
    462
    463static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
    464		struct request *rq, unsigned int hctx_idx,
    465		unsigned int numa_node)
    466{
    467	struct nvme_tcp_ctrl *ctrl = set->driver_data;
    468	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
    469	struct nvme_tcp_cmd_pdu *pdu;
    470	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
    471	struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
    472	u8 hdgst = nvme_tcp_hdgst_len(queue);
    473
    474	req->pdu = page_frag_alloc(&queue->pf_cache,
    475		sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
    476		GFP_KERNEL | __GFP_ZERO);
    477	if (!req->pdu)
    478		return -ENOMEM;
    479
    480	pdu = req->pdu;
    481	req->queue = queue;
    482	nvme_req(rq)->ctrl = &ctrl->ctrl;
    483	nvme_req(rq)->cmd = &pdu->cmd;
    484
    485	return 0;
    486}
    487
    488static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
    489		unsigned int hctx_idx)
    490{
    491	struct nvme_tcp_ctrl *ctrl = data;
    492	struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
    493
    494	hctx->driver_data = queue;
    495	return 0;
    496}
    497
    498static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
    499		unsigned int hctx_idx)
    500{
    501	struct nvme_tcp_ctrl *ctrl = data;
    502	struct nvme_tcp_queue *queue = &ctrl->queues[0];
    503
    504	hctx->driver_data = queue;
    505	return 0;
    506}
    507
    508static enum nvme_tcp_recv_state
    509nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
    510{
    511	return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
    512		(queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
    513		NVME_TCP_RECV_DATA;
    514}
    515
    516static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
    517{
    518	queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
    519				nvme_tcp_hdgst_len(queue);
    520	queue->pdu_offset = 0;
    521	queue->data_remaining = -1;
    522	queue->ddgst_remaining = 0;
    523}
    524
    525static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
    526{
    527	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
    528		return;
    529
    530	dev_warn(ctrl->device, "starting error recovery\n");
    531	queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
    532}
    533
    534static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
    535		struct nvme_completion *cqe)
    536{
    537	struct nvme_tcp_request *req;
    538	struct request *rq;
    539
    540	rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
    541	if (!rq) {
    542		dev_err(queue->ctrl->ctrl.device,
    543			"got bad cqe.command_id %#x on queue %d\n",
    544			cqe->command_id, nvme_tcp_queue_id(queue));
    545		nvme_tcp_error_recovery(&queue->ctrl->ctrl);
    546		return -EINVAL;
    547	}
    548
    549	req = blk_mq_rq_to_pdu(rq);
    550	if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
    551		req->status = cqe->status;
    552
    553	if (!nvme_try_complete_req(rq, req->status, cqe->result))
    554		nvme_complete_rq(rq);
    555	queue->nr_cqe++;
    556
    557	return 0;
    558}
    559
    560static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
    561		struct nvme_tcp_data_pdu *pdu)
    562{
    563	struct request *rq;
    564
    565	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
    566	if (!rq) {
    567		dev_err(queue->ctrl->ctrl.device,
    568			"got bad c2hdata.command_id %#x on queue %d\n",
    569			pdu->command_id, nvme_tcp_queue_id(queue));
    570		return -ENOENT;
    571	}
    572
    573	if (!blk_rq_payload_bytes(rq)) {
    574		dev_err(queue->ctrl->ctrl.device,
    575			"queue %d tag %#x unexpected data\n",
    576			nvme_tcp_queue_id(queue), rq->tag);
    577		return -EIO;
    578	}
    579
    580	queue->data_remaining = le32_to_cpu(pdu->data_length);
    581
    582	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
    583	    unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
    584		dev_err(queue->ctrl->ctrl.device,
    585			"queue %d tag %#x SUCCESS set but not last PDU\n",
    586			nvme_tcp_queue_id(queue), rq->tag);
    587		nvme_tcp_error_recovery(&queue->ctrl->ctrl);
    588		return -EPROTO;
    589	}
    590
    591	return 0;
    592}
    593
    594static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
    595		struct nvme_tcp_rsp_pdu *pdu)
    596{
    597	struct nvme_completion *cqe = &pdu->cqe;
    598	int ret = 0;
    599
    600	/*
    601	 * AEN requests are special as they don't time out and can
    602	 * survive any kind of queue freeze and often don't respond to
    603	 * aborts.  We don't even bother to allocate a struct request
    604	 * for them but rather special case them here.
    605	 */
    606	if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
    607				     cqe->command_id)))
    608		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
    609				&cqe->result);
    610	else
    611		ret = nvme_tcp_process_nvme_cqe(queue, cqe);
    612
    613	return ret;
    614}
    615
    616static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
    617{
    618	struct nvme_tcp_data_pdu *data = req->pdu;
    619	struct nvme_tcp_queue *queue = req->queue;
    620	struct request *rq = blk_mq_rq_from_pdu(req);
    621	u32 h2cdata_sent = req->pdu_len;
    622	u8 hdgst = nvme_tcp_hdgst_len(queue);
    623	u8 ddgst = nvme_tcp_ddgst_len(queue);
    624
    625	req->state = NVME_TCP_SEND_H2C_PDU;
    626	req->offset = 0;
    627	req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
    628	req->pdu_sent = 0;
    629	req->h2cdata_left -= req->pdu_len;
    630	req->h2cdata_offset += h2cdata_sent;
    631
    632	memset(data, 0, sizeof(*data));
    633	data->hdr.type = nvme_tcp_h2c_data;
    634	if (!req->h2cdata_left)
    635		data->hdr.flags = NVME_TCP_F_DATA_LAST;
    636	if (queue->hdr_digest)
    637		data->hdr.flags |= NVME_TCP_F_HDGST;
    638	if (queue->data_digest)
    639		data->hdr.flags |= NVME_TCP_F_DDGST;
    640	data->hdr.hlen = sizeof(*data);
    641	data->hdr.pdo = data->hdr.hlen + hdgst;
    642	data->hdr.plen =
    643		cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
    644	data->ttag = req->ttag;
    645	data->command_id = nvme_cid(rq);
    646	data->data_offset = cpu_to_le32(req->h2cdata_offset);
    647	data->data_length = cpu_to_le32(req->pdu_len);
    648}
    649
    650static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
    651		struct nvme_tcp_r2t_pdu *pdu)
    652{
    653	struct nvme_tcp_request *req;
    654	struct request *rq;
    655	u32 r2t_length = le32_to_cpu(pdu->r2t_length);
    656	u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
    657
    658	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
    659	if (!rq) {
    660		dev_err(queue->ctrl->ctrl.device,
    661			"got bad r2t.command_id %#x on queue %d\n",
    662			pdu->command_id, nvme_tcp_queue_id(queue));
    663		return -ENOENT;
    664	}
    665	req = blk_mq_rq_to_pdu(rq);
    666
    667	if (unlikely(!r2t_length)) {
    668		dev_err(queue->ctrl->ctrl.device,
    669			"req %d r2t len is %u, probably a bug...\n",
    670			rq->tag, r2t_length);
    671		return -EPROTO;
    672	}
    673
    674	if (unlikely(req->data_sent + r2t_length > req->data_len)) {
    675		dev_err(queue->ctrl->ctrl.device,
    676			"req %d r2t len %u exceeded data len %u (%zu sent)\n",
    677			rq->tag, r2t_length, req->data_len, req->data_sent);
    678		return -EPROTO;
    679	}
    680
    681	if (unlikely(r2t_offset < req->data_sent)) {
    682		dev_err(queue->ctrl->ctrl.device,
    683			"req %d unexpected r2t offset %u (expected %zu)\n",
    684			rq->tag, r2t_offset, req->data_sent);
    685		return -EPROTO;
    686	}
    687
    688	req->pdu_len = 0;
    689	req->h2cdata_left = r2t_length;
    690	req->h2cdata_offset = r2t_offset;
    691	req->ttag = pdu->ttag;
    692
    693	nvme_tcp_setup_h2c_data_pdu(req);
    694	nvme_tcp_queue_request(req, false, true);
    695
    696	return 0;
    697}
    698
    699static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
    700		unsigned int *offset, size_t *len)
    701{
    702	struct nvme_tcp_hdr *hdr;
    703	char *pdu = queue->pdu;
    704	size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
    705	int ret;
    706
    707	ret = skb_copy_bits(skb, *offset,
    708		&pdu[queue->pdu_offset], rcv_len);
    709	if (unlikely(ret))
    710		return ret;
    711
    712	queue->pdu_remaining -= rcv_len;
    713	queue->pdu_offset += rcv_len;
    714	*offset += rcv_len;
    715	*len -= rcv_len;
    716	if (queue->pdu_remaining)
    717		return 0;
    718
    719	hdr = queue->pdu;
    720	if (queue->hdr_digest) {
    721		ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
    722		if (unlikely(ret))
    723			return ret;
    724	}
    725
    726
    727	if (queue->data_digest) {
    728		ret = nvme_tcp_check_ddgst(queue, queue->pdu);
    729		if (unlikely(ret))
    730			return ret;
    731	}
    732
    733	switch (hdr->type) {
    734	case nvme_tcp_c2h_data:
    735		return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
    736	case nvme_tcp_rsp:
    737		nvme_tcp_init_recv_ctx(queue);
    738		return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
    739	case nvme_tcp_r2t:
    740		nvme_tcp_init_recv_ctx(queue);
    741		return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
    742	default:
    743		dev_err(queue->ctrl->ctrl.device,
    744			"unsupported pdu type (%d)\n", hdr->type);
    745		return -EINVAL;
    746	}
    747}
    748
    749static inline void nvme_tcp_end_request(struct request *rq, u16 status)
    750{
    751	union nvme_result res = {};
    752
    753	if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
    754		nvme_complete_rq(rq);
    755}
    756
    757static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
    758			      unsigned int *offset, size_t *len)
    759{
    760	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
    761	struct request *rq =
    762		nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
    763	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
    764
    765	while (true) {
    766		int recv_len, ret;
    767
    768		recv_len = min_t(size_t, *len, queue->data_remaining);
    769		if (!recv_len)
    770			break;
    771
    772		if (!iov_iter_count(&req->iter)) {
    773			req->curr_bio = req->curr_bio->bi_next;
    774
    775			/*
    776			 * If we don`t have any bios it means that controller
    777			 * sent more data than we requested, hence error
    778			 */
    779			if (!req->curr_bio) {
    780				dev_err(queue->ctrl->ctrl.device,
    781					"queue %d no space in request %#x",
    782					nvme_tcp_queue_id(queue), rq->tag);
    783				nvme_tcp_init_recv_ctx(queue);
    784				return -EIO;
    785			}
    786			nvme_tcp_init_iter(req, READ);
    787		}
    788
    789		/* we can read only from what is left in this bio */
    790		recv_len = min_t(size_t, recv_len,
    791				iov_iter_count(&req->iter));
    792
    793		if (queue->data_digest)
    794			ret = skb_copy_and_hash_datagram_iter(skb, *offset,
    795				&req->iter, recv_len, queue->rcv_hash);
    796		else
    797			ret = skb_copy_datagram_iter(skb, *offset,
    798					&req->iter, recv_len);
    799		if (ret) {
    800			dev_err(queue->ctrl->ctrl.device,
    801				"queue %d failed to copy request %#x data",
    802				nvme_tcp_queue_id(queue), rq->tag);
    803			return ret;
    804		}
    805
    806		*len -= recv_len;
    807		*offset += recv_len;
    808		queue->data_remaining -= recv_len;
    809	}
    810
    811	if (!queue->data_remaining) {
    812		if (queue->data_digest) {
    813			nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
    814			queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
    815		} else {
    816			if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
    817				nvme_tcp_end_request(rq,
    818						le16_to_cpu(req->status));
    819				queue->nr_cqe++;
    820			}
    821			nvme_tcp_init_recv_ctx(queue);
    822		}
    823	}
    824
    825	return 0;
    826}
    827
    828static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
    829		struct sk_buff *skb, unsigned int *offset, size_t *len)
    830{
    831	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
    832	char *ddgst = (char *)&queue->recv_ddgst;
    833	size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
    834	off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
    835	int ret;
    836
    837	ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
    838	if (unlikely(ret))
    839		return ret;
    840
    841	queue->ddgst_remaining -= recv_len;
    842	*offset += recv_len;
    843	*len -= recv_len;
    844	if (queue->ddgst_remaining)
    845		return 0;
    846
    847	if (queue->recv_ddgst != queue->exp_ddgst) {
    848		struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
    849					pdu->command_id);
    850		struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
    851
    852		req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
    853
    854		dev_err(queue->ctrl->ctrl.device,
    855			"data digest error: recv %#x expected %#x\n",
    856			le32_to_cpu(queue->recv_ddgst),
    857			le32_to_cpu(queue->exp_ddgst));
    858	}
    859
    860	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
    861		struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
    862					pdu->command_id);
    863		struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
    864
    865		nvme_tcp_end_request(rq, le16_to_cpu(req->status));
    866		queue->nr_cqe++;
    867	}
    868
    869	nvme_tcp_init_recv_ctx(queue);
    870	return 0;
    871}
    872
    873static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
    874			     unsigned int offset, size_t len)
    875{
    876	struct nvme_tcp_queue *queue = desc->arg.data;
    877	size_t consumed = len;
    878	int result;
    879
    880	while (len) {
    881		switch (nvme_tcp_recv_state(queue)) {
    882		case NVME_TCP_RECV_PDU:
    883			result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
    884			break;
    885		case NVME_TCP_RECV_DATA:
    886			result = nvme_tcp_recv_data(queue, skb, &offset, &len);
    887			break;
    888		case NVME_TCP_RECV_DDGST:
    889			result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
    890			break;
    891		default:
    892			result = -EFAULT;
    893		}
    894		if (result) {
    895			dev_err(queue->ctrl->ctrl.device,
    896				"receive failed:  %d\n", result);
    897			queue->rd_enabled = false;
    898			nvme_tcp_error_recovery(&queue->ctrl->ctrl);
    899			return result;
    900		}
    901	}
    902
    903	return consumed;
    904}
    905
    906static void nvme_tcp_data_ready(struct sock *sk)
    907{
    908	struct nvme_tcp_queue *queue;
    909
    910	read_lock_bh(&sk->sk_callback_lock);
    911	queue = sk->sk_user_data;
    912	if (likely(queue && queue->rd_enabled) &&
    913	    !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
    914		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
    915	read_unlock_bh(&sk->sk_callback_lock);
    916}
    917
    918static void nvme_tcp_write_space(struct sock *sk)
    919{
    920	struct nvme_tcp_queue *queue;
    921
    922	read_lock_bh(&sk->sk_callback_lock);
    923	queue = sk->sk_user_data;
    924	if (likely(queue && sk_stream_is_writeable(sk))) {
    925		clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
    926		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
    927	}
    928	read_unlock_bh(&sk->sk_callback_lock);
    929}
    930
    931static void nvme_tcp_state_change(struct sock *sk)
    932{
    933	struct nvme_tcp_queue *queue;
    934
    935	read_lock_bh(&sk->sk_callback_lock);
    936	queue = sk->sk_user_data;
    937	if (!queue)
    938		goto done;
    939
    940	switch (sk->sk_state) {
    941	case TCP_CLOSE:
    942	case TCP_CLOSE_WAIT:
    943	case TCP_LAST_ACK:
    944	case TCP_FIN_WAIT1:
    945	case TCP_FIN_WAIT2:
    946		nvme_tcp_error_recovery(&queue->ctrl->ctrl);
    947		break;
    948	default:
    949		dev_info(queue->ctrl->ctrl.device,
    950			"queue %d socket state %d\n",
    951			nvme_tcp_queue_id(queue), sk->sk_state);
    952	}
    953
    954	queue->state_change(sk);
    955done:
    956	read_unlock_bh(&sk->sk_callback_lock);
    957}
    958
    959static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
    960{
    961	queue->request = NULL;
    962}
    963
    964static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
    965{
    966	if (nvme_tcp_async_req(req)) {
    967		union nvme_result res = {};
    968
    969		nvme_complete_async_event(&req->queue->ctrl->ctrl,
    970				cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
    971	} else {
    972		nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
    973				NVME_SC_HOST_PATH_ERROR);
    974	}
    975}
    976
    977static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
    978{
    979	struct nvme_tcp_queue *queue = req->queue;
    980	int req_data_len = req->data_len;
    981	u32 h2cdata_left = req->h2cdata_left;
    982
    983	while (true) {
    984		struct page *page = nvme_tcp_req_cur_page(req);
    985		size_t offset = nvme_tcp_req_cur_offset(req);
    986		size_t len = nvme_tcp_req_cur_length(req);
    987		bool last = nvme_tcp_pdu_last_send(req, len);
    988		int req_data_sent = req->data_sent;
    989		int ret, flags = MSG_DONTWAIT;
    990
    991		if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
    992			flags |= MSG_EOR;
    993		else
    994			flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
    995
    996		if (sendpage_ok(page)) {
    997			ret = kernel_sendpage(queue->sock, page, offset, len,
    998					flags);
    999		} else {
   1000			ret = sock_no_sendpage(queue->sock, page, offset, len,
   1001					flags);
   1002		}
   1003		if (ret <= 0)
   1004			return ret;
   1005
   1006		if (queue->data_digest)
   1007			nvme_tcp_ddgst_update(queue->snd_hash, page,
   1008					offset, ret);
   1009
   1010		/*
   1011		 * update the request iterator except for the last payload send
   1012		 * in the request where we don't want to modify it as we may
   1013		 * compete with the RX path completing the request.
   1014		 */
   1015		if (req_data_sent + ret < req_data_len)
   1016			nvme_tcp_advance_req(req, ret);
   1017
   1018		/* fully successful last send in current PDU */
   1019		if (last && ret == len) {
   1020			if (queue->data_digest) {
   1021				nvme_tcp_ddgst_final(queue->snd_hash,
   1022					&req->ddgst);
   1023				req->state = NVME_TCP_SEND_DDGST;
   1024				req->offset = 0;
   1025			} else {
   1026				if (h2cdata_left)
   1027					nvme_tcp_setup_h2c_data_pdu(req);
   1028				else
   1029					nvme_tcp_done_send_req(queue);
   1030			}
   1031			return 1;
   1032		}
   1033	}
   1034	return -EAGAIN;
   1035}
   1036
   1037static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
   1038{
   1039	struct nvme_tcp_queue *queue = req->queue;
   1040	struct nvme_tcp_cmd_pdu *pdu = req->pdu;
   1041	bool inline_data = nvme_tcp_has_inline_data(req);
   1042	u8 hdgst = nvme_tcp_hdgst_len(queue);
   1043	int len = sizeof(*pdu) + hdgst - req->offset;
   1044	int flags = MSG_DONTWAIT;
   1045	int ret;
   1046
   1047	if (inline_data || nvme_tcp_queue_more(queue))
   1048		flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
   1049	else
   1050		flags |= MSG_EOR;
   1051
   1052	if (queue->hdr_digest && !req->offset)
   1053		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
   1054
   1055	ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
   1056			offset_in_page(pdu) + req->offset, len,  flags);
   1057	if (unlikely(ret <= 0))
   1058		return ret;
   1059
   1060	len -= ret;
   1061	if (!len) {
   1062		if (inline_data) {
   1063			req->state = NVME_TCP_SEND_DATA;
   1064			if (queue->data_digest)
   1065				crypto_ahash_init(queue->snd_hash);
   1066		} else {
   1067			nvme_tcp_done_send_req(queue);
   1068		}
   1069		return 1;
   1070	}
   1071	req->offset += ret;
   1072
   1073	return -EAGAIN;
   1074}
   1075
   1076static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
   1077{
   1078	struct nvme_tcp_queue *queue = req->queue;
   1079	struct nvme_tcp_data_pdu *pdu = req->pdu;
   1080	u8 hdgst = nvme_tcp_hdgst_len(queue);
   1081	int len = sizeof(*pdu) - req->offset + hdgst;
   1082	int ret;
   1083
   1084	if (queue->hdr_digest && !req->offset)
   1085		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
   1086
   1087	if (!req->h2cdata_left)
   1088		ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
   1089				offset_in_page(pdu) + req->offset, len,
   1090				MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
   1091	else
   1092		ret = sock_no_sendpage(queue->sock, virt_to_page(pdu),
   1093				offset_in_page(pdu) + req->offset, len,
   1094				MSG_DONTWAIT | MSG_MORE);
   1095	if (unlikely(ret <= 0))
   1096		return ret;
   1097
   1098	len -= ret;
   1099	if (!len) {
   1100		req->state = NVME_TCP_SEND_DATA;
   1101		if (queue->data_digest)
   1102			crypto_ahash_init(queue->snd_hash);
   1103		return 1;
   1104	}
   1105	req->offset += ret;
   1106
   1107	return -EAGAIN;
   1108}
   1109
   1110static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
   1111{
   1112	struct nvme_tcp_queue *queue = req->queue;
   1113	size_t offset = req->offset;
   1114	u32 h2cdata_left = req->h2cdata_left;
   1115	int ret;
   1116	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
   1117	struct kvec iov = {
   1118		.iov_base = (u8 *)&req->ddgst + req->offset,
   1119		.iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
   1120	};
   1121
   1122	if (nvme_tcp_queue_more(queue))
   1123		msg.msg_flags |= MSG_MORE;
   1124	else
   1125		msg.msg_flags |= MSG_EOR;
   1126
   1127	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
   1128	if (unlikely(ret <= 0))
   1129		return ret;
   1130
   1131	if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
   1132		if (h2cdata_left)
   1133			nvme_tcp_setup_h2c_data_pdu(req);
   1134		else
   1135			nvme_tcp_done_send_req(queue);
   1136		return 1;
   1137	}
   1138
   1139	req->offset += ret;
   1140	return -EAGAIN;
   1141}
   1142
   1143static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
   1144{
   1145	struct nvme_tcp_request *req;
   1146	int ret = 1;
   1147
   1148	if (!queue->request) {
   1149		queue->request = nvme_tcp_fetch_request(queue);
   1150		if (!queue->request)
   1151			return 0;
   1152	}
   1153	req = queue->request;
   1154
   1155	if (req->state == NVME_TCP_SEND_CMD_PDU) {
   1156		ret = nvme_tcp_try_send_cmd_pdu(req);
   1157		if (ret <= 0)
   1158			goto done;
   1159		if (!nvme_tcp_has_inline_data(req))
   1160			return ret;
   1161	}
   1162
   1163	if (req->state == NVME_TCP_SEND_H2C_PDU) {
   1164		ret = nvme_tcp_try_send_data_pdu(req);
   1165		if (ret <= 0)
   1166			goto done;
   1167	}
   1168
   1169	if (req->state == NVME_TCP_SEND_DATA) {
   1170		ret = nvme_tcp_try_send_data(req);
   1171		if (ret <= 0)
   1172			goto done;
   1173	}
   1174
   1175	if (req->state == NVME_TCP_SEND_DDGST)
   1176		ret = nvme_tcp_try_send_ddgst(req);
   1177done:
   1178	if (ret == -EAGAIN) {
   1179		ret = 0;
   1180	} else if (ret < 0) {
   1181		dev_err(queue->ctrl->ctrl.device,
   1182			"failed to send request %d\n", ret);
   1183		nvme_tcp_fail_request(queue->request);
   1184		nvme_tcp_done_send_req(queue);
   1185	}
   1186	return ret;
   1187}
   1188
   1189static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
   1190{
   1191	struct socket *sock = queue->sock;
   1192	struct sock *sk = sock->sk;
   1193	read_descriptor_t rd_desc;
   1194	int consumed;
   1195
   1196	rd_desc.arg.data = queue;
   1197	rd_desc.count = 1;
   1198	lock_sock(sk);
   1199	queue->nr_cqe = 0;
   1200	consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
   1201	release_sock(sk);
   1202	return consumed;
   1203}
   1204
   1205static void nvme_tcp_io_work(struct work_struct *w)
   1206{
   1207	struct nvme_tcp_queue *queue =
   1208		container_of(w, struct nvme_tcp_queue, io_work);
   1209	unsigned long deadline = jiffies + msecs_to_jiffies(1);
   1210
   1211	do {
   1212		bool pending = false;
   1213		int result;
   1214
   1215		if (mutex_trylock(&queue->send_mutex)) {
   1216			result = nvme_tcp_try_send(queue);
   1217			mutex_unlock(&queue->send_mutex);
   1218			if (result > 0)
   1219				pending = true;
   1220			else if (unlikely(result < 0))
   1221				break;
   1222		}
   1223
   1224		result = nvme_tcp_try_recv(queue);
   1225		if (result > 0)
   1226			pending = true;
   1227		else if (unlikely(result < 0))
   1228			return;
   1229
   1230		if (!pending)
   1231			return;
   1232
   1233	} while (!time_after(jiffies, deadline)); /* quota is exhausted */
   1234
   1235	queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
   1236}
   1237
   1238static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
   1239{
   1240	struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
   1241
   1242	ahash_request_free(queue->rcv_hash);
   1243	ahash_request_free(queue->snd_hash);
   1244	crypto_free_ahash(tfm);
   1245}
   1246
   1247static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
   1248{
   1249	struct crypto_ahash *tfm;
   1250
   1251	tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
   1252	if (IS_ERR(tfm))
   1253		return PTR_ERR(tfm);
   1254
   1255	queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
   1256	if (!queue->snd_hash)
   1257		goto free_tfm;
   1258	ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
   1259
   1260	queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
   1261	if (!queue->rcv_hash)
   1262		goto free_snd_hash;
   1263	ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
   1264
   1265	return 0;
   1266free_snd_hash:
   1267	ahash_request_free(queue->snd_hash);
   1268free_tfm:
   1269	crypto_free_ahash(tfm);
   1270	return -ENOMEM;
   1271}
   1272
   1273static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
   1274{
   1275	struct nvme_tcp_request *async = &ctrl->async_req;
   1276
   1277	page_frag_free(async->pdu);
   1278}
   1279
   1280static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
   1281{
   1282	struct nvme_tcp_queue *queue = &ctrl->queues[0];
   1283	struct nvme_tcp_request *async = &ctrl->async_req;
   1284	u8 hdgst = nvme_tcp_hdgst_len(queue);
   1285
   1286	async->pdu = page_frag_alloc(&queue->pf_cache,
   1287		sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
   1288		GFP_KERNEL | __GFP_ZERO);
   1289	if (!async->pdu)
   1290		return -ENOMEM;
   1291
   1292	async->queue = &ctrl->queues[0];
   1293	return 0;
   1294}
   1295
   1296static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
   1297{
   1298	struct page *page;
   1299	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   1300	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
   1301
   1302	if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
   1303		return;
   1304
   1305	if (queue->hdr_digest || queue->data_digest)
   1306		nvme_tcp_free_crypto(queue);
   1307
   1308	if (queue->pf_cache.va) {
   1309		page = virt_to_head_page(queue->pf_cache.va);
   1310		__page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
   1311		queue->pf_cache.va = NULL;
   1312	}
   1313	sock_release(queue->sock);
   1314	kfree(queue->pdu);
   1315	mutex_destroy(&queue->send_mutex);
   1316	mutex_destroy(&queue->queue_lock);
   1317}
   1318
   1319static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
   1320{
   1321	struct nvme_tcp_icreq_pdu *icreq;
   1322	struct nvme_tcp_icresp_pdu *icresp;
   1323	struct msghdr msg = {};
   1324	struct kvec iov;
   1325	bool ctrl_hdgst, ctrl_ddgst;
   1326	u32 maxh2cdata;
   1327	int ret;
   1328
   1329	icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
   1330	if (!icreq)
   1331		return -ENOMEM;
   1332
   1333	icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
   1334	if (!icresp) {
   1335		ret = -ENOMEM;
   1336		goto free_icreq;
   1337	}
   1338
   1339	icreq->hdr.type = nvme_tcp_icreq;
   1340	icreq->hdr.hlen = sizeof(*icreq);
   1341	icreq->hdr.pdo = 0;
   1342	icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
   1343	icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
   1344	icreq->maxr2t = 0; /* single inflight r2t supported */
   1345	icreq->hpda = 0; /* no alignment constraint */
   1346	if (queue->hdr_digest)
   1347		icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
   1348	if (queue->data_digest)
   1349		icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
   1350
   1351	iov.iov_base = icreq;
   1352	iov.iov_len = sizeof(*icreq);
   1353	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
   1354	if (ret < 0)
   1355		goto free_icresp;
   1356
   1357	memset(&msg, 0, sizeof(msg));
   1358	iov.iov_base = icresp;
   1359	iov.iov_len = sizeof(*icresp);
   1360	ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
   1361			iov.iov_len, msg.msg_flags);
   1362	if (ret < 0)
   1363		goto free_icresp;
   1364
   1365	ret = -EINVAL;
   1366	if (icresp->hdr.type != nvme_tcp_icresp) {
   1367		pr_err("queue %d: bad type returned %d\n",
   1368			nvme_tcp_queue_id(queue), icresp->hdr.type);
   1369		goto free_icresp;
   1370	}
   1371
   1372	if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
   1373		pr_err("queue %d: bad pdu length returned %d\n",
   1374			nvme_tcp_queue_id(queue), icresp->hdr.plen);
   1375		goto free_icresp;
   1376	}
   1377
   1378	if (icresp->pfv != NVME_TCP_PFV_1_0) {
   1379		pr_err("queue %d: bad pfv returned %d\n",
   1380			nvme_tcp_queue_id(queue), icresp->pfv);
   1381		goto free_icresp;
   1382	}
   1383
   1384	ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
   1385	if ((queue->data_digest && !ctrl_ddgst) ||
   1386	    (!queue->data_digest && ctrl_ddgst)) {
   1387		pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
   1388			nvme_tcp_queue_id(queue),
   1389			queue->data_digest ? "enabled" : "disabled",
   1390			ctrl_ddgst ? "enabled" : "disabled");
   1391		goto free_icresp;
   1392	}
   1393
   1394	ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
   1395	if ((queue->hdr_digest && !ctrl_hdgst) ||
   1396	    (!queue->hdr_digest && ctrl_hdgst)) {
   1397		pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
   1398			nvme_tcp_queue_id(queue),
   1399			queue->hdr_digest ? "enabled" : "disabled",
   1400			ctrl_hdgst ? "enabled" : "disabled");
   1401		goto free_icresp;
   1402	}
   1403
   1404	if (icresp->cpda != 0) {
   1405		pr_err("queue %d: unsupported cpda returned %d\n",
   1406			nvme_tcp_queue_id(queue), icresp->cpda);
   1407		goto free_icresp;
   1408	}
   1409
   1410	maxh2cdata = le32_to_cpu(icresp->maxdata);
   1411	if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
   1412		pr_err("queue %d: invalid maxh2cdata returned %u\n",
   1413		       nvme_tcp_queue_id(queue), maxh2cdata);
   1414		goto free_icresp;
   1415	}
   1416	queue->maxh2cdata = maxh2cdata;
   1417
   1418	ret = 0;
   1419free_icresp:
   1420	kfree(icresp);
   1421free_icreq:
   1422	kfree(icreq);
   1423	return ret;
   1424}
   1425
   1426static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
   1427{
   1428	return nvme_tcp_queue_id(queue) == 0;
   1429}
   1430
   1431static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
   1432{
   1433	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
   1434	int qid = nvme_tcp_queue_id(queue);
   1435
   1436	return !nvme_tcp_admin_queue(queue) &&
   1437		qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
   1438}
   1439
   1440static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
   1441{
   1442	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
   1443	int qid = nvme_tcp_queue_id(queue);
   1444
   1445	return !nvme_tcp_admin_queue(queue) &&
   1446		!nvme_tcp_default_queue(queue) &&
   1447		qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
   1448			  ctrl->io_queues[HCTX_TYPE_READ];
   1449}
   1450
   1451static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
   1452{
   1453	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
   1454	int qid = nvme_tcp_queue_id(queue);
   1455
   1456	return !nvme_tcp_admin_queue(queue) &&
   1457		!nvme_tcp_default_queue(queue) &&
   1458		!nvme_tcp_read_queue(queue) &&
   1459		qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
   1460			  ctrl->io_queues[HCTX_TYPE_READ] +
   1461			  ctrl->io_queues[HCTX_TYPE_POLL];
   1462}
   1463
   1464static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
   1465{
   1466	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
   1467	int qid = nvme_tcp_queue_id(queue);
   1468	int n = 0;
   1469
   1470	if (nvme_tcp_default_queue(queue))
   1471		n = qid - 1;
   1472	else if (nvme_tcp_read_queue(queue))
   1473		n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
   1474	else if (nvme_tcp_poll_queue(queue))
   1475		n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
   1476				ctrl->io_queues[HCTX_TYPE_READ] - 1;
   1477	queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
   1478}
   1479
   1480static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
   1481		int qid, size_t queue_size)
   1482{
   1483	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   1484	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
   1485	int ret, rcv_pdu_size;
   1486
   1487	mutex_init(&queue->queue_lock);
   1488	queue->ctrl = ctrl;
   1489	init_llist_head(&queue->req_list);
   1490	INIT_LIST_HEAD(&queue->send_list);
   1491	mutex_init(&queue->send_mutex);
   1492	INIT_WORK(&queue->io_work, nvme_tcp_io_work);
   1493	queue->queue_size = queue_size;
   1494
   1495	if (qid > 0)
   1496		queue->cmnd_capsule_len = nctrl->ioccsz * 16;
   1497	else
   1498		queue->cmnd_capsule_len = sizeof(struct nvme_command) +
   1499						NVME_TCP_ADMIN_CCSZ;
   1500
   1501	ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
   1502			IPPROTO_TCP, &queue->sock);
   1503	if (ret) {
   1504		dev_err(nctrl->device,
   1505			"failed to create socket: %d\n", ret);
   1506		goto err_destroy_mutex;
   1507	}
   1508
   1509	nvme_tcp_reclassify_socket(queue->sock);
   1510
   1511	/* Single syn retry */
   1512	tcp_sock_set_syncnt(queue->sock->sk, 1);
   1513
   1514	/* Set TCP no delay */
   1515	tcp_sock_set_nodelay(queue->sock->sk);
   1516
   1517	/*
   1518	 * Cleanup whatever is sitting in the TCP transmit queue on socket
   1519	 * close. This is done to prevent stale data from being sent should
   1520	 * the network connection be restored before TCP times out.
   1521	 */
   1522	sock_no_linger(queue->sock->sk);
   1523
   1524	if (so_priority > 0)
   1525		sock_set_priority(queue->sock->sk, so_priority);
   1526
   1527	/* Set socket type of service */
   1528	if (nctrl->opts->tos >= 0)
   1529		ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
   1530
   1531	/* Set 10 seconds timeout for icresp recvmsg */
   1532	queue->sock->sk->sk_rcvtimeo = 10 * HZ;
   1533
   1534	queue->sock->sk->sk_allocation = GFP_ATOMIC;
   1535	nvme_tcp_set_queue_io_cpu(queue);
   1536	queue->request = NULL;
   1537	queue->data_remaining = 0;
   1538	queue->ddgst_remaining = 0;
   1539	queue->pdu_remaining = 0;
   1540	queue->pdu_offset = 0;
   1541	sk_set_memalloc(queue->sock->sk);
   1542
   1543	if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
   1544		ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
   1545			sizeof(ctrl->src_addr));
   1546		if (ret) {
   1547			dev_err(nctrl->device,
   1548				"failed to bind queue %d socket %d\n",
   1549				qid, ret);
   1550			goto err_sock;
   1551		}
   1552	}
   1553
   1554	if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
   1555		char *iface = nctrl->opts->host_iface;
   1556		sockptr_t optval = KERNEL_SOCKPTR(iface);
   1557
   1558		ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
   1559				      optval, strlen(iface));
   1560		if (ret) {
   1561			dev_err(nctrl->device,
   1562			  "failed to bind to interface %s queue %d err %d\n",
   1563			  iface, qid, ret);
   1564			goto err_sock;
   1565		}
   1566	}
   1567
   1568	queue->hdr_digest = nctrl->opts->hdr_digest;
   1569	queue->data_digest = nctrl->opts->data_digest;
   1570	if (queue->hdr_digest || queue->data_digest) {
   1571		ret = nvme_tcp_alloc_crypto(queue);
   1572		if (ret) {
   1573			dev_err(nctrl->device,
   1574				"failed to allocate queue %d crypto\n", qid);
   1575			goto err_sock;
   1576		}
   1577	}
   1578
   1579	rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
   1580			nvme_tcp_hdgst_len(queue);
   1581	queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
   1582	if (!queue->pdu) {
   1583		ret = -ENOMEM;
   1584		goto err_crypto;
   1585	}
   1586
   1587	dev_dbg(nctrl->device, "connecting queue %d\n",
   1588			nvme_tcp_queue_id(queue));
   1589
   1590	ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
   1591		sizeof(ctrl->addr), 0);
   1592	if (ret) {
   1593		dev_err(nctrl->device,
   1594			"failed to connect socket: %d\n", ret);
   1595		goto err_rcv_pdu;
   1596	}
   1597
   1598	ret = nvme_tcp_init_connection(queue);
   1599	if (ret)
   1600		goto err_init_connect;
   1601
   1602	queue->rd_enabled = true;
   1603	set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
   1604	nvme_tcp_init_recv_ctx(queue);
   1605
   1606	write_lock_bh(&queue->sock->sk->sk_callback_lock);
   1607	queue->sock->sk->sk_user_data = queue;
   1608	queue->state_change = queue->sock->sk->sk_state_change;
   1609	queue->data_ready = queue->sock->sk->sk_data_ready;
   1610	queue->write_space = queue->sock->sk->sk_write_space;
   1611	queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
   1612	queue->sock->sk->sk_state_change = nvme_tcp_state_change;
   1613	queue->sock->sk->sk_write_space = nvme_tcp_write_space;
   1614#ifdef CONFIG_NET_RX_BUSY_POLL
   1615	queue->sock->sk->sk_ll_usec = 1;
   1616#endif
   1617	write_unlock_bh(&queue->sock->sk->sk_callback_lock);
   1618
   1619	return 0;
   1620
   1621err_init_connect:
   1622	kernel_sock_shutdown(queue->sock, SHUT_RDWR);
   1623err_rcv_pdu:
   1624	kfree(queue->pdu);
   1625err_crypto:
   1626	if (queue->hdr_digest || queue->data_digest)
   1627		nvme_tcp_free_crypto(queue);
   1628err_sock:
   1629	sock_release(queue->sock);
   1630	queue->sock = NULL;
   1631err_destroy_mutex:
   1632	mutex_destroy(&queue->send_mutex);
   1633	mutex_destroy(&queue->queue_lock);
   1634	return ret;
   1635}
   1636
   1637static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
   1638{
   1639	struct socket *sock = queue->sock;
   1640
   1641	write_lock_bh(&sock->sk->sk_callback_lock);
   1642	sock->sk->sk_user_data  = NULL;
   1643	sock->sk->sk_data_ready = queue->data_ready;
   1644	sock->sk->sk_state_change = queue->state_change;
   1645	sock->sk->sk_write_space  = queue->write_space;
   1646	write_unlock_bh(&sock->sk->sk_callback_lock);
   1647}
   1648
   1649static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
   1650{
   1651	kernel_sock_shutdown(queue->sock, SHUT_RDWR);
   1652	nvme_tcp_restore_sock_calls(queue);
   1653	cancel_work_sync(&queue->io_work);
   1654}
   1655
   1656static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
   1657{
   1658	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   1659	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
   1660
   1661	mutex_lock(&queue->queue_lock);
   1662	if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
   1663		__nvme_tcp_stop_queue(queue);
   1664	mutex_unlock(&queue->queue_lock);
   1665}
   1666
   1667static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
   1668{
   1669	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   1670	int ret;
   1671
   1672	if (idx)
   1673		ret = nvmf_connect_io_queue(nctrl, idx);
   1674	else
   1675		ret = nvmf_connect_admin_queue(nctrl);
   1676
   1677	if (!ret) {
   1678		set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
   1679	} else {
   1680		if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
   1681			__nvme_tcp_stop_queue(&ctrl->queues[idx]);
   1682		dev_err(nctrl->device,
   1683			"failed to connect queue: %d ret=%d\n", idx, ret);
   1684	}
   1685	return ret;
   1686}
   1687
   1688static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
   1689		bool admin)
   1690{
   1691	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   1692	struct blk_mq_tag_set *set;
   1693	int ret;
   1694
   1695	if (admin) {
   1696		set = &ctrl->admin_tag_set;
   1697		memset(set, 0, sizeof(*set));
   1698		set->ops = &nvme_tcp_admin_mq_ops;
   1699		set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
   1700		set->reserved_tags = NVMF_RESERVED_TAGS;
   1701		set->numa_node = nctrl->numa_node;
   1702		set->flags = BLK_MQ_F_BLOCKING;
   1703		set->cmd_size = sizeof(struct nvme_tcp_request);
   1704		set->driver_data = ctrl;
   1705		set->nr_hw_queues = 1;
   1706		set->timeout = NVME_ADMIN_TIMEOUT;
   1707	} else {
   1708		set = &ctrl->tag_set;
   1709		memset(set, 0, sizeof(*set));
   1710		set->ops = &nvme_tcp_mq_ops;
   1711		set->queue_depth = nctrl->sqsize + 1;
   1712		set->reserved_tags = NVMF_RESERVED_TAGS;
   1713		set->numa_node = nctrl->numa_node;
   1714		set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
   1715		set->cmd_size = sizeof(struct nvme_tcp_request);
   1716		set->driver_data = ctrl;
   1717		set->nr_hw_queues = nctrl->queue_count - 1;
   1718		set->timeout = NVME_IO_TIMEOUT;
   1719		set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
   1720	}
   1721
   1722	ret = blk_mq_alloc_tag_set(set);
   1723	if (ret)
   1724		return ERR_PTR(ret);
   1725
   1726	return set;
   1727}
   1728
   1729static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
   1730{
   1731	if (to_tcp_ctrl(ctrl)->async_req.pdu) {
   1732		cancel_work_sync(&ctrl->async_event_work);
   1733		nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
   1734		to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
   1735	}
   1736
   1737	nvme_tcp_free_queue(ctrl, 0);
   1738}
   1739
   1740static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
   1741{
   1742	int i;
   1743
   1744	for (i = 1; i < ctrl->queue_count; i++)
   1745		nvme_tcp_free_queue(ctrl, i);
   1746}
   1747
   1748static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
   1749{
   1750	int i;
   1751
   1752	for (i = 1; i < ctrl->queue_count; i++)
   1753		nvme_tcp_stop_queue(ctrl, i);
   1754}
   1755
   1756static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
   1757{
   1758	int i, ret;
   1759
   1760	for (i = 1; i < ctrl->queue_count; i++) {
   1761		ret = nvme_tcp_start_queue(ctrl, i);
   1762		if (ret)
   1763			goto out_stop_queues;
   1764	}
   1765
   1766	return 0;
   1767
   1768out_stop_queues:
   1769	for (i--; i >= 1; i--)
   1770		nvme_tcp_stop_queue(ctrl, i);
   1771	return ret;
   1772}
   1773
   1774static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
   1775{
   1776	int ret;
   1777
   1778	ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
   1779	if (ret)
   1780		return ret;
   1781
   1782	ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
   1783	if (ret)
   1784		goto out_free_queue;
   1785
   1786	return 0;
   1787
   1788out_free_queue:
   1789	nvme_tcp_free_queue(ctrl, 0);
   1790	return ret;
   1791}
   1792
   1793static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
   1794{
   1795	int i, ret;
   1796
   1797	for (i = 1; i < ctrl->queue_count; i++) {
   1798		ret = nvme_tcp_alloc_queue(ctrl, i, ctrl->sqsize + 1);
   1799		if (ret)
   1800			goto out_free_queues;
   1801	}
   1802
   1803	return 0;
   1804
   1805out_free_queues:
   1806	for (i--; i >= 1; i--)
   1807		nvme_tcp_free_queue(ctrl, i);
   1808
   1809	return ret;
   1810}
   1811
   1812static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
   1813{
   1814	unsigned int nr_io_queues;
   1815
   1816	nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
   1817	nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
   1818	nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
   1819
   1820	return nr_io_queues;
   1821}
   1822
   1823static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
   1824		unsigned int nr_io_queues)
   1825{
   1826	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   1827	struct nvmf_ctrl_options *opts = nctrl->opts;
   1828
   1829	if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
   1830		/*
   1831		 * separate read/write queues
   1832		 * hand out dedicated default queues only after we have
   1833		 * sufficient read queues.
   1834		 */
   1835		ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
   1836		nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
   1837		ctrl->io_queues[HCTX_TYPE_DEFAULT] =
   1838			min(opts->nr_write_queues, nr_io_queues);
   1839		nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
   1840	} else {
   1841		/*
   1842		 * shared read/write queues
   1843		 * either no write queues were requested, or we don't have
   1844		 * sufficient queue count to have dedicated default queues.
   1845		 */
   1846		ctrl->io_queues[HCTX_TYPE_DEFAULT] =
   1847			min(opts->nr_io_queues, nr_io_queues);
   1848		nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
   1849	}
   1850
   1851	if (opts->nr_poll_queues && nr_io_queues) {
   1852		/* map dedicated poll queues only if we have queues left */
   1853		ctrl->io_queues[HCTX_TYPE_POLL] =
   1854			min(opts->nr_poll_queues, nr_io_queues);
   1855	}
   1856}
   1857
   1858static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
   1859{
   1860	unsigned int nr_io_queues;
   1861	int ret;
   1862
   1863	nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
   1864	ret = nvme_set_queue_count(ctrl, &nr_io_queues);
   1865	if (ret)
   1866		return ret;
   1867
   1868	if (nr_io_queues == 0) {
   1869		dev_err(ctrl->device,
   1870			"unable to set any I/O queues\n");
   1871		return -ENOMEM;
   1872	}
   1873
   1874	ctrl->queue_count = nr_io_queues + 1;
   1875	dev_info(ctrl->device,
   1876		"creating %d I/O queues.\n", nr_io_queues);
   1877
   1878	nvme_tcp_set_io_queues(ctrl, nr_io_queues);
   1879
   1880	return __nvme_tcp_alloc_io_queues(ctrl);
   1881}
   1882
   1883static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
   1884{
   1885	nvme_tcp_stop_io_queues(ctrl);
   1886	if (remove) {
   1887		blk_cleanup_queue(ctrl->connect_q);
   1888		blk_mq_free_tag_set(ctrl->tagset);
   1889	}
   1890	nvme_tcp_free_io_queues(ctrl);
   1891}
   1892
   1893static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
   1894{
   1895	int ret;
   1896
   1897	ret = nvme_tcp_alloc_io_queues(ctrl);
   1898	if (ret)
   1899		return ret;
   1900
   1901	if (new) {
   1902		ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
   1903		if (IS_ERR(ctrl->tagset)) {
   1904			ret = PTR_ERR(ctrl->tagset);
   1905			goto out_free_io_queues;
   1906		}
   1907
   1908		ret = nvme_ctrl_init_connect_q(ctrl);
   1909		if (ret)
   1910			goto out_free_tag_set;
   1911	}
   1912
   1913	ret = nvme_tcp_start_io_queues(ctrl);
   1914	if (ret)
   1915		goto out_cleanup_connect_q;
   1916
   1917	if (!new) {
   1918		nvme_start_queues(ctrl);
   1919		if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
   1920			/*
   1921			 * If we timed out waiting for freeze we are likely to
   1922			 * be stuck.  Fail the controller initialization just
   1923			 * to be safe.
   1924			 */
   1925			ret = -ENODEV;
   1926			goto out_wait_freeze_timed_out;
   1927		}
   1928		blk_mq_update_nr_hw_queues(ctrl->tagset,
   1929			ctrl->queue_count - 1);
   1930		nvme_unfreeze(ctrl);
   1931	}
   1932
   1933	return 0;
   1934
   1935out_wait_freeze_timed_out:
   1936	nvme_stop_queues(ctrl);
   1937	nvme_sync_io_queues(ctrl);
   1938	nvme_tcp_stop_io_queues(ctrl);
   1939out_cleanup_connect_q:
   1940	nvme_cancel_tagset(ctrl);
   1941	if (new)
   1942		blk_cleanup_queue(ctrl->connect_q);
   1943out_free_tag_set:
   1944	if (new)
   1945		blk_mq_free_tag_set(ctrl->tagset);
   1946out_free_io_queues:
   1947	nvme_tcp_free_io_queues(ctrl);
   1948	return ret;
   1949}
   1950
   1951static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
   1952{
   1953	nvme_tcp_stop_queue(ctrl, 0);
   1954	if (remove) {
   1955		blk_cleanup_queue(ctrl->admin_q);
   1956		blk_cleanup_queue(ctrl->fabrics_q);
   1957		blk_mq_free_tag_set(ctrl->admin_tagset);
   1958	}
   1959	nvme_tcp_free_admin_queue(ctrl);
   1960}
   1961
   1962static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
   1963{
   1964	int error;
   1965
   1966	error = nvme_tcp_alloc_admin_queue(ctrl);
   1967	if (error)
   1968		return error;
   1969
   1970	if (new) {
   1971		ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
   1972		if (IS_ERR(ctrl->admin_tagset)) {
   1973			error = PTR_ERR(ctrl->admin_tagset);
   1974			goto out_free_queue;
   1975		}
   1976
   1977		ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
   1978		if (IS_ERR(ctrl->fabrics_q)) {
   1979			error = PTR_ERR(ctrl->fabrics_q);
   1980			goto out_free_tagset;
   1981		}
   1982
   1983		ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
   1984		if (IS_ERR(ctrl->admin_q)) {
   1985			error = PTR_ERR(ctrl->admin_q);
   1986			goto out_cleanup_fabrics_q;
   1987		}
   1988	}
   1989
   1990	error = nvme_tcp_start_queue(ctrl, 0);
   1991	if (error)
   1992		goto out_cleanup_queue;
   1993
   1994	error = nvme_enable_ctrl(ctrl);
   1995	if (error)
   1996		goto out_stop_queue;
   1997
   1998	nvme_start_admin_queue(ctrl);
   1999
   2000	error = nvme_init_ctrl_finish(ctrl);
   2001	if (error)
   2002		goto out_quiesce_queue;
   2003
   2004	return 0;
   2005
   2006out_quiesce_queue:
   2007	nvme_stop_admin_queue(ctrl);
   2008	blk_sync_queue(ctrl->admin_q);
   2009out_stop_queue:
   2010	nvme_tcp_stop_queue(ctrl, 0);
   2011	nvme_cancel_admin_tagset(ctrl);
   2012out_cleanup_queue:
   2013	if (new)
   2014		blk_cleanup_queue(ctrl->admin_q);
   2015out_cleanup_fabrics_q:
   2016	if (new)
   2017		blk_cleanup_queue(ctrl->fabrics_q);
   2018out_free_tagset:
   2019	if (new)
   2020		blk_mq_free_tag_set(ctrl->admin_tagset);
   2021out_free_queue:
   2022	nvme_tcp_free_admin_queue(ctrl);
   2023	return error;
   2024}
   2025
   2026static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
   2027		bool remove)
   2028{
   2029	nvme_stop_admin_queue(ctrl);
   2030	blk_sync_queue(ctrl->admin_q);
   2031	nvme_tcp_stop_queue(ctrl, 0);
   2032	nvme_cancel_admin_tagset(ctrl);
   2033	if (remove)
   2034		nvme_start_admin_queue(ctrl);
   2035	nvme_tcp_destroy_admin_queue(ctrl, remove);
   2036}
   2037
   2038static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
   2039		bool remove)
   2040{
   2041	if (ctrl->queue_count <= 1)
   2042		return;
   2043	nvme_stop_admin_queue(ctrl);
   2044	nvme_start_freeze(ctrl);
   2045	nvme_stop_queues(ctrl);
   2046	nvme_sync_io_queues(ctrl);
   2047	nvme_tcp_stop_io_queues(ctrl);
   2048	nvme_cancel_tagset(ctrl);
   2049	if (remove)
   2050		nvme_start_queues(ctrl);
   2051	nvme_tcp_destroy_io_queues(ctrl, remove);
   2052}
   2053
   2054static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
   2055{
   2056	/* If we are resetting/deleting then do nothing */
   2057	if (ctrl->state != NVME_CTRL_CONNECTING) {
   2058		WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
   2059			ctrl->state == NVME_CTRL_LIVE);
   2060		return;
   2061	}
   2062
   2063	if (nvmf_should_reconnect(ctrl)) {
   2064		dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
   2065			ctrl->opts->reconnect_delay);
   2066		queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
   2067				ctrl->opts->reconnect_delay * HZ);
   2068	} else {
   2069		dev_info(ctrl->device, "Removing controller...\n");
   2070		nvme_delete_ctrl(ctrl);
   2071	}
   2072}
   2073
   2074static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
   2075{
   2076	struct nvmf_ctrl_options *opts = ctrl->opts;
   2077	int ret;
   2078
   2079	ret = nvme_tcp_configure_admin_queue(ctrl, new);
   2080	if (ret)
   2081		return ret;
   2082
   2083	if (ctrl->icdoff) {
   2084		ret = -EOPNOTSUPP;
   2085		dev_err(ctrl->device, "icdoff is not supported!\n");
   2086		goto destroy_admin;
   2087	}
   2088
   2089	if (!nvme_ctrl_sgl_supported(ctrl)) {
   2090		ret = -EOPNOTSUPP;
   2091		dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
   2092		goto destroy_admin;
   2093	}
   2094
   2095	if (opts->queue_size > ctrl->sqsize + 1)
   2096		dev_warn(ctrl->device,
   2097			"queue_size %zu > ctrl sqsize %u, clamping down\n",
   2098			opts->queue_size, ctrl->sqsize + 1);
   2099
   2100	if (ctrl->sqsize + 1 > ctrl->maxcmd) {
   2101		dev_warn(ctrl->device,
   2102			"sqsize %u > ctrl maxcmd %u, clamping down\n",
   2103			ctrl->sqsize + 1, ctrl->maxcmd);
   2104		ctrl->sqsize = ctrl->maxcmd - 1;
   2105	}
   2106
   2107	if (ctrl->queue_count > 1) {
   2108		ret = nvme_tcp_configure_io_queues(ctrl, new);
   2109		if (ret)
   2110			goto destroy_admin;
   2111	}
   2112
   2113	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
   2114		/*
   2115		 * state change failure is ok if we started ctrl delete,
   2116		 * unless we're during creation of a new controller to
   2117		 * avoid races with teardown flow.
   2118		 */
   2119		WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
   2120			     ctrl->state != NVME_CTRL_DELETING_NOIO);
   2121		WARN_ON_ONCE(new);
   2122		ret = -EINVAL;
   2123		goto destroy_io;
   2124	}
   2125
   2126	nvme_start_ctrl(ctrl);
   2127	return 0;
   2128
   2129destroy_io:
   2130	if (ctrl->queue_count > 1) {
   2131		nvme_stop_queues(ctrl);
   2132		nvme_sync_io_queues(ctrl);
   2133		nvme_tcp_stop_io_queues(ctrl);
   2134		nvme_cancel_tagset(ctrl);
   2135		nvme_tcp_destroy_io_queues(ctrl, new);
   2136	}
   2137destroy_admin:
   2138	nvme_stop_admin_queue(ctrl);
   2139	blk_sync_queue(ctrl->admin_q);
   2140	nvme_tcp_stop_queue(ctrl, 0);
   2141	nvme_cancel_admin_tagset(ctrl);
   2142	nvme_tcp_destroy_admin_queue(ctrl, new);
   2143	return ret;
   2144}
   2145
   2146static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
   2147{
   2148	struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
   2149			struct nvme_tcp_ctrl, connect_work);
   2150	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
   2151
   2152	++ctrl->nr_reconnects;
   2153
   2154	if (nvme_tcp_setup_ctrl(ctrl, false))
   2155		goto requeue;
   2156
   2157	dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
   2158			ctrl->nr_reconnects);
   2159
   2160	ctrl->nr_reconnects = 0;
   2161
   2162	return;
   2163
   2164requeue:
   2165	dev_info(ctrl->device, "Failed reconnect attempt %d\n",
   2166			ctrl->nr_reconnects);
   2167	nvme_tcp_reconnect_or_remove(ctrl);
   2168}
   2169
   2170static void nvme_tcp_error_recovery_work(struct work_struct *work)
   2171{
   2172	struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
   2173				struct nvme_tcp_ctrl, err_work);
   2174	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
   2175
   2176	nvme_stop_keep_alive(ctrl);
   2177	flush_work(&ctrl->async_event_work);
   2178	nvme_tcp_teardown_io_queues(ctrl, false);
   2179	/* unquiesce to fail fast pending requests */
   2180	nvme_start_queues(ctrl);
   2181	nvme_tcp_teardown_admin_queue(ctrl, false);
   2182	nvme_start_admin_queue(ctrl);
   2183
   2184	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
   2185		/* state change failure is ok if we started ctrl delete */
   2186		WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
   2187			     ctrl->state != NVME_CTRL_DELETING_NOIO);
   2188		return;
   2189	}
   2190
   2191	nvme_tcp_reconnect_or_remove(ctrl);
   2192}
   2193
   2194static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
   2195{
   2196	nvme_tcp_teardown_io_queues(ctrl, shutdown);
   2197	nvme_stop_admin_queue(ctrl);
   2198	if (shutdown)
   2199		nvme_shutdown_ctrl(ctrl);
   2200	else
   2201		nvme_disable_ctrl(ctrl);
   2202	nvme_tcp_teardown_admin_queue(ctrl, shutdown);
   2203}
   2204
   2205static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
   2206{
   2207	nvme_tcp_teardown_ctrl(ctrl, true);
   2208}
   2209
   2210static void nvme_reset_ctrl_work(struct work_struct *work)
   2211{
   2212	struct nvme_ctrl *ctrl =
   2213		container_of(work, struct nvme_ctrl, reset_work);
   2214
   2215	nvme_stop_ctrl(ctrl);
   2216	nvme_tcp_teardown_ctrl(ctrl, false);
   2217
   2218	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
   2219		/* state change failure is ok if we started ctrl delete */
   2220		WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
   2221			     ctrl->state != NVME_CTRL_DELETING_NOIO);
   2222		return;
   2223	}
   2224
   2225	if (nvme_tcp_setup_ctrl(ctrl, false))
   2226		goto out_fail;
   2227
   2228	return;
   2229
   2230out_fail:
   2231	++ctrl->nr_reconnects;
   2232	nvme_tcp_reconnect_or_remove(ctrl);
   2233}
   2234
   2235static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
   2236{
   2237	cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
   2238	cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
   2239}
   2240
   2241static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
   2242{
   2243	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
   2244
   2245	if (list_empty(&ctrl->list))
   2246		goto free_ctrl;
   2247
   2248	mutex_lock(&nvme_tcp_ctrl_mutex);
   2249	list_del(&ctrl->list);
   2250	mutex_unlock(&nvme_tcp_ctrl_mutex);
   2251
   2252	nvmf_free_options(nctrl->opts);
   2253free_ctrl:
   2254	kfree(ctrl->queues);
   2255	kfree(ctrl);
   2256}
   2257
   2258static void nvme_tcp_set_sg_null(struct nvme_command *c)
   2259{
   2260	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
   2261
   2262	sg->addr = 0;
   2263	sg->length = 0;
   2264	sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
   2265			NVME_SGL_FMT_TRANSPORT_A;
   2266}
   2267
   2268static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
   2269		struct nvme_command *c, u32 data_len)
   2270{
   2271	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
   2272
   2273	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
   2274	sg->length = cpu_to_le32(data_len);
   2275	sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
   2276}
   2277
   2278static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
   2279		u32 data_len)
   2280{
   2281	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
   2282
   2283	sg->addr = 0;
   2284	sg->length = cpu_to_le32(data_len);
   2285	sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
   2286			NVME_SGL_FMT_TRANSPORT_A;
   2287}
   2288
   2289static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
   2290{
   2291	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
   2292	struct nvme_tcp_queue *queue = &ctrl->queues[0];
   2293	struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
   2294	struct nvme_command *cmd = &pdu->cmd;
   2295	u8 hdgst = nvme_tcp_hdgst_len(queue);
   2296
   2297	memset(pdu, 0, sizeof(*pdu));
   2298	pdu->hdr.type = nvme_tcp_cmd;
   2299	if (queue->hdr_digest)
   2300		pdu->hdr.flags |= NVME_TCP_F_HDGST;
   2301	pdu->hdr.hlen = sizeof(*pdu);
   2302	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
   2303
   2304	cmd->common.opcode = nvme_admin_async_event;
   2305	cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
   2306	cmd->common.flags |= NVME_CMD_SGL_METABUF;
   2307	nvme_tcp_set_sg_null(cmd);
   2308
   2309	ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
   2310	ctrl->async_req.offset = 0;
   2311	ctrl->async_req.curr_bio = NULL;
   2312	ctrl->async_req.data_len = 0;
   2313
   2314	nvme_tcp_queue_request(&ctrl->async_req, true, true);
   2315}
   2316
   2317static void nvme_tcp_complete_timed_out(struct request *rq)
   2318{
   2319	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
   2320	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
   2321
   2322	nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
   2323	nvmf_complete_timed_out_request(rq);
   2324}
   2325
   2326static enum blk_eh_timer_return
   2327nvme_tcp_timeout(struct request *rq, bool reserved)
   2328{
   2329	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
   2330	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
   2331	struct nvme_tcp_cmd_pdu *pdu = req->pdu;
   2332
   2333	dev_warn(ctrl->device,
   2334		"queue %d: timeout request %#x type %d\n",
   2335		nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
   2336
   2337	if (ctrl->state != NVME_CTRL_LIVE) {
   2338		/*
   2339		 * If we are resetting, connecting or deleting we should
   2340		 * complete immediately because we may block controller
   2341		 * teardown or setup sequence
   2342		 * - ctrl disable/shutdown fabrics requests
   2343		 * - connect requests
   2344		 * - initialization admin requests
   2345		 * - I/O requests that entered after unquiescing and
   2346		 *   the controller stopped responding
   2347		 *
   2348		 * All other requests should be cancelled by the error
   2349		 * recovery work, so it's fine that we fail it here.
   2350		 */
   2351		nvme_tcp_complete_timed_out(rq);
   2352		return BLK_EH_DONE;
   2353	}
   2354
   2355	/*
   2356	 * LIVE state should trigger the normal error recovery which will
   2357	 * handle completing this request.
   2358	 */
   2359	nvme_tcp_error_recovery(ctrl);
   2360	return BLK_EH_RESET_TIMER;
   2361}
   2362
   2363static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
   2364			struct request *rq)
   2365{
   2366	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
   2367	struct nvme_tcp_cmd_pdu *pdu = req->pdu;
   2368	struct nvme_command *c = &pdu->cmd;
   2369
   2370	c->common.flags |= NVME_CMD_SGL_METABUF;
   2371
   2372	if (!blk_rq_nr_phys_segments(rq))
   2373		nvme_tcp_set_sg_null(c);
   2374	else if (rq_data_dir(rq) == WRITE &&
   2375	    req->data_len <= nvme_tcp_inline_data_size(queue))
   2376		nvme_tcp_set_sg_inline(queue, c, req->data_len);
   2377	else
   2378		nvme_tcp_set_sg_host_data(c, req->data_len);
   2379
   2380	return 0;
   2381}
   2382
   2383static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
   2384		struct request *rq)
   2385{
   2386	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
   2387	struct nvme_tcp_cmd_pdu *pdu = req->pdu;
   2388	struct nvme_tcp_queue *queue = req->queue;
   2389	u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
   2390	blk_status_t ret;
   2391
   2392	ret = nvme_setup_cmd(ns, rq);
   2393	if (ret)
   2394		return ret;
   2395
   2396	req->state = NVME_TCP_SEND_CMD_PDU;
   2397	req->status = cpu_to_le16(NVME_SC_SUCCESS);
   2398	req->offset = 0;
   2399	req->data_sent = 0;
   2400	req->pdu_len = 0;
   2401	req->pdu_sent = 0;
   2402	req->h2cdata_left = 0;
   2403	req->data_len = blk_rq_nr_phys_segments(rq) ?
   2404				blk_rq_payload_bytes(rq) : 0;
   2405	req->curr_bio = rq->bio;
   2406	if (req->curr_bio && req->data_len)
   2407		nvme_tcp_init_iter(req, rq_data_dir(rq));
   2408
   2409	if (rq_data_dir(rq) == WRITE &&
   2410	    req->data_len <= nvme_tcp_inline_data_size(queue))
   2411		req->pdu_len = req->data_len;
   2412
   2413	pdu->hdr.type = nvme_tcp_cmd;
   2414	pdu->hdr.flags = 0;
   2415	if (queue->hdr_digest)
   2416		pdu->hdr.flags |= NVME_TCP_F_HDGST;
   2417	if (queue->data_digest && req->pdu_len) {
   2418		pdu->hdr.flags |= NVME_TCP_F_DDGST;
   2419		ddgst = nvme_tcp_ddgst_len(queue);
   2420	}
   2421	pdu->hdr.hlen = sizeof(*pdu);
   2422	pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
   2423	pdu->hdr.plen =
   2424		cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
   2425
   2426	ret = nvme_tcp_map_data(queue, rq);
   2427	if (unlikely(ret)) {
   2428		nvme_cleanup_cmd(rq);
   2429		dev_err(queue->ctrl->ctrl.device,
   2430			"Failed to map data (%d)\n", ret);
   2431		return ret;
   2432	}
   2433
   2434	return 0;
   2435}
   2436
   2437static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
   2438{
   2439	struct nvme_tcp_queue *queue = hctx->driver_data;
   2440
   2441	if (!llist_empty(&queue->req_list))
   2442		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
   2443}
   2444
   2445static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
   2446		const struct blk_mq_queue_data *bd)
   2447{
   2448	struct nvme_ns *ns = hctx->queue->queuedata;
   2449	struct nvme_tcp_queue *queue = hctx->driver_data;
   2450	struct request *rq = bd->rq;
   2451	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
   2452	bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
   2453	blk_status_t ret;
   2454
   2455	if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
   2456		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
   2457
   2458	ret = nvme_tcp_setup_cmd_pdu(ns, rq);
   2459	if (unlikely(ret))
   2460		return ret;
   2461
   2462	blk_mq_start_request(rq);
   2463
   2464	nvme_tcp_queue_request(req, true, bd->last);
   2465
   2466	return BLK_STS_OK;
   2467}
   2468
   2469static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
   2470{
   2471	struct nvme_tcp_ctrl *ctrl = set->driver_data;
   2472	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
   2473
   2474	if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
   2475		/* separate read/write queues */
   2476		set->map[HCTX_TYPE_DEFAULT].nr_queues =
   2477			ctrl->io_queues[HCTX_TYPE_DEFAULT];
   2478		set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
   2479		set->map[HCTX_TYPE_READ].nr_queues =
   2480			ctrl->io_queues[HCTX_TYPE_READ];
   2481		set->map[HCTX_TYPE_READ].queue_offset =
   2482			ctrl->io_queues[HCTX_TYPE_DEFAULT];
   2483	} else {
   2484		/* shared read/write queues */
   2485		set->map[HCTX_TYPE_DEFAULT].nr_queues =
   2486			ctrl->io_queues[HCTX_TYPE_DEFAULT];
   2487		set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
   2488		set->map[HCTX_TYPE_READ].nr_queues =
   2489			ctrl->io_queues[HCTX_TYPE_DEFAULT];
   2490		set->map[HCTX_TYPE_READ].queue_offset = 0;
   2491	}
   2492	blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
   2493	blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
   2494
   2495	if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
   2496		/* map dedicated poll queues only if we have queues left */
   2497		set->map[HCTX_TYPE_POLL].nr_queues =
   2498				ctrl->io_queues[HCTX_TYPE_POLL];
   2499		set->map[HCTX_TYPE_POLL].queue_offset =
   2500			ctrl->io_queues[HCTX_TYPE_DEFAULT] +
   2501			ctrl->io_queues[HCTX_TYPE_READ];
   2502		blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
   2503	}
   2504
   2505	dev_info(ctrl->ctrl.device,
   2506		"mapped %d/%d/%d default/read/poll queues.\n",
   2507		ctrl->io_queues[HCTX_TYPE_DEFAULT],
   2508		ctrl->io_queues[HCTX_TYPE_READ],
   2509		ctrl->io_queues[HCTX_TYPE_POLL]);
   2510
   2511	return 0;
   2512}
   2513
   2514static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
   2515{
   2516	struct nvme_tcp_queue *queue = hctx->driver_data;
   2517	struct sock *sk = queue->sock->sk;
   2518
   2519	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
   2520		return 0;
   2521
   2522	set_bit(NVME_TCP_Q_POLLING, &queue->flags);
   2523	if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
   2524		sk_busy_loop(sk, true);
   2525	nvme_tcp_try_recv(queue);
   2526	clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
   2527	return queue->nr_cqe;
   2528}
   2529
   2530static const struct blk_mq_ops nvme_tcp_mq_ops = {
   2531	.queue_rq	= nvme_tcp_queue_rq,
   2532	.commit_rqs	= nvme_tcp_commit_rqs,
   2533	.complete	= nvme_complete_rq,
   2534	.init_request	= nvme_tcp_init_request,
   2535	.exit_request	= nvme_tcp_exit_request,
   2536	.init_hctx	= nvme_tcp_init_hctx,
   2537	.timeout	= nvme_tcp_timeout,
   2538	.map_queues	= nvme_tcp_map_queues,
   2539	.poll		= nvme_tcp_poll,
   2540};
   2541
   2542static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
   2543	.queue_rq	= nvme_tcp_queue_rq,
   2544	.complete	= nvme_complete_rq,
   2545	.init_request	= nvme_tcp_init_request,
   2546	.exit_request	= nvme_tcp_exit_request,
   2547	.init_hctx	= nvme_tcp_init_admin_hctx,
   2548	.timeout	= nvme_tcp_timeout,
   2549};
   2550
   2551static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
   2552	.name			= "tcp",
   2553	.module			= THIS_MODULE,
   2554	.flags			= NVME_F_FABRICS,
   2555	.reg_read32		= nvmf_reg_read32,
   2556	.reg_read64		= nvmf_reg_read64,
   2557	.reg_write32		= nvmf_reg_write32,
   2558	.free_ctrl		= nvme_tcp_free_ctrl,
   2559	.submit_async_event	= nvme_tcp_submit_async_event,
   2560	.delete_ctrl		= nvme_tcp_delete_ctrl,
   2561	.get_address		= nvmf_get_address,
   2562	.stop_ctrl		= nvme_tcp_stop_ctrl,
   2563};
   2564
   2565static bool
   2566nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
   2567{
   2568	struct nvme_tcp_ctrl *ctrl;
   2569	bool found = false;
   2570
   2571	mutex_lock(&nvme_tcp_ctrl_mutex);
   2572	list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
   2573		found = nvmf_ip_options_match(&ctrl->ctrl, opts);
   2574		if (found)
   2575			break;
   2576	}
   2577	mutex_unlock(&nvme_tcp_ctrl_mutex);
   2578
   2579	return found;
   2580}
   2581
   2582static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
   2583		struct nvmf_ctrl_options *opts)
   2584{
   2585	struct nvme_tcp_ctrl *ctrl;
   2586	int ret;
   2587
   2588	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
   2589	if (!ctrl)
   2590		return ERR_PTR(-ENOMEM);
   2591
   2592	INIT_LIST_HEAD(&ctrl->list);
   2593	ctrl->ctrl.opts = opts;
   2594	ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
   2595				opts->nr_poll_queues + 1;
   2596	ctrl->ctrl.sqsize = opts->queue_size - 1;
   2597	ctrl->ctrl.kato = opts->kato;
   2598
   2599	INIT_DELAYED_WORK(&ctrl->connect_work,
   2600			nvme_tcp_reconnect_ctrl_work);
   2601	INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
   2602	INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
   2603
   2604	if (!(opts->mask & NVMF_OPT_TRSVCID)) {
   2605		opts->trsvcid =
   2606			kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
   2607		if (!opts->trsvcid) {
   2608			ret = -ENOMEM;
   2609			goto out_free_ctrl;
   2610		}
   2611		opts->mask |= NVMF_OPT_TRSVCID;
   2612	}
   2613
   2614	ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
   2615			opts->traddr, opts->trsvcid, &ctrl->addr);
   2616	if (ret) {
   2617		pr_err("malformed address passed: %s:%s\n",
   2618			opts->traddr, opts->trsvcid);
   2619		goto out_free_ctrl;
   2620	}
   2621
   2622	if (opts->mask & NVMF_OPT_HOST_TRADDR) {
   2623		ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
   2624			opts->host_traddr, NULL, &ctrl->src_addr);
   2625		if (ret) {
   2626			pr_err("malformed src address passed: %s\n",
   2627			       opts->host_traddr);
   2628			goto out_free_ctrl;
   2629		}
   2630	}
   2631
   2632	if (opts->mask & NVMF_OPT_HOST_IFACE) {
   2633		if (!__dev_get_by_name(&init_net, opts->host_iface)) {
   2634			pr_err("invalid interface passed: %s\n",
   2635			       opts->host_iface);
   2636			ret = -ENODEV;
   2637			goto out_free_ctrl;
   2638		}
   2639	}
   2640
   2641	if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
   2642		ret = -EALREADY;
   2643		goto out_free_ctrl;
   2644	}
   2645
   2646	ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
   2647				GFP_KERNEL);
   2648	if (!ctrl->queues) {
   2649		ret = -ENOMEM;
   2650		goto out_free_ctrl;
   2651	}
   2652
   2653	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
   2654	if (ret)
   2655		goto out_kfree_queues;
   2656
   2657	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
   2658		WARN_ON_ONCE(1);
   2659		ret = -EINTR;
   2660		goto out_uninit_ctrl;
   2661	}
   2662
   2663	ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
   2664	if (ret)
   2665		goto out_uninit_ctrl;
   2666
   2667	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
   2668		nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr);
   2669
   2670	mutex_lock(&nvme_tcp_ctrl_mutex);
   2671	list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
   2672	mutex_unlock(&nvme_tcp_ctrl_mutex);
   2673
   2674	return &ctrl->ctrl;
   2675
   2676out_uninit_ctrl:
   2677	nvme_uninit_ctrl(&ctrl->ctrl);
   2678	nvme_put_ctrl(&ctrl->ctrl);
   2679	if (ret > 0)
   2680		ret = -EIO;
   2681	return ERR_PTR(ret);
   2682out_kfree_queues:
   2683	kfree(ctrl->queues);
   2684out_free_ctrl:
   2685	kfree(ctrl);
   2686	return ERR_PTR(ret);
   2687}
   2688
   2689static struct nvmf_transport_ops nvme_tcp_transport = {
   2690	.name		= "tcp",
   2691	.module		= THIS_MODULE,
   2692	.required_opts	= NVMF_OPT_TRADDR,
   2693	.allowed_opts	= NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
   2694			  NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
   2695			  NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
   2696			  NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
   2697			  NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE,
   2698	.create_ctrl	= nvme_tcp_create_ctrl,
   2699};
   2700
   2701static int __init nvme_tcp_init_module(void)
   2702{
   2703	nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
   2704			WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
   2705	if (!nvme_tcp_wq)
   2706		return -ENOMEM;
   2707
   2708	nvmf_register_transport(&nvme_tcp_transport);
   2709	return 0;
   2710}
   2711
   2712static void __exit nvme_tcp_cleanup_module(void)
   2713{
   2714	struct nvme_tcp_ctrl *ctrl;
   2715
   2716	nvmf_unregister_transport(&nvme_tcp_transport);
   2717
   2718	mutex_lock(&nvme_tcp_ctrl_mutex);
   2719	list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
   2720		nvme_delete_ctrl(&ctrl->ctrl);
   2721	mutex_unlock(&nvme_tcp_ctrl_mutex);
   2722	flush_workqueue(nvme_delete_wq);
   2723
   2724	destroy_workqueue(nvme_tcp_wq);
   2725}
   2726
   2727module_init(nvme_tcp_init_module);
   2728module_exit(nvme_tcp_cleanup_module);
   2729
   2730MODULE_LICENSE("GPL v2");