cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

cxgbit_ddp.c (8335B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (c) 2016 Chelsio Communications, Inc.
      4 */
      5
      6#include "cxgbit.h"
      7
      8static void
      9cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod,
     10		    struct cxgbi_task_tag_info *ttinfo,
     11		    struct scatterlist **sg_pp, unsigned int *sg_off)
     12{
     13	struct scatterlist *sg = sg_pp ? *sg_pp : NULL;
     14	unsigned int offset = sg_off ? *sg_off : 0;
     15	dma_addr_t addr = 0UL;
     16	unsigned int len = 0;
     17	int i;
     18
     19	memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr));
     20
     21	if (sg) {
     22		addr = sg_dma_address(sg);
     23		len = sg_dma_len(sg);
     24	}
     25
     26	for (i = 0; i < PPOD_PAGES_MAX; i++) {
     27		if (sg) {
     28			ppod->addr[i] = cpu_to_be64(addr + offset);
     29			offset += PAGE_SIZE;
     30			if (offset == (len + sg->offset)) {
     31				offset = 0;
     32				sg = sg_next(sg);
     33				if (sg) {
     34					addr = sg_dma_address(sg);
     35					len = sg_dma_len(sg);
     36				}
     37			}
     38		} else {
     39			ppod->addr[i] = 0ULL;
     40		}
     41	}
     42
     43	/*
     44	 * the fifth address needs to be repeated in the next ppod, so do
     45	 * not move sg
     46	 */
     47	if (sg_pp) {
     48		*sg_pp = sg;
     49		*sg_off = offset;
     50	}
     51
     52	if (offset == len) {
     53		offset = 0;
     54		if (sg) {
     55			sg = sg_next(sg);
     56			if (sg)
     57				addr = sg_dma_address(sg);
     58		}
     59	}
     60	ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL;
     61}
     62
     63static struct sk_buff *
     64cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm,
     65		       unsigned int idx, unsigned int npods, unsigned int tid)
     66{
     67	struct ulp_mem_io *req;
     68	struct ulptx_idata *idata;
     69	unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit;
     70	unsigned int dlen = npods << PPOD_SIZE_SHIFT;
     71	unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
     72				sizeof(struct ulptx_idata) + dlen, 16);
     73	struct sk_buff *skb;
     74
     75	skb  = alloc_skb(wr_len, GFP_KERNEL);
     76	if (!skb)
     77		return NULL;
     78
     79	req = __skb_put(skb, wr_len);
     80	INIT_ULPTX_WR(req, wr_len, 0, tid);
     81	req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) |
     82		FW_WR_ATOMIC_V(0));
     83	req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
     84		ULP_MEMIO_ORDER_V(0) |
     85		T5_ULP_MEMIO_IMM_V(1));
     86	req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
     87	req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
     88	req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
     89
     90	idata = (struct ulptx_idata *)(req + 1);
     91	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
     92	idata->len = htonl(dlen);
     93
     94	return skb;
     95}
     96
     97static int
     98cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
     99			struct cxgbi_task_tag_info *ttinfo, unsigned int idx,
    100			unsigned int npods, struct scatterlist **sg_pp,
    101			unsigned int *sg_off)
    102{
    103	struct cxgbit_device *cdev = csk->com.cdev;
    104	struct sk_buff *skb;
    105	struct ulp_mem_io *req;
    106	struct ulptx_idata *idata;
    107	struct cxgbi_pagepod *ppod;
    108	unsigned int i;
    109
    110	skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, csk->tid);
    111	if (!skb)
    112		return -ENOMEM;
    113
    114	req = (struct ulp_mem_io *)skb->data;
    115	idata = (struct ulptx_idata *)(req + 1);
    116	ppod = (struct cxgbi_pagepod *)(idata + 1);
    117
    118	for (i = 0; i < npods; i++, ppod++)
    119		cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off);
    120
    121	__skb_queue_tail(&csk->ppodq, skb);
    122
    123	return 0;
    124}
    125
    126static int
    127cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk,
    128		   struct cxgbi_task_tag_info *ttinfo)
    129{
    130	unsigned int pidx = ttinfo->idx;
    131	unsigned int npods = ttinfo->npods;
    132	unsigned int i, cnt;
    133	struct scatterlist *sg = ttinfo->sgl;
    134	unsigned int offset = 0;
    135	int ret = 0;
    136
    137	for (i = 0; i < npods; i += cnt, pidx += cnt) {
    138		cnt = npods - i;
    139
    140		if (cnt > ULPMEM_IDATA_MAX_NPPODS)
    141			cnt = ULPMEM_IDATA_MAX_NPPODS;
    142
    143		ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt,
    144					      &sg, &offset);
    145		if (ret < 0)
    146			break;
    147	}
    148
    149	return ret;
    150}
    151
    152static int cxgbit_ddp_sgl_check(struct scatterlist *sg,
    153				unsigned int nents)
    154{
    155	unsigned int last_sgidx = nents - 1;
    156	unsigned int i;
    157
    158	for (i = 0; i < nents; i++, sg = sg_next(sg)) {
    159		unsigned int len = sg->length + sg->offset;
    160
    161		if ((sg->offset & 0x3) || (i && sg->offset) ||
    162		    ((i != last_sgidx) && (len != PAGE_SIZE))) {
    163			return -EINVAL;
    164		}
    165	}
    166
    167	return 0;
    168}
    169
    170static int
    171cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo,
    172		   unsigned int xferlen)
    173{
    174	struct cxgbit_device *cdev = csk->com.cdev;
    175	struct cxgbi_ppm *ppm = cdev2ppm(cdev);
    176	struct scatterlist *sgl = ttinfo->sgl;
    177	unsigned int sgcnt = ttinfo->nents;
    178	unsigned int sg_offset = sgl->offset;
    179	int ret;
    180
    181	if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) {
    182		pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n",
    183			 ppm, ppm->tformat.pgsz_idx_dflt,
    184			 xferlen, ttinfo->nents);
    185		return -EINVAL;
    186	}
    187
    188	if (cxgbit_ddp_sgl_check(sgl, sgcnt) < 0)
    189		return -EINVAL;
    190
    191	ttinfo->nr_pages = (xferlen + sgl->offset +
    192			    (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT;
    193
    194	/*
    195	 * the ddp tag will be used for the ttt in the outgoing r2t pdu
    196	 */
    197	ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx,
    198				      &ttinfo->tag, 0);
    199	if (ret < 0)
    200		return ret;
    201	ttinfo->npods = ret;
    202
    203	sgl->offset = 0;
    204	ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
    205	sgl->offset = sg_offset;
    206	if (!ret) {
    207		pr_debug("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n",
    208			 __func__, 0, xferlen, sgcnt);
    209		goto rel_ppods;
    210	}
    211
    212	cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset,
    213				xferlen, &ttinfo->hdr);
    214
    215	ret = cxgbit_ddp_set_map(ppm, csk, ttinfo);
    216	if (ret < 0) {
    217		__skb_queue_purge(&csk->ppodq);
    218		dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE);
    219		goto rel_ppods;
    220	}
    221
    222	return 0;
    223
    224rel_ppods:
    225	cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
    226	return -EINVAL;
    227}
    228
    229void
    230cxgbit_get_r2t_ttt(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
    231		   struct iscsi_r2t *r2t)
    232{
    233	struct cxgbit_sock *csk = conn->context;
    234	struct cxgbit_device *cdev = csk->com.cdev;
    235	struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
    236	struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
    237	int ret;
    238
    239	if ((!ccmd->setup_ddp) ||
    240	    (!test_bit(CSK_DDP_ENABLE, &csk->com.flags)))
    241		goto out;
    242
    243	ccmd->setup_ddp = false;
    244
    245	ttinfo->sgl = cmd->se_cmd.t_data_sg;
    246	ttinfo->nents = cmd->se_cmd.t_data_nents;
    247
    248	ret = cxgbit_ddp_reserve(csk, ttinfo, cmd->se_cmd.data_length);
    249	if (ret < 0) {
    250		pr_debug("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n",
    251			 csk, cmd, cmd->se_cmd.data_length, ttinfo->nents);
    252
    253		ttinfo->sgl = NULL;
    254		ttinfo->nents = 0;
    255	} else {
    256		ccmd->release = true;
    257	}
    258out:
    259	pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n", cdev, cmd, ttinfo->tag);
    260	r2t->targ_xfer_tag = ttinfo->tag;
    261}
    262
    263void cxgbit_unmap_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
    264{
    265	struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
    266
    267	if (ccmd->release) {
    268		if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
    269			put_page(sg_page(&ccmd->sg));
    270		} else {
    271			struct cxgbit_sock *csk = conn->context;
    272			struct cxgbit_device *cdev = csk->com.cdev;
    273			struct cxgbi_ppm *ppm = cdev2ppm(cdev);
    274			struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo;
    275
    276			/* Abort the TCP conn if DDP is not complete to
    277			 * avoid any possibility of DDP after freeing
    278			 * the cmd.
    279			 */
    280			if (unlikely(cmd->write_data_done !=
    281				     cmd->se_cmd.data_length))
    282				cxgbit_abort_conn(csk);
    283
    284			if (unlikely(ttinfo->sgl)) {
    285				dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl,
    286					     ttinfo->nents, DMA_FROM_DEVICE);
    287				ttinfo->nents = 0;
    288				ttinfo->sgl = NULL;
    289			}
    290			cxgbi_ppm_ppod_release(ppm, ttinfo->idx);
    291		}
    292		ccmd->release = false;
    293	}
    294}
    295
    296int cxgbit_ddp_init(struct cxgbit_device *cdev)
    297{
    298	struct cxgb4_lld_info *lldi = &cdev->lldi;
    299	struct net_device *ndev = cdev->lldi.ports[0];
    300	struct cxgbi_tag_format tformat;
    301	int ret, i;
    302
    303	if (!lldi->vr->iscsi.size) {
    304		pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name);
    305		return -EACCES;
    306	}
    307
    308	memset(&tformat, 0, sizeof(struct cxgbi_tag_format));
    309	for (i = 0; i < 4; i++)
    310		tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3))
    311					 & 0xF;
    312	cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat);
    313
    314	ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0],
    315			     cdev->lldi.pdev, &cdev->lldi, &tformat,
    316			     lldi->vr->iscsi.size, lldi->iscsi_llimit,
    317			     lldi->vr->iscsi.start, 2,
    318			     lldi->vr->ppod_edram.start,
    319			     lldi->vr->ppod_edram.size);
    320	if (ret >= 0) {
    321		struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm);
    322
    323		if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) &&
    324		    (ppm->ppmax >= 1024))
    325			set_bit(CDEV_DDP_ENABLE, &cdev->flags);
    326		ret = 0;
    327	}
    328
    329	return ret;
    330}