cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

otx2_cptvf_reqmgr.c (14907B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/* Copyright (C) 2020 Marvell. */
      3
      4#include "otx2_cptvf.h"
      5#include "otx2_cpt_common.h"
      6
      7/* SG list header size in bytes */
      8#define SG_LIST_HDR_SIZE	8
      9
     10/* Default timeout when waiting for free pending entry in us */
     11#define CPT_PENTRY_TIMEOUT	1000
     12#define CPT_PENTRY_STEP		50
     13
     14/* Default threshold for stopping and resuming sender requests */
     15#define CPT_IQ_STOP_MARGIN	128
     16#define CPT_IQ_RESUME_MARGIN	512
     17
     18/* Default command timeout in seconds */
     19#define CPT_COMMAND_TIMEOUT	4
     20#define CPT_TIME_IN_RESET_COUNT 5
     21
     22static void otx2_cpt_dump_sg_list(struct pci_dev *pdev,
     23				  struct otx2_cpt_req_info *req)
     24{
     25	int i;
     26
     27	pr_debug("Gather list size %d\n", req->in_cnt);
     28	for (i = 0; i < req->in_cnt; i++) {
     29		pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
     30			 req->in[i].size, req->in[i].vptr,
     31			 (void *) req->in[i].dma_addr);
     32		pr_debug("Buffer hexdump (%d bytes)\n",
     33			 req->in[i].size);
     34		print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
     35				     req->in[i].vptr, req->in[i].size, false);
     36	}
     37	pr_debug("Scatter list size %d\n", req->out_cnt);
     38	for (i = 0; i < req->out_cnt; i++) {
     39		pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i,
     40			 req->out[i].size, req->out[i].vptr,
     41			 (void *) req->out[i].dma_addr);
     42		pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size);
     43		print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1,
     44				     req->out[i].vptr, req->out[i].size, false);
     45	}
     46}
     47
     48static inline struct otx2_cpt_pending_entry *get_free_pending_entry(
     49					struct otx2_cpt_pending_queue *q,
     50					int qlen)
     51{
     52	struct otx2_cpt_pending_entry *ent = NULL;
     53
     54	ent = &q->head[q->rear];
     55	if (unlikely(ent->busy))
     56		return NULL;
     57
     58	q->rear++;
     59	if (unlikely(q->rear == qlen))
     60		q->rear = 0;
     61
     62	return ent;
     63}
     64
     65static inline u32 modulo_inc(u32 index, u32 length, u32 inc)
     66{
     67	if (WARN_ON(inc > length))
     68		inc = length;
     69
     70	index += inc;
     71	if (unlikely(index >= length))
     72		index -= length;
     73
     74	return index;
     75}
     76
     77static inline void free_pentry(struct otx2_cpt_pending_entry *pentry)
     78{
     79	pentry->completion_addr = NULL;
     80	pentry->info = NULL;
     81	pentry->callback = NULL;
     82	pentry->areq = NULL;
     83	pentry->resume_sender = false;
     84	pentry->busy = false;
     85}
     86
     87static inline int setup_sgio_components(struct pci_dev *pdev,
     88					struct otx2_cpt_buf_ptr *list,
     89					int buf_count, u8 *buffer)
     90{
     91	struct otx2_cpt_sglist_component *sg_ptr = NULL;
     92	int ret = 0, i, j;
     93	int components;
     94
     95	if (unlikely(!list)) {
     96		dev_err(&pdev->dev, "Input list pointer is NULL\n");
     97		return -EFAULT;
     98	}
     99
    100	for (i = 0; i < buf_count; i++) {
    101		if (unlikely(!list[i].vptr))
    102			continue;
    103		list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr,
    104						  list[i].size,
    105						  DMA_BIDIRECTIONAL);
    106		if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) {
    107			dev_err(&pdev->dev, "Dma mapping failed\n");
    108			ret = -EIO;
    109			goto sg_cleanup;
    110		}
    111	}
    112	components = buf_count / 4;
    113	sg_ptr = (struct otx2_cpt_sglist_component *)buffer;
    114	for (i = 0; i < components; i++) {
    115		sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size);
    116		sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size);
    117		sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size);
    118		sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size);
    119		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
    120		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
    121		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
    122		sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
    123		sg_ptr++;
    124	}
    125	components = buf_count % 4;
    126
    127	switch (components) {
    128	case 3:
    129		sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size);
    130		sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
    131		fallthrough;
    132	case 2:
    133		sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size);
    134		sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
    135		fallthrough;
    136	case 1:
    137		sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size);
    138		sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
    139		break;
    140	default:
    141		break;
    142	}
    143	return ret;
    144
    145sg_cleanup:
    146	for (j = 0; j < i; j++) {
    147		if (list[j].dma_addr) {
    148			dma_unmap_single(&pdev->dev, list[j].dma_addr,
    149					 list[j].size, DMA_BIDIRECTIONAL);
    150		}
    151
    152		list[j].dma_addr = 0;
    153	}
    154	return ret;
    155}
    156
    157static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev,
    158					      struct otx2_cpt_req_info *req,
    159					      gfp_t gfp)
    160{
    161	int align = OTX2_CPT_DMA_MINALIGN;
    162	struct otx2_cpt_inst_info *info;
    163	u32 dlen, align_dlen, info_len;
    164	u16 g_sz_bytes, s_sz_bytes;
    165	u32 total_mem_len;
    166
    167	if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT ||
    168		     req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) {
    169		dev_err(&pdev->dev, "Error too many sg components\n");
    170		return NULL;
    171	}
    172
    173	g_sz_bytes = ((req->in_cnt + 3) / 4) *
    174		      sizeof(struct otx2_cpt_sglist_component);
    175	s_sz_bytes = ((req->out_cnt + 3) / 4) *
    176		      sizeof(struct otx2_cpt_sglist_component);
    177
    178	dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
    179	align_dlen = ALIGN(dlen, align);
    180	info_len = ALIGN(sizeof(*info), align);
    181	total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s);
    182
    183	info = kzalloc(total_mem_len, gfp);
    184	if (unlikely(!info))
    185		return NULL;
    186
    187	info->dlen = dlen;
    188	info->in_buffer = (u8 *)info + info_len;
    189
    190	((u16 *)info->in_buffer)[0] = req->out_cnt;
    191	((u16 *)info->in_buffer)[1] = req->in_cnt;
    192	((u16 *)info->in_buffer)[2] = 0;
    193	((u16 *)info->in_buffer)[3] = 0;
    194	cpu_to_be64s((u64 *)info->in_buffer);
    195
    196	/* Setup gather (input) components */
    197	if (setup_sgio_components(pdev, req->in, req->in_cnt,
    198				  &info->in_buffer[8])) {
    199		dev_err(&pdev->dev, "Failed to setup gather list\n");
    200		goto destroy_info;
    201	}
    202
    203	if (setup_sgio_components(pdev, req->out, req->out_cnt,
    204				  &info->in_buffer[8 + g_sz_bytes])) {
    205		dev_err(&pdev->dev, "Failed to setup scatter list\n");
    206		goto destroy_info;
    207	}
    208
    209	info->dma_len = total_mem_len - info_len;
    210	info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer,
    211					  info->dma_len, DMA_BIDIRECTIONAL);
    212	if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) {
    213		dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n");
    214		goto destroy_info;
    215	}
    216	/*
    217	 * Get buffer for union otx2_cpt_res_s response
    218	 * structure and its physical address
    219	 */
    220	info->completion_addr = info->in_buffer + align_dlen;
    221	info->comp_baddr = info->dptr_baddr + align_dlen;
    222
    223	return info;
    224
    225destroy_info:
    226	otx2_cpt_info_destroy(pdev, info);
    227	return NULL;
    228}
    229
    230static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
    231			   struct otx2_cpt_pending_queue *pqueue,
    232			   struct otx2_cptlf_info *lf)
    233{
    234	struct otx2_cptvf_request *cpt_req = &req->req;
    235	struct otx2_cpt_pending_entry *pentry = NULL;
    236	union otx2_cpt_ctrl_info *ctrl = &req->ctrl;
    237	struct otx2_cpt_inst_info *info = NULL;
    238	union otx2_cpt_res_s *result = NULL;
    239	struct otx2_cpt_iq_command iq_cmd;
    240	union otx2_cpt_inst_s cptinst;
    241	int retry, ret = 0;
    242	u8 resume_sender;
    243	gfp_t gfp;
    244
    245	gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
    246							      GFP_ATOMIC;
    247	if (unlikely(!otx2_cptlf_started(lf->lfs)))
    248		return -ENODEV;
    249
    250	info = info_create(pdev, req, gfp);
    251	if (unlikely(!info)) {
    252		dev_err(&pdev->dev, "Setting up cpt inst info failed");
    253		return -ENOMEM;
    254	}
    255	cpt_req->dlen = info->dlen;
    256
    257	result = info->completion_addr;
    258	result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
    259
    260	spin_lock_bh(&pqueue->lock);
    261	pentry = get_free_pending_entry(pqueue, pqueue->qlen);
    262	retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP;
    263	while (unlikely(!pentry) && retry--) {
    264		spin_unlock_bh(&pqueue->lock);
    265		udelay(CPT_PENTRY_STEP);
    266		spin_lock_bh(&pqueue->lock);
    267		pentry = get_free_pending_entry(pqueue, pqueue->qlen);
    268	}
    269
    270	if (unlikely(!pentry)) {
    271		ret = -ENOSPC;
    272		goto destroy_info;
    273	}
    274
    275	/*
    276	 * Check if we are close to filling in entire pending queue,
    277	 * if so then tell the sender to stop/sleep by returning -EBUSY
    278	 * We do it only for context which can sleep (GFP_KERNEL)
    279	 */
    280	if (gfp == GFP_KERNEL &&
    281	    pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) {
    282		pentry->resume_sender = true;
    283	} else
    284		pentry->resume_sender = false;
    285	resume_sender = pentry->resume_sender;
    286	pqueue->pending_count++;
    287
    288	pentry->completion_addr = info->completion_addr;
    289	pentry->info = info;
    290	pentry->callback = req->callback;
    291	pentry->areq = req->areq;
    292	pentry->busy = true;
    293	info->pentry = pentry;
    294	info->time_in = jiffies;
    295	info->req = req;
    296
    297	/* Fill in the command */
    298	iq_cmd.cmd.u = 0;
    299	iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
    300	iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
    301	iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
    302	iq_cmd.cmd.s.dlen   = cpu_to_be16(cpt_req->dlen);
    303
    304	/* 64-bit swap for microcode data reads, not needed for addresses*/
    305	cpu_to_be64s(&iq_cmd.cmd.u);
    306	iq_cmd.dptr = info->dptr_baddr;
    307	iq_cmd.rptr = 0;
    308	iq_cmd.cptr.u = 0;
    309	iq_cmd.cptr.s.grp = ctrl->s.grp;
    310
    311	/* Fill in the CPT_INST_S type command for HW interpretation */
    312	otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr);
    313
    314	/* Print debug info if enabled */
    315	otx2_cpt_dump_sg_list(pdev, req);
    316	pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE);
    317	print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false);
    318	pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen);
    319	print_hex_dump_debug("", 0, 16, 1, info->in_buffer,
    320			     cpt_req->dlen, false);
    321
    322	/* Send CPT command */
    323	lf->lfs->ops->send_cmd(&cptinst, 1, lf);
    324
    325	/*
    326	 * We allocate and prepare pending queue entry in critical section
    327	 * together with submitting CPT instruction to CPT instruction queue
    328	 * to make sure that order of CPT requests is the same in both
    329	 * pending and instruction queues
    330	 */
    331	spin_unlock_bh(&pqueue->lock);
    332
    333	ret = resume_sender ? -EBUSY : -EINPROGRESS;
    334	return ret;
    335
    336destroy_info:
    337	spin_unlock_bh(&pqueue->lock);
    338	otx2_cpt_info_destroy(pdev, info);
    339	return ret;
    340}
    341
    342int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
    343			int cpu_num)
    344{
    345	struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
    346	struct otx2_cptlfs_info *lfs = &cptvf->lfs;
    347
    348	return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue,
    349			       &lfs->lf[cpu_num]);
    350}
    351
    352static int cpt_process_ccode(struct otx2_cptlfs_info *lfs,
    353			     union otx2_cpt_res_s *cpt_status,
    354			     struct otx2_cpt_inst_info *info,
    355			     u32 *res_code)
    356{
    357	u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status);
    358	u8 ccode = lfs->ops->cpt_get_compcode(cpt_status);
    359	struct pci_dev *pdev = lfs->pdev;
    360
    361	switch (ccode) {
    362	case OTX2_CPT_COMP_E_FAULT:
    363		dev_err(&pdev->dev,
    364			"Request failed with DMA fault\n");
    365		otx2_cpt_dump_sg_list(pdev, info->req);
    366		break;
    367
    368	case OTX2_CPT_COMP_E_HWERR:
    369		dev_err(&pdev->dev,
    370			"Request failed with hardware error\n");
    371		otx2_cpt_dump_sg_list(pdev, info->req);
    372		break;
    373
    374	case OTX2_CPT_COMP_E_INSTERR:
    375		dev_err(&pdev->dev,
    376			"Request failed with instruction error\n");
    377		otx2_cpt_dump_sg_list(pdev, info->req);
    378		break;
    379
    380	case OTX2_CPT_COMP_E_NOTDONE:
    381		/* check for timeout */
    382		if (time_after_eq(jiffies, info->time_in +
    383				  CPT_COMMAND_TIMEOUT * HZ))
    384			dev_warn(&pdev->dev,
    385				 "Request timed out 0x%p", info->req);
    386		else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) {
    387			info->time_in = jiffies;
    388			info->extra_time++;
    389		}
    390		return 1;
    391
    392	case OTX2_CPT_COMP_E_GOOD:
    393	case OTX2_CPT_COMP_E_WARN:
    394		/*
    395		 * Check microcode completion code, it is only valid
    396		 * when completion code is CPT_COMP_E::GOOD
    397		 */
    398		if (uc_ccode != OTX2_CPT_UCC_SUCCESS) {
    399			/*
    400			 * If requested hmac is truncated and ucode returns
    401			 * s/g write length error then we report success
    402			 * because ucode writes as many bytes of calculated
    403			 * hmac as available in gather buffer and reports
    404			 * s/g write length error if number of bytes in gather
    405			 * buffer is less than full hmac size.
    406			 */
    407			if (info->req->is_trunc_hmac &&
    408			    uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) {
    409				*res_code = 0;
    410				break;
    411			}
    412
    413			dev_err(&pdev->dev,
    414				"Request failed with software error code 0x%x\n",
    415				cpt_status->s.uc_compcode);
    416			otx2_cpt_dump_sg_list(pdev, info->req);
    417			break;
    418		}
    419		/* Request has been processed with success */
    420		*res_code = 0;
    421		break;
    422
    423	default:
    424		dev_err(&pdev->dev,
    425			"Request returned invalid status %d\n", ccode);
    426		break;
    427	}
    428	return 0;
    429}
    430
    431static inline void process_pending_queue(struct otx2_cptlfs_info *lfs,
    432					 struct otx2_cpt_pending_queue *pqueue)
    433{
    434	struct otx2_cpt_pending_entry *resume_pentry = NULL;
    435	void (*callback)(int status, void *arg, void *req);
    436	struct otx2_cpt_pending_entry *pentry = NULL;
    437	union otx2_cpt_res_s *cpt_status = NULL;
    438	struct otx2_cpt_inst_info *info = NULL;
    439	struct otx2_cpt_req_info *req = NULL;
    440	struct crypto_async_request *areq;
    441	struct pci_dev *pdev = lfs->pdev;
    442	u32 res_code, resume_index;
    443
    444	while (1) {
    445		spin_lock_bh(&pqueue->lock);
    446		pentry = &pqueue->head[pqueue->front];
    447
    448		if (WARN_ON(!pentry)) {
    449			spin_unlock_bh(&pqueue->lock);
    450			break;
    451		}
    452
    453		res_code = -EINVAL;
    454		if (unlikely(!pentry->busy)) {
    455			spin_unlock_bh(&pqueue->lock);
    456			break;
    457		}
    458
    459		if (unlikely(!pentry->callback)) {
    460			dev_err(&pdev->dev, "Callback NULL\n");
    461			goto process_pentry;
    462		}
    463
    464		info = pentry->info;
    465		if (unlikely(!info)) {
    466			dev_err(&pdev->dev, "Pending entry post arg NULL\n");
    467			goto process_pentry;
    468		}
    469
    470		req = info->req;
    471		if (unlikely(!req)) {
    472			dev_err(&pdev->dev, "Request NULL\n");
    473			goto process_pentry;
    474		}
    475
    476		cpt_status = pentry->completion_addr;
    477		if (unlikely(!cpt_status)) {
    478			dev_err(&pdev->dev, "Completion address NULL\n");
    479			goto process_pentry;
    480		}
    481
    482		if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) {
    483			spin_unlock_bh(&pqueue->lock);
    484			return;
    485		}
    486		info->pdev = pdev;
    487
    488process_pentry:
    489		/*
    490		 * Check if we should inform sending side to resume
    491		 * We do it CPT_IQ_RESUME_MARGIN elements in advance before
    492		 * pending queue becomes empty
    493		 */
    494		resume_index = modulo_inc(pqueue->front, pqueue->qlen,
    495					  CPT_IQ_RESUME_MARGIN);
    496		resume_pentry = &pqueue->head[resume_index];
    497		if (resume_pentry &&
    498		    resume_pentry->resume_sender) {
    499			resume_pentry->resume_sender = false;
    500			callback = resume_pentry->callback;
    501			areq = resume_pentry->areq;
    502
    503			if (callback) {
    504				spin_unlock_bh(&pqueue->lock);
    505
    506				/*
    507				 * EINPROGRESS is an indication for sending
    508				 * side that it can resume sending requests
    509				 */
    510				callback(-EINPROGRESS, areq, info);
    511				spin_lock_bh(&pqueue->lock);
    512			}
    513		}
    514
    515		callback = pentry->callback;
    516		areq = pentry->areq;
    517		free_pentry(pentry);
    518
    519		pqueue->pending_count--;
    520		pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1);
    521		spin_unlock_bh(&pqueue->lock);
    522
    523		/*
    524		 * Call callback after current pending entry has been
    525		 * processed, we don't do it if the callback pointer is
    526		 * invalid.
    527		 */
    528		if (callback)
    529			callback(res_code, areq, info);
    530	}
    531}
    532
    533void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe)
    534{
    535	process_pending_queue(wqe->lfs,
    536			      &wqe->lfs->lf[wqe->lf_num].pqueue);
    537}
    538
    539int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)
    540{
    541	struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
    542
    543	return cptvf->lfs.kcrypto_eng_grp_num;
    544}