cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

af_alg.c (27356B)


      1// SPDX-License-Identifier: GPL-2.0-or-later
      2/*
      3 * af_alg: User-space algorithm interface
      4 *
      5 * This file provides the user-space API for algorithms.
      6 *
      7 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
      8 */
      9
     10#include <linux/atomic.h>
     11#include <crypto/if_alg.h>
     12#include <linux/crypto.h>
     13#include <linux/init.h>
     14#include <linux/kernel.h>
     15#include <linux/list.h>
     16#include <linux/module.h>
     17#include <linux/net.h>
     18#include <linux/rwsem.h>
     19#include <linux/sched.h>
     20#include <linux/sched/signal.h>
     21#include <linux/security.h>
     22
     23struct alg_type_list {
     24	const struct af_alg_type *type;
     25	struct list_head list;
     26};
     27
     28static struct proto alg_proto = {
     29	.name			= "ALG",
     30	.owner			= THIS_MODULE,
     31	.obj_size		= sizeof(struct alg_sock),
     32};
     33
     34static LIST_HEAD(alg_types);
     35static DECLARE_RWSEM(alg_types_sem);
     36
     37static const struct af_alg_type *alg_get_type(const char *name)
     38{
     39	const struct af_alg_type *type = ERR_PTR(-ENOENT);
     40	struct alg_type_list *node;
     41
     42	down_read(&alg_types_sem);
     43	list_for_each_entry(node, &alg_types, list) {
     44		if (strcmp(node->type->name, name))
     45			continue;
     46
     47		if (try_module_get(node->type->owner))
     48			type = node->type;
     49		break;
     50	}
     51	up_read(&alg_types_sem);
     52
     53	return type;
     54}
     55
     56int af_alg_register_type(const struct af_alg_type *type)
     57{
     58	struct alg_type_list *node;
     59	int err = -EEXIST;
     60
     61	down_write(&alg_types_sem);
     62	list_for_each_entry(node, &alg_types, list) {
     63		if (!strcmp(node->type->name, type->name))
     64			goto unlock;
     65	}
     66
     67	node = kmalloc(sizeof(*node), GFP_KERNEL);
     68	err = -ENOMEM;
     69	if (!node)
     70		goto unlock;
     71
     72	type->ops->owner = THIS_MODULE;
     73	if (type->ops_nokey)
     74		type->ops_nokey->owner = THIS_MODULE;
     75	node->type = type;
     76	list_add(&node->list, &alg_types);
     77	err = 0;
     78
     79unlock:
     80	up_write(&alg_types_sem);
     81
     82	return err;
     83}
     84EXPORT_SYMBOL_GPL(af_alg_register_type);
     85
     86int af_alg_unregister_type(const struct af_alg_type *type)
     87{
     88	struct alg_type_list *node;
     89	int err = -ENOENT;
     90
     91	down_write(&alg_types_sem);
     92	list_for_each_entry(node, &alg_types, list) {
     93		if (strcmp(node->type->name, type->name))
     94			continue;
     95
     96		list_del(&node->list);
     97		kfree(node);
     98		err = 0;
     99		break;
    100	}
    101	up_write(&alg_types_sem);
    102
    103	return err;
    104}
    105EXPORT_SYMBOL_GPL(af_alg_unregister_type);
    106
    107static void alg_do_release(const struct af_alg_type *type, void *private)
    108{
    109	if (!type)
    110		return;
    111
    112	type->release(private);
    113	module_put(type->owner);
    114}
    115
    116int af_alg_release(struct socket *sock)
    117{
    118	if (sock->sk) {
    119		sock_put(sock->sk);
    120		sock->sk = NULL;
    121	}
    122	return 0;
    123}
    124EXPORT_SYMBOL_GPL(af_alg_release);
    125
    126void af_alg_release_parent(struct sock *sk)
    127{
    128	struct alg_sock *ask = alg_sk(sk);
    129	unsigned int nokey = atomic_read(&ask->nokey_refcnt);
    130
    131	sk = ask->parent;
    132	ask = alg_sk(sk);
    133
    134	if (nokey)
    135		atomic_dec(&ask->nokey_refcnt);
    136
    137	if (atomic_dec_and_test(&ask->refcnt))
    138		sock_put(sk);
    139}
    140EXPORT_SYMBOL_GPL(af_alg_release_parent);
    141
    142static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
    143{
    144	const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY;
    145	struct sock *sk = sock->sk;
    146	struct alg_sock *ask = alg_sk(sk);
    147	struct sockaddr_alg_new *sa = (void *)uaddr;
    148	const struct af_alg_type *type;
    149	void *private;
    150	int err;
    151
    152	if (sock->state == SS_CONNECTED)
    153		return -EINVAL;
    154
    155	BUILD_BUG_ON(offsetof(struct sockaddr_alg_new, salg_name) !=
    156		     offsetof(struct sockaddr_alg, salg_name));
    157	BUILD_BUG_ON(offsetof(struct sockaddr_alg, salg_name) != sizeof(*sa));
    158
    159	if (addr_len < sizeof(*sa) + 1)
    160		return -EINVAL;
    161
    162	/* If caller uses non-allowed flag, return error. */
    163	if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed))
    164		return -EINVAL;
    165
    166	sa->salg_type[sizeof(sa->salg_type) - 1] = 0;
    167	sa->salg_name[addr_len - sizeof(*sa) - 1] = 0;
    168
    169	type = alg_get_type(sa->salg_type);
    170	if (PTR_ERR(type) == -ENOENT) {
    171		request_module("algif-%s", sa->salg_type);
    172		type = alg_get_type(sa->salg_type);
    173	}
    174
    175	if (IS_ERR(type))
    176		return PTR_ERR(type);
    177
    178	private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask);
    179	if (IS_ERR(private)) {
    180		module_put(type->owner);
    181		return PTR_ERR(private);
    182	}
    183
    184	err = -EBUSY;
    185	lock_sock(sk);
    186	if (atomic_read(&ask->refcnt))
    187		goto unlock;
    188
    189	swap(ask->type, type);
    190	swap(ask->private, private);
    191
    192	err = 0;
    193
    194unlock:
    195	release_sock(sk);
    196
    197	alg_do_release(type, private);
    198
    199	return err;
    200}
    201
    202static int alg_setkey(struct sock *sk, sockptr_t ukey, unsigned int keylen)
    203{
    204	struct alg_sock *ask = alg_sk(sk);
    205	const struct af_alg_type *type = ask->type;
    206	u8 *key;
    207	int err;
    208
    209	key = sock_kmalloc(sk, keylen, GFP_KERNEL);
    210	if (!key)
    211		return -ENOMEM;
    212
    213	err = -EFAULT;
    214	if (copy_from_sockptr(key, ukey, keylen))
    215		goto out;
    216
    217	err = type->setkey(ask->private, key, keylen);
    218
    219out:
    220	sock_kzfree_s(sk, key, keylen);
    221
    222	return err;
    223}
    224
    225static int alg_setsockopt(struct socket *sock, int level, int optname,
    226			  sockptr_t optval, unsigned int optlen)
    227{
    228	struct sock *sk = sock->sk;
    229	struct alg_sock *ask = alg_sk(sk);
    230	const struct af_alg_type *type;
    231	int err = -EBUSY;
    232
    233	lock_sock(sk);
    234	if (atomic_read(&ask->refcnt) != atomic_read(&ask->nokey_refcnt))
    235		goto unlock;
    236
    237	type = ask->type;
    238
    239	err = -ENOPROTOOPT;
    240	if (level != SOL_ALG || !type)
    241		goto unlock;
    242
    243	switch (optname) {
    244	case ALG_SET_KEY:
    245		if (sock->state == SS_CONNECTED)
    246			goto unlock;
    247		if (!type->setkey)
    248			goto unlock;
    249
    250		err = alg_setkey(sk, optval, optlen);
    251		break;
    252	case ALG_SET_AEAD_AUTHSIZE:
    253		if (sock->state == SS_CONNECTED)
    254			goto unlock;
    255		if (!type->setauthsize)
    256			goto unlock;
    257		err = type->setauthsize(ask->private, optlen);
    258		break;
    259	case ALG_SET_DRBG_ENTROPY:
    260		if (sock->state == SS_CONNECTED)
    261			goto unlock;
    262		if (!type->setentropy)
    263			goto unlock;
    264
    265		err = type->setentropy(ask->private, optval, optlen);
    266	}
    267
    268unlock:
    269	release_sock(sk);
    270
    271	return err;
    272}
    273
    274int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern)
    275{
    276	struct alg_sock *ask = alg_sk(sk);
    277	const struct af_alg_type *type;
    278	struct sock *sk2;
    279	unsigned int nokey;
    280	int err;
    281
    282	lock_sock(sk);
    283	type = ask->type;
    284
    285	err = -EINVAL;
    286	if (!type)
    287		goto unlock;
    288
    289	sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, kern);
    290	err = -ENOMEM;
    291	if (!sk2)
    292		goto unlock;
    293
    294	sock_init_data(newsock, sk2);
    295	security_sock_graft(sk2, newsock);
    296	security_sk_clone(sk, sk2);
    297
    298	/*
    299	 * newsock->ops assigned here to allow type->accept call to override
    300	 * them when required.
    301	 */
    302	newsock->ops = type->ops;
    303	err = type->accept(ask->private, sk2);
    304
    305	nokey = err == -ENOKEY;
    306	if (nokey && type->accept_nokey)
    307		err = type->accept_nokey(ask->private, sk2);
    308
    309	if (err)
    310		goto unlock;
    311
    312	if (atomic_inc_return_relaxed(&ask->refcnt) == 1)
    313		sock_hold(sk);
    314	if (nokey) {
    315		atomic_inc(&ask->nokey_refcnt);
    316		atomic_set(&alg_sk(sk2)->nokey_refcnt, 1);
    317	}
    318	alg_sk(sk2)->parent = sk;
    319	alg_sk(sk2)->type = type;
    320
    321	newsock->state = SS_CONNECTED;
    322
    323	if (nokey)
    324		newsock->ops = type->ops_nokey;
    325
    326	err = 0;
    327
    328unlock:
    329	release_sock(sk);
    330
    331	return err;
    332}
    333EXPORT_SYMBOL_GPL(af_alg_accept);
    334
    335static int alg_accept(struct socket *sock, struct socket *newsock, int flags,
    336		      bool kern)
    337{
    338	return af_alg_accept(sock->sk, newsock, kern);
    339}
    340
    341static const struct proto_ops alg_proto_ops = {
    342	.family		=	PF_ALG,
    343	.owner		=	THIS_MODULE,
    344
    345	.connect	=	sock_no_connect,
    346	.socketpair	=	sock_no_socketpair,
    347	.getname	=	sock_no_getname,
    348	.ioctl		=	sock_no_ioctl,
    349	.listen		=	sock_no_listen,
    350	.shutdown	=	sock_no_shutdown,
    351	.mmap		=	sock_no_mmap,
    352	.sendpage	=	sock_no_sendpage,
    353	.sendmsg	=	sock_no_sendmsg,
    354	.recvmsg	=	sock_no_recvmsg,
    355
    356	.bind		=	alg_bind,
    357	.release	=	af_alg_release,
    358	.setsockopt	=	alg_setsockopt,
    359	.accept		=	alg_accept,
    360};
    361
    362static void alg_sock_destruct(struct sock *sk)
    363{
    364	struct alg_sock *ask = alg_sk(sk);
    365
    366	alg_do_release(ask->type, ask->private);
    367}
    368
    369static int alg_create(struct net *net, struct socket *sock, int protocol,
    370		      int kern)
    371{
    372	struct sock *sk;
    373	int err;
    374
    375	if (sock->type != SOCK_SEQPACKET)
    376		return -ESOCKTNOSUPPORT;
    377	if (protocol != 0)
    378		return -EPROTONOSUPPORT;
    379
    380	err = -ENOMEM;
    381	sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto, kern);
    382	if (!sk)
    383		goto out;
    384
    385	sock->ops = &alg_proto_ops;
    386	sock_init_data(sock, sk);
    387
    388	sk->sk_destruct = alg_sock_destruct;
    389
    390	return 0;
    391out:
    392	return err;
    393}
    394
    395static const struct net_proto_family alg_family = {
    396	.family	=	PF_ALG,
    397	.create	=	alg_create,
    398	.owner	=	THIS_MODULE,
    399};
    400
    401int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len)
    402{
    403	size_t off;
    404	ssize_t n;
    405	int npages, i;
    406
    407	n = iov_iter_get_pages(iter, sgl->pages, len, ALG_MAX_PAGES, &off);
    408	if (n < 0)
    409		return n;
    410
    411	npages = DIV_ROUND_UP(off + n, PAGE_SIZE);
    412	if (WARN_ON(npages == 0))
    413		return -EINVAL;
    414	/* Add one extra for linking */
    415	sg_init_table(sgl->sg, npages + 1);
    416
    417	for (i = 0, len = n; i < npages; i++) {
    418		int plen = min_t(int, len, PAGE_SIZE - off);
    419
    420		sg_set_page(sgl->sg + i, sgl->pages[i], plen, off);
    421
    422		off = 0;
    423		len -= plen;
    424	}
    425	sg_mark_end(sgl->sg + npages - 1);
    426	sgl->npages = npages;
    427
    428	return n;
    429}
    430EXPORT_SYMBOL_GPL(af_alg_make_sg);
    431
    432static void af_alg_link_sg(struct af_alg_sgl *sgl_prev,
    433			   struct af_alg_sgl *sgl_new)
    434{
    435	sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1);
    436	sg_chain(sgl_prev->sg, sgl_prev->npages + 1, sgl_new->sg);
    437}
    438
    439void af_alg_free_sg(struct af_alg_sgl *sgl)
    440{
    441	int i;
    442
    443	for (i = 0; i < sgl->npages; i++)
    444		put_page(sgl->pages[i]);
    445}
    446EXPORT_SYMBOL_GPL(af_alg_free_sg);
    447
    448static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con)
    449{
    450	struct cmsghdr *cmsg;
    451
    452	for_each_cmsghdr(cmsg, msg) {
    453		if (!CMSG_OK(msg, cmsg))
    454			return -EINVAL;
    455		if (cmsg->cmsg_level != SOL_ALG)
    456			continue;
    457
    458		switch (cmsg->cmsg_type) {
    459		case ALG_SET_IV:
    460			if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv)))
    461				return -EINVAL;
    462			con->iv = (void *)CMSG_DATA(cmsg);
    463			if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen +
    464						      sizeof(*con->iv)))
    465				return -EINVAL;
    466			break;
    467
    468		case ALG_SET_OP:
    469			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
    470				return -EINVAL;
    471			con->op = *(u32 *)CMSG_DATA(cmsg);
    472			break;
    473
    474		case ALG_SET_AEAD_ASSOCLEN:
    475			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
    476				return -EINVAL;
    477			con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg);
    478			break;
    479
    480		default:
    481			return -EINVAL;
    482		}
    483	}
    484
    485	return 0;
    486}
    487
    488/**
    489 * af_alg_alloc_tsgl - allocate the TX SGL
    490 *
    491 * @sk: socket of connection to user space
    492 * Return: 0 upon success, < 0 upon error
    493 */
    494static int af_alg_alloc_tsgl(struct sock *sk)
    495{
    496	struct alg_sock *ask = alg_sk(sk);
    497	struct af_alg_ctx *ctx = ask->private;
    498	struct af_alg_tsgl *sgl;
    499	struct scatterlist *sg = NULL;
    500
    501	sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
    502	if (!list_empty(&ctx->tsgl_list))
    503		sg = sgl->sg;
    504
    505	if (!sg || sgl->cur >= MAX_SGL_ENTS) {
    506		sgl = sock_kmalloc(sk,
    507				   struct_size(sgl, sg, (MAX_SGL_ENTS + 1)),
    508				   GFP_KERNEL);
    509		if (!sgl)
    510			return -ENOMEM;
    511
    512		sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
    513		sgl->cur = 0;
    514
    515		if (sg)
    516			sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
    517
    518		list_add_tail(&sgl->list, &ctx->tsgl_list);
    519	}
    520
    521	return 0;
    522}
    523
    524/**
    525 * af_alg_count_tsgl - Count number of TX SG entries
    526 *
    527 * The counting starts from the beginning of the SGL to @bytes. If
    528 * an @offset is provided, the counting of the SG entries starts at the @offset.
    529 *
    530 * @sk: socket of connection to user space
    531 * @bytes: Count the number of SG entries holding given number of bytes.
    532 * @offset: Start the counting of SG entries from the given offset.
    533 * Return: Number of TX SG entries found given the constraints
    534 */
    535unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset)
    536{
    537	const struct alg_sock *ask = alg_sk(sk);
    538	const struct af_alg_ctx *ctx = ask->private;
    539	const struct af_alg_tsgl *sgl;
    540	unsigned int i;
    541	unsigned int sgl_count = 0;
    542
    543	if (!bytes)
    544		return 0;
    545
    546	list_for_each_entry(sgl, &ctx->tsgl_list, list) {
    547		const struct scatterlist *sg = sgl->sg;
    548
    549		for (i = 0; i < sgl->cur; i++) {
    550			size_t bytes_count;
    551
    552			/* Skip offset */
    553			if (offset >= sg[i].length) {
    554				offset -= sg[i].length;
    555				bytes -= sg[i].length;
    556				continue;
    557			}
    558
    559			bytes_count = sg[i].length - offset;
    560
    561			offset = 0;
    562			sgl_count++;
    563
    564			/* If we have seen requested number of bytes, stop */
    565			if (bytes_count >= bytes)
    566				return sgl_count;
    567
    568			bytes -= bytes_count;
    569		}
    570	}
    571
    572	return sgl_count;
    573}
    574EXPORT_SYMBOL_GPL(af_alg_count_tsgl);
    575
    576/**
    577 * af_alg_pull_tsgl - Release the specified buffers from TX SGL
    578 *
    579 * If @dst is non-null, reassign the pages to @dst. The caller must release
    580 * the pages. If @dst_offset is given only reassign the pages to @dst starting
    581 * at the @dst_offset (byte). The caller must ensure that @dst is large
    582 * enough (e.g. by using af_alg_count_tsgl with the same offset).
    583 *
    584 * @sk: socket of connection to user space
    585 * @used: Number of bytes to pull from TX SGL
    586 * @dst: If non-NULL, buffer is reassigned to dst SGL instead of releasing. The
    587 *	 caller must release the buffers in dst.
    588 * @dst_offset: Reassign the TX SGL from given offset. All buffers before
    589 *	        reaching the offset is released.
    590 */
    591void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst,
    592		      size_t dst_offset)
    593{
    594	struct alg_sock *ask = alg_sk(sk);
    595	struct af_alg_ctx *ctx = ask->private;
    596	struct af_alg_tsgl *sgl;
    597	struct scatterlist *sg;
    598	unsigned int i, j = 0;
    599
    600	while (!list_empty(&ctx->tsgl_list)) {
    601		sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl,
    602				       list);
    603		sg = sgl->sg;
    604
    605		for (i = 0; i < sgl->cur; i++) {
    606			size_t plen = min_t(size_t, used, sg[i].length);
    607			struct page *page = sg_page(sg + i);
    608
    609			if (!page)
    610				continue;
    611
    612			/*
    613			 * Assumption: caller created af_alg_count_tsgl(len)
    614			 * SG entries in dst.
    615			 */
    616			if (dst) {
    617				if (dst_offset >= plen) {
    618					/* discard page before offset */
    619					dst_offset -= plen;
    620				} else {
    621					/* reassign page to dst after offset */
    622					get_page(page);
    623					sg_set_page(dst + j, page,
    624						    plen - dst_offset,
    625						    sg[i].offset + dst_offset);
    626					dst_offset = 0;
    627					j++;
    628				}
    629			}
    630
    631			sg[i].length -= plen;
    632			sg[i].offset += plen;
    633
    634			used -= plen;
    635			ctx->used -= plen;
    636
    637			if (sg[i].length)
    638				return;
    639
    640			put_page(page);
    641			sg_assign_page(sg + i, NULL);
    642		}
    643
    644		list_del(&sgl->list);
    645		sock_kfree_s(sk, sgl, struct_size(sgl, sg, MAX_SGL_ENTS + 1));
    646	}
    647
    648	if (!ctx->used)
    649		ctx->merge = 0;
    650	ctx->init = ctx->more;
    651}
    652EXPORT_SYMBOL_GPL(af_alg_pull_tsgl);
    653
    654/**
    655 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request
    656 *
    657 * @areq: Request holding the TX and RX SGL
    658 */
    659static void af_alg_free_areq_sgls(struct af_alg_async_req *areq)
    660{
    661	struct sock *sk = areq->sk;
    662	struct alg_sock *ask = alg_sk(sk);
    663	struct af_alg_ctx *ctx = ask->private;
    664	struct af_alg_rsgl *rsgl, *tmp;
    665	struct scatterlist *tsgl;
    666	struct scatterlist *sg;
    667	unsigned int i;
    668
    669	list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) {
    670		atomic_sub(rsgl->sg_num_bytes, &ctx->rcvused);
    671		af_alg_free_sg(&rsgl->sgl);
    672		list_del(&rsgl->list);
    673		if (rsgl != &areq->first_rsgl)
    674			sock_kfree_s(sk, rsgl, sizeof(*rsgl));
    675	}
    676
    677	tsgl = areq->tsgl;
    678	if (tsgl) {
    679		for_each_sg(tsgl, sg, areq->tsgl_entries, i) {
    680			if (!sg_page(sg))
    681				continue;
    682			put_page(sg_page(sg));
    683		}
    684
    685		sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl));
    686	}
    687}
    688
    689/**
    690 * af_alg_wait_for_wmem - wait for availability of writable memory
    691 *
    692 * @sk: socket of connection to user space
    693 * @flags: If MSG_DONTWAIT is set, then only report if function would sleep
    694 * Return: 0 when writable memory is available, < 0 upon error
    695 */
    696static int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags)
    697{
    698	DEFINE_WAIT_FUNC(wait, woken_wake_function);
    699	int err = -ERESTARTSYS;
    700	long timeout;
    701
    702	if (flags & MSG_DONTWAIT)
    703		return -EAGAIN;
    704
    705	sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
    706
    707	add_wait_queue(sk_sleep(sk), &wait);
    708	for (;;) {
    709		if (signal_pending(current))
    710			break;
    711		timeout = MAX_SCHEDULE_TIMEOUT;
    712		if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) {
    713			err = 0;
    714			break;
    715		}
    716	}
    717	remove_wait_queue(sk_sleep(sk), &wait);
    718
    719	return err;
    720}
    721
    722/**
    723 * af_alg_wmem_wakeup - wakeup caller when writable memory is available
    724 *
    725 * @sk: socket of connection to user space
    726 */
    727void af_alg_wmem_wakeup(struct sock *sk)
    728{
    729	struct socket_wq *wq;
    730
    731	if (!af_alg_writable(sk))
    732		return;
    733
    734	rcu_read_lock();
    735	wq = rcu_dereference(sk->sk_wq);
    736	if (skwq_has_sleeper(wq))
    737		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
    738							   EPOLLRDNORM |
    739							   EPOLLRDBAND);
    740	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
    741	rcu_read_unlock();
    742}
    743EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup);
    744
    745/**
    746 * af_alg_wait_for_data - wait for availability of TX data
    747 *
    748 * @sk: socket of connection to user space
    749 * @flags: If MSG_DONTWAIT is set, then only report if function would sleep
    750 * @min: Set to minimum request size if partial requests are allowed.
    751 * Return: 0 when writable memory is available, < 0 upon error
    752 */
    753int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min)
    754{
    755	DEFINE_WAIT_FUNC(wait, woken_wake_function);
    756	struct alg_sock *ask = alg_sk(sk);
    757	struct af_alg_ctx *ctx = ask->private;
    758	long timeout;
    759	int err = -ERESTARTSYS;
    760
    761	if (flags & MSG_DONTWAIT)
    762		return -EAGAIN;
    763
    764	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
    765
    766	add_wait_queue(sk_sleep(sk), &wait);
    767	for (;;) {
    768		if (signal_pending(current))
    769			break;
    770		timeout = MAX_SCHEDULE_TIMEOUT;
    771		if (sk_wait_event(sk, &timeout,
    772				  ctx->init && (!ctx->more ||
    773						(min && ctx->used >= min)),
    774				  &wait)) {
    775			err = 0;
    776			break;
    777		}
    778	}
    779	remove_wait_queue(sk_sleep(sk), &wait);
    780
    781	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
    782
    783	return err;
    784}
    785EXPORT_SYMBOL_GPL(af_alg_wait_for_data);
    786
    787/**
    788 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel
    789 *
    790 * @sk: socket of connection to user space
    791 */
    792static void af_alg_data_wakeup(struct sock *sk)
    793{
    794	struct alg_sock *ask = alg_sk(sk);
    795	struct af_alg_ctx *ctx = ask->private;
    796	struct socket_wq *wq;
    797
    798	if (!ctx->used)
    799		return;
    800
    801	rcu_read_lock();
    802	wq = rcu_dereference(sk->sk_wq);
    803	if (skwq_has_sleeper(wq))
    804		wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
    805							   EPOLLRDNORM |
    806							   EPOLLRDBAND);
    807	sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
    808	rcu_read_unlock();
    809}
    810
    811/**
    812 * af_alg_sendmsg - implementation of sendmsg system call handler
    813 *
    814 * The sendmsg system call handler obtains the user data and stores it
    815 * in ctx->tsgl_list. This implies allocation of the required numbers of
    816 * struct af_alg_tsgl.
    817 *
    818 * In addition, the ctx is filled with the information sent via CMSG.
    819 *
    820 * @sock: socket of connection to user space
    821 * @msg: message from user space
    822 * @size: size of message from user space
    823 * @ivsize: the size of the IV for the cipher operation to verify that the
    824 *	   user-space-provided IV has the right size
    825 * Return: the number of copied data upon success, < 0 upon error
    826 */
    827int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
    828		   unsigned int ivsize)
    829{
    830	struct sock *sk = sock->sk;
    831	struct alg_sock *ask = alg_sk(sk);
    832	struct af_alg_ctx *ctx = ask->private;
    833	struct af_alg_tsgl *sgl;
    834	struct af_alg_control con = {};
    835	long copied = 0;
    836	bool enc = false;
    837	bool init = false;
    838	int err = 0;
    839
    840	if (msg->msg_controllen) {
    841		err = af_alg_cmsg_send(msg, &con);
    842		if (err)
    843			return err;
    844
    845		init = true;
    846		switch (con.op) {
    847		case ALG_OP_ENCRYPT:
    848			enc = true;
    849			break;
    850		case ALG_OP_DECRYPT:
    851			enc = false;
    852			break;
    853		default:
    854			return -EINVAL;
    855		}
    856
    857		if (con.iv && con.iv->ivlen != ivsize)
    858			return -EINVAL;
    859	}
    860
    861	lock_sock(sk);
    862	if (ctx->init && !ctx->more) {
    863		if (ctx->used) {
    864			err = -EINVAL;
    865			goto unlock;
    866		}
    867
    868		pr_info_once(
    869			"%s sent an empty control message without MSG_MORE.\n",
    870			current->comm);
    871	}
    872	ctx->init = true;
    873
    874	if (init) {
    875		ctx->enc = enc;
    876		if (con.iv)
    877			memcpy(ctx->iv, con.iv->iv, ivsize);
    878
    879		ctx->aead_assoclen = con.aead_assoclen;
    880	}
    881
    882	while (size) {
    883		struct scatterlist *sg;
    884		size_t len = size;
    885		size_t plen;
    886
    887		/* use the existing memory in an allocated page */
    888		if (ctx->merge) {
    889			sgl = list_entry(ctx->tsgl_list.prev,
    890					 struct af_alg_tsgl, list);
    891			sg = sgl->sg + sgl->cur - 1;
    892			len = min_t(size_t, len,
    893				    PAGE_SIZE - sg->offset - sg->length);
    894
    895			err = memcpy_from_msg(page_address(sg_page(sg)) +
    896					      sg->offset + sg->length,
    897					      msg, len);
    898			if (err)
    899				goto unlock;
    900
    901			sg->length += len;
    902			ctx->merge = (sg->offset + sg->length) &
    903				     (PAGE_SIZE - 1);
    904
    905			ctx->used += len;
    906			copied += len;
    907			size -= len;
    908			continue;
    909		}
    910
    911		if (!af_alg_writable(sk)) {
    912			err = af_alg_wait_for_wmem(sk, msg->msg_flags);
    913			if (err)
    914				goto unlock;
    915		}
    916
    917		/* allocate a new page */
    918		len = min_t(unsigned long, len, af_alg_sndbuf(sk));
    919
    920		err = af_alg_alloc_tsgl(sk);
    921		if (err)
    922			goto unlock;
    923
    924		sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl,
    925				 list);
    926		sg = sgl->sg;
    927		if (sgl->cur)
    928			sg_unmark_end(sg + sgl->cur - 1);
    929
    930		do {
    931			struct page *pg;
    932			unsigned int i = sgl->cur;
    933
    934			plen = min_t(size_t, len, PAGE_SIZE);
    935
    936			pg = alloc_page(GFP_KERNEL);
    937			if (!pg) {
    938				err = -ENOMEM;
    939				goto unlock;
    940			}
    941
    942			sg_assign_page(sg + i, pg);
    943
    944			err = memcpy_from_msg(page_address(sg_page(sg + i)),
    945					      msg, plen);
    946			if (err) {
    947				__free_page(sg_page(sg + i));
    948				sg_assign_page(sg + i, NULL);
    949				goto unlock;
    950			}
    951
    952			sg[i].length = plen;
    953			len -= plen;
    954			ctx->used += plen;
    955			copied += plen;
    956			size -= plen;
    957			sgl->cur++;
    958		} while (len && sgl->cur < MAX_SGL_ENTS);
    959
    960		if (!size)
    961			sg_mark_end(sg + sgl->cur - 1);
    962
    963		ctx->merge = plen & (PAGE_SIZE - 1);
    964	}
    965
    966	err = 0;
    967
    968	ctx->more = msg->msg_flags & MSG_MORE;
    969
    970unlock:
    971	af_alg_data_wakeup(sk);
    972	release_sock(sk);
    973
    974	return copied ?: err;
    975}
    976EXPORT_SYMBOL_GPL(af_alg_sendmsg);
    977
    978/**
    979 * af_alg_sendpage - sendpage system call handler
    980 * @sock: socket of connection to user space to write to
    981 * @page: data to send
    982 * @offset: offset into page to begin sending
    983 * @size: length of data
    984 * @flags: message send/receive flags
    985 *
    986 * This is a generic implementation of sendpage to fill ctx->tsgl_list.
    987 */
    988ssize_t af_alg_sendpage(struct socket *sock, struct page *page,
    989			int offset, size_t size, int flags)
    990{
    991	struct sock *sk = sock->sk;
    992	struct alg_sock *ask = alg_sk(sk);
    993	struct af_alg_ctx *ctx = ask->private;
    994	struct af_alg_tsgl *sgl;
    995	int err = -EINVAL;
    996
    997	if (flags & MSG_SENDPAGE_NOTLAST)
    998		flags |= MSG_MORE;
    999
   1000	lock_sock(sk);
   1001	if (!ctx->more && ctx->used)
   1002		goto unlock;
   1003
   1004	if (!size)
   1005		goto done;
   1006
   1007	if (!af_alg_writable(sk)) {
   1008		err = af_alg_wait_for_wmem(sk, flags);
   1009		if (err)
   1010			goto unlock;
   1011	}
   1012
   1013	err = af_alg_alloc_tsgl(sk);
   1014	if (err)
   1015		goto unlock;
   1016
   1017	ctx->merge = 0;
   1018	sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list);
   1019
   1020	if (sgl->cur)
   1021		sg_unmark_end(sgl->sg + sgl->cur - 1);
   1022
   1023	sg_mark_end(sgl->sg + sgl->cur);
   1024
   1025	get_page(page);
   1026	sg_set_page(sgl->sg + sgl->cur, page, size, offset);
   1027	sgl->cur++;
   1028	ctx->used += size;
   1029
   1030done:
   1031	ctx->more = flags & MSG_MORE;
   1032
   1033unlock:
   1034	af_alg_data_wakeup(sk);
   1035	release_sock(sk);
   1036
   1037	return err ?: size;
   1038}
   1039EXPORT_SYMBOL_GPL(af_alg_sendpage);
   1040
   1041/**
   1042 * af_alg_free_resources - release resources required for crypto request
   1043 * @areq: Request holding the TX and RX SGL
   1044 */
   1045void af_alg_free_resources(struct af_alg_async_req *areq)
   1046{
   1047	struct sock *sk = areq->sk;
   1048
   1049	af_alg_free_areq_sgls(areq);
   1050	sock_kfree_s(sk, areq, areq->areqlen);
   1051}
   1052EXPORT_SYMBOL_GPL(af_alg_free_resources);
   1053
   1054/**
   1055 * af_alg_async_cb - AIO callback handler
   1056 * @_req: async request info
   1057 * @err: if non-zero, error result to be returned via ki_complete();
   1058 *       otherwise return the AIO output length via ki_complete().
   1059 *
   1060 * This handler cleans up the struct af_alg_async_req upon completion of the
   1061 * AIO operation.
   1062 *
   1063 * The number of bytes to be generated with the AIO operation must be set
   1064 * in areq->outlen before the AIO callback handler is invoked.
   1065 */
   1066void af_alg_async_cb(struct crypto_async_request *_req, int err)
   1067{
   1068	struct af_alg_async_req *areq = _req->data;
   1069	struct sock *sk = areq->sk;
   1070	struct kiocb *iocb = areq->iocb;
   1071	unsigned int resultlen;
   1072
   1073	/* Buffer size written by crypto operation. */
   1074	resultlen = areq->outlen;
   1075
   1076	af_alg_free_resources(areq);
   1077	sock_put(sk);
   1078
   1079	iocb->ki_complete(iocb, err ? err : (int)resultlen);
   1080}
   1081EXPORT_SYMBOL_GPL(af_alg_async_cb);
   1082
   1083/**
   1084 * af_alg_poll - poll system call handler
   1085 * @file: file pointer
   1086 * @sock: socket to poll
   1087 * @wait: poll_table
   1088 */
   1089__poll_t af_alg_poll(struct file *file, struct socket *sock,
   1090			 poll_table *wait)
   1091{
   1092	struct sock *sk = sock->sk;
   1093	struct alg_sock *ask = alg_sk(sk);
   1094	struct af_alg_ctx *ctx = ask->private;
   1095	__poll_t mask;
   1096
   1097	sock_poll_wait(file, sock, wait);
   1098	mask = 0;
   1099
   1100	if (!ctx->more || ctx->used)
   1101		mask |= EPOLLIN | EPOLLRDNORM;
   1102
   1103	if (af_alg_writable(sk))
   1104		mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
   1105
   1106	return mask;
   1107}
   1108EXPORT_SYMBOL_GPL(af_alg_poll);
   1109
   1110/**
   1111 * af_alg_alloc_areq - allocate struct af_alg_async_req
   1112 *
   1113 * @sk: socket of connection to user space
   1114 * @areqlen: size of struct af_alg_async_req + crypto_*_reqsize
   1115 * Return: allocated data structure or ERR_PTR upon error
   1116 */
   1117struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk,
   1118					   unsigned int areqlen)
   1119{
   1120	struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL);
   1121
   1122	if (unlikely(!areq))
   1123		return ERR_PTR(-ENOMEM);
   1124
   1125	areq->areqlen = areqlen;
   1126	areq->sk = sk;
   1127	areq->last_rsgl = NULL;
   1128	INIT_LIST_HEAD(&areq->rsgl_list);
   1129	areq->tsgl = NULL;
   1130	areq->tsgl_entries = 0;
   1131
   1132	return areq;
   1133}
   1134EXPORT_SYMBOL_GPL(af_alg_alloc_areq);
   1135
   1136/**
   1137 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto
   1138 *		     operation
   1139 *
   1140 * @sk: socket of connection to user space
   1141 * @msg: user space message
   1142 * @flags: flags used to invoke recvmsg with
   1143 * @areq: instance of the cryptographic request that will hold the RX SGL
   1144 * @maxsize: maximum number of bytes to be pulled from user space
   1145 * @outlen: number of bytes in the RX SGL
   1146 * Return: 0 on success, < 0 upon error
   1147 */
   1148int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags,
   1149		    struct af_alg_async_req *areq, size_t maxsize,
   1150		    size_t *outlen)
   1151{
   1152	struct alg_sock *ask = alg_sk(sk);
   1153	struct af_alg_ctx *ctx = ask->private;
   1154	size_t len = 0;
   1155
   1156	while (maxsize > len && msg_data_left(msg)) {
   1157		struct af_alg_rsgl *rsgl;
   1158		size_t seglen;
   1159		int err;
   1160
   1161		/* limit the amount of readable buffers */
   1162		if (!af_alg_readable(sk))
   1163			break;
   1164
   1165		seglen = min_t(size_t, (maxsize - len),
   1166			       msg_data_left(msg));
   1167
   1168		if (list_empty(&areq->rsgl_list)) {
   1169			rsgl = &areq->first_rsgl;
   1170		} else {
   1171			rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
   1172			if (unlikely(!rsgl))
   1173				return -ENOMEM;
   1174		}
   1175
   1176		rsgl->sgl.npages = 0;
   1177		list_add_tail(&rsgl->list, &areq->rsgl_list);
   1178
   1179		/* make one iovec available as scatterlist */
   1180		err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
   1181		if (err < 0) {
   1182			rsgl->sg_num_bytes = 0;
   1183			return err;
   1184		}
   1185
   1186		/* chain the new scatterlist with previous one */
   1187		if (areq->last_rsgl)
   1188			af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl);
   1189
   1190		areq->last_rsgl = rsgl;
   1191		len += err;
   1192		atomic_add(err, &ctx->rcvused);
   1193		rsgl->sg_num_bytes = err;
   1194		iov_iter_advance(&msg->msg_iter, err);
   1195	}
   1196
   1197	*outlen = len;
   1198	return 0;
   1199}
   1200EXPORT_SYMBOL_GPL(af_alg_get_rsgl);
   1201
   1202static int __init af_alg_init(void)
   1203{
   1204	int err = proto_register(&alg_proto, 0);
   1205
   1206	if (err)
   1207		goto out;
   1208
   1209	err = sock_register(&alg_family);
   1210	if (err != 0)
   1211		goto out_unregister_proto;
   1212
   1213out:
   1214	return err;
   1215
   1216out_unregister_proto:
   1217	proto_unregister(&alg_proto);
   1218	goto out;
   1219}
   1220
   1221static void __exit af_alg_exit(void)
   1222{
   1223	sock_unregister(PF_ALG);
   1224	proto_unregister(&alg_proto);
   1225}
   1226
   1227module_init(af_alg_init);
   1228module_exit(af_alg_exit);
   1229MODULE_LICENSE("GPL");
   1230MODULE_ALIAS_NETPROTO(AF_ALG);