cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

nft_bitwise.c (13359B)


      1// SPDX-License-Identifier: GPL-2.0-only
      2/*
      3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
      4 *
      5 * Development of this code funded by Astaro AG (http://www.astaro.com/)
      6 */
      7
      8#include <linux/kernel.h>
      9#include <linux/init.h>
     10#include <linux/module.h>
     11#include <linux/netlink.h>
     12#include <linux/netfilter.h>
     13#include <linux/netfilter/nf_tables.h>
     14#include <net/netfilter/nf_tables_core.h>
     15#include <net/netfilter/nf_tables.h>
     16#include <net/netfilter/nf_tables_offload.h>
     17
     18struct nft_bitwise {
     19	u8			sreg;
     20	u8			dreg;
     21	enum nft_bitwise_ops	op:8;
     22	u8			len;
     23	struct nft_data		mask;
     24	struct nft_data		xor;
     25	struct nft_data		data;
     26};
     27
     28static void nft_bitwise_eval_bool(u32 *dst, const u32 *src,
     29				  const struct nft_bitwise *priv)
     30{
     31	unsigned int i;
     32
     33	for (i = 0; i < DIV_ROUND_UP(priv->len, sizeof(u32)); i++)
     34		dst[i] = (src[i] & priv->mask.data[i]) ^ priv->xor.data[i];
     35}
     36
     37static void nft_bitwise_eval_lshift(u32 *dst, const u32 *src,
     38				    const struct nft_bitwise *priv)
     39{
     40	u32 shift = priv->data.data[0];
     41	unsigned int i;
     42	u32 carry = 0;
     43
     44	for (i = DIV_ROUND_UP(priv->len, sizeof(u32)); i > 0; i--) {
     45		dst[i - 1] = (src[i - 1] << shift) | carry;
     46		carry = src[i - 1] >> (BITS_PER_TYPE(u32) - shift);
     47	}
     48}
     49
     50static void nft_bitwise_eval_rshift(u32 *dst, const u32 *src,
     51				    const struct nft_bitwise *priv)
     52{
     53	u32 shift = priv->data.data[0];
     54	unsigned int i;
     55	u32 carry = 0;
     56
     57	for (i = 0; i < DIV_ROUND_UP(priv->len, sizeof(u32)); i++) {
     58		dst[i] = carry | (src[i] >> shift);
     59		carry = src[i] << (BITS_PER_TYPE(u32) - shift);
     60	}
     61}
     62
     63void nft_bitwise_eval(const struct nft_expr *expr,
     64		      struct nft_regs *regs, const struct nft_pktinfo *pkt)
     65{
     66	const struct nft_bitwise *priv = nft_expr_priv(expr);
     67	const u32 *src = &regs->data[priv->sreg];
     68	u32 *dst = &regs->data[priv->dreg];
     69
     70	switch (priv->op) {
     71	case NFT_BITWISE_BOOL:
     72		nft_bitwise_eval_bool(dst, src, priv);
     73		break;
     74	case NFT_BITWISE_LSHIFT:
     75		nft_bitwise_eval_lshift(dst, src, priv);
     76		break;
     77	case NFT_BITWISE_RSHIFT:
     78		nft_bitwise_eval_rshift(dst, src, priv);
     79		break;
     80	}
     81}
     82
     83static const struct nla_policy nft_bitwise_policy[NFTA_BITWISE_MAX + 1] = {
     84	[NFTA_BITWISE_SREG]	= { .type = NLA_U32 },
     85	[NFTA_BITWISE_DREG]	= { .type = NLA_U32 },
     86	[NFTA_BITWISE_LEN]	= { .type = NLA_U32 },
     87	[NFTA_BITWISE_MASK]	= { .type = NLA_NESTED },
     88	[NFTA_BITWISE_XOR]	= { .type = NLA_NESTED },
     89	[NFTA_BITWISE_OP]	= { .type = NLA_U32 },
     90	[NFTA_BITWISE_DATA]	= { .type = NLA_NESTED },
     91};
     92
     93static int nft_bitwise_init_bool(struct nft_bitwise *priv,
     94				 const struct nlattr *const tb[])
     95{
     96	struct nft_data_desc mask, xor;
     97	int err;
     98
     99	if (tb[NFTA_BITWISE_DATA])
    100		return -EINVAL;
    101
    102	if (!tb[NFTA_BITWISE_MASK] ||
    103	    !tb[NFTA_BITWISE_XOR])
    104		return -EINVAL;
    105
    106	err = nft_data_init(NULL, &priv->mask, sizeof(priv->mask), &mask,
    107			    tb[NFTA_BITWISE_MASK]);
    108	if (err < 0)
    109		return err;
    110	if (mask.type != NFT_DATA_VALUE || mask.len != priv->len) {
    111		err = -EINVAL;
    112		goto err_mask_release;
    113	}
    114
    115	err = nft_data_init(NULL, &priv->xor, sizeof(priv->xor), &xor,
    116			    tb[NFTA_BITWISE_XOR]);
    117	if (err < 0)
    118		goto err_mask_release;
    119	if (xor.type != NFT_DATA_VALUE || xor.len != priv->len) {
    120		err = -EINVAL;
    121		goto err_xor_release;
    122	}
    123
    124	return 0;
    125
    126err_xor_release:
    127	nft_data_release(&priv->xor, xor.type);
    128err_mask_release:
    129	nft_data_release(&priv->mask, mask.type);
    130	return err;
    131}
    132
    133static int nft_bitwise_init_shift(struct nft_bitwise *priv,
    134				  const struct nlattr *const tb[])
    135{
    136	struct nft_data_desc d;
    137	int err;
    138
    139	if (tb[NFTA_BITWISE_MASK] ||
    140	    tb[NFTA_BITWISE_XOR])
    141		return -EINVAL;
    142
    143	if (!tb[NFTA_BITWISE_DATA])
    144		return -EINVAL;
    145
    146	err = nft_data_init(NULL, &priv->data, sizeof(priv->data), &d,
    147			    tb[NFTA_BITWISE_DATA]);
    148	if (err < 0)
    149		return err;
    150	if (d.type != NFT_DATA_VALUE || d.len != sizeof(u32) ||
    151	    priv->data.data[0] >= BITS_PER_TYPE(u32)) {
    152		nft_data_release(&priv->data, d.type);
    153		return -EINVAL;
    154	}
    155
    156	return 0;
    157}
    158
    159static int nft_bitwise_init(const struct nft_ctx *ctx,
    160			    const struct nft_expr *expr,
    161			    const struct nlattr * const tb[])
    162{
    163	struct nft_bitwise *priv = nft_expr_priv(expr);
    164	u32 len;
    165	int err;
    166
    167	err = nft_parse_u32_check(tb[NFTA_BITWISE_LEN], U8_MAX, &len);
    168	if (err < 0)
    169		return err;
    170
    171	priv->len = len;
    172
    173	err = nft_parse_register_load(tb[NFTA_BITWISE_SREG], &priv->sreg,
    174				      priv->len);
    175	if (err < 0)
    176		return err;
    177
    178	err = nft_parse_register_store(ctx, tb[NFTA_BITWISE_DREG],
    179				       &priv->dreg, NULL, NFT_DATA_VALUE,
    180				       priv->len);
    181	if (err < 0)
    182		return err;
    183
    184	if (tb[NFTA_BITWISE_OP]) {
    185		priv->op = ntohl(nla_get_be32(tb[NFTA_BITWISE_OP]));
    186		switch (priv->op) {
    187		case NFT_BITWISE_BOOL:
    188		case NFT_BITWISE_LSHIFT:
    189		case NFT_BITWISE_RSHIFT:
    190			break;
    191		default:
    192			return -EOPNOTSUPP;
    193		}
    194	} else {
    195		priv->op = NFT_BITWISE_BOOL;
    196	}
    197
    198	switch(priv->op) {
    199	case NFT_BITWISE_BOOL:
    200		err = nft_bitwise_init_bool(priv, tb);
    201		break;
    202	case NFT_BITWISE_LSHIFT:
    203	case NFT_BITWISE_RSHIFT:
    204		err = nft_bitwise_init_shift(priv, tb);
    205		break;
    206	}
    207
    208	return err;
    209}
    210
    211static int nft_bitwise_dump_bool(struct sk_buff *skb,
    212				 const struct nft_bitwise *priv)
    213{
    214	if (nft_data_dump(skb, NFTA_BITWISE_MASK, &priv->mask,
    215			  NFT_DATA_VALUE, priv->len) < 0)
    216		return -1;
    217
    218	if (nft_data_dump(skb, NFTA_BITWISE_XOR, &priv->xor,
    219			  NFT_DATA_VALUE, priv->len) < 0)
    220		return -1;
    221
    222	return 0;
    223}
    224
    225static int nft_bitwise_dump_shift(struct sk_buff *skb,
    226				  const struct nft_bitwise *priv)
    227{
    228	if (nft_data_dump(skb, NFTA_BITWISE_DATA, &priv->data,
    229			  NFT_DATA_VALUE, sizeof(u32)) < 0)
    230		return -1;
    231	return 0;
    232}
    233
    234static int nft_bitwise_dump(struct sk_buff *skb, const struct nft_expr *expr)
    235{
    236	const struct nft_bitwise *priv = nft_expr_priv(expr);
    237	int err = 0;
    238
    239	if (nft_dump_register(skb, NFTA_BITWISE_SREG, priv->sreg))
    240		return -1;
    241	if (nft_dump_register(skb, NFTA_BITWISE_DREG, priv->dreg))
    242		return -1;
    243	if (nla_put_be32(skb, NFTA_BITWISE_LEN, htonl(priv->len)))
    244		return -1;
    245	if (nla_put_be32(skb, NFTA_BITWISE_OP, htonl(priv->op)))
    246		return -1;
    247
    248	switch (priv->op) {
    249	case NFT_BITWISE_BOOL:
    250		err = nft_bitwise_dump_bool(skb, priv);
    251		break;
    252	case NFT_BITWISE_LSHIFT:
    253	case NFT_BITWISE_RSHIFT:
    254		err = nft_bitwise_dump_shift(skb, priv);
    255		break;
    256	}
    257
    258	return err;
    259}
    260
    261static struct nft_data zero;
    262
    263static int nft_bitwise_offload(struct nft_offload_ctx *ctx,
    264			       struct nft_flow_rule *flow,
    265			       const struct nft_expr *expr)
    266{
    267	const struct nft_bitwise *priv = nft_expr_priv(expr);
    268	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
    269
    270	if (priv->op != NFT_BITWISE_BOOL)
    271		return -EOPNOTSUPP;
    272
    273	if (memcmp(&priv->xor, &zero, sizeof(priv->xor)) ||
    274	    priv->sreg != priv->dreg || priv->len != reg->len)
    275		return -EOPNOTSUPP;
    276
    277	memcpy(&reg->mask, &priv->mask, sizeof(priv->mask));
    278
    279	return 0;
    280}
    281
    282static bool nft_bitwise_reduce(struct nft_regs_track *track,
    283			       const struct nft_expr *expr)
    284{
    285	const struct nft_bitwise *priv = nft_expr_priv(expr);
    286	const struct nft_bitwise *bitwise;
    287	unsigned int regcount;
    288	u8 dreg;
    289	int i;
    290
    291	if (!track->regs[priv->sreg].selector)
    292		return false;
    293
    294	bitwise = nft_expr_priv(track->regs[priv->dreg].selector);
    295	if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector &&
    296	    track->regs[priv->sreg].num_reg == 0 &&
    297	    track->regs[priv->dreg].bitwise &&
    298	    track->regs[priv->dreg].bitwise->ops == expr->ops &&
    299	    priv->sreg == bitwise->sreg &&
    300	    priv->dreg == bitwise->dreg &&
    301	    priv->op == bitwise->op &&
    302	    priv->len == bitwise->len &&
    303	    !memcmp(&priv->mask, &bitwise->mask, sizeof(priv->mask)) &&
    304	    !memcmp(&priv->xor, &bitwise->xor, sizeof(priv->xor)) &&
    305	    !memcmp(&priv->data, &bitwise->data, sizeof(priv->data))) {
    306		track->cur = expr;
    307		return true;
    308	}
    309
    310	if (track->regs[priv->sreg].bitwise ||
    311	    track->regs[priv->sreg].num_reg != 0) {
    312		nft_reg_track_cancel(track, priv->dreg, priv->len);
    313		return false;
    314	}
    315
    316	if (priv->sreg != priv->dreg) {
    317		nft_reg_track_update(track, track->regs[priv->sreg].selector,
    318				     priv->dreg, priv->len);
    319	}
    320
    321	dreg = priv->dreg;
    322	regcount = DIV_ROUND_UP(priv->len, NFT_REG32_SIZE);
    323	for (i = 0; i < regcount; i++, dreg++)
    324		track->regs[priv->dreg].bitwise = expr;
    325
    326	return false;
    327}
    328
    329static const struct nft_expr_ops nft_bitwise_ops = {
    330	.type		= &nft_bitwise_type,
    331	.size		= NFT_EXPR_SIZE(sizeof(struct nft_bitwise)),
    332	.eval		= nft_bitwise_eval,
    333	.init		= nft_bitwise_init,
    334	.dump		= nft_bitwise_dump,
    335	.reduce		= nft_bitwise_reduce,
    336	.offload	= nft_bitwise_offload,
    337};
    338
    339static int
    340nft_bitwise_extract_u32_data(const struct nlattr * const tb, u32 *out)
    341{
    342	struct nft_data_desc desc;
    343	struct nft_data data;
    344	int err = 0;
    345
    346	err = nft_data_init(NULL, &data, sizeof(data), &desc, tb);
    347	if (err < 0)
    348		return err;
    349
    350	if (desc.type != NFT_DATA_VALUE || desc.len != sizeof(u32)) {
    351		err = -EINVAL;
    352		goto err;
    353	}
    354	*out = data.data[0];
    355err:
    356	nft_data_release(&data, desc.type);
    357	return err;
    358}
    359
    360static int nft_bitwise_fast_init(const struct nft_ctx *ctx,
    361				 const struct nft_expr *expr,
    362				 const struct nlattr * const tb[])
    363{
    364	struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
    365	int err;
    366
    367	err = nft_parse_register_load(tb[NFTA_BITWISE_SREG], &priv->sreg,
    368				      sizeof(u32));
    369	if (err < 0)
    370		return err;
    371
    372	err = nft_parse_register_store(ctx, tb[NFTA_BITWISE_DREG], &priv->dreg,
    373				       NULL, NFT_DATA_VALUE, sizeof(u32));
    374	if (err < 0)
    375		return err;
    376
    377	if (tb[NFTA_BITWISE_DATA])
    378		return -EINVAL;
    379
    380	if (!tb[NFTA_BITWISE_MASK] ||
    381	    !tb[NFTA_BITWISE_XOR])
    382		return -EINVAL;
    383
    384	err = nft_bitwise_extract_u32_data(tb[NFTA_BITWISE_MASK], &priv->mask);
    385	if (err < 0)
    386		return err;
    387
    388	err = nft_bitwise_extract_u32_data(tb[NFTA_BITWISE_XOR], &priv->xor);
    389	if (err < 0)
    390		return err;
    391
    392	return 0;
    393}
    394
    395static int
    396nft_bitwise_fast_dump(struct sk_buff *skb, const struct nft_expr *expr)
    397{
    398	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
    399	struct nft_data data;
    400
    401	if (nft_dump_register(skb, NFTA_BITWISE_SREG, priv->sreg))
    402		return -1;
    403	if (nft_dump_register(skb, NFTA_BITWISE_DREG, priv->dreg))
    404		return -1;
    405	if (nla_put_be32(skb, NFTA_BITWISE_LEN, htonl(sizeof(u32))))
    406		return -1;
    407	if (nla_put_be32(skb, NFTA_BITWISE_OP, htonl(NFT_BITWISE_BOOL)))
    408		return -1;
    409
    410	data.data[0] = priv->mask;
    411	if (nft_data_dump(skb, NFTA_BITWISE_MASK, &data,
    412			  NFT_DATA_VALUE, sizeof(u32)) < 0)
    413		return -1;
    414
    415	data.data[0] = priv->xor;
    416	if (nft_data_dump(skb, NFTA_BITWISE_XOR, &data,
    417			  NFT_DATA_VALUE, sizeof(u32)) < 0)
    418		return -1;
    419
    420	return 0;
    421}
    422
    423static int nft_bitwise_fast_offload(struct nft_offload_ctx *ctx,
    424				    struct nft_flow_rule *flow,
    425				    const struct nft_expr *expr)
    426{
    427	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
    428	struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
    429
    430	if (priv->xor || priv->sreg != priv->dreg || reg->len != sizeof(u32))
    431		return -EOPNOTSUPP;
    432
    433	reg->mask.data[0] = priv->mask;
    434	return 0;
    435}
    436
    437static bool nft_bitwise_fast_reduce(struct nft_regs_track *track,
    438				    const struct nft_expr *expr)
    439{
    440	const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
    441	const struct nft_bitwise_fast_expr *bitwise;
    442
    443	if (!track->regs[priv->sreg].selector)
    444		return false;
    445
    446	bitwise = nft_expr_priv(track->regs[priv->dreg].selector);
    447	if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector &&
    448	    track->regs[priv->dreg].bitwise &&
    449	    track->regs[priv->dreg].bitwise->ops == expr->ops &&
    450	    priv->sreg == bitwise->sreg &&
    451	    priv->dreg == bitwise->dreg &&
    452	    priv->mask == bitwise->mask &&
    453	    priv->xor == bitwise->xor) {
    454		track->cur = expr;
    455		return true;
    456	}
    457
    458	if (track->regs[priv->sreg].bitwise) {
    459		nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE);
    460		return false;
    461	}
    462
    463	if (priv->sreg != priv->dreg) {
    464		track->regs[priv->dreg].selector =
    465			track->regs[priv->sreg].selector;
    466	}
    467	track->regs[priv->dreg].bitwise = expr;
    468
    469	return false;
    470}
    471
    472const struct nft_expr_ops nft_bitwise_fast_ops = {
    473	.type		= &nft_bitwise_type,
    474	.size		= NFT_EXPR_SIZE(sizeof(struct nft_bitwise_fast_expr)),
    475	.eval		= NULL, /* inlined */
    476	.init		= nft_bitwise_fast_init,
    477	.dump		= nft_bitwise_fast_dump,
    478	.reduce		= nft_bitwise_fast_reduce,
    479	.offload	= nft_bitwise_fast_offload,
    480};
    481
    482static const struct nft_expr_ops *
    483nft_bitwise_select_ops(const struct nft_ctx *ctx,
    484		       const struct nlattr * const tb[])
    485{
    486	int err;
    487	u32 len;
    488
    489	if (!tb[NFTA_BITWISE_LEN] ||
    490	    !tb[NFTA_BITWISE_SREG] ||
    491	    !tb[NFTA_BITWISE_DREG])
    492		return ERR_PTR(-EINVAL);
    493
    494	err = nft_parse_u32_check(tb[NFTA_BITWISE_LEN], U8_MAX, &len);
    495	if (err < 0)
    496		return ERR_PTR(err);
    497
    498	if (len != sizeof(u32))
    499		return &nft_bitwise_ops;
    500
    501	if (tb[NFTA_BITWISE_OP] &&
    502	    ntohl(nla_get_be32(tb[NFTA_BITWISE_OP])) != NFT_BITWISE_BOOL)
    503		return &nft_bitwise_ops;
    504
    505	return &nft_bitwise_fast_ops;
    506}
    507
    508struct nft_expr_type nft_bitwise_type __read_mostly = {
    509	.name		= "bitwise",
    510	.select_ops	= nft_bitwise_select_ops,
    511	.policy		= nft_bitwise_policy,
    512	.maxattr	= NFTA_BITWISE_MAX,
    513	.owner		= THIS_MODULE,
    514};
    515
    516bool nft_expr_reduce_bitwise(struct nft_regs_track *track,
    517			     const struct nft_expr *expr)
    518{
    519	const struct nft_expr *last = track->last;
    520	const struct nft_expr *next;
    521
    522	if (expr == last)
    523		return false;
    524
    525	next = nft_expr_next(expr);
    526	if (next->ops == &nft_bitwise_ops)
    527		return nft_bitwise_reduce(track, next);
    528	else if (next->ops == &nft_bitwise_fast_ops)
    529		return nft_bitwise_fast_reduce(track, next);
    530
    531	return false;
    532}
    533EXPORT_SYMBOL_GPL(nft_expr_reduce_bitwise);