cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

jit.c (128084B)


      1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
      2/* Copyright (C) 2016-2018 Netronome Systems, Inc. */
      3
      4#define pr_fmt(fmt)	"NFP net bpf: " fmt
      5
      6#include <linux/bug.h>
      7#include <linux/bpf.h>
      8#include <linux/filter.h>
      9#include <linux/kernel.h>
     10#include <linux/pkt_cls.h>
     11#include <linux/reciprocal_div.h>
     12#include <linux/unistd.h>
     13
     14#include "main.h"
     15#include "../nfp_asm.h"
     16#include "../nfp_net_ctrl.h"
     17
     18/* --- NFP prog --- */
     19/* Foreach "multiple" entries macros provide pos and next<n> pointers.
     20 * It's safe to modify the next pointers (but not pos).
     21 */
     22#define nfp_for_each_insn_walk2(nfp_prog, pos, next)			\
     23	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
     24	     next = list_next_entry(pos, l);			\
     25	     &(nfp_prog)->insns != &pos->l &&			\
     26	     &(nfp_prog)->insns != &next->l;			\
     27	     pos = nfp_meta_next(pos),				\
     28	     next = nfp_meta_next(pos))
     29
     30#define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2)		\
     31	for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
     32	     next = list_next_entry(pos, l),			\
     33	     next2 = list_next_entry(next, l);			\
     34	     &(nfp_prog)->insns != &pos->l &&			\
     35	     &(nfp_prog)->insns != &next->l &&			\
     36	     &(nfp_prog)->insns != &next2->l;			\
     37	     pos = nfp_meta_next(pos),				\
     38	     next = nfp_meta_next(pos),				\
     39	     next2 = nfp_meta_next(next))
     40
     41static bool
     42nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
     43{
     44	return meta->l.prev != &nfp_prog->insns;
     45}
     46
     47static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
     48{
     49	if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) {
     50		pr_warn("instruction limit reached (%u NFP instructions)\n",
     51			nfp_prog->prog_len);
     52		nfp_prog->error = -ENOSPC;
     53		return;
     54	}
     55
     56	nfp_prog->prog[nfp_prog->prog_len] = insn;
     57	nfp_prog->prog_len++;
     58}
     59
     60static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
     61{
     62	return nfp_prog->prog_len;
     63}
     64
     65static bool
     66nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
     67{
     68	/* If there is a recorded error we may have dropped instructions;
     69	 * that doesn't have to be due to translator bug, and the translation
     70	 * will fail anyway, so just return OK.
     71	 */
     72	if (nfp_prog->error)
     73		return true;
     74	return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
     75}
     76
     77/* --- Emitters --- */
     78static void
     79__emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
     80	   u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, enum cmd_ctx_swap ctx,
     81	   bool indir)
     82{
     83	u64 insn;
     84
     85	insn =	FIELD_PREP(OP_CMD_A_SRC, areg) |
     86		FIELD_PREP(OP_CMD_CTX, ctx) |
     87		FIELD_PREP(OP_CMD_B_SRC, breg) |
     88		FIELD_PREP(OP_CMD_TOKEN, cmd_tgt_act[op].token) |
     89		FIELD_PREP(OP_CMD_XFER, xfer) |
     90		FIELD_PREP(OP_CMD_CNT, size) |
     91		FIELD_PREP(OP_CMD_SIG, ctx != CMD_CTX_NO_SWAP) |
     92		FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) |
     93		FIELD_PREP(OP_CMD_INDIR, indir) |
     94		FIELD_PREP(OP_CMD_MODE, mode);
     95
     96	nfp_prog_push(nfp_prog, insn);
     97}
     98
     99static void
    100emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
    101	     swreg lreg, swreg rreg, u8 size, enum cmd_ctx_swap ctx, bool indir)
    102{
    103	struct nfp_insn_re_regs reg;
    104	int err;
    105
    106	err = swreg_to_restricted(reg_none(), lreg, rreg, &reg, false);
    107	if (err) {
    108		nfp_prog->error = err;
    109		return;
    110	}
    111	if (reg.swap) {
    112		pr_err("cmd can't swap arguments\n");
    113		nfp_prog->error = -EFAULT;
    114		return;
    115	}
    116	if (reg.dst_lmextn || reg.src_lmextn) {
    117		pr_err("cmd can't use LMextn\n");
    118		nfp_prog->error = -EFAULT;
    119		return;
    120	}
    121
    122	__emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, ctx,
    123		   indir);
    124}
    125
    126static void
    127emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
    128	 swreg lreg, swreg rreg, u8 size, enum cmd_ctx_swap ctx)
    129{
    130	emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, false);
    131}
    132
    133static void
    134emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
    135	       swreg lreg, swreg rreg, u8 size, enum cmd_ctx_swap ctx)
    136{
    137	emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, true);
    138}
    139
    140static void
    141__emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
    142	  enum br_ctx_signal_state css, u16 addr, u8 defer)
    143{
    144	u16 addr_lo, addr_hi;
    145	u64 insn;
    146
    147	addr_lo = addr & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
    148	addr_hi = addr != addr_lo;
    149
    150	insn = OP_BR_BASE |
    151		FIELD_PREP(OP_BR_MASK, mask) |
    152		FIELD_PREP(OP_BR_EV_PIP, ev_pip) |
    153		FIELD_PREP(OP_BR_CSS, css) |
    154		FIELD_PREP(OP_BR_DEFBR, defer) |
    155		FIELD_PREP(OP_BR_ADDR_LO, addr_lo) |
    156		FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
    157
    158	nfp_prog_push(nfp_prog, insn);
    159}
    160
    161static void
    162emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,
    163	     enum nfp_relo_type relo)
    164{
    165	if (mask == BR_UNC && defer > 2) {
    166		pr_err("BUG: branch defer out of bounds %d\n", defer);
    167		nfp_prog->error = -EFAULT;
    168		return;
    169	}
    170
    171	__emit_br(nfp_prog, mask,
    172		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
    173		  BR_CSS_NONE, addr, defer);
    174
    175	nfp_prog->prog[nfp_prog->prog_len - 1] |=
    176		FIELD_PREP(OP_RELO_TYPE, relo);
    177}
    178
    179static void
    180emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
    181{
    182	emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
    183}
    184
    185static void
    186__emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer,
    187	      bool set, bool src_lmextn)
    188{
    189	u16 addr_lo, addr_hi;
    190	u64 insn;
    191
    192	addr_lo = addr & (OP_BR_BIT_ADDR_LO >> __bf_shf(OP_BR_BIT_ADDR_LO));
    193	addr_hi = addr != addr_lo;
    194
    195	insn = OP_BR_BIT_BASE |
    196		FIELD_PREP(OP_BR_BIT_A_SRC, areg) |
    197		FIELD_PREP(OP_BR_BIT_B_SRC, breg) |
    198		FIELD_PREP(OP_BR_BIT_BV, set) |
    199		FIELD_PREP(OP_BR_BIT_DEFBR, defer) |
    200		FIELD_PREP(OP_BR_BIT_ADDR_LO, addr_lo) |
    201		FIELD_PREP(OP_BR_BIT_ADDR_HI, addr_hi) |
    202		FIELD_PREP(OP_BR_BIT_SRC_LMEXTN, src_lmextn);
    203
    204	nfp_prog_push(nfp_prog, insn);
    205}
    206
    207static void
    208emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr,
    209		 u8 defer, bool set, enum nfp_relo_type relo)
    210{
    211	struct nfp_insn_re_regs reg;
    212	int err;
    213
    214	/* NOTE: The bit to test is specified as an rotation amount, such that
    215	 *	 the bit to test will be placed on the MSB of the result when
    216	 *	 doing a rotate right. For bit X, we need right rotate X + 1.
    217	 */
    218	bit += 1;
    219
    220	err = swreg_to_restricted(reg_none(), src, reg_imm(bit), &reg, false);
    221	if (err) {
    222		nfp_prog->error = err;
    223		return;
    224	}
    225
    226	__emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set,
    227		      reg.src_lmextn);
    228
    229	nfp_prog->prog[nfp_prog->prog_len - 1] |=
    230		FIELD_PREP(OP_RELO_TYPE, relo);
    231}
    232
    233static void
    234emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer)
    235{
    236	emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL);
    237}
    238
    239static void
    240__emit_br_alu(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
    241	      u8 defer, bool dst_lmextn, bool src_lmextn)
    242{
    243	u64 insn;
    244
    245	insn = OP_BR_ALU_BASE |
    246		FIELD_PREP(OP_BR_ALU_A_SRC, areg) |
    247		FIELD_PREP(OP_BR_ALU_B_SRC, breg) |
    248		FIELD_PREP(OP_BR_ALU_DEFBR, defer) |
    249		FIELD_PREP(OP_BR_ALU_IMM_HI, imm_hi) |
    250		FIELD_PREP(OP_BR_ALU_SRC_LMEXTN, src_lmextn) |
    251		FIELD_PREP(OP_BR_ALU_DST_LMEXTN, dst_lmextn);
    252
    253	nfp_prog_push(nfp_prog, insn);
    254}
    255
    256static void emit_rtn(struct nfp_prog *nfp_prog, swreg base, u8 defer)
    257{
    258	struct nfp_insn_ur_regs reg;
    259	int err;
    260
    261	err = swreg_to_unrestricted(reg_none(), base, reg_imm(0), &reg);
    262	if (err) {
    263		nfp_prog->error = err;
    264		return;
    265	}
    266
    267	__emit_br_alu(nfp_prog, reg.areg, reg.breg, 0, defer, reg.dst_lmextn,
    268		      reg.src_lmextn);
    269}
    270
    271static void
    272__emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
    273	     enum immed_width width, bool invert,
    274	     enum immed_shift shift, bool wr_both,
    275	     bool dst_lmextn, bool src_lmextn)
    276{
    277	u64 insn;
    278
    279	insn = OP_IMMED_BASE |
    280		FIELD_PREP(OP_IMMED_A_SRC, areg) |
    281		FIELD_PREP(OP_IMMED_B_SRC, breg) |
    282		FIELD_PREP(OP_IMMED_IMM, imm_hi) |
    283		FIELD_PREP(OP_IMMED_WIDTH, width) |
    284		FIELD_PREP(OP_IMMED_INV, invert) |
    285		FIELD_PREP(OP_IMMED_SHIFT, shift) |
    286		FIELD_PREP(OP_IMMED_WR_AB, wr_both) |
    287		FIELD_PREP(OP_IMMED_SRC_LMEXTN, src_lmextn) |
    288		FIELD_PREP(OP_IMMED_DST_LMEXTN, dst_lmextn);
    289
    290	nfp_prog_push(nfp_prog, insn);
    291}
    292
    293static void
    294emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm,
    295	   enum immed_width width, bool invert, enum immed_shift shift)
    296{
    297	struct nfp_insn_ur_regs reg;
    298	int err;
    299
    300	if (swreg_type(dst) == NN_REG_IMM) {
    301		nfp_prog->error = -EFAULT;
    302		return;
    303	}
    304
    305	err = swreg_to_unrestricted(dst, dst, reg_imm(imm & 0xff), &reg);
    306	if (err) {
    307		nfp_prog->error = err;
    308		return;
    309	}
    310
    311	/* Use reg.dst when destination is No-Dest. */
    312	__emit_immed(nfp_prog,
    313		     swreg_type(dst) == NN_REG_NONE ? reg.dst : reg.areg,
    314		     reg.breg, imm >> 8, width, invert, shift,
    315		     reg.wr_both, reg.dst_lmextn, reg.src_lmextn);
    316}
    317
    318static void
    319__emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
    320	   enum shf_sc sc, u8 shift,
    321	   u16 areg, enum shf_op op, u16 breg, bool i8, bool sw, bool wr_both,
    322	   bool dst_lmextn, bool src_lmextn)
    323{
    324	u64 insn;
    325
    326	if (!FIELD_FIT(OP_SHF_SHIFT, shift)) {
    327		nfp_prog->error = -EFAULT;
    328		return;
    329	}
    330
    331	/* NFP shift instruction has something special. If shift direction is
    332	 * left then shift amount of 1 to 31 is specified as 32 minus the amount
    333	 * to shift.
    334	 *
    335	 * But no need to do this for indirect shift which has shift amount be
    336	 * 0. Even after we do this subtraction, shift amount 0 will be turned
    337	 * into 32 which will eventually be encoded the same as 0 because only
    338	 * low 5 bits are encoded, but shift amount be 32 will fail the
    339	 * FIELD_PREP check done later on shift mask (0x1f), due to 32 is out of
    340	 * mask range.
    341	 */
    342	if (sc == SHF_SC_L_SHF && shift)
    343		shift = 32 - shift;
    344
    345	insn = OP_SHF_BASE |
    346		FIELD_PREP(OP_SHF_A_SRC, areg) |
    347		FIELD_PREP(OP_SHF_SC, sc) |
    348		FIELD_PREP(OP_SHF_B_SRC, breg) |
    349		FIELD_PREP(OP_SHF_I8, i8) |
    350		FIELD_PREP(OP_SHF_SW, sw) |
    351		FIELD_PREP(OP_SHF_DST, dst) |
    352		FIELD_PREP(OP_SHF_SHIFT, shift) |
    353		FIELD_PREP(OP_SHF_OP, op) |
    354		FIELD_PREP(OP_SHF_DST_AB, dst_ab) |
    355		FIELD_PREP(OP_SHF_WR_AB, wr_both) |
    356		FIELD_PREP(OP_SHF_SRC_LMEXTN, src_lmextn) |
    357		FIELD_PREP(OP_SHF_DST_LMEXTN, dst_lmextn);
    358
    359	nfp_prog_push(nfp_prog, insn);
    360}
    361
    362static void
    363emit_shf(struct nfp_prog *nfp_prog, swreg dst,
    364	 swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc, u8 shift)
    365{
    366	struct nfp_insn_re_regs reg;
    367	int err;
    368
    369	err = swreg_to_restricted(dst, lreg, rreg, &reg, true);
    370	if (err) {
    371		nfp_prog->error = err;
    372		return;
    373	}
    374
    375	__emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift,
    376		   reg.areg, op, reg.breg, reg.i8, reg.swap, reg.wr_both,
    377		   reg.dst_lmextn, reg.src_lmextn);
    378}
    379
    380static void
    381emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst,
    382	       swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc)
    383{
    384	if (sc == SHF_SC_R_ROT) {
    385		pr_err("indirect shift is not allowed on rotation\n");
    386		nfp_prog->error = -EFAULT;
    387		return;
    388	}
    389
    390	emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0);
    391}
    392
    393static void
    394__emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
    395	   u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both,
    396	   bool dst_lmextn, bool src_lmextn)
    397{
    398	u64 insn;
    399
    400	insn = OP_ALU_BASE |
    401		FIELD_PREP(OP_ALU_A_SRC, areg) |
    402		FIELD_PREP(OP_ALU_B_SRC, breg) |
    403		FIELD_PREP(OP_ALU_DST, dst) |
    404		FIELD_PREP(OP_ALU_SW, swap) |
    405		FIELD_PREP(OP_ALU_OP, op) |
    406		FIELD_PREP(OP_ALU_DST_AB, dst_ab) |
    407		FIELD_PREP(OP_ALU_WR_AB, wr_both) |
    408		FIELD_PREP(OP_ALU_SRC_LMEXTN, src_lmextn) |
    409		FIELD_PREP(OP_ALU_DST_LMEXTN, dst_lmextn);
    410
    411	nfp_prog_push(nfp_prog, insn);
    412}
    413
    414static void
    415emit_alu(struct nfp_prog *nfp_prog, swreg dst,
    416	 swreg lreg, enum alu_op op, swreg rreg)
    417{
    418	struct nfp_insn_ur_regs reg;
    419	int err;
    420
    421	err = swreg_to_unrestricted(dst, lreg, rreg, &reg);
    422	if (err) {
    423		nfp_prog->error = err;
    424		return;
    425	}
    426
    427	__emit_alu(nfp_prog, reg.dst, reg.dst_ab,
    428		   reg.areg, op, reg.breg, reg.swap, reg.wr_both,
    429		   reg.dst_lmextn, reg.src_lmextn);
    430}
    431
    432static void
    433__emit_mul(struct nfp_prog *nfp_prog, enum alu_dst_ab dst_ab, u16 areg,
    434	   enum mul_type type, enum mul_step step, u16 breg, bool swap,
    435	   bool wr_both, bool dst_lmextn, bool src_lmextn)
    436{
    437	u64 insn;
    438
    439	insn = OP_MUL_BASE |
    440		FIELD_PREP(OP_MUL_A_SRC, areg) |
    441		FIELD_PREP(OP_MUL_B_SRC, breg) |
    442		FIELD_PREP(OP_MUL_STEP, step) |
    443		FIELD_PREP(OP_MUL_DST_AB, dst_ab) |
    444		FIELD_PREP(OP_MUL_SW, swap) |
    445		FIELD_PREP(OP_MUL_TYPE, type) |
    446		FIELD_PREP(OP_MUL_WR_AB, wr_both) |
    447		FIELD_PREP(OP_MUL_SRC_LMEXTN, src_lmextn) |
    448		FIELD_PREP(OP_MUL_DST_LMEXTN, dst_lmextn);
    449
    450	nfp_prog_push(nfp_prog, insn);
    451}
    452
    453static void
    454emit_mul(struct nfp_prog *nfp_prog, swreg lreg, enum mul_type type,
    455	 enum mul_step step, swreg rreg)
    456{
    457	struct nfp_insn_ur_regs reg;
    458	u16 areg;
    459	int err;
    460
    461	if (type == MUL_TYPE_START && step != MUL_STEP_NONE) {
    462		nfp_prog->error = -EINVAL;
    463		return;
    464	}
    465
    466	if (step == MUL_LAST || step == MUL_LAST_2) {
    467		/* When type is step and step Number is LAST or LAST2, left
    468		 * source is used as destination.
    469		 */
    470		err = swreg_to_unrestricted(lreg, reg_none(), rreg, &reg);
    471		areg = reg.dst;
    472	} else {
    473		err = swreg_to_unrestricted(reg_none(), lreg, rreg, &reg);
    474		areg = reg.areg;
    475	}
    476
    477	if (err) {
    478		nfp_prog->error = err;
    479		return;
    480	}
    481
    482	__emit_mul(nfp_prog, reg.dst_ab, areg, type, step, reg.breg, reg.swap,
    483		   reg.wr_both, reg.dst_lmextn, reg.src_lmextn);
    484}
    485
    486static void
    487__emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc,
    488		u8 areg, u8 bmask, u8 breg, u8 shift, bool imm8,
    489		bool zero, bool swap, bool wr_both,
    490		bool dst_lmextn, bool src_lmextn)
    491{
    492	u64 insn;
    493
    494	insn = OP_LDF_BASE |
    495		FIELD_PREP(OP_LDF_A_SRC, areg) |
    496		FIELD_PREP(OP_LDF_SC, sc) |
    497		FIELD_PREP(OP_LDF_B_SRC, breg) |
    498		FIELD_PREP(OP_LDF_I8, imm8) |
    499		FIELD_PREP(OP_LDF_SW, swap) |
    500		FIELD_PREP(OP_LDF_ZF, zero) |
    501		FIELD_PREP(OP_LDF_BMASK, bmask) |
    502		FIELD_PREP(OP_LDF_SHF, shift) |
    503		FIELD_PREP(OP_LDF_WR_AB, wr_both) |
    504		FIELD_PREP(OP_LDF_SRC_LMEXTN, src_lmextn) |
    505		FIELD_PREP(OP_LDF_DST_LMEXTN, dst_lmextn);
    506
    507	nfp_prog_push(nfp_prog, insn);
    508}
    509
    510static void
    511emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
    512		  enum shf_sc sc, u8 shift, bool zero)
    513{
    514	struct nfp_insn_re_regs reg;
    515	int err;
    516
    517	/* Note: ld_field is special as it uses one of the src regs as dst */
    518	err = swreg_to_restricted(dst, dst, src, &reg, true);
    519	if (err) {
    520		nfp_prog->error = err;
    521		return;
    522	}
    523
    524	__emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift,
    525			reg.i8, zero, reg.swap, reg.wr_both,
    526			reg.dst_lmextn, reg.src_lmextn);
    527}
    528
    529static void
    530emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
    531	      enum shf_sc sc, u8 shift)
    532{
    533	emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false);
    534}
    535
    536static void
    537__emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr,
    538	    bool dst_lmextn, bool src_lmextn)
    539{
    540	u64 insn;
    541
    542	insn = OP_LCSR_BASE |
    543		FIELD_PREP(OP_LCSR_A_SRC, areg) |
    544		FIELD_PREP(OP_LCSR_B_SRC, breg) |
    545		FIELD_PREP(OP_LCSR_WRITE, wr) |
    546		FIELD_PREP(OP_LCSR_ADDR, addr / 4) |
    547		FIELD_PREP(OP_LCSR_SRC_LMEXTN, src_lmextn) |
    548		FIELD_PREP(OP_LCSR_DST_LMEXTN, dst_lmextn);
    549
    550	nfp_prog_push(nfp_prog, insn);
    551}
    552
    553static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr)
    554{
    555	struct nfp_insn_ur_regs reg;
    556	int err;
    557
    558	/* This instruction takes immeds instead of reg_none() for the ignored
    559	 * operand, but we can't encode 2 immeds in one instr with our normal
    560	 * swreg infra so if param is an immed, we encode as reg_none() and
    561	 * copy the immed to both operands.
    562	 */
    563	if (swreg_type(src) == NN_REG_IMM) {
    564		err = swreg_to_unrestricted(reg_none(), src, reg_none(), &reg);
    565		reg.breg = reg.areg;
    566	} else {
    567		err = swreg_to_unrestricted(reg_none(), src, reg_imm(0), &reg);
    568	}
    569	if (err) {
    570		nfp_prog->error = err;
    571		return;
    572	}
    573
    574	__emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr,
    575		    false, reg.src_lmextn);
    576}
    577
    578/* CSR value is read in following immed[gpr, 0] */
    579static void __emit_csr_rd(struct nfp_prog *nfp_prog, u16 addr)
    580{
    581	__emit_lcsr(nfp_prog, 0, 0, false, addr, false, false);
    582}
    583
    584static void emit_nop(struct nfp_prog *nfp_prog)
    585{
    586	__emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0);
    587}
    588
    589/* --- Wrappers --- */
    590static bool pack_immed(u32 imm, u16 *val, enum immed_shift *shift)
    591{
    592	if (!(imm & 0xffff0000)) {
    593		*val = imm;
    594		*shift = IMMED_SHIFT_0B;
    595	} else if (!(imm & 0xff0000ff)) {
    596		*val = imm >> 8;
    597		*shift = IMMED_SHIFT_1B;
    598	} else if (!(imm & 0x0000ffff)) {
    599		*val = imm >> 16;
    600		*shift = IMMED_SHIFT_2B;
    601	} else {
    602		return false;
    603	}
    604
    605	return true;
    606}
    607
    608static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
    609{
    610	enum immed_shift shift;
    611	u16 val;
    612
    613	if (pack_immed(imm, &val, &shift)) {
    614		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift);
    615	} else if (pack_immed(~imm, &val, &shift)) {
    616		emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift);
    617	} else {
    618		emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL,
    619			   false, IMMED_SHIFT_0B);
    620		emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD,
    621			   false, IMMED_SHIFT_2B);
    622	}
    623}
    624
    625static void
    626wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst)
    627{
    628	if (meta->flags & FLAG_INSN_DO_ZEXT)
    629		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
    630}
    631
    632static void
    633wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
    634	       enum nfp_relo_type relo)
    635{
    636	if (imm > 0xffff) {
    637		pr_err("relocation of a large immediate!\n");
    638		nfp_prog->error = -EFAULT;
    639		return;
    640	}
    641	emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B);
    642
    643	nfp_prog->prog[nfp_prog->prog_len - 1] |=
    644		FIELD_PREP(OP_RELO_TYPE, relo);
    645}
    646
    647/* ur_load_imm_any() - encode immediate or use tmp register (unrestricted)
    648 * If the @imm is small enough encode it directly in operand and return
    649 * otherwise load @imm to a spare register and return its encoding.
    650 */
    651static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
    652{
    653	if (FIELD_FIT(UR_REG_IMM_MAX, imm))
    654		return reg_imm(imm);
    655
    656	wrp_immed(nfp_prog, tmp_reg, imm);
    657	return tmp_reg;
    658}
    659
    660/* re_load_imm_any() - encode immediate or use tmp register (restricted)
    661 * If the @imm is small enough encode it directly in operand and return
    662 * otherwise load @imm to a spare register and return its encoding.
    663 */
    664static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
    665{
    666	if (FIELD_FIT(RE_REG_IMM_MAX, imm))
    667		return reg_imm(imm);
    668
    669	wrp_immed(nfp_prog, tmp_reg, imm);
    670	return tmp_reg;
    671}
    672
    673static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
    674{
    675	while (count--)
    676		emit_nop(nfp_prog);
    677}
    678
    679static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
    680{
    681	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
    682}
    683
    684static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
    685{
    686	wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
    687}
    688
    689/* wrp_reg_subpart() - load @field_len bytes from @offset of @src, write the
    690 * result to @dst from low end.
    691 */
    692static void
    693wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len,
    694		u8 offset)
    695{
    696	enum shf_sc sc = offset ? SHF_SC_R_SHF : SHF_SC_NONE;
    697	u8 mask = (1 << field_len) - 1;
    698
    699	emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true);
    700}
    701
    702/* wrp_reg_or_subpart() - load @field_len bytes from low end of @src, or the
    703 * result to @dst from offset, there is no change on the other bits of @dst.
    704 */
    705static void
    706wrp_reg_or_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src,
    707		   u8 field_len, u8 offset)
    708{
    709	enum shf_sc sc = offset ? SHF_SC_L_SHF : SHF_SC_NONE;
    710	u8 mask = ((1 << field_len) - 1) << offset;
    711
    712	emit_ld_field(nfp_prog, dst, mask, src, sc, 32 - offset * 8);
    713}
    714
    715static void
    716addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
    717	      swreg *rega, swreg *regb)
    718{
    719	if (offset == reg_imm(0)) {
    720		*rega = reg_a(src_gpr);
    721		*regb = reg_b(src_gpr + 1);
    722		return;
    723	}
    724
    725	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset);
    726	emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C,
    727		 reg_imm(0));
    728	*rega = imm_a(nfp_prog);
    729	*regb = imm_b(nfp_prog);
    730}
    731
    732/* NFP has Command Push Pull bus which supports bluk memory operations. */
    733static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
    734{
    735	bool descending_seq = meta->ldst_gather_len < 0;
    736	s16 len = abs(meta->ldst_gather_len);
    737	swreg src_base, off;
    738	bool src_40bit_addr;
    739	unsigned int i;
    740	u8 xfer_num;
    741
    742	off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
    743	src_40bit_addr = meta->ptr.type == PTR_TO_MAP_VALUE;
    744	src_base = reg_a(meta->insn.src_reg * 2);
    745	xfer_num = round_up(len, 4) / 4;
    746
    747	if (src_40bit_addr)
    748		addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base,
    749			      &off);
    750
    751	/* Setup PREV_ALU fields to override memory read length. */
    752	if (len > 32)
    753		wrp_immed(nfp_prog, reg_none(),
    754			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
    755
    756	/* Memory read from source addr into transfer-in registers. */
    757	emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP,
    758		     src_40bit_addr ? CMD_MODE_40b_BA : CMD_MODE_32b, 0,
    759		     src_base, off, xfer_num - 1, CMD_CTX_SWAP, len > 32);
    760
    761	/* Move from transfer-in to transfer-out. */
    762	for (i = 0; i < xfer_num; i++)
    763		wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i));
    764
    765	off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog));
    766
    767	if (len <= 8) {
    768		/* Use single direct_ref write8. */
    769		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
    770			 reg_a(meta->paired_st->dst_reg * 2), off, len - 1,
    771			 CMD_CTX_SWAP);
    772	} else if (len <= 32 && IS_ALIGNED(len, 4)) {
    773		/* Use single direct_ref write32. */
    774		emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
    775			 reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1,
    776			 CMD_CTX_SWAP);
    777	} else if (len <= 32) {
    778		/* Use single indirect_ref write8. */
    779		wrp_immed(nfp_prog, reg_none(),
    780			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, len - 1));
    781		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
    782			       reg_a(meta->paired_st->dst_reg * 2), off,
    783			       len - 1, CMD_CTX_SWAP);
    784	} else if (IS_ALIGNED(len, 4)) {
    785		/* Use single indirect_ref write32. */
    786		wrp_immed(nfp_prog, reg_none(),
    787			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
    788		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
    789			       reg_a(meta->paired_st->dst_reg * 2), off,
    790			       xfer_num - 1, CMD_CTX_SWAP);
    791	} else if (len <= 40) {
    792		/* Use one direct_ref write32 to write the first 32-bytes, then
    793		 * another direct_ref write8 to write the remaining bytes.
    794		 */
    795		emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
    796			 reg_a(meta->paired_st->dst_reg * 2), off, 7,
    797			 CMD_CTX_SWAP);
    798
    799		off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32,
    800				      imm_b(nfp_prog));
    801		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8,
    802			 reg_a(meta->paired_st->dst_reg * 2), off, len - 33,
    803			 CMD_CTX_SWAP);
    804	} else {
    805		/* Use one indirect_ref write32 to write 4-bytes aligned length,
    806		 * then another direct_ref write8 to write the remaining bytes.
    807		 */
    808		u8 new_off;
    809
    810		wrp_immed(nfp_prog, reg_none(),
    811			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 2));
    812		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
    813			       reg_a(meta->paired_st->dst_reg * 2), off,
    814			       xfer_num - 2, CMD_CTX_SWAP);
    815		new_off = meta->paired_st->off + (xfer_num - 1) * 4;
    816		off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog));
    817		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b,
    818			 xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off,
    819			 (len & 0x3) - 1, CMD_CTX_SWAP);
    820	}
    821
    822	/* TODO: The following extra load is to make sure data flow be identical
    823	 *  before and after we do memory copy optimization.
    824	 *
    825	 *  The load destination register is not guaranteed to be dead, so we
    826	 *  need to make sure it is loaded with the value the same as before
    827	 *  this transformation.
    828	 *
    829	 *  These extra loads could be removed once we have accurate register
    830	 *  usage information.
    831	 */
    832	if (descending_seq)
    833		xfer_num = 0;
    834	else if (BPF_SIZE(meta->insn.code) != BPF_DW)
    835		xfer_num = xfer_num - 1;
    836	else
    837		xfer_num = xfer_num - 2;
    838
    839	switch (BPF_SIZE(meta->insn.code)) {
    840	case BPF_B:
    841		wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
    842				reg_xfer(xfer_num), 1,
    843				IS_ALIGNED(len, 4) ? 3 : (len & 3) - 1);
    844		break;
    845	case BPF_H:
    846		wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
    847				reg_xfer(xfer_num), 2, (len & 3) ^ 2);
    848		break;
    849	case BPF_W:
    850		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
    851			reg_xfer(0));
    852		break;
    853	case BPF_DW:
    854		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
    855			reg_xfer(xfer_num));
    856		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1),
    857			reg_xfer(xfer_num + 1));
    858		break;
    859	}
    860
    861	if (BPF_SIZE(meta->insn.code) != BPF_DW)
    862		wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
    863
    864	return 0;
    865}
    866
    867static int
    868data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset,
    869	u8 dst_gpr, int size)
    870{
    871	unsigned int i;
    872	u16 shift, sz;
    873
    874	/* We load the value from the address indicated in @offset and then
    875	 * shift out the data we don't need.  Note: this is big endian!
    876	 */
    877	sz = max(size, 4);
    878	shift = size < 4 ? 4 - size : 0;
    879
    880	emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
    881		 pptr_reg(nfp_prog), offset, sz - 1, CMD_CTX_SWAP);
    882
    883	i = 0;
    884	if (shift)
    885		emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE,
    886			 reg_xfer(0), SHF_SC_R_SHF, shift * 8);
    887	else
    888		for (; i * 4 < size; i++)
    889			wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
    890
    891	if (i < 2)
    892		wrp_zext(nfp_prog, meta, dst_gpr);
    893
    894	return 0;
    895}
    896
    897static int
    898data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
    899		   u8 dst_gpr, swreg lreg, swreg rreg, int size,
    900		   enum cmd_mode mode)
    901{
    902	unsigned int i;
    903	u8 mask, sz;
    904
    905	/* We load the value from the address indicated in rreg + lreg and then
    906	 * mask out the data we don't need.  Note: this is little endian!
    907	 */
    908	sz = max(size, 4);
    909	mask = size < 4 ? GENMASK(size - 1, 0) : 0;
    910
    911	emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0,
    912		 lreg, rreg, sz / 4 - 1, CMD_CTX_SWAP);
    913
    914	i = 0;
    915	if (mask)
    916		emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask,
    917				  reg_xfer(0), SHF_SC_NONE, 0, true);
    918	else
    919		for (; i * 4 < size; i++)
    920			wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
    921
    922	if (i < 2)
    923		wrp_zext(nfp_prog, meta, dst_gpr);
    924
    925	return 0;
    926}
    927
    928static int
    929data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
    930			  u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
    931{
    932	return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
    933				  offset, size, CMD_MODE_32b);
    934}
    935
    936static int
    937data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
    938			  u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
    939{
    940	swreg rega, regb;
    941
    942	addr40_offset(nfp_prog, src_gpr, offset, &rega, &regb);
    943
    944	return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
    945				  size, CMD_MODE_40b_BA);
    946}
    947
    948static int
    949construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
    950		      u16 offset, u16 src, u8 size)
    951{
    952	swreg tmp_reg;
    953
    954	/* Calculate the true offset (src_reg + imm) */
    955	tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
    956	emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg);
    957
    958	/* Check packet length (size guaranteed to fit b/c it's u8) */
    959	emit_alu(nfp_prog, imm_a(nfp_prog),
    960		 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
    961	emit_alu(nfp_prog, reg_none(),
    962		 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
    963	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
    964
    965	/* Load data */
    966	return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
    967}
    968
    969static int
    970construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
    971		  u16 offset, u8 size)
    972{
    973	swreg tmp_reg;
    974
    975	/* Check packet length */
    976	tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
    977	emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
    978	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
    979
    980	/* Load data */
    981	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
    982	return data_ld(nfp_prog, meta, tmp_reg, 0, size);
    983}
    984
    985static int
    986data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
    987		    u8 src_gpr, u8 size)
    988{
    989	unsigned int i;
    990
    991	for (i = 0; i * 4 < size; i++)
    992		wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i));
    993
    994	emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
    995		 reg_a(dst_gpr), offset, size - 1, CMD_CTX_SWAP);
    996
    997	return 0;
    998}
    999
   1000static int
   1001data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
   1002		   u64 imm, u8 size)
   1003{
   1004	wrp_immed(nfp_prog, reg_xfer(0), imm);
   1005	if (size == 8)
   1006		wrp_immed(nfp_prog, reg_xfer(1), imm >> 32);
   1007
   1008	emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
   1009		 reg_a(dst_gpr), offset, size - 1, CMD_CTX_SWAP);
   1010
   1011	return 0;
   1012}
   1013
   1014typedef int
   1015(*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
   1016	     unsigned int size, bool first, bool new_gpr, bool last, bool lm3,
   1017	     bool needs_inc);
   1018
   1019static int
   1020wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off,
   1021	      unsigned int size, bool first, bool new_gpr, bool last, bool lm3,
   1022	      bool needs_inc)
   1023{
   1024	bool should_inc = needs_inc && new_gpr && !last;
   1025	u32 idx, src_byte;
   1026	enum shf_sc sc;
   1027	swreg reg;
   1028	int shf;
   1029	u8 mask;
   1030
   1031	if (WARN_ON_ONCE(dst_byte + size > 4 || off % 4 + size > 4))
   1032		return -EOPNOTSUPP;
   1033
   1034	idx = off / 4;
   1035
   1036	/* Move the entire word */
   1037	if (size == 4) {
   1038		wrp_mov(nfp_prog, reg_both(dst),
   1039			should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx));
   1040		return 0;
   1041	}
   1042
   1043	if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX))
   1044		return -EOPNOTSUPP;
   1045
   1046	src_byte = off % 4;
   1047
   1048	mask = (1 << size) - 1;
   1049	mask <<= dst_byte;
   1050
   1051	if (WARN_ON_ONCE(mask > 0xf))
   1052		return -EOPNOTSUPP;
   1053
   1054	shf = abs(src_byte - dst_byte) * 8;
   1055	if (src_byte == dst_byte) {
   1056		sc = SHF_SC_NONE;
   1057	} else if (src_byte < dst_byte) {
   1058		shf = 32 - shf;
   1059		sc = SHF_SC_L_SHF;
   1060	} else {
   1061		sc = SHF_SC_R_SHF;
   1062	}
   1063
   1064	/* ld_field can address fewer indexes, if offset too large do RMW.
   1065	 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes.
   1066	 */
   1067	if (idx <= RE_REG_LM_IDX_MAX) {
   1068		reg = reg_lm(lm3 ? 3 : 0, idx);
   1069	} else {
   1070		reg = imm_a(nfp_prog);
   1071		/* If it's not the first part of the load and we start a new GPR
   1072		 * that means we are loading a second part of the LMEM word into
   1073		 * a new GPR.  IOW we've already looked that LMEM word and
   1074		 * therefore it has been loaded into imm_a().
   1075		 */
   1076		if (first || !new_gpr)
   1077			wrp_mov(nfp_prog, reg, reg_lm(0, idx));
   1078	}
   1079
   1080	emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr);
   1081
   1082	if (should_inc)
   1083		wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3));
   1084
   1085	return 0;
   1086}
   1087
   1088static int
   1089wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off,
   1090	       unsigned int size, bool first, bool new_gpr, bool last, bool lm3,
   1091	       bool needs_inc)
   1092{
   1093	bool should_inc = needs_inc && new_gpr && !last;
   1094	u32 idx, dst_byte;
   1095	enum shf_sc sc;
   1096	swreg reg;
   1097	int shf;
   1098	u8 mask;
   1099
   1100	if (WARN_ON_ONCE(src_byte + size > 4 || off % 4 + size > 4))
   1101		return -EOPNOTSUPP;
   1102
   1103	idx = off / 4;
   1104
   1105	/* Move the entire word */
   1106	if (size == 4) {
   1107		wrp_mov(nfp_prog,
   1108			should_inc ? reg_lm_inc(3) : reg_lm(lm3 ? 3 : 0, idx),
   1109			reg_b(src));
   1110		return 0;
   1111	}
   1112
   1113	if (WARN_ON_ONCE(lm3 && idx > RE_REG_LM_IDX_MAX))
   1114		return -EOPNOTSUPP;
   1115
   1116	dst_byte = off % 4;
   1117
   1118	mask = (1 << size) - 1;
   1119	mask <<= dst_byte;
   1120
   1121	if (WARN_ON_ONCE(mask > 0xf))
   1122		return -EOPNOTSUPP;
   1123
   1124	shf = abs(src_byte - dst_byte) * 8;
   1125	if (src_byte == dst_byte) {
   1126		sc = SHF_SC_NONE;
   1127	} else if (src_byte < dst_byte) {
   1128		shf = 32 - shf;
   1129		sc = SHF_SC_L_SHF;
   1130	} else {
   1131		sc = SHF_SC_R_SHF;
   1132	}
   1133
   1134	/* ld_field can address fewer indexes, if offset too large do RMW.
   1135	 * Because we RMV twice we waste 2 cycles on unaligned 8 byte writes.
   1136	 */
   1137	if (idx <= RE_REG_LM_IDX_MAX) {
   1138		reg = reg_lm(lm3 ? 3 : 0, idx);
   1139	} else {
   1140		reg = imm_a(nfp_prog);
   1141		/* Only first and last LMEM locations are going to need RMW,
   1142		 * the middle location will be overwritten fully.
   1143		 */
   1144		if (first || last)
   1145			wrp_mov(nfp_prog, reg, reg_lm(0, idx));
   1146	}
   1147
   1148	emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf);
   1149
   1150	if (new_gpr || last) {
   1151		if (idx > RE_REG_LM_IDX_MAX)
   1152			wrp_mov(nfp_prog, reg_lm(0, idx), reg);
   1153		if (should_inc)
   1154			wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3));
   1155	}
   1156
   1157	return 0;
   1158}
   1159
   1160static int
   1161mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1162	     unsigned int size, unsigned int ptr_off, u8 gpr, u8 ptr_gpr,
   1163	     bool clr_gpr, lmem_step step)
   1164{
   1165	s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off;
   1166	bool first = true, narrow_ld, last;
   1167	bool needs_inc = false;
   1168	swreg stack_off_reg;
   1169	u8 prev_gpr = 255;
   1170	u32 gpr_byte = 0;
   1171	bool lm3 = true;
   1172	int ret;
   1173
   1174	if (meta->ptr_not_const ||
   1175	    meta->flags & FLAG_INSN_PTR_CALLER_STACK_FRAME) {
   1176		/* Use of the last encountered ptr_off is OK, they all have
   1177		 * the same alignment.  Depend on low bits of value being
   1178		 * discarded when written to LMaddr register.
   1179		 */
   1180		stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off,
   1181						stack_imm(nfp_prog));
   1182
   1183		emit_alu(nfp_prog, imm_b(nfp_prog),
   1184			 reg_a(ptr_gpr), ALU_OP_ADD, stack_off_reg);
   1185
   1186		needs_inc = true;
   1187	} else if (off + size <= 64) {
   1188		/* We can reach bottom 64B with LMaddr0 */
   1189		lm3 = false;
   1190	} else if (round_down(off, 32) == round_down(off + size - 1, 32)) {
   1191		/* We have to set up a new pointer.  If we know the offset
   1192		 * and the entire access falls into a single 32 byte aligned
   1193		 * window we won't have to increment the LM pointer.
   1194		 * The 32 byte alignment is imporant because offset is ORed in
   1195		 * not added when doing *l$indexN[off].
   1196		 */
   1197		stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32),
   1198						stack_imm(nfp_prog));
   1199		emit_alu(nfp_prog, imm_b(nfp_prog),
   1200			 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg);
   1201
   1202		off %= 32;
   1203	} else {
   1204		stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4),
   1205						stack_imm(nfp_prog));
   1206
   1207		emit_alu(nfp_prog, imm_b(nfp_prog),
   1208			 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg);
   1209
   1210		needs_inc = true;
   1211	}
   1212
   1213	narrow_ld = clr_gpr && size < 8;
   1214
   1215	if (lm3) {
   1216		unsigned int nop_cnt;
   1217
   1218		emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3);
   1219		/* For size < 4 one slot will be filled by zeroing of upper,
   1220		 * but be careful, that zeroing could be eliminated by zext
   1221		 * optimization.
   1222		 */
   1223		nop_cnt = narrow_ld && meta->flags & FLAG_INSN_DO_ZEXT ? 2 : 3;
   1224		wrp_nops(nfp_prog, nop_cnt);
   1225	}
   1226
   1227	if (narrow_ld)
   1228		wrp_zext(nfp_prog, meta, gpr);
   1229
   1230	while (size) {
   1231		u32 slice_end;
   1232		u8 slice_size;
   1233
   1234		slice_size = min(size, 4 - gpr_byte);
   1235		slice_end = min(off + slice_size, round_up(off + 1, 4));
   1236		slice_size = slice_end - off;
   1237
   1238		last = slice_size == size;
   1239
   1240		if (needs_inc)
   1241			off %= 4;
   1242
   1243		ret = step(nfp_prog, gpr, gpr_byte, off, slice_size,
   1244			   first, gpr != prev_gpr, last, lm3, needs_inc);
   1245		if (ret)
   1246			return ret;
   1247
   1248		prev_gpr = gpr;
   1249		first = false;
   1250
   1251		gpr_byte += slice_size;
   1252		if (gpr_byte >= 4) {
   1253			gpr_byte -= 4;
   1254			gpr++;
   1255		}
   1256
   1257		size -= slice_size;
   1258		off += slice_size;
   1259	}
   1260
   1261	return 0;
   1262}
   1263
   1264static void
   1265wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
   1266{
   1267	swreg tmp_reg;
   1268
   1269	if (alu_op == ALU_OP_AND) {
   1270		if (!imm)
   1271			wrp_immed(nfp_prog, reg_both(dst), 0);
   1272		if (!imm || !~imm)
   1273			return;
   1274	}
   1275	if (alu_op == ALU_OP_OR) {
   1276		if (!~imm)
   1277			wrp_immed(nfp_prog, reg_both(dst), ~0U);
   1278		if (!imm || !~imm)
   1279			return;
   1280	}
   1281	if (alu_op == ALU_OP_XOR) {
   1282		if (!~imm)
   1283			emit_alu(nfp_prog, reg_both(dst), reg_none(),
   1284				 ALU_OP_NOT, reg_b(dst));
   1285		if (!imm || !~imm)
   1286			return;
   1287	}
   1288
   1289	tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
   1290	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg);
   1291}
   1292
   1293static int
   1294wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1295	      enum alu_op alu_op, bool skip)
   1296{
   1297	const struct bpf_insn *insn = &meta->insn;
   1298	u64 imm = insn->imm; /* sign extend */
   1299
   1300	if (skip) {
   1301		meta->flags |= FLAG_INSN_SKIP_NOOP;
   1302		return 0;
   1303	}
   1304
   1305	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U);
   1306	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32);
   1307
   1308	return 0;
   1309}
   1310
   1311static int
   1312wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1313	      enum alu_op alu_op)
   1314{
   1315	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
   1316
   1317	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
   1318	emit_alu(nfp_prog, reg_both(dst + 1),
   1319		 reg_a(dst + 1), alu_op, reg_b(src + 1));
   1320
   1321	return 0;
   1322}
   1323
   1324static int
   1325wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1326	      enum alu_op alu_op)
   1327{
   1328	const struct bpf_insn *insn = &meta->insn;
   1329	u8 dst = insn->dst_reg * 2;
   1330
   1331	wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm);
   1332	wrp_zext(nfp_prog, meta, dst);
   1333
   1334	return 0;
   1335}
   1336
   1337static int
   1338wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1339	      enum alu_op alu_op)
   1340{
   1341	u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
   1342
   1343	emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
   1344	wrp_zext(nfp_prog, meta, dst);
   1345
   1346	return 0;
   1347}
   1348
   1349static void
   1350wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src,
   1351		 enum br_mask br_mask, u16 off)
   1352{
   1353	emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src));
   1354	emit_br(nfp_prog, br_mask, off, 0);
   1355}
   1356
   1357static int
   1358wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1359	     enum alu_op alu_op, enum br_mask br_mask)
   1360{
   1361	const struct bpf_insn *insn = &meta->insn;
   1362
   1363	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
   1364			 insn->src_reg * 2, br_mask, insn->off);
   1365	if (is_mbpf_jmp64(meta))
   1366		wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
   1367				 insn->src_reg * 2 + 1, br_mask, insn->off);
   1368
   1369	return 0;
   1370}
   1371
   1372static const struct jmp_code_map {
   1373	enum br_mask br_mask;
   1374	bool swap;
   1375} jmp_code_map[] = {
   1376	[BPF_JGT >> 4]	= { BR_BLO, true },
   1377	[BPF_JGE >> 4]	= { BR_BHS, false },
   1378	[BPF_JLT >> 4]	= { BR_BLO, false },
   1379	[BPF_JLE >> 4]	= { BR_BHS, true },
   1380	[BPF_JSGT >> 4]	= { BR_BLT, true },
   1381	[BPF_JSGE >> 4]	= { BR_BGE, false },
   1382	[BPF_JSLT >> 4]	= { BR_BLT, false },
   1383	[BPF_JSLE >> 4]	= { BR_BGE, true },
   1384};
   1385
   1386static const struct jmp_code_map *nfp_jmp_code_get(struct nfp_insn_meta *meta)
   1387{
   1388	unsigned int op;
   1389
   1390	op = BPF_OP(meta->insn.code) >> 4;
   1391	/* br_mask of 0 is BR_BEQ which we don't use in jump code table */
   1392	if (WARN_ONCE(op >= ARRAY_SIZE(jmp_code_map) ||
   1393		      !jmp_code_map[op].br_mask,
   1394		      "no code found for jump instruction"))
   1395		return NULL;
   1396
   1397	return &jmp_code_map[op];
   1398}
   1399
   1400static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1401{
   1402	const struct bpf_insn *insn = &meta->insn;
   1403	u64 imm = insn->imm; /* sign extend */
   1404	const struct jmp_code_map *code;
   1405	enum alu_op alu_op, carry_op;
   1406	u8 reg = insn->dst_reg * 2;
   1407	swreg tmp_reg;
   1408
   1409	code = nfp_jmp_code_get(meta);
   1410	if (!code)
   1411		return -EINVAL;
   1412
   1413	alu_op = meta->jump_neg_op ? ALU_OP_ADD : ALU_OP_SUB;
   1414	carry_op = meta->jump_neg_op ? ALU_OP_ADD_C : ALU_OP_SUB_C;
   1415
   1416	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
   1417	if (!code->swap)
   1418		emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg);
   1419	else
   1420		emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg));
   1421
   1422	if (is_mbpf_jmp64(meta)) {
   1423		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
   1424		if (!code->swap)
   1425			emit_alu(nfp_prog, reg_none(),
   1426				 reg_a(reg + 1), carry_op, tmp_reg);
   1427		else
   1428			emit_alu(nfp_prog, reg_none(),
   1429				 tmp_reg, carry_op, reg_a(reg + 1));
   1430	}
   1431
   1432	emit_br(nfp_prog, code->br_mask, insn->off, 0);
   1433
   1434	return 0;
   1435}
   1436
   1437static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1438{
   1439	const struct bpf_insn *insn = &meta->insn;
   1440	const struct jmp_code_map *code;
   1441	u8 areg, breg;
   1442
   1443	code = nfp_jmp_code_get(meta);
   1444	if (!code)
   1445		return -EINVAL;
   1446
   1447	areg = insn->dst_reg * 2;
   1448	breg = insn->src_reg * 2;
   1449
   1450	if (code->swap) {
   1451		areg ^= breg;
   1452		breg ^= areg;
   1453		areg ^= breg;
   1454	}
   1455
   1456	emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
   1457	if (is_mbpf_jmp64(meta))
   1458		emit_alu(nfp_prog, reg_none(),
   1459			 reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1));
   1460	emit_br(nfp_prog, code->br_mask, insn->off, 0);
   1461
   1462	return 0;
   1463}
   1464
   1465static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
   1466{
   1467	emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in,
   1468		      SHF_SC_R_ROT, 8);
   1469	emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out),
   1470		      SHF_SC_R_ROT, 16);
   1471}
   1472
   1473static void
   1474wrp_mul_u32(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg,
   1475	    swreg rreg, bool gen_high_half)
   1476{
   1477	emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg);
   1478	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_1, rreg);
   1479	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_2, rreg);
   1480	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_3, rreg);
   1481	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_4, rreg);
   1482	emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_32x32, MUL_LAST, reg_none());
   1483	if (gen_high_half)
   1484		emit_mul(nfp_prog, dst_hi, MUL_TYPE_STEP_32x32, MUL_LAST_2,
   1485			 reg_none());
   1486	else
   1487		wrp_immed(nfp_prog, dst_hi, 0);
   1488}
   1489
   1490static void
   1491wrp_mul_u16(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg,
   1492	    swreg rreg)
   1493{
   1494	emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg);
   1495	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_1, rreg);
   1496	emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_2, rreg);
   1497	emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_16x16, MUL_LAST, reg_none());
   1498}
   1499
   1500static int
   1501wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   1502	bool gen_high_half, bool ropnd_from_reg)
   1503{
   1504	swreg multiplier, multiplicand, dst_hi, dst_lo;
   1505	const struct bpf_insn *insn = &meta->insn;
   1506	u32 lopnd_max, ropnd_max;
   1507	u8 dst_reg;
   1508
   1509	dst_reg = insn->dst_reg;
   1510	multiplicand = reg_a(dst_reg * 2);
   1511	dst_hi = reg_both(dst_reg * 2 + 1);
   1512	dst_lo = reg_both(dst_reg * 2);
   1513	lopnd_max = meta->umax_dst;
   1514	if (ropnd_from_reg) {
   1515		multiplier = reg_b(insn->src_reg * 2);
   1516		ropnd_max = meta->umax_src;
   1517	} else {
   1518		u32 imm = insn->imm;
   1519
   1520		multiplier = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
   1521		ropnd_max = imm;
   1522	}
   1523	if (lopnd_max > U16_MAX || ropnd_max > U16_MAX)
   1524		wrp_mul_u32(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier,
   1525			    gen_high_half);
   1526	else
   1527		wrp_mul_u16(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier);
   1528
   1529	return 0;
   1530}
   1531
   1532static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm)
   1533{
   1534	swreg dst_both = reg_both(dst), dst_a = reg_a(dst), dst_b = reg_a(dst);
   1535	struct reciprocal_value_adv rvalue;
   1536	u8 pre_shift, exp;
   1537	swreg magic;
   1538
   1539	if (imm > U32_MAX) {
   1540		wrp_immed(nfp_prog, dst_both, 0);
   1541		return 0;
   1542	}
   1543
   1544	/* NOTE: because we are using "reciprocal_value_adv" which doesn't
   1545	 * support "divisor > (1u << 31)", we need to JIT separate NFP sequence
   1546	 * to handle such case which actually equals to the result of unsigned
   1547	 * comparison "dst >= imm" which could be calculated using the following
   1548	 * NFP sequence:
   1549	 *
   1550	 *  alu[--, dst, -, imm]
   1551	 *  immed[imm, 0]
   1552	 *  alu[dst, imm, +carry, 0]
   1553	 *
   1554	 */
   1555	if (imm > 1U << 31) {
   1556		swreg tmp_b = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
   1557
   1558		emit_alu(nfp_prog, reg_none(), dst_a, ALU_OP_SUB, tmp_b);
   1559		wrp_immed(nfp_prog, imm_a(nfp_prog), 0);
   1560		emit_alu(nfp_prog, dst_both, imm_a(nfp_prog), ALU_OP_ADD_C,
   1561			 reg_imm(0));
   1562		return 0;
   1563	}
   1564
   1565	rvalue = reciprocal_value_adv(imm, 32);
   1566	exp = rvalue.exp;
   1567	if (rvalue.is_wide_m && !(imm & 1)) {
   1568		pre_shift = fls(imm & -imm) - 1;
   1569		rvalue = reciprocal_value_adv(imm >> pre_shift, 32 - pre_shift);
   1570	} else {
   1571		pre_shift = 0;
   1572	}
   1573	magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog));
   1574	if (imm == 1U << exp) {
   1575		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
   1576			 SHF_SC_R_SHF, exp);
   1577	} else if (rvalue.is_wide_m) {
   1578		wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a,
   1579			    magic, true);
   1580		emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB,
   1581			 imm_b(nfp_prog));
   1582		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
   1583			 SHF_SC_R_SHF, 1);
   1584		emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD,
   1585			 imm_b(nfp_prog));
   1586		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
   1587			 SHF_SC_R_SHF, rvalue.sh - 1);
   1588	} else {
   1589		if (pre_shift)
   1590			emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE,
   1591				 dst_b, SHF_SC_R_SHF, pre_shift);
   1592		wrp_mul_u32(nfp_prog, dst_both, reg_none(), dst_a, magic, true);
   1593		emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE,
   1594			 dst_b, SHF_SC_R_SHF, rvalue.sh);
   1595	}
   1596
   1597	return 0;
   1598}
   1599
   1600static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1601{
   1602	swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog);
   1603	struct nfp_bpf_cap_adjust_head *adjust_head;
   1604	u32 ret_einval, end;
   1605
   1606	adjust_head = &nfp_prog->bpf->adjust_head;
   1607
   1608	/* Optimized version - 5 vs 14 cycles */
   1609	if (nfp_prog->adjust_head_location != UINT_MAX) {
   1610		if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n))
   1611			return -EINVAL;
   1612
   1613		emit_alu(nfp_prog, pptr_reg(nfp_prog),
   1614			 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog));
   1615		emit_alu(nfp_prog, plen_reg(nfp_prog),
   1616			 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
   1617		emit_alu(nfp_prog, pv_len(nfp_prog),
   1618			 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
   1619
   1620		wrp_immed(nfp_prog, reg_both(0), 0);
   1621		wrp_immed(nfp_prog, reg_both(1), 0);
   1622
   1623		/* TODO: when adjust head is guaranteed to succeed we can
   1624		 * also eliminate the following if (r0 == 0) branch.
   1625		 */
   1626
   1627		return 0;
   1628	}
   1629
   1630	ret_einval = nfp_prog_current_offset(nfp_prog) + 14;
   1631	end = ret_einval + 2;
   1632
   1633	/* We need to use a temp because offset is just a part of the pkt ptr */
   1634	emit_alu(nfp_prog, tmp,
   1635		 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog));
   1636
   1637	/* Validate result will fit within FW datapath constraints */
   1638	emit_alu(nfp_prog, reg_none(),
   1639		 tmp, ALU_OP_SUB, reg_imm(adjust_head->off_min));
   1640	emit_br(nfp_prog, BR_BLO, ret_einval, 0);
   1641	emit_alu(nfp_prog, reg_none(),
   1642		 reg_imm(adjust_head->off_max), ALU_OP_SUB, tmp);
   1643	emit_br(nfp_prog, BR_BLO, ret_einval, 0);
   1644
   1645	/* Validate the length is at least ETH_HLEN */
   1646	emit_alu(nfp_prog, tmp_len,
   1647		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
   1648	emit_alu(nfp_prog, reg_none(),
   1649		 tmp_len, ALU_OP_SUB, reg_imm(ETH_HLEN));
   1650	emit_br(nfp_prog, BR_BMI, ret_einval, 0);
   1651
   1652	/* Load the ret code */
   1653	wrp_immed(nfp_prog, reg_both(0), 0);
   1654	wrp_immed(nfp_prog, reg_both(1), 0);
   1655
   1656	/* Modify the packet metadata */
   1657	emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);
   1658
   1659	/* Skip over the -EINVAL ret code (defer 2) */
   1660	emit_br(nfp_prog, BR_UNC, end, 2);
   1661
   1662	emit_alu(nfp_prog, plen_reg(nfp_prog),
   1663		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
   1664	emit_alu(nfp_prog, pv_len(nfp_prog),
   1665		 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
   1666
   1667	/* return -EINVAL target */
   1668	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
   1669		return -EINVAL;
   1670
   1671	wrp_immed(nfp_prog, reg_both(0), -22);
   1672	wrp_immed(nfp_prog, reg_both(1), ~0);
   1673
   1674	if (!nfp_prog_confirm_current_offset(nfp_prog, end))
   1675		return -EINVAL;
   1676
   1677	return 0;
   1678}
   1679
   1680static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1681{
   1682	u32 ret_einval, end;
   1683	swreg plen, delta;
   1684
   1685	BUILD_BUG_ON(plen_reg(nfp_prog) != reg_b(STATIC_REG_PKT_LEN));
   1686
   1687	plen = imm_a(nfp_prog);
   1688	delta = reg_a(2 * 2);
   1689
   1690	ret_einval = nfp_prog_current_offset(nfp_prog) + 9;
   1691	end = nfp_prog_current_offset(nfp_prog) + 11;
   1692
   1693	/* Calculate resulting length */
   1694	emit_alu(nfp_prog, plen, plen_reg(nfp_prog), ALU_OP_ADD, delta);
   1695	/* delta == 0 is not allowed by the kernel, add must overflow to make
   1696	 * length smaller.
   1697	 */
   1698	emit_br(nfp_prog, BR_BCC, ret_einval, 0);
   1699
   1700	/* if (new_len < 14) then -EINVAL */
   1701	emit_alu(nfp_prog, reg_none(), plen, ALU_OP_SUB, reg_imm(ETH_HLEN));
   1702	emit_br(nfp_prog, BR_BMI, ret_einval, 0);
   1703
   1704	emit_alu(nfp_prog, plen_reg(nfp_prog),
   1705		 plen_reg(nfp_prog), ALU_OP_ADD, delta);
   1706	emit_alu(nfp_prog, pv_len(nfp_prog),
   1707		 pv_len(nfp_prog), ALU_OP_ADD, delta);
   1708
   1709	emit_br(nfp_prog, BR_UNC, end, 2);
   1710	wrp_immed(nfp_prog, reg_both(0), 0);
   1711	wrp_immed(nfp_prog, reg_both(1), 0);
   1712
   1713	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
   1714		return -EINVAL;
   1715
   1716	wrp_immed(nfp_prog, reg_both(0), -22);
   1717	wrp_immed(nfp_prog, reg_both(1), ~0);
   1718
   1719	if (!nfp_prog_confirm_current_offset(nfp_prog, end))
   1720		return -EINVAL;
   1721
   1722	return 0;
   1723}
   1724
   1725static int
   1726map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1727{
   1728	bool load_lm_ptr;
   1729	u32 ret_tgt;
   1730	s64 lm_off;
   1731
   1732	/* We only have to reload LM0 if the key is not at start of stack */
   1733	lm_off = nfp_prog->stack_frame_depth;
   1734	lm_off += meta->arg2.reg.var_off.value + meta->arg2.reg.off;
   1735	load_lm_ptr = meta->arg2.var_off || lm_off;
   1736
   1737	/* Set LM0 to start of key */
   1738	if (load_lm_ptr)
   1739		emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0);
   1740	if (meta->func_id == BPF_FUNC_map_update_elem)
   1741		emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2);
   1742
   1743	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
   1744		     2, RELO_BR_HELPER);
   1745	ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
   1746
   1747	/* Load map ID into A0 */
   1748	wrp_mov(nfp_prog, reg_a(0), reg_a(2));
   1749
   1750	/* Load the return address into B0 */
   1751	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
   1752
   1753	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
   1754		return -EINVAL;
   1755
   1756	/* Reset the LM0 pointer */
   1757	if (!load_lm_ptr)
   1758		return 0;
   1759
   1760	emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0);
   1761	wrp_nops(nfp_prog, 3);
   1762
   1763	return 0;
   1764}
   1765
   1766static int
   1767nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1768{
   1769	__emit_csr_rd(nfp_prog, NFP_CSR_PSEUDO_RND_NUM);
   1770	/* CSR value is read in following immed[gpr, 0] */
   1771	emit_immed(nfp_prog, reg_both(0), 0,
   1772		   IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B);
   1773	emit_immed(nfp_prog, reg_both(1), 0,
   1774		   IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B);
   1775	return 0;
   1776}
   1777
   1778static int
   1779nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1780{
   1781	swreg ptr_type;
   1782	u32 ret_tgt;
   1783
   1784	ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog));
   1785
   1786	ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
   1787
   1788	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
   1789		     2, RELO_BR_HELPER);
   1790
   1791	/* Load ptr type into A1 */
   1792	wrp_mov(nfp_prog, reg_a(1), ptr_type);
   1793
   1794	/* Load the return address into B0 */
   1795	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
   1796
   1797	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
   1798		return -EINVAL;
   1799
   1800	return 0;
   1801}
   1802
   1803static int
   1804nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1805{
   1806	u32 jmp_tgt;
   1807
   1808	jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5;
   1809
   1810	/* Make sure the queue id fits into FW field */
   1811	emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2),
   1812		 ALU_OP_AND_NOT_B, reg_imm(0xff));
   1813	emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2);
   1814
   1815	/* Set the 'queue selected' bit and the queue value */
   1816	emit_shf(nfp_prog, pv_qsel_set(nfp_prog),
   1817		 pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1),
   1818		 SHF_SC_L_SHF, PKT_VEL_QSEL_SET_BIT);
   1819	emit_ld_field(nfp_prog,
   1820		      pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2),
   1821		      SHF_SC_NONE, 0);
   1822	/* Delay slots end here, we will jump over next instruction if queue
   1823	 * value fits into the field.
   1824	 */
   1825	emit_ld_field(nfp_prog,
   1826		      pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX),
   1827		      SHF_SC_NONE, 0);
   1828
   1829	if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt))
   1830		return -EINVAL;
   1831
   1832	return 0;
   1833}
   1834
   1835/* --- Callbacks --- */
   1836static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1837{
   1838	const struct bpf_insn *insn = &meta->insn;
   1839	u8 dst = insn->dst_reg * 2;
   1840	u8 src = insn->src_reg * 2;
   1841
   1842	if (insn->src_reg == BPF_REG_10) {
   1843		swreg stack_depth_reg;
   1844
   1845		stack_depth_reg = ur_load_imm_any(nfp_prog,
   1846						  nfp_prog->stack_frame_depth,
   1847						  stack_imm(nfp_prog));
   1848		emit_alu(nfp_prog, reg_both(dst), stack_reg(nfp_prog),
   1849			 ALU_OP_ADD, stack_depth_reg);
   1850		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
   1851	} else {
   1852		wrp_reg_mov(nfp_prog, dst, src);
   1853		wrp_reg_mov(nfp_prog, dst + 1, src + 1);
   1854	}
   1855
   1856	return 0;
   1857}
   1858
   1859static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1860{
   1861	u64 imm = meta->insn.imm; /* sign extend */
   1862
   1863	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
   1864	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
   1865
   1866	return 0;
   1867}
   1868
   1869static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1870{
   1871	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
   1872}
   1873
   1874static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1875{
   1876	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
   1877}
   1878
   1879static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1880{
   1881	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
   1882}
   1883
   1884static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1885{
   1886	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
   1887}
   1888
   1889static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1890{
   1891	return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
   1892}
   1893
   1894static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1895{
   1896	return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
   1897}
   1898
   1899static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1900{
   1901	const struct bpf_insn *insn = &meta->insn;
   1902
   1903	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
   1904		 reg_a(insn->dst_reg * 2), ALU_OP_ADD,
   1905		 reg_b(insn->src_reg * 2));
   1906	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
   1907		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_ADD_C,
   1908		 reg_b(insn->src_reg * 2 + 1));
   1909
   1910	return 0;
   1911}
   1912
   1913static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1914{
   1915	const struct bpf_insn *insn = &meta->insn;
   1916	u64 imm = insn->imm; /* sign extend */
   1917
   1918	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U);
   1919	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32);
   1920
   1921	return 0;
   1922}
   1923
   1924static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1925{
   1926	const struct bpf_insn *insn = &meta->insn;
   1927
   1928	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
   1929		 reg_a(insn->dst_reg * 2), ALU_OP_SUB,
   1930		 reg_b(insn->src_reg * 2));
   1931	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
   1932		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_SUB_C,
   1933		 reg_b(insn->src_reg * 2 + 1));
   1934
   1935	return 0;
   1936}
   1937
   1938static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1939{
   1940	const struct bpf_insn *insn = &meta->insn;
   1941	u64 imm = insn->imm; /* sign extend */
   1942
   1943	wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U);
   1944	wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32);
   1945
   1946	return 0;
   1947}
   1948
   1949static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1950{
   1951	return wrp_mul(nfp_prog, meta, true, true);
   1952}
   1953
   1954static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1955{
   1956	return wrp_mul(nfp_prog, meta, true, false);
   1957}
   1958
   1959static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1960{
   1961	const struct bpf_insn *insn = &meta->insn;
   1962
   1963	return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm);
   1964}
   1965
   1966static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1967{
   1968	/* NOTE: verifier hook has rejected cases for which verifier doesn't
   1969	 * know whether the source operand is constant or not.
   1970	 */
   1971	return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src);
   1972}
   1973
   1974static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   1975{
   1976	const struct bpf_insn *insn = &meta->insn;
   1977
   1978	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0),
   1979		 ALU_OP_SUB, reg_b(insn->dst_reg * 2));
   1980	emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0),
   1981		 ALU_OP_SUB_C, reg_b(insn->dst_reg * 2 + 1));
   1982
   1983	return 0;
   1984}
   1985
   1986/* Pseudo code:
   1987 *   if shift_amt >= 32
   1988 *     dst_high = dst_low << shift_amt[4:0]
   1989 *     dst_low = 0;
   1990 *   else
   1991 *     dst_high = (dst_high, dst_low) >> (32 - shift_amt)
   1992 *     dst_low = dst_low << shift_amt
   1993 *
   1994 * The indirect shift will use the same logic at runtime.
   1995 */
   1996static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
   1997{
   1998	if (!shift_amt)
   1999		return 0;
   2000
   2001	if (shift_amt < 32) {
   2002		emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1),
   2003			 SHF_OP_NONE, reg_b(dst), SHF_SC_R_DSHF,
   2004			 32 - shift_amt);
   2005		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2006			 reg_b(dst), SHF_SC_L_SHF, shift_amt);
   2007	} else if (shift_amt == 32) {
   2008		wrp_reg_mov(nfp_prog, dst + 1, dst);
   2009		wrp_immed(nfp_prog, reg_both(dst), 0);
   2010	} else if (shift_amt > 32) {
   2011		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
   2012			 reg_b(dst), SHF_SC_L_SHF, shift_amt - 32);
   2013		wrp_immed(nfp_prog, reg_both(dst), 0);
   2014	}
   2015
   2016	return 0;
   2017}
   2018
   2019static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2020{
   2021	const struct bpf_insn *insn = &meta->insn;
   2022	u8 dst = insn->dst_reg * 2;
   2023
   2024	return __shl_imm64(nfp_prog, dst, insn->imm);
   2025}
   2026
   2027static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2028{
   2029	emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB,
   2030		 reg_b(src));
   2031	emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0));
   2032	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE,
   2033		       reg_b(dst), SHF_SC_R_DSHF);
   2034}
   2035
   2036/* NOTE: for indirect left shift, HIGH part should be calculated first. */
   2037static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2038{
   2039	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
   2040	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2041		       reg_b(dst), SHF_SC_L_SHF);
   2042}
   2043
   2044static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2045{
   2046	shl_reg64_lt32_high(nfp_prog, dst, src);
   2047	shl_reg64_lt32_low(nfp_prog, dst, src);
   2048}
   2049
   2050static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2051{
   2052	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
   2053	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
   2054		       reg_b(dst), SHF_SC_L_SHF);
   2055	wrp_immed(nfp_prog, reg_both(dst), 0);
   2056}
   2057
   2058static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2059{
   2060	const struct bpf_insn *insn = &meta->insn;
   2061	u64 umin, umax;
   2062	u8 dst, src;
   2063
   2064	dst = insn->dst_reg * 2;
   2065	umin = meta->umin_src;
   2066	umax = meta->umax_src;
   2067	if (umin == umax)
   2068		return __shl_imm64(nfp_prog, dst, umin);
   2069
   2070	src = insn->src_reg * 2;
   2071	if (umax < 32) {
   2072		shl_reg64_lt32(nfp_prog, dst, src);
   2073	} else if (umin >= 32) {
   2074		shl_reg64_ge32(nfp_prog, dst, src);
   2075	} else {
   2076		/* Generate different instruction sequences depending on runtime
   2077		 * value of shift amount.
   2078		 */
   2079		u16 label_ge32, label_end;
   2080
   2081		label_ge32 = nfp_prog_current_offset(nfp_prog) + 7;
   2082		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
   2083
   2084		shl_reg64_lt32_high(nfp_prog, dst, src);
   2085		label_end = nfp_prog_current_offset(nfp_prog) + 6;
   2086		emit_br(nfp_prog, BR_UNC, label_end, 2);
   2087		/* shl_reg64_lt32_low packed in delay slot. */
   2088		shl_reg64_lt32_low(nfp_prog, dst, src);
   2089
   2090		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
   2091			return -EINVAL;
   2092		shl_reg64_ge32(nfp_prog, dst, src);
   2093
   2094		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
   2095			return -EINVAL;
   2096	}
   2097
   2098	return 0;
   2099}
   2100
   2101/* Pseudo code:
   2102 *   if shift_amt >= 32
   2103 *     dst_high = 0;
   2104 *     dst_low = dst_high >> shift_amt[4:0]
   2105 *   else
   2106 *     dst_high = dst_high >> shift_amt
   2107 *     dst_low = (dst_high, dst_low) >> shift_amt
   2108 *
   2109 * The indirect shift will use the same logic at runtime.
   2110 */
   2111static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
   2112{
   2113	if (!shift_amt)
   2114		return 0;
   2115
   2116	if (shift_amt < 32) {
   2117		emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
   2118			 reg_b(dst), SHF_SC_R_DSHF, shift_amt);
   2119		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
   2120			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt);
   2121	} else if (shift_amt == 32) {
   2122		wrp_reg_mov(nfp_prog, dst, dst + 1);
   2123		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
   2124	} else if (shift_amt > 32) {
   2125		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2126			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt - 32);
   2127		wrp_immed(nfp_prog, reg_both(dst + 1), 0);
   2128	}
   2129
   2130	return 0;
   2131}
   2132
   2133static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2134{
   2135	const struct bpf_insn *insn = &meta->insn;
   2136	u8 dst = insn->dst_reg * 2;
   2137
   2138	return __shr_imm64(nfp_prog, dst, insn->imm);
   2139}
   2140
   2141/* NOTE: for indirect right shift, LOW part should be calculated first. */
   2142static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2143{
   2144	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
   2145	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
   2146		       reg_b(dst + 1), SHF_SC_R_SHF);
   2147}
   2148
   2149static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2150{
   2151	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
   2152	emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
   2153		       reg_b(dst), SHF_SC_R_DSHF);
   2154}
   2155
   2156static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2157{
   2158	shr_reg64_lt32_low(nfp_prog, dst, src);
   2159	shr_reg64_lt32_high(nfp_prog, dst, src);
   2160}
   2161
   2162static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2163{
   2164	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
   2165	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2166		       reg_b(dst + 1), SHF_SC_R_SHF);
   2167	wrp_immed(nfp_prog, reg_both(dst + 1), 0);
   2168}
   2169
   2170static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2171{
   2172	const struct bpf_insn *insn = &meta->insn;
   2173	u64 umin, umax;
   2174	u8 dst, src;
   2175
   2176	dst = insn->dst_reg * 2;
   2177	umin = meta->umin_src;
   2178	umax = meta->umax_src;
   2179	if (umin == umax)
   2180		return __shr_imm64(nfp_prog, dst, umin);
   2181
   2182	src = insn->src_reg * 2;
   2183	if (umax < 32) {
   2184		shr_reg64_lt32(nfp_prog, dst, src);
   2185	} else if (umin >= 32) {
   2186		shr_reg64_ge32(nfp_prog, dst, src);
   2187	} else {
   2188		/* Generate different instruction sequences depending on runtime
   2189		 * value of shift amount.
   2190		 */
   2191		u16 label_ge32, label_end;
   2192
   2193		label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
   2194		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
   2195		shr_reg64_lt32_low(nfp_prog, dst, src);
   2196		label_end = nfp_prog_current_offset(nfp_prog) + 6;
   2197		emit_br(nfp_prog, BR_UNC, label_end, 2);
   2198		/* shr_reg64_lt32_high packed in delay slot. */
   2199		shr_reg64_lt32_high(nfp_prog, dst, src);
   2200
   2201		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
   2202			return -EINVAL;
   2203		shr_reg64_ge32(nfp_prog, dst, src);
   2204
   2205		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
   2206			return -EINVAL;
   2207	}
   2208
   2209	return 0;
   2210}
   2211
   2212/* Code logic is the same as __shr_imm64 except ashr requires signedness bit
   2213 * told through PREV_ALU result.
   2214 */
   2215static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
   2216{
   2217	if (!shift_amt)
   2218		return 0;
   2219
   2220	if (shift_amt < 32) {
   2221		emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
   2222			 reg_b(dst), SHF_SC_R_DSHF, shift_amt);
   2223		/* Set signedness bit. */
   2224		emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
   2225			 reg_imm(0));
   2226		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
   2227			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt);
   2228	} else if (shift_amt == 32) {
   2229		/* NOTE: this also helps setting signedness bit. */
   2230		wrp_reg_mov(nfp_prog, dst, dst + 1);
   2231		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
   2232			 reg_b(dst + 1), SHF_SC_R_SHF, 31);
   2233	} else if (shift_amt > 32) {
   2234		emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
   2235			 reg_imm(0));
   2236		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
   2237			 reg_b(dst + 1), SHF_SC_R_SHF, shift_amt - 32);
   2238		emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
   2239			 reg_b(dst + 1), SHF_SC_R_SHF, 31);
   2240	}
   2241
   2242	return 0;
   2243}
   2244
   2245static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2246{
   2247	const struct bpf_insn *insn = &meta->insn;
   2248	u8 dst = insn->dst_reg * 2;
   2249
   2250	return __ashr_imm64(nfp_prog, dst, insn->imm);
   2251}
   2252
   2253static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2254{
   2255	/* NOTE: the first insn will set both indirect shift amount (source A)
   2256	 * and signedness bit (MSB of result).
   2257	 */
   2258	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
   2259	emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
   2260		       reg_b(dst + 1), SHF_SC_R_SHF);
   2261}
   2262
   2263static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2264{
   2265	/* NOTE: it is the same as logic shift because we don't need to shift in
   2266	 * signedness bit when the shift amount is less than 32.
   2267	 */
   2268	return shr_reg64_lt32_low(nfp_prog, dst, src);
   2269}
   2270
   2271static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2272{
   2273	ashr_reg64_lt32_low(nfp_prog, dst, src);
   2274	ashr_reg64_lt32_high(nfp_prog, dst, src);
   2275}
   2276
   2277static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
   2278{
   2279	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
   2280	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
   2281		       reg_b(dst + 1), SHF_SC_R_SHF);
   2282	emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
   2283		 reg_b(dst + 1), SHF_SC_R_SHF, 31);
   2284}
   2285
   2286/* Like ashr_imm64, but need to use indirect shift. */
   2287static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2288{
   2289	const struct bpf_insn *insn = &meta->insn;
   2290	u64 umin, umax;
   2291	u8 dst, src;
   2292
   2293	dst = insn->dst_reg * 2;
   2294	umin = meta->umin_src;
   2295	umax = meta->umax_src;
   2296	if (umin == umax)
   2297		return __ashr_imm64(nfp_prog, dst, umin);
   2298
   2299	src = insn->src_reg * 2;
   2300	if (umax < 32) {
   2301		ashr_reg64_lt32(nfp_prog, dst, src);
   2302	} else if (umin >= 32) {
   2303		ashr_reg64_ge32(nfp_prog, dst, src);
   2304	} else {
   2305		u16 label_ge32, label_end;
   2306
   2307		label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
   2308		emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
   2309		ashr_reg64_lt32_low(nfp_prog, dst, src);
   2310		label_end = nfp_prog_current_offset(nfp_prog) + 6;
   2311		emit_br(nfp_prog, BR_UNC, label_end, 2);
   2312		/* ashr_reg64_lt32_high packed in delay slot. */
   2313		ashr_reg64_lt32_high(nfp_prog, dst, src);
   2314
   2315		if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
   2316			return -EINVAL;
   2317		ashr_reg64_ge32(nfp_prog, dst, src);
   2318
   2319		if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
   2320			return -EINVAL;
   2321	}
   2322
   2323	return 0;
   2324}
   2325
   2326static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2327{
   2328	const struct bpf_insn *insn = &meta->insn;
   2329
   2330	wrp_reg_mov(nfp_prog, insn->dst_reg * 2,  insn->src_reg * 2);
   2331	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
   2332
   2333	return 0;
   2334}
   2335
   2336static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2337{
   2338	const struct bpf_insn *insn = &meta->insn;
   2339
   2340	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
   2341	wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
   2342
   2343	return 0;
   2344}
   2345
   2346static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2347{
   2348	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
   2349}
   2350
   2351static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2352{
   2353	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR);
   2354}
   2355
   2356static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2357{
   2358	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
   2359}
   2360
   2361static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2362{
   2363	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND);
   2364}
   2365
   2366static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2367{
   2368	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
   2369}
   2370
   2371static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2372{
   2373	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR);
   2374}
   2375
   2376static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2377{
   2378	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
   2379}
   2380
   2381static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2382{
   2383	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD);
   2384}
   2385
   2386static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2387{
   2388	return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
   2389}
   2390
   2391static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2392{
   2393	return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB);
   2394}
   2395
   2396static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2397{
   2398	return wrp_mul(nfp_prog, meta, false, true);
   2399}
   2400
   2401static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2402{
   2403	return wrp_mul(nfp_prog, meta, false, false);
   2404}
   2405
   2406static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2407{
   2408	return div_reg64(nfp_prog, meta);
   2409}
   2410
   2411static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2412{
   2413	return div_imm64(nfp_prog, meta);
   2414}
   2415
   2416static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2417{
   2418	u8 dst = meta->insn.dst_reg * 2;
   2419
   2420	emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst));
   2421	wrp_zext(nfp_prog, meta, dst);
   2422
   2423	return 0;
   2424}
   2425
   2426static int
   2427__ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
   2428	   u8 shift_amt)
   2429{
   2430	if (shift_amt) {
   2431		/* Set signedness bit (MSB of result). */
   2432		emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR,
   2433			 reg_imm(0));
   2434		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
   2435			 reg_b(dst), SHF_SC_R_SHF, shift_amt);
   2436	}
   2437	wrp_zext(nfp_prog, meta, dst);
   2438
   2439	return 0;
   2440}
   2441
   2442static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2443{
   2444	const struct bpf_insn *insn = &meta->insn;
   2445	u64 umin, umax;
   2446	u8 dst, src;
   2447
   2448	dst = insn->dst_reg * 2;
   2449	umin = meta->umin_src;
   2450	umax = meta->umax_src;
   2451	if (umin == umax)
   2452		return __ashr_imm(nfp_prog, meta, dst, umin);
   2453
   2454	src = insn->src_reg * 2;
   2455	/* NOTE: the first insn will set both indirect shift amount (source A)
   2456	 * and signedness bit (MSB of result).
   2457	 */
   2458	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
   2459	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
   2460		       reg_b(dst), SHF_SC_R_SHF);
   2461	wrp_zext(nfp_prog, meta, dst);
   2462
   2463	return 0;
   2464}
   2465
   2466static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2467{
   2468	const struct bpf_insn *insn = &meta->insn;
   2469	u8 dst = insn->dst_reg * 2;
   2470
   2471	return __ashr_imm(nfp_prog, meta, dst, insn->imm);
   2472}
   2473
   2474static int
   2475__shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
   2476	  u8 shift_amt)
   2477{
   2478	if (shift_amt)
   2479		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2480			 reg_b(dst), SHF_SC_R_SHF, shift_amt);
   2481	wrp_zext(nfp_prog, meta, dst);
   2482	return 0;
   2483}
   2484
   2485static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2486{
   2487	const struct bpf_insn *insn = &meta->insn;
   2488	u8 dst = insn->dst_reg * 2;
   2489
   2490	return __shr_imm(nfp_prog, meta, dst, insn->imm);
   2491}
   2492
   2493static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2494{
   2495	const struct bpf_insn *insn = &meta->insn;
   2496	u64 umin, umax;
   2497	u8 dst, src;
   2498
   2499	dst = insn->dst_reg * 2;
   2500	umin = meta->umin_src;
   2501	umax = meta->umax_src;
   2502	if (umin == umax)
   2503		return __shr_imm(nfp_prog, meta, dst, umin);
   2504
   2505	src = insn->src_reg * 2;
   2506	emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
   2507	emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2508		       reg_b(dst), SHF_SC_R_SHF);
   2509	wrp_zext(nfp_prog, meta, dst);
   2510	return 0;
   2511}
   2512
   2513static int
   2514__shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
   2515	  u8 shift_amt)
   2516{
   2517	if (shift_amt)
   2518		emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
   2519			 reg_b(dst), SHF_SC_L_SHF, shift_amt);
   2520	wrp_zext(nfp_prog, meta, dst);
   2521	return 0;
   2522}
   2523
   2524static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2525{
   2526	const struct bpf_insn *insn = &meta->insn;
   2527	u8 dst = insn->dst_reg * 2;
   2528
   2529	return __shl_imm(nfp_prog, meta, dst, insn->imm);
   2530}
   2531
   2532static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2533{
   2534	const struct bpf_insn *insn = &meta->insn;
   2535	u64 umin, umax;
   2536	u8 dst, src;
   2537
   2538	dst = insn->dst_reg * 2;
   2539	umin = meta->umin_src;
   2540	umax = meta->umax_src;
   2541	if (umin == umax)
   2542		return __shl_imm(nfp_prog, meta, dst, umin);
   2543
   2544	src = insn->src_reg * 2;
   2545	shl_reg64_lt32_low(nfp_prog, dst, src);
   2546	wrp_zext(nfp_prog, meta, dst);
   2547	return 0;
   2548}
   2549
   2550static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2551{
   2552	const struct bpf_insn *insn = &meta->insn;
   2553	u8 gpr = insn->dst_reg * 2;
   2554
   2555	switch (insn->imm) {
   2556	case 16:
   2557		emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr),
   2558			      SHF_SC_R_ROT, 8);
   2559		emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr),
   2560			      SHF_SC_R_SHF, 16);
   2561
   2562		wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
   2563		break;
   2564	case 32:
   2565		wrp_end32(nfp_prog, reg_a(gpr), gpr);
   2566		wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
   2567		break;
   2568	case 64:
   2569		wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1));
   2570
   2571		wrp_end32(nfp_prog, reg_a(gpr), gpr + 1);
   2572		wrp_end32(nfp_prog, imm_a(nfp_prog), gpr);
   2573		break;
   2574	}
   2575
   2576	return 0;
   2577}
   2578
   2579static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2580{
   2581	struct nfp_insn_meta *prev = nfp_meta_prev(meta);
   2582	u32 imm_lo, imm_hi;
   2583	u8 dst;
   2584
   2585	dst = prev->insn.dst_reg * 2;
   2586	imm_lo = prev->insn.imm;
   2587	imm_hi = meta->insn.imm;
   2588
   2589	wrp_immed(nfp_prog, reg_both(dst), imm_lo);
   2590
   2591	/* mov is always 1 insn, load imm may be two, so try to use mov */
   2592	if (imm_hi == imm_lo)
   2593		wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst));
   2594	else
   2595		wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi);
   2596
   2597	return 0;
   2598}
   2599
   2600static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2601{
   2602	meta->double_cb = imm_ld8_part2;
   2603	return 0;
   2604}
   2605
   2606static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2607{
   2608	return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
   2609}
   2610
   2611static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2612{
   2613	return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
   2614}
   2615
   2616static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2617{
   2618	return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
   2619}
   2620
   2621static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2622{
   2623	return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
   2624				     meta->insn.src_reg * 2, 1);
   2625}
   2626
   2627static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2628{
   2629	return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
   2630				     meta->insn.src_reg * 2, 2);
   2631}
   2632
   2633static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2634{
   2635	return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
   2636				     meta->insn.src_reg * 2, 4);
   2637}
   2638
   2639static int
   2640mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2641	      unsigned int size, unsigned int ptr_off)
   2642{
   2643	return mem_op_stack(nfp_prog, meta, size, ptr_off,
   2644			    meta->insn.dst_reg * 2, meta->insn.src_reg * 2,
   2645			    true, wrp_lmem_load);
   2646}
   2647
   2648static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2649		       u8 size)
   2650{
   2651	swreg dst = reg_both(meta->insn.dst_reg * 2);
   2652
   2653	switch (meta->insn.off) {
   2654	case offsetof(struct __sk_buff, len):
   2655		if (size != sizeof_field(struct __sk_buff, len))
   2656			return -EOPNOTSUPP;
   2657		wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
   2658		break;
   2659	case offsetof(struct __sk_buff, data):
   2660		if (size != sizeof_field(struct __sk_buff, data))
   2661			return -EOPNOTSUPP;
   2662		wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
   2663		break;
   2664	case offsetof(struct __sk_buff, data_end):
   2665		if (size != sizeof_field(struct __sk_buff, data_end))
   2666			return -EOPNOTSUPP;
   2667		emit_alu(nfp_prog, dst,
   2668			 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
   2669		break;
   2670	default:
   2671		return -EOPNOTSUPP;
   2672	}
   2673
   2674	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
   2675
   2676	return 0;
   2677}
   2678
   2679static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2680		       u8 size)
   2681{
   2682	swreg dst = reg_both(meta->insn.dst_reg * 2);
   2683
   2684	switch (meta->insn.off) {
   2685	case offsetof(struct xdp_md, data):
   2686		if (size != sizeof_field(struct xdp_md, data))
   2687			return -EOPNOTSUPP;
   2688		wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
   2689		break;
   2690	case offsetof(struct xdp_md, data_end):
   2691		if (size != sizeof_field(struct xdp_md, data_end))
   2692			return -EOPNOTSUPP;
   2693		emit_alu(nfp_prog, dst,
   2694			 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
   2695		break;
   2696	default:
   2697		return -EOPNOTSUPP;
   2698	}
   2699
   2700	wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
   2701
   2702	return 0;
   2703}
   2704
   2705static int
   2706mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2707	     unsigned int size)
   2708{
   2709	swreg tmp_reg;
   2710
   2711	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
   2712
   2713	return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
   2714					 tmp_reg, meta->insn.dst_reg * 2, size);
   2715}
   2716
   2717static int
   2718mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2719	     unsigned int size)
   2720{
   2721	swreg tmp_reg;
   2722
   2723	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
   2724
   2725	return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
   2726					 tmp_reg, meta->insn.dst_reg * 2, size);
   2727}
   2728
   2729static void
   2730mem_ldx_data_init_pktcache(struct nfp_prog *nfp_prog,
   2731			   struct nfp_insn_meta *meta)
   2732{
   2733	s16 range_start = meta->pkt_cache.range_start;
   2734	s16 range_end = meta->pkt_cache.range_end;
   2735	swreg src_base, off;
   2736	u8 xfer_num, len;
   2737	bool indir;
   2738
   2739	off = re_load_imm_any(nfp_prog, range_start, imm_b(nfp_prog));
   2740	src_base = reg_a(meta->insn.src_reg * 2);
   2741	len = range_end - range_start;
   2742	xfer_num = round_up(len, REG_WIDTH) / REG_WIDTH;
   2743
   2744	indir = len > 8 * REG_WIDTH;
   2745	/* Setup PREV_ALU for indirect mode. */
   2746	if (indir)
   2747		wrp_immed(nfp_prog, reg_none(),
   2748			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
   2749
   2750	/* Cache memory into transfer-in registers. */
   2751	emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base,
   2752		     off, xfer_num - 1, CMD_CTX_SWAP, indir);
   2753}
   2754
   2755static int
   2756mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog,
   2757				     struct nfp_insn_meta *meta,
   2758				     unsigned int size)
   2759{
   2760	s16 range_start = meta->pkt_cache.range_start;
   2761	s16 insn_off = meta->insn.off - range_start;
   2762	swreg dst_lo, dst_hi, src_lo, src_mid;
   2763	u8 dst_gpr = meta->insn.dst_reg * 2;
   2764	u8 len_lo = size, len_mid = 0;
   2765	u8 idx = insn_off / REG_WIDTH;
   2766	u8 off = insn_off % REG_WIDTH;
   2767
   2768	dst_hi = reg_both(dst_gpr + 1);
   2769	dst_lo = reg_both(dst_gpr);
   2770	src_lo = reg_xfer(idx);
   2771
   2772	/* The read length could involve as many as three registers. */
   2773	if (size > REG_WIDTH - off) {
   2774		/* Calculate the part in the second register. */
   2775		len_lo = REG_WIDTH - off;
   2776		len_mid = size - len_lo;
   2777
   2778		/* Calculate the part in the third register. */
   2779		if (size > 2 * REG_WIDTH - off)
   2780			len_mid = REG_WIDTH;
   2781	}
   2782
   2783	wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off);
   2784
   2785	if (!len_mid) {
   2786		wrp_zext(nfp_prog, meta, dst_gpr);
   2787		return 0;
   2788	}
   2789
   2790	src_mid = reg_xfer(idx + 1);
   2791
   2792	if (size <= REG_WIDTH) {
   2793		wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo);
   2794		wrp_zext(nfp_prog, meta, dst_gpr);
   2795	} else {
   2796		swreg src_hi = reg_xfer(idx + 2);
   2797
   2798		wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid,
   2799				   REG_WIDTH - len_lo, len_lo);
   2800		wrp_reg_subpart(nfp_prog, dst_hi, src_mid, len_lo,
   2801				REG_WIDTH - len_lo);
   2802		wrp_reg_or_subpart(nfp_prog, dst_hi, src_hi, REG_WIDTH - len_lo,
   2803				   len_lo);
   2804	}
   2805
   2806	return 0;
   2807}
   2808
   2809static int
   2810mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog,
   2811				   struct nfp_insn_meta *meta,
   2812				   unsigned int size)
   2813{
   2814	swreg dst_lo, dst_hi, src_lo;
   2815	u8 dst_gpr, idx;
   2816
   2817	idx = (meta->insn.off - meta->pkt_cache.range_start) / REG_WIDTH;
   2818	dst_gpr = meta->insn.dst_reg * 2;
   2819	dst_hi = reg_both(dst_gpr + 1);
   2820	dst_lo = reg_both(dst_gpr);
   2821	src_lo = reg_xfer(idx);
   2822
   2823	if (size < REG_WIDTH) {
   2824		wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0);
   2825		wrp_zext(nfp_prog, meta, dst_gpr);
   2826	} else if (size == REG_WIDTH) {
   2827		wrp_mov(nfp_prog, dst_lo, src_lo);
   2828		wrp_zext(nfp_prog, meta, dst_gpr);
   2829	} else {
   2830		swreg src_hi = reg_xfer(idx + 1);
   2831
   2832		wrp_mov(nfp_prog, dst_lo, src_lo);
   2833		wrp_mov(nfp_prog, dst_hi, src_hi);
   2834	}
   2835
   2836	return 0;
   2837}
   2838
   2839static int
   2840mem_ldx_data_from_pktcache(struct nfp_prog *nfp_prog,
   2841			   struct nfp_insn_meta *meta, unsigned int size)
   2842{
   2843	u8 off = meta->insn.off - meta->pkt_cache.range_start;
   2844
   2845	if (IS_ALIGNED(off, REG_WIDTH))
   2846		return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size);
   2847
   2848	return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size);
   2849}
   2850
   2851static int
   2852mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2853	unsigned int size)
   2854{
   2855	if (meta->ldst_gather_len)
   2856		return nfp_cpp_memcpy(nfp_prog, meta);
   2857
   2858	if (meta->ptr.type == PTR_TO_CTX) {
   2859		if (nfp_prog->type == BPF_PROG_TYPE_XDP)
   2860			return mem_ldx_xdp(nfp_prog, meta, size);
   2861		else
   2862			return mem_ldx_skb(nfp_prog, meta, size);
   2863	}
   2864
   2865	if (meta->ptr.type == PTR_TO_PACKET) {
   2866		if (meta->pkt_cache.range_end) {
   2867			if (meta->pkt_cache.do_init)
   2868				mem_ldx_data_init_pktcache(nfp_prog, meta);
   2869
   2870			return mem_ldx_data_from_pktcache(nfp_prog, meta, size);
   2871		} else {
   2872			return mem_ldx_data(nfp_prog, meta, size);
   2873		}
   2874	}
   2875
   2876	if (meta->ptr.type == PTR_TO_STACK)
   2877		return mem_ldx_stack(nfp_prog, meta, size,
   2878				     meta->ptr.off + meta->ptr.var_off.value);
   2879
   2880	if (meta->ptr.type == PTR_TO_MAP_VALUE)
   2881		return mem_ldx_emem(nfp_prog, meta, size);
   2882
   2883	return -EOPNOTSUPP;
   2884}
   2885
   2886static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2887{
   2888	return mem_ldx(nfp_prog, meta, 1);
   2889}
   2890
   2891static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2892{
   2893	return mem_ldx(nfp_prog, meta, 2);
   2894}
   2895
   2896static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2897{
   2898	return mem_ldx(nfp_prog, meta, 4);
   2899}
   2900
   2901static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2902{
   2903	return mem_ldx(nfp_prog, meta, 8);
   2904}
   2905
   2906static int
   2907mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2908	    unsigned int size)
   2909{
   2910	u64 imm = meta->insn.imm; /* sign extend */
   2911	swreg off_reg;
   2912
   2913	off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
   2914
   2915	return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
   2916				  imm, size);
   2917}
   2918
   2919static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2920		  unsigned int size)
   2921{
   2922	if (meta->ptr.type == PTR_TO_PACKET)
   2923		return mem_st_data(nfp_prog, meta, size);
   2924
   2925	return -EOPNOTSUPP;
   2926}
   2927
   2928static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2929{
   2930	return mem_st(nfp_prog, meta, 1);
   2931}
   2932
   2933static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2934{
   2935	return mem_st(nfp_prog, meta, 2);
   2936}
   2937
   2938static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2939{
   2940	return mem_st(nfp_prog, meta, 4);
   2941}
   2942
   2943static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2944{
   2945	return mem_st(nfp_prog, meta, 8);
   2946}
   2947
   2948static int
   2949mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2950	     unsigned int size)
   2951{
   2952	swreg off_reg;
   2953
   2954	off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
   2955
   2956	return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
   2957				   meta->insn.src_reg * 2, size);
   2958}
   2959
   2960static int
   2961mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2962	      unsigned int size, unsigned int ptr_off)
   2963{
   2964	return mem_op_stack(nfp_prog, meta, size, ptr_off,
   2965			    meta->insn.src_reg * 2, meta->insn.dst_reg * 2,
   2966			    false, wrp_lmem_store);
   2967}
   2968
   2969static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2970{
   2971	switch (meta->insn.off) {
   2972	case offsetof(struct xdp_md, rx_queue_index):
   2973		return nfp_queue_select(nfp_prog, meta);
   2974	}
   2975
   2976	WARN_ON_ONCE(1); /* verifier should have rejected bad accesses */
   2977	return -EOPNOTSUPP;
   2978}
   2979
   2980static int
   2981mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   2982	unsigned int size)
   2983{
   2984	if (meta->ptr.type == PTR_TO_PACKET)
   2985		return mem_stx_data(nfp_prog, meta, size);
   2986
   2987	if (meta->ptr.type == PTR_TO_STACK)
   2988		return mem_stx_stack(nfp_prog, meta, size,
   2989				     meta->ptr.off + meta->ptr.var_off.value);
   2990
   2991	return -EOPNOTSUPP;
   2992}
   2993
   2994static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   2995{
   2996	return mem_stx(nfp_prog, meta, 1);
   2997}
   2998
   2999static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3000{
   3001	return mem_stx(nfp_prog, meta, 2);
   3002}
   3003
   3004static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3005{
   3006	if (meta->ptr.type == PTR_TO_CTX)
   3007		if (nfp_prog->type == BPF_PROG_TYPE_XDP)
   3008			return mem_stx_xdp(nfp_prog, meta);
   3009	return mem_stx(nfp_prog, meta, 4);
   3010}
   3011
   3012static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3013{
   3014	return mem_stx(nfp_prog, meta, 8);
   3015}
   3016
   3017static int
   3018mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64)
   3019{
   3020	u8 dst_gpr = meta->insn.dst_reg * 2;
   3021	u8 src_gpr = meta->insn.src_reg * 2;
   3022	unsigned int full_add, out;
   3023	swreg addra, addrb, off;
   3024
   3025	off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
   3026
   3027	/* We can fit 16 bits into command immediate, if we know the immediate
   3028	 * is guaranteed to either always or never fit into 16 bit we only
   3029	 * generate code to handle that particular case, otherwise generate
   3030	 * code for both.
   3031	 */
   3032	out = nfp_prog_current_offset(nfp_prog);
   3033	full_add = nfp_prog_current_offset(nfp_prog);
   3034
   3035	if (meta->insn.off) {
   3036		out += 2;
   3037		full_add += 2;
   3038	}
   3039	if (meta->xadd_maybe_16bit) {
   3040		out += 3;
   3041		full_add += 3;
   3042	}
   3043	if (meta->xadd_over_16bit)
   3044		out += 2 + is64;
   3045	if (meta->xadd_maybe_16bit && meta->xadd_over_16bit) {
   3046		out += 5;
   3047		full_add += 5;
   3048	}
   3049
   3050	/* Generate the branch for choosing add_imm vs add */
   3051	if (meta->xadd_maybe_16bit && meta->xadd_over_16bit) {
   3052		swreg max_imm = imm_a(nfp_prog);
   3053
   3054		wrp_immed(nfp_prog, max_imm, 0xffff);
   3055		emit_alu(nfp_prog, reg_none(),
   3056			 max_imm, ALU_OP_SUB, reg_b(src_gpr));
   3057		emit_alu(nfp_prog, reg_none(),
   3058			 reg_imm(0), ALU_OP_SUB_C, reg_b(src_gpr + 1));
   3059		emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0);
   3060		/* defer for add */
   3061	}
   3062
   3063	/* If insn has an offset add to the address */
   3064	if (!meta->insn.off) {
   3065		addra = reg_a(dst_gpr);
   3066		addrb = reg_b(dst_gpr + 1);
   3067	} else {
   3068		emit_alu(nfp_prog, imma_a(nfp_prog),
   3069			 reg_a(dst_gpr), ALU_OP_ADD, off);
   3070		emit_alu(nfp_prog, imma_b(nfp_prog),
   3071			 reg_a(dst_gpr + 1), ALU_OP_ADD_C, reg_imm(0));
   3072		addra = imma_a(nfp_prog);
   3073		addrb = imma_b(nfp_prog);
   3074	}
   3075
   3076	/* Generate the add_imm if 16 bits are possible */
   3077	if (meta->xadd_maybe_16bit) {
   3078		swreg prev_alu = imm_a(nfp_prog);
   3079
   3080		wrp_immed(nfp_prog, prev_alu,
   3081			  FIELD_PREP(CMD_OVE_DATA, 2) |
   3082			  CMD_OVE_LEN |
   3083			  FIELD_PREP(CMD_OV_LEN, 0x8 | is64 << 2));
   3084		wrp_reg_or_subpart(nfp_prog, prev_alu, reg_b(src_gpr), 2, 2);
   3085		emit_cmd_indir(nfp_prog, CMD_TGT_ADD_IMM, CMD_MODE_40b_BA, 0,
   3086			       addra, addrb, 0, CMD_CTX_NO_SWAP);
   3087
   3088		if (meta->xadd_over_16bit)
   3089			emit_br(nfp_prog, BR_UNC, out, 0);
   3090	}
   3091
   3092	if (!nfp_prog_confirm_current_offset(nfp_prog, full_add))
   3093		return -EINVAL;
   3094
   3095	/* Generate the add if 16 bits are not guaranteed */
   3096	if (meta->xadd_over_16bit) {
   3097		emit_cmd(nfp_prog, CMD_TGT_ADD, CMD_MODE_40b_BA, 0,
   3098			 addra, addrb, is64 << 2,
   3099			 is64 ? CMD_CTX_SWAP_DEFER2 : CMD_CTX_SWAP_DEFER1);
   3100
   3101		wrp_mov(nfp_prog, reg_xfer(0), reg_a(src_gpr));
   3102		if (is64)
   3103			wrp_mov(nfp_prog, reg_xfer(1), reg_a(src_gpr + 1));
   3104	}
   3105
   3106	if (!nfp_prog_confirm_current_offset(nfp_prog, out))
   3107		return -EINVAL;
   3108
   3109	return 0;
   3110}
   3111
   3112static int mem_atomic4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3113{
   3114	if (meta->insn.imm != BPF_ADD)
   3115		return -EOPNOTSUPP;
   3116
   3117	return mem_xadd(nfp_prog, meta, false);
   3118}
   3119
   3120static int mem_atomic8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3121{
   3122	if (meta->insn.imm != BPF_ADD)
   3123		return -EOPNOTSUPP;
   3124
   3125	return mem_xadd(nfp_prog, meta, true);
   3126}
   3127
   3128static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3129{
   3130	emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
   3131
   3132	return 0;
   3133}
   3134
   3135static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3136{
   3137	const struct bpf_insn *insn = &meta->insn;
   3138	u64 imm = insn->imm; /* sign extend */
   3139	swreg or1, or2, tmp_reg;
   3140
   3141	or1 = reg_a(insn->dst_reg * 2);
   3142	or2 = reg_b(insn->dst_reg * 2 + 1);
   3143
   3144	if (imm & ~0U) {
   3145		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
   3146		emit_alu(nfp_prog, imm_a(nfp_prog),
   3147			 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
   3148		or1 = imm_a(nfp_prog);
   3149	}
   3150
   3151	if (imm >> 32) {
   3152		tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
   3153		emit_alu(nfp_prog, imm_b(nfp_prog),
   3154			 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
   3155		or2 = imm_b(nfp_prog);
   3156	}
   3157
   3158	emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2);
   3159	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
   3160
   3161	return 0;
   3162}
   3163
   3164static int jeq32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3165{
   3166	const struct bpf_insn *insn = &meta->insn;
   3167	swreg tmp_reg;
   3168
   3169	tmp_reg = ur_load_imm_any(nfp_prog, insn->imm, imm_b(nfp_prog));
   3170	emit_alu(nfp_prog, reg_none(),
   3171		 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
   3172	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
   3173
   3174	return 0;
   3175}
   3176
   3177static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3178{
   3179	const struct bpf_insn *insn = &meta->insn;
   3180	u64 imm = insn->imm; /* sign extend */
   3181	u8 dst_gpr = insn->dst_reg * 2;
   3182	swreg tmp_reg;
   3183
   3184	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
   3185	emit_alu(nfp_prog, imm_b(nfp_prog),
   3186		 reg_a(dst_gpr), ALU_OP_AND, tmp_reg);
   3187	/* Upper word of the mask can only be 0 or ~0 from sign extension,
   3188	 * so either ignore it or OR the whole thing in.
   3189	 */
   3190	if (is_mbpf_jmp64(meta) && imm >> 32) {
   3191		emit_alu(nfp_prog, reg_none(),
   3192			 reg_a(dst_gpr + 1), ALU_OP_OR, imm_b(nfp_prog));
   3193	}
   3194	emit_br(nfp_prog, BR_BNE, insn->off, 0);
   3195
   3196	return 0;
   3197}
   3198
   3199static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3200{
   3201	const struct bpf_insn *insn = &meta->insn;
   3202	u64 imm = insn->imm; /* sign extend */
   3203	bool is_jmp32 = is_mbpf_jmp32(meta);
   3204	swreg tmp_reg;
   3205
   3206	if (!imm) {
   3207		if (is_jmp32)
   3208			emit_alu(nfp_prog, reg_none(), reg_none(), ALU_OP_NONE,
   3209				 reg_b(insn->dst_reg * 2));
   3210		else
   3211			emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
   3212				 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
   3213		emit_br(nfp_prog, BR_BNE, insn->off, 0);
   3214		return 0;
   3215	}
   3216
   3217	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
   3218	emit_alu(nfp_prog, reg_none(),
   3219		 reg_a(insn->dst_reg * 2), ALU_OP_XOR, tmp_reg);
   3220	emit_br(nfp_prog, BR_BNE, insn->off, 0);
   3221
   3222	if (is_jmp32)
   3223		return 0;
   3224
   3225	tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
   3226	emit_alu(nfp_prog, reg_none(),
   3227		 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR, tmp_reg);
   3228	emit_br(nfp_prog, BR_BNE, insn->off, 0);
   3229
   3230	return 0;
   3231}
   3232
   3233static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3234{
   3235	const struct bpf_insn *insn = &meta->insn;
   3236
   3237	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
   3238		 ALU_OP_XOR, reg_b(insn->src_reg * 2));
   3239	if (is_mbpf_jmp64(meta)) {
   3240		emit_alu(nfp_prog, imm_b(nfp_prog),
   3241			 reg_a(insn->dst_reg * 2 + 1), ALU_OP_XOR,
   3242			 reg_b(insn->src_reg * 2 + 1));
   3243		emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR,
   3244			 imm_b(nfp_prog));
   3245	}
   3246	emit_br(nfp_prog, BR_BEQ, insn->off, 0);
   3247
   3248	return 0;
   3249}
   3250
   3251static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3252{
   3253	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
   3254}
   3255
   3256static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3257{
   3258	return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
   3259}
   3260
   3261static int
   3262bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3263{
   3264	u32 ret_tgt, stack_depth, offset_br;
   3265	swreg tmp_reg;
   3266
   3267	stack_depth = round_up(nfp_prog->stack_frame_depth, STACK_FRAME_ALIGN);
   3268	/* Space for saving the return address is accounted for by the callee,
   3269	 * so stack_depth can be zero for the main function.
   3270	 */
   3271	if (stack_depth) {
   3272		tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
   3273					  stack_imm(nfp_prog));
   3274		emit_alu(nfp_prog, stack_reg(nfp_prog),
   3275			 stack_reg(nfp_prog), ALU_OP_ADD, tmp_reg);
   3276		emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
   3277			    NFP_CSR_ACT_LM_ADDR0);
   3278	}
   3279
   3280	/* Two cases for jumping to the callee:
   3281	 *
   3282	 * - If callee uses and needs to save R6~R9 then:
   3283	 *     1. Put the start offset of the callee into imm_b(). This will
   3284	 *        require a fixup step, as we do not necessarily know this
   3285	 *        address yet.
   3286	 *     2. Put the return address from the callee to the caller into
   3287	 *        register ret_reg().
   3288	 *     3. (After defer slots are consumed) Jump to the subroutine that
   3289	 *        pushes the registers to the stack.
   3290	 *   The subroutine acts as a trampoline, and returns to the address in
   3291	 *   imm_b(), i.e. jumps to the callee.
   3292	 *
   3293	 * - If callee does not need to save R6~R9 then just load return
   3294	 *   address to the caller in ret_reg(), and jump to the callee
   3295	 *   directly.
   3296	 *
   3297	 * Using ret_reg() to pass the return address to the callee is set here
   3298	 * as a convention. The callee can then push this address onto its
   3299	 * stack frame in its prologue. The advantages of passing the return
   3300	 * address through ret_reg(), instead of pushing it to the stack right
   3301	 * here, are the following:
   3302	 * - It looks cleaner.
   3303	 * - If the called function is called multiple time, we get a lower
   3304	 *   program size.
   3305	 * - We save two no-op instructions that should be added just before
   3306	 *   the emit_br() when stack depth is not null otherwise.
   3307	 * - If we ever find a register to hold the return address during whole
   3308	 *   execution of the callee, we will not have to push the return
   3309	 *   address to the stack for leaf functions.
   3310	 */
   3311	if (!meta->jmp_dst) {
   3312		pr_err("BUG: BPF-to-BPF call has no destination recorded\n");
   3313		return -ELOOP;
   3314	}
   3315	if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) {
   3316		ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
   3317		emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2,
   3318			     RELO_BR_GO_CALL_PUSH_REGS);
   3319		offset_br = nfp_prog_current_offset(nfp_prog);
   3320		wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL);
   3321	} else {
   3322		ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
   3323		emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1);
   3324		offset_br = nfp_prog_current_offset(nfp_prog);
   3325	}
   3326	wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL);
   3327
   3328	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
   3329		return -EINVAL;
   3330
   3331	if (stack_depth) {
   3332		tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
   3333					  stack_imm(nfp_prog));
   3334		emit_alu(nfp_prog, stack_reg(nfp_prog),
   3335			 stack_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
   3336		emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
   3337			    NFP_CSR_ACT_LM_ADDR0);
   3338		wrp_nops(nfp_prog, 3);
   3339	}
   3340
   3341	meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog);
   3342	meta->num_insns_after_br -= offset_br;
   3343
   3344	return 0;
   3345}
   3346
   3347static int helper_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3348{
   3349	switch (meta->insn.imm) {
   3350	case BPF_FUNC_xdp_adjust_head:
   3351		return adjust_head(nfp_prog, meta);
   3352	case BPF_FUNC_xdp_adjust_tail:
   3353		return adjust_tail(nfp_prog, meta);
   3354	case BPF_FUNC_map_lookup_elem:
   3355	case BPF_FUNC_map_update_elem:
   3356	case BPF_FUNC_map_delete_elem:
   3357		return map_call_stack_common(nfp_prog, meta);
   3358	case BPF_FUNC_get_prandom_u32:
   3359		return nfp_get_prandom_u32(nfp_prog, meta);
   3360	case BPF_FUNC_perf_event_output:
   3361		return nfp_perf_event_output(nfp_prog, meta);
   3362	default:
   3363		WARN_ONCE(1, "verifier allowed unsupported function\n");
   3364		return -EOPNOTSUPP;
   3365	}
   3366}
   3367
   3368static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3369{
   3370	if (is_mbpf_pseudo_call(meta))
   3371		return bpf_to_bpf_call(nfp_prog, meta);
   3372	else
   3373		return helper_call(nfp_prog, meta);
   3374}
   3375
   3376static bool nfp_is_main_function(struct nfp_insn_meta *meta)
   3377{
   3378	return meta->subprog_idx == 0;
   3379}
   3380
   3381static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3382{
   3383	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT);
   3384
   3385	return 0;
   3386}
   3387
   3388static int
   3389nfp_subprog_epilogue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3390{
   3391	if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) {
   3392		/* Pop R6~R9 to the stack via related subroutine.
   3393		 * We loaded the return address to the caller into ret_reg().
   3394		 * This means that the subroutine does not come back here, we
   3395		 * make it jump back to the subprogram caller directly!
   3396		 */
   3397		emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 1,
   3398			     RELO_BR_GO_CALL_POP_REGS);
   3399		/* Pop return address from the stack. */
   3400		wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
   3401	} else {
   3402		/* Pop return address from the stack. */
   3403		wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
   3404		/* Jump back to caller if no callee-saved registers were used
   3405		 * by the subprogram.
   3406		 */
   3407		emit_rtn(nfp_prog, ret_reg(nfp_prog), 0);
   3408	}
   3409
   3410	return 0;
   3411}
   3412
   3413static int jmp_exit(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3414{
   3415	if (nfp_is_main_function(meta))
   3416		return goto_out(nfp_prog, meta);
   3417	else
   3418		return nfp_subprog_epilogue(nfp_prog, meta);
   3419}
   3420
   3421static const instr_cb_t instr_cb[256] = {
   3422	[BPF_ALU64 | BPF_MOV | BPF_X] =	mov_reg64,
   3423	[BPF_ALU64 | BPF_MOV | BPF_K] =	mov_imm64,
   3424	[BPF_ALU64 | BPF_XOR | BPF_X] =	xor_reg64,
   3425	[BPF_ALU64 | BPF_XOR | BPF_K] =	xor_imm64,
   3426	[BPF_ALU64 | BPF_AND | BPF_X] =	and_reg64,
   3427	[BPF_ALU64 | BPF_AND | BPF_K] =	and_imm64,
   3428	[BPF_ALU64 | BPF_OR | BPF_X] =	or_reg64,
   3429	[BPF_ALU64 | BPF_OR | BPF_K] =	or_imm64,
   3430	[BPF_ALU64 | BPF_ADD | BPF_X] =	add_reg64,
   3431	[BPF_ALU64 | BPF_ADD | BPF_K] =	add_imm64,
   3432	[BPF_ALU64 | BPF_SUB | BPF_X] =	sub_reg64,
   3433	[BPF_ALU64 | BPF_SUB | BPF_K] =	sub_imm64,
   3434	[BPF_ALU64 | BPF_MUL | BPF_X] =	mul_reg64,
   3435	[BPF_ALU64 | BPF_MUL | BPF_K] =	mul_imm64,
   3436	[BPF_ALU64 | BPF_DIV | BPF_X] =	div_reg64,
   3437	[BPF_ALU64 | BPF_DIV | BPF_K] =	div_imm64,
   3438	[BPF_ALU64 | BPF_NEG] =		neg_reg64,
   3439	[BPF_ALU64 | BPF_LSH | BPF_X] =	shl_reg64,
   3440	[BPF_ALU64 | BPF_LSH | BPF_K] =	shl_imm64,
   3441	[BPF_ALU64 | BPF_RSH | BPF_X] =	shr_reg64,
   3442	[BPF_ALU64 | BPF_RSH | BPF_K] =	shr_imm64,
   3443	[BPF_ALU64 | BPF_ARSH | BPF_X] = ashr_reg64,
   3444	[BPF_ALU64 | BPF_ARSH | BPF_K] = ashr_imm64,
   3445	[BPF_ALU | BPF_MOV | BPF_X] =	mov_reg,
   3446	[BPF_ALU | BPF_MOV | BPF_K] =	mov_imm,
   3447	[BPF_ALU | BPF_XOR | BPF_X] =	xor_reg,
   3448	[BPF_ALU | BPF_XOR | BPF_K] =	xor_imm,
   3449	[BPF_ALU | BPF_AND | BPF_X] =	and_reg,
   3450	[BPF_ALU | BPF_AND | BPF_K] =	and_imm,
   3451	[BPF_ALU | BPF_OR | BPF_X] =	or_reg,
   3452	[BPF_ALU | BPF_OR | BPF_K] =	or_imm,
   3453	[BPF_ALU | BPF_ADD | BPF_X] =	add_reg,
   3454	[BPF_ALU | BPF_ADD | BPF_K] =	add_imm,
   3455	[BPF_ALU | BPF_SUB | BPF_X] =	sub_reg,
   3456	[BPF_ALU | BPF_SUB | BPF_K] =	sub_imm,
   3457	[BPF_ALU | BPF_MUL | BPF_X] =	mul_reg,
   3458	[BPF_ALU | BPF_MUL | BPF_K] =	mul_imm,
   3459	[BPF_ALU | BPF_DIV | BPF_X] =	div_reg,
   3460	[BPF_ALU | BPF_DIV | BPF_K] =	div_imm,
   3461	[BPF_ALU | BPF_NEG] =		neg_reg,
   3462	[BPF_ALU | BPF_LSH | BPF_X] =	shl_reg,
   3463	[BPF_ALU | BPF_LSH | BPF_K] =	shl_imm,
   3464	[BPF_ALU | BPF_RSH | BPF_X] =	shr_reg,
   3465	[BPF_ALU | BPF_RSH | BPF_K] =	shr_imm,
   3466	[BPF_ALU | BPF_ARSH | BPF_X] =	ashr_reg,
   3467	[BPF_ALU | BPF_ARSH | BPF_K] =	ashr_imm,
   3468	[BPF_ALU | BPF_END | BPF_X] =	end_reg32,
   3469	[BPF_LD | BPF_IMM | BPF_DW] =	imm_ld8,
   3470	[BPF_LD | BPF_ABS | BPF_B] =	data_ld1,
   3471	[BPF_LD | BPF_ABS | BPF_H] =	data_ld2,
   3472	[BPF_LD | BPF_ABS | BPF_W] =	data_ld4,
   3473	[BPF_LD | BPF_IND | BPF_B] =	data_ind_ld1,
   3474	[BPF_LD | BPF_IND | BPF_H] =	data_ind_ld2,
   3475	[BPF_LD | BPF_IND | BPF_W] =	data_ind_ld4,
   3476	[BPF_LDX | BPF_MEM | BPF_B] =	mem_ldx1,
   3477	[BPF_LDX | BPF_MEM | BPF_H] =	mem_ldx2,
   3478	[BPF_LDX | BPF_MEM | BPF_W] =	mem_ldx4,
   3479	[BPF_LDX | BPF_MEM | BPF_DW] =	mem_ldx8,
   3480	[BPF_STX | BPF_MEM | BPF_B] =	mem_stx1,
   3481	[BPF_STX | BPF_MEM | BPF_H] =	mem_stx2,
   3482	[BPF_STX | BPF_MEM | BPF_W] =	mem_stx4,
   3483	[BPF_STX | BPF_MEM | BPF_DW] =	mem_stx8,
   3484	[BPF_STX | BPF_ATOMIC | BPF_W] =	mem_atomic4,
   3485	[BPF_STX | BPF_ATOMIC | BPF_DW] =	mem_atomic8,
   3486	[BPF_ST | BPF_MEM | BPF_B] =	mem_st1,
   3487	[BPF_ST | BPF_MEM | BPF_H] =	mem_st2,
   3488	[BPF_ST | BPF_MEM | BPF_W] =	mem_st4,
   3489	[BPF_ST | BPF_MEM | BPF_DW] =	mem_st8,
   3490	[BPF_JMP | BPF_JA | BPF_K] =	jump,
   3491	[BPF_JMP | BPF_JEQ | BPF_K] =	jeq_imm,
   3492	[BPF_JMP | BPF_JGT | BPF_K] =	cmp_imm,
   3493	[BPF_JMP | BPF_JGE | BPF_K] =	cmp_imm,
   3494	[BPF_JMP | BPF_JLT | BPF_K] =	cmp_imm,
   3495	[BPF_JMP | BPF_JLE | BPF_K] =	cmp_imm,
   3496	[BPF_JMP | BPF_JSGT | BPF_K] =  cmp_imm,
   3497	[BPF_JMP | BPF_JSGE | BPF_K] =  cmp_imm,
   3498	[BPF_JMP | BPF_JSLT | BPF_K] =  cmp_imm,
   3499	[BPF_JMP | BPF_JSLE | BPF_K] =  cmp_imm,
   3500	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,
   3501	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,
   3502	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
   3503	[BPF_JMP | BPF_JGT | BPF_X] =	cmp_reg,
   3504	[BPF_JMP | BPF_JGE | BPF_X] =	cmp_reg,
   3505	[BPF_JMP | BPF_JLT | BPF_X] =	cmp_reg,
   3506	[BPF_JMP | BPF_JLE | BPF_X] =	cmp_reg,
   3507	[BPF_JMP | BPF_JSGT | BPF_X] =  cmp_reg,
   3508	[BPF_JMP | BPF_JSGE | BPF_X] =  cmp_reg,
   3509	[BPF_JMP | BPF_JSLT | BPF_X] =  cmp_reg,
   3510	[BPF_JMP | BPF_JSLE | BPF_X] =  cmp_reg,
   3511	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,
   3512	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,
   3513	[BPF_JMP32 | BPF_JEQ | BPF_K] =	jeq32_imm,
   3514	[BPF_JMP32 | BPF_JGT | BPF_K] =	cmp_imm,
   3515	[BPF_JMP32 | BPF_JGE | BPF_K] =	cmp_imm,
   3516	[BPF_JMP32 | BPF_JLT | BPF_K] =	cmp_imm,
   3517	[BPF_JMP32 | BPF_JLE | BPF_K] =	cmp_imm,
   3518	[BPF_JMP32 | BPF_JSGT | BPF_K] =cmp_imm,
   3519	[BPF_JMP32 | BPF_JSGE | BPF_K] =cmp_imm,
   3520	[BPF_JMP32 | BPF_JSLT | BPF_K] =cmp_imm,
   3521	[BPF_JMP32 | BPF_JSLE | BPF_K] =cmp_imm,
   3522	[BPF_JMP32 | BPF_JSET | BPF_K] =jset_imm,
   3523	[BPF_JMP32 | BPF_JNE | BPF_K] =	jne_imm,
   3524	[BPF_JMP32 | BPF_JEQ | BPF_X] =	jeq_reg,
   3525	[BPF_JMP32 | BPF_JGT | BPF_X] =	cmp_reg,
   3526	[BPF_JMP32 | BPF_JGE | BPF_X] =	cmp_reg,
   3527	[BPF_JMP32 | BPF_JLT | BPF_X] =	cmp_reg,
   3528	[BPF_JMP32 | BPF_JLE | BPF_X] =	cmp_reg,
   3529	[BPF_JMP32 | BPF_JSGT | BPF_X] =cmp_reg,
   3530	[BPF_JMP32 | BPF_JSGE | BPF_X] =cmp_reg,
   3531	[BPF_JMP32 | BPF_JSLT | BPF_X] =cmp_reg,
   3532	[BPF_JMP32 | BPF_JSLE | BPF_X] =cmp_reg,
   3533	[BPF_JMP32 | BPF_JSET | BPF_X] =jset_reg,
   3534	[BPF_JMP32 | BPF_JNE | BPF_X] =	jne_reg,
   3535	[BPF_JMP | BPF_CALL] =		call,
   3536	[BPF_JMP | BPF_EXIT] =		jmp_exit,
   3537};
   3538
   3539/* --- Assembler logic --- */
   3540static int
   3541nfp_fixup_immed_relo(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
   3542		     struct nfp_insn_meta *jmp_dst, u32 br_idx)
   3543{
   3544	if (immed_get_value(nfp_prog->prog[br_idx + 1])) {
   3545		pr_err("BUG: failed to fix up callee register saving\n");
   3546		return -EINVAL;
   3547	}
   3548
   3549	immed_set_value(&nfp_prog->prog[br_idx + 1], jmp_dst->off);
   3550
   3551	return 0;
   3552}
   3553
   3554static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
   3555{
   3556	struct nfp_insn_meta *meta, *jmp_dst;
   3557	u32 idx, br_idx;
   3558	int err;
   3559
   3560	list_for_each_entry(meta, &nfp_prog->insns, l) {
   3561		if (meta->flags & FLAG_INSN_SKIP_MASK)
   3562			continue;
   3563		if (!is_mbpf_jmp(meta))
   3564			continue;
   3565		if (meta->insn.code == (BPF_JMP | BPF_EXIT) &&
   3566		    !nfp_is_main_function(meta))
   3567			continue;
   3568		if (is_mbpf_helper_call(meta))
   3569			continue;
   3570
   3571		if (list_is_last(&meta->l, &nfp_prog->insns))
   3572			br_idx = nfp_prog->last_bpf_off;
   3573		else
   3574			br_idx = list_next_entry(meta, l)->off - 1;
   3575
   3576		/* For BPF-to-BPF function call, a stack adjustment sequence is
   3577		 * generated after the return instruction. Therefore, we must
   3578		 * withdraw the length of this sequence to have br_idx pointing
   3579		 * to where the "branch" NFP instruction is expected to be.
   3580		 */
   3581		if (is_mbpf_pseudo_call(meta))
   3582			br_idx -= meta->num_insns_after_br;
   3583
   3584		if (!nfp_is_br(nfp_prog->prog[br_idx])) {
   3585			pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
   3586			       br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
   3587			return -ELOOP;
   3588		}
   3589
   3590		if (meta->insn.code == (BPF_JMP | BPF_EXIT))
   3591			continue;
   3592
   3593		/* Leave special branches for later */
   3594		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
   3595		    RELO_BR_REL && !is_mbpf_pseudo_call(meta))
   3596			continue;
   3597
   3598		if (!meta->jmp_dst) {
   3599			pr_err("Non-exit jump doesn't have destination info recorded!!\n");
   3600			return -ELOOP;
   3601		}
   3602
   3603		jmp_dst = meta->jmp_dst;
   3604
   3605		if (jmp_dst->flags & FLAG_INSN_SKIP_PREC_DEPENDENT) {
   3606			pr_err("Branch landing on removed instruction!!\n");
   3607			return -ELOOP;
   3608		}
   3609
   3610		if (is_mbpf_pseudo_call(meta) &&
   3611		    nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) {
   3612			err = nfp_fixup_immed_relo(nfp_prog, meta,
   3613						   jmp_dst, br_idx);
   3614			if (err)
   3615				return err;
   3616		}
   3617
   3618		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
   3619		    RELO_BR_REL)
   3620			continue;
   3621
   3622		for (idx = meta->off; idx <= br_idx; idx++) {
   3623			if (!nfp_is_br(nfp_prog->prog[idx]))
   3624				continue;
   3625			br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);
   3626		}
   3627	}
   3628
   3629	return 0;
   3630}
   3631
   3632static void nfp_intro(struct nfp_prog *nfp_prog)
   3633{
   3634	wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0));
   3635	emit_alu(nfp_prog, plen_reg(nfp_prog),
   3636		 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog));
   3637}
   3638
   3639static void
   3640nfp_subprog_prologue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3641{
   3642	/* Save return address into the stack. */
   3643	wrp_mov(nfp_prog, reg_lm(0, 0), ret_reg(nfp_prog));
   3644}
   3645
   3646static void
   3647nfp_start_subprog(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
   3648{
   3649	unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth;
   3650
   3651	nfp_prog->stack_frame_depth = round_up(depth, 4);
   3652	nfp_subprog_prologue(nfp_prog, meta);
   3653}
   3654
   3655bool nfp_is_subprog_start(struct nfp_insn_meta *meta)
   3656{
   3657	return meta->flags & FLAG_INSN_IS_SUBPROG_START;
   3658}
   3659
   3660static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
   3661{
   3662	/* TC direct-action mode:
   3663	 *   0,1   ok        NOT SUPPORTED[1]
   3664	 *   2   drop  0x22 -> drop,  count as stat1
   3665	 *   4,5 nuke  0x02 -> drop
   3666	 *   7  redir  0x44 -> redir, count as stat2
   3667	 *   * unspec  0x11 -> pass,  count as stat0
   3668	 *
   3669	 * [1] We can't support OK and RECLASSIFY because we can't tell TC
   3670	 *     the exact decision made.  We are forced to support UNSPEC
   3671	 *     to handle aborts so that's the only one we handle for passing
   3672	 *     packets up the stack.
   3673	 */
   3674	/* Target for aborts */
   3675	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
   3676
   3677	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
   3678
   3679	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
   3680	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
   3681
   3682	/* Target for normal exits */
   3683	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
   3684
   3685	/* if R0 > 7 jump to abort */
   3686	emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0));
   3687	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
   3688	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
   3689
   3690	wrp_immed(nfp_prog, reg_b(2), 0x41221211);
   3691	wrp_immed(nfp_prog, reg_b(3), 0x41001211);
   3692
   3693	emit_shf(nfp_prog, reg_a(1),
   3694		 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 2);
   3695
   3696	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
   3697	emit_shf(nfp_prog, reg_a(2),
   3698		 reg_imm(0xf), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
   3699
   3700	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
   3701	emit_shf(nfp_prog, reg_b(2),
   3702		 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);
   3703
   3704	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
   3705
   3706	emit_shf(nfp_prog, reg_b(2),
   3707		 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
   3708	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
   3709}
   3710
   3711static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
   3712{
   3713	/* XDP return codes:
   3714	 *   0 aborted  0x82 -> drop,  count as stat3
   3715	 *   1    drop  0x22 -> drop,  count as stat1
   3716	 *   2    pass  0x11 -> pass,  count as stat0
   3717	 *   3      tx  0x44 -> redir, count as stat2
   3718	 *   * unknown  0x82 -> drop,  count as stat3
   3719	 */
   3720	/* Target for aborts */
   3721	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
   3722
   3723	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
   3724
   3725	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
   3726	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
   3727
   3728	/* Target for normal exits */
   3729	nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
   3730
   3731	/* if R0 > 3 jump to abort */
   3732	emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0));
   3733	emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
   3734
   3735	wrp_immed(nfp_prog, reg_b(2), 0x44112282);
   3736
   3737	emit_shf(nfp_prog, reg_a(1),
   3738		 reg_none(), SHF_OP_NONE, reg_b(0), SHF_SC_L_SHF, 3);
   3739
   3740	emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
   3741	emit_shf(nfp_prog, reg_b(2),
   3742		 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
   3743
   3744	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
   3745
   3746	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
   3747	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
   3748}
   3749
   3750static bool nfp_prog_needs_callee_reg_save(struct nfp_prog *nfp_prog)
   3751{
   3752	unsigned int idx;
   3753
   3754	for (idx = 1; idx < nfp_prog->subprog_cnt; idx++)
   3755		if (nfp_prog->subprog[idx].needs_reg_push)
   3756			return true;
   3757
   3758	return false;
   3759}
   3760
   3761static void nfp_push_callee_registers(struct nfp_prog *nfp_prog)
   3762{
   3763	u8 reg;
   3764
   3765	/* Subroutine: Save all callee saved registers (R6 ~ R9).
   3766	 * imm_b() holds the return address.
   3767	 */
   3768	nfp_prog->tgt_call_push_regs = nfp_prog_current_offset(nfp_prog);
   3769	for (reg = BPF_REG_6; reg <= BPF_REG_9; reg++) {
   3770		u8 adj = (reg - BPF_REG_0) * 2;
   3771		u8 idx = (reg - BPF_REG_6) * 2;
   3772
   3773		/* The first slot in the stack frame is used to push the return
   3774		 * address in bpf_to_bpf_call(), start just after.
   3775		 */
   3776		wrp_mov(nfp_prog, reg_lm(0, 1 + idx), reg_b(adj));
   3777
   3778		if (reg == BPF_REG_8)
   3779			/* Prepare to jump back, last 3 insns use defer slots */
   3780			emit_rtn(nfp_prog, imm_b(nfp_prog), 3);
   3781
   3782		wrp_mov(nfp_prog, reg_lm(0, 1 + idx + 1), reg_b(adj + 1));
   3783	}
   3784}
   3785
   3786static void nfp_pop_callee_registers(struct nfp_prog *nfp_prog)
   3787{
   3788	u8 reg;
   3789
   3790	/* Subroutine: Restore all callee saved registers (R6 ~ R9).
   3791	 * ret_reg() holds the return address.
   3792	 */
   3793	nfp_prog->tgt_call_pop_regs = nfp_prog_current_offset(nfp_prog);
   3794	for (reg = BPF_REG_6; reg <= BPF_REG_9; reg++) {
   3795		u8 adj = (reg - BPF_REG_0) * 2;
   3796		u8 idx = (reg - BPF_REG_6) * 2;
   3797
   3798		/* The first slot in the stack frame holds the return address,
   3799		 * start popping just after that.
   3800		 */
   3801		wrp_mov(nfp_prog, reg_both(adj), reg_lm(0, 1 + idx));
   3802
   3803		if (reg == BPF_REG_8)
   3804			/* Prepare to jump back, last 3 insns use defer slots */
   3805			emit_rtn(nfp_prog, ret_reg(nfp_prog), 3);
   3806
   3807		wrp_mov(nfp_prog, reg_both(adj + 1), reg_lm(0, 1 + idx + 1));
   3808	}
   3809}
   3810
   3811static void nfp_outro(struct nfp_prog *nfp_prog)
   3812{
   3813	switch (nfp_prog->type) {
   3814	case BPF_PROG_TYPE_SCHED_CLS:
   3815		nfp_outro_tc_da(nfp_prog);
   3816		break;
   3817	case BPF_PROG_TYPE_XDP:
   3818		nfp_outro_xdp(nfp_prog);
   3819		break;
   3820	default:
   3821		WARN_ON(1);
   3822	}
   3823
   3824	if (!nfp_prog_needs_callee_reg_save(nfp_prog))
   3825		return;
   3826
   3827	nfp_push_callee_registers(nfp_prog);
   3828	nfp_pop_callee_registers(nfp_prog);
   3829}
   3830
   3831static int nfp_translate(struct nfp_prog *nfp_prog)
   3832{
   3833	struct nfp_insn_meta *meta;
   3834	unsigned int depth;
   3835	int err;
   3836
   3837	depth = nfp_prog->subprog[0].stack_depth;
   3838	nfp_prog->stack_frame_depth = round_up(depth, 4);
   3839
   3840	nfp_intro(nfp_prog);
   3841	if (nfp_prog->error)
   3842		return nfp_prog->error;
   3843
   3844	list_for_each_entry(meta, &nfp_prog->insns, l) {
   3845		instr_cb_t cb = instr_cb[meta->insn.code];
   3846
   3847		meta->off = nfp_prog_current_offset(nfp_prog);
   3848
   3849		if (nfp_is_subprog_start(meta)) {
   3850			nfp_start_subprog(nfp_prog, meta);
   3851			if (nfp_prog->error)
   3852				return nfp_prog->error;
   3853		}
   3854
   3855		if (meta->flags & FLAG_INSN_SKIP_MASK) {
   3856			nfp_prog->n_translated++;
   3857			continue;
   3858		}
   3859
   3860		if (nfp_meta_has_prev(nfp_prog, meta) &&
   3861		    nfp_meta_prev(meta)->double_cb)
   3862			cb = nfp_meta_prev(meta)->double_cb;
   3863		if (!cb)
   3864			return -ENOENT;
   3865		err = cb(nfp_prog, meta);
   3866		if (err)
   3867			return err;
   3868		if (nfp_prog->error)
   3869			return nfp_prog->error;
   3870
   3871		nfp_prog->n_translated++;
   3872	}
   3873
   3874	nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1;
   3875
   3876	nfp_outro(nfp_prog);
   3877	if (nfp_prog->error)
   3878		return nfp_prog->error;
   3879
   3880	wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW);
   3881	if (nfp_prog->error)
   3882		return nfp_prog->error;
   3883
   3884	return nfp_fixup_branches(nfp_prog);
   3885}
   3886
   3887/* --- Optimizations --- */
   3888static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
   3889{
   3890	struct nfp_insn_meta *meta;
   3891
   3892	list_for_each_entry(meta, &nfp_prog->insns, l) {
   3893		struct bpf_insn insn = meta->insn;
   3894
   3895		/* Programs converted from cBPF start with register xoring */
   3896		if (insn.code == (BPF_ALU64 | BPF_XOR | BPF_X) &&
   3897		    insn.src_reg == insn.dst_reg)
   3898			continue;
   3899
   3900		/* Programs start with R6 = R1 but we ignore the skb pointer */
   3901		if (insn.code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
   3902		    insn.src_reg == 1 && insn.dst_reg == 6)
   3903			meta->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
   3904
   3905		/* Return as soon as something doesn't match */
   3906		if (!(meta->flags & FLAG_INSN_SKIP_MASK))
   3907			return;
   3908	}
   3909}
   3910
   3911/* abs(insn.imm) will fit better into unrestricted reg immediate -
   3912 * convert add/sub of a negative number into a sub/add of a positive one.
   3913 */
   3914static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog)
   3915{
   3916	struct nfp_insn_meta *meta;
   3917
   3918	list_for_each_entry(meta, &nfp_prog->insns, l) {
   3919		struct bpf_insn insn = meta->insn;
   3920
   3921		if (meta->flags & FLAG_INSN_SKIP_MASK)
   3922			continue;
   3923
   3924		if (!is_mbpf_alu(meta) && !is_mbpf_jmp(meta))
   3925			continue;
   3926		if (BPF_SRC(insn.code) != BPF_K)
   3927			continue;
   3928		if (insn.imm >= 0)
   3929			continue;
   3930
   3931		if (is_mbpf_jmp(meta)) {
   3932			switch (BPF_OP(insn.code)) {
   3933			case BPF_JGE:
   3934			case BPF_JSGE:
   3935			case BPF_JLT:
   3936			case BPF_JSLT:
   3937				meta->jump_neg_op = true;
   3938				break;
   3939			default:
   3940				continue;
   3941			}
   3942		} else {
   3943			if (BPF_OP(insn.code) == BPF_ADD)
   3944				insn.code = BPF_CLASS(insn.code) | BPF_SUB;
   3945			else if (BPF_OP(insn.code) == BPF_SUB)
   3946				insn.code = BPF_CLASS(insn.code) | BPF_ADD;
   3947			else
   3948				continue;
   3949
   3950			meta->insn.code = insn.code | BPF_K;
   3951		}
   3952
   3953		meta->insn.imm = -insn.imm;
   3954	}
   3955}
   3956
   3957/* Remove masking after load since our load guarantees this is not needed */
   3958static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
   3959{
   3960	struct nfp_insn_meta *meta1, *meta2;
   3961	static const s32 exp_mask[] = {
   3962		[BPF_B] = 0x000000ffU,
   3963		[BPF_H] = 0x0000ffffU,
   3964		[BPF_W] = 0xffffffffU,
   3965	};
   3966
   3967	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
   3968		struct bpf_insn insn, next;
   3969
   3970		insn = meta1->insn;
   3971		next = meta2->insn;
   3972
   3973		if (BPF_CLASS(insn.code) != BPF_LD)
   3974			continue;
   3975		if (BPF_MODE(insn.code) != BPF_ABS &&
   3976		    BPF_MODE(insn.code) != BPF_IND)
   3977			continue;
   3978
   3979		if (next.code != (BPF_ALU64 | BPF_AND | BPF_K))
   3980			continue;
   3981
   3982		if (!exp_mask[BPF_SIZE(insn.code)])
   3983			continue;
   3984		if (exp_mask[BPF_SIZE(insn.code)] != next.imm)
   3985			continue;
   3986
   3987		if (next.src_reg || next.dst_reg)
   3988			continue;
   3989
   3990		if (meta2->flags & FLAG_INSN_IS_JUMP_DST)
   3991			continue;
   3992
   3993		meta2->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
   3994	}
   3995}
   3996
   3997static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
   3998{
   3999	struct nfp_insn_meta *meta1, *meta2, *meta3;
   4000
   4001	nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) {
   4002		struct bpf_insn insn, next1, next2;
   4003
   4004		insn = meta1->insn;
   4005		next1 = meta2->insn;
   4006		next2 = meta3->insn;
   4007
   4008		if (BPF_CLASS(insn.code) != BPF_LD)
   4009			continue;
   4010		if (BPF_MODE(insn.code) != BPF_ABS &&
   4011		    BPF_MODE(insn.code) != BPF_IND)
   4012			continue;
   4013		if (BPF_SIZE(insn.code) != BPF_W)
   4014			continue;
   4015
   4016		if (!(next1.code == (BPF_LSH | BPF_K | BPF_ALU64) &&
   4017		      next2.code == (BPF_RSH | BPF_K | BPF_ALU64)) &&
   4018		    !(next1.code == (BPF_RSH | BPF_K | BPF_ALU64) &&
   4019		      next2.code == (BPF_LSH | BPF_K | BPF_ALU64)))
   4020			continue;
   4021
   4022		if (next1.src_reg || next1.dst_reg ||
   4023		    next2.src_reg || next2.dst_reg)
   4024			continue;
   4025
   4026		if (next1.imm != 0x20 || next2.imm != 0x20)
   4027			continue;
   4028
   4029		if (meta2->flags & FLAG_INSN_IS_JUMP_DST ||
   4030		    meta3->flags & FLAG_INSN_IS_JUMP_DST)
   4031			continue;
   4032
   4033		meta2->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
   4034		meta3->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
   4035	}
   4036}
   4037
   4038/* load/store pair that forms memory copy sould look like the following:
   4039 *
   4040 *   ld_width R, [addr_src + offset_src]
   4041 *   st_width [addr_dest + offset_dest], R
   4042 *
   4043 * The destination register of load and source register of store should
   4044 * be the same, load and store should also perform at the same width.
   4045 * If either of addr_src or addr_dest is stack pointer, we don't do the
   4046 * CPP optimization as stack is modelled by registers on NFP.
   4047 */
   4048static bool
   4049curr_pair_is_memcpy(struct nfp_insn_meta *ld_meta,
   4050		    struct nfp_insn_meta *st_meta)
   4051{
   4052	struct bpf_insn *ld = &ld_meta->insn;
   4053	struct bpf_insn *st = &st_meta->insn;
   4054
   4055	if (!is_mbpf_load(ld_meta) || !is_mbpf_store(st_meta))
   4056		return false;
   4057
   4058	if (ld_meta->ptr.type != PTR_TO_PACKET &&
   4059	    ld_meta->ptr.type != PTR_TO_MAP_VALUE)
   4060		return false;
   4061
   4062	if (st_meta->ptr.type != PTR_TO_PACKET)
   4063		return false;
   4064
   4065	if (BPF_SIZE(ld->code) != BPF_SIZE(st->code))
   4066		return false;
   4067
   4068	if (ld->dst_reg != st->src_reg)
   4069		return false;
   4070
   4071	/* There is jump to the store insn in this pair. */
   4072	if (st_meta->flags & FLAG_INSN_IS_JUMP_DST)
   4073		return false;
   4074
   4075	return true;
   4076}
   4077
   4078/* Currently, we only support chaining load/store pairs if:
   4079 *
   4080 *  - Their address base registers are the same.
   4081 *  - Their address offsets are in the same order.
   4082 *  - They operate at the same memory width.
   4083 *  - There is no jump into the middle of them.
   4084 */
   4085static bool
   4086curr_pair_chain_with_previous(struct nfp_insn_meta *ld_meta,
   4087			      struct nfp_insn_meta *st_meta,
   4088			      struct bpf_insn *prev_ld,
   4089			      struct bpf_insn *prev_st)
   4090{
   4091	u8 prev_size, curr_size, prev_ld_base, prev_st_base, prev_ld_dst;
   4092	struct bpf_insn *ld = &ld_meta->insn;
   4093	struct bpf_insn *st = &st_meta->insn;
   4094	s16 prev_ld_off, prev_st_off;
   4095
   4096	/* This pair is the start pair. */
   4097	if (!prev_ld)
   4098		return true;
   4099
   4100	prev_size = BPF_LDST_BYTES(prev_ld);
   4101	curr_size = BPF_LDST_BYTES(ld);
   4102	prev_ld_base = prev_ld->src_reg;
   4103	prev_st_base = prev_st->dst_reg;
   4104	prev_ld_dst = prev_ld->dst_reg;
   4105	prev_ld_off = prev_ld->off;
   4106	prev_st_off = prev_st->off;
   4107
   4108	if (ld->dst_reg != prev_ld_dst)
   4109		return false;
   4110
   4111	if (ld->src_reg != prev_ld_base || st->dst_reg != prev_st_base)
   4112		return false;
   4113
   4114	if (curr_size != prev_size)
   4115		return false;
   4116
   4117	/* There is jump to the head of this pair. */
   4118	if (ld_meta->flags & FLAG_INSN_IS_JUMP_DST)
   4119		return false;
   4120
   4121	/* Both in ascending order. */
   4122	if (prev_ld_off + prev_size == ld->off &&
   4123	    prev_st_off + prev_size == st->off)
   4124		return true;
   4125
   4126	/* Both in descending order. */
   4127	if (ld->off + curr_size == prev_ld_off &&
   4128	    st->off + curr_size == prev_st_off)
   4129		return true;
   4130
   4131	return false;
   4132}
   4133
   4134/* Return TRUE if cross memory access happens. Cross memory access means
   4135 * store area is overlapping with load area that a later load might load
   4136 * the value from previous store, for this case we can't treat the sequence
   4137 * as an memory copy.
   4138 */
   4139static bool
   4140cross_mem_access(struct bpf_insn *ld, struct nfp_insn_meta *head_ld_meta,
   4141		 struct nfp_insn_meta *head_st_meta)
   4142{
   4143	s16 head_ld_off, head_st_off, ld_off;
   4144
   4145	/* Different pointer types does not overlap. */
   4146	if (head_ld_meta->ptr.type != head_st_meta->ptr.type)
   4147		return false;
   4148
   4149	/* load and store are both PTR_TO_PACKET, check ID info.  */
   4150	if (head_ld_meta->ptr.id != head_st_meta->ptr.id)
   4151		return true;
   4152
   4153	/* Canonicalize the offsets. Turn all of them against the original
   4154	 * base register.
   4155	 */
   4156	head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.off;
   4157	head_st_off = head_st_meta->insn.off + head_st_meta->ptr.off;
   4158	ld_off = ld->off + head_ld_meta->ptr.off;
   4159
   4160	/* Ascending order cross. */
   4161	if (ld_off > head_ld_off &&
   4162	    head_ld_off < head_st_off && ld_off >= head_st_off)
   4163		return true;
   4164
   4165	/* Descending order cross. */
   4166	if (ld_off < head_ld_off &&
   4167	    head_ld_off > head_st_off && ld_off <= head_st_off)
   4168		return true;
   4169
   4170	return false;
   4171}
   4172
   4173/* This pass try to identify the following instructoin sequences.
   4174 *
   4175 *   load R, [regA + offA]
   4176 *   store [regB + offB], R
   4177 *   load R, [regA + offA + const_imm_A]
   4178 *   store [regB + offB + const_imm_A], R
   4179 *   load R, [regA + offA + 2 * const_imm_A]
   4180 *   store [regB + offB + 2 * const_imm_A], R
   4181 *   ...
   4182 *
   4183 * Above sequence is typically generated by compiler when lowering
   4184 * memcpy. NFP prefer using CPP instructions to accelerate it.
   4185 */
   4186static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog)
   4187{
   4188	struct nfp_insn_meta *head_ld_meta = NULL;
   4189	struct nfp_insn_meta *head_st_meta = NULL;
   4190	struct nfp_insn_meta *meta1, *meta2;
   4191	struct bpf_insn *prev_ld = NULL;
   4192	struct bpf_insn *prev_st = NULL;
   4193	u8 count = 0;
   4194
   4195	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
   4196		struct bpf_insn *ld = &meta1->insn;
   4197		struct bpf_insn *st = &meta2->insn;
   4198
   4199		/* Reset record status if any of the following if true:
   4200		 *   - The current insn pair is not load/store.
   4201		 *   - The load/store pair doesn't chain with previous one.
   4202		 *   - The chained load/store pair crossed with previous pair.
   4203		 *   - The chained load/store pair has a total size of memory
   4204		 *     copy beyond 128 bytes which is the maximum length a
   4205		 *     single NFP CPP command can transfer.
   4206		 */
   4207		if (!curr_pair_is_memcpy(meta1, meta2) ||
   4208		    !curr_pair_chain_with_previous(meta1, meta2, prev_ld,
   4209						   prev_st) ||
   4210		    (head_ld_meta && (cross_mem_access(ld, head_ld_meta,
   4211						       head_st_meta) ||
   4212				      head_ld_meta->ldst_gather_len >= 128))) {
   4213			if (!count)
   4214				continue;
   4215
   4216			if (count > 1) {
   4217				s16 prev_ld_off = prev_ld->off;
   4218				s16 prev_st_off = prev_st->off;
   4219				s16 head_ld_off = head_ld_meta->insn.off;
   4220
   4221				if (prev_ld_off < head_ld_off) {
   4222					head_ld_meta->insn.off = prev_ld_off;
   4223					head_st_meta->insn.off = prev_st_off;
   4224					head_ld_meta->ldst_gather_len =
   4225						-head_ld_meta->ldst_gather_len;
   4226				}
   4227
   4228				head_ld_meta->paired_st = &head_st_meta->insn;
   4229				head_st_meta->flags |=
   4230					FLAG_INSN_SKIP_PREC_DEPENDENT;
   4231			} else {
   4232				head_ld_meta->ldst_gather_len = 0;
   4233			}
   4234
   4235			/* If the chain is ended by an load/store pair then this
   4236			 * could serve as the new head of the the next chain.
   4237			 */
   4238			if (curr_pair_is_memcpy(meta1, meta2)) {
   4239				head_ld_meta = meta1;
   4240				head_st_meta = meta2;
   4241				head_ld_meta->ldst_gather_len =
   4242					BPF_LDST_BYTES(ld);
   4243				meta1 = nfp_meta_next(meta1);
   4244				meta2 = nfp_meta_next(meta2);
   4245				prev_ld = ld;
   4246				prev_st = st;
   4247				count = 1;
   4248			} else {
   4249				head_ld_meta = NULL;
   4250				head_st_meta = NULL;
   4251				prev_ld = NULL;
   4252				prev_st = NULL;
   4253				count = 0;
   4254			}
   4255
   4256			continue;
   4257		}
   4258
   4259		if (!head_ld_meta) {
   4260			head_ld_meta = meta1;
   4261			head_st_meta = meta2;
   4262		} else {
   4263			meta1->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
   4264			meta2->flags |= FLAG_INSN_SKIP_PREC_DEPENDENT;
   4265		}
   4266
   4267		head_ld_meta->ldst_gather_len += BPF_LDST_BYTES(ld);
   4268		meta1 = nfp_meta_next(meta1);
   4269		meta2 = nfp_meta_next(meta2);
   4270		prev_ld = ld;
   4271		prev_st = st;
   4272		count++;
   4273	}
   4274}
   4275
   4276static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog)
   4277{
   4278	struct nfp_insn_meta *meta, *range_node = NULL;
   4279	s16 range_start = 0, range_end = 0;
   4280	bool cache_avail = false;
   4281	struct bpf_insn *insn;
   4282	s32 range_ptr_off = 0;
   4283	u32 range_ptr_id = 0;
   4284
   4285	list_for_each_entry(meta, &nfp_prog->insns, l) {
   4286		if (meta->flags & FLAG_INSN_IS_JUMP_DST)
   4287			cache_avail = false;
   4288
   4289		if (meta->flags & FLAG_INSN_SKIP_MASK)
   4290			continue;
   4291
   4292		insn = &meta->insn;
   4293
   4294		if (is_mbpf_store_pkt(meta) ||
   4295		    insn->code == (BPF_JMP | BPF_CALL) ||
   4296		    is_mbpf_classic_store_pkt(meta) ||
   4297		    is_mbpf_classic_load(meta)) {
   4298			cache_avail = false;
   4299			continue;
   4300		}
   4301
   4302		if (!is_mbpf_load(meta))
   4303			continue;
   4304
   4305		if (meta->ptr.type != PTR_TO_PACKET || meta->ldst_gather_len) {
   4306			cache_avail = false;
   4307			continue;
   4308		}
   4309
   4310		if (!cache_avail) {
   4311			cache_avail = true;
   4312			if (range_node)
   4313				goto end_current_then_start_new;
   4314			goto start_new;
   4315		}
   4316
   4317		/* Check ID to make sure two reads share the same
   4318		 * variable offset against PTR_TO_PACKET, and check OFF
   4319		 * to make sure they also share the same constant
   4320		 * offset.
   4321		 *
   4322		 * OFFs don't really need to be the same, because they
   4323		 * are the constant offsets against PTR_TO_PACKET, so
   4324		 * for different OFFs, we could canonicalize them to
   4325		 * offsets against original packet pointer. We don't
   4326		 * support this.
   4327		 */
   4328		if (meta->ptr.id == range_ptr_id &&
   4329		    meta->ptr.off == range_ptr_off) {
   4330			s16 new_start = range_start;
   4331			s16 end, off = insn->off;
   4332			s16 new_end = range_end;
   4333			bool changed = false;
   4334
   4335			if (off < range_start) {
   4336				new_start = off;
   4337				changed = true;
   4338			}
   4339
   4340			end = off + BPF_LDST_BYTES(insn);
   4341			if (end > range_end) {
   4342				new_end = end;
   4343				changed = true;
   4344			}
   4345
   4346			if (!changed)
   4347				continue;
   4348
   4349			if (new_end - new_start <= 64) {
   4350				/* Install new range. */
   4351				range_start = new_start;
   4352				range_end = new_end;
   4353				continue;
   4354			}
   4355		}
   4356
   4357end_current_then_start_new:
   4358		range_node->pkt_cache.range_start = range_start;
   4359		range_node->pkt_cache.range_end = range_end;
   4360start_new:
   4361		range_node = meta;
   4362		range_node->pkt_cache.do_init = true;
   4363		range_ptr_id = range_node->ptr.id;
   4364		range_ptr_off = range_node->ptr.off;
   4365		range_start = insn->off;
   4366		range_end = insn->off + BPF_LDST_BYTES(insn);
   4367	}
   4368
   4369	if (range_node) {
   4370		range_node->pkt_cache.range_start = range_start;
   4371		range_node->pkt_cache.range_end = range_end;
   4372	}
   4373
   4374	list_for_each_entry(meta, &nfp_prog->insns, l) {
   4375		if (meta->flags & FLAG_INSN_SKIP_MASK)
   4376			continue;
   4377
   4378		if (is_mbpf_load_pkt(meta) && !meta->ldst_gather_len) {
   4379			if (meta->pkt_cache.do_init) {
   4380				range_start = meta->pkt_cache.range_start;
   4381				range_end = meta->pkt_cache.range_end;
   4382			} else {
   4383				meta->pkt_cache.range_start = range_start;
   4384				meta->pkt_cache.range_end = range_end;
   4385			}
   4386		}
   4387	}
   4388}
   4389
   4390static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
   4391{
   4392	nfp_bpf_opt_reg_init(nfp_prog);
   4393
   4394	nfp_bpf_opt_neg_add_sub(nfp_prog);
   4395	nfp_bpf_opt_ld_mask(nfp_prog);
   4396	nfp_bpf_opt_ld_shift(nfp_prog);
   4397	nfp_bpf_opt_ldst_gather(nfp_prog);
   4398	nfp_bpf_opt_pkt_cache(nfp_prog);
   4399
   4400	return 0;
   4401}
   4402
   4403static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog)
   4404{
   4405	struct nfp_insn_meta *meta1, *meta2;
   4406	struct nfp_bpf_map *nfp_map;
   4407	struct bpf_map *map;
   4408	u32 id;
   4409
   4410	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
   4411		if (meta1->flags & FLAG_INSN_SKIP_MASK ||
   4412		    meta2->flags & FLAG_INSN_SKIP_MASK)
   4413			continue;
   4414
   4415		if (meta1->insn.code != (BPF_LD | BPF_IMM | BPF_DW) ||
   4416		    meta1->insn.src_reg != BPF_PSEUDO_MAP_FD)
   4417			continue;
   4418
   4419		map = (void *)(unsigned long)((u32)meta1->insn.imm |
   4420					      (u64)meta2->insn.imm << 32);
   4421		if (bpf_map_offload_neutral(map)) {
   4422			id = map->id;
   4423		} else {
   4424			nfp_map = map_to_offmap(map)->dev_priv;
   4425			id = nfp_map->tid;
   4426		}
   4427
   4428		meta1->insn.imm = id;
   4429		meta2->insn.imm = 0;
   4430	}
   4431
   4432	return 0;
   4433}
   4434
   4435static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len)
   4436{
   4437	__le64 *ustore = (__force __le64 *)prog;
   4438	int i;
   4439
   4440	for (i = 0; i < len; i++) {
   4441		int err;
   4442
   4443		err = nfp_ustore_check_valid_no_ecc(prog[i]);
   4444		if (err)
   4445			return err;
   4446
   4447		ustore[i] = cpu_to_le64(nfp_ustore_calc_ecc_insn(prog[i]));
   4448	}
   4449
   4450	return 0;
   4451}
   4452
   4453static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog)
   4454{
   4455	void *prog;
   4456
   4457	prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL);
   4458	if (!prog)
   4459		return;
   4460
   4461	nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64);
   4462	memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len);
   4463	kvfree(nfp_prog->prog);
   4464	nfp_prog->prog = prog;
   4465}
   4466
   4467int nfp_bpf_jit(struct nfp_prog *nfp_prog)
   4468{
   4469	int ret;
   4470
   4471	ret = nfp_bpf_replace_map_ptrs(nfp_prog);
   4472	if (ret)
   4473		return ret;
   4474
   4475	ret = nfp_bpf_optimize(nfp_prog);
   4476	if (ret)
   4477		return ret;
   4478
   4479	ret = nfp_translate(nfp_prog);
   4480	if (ret) {
   4481		pr_err("Translation failed with error %d (translated: %u)\n",
   4482		       ret, nfp_prog->n_translated);
   4483		return -EINVAL;
   4484	}
   4485
   4486	nfp_bpf_prog_trim(nfp_prog);
   4487
   4488	return ret;
   4489}
   4490
   4491void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog)
   4492{
   4493	struct nfp_insn_meta *meta;
   4494
   4495	/* Another pass to record jump information. */
   4496	list_for_each_entry(meta, &nfp_prog->insns, l) {
   4497		struct nfp_insn_meta *dst_meta;
   4498		u64 code = meta->insn.code;
   4499		unsigned int dst_idx;
   4500		bool pseudo_call;
   4501
   4502		if (!is_mbpf_jmp(meta))
   4503			continue;
   4504		if (BPF_OP(code) == BPF_EXIT)
   4505			continue;
   4506		if (is_mbpf_helper_call(meta))
   4507			continue;
   4508
   4509		/* If opcode is BPF_CALL at this point, this can only be a
   4510		 * BPF-to-BPF call (a.k.a pseudo call).
   4511		 */
   4512		pseudo_call = BPF_OP(code) == BPF_CALL;
   4513
   4514		if (pseudo_call)
   4515			dst_idx = meta->n + 1 + meta->insn.imm;
   4516		else
   4517			dst_idx = meta->n + 1 + meta->insn.off;
   4518
   4519		dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx);
   4520
   4521		if (pseudo_call)
   4522			dst_meta->flags |= FLAG_INSN_IS_SUBPROG_START;
   4523
   4524		dst_meta->flags |= FLAG_INSN_IS_JUMP_DST;
   4525		meta->jmp_dst = dst_meta;
   4526	}
   4527}
   4528
   4529bool nfp_bpf_supported_opcode(u8 code)
   4530{
   4531	return !!instr_cb[code];
   4532}
   4533
   4534void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
   4535{
   4536	unsigned int i;
   4537	u64 *prog;
   4538	int err;
   4539
   4540	prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64),
   4541		       GFP_KERNEL);
   4542	if (!prog)
   4543		return ERR_PTR(-ENOMEM);
   4544
   4545	for (i = 0; i < nfp_prog->prog_len; i++) {
   4546		enum nfp_relo_type special;
   4547		u32 val;
   4548		u16 off;
   4549
   4550		special = FIELD_GET(OP_RELO_TYPE, prog[i]);
   4551		switch (special) {
   4552		case RELO_NONE:
   4553			continue;
   4554		case RELO_BR_REL:
   4555			br_add_offset(&prog[i], bv->start_off);
   4556			break;
   4557		case RELO_BR_GO_OUT:
   4558			br_set_offset(&prog[i],
   4559				      nfp_prog->tgt_out + bv->start_off);
   4560			break;
   4561		case RELO_BR_GO_ABORT:
   4562			br_set_offset(&prog[i],
   4563				      nfp_prog->tgt_abort + bv->start_off);
   4564			break;
   4565		case RELO_BR_GO_CALL_PUSH_REGS:
   4566			if (!nfp_prog->tgt_call_push_regs) {
   4567				pr_err("BUG: failed to detect subprogram registers needs\n");
   4568				err = -EINVAL;
   4569				goto err_free_prog;
   4570			}
   4571			off = nfp_prog->tgt_call_push_regs + bv->start_off;
   4572			br_set_offset(&prog[i], off);
   4573			break;
   4574		case RELO_BR_GO_CALL_POP_REGS:
   4575			if (!nfp_prog->tgt_call_pop_regs) {
   4576				pr_err("BUG: failed to detect subprogram registers needs\n");
   4577				err = -EINVAL;
   4578				goto err_free_prog;
   4579			}
   4580			off = nfp_prog->tgt_call_pop_regs + bv->start_off;
   4581			br_set_offset(&prog[i], off);
   4582			break;
   4583		case RELO_BR_NEXT_PKT:
   4584			br_set_offset(&prog[i], bv->tgt_done);
   4585			break;
   4586		case RELO_BR_HELPER:
   4587			val = br_get_offset(prog[i]);
   4588			val -= BR_OFF_RELO;
   4589			switch (val) {
   4590			case BPF_FUNC_map_lookup_elem:
   4591				val = nfp_prog->bpf->helpers.map_lookup;
   4592				break;
   4593			case BPF_FUNC_map_update_elem:
   4594				val = nfp_prog->bpf->helpers.map_update;
   4595				break;
   4596			case BPF_FUNC_map_delete_elem:
   4597				val = nfp_prog->bpf->helpers.map_delete;
   4598				break;
   4599			case BPF_FUNC_perf_event_output:
   4600				val = nfp_prog->bpf->helpers.perf_event_output;
   4601				break;
   4602			default:
   4603				pr_err("relocation of unknown helper %d\n",
   4604				       val);
   4605				err = -EINVAL;
   4606				goto err_free_prog;
   4607			}
   4608			br_set_offset(&prog[i], val);
   4609			break;
   4610		case RELO_IMMED_REL:
   4611			immed_add_value(&prog[i], bv->start_off);
   4612			break;
   4613		}
   4614
   4615		prog[i] &= ~OP_RELO_TYPE;
   4616	}
   4617
   4618	err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);
   4619	if (err)
   4620		goto err_free_prog;
   4621
   4622	return prog;
   4623
   4624err_free_prog:
   4625	kfree(prog);
   4626	return ERR_PTR(err);
   4627}