cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

spectrum_router.c (301400B)


      1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
      2/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
      3
      4#include <linux/kernel.h>
      5#include <linux/types.h>
      6#include <linux/rhashtable.h>
      7#include <linux/bitops.h>
      8#include <linux/in6.h>
      9#include <linux/notifier.h>
     10#include <linux/inetdevice.h>
     11#include <linux/netdevice.h>
     12#include <linux/if_bridge.h>
     13#include <linux/socket.h>
     14#include <linux/route.h>
     15#include <linux/gcd.h>
     16#include <linux/if_macvlan.h>
     17#include <linux/refcount.h>
     18#include <linux/jhash.h>
     19#include <linux/net_namespace.h>
     20#include <linux/mutex.h>
     21#include <net/netevent.h>
     22#include <net/neighbour.h>
     23#include <net/arp.h>
     24#include <net/inet_dscp.h>
     25#include <net/ip_fib.h>
     26#include <net/ip6_fib.h>
     27#include <net/nexthop.h>
     28#include <net/fib_rules.h>
     29#include <net/ip_tunnels.h>
     30#include <net/l3mdev.h>
     31#include <net/addrconf.h>
     32#include <net/ndisc.h>
     33#include <net/ipv6.h>
     34#include <net/fib_notifier.h>
     35#include <net/switchdev.h>
     36
     37#include "spectrum.h"
     38#include "core.h"
     39#include "reg.h"
     40#include "spectrum_cnt.h"
     41#include "spectrum_dpipe.h"
     42#include "spectrum_ipip.h"
     43#include "spectrum_mr.h"
     44#include "spectrum_mr_tcam.h"
     45#include "spectrum_router.h"
     46#include "spectrum_span.h"
     47
     48struct mlxsw_sp_fib;
     49struct mlxsw_sp_vr;
     50struct mlxsw_sp_lpm_tree;
     51struct mlxsw_sp_rif_ops;
     52
     53struct mlxsw_sp_rif {
     54	struct list_head nexthop_list;
     55	struct list_head neigh_list;
     56	struct net_device *dev; /* NULL for underlay RIF */
     57	struct mlxsw_sp_fid *fid;
     58	unsigned char addr[ETH_ALEN];
     59	int mtu;
     60	u16 rif_index;
     61	u8 mac_profile_id;
     62	u16 vr_id;
     63	const struct mlxsw_sp_rif_ops *ops;
     64	struct mlxsw_sp *mlxsw_sp;
     65
     66	unsigned int counter_ingress;
     67	bool counter_ingress_valid;
     68	unsigned int counter_egress;
     69	bool counter_egress_valid;
     70};
     71
     72struct mlxsw_sp_rif_params {
     73	struct net_device *dev;
     74	union {
     75		u16 system_port;
     76		u16 lag_id;
     77	};
     78	u16 vid;
     79	bool lag;
     80};
     81
     82struct mlxsw_sp_rif_subport {
     83	struct mlxsw_sp_rif common;
     84	refcount_t ref_count;
     85	union {
     86		u16 system_port;
     87		u16 lag_id;
     88	};
     89	u16 vid;
     90	bool lag;
     91};
     92
     93struct mlxsw_sp_rif_ipip_lb {
     94	struct mlxsw_sp_rif common;
     95	struct mlxsw_sp_rif_ipip_lb_config lb_config;
     96	u16 ul_vr_id; /* Reserved for Spectrum-2. */
     97	u16 ul_rif_id; /* Reserved for Spectrum. */
     98};
     99
    100struct mlxsw_sp_rif_params_ipip_lb {
    101	struct mlxsw_sp_rif_params common;
    102	struct mlxsw_sp_rif_ipip_lb_config lb_config;
    103};
    104
    105struct mlxsw_sp_rif_ops {
    106	enum mlxsw_sp_rif_type type;
    107	size_t rif_size;
    108
    109	void (*setup)(struct mlxsw_sp_rif *rif,
    110		      const struct mlxsw_sp_rif_params *params);
    111	int (*configure)(struct mlxsw_sp_rif *rif,
    112			 struct netlink_ext_ack *extack);
    113	void (*deconfigure)(struct mlxsw_sp_rif *rif);
    114	struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
    115					 struct netlink_ext_ack *extack);
    116	void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
    117};
    118
    119struct mlxsw_sp_rif_mac_profile {
    120	unsigned char mac_prefix[ETH_ALEN];
    121	refcount_t ref_count;
    122	u8 id;
    123};
    124
    125struct mlxsw_sp_router_ops {
    126	int (*init)(struct mlxsw_sp *mlxsw_sp);
    127	int (*ipips_init)(struct mlxsw_sp *mlxsw_sp);
    128};
    129
    130static struct mlxsw_sp_rif *
    131mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
    132			 const struct net_device *dev);
    133static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
    134static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
    135static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
    136				  struct mlxsw_sp_lpm_tree *lpm_tree);
    137static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
    138				     const struct mlxsw_sp_fib *fib,
    139				     u8 tree_id);
    140static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
    141				       const struct mlxsw_sp_fib *fib);
    142
    143static unsigned int *
    144mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
    145			   enum mlxsw_sp_rif_counter_dir dir)
    146{
    147	switch (dir) {
    148	case MLXSW_SP_RIF_COUNTER_EGRESS:
    149		return &rif->counter_egress;
    150	case MLXSW_SP_RIF_COUNTER_INGRESS:
    151		return &rif->counter_ingress;
    152	}
    153	return NULL;
    154}
    155
    156static bool
    157mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
    158			       enum mlxsw_sp_rif_counter_dir dir)
    159{
    160	switch (dir) {
    161	case MLXSW_SP_RIF_COUNTER_EGRESS:
    162		return rif->counter_egress_valid;
    163	case MLXSW_SP_RIF_COUNTER_INGRESS:
    164		return rif->counter_ingress_valid;
    165	}
    166	return false;
    167}
    168
    169static void
    170mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
    171			       enum mlxsw_sp_rif_counter_dir dir,
    172			       bool valid)
    173{
    174	switch (dir) {
    175	case MLXSW_SP_RIF_COUNTER_EGRESS:
    176		rif->counter_egress_valid = valid;
    177		break;
    178	case MLXSW_SP_RIF_COUNTER_INGRESS:
    179		rif->counter_ingress_valid = valid;
    180		break;
    181	}
    182}
    183
    184static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
    185				     unsigned int counter_index, bool enable,
    186				     enum mlxsw_sp_rif_counter_dir dir)
    187{
    188	char ritr_pl[MLXSW_REG_RITR_LEN];
    189	bool is_egress = false;
    190	int err;
    191
    192	if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
    193		is_egress = true;
    194	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
    195	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
    196	if (err)
    197		return err;
    198
    199	mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
    200				    is_egress);
    201	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
    202}
    203
    204int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
    205				   struct mlxsw_sp_rif *rif,
    206				   enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
    207{
    208	char ricnt_pl[MLXSW_REG_RICNT_LEN];
    209	unsigned int *p_counter_index;
    210	bool valid;
    211	int err;
    212
    213	valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
    214	if (!valid)
    215		return -EINVAL;
    216
    217	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
    218	if (!p_counter_index)
    219		return -EINVAL;
    220	mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
    221			     MLXSW_REG_RICNT_OPCODE_NOP);
    222	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
    223	if (err)
    224		return err;
    225	*cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
    226	return 0;
    227}
    228
    229struct mlxsw_sp_rif_counter_set_basic {
    230	u64 good_unicast_packets;
    231	u64 good_multicast_packets;
    232	u64 good_broadcast_packets;
    233	u64 good_unicast_bytes;
    234	u64 good_multicast_bytes;
    235	u64 good_broadcast_bytes;
    236	u64 error_packets;
    237	u64 discard_packets;
    238	u64 error_bytes;
    239	u64 discard_bytes;
    240};
    241
    242static int
    243mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif,
    244				 enum mlxsw_sp_rif_counter_dir dir,
    245				 struct mlxsw_sp_rif_counter_set_basic *set)
    246{
    247	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
    248	char ricnt_pl[MLXSW_REG_RICNT_LEN];
    249	unsigned int *p_counter_index;
    250	int err;
    251
    252	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
    253		return -EINVAL;
    254
    255	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
    256	if (!p_counter_index)
    257		return -EINVAL;
    258
    259	mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
    260			     MLXSW_REG_RICNT_OPCODE_CLEAR);
    261	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
    262	if (err)
    263		return err;
    264
    265	if (!set)
    266		return 0;
    267
    268#define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME)				\
    269		(set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl))
    270
    271	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets);
    272	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets);
    273	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets);
    274	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes);
    275	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes);
    276	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes);
    277	MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets);
    278	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets);
    279	MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes);
    280	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes);
    281
    282#undef MLXSW_SP_RIF_COUNTER_EXTRACT
    283
    284	return 0;
    285}
    286
    287static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
    288				      unsigned int counter_index)
    289{
    290	char ricnt_pl[MLXSW_REG_RICNT_LEN];
    291
    292	mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
    293			     MLXSW_REG_RICNT_OPCODE_CLEAR);
    294	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
    295}
    296
    297int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif,
    298			       enum mlxsw_sp_rif_counter_dir dir)
    299{
    300	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
    301	unsigned int *p_counter_index;
    302	int err;
    303
    304	if (mlxsw_sp_rif_counter_valid_get(rif, dir))
    305		return 0;
    306
    307	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
    308	if (!p_counter_index)
    309		return -EINVAL;
    310
    311	err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
    312				     p_counter_index);
    313	if (err)
    314		return err;
    315
    316	err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
    317	if (err)
    318		goto err_counter_clear;
    319
    320	err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
    321					*p_counter_index, true, dir);
    322	if (err)
    323		goto err_counter_edit;
    324	mlxsw_sp_rif_counter_valid_set(rif, dir, true);
    325	return 0;
    326
    327err_counter_edit:
    328err_counter_clear:
    329	mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
    330			      *p_counter_index);
    331	return err;
    332}
    333
    334void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif,
    335			       enum mlxsw_sp_rif_counter_dir dir)
    336{
    337	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
    338	unsigned int *p_counter_index;
    339
    340	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
    341		return;
    342
    343	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
    344	if (WARN_ON(!p_counter_index))
    345		return;
    346	mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
    347				  *p_counter_index, false, dir);
    348	mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
    349			      *p_counter_index);
    350	mlxsw_sp_rif_counter_valid_set(rif, dir, false);
    351}
    352
    353static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
    354{
    355	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
    356	struct devlink *devlink;
    357
    358	devlink = priv_to_devlink(mlxsw_sp->core);
    359	if (!devlink_dpipe_table_counter_enabled(devlink,
    360						 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
    361		return;
    362	mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
    363}
    364
    365static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
    366{
    367	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
    368}
    369
    370#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
    371
    372struct mlxsw_sp_prefix_usage {
    373	DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
    374};
    375
    376#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
    377	for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
    378
    379static bool
    380mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
    381			 struct mlxsw_sp_prefix_usage *prefix_usage2)
    382{
    383	return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
    384}
    385
    386static void
    387mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
    388			  struct mlxsw_sp_prefix_usage *prefix_usage2)
    389{
    390	memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
    391}
    392
    393static void
    394mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
    395			  unsigned char prefix_len)
    396{
    397	set_bit(prefix_len, prefix_usage->b);
    398}
    399
    400static void
    401mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
    402			    unsigned char prefix_len)
    403{
    404	clear_bit(prefix_len, prefix_usage->b);
    405}
    406
    407struct mlxsw_sp_fib_key {
    408	unsigned char addr[sizeof(struct in6_addr)];
    409	unsigned char prefix_len;
    410};
    411
    412enum mlxsw_sp_fib_entry_type {
    413	MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
    414	MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
    415	MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
    416	MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
    417	MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
    418
    419	/* This is a special case of local delivery, where a packet should be
    420	 * decapsulated on reception. Note that there is no corresponding ENCAP,
    421	 * because that's a type of next hop, not of FIB entry. (There can be
    422	 * several next hops in a REMOTE entry, and some of them may be
    423	 * encapsulating entries.)
    424	 */
    425	MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
    426	MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
    427};
    428
    429struct mlxsw_sp_nexthop_group_info;
    430struct mlxsw_sp_nexthop_group;
    431struct mlxsw_sp_fib_entry;
    432
    433struct mlxsw_sp_fib_node {
    434	struct mlxsw_sp_fib_entry *fib_entry;
    435	struct list_head list;
    436	struct rhash_head ht_node;
    437	struct mlxsw_sp_fib *fib;
    438	struct mlxsw_sp_fib_key key;
    439};
    440
    441struct mlxsw_sp_fib_entry_decap {
    442	struct mlxsw_sp_ipip_entry *ipip_entry;
    443	u32 tunnel_index;
    444};
    445
    446static struct mlxsw_sp_fib_entry_priv *
    447mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops)
    448{
    449	struct mlxsw_sp_fib_entry_priv *priv;
    450
    451	if (!ll_ops->fib_entry_priv_size)
    452		/* No need to have priv */
    453		return NULL;
    454
    455	priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL);
    456	if (!priv)
    457		return ERR_PTR(-ENOMEM);
    458	refcount_set(&priv->refcnt, 1);
    459	return priv;
    460}
    461
    462static void
    463mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv)
    464{
    465	kfree(priv);
    466}
    467
    468static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv)
    469{
    470	refcount_inc(&priv->refcnt);
    471}
    472
    473static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv)
    474{
    475	if (!priv || !refcount_dec_and_test(&priv->refcnt))
    476		return;
    477	mlxsw_sp_fib_entry_priv_destroy(priv);
    478}
    479
    480static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
    481						struct mlxsw_sp_fib_entry_priv *priv)
    482{
    483	if (!priv)
    484		return;
    485	mlxsw_sp_fib_entry_priv_hold(priv);
    486	list_add(&priv->list, &op_ctx->fib_entry_priv_list);
    487}
    488
    489static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
    490{
    491	struct mlxsw_sp_fib_entry_priv *priv, *tmp;
    492
    493	list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list)
    494		mlxsw_sp_fib_entry_priv_put(priv);
    495	INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list);
    496}
    497
    498struct mlxsw_sp_fib_entry {
    499	struct mlxsw_sp_fib_node *fib_node;
    500	enum mlxsw_sp_fib_entry_type type;
    501	struct list_head nexthop_group_node;
    502	struct mlxsw_sp_nexthop_group *nh_group;
    503	struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
    504	struct mlxsw_sp_fib_entry_priv *priv;
    505};
    506
    507struct mlxsw_sp_fib4_entry {
    508	struct mlxsw_sp_fib_entry common;
    509	struct fib_info *fi;
    510	u32 tb_id;
    511	dscp_t dscp;
    512	u8 type;
    513};
    514
    515struct mlxsw_sp_fib6_entry {
    516	struct mlxsw_sp_fib_entry common;
    517	struct list_head rt6_list;
    518	unsigned int nrt6;
    519};
    520
    521struct mlxsw_sp_rt6 {
    522	struct list_head list;
    523	struct fib6_info *rt;
    524};
    525
    526struct mlxsw_sp_lpm_tree {
    527	u8 id; /* tree ID */
    528	unsigned int ref_count;
    529	enum mlxsw_sp_l3proto proto;
    530	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
    531	struct mlxsw_sp_prefix_usage prefix_usage;
    532};
    533
    534struct mlxsw_sp_fib {
    535	struct rhashtable ht;
    536	struct list_head node_list;
    537	struct mlxsw_sp_vr *vr;
    538	struct mlxsw_sp_lpm_tree *lpm_tree;
    539	enum mlxsw_sp_l3proto proto;
    540	const struct mlxsw_sp_router_ll_ops *ll_ops;
    541};
    542
    543struct mlxsw_sp_vr {
    544	u16 id; /* virtual router ID */
    545	u32 tb_id; /* kernel fib table id */
    546	unsigned int rif_count;
    547	struct mlxsw_sp_fib *fib4;
    548	struct mlxsw_sp_fib *fib6;
    549	struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
    550	struct mlxsw_sp_rif *ul_rif;
    551	refcount_t ul_rif_refcnt;
    552};
    553
    554static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
    555					 enum mlxsw_sp_l3proto proto)
    556{
    557	return 0;
    558}
    559
    560static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl)
    561{
    562	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta),
    563			       xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET);
    564}
    565
    566static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl)
    567{
    568	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst),
    569			       xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET);
    570}
    571
    572static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl)
    573{
    574	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
    575			       xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET);
    576}
    577
    578static const struct rhashtable_params mlxsw_sp_fib_ht_params;
    579
    580static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
    581						struct mlxsw_sp_vr *vr,
    582						enum mlxsw_sp_l3proto proto)
    583{
    584	const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
    585	struct mlxsw_sp_lpm_tree *lpm_tree;
    586	struct mlxsw_sp_fib *fib;
    587	int err;
    588
    589	err = ll_ops->init(mlxsw_sp, vr->id, proto);
    590	if (err)
    591		return ERR_PTR(err);
    592
    593	lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
    594	fib = kzalloc(sizeof(*fib), GFP_KERNEL);
    595	if (!fib)
    596		return ERR_PTR(-ENOMEM);
    597	err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
    598	if (err)
    599		goto err_rhashtable_init;
    600	INIT_LIST_HEAD(&fib->node_list);
    601	fib->proto = proto;
    602	fib->vr = vr;
    603	fib->lpm_tree = lpm_tree;
    604	fib->ll_ops = ll_ops;
    605	mlxsw_sp_lpm_tree_hold(lpm_tree);
    606	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
    607	if (err)
    608		goto err_lpm_tree_bind;
    609	return fib;
    610
    611err_lpm_tree_bind:
    612	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
    613err_rhashtable_init:
    614	kfree(fib);
    615	return ERR_PTR(err);
    616}
    617
    618static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
    619				 struct mlxsw_sp_fib *fib)
    620{
    621	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
    622	mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
    623	WARN_ON(!list_empty(&fib->node_list));
    624	rhashtable_destroy(&fib->ht);
    625	kfree(fib);
    626}
    627
    628static struct mlxsw_sp_lpm_tree *
    629mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
    630{
    631	static struct mlxsw_sp_lpm_tree *lpm_tree;
    632	int i;
    633
    634	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
    635		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
    636		if (lpm_tree->ref_count == 0)
    637			return lpm_tree;
    638	}
    639	return NULL;
    640}
    641
    642static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
    643				   const struct mlxsw_sp_router_ll_ops *ll_ops,
    644				   struct mlxsw_sp_lpm_tree *lpm_tree)
    645{
    646	char xralta_pl[MLXSW_REG_XRALTA_LEN];
    647
    648	mlxsw_reg_xralta_pack(xralta_pl, true,
    649			      (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
    650			      lpm_tree->id);
    651	return ll_ops->ralta_write(mlxsw_sp, xralta_pl);
    652}
    653
    654static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
    655				   const struct mlxsw_sp_router_ll_ops *ll_ops,
    656				   struct mlxsw_sp_lpm_tree *lpm_tree)
    657{
    658	char xralta_pl[MLXSW_REG_XRALTA_LEN];
    659
    660	mlxsw_reg_xralta_pack(xralta_pl, false,
    661			      (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
    662			      lpm_tree->id);
    663	ll_ops->ralta_write(mlxsw_sp, xralta_pl);
    664}
    665
    666static int
    667mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
    668				  const struct mlxsw_sp_router_ll_ops *ll_ops,
    669				  struct mlxsw_sp_prefix_usage *prefix_usage,
    670				  struct mlxsw_sp_lpm_tree *lpm_tree)
    671{
    672	char xralst_pl[MLXSW_REG_XRALST_LEN];
    673	u8 root_bin = 0;
    674	u8 prefix;
    675	u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
    676
    677	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
    678		root_bin = prefix;
    679
    680	mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id);
    681	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
    682		if (prefix == 0)
    683			continue;
    684		mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix,
    685					  MLXSW_REG_RALST_BIN_NO_CHILD);
    686		last_prefix = prefix;
    687	}
    688	return ll_ops->ralst_write(mlxsw_sp, xralst_pl);
    689}
    690
    691static struct mlxsw_sp_lpm_tree *
    692mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
    693			 const struct mlxsw_sp_router_ll_ops *ll_ops,
    694			 struct mlxsw_sp_prefix_usage *prefix_usage,
    695			 enum mlxsw_sp_l3proto proto)
    696{
    697	struct mlxsw_sp_lpm_tree *lpm_tree;
    698	int err;
    699
    700	lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
    701	if (!lpm_tree)
    702		return ERR_PTR(-EBUSY);
    703	lpm_tree->proto = proto;
    704	err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree);
    705	if (err)
    706		return ERR_PTR(err);
    707
    708	err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree);
    709	if (err)
    710		goto err_left_struct_set;
    711	memcpy(&lpm_tree->prefix_usage, prefix_usage,
    712	       sizeof(lpm_tree->prefix_usage));
    713	memset(&lpm_tree->prefix_ref_count, 0,
    714	       sizeof(lpm_tree->prefix_ref_count));
    715	lpm_tree->ref_count = 1;
    716	return lpm_tree;
    717
    718err_left_struct_set:
    719	mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
    720	return ERR_PTR(err);
    721}
    722
    723static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
    724				      const struct mlxsw_sp_router_ll_ops *ll_ops,
    725				      struct mlxsw_sp_lpm_tree *lpm_tree)
    726{
    727	mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree);
    728}
    729
    730static struct mlxsw_sp_lpm_tree *
    731mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
    732		      struct mlxsw_sp_prefix_usage *prefix_usage,
    733		      enum mlxsw_sp_l3proto proto)
    734{
    735	const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto];
    736	struct mlxsw_sp_lpm_tree *lpm_tree;
    737	int i;
    738
    739	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
    740		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
    741		if (lpm_tree->ref_count != 0 &&
    742		    lpm_tree->proto == proto &&
    743		    mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
    744					     prefix_usage)) {
    745			mlxsw_sp_lpm_tree_hold(lpm_tree);
    746			return lpm_tree;
    747		}
    748	}
    749	return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto);
    750}
    751
    752static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
    753{
    754	lpm_tree->ref_count++;
    755}
    756
    757static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
    758				  struct mlxsw_sp_lpm_tree *lpm_tree)
    759{
    760	const struct mlxsw_sp_router_ll_ops *ll_ops =
    761				mlxsw_sp->router->proto_ll_ops[lpm_tree->proto];
    762
    763	if (--lpm_tree->ref_count == 0)
    764		mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree);
    765}
    766
    767#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
    768
    769static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
    770{
    771	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
    772	struct mlxsw_sp_lpm_tree *lpm_tree;
    773	u64 max_trees;
    774	int err, i;
    775
    776	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
    777		return -EIO;
    778
    779	max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
    780	mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
    781	mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
    782					     sizeof(struct mlxsw_sp_lpm_tree),
    783					     GFP_KERNEL);
    784	if (!mlxsw_sp->router->lpm.trees)
    785		return -ENOMEM;
    786
    787	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
    788		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
    789		lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
    790	}
    791
    792	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
    793					 MLXSW_SP_L3_PROTO_IPV4);
    794	if (IS_ERR(lpm_tree)) {
    795		err = PTR_ERR(lpm_tree);
    796		goto err_ipv4_tree_get;
    797	}
    798	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
    799
    800	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
    801					 MLXSW_SP_L3_PROTO_IPV6);
    802	if (IS_ERR(lpm_tree)) {
    803		err = PTR_ERR(lpm_tree);
    804		goto err_ipv6_tree_get;
    805	}
    806	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
    807
    808	return 0;
    809
    810err_ipv6_tree_get:
    811	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
    812	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
    813err_ipv4_tree_get:
    814	kfree(mlxsw_sp->router->lpm.trees);
    815	return err;
    816}
    817
    818static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
    819{
    820	struct mlxsw_sp_lpm_tree *lpm_tree;
    821
    822	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
    823	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
    824
    825	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
    826	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
    827
    828	kfree(mlxsw_sp->router->lpm.trees);
    829}
    830
    831static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
    832{
    833	return !!vr->fib4 || !!vr->fib6 ||
    834	       !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
    835	       !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
    836}
    837
    838static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
    839{
    840	struct mlxsw_sp_vr *vr;
    841	int i;
    842
    843	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
    844		vr = &mlxsw_sp->router->vrs[i];
    845		if (!mlxsw_sp_vr_is_used(vr))
    846			return vr;
    847	}
    848	return NULL;
    849}
    850
    851static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
    852				     const struct mlxsw_sp_fib *fib, u8 tree_id)
    853{
    854	char xraltb_pl[MLXSW_REG_XRALTB_LEN];
    855
    856	mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
    857			      (enum mlxsw_reg_ralxx_protocol) fib->proto,
    858			      tree_id);
    859	return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
    860}
    861
    862static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
    863				       const struct mlxsw_sp_fib *fib)
    864{
    865	char xraltb_pl[MLXSW_REG_XRALTB_LEN];
    866
    867	/* Bind to tree 0 which is default */
    868	mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id,
    869			      (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
    870	return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl);
    871}
    872
    873static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
    874{
    875	/* For our purpose, squash main, default and local tables into one */
    876	if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
    877		tb_id = RT_TABLE_MAIN;
    878	return tb_id;
    879}
    880
    881static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
    882					    u32 tb_id)
    883{
    884	struct mlxsw_sp_vr *vr;
    885	int i;
    886
    887	tb_id = mlxsw_sp_fix_tb_id(tb_id);
    888
    889	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
    890		vr = &mlxsw_sp->router->vrs[i];
    891		if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
    892			return vr;
    893	}
    894	return NULL;
    895}
    896
    897int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
    898				u16 *vr_id)
    899{
    900	struct mlxsw_sp_vr *vr;
    901	int err = 0;
    902
    903	mutex_lock(&mlxsw_sp->router->lock);
    904	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
    905	if (!vr) {
    906		err = -ESRCH;
    907		goto out;
    908	}
    909	*vr_id = vr->id;
    910out:
    911	mutex_unlock(&mlxsw_sp->router->lock);
    912	return err;
    913}
    914
    915static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
    916					    enum mlxsw_sp_l3proto proto)
    917{
    918	switch (proto) {
    919	case MLXSW_SP_L3_PROTO_IPV4:
    920		return vr->fib4;
    921	case MLXSW_SP_L3_PROTO_IPV6:
    922		return vr->fib6;
    923	}
    924	return NULL;
    925}
    926
    927static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
    928					      u32 tb_id,
    929					      struct netlink_ext_ack *extack)
    930{
    931	struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
    932	struct mlxsw_sp_fib *fib4;
    933	struct mlxsw_sp_fib *fib6;
    934	struct mlxsw_sp_vr *vr;
    935	int err;
    936
    937	vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
    938	if (!vr) {
    939		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
    940		return ERR_PTR(-EBUSY);
    941	}
    942	fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
    943	if (IS_ERR(fib4))
    944		return ERR_CAST(fib4);
    945	fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
    946	if (IS_ERR(fib6)) {
    947		err = PTR_ERR(fib6);
    948		goto err_fib6_create;
    949	}
    950	mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
    951					     MLXSW_SP_L3_PROTO_IPV4);
    952	if (IS_ERR(mr4_table)) {
    953		err = PTR_ERR(mr4_table);
    954		goto err_mr4_table_create;
    955	}
    956	mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
    957					     MLXSW_SP_L3_PROTO_IPV6);
    958	if (IS_ERR(mr6_table)) {
    959		err = PTR_ERR(mr6_table);
    960		goto err_mr6_table_create;
    961	}
    962
    963	vr->fib4 = fib4;
    964	vr->fib6 = fib6;
    965	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
    966	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
    967	vr->tb_id = tb_id;
    968	return vr;
    969
    970err_mr6_table_create:
    971	mlxsw_sp_mr_table_destroy(mr4_table);
    972err_mr4_table_create:
    973	mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
    974err_fib6_create:
    975	mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
    976	return ERR_PTR(err);
    977}
    978
    979static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
    980				struct mlxsw_sp_vr *vr)
    981{
    982	mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
    983	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
    984	mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
    985	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
    986	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
    987	vr->fib6 = NULL;
    988	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
    989	vr->fib4 = NULL;
    990}
    991
    992static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
    993					   struct netlink_ext_ack *extack)
    994{
    995	struct mlxsw_sp_vr *vr;
    996
    997	tb_id = mlxsw_sp_fix_tb_id(tb_id);
    998	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
    999	if (!vr)
   1000		vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
   1001	return vr;
   1002}
   1003
   1004static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
   1005{
   1006	if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
   1007	    list_empty(&vr->fib6->node_list) &&
   1008	    mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
   1009	    mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
   1010		mlxsw_sp_vr_destroy(mlxsw_sp, vr);
   1011}
   1012
   1013static bool
   1014mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
   1015				    enum mlxsw_sp_l3proto proto, u8 tree_id)
   1016{
   1017	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
   1018
   1019	if (!mlxsw_sp_vr_is_used(vr))
   1020		return false;
   1021	if (fib->lpm_tree->id == tree_id)
   1022		return true;
   1023	return false;
   1024}
   1025
   1026static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
   1027					struct mlxsw_sp_fib *fib,
   1028					struct mlxsw_sp_lpm_tree *new_tree)
   1029{
   1030	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
   1031	int err;
   1032
   1033	fib->lpm_tree = new_tree;
   1034	mlxsw_sp_lpm_tree_hold(new_tree);
   1035	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
   1036	if (err)
   1037		goto err_tree_bind;
   1038	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
   1039	return 0;
   1040
   1041err_tree_bind:
   1042	mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
   1043	fib->lpm_tree = old_tree;
   1044	return err;
   1045}
   1046
   1047static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
   1048					 struct mlxsw_sp_fib *fib,
   1049					 struct mlxsw_sp_lpm_tree *new_tree)
   1050{
   1051	enum mlxsw_sp_l3proto proto = fib->proto;
   1052	struct mlxsw_sp_lpm_tree *old_tree;
   1053	u8 old_id, new_id = new_tree->id;
   1054	struct mlxsw_sp_vr *vr;
   1055	int i, err;
   1056
   1057	old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
   1058	old_id = old_tree->id;
   1059
   1060	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
   1061		vr = &mlxsw_sp->router->vrs[i];
   1062		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
   1063			continue;
   1064		err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
   1065						   mlxsw_sp_vr_fib(vr, proto),
   1066						   new_tree);
   1067		if (err)
   1068			goto err_tree_replace;
   1069	}
   1070
   1071	memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
   1072	       sizeof(new_tree->prefix_ref_count));
   1073	mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
   1074	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
   1075
   1076	return 0;
   1077
   1078err_tree_replace:
   1079	for (i--; i >= 0; i--) {
   1080		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
   1081			continue;
   1082		mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
   1083					     mlxsw_sp_vr_fib(vr, proto),
   1084					     old_tree);
   1085	}
   1086	return err;
   1087}
   1088
   1089static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
   1090{
   1091	struct mlxsw_sp_vr *vr;
   1092	u64 max_vrs;
   1093	int i;
   1094
   1095	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
   1096		return -EIO;
   1097
   1098	max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
   1099	mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
   1100					GFP_KERNEL);
   1101	if (!mlxsw_sp->router->vrs)
   1102		return -ENOMEM;
   1103
   1104	for (i = 0; i < max_vrs; i++) {
   1105		vr = &mlxsw_sp->router->vrs[i];
   1106		vr->id = i;
   1107	}
   1108
   1109	return 0;
   1110}
   1111
   1112static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
   1113
   1114static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
   1115{
   1116	/* At this stage we're guaranteed not to have new incoming
   1117	 * FIB notifications and the work queue is free from FIBs
   1118	 * sitting on top of mlxsw netdevs. However, we can still
   1119	 * have other FIBs queued. Flush the queue before flushing
   1120	 * the device's tables. No need for locks, as we're the only
   1121	 * writer.
   1122	 */
   1123	mlxsw_core_flush_owq();
   1124	mlxsw_sp_router_fib_flush(mlxsw_sp);
   1125	kfree(mlxsw_sp->router->vrs);
   1126}
   1127
   1128u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
   1129{
   1130	struct net_device *d;
   1131	u32 tb_id;
   1132
   1133	rcu_read_lock();
   1134	d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
   1135	if (d)
   1136		tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
   1137	else
   1138		tb_id = RT_TABLE_MAIN;
   1139	rcu_read_unlock();
   1140
   1141	return tb_id;
   1142}
   1143
   1144static struct mlxsw_sp_rif *
   1145mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
   1146		    const struct mlxsw_sp_rif_params *params,
   1147		    struct netlink_ext_ack *extack);
   1148
   1149static struct mlxsw_sp_rif_ipip_lb *
   1150mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
   1151				enum mlxsw_sp_ipip_type ipipt,
   1152				struct net_device *ol_dev,
   1153				struct netlink_ext_ack *extack)
   1154{
   1155	struct mlxsw_sp_rif_params_ipip_lb lb_params;
   1156	const struct mlxsw_sp_ipip_ops *ipip_ops;
   1157	struct mlxsw_sp_rif *rif;
   1158
   1159	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
   1160	lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
   1161		.common.dev = ol_dev,
   1162		.common.lag = false,
   1163		.lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
   1164	};
   1165
   1166	rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
   1167	if (IS_ERR(rif))
   1168		return ERR_CAST(rif);
   1169	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
   1170}
   1171
   1172static struct mlxsw_sp_ipip_entry *
   1173mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
   1174			  enum mlxsw_sp_ipip_type ipipt,
   1175			  struct net_device *ol_dev)
   1176{
   1177	const struct mlxsw_sp_ipip_ops *ipip_ops;
   1178	struct mlxsw_sp_ipip_entry *ipip_entry;
   1179	struct mlxsw_sp_ipip_entry *ret = NULL;
   1180	int err;
   1181
   1182	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
   1183	ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
   1184	if (!ipip_entry)
   1185		return ERR_PTR(-ENOMEM);
   1186
   1187	ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
   1188							    ol_dev, NULL);
   1189	if (IS_ERR(ipip_entry->ol_lb)) {
   1190		ret = ERR_CAST(ipip_entry->ol_lb);
   1191		goto err_ol_ipip_lb_create;
   1192	}
   1193
   1194	ipip_entry->ipipt = ipipt;
   1195	ipip_entry->ol_dev = ol_dev;
   1196	ipip_entry->parms = ipip_ops->parms_init(ol_dev);
   1197
   1198	err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry);
   1199	if (err) {
   1200		ret = ERR_PTR(err);
   1201		goto err_rem_ip_addr_set;
   1202	}
   1203
   1204	return ipip_entry;
   1205
   1206err_rem_ip_addr_set:
   1207	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
   1208err_ol_ipip_lb_create:
   1209	kfree(ipip_entry);
   1210	return ret;
   1211}
   1212
   1213static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp,
   1214					struct mlxsw_sp_ipip_entry *ipip_entry)
   1215{
   1216	const struct mlxsw_sp_ipip_ops *ipip_ops =
   1217		mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
   1218
   1219	ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry);
   1220	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
   1221	kfree(ipip_entry);
   1222}
   1223
   1224static bool
   1225mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
   1226				  const enum mlxsw_sp_l3proto ul_proto,
   1227				  union mlxsw_sp_l3addr saddr,
   1228				  u32 ul_tb_id,
   1229				  struct mlxsw_sp_ipip_entry *ipip_entry)
   1230{
   1231	u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
   1232	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
   1233	union mlxsw_sp_l3addr tun_saddr;
   1234
   1235	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
   1236		return false;
   1237
   1238	tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
   1239	return tun_ul_tb_id == ul_tb_id &&
   1240	       mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
   1241}
   1242
   1243static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp,
   1244						 enum mlxsw_sp_ipip_type ipipt)
   1245{
   1246	const struct mlxsw_sp_ipip_ops *ipip_ops;
   1247
   1248	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
   1249
   1250	/* Not all tunnels require to increase the default pasing depth
   1251	 * (96 bytes).
   1252	 */
   1253	if (ipip_ops->inc_parsing_depth)
   1254		return mlxsw_sp_parsing_depth_inc(mlxsw_sp);
   1255
   1256	return 0;
   1257}
   1258
   1259static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp,
   1260						  enum mlxsw_sp_ipip_type ipipt)
   1261{
   1262	const struct mlxsw_sp_ipip_ops *ipip_ops =
   1263		mlxsw_sp->router->ipip_ops_arr[ipipt];
   1264
   1265	if (ipip_ops->inc_parsing_depth)
   1266		mlxsw_sp_parsing_depth_dec(mlxsw_sp);
   1267}
   1268
   1269static int
   1270mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
   1271			      struct mlxsw_sp_fib_entry *fib_entry,
   1272			      struct mlxsw_sp_ipip_entry *ipip_entry)
   1273{
   1274	u32 tunnel_index;
   1275	int err;
   1276
   1277	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   1278				  1, &tunnel_index);
   1279	if (err)
   1280		return err;
   1281
   1282	err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp,
   1283						    ipip_entry->ipipt);
   1284	if (err)
   1285		goto err_parsing_depth_inc;
   1286
   1287	ipip_entry->decap_fib_entry = fib_entry;
   1288	fib_entry->decap.ipip_entry = ipip_entry;
   1289	fib_entry->decap.tunnel_index = tunnel_index;
   1290
   1291	return 0;
   1292
   1293err_parsing_depth_inc:
   1294	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
   1295			   fib_entry->decap.tunnel_index);
   1296	return err;
   1297}
   1298
   1299static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
   1300					  struct mlxsw_sp_fib_entry *fib_entry)
   1301{
   1302	enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt;
   1303
   1304	/* Unlink this node from the IPIP entry that it's the decap entry of. */
   1305	fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
   1306	fib_entry->decap.ipip_entry = NULL;
   1307	mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt);
   1308	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   1309			   1, fib_entry->decap.tunnel_index);
   1310}
   1311
   1312static struct mlxsw_sp_fib_node *
   1313mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
   1314			 size_t addr_len, unsigned char prefix_len);
   1315static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
   1316				     struct mlxsw_sp_fib_entry *fib_entry);
   1317
   1318static void
   1319mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
   1320				 struct mlxsw_sp_ipip_entry *ipip_entry)
   1321{
   1322	struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
   1323
   1324	mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
   1325	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   1326
   1327	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
   1328}
   1329
   1330static void
   1331mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
   1332				  struct mlxsw_sp_ipip_entry *ipip_entry,
   1333				  struct mlxsw_sp_fib_entry *decap_fib_entry)
   1334{
   1335	if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
   1336					  ipip_entry))
   1337		return;
   1338	decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
   1339
   1340	if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
   1341		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
   1342}
   1343
   1344static struct mlxsw_sp_fib_entry *
   1345mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
   1346				     enum mlxsw_sp_l3proto proto,
   1347				     const union mlxsw_sp_l3addr *addr,
   1348				     enum mlxsw_sp_fib_entry_type type)
   1349{
   1350	struct mlxsw_sp_fib_node *fib_node;
   1351	unsigned char addr_prefix_len;
   1352	struct mlxsw_sp_fib *fib;
   1353	struct mlxsw_sp_vr *vr;
   1354	const void *addrp;
   1355	size_t addr_len;
   1356	u32 addr4;
   1357
   1358	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
   1359	if (!vr)
   1360		return NULL;
   1361	fib = mlxsw_sp_vr_fib(vr, proto);
   1362
   1363	switch (proto) {
   1364	case MLXSW_SP_L3_PROTO_IPV4:
   1365		addr4 = be32_to_cpu(addr->addr4);
   1366		addrp = &addr4;
   1367		addr_len = 4;
   1368		addr_prefix_len = 32;
   1369		break;
   1370	case MLXSW_SP_L3_PROTO_IPV6:
   1371		addrp = &addr->addr6;
   1372		addr_len = 16;
   1373		addr_prefix_len = 128;
   1374		break;
   1375	default:
   1376		WARN_ON(1);
   1377		return NULL;
   1378	}
   1379
   1380	fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
   1381					    addr_prefix_len);
   1382	if (!fib_node || fib_node->fib_entry->type != type)
   1383		return NULL;
   1384
   1385	return fib_node->fib_entry;
   1386}
   1387
   1388/* Given an IPIP entry, find the corresponding decap route. */
   1389static struct mlxsw_sp_fib_entry *
   1390mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
   1391			       struct mlxsw_sp_ipip_entry *ipip_entry)
   1392{
   1393	static struct mlxsw_sp_fib_node *fib_node;
   1394	const struct mlxsw_sp_ipip_ops *ipip_ops;
   1395	unsigned char saddr_prefix_len;
   1396	union mlxsw_sp_l3addr saddr;
   1397	struct mlxsw_sp_fib *ul_fib;
   1398	struct mlxsw_sp_vr *ul_vr;
   1399	const void *saddrp;
   1400	size_t saddr_len;
   1401	u32 ul_tb_id;
   1402	u32 saddr4;
   1403
   1404	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
   1405
   1406	ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
   1407	ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
   1408	if (!ul_vr)
   1409		return NULL;
   1410
   1411	ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
   1412	saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
   1413					   ipip_entry->ol_dev);
   1414
   1415	switch (ipip_ops->ul_proto) {
   1416	case MLXSW_SP_L3_PROTO_IPV4:
   1417		saddr4 = be32_to_cpu(saddr.addr4);
   1418		saddrp = &saddr4;
   1419		saddr_len = 4;
   1420		saddr_prefix_len = 32;
   1421		break;
   1422	case MLXSW_SP_L3_PROTO_IPV6:
   1423		saddrp = &saddr.addr6;
   1424		saddr_len = 16;
   1425		saddr_prefix_len = 128;
   1426		break;
   1427	default:
   1428		WARN_ON(1);
   1429		return NULL;
   1430	}
   1431
   1432	fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
   1433					    saddr_prefix_len);
   1434	if (!fib_node ||
   1435	    fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
   1436		return NULL;
   1437
   1438	return fib_node->fib_entry;
   1439}
   1440
   1441static struct mlxsw_sp_ipip_entry *
   1442mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
   1443			   enum mlxsw_sp_ipip_type ipipt,
   1444			   struct net_device *ol_dev)
   1445{
   1446	struct mlxsw_sp_ipip_entry *ipip_entry;
   1447
   1448	ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
   1449	if (IS_ERR(ipip_entry))
   1450		return ipip_entry;
   1451
   1452	list_add_tail(&ipip_entry->ipip_list_node,
   1453		      &mlxsw_sp->router->ipip_list);
   1454
   1455	return ipip_entry;
   1456}
   1457
   1458static void
   1459mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
   1460			    struct mlxsw_sp_ipip_entry *ipip_entry)
   1461{
   1462	list_del(&ipip_entry->ipip_list_node);
   1463	mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry);
   1464}
   1465
   1466static bool
   1467mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
   1468				  const struct net_device *ul_dev,
   1469				  enum mlxsw_sp_l3proto ul_proto,
   1470				  union mlxsw_sp_l3addr ul_dip,
   1471				  struct mlxsw_sp_ipip_entry *ipip_entry)
   1472{
   1473	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
   1474	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
   1475
   1476	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
   1477		return false;
   1478
   1479	return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
   1480						 ul_tb_id, ipip_entry);
   1481}
   1482
   1483/* Given decap parameters, find the corresponding IPIP entry. */
   1484static struct mlxsw_sp_ipip_entry *
   1485mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
   1486				  enum mlxsw_sp_l3proto ul_proto,
   1487				  union mlxsw_sp_l3addr ul_dip)
   1488{
   1489	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
   1490	struct net_device *ul_dev;
   1491
   1492	rcu_read_lock();
   1493
   1494	ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
   1495	if (!ul_dev)
   1496		goto out_unlock;
   1497
   1498	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
   1499			    ipip_list_node)
   1500		if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
   1501						      ul_proto, ul_dip,
   1502						      ipip_entry))
   1503			goto out_unlock;
   1504
   1505	rcu_read_unlock();
   1506
   1507	return NULL;
   1508
   1509out_unlock:
   1510	rcu_read_unlock();
   1511	return ipip_entry;
   1512}
   1513
   1514static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
   1515				      const struct net_device *dev,
   1516				      enum mlxsw_sp_ipip_type *p_type)
   1517{
   1518	struct mlxsw_sp_router *router = mlxsw_sp->router;
   1519	const struct mlxsw_sp_ipip_ops *ipip_ops;
   1520	enum mlxsw_sp_ipip_type ipipt;
   1521
   1522	for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
   1523		ipip_ops = router->ipip_ops_arr[ipipt];
   1524		if (dev->type == ipip_ops->dev_type) {
   1525			if (p_type)
   1526				*p_type = ipipt;
   1527			return true;
   1528		}
   1529	}
   1530	return false;
   1531}
   1532
   1533static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
   1534				       const struct net_device *dev)
   1535{
   1536	return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
   1537}
   1538
   1539static struct mlxsw_sp_ipip_entry *
   1540mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
   1541				   const struct net_device *ol_dev)
   1542{
   1543	struct mlxsw_sp_ipip_entry *ipip_entry;
   1544
   1545	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
   1546			    ipip_list_node)
   1547		if (ipip_entry->ol_dev == ol_dev)
   1548			return ipip_entry;
   1549
   1550	return NULL;
   1551}
   1552
   1553static struct mlxsw_sp_ipip_entry *
   1554mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
   1555				   const struct net_device *ul_dev,
   1556				   struct mlxsw_sp_ipip_entry *start)
   1557{
   1558	struct mlxsw_sp_ipip_entry *ipip_entry;
   1559
   1560	ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
   1561					ipip_list_node);
   1562	list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
   1563				     ipip_list_node) {
   1564		struct net_device *ol_dev = ipip_entry->ol_dev;
   1565		struct net_device *ipip_ul_dev;
   1566
   1567		rcu_read_lock();
   1568		ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
   1569		rcu_read_unlock();
   1570
   1571		if (ipip_ul_dev == ul_dev)
   1572			return ipip_entry;
   1573	}
   1574
   1575	return NULL;
   1576}
   1577
   1578static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
   1579				       const struct net_device *dev)
   1580{
   1581	return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
   1582}
   1583
   1584static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
   1585						const struct net_device *ol_dev,
   1586						enum mlxsw_sp_ipip_type ipipt)
   1587{
   1588	const struct mlxsw_sp_ipip_ops *ops
   1589		= mlxsw_sp->router->ipip_ops_arr[ipipt];
   1590
   1591	return ops->can_offload(mlxsw_sp, ol_dev);
   1592}
   1593
   1594static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
   1595						struct net_device *ol_dev)
   1596{
   1597	enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
   1598	struct mlxsw_sp_ipip_entry *ipip_entry;
   1599	enum mlxsw_sp_l3proto ul_proto;
   1600	union mlxsw_sp_l3addr saddr;
   1601	u32 ul_tb_id;
   1602
   1603	mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
   1604	if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
   1605		ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
   1606		ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
   1607		saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
   1608		if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
   1609							  saddr, ul_tb_id,
   1610							  NULL)) {
   1611			ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
   1612								ol_dev);
   1613			if (IS_ERR(ipip_entry))
   1614				return PTR_ERR(ipip_entry);
   1615		}
   1616	}
   1617
   1618	return 0;
   1619}
   1620
   1621static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
   1622						   struct net_device *ol_dev)
   1623{
   1624	struct mlxsw_sp_ipip_entry *ipip_entry;
   1625
   1626	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
   1627	if (ipip_entry)
   1628		mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
   1629}
   1630
   1631static void
   1632mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
   1633				struct mlxsw_sp_ipip_entry *ipip_entry)
   1634{
   1635	struct mlxsw_sp_fib_entry *decap_fib_entry;
   1636
   1637	decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
   1638	if (decap_fib_entry)
   1639		mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
   1640						  decap_fib_entry);
   1641}
   1642
   1643static int
   1644mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
   1645			u16 ul_rif_id, bool enable)
   1646{
   1647	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
   1648	enum mlxsw_reg_ritr_loopback_ipip_options ipip_options;
   1649	struct mlxsw_sp_rif *rif = &lb_rif->common;
   1650	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   1651	char ritr_pl[MLXSW_REG_RITR_LEN];
   1652	struct in6_addr *saddr6;
   1653	u32 saddr4;
   1654
   1655	ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET;
   1656	switch (lb_cf.ul_protocol) {
   1657	case MLXSW_SP_L3_PROTO_IPV4:
   1658		saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
   1659		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
   1660				    rif->rif_index, rif->vr_id, rif->dev->mtu);
   1661		mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
   1662						   ipip_options, ul_vr_id,
   1663						   ul_rif_id, saddr4,
   1664						   lb_cf.okey);
   1665		break;
   1666
   1667	case MLXSW_SP_L3_PROTO_IPV6:
   1668		saddr6 = &lb_cf.saddr.addr6;
   1669		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
   1670				    rif->rif_index, rif->vr_id, rif->dev->mtu);
   1671		mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt,
   1672						   ipip_options, ul_vr_id,
   1673						   ul_rif_id, saddr6,
   1674						   lb_cf.okey);
   1675		break;
   1676	}
   1677
   1678	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   1679}
   1680
   1681static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
   1682						 struct net_device *ol_dev)
   1683{
   1684	struct mlxsw_sp_ipip_entry *ipip_entry;
   1685	struct mlxsw_sp_rif_ipip_lb *lb_rif;
   1686	int err = 0;
   1687
   1688	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
   1689	if (ipip_entry) {
   1690		lb_rif = ipip_entry->ol_lb;
   1691		err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
   1692					      lb_rif->ul_rif_id, true);
   1693		if (err)
   1694			goto out;
   1695		lb_rif->common.mtu = ol_dev->mtu;
   1696	}
   1697
   1698out:
   1699	return err;
   1700}
   1701
   1702static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
   1703						struct net_device *ol_dev)
   1704{
   1705	struct mlxsw_sp_ipip_entry *ipip_entry;
   1706
   1707	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
   1708	if (ipip_entry)
   1709		mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
   1710}
   1711
   1712static void
   1713mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
   1714				  struct mlxsw_sp_ipip_entry *ipip_entry)
   1715{
   1716	if (ipip_entry->decap_fib_entry)
   1717		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
   1718}
   1719
   1720static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
   1721						  struct net_device *ol_dev)
   1722{
   1723	struct mlxsw_sp_ipip_entry *ipip_entry;
   1724
   1725	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
   1726	if (ipip_entry)
   1727		mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
   1728}
   1729
   1730static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
   1731					 struct mlxsw_sp_rif *old_rif,
   1732					 struct mlxsw_sp_rif *new_rif);
   1733static int
   1734mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
   1735				 struct mlxsw_sp_ipip_entry *ipip_entry,
   1736				 bool keep_encap,
   1737				 struct netlink_ext_ack *extack)
   1738{
   1739	struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
   1740	struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
   1741
   1742	new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
   1743						     ipip_entry->ipipt,
   1744						     ipip_entry->ol_dev,
   1745						     extack);
   1746	if (IS_ERR(new_lb_rif))
   1747		return PTR_ERR(new_lb_rif);
   1748	ipip_entry->ol_lb = new_lb_rif;
   1749
   1750	if (keep_encap)
   1751		mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
   1752					     &new_lb_rif->common);
   1753
   1754	mlxsw_sp_rif_destroy(&old_lb_rif->common);
   1755
   1756	return 0;
   1757}
   1758
   1759static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
   1760					struct mlxsw_sp_rif *rif);
   1761
   1762/**
   1763 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
   1764 * @mlxsw_sp: mlxsw_sp.
   1765 * @ipip_entry: IPIP entry.
   1766 * @recreate_loopback: Recreates the associated loopback RIF.
   1767 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
   1768 *              relevant when recreate_loopback is true.
   1769 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
   1770 *                   is only relevant when recreate_loopback is false.
   1771 * @extack: extack.
   1772 *
   1773 * Return: Non-zero value on failure.
   1774 */
   1775int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
   1776					struct mlxsw_sp_ipip_entry *ipip_entry,
   1777					bool recreate_loopback,
   1778					bool keep_encap,
   1779					bool update_nexthops,
   1780					struct netlink_ext_ack *extack)
   1781{
   1782	int err;
   1783
   1784	/* RIFs can't be edited, so to update loopback, we need to destroy and
   1785	 * recreate it. That creates a window of opportunity where RALUE and
   1786	 * RATR registers end up referencing a RIF that's already gone. RATRs
   1787	 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
   1788	 * of RALUE, demote the decap route back.
   1789	 */
   1790	if (ipip_entry->decap_fib_entry)
   1791		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
   1792
   1793	if (recreate_loopback) {
   1794		err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
   1795						       keep_encap, extack);
   1796		if (err)
   1797			return err;
   1798	} else if (update_nexthops) {
   1799		mlxsw_sp_nexthop_rif_update(mlxsw_sp,
   1800					    &ipip_entry->ol_lb->common);
   1801	}
   1802
   1803	if (ipip_entry->ol_dev->flags & IFF_UP)
   1804		mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
   1805
   1806	return 0;
   1807}
   1808
   1809static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
   1810						struct net_device *ol_dev,
   1811						struct netlink_ext_ack *extack)
   1812{
   1813	struct mlxsw_sp_ipip_entry *ipip_entry =
   1814		mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
   1815
   1816	if (!ipip_entry)
   1817		return 0;
   1818
   1819	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
   1820						   true, false, false, extack);
   1821}
   1822
   1823static int
   1824mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
   1825				     struct mlxsw_sp_ipip_entry *ipip_entry,
   1826				     struct net_device *ul_dev,
   1827				     bool *demote_this,
   1828				     struct netlink_ext_ack *extack)
   1829{
   1830	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
   1831	enum mlxsw_sp_l3proto ul_proto;
   1832	union mlxsw_sp_l3addr saddr;
   1833
   1834	/* Moving underlay to a different VRF might cause local address
   1835	 * conflict, and the conflicting tunnels need to be demoted.
   1836	 */
   1837	ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
   1838	saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
   1839	if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
   1840						 saddr, ul_tb_id,
   1841						 ipip_entry)) {
   1842		*demote_this = true;
   1843		return 0;
   1844	}
   1845
   1846	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
   1847						   true, true, false, extack);
   1848}
   1849
   1850static int
   1851mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
   1852				    struct mlxsw_sp_ipip_entry *ipip_entry,
   1853				    struct net_device *ul_dev)
   1854{
   1855	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
   1856						   false, false, true, NULL);
   1857}
   1858
   1859static int
   1860mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
   1861				      struct mlxsw_sp_ipip_entry *ipip_entry,
   1862				      struct net_device *ul_dev)
   1863{
   1864	/* A down underlay device causes encapsulated packets to not be
   1865	 * forwarded, but decap still works. So refresh next hops without
   1866	 * touching anything else.
   1867	 */
   1868	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
   1869						   false, false, true, NULL);
   1870}
   1871
   1872static int
   1873mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
   1874					struct net_device *ol_dev,
   1875					struct netlink_ext_ack *extack)
   1876{
   1877	const struct mlxsw_sp_ipip_ops *ipip_ops;
   1878	struct mlxsw_sp_ipip_entry *ipip_entry;
   1879	int err;
   1880
   1881	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
   1882	if (!ipip_entry)
   1883		/* A change might make a tunnel eligible for offloading, but
   1884		 * that is currently not implemented. What falls to slow path
   1885		 * stays there.
   1886		 */
   1887		return 0;
   1888
   1889	/* A change might make a tunnel not eligible for offloading. */
   1890	if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
   1891						 ipip_entry->ipipt)) {
   1892		mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
   1893		return 0;
   1894	}
   1895
   1896	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
   1897	err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
   1898	return err;
   1899}
   1900
   1901void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
   1902				       struct mlxsw_sp_ipip_entry *ipip_entry)
   1903{
   1904	struct net_device *ol_dev = ipip_entry->ol_dev;
   1905
   1906	if (ol_dev->flags & IFF_UP)
   1907		mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
   1908	mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
   1909}
   1910
   1911/* The configuration where several tunnels have the same local address in the
   1912 * same underlay table needs special treatment in the HW. That is currently not
   1913 * implemented in the driver. This function finds and demotes the first tunnel
   1914 * with a given source address, except the one passed in in the argument
   1915 * `except'.
   1916 */
   1917bool
   1918mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
   1919				     enum mlxsw_sp_l3proto ul_proto,
   1920				     union mlxsw_sp_l3addr saddr,
   1921				     u32 ul_tb_id,
   1922				     const struct mlxsw_sp_ipip_entry *except)
   1923{
   1924	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
   1925
   1926	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
   1927				 ipip_list_node) {
   1928		if (ipip_entry != except &&
   1929		    mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
   1930						      ul_tb_id, ipip_entry)) {
   1931			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
   1932			return true;
   1933		}
   1934	}
   1935
   1936	return false;
   1937}
   1938
   1939static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
   1940						     struct net_device *ul_dev)
   1941{
   1942	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
   1943
   1944	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
   1945				 ipip_list_node) {
   1946		struct net_device *ol_dev = ipip_entry->ol_dev;
   1947		struct net_device *ipip_ul_dev;
   1948
   1949		rcu_read_lock();
   1950		ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
   1951		rcu_read_unlock();
   1952		if (ipip_ul_dev == ul_dev)
   1953			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
   1954	}
   1955}
   1956
   1957static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
   1958					    struct net_device *ol_dev,
   1959					    unsigned long event,
   1960					    struct netdev_notifier_info *info)
   1961{
   1962	struct netdev_notifier_changeupper_info *chup;
   1963	struct netlink_ext_ack *extack;
   1964	int err = 0;
   1965
   1966	switch (event) {
   1967	case NETDEV_REGISTER:
   1968		err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
   1969		break;
   1970	case NETDEV_UNREGISTER:
   1971		mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
   1972		break;
   1973	case NETDEV_UP:
   1974		mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
   1975		break;
   1976	case NETDEV_DOWN:
   1977		mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
   1978		break;
   1979	case NETDEV_CHANGEUPPER:
   1980		chup = container_of(info, typeof(*chup), info);
   1981		extack = info->extack;
   1982		if (netif_is_l3_master(chup->upper_dev))
   1983			err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
   1984								   ol_dev,
   1985								   extack);
   1986		break;
   1987	case NETDEV_CHANGE:
   1988		extack = info->extack;
   1989		err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
   1990							      ol_dev, extack);
   1991		break;
   1992	case NETDEV_CHANGEMTU:
   1993		err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
   1994		break;
   1995	}
   1996	return err;
   1997}
   1998
   1999static int
   2000__mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
   2001				   struct mlxsw_sp_ipip_entry *ipip_entry,
   2002				   struct net_device *ul_dev,
   2003				   bool *demote_this,
   2004				   unsigned long event,
   2005				   struct netdev_notifier_info *info)
   2006{
   2007	struct netdev_notifier_changeupper_info *chup;
   2008	struct netlink_ext_ack *extack;
   2009
   2010	switch (event) {
   2011	case NETDEV_CHANGEUPPER:
   2012		chup = container_of(info, typeof(*chup), info);
   2013		extack = info->extack;
   2014		if (netif_is_l3_master(chup->upper_dev))
   2015			return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
   2016								    ipip_entry,
   2017								    ul_dev,
   2018								    demote_this,
   2019								    extack);
   2020		break;
   2021
   2022	case NETDEV_UP:
   2023		return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
   2024							   ul_dev);
   2025	case NETDEV_DOWN:
   2026		return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
   2027							     ipip_entry,
   2028							     ul_dev);
   2029	}
   2030	return 0;
   2031}
   2032
   2033static int
   2034mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
   2035				 struct net_device *ul_dev,
   2036				 unsigned long event,
   2037				 struct netdev_notifier_info *info)
   2038{
   2039	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
   2040	int err;
   2041
   2042	while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
   2043								ul_dev,
   2044								ipip_entry))) {
   2045		struct mlxsw_sp_ipip_entry *prev;
   2046		bool demote_this = false;
   2047
   2048		err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
   2049							 ul_dev, &demote_this,
   2050							 event, info);
   2051		if (err) {
   2052			mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
   2053								 ul_dev);
   2054			return err;
   2055		}
   2056
   2057		if (demote_this) {
   2058			if (list_is_first(&ipip_entry->ipip_list_node,
   2059					  &mlxsw_sp->router->ipip_list))
   2060				prev = NULL;
   2061			else
   2062				/* This can't be cached from previous iteration,
   2063				 * because that entry could be gone now.
   2064				 */
   2065				prev = list_prev_entry(ipip_entry,
   2066						       ipip_list_node);
   2067			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
   2068			ipip_entry = prev;
   2069		}
   2070	}
   2071
   2072	return 0;
   2073}
   2074
   2075int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
   2076				      enum mlxsw_sp_l3proto ul_proto,
   2077				      const union mlxsw_sp_l3addr *ul_sip,
   2078				      u32 tunnel_index)
   2079{
   2080	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   2081	struct mlxsw_sp_router *router = mlxsw_sp->router;
   2082	struct mlxsw_sp_fib_entry *fib_entry;
   2083	int err = 0;
   2084
   2085	mutex_lock(&mlxsw_sp->router->lock);
   2086
   2087	if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
   2088		err = -EINVAL;
   2089		goto out;
   2090	}
   2091
   2092	router->nve_decap_config.ul_tb_id = ul_tb_id;
   2093	router->nve_decap_config.tunnel_index = tunnel_index;
   2094	router->nve_decap_config.ul_proto = ul_proto;
   2095	router->nve_decap_config.ul_sip = *ul_sip;
   2096	router->nve_decap_config.valid = true;
   2097
   2098	/* It is valid to create a tunnel with a local IP and only later
   2099	 * assign this IP address to a local interface
   2100	 */
   2101	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
   2102							 ul_proto, ul_sip,
   2103							 type);
   2104	if (!fib_entry)
   2105		goto out;
   2106
   2107	fib_entry->decap.tunnel_index = tunnel_index;
   2108	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
   2109
   2110	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
   2111	if (err)
   2112		goto err_fib_entry_update;
   2113
   2114	goto out;
   2115
   2116err_fib_entry_update:
   2117	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   2118	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
   2119out:
   2120	mutex_unlock(&mlxsw_sp->router->lock);
   2121	return err;
   2122}
   2123
   2124void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
   2125				      enum mlxsw_sp_l3proto ul_proto,
   2126				      const union mlxsw_sp_l3addr *ul_sip)
   2127{
   2128	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
   2129	struct mlxsw_sp_router *router = mlxsw_sp->router;
   2130	struct mlxsw_sp_fib_entry *fib_entry;
   2131
   2132	mutex_lock(&mlxsw_sp->router->lock);
   2133
   2134	if (WARN_ON_ONCE(!router->nve_decap_config.valid))
   2135		goto out;
   2136
   2137	router->nve_decap_config.valid = false;
   2138
   2139	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
   2140							 ul_proto, ul_sip,
   2141							 type);
   2142	if (!fib_entry)
   2143		goto out;
   2144
   2145	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   2146	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
   2147out:
   2148	mutex_unlock(&mlxsw_sp->router->lock);
   2149}
   2150
   2151static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
   2152					 u32 ul_tb_id,
   2153					 enum mlxsw_sp_l3proto ul_proto,
   2154					 const union mlxsw_sp_l3addr *ul_sip)
   2155{
   2156	struct mlxsw_sp_router *router = mlxsw_sp->router;
   2157
   2158	return router->nve_decap_config.valid &&
   2159	       router->nve_decap_config.ul_tb_id == ul_tb_id &&
   2160	       router->nve_decap_config.ul_proto == ul_proto &&
   2161	       !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
   2162		       sizeof(*ul_sip));
   2163}
   2164
   2165struct mlxsw_sp_neigh_key {
   2166	struct neighbour *n;
   2167};
   2168
   2169struct mlxsw_sp_neigh_entry {
   2170	struct list_head rif_list_node;
   2171	struct rhash_head ht_node;
   2172	struct mlxsw_sp_neigh_key key;
   2173	u16 rif;
   2174	bool connected;
   2175	unsigned char ha[ETH_ALEN];
   2176	struct list_head nexthop_list; /* list of nexthops using
   2177					* this neigh entry
   2178					*/
   2179	struct list_head nexthop_neighs_list_node;
   2180	unsigned int counter_index;
   2181	bool counter_valid;
   2182};
   2183
   2184static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
   2185	.key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
   2186	.head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
   2187	.key_len = sizeof(struct mlxsw_sp_neigh_key),
   2188};
   2189
   2190struct mlxsw_sp_neigh_entry *
   2191mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
   2192			struct mlxsw_sp_neigh_entry *neigh_entry)
   2193{
   2194	if (!neigh_entry) {
   2195		if (list_empty(&rif->neigh_list))
   2196			return NULL;
   2197		else
   2198			return list_first_entry(&rif->neigh_list,
   2199						typeof(*neigh_entry),
   2200						rif_list_node);
   2201	}
   2202	if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
   2203		return NULL;
   2204	return list_next_entry(neigh_entry, rif_list_node);
   2205}
   2206
   2207int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
   2208{
   2209	return neigh_entry->key.n->tbl->family;
   2210}
   2211
   2212unsigned char *
   2213mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
   2214{
   2215	return neigh_entry->ha;
   2216}
   2217
   2218u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
   2219{
   2220	struct neighbour *n;
   2221
   2222	n = neigh_entry->key.n;
   2223	return ntohl(*((__be32 *) n->primary_key));
   2224}
   2225
   2226struct in6_addr *
   2227mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
   2228{
   2229	struct neighbour *n;
   2230
   2231	n = neigh_entry->key.n;
   2232	return (struct in6_addr *) &n->primary_key;
   2233}
   2234
   2235int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
   2236			       struct mlxsw_sp_neigh_entry *neigh_entry,
   2237			       u64 *p_counter)
   2238{
   2239	if (!neigh_entry->counter_valid)
   2240		return -EINVAL;
   2241
   2242	return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
   2243					 p_counter, NULL);
   2244}
   2245
   2246static struct mlxsw_sp_neigh_entry *
   2247mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
   2248			   u16 rif)
   2249{
   2250	struct mlxsw_sp_neigh_entry *neigh_entry;
   2251
   2252	neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
   2253	if (!neigh_entry)
   2254		return NULL;
   2255
   2256	neigh_entry->key.n = n;
   2257	neigh_entry->rif = rif;
   2258	INIT_LIST_HEAD(&neigh_entry->nexthop_list);
   2259
   2260	return neigh_entry;
   2261}
   2262
   2263static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
   2264{
   2265	kfree(neigh_entry);
   2266}
   2267
   2268static int
   2269mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
   2270			    struct mlxsw_sp_neigh_entry *neigh_entry)
   2271{
   2272	return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
   2273				      &neigh_entry->ht_node,
   2274				      mlxsw_sp_neigh_ht_params);
   2275}
   2276
   2277static void
   2278mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
   2279			    struct mlxsw_sp_neigh_entry *neigh_entry)
   2280{
   2281	rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
   2282			       &neigh_entry->ht_node,
   2283			       mlxsw_sp_neigh_ht_params);
   2284}
   2285
   2286static bool
   2287mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
   2288				    struct mlxsw_sp_neigh_entry *neigh_entry)
   2289{
   2290	struct devlink *devlink;
   2291	const char *table_name;
   2292
   2293	switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
   2294	case AF_INET:
   2295		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
   2296		break;
   2297	case AF_INET6:
   2298		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
   2299		break;
   2300	default:
   2301		WARN_ON(1);
   2302		return false;
   2303	}
   2304
   2305	devlink = priv_to_devlink(mlxsw_sp->core);
   2306	return devlink_dpipe_table_counter_enabled(devlink, table_name);
   2307}
   2308
   2309static void
   2310mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
   2311			     struct mlxsw_sp_neigh_entry *neigh_entry)
   2312{
   2313	if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
   2314		return;
   2315
   2316	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
   2317		return;
   2318
   2319	neigh_entry->counter_valid = true;
   2320}
   2321
   2322static void
   2323mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
   2324			    struct mlxsw_sp_neigh_entry *neigh_entry)
   2325{
   2326	if (!neigh_entry->counter_valid)
   2327		return;
   2328	mlxsw_sp_flow_counter_free(mlxsw_sp,
   2329				   neigh_entry->counter_index);
   2330	neigh_entry->counter_valid = false;
   2331}
   2332
   2333static struct mlxsw_sp_neigh_entry *
   2334mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
   2335{
   2336	struct mlxsw_sp_neigh_entry *neigh_entry;
   2337	struct mlxsw_sp_rif *rif;
   2338	int err;
   2339
   2340	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
   2341	if (!rif)
   2342		return ERR_PTR(-EINVAL);
   2343
   2344	neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
   2345	if (!neigh_entry)
   2346		return ERR_PTR(-ENOMEM);
   2347
   2348	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
   2349	if (err)
   2350		goto err_neigh_entry_insert;
   2351
   2352	mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
   2353	atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count);
   2354	list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
   2355
   2356	return neigh_entry;
   2357
   2358err_neigh_entry_insert:
   2359	mlxsw_sp_neigh_entry_free(neigh_entry);
   2360	return ERR_PTR(err);
   2361}
   2362
   2363static void
   2364mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
   2365			     struct mlxsw_sp_neigh_entry *neigh_entry)
   2366{
   2367	list_del(&neigh_entry->rif_list_node);
   2368	atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count);
   2369	mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
   2370	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
   2371	mlxsw_sp_neigh_entry_free(neigh_entry);
   2372}
   2373
   2374static struct mlxsw_sp_neigh_entry *
   2375mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
   2376{
   2377	struct mlxsw_sp_neigh_key key;
   2378
   2379	key.n = n;
   2380	return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
   2381				      &key, mlxsw_sp_neigh_ht_params);
   2382}
   2383
   2384static void
   2385mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
   2386{
   2387	unsigned long interval;
   2388
   2389#if IS_ENABLED(CONFIG_IPV6)
   2390	interval = min_t(unsigned long,
   2391			 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
   2392			 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
   2393#else
   2394	interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
   2395#endif
   2396	mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
   2397}
   2398
   2399static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
   2400						   char *rauhtd_pl,
   2401						   int ent_index)
   2402{
   2403	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
   2404	struct net_device *dev;
   2405	struct neighbour *n;
   2406	__be32 dipn;
   2407	u32 dip;
   2408	u16 rif;
   2409
   2410	mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
   2411
   2412	if (WARN_ON_ONCE(rif >= max_rifs))
   2413		return;
   2414	if (!mlxsw_sp->router->rifs[rif]) {
   2415		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
   2416		return;
   2417	}
   2418
   2419	dipn = htonl(dip);
   2420	dev = mlxsw_sp->router->rifs[rif]->dev;
   2421	n = neigh_lookup(&arp_tbl, &dipn, dev);
   2422	if (!n)
   2423		return;
   2424
   2425	netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
   2426	neigh_event_send(n, NULL);
   2427	neigh_release(n);
   2428}
   2429
   2430#if IS_ENABLED(CONFIG_IPV6)
   2431static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
   2432						   char *rauhtd_pl,
   2433						   int rec_index)
   2434{
   2435	struct net_device *dev;
   2436	struct neighbour *n;
   2437	struct in6_addr dip;
   2438	u16 rif;
   2439
   2440	mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
   2441					 (char *) &dip);
   2442
   2443	if (!mlxsw_sp->router->rifs[rif]) {
   2444		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
   2445		return;
   2446	}
   2447
   2448	dev = mlxsw_sp->router->rifs[rif]->dev;
   2449	n = neigh_lookup(&nd_tbl, &dip, dev);
   2450	if (!n)
   2451		return;
   2452
   2453	netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
   2454	neigh_event_send(n, NULL);
   2455	neigh_release(n);
   2456}
   2457#else
   2458static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
   2459						   char *rauhtd_pl,
   2460						   int rec_index)
   2461{
   2462}
   2463#endif
   2464
   2465static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
   2466						   char *rauhtd_pl,
   2467						   int rec_index)
   2468{
   2469	u8 num_entries;
   2470	int i;
   2471
   2472	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
   2473								rec_index);
   2474	/* Hardware starts counting at 0, so add 1. */
   2475	num_entries++;
   2476
   2477	/* Each record consists of several neighbour entries. */
   2478	for (i = 0; i < num_entries; i++) {
   2479		int ent_index;
   2480
   2481		ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
   2482		mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
   2483						       ent_index);
   2484	}
   2485
   2486}
   2487
   2488static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
   2489						   char *rauhtd_pl,
   2490						   int rec_index)
   2491{
   2492	/* One record contains one entry. */
   2493	mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
   2494					       rec_index);
   2495}
   2496
   2497static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
   2498					      char *rauhtd_pl, int rec_index)
   2499{
   2500	switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
   2501	case MLXSW_REG_RAUHTD_TYPE_IPV4:
   2502		mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
   2503						       rec_index);
   2504		break;
   2505	case MLXSW_REG_RAUHTD_TYPE_IPV6:
   2506		mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
   2507						       rec_index);
   2508		break;
   2509	}
   2510}
   2511
   2512static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
   2513{
   2514	u8 num_rec, last_rec_index, num_entries;
   2515
   2516	num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
   2517	last_rec_index = num_rec - 1;
   2518
   2519	if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
   2520		return false;
   2521	if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
   2522	    MLXSW_REG_RAUHTD_TYPE_IPV6)
   2523		return true;
   2524
   2525	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
   2526								last_rec_index);
   2527	if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
   2528		return true;
   2529	return false;
   2530}
   2531
   2532static int
   2533__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
   2534				       char *rauhtd_pl,
   2535				       enum mlxsw_reg_rauhtd_type type)
   2536{
   2537	int i, num_rec;
   2538	int err;
   2539
   2540	/* Ensure the RIF we read from the device does not change mid-dump. */
   2541	mutex_lock(&mlxsw_sp->router->lock);
   2542	do {
   2543		mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
   2544		err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
   2545				      rauhtd_pl);
   2546		if (err) {
   2547			dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
   2548			break;
   2549		}
   2550		num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
   2551		for (i = 0; i < num_rec; i++)
   2552			mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
   2553							  i);
   2554	} while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
   2555	mutex_unlock(&mlxsw_sp->router->lock);
   2556
   2557	return err;
   2558}
   2559
   2560static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
   2561{
   2562	enum mlxsw_reg_rauhtd_type type;
   2563	char *rauhtd_pl;
   2564	int err;
   2565
   2566	if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count))
   2567		return 0;
   2568
   2569	rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
   2570	if (!rauhtd_pl)
   2571		return -ENOMEM;
   2572
   2573	type = MLXSW_REG_RAUHTD_TYPE_IPV4;
   2574	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
   2575	if (err)
   2576		goto out;
   2577
   2578	type = MLXSW_REG_RAUHTD_TYPE_IPV6;
   2579	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
   2580out:
   2581	kfree(rauhtd_pl);
   2582	return err;
   2583}
   2584
   2585static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
   2586{
   2587	struct mlxsw_sp_neigh_entry *neigh_entry;
   2588
   2589	mutex_lock(&mlxsw_sp->router->lock);
   2590	list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
   2591			    nexthop_neighs_list_node)
   2592		/* If this neigh have nexthops, make the kernel think this neigh
   2593		 * is active regardless of the traffic.
   2594		 */
   2595		neigh_event_send(neigh_entry->key.n, NULL);
   2596	mutex_unlock(&mlxsw_sp->router->lock);
   2597}
   2598
   2599static void
   2600mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
   2601{
   2602	unsigned long interval = mlxsw_sp->router->neighs_update.interval;
   2603
   2604	mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
   2605			       msecs_to_jiffies(interval));
   2606}
   2607
   2608static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
   2609{
   2610	struct mlxsw_sp_router *router;
   2611	int err;
   2612
   2613	router = container_of(work, struct mlxsw_sp_router,
   2614			      neighs_update.dw.work);
   2615	err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
   2616	if (err)
   2617		dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
   2618
   2619	mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
   2620
   2621	mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
   2622}
   2623
   2624static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
   2625{
   2626	struct mlxsw_sp_neigh_entry *neigh_entry;
   2627	struct mlxsw_sp_router *router;
   2628
   2629	router = container_of(work, struct mlxsw_sp_router,
   2630			      nexthop_probe_dw.work);
   2631	/* Iterate over nexthop neighbours, find those who are unresolved and
   2632	 * send arp on them. This solves the chicken-egg problem when
   2633	 * the nexthop wouldn't get offloaded until the neighbor is resolved
   2634	 * but it wouldn't get resolved ever in case traffic is flowing in HW
   2635	 * using different nexthop.
   2636	 */
   2637	mutex_lock(&router->lock);
   2638	list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
   2639			    nexthop_neighs_list_node)
   2640		if (!neigh_entry->connected)
   2641			neigh_event_send(neigh_entry->key.n, NULL);
   2642	mutex_unlock(&router->lock);
   2643
   2644	mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
   2645			       MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
   2646}
   2647
   2648static void
   2649mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
   2650			      struct mlxsw_sp_neigh_entry *neigh_entry,
   2651			      bool removing, bool dead);
   2652
   2653static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
   2654{
   2655	return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
   2656			MLXSW_REG_RAUHT_OP_WRITE_DELETE;
   2657}
   2658
   2659static int
   2660mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
   2661				struct mlxsw_sp_neigh_entry *neigh_entry,
   2662				enum mlxsw_reg_rauht_op op)
   2663{
   2664	struct neighbour *n = neigh_entry->key.n;
   2665	u32 dip = ntohl(*((__be32 *) n->primary_key));
   2666	char rauht_pl[MLXSW_REG_RAUHT_LEN];
   2667
   2668	mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
   2669			      dip);
   2670	if (neigh_entry->counter_valid)
   2671		mlxsw_reg_rauht_pack_counter(rauht_pl,
   2672					     neigh_entry->counter_index);
   2673	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
   2674}
   2675
   2676static int
   2677mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
   2678				struct mlxsw_sp_neigh_entry *neigh_entry,
   2679				enum mlxsw_reg_rauht_op op)
   2680{
   2681	struct neighbour *n = neigh_entry->key.n;
   2682	char rauht_pl[MLXSW_REG_RAUHT_LEN];
   2683	const char *dip = n->primary_key;
   2684
   2685	mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
   2686			      dip);
   2687	if (neigh_entry->counter_valid)
   2688		mlxsw_reg_rauht_pack_counter(rauht_pl,
   2689					     neigh_entry->counter_index);
   2690	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
   2691}
   2692
   2693bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
   2694{
   2695	struct neighbour *n = neigh_entry->key.n;
   2696
   2697	/* Packets with a link-local destination address are trapped
   2698	 * after LPM lookup and never reach the neighbour table, so
   2699	 * there is no need to program such neighbours to the device.
   2700	 */
   2701	if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
   2702	    IPV6_ADDR_LINKLOCAL)
   2703		return true;
   2704	return false;
   2705}
   2706
   2707static void
   2708mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
   2709			    struct mlxsw_sp_neigh_entry *neigh_entry,
   2710			    bool adding)
   2711{
   2712	enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
   2713	int err;
   2714
   2715	if (!adding && !neigh_entry->connected)
   2716		return;
   2717	neigh_entry->connected = adding;
   2718	if (neigh_entry->key.n->tbl->family == AF_INET) {
   2719		err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
   2720						      op);
   2721		if (err)
   2722			return;
   2723	} else if (neigh_entry->key.n->tbl->family == AF_INET6) {
   2724		if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
   2725			return;
   2726		err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
   2727						      op);
   2728		if (err)
   2729			return;
   2730	} else {
   2731		WARN_ON_ONCE(1);
   2732		return;
   2733	}
   2734
   2735	if (adding)
   2736		neigh_entry->key.n->flags |= NTF_OFFLOADED;
   2737	else
   2738		neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
   2739}
   2740
   2741void
   2742mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
   2743				    struct mlxsw_sp_neigh_entry *neigh_entry,
   2744				    bool adding)
   2745{
   2746	if (adding)
   2747		mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
   2748	else
   2749		mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
   2750	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
   2751}
   2752
   2753struct mlxsw_sp_netevent_work {
   2754	struct work_struct work;
   2755	struct mlxsw_sp *mlxsw_sp;
   2756	struct neighbour *n;
   2757};
   2758
   2759static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
   2760{
   2761	struct mlxsw_sp_netevent_work *net_work =
   2762		container_of(work, struct mlxsw_sp_netevent_work, work);
   2763	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
   2764	struct mlxsw_sp_neigh_entry *neigh_entry;
   2765	struct neighbour *n = net_work->n;
   2766	unsigned char ha[ETH_ALEN];
   2767	bool entry_connected;
   2768	u8 nud_state, dead;
   2769
   2770	/* If these parameters are changed after we release the lock,
   2771	 * then we are guaranteed to receive another event letting us
   2772	 * know about it.
   2773	 */
   2774	read_lock_bh(&n->lock);
   2775	memcpy(ha, n->ha, ETH_ALEN);
   2776	nud_state = n->nud_state;
   2777	dead = n->dead;
   2778	read_unlock_bh(&n->lock);
   2779
   2780	mutex_lock(&mlxsw_sp->router->lock);
   2781	mlxsw_sp_span_respin(mlxsw_sp);
   2782
   2783	entry_connected = nud_state & NUD_VALID && !dead;
   2784	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
   2785	if (!entry_connected && !neigh_entry)
   2786		goto out;
   2787	if (!neigh_entry) {
   2788		neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
   2789		if (IS_ERR(neigh_entry))
   2790			goto out;
   2791	}
   2792
   2793	if (neigh_entry->connected && entry_connected &&
   2794	    !memcmp(neigh_entry->ha, ha, ETH_ALEN))
   2795		goto out;
   2796
   2797	memcpy(neigh_entry->ha, ha, ETH_ALEN);
   2798	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
   2799	mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
   2800				      dead);
   2801
   2802	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
   2803		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
   2804
   2805out:
   2806	mutex_unlock(&mlxsw_sp->router->lock);
   2807	neigh_release(n);
   2808	kfree(net_work);
   2809}
   2810
   2811static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
   2812
   2813static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
   2814{
   2815	struct mlxsw_sp_netevent_work *net_work =
   2816		container_of(work, struct mlxsw_sp_netevent_work, work);
   2817	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
   2818
   2819	mlxsw_sp_mp_hash_init(mlxsw_sp);
   2820	kfree(net_work);
   2821}
   2822
   2823static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
   2824
   2825static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
   2826{
   2827	struct mlxsw_sp_netevent_work *net_work =
   2828		container_of(work, struct mlxsw_sp_netevent_work, work);
   2829	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
   2830
   2831	__mlxsw_sp_router_init(mlxsw_sp);
   2832	kfree(net_work);
   2833}
   2834
   2835static int mlxsw_sp_router_schedule_work(struct net *net,
   2836					 struct notifier_block *nb,
   2837					 void (*cb)(struct work_struct *))
   2838{
   2839	struct mlxsw_sp_netevent_work *net_work;
   2840	struct mlxsw_sp_router *router;
   2841
   2842	router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
   2843	if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
   2844		return NOTIFY_DONE;
   2845
   2846	net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
   2847	if (!net_work)
   2848		return NOTIFY_BAD;
   2849
   2850	INIT_WORK(&net_work->work, cb);
   2851	net_work->mlxsw_sp = router->mlxsw_sp;
   2852	mlxsw_core_schedule_work(&net_work->work);
   2853	return NOTIFY_DONE;
   2854}
   2855
   2856static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
   2857					  unsigned long event, void *ptr)
   2858{
   2859	struct mlxsw_sp_netevent_work *net_work;
   2860	struct mlxsw_sp_port *mlxsw_sp_port;
   2861	struct mlxsw_sp *mlxsw_sp;
   2862	unsigned long interval;
   2863	struct neigh_parms *p;
   2864	struct neighbour *n;
   2865
   2866	switch (event) {
   2867	case NETEVENT_DELAY_PROBE_TIME_UPDATE:
   2868		p = ptr;
   2869
   2870		/* We don't care about changes in the default table. */
   2871		if (!p->dev || (p->tbl->family != AF_INET &&
   2872				p->tbl->family != AF_INET6))
   2873			return NOTIFY_DONE;
   2874
   2875		/* We are in atomic context and can't take RTNL mutex,
   2876		 * so use RCU variant to walk the device chain.
   2877		 */
   2878		mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
   2879		if (!mlxsw_sp_port)
   2880			return NOTIFY_DONE;
   2881
   2882		mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
   2883		interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
   2884		mlxsw_sp->router->neighs_update.interval = interval;
   2885
   2886		mlxsw_sp_port_dev_put(mlxsw_sp_port);
   2887		break;
   2888	case NETEVENT_NEIGH_UPDATE:
   2889		n = ptr;
   2890
   2891		if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
   2892			return NOTIFY_DONE;
   2893
   2894		mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
   2895		if (!mlxsw_sp_port)
   2896			return NOTIFY_DONE;
   2897
   2898		net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
   2899		if (!net_work) {
   2900			mlxsw_sp_port_dev_put(mlxsw_sp_port);
   2901			return NOTIFY_BAD;
   2902		}
   2903
   2904		INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
   2905		net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
   2906		net_work->n = n;
   2907
   2908		/* Take a reference to ensure the neighbour won't be
   2909		 * destructed until we drop the reference in delayed
   2910		 * work.
   2911		 */
   2912		neigh_clone(n);
   2913		mlxsw_core_schedule_work(&net_work->work);
   2914		mlxsw_sp_port_dev_put(mlxsw_sp_port);
   2915		break;
   2916	case NETEVENT_IPV4_MPATH_HASH_UPDATE:
   2917	case NETEVENT_IPV6_MPATH_HASH_UPDATE:
   2918		return mlxsw_sp_router_schedule_work(ptr, nb,
   2919				mlxsw_sp_router_mp_hash_event_work);
   2920
   2921	case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
   2922		return mlxsw_sp_router_schedule_work(ptr, nb,
   2923				mlxsw_sp_router_update_priority_work);
   2924	}
   2925
   2926	return NOTIFY_DONE;
   2927}
   2928
   2929static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
   2930{
   2931	int err;
   2932
   2933	err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
   2934			      &mlxsw_sp_neigh_ht_params);
   2935	if (err)
   2936		return err;
   2937
   2938	/* Initialize the polling interval according to the default
   2939	 * table.
   2940	 */
   2941	mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
   2942
   2943	/* Create the delayed works for the activity_update */
   2944	INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
   2945			  mlxsw_sp_router_neighs_update_work);
   2946	INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
   2947			  mlxsw_sp_router_probe_unresolved_nexthops);
   2948	atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0);
   2949	mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
   2950	mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
   2951	return 0;
   2952}
   2953
   2954static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
   2955{
   2956	cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
   2957	cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
   2958	rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
   2959}
   2960
   2961static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
   2962					 struct mlxsw_sp_rif *rif)
   2963{
   2964	struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
   2965
   2966	list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
   2967				 rif_list_node) {
   2968		mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
   2969		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
   2970	}
   2971}
   2972
   2973enum mlxsw_sp_nexthop_type {
   2974	MLXSW_SP_NEXTHOP_TYPE_ETH,
   2975	MLXSW_SP_NEXTHOP_TYPE_IPIP,
   2976};
   2977
   2978enum mlxsw_sp_nexthop_action {
   2979	/* Nexthop forwards packets to an egress RIF */
   2980	MLXSW_SP_NEXTHOP_ACTION_FORWARD,
   2981	/* Nexthop discards packets */
   2982	MLXSW_SP_NEXTHOP_ACTION_DISCARD,
   2983	/* Nexthop traps packets */
   2984	MLXSW_SP_NEXTHOP_ACTION_TRAP,
   2985};
   2986
   2987struct mlxsw_sp_nexthop_key {
   2988	struct fib_nh *fib_nh;
   2989};
   2990
   2991struct mlxsw_sp_nexthop {
   2992	struct list_head neigh_list_node; /* member of neigh entry list */
   2993	struct list_head rif_list_node;
   2994	struct list_head router_list_node;
   2995	struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
   2996						   * this nexthop belongs to
   2997						   */
   2998	struct rhash_head ht_node;
   2999	struct neigh_table *neigh_tbl;
   3000	struct mlxsw_sp_nexthop_key key;
   3001	unsigned char gw_addr[sizeof(struct in6_addr)];
   3002	int ifindex;
   3003	int nh_weight;
   3004	int norm_nh_weight;
   3005	int num_adj_entries;
   3006	struct mlxsw_sp_rif *rif;
   3007	u8 should_offload:1, /* set indicates this nexthop should be written
   3008			      * to the adjacency table.
   3009			      */
   3010	   offloaded:1, /* set indicates this nexthop was written to the
   3011			 * adjacency table.
   3012			 */
   3013	   update:1; /* set indicates this nexthop should be updated in the
   3014		      * adjacency table (f.e., its MAC changed).
   3015		      */
   3016	enum mlxsw_sp_nexthop_action action;
   3017	enum mlxsw_sp_nexthop_type type;
   3018	union {
   3019		struct mlxsw_sp_neigh_entry *neigh_entry;
   3020		struct mlxsw_sp_ipip_entry *ipip_entry;
   3021	};
   3022	unsigned int counter_index;
   3023	bool counter_valid;
   3024};
   3025
   3026enum mlxsw_sp_nexthop_group_type {
   3027	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
   3028	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
   3029	MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
   3030};
   3031
   3032struct mlxsw_sp_nexthop_group_info {
   3033	struct mlxsw_sp_nexthop_group *nh_grp;
   3034	u32 adj_index;
   3035	u16 ecmp_size;
   3036	u16 count;
   3037	int sum_norm_weight;
   3038	u8 adj_index_valid:1,
   3039	   gateway:1, /* routes using the group use a gateway */
   3040	   is_resilient:1;
   3041	struct list_head list; /* member in nh_res_grp_list */
   3042	struct mlxsw_sp_nexthop nexthops[0];
   3043#define nh_rif	nexthops[0].rif
   3044};
   3045
   3046struct mlxsw_sp_nexthop_group_vr_key {
   3047	u16 vr_id;
   3048	enum mlxsw_sp_l3proto proto;
   3049};
   3050
   3051struct mlxsw_sp_nexthop_group_vr_entry {
   3052	struct list_head list; /* member in vr_list */
   3053	struct rhash_head ht_node; /* member in vr_ht */
   3054	refcount_t ref_count;
   3055	struct mlxsw_sp_nexthop_group_vr_key key;
   3056};
   3057
   3058struct mlxsw_sp_nexthop_group {
   3059	struct rhash_head ht_node;
   3060	struct list_head fib_list; /* list of fib entries that use this group */
   3061	union {
   3062		struct {
   3063			struct fib_info *fi;
   3064		} ipv4;
   3065		struct {
   3066			u32 id;
   3067		} obj;
   3068	};
   3069	struct mlxsw_sp_nexthop_group_info *nhgi;
   3070	struct list_head vr_list;
   3071	struct rhashtable vr_ht;
   3072	enum mlxsw_sp_nexthop_group_type type;
   3073	bool can_destroy;
   3074};
   3075
   3076void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
   3077				    struct mlxsw_sp_nexthop *nh)
   3078{
   3079	struct devlink *devlink;
   3080
   3081	devlink = priv_to_devlink(mlxsw_sp->core);
   3082	if (!devlink_dpipe_table_counter_enabled(devlink,
   3083						 MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
   3084		return;
   3085
   3086	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
   3087		return;
   3088
   3089	nh->counter_valid = true;
   3090}
   3091
   3092void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
   3093				   struct mlxsw_sp_nexthop *nh)
   3094{
   3095	if (!nh->counter_valid)
   3096		return;
   3097	mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
   3098	nh->counter_valid = false;
   3099}
   3100
   3101int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
   3102				 struct mlxsw_sp_nexthop *nh, u64 *p_counter)
   3103{
   3104	if (!nh->counter_valid)
   3105		return -EINVAL;
   3106
   3107	return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
   3108					 p_counter, NULL);
   3109}
   3110
   3111struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
   3112					       struct mlxsw_sp_nexthop *nh)
   3113{
   3114	if (!nh) {
   3115		if (list_empty(&router->nexthop_list))
   3116			return NULL;
   3117		else
   3118			return list_first_entry(&router->nexthop_list,
   3119						typeof(*nh), router_list_node);
   3120	}
   3121	if (list_is_last(&nh->router_list_node, &router->nexthop_list))
   3122		return NULL;
   3123	return list_next_entry(nh, router_list_node);
   3124}
   3125
   3126bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
   3127{
   3128	return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
   3129}
   3130
   3131unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
   3132{
   3133	if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
   3134	    !mlxsw_sp_nexthop_is_forward(nh))
   3135		return NULL;
   3136	return nh->neigh_entry->ha;
   3137}
   3138
   3139int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
   3140			     u32 *p_adj_size, u32 *p_adj_hash_index)
   3141{
   3142	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
   3143	u32 adj_hash_index = 0;
   3144	int i;
   3145
   3146	if (!nh->offloaded || !nhgi->adj_index_valid)
   3147		return -EINVAL;
   3148
   3149	*p_adj_index = nhgi->adj_index;
   3150	*p_adj_size = nhgi->ecmp_size;
   3151
   3152	for (i = 0; i < nhgi->count; i++) {
   3153		struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
   3154
   3155		if (nh_iter == nh)
   3156			break;
   3157		if (nh_iter->offloaded)
   3158			adj_hash_index += nh_iter->num_adj_entries;
   3159	}
   3160
   3161	*p_adj_hash_index = adj_hash_index;
   3162	return 0;
   3163}
   3164
   3165struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
   3166{
   3167	return nh->rif;
   3168}
   3169
   3170bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
   3171{
   3172	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
   3173	int i;
   3174
   3175	for (i = 0; i < nhgi->count; i++) {
   3176		struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
   3177
   3178		if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
   3179			return true;
   3180	}
   3181	return false;
   3182}
   3183
   3184static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
   3185	.key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
   3186	.head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
   3187	.key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
   3188	.automatic_shrinking = true,
   3189};
   3190
   3191static struct mlxsw_sp_nexthop_group_vr_entry *
   3192mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
   3193				       const struct mlxsw_sp_fib *fib)
   3194{
   3195	struct mlxsw_sp_nexthop_group_vr_key key;
   3196
   3197	memset(&key, 0, sizeof(key));
   3198	key.vr_id = fib->vr->id;
   3199	key.proto = fib->proto;
   3200	return rhashtable_lookup_fast(&nh_grp->vr_ht, &key,
   3201				      mlxsw_sp_nexthop_group_vr_ht_params);
   3202}
   3203
   3204static int
   3205mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
   3206				       const struct mlxsw_sp_fib *fib)
   3207{
   3208	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
   3209	int err;
   3210
   3211	vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL);
   3212	if (!vr_entry)
   3213		return -ENOMEM;
   3214
   3215	vr_entry->key.vr_id = fib->vr->id;
   3216	vr_entry->key.proto = fib->proto;
   3217	refcount_set(&vr_entry->ref_count, 1);
   3218
   3219	err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
   3220				     mlxsw_sp_nexthop_group_vr_ht_params);
   3221	if (err)
   3222		goto err_hashtable_insert;
   3223
   3224	list_add(&vr_entry->list, &nh_grp->vr_list);
   3225
   3226	return 0;
   3227
   3228err_hashtable_insert:
   3229	kfree(vr_entry);
   3230	return err;
   3231}
   3232
   3233static void
   3234mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
   3235					struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
   3236{
   3237	list_del(&vr_entry->list);
   3238	rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
   3239			       mlxsw_sp_nexthop_group_vr_ht_params);
   3240	kfree(vr_entry);
   3241}
   3242
   3243static int
   3244mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
   3245			       const struct mlxsw_sp_fib *fib)
   3246{
   3247	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
   3248
   3249	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
   3250	if (vr_entry) {
   3251		refcount_inc(&vr_entry->ref_count);
   3252		return 0;
   3253	}
   3254
   3255	return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
   3256}
   3257
   3258static void
   3259mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
   3260				 const struct mlxsw_sp_fib *fib)
   3261{
   3262	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
   3263
   3264	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
   3265	if (WARN_ON_ONCE(!vr_entry))
   3266		return;
   3267
   3268	if (!refcount_dec_and_test(&vr_entry->ref_count))
   3269		return;
   3270
   3271	mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
   3272}
   3273
   3274struct mlxsw_sp_nexthop_group_cmp_arg {
   3275	enum mlxsw_sp_nexthop_group_type type;
   3276	union {
   3277		struct fib_info *fi;
   3278		struct mlxsw_sp_fib6_entry *fib6_entry;
   3279		u32 id;
   3280	};
   3281};
   3282
   3283static bool
   3284mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
   3285				    const struct in6_addr *gw, int ifindex,
   3286				    int weight)
   3287{
   3288	int i;
   3289
   3290	for (i = 0; i < nh_grp->nhgi->count; i++) {
   3291		const struct mlxsw_sp_nexthop *nh;
   3292
   3293		nh = &nh_grp->nhgi->nexthops[i];
   3294		if (nh->ifindex == ifindex && nh->nh_weight == weight &&
   3295		    ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
   3296			return true;
   3297	}
   3298
   3299	return false;
   3300}
   3301
   3302static bool
   3303mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
   3304			    const struct mlxsw_sp_fib6_entry *fib6_entry)
   3305{
   3306	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   3307
   3308	if (nh_grp->nhgi->count != fib6_entry->nrt6)
   3309		return false;
   3310
   3311	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
   3312		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
   3313		struct in6_addr *gw;
   3314		int ifindex, weight;
   3315
   3316		ifindex = fib6_nh->fib_nh_dev->ifindex;
   3317		weight = fib6_nh->fib_nh_weight;
   3318		gw = &fib6_nh->fib_nh_gw6;
   3319		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
   3320							 weight))
   3321			return false;
   3322	}
   3323
   3324	return true;
   3325}
   3326
   3327static int
   3328mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
   3329{
   3330	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
   3331	const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
   3332
   3333	if (nh_grp->type != cmp_arg->type)
   3334		return 1;
   3335
   3336	switch (cmp_arg->type) {
   3337	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
   3338		return cmp_arg->fi != nh_grp->ipv4.fi;
   3339	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
   3340		return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
   3341						    cmp_arg->fib6_entry);
   3342	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
   3343		return cmp_arg->id != nh_grp->obj.id;
   3344	default:
   3345		WARN_ON(1);
   3346		return 1;
   3347	}
   3348}
   3349
   3350static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
   3351{
   3352	const struct mlxsw_sp_nexthop_group *nh_grp = data;
   3353	const struct mlxsw_sp_nexthop *nh;
   3354	struct fib_info *fi;
   3355	unsigned int val;
   3356	int i;
   3357
   3358	switch (nh_grp->type) {
   3359	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
   3360		fi = nh_grp->ipv4.fi;
   3361		return jhash(&fi, sizeof(fi), seed);
   3362	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
   3363		val = nh_grp->nhgi->count;
   3364		for (i = 0; i < nh_grp->nhgi->count; i++) {
   3365			nh = &nh_grp->nhgi->nexthops[i];
   3366			val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
   3367			val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
   3368		}
   3369		return jhash(&val, sizeof(val), seed);
   3370	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
   3371		return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed);
   3372	default:
   3373		WARN_ON(1);
   3374		return 0;
   3375	}
   3376}
   3377
   3378static u32
   3379mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
   3380{
   3381	unsigned int val = fib6_entry->nrt6;
   3382	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   3383
   3384	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
   3385		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
   3386		struct net_device *dev = fib6_nh->fib_nh_dev;
   3387		struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
   3388
   3389		val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
   3390		val ^= jhash(gw, sizeof(*gw), seed);
   3391	}
   3392
   3393	return jhash(&val, sizeof(val), seed);
   3394}
   3395
   3396static u32
   3397mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
   3398{
   3399	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
   3400
   3401	switch (cmp_arg->type) {
   3402	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
   3403		return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
   3404	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
   3405		return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
   3406	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
   3407		return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed);
   3408	default:
   3409		WARN_ON(1);
   3410		return 0;
   3411	}
   3412}
   3413
   3414static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
   3415	.head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
   3416	.hashfn	     = mlxsw_sp_nexthop_group_hash,
   3417	.obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
   3418	.obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
   3419};
   3420
   3421static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
   3422					 struct mlxsw_sp_nexthop_group *nh_grp)
   3423{
   3424	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
   3425	    !nh_grp->nhgi->gateway)
   3426		return 0;
   3427
   3428	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
   3429				      &nh_grp->ht_node,
   3430				      mlxsw_sp_nexthop_group_ht_params);
   3431}
   3432
   3433static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
   3434					  struct mlxsw_sp_nexthop_group *nh_grp)
   3435{
   3436	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
   3437	    !nh_grp->nhgi->gateway)
   3438		return;
   3439
   3440	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
   3441			       &nh_grp->ht_node,
   3442			       mlxsw_sp_nexthop_group_ht_params);
   3443}
   3444
   3445static struct mlxsw_sp_nexthop_group *
   3446mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
   3447			       struct fib_info *fi)
   3448{
   3449	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
   3450
   3451	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
   3452	cmp_arg.fi = fi;
   3453	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
   3454				      &cmp_arg,
   3455				      mlxsw_sp_nexthop_group_ht_params);
   3456}
   3457
   3458static struct mlxsw_sp_nexthop_group *
   3459mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
   3460			       struct mlxsw_sp_fib6_entry *fib6_entry)
   3461{
   3462	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
   3463
   3464	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
   3465	cmp_arg.fib6_entry = fib6_entry;
   3466	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
   3467				      &cmp_arg,
   3468				      mlxsw_sp_nexthop_group_ht_params);
   3469}
   3470
   3471static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
   3472	.key_offset = offsetof(struct mlxsw_sp_nexthop, key),
   3473	.head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
   3474	.key_len = sizeof(struct mlxsw_sp_nexthop_key),
   3475};
   3476
   3477static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
   3478				   struct mlxsw_sp_nexthop *nh)
   3479{
   3480	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
   3481				      &nh->ht_node, mlxsw_sp_nexthop_ht_params);
   3482}
   3483
   3484static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
   3485				    struct mlxsw_sp_nexthop *nh)
   3486{
   3487	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
   3488			       mlxsw_sp_nexthop_ht_params);
   3489}
   3490
   3491static struct mlxsw_sp_nexthop *
   3492mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
   3493			struct mlxsw_sp_nexthop_key key)
   3494{
   3495	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
   3496				      mlxsw_sp_nexthop_ht_params);
   3497}
   3498
   3499static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
   3500					     enum mlxsw_sp_l3proto proto,
   3501					     u16 vr_id,
   3502					     u32 adj_index, u16 ecmp_size,
   3503					     u32 new_adj_index,
   3504					     u16 new_ecmp_size)
   3505{
   3506	char raleu_pl[MLXSW_REG_RALEU_LEN];
   3507
   3508	mlxsw_reg_raleu_pack(raleu_pl,
   3509			     (enum mlxsw_reg_ralxx_protocol) proto, vr_id,
   3510			     adj_index, ecmp_size, new_adj_index,
   3511			     new_ecmp_size);
   3512	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
   3513}
   3514
   3515static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
   3516					  struct mlxsw_sp_nexthop_group *nh_grp,
   3517					  u32 old_adj_index, u16 old_ecmp_size)
   3518{
   3519	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
   3520	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
   3521	int err;
   3522
   3523	list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
   3524		err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
   3525							vr_entry->key.proto,
   3526							vr_entry->key.vr_id,
   3527							old_adj_index,
   3528							old_ecmp_size,
   3529							nhgi->adj_index,
   3530							nhgi->ecmp_size);
   3531		if (err)
   3532			goto err_mass_update_vr;
   3533	}
   3534	return 0;
   3535
   3536err_mass_update_vr:
   3537	list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
   3538		mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto,
   3539						  vr_entry->key.vr_id,
   3540						  nhgi->adj_index,
   3541						  nhgi->ecmp_size,
   3542						  old_adj_index, old_ecmp_size);
   3543	return err;
   3544}
   3545
   3546static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
   3547					 u32 adj_index,
   3548					 struct mlxsw_sp_nexthop *nh,
   3549					 bool force, char *ratr_pl)
   3550{
   3551	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
   3552	enum mlxsw_reg_ratr_op op;
   3553	u16 rif_index;
   3554
   3555	rif_index = nh->rif ? nh->rif->rif_index :
   3556			      mlxsw_sp->router->lb_rif_index;
   3557	op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY :
   3558		     MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY;
   3559	mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET,
   3560			    adj_index, rif_index);
   3561	switch (nh->action) {
   3562	case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
   3563		mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
   3564		break;
   3565	case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
   3566		mlxsw_reg_ratr_trap_action_set(ratr_pl,
   3567					       MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
   3568		break;
   3569	case MLXSW_SP_NEXTHOP_ACTION_TRAP:
   3570		mlxsw_reg_ratr_trap_action_set(ratr_pl,
   3571					       MLXSW_REG_RATR_TRAP_ACTION_TRAP);
   3572		mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
   3573		break;
   3574	default:
   3575		WARN_ON_ONCE(1);
   3576		return -EINVAL;
   3577	}
   3578	if (nh->counter_valid)
   3579		mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
   3580	else
   3581		mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
   3582
   3583	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
   3584}
   3585
   3586int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
   3587				struct mlxsw_sp_nexthop *nh, bool force,
   3588				char *ratr_pl)
   3589{
   3590	int i;
   3591
   3592	for (i = 0; i < nh->num_adj_entries; i++) {
   3593		int err;
   3594
   3595		err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i,
   3596						    nh, force, ratr_pl);
   3597		if (err)
   3598			return err;
   3599	}
   3600
   3601	return 0;
   3602}
   3603
   3604static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
   3605					  u32 adj_index,
   3606					  struct mlxsw_sp_nexthop *nh,
   3607					  bool force, char *ratr_pl)
   3608{
   3609	const struct mlxsw_sp_ipip_ops *ipip_ops;
   3610
   3611	ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
   3612	return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry,
   3613					force, ratr_pl);
   3614}
   3615
   3616static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
   3617					u32 adj_index,
   3618					struct mlxsw_sp_nexthop *nh, bool force,
   3619					char *ratr_pl)
   3620{
   3621	int i;
   3622
   3623	for (i = 0; i < nh->num_adj_entries; i++) {
   3624		int err;
   3625
   3626		err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
   3627						     nh, force, ratr_pl);
   3628		if (err)
   3629			return err;
   3630	}
   3631
   3632	return 0;
   3633}
   3634
   3635static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
   3636				   struct mlxsw_sp_nexthop *nh, bool force,
   3637				   char *ratr_pl)
   3638{
   3639	/* When action is discard or trap, the nexthop must be
   3640	 * programmed as an Ethernet nexthop.
   3641	 */
   3642	if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
   3643	    nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
   3644	    nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
   3645		return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh,
   3646						   force, ratr_pl);
   3647	else
   3648		return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh,
   3649						    force, ratr_pl);
   3650}
   3651
   3652static int
   3653mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
   3654			      struct mlxsw_sp_nexthop_group_info *nhgi,
   3655			      bool reallocate)
   3656{
   3657	char ratr_pl[MLXSW_REG_RATR_LEN];
   3658	u32 adj_index = nhgi->adj_index; /* base */
   3659	struct mlxsw_sp_nexthop *nh;
   3660	int i;
   3661
   3662	for (i = 0; i < nhgi->count; i++) {
   3663		nh = &nhgi->nexthops[i];
   3664
   3665		if (!nh->should_offload) {
   3666			nh->offloaded = 0;
   3667			continue;
   3668		}
   3669
   3670		if (nh->update || reallocate) {
   3671			int err = 0;
   3672
   3673			err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh,
   3674						      true, ratr_pl);
   3675			if (err)
   3676				return err;
   3677			nh->update = 0;
   3678			nh->offloaded = 1;
   3679		}
   3680		adj_index += nh->num_adj_entries;
   3681	}
   3682	return 0;
   3683}
   3684
   3685static int
   3686mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
   3687				    struct mlxsw_sp_nexthop_group *nh_grp)
   3688{
   3689	struct mlxsw_sp_fib_entry *fib_entry;
   3690	int err;
   3691
   3692	list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
   3693		err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
   3694		if (err)
   3695			return err;
   3696	}
   3697	return 0;
   3698}
   3699
   3700struct mlxsw_sp_adj_grp_size_range {
   3701	u16 start; /* Inclusive */
   3702	u16 end; /* Inclusive */
   3703};
   3704
   3705/* Ordered by range start value */
   3706static const struct mlxsw_sp_adj_grp_size_range
   3707mlxsw_sp1_adj_grp_size_ranges[] = {
   3708	{ .start = 1, .end = 64 },
   3709	{ .start = 512, .end = 512 },
   3710	{ .start = 1024, .end = 1024 },
   3711	{ .start = 2048, .end = 2048 },
   3712	{ .start = 4096, .end = 4096 },
   3713};
   3714
   3715/* Ordered by range start value */
   3716static const struct mlxsw_sp_adj_grp_size_range
   3717mlxsw_sp2_adj_grp_size_ranges[] = {
   3718	{ .start = 1, .end = 128 },
   3719	{ .start = 256, .end = 256 },
   3720	{ .start = 512, .end = 512 },
   3721	{ .start = 1024, .end = 1024 },
   3722	{ .start = 2048, .end = 2048 },
   3723	{ .start = 4096, .end = 4096 },
   3724};
   3725
   3726static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp,
   3727					   u16 *p_adj_grp_size)
   3728{
   3729	int i;
   3730
   3731	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
   3732		const struct mlxsw_sp_adj_grp_size_range *size_range;
   3733
   3734		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
   3735
   3736		if (*p_adj_grp_size >= size_range->start &&
   3737		    *p_adj_grp_size <= size_range->end)
   3738			return;
   3739
   3740		if (*p_adj_grp_size <= size_range->end) {
   3741			*p_adj_grp_size = size_range->end;
   3742			return;
   3743		}
   3744	}
   3745}
   3746
   3747static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp,
   3748					     u16 *p_adj_grp_size,
   3749					     unsigned int alloc_size)
   3750{
   3751	int i;
   3752
   3753	for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) {
   3754		const struct mlxsw_sp_adj_grp_size_range *size_range;
   3755
   3756		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
   3757
   3758		if (alloc_size >= size_range->end) {
   3759			*p_adj_grp_size = size_range->end;
   3760			return;
   3761		}
   3762	}
   3763}
   3764
   3765static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
   3766				     u16 *p_adj_grp_size)
   3767{
   3768	unsigned int alloc_size;
   3769	int err;
   3770
   3771	/* Round up the requested group size to the next size supported
   3772	 * by the device and make sure the request can be satisfied.
   3773	 */
   3774	mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size);
   3775	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
   3776					      MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   3777					      *p_adj_grp_size, &alloc_size);
   3778	if (err)
   3779		return err;
   3780	/* It is possible the allocation results in more allocated
   3781	 * entries than requested. Try to use as much of them as
   3782	 * possible.
   3783	 */
   3784	mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size);
   3785
   3786	return 0;
   3787}
   3788
   3789static void
   3790mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
   3791{
   3792	int i, g = 0, sum_norm_weight = 0;
   3793	struct mlxsw_sp_nexthop *nh;
   3794
   3795	for (i = 0; i < nhgi->count; i++) {
   3796		nh = &nhgi->nexthops[i];
   3797
   3798		if (!nh->should_offload)
   3799			continue;
   3800		if (g > 0)
   3801			g = gcd(nh->nh_weight, g);
   3802		else
   3803			g = nh->nh_weight;
   3804	}
   3805
   3806	for (i = 0; i < nhgi->count; i++) {
   3807		nh = &nhgi->nexthops[i];
   3808
   3809		if (!nh->should_offload)
   3810			continue;
   3811		nh->norm_nh_weight = nh->nh_weight / g;
   3812		sum_norm_weight += nh->norm_nh_weight;
   3813	}
   3814
   3815	nhgi->sum_norm_weight = sum_norm_weight;
   3816}
   3817
   3818static void
   3819mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
   3820{
   3821	int i, weight = 0, lower_bound = 0;
   3822	int total = nhgi->sum_norm_weight;
   3823	u16 ecmp_size = nhgi->ecmp_size;
   3824
   3825	for (i = 0; i < nhgi->count; i++) {
   3826		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
   3827		int upper_bound;
   3828
   3829		if (!nh->should_offload)
   3830			continue;
   3831		weight += nh->norm_nh_weight;
   3832		upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
   3833		nh->num_adj_entries = upper_bound - lower_bound;
   3834		lower_bound = upper_bound;
   3835	}
   3836}
   3837
   3838static struct mlxsw_sp_nexthop *
   3839mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
   3840		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
   3841
   3842static void
   3843mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
   3844					struct mlxsw_sp_nexthop_group *nh_grp)
   3845{
   3846	int i;
   3847
   3848	for (i = 0; i < nh_grp->nhgi->count; i++) {
   3849		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
   3850
   3851		if (nh->offloaded)
   3852			nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
   3853		else
   3854			nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
   3855	}
   3856}
   3857
   3858static void
   3859__mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
   3860					  struct mlxsw_sp_fib6_entry *fib6_entry)
   3861{
   3862	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   3863
   3864	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
   3865		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
   3866		struct mlxsw_sp_nexthop *nh;
   3867
   3868		nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
   3869		if (nh && nh->offloaded)
   3870			fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
   3871		else
   3872			fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
   3873	}
   3874}
   3875
   3876static void
   3877mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
   3878					struct mlxsw_sp_nexthop_group *nh_grp)
   3879{
   3880	struct mlxsw_sp_fib6_entry *fib6_entry;
   3881
   3882	/* Unfortunately, in IPv6 the route and the nexthop are described by
   3883	 * the same struct, so we need to iterate over all the routes using the
   3884	 * nexthop group and set / clear the offload indication for them.
   3885	 */
   3886	list_for_each_entry(fib6_entry, &nh_grp->fib_list,
   3887			    common.nexthop_group_node)
   3888		__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
   3889}
   3890
   3891static void
   3892mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp,
   3893					const struct mlxsw_sp_nexthop *nh,
   3894					u16 bucket_index)
   3895{
   3896	struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp;
   3897	bool offload = false, trap = false;
   3898
   3899	if (nh->offloaded) {
   3900		if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
   3901			trap = true;
   3902		else
   3903			offload = true;
   3904	}
   3905	nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
   3906				    bucket_index, offload, trap);
   3907}
   3908
   3909static void
   3910mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
   3911					   struct mlxsw_sp_nexthop_group *nh_grp)
   3912{
   3913	int i;
   3914
   3915	/* Do not update the flags if the nexthop group is being destroyed
   3916	 * since:
   3917	 * 1. The nexthop objects is being deleted, in which case the flags are
   3918	 * irrelevant.
   3919	 * 2. The nexthop group was replaced by a newer group, in which case
   3920	 * the flags of the nexthop object were already updated based on the
   3921	 * new group.
   3922	 */
   3923	if (nh_grp->can_destroy)
   3924		return;
   3925
   3926	nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
   3927			     nh_grp->nhgi->adj_index_valid, false);
   3928
   3929	/* Update flags of individual nexthop buckets in case of a resilient
   3930	 * nexthop group.
   3931	 */
   3932	if (!nh_grp->nhgi->is_resilient)
   3933		return;
   3934
   3935	for (i = 0; i < nh_grp->nhgi->count; i++) {
   3936		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
   3937
   3938		mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i);
   3939	}
   3940}
   3941
   3942static void
   3943mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
   3944				       struct mlxsw_sp_nexthop_group *nh_grp)
   3945{
   3946	switch (nh_grp->type) {
   3947	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
   3948		mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
   3949		break;
   3950	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
   3951		mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
   3952		break;
   3953	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
   3954		mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
   3955		break;
   3956	}
   3957}
   3958
   3959static int
   3960mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
   3961			       struct mlxsw_sp_nexthop_group *nh_grp)
   3962{
   3963	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
   3964	u16 ecmp_size, old_ecmp_size;
   3965	struct mlxsw_sp_nexthop *nh;
   3966	bool offload_change = false;
   3967	u32 adj_index;
   3968	bool old_adj_index_valid;
   3969	u32 old_adj_index;
   3970	int i, err2, err;
   3971
   3972	if (!nhgi->gateway)
   3973		return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
   3974
   3975	for (i = 0; i < nhgi->count; i++) {
   3976		nh = &nhgi->nexthops[i];
   3977
   3978		if (nh->should_offload != nh->offloaded) {
   3979			offload_change = true;
   3980			if (nh->should_offload)
   3981				nh->update = 1;
   3982		}
   3983	}
   3984	if (!offload_change) {
   3985		/* Nothing was added or removed, so no need to reallocate. Just
   3986		 * update MAC on existing adjacency indexes.
   3987		 */
   3988		err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
   3989		if (err) {
   3990			dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
   3991			goto set_trap;
   3992		}
   3993		/* Flags of individual nexthop buckets might need to be
   3994		 * updated.
   3995		 */
   3996		mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
   3997		return 0;
   3998	}
   3999	mlxsw_sp_nexthop_group_normalize(nhgi);
   4000	if (!nhgi->sum_norm_weight) {
   4001		/* No neigh of this group is connected so we just set
   4002		 * the trap and let everthing flow through kernel.
   4003		 */
   4004		err = 0;
   4005		goto set_trap;
   4006	}
   4007
   4008	ecmp_size = nhgi->sum_norm_weight;
   4009	err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
   4010	if (err)
   4011		/* No valid allocation size available. */
   4012		goto set_trap;
   4013
   4014	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   4015				  ecmp_size, &adj_index);
   4016	if (err) {
   4017		/* We ran out of KVD linear space, just set the
   4018		 * trap and let everything flow through kernel.
   4019		 */
   4020		dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
   4021		goto set_trap;
   4022	}
   4023	old_adj_index_valid = nhgi->adj_index_valid;
   4024	old_adj_index = nhgi->adj_index;
   4025	old_ecmp_size = nhgi->ecmp_size;
   4026	nhgi->adj_index_valid = 1;
   4027	nhgi->adj_index = adj_index;
   4028	nhgi->ecmp_size = ecmp_size;
   4029	mlxsw_sp_nexthop_group_rebalance(nhgi);
   4030	err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
   4031	if (err) {
   4032		dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
   4033		goto set_trap;
   4034	}
   4035
   4036	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
   4037
   4038	if (!old_adj_index_valid) {
   4039		/* The trap was set for fib entries, so we have to call
   4040		 * fib entry update to unset it and use adjacency index.
   4041		 */
   4042		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
   4043		if (err) {
   4044			dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
   4045			goto set_trap;
   4046		}
   4047		return 0;
   4048	}
   4049
   4050	err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
   4051					     old_adj_index, old_ecmp_size);
   4052	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   4053			   old_ecmp_size, old_adj_index);
   4054	if (err) {
   4055		dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
   4056		goto set_trap;
   4057	}
   4058
   4059	return 0;
   4060
   4061set_trap:
   4062	old_adj_index_valid = nhgi->adj_index_valid;
   4063	nhgi->adj_index_valid = 0;
   4064	for (i = 0; i < nhgi->count; i++) {
   4065		nh = &nhgi->nexthops[i];
   4066		nh->offloaded = 0;
   4067	}
   4068	err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
   4069	if (err2)
   4070		dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
   4071	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
   4072	if (old_adj_index_valid)
   4073		mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   4074				   nhgi->ecmp_size, nhgi->adj_index);
   4075	return err;
   4076}
   4077
   4078static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
   4079					    bool removing)
   4080{
   4081	if (!removing) {
   4082		nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
   4083		nh->should_offload = 1;
   4084	} else if (nh->nhgi->is_resilient) {
   4085		nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
   4086		nh->should_offload = 1;
   4087	} else {
   4088		nh->should_offload = 0;
   4089	}
   4090	nh->update = 1;
   4091}
   4092
   4093static int
   4094mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
   4095				    struct mlxsw_sp_neigh_entry *neigh_entry)
   4096{
   4097	struct neighbour *n, *old_n = neigh_entry->key.n;
   4098	struct mlxsw_sp_nexthop *nh;
   4099	bool entry_connected;
   4100	u8 nud_state, dead;
   4101	int err;
   4102
   4103	nh = list_first_entry(&neigh_entry->nexthop_list,
   4104			      struct mlxsw_sp_nexthop, neigh_list_node);
   4105
   4106	n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
   4107	if (!n) {
   4108		n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
   4109		if (IS_ERR(n))
   4110			return PTR_ERR(n);
   4111		neigh_event_send(n, NULL);
   4112	}
   4113
   4114	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
   4115	neigh_entry->key.n = n;
   4116	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
   4117	if (err)
   4118		goto err_neigh_entry_insert;
   4119
   4120	read_lock_bh(&n->lock);
   4121	nud_state = n->nud_state;
   4122	dead = n->dead;
   4123	read_unlock_bh(&n->lock);
   4124	entry_connected = nud_state & NUD_VALID && !dead;
   4125
   4126	list_for_each_entry(nh, &neigh_entry->nexthop_list,
   4127			    neigh_list_node) {
   4128		neigh_release(old_n);
   4129		neigh_clone(n);
   4130		__mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
   4131		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
   4132	}
   4133
   4134	neigh_release(n);
   4135
   4136	return 0;
   4137
   4138err_neigh_entry_insert:
   4139	neigh_entry->key.n = old_n;
   4140	mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
   4141	neigh_release(n);
   4142	return err;
   4143}
   4144
   4145static void
   4146mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
   4147			      struct mlxsw_sp_neigh_entry *neigh_entry,
   4148			      bool removing, bool dead)
   4149{
   4150	struct mlxsw_sp_nexthop *nh;
   4151
   4152	if (list_empty(&neigh_entry->nexthop_list))
   4153		return;
   4154
   4155	if (dead) {
   4156		int err;
   4157
   4158		err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
   4159							  neigh_entry);
   4160		if (err)
   4161			dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
   4162		return;
   4163	}
   4164
   4165	list_for_each_entry(nh, &neigh_entry->nexthop_list,
   4166			    neigh_list_node) {
   4167		__mlxsw_sp_nexthop_neigh_update(nh, removing);
   4168		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
   4169	}
   4170}
   4171
   4172static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
   4173				      struct mlxsw_sp_rif *rif)
   4174{
   4175	if (nh->rif)
   4176		return;
   4177
   4178	nh->rif = rif;
   4179	list_add(&nh->rif_list_node, &rif->nexthop_list);
   4180}
   4181
   4182static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
   4183{
   4184	if (!nh->rif)
   4185		return;
   4186
   4187	list_del(&nh->rif_list_node);
   4188	nh->rif = NULL;
   4189}
   4190
   4191static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
   4192				       struct mlxsw_sp_nexthop *nh)
   4193{
   4194	struct mlxsw_sp_neigh_entry *neigh_entry;
   4195	struct neighbour *n;
   4196	u8 nud_state, dead;
   4197	int err;
   4198
   4199	if (!nh->nhgi->gateway || nh->neigh_entry)
   4200		return 0;
   4201
   4202	/* Take a reference of neigh here ensuring that neigh would
   4203	 * not be destructed before the nexthop entry is finished.
   4204	 * The reference is taken either in neigh_lookup() or
   4205	 * in neigh_create() in case n is not found.
   4206	 */
   4207	n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
   4208	if (!n) {
   4209		n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
   4210		if (IS_ERR(n))
   4211			return PTR_ERR(n);
   4212		neigh_event_send(n, NULL);
   4213	}
   4214	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
   4215	if (!neigh_entry) {
   4216		neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
   4217		if (IS_ERR(neigh_entry)) {
   4218			err = -EINVAL;
   4219			goto err_neigh_entry_create;
   4220		}
   4221	}
   4222
   4223	/* If that is the first nexthop connected to that neigh, add to
   4224	 * nexthop_neighs_list
   4225	 */
   4226	if (list_empty(&neigh_entry->nexthop_list))
   4227		list_add_tail(&neigh_entry->nexthop_neighs_list_node,
   4228			      &mlxsw_sp->router->nexthop_neighs_list);
   4229
   4230	nh->neigh_entry = neigh_entry;
   4231	list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
   4232	read_lock_bh(&n->lock);
   4233	nud_state = n->nud_state;
   4234	dead = n->dead;
   4235	read_unlock_bh(&n->lock);
   4236	__mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
   4237
   4238	return 0;
   4239
   4240err_neigh_entry_create:
   4241	neigh_release(n);
   4242	return err;
   4243}
   4244
   4245static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
   4246					struct mlxsw_sp_nexthop *nh)
   4247{
   4248	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
   4249	struct neighbour *n;
   4250
   4251	if (!neigh_entry)
   4252		return;
   4253	n = neigh_entry->key.n;
   4254
   4255	__mlxsw_sp_nexthop_neigh_update(nh, true);
   4256	list_del(&nh->neigh_list_node);
   4257	nh->neigh_entry = NULL;
   4258
   4259	/* If that is the last nexthop connected to that neigh, remove from
   4260	 * nexthop_neighs_list
   4261	 */
   4262	if (list_empty(&neigh_entry->nexthop_list))
   4263		list_del(&neigh_entry->nexthop_neighs_list_node);
   4264
   4265	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
   4266		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
   4267
   4268	neigh_release(n);
   4269}
   4270
   4271static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
   4272{
   4273	struct net_device *ul_dev;
   4274	bool is_up;
   4275
   4276	rcu_read_lock();
   4277	ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
   4278	is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
   4279	rcu_read_unlock();
   4280
   4281	return is_up;
   4282}
   4283
   4284static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
   4285				       struct mlxsw_sp_nexthop *nh,
   4286				       struct mlxsw_sp_ipip_entry *ipip_entry)
   4287{
   4288	bool removing;
   4289
   4290	if (!nh->nhgi->gateway || nh->ipip_entry)
   4291		return;
   4292
   4293	nh->ipip_entry = ipip_entry;
   4294	removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
   4295	__mlxsw_sp_nexthop_neigh_update(nh, removing);
   4296	mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
   4297}
   4298
   4299static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
   4300				       struct mlxsw_sp_nexthop *nh)
   4301{
   4302	struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
   4303
   4304	if (!ipip_entry)
   4305		return;
   4306
   4307	__mlxsw_sp_nexthop_neigh_update(nh, true);
   4308	nh->ipip_entry = NULL;
   4309}
   4310
   4311static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
   4312					const struct fib_nh *fib_nh,
   4313					enum mlxsw_sp_ipip_type *p_ipipt)
   4314{
   4315	struct net_device *dev = fib_nh->fib_nh_dev;
   4316
   4317	return dev &&
   4318	       fib_nh->nh_parent->fib_type == RTN_UNICAST &&
   4319	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
   4320}
   4321
   4322static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
   4323				      struct mlxsw_sp_nexthop *nh,
   4324				      const struct net_device *dev)
   4325{
   4326	const struct mlxsw_sp_ipip_ops *ipip_ops;
   4327	struct mlxsw_sp_ipip_entry *ipip_entry;
   4328	struct mlxsw_sp_rif *rif;
   4329	int err;
   4330
   4331	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
   4332	if (ipip_entry) {
   4333		ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
   4334		if (ipip_ops->can_offload(mlxsw_sp, dev)) {
   4335			nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
   4336			mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
   4337			return 0;
   4338		}
   4339	}
   4340
   4341	nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
   4342	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   4343	if (!rif)
   4344		return 0;
   4345
   4346	mlxsw_sp_nexthop_rif_init(nh, rif);
   4347	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
   4348	if (err)
   4349		goto err_neigh_init;
   4350
   4351	return 0;
   4352
   4353err_neigh_init:
   4354	mlxsw_sp_nexthop_rif_fini(nh);
   4355	return err;
   4356}
   4357
   4358static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
   4359				       struct mlxsw_sp_nexthop *nh)
   4360{
   4361	switch (nh->type) {
   4362	case MLXSW_SP_NEXTHOP_TYPE_ETH:
   4363		mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
   4364		mlxsw_sp_nexthop_rif_fini(nh);
   4365		break;
   4366	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
   4367		mlxsw_sp_nexthop_rif_fini(nh);
   4368		mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
   4369		break;
   4370	}
   4371}
   4372
   4373static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
   4374				  struct mlxsw_sp_nexthop_group *nh_grp,
   4375				  struct mlxsw_sp_nexthop *nh,
   4376				  struct fib_nh *fib_nh)
   4377{
   4378	struct net_device *dev = fib_nh->fib_nh_dev;
   4379	struct in_device *in_dev;
   4380	int err;
   4381
   4382	nh->nhgi = nh_grp->nhgi;
   4383	nh->key.fib_nh = fib_nh;
   4384#ifdef CONFIG_IP_ROUTE_MULTIPATH
   4385	nh->nh_weight = fib_nh->fib_nh_weight;
   4386#else
   4387	nh->nh_weight = 1;
   4388#endif
   4389	memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
   4390	nh->neigh_tbl = &arp_tbl;
   4391	err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
   4392	if (err)
   4393		return err;
   4394
   4395	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
   4396	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
   4397
   4398	if (!dev)
   4399		return 0;
   4400	nh->ifindex = dev->ifindex;
   4401
   4402	rcu_read_lock();
   4403	in_dev = __in_dev_get_rcu(dev);
   4404	if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
   4405	    fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
   4406		rcu_read_unlock();
   4407		return 0;
   4408	}
   4409	rcu_read_unlock();
   4410
   4411	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
   4412	if (err)
   4413		goto err_nexthop_neigh_init;
   4414
   4415	return 0;
   4416
   4417err_nexthop_neigh_init:
   4418	list_del(&nh->router_list_node);
   4419	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
   4420	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
   4421	return err;
   4422}
   4423
   4424static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
   4425				   struct mlxsw_sp_nexthop *nh)
   4426{
   4427	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
   4428	list_del(&nh->router_list_node);
   4429	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
   4430	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
   4431}
   4432
   4433static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
   4434				    unsigned long event, struct fib_nh *fib_nh)
   4435{
   4436	struct mlxsw_sp_nexthop_key key;
   4437	struct mlxsw_sp_nexthop *nh;
   4438
   4439	key.fib_nh = fib_nh;
   4440	nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
   4441	if (!nh)
   4442		return;
   4443
   4444	switch (event) {
   4445	case FIB_EVENT_NH_ADD:
   4446		mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
   4447		break;
   4448	case FIB_EVENT_NH_DEL:
   4449		mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
   4450		break;
   4451	}
   4452
   4453	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
   4454}
   4455
   4456static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
   4457					struct mlxsw_sp_rif *rif)
   4458{
   4459	struct mlxsw_sp_nexthop *nh;
   4460	bool removing;
   4461
   4462	list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
   4463		switch (nh->type) {
   4464		case MLXSW_SP_NEXTHOP_TYPE_ETH:
   4465			removing = false;
   4466			break;
   4467		case MLXSW_SP_NEXTHOP_TYPE_IPIP:
   4468			removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
   4469			break;
   4470		default:
   4471			WARN_ON(1);
   4472			continue;
   4473		}
   4474
   4475		__mlxsw_sp_nexthop_neigh_update(nh, removing);
   4476		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
   4477	}
   4478}
   4479
   4480static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
   4481					 struct mlxsw_sp_rif *old_rif,
   4482					 struct mlxsw_sp_rif *new_rif)
   4483{
   4484	struct mlxsw_sp_nexthop *nh;
   4485
   4486	list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
   4487	list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
   4488		nh->rif = new_rif;
   4489	mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
   4490}
   4491
   4492static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
   4493					   struct mlxsw_sp_rif *rif)
   4494{
   4495	struct mlxsw_sp_nexthop *nh, *tmp;
   4496
   4497	list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
   4498		mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
   4499		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
   4500	}
   4501}
   4502
   4503static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp)
   4504{
   4505	enum mlxsw_reg_ratr_trap_action trap_action;
   4506	char ratr_pl[MLXSW_REG_RATR_LEN];
   4507	int err;
   4508
   4509	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
   4510				  &mlxsw_sp->router->adj_trap_index);
   4511	if (err)
   4512		return err;
   4513
   4514	trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP;
   4515	mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
   4516			    MLXSW_REG_RATR_TYPE_ETHERNET,
   4517			    mlxsw_sp->router->adj_trap_index,
   4518			    mlxsw_sp->router->lb_rif_index);
   4519	mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
   4520	mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
   4521	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
   4522	if (err)
   4523		goto err_ratr_write;
   4524
   4525	return 0;
   4526
   4527err_ratr_write:
   4528	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
   4529			   mlxsw_sp->router->adj_trap_index);
   4530	return err;
   4531}
   4532
   4533static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp)
   4534{
   4535	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
   4536			   mlxsw_sp->router->adj_trap_index);
   4537}
   4538
   4539static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp)
   4540{
   4541	int err;
   4542
   4543	if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups))
   4544		return 0;
   4545
   4546	err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp);
   4547	if (err)
   4548		return err;
   4549
   4550	refcount_set(&mlxsw_sp->router->num_groups, 1);
   4551
   4552	return 0;
   4553}
   4554
   4555static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp)
   4556{
   4557	if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups))
   4558		return;
   4559
   4560	mlxsw_sp_adj_trap_entry_fini(mlxsw_sp);
   4561}
   4562
   4563static void
   4564mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp,
   4565			     const struct mlxsw_sp_nexthop_group *nh_grp,
   4566			     unsigned long *activity)
   4567{
   4568	char *ratrad_pl;
   4569	int i, err;
   4570
   4571	ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL);
   4572	if (!ratrad_pl)
   4573		return;
   4574
   4575	mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index,
   4576			      nh_grp->nhgi->count);
   4577	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl);
   4578	if (err)
   4579		goto out;
   4580
   4581	for (i = 0; i < nh_grp->nhgi->count; i++) {
   4582		if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i))
   4583			continue;
   4584		bitmap_set(activity, i, 1);
   4585	}
   4586
   4587out:
   4588	kfree(ratrad_pl);
   4589}
   4590
   4591#define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */
   4592
   4593static void
   4594mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp,
   4595				const struct mlxsw_sp_nexthop_group *nh_grp)
   4596{
   4597	unsigned long *activity;
   4598
   4599	activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL);
   4600	if (!activity)
   4601		return;
   4602
   4603	mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity);
   4604	nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
   4605					nh_grp->nhgi->count, activity);
   4606
   4607	bitmap_free(activity);
   4608}
   4609
   4610static void
   4611mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp)
   4612{
   4613	unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL;
   4614
   4615	mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw,
   4616			       msecs_to_jiffies(interval));
   4617}
   4618
   4619static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work)
   4620{
   4621	struct mlxsw_sp_nexthop_group_info *nhgi;
   4622	struct mlxsw_sp_router *router;
   4623	bool reschedule = false;
   4624
   4625	router = container_of(work, struct mlxsw_sp_router,
   4626			      nh_grp_activity_dw.work);
   4627
   4628	mutex_lock(&router->lock);
   4629
   4630	list_for_each_entry(nhgi, &router->nh_res_grp_list, list) {
   4631		mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp);
   4632		reschedule = true;
   4633	}
   4634
   4635	mutex_unlock(&router->lock);
   4636
   4637	if (!reschedule)
   4638		return;
   4639	mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp);
   4640}
   4641
   4642static int
   4643mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
   4644				     const struct nh_notifier_single_info *nh,
   4645				     struct netlink_ext_ack *extack)
   4646{
   4647	int err = -EINVAL;
   4648
   4649	if (nh->is_fdb)
   4650		NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
   4651	else if (nh->has_encap)
   4652		NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
   4653	else
   4654		err = 0;
   4655
   4656	return err;
   4657}
   4658
   4659static int
   4660mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
   4661					  const struct nh_notifier_single_info *nh,
   4662					  struct netlink_ext_ack *extack)
   4663{
   4664	int err;
   4665
   4666	err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
   4667	if (err)
   4668		return err;
   4669
   4670	/* Device only nexthops with an IPIP device are programmed as
   4671	 * encapsulating adjacency entries.
   4672	 */
   4673	if (!nh->gw_family && !nh->is_reject &&
   4674	    !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) {
   4675		NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
   4676		return -EINVAL;
   4677	}
   4678
   4679	return 0;
   4680}
   4681
   4682static int
   4683mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
   4684				    const struct nh_notifier_grp_info *nh_grp,
   4685				    struct netlink_ext_ack *extack)
   4686{
   4687	int i;
   4688
   4689	if (nh_grp->is_fdb) {
   4690		NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
   4691		return -EINVAL;
   4692	}
   4693
   4694	for (i = 0; i < nh_grp->num_nh; i++) {
   4695		const struct nh_notifier_single_info *nh;
   4696		int err;
   4697
   4698		nh = &nh_grp->nh_entries[i].nh;
   4699		err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
   4700								extack);
   4701		if (err)
   4702			return err;
   4703	}
   4704
   4705	return 0;
   4706}
   4707
   4708static int
   4709mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp,
   4710					     const struct nh_notifier_res_table_info *nh_res_table,
   4711					     struct netlink_ext_ack *extack)
   4712{
   4713	unsigned int alloc_size;
   4714	bool valid_size = false;
   4715	int err, i;
   4716
   4717	if (nh_res_table->num_nh_buckets < 32) {
   4718		NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32");
   4719		return -EINVAL;
   4720	}
   4721
   4722	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
   4723		const struct mlxsw_sp_adj_grp_size_range *size_range;
   4724
   4725		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
   4726
   4727		if (nh_res_table->num_nh_buckets >= size_range->start &&
   4728		    nh_res_table->num_nh_buckets <= size_range->end) {
   4729			valid_size = true;
   4730			break;
   4731		}
   4732	}
   4733
   4734	if (!valid_size) {
   4735		NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets");
   4736		return -EINVAL;
   4737	}
   4738
   4739	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
   4740					      MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
   4741					      nh_res_table->num_nh_buckets,
   4742					      &alloc_size);
   4743	if (err || nh_res_table->num_nh_buckets != alloc_size) {
   4744		NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition");
   4745		return -EINVAL;
   4746	}
   4747
   4748	return 0;
   4749}
   4750
   4751static int
   4752mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp,
   4753					const struct nh_notifier_res_table_info *nh_res_table,
   4754					struct netlink_ext_ack *extack)
   4755{
   4756	int err;
   4757	u16 i;
   4758
   4759	err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp,
   4760							   nh_res_table,
   4761							   extack);
   4762	if (err)
   4763		return err;
   4764
   4765	for (i = 0; i < nh_res_table->num_nh_buckets; i++) {
   4766		const struct nh_notifier_single_info *nh;
   4767		int err;
   4768
   4769		nh = &nh_res_table->nhs[i];
   4770		err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
   4771								extack);
   4772		if (err)
   4773			return err;
   4774	}
   4775
   4776	return 0;
   4777}
   4778
   4779static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
   4780					 unsigned long event,
   4781					 struct nh_notifier_info *info)
   4782{
   4783	struct nh_notifier_single_info *nh;
   4784
   4785	if (event != NEXTHOP_EVENT_REPLACE &&
   4786	    event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE &&
   4787	    event != NEXTHOP_EVENT_BUCKET_REPLACE)
   4788		return 0;
   4789
   4790	switch (info->type) {
   4791	case NH_NOTIFIER_INFO_TYPE_SINGLE:
   4792		return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh,
   4793							    info->extack);
   4794	case NH_NOTIFIER_INFO_TYPE_GRP:
   4795		return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
   4796							   info->nh_grp,
   4797							   info->extack);
   4798	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
   4799		return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp,
   4800							       info->nh_res_table,
   4801							       info->extack);
   4802	case NH_NOTIFIER_INFO_TYPE_RES_BUCKET:
   4803		nh = &info->nh_res_bucket->new_nh;
   4804		return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
   4805								 info->extack);
   4806	default:
   4807		NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
   4808		return -EOPNOTSUPP;
   4809	}
   4810}
   4811
   4812static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
   4813					    const struct nh_notifier_info *info)
   4814{
   4815	const struct net_device *dev;
   4816
   4817	switch (info->type) {
   4818	case NH_NOTIFIER_INFO_TYPE_SINGLE:
   4819		dev = info->nh->dev;
   4820		return info->nh->gw_family || info->nh->is_reject ||
   4821		       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
   4822	case NH_NOTIFIER_INFO_TYPE_GRP:
   4823	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
   4824		/* Already validated earlier. */
   4825		return true;
   4826	default:
   4827		return false;
   4828	}
   4829}
   4830
   4831static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp,
   4832						struct mlxsw_sp_nexthop *nh)
   4833{
   4834	u16 lb_rif_index = mlxsw_sp->router->lb_rif_index;
   4835
   4836	nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD;
   4837	nh->should_offload = 1;
   4838	/* While nexthops that discard packets do not forward packets
   4839	 * via an egress RIF, they still need to be programmed using a
   4840	 * valid RIF, so use the loopback RIF created during init.
   4841	 */
   4842	nh->rif = mlxsw_sp->router->rifs[lb_rif_index];
   4843}
   4844
   4845static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp,
   4846						struct mlxsw_sp_nexthop *nh)
   4847{
   4848	nh->rif = NULL;
   4849	nh->should_offload = 0;
   4850}
   4851
   4852static int
   4853mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
   4854			  struct mlxsw_sp_nexthop_group *nh_grp,
   4855			  struct mlxsw_sp_nexthop *nh,
   4856			  struct nh_notifier_single_info *nh_obj, int weight)
   4857{
   4858	struct net_device *dev = nh_obj->dev;
   4859	int err;
   4860
   4861	nh->nhgi = nh_grp->nhgi;
   4862	nh->nh_weight = weight;
   4863
   4864	switch (nh_obj->gw_family) {
   4865	case AF_INET:
   4866		memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
   4867		nh->neigh_tbl = &arp_tbl;
   4868		break;
   4869	case AF_INET6:
   4870		memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
   4871#if IS_ENABLED(CONFIG_IPV6)
   4872		nh->neigh_tbl = &nd_tbl;
   4873#endif
   4874		break;
   4875	}
   4876
   4877	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
   4878	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
   4879	nh->ifindex = dev->ifindex;
   4880
   4881	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
   4882	if (err)
   4883		goto err_type_init;
   4884
   4885	if (nh_obj->is_reject)
   4886		mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh);
   4887
   4888	/* In a resilient nexthop group, all the nexthops must be written to
   4889	 * the adjacency table. Even if they do not have a valid neighbour or
   4890	 * RIF.
   4891	 */
   4892	if (nh_grp->nhgi->is_resilient && !nh->should_offload) {
   4893		nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
   4894		nh->should_offload = 1;
   4895	}
   4896
   4897	return 0;
   4898
   4899err_type_init:
   4900	list_del(&nh->router_list_node);
   4901	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
   4902	return err;
   4903}
   4904
   4905static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp,
   4906				      struct mlxsw_sp_nexthop *nh)
   4907{
   4908	if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD)
   4909		mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh);
   4910	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
   4911	list_del(&nh->router_list_node);
   4912	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
   4913	nh->should_offload = 0;
   4914}
   4915
   4916static int
   4917mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp,
   4918				     struct mlxsw_sp_nexthop_group *nh_grp,
   4919				     struct nh_notifier_info *info)
   4920{
   4921	struct mlxsw_sp_nexthop_group_info *nhgi;
   4922	struct mlxsw_sp_nexthop *nh;
   4923	bool is_resilient = false;
   4924	unsigned int nhs;
   4925	int err, i;
   4926
   4927	switch (info->type) {
   4928	case NH_NOTIFIER_INFO_TYPE_SINGLE:
   4929		nhs = 1;
   4930		break;
   4931	case NH_NOTIFIER_INFO_TYPE_GRP:
   4932		nhs = info->nh_grp->num_nh;
   4933		break;
   4934	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
   4935		nhs = info->nh_res_table->num_nh_buckets;
   4936		is_resilient = true;
   4937		break;
   4938	default:
   4939		return -EINVAL;
   4940	}
   4941
   4942	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
   4943	if (!nhgi)
   4944		return -ENOMEM;
   4945	nh_grp->nhgi = nhgi;
   4946	nhgi->nh_grp = nh_grp;
   4947	nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info);
   4948	nhgi->is_resilient = is_resilient;
   4949	nhgi->count = nhs;
   4950	for (i = 0; i < nhgi->count; i++) {
   4951		struct nh_notifier_single_info *nh_obj;
   4952		int weight;
   4953
   4954		nh = &nhgi->nexthops[i];
   4955		switch (info->type) {
   4956		case NH_NOTIFIER_INFO_TYPE_SINGLE:
   4957			nh_obj = info->nh;
   4958			weight = 1;
   4959			break;
   4960		case NH_NOTIFIER_INFO_TYPE_GRP:
   4961			nh_obj = &info->nh_grp->nh_entries[i].nh;
   4962			weight = info->nh_grp->nh_entries[i].weight;
   4963			break;
   4964		case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
   4965			nh_obj = &info->nh_res_table->nhs[i];
   4966			weight = 1;
   4967			break;
   4968		default:
   4969			err = -EINVAL;
   4970			goto err_nexthop_obj_init;
   4971		}
   4972		err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj,
   4973						weight);
   4974		if (err)
   4975			goto err_nexthop_obj_init;
   4976	}
   4977	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
   4978	if (err)
   4979		goto err_group_inc;
   4980	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
   4981	if (err) {
   4982		NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device");
   4983		goto err_group_refresh;
   4984	}
   4985
   4986	/* Add resilient nexthop groups to a list so that the activity of their
   4987	 * nexthop buckets will be periodically queried and cleared.
   4988	 */
   4989	if (nhgi->is_resilient) {
   4990		if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
   4991			mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp);
   4992		list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list);
   4993	}
   4994
   4995	return 0;
   4996
   4997err_group_refresh:
   4998	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
   4999err_group_inc:
   5000	i = nhgi->count;
   5001err_nexthop_obj_init:
   5002	for (i--; i >= 0; i--) {
   5003		nh = &nhgi->nexthops[i];
   5004		mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
   5005	}
   5006	kfree(nhgi);
   5007	return err;
   5008}
   5009
   5010static void
   5011mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp,
   5012				     struct mlxsw_sp_nexthop_group *nh_grp)
   5013{
   5014	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
   5015	struct mlxsw_sp_router *router = mlxsw_sp->router;
   5016	int i;
   5017
   5018	if (nhgi->is_resilient) {
   5019		list_del(&nhgi->list);
   5020		if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
   5021			cancel_delayed_work(&router->nh_grp_activity_dw);
   5022	}
   5023
   5024	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
   5025	for (i = nhgi->count - 1; i >= 0; i--) {
   5026		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
   5027
   5028		mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
   5029	}
   5030	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
   5031	WARN_ON_ONCE(nhgi->adj_index_valid);
   5032	kfree(nhgi);
   5033}
   5034
   5035static struct mlxsw_sp_nexthop_group *
   5036mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp,
   5037				  struct nh_notifier_info *info)
   5038{
   5039	struct mlxsw_sp_nexthop_group *nh_grp;
   5040	int err;
   5041
   5042	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
   5043	if (!nh_grp)
   5044		return ERR_PTR(-ENOMEM);
   5045	INIT_LIST_HEAD(&nh_grp->vr_list);
   5046	err = rhashtable_init(&nh_grp->vr_ht,
   5047			      &mlxsw_sp_nexthop_group_vr_ht_params);
   5048	if (err)
   5049		goto err_nexthop_group_vr_ht_init;
   5050	INIT_LIST_HEAD(&nh_grp->fib_list);
   5051	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
   5052	nh_grp->obj.id = info->id;
   5053
   5054	err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info);
   5055	if (err)
   5056		goto err_nexthop_group_info_init;
   5057
   5058	nh_grp->can_destroy = false;
   5059
   5060	return nh_grp;
   5061
   5062err_nexthop_group_info_init:
   5063	rhashtable_destroy(&nh_grp->vr_ht);
   5064err_nexthop_group_vr_ht_init:
   5065	kfree(nh_grp);
   5066	return ERR_PTR(err);
   5067}
   5068
   5069static void
   5070mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp,
   5071				   struct mlxsw_sp_nexthop_group *nh_grp)
   5072{
   5073	if (!nh_grp->can_destroy)
   5074		return;
   5075	mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp);
   5076	WARN_ON_ONCE(!list_empty(&nh_grp->fib_list));
   5077	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
   5078	rhashtable_destroy(&nh_grp->vr_ht);
   5079	kfree(nh_grp);
   5080}
   5081
   5082static struct mlxsw_sp_nexthop_group *
   5083mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id)
   5084{
   5085	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
   5086
   5087	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
   5088	cmp_arg.id = id;
   5089	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
   5090				      &cmp_arg,
   5091				      mlxsw_sp_nexthop_group_ht_params);
   5092}
   5093
   5094static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp,
   5095					  struct mlxsw_sp_nexthop_group *nh_grp)
   5096{
   5097	return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
   5098}
   5099
   5100static int
   5101mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp,
   5102				   struct mlxsw_sp_nexthop_group *nh_grp,
   5103				   struct mlxsw_sp_nexthop_group *old_nh_grp,
   5104				   struct netlink_ext_ack *extack)
   5105{
   5106	struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi;
   5107	struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi;
   5108	int err;
   5109
   5110	old_nh_grp->nhgi = new_nhgi;
   5111	new_nhgi->nh_grp = old_nh_grp;
   5112	nh_grp->nhgi = old_nhgi;
   5113	old_nhgi->nh_grp = nh_grp;
   5114
   5115	if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
   5116		/* Both the old adjacency index and the new one are valid.
   5117		 * Routes are currently using the old one. Tell the device to
   5118		 * replace the old adjacency index with the new one.
   5119		 */
   5120		err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp,
   5121						     old_nhgi->adj_index,
   5122						     old_nhgi->ecmp_size);
   5123		if (err) {
   5124			NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one");
   5125			goto err_out;
   5126		}
   5127	} else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) {
   5128		/* The old adjacency index is valid, while the new one is not.
   5129		 * Iterate over all the routes using the group and change them
   5130		 * to trap packets to the CPU.
   5131		 */
   5132		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
   5133		if (err) {
   5134			NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets");
   5135			goto err_out;
   5136		}
   5137	} else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
   5138		/* The old adjacency index is invalid, while the new one is.
   5139		 * Iterate over all the routes using the group and change them
   5140		 * to forward packets using the new valid index.
   5141		 */
   5142		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
   5143		if (err) {
   5144			NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets");
   5145			goto err_out;
   5146		}
   5147	}
   5148
   5149	/* Make sure the flags are set / cleared based on the new nexthop group
   5150	 * information.
   5151	 */
   5152	mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp);
   5153
   5154	/* At this point 'nh_grp' is just a shell that is not used by anyone
   5155	 * and its nexthop group info is the old info that was just replaced
   5156	 * with the new one. Remove it.
   5157	 */
   5158	nh_grp->can_destroy = true;
   5159	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
   5160
   5161	return 0;
   5162
   5163err_out:
   5164	old_nhgi->nh_grp = old_nh_grp;
   5165	nh_grp->nhgi = new_nhgi;
   5166	new_nhgi->nh_grp = nh_grp;
   5167	old_nh_grp->nhgi = old_nhgi;
   5168	return err;
   5169}
   5170
   5171static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp,
   5172				    struct nh_notifier_info *info)
   5173{
   5174	struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp;
   5175	struct netlink_ext_ack *extack = info->extack;
   5176	int err;
   5177
   5178	nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info);
   5179	if (IS_ERR(nh_grp))
   5180		return PTR_ERR(nh_grp);
   5181
   5182	old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
   5183	if (!old_nh_grp)
   5184		err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp);
   5185	else
   5186		err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp,
   5187							 old_nh_grp, extack);
   5188
   5189	if (err) {
   5190		nh_grp->can_destroy = true;
   5191		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
   5192	}
   5193
   5194	return err;
   5195}
   5196
   5197static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp,
   5198				     struct nh_notifier_info *info)
   5199{
   5200	struct mlxsw_sp_nexthop_group *nh_grp;
   5201
   5202	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
   5203	if (!nh_grp)
   5204		return;
   5205
   5206	nh_grp->can_destroy = true;
   5207	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
   5208
   5209	/* If the group still has routes using it, then defer the delete
   5210	 * operation until the last route using it is deleted.
   5211	 */
   5212	if (!list_empty(&nh_grp->fib_list))
   5213		return;
   5214	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
   5215}
   5216
   5217static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp,
   5218					     u32 adj_index, char *ratr_pl)
   5219{
   5220	MLXSW_REG_ZERO(ratr, ratr_pl);
   5221	mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
   5222	mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index);
   5223	mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16);
   5224
   5225	return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
   5226}
   5227
   5228static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new)
   5229{
   5230	/* Clear the opcode and activity on both the old and new payload as
   5231	 * they are irrelevant for the comparison.
   5232	 */
   5233	mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
   5234	mlxsw_reg_ratr_a_set(ratr_pl, 0);
   5235	mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ);
   5236	mlxsw_reg_ratr_a_set(ratr_pl_new, 0);
   5237
   5238	/* If the contents of the adjacency entry are consistent with the
   5239	 * replacement request, then replacement was successful.
   5240	 */
   5241	if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN))
   5242		return 0;
   5243
   5244	return -EINVAL;
   5245}
   5246
   5247static int
   5248mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp,
   5249				       struct mlxsw_sp_nexthop *nh,
   5250				       struct nh_notifier_info *info)
   5251{
   5252	u16 bucket_index = info->nh_res_bucket->bucket_index;
   5253	struct netlink_ext_ack *extack = info->extack;
   5254	bool force = info->nh_res_bucket->force;
   5255	char ratr_pl_new[MLXSW_REG_RATR_LEN];
   5256	char ratr_pl[MLXSW_REG_RATR_LEN];
   5257	u32 adj_index;
   5258	int err;
   5259
   5260	/* No point in trying an atomic replacement if the idle timer interval
   5261	 * is smaller than the interval in which we query and clear activity.
   5262	 */
   5263	if (!force && info->nh_res_bucket->idle_timer_ms <
   5264	    MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL)
   5265		force = true;
   5266
   5267	adj_index = nh->nhgi->adj_index + bucket_index;
   5268	err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl);
   5269	if (err) {
   5270		NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket");
   5271		return err;
   5272	}
   5273
   5274	if (!force) {
   5275		err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index,
   5276							ratr_pl_new);
   5277		if (err) {
   5278			NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent");
   5279			return err;
   5280		}
   5281
   5282		err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new);
   5283		if (err) {
   5284			NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement");
   5285			return err;
   5286		}
   5287	}
   5288
   5289	nh->update = 0;
   5290	nh->offloaded = 1;
   5291	mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index);
   5292
   5293	return 0;
   5294}
   5295
   5296static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp,
   5297					       struct nh_notifier_info *info)
   5298{
   5299	u16 bucket_index = info->nh_res_bucket->bucket_index;
   5300	struct netlink_ext_ack *extack = info->extack;
   5301	struct mlxsw_sp_nexthop_group_info *nhgi;
   5302	struct nh_notifier_single_info *nh_obj;
   5303	struct mlxsw_sp_nexthop_group *nh_grp;
   5304	struct mlxsw_sp_nexthop *nh;
   5305	int err;
   5306
   5307	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
   5308	if (!nh_grp) {
   5309		NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found");
   5310		return -EINVAL;
   5311	}
   5312
   5313	nhgi = nh_grp->nhgi;
   5314
   5315	if (bucket_index >= nhgi->count) {
   5316		NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range");
   5317		return -EINVAL;
   5318	}
   5319
   5320	nh = &nhgi->nexthops[bucket_index];
   5321	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
   5322
   5323	nh_obj = &info->nh_res_bucket->new_nh;
   5324	err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
   5325	if (err) {
   5326		NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement");
   5327		goto err_nexthop_obj_init;
   5328	}
   5329
   5330	err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info);
   5331	if (err)
   5332		goto err_nexthop_obj_bucket_adj_update;
   5333
   5334	return 0;
   5335
   5336err_nexthop_obj_bucket_adj_update:
   5337	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
   5338err_nexthop_obj_init:
   5339	nh_obj = &info->nh_res_bucket->old_nh;
   5340	mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
   5341	/* The old adjacency entry was not overwritten */
   5342	nh->update = 0;
   5343	nh->offloaded = 1;
   5344	return err;
   5345}
   5346
   5347static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb,
   5348				      unsigned long event, void *ptr)
   5349{
   5350	struct nh_notifier_info *info = ptr;
   5351	struct mlxsw_sp_router *router;
   5352	int err = 0;
   5353
   5354	router = container_of(nb, struct mlxsw_sp_router, nexthop_nb);
   5355	err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info);
   5356	if (err)
   5357		goto out;
   5358
   5359	mutex_lock(&router->lock);
   5360
   5361	switch (event) {
   5362	case NEXTHOP_EVENT_REPLACE:
   5363		err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info);
   5364		break;
   5365	case NEXTHOP_EVENT_DEL:
   5366		mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info);
   5367		break;
   5368	case NEXTHOP_EVENT_BUCKET_REPLACE:
   5369		err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp,
   5370							  info);
   5371		break;
   5372	default:
   5373		break;
   5374	}
   5375
   5376	mutex_unlock(&router->lock);
   5377
   5378out:
   5379	return notifier_from_errno(err);
   5380}
   5381
   5382static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
   5383				   struct fib_info *fi)
   5384{
   5385	const struct fib_nh *nh = fib_info_nh(fi, 0);
   5386
   5387	return nh->fib_nh_scope == RT_SCOPE_LINK ||
   5388	       mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
   5389}
   5390
   5391static int
   5392mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
   5393				  struct mlxsw_sp_nexthop_group *nh_grp)
   5394{
   5395	unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
   5396	struct mlxsw_sp_nexthop_group_info *nhgi;
   5397	struct mlxsw_sp_nexthop *nh;
   5398	int err, i;
   5399
   5400	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
   5401	if (!nhgi)
   5402		return -ENOMEM;
   5403	nh_grp->nhgi = nhgi;
   5404	nhgi->nh_grp = nh_grp;
   5405	nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
   5406	nhgi->count = nhs;
   5407	for (i = 0; i < nhgi->count; i++) {
   5408		struct fib_nh *fib_nh;
   5409
   5410		nh = &nhgi->nexthops[i];
   5411		fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
   5412		err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
   5413		if (err)
   5414			goto err_nexthop4_init;
   5415	}
   5416	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
   5417	if (err)
   5418		goto err_group_inc;
   5419	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
   5420	if (err)
   5421		goto err_group_refresh;
   5422
   5423	return 0;
   5424
   5425err_group_refresh:
   5426	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
   5427err_group_inc:
   5428	i = nhgi->count;
   5429err_nexthop4_init:
   5430	for (i--; i >= 0; i--) {
   5431		nh = &nhgi->nexthops[i];
   5432		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
   5433	}
   5434	kfree(nhgi);
   5435	return err;
   5436}
   5437
   5438static void
   5439mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
   5440				  struct mlxsw_sp_nexthop_group *nh_grp)
   5441{
   5442	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
   5443	int i;
   5444
   5445	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
   5446	for (i = nhgi->count - 1; i >= 0; i--) {
   5447		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
   5448
   5449		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
   5450	}
   5451	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
   5452	WARN_ON_ONCE(nhgi->adj_index_valid);
   5453	kfree(nhgi);
   5454}
   5455
   5456static struct mlxsw_sp_nexthop_group *
   5457mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
   5458{
   5459	struct mlxsw_sp_nexthop_group *nh_grp;
   5460	int err;
   5461
   5462	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
   5463	if (!nh_grp)
   5464		return ERR_PTR(-ENOMEM);
   5465	INIT_LIST_HEAD(&nh_grp->vr_list);
   5466	err = rhashtable_init(&nh_grp->vr_ht,
   5467			      &mlxsw_sp_nexthop_group_vr_ht_params);
   5468	if (err)
   5469		goto err_nexthop_group_vr_ht_init;
   5470	INIT_LIST_HEAD(&nh_grp->fib_list);
   5471	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
   5472	nh_grp->ipv4.fi = fi;
   5473	fib_info_hold(fi);
   5474
   5475	err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
   5476	if (err)
   5477		goto err_nexthop_group_info_init;
   5478
   5479	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
   5480	if (err)
   5481		goto err_nexthop_group_insert;
   5482
   5483	nh_grp->can_destroy = true;
   5484
   5485	return nh_grp;
   5486
   5487err_nexthop_group_insert:
   5488	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
   5489err_nexthop_group_info_init:
   5490	fib_info_put(fi);
   5491	rhashtable_destroy(&nh_grp->vr_ht);
   5492err_nexthop_group_vr_ht_init:
   5493	kfree(nh_grp);
   5494	return ERR_PTR(err);
   5495}
   5496
   5497static void
   5498mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
   5499				struct mlxsw_sp_nexthop_group *nh_grp)
   5500{
   5501	if (!nh_grp->can_destroy)
   5502		return;
   5503	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
   5504	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
   5505	fib_info_put(nh_grp->ipv4.fi);
   5506	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
   5507	rhashtable_destroy(&nh_grp->vr_ht);
   5508	kfree(nh_grp);
   5509}
   5510
   5511static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
   5512				       struct mlxsw_sp_fib_entry *fib_entry,
   5513				       struct fib_info *fi)
   5514{
   5515	struct mlxsw_sp_nexthop_group *nh_grp;
   5516
   5517	if (fi->nh) {
   5518		nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
   5519							   fi->nh->id);
   5520		if (WARN_ON_ONCE(!nh_grp))
   5521			return -EINVAL;
   5522		goto out;
   5523	}
   5524
   5525	nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
   5526	if (!nh_grp) {
   5527		nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
   5528		if (IS_ERR(nh_grp))
   5529			return PTR_ERR(nh_grp);
   5530	}
   5531out:
   5532	list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
   5533	fib_entry->nh_group = nh_grp;
   5534	return 0;
   5535}
   5536
   5537static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
   5538					struct mlxsw_sp_fib_entry *fib_entry)
   5539{
   5540	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
   5541
   5542	list_del(&fib_entry->nexthop_group_node);
   5543	if (!list_empty(&nh_grp->fib_list))
   5544		return;
   5545
   5546	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
   5547		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
   5548		return;
   5549	}
   5550
   5551	mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
   5552}
   5553
   5554static bool
   5555mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
   5556{
   5557	struct mlxsw_sp_fib4_entry *fib4_entry;
   5558
   5559	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
   5560				  common);
   5561	return !fib4_entry->dscp;
   5562}
   5563
   5564static bool
   5565mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
   5566{
   5567	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
   5568
   5569	switch (fib_entry->fib_node->fib->proto) {
   5570	case MLXSW_SP_L3_PROTO_IPV4:
   5571		if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
   5572			return false;
   5573		break;
   5574	case MLXSW_SP_L3_PROTO_IPV6:
   5575		break;
   5576	}
   5577
   5578	switch (fib_entry->type) {
   5579	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
   5580		return !!nh_group->nhgi->adj_index_valid;
   5581	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
   5582		return !!nh_group->nhgi->nh_rif;
   5583	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
   5584	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
   5585	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
   5586		return true;
   5587	default:
   5588		return false;
   5589	}
   5590}
   5591
   5592static struct mlxsw_sp_nexthop *
   5593mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
   5594		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
   5595{
   5596	int i;
   5597
   5598	for (i = 0; i < nh_grp->nhgi->count; i++) {
   5599		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
   5600		struct fib6_info *rt = mlxsw_sp_rt6->rt;
   5601
   5602		if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
   5603		    ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
   5604				    &rt->fib6_nh->fib_nh_gw6))
   5605			return nh;
   5606	}
   5607
   5608	return NULL;
   5609}
   5610
   5611static void
   5612mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
   5613				      struct fib_entry_notifier_info *fen_info)
   5614{
   5615	u32 *p_dst = (u32 *) &fen_info->dst;
   5616	struct fib_rt_info fri;
   5617
   5618	fri.fi = fen_info->fi;
   5619	fri.tb_id = fen_info->tb_id;
   5620	fri.dst = cpu_to_be32(*p_dst);
   5621	fri.dst_len = fen_info->dst_len;
   5622	fri.dscp = fen_info->dscp;
   5623	fri.type = fen_info->type;
   5624	fri.offload = false;
   5625	fri.trap = false;
   5626	fri.offload_failed = true;
   5627	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
   5628}
   5629
   5630static void
   5631mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
   5632				 struct mlxsw_sp_fib_entry *fib_entry)
   5633{
   5634	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
   5635	int dst_len = fib_entry->fib_node->key.prefix_len;
   5636	struct mlxsw_sp_fib4_entry *fib4_entry;
   5637	struct fib_rt_info fri;
   5638	bool should_offload;
   5639
   5640	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
   5641	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
   5642				  common);
   5643	fri.fi = fib4_entry->fi;
   5644	fri.tb_id = fib4_entry->tb_id;
   5645	fri.dst = cpu_to_be32(*p_dst);
   5646	fri.dst_len = dst_len;
   5647	fri.dscp = fib4_entry->dscp;
   5648	fri.type = fib4_entry->type;
   5649	fri.offload = should_offload;
   5650	fri.trap = !should_offload;
   5651	fri.offload_failed = false;
   5652	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
   5653}
   5654
   5655static void
   5656mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
   5657				   struct mlxsw_sp_fib_entry *fib_entry)
   5658{
   5659	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
   5660	int dst_len = fib_entry->fib_node->key.prefix_len;
   5661	struct mlxsw_sp_fib4_entry *fib4_entry;
   5662	struct fib_rt_info fri;
   5663
   5664	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
   5665				  common);
   5666	fri.fi = fib4_entry->fi;
   5667	fri.tb_id = fib4_entry->tb_id;
   5668	fri.dst = cpu_to_be32(*p_dst);
   5669	fri.dst_len = dst_len;
   5670	fri.dscp = fib4_entry->dscp;
   5671	fri.type = fib4_entry->type;
   5672	fri.offload = false;
   5673	fri.trap = false;
   5674	fri.offload_failed = false;
   5675	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
   5676}
   5677
   5678#if IS_ENABLED(CONFIG_IPV6)
   5679static void
   5680mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
   5681				      struct fib6_info **rt_arr,
   5682				      unsigned int nrt6)
   5683{
   5684	int i;
   5685
   5686	/* In IPv6 a multipath route is represented using multiple routes, so
   5687	 * we need to set the flags on all of them.
   5688	 */
   5689	for (i = 0; i < nrt6; i++)
   5690		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i],
   5691				       false, false, true);
   5692}
   5693#else
   5694static void
   5695mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
   5696				      struct fib6_info **rt_arr,
   5697				      unsigned int nrt6)
   5698{
   5699}
   5700#endif
   5701
   5702#if IS_ENABLED(CONFIG_IPV6)
   5703static void
   5704mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
   5705				 struct mlxsw_sp_fib_entry *fib_entry)
   5706{
   5707	struct mlxsw_sp_fib6_entry *fib6_entry;
   5708	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   5709	bool should_offload;
   5710
   5711	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
   5712
   5713	/* In IPv6 a multipath route is represented using multiple routes, so
   5714	 * we need to set the flags on all of them.
   5715	 */
   5716	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
   5717				  common);
   5718	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
   5719		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
   5720				       should_offload, !should_offload, false);
   5721}
   5722#else
   5723static void
   5724mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
   5725				 struct mlxsw_sp_fib_entry *fib_entry)
   5726{
   5727}
   5728#endif
   5729
   5730#if IS_ENABLED(CONFIG_IPV6)
   5731static void
   5732mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
   5733				   struct mlxsw_sp_fib_entry *fib_entry)
   5734{
   5735	struct mlxsw_sp_fib6_entry *fib6_entry;
   5736	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   5737
   5738	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
   5739				  common);
   5740	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
   5741		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
   5742				       false, false, false);
   5743}
   5744#else
   5745static void
   5746mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
   5747				   struct mlxsw_sp_fib_entry *fib_entry)
   5748{
   5749}
   5750#endif
   5751
   5752static void
   5753mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
   5754				struct mlxsw_sp_fib_entry *fib_entry)
   5755{
   5756	switch (fib_entry->fib_node->fib->proto) {
   5757	case MLXSW_SP_L3_PROTO_IPV4:
   5758		mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
   5759		break;
   5760	case MLXSW_SP_L3_PROTO_IPV6:
   5761		mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
   5762		break;
   5763	}
   5764}
   5765
   5766static void
   5767mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
   5768				  struct mlxsw_sp_fib_entry *fib_entry)
   5769{
   5770	switch (fib_entry->fib_node->fib->proto) {
   5771	case MLXSW_SP_L3_PROTO_IPV4:
   5772		mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
   5773		break;
   5774	case MLXSW_SP_L3_PROTO_IPV6:
   5775		mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
   5776		break;
   5777	}
   5778}
   5779
   5780static void
   5781mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
   5782				    struct mlxsw_sp_fib_entry *fib_entry,
   5783				    enum mlxsw_sp_fib_entry_op op)
   5784{
   5785	switch (op) {
   5786	case MLXSW_SP_FIB_ENTRY_OP_WRITE:
   5787	case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
   5788		mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
   5789		break;
   5790	case MLXSW_SP_FIB_ENTRY_OP_DELETE:
   5791		mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
   5792		break;
   5793	default:
   5794		break;
   5795	}
   5796}
   5797
   5798struct mlxsw_sp_fib_entry_op_ctx_basic {
   5799	char ralue_pl[MLXSW_REG_RALUE_LEN];
   5800};
   5801
   5802static void
   5803mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5804					enum mlxsw_sp_l3proto proto,
   5805					enum mlxsw_sp_fib_entry_op op,
   5806					u16 virtual_router, u8 prefix_len,
   5807					unsigned char *addr,
   5808					struct mlxsw_sp_fib_entry_priv *priv)
   5809{
   5810	struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
   5811	enum mlxsw_reg_ralxx_protocol ralxx_proto;
   5812	char *ralue_pl = op_ctx_basic->ralue_pl;
   5813	enum mlxsw_reg_ralue_op ralue_op;
   5814
   5815	ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto;
   5816
   5817	switch (op) {
   5818	case MLXSW_SP_FIB_ENTRY_OP_WRITE:
   5819	case MLXSW_SP_FIB_ENTRY_OP_UPDATE:
   5820		ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
   5821		break;
   5822	case MLXSW_SP_FIB_ENTRY_OP_DELETE:
   5823		ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
   5824		break;
   5825	default:
   5826		WARN_ON_ONCE(1);
   5827		return;
   5828	}
   5829
   5830	switch (proto) {
   5831	case MLXSW_SP_L3_PROTO_IPV4:
   5832		mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op,
   5833				      virtual_router, prefix_len, (u32 *) addr);
   5834		break;
   5835	case MLXSW_SP_L3_PROTO_IPV6:
   5836		mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op,
   5837				      virtual_router, prefix_len, addr);
   5838		break;
   5839	}
   5840}
   5841
   5842static void
   5843mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5844						   enum mlxsw_reg_ralue_trap_action trap_action,
   5845						   u16 trap_id, u32 adjacency_index, u16 ecmp_size)
   5846{
   5847	struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
   5848
   5849	mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action,
   5850					trap_id, adjacency_index, ecmp_size);
   5851}
   5852
   5853static void
   5854mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5855						  enum mlxsw_reg_ralue_trap_action trap_action,
   5856						  u16 trap_id, u16 local_erif)
   5857{
   5858	struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
   5859
   5860	mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action,
   5861				       trap_id, local_erif);
   5862}
   5863
   5864static void
   5865mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx)
   5866{
   5867	struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
   5868
   5869	mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl);
   5870}
   5871
   5872static void
   5873mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5874						      u32 tunnel_ptr)
   5875{
   5876	struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
   5877
   5878	mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr);
   5879}
   5880
   5881static int
   5882mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
   5883					  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5884					  bool *postponed_for_bulk)
   5885{
   5886	struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv;
   5887
   5888	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
   5889			       op_ctx_basic->ralue_pl);
   5890}
   5891
   5892static bool
   5893mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv)
   5894{
   5895	return true;
   5896}
   5897
   5898static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5899				    struct mlxsw_sp_fib_entry *fib_entry,
   5900				    enum mlxsw_sp_fib_entry_op op)
   5901{
   5902	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
   5903
   5904	mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv);
   5905	fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id,
   5906				    fib_entry->fib_node->key.prefix_len,
   5907				    fib_entry->fib_node->key.addr,
   5908				    fib_entry->priv);
   5909}
   5910
   5911static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp,
   5912				     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5913				     const struct mlxsw_sp_router_ll_ops *ll_ops)
   5914{
   5915	bool postponed_for_bulk = false;
   5916	int err;
   5917
   5918	err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk);
   5919	if (!postponed_for_bulk)
   5920		mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
   5921	return err;
   5922}
   5923
   5924static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
   5925					struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5926					struct mlxsw_sp_fib_entry *fib_entry,
   5927					enum mlxsw_sp_fib_entry_op op)
   5928{
   5929	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   5930	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
   5931	struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
   5932	enum mlxsw_reg_ralue_trap_action trap_action;
   5933	u16 trap_id = 0;
   5934	u32 adjacency_index = 0;
   5935	u16 ecmp_size = 0;
   5936
   5937	/* In case the nexthop group adjacency index is valid, use it
   5938	 * with provided ECMP size. Otherwise, setup trap and pass
   5939	 * traffic to kernel.
   5940	 */
   5941	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
   5942		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
   5943		adjacency_index = nhgi->adj_index;
   5944		ecmp_size = nhgi->ecmp_size;
   5945	} else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) {
   5946		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
   5947		adjacency_index = mlxsw_sp->router->adj_trap_index;
   5948		ecmp_size = 1;
   5949	} else {
   5950		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
   5951		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
   5952	}
   5953
   5954	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   5955	ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id,
   5956					  adjacency_index, ecmp_size);
   5957	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   5958}
   5959
   5960static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
   5961				       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5962				       struct mlxsw_sp_fib_entry *fib_entry,
   5963				       enum mlxsw_sp_fib_entry_op op)
   5964{
   5965	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   5966	struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif;
   5967	enum mlxsw_reg_ralue_trap_action trap_action;
   5968	u16 trap_id = 0;
   5969	u16 rif_index = 0;
   5970
   5971	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
   5972		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
   5973		rif_index = rif->rif_index;
   5974	} else {
   5975		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
   5976		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
   5977	}
   5978
   5979	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   5980	ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index);
   5981	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   5982}
   5983
   5984static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
   5985				      struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5986				      struct mlxsw_sp_fib_entry *fib_entry,
   5987				      enum mlxsw_sp_fib_entry_op op)
   5988{
   5989	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   5990
   5991	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   5992	ll_ops->fib_entry_act_ip2me_pack(op_ctx);
   5993	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   5994}
   5995
   5996static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
   5997					   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   5998					   struct mlxsw_sp_fib_entry *fib_entry,
   5999					   enum mlxsw_sp_fib_entry_op op)
   6000{
   6001	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   6002	enum mlxsw_reg_ralue_trap_action trap_action;
   6003
   6004	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
   6005	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   6006	ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0);
   6007	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   6008}
   6009
   6010static int
   6011mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
   6012				  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6013				  struct mlxsw_sp_fib_entry *fib_entry,
   6014				  enum mlxsw_sp_fib_entry_op op)
   6015{
   6016	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   6017	enum mlxsw_reg_ralue_trap_action trap_action;
   6018	u16 trap_id;
   6019
   6020	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
   6021	trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
   6022
   6023	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   6024	ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0);
   6025	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   6026}
   6027
   6028static int
   6029mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
   6030				 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6031				 struct mlxsw_sp_fib_entry *fib_entry,
   6032				 enum mlxsw_sp_fib_entry_op op)
   6033{
   6034	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   6035	struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
   6036	const struct mlxsw_sp_ipip_ops *ipip_ops;
   6037	int err;
   6038
   6039	if (WARN_ON(!ipip_entry))
   6040		return -EINVAL;
   6041
   6042	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
   6043	err = ipip_ops->decap_config(mlxsw_sp, ipip_entry,
   6044				     fib_entry->decap.tunnel_index);
   6045	if (err)
   6046		return err;
   6047
   6048	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   6049	ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
   6050					     fib_entry->decap.tunnel_index);
   6051	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   6052}
   6053
   6054static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
   6055					   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6056					   struct mlxsw_sp_fib_entry *fib_entry,
   6057					   enum mlxsw_sp_fib_entry_op op)
   6058{
   6059	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   6060
   6061	mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op);
   6062	ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx,
   6063					     fib_entry->decap.tunnel_index);
   6064	return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops);
   6065}
   6066
   6067static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
   6068				   struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6069				   struct mlxsw_sp_fib_entry *fib_entry,
   6070				   enum mlxsw_sp_fib_entry_op op)
   6071{
   6072	switch (fib_entry->type) {
   6073	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
   6074		return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op);
   6075	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
   6076		return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op);
   6077	case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
   6078		return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op);
   6079	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
   6080		return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op);
   6081	case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
   6082		return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op);
   6083	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
   6084		return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op);
   6085	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
   6086		return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op);
   6087	}
   6088	return -EINVAL;
   6089}
   6090
   6091static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
   6092				 struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6093				 struct mlxsw_sp_fib_entry *fib_entry,
   6094				 enum mlxsw_sp_fib_entry_op op)
   6095{
   6096	int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op);
   6097
   6098	if (err)
   6099		return err;
   6100
   6101	mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
   6102
   6103	return err;
   6104}
   6105
   6106static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
   6107				       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6108				       struct mlxsw_sp_fib_entry *fib_entry,
   6109				       bool is_new)
   6110{
   6111	return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
   6112				     is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE :
   6113					      MLXSW_SP_FIB_ENTRY_OP_UPDATE);
   6114}
   6115
   6116static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
   6117				     struct mlxsw_sp_fib_entry *fib_entry)
   6118{
   6119	struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
   6120
   6121	mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
   6122	return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false);
   6123}
   6124
   6125static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
   6126				  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6127				  struct mlxsw_sp_fib_entry *fib_entry)
   6128{
   6129	const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops;
   6130
   6131	if (!ll_ops->fib_entry_is_committed(fib_entry->priv))
   6132		return 0;
   6133	return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry,
   6134				     MLXSW_SP_FIB_ENTRY_OP_DELETE);
   6135}
   6136
   6137static int
   6138mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
   6139			     const struct fib_entry_notifier_info *fen_info,
   6140			     struct mlxsw_sp_fib_entry *fib_entry)
   6141{
   6142	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
   6143	union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
   6144	struct mlxsw_sp_router *router = mlxsw_sp->router;
   6145	u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
   6146	int ifindex = nhgi->nexthops[0].ifindex;
   6147	struct mlxsw_sp_ipip_entry *ipip_entry;
   6148
   6149	switch (fen_info->type) {
   6150	case RTN_LOCAL:
   6151		ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
   6152							       MLXSW_SP_L3_PROTO_IPV4, dip);
   6153		if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
   6154			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
   6155			return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
   6156							     fib_entry,
   6157							     ipip_entry);
   6158		}
   6159		if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
   6160						 MLXSW_SP_L3_PROTO_IPV4,
   6161						 &dip)) {
   6162			u32 tunnel_index;
   6163
   6164			tunnel_index = router->nve_decap_config.tunnel_index;
   6165			fib_entry->decap.tunnel_index = tunnel_index;
   6166			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
   6167			return 0;
   6168		}
   6169		fallthrough;
   6170	case RTN_BROADCAST:
   6171		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   6172		return 0;
   6173	case RTN_BLACKHOLE:
   6174		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
   6175		return 0;
   6176	case RTN_UNREACHABLE:
   6177	case RTN_PROHIBIT:
   6178		/* Packets hitting these routes need to be trapped, but
   6179		 * can do so with a lower priority than packets directed
   6180		 * at the host, so use action type local instead of trap.
   6181		 */
   6182		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
   6183		return 0;
   6184	case RTN_UNICAST:
   6185		if (nhgi->gateway)
   6186			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
   6187		else
   6188			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
   6189		return 0;
   6190	default:
   6191		return -EINVAL;
   6192	}
   6193}
   6194
   6195static void
   6196mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
   6197			      struct mlxsw_sp_fib_entry *fib_entry)
   6198{
   6199	switch (fib_entry->type) {
   6200	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
   6201		mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
   6202		break;
   6203	default:
   6204		break;
   6205	}
   6206}
   6207
   6208static void
   6209mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
   6210			       struct mlxsw_sp_fib4_entry *fib4_entry)
   6211{
   6212	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common);
   6213}
   6214
   6215static struct mlxsw_sp_fib4_entry *
   6216mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
   6217			   struct mlxsw_sp_fib_node *fib_node,
   6218			   const struct fib_entry_notifier_info *fen_info)
   6219{
   6220	struct mlxsw_sp_fib4_entry *fib4_entry;
   6221	struct mlxsw_sp_fib_entry *fib_entry;
   6222	int err;
   6223
   6224	fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
   6225	if (!fib4_entry)
   6226		return ERR_PTR(-ENOMEM);
   6227	fib_entry = &fib4_entry->common;
   6228
   6229	fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
   6230	if (IS_ERR(fib_entry->priv)) {
   6231		err = PTR_ERR(fib_entry->priv);
   6232		goto err_fib_entry_priv_create;
   6233	}
   6234
   6235	err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
   6236	if (err)
   6237		goto err_nexthop4_group_get;
   6238
   6239	err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
   6240					     fib_node->fib);
   6241	if (err)
   6242		goto err_nexthop_group_vr_link;
   6243
   6244	err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
   6245	if (err)
   6246		goto err_fib4_entry_type_set;
   6247
   6248	fib4_entry->fi = fen_info->fi;
   6249	fib_info_hold(fib4_entry->fi);
   6250	fib4_entry->tb_id = fen_info->tb_id;
   6251	fib4_entry->type = fen_info->type;
   6252	fib4_entry->dscp = fen_info->dscp;
   6253
   6254	fib_entry->fib_node = fib_node;
   6255
   6256	return fib4_entry;
   6257
   6258err_fib4_entry_type_set:
   6259	mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
   6260err_nexthop_group_vr_link:
   6261	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
   6262err_nexthop4_group_get:
   6263	mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
   6264err_fib_entry_priv_create:
   6265	kfree(fib4_entry);
   6266	return ERR_PTR(err);
   6267}
   6268
   6269static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
   6270					struct mlxsw_sp_fib4_entry *fib4_entry)
   6271{
   6272	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
   6273
   6274	fib_info_put(fib4_entry->fi);
   6275	mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry);
   6276	mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group,
   6277					 fib_node->fib);
   6278	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
   6279	mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv);
   6280	kfree(fib4_entry);
   6281}
   6282
   6283static struct mlxsw_sp_fib4_entry *
   6284mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
   6285			   const struct fib_entry_notifier_info *fen_info)
   6286{
   6287	struct mlxsw_sp_fib4_entry *fib4_entry;
   6288	struct mlxsw_sp_fib_node *fib_node;
   6289	struct mlxsw_sp_fib *fib;
   6290	struct mlxsw_sp_vr *vr;
   6291
   6292	vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
   6293	if (!vr)
   6294		return NULL;
   6295	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
   6296
   6297	fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
   6298					    sizeof(fen_info->dst),
   6299					    fen_info->dst_len);
   6300	if (!fib_node)
   6301		return NULL;
   6302
   6303	fib4_entry = container_of(fib_node->fib_entry,
   6304				  struct mlxsw_sp_fib4_entry, common);
   6305	if (fib4_entry->tb_id == fen_info->tb_id &&
   6306	    fib4_entry->dscp == fen_info->dscp &&
   6307	    fib4_entry->type == fen_info->type &&
   6308	    fib4_entry->fi == fen_info->fi)
   6309		return fib4_entry;
   6310
   6311	return NULL;
   6312}
   6313
   6314static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
   6315	.key_offset = offsetof(struct mlxsw_sp_fib_node, key),
   6316	.head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
   6317	.key_len = sizeof(struct mlxsw_sp_fib_key),
   6318	.automatic_shrinking = true,
   6319};
   6320
   6321static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
   6322				    struct mlxsw_sp_fib_node *fib_node)
   6323{
   6324	return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
   6325				      mlxsw_sp_fib_ht_params);
   6326}
   6327
   6328static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
   6329				     struct mlxsw_sp_fib_node *fib_node)
   6330{
   6331	rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
   6332			       mlxsw_sp_fib_ht_params);
   6333}
   6334
   6335static struct mlxsw_sp_fib_node *
   6336mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
   6337			 size_t addr_len, unsigned char prefix_len)
   6338{
   6339	struct mlxsw_sp_fib_key key;
   6340
   6341	memset(&key, 0, sizeof(key));
   6342	memcpy(key.addr, addr, addr_len);
   6343	key.prefix_len = prefix_len;
   6344	return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
   6345}
   6346
   6347static struct mlxsw_sp_fib_node *
   6348mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
   6349			 size_t addr_len, unsigned char prefix_len)
   6350{
   6351	struct mlxsw_sp_fib_node *fib_node;
   6352
   6353	fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
   6354	if (!fib_node)
   6355		return NULL;
   6356
   6357	list_add(&fib_node->list, &fib->node_list);
   6358	memcpy(fib_node->key.addr, addr, addr_len);
   6359	fib_node->key.prefix_len = prefix_len;
   6360
   6361	return fib_node;
   6362}
   6363
   6364static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
   6365{
   6366	list_del(&fib_node->list);
   6367	kfree(fib_node);
   6368}
   6369
   6370static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
   6371				      struct mlxsw_sp_fib_node *fib_node)
   6372{
   6373	struct mlxsw_sp_prefix_usage req_prefix_usage;
   6374	struct mlxsw_sp_fib *fib = fib_node->fib;
   6375	struct mlxsw_sp_lpm_tree *lpm_tree;
   6376	int err;
   6377
   6378	lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
   6379	if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
   6380		goto out;
   6381
   6382	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
   6383	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
   6384	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
   6385					 fib->proto);
   6386	if (IS_ERR(lpm_tree))
   6387		return PTR_ERR(lpm_tree);
   6388
   6389	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
   6390	if (err)
   6391		goto err_lpm_tree_replace;
   6392
   6393out:
   6394	lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
   6395	return 0;
   6396
   6397err_lpm_tree_replace:
   6398	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
   6399	return err;
   6400}
   6401
   6402static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
   6403					 struct mlxsw_sp_fib_node *fib_node)
   6404{
   6405	struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
   6406	struct mlxsw_sp_prefix_usage req_prefix_usage;
   6407	struct mlxsw_sp_fib *fib = fib_node->fib;
   6408	int err;
   6409
   6410	if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
   6411		return;
   6412	/* Try to construct a new LPM tree from the current prefix usage
   6413	 * minus the unused one. If we fail, continue using the old one.
   6414	 */
   6415	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
   6416	mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
   6417				    fib_node->key.prefix_len);
   6418	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
   6419					 fib->proto);
   6420	if (IS_ERR(lpm_tree))
   6421		return;
   6422
   6423	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
   6424	if (err)
   6425		goto err_lpm_tree_replace;
   6426
   6427	return;
   6428
   6429err_lpm_tree_replace:
   6430	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
   6431}
   6432
   6433static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
   6434				  struct mlxsw_sp_fib_node *fib_node,
   6435				  struct mlxsw_sp_fib *fib)
   6436{
   6437	int err;
   6438
   6439	err = mlxsw_sp_fib_node_insert(fib, fib_node);
   6440	if (err)
   6441		return err;
   6442	fib_node->fib = fib;
   6443
   6444	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
   6445	if (err)
   6446		goto err_fib_lpm_tree_link;
   6447
   6448	return 0;
   6449
   6450err_fib_lpm_tree_link:
   6451	fib_node->fib = NULL;
   6452	mlxsw_sp_fib_node_remove(fib, fib_node);
   6453	return err;
   6454}
   6455
   6456static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
   6457				   struct mlxsw_sp_fib_node *fib_node)
   6458{
   6459	struct mlxsw_sp_fib *fib = fib_node->fib;
   6460
   6461	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
   6462	fib_node->fib = NULL;
   6463	mlxsw_sp_fib_node_remove(fib, fib_node);
   6464}
   6465
   6466static struct mlxsw_sp_fib_node *
   6467mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
   6468		      size_t addr_len, unsigned char prefix_len,
   6469		      enum mlxsw_sp_l3proto proto)
   6470{
   6471	struct mlxsw_sp_fib_node *fib_node;
   6472	struct mlxsw_sp_fib *fib;
   6473	struct mlxsw_sp_vr *vr;
   6474	int err;
   6475
   6476	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
   6477	if (IS_ERR(vr))
   6478		return ERR_CAST(vr);
   6479	fib = mlxsw_sp_vr_fib(vr, proto);
   6480
   6481	fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
   6482	if (fib_node)
   6483		return fib_node;
   6484
   6485	fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
   6486	if (!fib_node) {
   6487		err = -ENOMEM;
   6488		goto err_fib_node_create;
   6489	}
   6490
   6491	err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
   6492	if (err)
   6493		goto err_fib_node_init;
   6494
   6495	return fib_node;
   6496
   6497err_fib_node_init:
   6498	mlxsw_sp_fib_node_destroy(fib_node);
   6499err_fib_node_create:
   6500	mlxsw_sp_vr_put(mlxsw_sp, vr);
   6501	return ERR_PTR(err);
   6502}
   6503
   6504static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
   6505				  struct mlxsw_sp_fib_node *fib_node)
   6506{
   6507	struct mlxsw_sp_vr *vr = fib_node->fib->vr;
   6508
   6509	if (fib_node->fib_entry)
   6510		return;
   6511	mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
   6512	mlxsw_sp_fib_node_destroy(fib_node);
   6513	mlxsw_sp_vr_put(mlxsw_sp, vr);
   6514}
   6515
   6516static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
   6517					struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6518					struct mlxsw_sp_fib_entry *fib_entry)
   6519{
   6520	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
   6521	bool is_new = !fib_node->fib_entry;
   6522	int err;
   6523
   6524	fib_node->fib_entry = fib_entry;
   6525
   6526	err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new);
   6527	if (err)
   6528		goto err_fib_entry_update;
   6529
   6530	return 0;
   6531
   6532err_fib_entry_update:
   6533	fib_node->fib_entry = NULL;
   6534	return err;
   6535}
   6536
   6537static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
   6538					    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6539					    struct mlxsw_sp_fib_entry *fib_entry)
   6540{
   6541	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
   6542	int err;
   6543
   6544	err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry);
   6545	fib_node->fib_entry = NULL;
   6546	return err;
   6547}
   6548
   6549static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
   6550					   struct mlxsw_sp_fib_entry *fib_entry)
   6551{
   6552	struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx;
   6553
   6554	mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
   6555	__mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry);
   6556}
   6557
   6558static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
   6559{
   6560	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
   6561	struct mlxsw_sp_fib4_entry *fib4_replaced;
   6562
   6563	if (!fib_node->fib_entry)
   6564		return true;
   6565
   6566	fib4_replaced = container_of(fib_node->fib_entry,
   6567				     struct mlxsw_sp_fib4_entry, common);
   6568	if (fib4_entry->tb_id == RT_TABLE_MAIN &&
   6569	    fib4_replaced->tb_id == RT_TABLE_LOCAL)
   6570		return false;
   6571
   6572	return true;
   6573}
   6574
   6575static int
   6576mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
   6577			     struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6578			     const struct fib_entry_notifier_info *fen_info)
   6579{
   6580	struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
   6581	struct mlxsw_sp_fib_entry *replaced;
   6582	struct mlxsw_sp_fib_node *fib_node;
   6583	int err;
   6584
   6585	if (fen_info->fi->nh &&
   6586	    !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id))
   6587		return 0;
   6588
   6589	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
   6590					 &fen_info->dst, sizeof(fen_info->dst),
   6591					 fen_info->dst_len,
   6592					 MLXSW_SP_L3_PROTO_IPV4);
   6593	if (IS_ERR(fib_node)) {
   6594		dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
   6595		return PTR_ERR(fib_node);
   6596	}
   6597
   6598	fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
   6599	if (IS_ERR(fib4_entry)) {
   6600		dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
   6601		err = PTR_ERR(fib4_entry);
   6602		goto err_fib4_entry_create;
   6603	}
   6604
   6605	if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
   6606		mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
   6607		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   6608		return 0;
   6609	}
   6610
   6611	replaced = fib_node->fib_entry;
   6612	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common);
   6613	if (err) {
   6614		dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
   6615		goto err_fib_node_entry_link;
   6616	}
   6617
   6618	/* Nothing to replace */
   6619	if (!replaced)
   6620		return 0;
   6621
   6622	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
   6623	fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
   6624				     common);
   6625	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
   6626
   6627	return 0;
   6628
   6629err_fib_node_entry_link:
   6630	fib_node->fib_entry = replaced;
   6631	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
   6632err_fib4_entry_create:
   6633	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   6634	return err;
   6635}
   6636
   6637static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
   6638				    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6639				    struct fib_entry_notifier_info *fen_info)
   6640{
   6641	struct mlxsw_sp_fib4_entry *fib4_entry;
   6642	struct mlxsw_sp_fib_node *fib_node;
   6643	int err;
   6644
   6645	fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
   6646	if (!fib4_entry)
   6647		return 0;
   6648	fib_node = fib4_entry->common.fib_node;
   6649
   6650	err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common);
   6651	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
   6652	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   6653	return err;
   6654}
   6655
   6656static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
   6657{
   6658	/* Multicast routes aren't supported, so ignore them. Neighbour
   6659	 * Discovery packets are specifically trapped.
   6660	 */
   6661	if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
   6662		return true;
   6663
   6664	/* Cloned routes are irrelevant in the forwarding path. */
   6665	if (rt->fib6_flags & RTF_CACHE)
   6666		return true;
   6667
   6668	return false;
   6669}
   6670
   6671static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
   6672{
   6673	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   6674
   6675	mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
   6676	if (!mlxsw_sp_rt6)
   6677		return ERR_PTR(-ENOMEM);
   6678
   6679	/* In case of route replace, replaced route is deleted with
   6680	 * no notification. Take reference to prevent accessing freed
   6681	 * memory.
   6682	 */
   6683	mlxsw_sp_rt6->rt = rt;
   6684	fib6_info_hold(rt);
   6685
   6686	return mlxsw_sp_rt6;
   6687}
   6688
   6689#if IS_ENABLED(CONFIG_IPV6)
   6690static void mlxsw_sp_rt6_release(struct fib6_info *rt)
   6691{
   6692	fib6_info_release(rt);
   6693}
   6694#else
   6695static void mlxsw_sp_rt6_release(struct fib6_info *rt)
   6696{
   6697}
   6698#endif
   6699
   6700static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
   6701{
   6702	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
   6703
   6704	if (!mlxsw_sp_rt6->rt->nh)
   6705		fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
   6706	mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
   6707	kfree(mlxsw_sp_rt6);
   6708}
   6709
   6710static struct fib6_info *
   6711mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
   6712{
   6713	return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
   6714				list)->rt;
   6715}
   6716
   6717static struct mlxsw_sp_rt6 *
   6718mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
   6719			    const struct fib6_info *rt)
   6720{
   6721	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   6722
   6723	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
   6724		if (mlxsw_sp_rt6->rt == rt)
   6725			return mlxsw_sp_rt6;
   6726	}
   6727
   6728	return NULL;
   6729}
   6730
   6731static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
   6732					const struct fib6_info *rt,
   6733					enum mlxsw_sp_ipip_type *ret)
   6734{
   6735	return rt->fib6_nh->fib_nh_dev &&
   6736	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
   6737}
   6738
   6739static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
   6740				  struct mlxsw_sp_nexthop_group *nh_grp,
   6741				  struct mlxsw_sp_nexthop *nh,
   6742				  const struct fib6_info *rt)
   6743{
   6744	struct net_device *dev = rt->fib6_nh->fib_nh_dev;
   6745	int err;
   6746
   6747	nh->nhgi = nh_grp->nhgi;
   6748	nh->nh_weight = rt->fib6_nh->fib_nh_weight;
   6749	memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
   6750#if IS_ENABLED(CONFIG_IPV6)
   6751	nh->neigh_tbl = &nd_tbl;
   6752#endif
   6753	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
   6754
   6755	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
   6756
   6757	if (!dev)
   6758		return 0;
   6759	nh->ifindex = dev->ifindex;
   6760
   6761	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
   6762	if (err)
   6763		goto err_nexthop_type_init;
   6764
   6765	return 0;
   6766
   6767err_nexthop_type_init:
   6768	list_del(&nh->router_list_node);
   6769	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
   6770	return err;
   6771}
   6772
   6773static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
   6774				   struct mlxsw_sp_nexthop *nh)
   6775{
   6776	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
   6777	list_del(&nh->router_list_node);
   6778	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
   6779}
   6780
   6781static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
   6782				    const struct fib6_info *rt)
   6783{
   6784	return rt->fib6_nh->fib_nh_gw_family ||
   6785	       mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
   6786}
   6787
   6788static int
   6789mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
   6790				  struct mlxsw_sp_nexthop_group *nh_grp,
   6791				  struct mlxsw_sp_fib6_entry *fib6_entry)
   6792{
   6793	struct mlxsw_sp_nexthop_group_info *nhgi;
   6794	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   6795	struct mlxsw_sp_nexthop *nh;
   6796	int err, i;
   6797
   6798	nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
   6799		       GFP_KERNEL);
   6800	if (!nhgi)
   6801		return -ENOMEM;
   6802	nh_grp->nhgi = nhgi;
   6803	nhgi->nh_grp = nh_grp;
   6804	mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
   6805					struct mlxsw_sp_rt6, list);
   6806	nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
   6807	nhgi->count = fib6_entry->nrt6;
   6808	for (i = 0; i < nhgi->count; i++) {
   6809		struct fib6_info *rt = mlxsw_sp_rt6->rt;
   6810
   6811		nh = &nhgi->nexthops[i];
   6812		err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
   6813		if (err)
   6814			goto err_nexthop6_init;
   6815		mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
   6816	}
   6817	nh_grp->nhgi = nhgi;
   6818	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
   6819	if (err)
   6820		goto err_group_inc;
   6821	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
   6822	if (err)
   6823		goto err_group_refresh;
   6824
   6825	return 0;
   6826
   6827err_group_refresh:
   6828	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
   6829err_group_inc:
   6830	i = nhgi->count;
   6831err_nexthop6_init:
   6832	for (i--; i >= 0; i--) {
   6833		nh = &nhgi->nexthops[i];
   6834		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
   6835	}
   6836	kfree(nhgi);
   6837	return err;
   6838}
   6839
   6840static void
   6841mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
   6842				  struct mlxsw_sp_nexthop_group *nh_grp)
   6843{
   6844	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
   6845	int i;
   6846
   6847	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
   6848	for (i = nhgi->count - 1; i >= 0; i--) {
   6849		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
   6850
   6851		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
   6852	}
   6853	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
   6854	WARN_ON_ONCE(nhgi->adj_index_valid);
   6855	kfree(nhgi);
   6856}
   6857
   6858static struct mlxsw_sp_nexthop_group *
   6859mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
   6860			       struct mlxsw_sp_fib6_entry *fib6_entry)
   6861{
   6862	struct mlxsw_sp_nexthop_group *nh_grp;
   6863	int err;
   6864
   6865	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
   6866	if (!nh_grp)
   6867		return ERR_PTR(-ENOMEM);
   6868	INIT_LIST_HEAD(&nh_grp->vr_list);
   6869	err = rhashtable_init(&nh_grp->vr_ht,
   6870			      &mlxsw_sp_nexthop_group_vr_ht_params);
   6871	if (err)
   6872		goto err_nexthop_group_vr_ht_init;
   6873	INIT_LIST_HEAD(&nh_grp->fib_list);
   6874	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
   6875
   6876	err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
   6877	if (err)
   6878		goto err_nexthop_group_info_init;
   6879
   6880	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
   6881	if (err)
   6882		goto err_nexthop_group_insert;
   6883
   6884	nh_grp->can_destroy = true;
   6885
   6886	return nh_grp;
   6887
   6888err_nexthop_group_insert:
   6889	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
   6890err_nexthop_group_info_init:
   6891	rhashtable_destroy(&nh_grp->vr_ht);
   6892err_nexthop_group_vr_ht_init:
   6893	kfree(nh_grp);
   6894	return ERR_PTR(err);
   6895}
   6896
   6897static void
   6898mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
   6899				struct mlxsw_sp_nexthop_group *nh_grp)
   6900{
   6901	if (!nh_grp->can_destroy)
   6902		return;
   6903	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
   6904	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
   6905	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
   6906	rhashtable_destroy(&nh_grp->vr_ht);
   6907	kfree(nh_grp);
   6908}
   6909
   6910static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
   6911				       struct mlxsw_sp_fib6_entry *fib6_entry)
   6912{
   6913	struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
   6914	struct mlxsw_sp_nexthop_group *nh_grp;
   6915
   6916	if (rt->nh) {
   6917		nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
   6918							   rt->nh->id);
   6919		if (WARN_ON_ONCE(!nh_grp))
   6920			return -EINVAL;
   6921		goto out;
   6922	}
   6923
   6924	nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
   6925	if (!nh_grp) {
   6926		nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
   6927		if (IS_ERR(nh_grp))
   6928			return PTR_ERR(nh_grp);
   6929	}
   6930
   6931	/* The route and the nexthop are described by the same struct, so we
   6932	 * need to the update the nexthop offload indication for the new route.
   6933	 */
   6934	__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
   6935
   6936out:
   6937	list_add_tail(&fib6_entry->common.nexthop_group_node,
   6938		      &nh_grp->fib_list);
   6939	fib6_entry->common.nh_group = nh_grp;
   6940
   6941	return 0;
   6942}
   6943
   6944static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
   6945					struct mlxsw_sp_fib_entry *fib_entry)
   6946{
   6947	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
   6948
   6949	list_del(&fib_entry->nexthop_group_node);
   6950	if (!list_empty(&nh_grp->fib_list))
   6951		return;
   6952
   6953	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
   6954		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
   6955		return;
   6956	}
   6957
   6958	mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
   6959}
   6960
   6961static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
   6962					  struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   6963					  struct mlxsw_sp_fib6_entry *fib6_entry)
   6964{
   6965	struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
   6966	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
   6967	int err;
   6968
   6969	mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib);
   6970	fib6_entry->common.nh_group = NULL;
   6971	list_del(&fib6_entry->common.nexthop_group_node);
   6972
   6973	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
   6974	if (err)
   6975		goto err_nexthop6_group_get;
   6976
   6977	err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group,
   6978					     fib_node->fib);
   6979	if (err)
   6980		goto err_nexthop_group_vr_link;
   6981
   6982	/* In case this entry is offloaded, then the adjacency index
   6983	 * currently associated with it in the device's table is that
   6984	 * of the old group. Start using the new one instead.
   6985	 */
   6986	err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx,
   6987					  &fib6_entry->common, false);
   6988	if (err)
   6989		goto err_fib_entry_update;
   6990
   6991	if (list_empty(&old_nh_grp->fib_list))
   6992		mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
   6993
   6994	return 0;
   6995
   6996err_fib_entry_update:
   6997	mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
   6998					 fib_node->fib);
   6999err_nexthop_group_vr_link:
   7000	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
   7001err_nexthop6_group_get:
   7002	list_add_tail(&fib6_entry->common.nexthop_group_node,
   7003		      &old_nh_grp->fib_list);
   7004	fib6_entry->common.nh_group = old_nh_grp;
   7005	mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib);
   7006	return err;
   7007}
   7008
   7009static int
   7010mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
   7011				struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7012				struct mlxsw_sp_fib6_entry *fib6_entry,
   7013				struct fib6_info **rt_arr, unsigned int nrt6)
   7014{
   7015	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   7016	int err, i;
   7017
   7018	for (i = 0; i < nrt6; i++) {
   7019		mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
   7020		if (IS_ERR(mlxsw_sp_rt6)) {
   7021			err = PTR_ERR(mlxsw_sp_rt6);
   7022			goto err_rt6_unwind;
   7023		}
   7024
   7025		list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
   7026		fib6_entry->nrt6++;
   7027	}
   7028
   7029	err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
   7030	if (err)
   7031		goto err_rt6_unwind;
   7032
   7033	return 0;
   7034
   7035err_rt6_unwind:
   7036	for (; i > 0; i--) {
   7037		fib6_entry->nrt6--;
   7038		mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
   7039					       struct mlxsw_sp_rt6, list);
   7040		list_del(&mlxsw_sp_rt6->list);
   7041		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
   7042	}
   7043	return err;
   7044}
   7045
   7046static void
   7047mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
   7048				struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7049				struct mlxsw_sp_fib6_entry *fib6_entry,
   7050				struct fib6_info **rt_arr, unsigned int nrt6)
   7051{
   7052	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   7053	int i;
   7054
   7055	for (i = 0; i < nrt6; i++) {
   7056		mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
   7057							   rt_arr[i]);
   7058		if (WARN_ON_ONCE(!mlxsw_sp_rt6))
   7059			continue;
   7060
   7061		fib6_entry->nrt6--;
   7062		list_del(&mlxsw_sp_rt6->list);
   7063		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
   7064	}
   7065
   7066	mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry);
   7067}
   7068
   7069static int
   7070mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp,
   7071				   struct mlxsw_sp_fib_entry *fib_entry,
   7072				   const struct fib6_info *rt)
   7073{
   7074	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
   7075	union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr };
   7076	u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id);
   7077	struct mlxsw_sp_router *router = mlxsw_sp->router;
   7078	int ifindex = nhgi->nexthops[0].ifindex;
   7079	struct mlxsw_sp_ipip_entry *ipip_entry;
   7080
   7081	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   7082	ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
   7083						       MLXSW_SP_L3_PROTO_IPV6,
   7084						       dip);
   7085
   7086	if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
   7087		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
   7088		return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry,
   7089						     ipip_entry);
   7090	}
   7091	if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
   7092					 MLXSW_SP_L3_PROTO_IPV6, &dip)) {
   7093		u32 tunnel_index;
   7094
   7095		tunnel_index = router->nve_decap_config.tunnel_index;
   7096		fib_entry->decap.tunnel_index = tunnel_index;
   7097		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
   7098	}
   7099
   7100	return 0;
   7101}
   7102
   7103static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
   7104					struct mlxsw_sp_fib_entry *fib_entry,
   7105					const struct fib6_info *rt)
   7106{
   7107	if (rt->fib6_flags & RTF_LOCAL)
   7108		return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry,
   7109							  rt);
   7110	if (rt->fib6_flags & RTF_ANYCAST)
   7111		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
   7112	else if (rt->fib6_type == RTN_BLACKHOLE)
   7113		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
   7114	else if (rt->fib6_flags & RTF_REJECT)
   7115		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
   7116	else if (fib_entry->nh_group->nhgi->gateway)
   7117		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
   7118	else
   7119		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
   7120
   7121	return 0;
   7122}
   7123
   7124static void
   7125mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
   7126{
   7127	struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
   7128
   7129	list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
   7130				 list) {
   7131		fib6_entry->nrt6--;
   7132		list_del(&mlxsw_sp_rt6->list);
   7133		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
   7134	}
   7135}
   7136
   7137static struct mlxsw_sp_fib6_entry *
   7138mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
   7139			   struct mlxsw_sp_fib_node *fib_node,
   7140			   struct fib6_info **rt_arr, unsigned int nrt6)
   7141{
   7142	struct mlxsw_sp_fib6_entry *fib6_entry;
   7143	struct mlxsw_sp_fib_entry *fib_entry;
   7144	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
   7145	int err, i;
   7146
   7147	fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
   7148	if (!fib6_entry)
   7149		return ERR_PTR(-ENOMEM);
   7150	fib_entry = &fib6_entry->common;
   7151
   7152	fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops);
   7153	if (IS_ERR(fib_entry->priv)) {
   7154		err = PTR_ERR(fib_entry->priv);
   7155		goto err_fib_entry_priv_create;
   7156	}
   7157
   7158	INIT_LIST_HEAD(&fib6_entry->rt6_list);
   7159
   7160	for (i = 0; i < nrt6; i++) {
   7161		mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
   7162		if (IS_ERR(mlxsw_sp_rt6)) {
   7163			err = PTR_ERR(mlxsw_sp_rt6);
   7164			goto err_rt6_unwind;
   7165		}
   7166		list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
   7167		fib6_entry->nrt6++;
   7168	}
   7169
   7170	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
   7171	if (err)
   7172		goto err_rt6_unwind;
   7173
   7174	err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
   7175					     fib_node->fib);
   7176	if (err)
   7177		goto err_nexthop_group_vr_link;
   7178
   7179	err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
   7180	if (err)
   7181		goto err_fib6_entry_type_set;
   7182
   7183	fib_entry->fib_node = fib_node;
   7184
   7185	return fib6_entry;
   7186
   7187err_fib6_entry_type_set:
   7188	mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
   7189err_nexthop_group_vr_link:
   7190	mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry);
   7191err_rt6_unwind:
   7192	for (; i > 0; i--) {
   7193		fib6_entry->nrt6--;
   7194		mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
   7195					       struct mlxsw_sp_rt6, list);
   7196		list_del(&mlxsw_sp_rt6->list);
   7197		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
   7198	}
   7199	mlxsw_sp_fib_entry_priv_put(fib_entry->priv);
   7200err_fib_entry_priv_create:
   7201	kfree(fib6_entry);
   7202	return ERR_PTR(err);
   7203}
   7204
   7205static void
   7206mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
   7207			       struct mlxsw_sp_fib6_entry *fib6_entry)
   7208{
   7209	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common);
   7210}
   7211
   7212static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
   7213					struct mlxsw_sp_fib6_entry *fib6_entry)
   7214{
   7215	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
   7216
   7217	mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry);
   7218	mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
   7219					 fib_node->fib);
   7220	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
   7221	mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
   7222	WARN_ON(fib6_entry->nrt6);
   7223	mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv);
   7224	kfree(fib6_entry);
   7225}
   7226
   7227static struct mlxsw_sp_fib6_entry *
   7228mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
   7229			   const struct fib6_info *rt)
   7230{
   7231	struct mlxsw_sp_fib6_entry *fib6_entry;
   7232	struct mlxsw_sp_fib_node *fib_node;
   7233	struct mlxsw_sp_fib *fib;
   7234	struct fib6_info *cmp_rt;
   7235	struct mlxsw_sp_vr *vr;
   7236
   7237	vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
   7238	if (!vr)
   7239		return NULL;
   7240	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
   7241
   7242	fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
   7243					    sizeof(rt->fib6_dst.addr),
   7244					    rt->fib6_dst.plen);
   7245	if (!fib_node)
   7246		return NULL;
   7247
   7248	fib6_entry = container_of(fib_node->fib_entry,
   7249				  struct mlxsw_sp_fib6_entry, common);
   7250	cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
   7251	if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
   7252	    rt->fib6_metric == cmp_rt->fib6_metric &&
   7253	    mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
   7254		return fib6_entry;
   7255
   7256	return NULL;
   7257}
   7258
   7259static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
   7260{
   7261	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
   7262	struct mlxsw_sp_fib6_entry *fib6_replaced;
   7263	struct fib6_info *rt, *rt_replaced;
   7264
   7265	if (!fib_node->fib_entry)
   7266		return true;
   7267
   7268	fib6_replaced = container_of(fib_node->fib_entry,
   7269				     struct mlxsw_sp_fib6_entry,
   7270				     common);
   7271	rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
   7272	rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
   7273	if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
   7274	    rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
   7275		return false;
   7276
   7277	return true;
   7278}
   7279
   7280static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
   7281					struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7282					struct fib6_info **rt_arr, unsigned int nrt6)
   7283{
   7284	struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
   7285	struct mlxsw_sp_fib_entry *replaced;
   7286	struct mlxsw_sp_fib_node *fib_node;
   7287	struct fib6_info *rt = rt_arr[0];
   7288	int err;
   7289
   7290	if (rt->fib6_src.plen)
   7291		return -EINVAL;
   7292
   7293	if (mlxsw_sp_fib6_rt_should_ignore(rt))
   7294		return 0;
   7295
   7296	if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id))
   7297		return 0;
   7298
   7299	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
   7300					 &rt->fib6_dst.addr,
   7301					 sizeof(rt->fib6_dst.addr),
   7302					 rt->fib6_dst.plen,
   7303					 MLXSW_SP_L3_PROTO_IPV6);
   7304	if (IS_ERR(fib_node))
   7305		return PTR_ERR(fib_node);
   7306
   7307	fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
   7308						nrt6);
   7309	if (IS_ERR(fib6_entry)) {
   7310		err = PTR_ERR(fib6_entry);
   7311		goto err_fib6_entry_create;
   7312	}
   7313
   7314	if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
   7315		mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
   7316		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7317		return 0;
   7318	}
   7319
   7320	replaced = fib_node->fib_entry;
   7321	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common);
   7322	if (err)
   7323		goto err_fib_node_entry_link;
   7324
   7325	/* Nothing to replace */
   7326	if (!replaced)
   7327		return 0;
   7328
   7329	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
   7330	fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
   7331				     common);
   7332	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
   7333
   7334	return 0;
   7335
   7336err_fib_node_entry_link:
   7337	fib_node->fib_entry = replaced;
   7338	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
   7339err_fib6_entry_create:
   7340	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7341	return err;
   7342}
   7343
   7344static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
   7345				       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7346				       struct fib6_info **rt_arr, unsigned int nrt6)
   7347{
   7348	struct mlxsw_sp_fib6_entry *fib6_entry;
   7349	struct mlxsw_sp_fib_node *fib_node;
   7350	struct fib6_info *rt = rt_arr[0];
   7351	int err;
   7352
   7353	if (rt->fib6_src.plen)
   7354		return -EINVAL;
   7355
   7356	if (mlxsw_sp_fib6_rt_should_ignore(rt))
   7357		return 0;
   7358
   7359	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
   7360					 &rt->fib6_dst.addr,
   7361					 sizeof(rt->fib6_dst.addr),
   7362					 rt->fib6_dst.plen,
   7363					 MLXSW_SP_L3_PROTO_IPV6);
   7364	if (IS_ERR(fib_node))
   7365		return PTR_ERR(fib_node);
   7366
   7367	if (WARN_ON_ONCE(!fib_node->fib_entry)) {
   7368		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7369		return -EINVAL;
   7370	}
   7371
   7372	fib6_entry = container_of(fib_node->fib_entry,
   7373				  struct mlxsw_sp_fib6_entry, common);
   7374	err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
   7375	if (err)
   7376		goto err_fib6_entry_nexthop_add;
   7377
   7378	return 0;
   7379
   7380err_fib6_entry_nexthop_add:
   7381	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7382	return err;
   7383}
   7384
   7385static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
   7386				    struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7387				    struct fib6_info **rt_arr, unsigned int nrt6)
   7388{
   7389	struct mlxsw_sp_fib6_entry *fib6_entry;
   7390	struct mlxsw_sp_fib_node *fib_node;
   7391	struct fib6_info *rt = rt_arr[0];
   7392	int err;
   7393
   7394	if (mlxsw_sp_fib6_rt_should_ignore(rt))
   7395		return 0;
   7396
   7397	/* Multipath routes are first added to the FIB trie and only then
   7398	 * notified. If we vetoed the addition, we will get a delete
   7399	 * notification for a route we do not have. Therefore, do not warn if
   7400	 * route was not found.
   7401	 */
   7402	fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
   7403	if (!fib6_entry)
   7404		return 0;
   7405
   7406	/* If not all the nexthops are deleted, then only reduce the nexthop
   7407	 * group.
   7408	 */
   7409	if (nrt6 != fib6_entry->nrt6) {
   7410		mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6);
   7411		return 0;
   7412	}
   7413
   7414	fib_node = fib6_entry->common.fib_node;
   7415
   7416	err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common);
   7417	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
   7418	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7419	return err;
   7420}
   7421
   7422static struct mlxsw_sp_mr_table *
   7423mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
   7424{
   7425	if (family == RTNL_FAMILY_IPMR)
   7426		return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
   7427	else
   7428		return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
   7429}
   7430
   7431static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
   7432				     struct mfc_entry_notifier_info *men_info,
   7433				     bool replace)
   7434{
   7435	struct mlxsw_sp_mr_table *mrt;
   7436	struct mlxsw_sp_vr *vr;
   7437
   7438	vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
   7439	if (IS_ERR(vr))
   7440		return PTR_ERR(vr);
   7441
   7442	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
   7443	return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
   7444}
   7445
   7446static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
   7447				      struct mfc_entry_notifier_info *men_info)
   7448{
   7449	struct mlxsw_sp_mr_table *mrt;
   7450	struct mlxsw_sp_vr *vr;
   7451
   7452	vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
   7453	if (WARN_ON(!vr))
   7454		return;
   7455
   7456	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
   7457	mlxsw_sp_mr_route_del(mrt, men_info->mfc);
   7458	mlxsw_sp_vr_put(mlxsw_sp, vr);
   7459}
   7460
   7461static int
   7462mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
   7463			      struct vif_entry_notifier_info *ven_info)
   7464{
   7465	struct mlxsw_sp_mr_table *mrt;
   7466	struct mlxsw_sp_rif *rif;
   7467	struct mlxsw_sp_vr *vr;
   7468
   7469	vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
   7470	if (IS_ERR(vr))
   7471		return PTR_ERR(vr);
   7472
   7473	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
   7474	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
   7475	return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
   7476				   ven_info->vif_index,
   7477				   ven_info->vif_flags, rif);
   7478}
   7479
   7480static void
   7481mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
   7482			      struct vif_entry_notifier_info *ven_info)
   7483{
   7484	struct mlxsw_sp_mr_table *mrt;
   7485	struct mlxsw_sp_vr *vr;
   7486
   7487	vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
   7488	if (WARN_ON(!vr))
   7489		return;
   7490
   7491	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
   7492	mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
   7493	mlxsw_sp_vr_put(mlxsw_sp, vr);
   7494}
   7495
   7496static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
   7497				     struct mlxsw_sp_fib_node *fib_node)
   7498{
   7499	struct mlxsw_sp_fib4_entry *fib4_entry;
   7500
   7501	fib4_entry = container_of(fib_node->fib_entry,
   7502				  struct mlxsw_sp_fib4_entry, common);
   7503	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
   7504	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
   7505	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7506}
   7507
   7508static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
   7509				     struct mlxsw_sp_fib_node *fib_node)
   7510{
   7511	struct mlxsw_sp_fib6_entry *fib6_entry;
   7512
   7513	fib6_entry = container_of(fib_node->fib_entry,
   7514				  struct mlxsw_sp_fib6_entry, common);
   7515	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
   7516	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
   7517	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
   7518}
   7519
   7520static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
   7521				    struct mlxsw_sp_fib_node *fib_node)
   7522{
   7523	switch (fib_node->fib->proto) {
   7524	case MLXSW_SP_L3_PROTO_IPV4:
   7525		mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
   7526		break;
   7527	case MLXSW_SP_L3_PROTO_IPV6:
   7528		mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
   7529		break;
   7530	}
   7531}
   7532
   7533static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
   7534				  struct mlxsw_sp_vr *vr,
   7535				  enum mlxsw_sp_l3proto proto)
   7536{
   7537	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
   7538	struct mlxsw_sp_fib_node *fib_node, *tmp;
   7539
   7540	list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
   7541		bool do_break = &tmp->list == &fib->node_list;
   7542
   7543		mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
   7544		if (do_break)
   7545			break;
   7546	}
   7547}
   7548
   7549static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
   7550{
   7551	int i, j;
   7552
   7553	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
   7554		struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
   7555
   7556		if (!mlxsw_sp_vr_is_used(vr))
   7557			continue;
   7558
   7559		for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
   7560			mlxsw_sp_mr_table_flush(vr->mr_table[j]);
   7561		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
   7562
   7563		/* If virtual router was only used for IPv4, then it's no
   7564		 * longer used.
   7565		 */
   7566		if (!mlxsw_sp_vr_is_used(vr))
   7567			continue;
   7568		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
   7569	}
   7570}
   7571
   7572struct mlxsw_sp_fib6_event {
   7573	struct fib6_info **rt_arr;
   7574	unsigned int nrt6;
   7575};
   7576
   7577struct mlxsw_sp_fib_event {
   7578	struct list_head list; /* node in fib queue */
   7579	union {
   7580		struct mlxsw_sp_fib6_event fib6_event;
   7581		struct fib_entry_notifier_info fen_info;
   7582		struct fib_rule_notifier_info fr_info;
   7583		struct fib_nh_notifier_info fnh_info;
   7584		struct mfc_entry_notifier_info men_info;
   7585		struct vif_entry_notifier_info ven_info;
   7586	};
   7587	struct mlxsw_sp *mlxsw_sp;
   7588	unsigned long event;
   7589	int family;
   7590};
   7591
   7592static int
   7593mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event,
   7594				struct fib6_entry_notifier_info *fen6_info)
   7595{
   7596	struct fib6_info *rt = fen6_info->rt;
   7597	struct fib6_info **rt_arr;
   7598	struct fib6_info *iter;
   7599	unsigned int nrt6;
   7600	int i = 0;
   7601
   7602	nrt6 = fen6_info->nsiblings + 1;
   7603
   7604	rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
   7605	if (!rt_arr)
   7606		return -ENOMEM;
   7607
   7608	fib6_event->rt_arr = rt_arr;
   7609	fib6_event->nrt6 = nrt6;
   7610
   7611	rt_arr[0] = rt;
   7612	fib6_info_hold(rt);
   7613
   7614	if (!fen6_info->nsiblings)
   7615		return 0;
   7616
   7617	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
   7618		if (i == fen6_info->nsiblings)
   7619			break;
   7620
   7621		rt_arr[i + 1] = iter;
   7622		fib6_info_hold(iter);
   7623		i++;
   7624	}
   7625	WARN_ON_ONCE(i != fen6_info->nsiblings);
   7626
   7627	return 0;
   7628}
   7629
   7630static void
   7631mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event)
   7632{
   7633	int i;
   7634
   7635	for (i = 0; i < fib6_event->nrt6; i++)
   7636		mlxsw_sp_rt6_release(fib6_event->rt_arr[i]);
   7637	kfree(fib6_event->rt_arr);
   7638}
   7639
   7640static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp,
   7641					       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7642					       struct mlxsw_sp_fib_event *fib_event)
   7643{
   7644	int err;
   7645
   7646	mlxsw_sp_span_respin(mlxsw_sp);
   7647
   7648	switch (fib_event->event) {
   7649	case FIB_EVENT_ENTRY_REPLACE:
   7650		err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info);
   7651		if (err) {
   7652			mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
   7653			dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
   7654			mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp,
   7655							      &fib_event->fen_info);
   7656		}
   7657		fib_info_put(fib_event->fen_info.fi);
   7658		break;
   7659	case FIB_EVENT_ENTRY_DEL:
   7660		err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info);
   7661		if (err)
   7662			mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
   7663		fib_info_put(fib_event->fen_info.fi);
   7664		break;
   7665	case FIB_EVENT_NH_ADD:
   7666	case FIB_EVENT_NH_DEL:
   7667		mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh);
   7668		fib_info_put(fib_event->fnh_info.fib_nh->nh_parent);
   7669		break;
   7670	}
   7671}
   7672
   7673static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp,
   7674					       struct mlxsw_sp_fib_entry_op_ctx *op_ctx,
   7675					       struct mlxsw_sp_fib_event *fib_event)
   7676{
   7677	struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event;
   7678	int err;
   7679
   7680	mlxsw_sp_span_respin(mlxsw_sp);
   7681
   7682	switch (fib_event->event) {
   7683	case FIB_EVENT_ENTRY_REPLACE:
   7684		err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
   7685						   fib_event->fib6_event.nrt6);
   7686		if (err) {
   7687			mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
   7688			dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
   7689			mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
   7690							      fib6_event->rt_arr,
   7691							      fib6_event->nrt6);
   7692		}
   7693		mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
   7694		break;
   7695	case FIB_EVENT_ENTRY_APPEND:
   7696		err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
   7697						  fib_event->fib6_event.nrt6);
   7698		if (err) {
   7699			mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
   7700			dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n");
   7701			mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
   7702							      fib6_event->rt_arr,
   7703							      fib6_event->nrt6);
   7704		}
   7705		mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
   7706		break;
   7707	case FIB_EVENT_ENTRY_DEL:
   7708		err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr,
   7709					       fib_event->fib6_event.nrt6);
   7710		if (err)
   7711			mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx);
   7712		mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event);
   7713		break;
   7714	}
   7715}
   7716
   7717static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp,
   7718						struct mlxsw_sp_fib_event *fib_event)
   7719{
   7720	bool replace;
   7721	int err;
   7722
   7723	rtnl_lock();
   7724	mutex_lock(&mlxsw_sp->router->lock);
   7725	switch (fib_event->event) {
   7726	case FIB_EVENT_ENTRY_REPLACE:
   7727	case FIB_EVENT_ENTRY_ADD:
   7728		replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE;
   7729
   7730		err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace);
   7731		if (err)
   7732			dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n");
   7733		mr_cache_put(fib_event->men_info.mfc);
   7734		break;
   7735	case FIB_EVENT_ENTRY_DEL:
   7736		mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info);
   7737		mr_cache_put(fib_event->men_info.mfc);
   7738		break;
   7739	case FIB_EVENT_VIF_ADD:
   7740		err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
   7741						    &fib_event->ven_info);
   7742		if (err)
   7743			dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n");
   7744		dev_put(fib_event->ven_info.dev);
   7745		break;
   7746	case FIB_EVENT_VIF_DEL:
   7747		mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info);
   7748		dev_put(fib_event->ven_info.dev);
   7749		break;
   7750	}
   7751	mutex_unlock(&mlxsw_sp->router->lock);
   7752	rtnl_unlock();
   7753}
   7754
   7755static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
   7756{
   7757	struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work);
   7758	struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx;
   7759	struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp;
   7760	struct mlxsw_sp_fib_event *next_fib_event;
   7761	struct mlxsw_sp_fib_event *fib_event;
   7762	int last_family = AF_UNSPEC;
   7763	LIST_HEAD(fib_event_queue);
   7764
   7765	spin_lock_bh(&router->fib_event_queue_lock);
   7766	list_splice_init(&router->fib_event_queue, &fib_event_queue);
   7767	spin_unlock_bh(&router->fib_event_queue_lock);
   7768
   7769	/* Router lock is held here to make sure per-instance
   7770	 * operation context is not used in between FIB4/6 events
   7771	 * processing.
   7772	 */
   7773	mutex_lock(&router->lock);
   7774	mlxsw_sp_fib_entry_op_ctx_clear(op_ctx);
   7775	list_for_each_entry_safe(fib_event, next_fib_event,
   7776				 &fib_event_queue, list) {
   7777		/* Check if the next entry in the queue exists and it is
   7778		 * of the same type (family and event) as the currect one.
   7779		 * In that case it is permitted to do the bulking
   7780		 * of multiple FIB entries to a single register write.
   7781		 */
   7782		op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) &&
   7783				  fib_event->family == next_fib_event->family &&
   7784				  fib_event->event == next_fib_event->event;
   7785		op_ctx->event = fib_event->event;
   7786
   7787		/* In case family of this and the previous entry are different, context
   7788		 * reinitialization is going to be needed now, indicate that.
   7789		 * Note that since last_family is initialized to AF_UNSPEC, this is always
   7790		 * going to happen for the first entry processed in the work.
   7791		 */
   7792		if (fib_event->family != last_family)
   7793			op_ctx->initialized = false;
   7794
   7795		switch (fib_event->family) {
   7796		case AF_INET:
   7797			mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx,
   7798							   fib_event);
   7799			break;
   7800		case AF_INET6:
   7801			mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx,
   7802							   fib_event);
   7803			break;
   7804		case RTNL_FAMILY_IP6MR:
   7805		case RTNL_FAMILY_IPMR:
   7806			/* Unlock here as inside FIBMR the lock is taken again
   7807			 * under RTNL. The per-instance operation context
   7808			 * is not used by FIBMR.
   7809			 */
   7810			mutex_unlock(&router->lock);
   7811			mlxsw_sp_router_fibmr_event_process(mlxsw_sp,
   7812							    fib_event);
   7813			mutex_lock(&router->lock);
   7814			break;
   7815		default:
   7816			WARN_ON_ONCE(1);
   7817		}
   7818		last_family = fib_event->family;
   7819		kfree(fib_event);
   7820		cond_resched();
   7821	}
   7822	WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
   7823	mutex_unlock(&router->lock);
   7824}
   7825
   7826static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event,
   7827				       struct fib_notifier_info *info)
   7828{
   7829	struct fib_entry_notifier_info *fen_info;
   7830	struct fib_nh_notifier_info *fnh_info;
   7831
   7832	switch (fib_event->event) {
   7833	case FIB_EVENT_ENTRY_REPLACE:
   7834	case FIB_EVENT_ENTRY_DEL:
   7835		fen_info = container_of(info, struct fib_entry_notifier_info,
   7836					info);
   7837		fib_event->fen_info = *fen_info;
   7838		/* Take reference on fib_info to prevent it from being
   7839		 * freed while event is queued. Release it afterwards.
   7840		 */
   7841		fib_info_hold(fib_event->fen_info.fi);
   7842		break;
   7843	case FIB_EVENT_NH_ADD:
   7844	case FIB_EVENT_NH_DEL:
   7845		fnh_info = container_of(info, struct fib_nh_notifier_info,
   7846					info);
   7847		fib_event->fnh_info = *fnh_info;
   7848		fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent);
   7849		break;
   7850	}
   7851}
   7852
   7853static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event,
   7854				      struct fib_notifier_info *info)
   7855{
   7856	struct fib6_entry_notifier_info *fen6_info;
   7857	int err;
   7858
   7859	switch (fib_event->event) {
   7860	case FIB_EVENT_ENTRY_REPLACE:
   7861	case FIB_EVENT_ENTRY_APPEND:
   7862	case FIB_EVENT_ENTRY_DEL:
   7863		fen6_info = container_of(info, struct fib6_entry_notifier_info,
   7864					 info);
   7865		err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event,
   7866						      fen6_info);
   7867		if (err)
   7868			return err;
   7869		break;
   7870	}
   7871
   7872	return 0;
   7873}
   7874
   7875static void
   7876mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event,
   7877			    struct fib_notifier_info *info)
   7878{
   7879	switch (fib_event->event) {
   7880	case FIB_EVENT_ENTRY_REPLACE:
   7881	case FIB_EVENT_ENTRY_ADD:
   7882	case FIB_EVENT_ENTRY_DEL:
   7883		memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info));
   7884		mr_cache_hold(fib_event->men_info.mfc);
   7885		break;
   7886	case FIB_EVENT_VIF_ADD:
   7887	case FIB_EVENT_VIF_DEL:
   7888		memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info));
   7889		dev_hold(fib_event->ven_info.dev);
   7890		break;
   7891	}
   7892}
   7893
   7894static int mlxsw_sp_router_fib_rule_event(unsigned long event,
   7895					  struct fib_notifier_info *info,
   7896					  struct mlxsw_sp *mlxsw_sp)
   7897{
   7898	struct netlink_ext_ack *extack = info->extack;
   7899	struct fib_rule_notifier_info *fr_info;
   7900	struct fib_rule *rule;
   7901	int err = 0;
   7902
   7903	/* nothing to do at the moment */
   7904	if (event == FIB_EVENT_RULE_DEL)
   7905		return 0;
   7906
   7907	fr_info = container_of(info, struct fib_rule_notifier_info, info);
   7908	rule = fr_info->rule;
   7909
   7910	/* Rule only affects locally generated traffic */
   7911	if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
   7912		return 0;
   7913
   7914	switch (info->family) {
   7915	case AF_INET:
   7916		if (!fib4_rule_default(rule) && !rule->l3mdev)
   7917			err = -EOPNOTSUPP;
   7918		break;
   7919	case AF_INET6:
   7920		if (!fib6_rule_default(rule) && !rule->l3mdev)
   7921			err = -EOPNOTSUPP;
   7922		break;
   7923	case RTNL_FAMILY_IPMR:
   7924		if (!ipmr_rule_default(rule) && !rule->l3mdev)
   7925			err = -EOPNOTSUPP;
   7926		break;
   7927	case RTNL_FAMILY_IP6MR:
   7928		if (!ip6mr_rule_default(rule) && !rule->l3mdev)
   7929			err = -EOPNOTSUPP;
   7930		break;
   7931	}
   7932
   7933	if (err < 0)
   7934		NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
   7935
   7936	return err;
   7937}
   7938
   7939/* Called with rcu_read_lock() */
   7940static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
   7941				     unsigned long event, void *ptr)
   7942{
   7943	struct mlxsw_sp_fib_event *fib_event;
   7944	struct fib_notifier_info *info = ptr;
   7945	struct mlxsw_sp_router *router;
   7946	int err;
   7947
   7948	if ((info->family != AF_INET && info->family != AF_INET6 &&
   7949	     info->family != RTNL_FAMILY_IPMR &&
   7950	     info->family != RTNL_FAMILY_IP6MR))
   7951		return NOTIFY_DONE;
   7952
   7953	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
   7954
   7955	switch (event) {
   7956	case FIB_EVENT_RULE_ADD:
   7957	case FIB_EVENT_RULE_DEL:
   7958		err = mlxsw_sp_router_fib_rule_event(event, info,
   7959						     router->mlxsw_sp);
   7960		return notifier_from_errno(err);
   7961	case FIB_EVENT_ENTRY_ADD:
   7962	case FIB_EVENT_ENTRY_REPLACE:
   7963	case FIB_EVENT_ENTRY_APPEND:
   7964		if (info->family == AF_INET) {
   7965			struct fib_entry_notifier_info *fen_info = ptr;
   7966
   7967			if (fen_info->fi->fib_nh_is_v6) {
   7968				NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
   7969				return notifier_from_errno(-EINVAL);
   7970			}
   7971		}
   7972		break;
   7973	}
   7974
   7975	fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC);
   7976	if (!fib_event)
   7977		return NOTIFY_BAD;
   7978
   7979	fib_event->mlxsw_sp = router->mlxsw_sp;
   7980	fib_event->event = event;
   7981	fib_event->family = info->family;
   7982
   7983	switch (info->family) {
   7984	case AF_INET:
   7985		mlxsw_sp_router_fib4_event(fib_event, info);
   7986		break;
   7987	case AF_INET6:
   7988		err = mlxsw_sp_router_fib6_event(fib_event, info);
   7989		if (err)
   7990			goto err_fib_event;
   7991		break;
   7992	case RTNL_FAMILY_IP6MR:
   7993	case RTNL_FAMILY_IPMR:
   7994		mlxsw_sp_router_fibmr_event(fib_event, info);
   7995		break;
   7996	}
   7997
   7998	/* Enqueue the event and trigger the work */
   7999	spin_lock_bh(&router->fib_event_queue_lock);
   8000	list_add_tail(&fib_event->list, &router->fib_event_queue);
   8001	spin_unlock_bh(&router->fib_event_queue_lock);
   8002	mlxsw_core_schedule_work(&router->fib_event_work);
   8003
   8004	return NOTIFY_DONE;
   8005
   8006err_fib_event:
   8007	kfree(fib_event);
   8008	return NOTIFY_BAD;
   8009}
   8010
   8011static struct mlxsw_sp_rif *
   8012mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
   8013			 const struct net_device *dev)
   8014{
   8015	int i;
   8016
   8017	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
   8018		if (mlxsw_sp->router->rifs[i] &&
   8019		    mlxsw_sp->router->rifs[i]->dev == dev)
   8020			return mlxsw_sp->router->rifs[i];
   8021
   8022	return NULL;
   8023}
   8024
   8025bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
   8026			 const struct net_device *dev)
   8027{
   8028	struct mlxsw_sp_rif *rif;
   8029
   8030	mutex_lock(&mlxsw_sp->router->lock);
   8031	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   8032	mutex_unlock(&mlxsw_sp->router->lock);
   8033
   8034	return rif;
   8035}
   8036
   8037u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
   8038{
   8039	struct mlxsw_sp_rif *rif;
   8040	u16 vid = 0;
   8041
   8042	mutex_lock(&mlxsw_sp->router->lock);
   8043	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   8044	if (!rif)
   8045		goto out;
   8046
   8047	/* We only return the VID for VLAN RIFs. Otherwise we return an
   8048	 * invalid value (0).
   8049	 */
   8050	if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
   8051		goto out;
   8052
   8053	vid = mlxsw_sp_fid_8021q_vid(rif->fid);
   8054
   8055out:
   8056	mutex_unlock(&mlxsw_sp->router->lock);
   8057	return vid;
   8058}
   8059
   8060static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
   8061{
   8062	char ritr_pl[MLXSW_REG_RITR_LEN];
   8063	int err;
   8064
   8065	mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
   8066	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   8067	if (err)
   8068		return err;
   8069
   8070	mlxsw_reg_ritr_enable_set(ritr_pl, false);
   8071	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   8072}
   8073
   8074static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
   8075					  struct mlxsw_sp_rif *rif)
   8076{
   8077	mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
   8078	mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
   8079	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
   8080}
   8081
   8082static bool
   8083mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
   8084			   unsigned long event)
   8085{
   8086	struct inet6_dev *inet6_dev;
   8087	bool addr_list_empty = true;
   8088	struct in_device *idev;
   8089
   8090	switch (event) {
   8091	case NETDEV_UP:
   8092		return rif == NULL;
   8093	case NETDEV_DOWN:
   8094		rcu_read_lock();
   8095		idev = __in_dev_get_rcu(dev);
   8096		if (idev && idev->ifa_list)
   8097			addr_list_empty = false;
   8098
   8099		inet6_dev = __in6_dev_get(dev);
   8100		if (addr_list_empty && inet6_dev &&
   8101		    !list_empty(&inet6_dev->addr_list))
   8102			addr_list_empty = false;
   8103		rcu_read_unlock();
   8104
   8105		/* macvlans do not have a RIF, but rather piggy back on the
   8106		 * RIF of their lower device.
   8107		 */
   8108		if (netif_is_macvlan(dev) && addr_list_empty)
   8109			return true;
   8110
   8111		if (rif && addr_list_empty &&
   8112		    !netif_is_l3_slave(rif->dev))
   8113			return true;
   8114		/* It is possible we already removed the RIF ourselves
   8115		 * if it was assigned to a netdev that is now a bridge
   8116		 * or LAG slave.
   8117		 */
   8118		return false;
   8119	}
   8120
   8121	return false;
   8122}
   8123
   8124static enum mlxsw_sp_rif_type
   8125mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
   8126		      const struct net_device *dev)
   8127{
   8128	enum mlxsw_sp_fid_type type;
   8129
   8130	if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
   8131		return MLXSW_SP_RIF_TYPE_IPIP_LB;
   8132
   8133	/* Otherwise RIF type is derived from the type of the underlying FID. */
   8134	if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
   8135		type = MLXSW_SP_FID_TYPE_8021Q;
   8136	else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
   8137		type = MLXSW_SP_FID_TYPE_8021Q;
   8138	else if (netif_is_bridge_master(dev))
   8139		type = MLXSW_SP_FID_TYPE_8021D;
   8140	else
   8141		type = MLXSW_SP_FID_TYPE_RFID;
   8142
   8143	return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
   8144}
   8145
   8146static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
   8147{
   8148	int i;
   8149
   8150	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
   8151		if (!mlxsw_sp->router->rifs[i]) {
   8152			*p_rif_index = i;
   8153			return 0;
   8154		}
   8155	}
   8156
   8157	return -ENOBUFS;
   8158}
   8159
   8160static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
   8161					       u16 vr_id,
   8162					       struct net_device *l3_dev)
   8163{
   8164	struct mlxsw_sp_rif *rif;
   8165
   8166	rif = kzalloc(rif_size, GFP_KERNEL);
   8167	if (!rif)
   8168		return NULL;
   8169
   8170	INIT_LIST_HEAD(&rif->nexthop_list);
   8171	INIT_LIST_HEAD(&rif->neigh_list);
   8172	if (l3_dev) {
   8173		ether_addr_copy(rif->addr, l3_dev->dev_addr);
   8174		rif->mtu = l3_dev->mtu;
   8175		rif->dev = l3_dev;
   8176	}
   8177	rif->vr_id = vr_id;
   8178	rif->rif_index = rif_index;
   8179
   8180	return rif;
   8181}
   8182
   8183struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
   8184					   u16 rif_index)
   8185{
   8186	return mlxsw_sp->router->rifs[rif_index];
   8187}
   8188
   8189u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
   8190{
   8191	return rif->rif_index;
   8192}
   8193
   8194u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
   8195{
   8196	return lb_rif->common.rif_index;
   8197}
   8198
   8199u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
   8200{
   8201	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
   8202	struct mlxsw_sp_vr *ul_vr;
   8203
   8204	ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
   8205	if (WARN_ON(IS_ERR(ul_vr)))
   8206		return 0;
   8207
   8208	return ul_vr->id;
   8209}
   8210
   8211u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
   8212{
   8213	return lb_rif->ul_rif_id;
   8214}
   8215
   8216static bool
   8217mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif)
   8218{
   8219	return mlxsw_sp_rif_counter_valid_get(rif,
   8220					      MLXSW_SP_RIF_COUNTER_EGRESS) &&
   8221	       mlxsw_sp_rif_counter_valid_get(rif,
   8222					      MLXSW_SP_RIF_COUNTER_INGRESS);
   8223}
   8224
   8225static int
   8226mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif)
   8227{
   8228	int err;
   8229
   8230	err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
   8231	if (err)
   8232		return err;
   8233
   8234	/* Clear stale data. */
   8235	err = mlxsw_sp_rif_counter_fetch_clear(rif,
   8236					       MLXSW_SP_RIF_COUNTER_INGRESS,
   8237					       NULL);
   8238	if (err)
   8239		goto err_clear_ingress;
   8240
   8241	err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
   8242	if (err)
   8243		goto err_alloc_egress;
   8244
   8245	/* Clear stale data. */
   8246	err = mlxsw_sp_rif_counter_fetch_clear(rif,
   8247					       MLXSW_SP_RIF_COUNTER_EGRESS,
   8248					       NULL);
   8249	if (err)
   8250		goto err_clear_egress;
   8251
   8252	return 0;
   8253
   8254err_clear_egress:
   8255	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
   8256err_alloc_egress:
   8257err_clear_ingress:
   8258	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
   8259	return err;
   8260}
   8261
   8262static void
   8263mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif)
   8264{
   8265	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
   8266	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
   8267}
   8268
   8269static void
   8270mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif,
   8271					  struct netdev_notifier_offload_xstats_info *info)
   8272{
   8273	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
   8274		return;
   8275	netdev_offload_xstats_report_used(info->report_used);
   8276}
   8277
   8278static int
   8279mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif,
   8280				    struct rtnl_hw_stats64 *p_stats)
   8281{
   8282	struct mlxsw_sp_rif_counter_set_basic ingress;
   8283	struct mlxsw_sp_rif_counter_set_basic egress;
   8284	int err;
   8285
   8286	err = mlxsw_sp_rif_counter_fetch_clear(rif,
   8287					       MLXSW_SP_RIF_COUNTER_INGRESS,
   8288					       &ingress);
   8289	if (err)
   8290		return err;
   8291
   8292	err = mlxsw_sp_rif_counter_fetch_clear(rif,
   8293					       MLXSW_SP_RIF_COUNTER_EGRESS,
   8294					       &egress);
   8295	if (err)
   8296		return err;
   8297
   8298#define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX)		\
   8299		((SET.good_unicast_ ## SFX) +		\
   8300		 (SET.good_multicast_ ## SFX) +		\
   8301		 (SET.good_broadcast_ ## SFX))
   8302
   8303	p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets);
   8304	p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets);
   8305	p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes);
   8306	p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes);
   8307	p_stats->rx_errors = ingress.error_packets;
   8308	p_stats->tx_errors = egress.error_packets;
   8309	p_stats->rx_dropped = ingress.discard_packets;
   8310	p_stats->tx_dropped = egress.discard_packets;
   8311	p_stats->multicast = ingress.good_multicast_packets +
   8312			     ingress.good_broadcast_packets;
   8313
   8314#undef MLXSW_SP_ROUTER_ALL_GOOD
   8315
   8316	return 0;
   8317}
   8318
   8319static int
   8320mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif,
   8321					   struct netdev_notifier_offload_xstats_info *info)
   8322{
   8323	struct rtnl_hw_stats64 stats = {};
   8324	int err;
   8325
   8326	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
   8327		return 0;
   8328
   8329	err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats);
   8330	if (err)
   8331		return err;
   8332
   8333	netdev_offload_xstats_report_delta(info->report_delta, &stats);
   8334	return 0;
   8335}
   8336
   8337struct mlxsw_sp_router_hwstats_notify_work {
   8338	struct work_struct work;
   8339	struct net_device *dev;
   8340};
   8341
   8342static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work)
   8343{
   8344	struct mlxsw_sp_router_hwstats_notify_work *hws_work =
   8345		container_of(work, struct mlxsw_sp_router_hwstats_notify_work,
   8346			     work);
   8347
   8348	rtnl_lock();
   8349	rtnl_offload_xstats_notify(hws_work->dev);
   8350	rtnl_unlock();
   8351	dev_put(hws_work->dev);
   8352	kfree(hws_work);
   8353}
   8354
   8355static void
   8356mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev)
   8357{
   8358	struct mlxsw_sp_router_hwstats_notify_work *hws_work;
   8359
   8360	/* To collect notification payload, the core ends up sending another
   8361	 * notifier block message, which would deadlock on the attempt to
   8362	 * acquire the router lock again. Just postpone the notification until
   8363	 * later.
   8364	 */
   8365
   8366	hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL);
   8367	if (!hws_work)
   8368		return;
   8369
   8370	INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work);
   8371	dev_hold(dev);
   8372	hws_work->dev = dev;
   8373	mlxsw_core_schedule_work(&hws_work->work);
   8374}
   8375
   8376int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
   8377{
   8378	return rif->dev->ifindex;
   8379}
   8380
   8381const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
   8382{
   8383	return rif->dev;
   8384}
   8385
   8386static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif)
   8387{
   8388	struct rtnl_hw_stats64 stats = {};
   8389
   8390	if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats))
   8391		netdev_offload_xstats_push_delta(rif->dev,
   8392						 NETDEV_OFFLOAD_XSTATS_TYPE_L3,
   8393						 &stats);
   8394}
   8395
   8396static struct mlxsw_sp_rif *
   8397mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
   8398		    const struct mlxsw_sp_rif_params *params,
   8399		    struct netlink_ext_ack *extack)
   8400{
   8401	u32 tb_id = l3mdev_fib_table(params->dev);
   8402	const struct mlxsw_sp_rif_ops *ops;
   8403	struct mlxsw_sp_fid *fid = NULL;
   8404	enum mlxsw_sp_rif_type type;
   8405	struct mlxsw_sp_rif *rif;
   8406	struct mlxsw_sp_vr *vr;
   8407	u16 rif_index;
   8408	int i, err;
   8409
   8410	type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
   8411	ops = mlxsw_sp->router->rif_ops_arr[type];
   8412
   8413	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
   8414	if (IS_ERR(vr))
   8415		return ERR_CAST(vr);
   8416	vr->rif_count++;
   8417
   8418	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
   8419	if (err) {
   8420		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
   8421		goto err_rif_index_alloc;
   8422	}
   8423
   8424	rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
   8425	if (!rif) {
   8426		err = -ENOMEM;
   8427		goto err_rif_alloc;
   8428	}
   8429	dev_hold(rif->dev);
   8430	mlxsw_sp->router->rifs[rif_index] = rif;
   8431	rif->mlxsw_sp = mlxsw_sp;
   8432	rif->ops = ops;
   8433
   8434	if (ops->fid_get) {
   8435		fid = ops->fid_get(rif, extack);
   8436		if (IS_ERR(fid)) {
   8437			err = PTR_ERR(fid);
   8438			goto err_fid_get;
   8439		}
   8440		rif->fid = fid;
   8441	}
   8442
   8443	if (ops->setup)
   8444		ops->setup(rif, params);
   8445
   8446	err = ops->configure(rif, extack);
   8447	if (err)
   8448		goto err_configure;
   8449
   8450	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
   8451		err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
   8452		if (err)
   8453			goto err_mr_rif_add;
   8454	}
   8455
   8456	if (netdev_offload_xstats_enabled(rif->dev,
   8457					  NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
   8458		err = mlxsw_sp_router_port_l3_stats_enable(rif);
   8459		if (err)
   8460			goto err_stats_enable;
   8461		mlxsw_sp_router_hwstats_notify_schedule(rif->dev);
   8462	} else {
   8463		mlxsw_sp_rif_counters_alloc(rif);
   8464	}
   8465
   8466	return rif;
   8467
   8468err_stats_enable:
   8469err_mr_rif_add:
   8470	for (i--; i >= 0; i--)
   8471		mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
   8472	ops->deconfigure(rif);
   8473err_configure:
   8474	if (fid)
   8475		mlxsw_sp_fid_put(fid);
   8476err_fid_get:
   8477	mlxsw_sp->router->rifs[rif_index] = NULL;
   8478	dev_put(rif->dev);
   8479	kfree(rif);
   8480err_rif_alloc:
   8481err_rif_index_alloc:
   8482	vr->rif_count--;
   8483	mlxsw_sp_vr_put(mlxsw_sp, vr);
   8484	return ERR_PTR(err);
   8485}
   8486
   8487static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
   8488{
   8489	const struct mlxsw_sp_rif_ops *ops = rif->ops;
   8490	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   8491	struct mlxsw_sp_fid *fid = rif->fid;
   8492	struct mlxsw_sp_vr *vr;
   8493	int i;
   8494
   8495	mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
   8496	vr = &mlxsw_sp->router->vrs[rif->vr_id];
   8497
   8498	if (netdev_offload_xstats_enabled(rif->dev,
   8499					  NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
   8500		mlxsw_sp_rif_push_l3_stats(rif);
   8501		mlxsw_sp_router_port_l3_stats_disable(rif);
   8502		mlxsw_sp_router_hwstats_notify_schedule(rif->dev);
   8503	} else {
   8504		mlxsw_sp_rif_counters_free(rif);
   8505	}
   8506
   8507	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
   8508		mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
   8509	ops->deconfigure(rif);
   8510	if (fid)
   8511		/* Loopback RIFs are not associated with a FID. */
   8512		mlxsw_sp_fid_put(fid);
   8513	mlxsw_sp->router->rifs[rif->rif_index] = NULL;
   8514	dev_put(rif->dev);
   8515	kfree(rif);
   8516	vr->rif_count--;
   8517	mlxsw_sp_vr_put(mlxsw_sp, vr);
   8518}
   8519
   8520void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
   8521				 struct net_device *dev)
   8522{
   8523	struct mlxsw_sp_rif *rif;
   8524
   8525	mutex_lock(&mlxsw_sp->router->lock);
   8526	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   8527	if (!rif)
   8528		goto out;
   8529	mlxsw_sp_rif_destroy(rif);
   8530out:
   8531	mutex_unlock(&mlxsw_sp->router->lock);
   8532}
   8533
   8534static void
   8535mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
   8536				 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
   8537{
   8538	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
   8539
   8540	params->vid = mlxsw_sp_port_vlan->vid;
   8541	params->lag = mlxsw_sp_port->lagged;
   8542	if (params->lag)
   8543		params->lag_id = mlxsw_sp_port->lag_id;
   8544	else
   8545		params->system_port = mlxsw_sp_port->local_port;
   8546}
   8547
   8548static struct mlxsw_sp_rif_subport *
   8549mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
   8550{
   8551	return container_of(rif, struct mlxsw_sp_rif_subport, common);
   8552}
   8553
   8554static struct mlxsw_sp_rif *
   8555mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
   8556			 const struct mlxsw_sp_rif_params *params,
   8557			 struct netlink_ext_ack *extack)
   8558{
   8559	struct mlxsw_sp_rif_subport *rif_subport;
   8560	struct mlxsw_sp_rif *rif;
   8561
   8562	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
   8563	if (!rif)
   8564		return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
   8565
   8566	rif_subport = mlxsw_sp_rif_subport_rif(rif);
   8567	refcount_inc(&rif_subport->ref_count);
   8568	return rif;
   8569}
   8570
   8571static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
   8572{
   8573	struct mlxsw_sp_rif_subport *rif_subport;
   8574
   8575	rif_subport = mlxsw_sp_rif_subport_rif(rif);
   8576	if (!refcount_dec_and_test(&rif_subport->ref_count))
   8577		return;
   8578
   8579	mlxsw_sp_rif_destroy(rif);
   8580}
   8581
   8582static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp,
   8583						struct mlxsw_sp_rif_mac_profile *profile,
   8584						struct netlink_ext_ack *extack)
   8585{
   8586	u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
   8587	struct mlxsw_sp_router *router = mlxsw_sp->router;
   8588	int id;
   8589
   8590	id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0,
   8591		       max_rif_mac_profiles, GFP_KERNEL);
   8592
   8593	if (id >= 0) {
   8594		profile->id = id;
   8595		return 0;
   8596	}
   8597
   8598	if (id == -ENOSPC)
   8599		NL_SET_ERR_MSG_MOD(extack,
   8600				   "Exceeded number of supported router interface MAC profiles");
   8601
   8602	return id;
   8603}
   8604
   8605static struct mlxsw_sp_rif_mac_profile *
   8606mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile)
   8607{
   8608	struct mlxsw_sp_rif_mac_profile *profile;
   8609
   8610	profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr,
   8611			     mac_profile);
   8612	WARN_ON(!profile);
   8613	return profile;
   8614}
   8615
   8616static struct mlxsw_sp_rif_mac_profile *
   8617mlxsw_sp_rif_mac_profile_alloc(const char *mac)
   8618{
   8619	struct mlxsw_sp_rif_mac_profile *profile;
   8620
   8621	profile = kzalloc(sizeof(*profile), GFP_KERNEL);
   8622	if (!profile)
   8623		return NULL;
   8624
   8625	ether_addr_copy(profile->mac_prefix, mac);
   8626	refcount_set(&profile->ref_count, 1);
   8627	return profile;
   8628}
   8629
   8630static struct mlxsw_sp_rif_mac_profile *
   8631mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac)
   8632{
   8633	struct mlxsw_sp_router *router = mlxsw_sp->router;
   8634	struct mlxsw_sp_rif_mac_profile *profile;
   8635	int id;
   8636
   8637	idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) {
   8638		if (ether_addr_equal_masked(profile->mac_prefix, mac,
   8639					    mlxsw_sp->mac_mask))
   8640			return profile;
   8641	}
   8642
   8643	return NULL;
   8644}
   8645
   8646static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv)
   8647{
   8648	const struct mlxsw_sp *mlxsw_sp = priv;
   8649
   8650	return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count);
   8651}
   8652
   8653static struct mlxsw_sp_rif_mac_profile *
   8654mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac,
   8655				struct netlink_ext_ack *extack)
   8656{
   8657	struct mlxsw_sp_rif_mac_profile *profile;
   8658	int err;
   8659
   8660	profile = mlxsw_sp_rif_mac_profile_alloc(mac);
   8661	if (!profile)
   8662		return ERR_PTR(-ENOMEM);
   8663
   8664	err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack);
   8665	if (err)
   8666		goto profile_index_alloc_err;
   8667
   8668	atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count);
   8669	return profile;
   8670
   8671profile_index_alloc_err:
   8672	kfree(profile);
   8673	return ERR_PTR(err);
   8674}
   8675
   8676static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp,
   8677					     u8 mac_profile)
   8678{
   8679	struct mlxsw_sp_rif_mac_profile *profile;
   8680
   8681	atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count);
   8682	profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile);
   8683	kfree(profile);
   8684}
   8685
   8686static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp,
   8687					const char *mac, u8 *p_mac_profile,
   8688					struct netlink_ext_ack *extack)
   8689{
   8690	struct mlxsw_sp_rif_mac_profile *profile;
   8691
   8692	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac);
   8693	if (profile) {
   8694		refcount_inc(&profile->ref_count);
   8695		goto out;
   8696	}
   8697
   8698	profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack);
   8699	if (IS_ERR(profile))
   8700		return PTR_ERR(profile);
   8701
   8702out:
   8703	*p_mac_profile = profile->id;
   8704	return 0;
   8705}
   8706
   8707static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp,
   8708					 u8 mac_profile)
   8709{
   8710	struct mlxsw_sp_rif_mac_profile *profile;
   8711
   8712	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
   8713			   mac_profile);
   8714	if (WARN_ON(!profile))
   8715		return;
   8716
   8717	if (!refcount_dec_and_test(&profile->ref_count))
   8718		return;
   8719
   8720	mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile);
   8721}
   8722
   8723static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif)
   8724{
   8725	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   8726	struct mlxsw_sp_rif_mac_profile *profile;
   8727
   8728	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
   8729			   rif->mac_profile_id);
   8730	if (WARN_ON(!profile))
   8731		return false;
   8732
   8733	return refcount_read(&profile->ref_count) > 1;
   8734}
   8735
   8736static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif,
   8737					 const char *new_mac)
   8738{
   8739	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   8740	struct mlxsw_sp_rif_mac_profile *profile;
   8741
   8742	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
   8743			   rif->mac_profile_id);
   8744	if (WARN_ON(!profile))
   8745		return -EINVAL;
   8746
   8747	ether_addr_copy(profile->mac_prefix, new_mac);
   8748	return 0;
   8749}
   8750
   8751static int
   8752mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp,
   8753				 struct mlxsw_sp_rif *rif,
   8754				 const char *new_mac,
   8755				 struct netlink_ext_ack *extack)
   8756{
   8757	u8 mac_profile;
   8758	int err;
   8759
   8760	if (!mlxsw_sp_rif_mac_profile_is_shared(rif) &&
   8761	    !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac))
   8762		return mlxsw_sp_rif_mac_profile_edit(rif, new_mac);
   8763
   8764	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac,
   8765					   &mac_profile, extack);
   8766	if (err)
   8767		return err;
   8768
   8769	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id);
   8770	rif->mac_profile_id = mac_profile;
   8771	return 0;
   8772}
   8773
   8774static int
   8775__mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
   8776				 struct net_device *l3_dev,
   8777				 struct netlink_ext_ack *extack)
   8778{
   8779	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
   8780	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
   8781	struct mlxsw_sp_rif_params params = {
   8782		.dev = l3_dev,
   8783	};
   8784	u16 vid = mlxsw_sp_port_vlan->vid;
   8785	struct mlxsw_sp_rif *rif;
   8786	struct mlxsw_sp_fid *fid;
   8787	int err;
   8788
   8789	mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
   8790	rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
   8791	if (IS_ERR(rif))
   8792		return PTR_ERR(rif);
   8793
   8794	/* FID was already created, just take a reference */
   8795	fid = rif->ops->fid_get(rif, extack);
   8796	err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
   8797	if (err)
   8798		goto err_fid_port_vid_map;
   8799
   8800	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
   8801	if (err)
   8802		goto err_port_vid_learning_set;
   8803
   8804	err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
   8805					BR_STATE_FORWARDING);
   8806	if (err)
   8807		goto err_port_vid_stp_set;
   8808
   8809	mlxsw_sp_port_vlan->fid = fid;
   8810
   8811	return 0;
   8812
   8813err_port_vid_stp_set:
   8814	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
   8815err_port_vid_learning_set:
   8816	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
   8817err_fid_port_vid_map:
   8818	mlxsw_sp_fid_put(fid);
   8819	mlxsw_sp_rif_subport_put(rif);
   8820	return err;
   8821}
   8822
   8823static void
   8824__mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
   8825{
   8826	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
   8827	struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
   8828	struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
   8829	u16 vid = mlxsw_sp_port_vlan->vid;
   8830
   8831	if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
   8832		return;
   8833
   8834	mlxsw_sp_port_vlan->fid = NULL;
   8835	mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
   8836	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
   8837	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
   8838	mlxsw_sp_fid_put(fid);
   8839	mlxsw_sp_rif_subport_put(rif);
   8840}
   8841
   8842int
   8843mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
   8844			       struct net_device *l3_dev,
   8845			       struct netlink_ext_ack *extack)
   8846{
   8847	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
   8848	struct mlxsw_sp_rif *rif;
   8849	int err = 0;
   8850
   8851	mutex_lock(&mlxsw_sp->router->lock);
   8852	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
   8853	if (!rif)
   8854		goto out;
   8855
   8856	err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev,
   8857					       extack);
   8858out:
   8859	mutex_unlock(&mlxsw_sp->router->lock);
   8860	return err;
   8861}
   8862
   8863void
   8864mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
   8865{
   8866	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
   8867
   8868	mutex_lock(&mlxsw_sp->router->lock);
   8869	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
   8870	mutex_unlock(&mlxsw_sp->router->lock);
   8871}
   8872
   8873static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
   8874					     struct net_device *port_dev,
   8875					     unsigned long event, u16 vid,
   8876					     struct netlink_ext_ack *extack)
   8877{
   8878	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
   8879	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
   8880
   8881	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
   8882	if (WARN_ON(!mlxsw_sp_port_vlan))
   8883		return -EINVAL;
   8884
   8885	switch (event) {
   8886	case NETDEV_UP:
   8887		return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
   8888							l3_dev, extack);
   8889	case NETDEV_DOWN:
   8890		__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
   8891		break;
   8892	}
   8893
   8894	return 0;
   8895}
   8896
   8897static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
   8898					unsigned long event,
   8899					struct netlink_ext_ack *extack)
   8900{
   8901	if (netif_is_bridge_port(port_dev) ||
   8902	    netif_is_lag_port(port_dev) ||
   8903	    netif_is_ovs_port(port_dev))
   8904		return 0;
   8905
   8906	return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
   8907						 MLXSW_SP_DEFAULT_VID, extack);
   8908}
   8909
   8910static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
   8911					 struct net_device *lag_dev,
   8912					 unsigned long event, u16 vid,
   8913					 struct netlink_ext_ack *extack)
   8914{
   8915	struct net_device *port_dev;
   8916	struct list_head *iter;
   8917	int err;
   8918
   8919	netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
   8920		if (mlxsw_sp_port_dev_check(port_dev)) {
   8921			err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
   8922								port_dev,
   8923								event, vid,
   8924								extack);
   8925			if (err)
   8926				return err;
   8927		}
   8928	}
   8929
   8930	return 0;
   8931}
   8932
   8933static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
   8934				       unsigned long event,
   8935				       struct netlink_ext_ack *extack)
   8936{
   8937	if (netif_is_bridge_port(lag_dev))
   8938		return 0;
   8939
   8940	return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
   8941					     MLXSW_SP_DEFAULT_VID, extack);
   8942}
   8943
   8944static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
   8945					  struct net_device *l3_dev,
   8946					  unsigned long event,
   8947					  struct netlink_ext_ack *extack)
   8948{
   8949	struct mlxsw_sp_rif_params params = {
   8950		.dev = l3_dev,
   8951	};
   8952	struct mlxsw_sp_rif *rif;
   8953
   8954	switch (event) {
   8955	case NETDEV_UP:
   8956		if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) {
   8957			u16 proto;
   8958
   8959			br_vlan_get_proto(l3_dev, &proto);
   8960			if (proto == ETH_P_8021AD) {
   8961				NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported");
   8962				return -EOPNOTSUPP;
   8963			}
   8964		}
   8965		rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
   8966		if (IS_ERR(rif))
   8967			return PTR_ERR(rif);
   8968		break;
   8969	case NETDEV_DOWN:
   8970		rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
   8971		mlxsw_sp_rif_destroy(rif);
   8972		break;
   8973	}
   8974
   8975	return 0;
   8976}
   8977
   8978static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
   8979					struct net_device *vlan_dev,
   8980					unsigned long event,
   8981					struct netlink_ext_ack *extack)
   8982{
   8983	struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
   8984	u16 vid = vlan_dev_vlan_id(vlan_dev);
   8985
   8986	if (netif_is_bridge_port(vlan_dev))
   8987		return 0;
   8988
   8989	if (mlxsw_sp_port_dev_check(real_dev))
   8990		return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
   8991							 event, vid, extack);
   8992	else if (netif_is_lag_master(real_dev))
   8993		return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
   8994						     vid, extack);
   8995	else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
   8996		return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
   8997						      extack);
   8998
   8999	return 0;
   9000}
   9001
   9002static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
   9003{
   9004	u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
   9005	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
   9006
   9007	return ether_addr_equal_masked(mac, vrrp4, mask);
   9008}
   9009
   9010static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
   9011{
   9012	u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
   9013	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
   9014
   9015	return ether_addr_equal_masked(mac, vrrp6, mask);
   9016}
   9017
   9018static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
   9019				const u8 *mac, bool adding)
   9020{
   9021	char ritr_pl[MLXSW_REG_RITR_LEN];
   9022	u8 vrrp_id = adding ? mac[5] : 0;
   9023	int err;
   9024
   9025	if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
   9026	    !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
   9027		return 0;
   9028
   9029	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
   9030	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9031	if (err)
   9032		return err;
   9033
   9034	if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
   9035		mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
   9036	else
   9037		mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
   9038
   9039	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9040}
   9041
   9042static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
   9043				    const struct net_device *macvlan_dev,
   9044				    struct netlink_ext_ack *extack)
   9045{
   9046	struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
   9047	struct mlxsw_sp_rif *rif;
   9048	int err;
   9049
   9050	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
   9051	if (!rif) {
   9052		NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
   9053		return -EOPNOTSUPP;
   9054	}
   9055
   9056	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
   9057				  mlxsw_sp_fid_index(rif->fid), true);
   9058	if (err)
   9059		return err;
   9060
   9061	err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
   9062				   macvlan_dev->dev_addr, true);
   9063	if (err)
   9064		goto err_rif_vrrp_add;
   9065
   9066	/* Make sure the bridge driver does not have this MAC pointing at
   9067	 * some other port.
   9068	 */
   9069	if (rif->ops->fdb_del)
   9070		rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
   9071
   9072	return 0;
   9073
   9074err_rif_vrrp_add:
   9075	mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
   9076			    mlxsw_sp_fid_index(rif->fid), false);
   9077	return err;
   9078}
   9079
   9080static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
   9081				       const struct net_device *macvlan_dev)
   9082{
   9083	struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
   9084	struct mlxsw_sp_rif *rif;
   9085
   9086	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
   9087	/* If we do not have a RIF, then we already took care of
   9088	 * removing the macvlan's MAC during RIF deletion.
   9089	 */
   9090	if (!rif)
   9091		return;
   9092	mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
   9093			     false);
   9094	mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
   9095			    mlxsw_sp_fid_index(rif->fid), false);
   9096}
   9097
   9098void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
   9099			      const struct net_device *macvlan_dev)
   9100{
   9101	mutex_lock(&mlxsw_sp->router->lock);
   9102	__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
   9103	mutex_unlock(&mlxsw_sp->router->lock);
   9104}
   9105
   9106static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
   9107					   struct net_device *macvlan_dev,
   9108					   unsigned long event,
   9109					   struct netlink_ext_ack *extack)
   9110{
   9111	switch (event) {
   9112	case NETDEV_UP:
   9113		return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
   9114	case NETDEV_DOWN:
   9115		__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
   9116		break;
   9117	}
   9118
   9119	return 0;
   9120}
   9121
   9122static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
   9123				     struct net_device *dev,
   9124				     unsigned long event,
   9125				     struct netlink_ext_ack *extack)
   9126{
   9127	if (mlxsw_sp_port_dev_check(dev))
   9128		return mlxsw_sp_inetaddr_port_event(dev, event, extack);
   9129	else if (netif_is_lag_master(dev))
   9130		return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
   9131	else if (netif_is_bridge_master(dev))
   9132		return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
   9133						      extack);
   9134	else if (is_vlan_dev(dev))
   9135		return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
   9136						    extack);
   9137	else if (netif_is_macvlan(dev))
   9138		return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
   9139						       extack);
   9140	else
   9141		return 0;
   9142}
   9143
   9144static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
   9145				   unsigned long event, void *ptr)
   9146{
   9147	struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
   9148	struct net_device *dev = ifa->ifa_dev->dev;
   9149	struct mlxsw_sp_router *router;
   9150	struct mlxsw_sp_rif *rif;
   9151	int err = 0;
   9152
   9153	/* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
   9154	if (event == NETDEV_UP)
   9155		return NOTIFY_DONE;
   9156
   9157	router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
   9158	mutex_lock(&router->lock);
   9159	rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
   9160	if (!mlxsw_sp_rif_should_config(rif, dev, event))
   9161		goto out;
   9162
   9163	err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
   9164out:
   9165	mutex_unlock(&router->lock);
   9166	return notifier_from_errno(err);
   9167}
   9168
   9169int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
   9170				  unsigned long event, void *ptr)
   9171{
   9172	struct in_validator_info *ivi = (struct in_validator_info *) ptr;
   9173	struct net_device *dev = ivi->ivi_dev->dev;
   9174	struct mlxsw_sp *mlxsw_sp;
   9175	struct mlxsw_sp_rif *rif;
   9176	int err = 0;
   9177
   9178	mlxsw_sp = mlxsw_sp_lower_get(dev);
   9179	if (!mlxsw_sp)
   9180		return NOTIFY_DONE;
   9181
   9182	mutex_lock(&mlxsw_sp->router->lock);
   9183	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   9184	if (!mlxsw_sp_rif_should_config(rif, dev, event))
   9185		goto out;
   9186
   9187	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
   9188out:
   9189	mutex_unlock(&mlxsw_sp->router->lock);
   9190	return notifier_from_errno(err);
   9191}
   9192
   9193struct mlxsw_sp_inet6addr_event_work {
   9194	struct work_struct work;
   9195	struct mlxsw_sp *mlxsw_sp;
   9196	struct net_device *dev;
   9197	unsigned long event;
   9198};
   9199
   9200static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
   9201{
   9202	struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
   9203		container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
   9204	struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
   9205	struct net_device *dev = inet6addr_work->dev;
   9206	unsigned long event = inet6addr_work->event;
   9207	struct mlxsw_sp_rif *rif;
   9208
   9209	rtnl_lock();
   9210	mutex_lock(&mlxsw_sp->router->lock);
   9211
   9212	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   9213	if (!mlxsw_sp_rif_should_config(rif, dev, event))
   9214		goto out;
   9215
   9216	__mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
   9217out:
   9218	mutex_unlock(&mlxsw_sp->router->lock);
   9219	rtnl_unlock();
   9220	dev_put(dev);
   9221	kfree(inet6addr_work);
   9222}
   9223
   9224/* Called with rcu_read_lock() */
   9225static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
   9226				    unsigned long event, void *ptr)
   9227{
   9228	struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
   9229	struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
   9230	struct net_device *dev = if6->idev->dev;
   9231	struct mlxsw_sp_router *router;
   9232
   9233	/* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
   9234	if (event == NETDEV_UP)
   9235		return NOTIFY_DONE;
   9236
   9237	inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
   9238	if (!inet6addr_work)
   9239		return NOTIFY_BAD;
   9240
   9241	router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
   9242	INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
   9243	inet6addr_work->mlxsw_sp = router->mlxsw_sp;
   9244	inet6addr_work->dev = dev;
   9245	inet6addr_work->event = event;
   9246	dev_hold(dev);
   9247	mlxsw_core_schedule_work(&inet6addr_work->work);
   9248
   9249	return NOTIFY_DONE;
   9250}
   9251
   9252int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
   9253				   unsigned long event, void *ptr)
   9254{
   9255	struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
   9256	struct net_device *dev = i6vi->i6vi_dev->dev;
   9257	struct mlxsw_sp *mlxsw_sp;
   9258	struct mlxsw_sp_rif *rif;
   9259	int err = 0;
   9260
   9261	mlxsw_sp = mlxsw_sp_lower_get(dev);
   9262	if (!mlxsw_sp)
   9263		return NOTIFY_DONE;
   9264
   9265	mutex_lock(&mlxsw_sp->router->lock);
   9266	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   9267	if (!mlxsw_sp_rif_should_config(rif, dev, event))
   9268		goto out;
   9269
   9270	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
   9271out:
   9272	mutex_unlock(&mlxsw_sp->router->lock);
   9273	return notifier_from_errno(err);
   9274}
   9275
   9276static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
   9277			     const char *mac, int mtu, u8 mac_profile)
   9278{
   9279	char ritr_pl[MLXSW_REG_RITR_LEN];
   9280	int err;
   9281
   9282	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
   9283	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9284	if (err)
   9285		return err;
   9286
   9287	mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
   9288	mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
   9289	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile);
   9290	mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
   9291	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9292}
   9293
   9294static int
   9295mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
   9296				  struct mlxsw_sp_rif *rif,
   9297				  struct netlink_ext_ack *extack)
   9298{
   9299	struct net_device *dev = rif->dev;
   9300	u8 old_mac_profile;
   9301	u16 fid_index;
   9302	int err;
   9303
   9304	fid_index = mlxsw_sp_fid_index(rif->fid);
   9305
   9306	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
   9307	if (err)
   9308		return err;
   9309
   9310	old_mac_profile = rif->mac_profile_id;
   9311	err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr,
   9312					       extack);
   9313	if (err)
   9314		goto err_rif_mac_profile_replace;
   9315
   9316	err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
   9317				dev->mtu, rif->mac_profile_id);
   9318	if (err)
   9319		goto err_rif_edit;
   9320
   9321	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
   9322	if (err)
   9323		goto err_rif_fdb_op;
   9324
   9325	if (rif->mtu != dev->mtu) {
   9326		struct mlxsw_sp_vr *vr;
   9327		int i;
   9328
   9329		/* The RIF is relevant only to its mr_table instance, as unlike
   9330		 * unicast routing, in multicast routing a RIF cannot be shared
   9331		 * between several multicast routing tables.
   9332		 */
   9333		vr = &mlxsw_sp->router->vrs[rif->vr_id];
   9334		for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
   9335			mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
   9336						   rif, dev->mtu);
   9337	}
   9338
   9339	ether_addr_copy(rif->addr, dev->dev_addr);
   9340	rif->mtu = dev->mtu;
   9341
   9342	netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
   9343
   9344	return 0;
   9345
   9346err_rif_fdb_op:
   9347	mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu,
   9348			  old_mac_profile);
   9349err_rif_edit:
   9350	mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack);
   9351err_rif_mac_profile_replace:
   9352	mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
   9353	return err;
   9354}
   9355
   9356static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
   9357			    struct netdev_notifier_pre_changeaddr_info *info)
   9358{
   9359	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9360	struct mlxsw_sp_rif_mac_profile *profile;
   9361	struct netlink_ext_ack *extack;
   9362	u8 max_rif_mac_profiles;
   9363	u64 occ;
   9364
   9365	extack = netdev_notifier_info_to_extack(&info->info);
   9366
   9367	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr);
   9368	if (profile)
   9369		return 0;
   9370
   9371	max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
   9372	occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp);
   9373	if (occ < max_rif_mac_profiles)
   9374		return 0;
   9375
   9376	if (!mlxsw_sp_rif_mac_profile_is_shared(rif))
   9377		return 0;
   9378
   9379	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles");
   9380	return -ENOBUFS;
   9381}
   9382
   9383static bool mlxsw_sp_is_offload_xstats_event(unsigned long event)
   9384{
   9385	switch (event) {
   9386	case NETDEV_OFFLOAD_XSTATS_ENABLE:
   9387	case NETDEV_OFFLOAD_XSTATS_DISABLE:
   9388	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
   9389	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
   9390		return true;
   9391	}
   9392
   9393	return false;
   9394}
   9395
   9396static int
   9397mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif,
   9398					unsigned long event,
   9399					struct netdev_notifier_offload_xstats_info *info)
   9400{
   9401	switch (info->type) {
   9402	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
   9403		break;
   9404	default:
   9405		return 0;
   9406	}
   9407
   9408	switch (event) {
   9409	case NETDEV_OFFLOAD_XSTATS_ENABLE:
   9410		return mlxsw_sp_router_port_l3_stats_enable(rif);
   9411	case NETDEV_OFFLOAD_XSTATS_DISABLE:
   9412		mlxsw_sp_router_port_l3_stats_disable(rif);
   9413		return 0;
   9414	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
   9415		mlxsw_sp_router_port_l3_stats_report_used(rif, info);
   9416		return 0;
   9417	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
   9418		return mlxsw_sp_router_port_l3_stats_report_delta(rif, info);
   9419	}
   9420
   9421	WARN_ON_ONCE(1);
   9422	return 0;
   9423}
   9424
   9425static int
   9426mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp,
   9427				      struct net_device *dev,
   9428				      unsigned long event,
   9429				      struct netdev_notifier_offload_xstats_info *info)
   9430{
   9431	struct mlxsw_sp_rif *rif;
   9432
   9433	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   9434	if (!rif)
   9435		return 0;
   9436
   9437	return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info);
   9438}
   9439
   9440static bool mlxsw_sp_is_router_event(unsigned long event)
   9441{
   9442	switch (event) {
   9443	case NETDEV_PRE_CHANGEADDR:
   9444	case NETDEV_CHANGEADDR:
   9445	case NETDEV_CHANGEMTU:
   9446		return true;
   9447	default:
   9448		return false;
   9449	}
   9450}
   9451
   9452static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
   9453						unsigned long event, void *ptr)
   9454{
   9455	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
   9456	struct mlxsw_sp *mlxsw_sp;
   9457	struct mlxsw_sp_rif *rif;
   9458
   9459	mlxsw_sp = mlxsw_sp_lower_get(dev);
   9460	if (!mlxsw_sp)
   9461		return 0;
   9462
   9463	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
   9464	if (!rif)
   9465		return 0;
   9466
   9467	switch (event) {
   9468	case NETDEV_CHANGEMTU:
   9469	case NETDEV_CHANGEADDR:
   9470		return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack);
   9471	case NETDEV_PRE_CHANGEADDR:
   9472		return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
   9473	default:
   9474		WARN_ON_ONCE(1);
   9475		break;
   9476	}
   9477
   9478	return 0;
   9479}
   9480
   9481static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
   9482				  struct net_device *l3_dev,
   9483				  struct netlink_ext_ack *extack)
   9484{
   9485	struct mlxsw_sp_rif *rif;
   9486
   9487	/* If netdev is already associated with a RIF, then we need to
   9488	 * destroy it and create a new one with the new virtual router ID.
   9489	 */
   9490	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
   9491	if (rif)
   9492		__mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
   9493					  extack);
   9494
   9495	return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
   9496}
   9497
   9498static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
   9499				    struct net_device *l3_dev)
   9500{
   9501	struct mlxsw_sp_rif *rif;
   9502
   9503	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
   9504	if (!rif)
   9505		return;
   9506	__mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
   9507}
   9508
   9509static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr)
   9510{
   9511	struct netdev_notifier_changeupper_info *info = ptr;
   9512
   9513	if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER)
   9514		return false;
   9515	return netif_is_l3_master(info->upper_dev);
   9516}
   9517
   9518static int
   9519mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
   9520			     struct netdev_notifier_changeupper_info *info)
   9521{
   9522	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
   9523	int err = 0;
   9524
   9525	/* We do not create a RIF for a macvlan, but only use it to
   9526	 * direct more MAC addresses to the router.
   9527	 */
   9528	if (!mlxsw_sp || netif_is_macvlan(l3_dev))
   9529		return 0;
   9530
   9531	switch (event) {
   9532	case NETDEV_PRECHANGEUPPER:
   9533		break;
   9534	case NETDEV_CHANGEUPPER:
   9535		if (info->linking) {
   9536			struct netlink_ext_ack *extack;
   9537
   9538			extack = netdev_notifier_info_to_extack(&info->info);
   9539			err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
   9540		} else {
   9541			mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
   9542		}
   9543		break;
   9544	}
   9545
   9546	return err;
   9547}
   9548
   9549static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb,
   9550					   unsigned long event, void *ptr)
   9551{
   9552	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
   9553	struct mlxsw_sp_router *router;
   9554	struct mlxsw_sp *mlxsw_sp;
   9555	int err = 0;
   9556
   9557	router = container_of(nb, struct mlxsw_sp_router, netdevice_nb);
   9558	mlxsw_sp = router->mlxsw_sp;
   9559
   9560	mutex_lock(&mlxsw_sp->router->lock);
   9561
   9562	if (mlxsw_sp_is_offload_xstats_event(event))
   9563		err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev,
   9564							    event, ptr);
   9565	else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev))
   9566		err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev,
   9567						       event, ptr);
   9568	else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev))
   9569		err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev,
   9570						       event, ptr);
   9571	else if (mlxsw_sp_is_router_event(event))
   9572		err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr);
   9573	else if (mlxsw_sp_is_vrf_event(event, ptr))
   9574		err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr);
   9575
   9576	mutex_unlock(&mlxsw_sp->router->lock);
   9577
   9578	return notifier_from_errno(err);
   9579}
   9580
   9581static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
   9582					struct netdev_nested_priv *priv)
   9583{
   9584	struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
   9585
   9586	if (!netif_is_macvlan(dev))
   9587		return 0;
   9588
   9589	return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
   9590				   mlxsw_sp_fid_index(rif->fid), false);
   9591}
   9592
   9593static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
   9594{
   9595	struct netdev_nested_priv priv = {
   9596		.data = (void *)rif,
   9597	};
   9598
   9599	if (!netif_is_macvlan_port(rif->dev))
   9600		return 0;
   9601
   9602	netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
   9603	return netdev_walk_all_upper_dev_rcu(rif->dev,
   9604					     __mlxsw_sp_rif_macvlan_flush, &priv);
   9605}
   9606
   9607static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
   9608				       const struct mlxsw_sp_rif_params *params)
   9609{
   9610	struct mlxsw_sp_rif_subport *rif_subport;
   9611
   9612	rif_subport = mlxsw_sp_rif_subport_rif(rif);
   9613	refcount_set(&rif_subport->ref_count, 1);
   9614	rif_subport->vid = params->vid;
   9615	rif_subport->lag = params->lag;
   9616	if (params->lag)
   9617		rif_subport->lag_id = params->lag_id;
   9618	else
   9619		rif_subport->system_port = params->system_port;
   9620}
   9621
   9622static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
   9623{
   9624	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9625	struct mlxsw_sp_rif_subport *rif_subport;
   9626	char ritr_pl[MLXSW_REG_RITR_LEN];
   9627
   9628	rif_subport = mlxsw_sp_rif_subport_rif(rif);
   9629	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
   9630			    rif->rif_index, rif->vr_id, rif->dev->mtu);
   9631	mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
   9632	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
   9633	mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
   9634				  rif_subport->lag ? rif_subport->lag_id :
   9635						     rif_subport->system_port,
   9636				  rif_subport->vid);
   9637
   9638	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9639}
   9640
   9641static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif,
   9642					  struct netlink_ext_ack *extack)
   9643{
   9644	u8 mac_profile;
   9645	int err;
   9646
   9647	err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr,
   9648					   &mac_profile, extack);
   9649	if (err)
   9650		return err;
   9651	rif->mac_profile_id = mac_profile;
   9652
   9653	err = mlxsw_sp_rif_subport_op(rif, true);
   9654	if (err)
   9655		goto err_rif_subport_op;
   9656
   9657	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
   9658				  mlxsw_sp_fid_index(rif->fid), true);
   9659	if (err)
   9660		goto err_rif_fdb_op;
   9661
   9662	mlxsw_sp_fid_rif_set(rif->fid, rif);
   9663	return 0;
   9664
   9665err_rif_fdb_op:
   9666	mlxsw_sp_rif_subport_op(rif, false);
   9667err_rif_subport_op:
   9668	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile);
   9669	return err;
   9670}
   9671
   9672static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
   9673{
   9674	struct mlxsw_sp_fid *fid = rif->fid;
   9675
   9676	mlxsw_sp_fid_rif_set(fid, NULL);
   9677	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
   9678			    mlxsw_sp_fid_index(fid), false);
   9679	mlxsw_sp_rif_macvlan_flush(rif);
   9680	mlxsw_sp_rif_subport_op(rif, false);
   9681	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
   9682}
   9683
   9684static struct mlxsw_sp_fid *
   9685mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
   9686			     struct netlink_ext_ack *extack)
   9687{
   9688	return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
   9689}
   9690
   9691static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
   9692	.type			= MLXSW_SP_RIF_TYPE_SUBPORT,
   9693	.rif_size		= sizeof(struct mlxsw_sp_rif_subport),
   9694	.setup			= mlxsw_sp_rif_subport_setup,
   9695	.configure		= mlxsw_sp_rif_subport_configure,
   9696	.deconfigure		= mlxsw_sp_rif_subport_deconfigure,
   9697	.fid_get		= mlxsw_sp_rif_subport_fid_get,
   9698};
   9699
   9700static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
   9701				    enum mlxsw_reg_ritr_if_type type,
   9702				    u16 vid_fid, bool enable)
   9703{
   9704	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9705	char ritr_pl[MLXSW_REG_RITR_LEN];
   9706
   9707	mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
   9708			    rif->dev->mtu);
   9709	mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
   9710	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
   9711	mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
   9712
   9713	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9714}
   9715
   9716u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
   9717{
   9718	return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
   9719}
   9720
   9721static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif,
   9722				      struct netlink_ext_ack *extack)
   9723{
   9724	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9725	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
   9726	u8 mac_profile;
   9727	int err;
   9728
   9729	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
   9730					   &mac_profile, extack);
   9731	if (err)
   9732		return err;
   9733	rif->mac_profile_id = mac_profile;
   9734
   9735	err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
   9736				       true);
   9737	if (err)
   9738		goto err_rif_vlan_fid_op;
   9739
   9740	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
   9741				     mlxsw_sp_router_port(mlxsw_sp), true);
   9742	if (err)
   9743		goto err_fid_mc_flood_set;
   9744
   9745	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
   9746				     mlxsw_sp_router_port(mlxsw_sp), true);
   9747	if (err)
   9748		goto err_fid_bc_flood_set;
   9749
   9750	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
   9751				  mlxsw_sp_fid_index(rif->fid), true);
   9752	if (err)
   9753		goto err_rif_fdb_op;
   9754
   9755	mlxsw_sp_fid_rif_set(rif->fid, rif);
   9756	return 0;
   9757
   9758err_rif_fdb_op:
   9759	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
   9760			       mlxsw_sp_router_port(mlxsw_sp), false);
   9761err_fid_bc_flood_set:
   9762	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
   9763			       mlxsw_sp_router_port(mlxsw_sp), false);
   9764err_fid_mc_flood_set:
   9765	mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
   9766err_rif_vlan_fid_op:
   9767	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
   9768	return err;
   9769}
   9770
   9771static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
   9772{
   9773	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
   9774	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9775	struct mlxsw_sp_fid *fid = rif->fid;
   9776
   9777	mlxsw_sp_fid_rif_set(fid, NULL);
   9778	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
   9779			    mlxsw_sp_fid_index(fid), false);
   9780	mlxsw_sp_rif_macvlan_flush(rif);
   9781	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
   9782			       mlxsw_sp_router_port(mlxsw_sp), false);
   9783	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
   9784			       mlxsw_sp_router_port(mlxsw_sp), false);
   9785	mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
   9786	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
   9787}
   9788
   9789static struct mlxsw_sp_fid *
   9790mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
   9791			 struct netlink_ext_ack *extack)
   9792{
   9793	return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
   9794}
   9795
   9796static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
   9797{
   9798	struct switchdev_notifier_fdb_info info = {};
   9799	struct net_device *dev;
   9800
   9801	dev = br_fdb_find_port(rif->dev, mac, 0);
   9802	if (!dev)
   9803		return;
   9804
   9805	info.addr = mac;
   9806	info.vid = 0;
   9807	call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
   9808				 NULL);
   9809}
   9810
   9811static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
   9812	.type			= MLXSW_SP_RIF_TYPE_FID,
   9813	.rif_size		= sizeof(struct mlxsw_sp_rif),
   9814	.configure		= mlxsw_sp_rif_fid_configure,
   9815	.deconfigure		= mlxsw_sp_rif_fid_deconfigure,
   9816	.fid_get		= mlxsw_sp_rif_fid_fid_get,
   9817	.fdb_del		= mlxsw_sp_rif_fid_fdb_del,
   9818};
   9819
   9820static struct mlxsw_sp_fid *
   9821mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
   9822			  struct netlink_ext_ack *extack)
   9823{
   9824	struct net_device *br_dev;
   9825	u16 vid;
   9826	int err;
   9827
   9828	if (is_vlan_dev(rif->dev)) {
   9829		vid = vlan_dev_vlan_id(rif->dev);
   9830		br_dev = vlan_dev_real_dev(rif->dev);
   9831		if (WARN_ON(!netif_is_bridge_master(br_dev)))
   9832			return ERR_PTR(-EINVAL);
   9833	} else {
   9834		err = br_vlan_get_pvid(rif->dev, &vid);
   9835		if (err < 0 || !vid) {
   9836			NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
   9837			return ERR_PTR(-EINVAL);
   9838		}
   9839	}
   9840
   9841	return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
   9842}
   9843
   9844static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
   9845{
   9846	struct switchdev_notifier_fdb_info info = {};
   9847	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
   9848	struct net_device *br_dev;
   9849	struct net_device *dev;
   9850
   9851	br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
   9852	dev = br_fdb_find_port(br_dev, mac, vid);
   9853	if (!dev)
   9854		return;
   9855
   9856	info.addr = mac;
   9857	info.vid = vid;
   9858	call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
   9859				 NULL);
   9860}
   9861
   9862static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
   9863	.type			= MLXSW_SP_RIF_TYPE_VLAN,
   9864	.rif_size		= sizeof(struct mlxsw_sp_rif),
   9865	.configure		= mlxsw_sp_rif_fid_configure,
   9866	.deconfigure		= mlxsw_sp_rif_fid_deconfigure,
   9867	.fid_get		= mlxsw_sp_rif_vlan_fid_get,
   9868	.fdb_del		= mlxsw_sp_rif_vlan_fdb_del,
   9869};
   9870
   9871static struct mlxsw_sp_rif_ipip_lb *
   9872mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
   9873{
   9874	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
   9875}
   9876
   9877static void
   9878mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
   9879			   const struct mlxsw_sp_rif_params *params)
   9880{
   9881	struct mlxsw_sp_rif_params_ipip_lb *params_lb;
   9882	struct mlxsw_sp_rif_ipip_lb *rif_lb;
   9883
   9884	params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
   9885				 common);
   9886	rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
   9887	rif_lb->lb_config = params_lb->lb_config;
   9888}
   9889
   9890static int
   9891mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
   9892				struct netlink_ext_ack *extack)
   9893{
   9894	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
   9895	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
   9896	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9897	struct mlxsw_sp_vr *ul_vr;
   9898	int err;
   9899
   9900	ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
   9901	if (IS_ERR(ul_vr))
   9902		return PTR_ERR(ul_vr);
   9903
   9904	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
   9905	if (err)
   9906		goto err_loopback_op;
   9907
   9908	lb_rif->ul_vr_id = ul_vr->id;
   9909	lb_rif->ul_rif_id = 0;
   9910	++ul_vr->rif_count;
   9911	return 0;
   9912
   9913err_loopback_op:
   9914	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
   9915	return err;
   9916}
   9917
   9918static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
   9919{
   9920	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
   9921	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
   9922	struct mlxsw_sp_vr *ul_vr;
   9923
   9924	ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
   9925	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
   9926
   9927	--ul_vr->rif_count;
   9928	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
   9929}
   9930
   9931static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
   9932	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
   9933	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
   9934	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
   9935	.configure		= mlxsw_sp1_rif_ipip_lb_configure,
   9936	.deconfigure		= mlxsw_sp1_rif_ipip_lb_deconfigure,
   9937};
   9938
   9939static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
   9940	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
   9941	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp_rif_vlan_emu_ops,
   9942	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
   9943	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp1_rif_ipip_lb_ops,
   9944};
   9945
   9946static int
   9947mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
   9948{
   9949	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
   9950	char ritr_pl[MLXSW_REG_RITR_LEN];
   9951
   9952	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
   9953			    ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
   9954	mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
   9955					     MLXSW_REG_RITR_LOOPBACK_GENERIC);
   9956
   9957	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
   9958}
   9959
   9960static struct mlxsw_sp_rif *
   9961mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
   9962		       struct netlink_ext_ack *extack)
   9963{
   9964	struct mlxsw_sp_rif *ul_rif;
   9965	u16 rif_index;
   9966	int err;
   9967
   9968	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
   9969	if (err) {
   9970		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
   9971		return ERR_PTR(err);
   9972	}
   9973
   9974	ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
   9975	if (!ul_rif)
   9976		return ERR_PTR(-ENOMEM);
   9977
   9978	mlxsw_sp->router->rifs[rif_index] = ul_rif;
   9979	ul_rif->mlxsw_sp = mlxsw_sp;
   9980	err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
   9981	if (err)
   9982		goto ul_rif_op_err;
   9983
   9984	return ul_rif;
   9985
   9986ul_rif_op_err:
   9987	mlxsw_sp->router->rifs[rif_index] = NULL;
   9988	kfree(ul_rif);
   9989	return ERR_PTR(err);
   9990}
   9991
   9992static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
   9993{
   9994	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
   9995
   9996	mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
   9997	mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
   9998	kfree(ul_rif);
   9999}
  10000
  10001static struct mlxsw_sp_rif *
  10002mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
  10003		    struct netlink_ext_ack *extack)
  10004{
  10005	struct mlxsw_sp_vr *vr;
  10006	int err;
  10007
  10008	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
  10009	if (IS_ERR(vr))
  10010		return ERR_CAST(vr);
  10011
  10012	if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
  10013		return vr->ul_rif;
  10014
  10015	vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
  10016	if (IS_ERR(vr->ul_rif)) {
  10017		err = PTR_ERR(vr->ul_rif);
  10018		goto err_ul_rif_create;
  10019	}
  10020
  10021	vr->rif_count++;
  10022	refcount_set(&vr->ul_rif_refcnt, 1);
  10023
  10024	return vr->ul_rif;
  10025
  10026err_ul_rif_create:
  10027	mlxsw_sp_vr_put(mlxsw_sp, vr);
  10028	return ERR_PTR(err);
  10029}
  10030
  10031static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
  10032{
  10033	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
  10034	struct mlxsw_sp_vr *vr;
  10035
  10036	vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
  10037
  10038	if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
  10039		return;
  10040
  10041	vr->rif_count--;
  10042	mlxsw_sp_ul_rif_destroy(ul_rif);
  10043	mlxsw_sp_vr_put(mlxsw_sp, vr);
  10044}
  10045
  10046int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
  10047			       u16 *ul_rif_index)
  10048{
  10049	struct mlxsw_sp_rif *ul_rif;
  10050	int err = 0;
  10051
  10052	mutex_lock(&mlxsw_sp->router->lock);
  10053	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
  10054	if (IS_ERR(ul_rif)) {
  10055		err = PTR_ERR(ul_rif);
  10056		goto out;
  10057	}
  10058	*ul_rif_index = ul_rif->rif_index;
  10059out:
  10060	mutex_unlock(&mlxsw_sp->router->lock);
  10061	return err;
  10062}
  10063
  10064void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
  10065{
  10066	struct mlxsw_sp_rif *ul_rif;
  10067
  10068	mutex_lock(&mlxsw_sp->router->lock);
  10069	ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
  10070	if (WARN_ON(!ul_rif))
  10071		goto out;
  10072
  10073	mlxsw_sp_ul_rif_put(ul_rif);
  10074out:
  10075	mutex_unlock(&mlxsw_sp->router->lock);
  10076}
  10077
  10078static int
  10079mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
  10080				struct netlink_ext_ack *extack)
  10081{
  10082	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
  10083	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
  10084	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
  10085	struct mlxsw_sp_rif *ul_rif;
  10086	int err;
  10087
  10088	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
  10089	if (IS_ERR(ul_rif))
  10090		return PTR_ERR(ul_rif);
  10091
  10092	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
  10093	if (err)
  10094		goto err_loopback_op;
  10095
  10096	lb_rif->ul_vr_id = 0;
  10097	lb_rif->ul_rif_id = ul_rif->rif_index;
  10098
  10099	return 0;
  10100
  10101err_loopback_op:
  10102	mlxsw_sp_ul_rif_put(ul_rif);
  10103	return err;
  10104}
  10105
  10106static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
  10107{
  10108	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
  10109	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
  10110	struct mlxsw_sp_rif *ul_rif;
  10111
  10112	ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
  10113	mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
  10114	mlxsw_sp_ul_rif_put(ul_rif);
  10115}
  10116
  10117static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
  10118	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
  10119	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
  10120	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
  10121	.configure		= mlxsw_sp2_rif_ipip_lb_configure,
  10122	.deconfigure		= mlxsw_sp2_rif_ipip_lb_deconfigure,
  10123};
  10124
  10125static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
  10126	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
  10127	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp_rif_vlan_emu_ops,
  10128	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
  10129	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp2_rif_ipip_lb_ops,
  10130};
  10131
  10132static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
  10133{
  10134	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
  10135	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
  10136	struct mlxsw_core *core = mlxsw_sp->core;
  10137
  10138	if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES))
  10139		return -EIO;
  10140	mlxsw_sp->router->max_rif_mac_profile =
  10141		MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES);
  10142
  10143	mlxsw_sp->router->rifs = kcalloc(max_rifs,
  10144					 sizeof(struct mlxsw_sp_rif *),
  10145					 GFP_KERNEL);
  10146	if (!mlxsw_sp->router->rifs)
  10147		return -ENOMEM;
  10148
  10149	idr_init(&mlxsw_sp->router->rif_mac_profiles_idr);
  10150	atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0);
  10151	devlink_resource_occ_get_register(devlink,
  10152					  MLXSW_SP_RESOURCE_RIF_MAC_PROFILES,
  10153					  mlxsw_sp_rif_mac_profiles_occ_get,
  10154					  mlxsw_sp);
  10155
  10156	return 0;
  10157}
  10158
  10159static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
  10160{
  10161	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
  10162	int i;
  10163
  10164	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
  10165		WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
  10166
  10167	devlink_resource_occ_get_unregister(devlink,
  10168					    MLXSW_SP_RESOURCE_RIF_MAC_PROFILES);
  10169	WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr));
  10170	idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr);
  10171	kfree(mlxsw_sp->router->rifs);
  10172}
  10173
  10174static int
  10175mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
  10176{
  10177	char tigcr_pl[MLXSW_REG_TIGCR_LEN];
  10178
  10179	mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
  10180	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
  10181}
  10182
  10183static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
  10184{
  10185	int err;
  10186
  10187	INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
  10188
  10189	err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
  10190	if (err)
  10191		return err;
  10192	err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
  10193	if (err)
  10194		return err;
  10195
  10196	return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
  10197}
  10198
  10199static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp)
  10200{
  10201	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr;
  10202	return mlxsw_sp_ipips_init(mlxsw_sp);
  10203}
  10204
  10205static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp)
  10206{
  10207	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr;
  10208	return mlxsw_sp_ipips_init(mlxsw_sp);
  10209}
  10210
  10211static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
  10212{
  10213	WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
  10214}
  10215
  10216static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
  10217{
  10218	struct mlxsw_sp_router *router;
  10219
  10220	/* Flush pending FIB notifications and then flush the device's
  10221	 * table before requesting another dump. The FIB notification
  10222	 * block is unregistered, so no need to take RTNL.
  10223	 */
  10224	mlxsw_core_flush_owq();
  10225	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
  10226	mlxsw_sp_router_fib_flush(router->mlxsw_sp);
  10227}
  10228
  10229#ifdef CONFIG_IP_ROUTE_MULTIPATH
  10230struct mlxsw_sp_mp_hash_config {
  10231	DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT);
  10232	DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT);
  10233	DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT);
  10234	DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT);
  10235	bool inc_parsing_depth;
  10236};
  10237
  10238#define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \
  10239	bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1)
  10240
  10241#define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \
  10242	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1)
  10243
  10244#define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \
  10245	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr)
  10246
  10247static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config)
  10248{
  10249	unsigned long *inner_headers = config->inner_headers;
  10250	unsigned long *inner_fields = config->inner_fields;
  10251
  10252	/* IPv4 inner */
  10253	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
  10254	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
  10255	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
  10256	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
  10257	/* IPv6 inner */
  10258	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
  10259	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
  10260	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
  10261	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
  10262	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
  10263	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
  10264	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
  10265	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
  10266}
  10267
  10268static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
  10269{
  10270	unsigned long *headers = config->headers;
  10271	unsigned long *fields = config->fields;
  10272
  10273	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
  10274	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
  10275	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
  10276	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
  10277}
  10278
  10279static void
  10280mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config,
  10281			      u32 hash_fields)
  10282{
  10283	unsigned long *inner_headers = config->inner_headers;
  10284	unsigned long *inner_fields = config->inner_fields;
  10285
  10286	/* IPv4 Inner */
  10287	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
  10288	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
  10289	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
  10290		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
  10291	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
  10292		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
  10293	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
  10294		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL);
  10295	/* IPv6 inner */
  10296	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
  10297	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
  10298	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) {
  10299		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
  10300		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
  10301	}
  10302	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) {
  10303		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
  10304		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
  10305	}
  10306	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
  10307		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
  10308	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
  10309		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
  10310	/* L4 inner */
  10311	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4);
  10312	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6);
  10313	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
  10314		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT);
  10315	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
  10316		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT);
  10317}
  10318
  10319static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp,
  10320				   struct mlxsw_sp_mp_hash_config *config)
  10321{
  10322	struct net *net = mlxsw_sp_net(mlxsw_sp);
  10323	unsigned long *headers = config->headers;
  10324	unsigned long *fields = config->fields;
  10325	u32 hash_fields;
  10326
  10327	switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
  10328	case 0:
  10329		mlxsw_sp_mp4_hash_outer_addr(config);
  10330		break;
  10331	case 1:
  10332		mlxsw_sp_mp4_hash_outer_addr(config);
  10333		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
  10334		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
  10335		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
  10336		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
  10337		break;
  10338	case 2:
  10339		/* Outer */
  10340		mlxsw_sp_mp4_hash_outer_addr(config);
  10341		/* Inner */
  10342		mlxsw_sp_mp_hash_inner_l3(config);
  10343		break;
  10344	case 3:
  10345		hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields;
  10346		/* Outer */
  10347		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
  10348		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
  10349		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
  10350		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
  10351			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
  10352		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
  10353			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
  10354		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
  10355			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
  10356		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
  10357			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
  10358		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
  10359			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
  10360		/* Inner */
  10361		mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
  10362		break;
  10363	}
  10364}
  10365
  10366static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
  10367{
  10368	unsigned long *headers = config->headers;
  10369	unsigned long *fields = config->fields;
  10370
  10371	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
  10372	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
  10373	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
  10374	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
  10375	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
  10376	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
  10377}
  10378
  10379static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp,
  10380				   struct mlxsw_sp_mp_hash_config *config)
  10381{
  10382	u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp));
  10383	unsigned long *headers = config->headers;
  10384	unsigned long *fields = config->fields;
  10385
  10386	switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) {
  10387	case 0:
  10388		mlxsw_sp_mp6_hash_outer_addr(config);
  10389		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
  10390		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
  10391		break;
  10392	case 1:
  10393		mlxsw_sp_mp6_hash_outer_addr(config);
  10394		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
  10395		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
  10396		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
  10397		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
  10398		break;
  10399	case 2:
  10400		/* Outer */
  10401		mlxsw_sp_mp6_hash_outer_addr(config);
  10402		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
  10403		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
  10404		/* Inner */
  10405		mlxsw_sp_mp_hash_inner_l3(config);
  10406		config->inc_parsing_depth = true;
  10407		break;
  10408	case 3:
  10409		/* Outer */
  10410		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
  10411		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
  10412		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
  10413		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) {
  10414			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
  10415			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
  10416		}
  10417		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) {
  10418			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
  10419			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
  10420		}
  10421		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
  10422			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
  10423		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
  10424			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
  10425		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
  10426			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
  10427		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
  10428			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
  10429		/* Inner */
  10430		mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
  10431		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)
  10432			config->inc_parsing_depth = true;
  10433		break;
  10434	}
  10435}
  10436
  10437static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp,
  10438						 bool old_inc_parsing_depth,
  10439						 bool new_inc_parsing_depth)
  10440{
  10441	int err;
  10442
  10443	if (!old_inc_parsing_depth && new_inc_parsing_depth) {
  10444		err = mlxsw_sp_parsing_depth_inc(mlxsw_sp);
  10445		if (err)
  10446			return err;
  10447		mlxsw_sp->router->inc_parsing_depth = true;
  10448	} else if (old_inc_parsing_depth && !new_inc_parsing_depth) {
  10449		mlxsw_sp_parsing_depth_dec(mlxsw_sp);
  10450		mlxsw_sp->router->inc_parsing_depth = false;
  10451	}
  10452
  10453	return 0;
  10454}
  10455
  10456static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
  10457{
  10458	bool old_inc_parsing_depth, new_inc_parsing_depth;
  10459	struct mlxsw_sp_mp_hash_config config = {};
  10460	char recr2_pl[MLXSW_REG_RECR2_LEN];
  10461	unsigned long bit;
  10462	u32 seed;
  10463	int err;
  10464
  10465	seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
  10466	mlxsw_reg_recr2_pack(recr2_pl, seed);
  10467	mlxsw_sp_mp4_hash_init(mlxsw_sp, &config);
  10468	mlxsw_sp_mp6_hash_init(mlxsw_sp, &config);
  10469
  10470	old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
  10471	new_inc_parsing_depth = config.inc_parsing_depth;
  10472	err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp,
  10473						    old_inc_parsing_depth,
  10474						    new_inc_parsing_depth);
  10475	if (err)
  10476		return err;
  10477
  10478	for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT)
  10479		mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1);
  10480	for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT)
  10481		mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1);
  10482	for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT)
  10483		mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1);
  10484	for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT)
  10485		mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1);
  10486
  10487	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
  10488	if (err)
  10489		goto err_reg_write;
  10490
  10491	return 0;
  10492
  10493err_reg_write:
  10494	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth,
  10495					      old_inc_parsing_depth);
  10496	return err;
  10497}
  10498#else
  10499static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
  10500{
  10501	return 0;
  10502}
  10503#endif
  10504
  10505static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
  10506{
  10507	char rdpm_pl[MLXSW_REG_RDPM_LEN];
  10508	unsigned int i;
  10509
  10510	MLXSW_REG_ZERO(rdpm, rdpm_pl);
  10511
  10512	/* HW is determining switch priority based on DSCP-bits, but the
  10513	 * kernel is still doing that based on the ToS. Since there's a
  10514	 * mismatch in bits we need to make sure to translate the right
  10515	 * value ToS would observe, skipping the 2 least-significant ECN bits.
  10516	 */
  10517	for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
  10518		mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
  10519
  10520	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
  10521}
  10522
  10523static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
  10524{
  10525	struct net *net = mlxsw_sp_net(mlxsw_sp);
  10526	bool usp = net->ipv4.sysctl_ip_fwd_update_priority;
  10527	char rgcr_pl[MLXSW_REG_RGCR_LEN];
  10528	u64 max_rifs;
  10529
  10530	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
  10531		return -EIO;
  10532	max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
  10533
  10534	mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
  10535	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
  10536	mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
  10537	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
  10538}
  10539
  10540static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
  10541{
  10542	char rgcr_pl[MLXSW_REG_RGCR_LEN];
  10543
  10544	mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
  10545	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
  10546}
  10547
  10548static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = {
  10549	.init = mlxsw_sp_router_ll_basic_init,
  10550	.ralta_write = mlxsw_sp_router_ll_basic_ralta_write,
  10551	.ralst_write = mlxsw_sp_router_ll_basic_ralst_write,
  10552	.raltb_write = mlxsw_sp_router_ll_basic_raltb_write,
  10553	.fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic),
  10554	.fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack,
  10555	.fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack,
  10556	.fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack,
  10557	.fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack,
  10558	.fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack,
  10559	.fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit,
  10560	.fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed,
  10561};
  10562
  10563static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router)
  10564{
  10565	size_t max_size = 0;
  10566	int i;
  10567
  10568	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
  10569		size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size;
  10570
  10571		if (size > max_size)
  10572			max_size = size;
  10573	}
  10574	router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size,
  10575				    GFP_KERNEL);
  10576	if (!router->ll_op_ctx)
  10577		return -ENOMEM;
  10578	INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list);
  10579	return 0;
  10580}
  10581
  10582static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router)
  10583{
  10584	WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list));
  10585	kfree(router->ll_op_ctx);
  10586}
  10587
  10588static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp)
  10589{
  10590	u16 lb_rif_index;
  10591	int err;
  10592
  10593	/* Create a generic loopback RIF associated with the main table
  10594	 * (default VRF). Any table can be used, but the main table exists
  10595	 * anyway, so we do not waste resources.
  10596	 */
  10597	err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN,
  10598					 &lb_rif_index);
  10599	if (err)
  10600		return err;
  10601
  10602	mlxsw_sp->router->lb_rif_index = lb_rif_index;
  10603
  10604	return 0;
  10605}
  10606
  10607static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp)
  10608{
  10609	mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index);
  10610}
  10611
  10612static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp)
  10613{
  10614	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges);
  10615
  10616	mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr;
  10617	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges;
  10618	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
  10619
  10620	return 0;
  10621}
  10622
  10623const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = {
  10624	.init = mlxsw_sp1_router_init,
  10625	.ipips_init = mlxsw_sp1_ipips_init,
  10626};
  10627
  10628static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp)
  10629{
  10630	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges);
  10631
  10632	mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr;
  10633	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges;
  10634	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
  10635
  10636	return 0;
  10637}
  10638
  10639const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = {
  10640	.init = mlxsw_sp2_router_init,
  10641	.ipips_init = mlxsw_sp2_ipips_init,
  10642};
  10643
  10644int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
  10645			 struct netlink_ext_ack *extack)
  10646{
  10647	struct mlxsw_sp_router *router;
  10648	int err;
  10649
  10650	router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
  10651	if (!router)
  10652		return -ENOMEM;
  10653	mutex_init(&router->lock);
  10654	mlxsw_sp->router = router;
  10655	router->mlxsw_sp = mlxsw_sp;
  10656
  10657	err = mlxsw_sp->router_ops->init(mlxsw_sp);
  10658	if (err)
  10659		goto err_router_ops_init;
  10660
  10661	err = mlxsw_sp_router_xm_init(mlxsw_sp);
  10662	if (err)
  10663		goto err_xm_init;
  10664
  10665	router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ?
  10666						       &mlxsw_sp_router_ll_xm_ops :
  10667						       &mlxsw_sp_router_ll_basic_ops;
  10668	router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops;
  10669
  10670	err = mlxsw_sp_router_ll_op_ctx_init(router);
  10671	if (err)
  10672		goto err_ll_op_ctx_init;
  10673
  10674	INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list);
  10675	INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw,
  10676			  mlxsw_sp_nh_grp_activity_work);
  10677
  10678	INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
  10679	err = __mlxsw_sp_router_init(mlxsw_sp);
  10680	if (err)
  10681		goto err_router_init;
  10682
  10683	err = mlxsw_sp_rifs_init(mlxsw_sp);
  10684	if (err)
  10685		goto err_rifs_init;
  10686
  10687	err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp);
  10688	if (err)
  10689		goto err_ipips_init;
  10690
  10691	err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
  10692			      &mlxsw_sp_nexthop_ht_params);
  10693	if (err)
  10694		goto err_nexthop_ht_init;
  10695
  10696	err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
  10697			      &mlxsw_sp_nexthop_group_ht_params);
  10698	if (err)
  10699		goto err_nexthop_group_ht_init;
  10700
  10701	INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
  10702	err = mlxsw_sp_lpm_init(mlxsw_sp);
  10703	if (err)
  10704		goto err_lpm_init;
  10705
  10706	err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
  10707	if (err)
  10708		goto err_mr_init;
  10709
  10710	err = mlxsw_sp_vrs_init(mlxsw_sp);
  10711	if (err)
  10712		goto err_vrs_init;
  10713
  10714	err = mlxsw_sp_lb_rif_init(mlxsw_sp);
  10715	if (err)
  10716		goto err_lb_rif_init;
  10717
  10718	err = mlxsw_sp_neigh_init(mlxsw_sp);
  10719	if (err)
  10720		goto err_neigh_init;
  10721
  10722	err = mlxsw_sp_mp_hash_init(mlxsw_sp);
  10723	if (err)
  10724		goto err_mp_hash_init;
  10725
  10726	err = mlxsw_sp_dscp_init(mlxsw_sp);
  10727	if (err)
  10728		goto err_dscp_init;
  10729
  10730	INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work);
  10731	INIT_LIST_HEAD(&router->fib_event_queue);
  10732	spin_lock_init(&router->fib_event_queue_lock);
  10733
  10734	router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
  10735	err = register_inetaddr_notifier(&router->inetaddr_nb);
  10736	if (err)
  10737		goto err_register_inetaddr_notifier;
  10738
  10739	router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
  10740	err = register_inet6addr_notifier(&router->inet6addr_nb);
  10741	if (err)
  10742		goto err_register_inet6addr_notifier;
  10743
  10744	mlxsw_sp->router->netevent_nb.notifier_call =
  10745		mlxsw_sp_router_netevent_event;
  10746	err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
  10747	if (err)
  10748		goto err_register_netevent_notifier;
  10749
  10750	mlxsw_sp->router->nexthop_nb.notifier_call =
  10751		mlxsw_sp_nexthop_obj_event;
  10752	err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
  10753					&mlxsw_sp->router->nexthop_nb,
  10754					extack);
  10755	if (err)
  10756		goto err_register_nexthop_notifier;
  10757
  10758	mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
  10759	err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
  10760				    &mlxsw_sp->router->fib_nb,
  10761				    mlxsw_sp_router_fib_dump_flush, extack);
  10762	if (err)
  10763		goto err_register_fib_notifier;
  10764
  10765	mlxsw_sp->router->netdevice_nb.notifier_call =
  10766		mlxsw_sp_router_netdevice_event;
  10767	err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
  10768					      &mlxsw_sp->router->netdevice_nb);
  10769	if (err)
  10770		goto err_register_netdev_notifier;
  10771
  10772	return 0;
  10773
  10774err_register_netdev_notifier:
  10775	unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
  10776				&mlxsw_sp->router->fib_nb);
  10777err_register_fib_notifier:
  10778	unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
  10779				    &mlxsw_sp->router->nexthop_nb);
  10780err_register_nexthop_notifier:
  10781	unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
  10782err_register_netevent_notifier:
  10783	unregister_inet6addr_notifier(&router->inet6addr_nb);
  10784err_register_inet6addr_notifier:
  10785	unregister_inetaddr_notifier(&router->inetaddr_nb);
  10786err_register_inetaddr_notifier:
  10787	mlxsw_core_flush_owq();
  10788	WARN_ON(!list_empty(&router->fib_event_queue));
  10789err_dscp_init:
  10790err_mp_hash_init:
  10791	mlxsw_sp_neigh_fini(mlxsw_sp);
  10792err_neigh_init:
  10793	mlxsw_sp_lb_rif_fini(mlxsw_sp);
  10794err_lb_rif_init:
  10795	mlxsw_sp_vrs_fini(mlxsw_sp);
  10796err_vrs_init:
  10797	mlxsw_sp_mr_fini(mlxsw_sp);
  10798err_mr_init:
  10799	mlxsw_sp_lpm_fini(mlxsw_sp);
  10800err_lpm_init:
  10801	rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
  10802err_nexthop_group_ht_init:
  10803	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
  10804err_nexthop_ht_init:
  10805	mlxsw_sp_ipips_fini(mlxsw_sp);
  10806err_ipips_init:
  10807	mlxsw_sp_rifs_fini(mlxsw_sp);
  10808err_rifs_init:
  10809	__mlxsw_sp_router_fini(mlxsw_sp);
  10810err_router_init:
  10811	cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw);
  10812	mlxsw_sp_router_ll_op_ctx_fini(router);
  10813err_ll_op_ctx_init:
  10814	mlxsw_sp_router_xm_fini(mlxsw_sp);
  10815err_xm_init:
  10816err_router_ops_init:
  10817	mutex_destroy(&mlxsw_sp->router->lock);
  10818	kfree(mlxsw_sp->router);
  10819	return err;
  10820}
  10821
  10822void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
  10823{
  10824	unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
  10825					  &mlxsw_sp->router->netdevice_nb);
  10826	unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
  10827				&mlxsw_sp->router->fib_nb);
  10828	unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
  10829				    &mlxsw_sp->router->nexthop_nb);
  10830	unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
  10831	unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
  10832	unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
  10833	mlxsw_core_flush_owq();
  10834	WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue));
  10835	mlxsw_sp_neigh_fini(mlxsw_sp);
  10836	mlxsw_sp_lb_rif_fini(mlxsw_sp);
  10837	mlxsw_sp_vrs_fini(mlxsw_sp);
  10838	mlxsw_sp_mr_fini(mlxsw_sp);
  10839	mlxsw_sp_lpm_fini(mlxsw_sp);
  10840	rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
  10841	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
  10842	mlxsw_sp_ipips_fini(mlxsw_sp);
  10843	mlxsw_sp_rifs_fini(mlxsw_sp);
  10844	__mlxsw_sp_router_fini(mlxsw_sp);
  10845	cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw);
  10846	mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router);
  10847	mlxsw_sp_router_xm_fini(mlxsw_sp);
  10848	mutex_destroy(&mlxsw_sp->router->lock);
  10849	kfree(mlxsw_sp->router);
  10850}