spectrum_router.c (301400B)
1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4#include <linux/kernel.h> 5#include <linux/types.h> 6#include <linux/rhashtable.h> 7#include <linux/bitops.h> 8#include <linux/in6.h> 9#include <linux/notifier.h> 10#include <linux/inetdevice.h> 11#include <linux/netdevice.h> 12#include <linux/if_bridge.h> 13#include <linux/socket.h> 14#include <linux/route.h> 15#include <linux/gcd.h> 16#include <linux/if_macvlan.h> 17#include <linux/refcount.h> 18#include <linux/jhash.h> 19#include <linux/net_namespace.h> 20#include <linux/mutex.h> 21#include <net/netevent.h> 22#include <net/neighbour.h> 23#include <net/arp.h> 24#include <net/inet_dscp.h> 25#include <net/ip_fib.h> 26#include <net/ip6_fib.h> 27#include <net/nexthop.h> 28#include <net/fib_rules.h> 29#include <net/ip_tunnels.h> 30#include <net/l3mdev.h> 31#include <net/addrconf.h> 32#include <net/ndisc.h> 33#include <net/ipv6.h> 34#include <net/fib_notifier.h> 35#include <net/switchdev.h> 36 37#include "spectrum.h" 38#include "core.h" 39#include "reg.h" 40#include "spectrum_cnt.h" 41#include "spectrum_dpipe.h" 42#include "spectrum_ipip.h" 43#include "spectrum_mr.h" 44#include "spectrum_mr_tcam.h" 45#include "spectrum_router.h" 46#include "spectrum_span.h" 47 48struct mlxsw_sp_fib; 49struct mlxsw_sp_vr; 50struct mlxsw_sp_lpm_tree; 51struct mlxsw_sp_rif_ops; 52 53struct mlxsw_sp_rif { 54 struct list_head nexthop_list; 55 struct list_head neigh_list; 56 struct net_device *dev; /* NULL for underlay RIF */ 57 struct mlxsw_sp_fid *fid; 58 unsigned char addr[ETH_ALEN]; 59 int mtu; 60 u16 rif_index; 61 u8 mac_profile_id; 62 u16 vr_id; 63 const struct mlxsw_sp_rif_ops *ops; 64 struct mlxsw_sp *mlxsw_sp; 65 66 unsigned int counter_ingress; 67 bool counter_ingress_valid; 68 unsigned int counter_egress; 69 bool counter_egress_valid; 70}; 71 72struct mlxsw_sp_rif_params { 73 struct net_device *dev; 74 union { 75 u16 system_port; 76 u16 lag_id; 77 }; 78 u16 vid; 79 bool lag; 80}; 81 82struct mlxsw_sp_rif_subport { 83 struct mlxsw_sp_rif common; 84 refcount_t ref_count; 85 union { 86 u16 system_port; 87 u16 lag_id; 88 }; 89 u16 vid; 90 bool lag; 91}; 92 93struct mlxsw_sp_rif_ipip_lb { 94 struct mlxsw_sp_rif common; 95 struct mlxsw_sp_rif_ipip_lb_config lb_config; 96 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 97 u16 ul_rif_id; /* Reserved for Spectrum. */ 98}; 99 100struct mlxsw_sp_rif_params_ipip_lb { 101 struct mlxsw_sp_rif_params common; 102 struct mlxsw_sp_rif_ipip_lb_config lb_config; 103}; 104 105struct mlxsw_sp_rif_ops { 106 enum mlxsw_sp_rif_type type; 107 size_t rif_size; 108 109 void (*setup)(struct mlxsw_sp_rif *rif, 110 const struct mlxsw_sp_rif_params *params); 111 int (*configure)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*deconfigure)(struct mlxsw_sp_rif *rif); 114 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 115 struct netlink_ext_ack *extack); 116 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 117}; 118 119struct mlxsw_sp_rif_mac_profile { 120 unsigned char mac_prefix[ETH_ALEN]; 121 refcount_t ref_count; 122 u8 id; 123}; 124 125struct mlxsw_sp_router_ops { 126 int (*init)(struct mlxsw_sp *mlxsw_sp); 127 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 128}; 129 130static struct mlxsw_sp_rif * 131mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 132 const struct net_device *dev); 133static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 134static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 135static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 136 struct mlxsw_sp_lpm_tree *lpm_tree); 137static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 138 const struct mlxsw_sp_fib *fib, 139 u8 tree_id); 140static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 141 const struct mlxsw_sp_fib *fib); 142 143static unsigned int * 144mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 145 enum mlxsw_sp_rif_counter_dir dir) 146{ 147 switch (dir) { 148 case MLXSW_SP_RIF_COUNTER_EGRESS: 149 return &rif->counter_egress; 150 case MLXSW_SP_RIF_COUNTER_INGRESS: 151 return &rif->counter_ingress; 152 } 153 return NULL; 154} 155 156static bool 157mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 158 enum mlxsw_sp_rif_counter_dir dir) 159{ 160 switch (dir) { 161 case MLXSW_SP_RIF_COUNTER_EGRESS: 162 return rif->counter_egress_valid; 163 case MLXSW_SP_RIF_COUNTER_INGRESS: 164 return rif->counter_ingress_valid; 165 } 166 return false; 167} 168 169static void 170mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 171 enum mlxsw_sp_rif_counter_dir dir, 172 bool valid) 173{ 174 switch (dir) { 175 case MLXSW_SP_RIF_COUNTER_EGRESS: 176 rif->counter_egress_valid = valid; 177 break; 178 case MLXSW_SP_RIF_COUNTER_INGRESS: 179 rif->counter_ingress_valid = valid; 180 break; 181 } 182} 183 184static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 185 unsigned int counter_index, bool enable, 186 enum mlxsw_sp_rif_counter_dir dir) 187{ 188 char ritr_pl[MLXSW_REG_RITR_LEN]; 189 bool is_egress = false; 190 int err; 191 192 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 193 is_egress = true; 194 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 195 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 196 if (err) 197 return err; 198 199 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 200 is_egress); 201 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 202} 203 204int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 205 struct mlxsw_sp_rif *rif, 206 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 207{ 208 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 209 unsigned int *p_counter_index; 210 bool valid; 211 int err; 212 213 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 214 if (!valid) 215 return -EINVAL; 216 217 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 218 if (!p_counter_index) 219 return -EINVAL; 220 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 221 MLXSW_REG_RICNT_OPCODE_NOP); 222 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 223 if (err) 224 return err; 225 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 226 return 0; 227} 228 229struct mlxsw_sp_rif_counter_set_basic { 230 u64 good_unicast_packets; 231 u64 good_multicast_packets; 232 u64 good_broadcast_packets; 233 u64 good_unicast_bytes; 234 u64 good_multicast_bytes; 235 u64 good_broadcast_bytes; 236 u64 error_packets; 237 u64 discard_packets; 238 u64 error_bytes; 239 u64 discard_bytes; 240}; 241 242static int 243mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 244 enum mlxsw_sp_rif_counter_dir dir, 245 struct mlxsw_sp_rif_counter_set_basic *set) 246{ 247 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 248 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 249 unsigned int *p_counter_index; 250 int err; 251 252 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 253 return -EINVAL; 254 255 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 256 if (!p_counter_index) 257 return -EINVAL; 258 259 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 260 MLXSW_REG_RICNT_OPCODE_CLEAR); 261 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 262 if (err) 263 return err; 264 265 if (!set) 266 return 0; 267 268#define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 269 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 270 271 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 272 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 273 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 274 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 275 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 276 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 277 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 278 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 279 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 280 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 281 282#undef MLXSW_SP_RIF_COUNTER_EXTRACT 283 284 return 0; 285} 286 287static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 288 unsigned int counter_index) 289{ 290 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 291 292 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 293 MLXSW_REG_RICNT_OPCODE_CLEAR); 294 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 295} 296 297int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 298 enum mlxsw_sp_rif_counter_dir dir) 299{ 300 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 301 unsigned int *p_counter_index; 302 int err; 303 304 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 305 return 0; 306 307 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 308 if (!p_counter_index) 309 return -EINVAL; 310 311 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 312 p_counter_index); 313 if (err) 314 return err; 315 316 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 317 if (err) 318 goto err_counter_clear; 319 320 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 321 *p_counter_index, true, dir); 322 if (err) 323 goto err_counter_edit; 324 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 325 return 0; 326 327err_counter_edit: 328err_counter_clear: 329 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 330 *p_counter_index); 331 return err; 332} 333 334void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 335 enum mlxsw_sp_rif_counter_dir dir) 336{ 337 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 338 unsigned int *p_counter_index; 339 340 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 341 return; 342 343 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 344 if (WARN_ON(!p_counter_index)) 345 return; 346 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 347 *p_counter_index, false, dir); 348 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 349 *p_counter_index); 350 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 351} 352 353static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 354{ 355 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 356 struct devlink *devlink; 357 358 devlink = priv_to_devlink(mlxsw_sp->core); 359 if (!devlink_dpipe_table_counter_enabled(devlink, 360 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 361 return; 362 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 363} 364 365static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 366{ 367 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 368} 369 370#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 371 372struct mlxsw_sp_prefix_usage { 373 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 374}; 375 376#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 377 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 378 379static bool 380mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 381 struct mlxsw_sp_prefix_usage *prefix_usage2) 382{ 383 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 384} 385 386static void 387mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 388 struct mlxsw_sp_prefix_usage *prefix_usage2) 389{ 390 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 391} 392 393static void 394mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 395 unsigned char prefix_len) 396{ 397 set_bit(prefix_len, prefix_usage->b); 398} 399 400static void 401mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 402 unsigned char prefix_len) 403{ 404 clear_bit(prefix_len, prefix_usage->b); 405} 406 407struct mlxsw_sp_fib_key { 408 unsigned char addr[sizeof(struct in6_addr)]; 409 unsigned char prefix_len; 410}; 411 412enum mlxsw_sp_fib_entry_type { 413 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 414 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 415 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 416 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 417 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 418 419 /* This is a special case of local delivery, where a packet should be 420 * decapsulated on reception. Note that there is no corresponding ENCAP, 421 * because that's a type of next hop, not of FIB entry. (There can be 422 * several next hops in a REMOTE entry, and some of them may be 423 * encapsulating entries.) 424 */ 425 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 426 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 427}; 428 429struct mlxsw_sp_nexthop_group_info; 430struct mlxsw_sp_nexthop_group; 431struct mlxsw_sp_fib_entry; 432 433struct mlxsw_sp_fib_node { 434 struct mlxsw_sp_fib_entry *fib_entry; 435 struct list_head list; 436 struct rhash_head ht_node; 437 struct mlxsw_sp_fib *fib; 438 struct mlxsw_sp_fib_key key; 439}; 440 441struct mlxsw_sp_fib_entry_decap { 442 struct mlxsw_sp_ipip_entry *ipip_entry; 443 u32 tunnel_index; 444}; 445 446static struct mlxsw_sp_fib_entry_priv * 447mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 448{ 449 struct mlxsw_sp_fib_entry_priv *priv; 450 451 if (!ll_ops->fib_entry_priv_size) 452 /* No need to have priv */ 453 return NULL; 454 455 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 456 if (!priv) 457 return ERR_PTR(-ENOMEM); 458 refcount_set(&priv->refcnt, 1); 459 return priv; 460} 461 462static void 463mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 464{ 465 kfree(priv); 466} 467 468static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 469{ 470 refcount_inc(&priv->refcnt); 471} 472 473static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 474{ 475 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 476 return; 477 mlxsw_sp_fib_entry_priv_destroy(priv); 478} 479 480static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 481 struct mlxsw_sp_fib_entry_priv *priv) 482{ 483 if (!priv) 484 return; 485 mlxsw_sp_fib_entry_priv_hold(priv); 486 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 487} 488 489static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 490{ 491 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 492 493 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 494 mlxsw_sp_fib_entry_priv_put(priv); 495 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 496} 497 498struct mlxsw_sp_fib_entry { 499 struct mlxsw_sp_fib_node *fib_node; 500 enum mlxsw_sp_fib_entry_type type; 501 struct list_head nexthop_group_node; 502 struct mlxsw_sp_nexthop_group *nh_group; 503 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 504 struct mlxsw_sp_fib_entry_priv *priv; 505}; 506 507struct mlxsw_sp_fib4_entry { 508 struct mlxsw_sp_fib_entry common; 509 struct fib_info *fi; 510 u32 tb_id; 511 dscp_t dscp; 512 u8 type; 513}; 514 515struct mlxsw_sp_fib6_entry { 516 struct mlxsw_sp_fib_entry common; 517 struct list_head rt6_list; 518 unsigned int nrt6; 519}; 520 521struct mlxsw_sp_rt6 { 522 struct list_head list; 523 struct fib6_info *rt; 524}; 525 526struct mlxsw_sp_lpm_tree { 527 u8 id; /* tree ID */ 528 unsigned int ref_count; 529 enum mlxsw_sp_l3proto proto; 530 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 531 struct mlxsw_sp_prefix_usage prefix_usage; 532}; 533 534struct mlxsw_sp_fib { 535 struct rhashtable ht; 536 struct list_head node_list; 537 struct mlxsw_sp_vr *vr; 538 struct mlxsw_sp_lpm_tree *lpm_tree; 539 enum mlxsw_sp_l3proto proto; 540 const struct mlxsw_sp_router_ll_ops *ll_ops; 541}; 542 543struct mlxsw_sp_vr { 544 u16 id; /* virtual router ID */ 545 u32 tb_id; /* kernel fib table id */ 546 unsigned int rif_count; 547 struct mlxsw_sp_fib *fib4; 548 struct mlxsw_sp_fib *fib6; 549 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 550 struct mlxsw_sp_rif *ul_rif; 551 refcount_t ul_rif_refcnt; 552}; 553 554static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 555 enum mlxsw_sp_l3proto proto) 556{ 557 return 0; 558} 559 560static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 561{ 562 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 563 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 564} 565 566static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 567{ 568 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 569 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 570} 571 572static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 573{ 574 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 575 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 576} 577 578static const struct rhashtable_params mlxsw_sp_fib_ht_params; 579 580static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 581 struct mlxsw_sp_vr *vr, 582 enum mlxsw_sp_l3proto proto) 583{ 584 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 585 struct mlxsw_sp_lpm_tree *lpm_tree; 586 struct mlxsw_sp_fib *fib; 587 int err; 588 589 err = ll_ops->init(mlxsw_sp, vr->id, proto); 590 if (err) 591 return ERR_PTR(err); 592 593 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 594 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 595 if (!fib) 596 return ERR_PTR(-ENOMEM); 597 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 598 if (err) 599 goto err_rhashtable_init; 600 INIT_LIST_HEAD(&fib->node_list); 601 fib->proto = proto; 602 fib->vr = vr; 603 fib->lpm_tree = lpm_tree; 604 fib->ll_ops = ll_ops; 605 mlxsw_sp_lpm_tree_hold(lpm_tree); 606 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 607 if (err) 608 goto err_lpm_tree_bind; 609 return fib; 610 611err_lpm_tree_bind: 612 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 613err_rhashtable_init: 614 kfree(fib); 615 return ERR_PTR(err); 616} 617 618static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 619 struct mlxsw_sp_fib *fib) 620{ 621 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 622 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 623 WARN_ON(!list_empty(&fib->node_list)); 624 rhashtable_destroy(&fib->ht); 625 kfree(fib); 626} 627 628static struct mlxsw_sp_lpm_tree * 629mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 630{ 631 static struct mlxsw_sp_lpm_tree *lpm_tree; 632 int i; 633 634 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 635 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 636 if (lpm_tree->ref_count == 0) 637 return lpm_tree; 638 } 639 return NULL; 640} 641 642static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 643 const struct mlxsw_sp_router_ll_ops *ll_ops, 644 struct mlxsw_sp_lpm_tree *lpm_tree) 645{ 646 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 647 648 mlxsw_reg_xralta_pack(xralta_pl, true, 649 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 650 lpm_tree->id); 651 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 652} 653 654static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 655 const struct mlxsw_sp_router_ll_ops *ll_ops, 656 struct mlxsw_sp_lpm_tree *lpm_tree) 657{ 658 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 659 660 mlxsw_reg_xralta_pack(xralta_pl, false, 661 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 662 lpm_tree->id); 663 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 664} 665 666static int 667mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 668 const struct mlxsw_sp_router_ll_ops *ll_ops, 669 struct mlxsw_sp_prefix_usage *prefix_usage, 670 struct mlxsw_sp_lpm_tree *lpm_tree) 671{ 672 char xralst_pl[MLXSW_REG_XRALST_LEN]; 673 u8 root_bin = 0; 674 u8 prefix; 675 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 676 677 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 678 root_bin = prefix; 679 680 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 681 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 682 if (prefix == 0) 683 continue; 684 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 685 MLXSW_REG_RALST_BIN_NO_CHILD); 686 last_prefix = prefix; 687 } 688 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 689} 690 691static struct mlxsw_sp_lpm_tree * 692mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 693 const struct mlxsw_sp_router_ll_ops *ll_ops, 694 struct mlxsw_sp_prefix_usage *prefix_usage, 695 enum mlxsw_sp_l3proto proto) 696{ 697 struct mlxsw_sp_lpm_tree *lpm_tree; 698 int err; 699 700 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 701 if (!lpm_tree) 702 return ERR_PTR(-EBUSY); 703 lpm_tree->proto = proto; 704 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 705 if (err) 706 return ERR_PTR(err); 707 708 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 709 if (err) 710 goto err_left_struct_set; 711 memcpy(&lpm_tree->prefix_usage, prefix_usage, 712 sizeof(lpm_tree->prefix_usage)); 713 memset(&lpm_tree->prefix_ref_count, 0, 714 sizeof(lpm_tree->prefix_ref_count)); 715 lpm_tree->ref_count = 1; 716 return lpm_tree; 717 718err_left_struct_set: 719 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 720 return ERR_PTR(err); 721} 722 723static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 724 const struct mlxsw_sp_router_ll_ops *ll_ops, 725 struct mlxsw_sp_lpm_tree *lpm_tree) 726{ 727 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 728} 729 730static struct mlxsw_sp_lpm_tree * 731mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 732 struct mlxsw_sp_prefix_usage *prefix_usage, 733 enum mlxsw_sp_l3proto proto) 734{ 735 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 736 struct mlxsw_sp_lpm_tree *lpm_tree; 737 int i; 738 739 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 740 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 741 if (lpm_tree->ref_count != 0 && 742 lpm_tree->proto == proto && 743 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 744 prefix_usage)) { 745 mlxsw_sp_lpm_tree_hold(lpm_tree); 746 return lpm_tree; 747 } 748 } 749 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 750} 751 752static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 753{ 754 lpm_tree->ref_count++; 755} 756 757static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 758 struct mlxsw_sp_lpm_tree *lpm_tree) 759{ 760 const struct mlxsw_sp_router_ll_ops *ll_ops = 761 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 762 763 if (--lpm_tree->ref_count == 0) 764 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 765} 766 767#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 768 769static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 770{ 771 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 772 struct mlxsw_sp_lpm_tree *lpm_tree; 773 u64 max_trees; 774 int err, i; 775 776 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 777 return -EIO; 778 779 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 780 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 781 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 782 sizeof(struct mlxsw_sp_lpm_tree), 783 GFP_KERNEL); 784 if (!mlxsw_sp->router->lpm.trees) 785 return -ENOMEM; 786 787 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 788 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 789 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 790 } 791 792 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 793 MLXSW_SP_L3_PROTO_IPV4); 794 if (IS_ERR(lpm_tree)) { 795 err = PTR_ERR(lpm_tree); 796 goto err_ipv4_tree_get; 797 } 798 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 799 800 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 801 MLXSW_SP_L3_PROTO_IPV6); 802 if (IS_ERR(lpm_tree)) { 803 err = PTR_ERR(lpm_tree); 804 goto err_ipv6_tree_get; 805 } 806 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 807 808 return 0; 809 810err_ipv6_tree_get: 811 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 812 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 813err_ipv4_tree_get: 814 kfree(mlxsw_sp->router->lpm.trees); 815 return err; 816} 817 818static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 819{ 820 struct mlxsw_sp_lpm_tree *lpm_tree; 821 822 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 823 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 824 825 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 826 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 827 828 kfree(mlxsw_sp->router->lpm.trees); 829} 830 831static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 832{ 833 return !!vr->fib4 || !!vr->fib6 || 834 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 835 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 836} 837 838static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 839{ 840 struct mlxsw_sp_vr *vr; 841 int i; 842 843 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 844 vr = &mlxsw_sp->router->vrs[i]; 845 if (!mlxsw_sp_vr_is_used(vr)) 846 return vr; 847 } 848 return NULL; 849} 850 851static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 852 const struct mlxsw_sp_fib *fib, u8 tree_id) 853{ 854 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 855 856 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 857 (enum mlxsw_reg_ralxx_protocol) fib->proto, 858 tree_id); 859 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 860} 861 862static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 863 const struct mlxsw_sp_fib *fib) 864{ 865 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 866 867 /* Bind to tree 0 which is default */ 868 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 869 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 870 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 871} 872 873static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 874{ 875 /* For our purpose, squash main, default and local tables into one */ 876 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 877 tb_id = RT_TABLE_MAIN; 878 return tb_id; 879} 880 881static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 882 u32 tb_id) 883{ 884 struct mlxsw_sp_vr *vr; 885 int i; 886 887 tb_id = mlxsw_sp_fix_tb_id(tb_id); 888 889 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 890 vr = &mlxsw_sp->router->vrs[i]; 891 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 892 return vr; 893 } 894 return NULL; 895} 896 897int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 898 u16 *vr_id) 899{ 900 struct mlxsw_sp_vr *vr; 901 int err = 0; 902 903 mutex_lock(&mlxsw_sp->router->lock); 904 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 905 if (!vr) { 906 err = -ESRCH; 907 goto out; 908 } 909 *vr_id = vr->id; 910out: 911 mutex_unlock(&mlxsw_sp->router->lock); 912 return err; 913} 914 915static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 916 enum mlxsw_sp_l3proto proto) 917{ 918 switch (proto) { 919 case MLXSW_SP_L3_PROTO_IPV4: 920 return vr->fib4; 921 case MLXSW_SP_L3_PROTO_IPV6: 922 return vr->fib6; 923 } 924 return NULL; 925} 926 927static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 928 u32 tb_id, 929 struct netlink_ext_ack *extack) 930{ 931 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 932 struct mlxsw_sp_fib *fib4; 933 struct mlxsw_sp_fib *fib6; 934 struct mlxsw_sp_vr *vr; 935 int err; 936 937 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 938 if (!vr) { 939 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 940 return ERR_PTR(-EBUSY); 941 } 942 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 943 if (IS_ERR(fib4)) 944 return ERR_CAST(fib4); 945 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 946 if (IS_ERR(fib6)) { 947 err = PTR_ERR(fib6); 948 goto err_fib6_create; 949 } 950 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 951 MLXSW_SP_L3_PROTO_IPV4); 952 if (IS_ERR(mr4_table)) { 953 err = PTR_ERR(mr4_table); 954 goto err_mr4_table_create; 955 } 956 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 957 MLXSW_SP_L3_PROTO_IPV6); 958 if (IS_ERR(mr6_table)) { 959 err = PTR_ERR(mr6_table); 960 goto err_mr6_table_create; 961 } 962 963 vr->fib4 = fib4; 964 vr->fib6 = fib6; 965 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 966 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 967 vr->tb_id = tb_id; 968 return vr; 969 970err_mr6_table_create: 971 mlxsw_sp_mr_table_destroy(mr4_table); 972err_mr4_table_create: 973 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 974err_fib6_create: 975 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 976 return ERR_PTR(err); 977} 978 979static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 980 struct mlxsw_sp_vr *vr) 981{ 982 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 983 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 984 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 985 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 986 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 987 vr->fib6 = NULL; 988 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 989 vr->fib4 = NULL; 990} 991 992static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 993 struct netlink_ext_ack *extack) 994{ 995 struct mlxsw_sp_vr *vr; 996 997 tb_id = mlxsw_sp_fix_tb_id(tb_id); 998 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 999 if (!vr) 1000 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 1001 return vr; 1002} 1003 1004static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 1005{ 1006 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 1007 list_empty(&vr->fib6->node_list) && 1008 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 1009 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 1010 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 1011} 1012 1013static bool 1014mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 1015 enum mlxsw_sp_l3proto proto, u8 tree_id) 1016{ 1017 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 1018 1019 if (!mlxsw_sp_vr_is_used(vr)) 1020 return false; 1021 if (fib->lpm_tree->id == tree_id) 1022 return true; 1023 return false; 1024} 1025 1026static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 1027 struct mlxsw_sp_fib *fib, 1028 struct mlxsw_sp_lpm_tree *new_tree) 1029{ 1030 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 1031 int err; 1032 1033 fib->lpm_tree = new_tree; 1034 mlxsw_sp_lpm_tree_hold(new_tree); 1035 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 1036 if (err) 1037 goto err_tree_bind; 1038 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1039 return 0; 1040 1041err_tree_bind: 1042 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 1043 fib->lpm_tree = old_tree; 1044 return err; 1045} 1046 1047static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 1048 struct mlxsw_sp_fib *fib, 1049 struct mlxsw_sp_lpm_tree *new_tree) 1050{ 1051 enum mlxsw_sp_l3proto proto = fib->proto; 1052 struct mlxsw_sp_lpm_tree *old_tree; 1053 u8 old_id, new_id = new_tree->id; 1054 struct mlxsw_sp_vr *vr; 1055 int i, err; 1056 1057 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 1058 old_id = old_tree->id; 1059 1060 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 1061 vr = &mlxsw_sp->router->vrs[i]; 1062 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 1063 continue; 1064 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1065 mlxsw_sp_vr_fib(vr, proto), 1066 new_tree); 1067 if (err) 1068 goto err_tree_replace; 1069 } 1070 1071 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1072 sizeof(new_tree->prefix_ref_count)); 1073 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1074 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1075 1076 return 0; 1077 1078err_tree_replace: 1079 for (i--; i >= 0; i--) { 1080 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1081 continue; 1082 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1083 mlxsw_sp_vr_fib(vr, proto), 1084 old_tree); 1085 } 1086 return err; 1087} 1088 1089static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1090{ 1091 struct mlxsw_sp_vr *vr; 1092 u64 max_vrs; 1093 int i; 1094 1095 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1096 return -EIO; 1097 1098 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1099 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1100 GFP_KERNEL); 1101 if (!mlxsw_sp->router->vrs) 1102 return -ENOMEM; 1103 1104 for (i = 0; i < max_vrs; i++) { 1105 vr = &mlxsw_sp->router->vrs[i]; 1106 vr->id = i; 1107 } 1108 1109 return 0; 1110} 1111 1112static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1113 1114static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1115{ 1116 /* At this stage we're guaranteed not to have new incoming 1117 * FIB notifications and the work queue is free from FIBs 1118 * sitting on top of mlxsw netdevs. However, we can still 1119 * have other FIBs queued. Flush the queue before flushing 1120 * the device's tables. No need for locks, as we're the only 1121 * writer. 1122 */ 1123 mlxsw_core_flush_owq(); 1124 mlxsw_sp_router_fib_flush(mlxsw_sp); 1125 kfree(mlxsw_sp->router->vrs); 1126} 1127 1128u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1129{ 1130 struct net_device *d; 1131 u32 tb_id; 1132 1133 rcu_read_lock(); 1134 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1135 if (d) 1136 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1137 else 1138 tb_id = RT_TABLE_MAIN; 1139 rcu_read_unlock(); 1140 1141 return tb_id; 1142} 1143 1144static struct mlxsw_sp_rif * 1145mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1146 const struct mlxsw_sp_rif_params *params, 1147 struct netlink_ext_ack *extack); 1148 1149static struct mlxsw_sp_rif_ipip_lb * 1150mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1151 enum mlxsw_sp_ipip_type ipipt, 1152 struct net_device *ol_dev, 1153 struct netlink_ext_ack *extack) 1154{ 1155 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1156 const struct mlxsw_sp_ipip_ops *ipip_ops; 1157 struct mlxsw_sp_rif *rif; 1158 1159 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1160 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1161 .common.dev = ol_dev, 1162 .common.lag = false, 1163 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1164 }; 1165 1166 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1167 if (IS_ERR(rif)) 1168 return ERR_CAST(rif); 1169 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1170} 1171 1172static struct mlxsw_sp_ipip_entry * 1173mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1174 enum mlxsw_sp_ipip_type ipipt, 1175 struct net_device *ol_dev) 1176{ 1177 const struct mlxsw_sp_ipip_ops *ipip_ops; 1178 struct mlxsw_sp_ipip_entry *ipip_entry; 1179 struct mlxsw_sp_ipip_entry *ret = NULL; 1180 int err; 1181 1182 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1183 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1184 if (!ipip_entry) 1185 return ERR_PTR(-ENOMEM); 1186 1187 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1188 ol_dev, NULL); 1189 if (IS_ERR(ipip_entry->ol_lb)) { 1190 ret = ERR_CAST(ipip_entry->ol_lb); 1191 goto err_ol_ipip_lb_create; 1192 } 1193 1194 ipip_entry->ipipt = ipipt; 1195 ipip_entry->ol_dev = ol_dev; 1196 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1197 1198 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1199 if (err) { 1200 ret = ERR_PTR(err); 1201 goto err_rem_ip_addr_set; 1202 } 1203 1204 return ipip_entry; 1205 1206err_rem_ip_addr_set: 1207 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1208err_ol_ipip_lb_create: 1209 kfree(ipip_entry); 1210 return ret; 1211} 1212 1213static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1214 struct mlxsw_sp_ipip_entry *ipip_entry) 1215{ 1216 const struct mlxsw_sp_ipip_ops *ipip_ops = 1217 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1218 1219 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1220 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1221 kfree(ipip_entry); 1222} 1223 1224static bool 1225mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1226 const enum mlxsw_sp_l3proto ul_proto, 1227 union mlxsw_sp_l3addr saddr, 1228 u32 ul_tb_id, 1229 struct mlxsw_sp_ipip_entry *ipip_entry) 1230{ 1231 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1232 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1233 union mlxsw_sp_l3addr tun_saddr; 1234 1235 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1236 return false; 1237 1238 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1239 return tun_ul_tb_id == ul_tb_id && 1240 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1241} 1242 1243static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1244 enum mlxsw_sp_ipip_type ipipt) 1245{ 1246 const struct mlxsw_sp_ipip_ops *ipip_ops; 1247 1248 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1249 1250 /* Not all tunnels require to increase the default pasing depth 1251 * (96 bytes). 1252 */ 1253 if (ipip_ops->inc_parsing_depth) 1254 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1255 1256 return 0; 1257} 1258 1259static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1260 enum mlxsw_sp_ipip_type ipipt) 1261{ 1262 const struct mlxsw_sp_ipip_ops *ipip_ops = 1263 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1264 1265 if (ipip_ops->inc_parsing_depth) 1266 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1267} 1268 1269static int 1270mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1271 struct mlxsw_sp_fib_entry *fib_entry, 1272 struct mlxsw_sp_ipip_entry *ipip_entry) 1273{ 1274 u32 tunnel_index; 1275 int err; 1276 1277 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1278 1, &tunnel_index); 1279 if (err) 1280 return err; 1281 1282 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1283 ipip_entry->ipipt); 1284 if (err) 1285 goto err_parsing_depth_inc; 1286 1287 ipip_entry->decap_fib_entry = fib_entry; 1288 fib_entry->decap.ipip_entry = ipip_entry; 1289 fib_entry->decap.tunnel_index = tunnel_index; 1290 1291 return 0; 1292 1293err_parsing_depth_inc: 1294 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1295 fib_entry->decap.tunnel_index); 1296 return err; 1297} 1298 1299static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1300 struct mlxsw_sp_fib_entry *fib_entry) 1301{ 1302 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1303 1304 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1305 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1306 fib_entry->decap.ipip_entry = NULL; 1307 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1308 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1309 1, fib_entry->decap.tunnel_index); 1310} 1311 1312static struct mlxsw_sp_fib_node * 1313mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1314 size_t addr_len, unsigned char prefix_len); 1315static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1316 struct mlxsw_sp_fib_entry *fib_entry); 1317 1318static void 1319mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1320 struct mlxsw_sp_ipip_entry *ipip_entry) 1321{ 1322 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1323 1324 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1325 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1326 1327 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1328} 1329 1330static void 1331mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1332 struct mlxsw_sp_ipip_entry *ipip_entry, 1333 struct mlxsw_sp_fib_entry *decap_fib_entry) 1334{ 1335 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1336 ipip_entry)) 1337 return; 1338 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1339 1340 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1341 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1342} 1343 1344static struct mlxsw_sp_fib_entry * 1345mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1346 enum mlxsw_sp_l3proto proto, 1347 const union mlxsw_sp_l3addr *addr, 1348 enum mlxsw_sp_fib_entry_type type) 1349{ 1350 struct mlxsw_sp_fib_node *fib_node; 1351 unsigned char addr_prefix_len; 1352 struct mlxsw_sp_fib *fib; 1353 struct mlxsw_sp_vr *vr; 1354 const void *addrp; 1355 size_t addr_len; 1356 u32 addr4; 1357 1358 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1359 if (!vr) 1360 return NULL; 1361 fib = mlxsw_sp_vr_fib(vr, proto); 1362 1363 switch (proto) { 1364 case MLXSW_SP_L3_PROTO_IPV4: 1365 addr4 = be32_to_cpu(addr->addr4); 1366 addrp = &addr4; 1367 addr_len = 4; 1368 addr_prefix_len = 32; 1369 break; 1370 case MLXSW_SP_L3_PROTO_IPV6: 1371 addrp = &addr->addr6; 1372 addr_len = 16; 1373 addr_prefix_len = 128; 1374 break; 1375 default: 1376 WARN_ON(1); 1377 return NULL; 1378 } 1379 1380 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1381 addr_prefix_len); 1382 if (!fib_node || fib_node->fib_entry->type != type) 1383 return NULL; 1384 1385 return fib_node->fib_entry; 1386} 1387 1388/* Given an IPIP entry, find the corresponding decap route. */ 1389static struct mlxsw_sp_fib_entry * 1390mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1391 struct mlxsw_sp_ipip_entry *ipip_entry) 1392{ 1393 static struct mlxsw_sp_fib_node *fib_node; 1394 const struct mlxsw_sp_ipip_ops *ipip_ops; 1395 unsigned char saddr_prefix_len; 1396 union mlxsw_sp_l3addr saddr; 1397 struct mlxsw_sp_fib *ul_fib; 1398 struct mlxsw_sp_vr *ul_vr; 1399 const void *saddrp; 1400 size_t saddr_len; 1401 u32 ul_tb_id; 1402 u32 saddr4; 1403 1404 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1405 1406 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1407 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1408 if (!ul_vr) 1409 return NULL; 1410 1411 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1412 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1413 ipip_entry->ol_dev); 1414 1415 switch (ipip_ops->ul_proto) { 1416 case MLXSW_SP_L3_PROTO_IPV4: 1417 saddr4 = be32_to_cpu(saddr.addr4); 1418 saddrp = &saddr4; 1419 saddr_len = 4; 1420 saddr_prefix_len = 32; 1421 break; 1422 case MLXSW_SP_L3_PROTO_IPV6: 1423 saddrp = &saddr.addr6; 1424 saddr_len = 16; 1425 saddr_prefix_len = 128; 1426 break; 1427 default: 1428 WARN_ON(1); 1429 return NULL; 1430 } 1431 1432 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1433 saddr_prefix_len); 1434 if (!fib_node || 1435 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1436 return NULL; 1437 1438 return fib_node->fib_entry; 1439} 1440 1441static struct mlxsw_sp_ipip_entry * 1442mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1443 enum mlxsw_sp_ipip_type ipipt, 1444 struct net_device *ol_dev) 1445{ 1446 struct mlxsw_sp_ipip_entry *ipip_entry; 1447 1448 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1449 if (IS_ERR(ipip_entry)) 1450 return ipip_entry; 1451 1452 list_add_tail(&ipip_entry->ipip_list_node, 1453 &mlxsw_sp->router->ipip_list); 1454 1455 return ipip_entry; 1456} 1457 1458static void 1459mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1460 struct mlxsw_sp_ipip_entry *ipip_entry) 1461{ 1462 list_del(&ipip_entry->ipip_list_node); 1463 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1464} 1465 1466static bool 1467mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1468 const struct net_device *ul_dev, 1469 enum mlxsw_sp_l3proto ul_proto, 1470 union mlxsw_sp_l3addr ul_dip, 1471 struct mlxsw_sp_ipip_entry *ipip_entry) 1472{ 1473 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1474 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1475 1476 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1477 return false; 1478 1479 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1480 ul_tb_id, ipip_entry); 1481} 1482 1483/* Given decap parameters, find the corresponding IPIP entry. */ 1484static struct mlxsw_sp_ipip_entry * 1485mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1486 enum mlxsw_sp_l3proto ul_proto, 1487 union mlxsw_sp_l3addr ul_dip) 1488{ 1489 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1490 struct net_device *ul_dev; 1491 1492 rcu_read_lock(); 1493 1494 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1495 if (!ul_dev) 1496 goto out_unlock; 1497 1498 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1499 ipip_list_node) 1500 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1501 ul_proto, ul_dip, 1502 ipip_entry)) 1503 goto out_unlock; 1504 1505 rcu_read_unlock(); 1506 1507 return NULL; 1508 1509out_unlock: 1510 rcu_read_unlock(); 1511 return ipip_entry; 1512} 1513 1514static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1515 const struct net_device *dev, 1516 enum mlxsw_sp_ipip_type *p_type) 1517{ 1518 struct mlxsw_sp_router *router = mlxsw_sp->router; 1519 const struct mlxsw_sp_ipip_ops *ipip_ops; 1520 enum mlxsw_sp_ipip_type ipipt; 1521 1522 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1523 ipip_ops = router->ipip_ops_arr[ipipt]; 1524 if (dev->type == ipip_ops->dev_type) { 1525 if (p_type) 1526 *p_type = ipipt; 1527 return true; 1528 } 1529 } 1530 return false; 1531} 1532 1533static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1534 const struct net_device *dev) 1535{ 1536 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1537} 1538 1539static struct mlxsw_sp_ipip_entry * 1540mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1541 const struct net_device *ol_dev) 1542{ 1543 struct mlxsw_sp_ipip_entry *ipip_entry; 1544 1545 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1546 ipip_list_node) 1547 if (ipip_entry->ol_dev == ol_dev) 1548 return ipip_entry; 1549 1550 return NULL; 1551} 1552 1553static struct mlxsw_sp_ipip_entry * 1554mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1555 const struct net_device *ul_dev, 1556 struct mlxsw_sp_ipip_entry *start) 1557{ 1558 struct mlxsw_sp_ipip_entry *ipip_entry; 1559 1560 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1561 ipip_list_node); 1562 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1563 ipip_list_node) { 1564 struct net_device *ol_dev = ipip_entry->ol_dev; 1565 struct net_device *ipip_ul_dev; 1566 1567 rcu_read_lock(); 1568 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1569 rcu_read_unlock(); 1570 1571 if (ipip_ul_dev == ul_dev) 1572 return ipip_entry; 1573 } 1574 1575 return NULL; 1576} 1577 1578static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1579 const struct net_device *dev) 1580{ 1581 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1582} 1583 1584static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1585 const struct net_device *ol_dev, 1586 enum mlxsw_sp_ipip_type ipipt) 1587{ 1588 const struct mlxsw_sp_ipip_ops *ops 1589 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1590 1591 return ops->can_offload(mlxsw_sp, ol_dev); 1592} 1593 1594static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1595 struct net_device *ol_dev) 1596{ 1597 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1598 struct mlxsw_sp_ipip_entry *ipip_entry; 1599 enum mlxsw_sp_l3proto ul_proto; 1600 union mlxsw_sp_l3addr saddr; 1601 u32 ul_tb_id; 1602 1603 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1604 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1605 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1606 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1607 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1608 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1609 saddr, ul_tb_id, 1610 NULL)) { 1611 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1612 ol_dev); 1613 if (IS_ERR(ipip_entry)) 1614 return PTR_ERR(ipip_entry); 1615 } 1616 } 1617 1618 return 0; 1619} 1620 1621static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1622 struct net_device *ol_dev) 1623{ 1624 struct mlxsw_sp_ipip_entry *ipip_entry; 1625 1626 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1627 if (ipip_entry) 1628 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1629} 1630 1631static void 1632mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1633 struct mlxsw_sp_ipip_entry *ipip_entry) 1634{ 1635 struct mlxsw_sp_fib_entry *decap_fib_entry; 1636 1637 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1638 if (decap_fib_entry) 1639 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1640 decap_fib_entry); 1641} 1642 1643static int 1644mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1645 u16 ul_rif_id, bool enable) 1646{ 1647 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1648 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1649 struct mlxsw_sp_rif *rif = &lb_rif->common; 1650 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1651 char ritr_pl[MLXSW_REG_RITR_LEN]; 1652 struct in6_addr *saddr6; 1653 u32 saddr4; 1654 1655 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1656 switch (lb_cf.ul_protocol) { 1657 case MLXSW_SP_L3_PROTO_IPV4: 1658 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1659 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1660 rif->rif_index, rif->vr_id, rif->dev->mtu); 1661 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1662 ipip_options, ul_vr_id, 1663 ul_rif_id, saddr4, 1664 lb_cf.okey); 1665 break; 1666 1667 case MLXSW_SP_L3_PROTO_IPV6: 1668 saddr6 = &lb_cf.saddr.addr6; 1669 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1670 rif->rif_index, rif->vr_id, rif->dev->mtu); 1671 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1672 ipip_options, ul_vr_id, 1673 ul_rif_id, saddr6, 1674 lb_cf.okey); 1675 break; 1676 } 1677 1678 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1679} 1680 1681static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1682 struct net_device *ol_dev) 1683{ 1684 struct mlxsw_sp_ipip_entry *ipip_entry; 1685 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1686 int err = 0; 1687 1688 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1689 if (ipip_entry) { 1690 lb_rif = ipip_entry->ol_lb; 1691 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1692 lb_rif->ul_rif_id, true); 1693 if (err) 1694 goto out; 1695 lb_rif->common.mtu = ol_dev->mtu; 1696 } 1697 1698out: 1699 return err; 1700} 1701 1702static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1703 struct net_device *ol_dev) 1704{ 1705 struct mlxsw_sp_ipip_entry *ipip_entry; 1706 1707 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1708 if (ipip_entry) 1709 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1710} 1711 1712static void 1713mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1714 struct mlxsw_sp_ipip_entry *ipip_entry) 1715{ 1716 if (ipip_entry->decap_fib_entry) 1717 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1718} 1719 1720static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1721 struct net_device *ol_dev) 1722{ 1723 struct mlxsw_sp_ipip_entry *ipip_entry; 1724 1725 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1726 if (ipip_entry) 1727 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1728} 1729 1730static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1731 struct mlxsw_sp_rif *old_rif, 1732 struct mlxsw_sp_rif *new_rif); 1733static int 1734mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1735 struct mlxsw_sp_ipip_entry *ipip_entry, 1736 bool keep_encap, 1737 struct netlink_ext_ack *extack) 1738{ 1739 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1740 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1741 1742 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1743 ipip_entry->ipipt, 1744 ipip_entry->ol_dev, 1745 extack); 1746 if (IS_ERR(new_lb_rif)) 1747 return PTR_ERR(new_lb_rif); 1748 ipip_entry->ol_lb = new_lb_rif; 1749 1750 if (keep_encap) 1751 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1752 &new_lb_rif->common); 1753 1754 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1755 1756 return 0; 1757} 1758 1759static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1760 struct mlxsw_sp_rif *rif); 1761 1762/** 1763 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1764 * @mlxsw_sp: mlxsw_sp. 1765 * @ipip_entry: IPIP entry. 1766 * @recreate_loopback: Recreates the associated loopback RIF. 1767 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1768 * relevant when recreate_loopback is true. 1769 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1770 * is only relevant when recreate_loopback is false. 1771 * @extack: extack. 1772 * 1773 * Return: Non-zero value on failure. 1774 */ 1775int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1776 struct mlxsw_sp_ipip_entry *ipip_entry, 1777 bool recreate_loopback, 1778 bool keep_encap, 1779 bool update_nexthops, 1780 struct netlink_ext_ack *extack) 1781{ 1782 int err; 1783 1784 /* RIFs can't be edited, so to update loopback, we need to destroy and 1785 * recreate it. That creates a window of opportunity where RALUE and 1786 * RATR registers end up referencing a RIF that's already gone. RATRs 1787 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1788 * of RALUE, demote the decap route back. 1789 */ 1790 if (ipip_entry->decap_fib_entry) 1791 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1792 1793 if (recreate_loopback) { 1794 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1795 keep_encap, extack); 1796 if (err) 1797 return err; 1798 } else if (update_nexthops) { 1799 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1800 &ipip_entry->ol_lb->common); 1801 } 1802 1803 if (ipip_entry->ol_dev->flags & IFF_UP) 1804 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1805 1806 return 0; 1807} 1808 1809static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1810 struct net_device *ol_dev, 1811 struct netlink_ext_ack *extack) 1812{ 1813 struct mlxsw_sp_ipip_entry *ipip_entry = 1814 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1815 1816 if (!ipip_entry) 1817 return 0; 1818 1819 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1820 true, false, false, extack); 1821} 1822 1823static int 1824mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1825 struct mlxsw_sp_ipip_entry *ipip_entry, 1826 struct net_device *ul_dev, 1827 bool *demote_this, 1828 struct netlink_ext_ack *extack) 1829{ 1830 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1831 enum mlxsw_sp_l3proto ul_proto; 1832 union mlxsw_sp_l3addr saddr; 1833 1834 /* Moving underlay to a different VRF might cause local address 1835 * conflict, and the conflicting tunnels need to be demoted. 1836 */ 1837 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1838 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1839 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1840 saddr, ul_tb_id, 1841 ipip_entry)) { 1842 *demote_this = true; 1843 return 0; 1844 } 1845 1846 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1847 true, true, false, extack); 1848} 1849 1850static int 1851mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1852 struct mlxsw_sp_ipip_entry *ipip_entry, 1853 struct net_device *ul_dev) 1854{ 1855 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1856 false, false, true, NULL); 1857} 1858 1859static int 1860mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1861 struct mlxsw_sp_ipip_entry *ipip_entry, 1862 struct net_device *ul_dev) 1863{ 1864 /* A down underlay device causes encapsulated packets to not be 1865 * forwarded, but decap still works. So refresh next hops without 1866 * touching anything else. 1867 */ 1868 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1869 false, false, true, NULL); 1870} 1871 1872static int 1873mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1874 struct net_device *ol_dev, 1875 struct netlink_ext_ack *extack) 1876{ 1877 const struct mlxsw_sp_ipip_ops *ipip_ops; 1878 struct mlxsw_sp_ipip_entry *ipip_entry; 1879 int err; 1880 1881 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1882 if (!ipip_entry) 1883 /* A change might make a tunnel eligible for offloading, but 1884 * that is currently not implemented. What falls to slow path 1885 * stays there. 1886 */ 1887 return 0; 1888 1889 /* A change might make a tunnel not eligible for offloading. */ 1890 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1891 ipip_entry->ipipt)) { 1892 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1893 return 0; 1894 } 1895 1896 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1897 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1898 return err; 1899} 1900 1901void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1902 struct mlxsw_sp_ipip_entry *ipip_entry) 1903{ 1904 struct net_device *ol_dev = ipip_entry->ol_dev; 1905 1906 if (ol_dev->flags & IFF_UP) 1907 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1908 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1909} 1910 1911/* The configuration where several tunnels have the same local address in the 1912 * same underlay table needs special treatment in the HW. That is currently not 1913 * implemented in the driver. This function finds and demotes the first tunnel 1914 * with a given source address, except the one passed in in the argument 1915 * `except'. 1916 */ 1917bool 1918mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1919 enum mlxsw_sp_l3proto ul_proto, 1920 union mlxsw_sp_l3addr saddr, 1921 u32 ul_tb_id, 1922 const struct mlxsw_sp_ipip_entry *except) 1923{ 1924 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1925 1926 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1927 ipip_list_node) { 1928 if (ipip_entry != except && 1929 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1930 ul_tb_id, ipip_entry)) { 1931 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1932 return true; 1933 } 1934 } 1935 1936 return false; 1937} 1938 1939static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1940 struct net_device *ul_dev) 1941{ 1942 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1943 1944 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1945 ipip_list_node) { 1946 struct net_device *ol_dev = ipip_entry->ol_dev; 1947 struct net_device *ipip_ul_dev; 1948 1949 rcu_read_lock(); 1950 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1951 rcu_read_unlock(); 1952 if (ipip_ul_dev == ul_dev) 1953 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1954 } 1955} 1956 1957static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1958 struct net_device *ol_dev, 1959 unsigned long event, 1960 struct netdev_notifier_info *info) 1961{ 1962 struct netdev_notifier_changeupper_info *chup; 1963 struct netlink_ext_ack *extack; 1964 int err = 0; 1965 1966 switch (event) { 1967 case NETDEV_REGISTER: 1968 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1969 break; 1970 case NETDEV_UNREGISTER: 1971 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1972 break; 1973 case NETDEV_UP: 1974 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1975 break; 1976 case NETDEV_DOWN: 1977 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1978 break; 1979 case NETDEV_CHANGEUPPER: 1980 chup = container_of(info, typeof(*chup), info); 1981 extack = info->extack; 1982 if (netif_is_l3_master(chup->upper_dev)) 1983 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1984 ol_dev, 1985 extack); 1986 break; 1987 case NETDEV_CHANGE: 1988 extack = info->extack; 1989 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1990 ol_dev, extack); 1991 break; 1992 case NETDEV_CHANGEMTU: 1993 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1994 break; 1995 } 1996 return err; 1997} 1998 1999static int 2000__mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2001 struct mlxsw_sp_ipip_entry *ipip_entry, 2002 struct net_device *ul_dev, 2003 bool *demote_this, 2004 unsigned long event, 2005 struct netdev_notifier_info *info) 2006{ 2007 struct netdev_notifier_changeupper_info *chup; 2008 struct netlink_ext_ack *extack; 2009 2010 switch (event) { 2011 case NETDEV_CHANGEUPPER: 2012 chup = container_of(info, typeof(*chup), info); 2013 extack = info->extack; 2014 if (netif_is_l3_master(chup->upper_dev)) 2015 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 2016 ipip_entry, 2017 ul_dev, 2018 demote_this, 2019 extack); 2020 break; 2021 2022 case NETDEV_UP: 2023 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 2024 ul_dev); 2025 case NETDEV_DOWN: 2026 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 2027 ipip_entry, 2028 ul_dev); 2029 } 2030 return 0; 2031} 2032 2033static int 2034mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2035 struct net_device *ul_dev, 2036 unsigned long event, 2037 struct netdev_notifier_info *info) 2038{ 2039 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 2040 int err; 2041 2042 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 2043 ul_dev, 2044 ipip_entry))) { 2045 struct mlxsw_sp_ipip_entry *prev; 2046 bool demote_this = false; 2047 2048 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 2049 ul_dev, &demote_this, 2050 event, info); 2051 if (err) { 2052 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 2053 ul_dev); 2054 return err; 2055 } 2056 2057 if (demote_this) { 2058 if (list_is_first(&ipip_entry->ipip_list_node, 2059 &mlxsw_sp->router->ipip_list)) 2060 prev = NULL; 2061 else 2062 /* This can't be cached from previous iteration, 2063 * because that entry could be gone now. 2064 */ 2065 prev = list_prev_entry(ipip_entry, 2066 ipip_list_node); 2067 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 2068 ipip_entry = prev; 2069 } 2070 } 2071 2072 return 0; 2073} 2074 2075int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2076 enum mlxsw_sp_l3proto ul_proto, 2077 const union mlxsw_sp_l3addr *ul_sip, 2078 u32 tunnel_index) 2079{ 2080 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2081 struct mlxsw_sp_router *router = mlxsw_sp->router; 2082 struct mlxsw_sp_fib_entry *fib_entry; 2083 int err = 0; 2084 2085 mutex_lock(&mlxsw_sp->router->lock); 2086 2087 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 2088 err = -EINVAL; 2089 goto out; 2090 } 2091 2092 router->nve_decap_config.ul_tb_id = ul_tb_id; 2093 router->nve_decap_config.tunnel_index = tunnel_index; 2094 router->nve_decap_config.ul_proto = ul_proto; 2095 router->nve_decap_config.ul_sip = *ul_sip; 2096 router->nve_decap_config.valid = true; 2097 2098 /* It is valid to create a tunnel with a local IP and only later 2099 * assign this IP address to a local interface 2100 */ 2101 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2102 ul_proto, ul_sip, 2103 type); 2104 if (!fib_entry) 2105 goto out; 2106 2107 fib_entry->decap.tunnel_index = tunnel_index; 2108 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2109 2110 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2111 if (err) 2112 goto err_fib_entry_update; 2113 2114 goto out; 2115 2116err_fib_entry_update: 2117 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2118 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2119out: 2120 mutex_unlock(&mlxsw_sp->router->lock); 2121 return err; 2122} 2123 2124void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2125 enum mlxsw_sp_l3proto ul_proto, 2126 const union mlxsw_sp_l3addr *ul_sip) 2127{ 2128 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2129 struct mlxsw_sp_router *router = mlxsw_sp->router; 2130 struct mlxsw_sp_fib_entry *fib_entry; 2131 2132 mutex_lock(&mlxsw_sp->router->lock); 2133 2134 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2135 goto out; 2136 2137 router->nve_decap_config.valid = false; 2138 2139 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2140 ul_proto, ul_sip, 2141 type); 2142 if (!fib_entry) 2143 goto out; 2144 2145 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2146 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2147out: 2148 mutex_unlock(&mlxsw_sp->router->lock); 2149} 2150 2151static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2152 u32 ul_tb_id, 2153 enum mlxsw_sp_l3proto ul_proto, 2154 const union mlxsw_sp_l3addr *ul_sip) 2155{ 2156 struct mlxsw_sp_router *router = mlxsw_sp->router; 2157 2158 return router->nve_decap_config.valid && 2159 router->nve_decap_config.ul_tb_id == ul_tb_id && 2160 router->nve_decap_config.ul_proto == ul_proto && 2161 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2162 sizeof(*ul_sip)); 2163} 2164 2165struct mlxsw_sp_neigh_key { 2166 struct neighbour *n; 2167}; 2168 2169struct mlxsw_sp_neigh_entry { 2170 struct list_head rif_list_node; 2171 struct rhash_head ht_node; 2172 struct mlxsw_sp_neigh_key key; 2173 u16 rif; 2174 bool connected; 2175 unsigned char ha[ETH_ALEN]; 2176 struct list_head nexthop_list; /* list of nexthops using 2177 * this neigh entry 2178 */ 2179 struct list_head nexthop_neighs_list_node; 2180 unsigned int counter_index; 2181 bool counter_valid; 2182}; 2183 2184static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2185 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2186 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2187 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2188}; 2189 2190struct mlxsw_sp_neigh_entry * 2191mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2192 struct mlxsw_sp_neigh_entry *neigh_entry) 2193{ 2194 if (!neigh_entry) { 2195 if (list_empty(&rif->neigh_list)) 2196 return NULL; 2197 else 2198 return list_first_entry(&rif->neigh_list, 2199 typeof(*neigh_entry), 2200 rif_list_node); 2201 } 2202 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2203 return NULL; 2204 return list_next_entry(neigh_entry, rif_list_node); 2205} 2206 2207int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2208{ 2209 return neigh_entry->key.n->tbl->family; 2210} 2211 2212unsigned char * 2213mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2214{ 2215 return neigh_entry->ha; 2216} 2217 2218u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2219{ 2220 struct neighbour *n; 2221 2222 n = neigh_entry->key.n; 2223 return ntohl(*((__be32 *) n->primary_key)); 2224} 2225 2226struct in6_addr * 2227mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2228{ 2229 struct neighbour *n; 2230 2231 n = neigh_entry->key.n; 2232 return (struct in6_addr *) &n->primary_key; 2233} 2234 2235int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2236 struct mlxsw_sp_neigh_entry *neigh_entry, 2237 u64 *p_counter) 2238{ 2239 if (!neigh_entry->counter_valid) 2240 return -EINVAL; 2241 2242 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2243 p_counter, NULL); 2244} 2245 2246static struct mlxsw_sp_neigh_entry * 2247mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2248 u16 rif) 2249{ 2250 struct mlxsw_sp_neigh_entry *neigh_entry; 2251 2252 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2253 if (!neigh_entry) 2254 return NULL; 2255 2256 neigh_entry->key.n = n; 2257 neigh_entry->rif = rif; 2258 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2259 2260 return neigh_entry; 2261} 2262 2263static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2264{ 2265 kfree(neigh_entry); 2266} 2267 2268static int 2269mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2270 struct mlxsw_sp_neigh_entry *neigh_entry) 2271{ 2272 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2273 &neigh_entry->ht_node, 2274 mlxsw_sp_neigh_ht_params); 2275} 2276 2277static void 2278mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2279 struct mlxsw_sp_neigh_entry *neigh_entry) 2280{ 2281 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2282 &neigh_entry->ht_node, 2283 mlxsw_sp_neigh_ht_params); 2284} 2285 2286static bool 2287mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2288 struct mlxsw_sp_neigh_entry *neigh_entry) 2289{ 2290 struct devlink *devlink; 2291 const char *table_name; 2292 2293 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2294 case AF_INET: 2295 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2296 break; 2297 case AF_INET6: 2298 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2299 break; 2300 default: 2301 WARN_ON(1); 2302 return false; 2303 } 2304 2305 devlink = priv_to_devlink(mlxsw_sp->core); 2306 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2307} 2308 2309static void 2310mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2311 struct mlxsw_sp_neigh_entry *neigh_entry) 2312{ 2313 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2314 return; 2315 2316 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2317 return; 2318 2319 neigh_entry->counter_valid = true; 2320} 2321 2322static void 2323mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2324 struct mlxsw_sp_neigh_entry *neigh_entry) 2325{ 2326 if (!neigh_entry->counter_valid) 2327 return; 2328 mlxsw_sp_flow_counter_free(mlxsw_sp, 2329 neigh_entry->counter_index); 2330 neigh_entry->counter_valid = false; 2331} 2332 2333static struct mlxsw_sp_neigh_entry * 2334mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2335{ 2336 struct mlxsw_sp_neigh_entry *neigh_entry; 2337 struct mlxsw_sp_rif *rif; 2338 int err; 2339 2340 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2341 if (!rif) 2342 return ERR_PTR(-EINVAL); 2343 2344 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2345 if (!neigh_entry) 2346 return ERR_PTR(-ENOMEM); 2347 2348 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2349 if (err) 2350 goto err_neigh_entry_insert; 2351 2352 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2353 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2354 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2355 2356 return neigh_entry; 2357 2358err_neigh_entry_insert: 2359 mlxsw_sp_neigh_entry_free(neigh_entry); 2360 return ERR_PTR(err); 2361} 2362 2363static void 2364mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2365 struct mlxsw_sp_neigh_entry *neigh_entry) 2366{ 2367 list_del(&neigh_entry->rif_list_node); 2368 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2369 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2370 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2371 mlxsw_sp_neigh_entry_free(neigh_entry); 2372} 2373 2374static struct mlxsw_sp_neigh_entry * 2375mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2376{ 2377 struct mlxsw_sp_neigh_key key; 2378 2379 key.n = n; 2380 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2381 &key, mlxsw_sp_neigh_ht_params); 2382} 2383 2384static void 2385mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2386{ 2387 unsigned long interval; 2388 2389#if IS_ENABLED(CONFIG_IPV6) 2390 interval = min_t(unsigned long, 2391 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2392 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2393#else 2394 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2395#endif 2396 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2397} 2398 2399static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2400 char *rauhtd_pl, 2401 int ent_index) 2402{ 2403 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2404 struct net_device *dev; 2405 struct neighbour *n; 2406 __be32 dipn; 2407 u32 dip; 2408 u16 rif; 2409 2410 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2411 2412 if (WARN_ON_ONCE(rif >= max_rifs)) 2413 return; 2414 if (!mlxsw_sp->router->rifs[rif]) { 2415 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2416 return; 2417 } 2418 2419 dipn = htonl(dip); 2420 dev = mlxsw_sp->router->rifs[rif]->dev; 2421 n = neigh_lookup(&arp_tbl, &dipn, dev); 2422 if (!n) 2423 return; 2424 2425 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2426 neigh_event_send(n, NULL); 2427 neigh_release(n); 2428} 2429 2430#if IS_ENABLED(CONFIG_IPV6) 2431static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2432 char *rauhtd_pl, 2433 int rec_index) 2434{ 2435 struct net_device *dev; 2436 struct neighbour *n; 2437 struct in6_addr dip; 2438 u16 rif; 2439 2440 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2441 (char *) &dip); 2442 2443 if (!mlxsw_sp->router->rifs[rif]) { 2444 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2445 return; 2446 } 2447 2448 dev = mlxsw_sp->router->rifs[rif]->dev; 2449 n = neigh_lookup(&nd_tbl, &dip, dev); 2450 if (!n) 2451 return; 2452 2453 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2454 neigh_event_send(n, NULL); 2455 neigh_release(n); 2456} 2457#else 2458static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2459 char *rauhtd_pl, 2460 int rec_index) 2461{ 2462} 2463#endif 2464 2465static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2466 char *rauhtd_pl, 2467 int rec_index) 2468{ 2469 u8 num_entries; 2470 int i; 2471 2472 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2473 rec_index); 2474 /* Hardware starts counting at 0, so add 1. */ 2475 num_entries++; 2476 2477 /* Each record consists of several neighbour entries. */ 2478 for (i = 0; i < num_entries; i++) { 2479 int ent_index; 2480 2481 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2482 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2483 ent_index); 2484 } 2485 2486} 2487 2488static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2489 char *rauhtd_pl, 2490 int rec_index) 2491{ 2492 /* One record contains one entry. */ 2493 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2494 rec_index); 2495} 2496 2497static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2498 char *rauhtd_pl, int rec_index) 2499{ 2500 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2501 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2502 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2503 rec_index); 2504 break; 2505 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2506 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2507 rec_index); 2508 break; 2509 } 2510} 2511 2512static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2513{ 2514 u8 num_rec, last_rec_index, num_entries; 2515 2516 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2517 last_rec_index = num_rec - 1; 2518 2519 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2520 return false; 2521 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2522 MLXSW_REG_RAUHTD_TYPE_IPV6) 2523 return true; 2524 2525 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2526 last_rec_index); 2527 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2528 return true; 2529 return false; 2530} 2531 2532static int 2533__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2534 char *rauhtd_pl, 2535 enum mlxsw_reg_rauhtd_type type) 2536{ 2537 int i, num_rec; 2538 int err; 2539 2540 /* Ensure the RIF we read from the device does not change mid-dump. */ 2541 mutex_lock(&mlxsw_sp->router->lock); 2542 do { 2543 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2544 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2545 rauhtd_pl); 2546 if (err) { 2547 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2548 break; 2549 } 2550 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2551 for (i = 0; i < num_rec; i++) 2552 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2553 i); 2554 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2555 mutex_unlock(&mlxsw_sp->router->lock); 2556 2557 return err; 2558} 2559 2560static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2561{ 2562 enum mlxsw_reg_rauhtd_type type; 2563 char *rauhtd_pl; 2564 int err; 2565 2566 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2567 return 0; 2568 2569 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2570 if (!rauhtd_pl) 2571 return -ENOMEM; 2572 2573 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2574 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2575 if (err) 2576 goto out; 2577 2578 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2579 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2580out: 2581 kfree(rauhtd_pl); 2582 return err; 2583} 2584 2585static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2586{ 2587 struct mlxsw_sp_neigh_entry *neigh_entry; 2588 2589 mutex_lock(&mlxsw_sp->router->lock); 2590 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2591 nexthop_neighs_list_node) 2592 /* If this neigh have nexthops, make the kernel think this neigh 2593 * is active regardless of the traffic. 2594 */ 2595 neigh_event_send(neigh_entry->key.n, NULL); 2596 mutex_unlock(&mlxsw_sp->router->lock); 2597} 2598 2599static void 2600mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2601{ 2602 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2603 2604 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2605 msecs_to_jiffies(interval)); 2606} 2607 2608static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2609{ 2610 struct mlxsw_sp_router *router; 2611 int err; 2612 2613 router = container_of(work, struct mlxsw_sp_router, 2614 neighs_update.dw.work); 2615 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2616 if (err) 2617 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2618 2619 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2620 2621 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2622} 2623 2624static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2625{ 2626 struct mlxsw_sp_neigh_entry *neigh_entry; 2627 struct mlxsw_sp_router *router; 2628 2629 router = container_of(work, struct mlxsw_sp_router, 2630 nexthop_probe_dw.work); 2631 /* Iterate over nexthop neighbours, find those who are unresolved and 2632 * send arp on them. This solves the chicken-egg problem when 2633 * the nexthop wouldn't get offloaded until the neighbor is resolved 2634 * but it wouldn't get resolved ever in case traffic is flowing in HW 2635 * using different nexthop. 2636 */ 2637 mutex_lock(&router->lock); 2638 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2639 nexthop_neighs_list_node) 2640 if (!neigh_entry->connected) 2641 neigh_event_send(neigh_entry->key.n, NULL); 2642 mutex_unlock(&router->lock); 2643 2644 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2645 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2646} 2647 2648static void 2649mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2650 struct mlxsw_sp_neigh_entry *neigh_entry, 2651 bool removing, bool dead); 2652 2653static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2654{ 2655 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2656 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2657} 2658 2659static int 2660mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2661 struct mlxsw_sp_neigh_entry *neigh_entry, 2662 enum mlxsw_reg_rauht_op op) 2663{ 2664 struct neighbour *n = neigh_entry->key.n; 2665 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2666 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2667 2668 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2669 dip); 2670 if (neigh_entry->counter_valid) 2671 mlxsw_reg_rauht_pack_counter(rauht_pl, 2672 neigh_entry->counter_index); 2673 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2674} 2675 2676static int 2677mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2678 struct mlxsw_sp_neigh_entry *neigh_entry, 2679 enum mlxsw_reg_rauht_op op) 2680{ 2681 struct neighbour *n = neigh_entry->key.n; 2682 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2683 const char *dip = n->primary_key; 2684 2685 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2686 dip); 2687 if (neigh_entry->counter_valid) 2688 mlxsw_reg_rauht_pack_counter(rauht_pl, 2689 neigh_entry->counter_index); 2690 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2691} 2692 2693bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2694{ 2695 struct neighbour *n = neigh_entry->key.n; 2696 2697 /* Packets with a link-local destination address are trapped 2698 * after LPM lookup and never reach the neighbour table, so 2699 * there is no need to program such neighbours to the device. 2700 */ 2701 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2702 IPV6_ADDR_LINKLOCAL) 2703 return true; 2704 return false; 2705} 2706 2707static void 2708mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2709 struct mlxsw_sp_neigh_entry *neigh_entry, 2710 bool adding) 2711{ 2712 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2713 int err; 2714 2715 if (!adding && !neigh_entry->connected) 2716 return; 2717 neigh_entry->connected = adding; 2718 if (neigh_entry->key.n->tbl->family == AF_INET) { 2719 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2720 op); 2721 if (err) 2722 return; 2723 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2724 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2725 return; 2726 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2727 op); 2728 if (err) 2729 return; 2730 } else { 2731 WARN_ON_ONCE(1); 2732 return; 2733 } 2734 2735 if (adding) 2736 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2737 else 2738 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2739} 2740 2741void 2742mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2743 struct mlxsw_sp_neigh_entry *neigh_entry, 2744 bool adding) 2745{ 2746 if (adding) 2747 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2748 else 2749 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2750 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2751} 2752 2753struct mlxsw_sp_netevent_work { 2754 struct work_struct work; 2755 struct mlxsw_sp *mlxsw_sp; 2756 struct neighbour *n; 2757}; 2758 2759static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2760{ 2761 struct mlxsw_sp_netevent_work *net_work = 2762 container_of(work, struct mlxsw_sp_netevent_work, work); 2763 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2764 struct mlxsw_sp_neigh_entry *neigh_entry; 2765 struct neighbour *n = net_work->n; 2766 unsigned char ha[ETH_ALEN]; 2767 bool entry_connected; 2768 u8 nud_state, dead; 2769 2770 /* If these parameters are changed after we release the lock, 2771 * then we are guaranteed to receive another event letting us 2772 * know about it. 2773 */ 2774 read_lock_bh(&n->lock); 2775 memcpy(ha, n->ha, ETH_ALEN); 2776 nud_state = n->nud_state; 2777 dead = n->dead; 2778 read_unlock_bh(&n->lock); 2779 2780 mutex_lock(&mlxsw_sp->router->lock); 2781 mlxsw_sp_span_respin(mlxsw_sp); 2782 2783 entry_connected = nud_state & NUD_VALID && !dead; 2784 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2785 if (!entry_connected && !neigh_entry) 2786 goto out; 2787 if (!neigh_entry) { 2788 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2789 if (IS_ERR(neigh_entry)) 2790 goto out; 2791 } 2792 2793 if (neigh_entry->connected && entry_connected && 2794 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2795 goto out; 2796 2797 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2798 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2799 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2800 dead); 2801 2802 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2803 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2804 2805out: 2806 mutex_unlock(&mlxsw_sp->router->lock); 2807 neigh_release(n); 2808 kfree(net_work); 2809} 2810 2811static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2812 2813static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2814{ 2815 struct mlxsw_sp_netevent_work *net_work = 2816 container_of(work, struct mlxsw_sp_netevent_work, work); 2817 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2818 2819 mlxsw_sp_mp_hash_init(mlxsw_sp); 2820 kfree(net_work); 2821} 2822 2823static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2824 2825static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2826{ 2827 struct mlxsw_sp_netevent_work *net_work = 2828 container_of(work, struct mlxsw_sp_netevent_work, work); 2829 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2830 2831 __mlxsw_sp_router_init(mlxsw_sp); 2832 kfree(net_work); 2833} 2834 2835static int mlxsw_sp_router_schedule_work(struct net *net, 2836 struct notifier_block *nb, 2837 void (*cb)(struct work_struct *)) 2838{ 2839 struct mlxsw_sp_netevent_work *net_work; 2840 struct mlxsw_sp_router *router; 2841 2842 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2843 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2844 return NOTIFY_DONE; 2845 2846 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2847 if (!net_work) 2848 return NOTIFY_BAD; 2849 2850 INIT_WORK(&net_work->work, cb); 2851 net_work->mlxsw_sp = router->mlxsw_sp; 2852 mlxsw_core_schedule_work(&net_work->work); 2853 return NOTIFY_DONE; 2854} 2855 2856static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2857 unsigned long event, void *ptr) 2858{ 2859 struct mlxsw_sp_netevent_work *net_work; 2860 struct mlxsw_sp_port *mlxsw_sp_port; 2861 struct mlxsw_sp *mlxsw_sp; 2862 unsigned long interval; 2863 struct neigh_parms *p; 2864 struct neighbour *n; 2865 2866 switch (event) { 2867 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2868 p = ptr; 2869 2870 /* We don't care about changes in the default table. */ 2871 if (!p->dev || (p->tbl->family != AF_INET && 2872 p->tbl->family != AF_INET6)) 2873 return NOTIFY_DONE; 2874 2875 /* We are in atomic context and can't take RTNL mutex, 2876 * so use RCU variant to walk the device chain. 2877 */ 2878 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2879 if (!mlxsw_sp_port) 2880 return NOTIFY_DONE; 2881 2882 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2883 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2884 mlxsw_sp->router->neighs_update.interval = interval; 2885 2886 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2887 break; 2888 case NETEVENT_NEIGH_UPDATE: 2889 n = ptr; 2890 2891 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2892 return NOTIFY_DONE; 2893 2894 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2895 if (!mlxsw_sp_port) 2896 return NOTIFY_DONE; 2897 2898 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2899 if (!net_work) { 2900 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2901 return NOTIFY_BAD; 2902 } 2903 2904 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2905 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2906 net_work->n = n; 2907 2908 /* Take a reference to ensure the neighbour won't be 2909 * destructed until we drop the reference in delayed 2910 * work. 2911 */ 2912 neigh_clone(n); 2913 mlxsw_core_schedule_work(&net_work->work); 2914 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2915 break; 2916 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2917 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2918 return mlxsw_sp_router_schedule_work(ptr, nb, 2919 mlxsw_sp_router_mp_hash_event_work); 2920 2921 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2922 return mlxsw_sp_router_schedule_work(ptr, nb, 2923 mlxsw_sp_router_update_priority_work); 2924 } 2925 2926 return NOTIFY_DONE; 2927} 2928 2929static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2930{ 2931 int err; 2932 2933 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2934 &mlxsw_sp_neigh_ht_params); 2935 if (err) 2936 return err; 2937 2938 /* Initialize the polling interval according to the default 2939 * table. 2940 */ 2941 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2942 2943 /* Create the delayed works for the activity_update */ 2944 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2945 mlxsw_sp_router_neighs_update_work); 2946 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2947 mlxsw_sp_router_probe_unresolved_nexthops); 2948 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2949 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2950 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2951 return 0; 2952} 2953 2954static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2955{ 2956 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2957 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2958 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2959} 2960 2961static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2962 struct mlxsw_sp_rif *rif) 2963{ 2964 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2965 2966 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2967 rif_list_node) { 2968 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2969 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2970 } 2971} 2972 2973enum mlxsw_sp_nexthop_type { 2974 MLXSW_SP_NEXTHOP_TYPE_ETH, 2975 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2976}; 2977 2978enum mlxsw_sp_nexthop_action { 2979 /* Nexthop forwards packets to an egress RIF */ 2980 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2981 /* Nexthop discards packets */ 2982 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2983 /* Nexthop traps packets */ 2984 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2985}; 2986 2987struct mlxsw_sp_nexthop_key { 2988 struct fib_nh *fib_nh; 2989}; 2990 2991struct mlxsw_sp_nexthop { 2992 struct list_head neigh_list_node; /* member of neigh entry list */ 2993 struct list_head rif_list_node; 2994 struct list_head router_list_node; 2995 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2996 * this nexthop belongs to 2997 */ 2998 struct rhash_head ht_node; 2999 struct neigh_table *neigh_tbl; 3000 struct mlxsw_sp_nexthop_key key; 3001 unsigned char gw_addr[sizeof(struct in6_addr)]; 3002 int ifindex; 3003 int nh_weight; 3004 int norm_nh_weight; 3005 int num_adj_entries; 3006 struct mlxsw_sp_rif *rif; 3007 u8 should_offload:1, /* set indicates this nexthop should be written 3008 * to the adjacency table. 3009 */ 3010 offloaded:1, /* set indicates this nexthop was written to the 3011 * adjacency table. 3012 */ 3013 update:1; /* set indicates this nexthop should be updated in the 3014 * adjacency table (f.e., its MAC changed). 3015 */ 3016 enum mlxsw_sp_nexthop_action action; 3017 enum mlxsw_sp_nexthop_type type; 3018 union { 3019 struct mlxsw_sp_neigh_entry *neigh_entry; 3020 struct mlxsw_sp_ipip_entry *ipip_entry; 3021 }; 3022 unsigned int counter_index; 3023 bool counter_valid; 3024}; 3025 3026enum mlxsw_sp_nexthop_group_type { 3027 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 3028 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 3029 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 3030}; 3031 3032struct mlxsw_sp_nexthop_group_info { 3033 struct mlxsw_sp_nexthop_group *nh_grp; 3034 u32 adj_index; 3035 u16 ecmp_size; 3036 u16 count; 3037 int sum_norm_weight; 3038 u8 adj_index_valid:1, 3039 gateway:1, /* routes using the group use a gateway */ 3040 is_resilient:1; 3041 struct list_head list; /* member in nh_res_grp_list */ 3042 struct mlxsw_sp_nexthop nexthops[0]; 3043#define nh_rif nexthops[0].rif 3044}; 3045 3046struct mlxsw_sp_nexthop_group_vr_key { 3047 u16 vr_id; 3048 enum mlxsw_sp_l3proto proto; 3049}; 3050 3051struct mlxsw_sp_nexthop_group_vr_entry { 3052 struct list_head list; /* member in vr_list */ 3053 struct rhash_head ht_node; /* member in vr_ht */ 3054 refcount_t ref_count; 3055 struct mlxsw_sp_nexthop_group_vr_key key; 3056}; 3057 3058struct mlxsw_sp_nexthop_group { 3059 struct rhash_head ht_node; 3060 struct list_head fib_list; /* list of fib entries that use this group */ 3061 union { 3062 struct { 3063 struct fib_info *fi; 3064 } ipv4; 3065 struct { 3066 u32 id; 3067 } obj; 3068 }; 3069 struct mlxsw_sp_nexthop_group_info *nhgi; 3070 struct list_head vr_list; 3071 struct rhashtable vr_ht; 3072 enum mlxsw_sp_nexthop_group_type type; 3073 bool can_destroy; 3074}; 3075 3076void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 3077 struct mlxsw_sp_nexthop *nh) 3078{ 3079 struct devlink *devlink; 3080 3081 devlink = priv_to_devlink(mlxsw_sp->core); 3082 if (!devlink_dpipe_table_counter_enabled(devlink, 3083 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 3084 return; 3085 3086 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 3087 return; 3088 3089 nh->counter_valid = true; 3090} 3091 3092void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3093 struct mlxsw_sp_nexthop *nh) 3094{ 3095 if (!nh->counter_valid) 3096 return; 3097 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 3098 nh->counter_valid = false; 3099} 3100 3101int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3102 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3103{ 3104 if (!nh->counter_valid) 3105 return -EINVAL; 3106 3107 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 3108 p_counter, NULL); 3109} 3110 3111struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3112 struct mlxsw_sp_nexthop *nh) 3113{ 3114 if (!nh) { 3115 if (list_empty(&router->nexthop_list)) 3116 return NULL; 3117 else 3118 return list_first_entry(&router->nexthop_list, 3119 typeof(*nh), router_list_node); 3120 } 3121 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3122 return NULL; 3123 return list_next_entry(nh, router_list_node); 3124} 3125 3126bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3127{ 3128 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3129} 3130 3131unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3132{ 3133 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3134 !mlxsw_sp_nexthop_is_forward(nh)) 3135 return NULL; 3136 return nh->neigh_entry->ha; 3137} 3138 3139int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3140 u32 *p_adj_size, u32 *p_adj_hash_index) 3141{ 3142 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3143 u32 adj_hash_index = 0; 3144 int i; 3145 3146 if (!nh->offloaded || !nhgi->adj_index_valid) 3147 return -EINVAL; 3148 3149 *p_adj_index = nhgi->adj_index; 3150 *p_adj_size = nhgi->ecmp_size; 3151 3152 for (i = 0; i < nhgi->count; i++) { 3153 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3154 3155 if (nh_iter == nh) 3156 break; 3157 if (nh_iter->offloaded) 3158 adj_hash_index += nh_iter->num_adj_entries; 3159 } 3160 3161 *p_adj_hash_index = adj_hash_index; 3162 return 0; 3163} 3164 3165struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3166{ 3167 return nh->rif; 3168} 3169 3170bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3171{ 3172 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3173 int i; 3174 3175 for (i = 0; i < nhgi->count; i++) { 3176 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3177 3178 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3179 return true; 3180 } 3181 return false; 3182} 3183 3184static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3185 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3186 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3187 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3188 .automatic_shrinking = true, 3189}; 3190 3191static struct mlxsw_sp_nexthop_group_vr_entry * 3192mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3193 const struct mlxsw_sp_fib *fib) 3194{ 3195 struct mlxsw_sp_nexthop_group_vr_key key; 3196 3197 memset(&key, 0, sizeof(key)); 3198 key.vr_id = fib->vr->id; 3199 key.proto = fib->proto; 3200 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3201 mlxsw_sp_nexthop_group_vr_ht_params); 3202} 3203 3204static int 3205mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3206 const struct mlxsw_sp_fib *fib) 3207{ 3208 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3209 int err; 3210 3211 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3212 if (!vr_entry) 3213 return -ENOMEM; 3214 3215 vr_entry->key.vr_id = fib->vr->id; 3216 vr_entry->key.proto = fib->proto; 3217 refcount_set(&vr_entry->ref_count, 1); 3218 3219 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3220 mlxsw_sp_nexthop_group_vr_ht_params); 3221 if (err) 3222 goto err_hashtable_insert; 3223 3224 list_add(&vr_entry->list, &nh_grp->vr_list); 3225 3226 return 0; 3227 3228err_hashtable_insert: 3229 kfree(vr_entry); 3230 return err; 3231} 3232 3233static void 3234mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3235 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3236{ 3237 list_del(&vr_entry->list); 3238 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3239 mlxsw_sp_nexthop_group_vr_ht_params); 3240 kfree(vr_entry); 3241} 3242 3243static int 3244mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3245 const struct mlxsw_sp_fib *fib) 3246{ 3247 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3248 3249 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3250 if (vr_entry) { 3251 refcount_inc(&vr_entry->ref_count); 3252 return 0; 3253 } 3254 3255 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3256} 3257 3258static void 3259mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3260 const struct mlxsw_sp_fib *fib) 3261{ 3262 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3263 3264 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3265 if (WARN_ON_ONCE(!vr_entry)) 3266 return; 3267 3268 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3269 return; 3270 3271 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3272} 3273 3274struct mlxsw_sp_nexthop_group_cmp_arg { 3275 enum mlxsw_sp_nexthop_group_type type; 3276 union { 3277 struct fib_info *fi; 3278 struct mlxsw_sp_fib6_entry *fib6_entry; 3279 u32 id; 3280 }; 3281}; 3282 3283static bool 3284mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3285 const struct in6_addr *gw, int ifindex, 3286 int weight) 3287{ 3288 int i; 3289 3290 for (i = 0; i < nh_grp->nhgi->count; i++) { 3291 const struct mlxsw_sp_nexthop *nh; 3292 3293 nh = &nh_grp->nhgi->nexthops[i]; 3294 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3295 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3296 return true; 3297 } 3298 3299 return false; 3300} 3301 3302static bool 3303mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3304 const struct mlxsw_sp_fib6_entry *fib6_entry) 3305{ 3306 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3307 3308 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3309 return false; 3310 3311 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3312 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3313 struct in6_addr *gw; 3314 int ifindex, weight; 3315 3316 ifindex = fib6_nh->fib_nh_dev->ifindex; 3317 weight = fib6_nh->fib_nh_weight; 3318 gw = &fib6_nh->fib_nh_gw6; 3319 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3320 weight)) 3321 return false; 3322 } 3323 3324 return true; 3325} 3326 3327static int 3328mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3329{ 3330 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3331 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3332 3333 if (nh_grp->type != cmp_arg->type) 3334 return 1; 3335 3336 switch (cmp_arg->type) { 3337 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3338 return cmp_arg->fi != nh_grp->ipv4.fi; 3339 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3340 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3341 cmp_arg->fib6_entry); 3342 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3343 return cmp_arg->id != nh_grp->obj.id; 3344 default: 3345 WARN_ON(1); 3346 return 1; 3347 } 3348} 3349 3350static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3351{ 3352 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3353 const struct mlxsw_sp_nexthop *nh; 3354 struct fib_info *fi; 3355 unsigned int val; 3356 int i; 3357 3358 switch (nh_grp->type) { 3359 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3360 fi = nh_grp->ipv4.fi; 3361 return jhash(&fi, sizeof(fi), seed); 3362 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3363 val = nh_grp->nhgi->count; 3364 for (i = 0; i < nh_grp->nhgi->count; i++) { 3365 nh = &nh_grp->nhgi->nexthops[i]; 3366 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3367 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3368 } 3369 return jhash(&val, sizeof(val), seed); 3370 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3371 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3372 default: 3373 WARN_ON(1); 3374 return 0; 3375 } 3376} 3377 3378static u32 3379mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3380{ 3381 unsigned int val = fib6_entry->nrt6; 3382 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3383 3384 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3385 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3386 struct net_device *dev = fib6_nh->fib_nh_dev; 3387 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3388 3389 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3390 val ^= jhash(gw, sizeof(*gw), seed); 3391 } 3392 3393 return jhash(&val, sizeof(val), seed); 3394} 3395 3396static u32 3397mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3398{ 3399 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3400 3401 switch (cmp_arg->type) { 3402 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3403 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3404 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3405 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3406 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3407 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3408 default: 3409 WARN_ON(1); 3410 return 0; 3411 } 3412} 3413 3414static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3415 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3416 .hashfn = mlxsw_sp_nexthop_group_hash, 3417 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3418 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3419}; 3420 3421static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3422 struct mlxsw_sp_nexthop_group *nh_grp) 3423{ 3424 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3425 !nh_grp->nhgi->gateway) 3426 return 0; 3427 3428 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3429 &nh_grp->ht_node, 3430 mlxsw_sp_nexthop_group_ht_params); 3431} 3432 3433static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3434 struct mlxsw_sp_nexthop_group *nh_grp) 3435{ 3436 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3437 !nh_grp->nhgi->gateway) 3438 return; 3439 3440 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3441 &nh_grp->ht_node, 3442 mlxsw_sp_nexthop_group_ht_params); 3443} 3444 3445static struct mlxsw_sp_nexthop_group * 3446mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3447 struct fib_info *fi) 3448{ 3449 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3450 3451 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3452 cmp_arg.fi = fi; 3453 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3454 &cmp_arg, 3455 mlxsw_sp_nexthop_group_ht_params); 3456} 3457 3458static struct mlxsw_sp_nexthop_group * 3459mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3460 struct mlxsw_sp_fib6_entry *fib6_entry) 3461{ 3462 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3463 3464 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3465 cmp_arg.fib6_entry = fib6_entry; 3466 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3467 &cmp_arg, 3468 mlxsw_sp_nexthop_group_ht_params); 3469} 3470 3471static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3472 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3473 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3474 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3475}; 3476 3477static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3478 struct mlxsw_sp_nexthop *nh) 3479{ 3480 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3481 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3482} 3483 3484static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3485 struct mlxsw_sp_nexthop *nh) 3486{ 3487 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3488 mlxsw_sp_nexthop_ht_params); 3489} 3490 3491static struct mlxsw_sp_nexthop * 3492mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3493 struct mlxsw_sp_nexthop_key key) 3494{ 3495 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3496 mlxsw_sp_nexthop_ht_params); 3497} 3498 3499static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3500 enum mlxsw_sp_l3proto proto, 3501 u16 vr_id, 3502 u32 adj_index, u16 ecmp_size, 3503 u32 new_adj_index, 3504 u16 new_ecmp_size) 3505{ 3506 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3507 3508 mlxsw_reg_raleu_pack(raleu_pl, 3509 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3510 adj_index, ecmp_size, new_adj_index, 3511 new_ecmp_size); 3512 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3513} 3514 3515static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3516 struct mlxsw_sp_nexthop_group *nh_grp, 3517 u32 old_adj_index, u16 old_ecmp_size) 3518{ 3519 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3520 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3521 int err; 3522 3523 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3524 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3525 vr_entry->key.proto, 3526 vr_entry->key.vr_id, 3527 old_adj_index, 3528 old_ecmp_size, 3529 nhgi->adj_index, 3530 nhgi->ecmp_size); 3531 if (err) 3532 goto err_mass_update_vr; 3533 } 3534 return 0; 3535 3536err_mass_update_vr: 3537 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3538 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3539 vr_entry->key.vr_id, 3540 nhgi->adj_index, 3541 nhgi->ecmp_size, 3542 old_adj_index, old_ecmp_size); 3543 return err; 3544} 3545 3546static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3547 u32 adj_index, 3548 struct mlxsw_sp_nexthop *nh, 3549 bool force, char *ratr_pl) 3550{ 3551 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3552 enum mlxsw_reg_ratr_op op; 3553 u16 rif_index; 3554 3555 rif_index = nh->rif ? nh->rif->rif_index : 3556 mlxsw_sp->router->lb_rif_index; 3557 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3558 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3559 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3560 adj_index, rif_index); 3561 switch (nh->action) { 3562 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3563 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3564 break; 3565 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3566 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3567 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3568 break; 3569 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3570 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3571 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3572 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3573 break; 3574 default: 3575 WARN_ON_ONCE(1); 3576 return -EINVAL; 3577 } 3578 if (nh->counter_valid) 3579 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3580 else 3581 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3582 3583 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3584} 3585 3586int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3587 struct mlxsw_sp_nexthop *nh, bool force, 3588 char *ratr_pl) 3589{ 3590 int i; 3591 3592 for (i = 0; i < nh->num_adj_entries; i++) { 3593 int err; 3594 3595 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3596 nh, force, ratr_pl); 3597 if (err) 3598 return err; 3599 } 3600 3601 return 0; 3602} 3603 3604static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3605 u32 adj_index, 3606 struct mlxsw_sp_nexthop *nh, 3607 bool force, char *ratr_pl) 3608{ 3609 const struct mlxsw_sp_ipip_ops *ipip_ops; 3610 3611 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3612 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3613 force, ratr_pl); 3614} 3615 3616static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3617 u32 adj_index, 3618 struct mlxsw_sp_nexthop *nh, bool force, 3619 char *ratr_pl) 3620{ 3621 int i; 3622 3623 for (i = 0; i < nh->num_adj_entries; i++) { 3624 int err; 3625 3626 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3627 nh, force, ratr_pl); 3628 if (err) 3629 return err; 3630 } 3631 3632 return 0; 3633} 3634 3635static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3636 struct mlxsw_sp_nexthop *nh, bool force, 3637 char *ratr_pl) 3638{ 3639 /* When action is discard or trap, the nexthop must be 3640 * programmed as an Ethernet nexthop. 3641 */ 3642 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3643 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3644 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3645 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3646 force, ratr_pl); 3647 else 3648 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3649 force, ratr_pl); 3650} 3651 3652static int 3653mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3654 struct mlxsw_sp_nexthop_group_info *nhgi, 3655 bool reallocate) 3656{ 3657 char ratr_pl[MLXSW_REG_RATR_LEN]; 3658 u32 adj_index = nhgi->adj_index; /* base */ 3659 struct mlxsw_sp_nexthop *nh; 3660 int i; 3661 3662 for (i = 0; i < nhgi->count; i++) { 3663 nh = &nhgi->nexthops[i]; 3664 3665 if (!nh->should_offload) { 3666 nh->offloaded = 0; 3667 continue; 3668 } 3669 3670 if (nh->update || reallocate) { 3671 int err = 0; 3672 3673 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3674 true, ratr_pl); 3675 if (err) 3676 return err; 3677 nh->update = 0; 3678 nh->offloaded = 1; 3679 } 3680 adj_index += nh->num_adj_entries; 3681 } 3682 return 0; 3683} 3684 3685static int 3686mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3687 struct mlxsw_sp_nexthop_group *nh_grp) 3688{ 3689 struct mlxsw_sp_fib_entry *fib_entry; 3690 int err; 3691 3692 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3693 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3694 if (err) 3695 return err; 3696 } 3697 return 0; 3698} 3699 3700struct mlxsw_sp_adj_grp_size_range { 3701 u16 start; /* Inclusive */ 3702 u16 end; /* Inclusive */ 3703}; 3704 3705/* Ordered by range start value */ 3706static const struct mlxsw_sp_adj_grp_size_range 3707mlxsw_sp1_adj_grp_size_ranges[] = { 3708 { .start = 1, .end = 64 }, 3709 { .start = 512, .end = 512 }, 3710 { .start = 1024, .end = 1024 }, 3711 { .start = 2048, .end = 2048 }, 3712 { .start = 4096, .end = 4096 }, 3713}; 3714 3715/* Ordered by range start value */ 3716static const struct mlxsw_sp_adj_grp_size_range 3717mlxsw_sp2_adj_grp_size_ranges[] = { 3718 { .start = 1, .end = 128 }, 3719 { .start = 256, .end = 256 }, 3720 { .start = 512, .end = 512 }, 3721 { .start = 1024, .end = 1024 }, 3722 { .start = 2048, .end = 2048 }, 3723 { .start = 4096, .end = 4096 }, 3724}; 3725 3726static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3727 u16 *p_adj_grp_size) 3728{ 3729 int i; 3730 3731 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3732 const struct mlxsw_sp_adj_grp_size_range *size_range; 3733 3734 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3735 3736 if (*p_adj_grp_size >= size_range->start && 3737 *p_adj_grp_size <= size_range->end) 3738 return; 3739 3740 if (*p_adj_grp_size <= size_range->end) { 3741 *p_adj_grp_size = size_range->end; 3742 return; 3743 } 3744 } 3745} 3746 3747static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3748 u16 *p_adj_grp_size, 3749 unsigned int alloc_size) 3750{ 3751 int i; 3752 3753 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3754 const struct mlxsw_sp_adj_grp_size_range *size_range; 3755 3756 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3757 3758 if (alloc_size >= size_range->end) { 3759 *p_adj_grp_size = size_range->end; 3760 return; 3761 } 3762 } 3763} 3764 3765static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3766 u16 *p_adj_grp_size) 3767{ 3768 unsigned int alloc_size; 3769 int err; 3770 3771 /* Round up the requested group size to the next size supported 3772 * by the device and make sure the request can be satisfied. 3773 */ 3774 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3775 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3776 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3777 *p_adj_grp_size, &alloc_size); 3778 if (err) 3779 return err; 3780 /* It is possible the allocation results in more allocated 3781 * entries than requested. Try to use as much of them as 3782 * possible. 3783 */ 3784 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3785 3786 return 0; 3787} 3788 3789static void 3790mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3791{ 3792 int i, g = 0, sum_norm_weight = 0; 3793 struct mlxsw_sp_nexthop *nh; 3794 3795 for (i = 0; i < nhgi->count; i++) { 3796 nh = &nhgi->nexthops[i]; 3797 3798 if (!nh->should_offload) 3799 continue; 3800 if (g > 0) 3801 g = gcd(nh->nh_weight, g); 3802 else 3803 g = nh->nh_weight; 3804 } 3805 3806 for (i = 0; i < nhgi->count; i++) { 3807 nh = &nhgi->nexthops[i]; 3808 3809 if (!nh->should_offload) 3810 continue; 3811 nh->norm_nh_weight = nh->nh_weight / g; 3812 sum_norm_weight += nh->norm_nh_weight; 3813 } 3814 3815 nhgi->sum_norm_weight = sum_norm_weight; 3816} 3817 3818static void 3819mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3820{ 3821 int i, weight = 0, lower_bound = 0; 3822 int total = nhgi->sum_norm_weight; 3823 u16 ecmp_size = nhgi->ecmp_size; 3824 3825 for (i = 0; i < nhgi->count; i++) { 3826 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3827 int upper_bound; 3828 3829 if (!nh->should_offload) 3830 continue; 3831 weight += nh->norm_nh_weight; 3832 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3833 nh->num_adj_entries = upper_bound - lower_bound; 3834 lower_bound = upper_bound; 3835 } 3836} 3837 3838static struct mlxsw_sp_nexthop * 3839mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3840 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3841 3842static void 3843mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3844 struct mlxsw_sp_nexthop_group *nh_grp) 3845{ 3846 int i; 3847 3848 for (i = 0; i < nh_grp->nhgi->count; i++) { 3849 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3850 3851 if (nh->offloaded) 3852 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3853 else 3854 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3855 } 3856} 3857 3858static void 3859__mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3860 struct mlxsw_sp_fib6_entry *fib6_entry) 3861{ 3862 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3863 3864 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3865 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3866 struct mlxsw_sp_nexthop *nh; 3867 3868 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3869 if (nh && nh->offloaded) 3870 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3871 else 3872 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3873 } 3874} 3875 3876static void 3877mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3878 struct mlxsw_sp_nexthop_group *nh_grp) 3879{ 3880 struct mlxsw_sp_fib6_entry *fib6_entry; 3881 3882 /* Unfortunately, in IPv6 the route and the nexthop are described by 3883 * the same struct, so we need to iterate over all the routes using the 3884 * nexthop group and set / clear the offload indication for them. 3885 */ 3886 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3887 common.nexthop_group_node) 3888 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3889} 3890 3891static void 3892mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3893 const struct mlxsw_sp_nexthop *nh, 3894 u16 bucket_index) 3895{ 3896 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3897 bool offload = false, trap = false; 3898 3899 if (nh->offloaded) { 3900 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3901 trap = true; 3902 else 3903 offload = true; 3904 } 3905 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3906 bucket_index, offload, trap); 3907} 3908 3909static void 3910mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3911 struct mlxsw_sp_nexthop_group *nh_grp) 3912{ 3913 int i; 3914 3915 /* Do not update the flags if the nexthop group is being destroyed 3916 * since: 3917 * 1. The nexthop objects is being deleted, in which case the flags are 3918 * irrelevant. 3919 * 2. The nexthop group was replaced by a newer group, in which case 3920 * the flags of the nexthop object were already updated based on the 3921 * new group. 3922 */ 3923 if (nh_grp->can_destroy) 3924 return; 3925 3926 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3927 nh_grp->nhgi->adj_index_valid, false); 3928 3929 /* Update flags of individual nexthop buckets in case of a resilient 3930 * nexthop group. 3931 */ 3932 if (!nh_grp->nhgi->is_resilient) 3933 return; 3934 3935 for (i = 0; i < nh_grp->nhgi->count; i++) { 3936 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3937 3938 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 3939 } 3940} 3941 3942static void 3943mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3944 struct mlxsw_sp_nexthop_group *nh_grp) 3945{ 3946 switch (nh_grp->type) { 3947 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3948 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3949 break; 3950 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3951 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3952 break; 3953 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3954 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3955 break; 3956 } 3957} 3958 3959static int 3960mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3961 struct mlxsw_sp_nexthop_group *nh_grp) 3962{ 3963 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3964 u16 ecmp_size, old_ecmp_size; 3965 struct mlxsw_sp_nexthop *nh; 3966 bool offload_change = false; 3967 u32 adj_index; 3968 bool old_adj_index_valid; 3969 u32 old_adj_index; 3970 int i, err2, err; 3971 3972 if (!nhgi->gateway) 3973 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3974 3975 for (i = 0; i < nhgi->count; i++) { 3976 nh = &nhgi->nexthops[i]; 3977 3978 if (nh->should_offload != nh->offloaded) { 3979 offload_change = true; 3980 if (nh->should_offload) 3981 nh->update = 1; 3982 } 3983 } 3984 if (!offload_change) { 3985 /* Nothing was added or removed, so no need to reallocate. Just 3986 * update MAC on existing adjacency indexes. 3987 */ 3988 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3989 if (err) { 3990 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3991 goto set_trap; 3992 } 3993 /* Flags of individual nexthop buckets might need to be 3994 * updated. 3995 */ 3996 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3997 return 0; 3998 } 3999 mlxsw_sp_nexthop_group_normalize(nhgi); 4000 if (!nhgi->sum_norm_weight) { 4001 /* No neigh of this group is connected so we just set 4002 * the trap and let everthing flow through kernel. 4003 */ 4004 err = 0; 4005 goto set_trap; 4006 } 4007 4008 ecmp_size = nhgi->sum_norm_weight; 4009 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 4010 if (err) 4011 /* No valid allocation size available. */ 4012 goto set_trap; 4013 4014 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4015 ecmp_size, &adj_index); 4016 if (err) { 4017 /* We ran out of KVD linear space, just set the 4018 * trap and let everything flow through kernel. 4019 */ 4020 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 4021 goto set_trap; 4022 } 4023 old_adj_index_valid = nhgi->adj_index_valid; 4024 old_adj_index = nhgi->adj_index; 4025 old_ecmp_size = nhgi->ecmp_size; 4026 nhgi->adj_index_valid = 1; 4027 nhgi->adj_index = adj_index; 4028 nhgi->ecmp_size = ecmp_size; 4029 mlxsw_sp_nexthop_group_rebalance(nhgi); 4030 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 4031 if (err) { 4032 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4033 goto set_trap; 4034 } 4035 4036 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4037 4038 if (!old_adj_index_valid) { 4039 /* The trap was set for fib entries, so we have to call 4040 * fib entry update to unset it and use adjacency index. 4041 */ 4042 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4043 if (err) { 4044 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 4045 goto set_trap; 4046 } 4047 return 0; 4048 } 4049 4050 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 4051 old_adj_index, old_ecmp_size); 4052 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4053 old_ecmp_size, old_adj_index); 4054 if (err) { 4055 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 4056 goto set_trap; 4057 } 4058 4059 return 0; 4060 4061set_trap: 4062 old_adj_index_valid = nhgi->adj_index_valid; 4063 nhgi->adj_index_valid = 0; 4064 for (i = 0; i < nhgi->count; i++) { 4065 nh = &nhgi->nexthops[i]; 4066 nh->offloaded = 0; 4067 } 4068 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4069 if (err2) 4070 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 4071 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4072 if (old_adj_index_valid) 4073 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4074 nhgi->ecmp_size, nhgi->adj_index); 4075 return err; 4076} 4077 4078static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 4079 bool removing) 4080{ 4081 if (!removing) { 4082 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 4083 nh->should_offload = 1; 4084 } else if (nh->nhgi->is_resilient) { 4085 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4086 nh->should_offload = 1; 4087 } else { 4088 nh->should_offload = 0; 4089 } 4090 nh->update = 1; 4091} 4092 4093static int 4094mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4095 struct mlxsw_sp_neigh_entry *neigh_entry) 4096{ 4097 struct neighbour *n, *old_n = neigh_entry->key.n; 4098 struct mlxsw_sp_nexthop *nh; 4099 bool entry_connected; 4100 u8 nud_state, dead; 4101 int err; 4102 4103 nh = list_first_entry(&neigh_entry->nexthop_list, 4104 struct mlxsw_sp_nexthop, neigh_list_node); 4105 4106 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4107 if (!n) { 4108 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4109 if (IS_ERR(n)) 4110 return PTR_ERR(n); 4111 neigh_event_send(n, NULL); 4112 } 4113 4114 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4115 neigh_entry->key.n = n; 4116 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4117 if (err) 4118 goto err_neigh_entry_insert; 4119 4120 read_lock_bh(&n->lock); 4121 nud_state = n->nud_state; 4122 dead = n->dead; 4123 read_unlock_bh(&n->lock); 4124 entry_connected = nud_state & NUD_VALID && !dead; 4125 4126 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4127 neigh_list_node) { 4128 neigh_release(old_n); 4129 neigh_clone(n); 4130 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4131 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4132 } 4133 4134 neigh_release(n); 4135 4136 return 0; 4137 4138err_neigh_entry_insert: 4139 neigh_entry->key.n = old_n; 4140 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4141 neigh_release(n); 4142 return err; 4143} 4144 4145static void 4146mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4147 struct mlxsw_sp_neigh_entry *neigh_entry, 4148 bool removing, bool dead) 4149{ 4150 struct mlxsw_sp_nexthop *nh; 4151 4152 if (list_empty(&neigh_entry->nexthop_list)) 4153 return; 4154 4155 if (dead) { 4156 int err; 4157 4158 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4159 neigh_entry); 4160 if (err) 4161 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4162 return; 4163 } 4164 4165 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4166 neigh_list_node) { 4167 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4168 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4169 } 4170} 4171 4172static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 4173 struct mlxsw_sp_rif *rif) 4174{ 4175 if (nh->rif) 4176 return; 4177 4178 nh->rif = rif; 4179 list_add(&nh->rif_list_node, &rif->nexthop_list); 4180} 4181 4182static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 4183{ 4184 if (!nh->rif) 4185 return; 4186 4187 list_del(&nh->rif_list_node); 4188 nh->rif = NULL; 4189} 4190 4191static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4192 struct mlxsw_sp_nexthop *nh) 4193{ 4194 struct mlxsw_sp_neigh_entry *neigh_entry; 4195 struct neighbour *n; 4196 u8 nud_state, dead; 4197 int err; 4198 4199 if (!nh->nhgi->gateway || nh->neigh_entry) 4200 return 0; 4201 4202 /* Take a reference of neigh here ensuring that neigh would 4203 * not be destructed before the nexthop entry is finished. 4204 * The reference is taken either in neigh_lookup() or 4205 * in neigh_create() in case n is not found. 4206 */ 4207 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4208 if (!n) { 4209 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4210 if (IS_ERR(n)) 4211 return PTR_ERR(n); 4212 neigh_event_send(n, NULL); 4213 } 4214 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4215 if (!neigh_entry) { 4216 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4217 if (IS_ERR(neigh_entry)) { 4218 err = -EINVAL; 4219 goto err_neigh_entry_create; 4220 } 4221 } 4222 4223 /* If that is the first nexthop connected to that neigh, add to 4224 * nexthop_neighs_list 4225 */ 4226 if (list_empty(&neigh_entry->nexthop_list)) 4227 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4228 &mlxsw_sp->router->nexthop_neighs_list); 4229 4230 nh->neigh_entry = neigh_entry; 4231 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4232 read_lock_bh(&n->lock); 4233 nud_state = n->nud_state; 4234 dead = n->dead; 4235 read_unlock_bh(&n->lock); 4236 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4237 4238 return 0; 4239 4240err_neigh_entry_create: 4241 neigh_release(n); 4242 return err; 4243} 4244 4245static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4246 struct mlxsw_sp_nexthop *nh) 4247{ 4248 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4249 struct neighbour *n; 4250 4251 if (!neigh_entry) 4252 return; 4253 n = neigh_entry->key.n; 4254 4255 __mlxsw_sp_nexthop_neigh_update(nh, true); 4256 list_del(&nh->neigh_list_node); 4257 nh->neigh_entry = NULL; 4258 4259 /* If that is the last nexthop connected to that neigh, remove from 4260 * nexthop_neighs_list 4261 */ 4262 if (list_empty(&neigh_entry->nexthop_list)) 4263 list_del(&neigh_entry->nexthop_neighs_list_node); 4264 4265 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4266 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4267 4268 neigh_release(n); 4269} 4270 4271static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4272{ 4273 struct net_device *ul_dev; 4274 bool is_up; 4275 4276 rcu_read_lock(); 4277 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4278 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4279 rcu_read_unlock(); 4280 4281 return is_up; 4282} 4283 4284static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4285 struct mlxsw_sp_nexthop *nh, 4286 struct mlxsw_sp_ipip_entry *ipip_entry) 4287{ 4288 bool removing; 4289 4290 if (!nh->nhgi->gateway || nh->ipip_entry) 4291 return; 4292 4293 nh->ipip_entry = ipip_entry; 4294 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4295 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4296 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4297} 4298 4299static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4300 struct mlxsw_sp_nexthop *nh) 4301{ 4302 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4303 4304 if (!ipip_entry) 4305 return; 4306 4307 __mlxsw_sp_nexthop_neigh_update(nh, true); 4308 nh->ipip_entry = NULL; 4309} 4310 4311static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4312 const struct fib_nh *fib_nh, 4313 enum mlxsw_sp_ipip_type *p_ipipt) 4314{ 4315 struct net_device *dev = fib_nh->fib_nh_dev; 4316 4317 return dev && 4318 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4319 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4320} 4321 4322static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4323 struct mlxsw_sp_nexthop *nh, 4324 const struct net_device *dev) 4325{ 4326 const struct mlxsw_sp_ipip_ops *ipip_ops; 4327 struct mlxsw_sp_ipip_entry *ipip_entry; 4328 struct mlxsw_sp_rif *rif; 4329 int err; 4330 4331 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4332 if (ipip_entry) { 4333 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4334 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4335 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4336 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4337 return 0; 4338 } 4339 } 4340 4341 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4342 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4343 if (!rif) 4344 return 0; 4345 4346 mlxsw_sp_nexthop_rif_init(nh, rif); 4347 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4348 if (err) 4349 goto err_neigh_init; 4350 4351 return 0; 4352 4353err_neigh_init: 4354 mlxsw_sp_nexthop_rif_fini(nh); 4355 return err; 4356} 4357 4358static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4359 struct mlxsw_sp_nexthop *nh) 4360{ 4361 switch (nh->type) { 4362 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4363 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4364 mlxsw_sp_nexthop_rif_fini(nh); 4365 break; 4366 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4367 mlxsw_sp_nexthop_rif_fini(nh); 4368 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4369 break; 4370 } 4371} 4372 4373static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4374 struct mlxsw_sp_nexthop_group *nh_grp, 4375 struct mlxsw_sp_nexthop *nh, 4376 struct fib_nh *fib_nh) 4377{ 4378 struct net_device *dev = fib_nh->fib_nh_dev; 4379 struct in_device *in_dev; 4380 int err; 4381 4382 nh->nhgi = nh_grp->nhgi; 4383 nh->key.fib_nh = fib_nh; 4384#ifdef CONFIG_IP_ROUTE_MULTIPATH 4385 nh->nh_weight = fib_nh->fib_nh_weight; 4386#else 4387 nh->nh_weight = 1; 4388#endif 4389 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4390 nh->neigh_tbl = &arp_tbl; 4391 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4392 if (err) 4393 return err; 4394 4395 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4396 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4397 4398 if (!dev) 4399 return 0; 4400 nh->ifindex = dev->ifindex; 4401 4402 rcu_read_lock(); 4403 in_dev = __in_dev_get_rcu(dev); 4404 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4405 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4406 rcu_read_unlock(); 4407 return 0; 4408 } 4409 rcu_read_unlock(); 4410 4411 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4412 if (err) 4413 goto err_nexthop_neigh_init; 4414 4415 return 0; 4416 4417err_nexthop_neigh_init: 4418 list_del(&nh->router_list_node); 4419 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4420 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4421 return err; 4422} 4423 4424static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4425 struct mlxsw_sp_nexthop *nh) 4426{ 4427 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4428 list_del(&nh->router_list_node); 4429 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4430 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4431} 4432 4433static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4434 unsigned long event, struct fib_nh *fib_nh) 4435{ 4436 struct mlxsw_sp_nexthop_key key; 4437 struct mlxsw_sp_nexthop *nh; 4438 4439 key.fib_nh = fib_nh; 4440 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4441 if (!nh) 4442 return; 4443 4444 switch (event) { 4445 case FIB_EVENT_NH_ADD: 4446 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4447 break; 4448 case FIB_EVENT_NH_DEL: 4449 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4450 break; 4451 } 4452 4453 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4454} 4455 4456static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4457 struct mlxsw_sp_rif *rif) 4458{ 4459 struct mlxsw_sp_nexthop *nh; 4460 bool removing; 4461 4462 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4463 switch (nh->type) { 4464 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4465 removing = false; 4466 break; 4467 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4468 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4469 break; 4470 default: 4471 WARN_ON(1); 4472 continue; 4473 } 4474 4475 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4476 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4477 } 4478} 4479 4480static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4481 struct mlxsw_sp_rif *old_rif, 4482 struct mlxsw_sp_rif *new_rif) 4483{ 4484 struct mlxsw_sp_nexthop *nh; 4485 4486 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4487 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4488 nh->rif = new_rif; 4489 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4490} 4491 4492static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4493 struct mlxsw_sp_rif *rif) 4494{ 4495 struct mlxsw_sp_nexthop *nh, *tmp; 4496 4497 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4498 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4499 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4500 } 4501} 4502 4503static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4504{ 4505 enum mlxsw_reg_ratr_trap_action trap_action; 4506 char ratr_pl[MLXSW_REG_RATR_LEN]; 4507 int err; 4508 4509 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4510 &mlxsw_sp->router->adj_trap_index); 4511 if (err) 4512 return err; 4513 4514 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4515 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4516 MLXSW_REG_RATR_TYPE_ETHERNET, 4517 mlxsw_sp->router->adj_trap_index, 4518 mlxsw_sp->router->lb_rif_index); 4519 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4520 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4521 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4522 if (err) 4523 goto err_ratr_write; 4524 4525 return 0; 4526 4527err_ratr_write: 4528 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4529 mlxsw_sp->router->adj_trap_index); 4530 return err; 4531} 4532 4533static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4534{ 4535 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4536 mlxsw_sp->router->adj_trap_index); 4537} 4538 4539static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4540{ 4541 int err; 4542 4543 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4544 return 0; 4545 4546 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4547 if (err) 4548 return err; 4549 4550 refcount_set(&mlxsw_sp->router->num_groups, 1); 4551 4552 return 0; 4553} 4554 4555static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4556{ 4557 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4558 return; 4559 4560 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4561} 4562 4563static void 4564mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4565 const struct mlxsw_sp_nexthop_group *nh_grp, 4566 unsigned long *activity) 4567{ 4568 char *ratrad_pl; 4569 int i, err; 4570 4571 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4572 if (!ratrad_pl) 4573 return; 4574 4575 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4576 nh_grp->nhgi->count); 4577 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4578 if (err) 4579 goto out; 4580 4581 for (i = 0; i < nh_grp->nhgi->count; i++) { 4582 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4583 continue; 4584 bitmap_set(activity, i, 1); 4585 } 4586 4587out: 4588 kfree(ratrad_pl); 4589} 4590 4591#define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4592 4593static void 4594mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4595 const struct mlxsw_sp_nexthop_group *nh_grp) 4596{ 4597 unsigned long *activity; 4598 4599 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4600 if (!activity) 4601 return; 4602 4603 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4604 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4605 nh_grp->nhgi->count, activity); 4606 4607 bitmap_free(activity); 4608} 4609 4610static void 4611mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4612{ 4613 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4614 4615 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4616 msecs_to_jiffies(interval)); 4617} 4618 4619static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4620{ 4621 struct mlxsw_sp_nexthop_group_info *nhgi; 4622 struct mlxsw_sp_router *router; 4623 bool reschedule = false; 4624 4625 router = container_of(work, struct mlxsw_sp_router, 4626 nh_grp_activity_dw.work); 4627 4628 mutex_lock(&router->lock); 4629 4630 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4631 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4632 reschedule = true; 4633 } 4634 4635 mutex_unlock(&router->lock); 4636 4637 if (!reschedule) 4638 return; 4639 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4640} 4641 4642static int 4643mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4644 const struct nh_notifier_single_info *nh, 4645 struct netlink_ext_ack *extack) 4646{ 4647 int err = -EINVAL; 4648 4649 if (nh->is_fdb) 4650 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4651 else if (nh->has_encap) 4652 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4653 else 4654 err = 0; 4655 4656 return err; 4657} 4658 4659static int 4660mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4661 const struct nh_notifier_single_info *nh, 4662 struct netlink_ext_ack *extack) 4663{ 4664 int err; 4665 4666 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4667 if (err) 4668 return err; 4669 4670 /* Device only nexthops with an IPIP device are programmed as 4671 * encapsulating adjacency entries. 4672 */ 4673 if (!nh->gw_family && !nh->is_reject && 4674 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4675 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4676 return -EINVAL; 4677 } 4678 4679 return 0; 4680} 4681 4682static int 4683mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4684 const struct nh_notifier_grp_info *nh_grp, 4685 struct netlink_ext_ack *extack) 4686{ 4687 int i; 4688 4689 if (nh_grp->is_fdb) { 4690 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4691 return -EINVAL; 4692 } 4693 4694 for (i = 0; i < nh_grp->num_nh; i++) { 4695 const struct nh_notifier_single_info *nh; 4696 int err; 4697 4698 nh = &nh_grp->nh_entries[i].nh; 4699 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4700 extack); 4701 if (err) 4702 return err; 4703 } 4704 4705 return 0; 4706} 4707 4708static int 4709mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4710 const struct nh_notifier_res_table_info *nh_res_table, 4711 struct netlink_ext_ack *extack) 4712{ 4713 unsigned int alloc_size; 4714 bool valid_size = false; 4715 int err, i; 4716 4717 if (nh_res_table->num_nh_buckets < 32) { 4718 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4719 return -EINVAL; 4720 } 4721 4722 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4723 const struct mlxsw_sp_adj_grp_size_range *size_range; 4724 4725 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4726 4727 if (nh_res_table->num_nh_buckets >= size_range->start && 4728 nh_res_table->num_nh_buckets <= size_range->end) { 4729 valid_size = true; 4730 break; 4731 } 4732 } 4733 4734 if (!valid_size) { 4735 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4736 return -EINVAL; 4737 } 4738 4739 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4740 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4741 nh_res_table->num_nh_buckets, 4742 &alloc_size); 4743 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4744 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4745 return -EINVAL; 4746 } 4747 4748 return 0; 4749} 4750 4751static int 4752mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4753 const struct nh_notifier_res_table_info *nh_res_table, 4754 struct netlink_ext_ack *extack) 4755{ 4756 int err; 4757 u16 i; 4758 4759 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4760 nh_res_table, 4761 extack); 4762 if (err) 4763 return err; 4764 4765 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4766 const struct nh_notifier_single_info *nh; 4767 int err; 4768 4769 nh = &nh_res_table->nhs[i]; 4770 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4771 extack); 4772 if (err) 4773 return err; 4774 } 4775 4776 return 0; 4777} 4778 4779static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4780 unsigned long event, 4781 struct nh_notifier_info *info) 4782{ 4783 struct nh_notifier_single_info *nh; 4784 4785 if (event != NEXTHOP_EVENT_REPLACE && 4786 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4787 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4788 return 0; 4789 4790 switch (info->type) { 4791 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4792 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4793 info->extack); 4794 case NH_NOTIFIER_INFO_TYPE_GRP: 4795 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4796 info->nh_grp, 4797 info->extack); 4798 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4799 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4800 info->nh_res_table, 4801 info->extack); 4802 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4803 nh = &info->nh_res_bucket->new_nh; 4804 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4805 info->extack); 4806 default: 4807 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4808 return -EOPNOTSUPP; 4809 } 4810} 4811 4812static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4813 const struct nh_notifier_info *info) 4814{ 4815 const struct net_device *dev; 4816 4817 switch (info->type) { 4818 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4819 dev = info->nh->dev; 4820 return info->nh->gw_family || info->nh->is_reject || 4821 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4822 case NH_NOTIFIER_INFO_TYPE_GRP: 4823 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4824 /* Already validated earlier. */ 4825 return true; 4826 default: 4827 return false; 4828 } 4829} 4830 4831static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4832 struct mlxsw_sp_nexthop *nh) 4833{ 4834 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4835 4836 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4837 nh->should_offload = 1; 4838 /* While nexthops that discard packets do not forward packets 4839 * via an egress RIF, they still need to be programmed using a 4840 * valid RIF, so use the loopback RIF created during init. 4841 */ 4842 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4843} 4844 4845static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4846 struct mlxsw_sp_nexthop *nh) 4847{ 4848 nh->rif = NULL; 4849 nh->should_offload = 0; 4850} 4851 4852static int 4853mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4854 struct mlxsw_sp_nexthop_group *nh_grp, 4855 struct mlxsw_sp_nexthop *nh, 4856 struct nh_notifier_single_info *nh_obj, int weight) 4857{ 4858 struct net_device *dev = nh_obj->dev; 4859 int err; 4860 4861 nh->nhgi = nh_grp->nhgi; 4862 nh->nh_weight = weight; 4863 4864 switch (nh_obj->gw_family) { 4865 case AF_INET: 4866 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4867 nh->neigh_tbl = &arp_tbl; 4868 break; 4869 case AF_INET6: 4870 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4871#if IS_ENABLED(CONFIG_IPV6) 4872 nh->neigh_tbl = &nd_tbl; 4873#endif 4874 break; 4875 } 4876 4877 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4878 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4879 nh->ifindex = dev->ifindex; 4880 4881 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4882 if (err) 4883 goto err_type_init; 4884 4885 if (nh_obj->is_reject) 4886 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4887 4888 /* In a resilient nexthop group, all the nexthops must be written to 4889 * the adjacency table. Even if they do not have a valid neighbour or 4890 * RIF. 4891 */ 4892 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 4893 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4894 nh->should_offload = 1; 4895 } 4896 4897 return 0; 4898 4899err_type_init: 4900 list_del(&nh->router_list_node); 4901 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4902 return err; 4903} 4904 4905static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4906 struct mlxsw_sp_nexthop *nh) 4907{ 4908 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4909 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4910 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4911 list_del(&nh->router_list_node); 4912 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4913 nh->should_offload = 0; 4914} 4915 4916static int 4917mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4918 struct mlxsw_sp_nexthop_group *nh_grp, 4919 struct nh_notifier_info *info) 4920{ 4921 struct mlxsw_sp_nexthop_group_info *nhgi; 4922 struct mlxsw_sp_nexthop *nh; 4923 bool is_resilient = false; 4924 unsigned int nhs; 4925 int err, i; 4926 4927 switch (info->type) { 4928 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4929 nhs = 1; 4930 break; 4931 case NH_NOTIFIER_INFO_TYPE_GRP: 4932 nhs = info->nh_grp->num_nh; 4933 break; 4934 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4935 nhs = info->nh_res_table->num_nh_buckets; 4936 is_resilient = true; 4937 break; 4938 default: 4939 return -EINVAL; 4940 } 4941 4942 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4943 if (!nhgi) 4944 return -ENOMEM; 4945 nh_grp->nhgi = nhgi; 4946 nhgi->nh_grp = nh_grp; 4947 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4948 nhgi->is_resilient = is_resilient; 4949 nhgi->count = nhs; 4950 for (i = 0; i < nhgi->count; i++) { 4951 struct nh_notifier_single_info *nh_obj; 4952 int weight; 4953 4954 nh = &nhgi->nexthops[i]; 4955 switch (info->type) { 4956 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4957 nh_obj = info->nh; 4958 weight = 1; 4959 break; 4960 case NH_NOTIFIER_INFO_TYPE_GRP: 4961 nh_obj = &info->nh_grp->nh_entries[i].nh; 4962 weight = info->nh_grp->nh_entries[i].weight; 4963 break; 4964 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4965 nh_obj = &info->nh_res_table->nhs[i]; 4966 weight = 1; 4967 break; 4968 default: 4969 err = -EINVAL; 4970 goto err_nexthop_obj_init; 4971 } 4972 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4973 weight); 4974 if (err) 4975 goto err_nexthop_obj_init; 4976 } 4977 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 4978 if (err) 4979 goto err_group_inc; 4980 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4981 if (err) { 4982 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4983 goto err_group_refresh; 4984 } 4985 4986 /* Add resilient nexthop groups to a list so that the activity of their 4987 * nexthop buckets will be periodically queried and cleared. 4988 */ 4989 if (nhgi->is_resilient) { 4990 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4991 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 4992 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 4993 } 4994 4995 return 0; 4996 4997err_group_refresh: 4998 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 4999err_group_inc: 5000 i = nhgi->count; 5001err_nexthop_obj_init: 5002 for (i--; i >= 0; i--) { 5003 nh = &nhgi->nexthops[i]; 5004 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5005 } 5006 kfree(nhgi); 5007 return err; 5008} 5009 5010static void 5011mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5012 struct mlxsw_sp_nexthop_group *nh_grp) 5013{ 5014 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5015 struct mlxsw_sp_router *router = mlxsw_sp->router; 5016 int i; 5017 5018 if (nhgi->is_resilient) { 5019 list_del(&nhgi->list); 5020 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5021 cancel_delayed_work(&router->nh_grp_activity_dw); 5022 } 5023 5024 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5025 for (i = nhgi->count - 1; i >= 0; i--) { 5026 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5027 5028 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5029 } 5030 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5031 WARN_ON_ONCE(nhgi->adj_index_valid); 5032 kfree(nhgi); 5033} 5034 5035static struct mlxsw_sp_nexthop_group * 5036mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 5037 struct nh_notifier_info *info) 5038{ 5039 struct mlxsw_sp_nexthop_group *nh_grp; 5040 int err; 5041 5042 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5043 if (!nh_grp) 5044 return ERR_PTR(-ENOMEM); 5045 INIT_LIST_HEAD(&nh_grp->vr_list); 5046 err = rhashtable_init(&nh_grp->vr_ht, 5047 &mlxsw_sp_nexthop_group_vr_ht_params); 5048 if (err) 5049 goto err_nexthop_group_vr_ht_init; 5050 INIT_LIST_HEAD(&nh_grp->fib_list); 5051 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5052 nh_grp->obj.id = info->id; 5053 5054 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 5055 if (err) 5056 goto err_nexthop_group_info_init; 5057 5058 nh_grp->can_destroy = false; 5059 5060 return nh_grp; 5061 5062err_nexthop_group_info_init: 5063 rhashtable_destroy(&nh_grp->vr_ht); 5064err_nexthop_group_vr_ht_init: 5065 kfree(nh_grp); 5066 return ERR_PTR(err); 5067} 5068 5069static void 5070mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 5071 struct mlxsw_sp_nexthop_group *nh_grp) 5072{ 5073 if (!nh_grp->can_destroy) 5074 return; 5075 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 5076 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 5077 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5078 rhashtable_destroy(&nh_grp->vr_ht); 5079 kfree(nh_grp); 5080} 5081 5082static struct mlxsw_sp_nexthop_group * 5083mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 5084{ 5085 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 5086 5087 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5088 cmp_arg.id = id; 5089 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 5090 &cmp_arg, 5091 mlxsw_sp_nexthop_group_ht_params); 5092} 5093 5094static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5095 struct mlxsw_sp_nexthop_group *nh_grp) 5096{ 5097 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5098} 5099 5100static int 5101mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5102 struct mlxsw_sp_nexthop_group *nh_grp, 5103 struct mlxsw_sp_nexthop_group *old_nh_grp, 5104 struct netlink_ext_ack *extack) 5105{ 5106 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5107 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5108 int err; 5109 5110 old_nh_grp->nhgi = new_nhgi; 5111 new_nhgi->nh_grp = old_nh_grp; 5112 nh_grp->nhgi = old_nhgi; 5113 old_nhgi->nh_grp = nh_grp; 5114 5115 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5116 /* Both the old adjacency index and the new one are valid. 5117 * Routes are currently using the old one. Tell the device to 5118 * replace the old adjacency index with the new one. 5119 */ 5120 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5121 old_nhgi->adj_index, 5122 old_nhgi->ecmp_size); 5123 if (err) { 5124 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5125 goto err_out; 5126 } 5127 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5128 /* The old adjacency index is valid, while the new one is not. 5129 * Iterate over all the routes using the group and change them 5130 * to trap packets to the CPU. 5131 */ 5132 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5133 if (err) { 5134 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5135 goto err_out; 5136 } 5137 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5138 /* The old adjacency index is invalid, while the new one is. 5139 * Iterate over all the routes using the group and change them 5140 * to forward packets using the new valid index. 5141 */ 5142 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5143 if (err) { 5144 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5145 goto err_out; 5146 } 5147 } 5148 5149 /* Make sure the flags are set / cleared based on the new nexthop group 5150 * information. 5151 */ 5152 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5153 5154 /* At this point 'nh_grp' is just a shell that is not used by anyone 5155 * and its nexthop group info is the old info that was just replaced 5156 * with the new one. Remove it. 5157 */ 5158 nh_grp->can_destroy = true; 5159 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5160 5161 return 0; 5162 5163err_out: 5164 old_nhgi->nh_grp = old_nh_grp; 5165 nh_grp->nhgi = new_nhgi; 5166 new_nhgi->nh_grp = nh_grp; 5167 old_nh_grp->nhgi = old_nhgi; 5168 return err; 5169} 5170 5171static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5172 struct nh_notifier_info *info) 5173{ 5174 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5175 struct netlink_ext_ack *extack = info->extack; 5176 int err; 5177 5178 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5179 if (IS_ERR(nh_grp)) 5180 return PTR_ERR(nh_grp); 5181 5182 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5183 if (!old_nh_grp) 5184 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5185 else 5186 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5187 old_nh_grp, extack); 5188 5189 if (err) { 5190 nh_grp->can_destroy = true; 5191 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5192 } 5193 5194 return err; 5195} 5196 5197static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5198 struct nh_notifier_info *info) 5199{ 5200 struct mlxsw_sp_nexthop_group *nh_grp; 5201 5202 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5203 if (!nh_grp) 5204 return; 5205 5206 nh_grp->can_destroy = true; 5207 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5208 5209 /* If the group still has routes using it, then defer the delete 5210 * operation until the last route using it is deleted. 5211 */ 5212 if (!list_empty(&nh_grp->fib_list)) 5213 return; 5214 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5215} 5216 5217static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5218 u32 adj_index, char *ratr_pl) 5219{ 5220 MLXSW_REG_ZERO(ratr, ratr_pl); 5221 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5222 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5223 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5224 5225 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5226} 5227 5228static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5229{ 5230 /* Clear the opcode and activity on both the old and new payload as 5231 * they are irrelevant for the comparison. 5232 */ 5233 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5234 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5235 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5236 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5237 5238 /* If the contents of the adjacency entry are consistent with the 5239 * replacement request, then replacement was successful. 5240 */ 5241 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5242 return 0; 5243 5244 return -EINVAL; 5245} 5246 5247static int 5248mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5249 struct mlxsw_sp_nexthop *nh, 5250 struct nh_notifier_info *info) 5251{ 5252 u16 bucket_index = info->nh_res_bucket->bucket_index; 5253 struct netlink_ext_ack *extack = info->extack; 5254 bool force = info->nh_res_bucket->force; 5255 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5256 char ratr_pl[MLXSW_REG_RATR_LEN]; 5257 u32 adj_index; 5258 int err; 5259 5260 /* No point in trying an atomic replacement if the idle timer interval 5261 * is smaller than the interval in which we query and clear activity. 5262 */ 5263 if (!force && info->nh_res_bucket->idle_timer_ms < 5264 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5265 force = true; 5266 5267 adj_index = nh->nhgi->adj_index + bucket_index; 5268 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5269 if (err) { 5270 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5271 return err; 5272 } 5273 5274 if (!force) { 5275 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5276 ratr_pl_new); 5277 if (err) { 5278 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5279 return err; 5280 } 5281 5282 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5283 if (err) { 5284 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5285 return err; 5286 } 5287 } 5288 5289 nh->update = 0; 5290 nh->offloaded = 1; 5291 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5292 5293 return 0; 5294} 5295 5296static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5297 struct nh_notifier_info *info) 5298{ 5299 u16 bucket_index = info->nh_res_bucket->bucket_index; 5300 struct netlink_ext_ack *extack = info->extack; 5301 struct mlxsw_sp_nexthop_group_info *nhgi; 5302 struct nh_notifier_single_info *nh_obj; 5303 struct mlxsw_sp_nexthop_group *nh_grp; 5304 struct mlxsw_sp_nexthop *nh; 5305 int err; 5306 5307 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5308 if (!nh_grp) { 5309 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5310 return -EINVAL; 5311 } 5312 5313 nhgi = nh_grp->nhgi; 5314 5315 if (bucket_index >= nhgi->count) { 5316 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5317 return -EINVAL; 5318 } 5319 5320 nh = &nhgi->nexthops[bucket_index]; 5321 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5322 5323 nh_obj = &info->nh_res_bucket->new_nh; 5324 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5325 if (err) { 5326 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5327 goto err_nexthop_obj_init; 5328 } 5329 5330 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5331 if (err) 5332 goto err_nexthop_obj_bucket_adj_update; 5333 5334 return 0; 5335 5336err_nexthop_obj_bucket_adj_update: 5337 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5338err_nexthop_obj_init: 5339 nh_obj = &info->nh_res_bucket->old_nh; 5340 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5341 /* The old adjacency entry was not overwritten */ 5342 nh->update = 0; 5343 nh->offloaded = 1; 5344 return err; 5345} 5346 5347static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5348 unsigned long event, void *ptr) 5349{ 5350 struct nh_notifier_info *info = ptr; 5351 struct mlxsw_sp_router *router; 5352 int err = 0; 5353 5354 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5355 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5356 if (err) 5357 goto out; 5358 5359 mutex_lock(&router->lock); 5360 5361 switch (event) { 5362 case NEXTHOP_EVENT_REPLACE: 5363 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5364 break; 5365 case NEXTHOP_EVENT_DEL: 5366 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5367 break; 5368 case NEXTHOP_EVENT_BUCKET_REPLACE: 5369 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5370 info); 5371 break; 5372 default: 5373 break; 5374 } 5375 5376 mutex_unlock(&router->lock); 5377 5378out: 5379 return notifier_from_errno(err); 5380} 5381 5382static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5383 struct fib_info *fi) 5384{ 5385 const struct fib_nh *nh = fib_info_nh(fi, 0); 5386 5387 return nh->fib_nh_scope == RT_SCOPE_LINK || 5388 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5389} 5390 5391static int 5392mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5393 struct mlxsw_sp_nexthop_group *nh_grp) 5394{ 5395 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5396 struct mlxsw_sp_nexthop_group_info *nhgi; 5397 struct mlxsw_sp_nexthop *nh; 5398 int err, i; 5399 5400 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5401 if (!nhgi) 5402 return -ENOMEM; 5403 nh_grp->nhgi = nhgi; 5404 nhgi->nh_grp = nh_grp; 5405 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5406 nhgi->count = nhs; 5407 for (i = 0; i < nhgi->count; i++) { 5408 struct fib_nh *fib_nh; 5409 5410 nh = &nhgi->nexthops[i]; 5411 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5412 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5413 if (err) 5414 goto err_nexthop4_init; 5415 } 5416 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5417 if (err) 5418 goto err_group_inc; 5419 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5420 if (err) 5421 goto err_group_refresh; 5422 5423 return 0; 5424 5425err_group_refresh: 5426 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5427err_group_inc: 5428 i = nhgi->count; 5429err_nexthop4_init: 5430 for (i--; i >= 0; i--) { 5431 nh = &nhgi->nexthops[i]; 5432 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5433 } 5434 kfree(nhgi); 5435 return err; 5436} 5437 5438static void 5439mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5440 struct mlxsw_sp_nexthop_group *nh_grp) 5441{ 5442 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5443 int i; 5444 5445 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5446 for (i = nhgi->count - 1; i >= 0; i--) { 5447 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5448 5449 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5450 } 5451 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5452 WARN_ON_ONCE(nhgi->adj_index_valid); 5453 kfree(nhgi); 5454} 5455 5456static struct mlxsw_sp_nexthop_group * 5457mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5458{ 5459 struct mlxsw_sp_nexthop_group *nh_grp; 5460 int err; 5461 5462 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5463 if (!nh_grp) 5464 return ERR_PTR(-ENOMEM); 5465 INIT_LIST_HEAD(&nh_grp->vr_list); 5466 err = rhashtable_init(&nh_grp->vr_ht, 5467 &mlxsw_sp_nexthop_group_vr_ht_params); 5468 if (err) 5469 goto err_nexthop_group_vr_ht_init; 5470 INIT_LIST_HEAD(&nh_grp->fib_list); 5471 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5472 nh_grp->ipv4.fi = fi; 5473 fib_info_hold(fi); 5474 5475 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5476 if (err) 5477 goto err_nexthop_group_info_init; 5478 5479 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5480 if (err) 5481 goto err_nexthop_group_insert; 5482 5483 nh_grp->can_destroy = true; 5484 5485 return nh_grp; 5486 5487err_nexthop_group_insert: 5488 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5489err_nexthop_group_info_init: 5490 fib_info_put(fi); 5491 rhashtable_destroy(&nh_grp->vr_ht); 5492err_nexthop_group_vr_ht_init: 5493 kfree(nh_grp); 5494 return ERR_PTR(err); 5495} 5496 5497static void 5498mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5499 struct mlxsw_sp_nexthop_group *nh_grp) 5500{ 5501 if (!nh_grp->can_destroy) 5502 return; 5503 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5504 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5505 fib_info_put(nh_grp->ipv4.fi); 5506 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5507 rhashtable_destroy(&nh_grp->vr_ht); 5508 kfree(nh_grp); 5509} 5510 5511static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5512 struct mlxsw_sp_fib_entry *fib_entry, 5513 struct fib_info *fi) 5514{ 5515 struct mlxsw_sp_nexthop_group *nh_grp; 5516 5517 if (fi->nh) { 5518 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5519 fi->nh->id); 5520 if (WARN_ON_ONCE(!nh_grp)) 5521 return -EINVAL; 5522 goto out; 5523 } 5524 5525 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5526 if (!nh_grp) { 5527 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5528 if (IS_ERR(nh_grp)) 5529 return PTR_ERR(nh_grp); 5530 } 5531out: 5532 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5533 fib_entry->nh_group = nh_grp; 5534 return 0; 5535} 5536 5537static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5538 struct mlxsw_sp_fib_entry *fib_entry) 5539{ 5540 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5541 5542 list_del(&fib_entry->nexthop_group_node); 5543 if (!list_empty(&nh_grp->fib_list)) 5544 return; 5545 5546 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5547 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5548 return; 5549 } 5550 5551 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5552} 5553 5554static bool 5555mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5556{ 5557 struct mlxsw_sp_fib4_entry *fib4_entry; 5558 5559 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5560 common); 5561 return !fib4_entry->dscp; 5562} 5563 5564static bool 5565mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5566{ 5567 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5568 5569 switch (fib_entry->fib_node->fib->proto) { 5570 case MLXSW_SP_L3_PROTO_IPV4: 5571 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5572 return false; 5573 break; 5574 case MLXSW_SP_L3_PROTO_IPV6: 5575 break; 5576 } 5577 5578 switch (fib_entry->type) { 5579 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5580 return !!nh_group->nhgi->adj_index_valid; 5581 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5582 return !!nh_group->nhgi->nh_rif; 5583 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5584 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5585 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5586 return true; 5587 default: 5588 return false; 5589 } 5590} 5591 5592static struct mlxsw_sp_nexthop * 5593mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5594 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5595{ 5596 int i; 5597 5598 for (i = 0; i < nh_grp->nhgi->count; i++) { 5599 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5600 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5601 5602 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 5603 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5604 &rt->fib6_nh->fib_nh_gw6)) 5605 return nh; 5606 } 5607 5608 return NULL; 5609} 5610 5611static void 5612mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5613 struct fib_entry_notifier_info *fen_info) 5614{ 5615 u32 *p_dst = (u32 *) &fen_info->dst; 5616 struct fib_rt_info fri; 5617 5618 fri.fi = fen_info->fi; 5619 fri.tb_id = fen_info->tb_id; 5620 fri.dst = cpu_to_be32(*p_dst); 5621 fri.dst_len = fen_info->dst_len; 5622 fri.dscp = fen_info->dscp; 5623 fri.type = fen_info->type; 5624 fri.offload = false; 5625 fri.trap = false; 5626 fri.offload_failed = true; 5627 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5628} 5629 5630static void 5631mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5632 struct mlxsw_sp_fib_entry *fib_entry) 5633{ 5634 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5635 int dst_len = fib_entry->fib_node->key.prefix_len; 5636 struct mlxsw_sp_fib4_entry *fib4_entry; 5637 struct fib_rt_info fri; 5638 bool should_offload; 5639 5640 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5641 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5642 common); 5643 fri.fi = fib4_entry->fi; 5644 fri.tb_id = fib4_entry->tb_id; 5645 fri.dst = cpu_to_be32(*p_dst); 5646 fri.dst_len = dst_len; 5647 fri.dscp = fib4_entry->dscp; 5648 fri.type = fib4_entry->type; 5649 fri.offload = should_offload; 5650 fri.trap = !should_offload; 5651 fri.offload_failed = false; 5652 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5653} 5654 5655static void 5656mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5657 struct mlxsw_sp_fib_entry *fib_entry) 5658{ 5659 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5660 int dst_len = fib_entry->fib_node->key.prefix_len; 5661 struct mlxsw_sp_fib4_entry *fib4_entry; 5662 struct fib_rt_info fri; 5663 5664 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5665 common); 5666 fri.fi = fib4_entry->fi; 5667 fri.tb_id = fib4_entry->tb_id; 5668 fri.dst = cpu_to_be32(*p_dst); 5669 fri.dst_len = dst_len; 5670 fri.dscp = fib4_entry->dscp; 5671 fri.type = fib4_entry->type; 5672 fri.offload = false; 5673 fri.trap = false; 5674 fri.offload_failed = false; 5675 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5676} 5677 5678#if IS_ENABLED(CONFIG_IPV6) 5679static void 5680mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5681 struct fib6_info **rt_arr, 5682 unsigned int nrt6) 5683{ 5684 int i; 5685 5686 /* In IPv6 a multipath route is represented using multiple routes, so 5687 * we need to set the flags on all of them. 5688 */ 5689 for (i = 0; i < nrt6; i++) 5690 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5691 false, false, true); 5692} 5693#else 5694static void 5695mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5696 struct fib6_info **rt_arr, 5697 unsigned int nrt6) 5698{ 5699} 5700#endif 5701 5702#if IS_ENABLED(CONFIG_IPV6) 5703static void 5704mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5705 struct mlxsw_sp_fib_entry *fib_entry) 5706{ 5707 struct mlxsw_sp_fib6_entry *fib6_entry; 5708 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5709 bool should_offload; 5710 5711 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5712 5713 /* In IPv6 a multipath route is represented using multiple routes, so 5714 * we need to set the flags on all of them. 5715 */ 5716 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5717 common); 5718 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5719 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5720 should_offload, !should_offload, false); 5721} 5722#else 5723static void 5724mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5725 struct mlxsw_sp_fib_entry *fib_entry) 5726{ 5727} 5728#endif 5729 5730#if IS_ENABLED(CONFIG_IPV6) 5731static void 5732mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5733 struct mlxsw_sp_fib_entry *fib_entry) 5734{ 5735 struct mlxsw_sp_fib6_entry *fib6_entry; 5736 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5737 5738 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5739 common); 5740 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5741 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5742 false, false, false); 5743} 5744#else 5745static void 5746mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5747 struct mlxsw_sp_fib_entry *fib_entry) 5748{ 5749} 5750#endif 5751 5752static void 5753mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5754 struct mlxsw_sp_fib_entry *fib_entry) 5755{ 5756 switch (fib_entry->fib_node->fib->proto) { 5757 case MLXSW_SP_L3_PROTO_IPV4: 5758 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5759 break; 5760 case MLXSW_SP_L3_PROTO_IPV6: 5761 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5762 break; 5763 } 5764} 5765 5766static void 5767mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5768 struct mlxsw_sp_fib_entry *fib_entry) 5769{ 5770 switch (fib_entry->fib_node->fib->proto) { 5771 case MLXSW_SP_L3_PROTO_IPV4: 5772 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5773 break; 5774 case MLXSW_SP_L3_PROTO_IPV6: 5775 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5776 break; 5777 } 5778} 5779 5780static void 5781mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5782 struct mlxsw_sp_fib_entry *fib_entry, 5783 enum mlxsw_sp_fib_entry_op op) 5784{ 5785 switch (op) { 5786 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5787 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5788 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5789 break; 5790 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5791 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5792 break; 5793 default: 5794 break; 5795 } 5796} 5797 5798struct mlxsw_sp_fib_entry_op_ctx_basic { 5799 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5800}; 5801 5802static void 5803mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5804 enum mlxsw_sp_l3proto proto, 5805 enum mlxsw_sp_fib_entry_op op, 5806 u16 virtual_router, u8 prefix_len, 5807 unsigned char *addr, 5808 struct mlxsw_sp_fib_entry_priv *priv) 5809{ 5810 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5811 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5812 char *ralue_pl = op_ctx_basic->ralue_pl; 5813 enum mlxsw_reg_ralue_op ralue_op; 5814 5815 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5816 5817 switch (op) { 5818 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5819 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5820 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5821 break; 5822 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5823 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5824 break; 5825 default: 5826 WARN_ON_ONCE(1); 5827 return; 5828 } 5829 5830 switch (proto) { 5831 case MLXSW_SP_L3_PROTO_IPV4: 5832 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5833 virtual_router, prefix_len, (u32 *) addr); 5834 break; 5835 case MLXSW_SP_L3_PROTO_IPV6: 5836 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5837 virtual_router, prefix_len, addr); 5838 break; 5839 } 5840} 5841 5842static void 5843mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5844 enum mlxsw_reg_ralue_trap_action trap_action, 5845 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5846{ 5847 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5848 5849 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5850 trap_id, adjacency_index, ecmp_size); 5851} 5852 5853static void 5854mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5855 enum mlxsw_reg_ralue_trap_action trap_action, 5856 u16 trap_id, u16 local_erif) 5857{ 5858 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5859 5860 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5861 trap_id, local_erif); 5862} 5863 5864static void 5865mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5866{ 5867 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5868 5869 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5870} 5871 5872static void 5873mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5874 u32 tunnel_ptr) 5875{ 5876 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5877 5878 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5879} 5880 5881static int 5882mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5883 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5884 bool *postponed_for_bulk) 5885{ 5886 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5887 5888 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5889 op_ctx_basic->ralue_pl); 5890} 5891 5892static bool 5893mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5894{ 5895 return true; 5896} 5897 5898static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5899 struct mlxsw_sp_fib_entry *fib_entry, 5900 enum mlxsw_sp_fib_entry_op op) 5901{ 5902 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5903 5904 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5905 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5906 fib_entry->fib_node->key.prefix_len, 5907 fib_entry->fib_node->key.addr, 5908 fib_entry->priv); 5909} 5910 5911static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5912 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5913 const struct mlxsw_sp_router_ll_ops *ll_ops) 5914{ 5915 bool postponed_for_bulk = false; 5916 int err; 5917 5918 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5919 if (!postponed_for_bulk) 5920 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5921 return err; 5922} 5923 5924static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5925 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5926 struct mlxsw_sp_fib_entry *fib_entry, 5927 enum mlxsw_sp_fib_entry_op op) 5928{ 5929 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5930 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5931 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5932 enum mlxsw_reg_ralue_trap_action trap_action; 5933 u16 trap_id = 0; 5934 u32 adjacency_index = 0; 5935 u16 ecmp_size = 0; 5936 5937 /* In case the nexthop group adjacency index is valid, use it 5938 * with provided ECMP size. Otherwise, setup trap and pass 5939 * traffic to kernel. 5940 */ 5941 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5942 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5943 adjacency_index = nhgi->adj_index; 5944 ecmp_size = nhgi->ecmp_size; 5945 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5946 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5947 adjacency_index = mlxsw_sp->router->adj_trap_index; 5948 ecmp_size = 1; 5949 } else { 5950 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5951 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5952 } 5953 5954 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5955 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5956 adjacency_index, ecmp_size); 5957 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5958} 5959 5960static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5961 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5962 struct mlxsw_sp_fib_entry *fib_entry, 5963 enum mlxsw_sp_fib_entry_op op) 5964{ 5965 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5966 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5967 enum mlxsw_reg_ralue_trap_action trap_action; 5968 u16 trap_id = 0; 5969 u16 rif_index = 0; 5970 5971 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5972 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5973 rif_index = rif->rif_index; 5974 } else { 5975 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5976 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5977 } 5978 5979 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5980 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5981 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5982} 5983 5984static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5985 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5986 struct mlxsw_sp_fib_entry *fib_entry, 5987 enum mlxsw_sp_fib_entry_op op) 5988{ 5989 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5990 5991 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5992 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5993 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5994} 5995 5996static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5997 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5998 struct mlxsw_sp_fib_entry *fib_entry, 5999 enum mlxsw_sp_fib_entry_op op) 6000{ 6001 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6002 enum mlxsw_reg_ralue_trap_action trap_action; 6003 6004 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 6005 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6006 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 6007 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6008} 6009 6010static int 6011mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 6012 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6013 struct mlxsw_sp_fib_entry *fib_entry, 6014 enum mlxsw_sp_fib_entry_op op) 6015{ 6016 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6017 enum mlxsw_reg_ralue_trap_action trap_action; 6018 u16 trap_id; 6019 6020 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6021 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 6022 6023 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6024 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 6025 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6026} 6027 6028static int 6029mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 6030 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6031 struct mlxsw_sp_fib_entry *fib_entry, 6032 enum mlxsw_sp_fib_entry_op op) 6033{ 6034 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6035 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 6036 const struct mlxsw_sp_ipip_ops *ipip_ops; 6037 int err; 6038 6039 if (WARN_ON(!ipip_entry)) 6040 return -EINVAL; 6041 6042 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 6043 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 6044 fib_entry->decap.tunnel_index); 6045 if (err) 6046 return err; 6047 6048 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6049 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 6050 fib_entry->decap.tunnel_index); 6051 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6052} 6053 6054static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 6055 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6056 struct mlxsw_sp_fib_entry *fib_entry, 6057 enum mlxsw_sp_fib_entry_op op) 6058{ 6059 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6060 6061 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6062 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 6063 fib_entry->decap.tunnel_index); 6064 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6065} 6066 6067static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6068 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6069 struct mlxsw_sp_fib_entry *fib_entry, 6070 enum mlxsw_sp_fib_entry_op op) 6071{ 6072 switch (fib_entry->type) { 6073 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 6074 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 6075 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 6076 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 6077 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 6078 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 6079 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 6080 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 6081 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 6082 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 6083 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6084 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 6085 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 6086 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 6087 } 6088 return -EINVAL; 6089} 6090 6091static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6092 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6093 struct mlxsw_sp_fib_entry *fib_entry, 6094 enum mlxsw_sp_fib_entry_op op) 6095{ 6096 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 6097 6098 if (err) 6099 return err; 6100 6101 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 6102 6103 return err; 6104} 6105 6106static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6107 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6108 struct mlxsw_sp_fib_entry *fib_entry, 6109 bool is_new) 6110{ 6111 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 6112 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 6113 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 6114} 6115 6116static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6117 struct mlxsw_sp_fib_entry *fib_entry) 6118{ 6119 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6120 6121 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6122 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 6123} 6124 6125static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 6126 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6127 struct mlxsw_sp_fib_entry *fib_entry) 6128{ 6129 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6130 6131 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 6132 return 0; 6133 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 6134 MLXSW_SP_FIB_ENTRY_OP_DELETE); 6135} 6136 6137static int 6138mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6139 const struct fib_entry_notifier_info *fen_info, 6140 struct mlxsw_sp_fib_entry *fib_entry) 6141{ 6142 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6143 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 6144 struct mlxsw_sp_router *router = mlxsw_sp->router; 6145 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 6146 int ifindex = nhgi->nexthops[0].ifindex; 6147 struct mlxsw_sp_ipip_entry *ipip_entry; 6148 6149 switch (fen_info->type) { 6150 case RTN_LOCAL: 6151 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6152 MLXSW_SP_L3_PROTO_IPV4, dip); 6153 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6154 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6155 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6156 fib_entry, 6157 ipip_entry); 6158 } 6159 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6160 MLXSW_SP_L3_PROTO_IPV4, 6161 &dip)) { 6162 u32 tunnel_index; 6163 6164 tunnel_index = router->nve_decap_config.tunnel_index; 6165 fib_entry->decap.tunnel_index = tunnel_index; 6166 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6167 return 0; 6168 } 6169 fallthrough; 6170 case RTN_BROADCAST: 6171 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6172 return 0; 6173 case RTN_BLACKHOLE: 6174 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6175 return 0; 6176 case RTN_UNREACHABLE: 6177 case RTN_PROHIBIT: 6178 /* Packets hitting these routes need to be trapped, but 6179 * can do so with a lower priority than packets directed 6180 * at the host, so use action type local instead of trap. 6181 */ 6182 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6183 return 0; 6184 case RTN_UNICAST: 6185 if (nhgi->gateway) 6186 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6187 else 6188 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6189 return 0; 6190 default: 6191 return -EINVAL; 6192 } 6193} 6194 6195static void 6196mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6197 struct mlxsw_sp_fib_entry *fib_entry) 6198{ 6199 switch (fib_entry->type) { 6200 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6201 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6202 break; 6203 default: 6204 break; 6205 } 6206} 6207 6208static void 6209mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6210 struct mlxsw_sp_fib4_entry *fib4_entry) 6211{ 6212 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6213} 6214 6215static struct mlxsw_sp_fib4_entry * 6216mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6217 struct mlxsw_sp_fib_node *fib_node, 6218 const struct fib_entry_notifier_info *fen_info) 6219{ 6220 struct mlxsw_sp_fib4_entry *fib4_entry; 6221 struct mlxsw_sp_fib_entry *fib_entry; 6222 int err; 6223 6224 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6225 if (!fib4_entry) 6226 return ERR_PTR(-ENOMEM); 6227 fib_entry = &fib4_entry->common; 6228 6229 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6230 if (IS_ERR(fib_entry->priv)) { 6231 err = PTR_ERR(fib_entry->priv); 6232 goto err_fib_entry_priv_create; 6233 } 6234 6235 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6236 if (err) 6237 goto err_nexthop4_group_get; 6238 6239 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6240 fib_node->fib); 6241 if (err) 6242 goto err_nexthop_group_vr_link; 6243 6244 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6245 if (err) 6246 goto err_fib4_entry_type_set; 6247 6248 fib4_entry->fi = fen_info->fi; 6249 fib_info_hold(fib4_entry->fi); 6250 fib4_entry->tb_id = fen_info->tb_id; 6251 fib4_entry->type = fen_info->type; 6252 fib4_entry->dscp = fen_info->dscp; 6253 6254 fib_entry->fib_node = fib_node; 6255 6256 return fib4_entry; 6257 6258err_fib4_entry_type_set: 6259 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6260err_nexthop_group_vr_link: 6261 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6262err_nexthop4_group_get: 6263 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6264err_fib_entry_priv_create: 6265 kfree(fib4_entry); 6266 return ERR_PTR(err); 6267} 6268 6269static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6270 struct mlxsw_sp_fib4_entry *fib4_entry) 6271{ 6272 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6273 6274 fib_info_put(fib4_entry->fi); 6275 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6276 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6277 fib_node->fib); 6278 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6279 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 6280 kfree(fib4_entry); 6281} 6282 6283static struct mlxsw_sp_fib4_entry * 6284mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6285 const struct fib_entry_notifier_info *fen_info) 6286{ 6287 struct mlxsw_sp_fib4_entry *fib4_entry; 6288 struct mlxsw_sp_fib_node *fib_node; 6289 struct mlxsw_sp_fib *fib; 6290 struct mlxsw_sp_vr *vr; 6291 6292 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6293 if (!vr) 6294 return NULL; 6295 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6296 6297 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6298 sizeof(fen_info->dst), 6299 fen_info->dst_len); 6300 if (!fib_node) 6301 return NULL; 6302 6303 fib4_entry = container_of(fib_node->fib_entry, 6304 struct mlxsw_sp_fib4_entry, common); 6305 if (fib4_entry->tb_id == fen_info->tb_id && 6306 fib4_entry->dscp == fen_info->dscp && 6307 fib4_entry->type == fen_info->type && 6308 fib4_entry->fi == fen_info->fi) 6309 return fib4_entry; 6310 6311 return NULL; 6312} 6313 6314static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6315 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6316 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6317 .key_len = sizeof(struct mlxsw_sp_fib_key), 6318 .automatic_shrinking = true, 6319}; 6320 6321static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6322 struct mlxsw_sp_fib_node *fib_node) 6323{ 6324 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6325 mlxsw_sp_fib_ht_params); 6326} 6327 6328static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6329 struct mlxsw_sp_fib_node *fib_node) 6330{ 6331 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6332 mlxsw_sp_fib_ht_params); 6333} 6334 6335static struct mlxsw_sp_fib_node * 6336mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6337 size_t addr_len, unsigned char prefix_len) 6338{ 6339 struct mlxsw_sp_fib_key key; 6340 6341 memset(&key, 0, sizeof(key)); 6342 memcpy(key.addr, addr, addr_len); 6343 key.prefix_len = prefix_len; 6344 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6345} 6346 6347static struct mlxsw_sp_fib_node * 6348mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6349 size_t addr_len, unsigned char prefix_len) 6350{ 6351 struct mlxsw_sp_fib_node *fib_node; 6352 6353 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6354 if (!fib_node) 6355 return NULL; 6356 6357 list_add(&fib_node->list, &fib->node_list); 6358 memcpy(fib_node->key.addr, addr, addr_len); 6359 fib_node->key.prefix_len = prefix_len; 6360 6361 return fib_node; 6362} 6363 6364static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6365{ 6366 list_del(&fib_node->list); 6367 kfree(fib_node); 6368} 6369 6370static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6371 struct mlxsw_sp_fib_node *fib_node) 6372{ 6373 struct mlxsw_sp_prefix_usage req_prefix_usage; 6374 struct mlxsw_sp_fib *fib = fib_node->fib; 6375 struct mlxsw_sp_lpm_tree *lpm_tree; 6376 int err; 6377 6378 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6379 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6380 goto out; 6381 6382 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6383 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6384 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6385 fib->proto); 6386 if (IS_ERR(lpm_tree)) 6387 return PTR_ERR(lpm_tree); 6388 6389 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6390 if (err) 6391 goto err_lpm_tree_replace; 6392 6393out: 6394 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6395 return 0; 6396 6397err_lpm_tree_replace: 6398 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6399 return err; 6400} 6401 6402static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6403 struct mlxsw_sp_fib_node *fib_node) 6404{ 6405 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6406 struct mlxsw_sp_prefix_usage req_prefix_usage; 6407 struct mlxsw_sp_fib *fib = fib_node->fib; 6408 int err; 6409 6410 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6411 return; 6412 /* Try to construct a new LPM tree from the current prefix usage 6413 * minus the unused one. If we fail, continue using the old one. 6414 */ 6415 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6416 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6417 fib_node->key.prefix_len); 6418 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6419 fib->proto); 6420 if (IS_ERR(lpm_tree)) 6421 return; 6422 6423 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6424 if (err) 6425 goto err_lpm_tree_replace; 6426 6427 return; 6428 6429err_lpm_tree_replace: 6430 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6431} 6432 6433static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6434 struct mlxsw_sp_fib_node *fib_node, 6435 struct mlxsw_sp_fib *fib) 6436{ 6437 int err; 6438 6439 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6440 if (err) 6441 return err; 6442 fib_node->fib = fib; 6443 6444 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6445 if (err) 6446 goto err_fib_lpm_tree_link; 6447 6448 return 0; 6449 6450err_fib_lpm_tree_link: 6451 fib_node->fib = NULL; 6452 mlxsw_sp_fib_node_remove(fib, fib_node); 6453 return err; 6454} 6455 6456static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6457 struct mlxsw_sp_fib_node *fib_node) 6458{ 6459 struct mlxsw_sp_fib *fib = fib_node->fib; 6460 6461 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6462 fib_node->fib = NULL; 6463 mlxsw_sp_fib_node_remove(fib, fib_node); 6464} 6465 6466static struct mlxsw_sp_fib_node * 6467mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6468 size_t addr_len, unsigned char prefix_len, 6469 enum mlxsw_sp_l3proto proto) 6470{ 6471 struct mlxsw_sp_fib_node *fib_node; 6472 struct mlxsw_sp_fib *fib; 6473 struct mlxsw_sp_vr *vr; 6474 int err; 6475 6476 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6477 if (IS_ERR(vr)) 6478 return ERR_CAST(vr); 6479 fib = mlxsw_sp_vr_fib(vr, proto); 6480 6481 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6482 if (fib_node) 6483 return fib_node; 6484 6485 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6486 if (!fib_node) { 6487 err = -ENOMEM; 6488 goto err_fib_node_create; 6489 } 6490 6491 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6492 if (err) 6493 goto err_fib_node_init; 6494 6495 return fib_node; 6496 6497err_fib_node_init: 6498 mlxsw_sp_fib_node_destroy(fib_node); 6499err_fib_node_create: 6500 mlxsw_sp_vr_put(mlxsw_sp, vr); 6501 return ERR_PTR(err); 6502} 6503 6504static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6505 struct mlxsw_sp_fib_node *fib_node) 6506{ 6507 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6508 6509 if (fib_node->fib_entry) 6510 return; 6511 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6512 mlxsw_sp_fib_node_destroy(fib_node); 6513 mlxsw_sp_vr_put(mlxsw_sp, vr); 6514} 6515 6516static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6517 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6518 struct mlxsw_sp_fib_entry *fib_entry) 6519{ 6520 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6521 bool is_new = !fib_node->fib_entry; 6522 int err; 6523 6524 fib_node->fib_entry = fib_entry; 6525 6526 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 6527 if (err) 6528 goto err_fib_entry_update; 6529 6530 return 0; 6531 6532err_fib_entry_update: 6533 fib_node->fib_entry = NULL; 6534 return err; 6535} 6536 6537static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6538 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6539 struct mlxsw_sp_fib_entry *fib_entry) 6540{ 6541 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6542 int err; 6543 6544 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 6545 fib_node->fib_entry = NULL; 6546 return err; 6547} 6548 6549static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6550 struct mlxsw_sp_fib_entry *fib_entry) 6551{ 6552 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6553 6554 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6555 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 6556} 6557 6558static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6559{ 6560 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6561 struct mlxsw_sp_fib4_entry *fib4_replaced; 6562 6563 if (!fib_node->fib_entry) 6564 return true; 6565 6566 fib4_replaced = container_of(fib_node->fib_entry, 6567 struct mlxsw_sp_fib4_entry, common); 6568 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6569 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6570 return false; 6571 6572 return true; 6573} 6574 6575static int 6576mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6577 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6578 const struct fib_entry_notifier_info *fen_info) 6579{ 6580 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6581 struct mlxsw_sp_fib_entry *replaced; 6582 struct mlxsw_sp_fib_node *fib_node; 6583 int err; 6584 6585 if (fen_info->fi->nh && 6586 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6587 return 0; 6588 6589 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6590 &fen_info->dst, sizeof(fen_info->dst), 6591 fen_info->dst_len, 6592 MLXSW_SP_L3_PROTO_IPV4); 6593 if (IS_ERR(fib_node)) { 6594 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6595 return PTR_ERR(fib_node); 6596 } 6597 6598 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6599 if (IS_ERR(fib4_entry)) { 6600 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6601 err = PTR_ERR(fib4_entry); 6602 goto err_fib4_entry_create; 6603 } 6604 6605 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6606 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6607 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6608 return 0; 6609 } 6610 6611 replaced = fib_node->fib_entry; 6612 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 6613 if (err) { 6614 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6615 goto err_fib_node_entry_link; 6616 } 6617 6618 /* Nothing to replace */ 6619 if (!replaced) 6620 return 0; 6621 6622 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6623 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6624 common); 6625 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6626 6627 return 0; 6628 6629err_fib_node_entry_link: 6630 fib_node->fib_entry = replaced; 6631 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6632err_fib4_entry_create: 6633 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6634 return err; 6635} 6636 6637static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6638 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6639 struct fib_entry_notifier_info *fen_info) 6640{ 6641 struct mlxsw_sp_fib4_entry *fib4_entry; 6642 struct mlxsw_sp_fib_node *fib_node; 6643 int err; 6644 6645 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6646 if (!fib4_entry) 6647 return 0; 6648 fib_node = fib4_entry->common.fib_node; 6649 6650 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 6651 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6652 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6653 return err; 6654} 6655 6656static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6657{ 6658 /* Multicast routes aren't supported, so ignore them. Neighbour 6659 * Discovery packets are specifically trapped. 6660 */ 6661 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6662 return true; 6663 6664 /* Cloned routes are irrelevant in the forwarding path. */ 6665 if (rt->fib6_flags & RTF_CACHE) 6666 return true; 6667 6668 return false; 6669} 6670 6671static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6672{ 6673 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6674 6675 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6676 if (!mlxsw_sp_rt6) 6677 return ERR_PTR(-ENOMEM); 6678 6679 /* In case of route replace, replaced route is deleted with 6680 * no notification. Take reference to prevent accessing freed 6681 * memory. 6682 */ 6683 mlxsw_sp_rt6->rt = rt; 6684 fib6_info_hold(rt); 6685 6686 return mlxsw_sp_rt6; 6687} 6688 6689#if IS_ENABLED(CONFIG_IPV6) 6690static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6691{ 6692 fib6_info_release(rt); 6693} 6694#else 6695static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6696{ 6697} 6698#endif 6699 6700static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6701{ 6702 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6703 6704 if (!mlxsw_sp_rt6->rt->nh) 6705 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6706 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6707 kfree(mlxsw_sp_rt6); 6708} 6709 6710static struct fib6_info * 6711mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6712{ 6713 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6714 list)->rt; 6715} 6716 6717static struct mlxsw_sp_rt6 * 6718mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6719 const struct fib6_info *rt) 6720{ 6721 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6722 6723 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6724 if (mlxsw_sp_rt6->rt == rt) 6725 return mlxsw_sp_rt6; 6726 } 6727 6728 return NULL; 6729} 6730 6731static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6732 const struct fib6_info *rt, 6733 enum mlxsw_sp_ipip_type *ret) 6734{ 6735 return rt->fib6_nh->fib_nh_dev && 6736 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6737} 6738 6739static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6740 struct mlxsw_sp_nexthop_group *nh_grp, 6741 struct mlxsw_sp_nexthop *nh, 6742 const struct fib6_info *rt) 6743{ 6744 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6745 int err; 6746 6747 nh->nhgi = nh_grp->nhgi; 6748 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6749 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6750#if IS_ENABLED(CONFIG_IPV6) 6751 nh->neigh_tbl = &nd_tbl; 6752#endif 6753 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6754 6755 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6756 6757 if (!dev) 6758 return 0; 6759 nh->ifindex = dev->ifindex; 6760 6761 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6762 if (err) 6763 goto err_nexthop_type_init; 6764 6765 return 0; 6766 6767err_nexthop_type_init: 6768 list_del(&nh->router_list_node); 6769 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6770 return err; 6771} 6772 6773static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6774 struct mlxsw_sp_nexthop *nh) 6775{ 6776 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6777 list_del(&nh->router_list_node); 6778 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6779} 6780 6781static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6782 const struct fib6_info *rt) 6783{ 6784 return rt->fib6_nh->fib_nh_gw_family || 6785 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6786} 6787 6788static int 6789mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6790 struct mlxsw_sp_nexthop_group *nh_grp, 6791 struct mlxsw_sp_fib6_entry *fib6_entry) 6792{ 6793 struct mlxsw_sp_nexthop_group_info *nhgi; 6794 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6795 struct mlxsw_sp_nexthop *nh; 6796 int err, i; 6797 6798 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6799 GFP_KERNEL); 6800 if (!nhgi) 6801 return -ENOMEM; 6802 nh_grp->nhgi = nhgi; 6803 nhgi->nh_grp = nh_grp; 6804 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6805 struct mlxsw_sp_rt6, list); 6806 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6807 nhgi->count = fib6_entry->nrt6; 6808 for (i = 0; i < nhgi->count; i++) { 6809 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6810 6811 nh = &nhgi->nexthops[i]; 6812 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6813 if (err) 6814 goto err_nexthop6_init; 6815 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6816 } 6817 nh_grp->nhgi = nhgi; 6818 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 6819 if (err) 6820 goto err_group_inc; 6821 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6822 if (err) 6823 goto err_group_refresh; 6824 6825 return 0; 6826 6827err_group_refresh: 6828 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6829err_group_inc: 6830 i = nhgi->count; 6831err_nexthop6_init: 6832 for (i--; i >= 0; i--) { 6833 nh = &nhgi->nexthops[i]; 6834 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6835 } 6836 kfree(nhgi); 6837 return err; 6838} 6839 6840static void 6841mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6842 struct mlxsw_sp_nexthop_group *nh_grp) 6843{ 6844 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6845 int i; 6846 6847 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6848 for (i = nhgi->count - 1; i >= 0; i--) { 6849 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6850 6851 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6852 } 6853 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6854 WARN_ON_ONCE(nhgi->adj_index_valid); 6855 kfree(nhgi); 6856} 6857 6858static struct mlxsw_sp_nexthop_group * 6859mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6860 struct mlxsw_sp_fib6_entry *fib6_entry) 6861{ 6862 struct mlxsw_sp_nexthop_group *nh_grp; 6863 int err; 6864 6865 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6866 if (!nh_grp) 6867 return ERR_PTR(-ENOMEM); 6868 INIT_LIST_HEAD(&nh_grp->vr_list); 6869 err = rhashtable_init(&nh_grp->vr_ht, 6870 &mlxsw_sp_nexthop_group_vr_ht_params); 6871 if (err) 6872 goto err_nexthop_group_vr_ht_init; 6873 INIT_LIST_HEAD(&nh_grp->fib_list); 6874 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6875 6876 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6877 if (err) 6878 goto err_nexthop_group_info_init; 6879 6880 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6881 if (err) 6882 goto err_nexthop_group_insert; 6883 6884 nh_grp->can_destroy = true; 6885 6886 return nh_grp; 6887 6888err_nexthop_group_insert: 6889 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6890err_nexthop_group_info_init: 6891 rhashtable_destroy(&nh_grp->vr_ht); 6892err_nexthop_group_vr_ht_init: 6893 kfree(nh_grp); 6894 return ERR_PTR(err); 6895} 6896 6897static void 6898mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6899 struct mlxsw_sp_nexthop_group *nh_grp) 6900{ 6901 if (!nh_grp->can_destroy) 6902 return; 6903 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6904 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6905 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6906 rhashtable_destroy(&nh_grp->vr_ht); 6907 kfree(nh_grp); 6908} 6909 6910static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6911 struct mlxsw_sp_fib6_entry *fib6_entry) 6912{ 6913 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6914 struct mlxsw_sp_nexthop_group *nh_grp; 6915 6916 if (rt->nh) { 6917 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6918 rt->nh->id); 6919 if (WARN_ON_ONCE(!nh_grp)) 6920 return -EINVAL; 6921 goto out; 6922 } 6923 6924 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6925 if (!nh_grp) { 6926 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6927 if (IS_ERR(nh_grp)) 6928 return PTR_ERR(nh_grp); 6929 } 6930 6931 /* The route and the nexthop are described by the same struct, so we 6932 * need to the update the nexthop offload indication for the new route. 6933 */ 6934 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6935 6936out: 6937 list_add_tail(&fib6_entry->common.nexthop_group_node, 6938 &nh_grp->fib_list); 6939 fib6_entry->common.nh_group = nh_grp; 6940 6941 return 0; 6942} 6943 6944static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6945 struct mlxsw_sp_fib_entry *fib_entry) 6946{ 6947 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6948 6949 list_del(&fib_entry->nexthop_group_node); 6950 if (!list_empty(&nh_grp->fib_list)) 6951 return; 6952 6953 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6954 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6955 return; 6956 } 6957 6958 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6959} 6960 6961static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6962 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6963 struct mlxsw_sp_fib6_entry *fib6_entry) 6964{ 6965 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6966 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6967 int err; 6968 6969 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6970 fib6_entry->common.nh_group = NULL; 6971 list_del(&fib6_entry->common.nexthop_group_node); 6972 6973 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6974 if (err) 6975 goto err_nexthop6_group_get; 6976 6977 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6978 fib_node->fib); 6979 if (err) 6980 goto err_nexthop_group_vr_link; 6981 6982 /* In case this entry is offloaded, then the adjacency index 6983 * currently associated with it in the device's table is that 6984 * of the old group. Start using the new one instead. 6985 */ 6986 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6987 &fib6_entry->common, false); 6988 if (err) 6989 goto err_fib_entry_update; 6990 6991 if (list_empty(&old_nh_grp->fib_list)) 6992 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6993 6994 return 0; 6995 6996err_fib_entry_update: 6997 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6998 fib_node->fib); 6999err_nexthop_group_vr_link: 7000 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7001err_nexthop6_group_get: 7002 list_add_tail(&fib6_entry->common.nexthop_group_node, 7003 &old_nh_grp->fib_list); 7004 fib6_entry->common.nh_group = old_nh_grp; 7005 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 7006 return err; 7007} 7008 7009static int 7010mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 7011 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7012 struct mlxsw_sp_fib6_entry *fib6_entry, 7013 struct fib6_info **rt_arr, unsigned int nrt6) 7014{ 7015 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7016 int err, i; 7017 7018 for (i = 0; i < nrt6; i++) { 7019 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7020 if (IS_ERR(mlxsw_sp_rt6)) { 7021 err = PTR_ERR(mlxsw_sp_rt6); 7022 goto err_rt6_unwind; 7023 } 7024 7025 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7026 fib6_entry->nrt6++; 7027 } 7028 7029 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 7030 if (err) 7031 goto err_rt6_unwind; 7032 7033 return 0; 7034 7035err_rt6_unwind: 7036 for (; i > 0; i--) { 7037 fib6_entry->nrt6--; 7038 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7039 struct mlxsw_sp_rt6, list); 7040 list_del(&mlxsw_sp_rt6->list); 7041 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7042 } 7043 return err; 7044} 7045 7046static void 7047mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 7048 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7049 struct mlxsw_sp_fib6_entry *fib6_entry, 7050 struct fib6_info **rt_arr, unsigned int nrt6) 7051{ 7052 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7053 int i; 7054 7055 for (i = 0; i < nrt6; i++) { 7056 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 7057 rt_arr[i]); 7058 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 7059 continue; 7060 7061 fib6_entry->nrt6--; 7062 list_del(&mlxsw_sp_rt6->list); 7063 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7064 } 7065 7066 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 7067} 7068 7069static int 7070mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 7071 struct mlxsw_sp_fib_entry *fib_entry, 7072 const struct fib6_info *rt) 7073{ 7074 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 7075 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 7076 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 7077 struct mlxsw_sp_router *router = mlxsw_sp->router; 7078 int ifindex = nhgi->nexthops[0].ifindex; 7079 struct mlxsw_sp_ipip_entry *ipip_entry; 7080 7081 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7082 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 7083 MLXSW_SP_L3_PROTO_IPV6, 7084 dip); 7085 7086 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 7087 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 7088 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 7089 ipip_entry); 7090 } 7091 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 7092 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 7093 u32 tunnel_index; 7094 7095 tunnel_index = router->nve_decap_config.tunnel_index; 7096 fib_entry->decap.tunnel_index = tunnel_index; 7097 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 7098 } 7099 7100 return 0; 7101} 7102 7103static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 7104 struct mlxsw_sp_fib_entry *fib_entry, 7105 const struct fib6_info *rt) 7106{ 7107 if (rt->fib6_flags & RTF_LOCAL) 7108 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 7109 rt); 7110 if (rt->fib6_flags & RTF_ANYCAST) 7111 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7112 else if (rt->fib6_type == RTN_BLACKHOLE) 7113 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 7114 else if (rt->fib6_flags & RTF_REJECT) 7115 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 7116 else if (fib_entry->nh_group->nhgi->gateway) 7117 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 7118 else 7119 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 7120 7121 return 0; 7122} 7123 7124static void 7125mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 7126{ 7127 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 7128 7129 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 7130 list) { 7131 fib6_entry->nrt6--; 7132 list_del(&mlxsw_sp_rt6->list); 7133 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7134 } 7135} 7136 7137static struct mlxsw_sp_fib6_entry * 7138mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 7139 struct mlxsw_sp_fib_node *fib_node, 7140 struct fib6_info **rt_arr, unsigned int nrt6) 7141{ 7142 struct mlxsw_sp_fib6_entry *fib6_entry; 7143 struct mlxsw_sp_fib_entry *fib_entry; 7144 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7145 int err, i; 7146 7147 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 7148 if (!fib6_entry) 7149 return ERR_PTR(-ENOMEM); 7150 fib_entry = &fib6_entry->common; 7151 7152 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 7153 if (IS_ERR(fib_entry->priv)) { 7154 err = PTR_ERR(fib_entry->priv); 7155 goto err_fib_entry_priv_create; 7156 } 7157 7158 INIT_LIST_HEAD(&fib6_entry->rt6_list); 7159 7160 for (i = 0; i < nrt6; i++) { 7161 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7162 if (IS_ERR(mlxsw_sp_rt6)) { 7163 err = PTR_ERR(mlxsw_sp_rt6); 7164 goto err_rt6_unwind; 7165 } 7166 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7167 fib6_entry->nrt6++; 7168 } 7169 7170 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7171 if (err) 7172 goto err_rt6_unwind; 7173 7174 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 7175 fib_node->fib); 7176 if (err) 7177 goto err_nexthop_group_vr_link; 7178 7179 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 7180 if (err) 7181 goto err_fib6_entry_type_set; 7182 7183 fib_entry->fib_node = fib_node; 7184 7185 return fib6_entry; 7186 7187err_fib6_entry_type_set: 7188 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 7189err_nexthop_group_vr_link: 7190 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 7191err_rt6_unwind: 7192 for (; i > 0; i--) { 7193 fib6_entry->nrt6--; 7194 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7195 struct mlxsw_sp_rt6, list); 7196 list_del(&mlxsw_sp_rt6->list); 7197 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7198 } 7199 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 7200err_fib_entry_priv_create: 7201 kfree(fib6_entry); 7202 return ERR_PTR(err); 7203} 7204 7205static void 7206mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 7207 struct mlxsw_sp_fib6_entry *fib6_entry) 7208{ 7209 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 7210} 7211 7212static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 7213 struct mlxsw_sp_fib6_entry *fib6_entry) 7214{ 7215 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7216 7217 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 7218 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7219 fib_node->fib); 7220 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7221 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7222 WARN_ON(fib6_entry->nrt6); 7223 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 7224 kfree(fib6_entry); 7225} 7226 7227static struct mlxsw_sp_fib6_entry * 7228mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7229 const struct fib6_info *rt) 7230{ 7231 struct mlxsw_sp_fib6_entry *fib6_entry; 7232 struct mlxsw_sp_fib_node *fib_node; 7233 struct mlxsw_sp_fib *fib; 7234 struct fib6_info *cmp_rt; 7235 struct mlxsw_sp_vr *vr; 7236 7237 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7238 if (!vr) 7239 return NULL; 7240 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7241 7242 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7243 sizeof(rt->fib6_dst.addr), 7244 rt->fib6_dst.plen); 7245 if (!fib_node) 7246 return NULL; 7247 7248 fib6_entry = container_of(fib_node->fib_entry, 7249 struct mlxsw_sp_fib6_entry, common); 7250 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7251 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7252 rt->fib6_metric == cmp_rt->fib6_metric && 7253 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7254 return fib6_entry; 7255 7256 return NULL; 7257} 7258 7259static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7260{ 7261 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7262 struct mlxsw_sp_fib6_entry *fib6_replaced; 7263 struct fib6_info *rt, *rt_replaced; 7264 7265 if (!fib_node->fib_entry) 7266 return true; 7267 7268 fib6_replaced = container_of(fib_node->fib_entry, 7269 struct mlxsw_sp_fib6_entry, 7270 common); 7271 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7272 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7273 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7274 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7275 return false; 7276 7277 return true; 7278} 7279 7280static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7281 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7282 struct fib6_info **rt_arr, unsigned int nrt6) 7283{ 7284 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7285 struct mlxsw_sp_fib_entry *replaced; 7286 struct mlxsw_sp_fib_node *fib_node; 7287 struct fib6_info *rt = rt_arr[0]; 7288 int err; 7289 7290 if (rt->fib6_src.plen) 7291 return -EINVAL; 7292 7293 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7294 return 0; 7295 7296 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7297 return 0; 7298 7299 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7300 &rt->fib6_dst.addr, 7301 sizeof(rt->fib6_dst.addr), 7302 rt->fib6_dst.plen, 7303 MLXSW_SP_L3_PROTO_IPV6); 7304 if (IS_ERR(fib_node)) 7305 return PTR_ERR(fib_node); 7306 7307 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7308 nrt6); 7309 if (IS_ERR(fib6_entry)) { 7310 err = PTR_ERR(fib6_entry); 7311 goto err_fib6_entry_create; 7312 } 7313 7314 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7315 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7316 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7317 return 0; 7318 } 7319 7320 replaced = fib_node->fib_entry; 7321 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 7322 if (err) 7323 goto err_fib_node_entry_link; 7324 7325 /* Nothing to replace */ 7326 if (!replaced) 7327 return 0; 7328 7329 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7330 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7331 common); 7332 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7333 7334 return 0; 7335 7336err_fib_node_entry_link: 7337 fib_node->fib_entry = replaced; 7338 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7339err_fib6_entry_create: 7340 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7341 return err; 7342} 7343 7344static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7345 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7346 struct fib6_info **rt_arr, unsigned int nrt6) 7347{ 7348 struct mlxsw_sp_fib6_entry *fib6_entry; 7349 struct mlxsw_sp_fib_node *fib_node; 7350 struct fib6_info *rt = rt_arr[0]; 7351 int err; 7352 7353 if (rt->fib6_src.plen) 7354 return -EINVAL; 7355 7356 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7357 return 0; 7358 7359 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7360 &rt->fib6_dst.addr, 7361 sizeof(rt->fib6_dst.addr), 7362 rt->fib6_dst.plen, 7363 MLXSW_SP_L3_PROTO_IPV6); 7364 if (IS_ERR(fib_node)) 7365 return PTR_ERR(fib_node); 7366 7367 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7368 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7369 return -EINVAL; 7370 } 7371 7372 fib6_entry = container_of(fib_node->fib_entry, 7373 struct mlxsw_sp_fib6_entry, common); 7374 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7375 if (err) 7376 goto err_fib6_entry_nexthop_add; 7377 7378 return 0; 7379 7380err_fib6_entry_nexthop_add: 7381 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7382 return err; 7383} 7384 7385static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7386 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7387 struct fib6_info **rt_arr, unsigned int nrt6) 7388{ 7389 struct mlxsw_sp_fib6_entry *fib6_entry; 7390 struct mlxsw_sp_fib_node *fib_node; 7391 struct fib6_info *rt = rt_arr[0]; 7392 int err; 7393 7394 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7395 return 0; 7396 7397 /* Multipath routes are first added to the FIB trie and only then 7398 * notified. If we vetoed the addition, we will get a delete 7399 * notification for a route we do not have. Therefore, do not warn if 7400 * route was not found. 7401 */ 7402 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7403 if (!fib6_entry) 7404 return 0; 7405 7406 /* If not all the nexthops are deleted, then only reduce the nexthop 7407 * group. 7408 */ 7409 if (nrt6 != fib6_entry->nrt6) { 7410 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7411 return 0; 7412 } 7413 7414 fib_node = fib6_entry->common.fib_node; 7415 7416 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 7417 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7418 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7419 return err; 7420} 7421 7422static struct mlxsw_sp_mr_table * 7423mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7424{ 7425 if (family == RTNL_FAMILY_IPMR) 7426 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7427 else 7428 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7429} 7430 7431static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7432 struct mfc_entry_notifier_info *men_info, 7433 bool replace) 7434{ 7435 struct mlxsw_sp_mr_table *mrt; 7436 struct mlxsw_sp_vr *vr; 7437 7438 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7439 if (IS_ERR(vr)) 7440 return PTR_ERR(vr); 7441 7442 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7443 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7444} 7445 7446static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7447 struct mfc_entry_notifier_info *men_info) 7448{ 7449 struct mlxsw_sp_mr_table *mrt; 7450 struct mlxsw_sp_vr *vr; 7451 7452 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7453 if (WARN_ON(!vr)) 7454 return; 7455 7456 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7457 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7458 mlxsw_sp_vr_put(mlxsw_sp, vr); 7459} 7460 7461static int 7462mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7463 struct vif_entry_notifier_info *ven_info) 7464{ 7465 struct mlxsw_sp_mr_table *mrt; 7466 struct mlxsw_sp_rif *rif; 7467 struct mlxsw_sp_vr *vr; 7468 7469 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7470 if (IS_ERR(vr)) 7471 return PTR_ERR(vr); 7472 7473 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7474 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7475 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7476 ven_info->vif_index, 7477 ven_info->vif_flags, rif); 7478} 7479 7480static void 7481mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7482 struct vif_entry_notifier_info *ven_info) 7483{ 7484 struct mlxsw_sp_mr_table *mrt; 7485 struct mlxsw_sp_vr *vr; 7486 7487 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7488 if (WARN_ON(!vr)) 7489 return; 7490 7491 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7492 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7493 mlxsw_sp_vr_put(mlxsw_sp, vr); 7494} 7495 7496static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7497 struct mlxsw_sp_fib_node *fib_node) 7498{ 7499 struct mlxsw_sp_fib4_entry *fib4_entry; 7500 7501 fib4_entry = container_of(fib_node->fib_entry, 7502 struct mlxsw_sp_fib4_entry, common); 7503 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7504 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7505 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7506} 7507 7508static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7509 struct mlxsw_sp_fib_node *fib_node) 7510{ 7511 struct mlxsw_sp_fib6_entry *fib6_entry; 7512 7513 fib6_entry = container_of(fib_node->fib_entry, 7514 struct mlxsw_sp_fib6_entry, common); 7515 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7516 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7517 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7518} 7519 7520static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7521 struct mlxsw_sp_fib_node *fib_node) 7522{ 7523 switch (fib_node->fib->proto) { 7524 case MLXSW_SP_L3_PROTO_IPV4: 7525 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7526 break; 7527 case MLXSW_SP_L3_PROTO_IPV6: 7528 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7529 break; 7530 } 7531} 7532 7533static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7534 struct mlxsw_sp_vr *vr, 7535 enum mlxsw_sp_l3proto proto) 7536{ 7537 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7538 struct mlxsw_sp_fib_node *fib_node, *tmp; 7539 7540 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7541 bool do_break = &tmp->list == &fib->node_list; 7542 7543 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7544 if (do_break) 7545 break; 7546 } 7547} 7548 7549static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7550{ 7551 int i, j; 7552 7553 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7554 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7555 7556 if (!mlxsw_sp_vr_is_used(vr)) 7557 continue; 7558 7559 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7560 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7561 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7562 7563 /* If virtual router was only used for IPv4, then it's no 7564 * longer used. 7565 */ 7566 if (!mlxsw_sp_vr_is_used(vr)) 7567 continue; 7568 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7569 } 7570} 7571 7572struct mlxsw_sp_fib6_event { 7573 struct fib6_info **rt_arr; 7574 unsigned int nrt6; 7575}; 7576 7577struct mlxsw_sp_fib_event { 7578 struct list_head list; /* node in fib queue */ 7579 union { 7580 struct mlxsw_sp_fib6_event fib6_event; 7581 struct fib_entry_notifier_info fen_info; 7582 struct fib_rule_notifier_info fr_info; 7583 struct fib_nh_notifier_info fnh_info; 7584 struct mfc_entry_notifier_info men_info; 7585 struct vif_entry_notifier_info ven_info; 7586 }; 7587 struct mlxsw_sp *mlxsw_sp; 7588 unsigned long event; 7589 int family; 7590}; 7591 7592static int 7593mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 7594 struct fib6_entry_notifier_info *fen6_info) 7595{ 7596 struct fib6_info *rt = fen6_info->rt; 7597 struct fib6_info **rt_arr; 7598 struct fib6_info *iter; 7599 unsigned int nrt6; 7600 int i = 0; 7601 7602 nrt6 = fen6_info->nsiblings + 1; 7603 7604 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7605 if (!rt_arr) 7606 return -ENOMEM; 7607 7608 fib6_event->rt_arr = rt_arr; 7609 fib6_event->nrt6 = nrt6; 7610 7611 rt_arr[0] = rt; 7612 fib6_info_hold(rt); 7613 7614 if (!fen6_info->nsiblings) 7615 return 0; 7616 7617 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7618 if (i == fen6_info->nsiblings) 7619 break; 7620 7621 rt_arr[i + 1] = iter; 7622 fib6_info_hold(iter); 7623 i++; 7624 } 7625 WARN_ON_ONCE(i != fen6_info->nsiblings); 7626 7627 return 0; 7628} 7629 7630static void 7631mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 7632{ 7633 int i; 7634 7635 for (i = 0; i < fib6_event->nrt6; i++) 7636 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 7637 kfree(fib6_event->rt_arr); 7638} 7639 7640static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 7641 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7642 struct mlxsw_sp_fib_event *fib_event) 7643{ 7644 int err; 7645 7646 mlxsw_sp_span_respin(mlxsw_sp); 7647 7648 switch (fib_event->event) { 7649 case FIB_EVENT_ENTRY_REPLACE: 7650 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 7651 if (err) { 7652 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7653 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7654 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7655 &fib_event->fen_info); 7656 } 7657 fib_info_put(fib_event->fen_info.fi); 7658 break; 7659 case FIB_EVENT_ENTRY_DEL: 7660 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 7661 if (err) 7662 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7663 fib_info_put(fib_event->fen_info.fi); 7664 break; 7665 case FIB_EVENT_NH_ADD: 7666 case FIB_EVENT_NH_DEL: 7667 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 7668 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 7669 break; 7670 } 7671} 7672 7673static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 7674 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7675 struct mlxsw_sp_fib_event *fib_event) 7676{ 7677 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event; 7678 int err; 7679 7680 mlxsw_sp_span_respin(mlxsw_sp); 7681 7682 switch (fib_event->event) { 7683 case FIB_EVENT_ENTRY_REPLACE: 7684 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7685 fib_event->fib6_event.nrt6); 7686 if (err) { 7687 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7688 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7689 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7690 fib6_event->rt_arr, 7691 fib6_event->nrt6); 7692 } 7693 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7694 break; 7695 case FIB_EVENT_ENTRY_APPEND: 7696 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7697 fib_event->fib6_event.nrt6); 7698 if (err) { 7699 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7700 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7701 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7702 fib6_event->rt_arr, 7703 fib6_event->nrt6); 7704 } 7705 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7706 break; 7707 case FIB_EVENT_ENTRY_DEL: 7708 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7709 fib_event->fib6_event.nrt6); 7710 if (err) 7711 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7712 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7713 break; 7714 } 7715} 7716 7717static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7718 struct mlxsw_sp_fib_event *fib_event) 7719{ 7720 bool replace; 7721 int err; 7722 7723 rtnl_lock(); 7724 mutex_lock(&mlxsw_sp->router->lock); 7725 switch (fib_event->event) { 7726 case FIB_EVENT_ENTRY_REPLACE: 7727 case FIB_EVENT_ENTRY_ADD: 7728 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7729 7730 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7731 if (err) 7732 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7733 mr_cache_put(fib_event->men_info.mfc); 7734 break; 7735 case FIB_EVENT_ENTRY_DEL: 7736 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7737 mr_cache_put(fib_event->men_info.mfc); 7738 break; 7739 case FIB_EVENT_VIF_ADD: 7740 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7741 &fib_event->ven_info); 7742 if (err) 7743 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7744 dev_put(fib_event->ven_info.dev); 7745 break; 7746 case FIB_EVENT_VIF_DEL: 7747 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7748 dev_put(fib_event->ven_info.dev); 7749 break; 7750 } 7751 mutex_unlock(&mlxsw_sp->router->lock); 7752 rtnl_unlock(); 7753} 7754 7755static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7756{ 7757 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7758 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7759 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7760 struct mlxsw_sp_fib_event *next_fib_event; 7761 struct mlxsw_sp_fib_event *fib_event; 7762 int last_family = AF_UNSPEC; 7763 LIST_HEAD(fib_event_queue); 7764 7765 spin_lock_bh(&router->fib_event_queue_lock); 7766 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7767 spin_unlock_bh(&router->fib_event_queue_lock); 7768 7769 /* Router lock is held here to make sure per-instance 7770 * operation context is not used in between FIB4/6 events 7771 * processing. 7772 */ 7773 mutex_lock(&router->lock); 7774 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7775 list_for_each_entry_safe(fib_event, next_fib_event, 7776 &fib_event_queue, list) { 7777 /* Check if the next entry in the queue exists and it is 7778 * of the same type (family and event) as the currect one. 7779 * In that case it is permitted to do the bulking 7780 * of multiple FIB entries to a single register write. 7781 */ 7782 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7783 fib_event->family == next_fib_event->family && 7784 fib_event->event == next_fib_event->event; 7785 op_ctx->event = fib_event->event; 7786 7787 /* In case family of this and the previous entry are different, context 7788 * reinitialization is going to be needed now, indicate that. 7789 * Note that since last_family is initialized to AF_UNSPEC, this is always 7790 * going to happen for the first entry processed in the work. 7791 */ 7792 if (fib_event->family != last_family) 7793 op_ctx->initialized = false; 7794 7795 switch (fib_event->family) { 7796 case AF_INET: 7797 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7798 fib_event); 7799 break; 7800 case AF_INET6: 7801 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7802 fib_event); 7803 break; 7804 case RTNL_FAMILY_IP6MR: 7805 case RTNL_FAMILY_IPMR: 7806 /* Unlock here as inside FIBMR the lock is taken again 7807 * under RTNL. The per-instance operation context 7808 * is not used by FIBMR. 7809 */ 7810 mutex_unlock(&router->lock); 7811 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7812 fib_event); 7813 mutex_lock(&router->lock); 7814 break; 7815 default: 7816 WARN_ON_ONCE(1); 7817 } 7818 last_family = fib_event->family; 7819 kfree(fib_event); 7820 cond_resched(); 7821 } 7822 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7823 mutex_unlock(&router->lock); 7824} 7825 7826static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7827 struct fib_notifier_info *info) 7828{ 7829 struct fib_entry_notifier_info *fen_info; 7830 struct fib_nh_notifier_info *fnh_info; 7831 7832 switch (fib_event->event) { 7833 case FIB_EVENT_ENTRY_REPLACE: 7834 case FIB_EVENT_ENTRY_DEL: 7835 fen_info = container_of(info, struct fib_entry_notifier_info, 7836 info); 7837 fib_event->fen_info = *fen_info; 7838 /* Take reference on fib_info to prevent it from being 7839 * freed while event is queued. Release it afterwards. 7840 */ 7841 fib_info_hold(fib_event->fen_info.fi); 7842 break; 7843 case FIB_EVENT_NH_ADD: 7844 case FIB_EVENT_NH_DEL: 7845 fnh_info = container_of(info, struct fib_nh_notifier_info, 7846 info); 7847 fib_event->fnh_info = *fnh_info; 7848 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7849 break; 7850 } 7851} 7852 7853static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7854 struct fib_notifier_info *info) 7855{ 7856 struct fib6_entry_notifier_info *fen6_info; 7857 int err; 7858 7859 switch (fib_event->event) { 7860 case FIB_EVENT_ENTRY_REPLACE: 7861 case FIB_EVENT_ENTRY_APPEND: 7862 case FIB_EVENT_ENTRY_DEL: 7863 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7864 info); 7865 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7866 fen6_info); 7867 if (err) 7868 return err; 7869 break; 7870 } 7871 7872 return 0; 7873} 7874 7875static void 7876mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7877 struct fib_notifier_info *info) 7878{ 7879 switch (fib_event->event) { 7880 case FIB_EVENT_ENTRY_REPLACE: 7881 case FIB_EVENT_ENTRY_ADD: 7882 case FIB_EVENT_ENTRY_DEL: 7883 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7884 mr_cache_hold(fib_event->men_info.mfc); 7885 break; 7886 case FIB_EVENT_VIF_ADD: 7887 case FIB_EVENT_VIF_DEL: 7888 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7889 dev_hold(fib_event->ven_info.dev); 7890 break; 7891 } 7892} 7893 7894static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7895 struct fib_notifier_info *info, 7896 struct mlxsw_sp *mlxsw_sp) 7897{ 7898 struct netlink_ext_ack *extack = info->extack; 7899 struct fib_rule_notifier_info *fr_info; 7900 struct fib_rule *rule; 7901 int err = 0; 7902 7903 /* nothing to do at the moment */ 7904 if (event == FIB_EVENT_RULE_DEL) 7905 return 0; 7906 7907 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7908 rule = fr_info->rule; 7909 7910 /* Rule only affects locally generated traffic */ 7911 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7912 return 0; 7913 7914 switch (info->family) { 7915 case AF_INET: 7916 if (!fib4_rule_default(rule) && !rule->l3mdev) 7917 err = -EOPNOTSUPP; 7918 break; 7919 case AF_INET6: 7920 if (!fib6_rule_default(rule) && !rule->l3mdev) 7921 err = -EOPNOTSUPP; 7922 break; 7923 case RTNL_FAMILY_IPMR: 7924 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7925 err = -EOPNOTSUPP; 7926 break; 7927 case RTNL_FAMILY_IP6MR: 7928 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7929 err = -EOPNOTSUPP; 7930 break; 7931 } 7932 7933 if (err < 0) 7934 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7935 7936 return err; 7937} 7938 7939/* Called with rcu_read_lock() */ 7940static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7941 unsigned long event, void *ptr) 7942{ 7943 struct mlxsw_sp_fib_event *fib_event; 7944 struct fib_notifier_info *info = ptr; 7945 struct mlxsw_sp_router *router; 7946 int err; 7947 7948 if ((info->family != AF_INET && info->family != AF_INET6 && 7949 info->family != RTNL_FAMILY_IPMR && 7950 info->family != RTNL_FAMILY_IP6MR)) 7951 return NOTIFY_DONE; 7952 7953 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7954 7955 switch (event) { 7956 case FIB_EVENT_RULE_ADD: 7957 case FIB_EVENT_RULE_DEL: 7958 err = mlxsw_sp_router_fib_rule_event(event, info, 7959 router->mlxsw_sp); 7960 return notifier_from_errno(err); 7961 case FIB_EVENT_ENTRY_ADD: 7962 case FIB_EVENT_ENTRY_REPLACE: 7963 case FIB_EVENT_ENTRY_APPEND: 7964 if (info->family == AF_INET) { 7965 struct fib_entry_notifier_info *fen_info = ptr; 7966 7967 if (fen_info->fi->fib_nh_is_v6) { 7968 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7969 return notifier_from_errno(-EINVAL); 7970 } 7971 } 7972 break; 7973 } 7974 7975 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7976 if (!fib_event) 7977 return NOTIFY_BAD; 7978 7979 fib_event->mlxsw_sp = router->mlxsw_sp; 7980 fib_event->event = event; 7981 fib_event->family = info->family; 7982 7983 switch (info->family) { 7984 case AF_INET: 7985 mlxsw_sp_router_fib4_event(fib_event, info); 7986 break; 7987 case AF_INET6: 7988 err = mlxsw_sp_router_fib6_event(fib_event, info); 7989 if (err) 7990 goto err_fib_event; 7991 break; 7992 case RTNL_FAMILY_IP6MR: 7993 case RTNL_FAMILY_IPMR: 7994 mlxsw_sp_router_fibmr_event(fib_event, info); 7995 break; 7996 } 7997 7998 /* Enqueue the event and trigger the work */ 7999 spin_lock_bh(&router->fib_event_queue_lock); 8000 list_add_tail(&fib_event->list, &router->fib_event_queue); 8001 spin_unlock_bh(&router->fib_event_queue_lock); 8002 mlxsw_core_schedule_work(&router->fib_event_work); 8003 8004 return NOTIFY_DONE; 8005 8006err_fib_event: 8007 kfree(fib_event); 8008 return NOTIFY_BAD; 8009} 8010 8011static struct mlxsw_sp_rif * 8012mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 8013 const struct net_device *dev) 8014{ 8015 int i; 8016 8017 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 8018 if (mlxsw_sp->router->rifs[i] && 8019 mlxsw_sp->router->rifs[i]->dev == dev) 8020 return mlxsw_sp->router->rifs[i]; 8021 8022 return NULL; 8023} 8024 8025bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 8026 const struct net_device *dev) 8027{ 8028 struct mlxsw_sp_rif *rif; 8029 8030 mutex_lock(&mlxsw_sp->router->lock); 8031 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8032 mutex_unlock(&mlxsw_sp->router->lock); 8033 8034 return rif; 8035} 8036 8037u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 8038{ 8039 struct mlxsw_sp_rif *rif; 8040 u16 vid = 0; 8041 8042 mutex_lock(&mlxsw_sp->router->lock); 8043 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8044 if (!rif) 8045 goto out; 8046 8047 /* We only return the VID for VLAN RIFs. Otherwise we return an 8048 * invalid value (0). 8049 */ 8050 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 8051 goto out; 8052 8053 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 8054 8055out: 8056 mutex_unlock(&mlxsw_sp->router->lock); 8057 return vid; 8058} 8059 8060static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 8061{ 8062 char ritr_pl[MLXSW_REG_RITR_LEN]; 8063 int err; 8064 8065 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 8066 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8067 if (err) 8068 return err; 8069 8070 mlxsw_reg_ritr_enable_set(ritr_pl, false); 8071 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8072} 8073 8074static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 8075 struct mlxsw_sp_rif *rif) 8076{ 8077 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 8078 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 8079 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8080} 8081 8082static bool 8083mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 8084 unsigned long event) 8085{ 8086 struct inet6_dev *inet6_dev; 8087 bool addr_list_empty = true; 8088 struct in_device *idev; 8089 8090 switch (event) { 8091 case NETDEV_UP: 8092 return rif == NULL; 8093 case NETDEV_DOWN: 8094 rcu_read_lock(); 8095 idev = __in_dev_get_rcu(dev); 8096 if (idev && idev->ifa_list) 8097 addr_list_empty = false; 8098 8099 inet6_dev = __in6_dev_get(dev); 8100 if (addr_list_empty && inet6_dev && 8101 !list_empty(&inet6_dev->addr_list)) 8102 addr_list_empty = false; 8103 rcu_read_unlock(); 8104 8105 /* macvlans do not have a RIF, but rather piggy back on the 8106 * RIF of their lower device. 8107 */ 8108 if (netif_is_macvlan(dev) && addr_list_empty) 8109 return true; 8110 8111 if (rif && addr_list_empty && 8112 !netif_is_l3_slave(rif->dev)) 8113 return true; 8114 /* It is possible we already removed the RIF ourselves 8115 * if it was assigned to a netdev that is now a bridge 8116 * or LAG slave. 8117 */ 8118 return false; 8119 } 8120 8121 return false; 8122} 8123 8124static enum mlxsw_sp_rif_type 8125mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 8126 const struct net_device *dev) 8127{ 8128 enum mlxsw_sp_fid_type type; 8129 8130 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 8131 return MLXSW_SP_RIF_TYPE_IPIP_LB; 8132 8133 /* Otherwise RIF type is derived from the type of the underlying FID. */ 8134 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 8135 type = MLXSW_SP_FID_TYPE_8021Q; 8136 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 8137 type = MLXSW_SP_FID_TYPE_8021Q; 8138 else if (netif_is_bridge_master(dev)) 8139 type = MLXSW_SP_FID_TYPE_8021D; 8140 else 8141 type = MLXSW_SP_FID_TYPE_RFID; 8142 8143 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 8144} 8145 8146static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 8147{ 8148 int i; 8149 8150 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8151 if (!mlxsw_sp->router->rifs[i]) { 8152 *p_rif_index = i; 8153 return 0; 8154 } 8155 } 8156 8157 return -ENOBUFS; 8158} 8159 8160static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8161 u16 vr_id, 8162 struct net_device *l3_dev) 8163{ 8164 struct mlxsw_sp_rif *rif; 8165 8166 rif = kzalloc(rif_size, GFP_KERNEL); 8167 if (!rif) 8168 return NULL; 8169 8170 INIT_LIST_HEAD(&rif->nexthop_list); 8171 INIT_LIST_HEAD(&rif->neigh_list); 8172 if (l3_dev) { 8173 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8174 rif->mtu = l3_dev->mtu; 8175 rif->dev = l3_dev; 8176 } 8177 rif->vr_id = vr_id; 8178 rif->rif_index = rif_index; 8179 8180 return rif; 8181} 8182 8183struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8184 u16 rif_index) 8185{ 8186 return mlxsw_sp->router->rifs[rif_index]; 8187} 8188 8189u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8190{ 8191 return rif->rif_index; 8192} 8193 8194u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8195{ 8196 return lb_rif->common.rif_index; 8197} 8198 8199u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8200{ 8201 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 8202 struct mlxsw_sp_vr *ul_vr; 8203 8204 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 8205 if (WARN_ON(IS_ERR(ul_vr))) 8206 return 0; 8207 8208 return ul_vr->id; 8209} 8210 8211u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8212{ 8213 return lb_rif->ul_rif_id; 8214} 8215 8216static bool 8217mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 8218{ 8219 return mlxsw_sp_rif_counter_valid_get(rif, 8220 MLXSW_SP_RIF_COUNTER_EGRESS) && 8221 mlxsw_sp_rif_counter_valid_get(rif, 8222 MLXSW_SP_RIF_COUNTER_INGRESS); 8223} 8224 8225static int 8226mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 8227{ 8228 int err; 8229 8230 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8231 if (err) 8232 return err; 8233 8234 /* Clear stale data. */ 8235 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8236 MLXSW_SP_RIF_COUNTER_INGRESS, 8237 NULL); 8238 if (err) 8239 goto err_clear_ingress; 8240 8241 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8242 if (err) 8243 goto err_alloc_egress; 8244 8245 /* Clear stale data. */ 8246 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8247 MLXSW_SP_RIF_COUNTER_EGRESS, 8248 NULL); 8249 if (err) 8250 goto err_clear_egress; 8251 8252 return 0; 8253 8254err_clear_egress: 8255 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8256err_alloc_egress: 8257err_clear_ingress: 8258 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8259 return err; 8260} 8261 8262static void 8263mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 8264{ 8265 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8266 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8267} 8268 8269static void 8270mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 8271 struct netdev_notifier_offload_xstats_info *info) 8272{ 8273 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8274 return; 8275 netdev_offload_xstats_report_used(info->report_used); 8276} 8277 8278static int 8279mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 8280 struct rtnl_hw_stats64 *p_stats) 8281{ 8282 struct mlxsw_sp_rif_counter_set_basic ingress; 8283 struct mlxsw_sp_rif_counter_set_basic egress; 8284 int err; 8285 8286 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8287 MLXSW_SP_RIF_COUNTER_INGRESS, 8288 &ingress); 8289 if (err) 8290 return err; 8291 8292 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8293 MLXSW_SP_RIF_COUNTER_EGRESS, 8294 &egress); 8295 if (err) 8296 return err; 8297 8298#define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 8299 ((SET.good_unicast_ ## SFX) + \ 8300 (SET.good_multicast_ ## SFX) + \ 8301 (SET.good_broadcast_ ## SFX)) 8302 8303 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 8304 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 8305 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 8306 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 8307 p_stats->rx_errors = ingress.error_packets; 8308 p_stats->tx_errors = egress.error_packets; 8309 p_stats->rx_dropped = ingress.discard_packets; 8310 p_stats->tx_dropped = egress.discard_packets; 8311 p_stats->multicast = ingress.good_multicast_packets + 8312 ingress.good_broadcast_packets; 8313 8314#undef MLXSW_SP_ROUTER_ALL_GOOD 8315 8316 return 0; 8317} 8318 8319static int 8320mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 8321 struct netdev_notifier_offload_xstats_info *info) 8322{ 8323 struct rtnl_hw_stats64 stats = {}; 8324 int err; 8325 8326 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8327 return 0; 8328 8329 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8330 if (err) 8331 return err; 8332 8333 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8334 return 0; 8335} 8336 8337struct mlxsw_sp_router_hwstats_notify_work { 8338 struct work_struct work; 8339 struct net_device *dev; 8340}; 8341 8342static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8343{ 8344 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8345 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8346 work); 8347 8348 rtnl_lock(); 8349 rtnl_offload_xstats_notify(hws_work->dev); 8350 rtnl_unlock(); 8351 dev_put(hws_work->dev); 8352 kfree(hws_work); 8353} 8354 8355static void 8356mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8357{ 8358 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8359 8360 /* To collect notification payload, the core ends up sending another 8361 * notifier block message, which would deadlock on the attempt to 8362 * acquire the router lock again. Just postpone the notification until 8363 * later. 8364 */ 8365 8366 hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL); 8367 if (!hws_work) 8368 return; 8369 8370 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8371 dev_hold(dev); 8372 hws_work->dev = dev; 8373 mlxsw_core_schedule_work(&hws_work->work); 8374} 8375 8376int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8377{ 8378 return rif->dev->ifindex; 8379} 8380 8381const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 8382{ 8383 return rif->dev; 8384} 8385 8386static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8387{ 8388 struct rtnl_hw_stats64 stats = {}; 8389 8390 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8391 netdev_offload_xstats_push_delta(rif->dev, 8392 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8393 &stats); 8394} 8395 8396static struct mlxsw_sp_rif * 8397mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8398 const struct mlxsw_sp_rif_params *params, 8399 struct netlink_ext_ack *extack) 8400{ 8401 u32 tb_id = l3mdev_fib_table(params->dev); 8402 const struct mlxsw_sp_rif_ops *ops; 8403 struct mlxsw_sp_fid *fid = NULL; 8404 enum mlxsw_sp_rif_type type; 8405 struct mlxsw_sp_rif *rif; 8406 struct mlxsw_sp_vr *vr; 8407 u16 rif_index; 8408 int i, err; 8409 8410 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8411 ops = mlxsw_sp->router->rif_ops_arr[type]; 8412 8413 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8414 if (IS_ERR(vr)) 8415 return ERR_CAST(vr); 8416 vr->rif_count++; 8417 8418 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8419 if (err) { 8420 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8421 goto err_rif_index_alloc; 8422 } 8423 8424 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 8425 if (!rif) { 8426 err = -ENOMEM; 8427 goto err_rif_alloc; 8428 } 8429 dev_hold(rif->dev); 8430 mlxsw_sp->router->rifs[rif_index] = rif; 8431 rif->mlxsw_sp = mlxsw_sp; 8432 rif->ops = ops; 8433 8434 if (ops->fid_get) { 8435 fid = ops->fid_get(rif, extack); 8436 if (IS_ERR(fid)) { 8437 err = PTR_ERR(fid); 8438 goto err_fid_get; 8439 } 8440 rif->fid = fid; 8441 } 8442 8443 if (ops->setup) 8444 ops->setup(rif, params); 8445 8446 err = ops->configure(rif, extack); 8447 if (err) 8448 goto err_configure; 8449 8450 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8451 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8452 if (err) 8453 goto err_mr_rif_add; 8454 } 8455 8456 if (netdev_offload_xstats_enabled(rif->dev, 8457 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8458 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8459 if (err) 8460 goto err_stats_enable; 8461 mlxsw_sp_router_hwstats_notify_schedule(rif->dev); 8462 } else { 8463 mlxsw_sp_rif_counters_alloc(rif); 8464 } 8465 8466 return rif; 8467 8468err_stats_enable: 8469err_mr_rif_add: 8470 for (i--; i >= 0; i--) 8471 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8472 ops->deconfigure(rif); 8473err_configure: 8474 if (fid) 8475 mlxsw_sp_fid_put(fid); 8476err_fid_get: 8477 mlxsw_sp->router->rifs[rif_index] = NULL; 8478 dev_put(rif->dev); 8479 kfree(rif); 8480err_rif_alloc: 8481err_rif_index_alloc: 8482 vr->rif_count--; 8483 mlxsw_sp_vr_put(mlxsw_sp, vr); 8484 return ERR_PTR(err); 8485} 8486 8487static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8488{ 8489 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8490 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8491 struct mlxsw_sp_fid *fid = rif->fid; 8492 struct mlxsw_sp_vr *vr; 8493 int i; 8494 8495 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8496 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8497 8498 if (netdev_offload_xstats_enabled(rif->dev, 8499 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8500 mlxsw_sp_rif_push_l3_stats(rif); 8501 mlxsw_sp_router_port_l3_stats_disable(rif); 8502 mlxsw_sp_router_hwstats_notify_schedule(rif->dev); 8503 } else { 8504 mlxsw_sp_rif_counters_free(rif); 8505 } 8506 8507 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8508 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8509 ops->deconfigure(rif); 8510 if (fid) 8511 /* Loopback RIFs are not associated with a FID. */ 8512 mlxsw_sp_fid_put(fid); 8513 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8514 dev_put(rif->dev); 8515 kfree(rif); 8516 vr->rif_count--; 8517 mlxsw_sp_vr_put(mlxsw_sp, vr); 8518} 8519 8520void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8521 struct net_device *dev) 8522{ 8523 struct mlxsw_sp_rif *rif; 8524 8525 mutex_lock(&mlxsw_sp->router->lock); 8526 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8527 if (!rif) 8528 goto out; 8529 mlxsw_sp_rif_destroy(rif); 8530out: 8531 mutex_unlock(&mlxsw_sp->router->lock); 8532} 8533 8534static void 8535mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8536 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8537{ 8538 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8539 8540 params->vid = mlxsw_sp_port_vlan->vid; 8541 params->lag = mlxsw_sp_port->lagged; 8542 if (params->lag) 8543 params->lag_id = mlxsw_sp_port->lag_id; 8544 else 8545 params->system_port = mlxsw_sp_port->local_port; 8546} 8547 8548static struct mlxsw_sp_rif_subport * 8549mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8550{ 8551 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8552} 8553 8554static struct mlxsw_sp_rif * 8555mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8556 const struct mlxsw_sp_rif_params *params, 8557 struct netlink_ext_ack *extack) 8558{ 8559 struct mlxsw_sp_rif_subport *rif_subport; 8560 struct mlxsw_sp_rif *rif; 8561 8562 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8563 if (!rif) 8564 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8565 8566 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8567 refcount_inc(&rif_subport->ref_count); 8568 return rif; 8569} 8570 8571static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8572{ 8573 struct mlxsw_sp_rif_subport *rif_subport; 8574 8575 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8576 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8577 return; 8578 8579 mlxsw_sp_rif_destroy(rif); 8580} 8581 8582static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8583 struct mlxsw_sp_rif_mac_profile *profile, 8584 struct netlink_ext_ack *extack) 8585{ 8586 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8587 struct mlxsw_sp_router *router = mlxsw_sp->router; 8588 int id; 8589 8590 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8591 max_rif_mac_profiles, GFP_KERNEL); 8592 8593 if (id >= 0) { 8594 profile->id = id; 8595 return 0; 8596 } 8597 8598 if (id == -ENOSPC) 8599 NL_SET_ERR_MSG_MOD(extack, 8600 "Exceeded number of supported router interface MAC profiles"); 8601 8602 return id; 8603} 8604 8605static struct mlxsw_sp_rif_mac_profile * 8606mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8607{ 8608 struct mlxsw_sp_rif_mac_profile *profile; 8609 8610 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8611 mac_profile); 8612 WARN_ON(!profile); 8613 return profile; 8614} 8615 8616static struct mlxsw_sp_rif_mac_profile * 8617mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8618{ 8619 struct mlxsw_sp_rif_mac_profile *profile; 8620 8621 profile = kzalloc(sizeof(*profile), GFP_KERNEL); 8622 if (!profile) 8623 return NULL; 8624 8625 ether_addr_copy(profile->mac_prefix, mac); 8626 refcount_set(&profile->ref_count, 1); 8627 return profile; 8628} 8629 8630static struct mlxsw_sp_rif_mac_profile * 8631mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8632{ 8633 struct mlxsw_sp_router *router = mlxsw_sp->router; 8634 struct mlxsw_sp_rif_mac_profile *profile; 8635 int id; 8636 8637 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8638 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8639 mlxsw_sp->mac_mask)) 8640 return profile; 8641 } 8642 8643 return NULL; 8644} 8645 8646static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8647{ 8648 const struct mlxsw_sp *mlxsw_sp = priv; 8649 8650 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8651} 8652 8653static struct mlxsw_sp_rif_mac_profile * 8654mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8655 struct netlink_ext_ack *extack) 8656{ 8657 struct mlxsw_sp_rif_mac_profile *profile; 8658 int err; 8659 8660 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8661 if (!profile) 8662 return ERR_PTR(-ENOMEM); 8663 8664 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8665 if (err) 8666 goto profile_index_alloc_err; 8667 8668 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 8669 return profile; 8670 8671profile_index_alloc_err: 8672 kfree(profile); 8673 return ERR_PTR(err); 8674} 8675 8676static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 8677 u8 mac_profile) 8678{ 8679 struct mlxsw_sp_rif_mac_profile *profile; 8680 8681 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 8682 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 8683 kfree(profile); 8684} 8685 8686static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 8687 const char *mac, u8 *p_mac_profile, 8688 struct netlink_ext_ack *extack) 8689{ 8690 struct mlxsw_sp_rif_mac_profile *profile; 8691 8692 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 8693 if (profile) { 8694 refcount_inc(&profile->ref_count); 8695 goto out; 8696 } 8697 8698 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 8699 if (IS_ERR(profile)) 8700 return PTR_ERR(profile); 8701 8702out: 8703 *p_mac_profile = profile->id; 8704 return 0; 8705} 8706 8707static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 8708 u8 mac_profile) 8709{ 8710 struct mlxsw_sp_rif_mac_profile *profile; 8711 8712 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8713 mac_profile); 8714 if (WARN_ON(!profile)) 8715 return; 8716 8717 if (!refcount_dec_and_test(&profile->ref_count)) 8718 return; 8719 8720 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 8721} 8722 8723static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 8724{ 8725 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8726 struct mlxsw_sp_rif_mac_profile *profile; 8727 8728 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8729 rif->mac_profile_id); 8730 if (WARN_ON(!profile)) 8731 return false; 8732 8733 return refcount_read(&profile->ref_count) > 1; 8734} 8735 8736static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 8737 const char *new_mac) 8738{ 8739 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8740 struct mlxsw_sp_rif_mac_profile *profile; 8741 8742 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8743 rif->mac_profile_id); 8744 if (WARN_ON(!profile)) 8745 return -EINVAL; 8746 8747 ether_addr_copy(profile->mac_prefix, new_mac); 8748 return 0; 8749} 8750 8751static int 8752mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 8753 struct mlxsw_sp_rif *rif, 8754 const char *new_mac, 8755 struct netlink_ext_ack *extack) 8756{ 8757 u8 mac_profile; 8758 int err; 8759 8760 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 8761 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 8762 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 8763 8764 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 8765 &mac_profile, extack); 8766 if (err) 8767 return err; 8768 8769 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 8770 rif->mac_profile_id = mac_profile; 8771 return 0; 8772} 8773 8774static int 8775__mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8776 struct net_device *l3_dev, 8777 struct netlink_ext_ack *extack) 8778{ 8779 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8780 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8781 struct mlxsw_sp_rif_params params = { 8782 .dev = l3_dev, 8783 }; 8784 u16 vid = mlxsw_sp_port_vlan->vid; 8785 struct mlxsw_sp_rif *rif; 8786 struct mlxsw_sp_fid *fid; 8787 int err; 8788 8789 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8790 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8791 if (IS_ERR(rif)) 8792 return PTR_ERR(rif); 8793 8794 /* FID was already created, just take a reference */ 8795 fid = rif->ops->fid_get(rif, extack); 8796 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8797 if (err) 8798 goto err_fid_port_vid_map; 8799 8800 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8801 if (err) 8802 goto err_port_vid_learning_set; 8803 8804 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8805 BR_STATE_FORWARDING); 8806 if (err) 8807 goto err_port_vid_stp_set; 8808 8809 mlxsw_sp_port_vlan->fid = fid; 8810 8811 return 0; 8812 8813err_port_vid_stp_set: 8814 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8815err_port_vid_learning_set: 8816 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8817err_fid_port_vid_map: 8818 mlxsw_sp_fid_put(fid); 8819 mlxsw_sp_rif_subport_put(rif); 8820 return err; 8821} 8822 8823static void 8824__mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8825{ 8826 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8827 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8828 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8829 u16 vid = mlxsw_sp_port_vlan->vid; 8830 8831 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8832 return; 8833 8834 mlxsw_sp_port_vlan->fid = NULL; 8835 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8836 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8837 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8838 mlxsw_sp_fid_put(fid); 8839 mlxsw_sp_rif_subport_put(rif); 8840} 8841 8842int 8843mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8844 struct net_device *l3_dev, 8845 struct netlink_ext_ack *extack) 8846{ 8847 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8848 struct mlxsw_sp_rif *rif; 8849 int err = 0; 8850 8851 mutex_lock(&mlxsw_sp->router->lock); 8852 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8853 if (!rif) 8854 goto out; 8855 8856 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8857 extack); 8858out: 8859 mutex_unlock(&mlxsw_sp->router->lock); 8860 return err; 8861} 8862 8863void 8864mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8865{ 8866 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8867 8868 mutex_lock(&mlxsw_sp->router->lock); 8869 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8870 mutex_unlock(&mlxsw_sp->router->lock); 8871} 8872 8873static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8874 struct net_device *port_dev, 8875 unsigned long event, u16 vid, 8876 struct netlink_ext_ack *extack) 8877{ 8878 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8879 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8880 8881 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8882 if (WARN_ON(!mlxsw_sp_port_vlan)) 8883 return -EINVAL; 8884 8885 switch (event) { 8886 case NETDEV_UP: 8887 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8888 l3_dev, extack); 8889 case NETDEV_DOWN: 8890 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8891 break; 8892 } 8893 8894 return 0; 8895} 8896 8897static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8898 unsigned long event, 8899 struct netlink_ext_ack *extack) 8900{ 8901 if (netif_is_bridge_port(port_dev) || 8902 netif_is_lag_port(port_dev) || 8903 netif_is_ovs_port(port_dev)) 8904 return 0; 8905 8906 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8907 MLXSW_SP_DEFAULT_VID, extack); 8908} 8909 8910static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8911 struct net_device *lag_dev, 8912 unsigned long event, u16 vid, 8913 struct netlink_ext_ack *extack) 8914{ 8915 struct net_device *port_dev; 8916 struct list_head *iter; 8917 int err; 8918 8919 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8920 if (mlxsw_sp_port_dev_check(port_dev)) { 8921 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8922 port_dev, 8923 event, vid, 8924 extack); 8925 if (err) 8926 return err; 8927 } 8928 } 8929 8930 return 0; 8931} 8932 8933static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8934 unsigned long event, 8935 struct netlink_ext_ack *extack) 8936{ 8937 if (netif_is_bridge_port(lag_dev)) 8938 return 0; 8939 8940 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8941 MLXSW_SP_DEFAULT_VID, extack); 8942} 8943 8944static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8945 struct net_device *l3_dev, 8946 unsigned long event, 8947 struct netlink_ext_ack *extack) 8948{ 8949 struct mlxsw_sp_rif_params params = { 8950 .dev = l3_dev, 8951 }; 8952 struct mlxsw_sp_rif *rif; 8953 8954 switch (event) { 8955 case NETDEV_UP: 8956 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8957 u16 proto; 8958 8959 br_vlan_get_proto(l3_dev, &proto); 8960 if (proto == ETH_P_8021AD) { 8961 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8962 return -EOPNOTSUPP; 8963 } 8964 } 8965 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8966 if (IS_ERR(rif)) 8967 return PTR_ERR(rif); 8968 break; 8969 case NETDEV_DOWN: 8970 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8971 mlxsw_sp_rif_destroy(rif); 8972 break; 8973 } 8974 8975 return 0; 8976} 8977 8978static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8979 struct net_device *vlan_dev, 8980 unsigned long event, 8981 struct netlink_ext_ack *extack) 8982{ 8983 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8984 u16 vid = vlan_dev_vlan_id(vlan_dev); 8985 8986 if (netif_is_bridge_port(vlan_dev)) 8987 return 0; 8988 8989 if (mlxsw_sp_port_dev_check(real_dev)) 8990 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8991 event, vid, extack); 8992 else if (netif_is_lag_master(real_dev)) 8993 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8994 vid, extack); 8995 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8996 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8997 extack); 8998 8999 return 0; 9000} 9001 9002static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 9003{ 9004 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 9005 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9006 9007 return ether_addr_equal_masked(mac, vrrp4, mask); 9008} 9009 9010static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 9011{ 9012 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 9013 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9014 9015 return ether_addr_equal_masked(mac, vrrp6, mask); 9016} 9017 9018static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9019 const u8 *mac, bool adding) 9020{ 9021 char ritr_pl[MLXSW_REG_RITR_LEN]; 9022 u8 vrrp_id = adding ? mac[5] : 0; 9023 int err; 9024 9025 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 9026 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 9027 return 0; 9028 9029 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9030 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9031 if (err) 9032 return err; 9033 9034 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 9035 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 9036 else 9037 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 9038 9039 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9040} 9041 9042static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 9043 const struct net_device *macvlan_dev, 9044 struct netlink_ext_ack *extack) 9045{ 9046 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9047 struct mlxsw_sp_rif *rif; 9048 int err; 9049 9050 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9051 if (!rif) { 9052 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 9053 return -EOPNOTSUPP; 9054 } 9055 9056 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9057 mlxsw_sp_fid_index(rif->fid), true); 9058 if (err) 9059 return err; 9060 9061 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 9062 macvlan_dev->dev_addr, true); 9063 if (err) 9064 goto err_rif_vrrp_add; 9065 9066 /* Make sure the bridge driver does not have this MAC pointing at 9067 * some other port. 9068 */ 9069 if (rif->ops->fdb_del) 9070 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 9071 9072 return 0; 9073 9074err_rif_vrrp_add: 9075 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9076 mlxsw_sp_fid_index(rif->fid), false); 9077 return err; 9078} 9079 9080static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9081 const struct net_device *macvlan_dev) 9082{ 9083 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9084 struct mlxsw_sp_rif *rif; 9085 9086 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9087 /* If we do not have a RIF, then we already took care of 9088 * removing the macvlan's MAC during RIF deletion. 9089 */ 9090 if (!rif) 9091 return; 9092 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 9093 false); 9094 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9095 mlxsw_sp_fid_index(rif->fid), false); 9096} 9097 9098void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9099 const struct net_device *macvlan_dev) 9100{ 9101 mutex_lock(&mlxsw_sp->router->lock); 9102 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9103 mutex_unlock(&mlxsw_sp->router->lock); 9104} 9105 9106static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 9107 struct net_device *macvlan_dev, 9108 unsigned long event, 9109 struct netlink_ext_ack *extack) 9110{ 9111 switch (event) { 9112 case NETDEV_UP: 9113 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 9114 case NETDEV_DOWN: 9115 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9116 break; 9117 } 9118 9119 return 0; 9120} 9121 9122static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 9123 struct net_device *dev, 9124 unsigned long event, 9125 struct netlink_ext_ack *extack) 9126{ 9127 if (mlxsw_sp_port_dev_check(dev)) 9128 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 9129 else if (netif_is_lag_master(dev)) 9130 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 9131 else if (netif_is_bridge_master(dev)) 9132 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 9133 extack); 9134 else if (is_vlan_dev(dev)) 9135 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 9136 extack); 9137 else if (netif_is_macvlan(dev)) 9138 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 9139 extack); 9140 else 9141 return 0; 9142} 9143 9144static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 9145 unsigned long event, void *ptr) 9146{ 9147 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 9148 struct net_device *dev = ifa->ifa_dev->dev; 9149 struct mlxsw_sp_router *router; 9150 struct mlxsw_sp_rif *rif; 9151 int err = 0; 9152 9153 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 9154 if (event == NETDEV_UP) 9155 return NOTIFY_DONE; 9156 9157 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 9158 mutex_lock(&router->lock); 9159 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 9160 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9161 goto out; 9162 9163 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 9164out: 9165 mutex_unlock(&router->lock); 9166 return notifier_from_errno(err); 9167} 9168 9169int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 9170 unsigned long event, void *ptr) 9171{ 9172 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 9173 struct net_device *dev = ivi->ivi_dev->dev; 9174 struct mlxsw_sp *mlxsw_sp; 9175 struct mlxsw_sp_rif *rif; 9176 int err = 0; 9177 9178 mlxsw_sp = mlxsw_sp_lower_get(dev); 9179 if (!mlxsw_sp) 9180 return NOTIFY_DONE; 9181 9182 mutex_lock(&mlxsw_sp->router->lock); 9183 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9184 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9185 goto out; 9186 9187 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 9188out: 9189 mutex_unlock(&mlxsw_sp->router->lock); 9190 return notifier_from_errno(err); 9191} 9192 9193struct mlxsw_sp_inet6addr_event_work { 9194 struct work_struct work; 9195 struct mlxsw_sp *mlxsw_sp; 9196 struct net_device *dev; 9197 unsigned long event; 9198}; 9199 9200static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 9201{ 9202 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 9203 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 9204 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 9205 struct net_device *dev = inet6addr_work->dev; 9206 unsigned long event = inet6addr_work->event; 9207 struct mlxsw_sp_rif *rif; 9208 9209 rtnl_lock(); 9210 mutex_lock(&mlxsw_sp->router->lock); 9211 9212 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9213 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9214 goto out; 9215 9216 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 9217out: 9218 mutex_unlock(&mlxsw_sp->router->lock); 9219 rtnl_unlock(); 9220 dev_put(dev); 9221 kfree(inet6addr_work); 9222} 9223 9224/* Called with rcu_read_lock() */ 9225static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 9226 unsigned long event, void *ptr) 9227{ 9228 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 9229 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 9230 struct net_device *dev = if6->idev->dev; 9231 struct mlxsw_sp_router *router; 9232 9233 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 9234 if (event == NETDEV_UP) 9235 return NOTIFY_DONE; 9236 9237 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 9238 if (!inet6addr_work) 9239 return NOTIFY_BAD; 9240 9241 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 9242 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 9243 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 9244 inet6addr_work->dev = dev; 9245 inet6addr_work->event = event; 9246 dev_hold(dev); 9247 mlxsw_core_schedule_work(&inet6addr_work->work); 9248 9249 return NOTIFY_DONE; 9250} 9251 9252int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 9253 unsigned long event, void *ptr) 9254{ 9255 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 9256 struct net_device *dev = i6vi->i6vi_dev->dev; 9257 struct mlxsw_sp *mlxsw_sp; 9258 struct mlxsw_sp_rif *rif; 9259 int err = 0; 9260 9261 mlxsw_sp = mlxsw_sp_lower_get(dev); 9262 if (!mlxsw_sp) 9263 return NOTIFY_DONE; 9264 9265 mutex_lock(&mlxsw_sp->router->lock); 9266 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9267 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9268 goto out; 9269 9270 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 9271out: 9272 mutex_unlock(&mlxsw_sp->router->lock); 9273 return notifier_from_errno(err); 9274} 9275 9276static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9277 const char *mac, int mtu, u8 mac_profile) 9278{ 9279 char ritr_pl[MLXSW_REG_RITR_LEN]; 9280 int err; 9281 9282 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9283 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9284 if (err) 9285 return err; 9286 9287 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 9288 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 9289 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 9290 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 9291 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9292} 9293 9294static int 9295mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 9296 struct mlxsw_sp_rif *rif, 9297 struct netlink_ext_ack *extack) 9298{ 9299 struct net_device *dev = rif->dev; 9300 u8 old_mac_profile; 9301 u16 fid_index; 9302 int err; 9303 9304 fid_index = mlxsw_sp_fid_index(rif->fid); 9305 9306 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 9307 if (err) 9308 return err; 9309 9310 old_mac_profile = rif->mac_profile_id; 9311 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 9312 extack); 9313 if (err) 9314 goto err_rif_mac_profile_replace; 9315 9316 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 9317 dev->mtu, rif->mac_profile_id); 9318 if (err) 9319 goto err_rif_edit; 9320 9321 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9322 if (err) 9323 goto err_rif_fdb_op; 9324 9325 if (rif->mtu != dev->mtu) { 9326 struct mlxsw_sp_vr *vr; 9327 int i; 9328 9329 /* The RIF is relevant only to its mr_table instance, as unlike 9330 * unicast routing, in multicast routing a RIF cannot be shared 9331 * between several multicast routing tables. 9332 */ 9333 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9334 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9335 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9336 rif, dev->mtu); 9337 } 9338 9339 ether_addr_copy(rif->addr, dev->dev_addr); 9340 rif->mtu = dev->mtu; 9341 9342 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9343 9344 return 0; 9345 9346err_rif_fdb_op: 9347 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9348 old_mac_profile); 9349err_rif_edit: 9350 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9351err_rif_mac_profile_replace: 9352 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9353 return err; 9354} 9355 9356static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9357 struct netdev_notifier_pre_changeaddr_info *info) 9358{ 9359 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9360 struct mlxsw_sp_rif_mac_profile *profile; 9361 struct netlink_ext_ack *extack; 9362 u8 max_rif_mac_profiles; 9363 u64 occ; 9364 9365 extack = netdev_notifier_info_to_extack(&info->info); 9366 9367 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9368 if (profile) 9369 return 0; 9370 9371 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9372 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9373 if (occ < max_rif_mac_profiles) 9374 return 0; 9375 9376 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9377 return 0; 9378 9379 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9380 return -ENOBUFS; 9381} 9382 9383static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9384{ 9385 switch (event) { 9386 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9387 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9388 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9389 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9390 return true; 9391 } 9392 9393 return false; 9394} 9395 9396static int 9397mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9398 unsigned long event, 9399 struct netdev_notifier_offload_xstats_info *info) 9400{ 9401 switch (info->type) { 9402 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9403 break; 9404 default: 9405 return 0; 9406 } 9407 9408 switch (event) { 9409 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9410 return mlxsw_sp_router_port_l3_stats_enable(rif); 9411 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9412 mlxsw_sp_router_port_l3_stats_disable(rif); 9413 return 0; 9414 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9415 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9416 return 0; 9417 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9418 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9419 } 9420 9421 WARN_ON_ONCE(1); 9422 return 0; 9423} 9424 9425static int 9426mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9427 struct net_device *dev, 9428 unsigned long event, 9429 struct netdev_notifier_offload_xstats_info *info) 9430{ 9431 struct mlxsw_sp_rif *rif; 9432 9433 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9434 if (!rif) 9435 return 0; 9436 9437 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9438} 9439 9440static bool mlxsw_sp_is_router_event(unsigned long event) 9441{ 9442 switch (event) { 9443 case NETDEV_PRE_CHANGEADDR: 9444 case NETDEV_CHANGEADDR: 9445 case NETDEV_CHANGEMTU: 9446 return true; 9447 default: 9448 return false; 9449 } 9450} 9451 9452static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9453 unsigned long event, void *ptr) 9454{ 9455 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9456 struct mlxsw_sp *mlxsw_sp; 9457 struct mlxsw_sp_rif *rif; 9458 9459 mlxsw_sp = mlxsw_sp_lower_get(dev); 9460 if (!mlxsw_sp) 9461 return 0; 9462 9463 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9464 if (!rif) 9465 return 0; 9466 9467 switch (event) { 9468 case NETDEV_CHANGEMTU: 9469 case NETDEV_CHANGEADDR: 9470 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9471 case NETDEV_PRE_CHANGEADDR: 9472 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9473 default: 9474 WARN_ON_ONCE(1); 9475 break; 9476 } 9477 9478 return 0; 9479} 9480 9481static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9482 struct net_device *l3_dev, 9483 struct netlink_ext_ack *extack) 9484{ 9485 struct mlxsw_sp_rif *rif; 9486 9487 /* If netdev is already associated with a RIF, then we need to 9488 * destroy it and create a new one with the new virtual router ID. 9489 */ 9490 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9491 if (rif) 9492 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 9493 extack); 9494 9495 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 9496} 9497 9498static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9499 struct net_device *l3_dev) 9500{ 9501 struct mlxsw_sp_rif *rif; 9502 9503 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9504 if (!rif) 9505 return; 9506 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 9507} 9508 9509static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9510{ 9511 struct netdev_notifier_changeupper_info *info = ptr; 9512 9513 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9514 return false; 9515 return netif_is_l3_master(info->upper_dev); 9516} 9517 9518static int 9519mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9520 struct netdev_notifier_changeupper_info *info) 9521{ 9522 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9523 int err = 0; 9524 9525 /* We do not create a RIF for a macvlan, but only use it to 9526 * direct more MAC addresses to the router. 9527 */ 9528 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9529 return 0; 9530 9531 switch (event) { 9532 case NETDEV_PRECHANGEUPPER: 9533 break; 9534 case NETDEV_CHANGEUPPER: 9535 if (info->linking) { 9536 struct netlink_ext_ack *extack; 9537 9538 extack = netdev_notifier_info_to_extack(&info->info); 9539 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9540 } else { 9541 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9542 } 9543 break; 9544 } 9545 9546 return err; 9547} 9548 9549static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 9550 unsigned long event, void *ptr) 9551{ 9552 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 9553 struct mlxsw_sp_router *router; 9554 struct mlxsw_sp *mlxsw_sp; 9555 int err = 0; 9556 9557 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 9558 mlxsw_sp = router->mlxsw_sp; 9559 9560 mutex_lock(&mlxsw_sp->router->lock); 9561 9562 if (mlxsw_sp_is_offload_xstats_event(event)) 9563 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 9564 event, ptr); 9565 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 9566 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 9567 event, ptr); 9568 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 9569 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 9570 event, ptr); 9571 else if (mlxsw_sp_is_router_event(event)) 9572 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 9573 else if (mlxsw_sp_is_vrf_event(event, ptr)) 9574 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 9575 9576 mutex_unlock(&mlxsw_sp->router->lock); 9577 9578 return notifier_from_errno(err); 9579} 9580 9581static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 9582 struct netdev_nested_priv *priv) 9583{ 9584 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 9585 9586 if (!netif_is_macvlan(dev)) 9587 return 0; 9588 9589 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9590 mlxsw_sp_fid_index(rif->fid), false); 9591} 9592 9593static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 9594{ 9595 struct netdev_nested_priv priv = { 9596 .data = (void *)rif, 9597 }; 9598 9599 if (!netif_is_macvlan_port(rif->dev)) 9600 return 0; 9601 9602 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 9603 return netdev_walk_all_upper_dev_rcu(rif->dev, 9604 __mlxsw_sp_rif_macvlan_flush, &priv); 9605} 9606 9607static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 9608 const struct mlxsw_sp_rif_params *params) 9609{ 9610 struct mlxsw_sp_rif_subport *rif_subport; 9611 9612 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9613 refcount_set(&rif_subport->ref_count, 1); 9614 rif_subport->vid = params->vid; 9615 rif_subport->lag = params->lag; 9616 if (params->lag) 9617 rif_subport->lag_id = params->lag_id; 9618 else 9619 rif_subport->system_port = params->system_port; 9620} 9621 9622static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 9623{ 9624 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9625 struct mlxsw_sp_rif_subport *rif_subport; 9626 char ritr_pl[MLXSW_REG_RITR_LEN]; 9627 9628 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9629 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 9630 rif->rif_index, rif->vr_id, rif->dev->mtu); 9631 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9632 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9633 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 9634 rif_subport->lag ? rif_subport->lag_id : 9635 rif_subport->system_port, 9636 rif_subport->vid); 9637 9638 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9639} 9640 9641static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 9642 struct netlink_ext_ack *extack) 9643{ 9644 u8 mac_profile; 9645 int err; 9646 9647 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 9648 &mac_profile, extack); 9649 if (err) 9650 return err; 9651 rif->mac_profile_id = mac_profile; 9652 9653 err = mlxsw_sp_rif_subport_op(rif, true); 9654 if (err) 9655 goto err_rif_subport_op; 9656 9657 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9658 mlxsw_sp_fid_index(rif->fid), true); 9659 if (err) 9660 goto err_rif_fdb_op; 9661 9662 mlxsw_sp_fid_rif_set(rif->fid, rif); 9663 return 0; 9664 9665err_rif_fdb_op: 9666 mlxsw_sp_rif_subport_op(rif, false); 9667err_rif_subport_op: 9668 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 9669 return err; 9670} 9671 9672static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 9673{ 9674 struct mlxsw_sp_fid *fid = rif->fid; 9675 9676 mlxsw_sp_fid_rif_set(fid, NULL); 9677 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9678 mlxsw_sp_fid_index(fid), false); 9679 mlxsw_sp_rif_macvlan_flush(rif); 9680 mlxsw_sp_rif_subport_op(rif, false); 9681 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9682} 9683 9684static struct mlxsw_sp_fid * 9685mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 9686 struct netlink_ext_ack *extack) 9687{ 9688 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 9689} 9690 9691static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 9692 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 9693 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 9694 .setup = mlxsw_sp_rif_subport_setup, 9695 .configure = mlxsw_sp_rif_subport_configure, 9696 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 9697 .fid_get = mlxsw_sp_rif_subport_fid_get, 9698}; 9699 9700static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 9701 enum mlxsw_reg_ritr_if_type type, 9702 u16 vid_fid, bool enable) 9703{ 9704 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9705 char ritr_pl[MLXSW_REG_RITR_LEN]; 9706 9707 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 9708 rif->dev->mtu); 9709 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9710 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9711 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 9712 9713 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9714} 9715 9716u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 9717{ 9718 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 9719} 9720 9721static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 9722 struct netlink_ext_ack *extack) 9723{ 9724 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9725 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9726 u8 mac_profile; 9727 int err; 9728 9729 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 9730 &mac_profile, extack); 9731 if (err) 9732 return err; 9733 rif->mac_profile_id = mac_profile; 9734 9735 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 9736 true); 9737 if (err) 9738 goto err_rif_vlan_fid_op; 9739 9740 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9741 mlxsw_sp_router_port(mlxsw_sp), true); 9742 if (err) 9743 goto err_fid_mc_flood_set; 9744 9745 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9746 mlxsw_sp_router_port(mlxsw_sp), true); 9747 if (err) 9748 goto err_fid_bc_flood_set; 9749 9750 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9751 mlxsw_sp_fid_index(rif->fid), true); 9752 if (err) 9753 goto err_rif_fdb_op; 9754 9755 mlxsw_sp_fid_rif_set(rif->fid, rif); 9756 return 0; 9757 9758err_rif_fdb_op: 9759 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9760 mlxsw_sp_router_port(mlxsw_sp), false); 9761err_fid_bc_flood_set: 9762 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9763 mlxsw_sp_router_port(mlxsw_sp), false); 9764err_fid_mc_flood_set: 9765 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9766err_rif_vlan_fid_op: 9767 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 9768 return err; 9769} 9770 9771static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 9772{ 9773 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9774 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9775 struct mlxsw_sp_fid *fid = rif->fid; 9776 9777 mlxsw_sp_fid_rif_set(fid, NULL); 9778 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9779 mlxsw_sp_fid_index(fid), false); 9780 mlxsw_sp_rif_macvlan_flush(rif); 9781 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9782 mlxsw_sp_router_port(mlxsw_sp), false); 9783 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9784 mlxsw_sp_router_port(mlxsw_sp), false); 9785 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9786 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9787} 9788 9789static struct mlxsw_sp_fid * 9790mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 9791 struct netlink_ext_ack *extack) 9792{ 9793 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 9794} 9795 9796static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9797{ 9798 struct switchdev_notifier_fdb_info info = {}; 9799 struct net_device *dev; 9800 9801 dev = br_fdb_find_port(rif->dev, mac, 0); 9802 if (!dev) 9803 return; 9804 9805 info.addr = mac; 9806 info.vid = 0; 9807 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9808 NULL); 9809} 9810 9811static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 9812 .type = MLXSW_SP_RIF_TYPE_FID, 9813 .rif_size = sizeof(struct mlxsw_sp_rif), 9814 .configure = mlxsw_sp_rif_fid_configure, 9815 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9816 .fid_get = mlxsw_sp_rif_fid_fid_get, 9817 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 9818}; 9819 9820static struct mlxsw_sp_fid * 9821mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 9822 struct netlink_ext_ack *extack) 9823{ 9824 struct net_device *br_dev; 9825 u16 vid; 9826 int err; 9827 9828 if (is_vlan_dev(rif->dev)) { 9829 vid = vlan_dev_vlan_id(rif->dev); 9830 br_dev = vlan_dev_real_dev(rif->dev); 9831 if (WARN_ON(!netif_is_bridge_master(br_dev))) 9832 return ERR_PTR(-EINVAL); 9833 } else { 9834 err = br_vlan_get_pvid(rif->dev, &vid); 9835 if (err < 0 || !vid) { 9836 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 9837 return ERR_PTR(-EINVAL); 9838 } 9839 } 9840 9841 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 9842} 9843 9844static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9845{ 9846 struct switchdev_notifier_fdb_info info = {}; 9847 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9848 struct net_device *br_dev; 9849 struct net_device *dev; 9850 9851 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 9852 dev = br_fdb_find_port(br_dev, mac, vid); 9853 if (!dev) 9854 return; 9855 9856 info.addr = mac; 9857 info.vid = vid; 9858 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9859 NULL); 9860} 9861 9862static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 9863 .type = MLXSW_SP_RIF_TYPE_VLAN, 9864 .rif_size = sizeof(struct mlxsw_sp_rif), 9865 .configure = mlxsw_sp_rif_fid_configure, 9866 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9867 .fid_get = mlxsw_sp_rif_vlan_fid_get, 9868 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 9869}; 9870 9871static struct mlxsw_sp_rif_ipip_lb * 9872mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 9873{ 9874 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 9875} 9876 9877static void 9878mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 9879 const struct mlxsw_sp_rif_params *params) 9880{ 9881 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 9882 struct mlxsw_sp_rif_ipip_lb *rif_lb; 9883 9884 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 9885 common); 9886 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 9887 rif_lb->lb_config = params_lb->lb_config; 9888} 9889 9890static int 9891mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 9892 struct netlink_ext_ack *extack) 9893{ 9894 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9895 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9896 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9897 struct mlxsw_sp_vr *ul_vr; 9898 int err; 9899 9900 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 9901 if (IS_ERR(ul_vr)) 9902 return PTR_ERR(ul_vr); 9903 9904 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 9905 if (err) 9906 goto err_loopback_op; 9907 9908 lb_rif->ul_vr_id = ul_vr->id; 9909 lb_rif->ul_rif_id = 0; 9910 ++ul_vr->rif_count; 9911 return 0; 9912 9913err_loopback_op: 9914 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9915 return err; 9916} 9917 9918static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9919{ 9920 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9921 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9922 struct mlxsw_sp_vr *ul_vr; 9923 9924 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 9925 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 9926 9927 --ul_vr->rif_count; 9928 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9929} 9930 9931static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 9932 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9933 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9934 .setup = mlxsw_sp_rif_ipip_lb_setup, 9935 .configure = mlxsw_sp1_rif_ipip_lb_configure, 9936 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 9937}; 9938 9939static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 9940 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9941 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9942 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9943 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 9944}; 9945 9946static int 9947mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 9948{ 9949 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9950 char ritr_pl[MLXSW_REG_RITR_LEN]; 9951 9952 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 9953 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 9954 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 9955 MLXSW_REG_RITR_LOOPBACK_GENERIC); 9956 9957 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9958} 9959 9960static struct mlxsw_sp_rif * 9961mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 9962 struct netlink_ext_ack *extack) 9963{ 9964 struct mlxsw_sp_rif *ul_rif; 9965 u16 rif_index; 9966 int err; 9967 9968 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 9969 if (err) { 9970 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 9971 return ERR_PTR(err); 9972 } 9973 9974 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 9975 if (!ul_rif) 9976 return ERR_PTR(-ENOMEM); 9977 9978 mlxsw_sp->router->rifs[rif_index] = ul_rif; 9979 ul_rif->mlxsw_sp = mlxsw_sp; 9980 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 9981 if (err) 9982 goto ul_rif_op_err; 9983 9984 return ul_rif; 9985 9986ul_rif_op_err: 9987 mlxsw_sp->router->rifs[rif_index] = NULL; 9988 kfree(ul_rif); 9989 return ERR_PTR(err); 9990} 9991 9992static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 9993{ 9994 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9995 9996 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 9997 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 9998 kfree(ul_rif); 9999} 10000 10001static struct mlxsw_sp_rif * 10002mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 10003 struct netlink_ext_ack *extack) 10004{ 10005 struct mlxsw_sp_vr *vr; 10006 int err; 10007 10008 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 10009 if (IS_ERR(vr)) 10010 return ERR_CAST(vr); 10011 10012 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 10013 return vr->ul_rif; 10014 10015 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 10016 if (IS_ERR(vr->ul_rif)) { 10017 err = PTR_ERR(vr->ul_rif); 10018 goto err_ul_rif_create; 10019 } 10020 10021 vr->rif_count++; 10022 refcount_set(&vr->ul_rif_refcnt, 1); 10023 10024 return vr->ul_rif; 10025 10026err_ul_rif_create: 10027 mlxsw_sp_vr_put(mlxsw_sp, vr); 10028 return ERR_PTR(err); 10029} 10030 10031static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 10032{ 10033 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10034 struct mlxsw_sp_vr *vr; 10035 10036 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 10037 10038 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 10039 return; 10040 10041 vr->rif_count--; 10042 mlxsw_sp_ul_rif_destroy(ul_rif); 10043 mlxsw_sp_vr_put(mlxsw_sp, vr); 10044} 10045 10046int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 10047 u16 *ul_rif_index) 10048{ 10049 struct mlxsw_sp_rif *ul_rif; 10050 int err = 0; 10051 10052 mutex_lock(&mlxsw_sp->router->lock); 10053 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 10054 if (IS_ERR(ul_rif)) { 10055 err = PTR_ERR(ul_rif); 10056 goto out; 10057 } 10058 *ul_rif_index = ul_rif->rif_index; 10059out: 10060 mutex_unlock(&mlxsw_sp->router->lock); 10061 return err; 10062} 10063 10064void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 10065{ 10066 struct mlxsw_sp_rif *ul_rif; 10067 10068 mutex_lock(&mlxsw_sp->router->lock); 10069 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 10070 if (WARN_ON(!ul_rif)) 10071 goto out; 10072 10073 mlxsw_sp_ul_rif_put(ul_rif); 10074out: 10075 mutex_unlock(&mlxsw_sp->router->lock); 10076} 10077 10078static int 10079mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10080 struct netlink_ext_ack *extack) 10081{ 10082 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10083 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 10084 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10085 struct mlxsw_sp_rif *ul_rif; 10086 int err; 10087 10088 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 10089 if (IS_ERR(ul_rif)) 10090 return PTR_ERR(ul_rif); 10091 10092 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 10093 if (err) 10094 goto err_loopback_op; 10095 10096 lb_rif->ul_vr_id = 0; 10097 lb_rif->ul_rif_id = ul_rif->rif_index; 10098 10099 return 0; 10100 10101err_loopback_op: 10102 mlxsw_sp_ul_rif_put(ul_rif); 10103 return err; 10104} 10105 10106static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10107{ 10108 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10109 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10110 struct mlxsw_sp_rif *ul_rif; 10111 10112 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 10113 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 10114 mlxsw_sp_ul_rif_put(ul_rif); 10115} 10116 10117static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 10118 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10119 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10120 .setup = mlxsw_sp_rif_ipip_lb_setup, 10121 .configure = mlxsw_sp2_rif_ipip_lb_configure, 10122 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 10123}; 10124 10125static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 10126 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10127 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 10128 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10129 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 10130}; 10131 10132static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 10133{ 10134 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10135 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10136 struct mlxsw_core *core = mlxsw_sp->core; 10137 10138 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 10139 return -EIO; 10140 mlxsw_sp->router->max_rif_mac_profile = 10141 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 10142 10143 mlxsw_sp->router->rifs = kcalloc(max_rifs, 10144 sizeof(struct mlxsw_sp_rif *), 10145 GFP_KERNEL); 10146 if (!mlxsw_sp->router->rifs) 10147 return -ENOMEM; 10148 10149 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 10150 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 10151 devlink_resource_occ_get_register(devlink, 10152 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 10153 mlxsw_sp_rif_mac_profiles_occ_get, 10154 mlxsw_sp); 10155 10156 return 0; 10157} 10158 10159static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 10160{ 10161 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10162 int i; 10163 10164 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 10165 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 10166 10167 devlink_resource_occ_get_unregister(devlink, 10168 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 10169 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 10170 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 10171 kfree(mlxsw_sp->router->rifs); 10172} 10173 10174static int 10175mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 10176{ 10177 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 10178 10179 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 10180 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 10181} 10182 10183static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 10184{ 10185 int err; 10186 10187 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 10188 10189 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 10190 if (err) 10191 return err; 10192 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 10193 if (err) 10194 return err; 10195 10196 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 10197} 10198 10199static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 10200{ 10201 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 10202 return mlxsw_sp_ipips_init(mlxsw_sp); 10203} 10204 10205static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 10206{ 10207 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 10208 return mlxsw_sp_ipips_init(mlxsw_sp); 10209} 10210 10211static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 10212{ 10213 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 10214} 10215 10216static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 10217{ 10218 struct mlxsw_sp_router *router; 10219 10220 /* Flush pending FIB notifications and then flush the device's 10221 * table before requesting another dump. The FIB notification 10222 * block is unregistered, so no need to take RTNL. 10223 */ 10224 mlxsw_core_flush_owq(); 10225 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 10226 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 10227} 10228 10229#ifdef CONFIG_IP_ROUTE_MULTIPATH 10230struct mlxsw_sp_mp_hash_config { 10231 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 10232 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 10233 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 10234 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 10235 bool inc_parsing_depth; 10236}; 10237 10238#define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 10239 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 10240 10241#define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 10242 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 10243 10244#define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 10245 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 10246 10247static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 10248{ 10249 unsigned long *inner_headers = config->inner_headers; 10250 unsigned long *inner_fields = config->inner_fields; 10251 10252 /* IPv4 inner */ 10253 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10254 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10255 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10256 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10257 /* IPv6 inner */ 10258 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10259 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10260 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10261 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10262 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10263 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10264 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10265 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10266} 10267 10268static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10269{ 10270 unsigned long *headers = config->headers; 10271 unsigned long *fields = config->fields; 10272 10273 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10274 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10275 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10276 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10277} 10278 10279static void 10280mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 10281 u32 hash_fields) 10282{ 10283 unsigned long *inner_headers = config->inner_headers; 10284 unsigned long *inner_fields = config->inner_fields; 10285 10286 /* IPv4 Inner */ 10287 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10288 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10289 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 10290 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10291 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 10292 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10293 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 10294 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 10295 /* IPv6 inner */ 10296 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10297 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10298 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 10299 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10300 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10301 } 10302 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 10303 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10304 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10305 } 10306 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 10307 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10308 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 10309 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10310 /* L4 inner */ 10311 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 10312 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 10313 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 10314 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 10315 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 10316 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 10317} 10318 10319static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 10320 struct mlxsw_sp_mp_hash_config *config) 10321{ 10322 struct net *net = mlxsw_sp_net(mlxsw_sp); 10323 unsigned long *headers = config->headers; 10324 unsigned long *fields = config->fields; 10325 u32 hash_fields; 10326 10327 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 10328 case 0: 10329 mlxsw_sp_mp4_hash_outer_addr(config); 10330 break; 10331 case 1: 10332 mlxsw_sp_mp4_hash_outer_addr(config); 10333 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10334 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10335 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10336 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10337 break; 10338 case 2: 10339 /* Outer */ 10340 mlxsw_sp_mp4_hash_outer_addr(config); 10341 /* Inner */ 10342 mlxsw_sp_mp_hash_inner_l3(config); 10343 break; 10344 case 3: 10345 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; 10346 /* Outer */ 10347 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10348 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10349 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10350 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 10351 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10352 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 10353 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10354 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10355 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10356 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10357 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10358 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10359 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10360 /* Inner */ 10361 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10362 break; 10363 } 10364} 10365 10366static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10367{ 10368 unsigned long *headers = config->headers; 10369 unsigned long *fields = config->fields; 10370 10371 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10372 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10373 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10374 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10375 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10376 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10377} 10378 10379static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 10380 struct mlxsw_sp_mp_hash_config *config) 10381{ 10382 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 10383 unsigned long *headers = config->headers; 10384 unsigned long *fields = config->fields; 10385 10386 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 10387 case 0: 10388 mlxsw_sp_mp6_hash_outer_addr(config); 10389 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10390 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10391 break; 10392 case 1: 10393 mlxsw_sp_mp6_hash_outer_addr(config); 10394 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10395 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10396 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10397 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10398 break; 10399 case 2: 10400 /* Outer */ 10401 mlxsw_sp_mp6_hash_outer_addr(config); 10402 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10403 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10404 /* Inner */ 10405 mlxsw_sp_mp_hash_inner_l3(config); 10406 config->inc_parsing_depth = true; 10407 break; 10408 case 3: 10409 /* Outer */ 10410 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10411 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10412 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10413 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 10414 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10415 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10416 } 10417 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 10418 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10419 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10420 } 10421 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10422 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10423 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 10424 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10425 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10426 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10427 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10428 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10429 /* Inner */ 10430 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10431 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 10432 config->inc_parsing_depth = true; 10433 break; 10434 } 10435} 10436 10437static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 10438 bool old_inc_parsing_depth, 10439 bool new_inc_parsing_depth) 10440{ 10441 int err; 10442 10443 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 10444 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 10445 if (err) 10446 return err; 10447 mlxsw_sp->router->inc_parsing_depth = true; 10448 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 10449 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 10450 mlxsw_sp->router->inc_parsing_depth = false; 10451 } 10452 10453 return 0; 10454} 10455 10456static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10457{ 10458 bool old_inc_parsing_depth, new_inc_parsing_depth; 10459 struct mlxsw_sp_mp_hash_config config = {}; 10460 char recr2_pl[MLXSW_REG_RECR2_LEN]; 10461 unsigned long bit; 10462 u32 seed; 10463 int err; 10464 10465 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 10466 mlxsw_reg_recr2_pack(recr2_pl, seed); 10467 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 10468 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 10469 10470 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 10471 new_inc_parsing_depth = config.inc_parsing_depth; 10472 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 10473 old_inc_parsing_depth, 10474 new_inc_parsing_depth); 10475 if (err) 10476 return err; 10477 10478 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 10479 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 10480 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 10481 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 10482 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 10483 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 10484 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 10485 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 10486 10487 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 10488 if (err) 10489 goto err_reg_write; 10490 10491 return 0; 10492 10493err_reg_write: 10494 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 10495 old_inc_parsing_depth); 10496 return err; 10497} 10498#else 10499static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10500{ 10501 return 0; 10502} 10503#endif 10504 10505static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 10506{ 10507 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 10508 unsigned int i; 10509 10510 MLXSW_REG_ZERO(rdpm, rdpm_pl); 10511 10512 /* HW is determining switch priority based on DSCP-bits, but the 10513 * kernel is still doing that based on the ToS. Since there's a 10514 * mismatch in bits we need to make sure to translate the right 10515 * value ToS would observe, skipping the 2 least-significant ECN bits. 10516 */ 10517 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 10518 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 10519 10520 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 10521} 10522 10523static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 10524{ 10525 struct net *net = mlxsw_sp_net(mlxsw_sp); 10526 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 10527 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10528 u64 max_rifs; 10529 10530 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 10531 return -EIO; 10532 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10533 10534 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 10535 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 10536 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 10537 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10538} 10539 10540static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10541{ 10542 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10543 10544 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 10545 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10546} 10547 10548static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 10549 .init = mlxsw_sp_router_ll_basic_init, 10550 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 10551 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 10552 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 10553 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 10554 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 10555 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 10556 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 10557 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 10558 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 10559 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 10560 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 10561}; 10562 10563static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 10564{ 10565 size_t max_size = 0; 10566 int i; 10567 10568 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 10569 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 10570 10571 if (size > max_size) 10572 max_size = size; 10573 } 10574 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 10575 GFP_KERNEL); 10576 if (!router->ll_op_ctx) 10577 return -ENOMEM; 10578 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 10579 return 0; 10580} 10581 10582static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 10583{ 10584 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 10585 kfree(router->ll_op_ctx); 10586} 10587 10588static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 10589{ 10590 u16 lb_rif_index; 10591 int err; 10592 10593 /* Create a generic loopback RIF associated with the main table 10594 * (default VRF). Any table can be used, but the main table exists 10595 * anyway, so we do not waste resources. 10596 */ 10597 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 10598 &lb_rif_index); 10599 if (err) 10600 return err; 10601 10602 mlxsw_sp->router->lb_rif_index = lb_rif_index; 10603 10604 return 0; 10605} 10606 10607static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 10608{ 10609 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 10610} 10611 10612static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 10613{ 10614 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 10615 10616 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 10617 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 10618 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10619 10620 return 0; 10621} 10622 10623const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 10624 .init = mlxsw_sp1_router_init, 10625 .ipips_init = mlxsw_sp1_ipips_init, 10626}; 10627 10628static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 10629{ 10630 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 10631 10632 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 10633 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 10634 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10635 10636 return 0; 10637} 10638 10639const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 10640 .init = mlxsw_sp2_router_init, 10641 .ipips_init = mlxsw_sp2_ipips_init, 10642}; 10643 10644int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 10645 struct netlink_ext_ack *extack) 10646{ 10647 struct mlxsw_sp_router *router; 10648 int err; 10649 10650 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 10651 if (!router) 10652 return -ENOMEM; 10653 mutex_init(&router->lock); 10654 mlxsw_sp->router = router; 10655 router->mlxsw_sp = mlxsw_sp; 10656 10657 err = mlxsw_sp->router_ops->init(mlxsw_sp); 10658 if (err) 10659 goto err_router_ops_init; 10660 10661 err = mlxsw_sp_router_xm_init(mlxsw_sp); 10662 if (err) 10663 goto err_xm_init; 10664 10665 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 10666 &mlxsw_sp_router_ll_xm_ops : 10667 &mlxsw_sp_router_ll_basic_ops; 10668 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 10669 10670 err = mlxsw_sp_router_ll_op_ctx_init(router); 10671 if (err) 10672 goto err_ll_op_ctx_init; 10673 10674 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 10675 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 10676 mlxsw_sp_nh_grp_activity_work); 10677 10678 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 10679 err = __mlxsw_sp_router_init(mlxsw_sp); 10680 if (err) 10681 goto err_router_init; 10682 10683 err = mlxsw_sp_rifs_init(mlxsw_sp); 10684 if (err) 10685 goto err_rifs_init; 10686 10687 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 10688 if (err) 10689 goto err_ipips_init; 10690 10691 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 10692 &mlxsw_sp_nexthop_ht_params); 10693 if (err) 10694 goto err_nexthop_ht_init; 10695 10696 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 10697 &mlxsw_sp_nexthop_group_ht_params); 10698 if (err) 10699 goto err_nexthop_group_ht_init; 10700 10701 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 10702 err = mlxsw_sp_lpm_init(mlxsw_sp); 10703 if (err) 10704 goto err_lpm_init; 10705 10706 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 10707 if (err) 10708 goto err_mr_init; 10709 10710 err = mlxsw_sp_vrs_init(mlxsw_sp); 10711 if (err) 10712 goto err_vrs_init; 10713 10714 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 10715 if (err) 10716 goto err_lb_rif_init; 10717 10718 err = mlxsw_sp_neigh_init(mlxsw_sp); 10719 if (err) 10720 goto err_neigh_init; 10721 10722 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 10723 if (err) 10724 goto err_mp_hash_init; 10725 10726 err = mlxsw_sp_dscp_init(mlxsw_sp); 10727 if (err) 10728 goto err_dscp_init; 10729 10730 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 10731 INIT_LIST_HEAD(&router->fib_event_queue); 10732 spin_lock_init(&router->fib_event_queue_lock); 10733 10734 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 10735 err = register_inetaddr_notifier(&router->inetaddr_nb); 10736 if (err) 10737 goto err_register_inetaddr_notifier; 10738 10739 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 10740 err = register_inet6addr_notifier(&router->inet6addr_nb); 10741 if (err) 10742 goto err_register_inet6addr_notifier; 10743 10744 mlxsw_sp->router->netevent_nb.notifier_call = 10745 mlxsw_sp_router_netevent_event; 10746 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10747 if (err) 10748 goto err_register_netevent_notifier; 10749 10750 mlxsw_sp->router->nexthop_nb.notifier_call = 10751 mlxsw_sp_nexthop_obj_event; 10752 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10753 &mlxsw_sp->router->nexthop_nb, 10754 extack); 10755 if (err) 10756 goto err_register_nexthop_notifier; 10757 10758 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 10759 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10760 &mlxsw_sp->router->fib_nb, 10761 mlxsw_sp_router_fib_dump_flush, extack); 10762 if (err) 10763 goto err_register_fib_notifier; 10764 10765 mlxsw_sp->router->netdevice_nb.notifier_call = 10766 mlxsw_sp_router_netdevice_event; 10767 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10768 &mlxsw_sp->router->netdevice_nb); 10769 if (err) 10770 goto err_register_netdev_notifier; 10771 10772 return 0; 10773 10774err_register_netdev_notifier: 10775 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10776 &mlxsw_sp->router->fib_nb); 10777err_register_fib_notifier: 10778 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10779 &mlxsw_sp->router->nexthop_nb); 10780err_register_nexthop_notifier: 10781 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10782err_register_netevent_notifier: 10783 unregister_inet6addr_notifier(&router->inet6addr_nb); 10784err_register_inet6addr_notifier: 10785 unregister_inetaddr_notifier(&router->inetaddr_nb); 10786err_register_inetaddr_notifier: 10787 mlxsw_core_flush_owq(); 10788 WARN_ON(!list_empty(&router->fib_event_queue)); 10789err_dscp_init: 10790err_mp_hash_init: 10791 mlxsw_sp_neigh_fini(mlxsw_sp); 10792err_neigh_init: 10793 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10794err_lb_rif_init: 10795 mlxsw_sp_vrs_fini(mlxsw_sp); 10796err_vrs_init: 10797 mlxsw_sp_mr_fini(mlxsw_sp); 10798err_mr_init: 10799 mlxsw_sp_lpm_fini(mlxsw_sp); 10800err_lpm_init: 10801 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10802err_nexthop_group_ht_init: 10803 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10804err_nexthop_ht_init: 10805 mlxsw_sp_ipips_fini(mlxsw_sp); 10806err_ipips_init: 10807 mlxsw_sp_rifs_fini(mlxsw_sp); 10808err_rifs_init: 10809 __mlxsw_sp_router_fini(mlxsw_sp); 10810err_router_init: 10811 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10812 mlxsw_sp_router_ll_op_ctx_fini(router); 10813err_ll_op_ctx_init: 10814 mlxsw_sp_router_xm_fini(mlxsw_sp); 10815err_xm_init: 10816err_router_ops_init: 10817 mutex_destroy(&mlxsw_sp->router->lock); 10818 kfree(mlxsw_sp->router); 10819 return err; 10820} 10821 10822void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10823{ 10824 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10825 &mlxsw_sp->router->netdevice_nb); 10826 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10827 &mlxsw_sp->router->fib_nb); 10828 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10829 &mlxsw_sp->router->nexthop_nb); 10830 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10831 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 10832 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 10833 mlxsw_core_flush_owq(); 10834 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 10835 mlxsw_sp_neigh_fini(mlxsw_sp); 10836 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10837 mlxsw_sp_vrs_fini(mlxsw_sp); 10838 mlxsw_sp_mr_fini(mlxsw_sp); 10839 mlxsw_sp_lpm_fini(mlxsw_sp); 10840 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10841 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10842 mlxsw_sp_ipips_fini(mlxsw_sp); 10843 mlxsw_sp_rifs_fini(mlxsw_sp); 10844 __mlxsw_sp_router_fini(mlxsw_sp); 10845 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10846 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 10847 mlxsw_sp_router_xm_fini(mlxsw_sp); 10848 mutex_destroy(&mlxsw_sp->router->lock); 10849 kfree(mlxsw_sp->router); 10850}