drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <linux/if_bridge.h>
  46 #include <net/netevent.h>
  47 #include <net/neighbour.h>
  48 #include <net/arp.h>
  49 #include <net/ip_fib.h>
  50 #include <net/fib_rules.h>
  51 #include <net/l3mdev.h>
  52
  53 #include "spectrum.h"
  54 #include "core.h"
  55 #include "reg.h"
  56 #include "spectrum_cnt.h"
  57 #include "spectrum_dpipe.h"
  58 #include "spectrum_router.h"
  59
  60 struct mlxsw_sp_vr;
  61 struct mlxsw_sp_lpm_tree;
  62 struct mlxsw_sp_rif_ops;
  63
  64 struct mlxsw_sp_router {
  65         struct mlxsw_sp *mlxsw_sp;
  66         struct mlxsw_sp_rif **rifs;
  67         struct mlxsw_sp_vr *vrs;
  68         struct rhashtable neigh_ht;
  69         struct rhashtable nexthop_group_ht;
  70         struct rhashtable nexthop_ht;
  71         struct {
  72                 struct mlxsw_sp_lpm_tree *trees;
  73                 unsigned int tree_count;
  74         } lpm;
  75         struct {
  76                 struct delayed_work dw;
  77                 unsigned long interval; /* ms */
  78         } neighs_update;
  79         struct delayed_work nexthop_probe_dw;
  80 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  81         struct list_head nexthop_neighs_list;
  82         bool aborted;
  83         struct notifier_block fib_nb;
  84         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  85 };
  86
  87 struct mlxsw_sp_rif {
  88         struct list_head nexthop_list;
  89         struct list_head neigh_list;
  90         struct net_device *dev;
  91         struct mlxsw_sp_fid *fid;
  92         unsigned char addr[ETH_ALEN];
  93         int mtu;
  94         u16 rif_index;
  95         u16 vr_id;
  96         const struct mlxsw_sp_rif_ops *ops;
  97         struct mlxsw_sp *mlxsw_sp;
  98
  99         unsigned int counter_ingress;
 100         bool counter_ingress_valid;
 101         unsigned int counter_egress;
 102         bool counter_egress_valid;
 103 };
 104
 105 struct mlxsw_sp_rif_params {
 106         struct net_device *dev;
 107         union {
 108                 u16 system_port;
 109                 u16 lag_id;
 110         };
 111         u16 vid;
 112         bool lag;
 113 };
 114
 115 struct mlxsw_sp_rif_subport {
 116         struct mlxsw_sp_rif common;
 117         union {
 118                 u16 system_port;
 119                 u16 lag_id;
 120         };
 121         u16 vid;
 122         bool lag;
 123 };
 124
 125 struct mlxsw_sp_rif_ops {
 126         enum mlxsw_sp_rif_type type;
 127         size_t rif_size;
 128
 129         void (*setup)(struct mlxsw_sp_rif *rif,
 130                       const struct mlxsw_sp_rif_params *params);
 131         int (*configure)(struct mlxsw_sp_rif *rif);
 132         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 133         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 134 };
 135
 136 static unsigned int *
 137 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 138                            enum mlxsw_sp_rif_counter_dir dir)
 139 {
 140         switch (dir) {
 141         case MLXSW_SP_RIF_COUNTER_EGRESS:
 142                 return &rif->counter_egress;
 143         case MLXSW_SP_RIF_COUNTER_INGRESS:
 144                 return &rif->counter_ingress;
 145         }
 146         return NULL;
 147 }
 148
 149 static bool
 150 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 151                                enum mlxsw_sp_rif_counter_dir dir)
 152 {
 153         switch (dir) {
 154         case MLXSW_SP_RIF_COUNTER_EGRESS:
 155                 return rif->counter_egress_valid;
 156         case MLXSW_SP_RIF_COUNTER_INGRESS:
 157                 return rif->counter_ingress_valid;
 158         }
 159         return false;
 160 }
 161
 162 static void
 163 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 164                                enum mlxsw_sp_rif_counter_dir dir,
 165                                bool valid)
 166 {
 167         switch (dir) {
 168         case MLXSW_SP_RIF_COUNTER_EGRESS:
 169                 rif->counter_egress_valid = valid;
 170                 break;
 171         case MLXSW_SP_RIF_COUNTER_INGRESS:
 172                 rif->counter_ingress_valid = valid;
 173                 break;
 174         }
 175 }
 176
 177 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 178                                      unsigned int counter_index, bool enable,
 179                                      enum mlxsw_sp_rif_counter_dir dir)
 180 {
 181         char ritr_pl[MLXSW_REG_RITR_LEN];
 182         bool is_egress = false;
 183         int err;
 184
 185         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 186                 is_egress = true;
 187         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 188         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 189         if (err)
 190                 return err;
 191
 192         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 193                                     is_egress);
 194         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 195 }
 196
 197 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 198                                    struct mlxsw_sp_rif *rif,
 199                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 200 {
 201         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 202         unsigned int *p_counter_index;
 203         bool valid;
 204         int err;
 205
 206         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 207         if (!valid)
 208                 return -EINVAL;
 209
 210         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 211         if (!p_counter_index)
 212                 return -EINVAL;
 213         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 214                              MLXSW_REG_RICNT_OPCODE_NOP);
 215         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 216         if (err)
 217                 return err;
 218         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 219         return 0;
 220 }
 221
 222 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 223                                       unsigned int counter_index)
 224 {
 225         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 226
 227         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 228                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 229         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 230 }
 231
 232 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 233                                struct mlxsw_sp_rif *rif,
 234                                enum mlxsw_sp_rif_counter_dir dir)
 235 {
 236         unsigned int *p_counter_index;
 237         int err;
 238
 239         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 240         if (!p_counter_index)
 241                 return -EINVAL;
 242         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 243                                      p_counter_index);
 244         if (err)
 245                 return err;
 246
 247         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 248         if (err)
 249                 goto err_counter_clear;
 250
 251         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 252                                         *p_counter_index, true, dir);
 253         if (err)
 254                 goto err_counter_edit;
 255         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 256         return 0;
 257
 258 err_counter_edit:
 259 err_counter_clear:
 260         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 261                               *p_counter_index);
 262         return err;
 263 }
 264
 265 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 266                                struct mlxsw_sp_rif *rif,
 267                                enum mlxsw_sp_rif_counter_dir dir)
 268 {
 269         unsigned int *p_counter_index;
 270
 271         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 272                 return;
 273
 274         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 275         if (WARN_ON(!p_counter_index))
 276                 return;
 277         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 278                                   *p_counter_index, false, dir);
 279         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 280                               *p_counter_index);
 281         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 282 }
 283
 284 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 285 {
 286         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 287         struct devlink *devlink;
 288
 289         devlink = priv_to_devlink(mlxsw_sp->core);
 290         if (!devlink_dpipe_table_counter_enabled(devlink,
 291                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 292                 return;
 293         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 297 {
 298         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 299
 300         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 301 }
 302
 303 static struct mlxsw_sp_rif *
 304 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 305                          const struct net_device *dev);
 306
 307 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
 308
 309 struct mlxsw_sp_prefix_usage {
 310         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 311 };
 312
 313 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 314         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 315
 316 static bool
 317 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
 318                              struct mlxsw_sp_prefix_usage *prefix_usage2)
 319 {
 320         unsigned char prefix;
 321
 322         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
 323                 if (!test_bit(prefix, prefix_usage2->b))
 324                         return false;
 325         }
 326         return true;
 327 }
 328
 329 static bool
 330 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 331                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 332 {
 333         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 334 }
 335
 336 static bool
 337 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 338 {
 339         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 340
 341         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 342 }
 343
 344 static void
 345 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 346                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 347 {
 348         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 349 }
 350
 351 static void
 352 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 353                           unsigned char prefix_len)
 354 {
 355         set_bit(prefix_len, prefix_usage->b);
 356 }
 357
 358 static void
 359 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 360                             unsigned char prefix_len)
 361 {
 362         clear_bit(prefix_len, prefix_usage->b);
 363 }
 364
 365 struct mlxsw_sp_fib_key {
 366         unsigned char addr[sizeof(struct in6_addr)];
 367         unsigned char prefix_len;
 368 };
 369
 370 enum mlxsw_sp_fib_entry_type {
 371         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 372         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 373         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 374 };
 375
 376 struct mlxsw_sp_nexthop_group;
 377 struct mlxsw_sp_fib;
 378
 379 struct mlxsw_sp_fib_node {
 380         struct list_head entry_list;
 381         struct list_head list;
 382         struct rhash_head ht_node;
 383         struct mlxsw_sp_fib *fib;
 384         struct mlxsw_sp_fib_key key;
 385 };
 386
 387 struct mlxsw_sp_fib_entry_params {
 388         u32 tb_id;
 389         u32 prio;
 390         u8 tos;
 391         u8 type;
 392 };
 393
 394 struct mlxsw_sp_fib_entry {
 395         struct list_head list;
 396         struct mlxsw_sp_fib_node *fib_node;
 397         enum mlxsw_sp_fib_entry_type type;
 398         struct list_head nexthop_group_node;
 399         struct mlxsw_sp_nexthop_group *nh_group;
 400         struct mlxsw_sp_fib_entry_params params;
 401         bool offloaded;
 402 };
 403
 404 enum mlxsw_sp_l3proto {
 405         MLXSW_SP_L3_PROTO_IPV4,
 406         MLXSW_SP_L3_PROTO_IPV6,
 407 };
 408
 409 struct mlxsw_sp_lpm_tree {
 410         u8 id; /* tree ID */
 411         unsigned int ref_count;
 412         enum mlxsw_sp_l3proto proto;
 413         struct mlxsw_sp_prefix_usage prefix_usage;
 414 };
 415
 416 struct mlxsw_sp_fib {
 417         struct rhashtable ht;
 418         struct list_head node_list;
 419         struct mlxsw_sp_vr *vr;
 420         struct mlxsw_sp_lpm_tree *lpm_tree;
 421         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 422         struct mlxsw_sp_prefix_usage prefix_usage;
 423         enum mlxsw_sp_l3proto proto;
 424 };
 425
 426 struct mlxsw_sp_vr {
 427         u16 id; /* virtual router ID */
 428         u32 tb_id; /* kernel fib table id */
 429         unsigned int rif_count;
 430         struct mlxsw_sp_fib *fib4;
 431 };
 432
 433 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 434
 435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 436                                                 enum mlxsw_sp_l3proto proto)
 437 {
 438         struct mlxsw_sp_fib *fib;
 439         int err;
 440
 441         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 442         if (!fib)
 443                 return ERR_PTR(-ENOMEM);
 444         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 445         if (err)
 446                 goto err_rhashtable_init;
 447         INIT_LIST_HEAD(&fib->node_list);
 448         fib->proto = proto;
 449         fib->vr = vr;
 450         return fib;
 451
 452 err_rhashtable_init:
 453         kfree(fib);
 454         return ERR_PTR(err);
 455 }
 456
 457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 458 {
 459         WARN_ON(!list_empty(&fib->node_list));
 460         WARN_ON(fib->lpm_tree);
 461         rhashtable_destroy(&fib->ht);
 462         kfree(fib);
 463 }
 464
 465 static struct mlxsw_sp_lpm_tree *
 466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 467 {
 468         static struct mlxsw_sp_lpm_tree *lpm_tree;
 469         int i;
 470
 471         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 472                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 473                 if (lpm_tree->ref_count == 0)
 474                         return lpm_tree;
 475         }
 476         return NULL;
 477 }
 478
 479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 480                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 481 {
 482         char ralta_pl[MLXSW_REG_RALTA_LEN];
 483
 484         mlxsw_reg_ralta_pack(ralta_pl, true,
 485                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 486                              lpm_tree->id);
 487         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 488 }
 489
 490 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 491                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 492 {
 493         char ralta_pl[MLXSW_REG_RALTA_LEN];
 494
 495         mlxsw_reg_ralta_pack(ralta_pl, false,
 496                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 497                              lpm_tree->id);
 498         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 499 }
 500
 501 static int
 502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 503                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 504                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 505 {
 506         char ralst_pl[MLXSW_REG_RALST_LEN];
 507         u8 root_bin = 0;
 508         u8 prefix;
 509         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 510
 511         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 512                 root_bin = prefix;
 513
 514         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 515         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 516                 if (prefix == 0)
 517                         continue;
 518                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 519                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 520                 last_prefix = prefix;
 521         }
 522         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 523 }
 524
 525 static struct mlxsw_sp_lpm_tree *
 526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 527                          struct mlxsw_sp_prefix_usage *prefix_usage,
 528                          enum mlxsw_sp_l3proto proto)
 529 {
 530         struct mlxsw_sp_lpm_tree *lpm_tree;
 531         int err;
 532
 533         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 534         if (!lpm_tree)
 535                 return ERR_PTR(-EBUSY);
 536         lpm_tree->proto = proto;
 537         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 538         if (err)
 539                 return ERR_PTR(err);
 540
 541         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 542                                                 lpm_tree);
 543         if (err)
 544                 goto err_left_struct_set;
 545         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 546                sizeof(lpm_tree->prefix_usage));
 547         return lpm_tree;
 548
 549 err_left_struct_set:
 550         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 551         return ERR_PTR(err);
 552 }
 553
 554 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 555                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 556 {
 557         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 558 }
 559
 560 static struct mlxsw_sp_lpm_tree *
 561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 562                       struct mlxsw_sp_prefix_usage *prefix_usage,
 563                       enum mlxsw_sp_l3proto proto)
 564 {
 565         struct mlxsw_sp_lpm_tree *lpm_tree;
 566         int i;
 567
 568         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 569                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 570                 if (lpm_tree->ref_count != 0 &&
 571                     lpm_tree->proto == proto &&
 572                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 573                                              prefix_usage))
 574                         goto inc_ref_count;
 575         }
 576         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 577                                             proto);
 578         if (IS_ERR(lpm_tree))
 579                 return lpm_tree;
 580
 581 inc_ref_count:
 582         lpm_tree->ref_count++;
 583         return lpm_tree;
 584 }
 585
 586 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 587                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 588 {
 589         if (--lpm_tree->ref_count == 0)
 590                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 591         return 0;
 592 }
 593
 594 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 595
 596 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 597 {
 598         struct mlxsw_sp_lpm_tree *lpm_tree;
 599         u64 max_trees;
 600         int i;
 601
 602         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 603                 return -EIO;
 604
 605         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 606         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 607         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 608                                              sizeof(struct mlxsw_sp_lpm_tree),
 609                                              GFP_KERNEL);
 610         if (!mlxsw_sp->router->lpm.trees)
 611                 return -ENOMEM;
 612
 613         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 614                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 615                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 616         }
 617
 618         return 0;
 619 }
 620
 621 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 622 {
 623         kfree(mlxsw_sp->router->lpm.trees);
 624 }
 625
 626 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 627 {
 628         return !!vr->fib4;
 629 }
 630
 631 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 632 {
 633         struct mlxsw_sp_vr *vr;
 634         int i;
 635
 636         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 637                 vr = &mlxsw_sp->router->vrs[i];
 638                 if (!mlxsw_sp_vr_is_used(vr))
 639                         return vr;
 640         }
 641         return NULL;
 642 }
 643
 644 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 645                                      const struct mlxsw_sp_fib *fib)
 646 {
 647         char raltb_pl[MLXSW_REG_RALTB_LEN];
 648
 649         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 650                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 651                              fib->lpm_tree->id);
 652         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 653 }
 654
 655 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 656                                        const struct mlxsw_sp_fib *fib)
 657 {
 658         char raltb_pl[MLXSW_REG_RALTB_LEN];
 659
 660         /* Bind to tree 0 which is default */
 661         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 662                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 663         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 664 }
 665
 666 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 667 {
 668         /* For our purpose, squash main and local table into one */
 669         if (tb_id == RT_TABLE_LOCAL)
 670                 tb_id = RT_TABLE_MAIN;
 671         return tb_id;
 672 }
 673
 674 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 675                                             u32 tb_id)
 676 {
 677         struct mlxsw_sp_vr *vr;
 678         int i;
 679
 680         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 681
 682         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 683                 vr = &mlxsw_sp->router->vrs[i];
 684                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 685                         return vr;
 686         }
 687         return NULL;
 688 }
 689
 690 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 691                                             enum mlxsw_sp_l3proto proto)
 692 {
 693         switch (proto) {
 694         case MLXSW_SP_L3_PROTO_IPV4:
 695                 return vr->fib4;
 696         case MLXSW_SP_L3_PROTO_IPV6:
 697                 BUG_ON(1);
 698         }
 699         return NULL;
 700 }
 701
 702 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 703                                               u32 tb_id)
 704 {
 705         struct mlxsw_sp_vr *vr;
 706
 707         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 708         if (!vr)
 709                 return ERR_PTR(-EBUSY);
 710         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 711         if (IS_ERR(vr->fib4))
 712                 return ERR_CAST(vr->fib4);
 713         vr->tb_id = tb_id;
 714         return vr;
 715 }
 716
 717 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 718 {
 719         mlxsw_sp_fib_destroy(vr->fib4);
 720         vr->fib4 = NULL;
 721 }
 722
 723 static int
 724 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 725                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 726 {
 727         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 728         struct mlxsw_sp_lpm_tree *new_tree;
 729         int err;
 730
 731         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 732                 return 0;
 733
 734         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 735                                          fib->proto);
 736         if (IS_ERR(new_tree)) {
 737                 /* We failed to get a tree according to the required
 738                  * prefix usage. However, the current tree might be still good
 739                  * for us if our requirement is subset of the prefixes used
 740                  * in the tree.
 741                  */
 742                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 743                                                  &lpm_tree->prefix_usage))
 744                         return 0;
 745                 return PTR_ERR(new_tree);
 746         }
 747
 748         /* Prevent packet loss by overwriting existing binding */
 749         fib->lpm_tree = new_tree;
 750         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 751         if (err)
 752                 goto err_tree_bind;
 753         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 754
 755         return 0;
 756
 757 err_tree_bind:
 758         fib->lpm_tree = lpm_tree;
 759         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 760         return err;
 761 }
 762
 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 764 {
 765         struct mlxsw_sp_vr *vr;
 766
 767         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 768         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 769         if (!vr)
 770                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 771         return vr;
 772 }
 773
 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 775 {
 776         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 777                 mlxsw_sp_vr_destroy(vr);
 778 }
 779
 780 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 781 {
 782         struct mlxsw_sp_vr *vr;
 783         u64 max_vrs;
 784         int i;
 785
 786         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 787                 return -EIO;
 788
 789         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 790         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 791                                         GFP_KERNEL);
 792         if (!mlxsw_sp->router->vrs)
 793                 return -ENOMEM;
 794
 795         for (i = 0; i < max_vrs; i++) {
 796                 vr = &mlxsw_sp->router->vrs[i];
 797                 vr->id = i;
 798         }
 799
 800         return 0;
 801 }
 802
 803 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 804
 805 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 806 {
 807         /* At this stage we're guaranteed not to have new incoming
 808          * FIB notifications and the work queue is free from FIBs
 809          * sitting on top of mlxsw netdevs. However, we can still
 810          * have other FIBs queued. Flush the queue before flushing
 811          * the device's tables. No need for locks, as we're the only
 812          * writer.
 813          */
 814         mlxsw_core_flush_owq();
 815         mlxsw_sp_router_fib_flush(mlxsw_sp);
 816         kfree(mlxsw_sp->router->vrs);
 817 }
 818
 819 struct mlxsw_sp_neigh_key {
 820         struct neighbour *n;
 821 };
 822
 823 struct mlxsw_sp_neigh_entry {
 824         struct list_head rif_list_node;
 825         struct rhash_head ht_node;
 826         struct mlxsw_sp_neigh_key key;
 827         u16 rif;
 828         bool connected;
 829         unsigned char ha[ETH_ALEN];
 830         struct list_head nexthop_list; /* list of nexthops using
 831                                         * this neigh entry
 832                                         */
 833         struct list_head nexthop_neighs_list_node;
 834 };
 835
 836 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 837         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 838         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 839         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 840 };
 841
 842 static struct mlxsw_sp_neigh_entry *
 843 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 844                            u16 rif)
 845 {
 846         struct mlxsw_sp_neigh_entry *neigh_entry;
 847
 848         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 849         if (!neigh_entry)
 850                 return NULL;
 851
 852         neigh_entry->key.n = n;
 853         neigh_entry->rif = rif;
 854         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 855
 856         return neigh_entry;
 857 }
 858
 859 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 860 {
 861         kfree(neigh_entry);
 862 }
 863
 864 static int
 865 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 866                             struct mlxsw_sp_neigh_entry *neigh_entry)
 867 {
 868         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
 869                                       &neigh_entry->ht_node,
 870                                       mlxsw_sp_neigh_ht_params);
 871 }
 872
 873 static void
 874 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 875                             struct mlxsw_sp_neigh_entry *neigh_entry)
 876 {
 877         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
 878                                &neigh_entry->ht_node,
 879                                mlxsw_sp_neigh_ht_params);
 880 }
 881
 882 static struct mlxsw_sp_neigh_entry *
 883 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 884 {
 885         struct mlxsw_sp_neigh_entry *neigh_entry;
 886         struct mlxsw_sp_rif *rif;
 887         int err;
 888
 889         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 890         if (!rif)
 891                 return ERR_PTR(-EINVAL);
 892
 893         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
 894         if (!neigh_entry)
 895                 return ERR_PTR(-ENOMEM);
 896
 897         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 898         if (err)
 899                 goto err_neigh_entry_insert;
 900
 901         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 902
 903         return neigh_entry;
 904
 905 err_neigh_entry_insert:
 906         mlxsw_sp_neigh_entry_free(neigh_entry);
 907         return ERR_PTR(err);
 908 }
 909
 910 static void
 911 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 912                              struct mlxsw_sp_neigh_entry *neigh_entry)
 913 {
 914         list_del(&neigh_entry->rif_list_node);
 915         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 916         mlxsw_sp_neigh_entry_free(neigh_entry);
 917 }
 918
 919 static struct mlxsw_sp_neigh_entry *
 920 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 921 {
 922         struct mlxsw_sp_neigh_key key;
 923
 924         key.n = n;
 925         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
 926                                       &key, mlxsw_sp_neigh_ht_params);
 927 }
 928
 929 static void
 930 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 931 {
 932         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 933
 934         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
 935 }
 936
 937 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 938                                                    char *rauhtd_pl,
 939                                                    int ent_index)
 940 {
 941         struct net_device *dev;
 942         struct neighbour *n;
 943         __be32 dipn;
 944         u32 dip;
 945         u16 rif;
 946
 947         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 948
 949         if (!mlxsw_sp->router->rifs[rif]) {
 950                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 951                 return;
 952         }
 953
 954         dipn = htonl(dip);
 955         dev = mlxsw_sp->router->rifs[rif]->dev;
 956         n = neigh_lookup(&arp_tbl, &dipn, dev);
 957         if (!n) {
 958                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 959                            &dip);
 960                 return;
 961         }
 962
 963         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 964         neigh_event_send(n, NULL);
 965         neigh_release(n);
 966 }
 967
 968 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 969                                                    char *rauhtd_pl,
 970                                                    int rec_index)
 971 {
 972         u8 num_entries;
 973         int i;
 974
 975         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 976                                                                 rec_index);
 977         /* Hardware starts counting at 0, so add 1. */
 978         num_entries++;
 979
 980         /* Each record consists of several neighbour entries. */
 981         for (i = 0; i < num_entries; i++) {
 982                 int ent_index;
 983
 984                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 985                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 986                                                        ent_index);
 987         }
 988
 989 }
 990
 991 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 992                                               char *rauhtd_pl, int rec_index)
 993 {
 994         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 995         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 996                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 997                                                        rec_index);
 998                 break;
 999         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1000                 WARN_ON_ONCE(1);
1001                 break;
1002         }
1003 }
1004
1005 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1006 {
1007         u8 num_rec, last_rec_index, num_entries;
1008
1009         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1010         last_rec_index = num_rec - 1;
1011
1012         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1013                 return false;
1014         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1015             MLXSW_REG_RAUHTD_TYPE_IPV6)
1016                 return true;
1017
1018         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1019                                                                 last_rec_index);
1020         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1021                 return true;
1022         return false;
1023 }
1024
1025 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1026 {
1027         char *rauhtd_pl;
1028         u8 num_rec;
1029         int i, err;
1030
1031         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1032         if (!rauhtd_pl)
1033                 return -ENOMEM;
1034
1035         /* Make sure the neighbour's netdev isn't removed in the
1036          * process.
1037          */
1038         rtnl_lock();
1039         do {
1040                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
1041                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1042                                       rauhtd_pl);
1043                 if (err) {
1044                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1045                         break;
1046                 }
1047                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1048                 for (i = 0; i < num_rec; i++)
1049                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1050                                                           i);
1051         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1052         rtnl_unlock();
1053
1054         kfree(rauhtd_pl);
1055         return err;
1056 }
1057
1058 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1059 {
1060         struct mlxsw_sp_neigh_entry *neigh_entry;
1061
1062         /* Take RTNL mutex here to prevent lists from changes */
1063         rtnl_lock();
1064         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1065                             nexthop_neighs_list_node)
1066                 /* If this neigh have nexthops, make the kernel think this neigh
1067                  * is active regardless of the traffic.
1068                  */
1069                 neigh_event_send(neigh_entry->key.n, NULL);
1070         rtnl_unlock();
1071 }
1072
1073 static void
1074 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1075 {
1076         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1077
1078         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1079                                msecs_to_jiffies(interval));
1080 }
1081
1082 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1083 {
1084         struct mlxsw_sp_router *router;
1085         int err;
1086
1087         router = container_of(work, struct mlxsw_sp_router,
1088                               neighs_update.dw.work);
1089         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1090         if (err)
1091                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1092
1093         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1094
1095         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1096 }
1097
1098 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1099 {
1100         struct mlxsw_sp_neigh_entry *neigh_entry;
1101         struct mlxsw_sp_router *router;
1102
1103         router = container_of(work, struct mlxsw_sp_router,
1104                               nexthop_probe_dw.work);
1105         /* Iterate over nexthop neighbours, find those who are unresolved and
1106          * send arp on them. This solves the chicken-egg problem when
1107          * the nexthop wouldn't get offloaded until the neighbor is resolved
1108          * but it wouldn't get resolved ever in case traffic is flowing in HW
1109          * using different nexthop.
1110          *
1111          * Take RTNL mutex here to prevent lists from changes.
1112          */
1113         rtnl_lock();
1114         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1115                             nexthop_neighs_list_node)
1116                 if (!neigh_entry->connected)
1117                         neigh_event_send(neigh_entry->key.n, NULL);
1118         rtnl_unlock();
1119
1120         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1121                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1122 }
1123
1124 static void
1125 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1126                               struct mlxsw_sp_neigh_entry *neigh_entry,
1127                               bool removing);
1128
1129 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1130 {
1131         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1132                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1133 }
1134
1135 static void
1136 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1137                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1138                                 enum mlxsw_reg_rauht_op op)
1139 {
1140         struct neighbour *n = neigh_entry->key.n;
1141         u32 dip = ntohl(*((__be32 *) n->primary_key));
1142         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1143
1144         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1145                               dip);
1146         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1147 }
1148
1149 static void
1150 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1151                             struct mlxsw_sp_neigh_entry *neigh_entry,
1152                             bool adding)
1153 {
1154         if (!adding && !neigh_entry->connected)
1155                 return;
1156         neigh_entry->connected = adding;
1157         if (neigh_entry->key.n->tbl == &arp_tbl)
1158                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1159                                                 mlxsw_sp_rauht_op(adding));
1160         else
1161                 WARN_ON_ONCE(1);
1162 }
1163
1164 struct mlxsw_sp_neigh_event_work {
1165         struct work_struct work;
1166         struct mlxsw_sp *mlxsw_sp;
1167         struct neighbour *n;
1168 };
1169
1170 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1171 {
1172         struct mlxsw_sp_neigh_event_work *neigh_work =
1173                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1174         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1175         struct mlxsw_sp_neigh_entry *neigh_entry;
1176         struct neighbour *n = neigh_work->n;
1177         unsigned char ha[ETH_ALEN];
1178         bool entry_connected;
1179         u8 nud_state, dead;
1180
1181         /* If these parameters are changed after we release the lock,
1182          * then we are guaranteed to receive another event letting us
1183          * know about it.
1184          */
1185         read_lock_bh(&n->lock);
1186         memcpy(ha, n->ha, ETH_ALEN);
1187         nud_state = n->nud_state;
1188         dead = n->dead;
1189         read_unlock_bh(&n->lock);
1190
1191         rtnl_lock();
1192         entry_connected = nud_state & NUD_VALID && !dead;
1193         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1194         if (!entry_connected && !neigh_entry)
1195                 goto out;
1196         if (!neigh_entry) {
1197                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1198                 if (IS_ERR(neigh_entry))
1199                         goto out;
1200         }
1201
1202         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1203         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1204         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1205
1206         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1207                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1208
1209 out:
1210         rtnl_unlock();
1211         neigh_release(n);
1212         kfree(neigh_work);
1213 }
1214
1215 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1216                                    unsigned long event, void *ptr)
1217 {
1218         struct mlxsw_sp_neigh_event_work *neigh_work;
1219         struct mlxsw_sp_port *mlxsw_sp_port;
1220         struct mlxsw_sp *mlxsw_sp;
1221         unsigned long interval;
1222         struct neigh_parms *p;
1223         struct neighbour *n;
1224
1225         switch (event) {
1226         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1227                 p = ptr;
1228
1229                 /* We don't care about changes in the default table. */
1230                 if (!p->dev || p->tbl != &arp_tbl)
1231                         return NOTIFY_DONE;
1232
1233                 /* We are in atomic context and can't take RTNL mutex,
1234                  * so use RCU variant to walk the device chain.
1235                  */
1236                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1237                 if (!mlxsw_sp_port)
1238                         return NOTIFY_DONE;
1239
1240                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1241                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1242                 mlxsw_sp->router->neighs_update.interval = interval;
1243
1244                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1245                 break;
1246         case NETEVENT_NEIGH_UPDATE:
1247                 n = ptr;
1248
1249                 if (n->tbl != &arp_tbl)
1250                         return NOTIFY_DONE;
1251
1252                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1253                 if (!mlxsw_sp_port)
1254                         return NOTIFY_DONE;
1255
1256                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1257                 if (!neigh_work) {
1258                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1259                         return NOTIFY_BAD;
1260                 }
1261
1262                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1263                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1264                 neigh_work->n = n;
1265
1266                 /* Take a reference to ensure the neighbour won't be
1267                  * destructed until we drop the reference in delayed
1268                  * work.
1269                  */
1270                 neigh_clone(n);
1271                 mlxsw_core_schedule_work(&neigh_work->work);
1272                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1273                 break;
1274         }
1275
1276         return NOTIFY_DONE;
1277 }
1278
1279 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1280 {
1281         int err;
1282
1283         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1284                               &mlxsw_sp_neigh_ht_params);
1285         if (err)
1286                 return err;
1287
1288         /* Initialize the polling interval according to the default
1289          * table.
1290          */
1291         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1292
1293         /* Create the delayed works for the activity_update */
1294         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1295                           mlxsw_sp_router_neighs_update_work);
1296         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1297                           mlxsw_sp_router_probe_unresolved_nexthops);
1298         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1299         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1300         return 0;
1301 }
1302
1303 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1304 {
1305         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1306         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1307         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1308 }
1309
1310 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1311                                     const struct mlxsw_sp_rif *rif)
1312 {
1313         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1314
1315         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1316                              rif->rif_index, rif->addr);
1317         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1318 }
1319
1320 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1321                                          struct mlxsw_sp_rif *rif)
1322 {
1323         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1324
1325         mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1326         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1327                                  rif_list_node)
1328                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1329 }
1330
1331 struct mlxsw_sp_nexthop_key {
1332         struct fib_nh *fib_nh;
1333 };
1334
1335 struct mlxsw_sp_nexthop {
1336         struct list_head neigh_list_node; /* member of neigh entry list */
1337         struct list_head rif_list_node;
1338         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1339                                                 * this belongs to
1340                                                 */
1341         struct rhash_head ht_node;
1342         struct mlxsw_sp_nexthop_key key;
1343         struct mlxsw_sp_rif *rif;
1344         u8 should_offload:1, /* set indicates this neigh is connected and
1345                               * should be put to KVD linear area of this group.
1346                               */
1347            offloaded:1, /* set in case the neigh is actually put into
1348                          * KVD linear area of this group.
1349                          */
1350            update:1; /* set indicates that MAC of this neigh should be
1351                       * updated in HW
1352                       */
1353         struct mlxsw_sp_neigh_entry *neigh_entry;
1354 };
1355
1356 struct mlxsw_sp_nexthop_group_key {
1357         struct fib_info *fi;
1358 };
1359
1360 struct mlxsw_sp_nexthop_group {
1361         struct rhash_head ht_node;
1362         struct list_head fib_list; /* list of fib entries that use this group */
1363         struct mlxsw_sp_nexthop_group_key key;
1364         u8 adj_index_valid:1,
1365            gateway:1; /* routes using the group use a gateway */
1366         u32 adj_index;
1367         u16 ecmp_size;
1368         u16 count;
1369         struct mlxsw_sp_nexthop nexthops[0];
1370 #define nh_rif  nexthops[0].rif
1371 };
1372
1373 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1374         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1375         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1376         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1377 };
1378
1379 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1380                                          struct mlxsw_sp_nexthop_group *nh_grp)
1381 {
1382         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1383                                       &nh_grp->ht_node,
1384                                       mlxsw_sp_nexthop_group_ht_params);
1385 }
1386
1387 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1388                                           struct mlxsw_sp_nexthop_group *nh_grp)
1389 {
1390         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1391                                &nh_grp->ht_node,
1392                                mlxsw_sp_nexthop_group_ht_params);
1393 }
1394
1395 static struct mlxsw_sp_nexthop_group *
1396 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1397                               struct mlxsw_sp_nexthop_group_key key)
1398 {
1399         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
1400                                       mlxsw_sp_nexthop_group_ht_params);
1401 }
1402
1403 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1404         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1405         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1406         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1407 };
1408
1409 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1410                                    struct mlxsw_sp_nexthop *nh)
1411 {
1412         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1413                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1414 }
1415
1416 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1417                                     struct mlxsw_sp_nexthop *nh)
1418 {
1419         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1420                                mlxsw_sp_nexthop_ht_params);
1421 }
1422
1423 static struct mlxsw_sp_nexthop *
1424 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1425                         struct mlxsw_sp_nexthop_key key)
1426 {
1427         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1428                                       mlxsw_sp_nexthop_ht_params);
1429 }
1430
1431 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1432                                              const struct mlxsw_sp_fib *fib,
1433                                              u32 adj_index, u16 ecmp_size,
1434                                              u32 new_adj_index,
1435                                              u16 new_ecmp_size)
1436 {
1437         char raleu_pl[MLXSW_REG_RALEU_LEN];
1438
1439         mlxsw_reg_raleu_pack(raleu_pl,
1440                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1441                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1442                              new_ecmp_size);
1443         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1444 }
1445
1446 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1447                                           struct mlxsw_sp_nexthop_group *nh_grp,
1448                                           u32 old_adj_index, u16 old_ecmp_size)
1449 {
1450         struct mlxsw_sp_fib_entry *fib_entry;
1451         struct mlxsw_sp_fib *fib = NULL;
1452         int err;
1453
1454         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1455                 if (fib == fib_entry->fib_node->fib)
1456                         continue;
1457                 fib = fib_entry->fib_node->fib;
1458                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1459                                                         old_adj_index,
1460                                                         old_ecmp_size,
1461                                                         nh_grp->adj_index,
1462                                                         nh_grp->ecmp_size);
1463                 if (err)
1464                         return err;
1465         }
1466         return 0;
1467 }
1468
1469 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1470                                        struct mlxsw_sp_nexthop *nh)
1471 {
1472         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473         char ratr_pl[MLXSW_REG_RATR_LEN];
1474
1475         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1476                             true, adj_index, neigh_entry->rif);
1477         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1478         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1479 }
1480
1481 static int
1482 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1483                                   struct mlxsw_sp_nexthop_group *nh_grp,
1484                                   bool reallocate)
1485 {
1486         u32 adj_index = nh_grp->adj_index; /* base */
1487         struct mlxsw_sp_nexthop *nh;
1488         int i;
1489         int err;
1490
1491         for (i = 0; i < nh_grp->count; i++) {
1492                 nh = &nh_grp->nexthops[i];
1493
1494                 if (!nh->should_offload) {
1495                         nh->offloaded = 0;
1496                         continue;
1497                 }
1498
1499                 if (nh->update || reallocate) {
1500                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1501                                                           adj_index, nh);
1502                         if (err)
1503                                 return err;
1504                         nh->update = 0;
1505                         nh->offloaded = 1;
1506                 }
1507                 adj_index++;
1508         }
1509         return 0;
1510 }
1511
1512 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1513                                      struct mlxsw_sp_fib_entry *fib_entry);
1514
1515 static bool
1516 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
1517                                  const struct mlxsw_sp_fib_entry *fib_entry);
1518
1519 static int
1520 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1521                                     struct mlxsw_sp_nexthop_group *nh_grp)
1522 {
1523         struct mlxsw_sp_fib_entry *fib_entry;
1524         int err;
1525
1526         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1527                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
1528                                                       fib_entry))
1529                         continue;
1530                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1531                 if (err)
1532                         return err;
1533         }
1534         return 0;
1535 }
1536
1537 static void
1538 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1539                                struct mlxsw_sp_nexthop_group *nh_grp)
1540 {
1541         struct mlxsw_sp_nexthop *nh;
1542         bool offload_change = false;
1543         u32 adj_index;
1544         u16 ecmp_size = 0;
1545         bool old_adj_index_valid;
1546         u32 old_adj_index;
1547         u16 old_ecmp_size;
1548         int i;
1549         int err;
1550
1551         if (!nh_grp->gateway) {
1552                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1553                 return;
1554         }
1555
1556         for (i = 0; i < nh_grp->count; i++) {
1557                 nh = &nh_grp->nexthops[i];
1558
1559                 if (nh->should_offload ^ nh->offloaded) {
1560                         offload_change = true;
1561                         if (nh->should_offload)
1562                                 nh->update = 1;
1563                 }
1564                 if (nh->should_offload)
1565                         ecmp_size++;
1566         }
1567         if (!offload_change) {
1568                 /* Nothing was added or removed, so no need to reallocate. Just
1569                  * update MAC on existing adjacency indexes.
1570                  */
1571                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1572                                                         false);
1573                 if (err) {
1574                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1575                         goto set_trap;
1576                 }
1577                 return;
1578         }
1579         if (!ecmp_size)
1580                 /* No neigh of this group is connected so we just set
1581                  * the trap and let everthing flow through kernel.
1582                  */
1583                 goto set_trap;
1584
1585         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1586         if (err) {
1587                 /* We ran out of KVD linear space, just set the
1588                  * trap and let everything flow through kernel.
1589                  */
1590                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1591                 goto set_trap;
1592         }
1593         old_adj_index_valid = nh_grp->adj_index_valid;
1594         old_adj_index = nh_grp->adj_index;
1595         old_ecmp_size = nh_grp->ecmp_size;
1596         nh_grp->adj_index_valid = 1;
1597         nh_grp->adj_index = adj_index;
1598         nh_grp->ecmp_size = ecmp_size;
1599         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1600         if (err) {
1601                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1602                 goto set_trap;
1603         }
1604
1605         if (!old_adj_index_valid) {
1606                 /* The trap was set for fib entries, so we have to call
1607                  * fib entry update to unset it and use adjacency index.
1608                  */
1609                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1610                 if (err) {
1611                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1612                         goto set_trap;
1613                 }
1614                 return;
1615         }
1616
1617         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1618                                              old_adj_index, old_ecmp_size);
1619         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1620         if (err) {
1621                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1622                 goto set_trap;
1623         }
1624         return;
1625
1626 set_trap:
1627         old_adj_index_valid = nh_grp->adj_index_valid;
1628         nh_grp->adj_index_valid = 0;
1629         for (i = 0; i < nh_grp->count; i++) {
1630                 nh = &nh_grp->nexthops[i];
1631                 nh->offloaded = 0;
1632         }
1633         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1634         if (err)
1635                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1636         if (old_adj_index_valid)
1637                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1638 }
1639
1640 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1641                                             bool removing)
1642 {
1643         if (!removing && !nh->should_offload)
1644                 nh->should_offload = 1;
1645         else if (removing && nh->offloaded)
1646                 nh->should_offload = 0;
1647         nh->update = 1;
1648 }
1649
1650 static void
1651 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1652                               struct mlxsw_sp_neigh_entry *neigh_entry,
1653                               bool removing)
1654 {
1655         struct mlxsw_sp_nexthop *nh;
1656
1657         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1658                             neigh_list_node) {
1659                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1660                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1661         }
1662 }
1663
1664 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1665                                       struct mlxsw_sp_rif *rif)
1666 {
1667         if (nh->rif)
1668                 return;
1669
1670         nh->rif = rif;
1671         list_add(&nh->rif_list_node, &rif->nexthop_list);
1672 }
1673
1674 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1675 {
1676         if (!nh->rif)
1677                 return;
1678
1679         list_del(&nh->rif_list_node);
1680         nh->rif = NULL;
1681 }
1682
1683 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1684                                        struct mlxsw_sp_nexthop *nh)
1685 {
1686         struct mlxsw_sp_neigh_entry *neigh_entry;
1687         struct fib_nh *fib_nh = nh->key.fib_nh;
1688         struct neighbour *n;
1689         u8 nud_state, dead;
1690         int err;
1691
1692         if (!nh->nh_grp->gateway || nh->neigh_entry)
1693                 return 0;
1694
1695         /* Take a reference of neigh here ensuring that neigh would
1696          * not be detructed before the nexthop entry is finished.
1697          * The reference is taken either in neigh_lookup() or
1698          * in neigh_create() in case n is not found.
1699          */
1700         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1701         if (!n) {
1702                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1703                 if (IS_ERR(n))
1704                         return PTR_ERR(n);
1705                 neigh_event_send(n, NULL);
1706         }
1707         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1708         if (!neigh_entry) {
1709                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1710                 if (IS_ERR(neigh_entry)) {
1711                         err = -EINVAL;
1712                         goto err_neigh_entry_create;
1713                 }
1714         }
1715
1716         /* If that is the first nexthop connected to that neigh, add to
1717          * nexthop_neighs_list
1718          */
1719         if (list_empty(&neigh_entry->nexthop_list))
1720                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1721                               &mlxsw_sp->router->nexthop_neighs_list);
1722
1723         nh->neigh_entry = neigh_entry;
1724         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1725         read_lock_bh(&n->lock);
1726         nud_state = n->nud_state;
1727         dead = n->dead;
1728         read_unlock_bh(&n->lock);
1729         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1730
1731         return 0;
1732
1733 err_neigh_entry_create:
1734         neigh_release(n);
1735         return err;
1736 }
1737
1738 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1739                                         struct mlxsw_sp_nexthop *nh)
1740 {
1741         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1742         struct neighbour *n;
1743
1744         if (!neigh_entry)
1745                 return;
1746         n = neigh_entry->key.n;
1747
1748         __mlxsw_sp_nexthop_neigh_update(nh, true);
1749         list_del(&nh->neigh_list_node);
1750         nh->neigh_entry = NULL;
1751
1752         /* If that is the last nexthop connected to that neigh, remove from
1753          * nexthop_neighs_list
1754          */
1755         if (list_empty(&neigh_entry->nexthop_list))
1756                 list_del(&neigh_entry->nexthop_neighs_list_node);
1757
1758         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1759                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1760
1761         neigh_release(n);
1762 }
1763
1764 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1765                                  struct mlxsw_sp_nexthop_group *nh_grp,
1766                                  struct mlxsw_sp_nexthop *nh,
1767                                  struct fib_nh *fib_nh)
1768 {
1769         struct net_device *dev = fib_nh->nh_dev;
1770         struct in_device *in_dev;
1771         struct mlxsw_sp_rif *rif;
1772         int err;
1773
1774         nh->nh_grp = nh_grp;
1775         nh->key.fib_nh = fib_nh;
1776         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1777         if (err)
1778                 return err;
1779
1780         if (!dev)
1781                 return 0;
1782
1783         in_dev = __in_dev_get_rtnl(dev);
1784         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1785             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1786                 return 0;
1787
1788         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1789         if (!rif)
1790                 return 0;
1791         mlxsw_sp_nexthop_rif_init(nh, rif);
1792
1793         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1794         if (err)
1795                 goto err_nexthop_neigh_init;
1796
1797         return 0;
1798
1799 err_nexthop_neigh_init:
1800         mlxsw_sp_nexthop_rif_fini(nh);
1801         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1802         return err;
1803 }
1804
1805 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1806                                   struct mlxsw_sp_nexthop *nh)
1807 {
1808         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1809         mlxsw_sp_nexthop_rif_fini(nh);
1810         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1811 }
1812
1813 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1814                                    unsigned long event, struct fib_nh *fib_nh)
1815 {
1816         struct mlxsw_sp_nexthop_key key;
1817         struct mlxsw_sp_nexthop *nh;
1818         struct mlxsw_sp_rif *rif;
1819
1820         if (mlxsw_sp->router->aborted)
1821                 return;
1822
1823         key.fib_nh = fib_nh;
1824         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1825         if (WARN_ON_ONCE(!nh))
1826                 return;
1827
1828         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1829         if (!rif)
1830                 return;
1831
1832         switch (event) {
1833         case FIB_EVENT_NH_ADD:
1834                 mlxsw_sp_nexthop_rif_init(nh, rif);
1835                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1836                 break;
1837         case FIB_EVENT_NH_DEL:
1838                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1839                 mlxsw_sp_nexthop_rif_fini(nh);
1840                 break;
1841         }
1842
1843         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1844 }
1845
1846 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1847                                            struct mlxsw_sp_rif *rif)
1848 {
1849         struct mlxsw_sp_nexthop *nh, *tmp;
1850
1851         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1852                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1853                 mlxsw_sp_nexthop_rif_fini(nh);
1854                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1855         }
1856 }
1857
1858 static struct mlxsw_sp_nexthop_group *
1859 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1860 {
1861         struct mlxsw_sp_nexthop_group *nh_grp;
1862         struct mlxsw_sp_nexthop *nh;
1863         struct fib_nh *fib_nh;
1864         size_t alloc_size;
1865         int i;
1866         int err;
1867
1868         alloc_size = sizeof(*nh_grp) +
1869                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1870         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1871         if (!nh_grp)
1872                 return ERR_PTR(-ENOMEM);
1873         INIT_LIST_HEAD(&nh_grp->fib_list);
1874         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1875         nh_grp->count = fi->fib_nhs;
1876         nh_grp->key.fi = fi;
1877         fib_info_hold(fi);
1878         for (i = 0; i < nh_grp->count; i++) {
1879                 nh = &nh_grp->nexthops[i];
1880                 fib_nh = &fi->fib_nh[i];
1881                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1882                 if (err)
1883                         goto err_nexthop_init;
1884         }
1885         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1886         if (err)
1887                 goto err_nexthop_group_insert;
1888         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1889         return nh_grp;
1890
1891 err_nexthop_group_insert:
1892 err_nexthop_init:
1893         for (i--; i >= 0; i--) {
1894                 nh = &nh_grp->nexthops[i];
1895                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1896         }
1897         fib_info_put(nh_grp->key.fi);
1898         kfree(nh_grp);
1899         return ERR_PTR(err);
1900 }
1901
1902 static void
1903 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1904                                struct mlxsw_sp_nexthop_group *nh_grp)
1905 {
1906         struct mlxsw_sp_nexthop *nh;
1907         int i;
1908
1909         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1910         for (i = 0; i < nh_grp->count; i++) {
1911                 nh = &nh_grp->nexthops[i];
1912                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1913         }
1914         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1915         WARN_ON_ONCE(nh_grp->adj_index_valid);
1916         fib_info_put(nh_grp->key.fi);
1917         kfree(nh_grp);
1918 }
1919
1920 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1921                                       struct mlxsw_sp_fib_entry *fib_entry,
1922                                       struct fib_info *fi)
1923 {
1924         struct mlxsw_sp_nexthop_group_key key;
1925         struct mlxsw_sp_nexthop_group *nh_grp;
1926
1927         key.fi = fi;
1928         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1929         if (!nh_grp) {
1930                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1931                 if (IS_ERR(nh_grp))
1932                         return PTR_ERR(nh_grp);
1933         }
1934         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1935         fib_entry->nh_group = nh_grp;
1936         return 0;
1937 }
1938
1939 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1940                                        struct mlxsw_sp_fib_entry *fib_entry)
1941 {
1942         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1943
1944         list_del(&fib_entry->nexthop_group_node);
1945         if (!list_empty(&nh_grp->fib_list))
1946                 return;
1947         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1948 }
1949
1950 static bool
1951 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1952 {
1953         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1954
1955         if (fib_entry->params.tos)
1956                 return false;
1957
1958         switch (fib_entry->type) {
1959         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1960                 return !!nh_group->adj_index_valid;
1961         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1962                 return !!nh_group->nh_rif;
1963         default:
1964                 return false;
1965         }
1966 }
1967
1968 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1969 {
1970         fib_entry->offloaded = true;
1971
1972         switch (fib_entry->fib_node->fib->proto) {
1973         case MLXSW_SP_L3_PROTO_IPV4:
1974                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1975                 break;
1976         case MLXSW_SP_L3_PROTO_IPV6:
1977                 WARN_ON_ONCE(1);
1978         }
1979 }
1980
1981 static void
1982 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1983 {
1984         switch (fib_entry->fib_node->fib->proto) {
1985         case MLXSW_SP_L3_PROTO_IPV4:
1986                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1987                 break;
1988         case MLXSW_SP_L3_PROTO_IPV6:
1989                 WARN_ON_ONCE(1);
1990         }
1991
1992         fib_entry->offloaded = false;
1993 }
1994
1995 static void
1996 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1997                                    enum mlxsw_reg_ralue_op op, int err)
1998 {
1999         switch (op) {
2000         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2001                 if (!fib_entry->offloaded)
2002                         return;
2003                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2004         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2005                 if (err)
2006                         return;
2007                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2008                     !fib_entry->offloaded)
2009                         mlxsw_sp_fib_entry_offload_set(fib_entry);
2010                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2011                          fib_entry->offloaded)
2012                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
2013                 return;
2014         default:
2015                 return;
2016         }
2017 }
2018
2019 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
2020                                          struct mlxsw_sp_fib_entry *fib_entry,
2021                                          enum mlxsw_reg_ralue_op op)
2022 {
2023         char ralue_pl[MLXSW_REG_RALUE_LEN];
2024         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2025         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2026         enum mlxsw_reg_ralue_trap_action trap_action;
2027         u16 trap_id = 0;
2028         u32 adjacency_index = 0;
2029         u16 ecmp_size = 0;
2030
2031         /* In case the nexthop group adjacency index is valid, use it
2032          * with provided ECMP size. Otherwise, setup trap and pass
2033          * traffic to kernel.
2034          */
2035         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2036                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2037                 adjacency_index = fib_entry->nh_group->adj_index;
2038                 ecmp_size = fib_entry->nh_group->ecmp_size;
2039         } else {
2040                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2041                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2042         }
2043
2044         mlxsw_reg_ralue_pack4(ralue_pl,
2045                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2046                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2047                               *p_dip);
2048         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2049                                         adjacency_index, ecmp_size);
2050         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2051 }
2052
2053 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
2054                                         struct mlxsw_sp_fib_entry *fib_entry,
2055                                         enum mlxsw_reg_ralue_op op)
2056 {
2057         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2058         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2059         enum mlxsw_reg_ralue_trap_action trap_action;
2060         char ralue_pl[MLXSW_REG_RALUE_LEN];
2061         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2062         u16 trap_id = 0;
2063         u16 rif_index = 0;
2064
2065         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2066                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2067                 rif_index = rif->rif_index;
2068         } else {
2069                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2070                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2071         }
2072
2073         mlxsw_reg_ralue_pack4(ralue_pl,
2074                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2075                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2076                               *p_dip);
2077         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2078                                        rif_index);
2079         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2080 }
2081
2082 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
2083                                        struct mlxsw_sp_fib_entry *fib_entry,
2084                                        enum mlxsw_reg_ralue_op op)
2085 {
2086         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2087         char ralue_pl[MLXSW_REG_RALUE_LEN];
2088         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2089
2090         mlxsw_reg_ralue_pack4(ralue_pl,
2091                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2092                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2093                               *p_dip);
2094         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2095         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2096 }
2097
2098 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
2099                                   struct mlxsw_sp_fib_entry *fib_entry,
2100                                   enum mlxsw_reg_ralue_op op)
2101 {
2102         switch (fib_entry->type) {
2103         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2104                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
2105         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2106                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
2107         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2108                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
2109         }
2110         return -EINVAL;
2111 }
2112
2113 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2114                                  struct mlxsw_sp_fib_entry *fib_entry,
2115                                  enum mlxsw_reg_ralue_op op)
2116 {
2117         int err = -EINVAL;
2118
2119         switch (fib_entry->fib_node->fib->proto) {
2120         case MLXSW_SP_L3_PROTO_IPV4:
2121                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
2122                 break;
2123         case MLXSW_SP_L3_PROTO_IPV6:
2124                 return err;
2125         }
2126         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2127         return err;
2128 }
2129
2130 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2131                                      struct mlxsw_sp_fib_entry *fib_entry)
2132 {
2133         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2134                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
2135 }
2136
2137 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2138                                   struct mlxsw_sp_fib_entry *fib_entry)
2139 {
2140         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2141                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
2142 }
2143
2144 static int
2145 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2146                              const struct fib_entry_notifier_info *fen_info,
2147                              struct mlxsw_sp_fib_entry *fib_entry)
2148 {
2149         struct fib_info *fi = fen_info->fi;
2150
2151         switch (fen_info->type) {
2152         case RTN_BROADCAST: /* fall through */
2153         case RTN_LOCAL:
2154                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2155                 return 0;
2156         case RTN_UNREACHABLE: /* fall through */
2157         case RTN_BLACKHOLE: /* fall through */
2158         case RTN_PROHIBIT:
2159                 /* Packets hitting these routes need to be trapped, but
2160                  * can do so with a lower priority than packets directed
2161                  * at the host, so use action type local instead of trap.
2162                  */
2163                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2164                 return 0;
2165         case RTN_UNICAST:
2166                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2167                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2168                 else
2169                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2170                 return 0;
2171         default:
2172                 return -EINVAL;
2173         }
2174 }
2175
2176 static struct mlxsw_sp_fib_entry *
2177 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2178                            struct mlxsw_sp_fib_node *fib_node,
2179                            const struct fib_entry_notifier_info *fen_info)
2180 {
2181         struct mlxsw_sp_fib_entry *fib_entry;
2182         int err;
2183
2184         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
2185         if (!fib_entry) {
2186                 err = -ENOMEM;
2187                 goto err_fib_entry_alloc;
2188         }
2189
2190         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2191         if (err)
2192                 goto err_fib4_entry_type_set;
2193
2194         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2195         if (err)
2196                 goto err_nexthop_group_get;
2197
2198         fib_entry->params.prio = fen_info->fi->fib_priority;
2199         fib_entry->params.tb_id = fen_info->tb_id;
2200         fib_entry->params.type = fen_info->type;
2201         fib_entry->params.tos = fen_info->tos;
2202
2203         fib_entry->fib_node = fib_node;
2204
2205         return fib_entry;
2206
2207 err_nexthop_group_get:
2208 err_fib4_entry_type_set:
2209         kfree(fib_entry);
2210 err_fib_entry_alloc:
2211         return ERR_PTR(err);
2212 }
2213
2214 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2215                                         struct mlxsw_sp_fib_entry *fib_entry)
2216 {
2217         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
2218         kfree(fib_entry);
2219 }
2220
2221 static struct mlxsw_sp_fib_node *
2222 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2223                        const struct fib_entry_notifier_info *fen_info);
2224
2225 static struct mlxsw_sp_fib_entry *
2226 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2227                            const struct fib_entry_notifier_info *fen_info)
2228 {
2229         struct mlxsw_sp_fib_entry *fib_entry;
2230         struct mlxsw_sp_fib_node *fib_node;
2231
2232         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2233         if (IS_ERR(fib_node))
2234                 return NULL;
2235
2236         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2237                 if (fib_entry->params.tb_id == fen_info->tb_id &&
2238                     fib_entry->params.tos == fen_info->tos &&
2239                     fib_entry->params.type == fen_info->type &&
2240                     fib_entry->nh_group->key.fi == fen_info->fi) {
2241                         return fib_entry;
2242                 }
2243         }
2244
2245         return NULL;
2246 }
2247
2248 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2249         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2250         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2251         .key_len = sizeof(struct mlxsw_sp_fib_key),
2252         .automatic_shrinking = true,
2253 };
2254
2255 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2256                                     struct mlxsw_sp_fib_node *fib_node)
2257 {
2258         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2259                                       mlxsw_sp_fib_ht_params);
2260 }
2261
2262 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2263                                      struct mlxsw_sp_fib_node *fib_node)
2264 {
2265         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2266                                mlxsw_sp_fib_ht_params);
2267 }
2268
2269 static struct mlxsw_sp_fib_node *
2270 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2271                          size_t addr_len, unsigned char prefix_len)
2272 {
2273         struct mlxsw_sp_fib_key key;
2274
2275         memset(&key, 0, sizeof(key));
2276         memcpy(key.addr, addr, addr_len);
2277         key.prefix_len = prefix_len;
2278         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2279 }
2280
2281 static struct mlxsw_sp_fib_node *
2282 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2283                          size_t addr_len, unsigned char prefix_len)
2284 {
2285         struct mlxsw_sp_fib_node *fib_node;
2286
2287         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2288         if (!fib_node)
2289                 return NULL;
2290
2291         INIT_LIST_HEAD(&fib_node->entry_list);
2292         list_add(&fib_node->list, &fib->node_list);
2293         memcpy(fib_node->key.addr, addr, addr_len);
2294         fib_node->key.prefix_len = prefix_len;
2295
2296         return fib_node;
2297 }
2298
2299 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2300 {
2301         list_del(&fib_node->list);
2302         WARN_ON(!list_empty(&fib_node->entry_list));
2303         kfree(fib_node);
2304 }
2305
2306 static bool
2307 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2308                                  const struct mlxsw_sp_fib_entry *fib_entry)
2309 {
2310         return list_first_entry(&fib_node->entry_list,
2311                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2312 }
2313
2314 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2315 {
2316         unsigned char prefix_len = fib_node->key.prefix_len;
2317         struct mlxsw_sp_fib *fib = fib_node->fib;
2318
2319         if (fib->prefix_ref_count[prefix_len]++ == 0)
2320                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2321 }
2322
2323 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2324 {
2325         unsigned char prefix_len = fib_node->key.prefix_len;
2326         struct mlxsw_sp_fib *fib = fib_node->fib;
2327
2328         if (--fib->prefix_ref_count[prefix_len] == 0)
2329                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2330 }
2331
2332 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2333                                   struct mlxsw_sp_fib_node *fib_node,
2334                                   struct mlxsw_sp_fib *fib)
2335 {
2336         struct mlxsw_sp_prefix_usage req_prefix_usage;
2337         struct mlxsw_sp_lpm_tree *lpm_tree;
2338         int err;
2339
2340         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2341         if (err)
2342                 return err;
2343         fib_node->fib = fib;
2344
2345         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2346         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2347
2348         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2349                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2350                                                  &req_prefix_usage);
2351                 if (err)
2352                         goto err_tree_check;
2353         } else {
2354                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2355                                                  fib->proto);
2356                 if (IS_ERR(lpm_tree))
2357                         return PTR_ERR(lpm_tree);
2358                 fib->lpm_tree = lpm_tree;
2359                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2360                 if (err)
2361                         goto err_tree_bind;
2362         }
2363
2364         mlxsw_sp_fib_node_prefix_inc(fib_node);
2365
2366         return 0;
2367
2368 err_tree_bind:
2369         fib->lpm_tree = NULL;
2370         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2371 err_tree_check:
2372         fib_node->fib = NULL;
2373         mlxsw_sp_fib_node_remove(fib, fib_node);
2374         return err;
2375 }
2376
2377 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2378                                    struct mlxsw_sp_fib_node *fib_node)
2379 {
2380         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2381         struct mlxsw_sp_fib *fib = fib_node->fib;
2382
2383         mlxsw_sp_fib_node_prefix_dec(fib_node);
2384
2385         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2386                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2387                 fib->lpm_tree = NULL;
2388                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2389         } else {
2390                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2391         }
2392
2393         fib_node->fib = NULL;
2394         mlxsw_sp_fib_node_remove(fib, fib_node);
2395 }
2396
2397 static struct mlxsw_sp_fib_node *
2398 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2399                        const struct fib_entry_notifier_info *fen_info)
2400 {
2401         struct mlxsw_sp_fib_node *fib_node;
2402         struct mlxsw_sp_fib *fib;
2403         struct mlxsw_sp_vr *vr;
2404         int err;
2405
2406         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2407         if (IS_ERR(vr))
2408                 return ERR_CAST(vr);
2409         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2410
2411         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2412                                             sizeof(fen_info->dst),
2413                                             fen_info->dst_len);
2414         if (fib_node)
2415                 return fib_node;
2416
2417         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2418                                             sizeof(fen_info->dst),
2419                                             fen_info->dst_len);
2420         if (!fib_node) {
2421                 err = -ENOMEM;
2422                 goto err_fib_node_create;
2423         }
2424
2425         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2426         if (err)
2427                 goto err_fib_node_init;
2428
2429         return fib_node;
2430
2431 err_fib_node_init:
2432         mlxsw_sp_fib_node_destroy(fib_node);
2433 err_fib_node_create:
2434         mlxsw_sp_vr_put(vr);
2435         return ERR_PTR(err);
2436 }
2437
2438 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2439                                    struct mlxsw_sp_fib_node *fib_node)
2440 {
2441         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2442
2443         if (!list_empty(&fib_node->entry_list))
2444                 return;
2445         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2446         mlxsw_sp_fib_node_destroy(fib_node);
2447         mlxsw_sp_vr_put(vr);
2448 }
2449
2450 static struct mlxsw_sp_fib_entry *
2451 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2452                               const struct mlxsw_sp_fib_entry_params *params)
2453 {
2454         struct mlxsw_sp_fib_entry *fib_entry;
2455
2456         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2457                 if (fib_entry->params.tb_id > params->tb_id)
2458                         continue;
2459                 if (fib_entry->params.tb_id != params->tb_id)
2460                         break;
2461                 if (fib_entry->params.tos > params->tos)
2462                         continue;
2463                 if (fib_entry->params.prio >= params->prio ||
2464                     fib_entry->params.tos < params->tos)
2465                         return fib_entry;
2466         }
2467
2468         return NULL;
2469 }
2470
2471 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2472                                           struct mlxsw_sp_fib_entry *new_entry)
2473 {
2474         struct mlxsw_sp_fib_node *fib_node;
2475
2476         if (WARN_ON(!fib_entry))
2477                 return -EINVAL;
2478
2479         fib_node = fib_entry->fib_node;
2480         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2481                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2482                     fib_entry->params.tos != new_entry->params.tos ||
2483                     fib_entry->params.prio != new_entry->params.prio)
2484                         break;
2485         }
2486
2487         list_add_tail(&new_entry->list, &fib_entry->list);
2488         return 0;
2489 }
2490
2491 static int
2492 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2493                                struct mlxsw_sp_fib_entry *new_entry,
2494                                bool replace, bool append)
2495 {
2496         struct mlxsw_sp_fib_entry *fib_entry;
2497
2498         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2499
2500         if (append)
2501                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2502         if (replace && WARN_ON(!fib_entry))
2503                 return -EINVAL;
2504
2505         /* Insert new entry before replaced one, so that we can later
2506          * remove the second.
2507          */
2508         if (fib_entry) {
2509                 list_add_tail(&new_entry->list, &fib_entry->list);
2510         } else {
2511                 struct mlxsw_sp_fib_entry *last;
2512
2513                 list_for_each_entry(last, &fib_node->entry_list, list) {
2514                         if (new_entry->params.tb_id > last->params.tb_id)
2515                                 break;
2516                         fib_entry = last;
2517                 }
2518
2519                 if (fib_entry)
2520                         list_add(&new_entry->list, &fib_entry->list);
2521                 else
2522                         list_add(&new_entry->list, &fib_node->entry_list);
2523         }
2524
2525         return 0;
2526 }
2527
2528 static void
2529 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2530 {
2531         list_del(&fib_entry->list);
2532 }
2533
2534 static int
2535 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2536                              const struct mlxsw_sp_fib_node *fib_node,
2537                              struct mlxsw_sp_fib_entry *fib_entry)
2538 {
2539         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2540                 return 0;
2541
2542         /* To prevent packet loss, overwrite the previously offloaded
2543          * entry.
2544          */
2545         if (!list_is_singular(&fib_node->entry_list)) {
2546                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2547                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2548
2549                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2550         }
2551
2552         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2553 }
2554
2555 static void
2556 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2557                              const struct mlxsw_sp_fib_node *fib_node,
2558                              struct mlxsw_sp_fib_entry *fib_entry)
2559 {
2560         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2561                 return;
2562
2563         /* Promote the next entry by overwriting the deleted entry */
2564         if (!list_is_singular(&fib_node->entry_list)) {
2565                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2566                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2567
2568                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2569                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2570                 return;
2571         }
2572
2573         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2574 }
2575
2576 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2577                                          struct mlxsw_sp_fib_entry *fib_entry,
2578                                          bool replace, bool append)
2579 {
2580         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2581         int err;
2582
2583         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2584                                              append);
2585         if (err)
2586                 return err;
2587
2588         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2589         if (err)
2590                 goto err_fib4_node_entry_add;
2591
2592         return 0;
2593
2594 err_fib4_node_entry_add:
2595         mlxsw_sp_fib4_node_list_remove(fib_entry);
2596         return err;
2597 }
2598
2599 static void
2600 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2601                                 struct mlxsw_sp_fib_entry *fib_entry)
2602 {
2603         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2604
2605         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2606         mlxsw_sp_fib4_node_list_remove(fib_entry);
2607 }
2608
2609 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2610                                         struct mlxsw_sp_fib_entry *fib_entry,
2611                                         bool replace)
2612 {
2613         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2614         struct mlxsw_sp_fib_entry *replaced;
2615
2616         if (!replace)
2617                 return;
2618
2619         /* We inserted the new entry before replaced one */
2620         replaced = list_next_entry(fib_entry, list);
2621
2622         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2623         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2624         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2625 }
2626
2627 static int
2628 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2629                          const struct fib_entry_notifier_info *fen_info,
2630                          bool replace, bool append)
2631 {
2632         struct mlxsw_sp_fib_entry *fib_entry;
2633         struct mlxsw_sp_fib_node *fib_node;
2634         int err;
2635
2636         if (mlxsw_sp->router->aborted)
2637                 return 0;
2638
2639         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2640         if (IS_ERR(fib_node)) {
2641                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2642                 return PTR_ERR(fib_node);
2643         }
2644
2645         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2646         if (IS_ERR(fib_entry)) {
2647                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2648                 err = PTR_ERR(fib_entry);
2649                 goto err_fib4_entry_create;
2650         }
2651
2652         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2653                                             append);
2654         if (err) {
2655                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2656                 goto err_fib4_node_entry_link;
2657         }
2658
2659         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2660
2661         return 0;
2662
2663 err_fib4_node_entry_link:
2664         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2665 err_fib4_entry_create:
2666         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2667         return err;
2668 }
2669
2670 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2671                                      struct fib_entry_notifier_info *fen_info)
2672 {
2673         struct mlxsw_sp_fib_entry *fib_entry;
2674         struct mlxsw_sp_fib_node *fib_node;
2675
2676         if (mlxsw_sp->router->aborted)
2677                 return;
2678
2679         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2680         if (WARN_ON(!fib_entry))
2681                 return;
2682         fib_node = fib_entry->fib_node;
2683
2684         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2685         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2686         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2687 }
2688
2689 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2690 {
2691         char ralta_pl[MLXSW_REG_RALTA_LEN];
2692         char ralst_pl[MLXSW_REG_RALST_LEN];
2693         int i, err;
2694
2695         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2696                              MLXSW_SP_LPM_TREE_MIN);
2697         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2698         if (err)
2699                 return err;
2700
2701         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2702         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2703         if (err)
2704                 return err;
2705
2706         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2707                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2708                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2709                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2710
2711                 if (!mlxsw_sp_vr_is_used(vr))
2712                         continue;
2713
2714                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2715                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2716                                      MLXSW_SP_LPM_TREE_MIN);
2717                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2718                                       raltb_pl);
2719                 if (err)
2720                         return err;
2721
2722                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2723                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2724                                       0);
2725                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2726                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2727                                       ralue_pl);
2728                 if (err)
2729                         return err;
2730         }
2731
2732         return 0;
2733 }
2734
2735 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2736                                      struct mlxsw_sp_fib_node *fib_node)
2737 {
2738         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2739
2740         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2741                 bool do_break = &tmp->list == &fib_node->entry_list;
2742
2743                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2744                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2745                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2746                 /* Break when entry list is empty and node was freed.
2747                  * Otherwise, we'll access freed memory in the next
2748                  * iteration.
2749                  */
2750                 if (do_break)
2751                         break;
2752         }
2753 }
2754
2755 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2756                                     struct mlxsw_sp_fib_node *fib_node)
2757 {
2758         switch (fib_node->fib->proto) {
2759         case MLXSW_SP_L3_PROTO_IPV4:
2760                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2761                 break;
2762         case MLXSW_SP_L3_PROTO_IPV6:
2763                 WARN_ON_ONCE(1);
2764                 break;
2765         }
2766 }
2767
2768 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2769                                   struct mlxsw_sp_vr *vr,
2770                                   enum mlxsw_sp_l3proto proto)
2771 {
2772         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2773         struct mlxsw_sp_fib_node *fib_node, *tmp;
2774
2775         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2776                 bool do_break = &tmp->list == &fib->node_list;
2777
2778                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2779                 if (do_break)
2780                         break;
2781         }
2782 }
2783
2784 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2785 {
2786         int i;
2787
2788         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2789                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2790
2791                 if (!mlxsw_sp_vr_is_used(vr))
2792                         continue;
2793                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2794         }
2795 }
2796
2797 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2798 {
2799         int err;
2800
2801         if (mlxsw_sp->router->aborted)
2802                 return;
2803         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2804         mlxsw_sp_router_fib_flush(mlxsw_sp);
2805         mlxsw_sp->router->aborted = true;
2806         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2807         if (err)
2808                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2809 }
2810
2811 struct mlxsw_sp_fib_event_work {
2812         struct work_struct work;
2813         union {
2814                 struct fib_entry_notifier_info fen_info;
2815                 struct fib_rule_notifier_info fr_info;
2816                 struct fib_nh_notifier_info fnh_info;
2817         };
2818         struct mlxsw_sp *mlxsw_sp;
2819         unsigned long event;
2820 };
2821
2822 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2823 {
2824         struct mlxsw_sp_fib_event_work *fib_work =
2825                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2826         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2827         struct fib_rule *rule;
2828         bool replace, append;
2829         int err;
2830
2831         /* Protect internal structures from changes */
2832         rtnl_lock();
2833         switch (fib_work->event) {
2834         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2835         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2836         case FIB_EVENT_ENTRY_ADD:
2837                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2838                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2839                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2840                                                replace, append);
2841                 if (err)
2842                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2843                 fib_info_put(fib_work->fen_info.fi);
2844                 break;
2845         case FIB_EVENT_ENTRY_DEL:
2846                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2847                 fib_info_put(fib_work->fen_info.fi);
2848                 break;
2849         case FIB_EVENT_RULE_ADD: /* fall through */
2850         case FIB_EVENT_RULE_DEL:
2851                 rule = fib_work->fr_info.rule;
2852                 if (!fib4_rule_default(rule) && !rule->l3mdev)
2853                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2854                 fib_rule_put(rule);
2855                 break;
2856         case FIB_EVENT_NH_ADD: /* fall through */
2857         case FIB_EVENT_NH_DEL:
2858                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2859                                        fib_work->fnh_info.fib_nh);
2860                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2861                 break;
2862         }
2863         rtnl_unlock();
2864         kfree(fib_work);
2865 }
2866
2867 /* Called with rcu_read_lock() */
2868 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2869                                      unsigned long event, void *ptr)
2870 {
2871         struct mlxsw_sp_fib_event_work *fib_work;
2872         struct fib_notifier_info *info = ptr;
2873         struct mlxsw_sp_router *router;
2874
2875         if (!net_eq(info->net, &init_net))
2876                 return NOTIFY_DONE;
2877
2878         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2879         if (WARN_ON(!fib_work))
2880                 return NOTIFY_BAD;
2881
2882         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2883         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
2884         fib_work->mlxsw_sp = router->mlxsw_sp;
2885         fib_work->event = event;
2886
2887         switch (event) {
2888         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2889         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2890         case FIB_EVENT_ENTRY_ADD: /* fall through */
2891         case FIB_EVENT_ENTRY_DEL:
2892                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2893                 /* Take referece on fib_info to prevent it from being
2894                  * freed while work is queued. Release it afterwards.
2895                  */
2896                 fib_info_hold(fib_work->fen_info.fi);
2897                 break;
2898         case FIB_EVENT_RULE_ADD: /* fall through */
2899         case FIB_EVENT_RULE_DEL:
2900                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2901                 fib_rule_get(fib_work->fr_info.rule);
2902                 break;
2903         case FIB_EVENT_NH_ADD: /* fall through */
2904         case FIB_EVENT_NH_DEL:
2905                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2906                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2907                 break;
2908         }
2909
2910         mlxsw_core_schedule_work(&fib_work->work);
2911
2912         return NOTIFY_DONE;
2913 }
2914
2915 static struct mlxsw_sp_rif *
2916 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2917                          const struct net_device *dev)
2918 {
2919         int i;
2920
2921         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2922                 if (mlxsw_sp->router->rifs[i] &&
2923                     mlxsw_sp->router->rifs[i]->dev == dev)
2924                         return mlxsw_sp->router->rifs[i];
2925
2926         return NULL;
2927 }
2928
2929 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2930 {
2931         char ritr_pl[MLXSW_REG_RITR_LEN];
2932         int err;
2933
2934         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2935         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2936         if (WARN_ON_ONCE(err))
2937                 return err;
2938
2939         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2940         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2941 }
2942
2943 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2944                                           struct mlxsw_sp_rif *rif)
2945 {
2946         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2947         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2948         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2949 }
2950
2951 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2952                                        const struct in_device *in_dev,
2953                                        unsigned long event)
2954 {
2955         switch (event) {
2956         case NETDEV_UP:
2957                 if (!rif)
2958                         return true;
2959                 return false;
2960         case NETDEV_DOWN:
2961                 if (rif && !in_dev->ifa_list &&
2962                     !netif_is_l3_slave(rif->dev))
2963                         return true;
2964                 /* It is possible we already removed the RIF ourselves
2965                  * if it was assigned to a netdev that is now a bridge
2966                  * or LAG slave.
2967                  */
2968                 return false;
2969         }
2970
2971         return false;
2972 }
2973
2974 static enum mlxsw_sp_rif_type
2975 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
2976                       const struct net_device *dev)
2977 {
2978         enum mlxsw_sp_fid_type type;
2979
2980         /* RIF type is derived from the type of the underlying FID */
2981         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
2982                 type = MLXSW_SP_FID_TYPE_8021Q;
2983         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
2984                 type = MLXSW_SP_FID_TYPE_8021Q;
2985         else if (netif_is_bridge_master(dev))
2986                 type = MLXSW_SP_FID_TYPE_8021D;
2987         else
2988                 type = MLXSW_SP_FID_TYPE_RFID;
2989
2990         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
2991 }
2992
2993 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
2994 {
2995         int i;
2996
2997         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
2998                 if (!mlxsw_sp->router->rifs[i]) {
2999                         *p_rif_index = i;
3000                         return 0;
3001                 }
3002         }
3003
3004         return -ENOBUFS;
3005 }
3006
3007 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
3008                                                u16 vr_id,
3009                                                struct net_device *l3_dev)
3010 {
3011         struct mlxsw_sp_rif *rif;
3012
3013         rif = kzalloc(rif_size, GFP_KERNEL);
3014         if (!rif)
3015                 return NULL;
3016
3017         INIT_LIST_HEAD(&rif->nexthop_list);
3018         INIT_LIST_HEAD(&rif->neigh_list);
3019         ether_addr_copy(rif->addr, l3_dev->dev_addr);
3020         rif->mtu = l3_dev->mtu;
3021         rif->vr_id = vr_id;
3022         rif->dev = l3_dev;
3023         rif->rif_index = rif_index;
3024
3025         return rif;
3026 }
3027
3028 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
3029                                            u16 rif_index)
3030 {
3031         return mlxsw_sp->router->rifs[rif_index];
3032 }
3033
3034 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
3035 {
3036         return rif->rif_index;
3037 }
3038
3039 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
3040 {
3041         return rif->dev->ifindex;
3042 }
3043
3044 static struct mlxsw_sp_rif *
3045 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
3046                     const struct mlxsw_sp_rif_params *params)
3047 {
3048         u32 tb_id = l3mdev_fib_table(params->dev);
3049         const struct mlxsw_sp_rif_ops *ops;
3050         enum mlxsw_sp_rif_type type;
3051         struct mlxsw_sp_rif *rif;
3052         struct mlxsw_sp_fid *fid;
3053         struct mlxsw_sp_vr *vr;
3054         u16 rif_index;
3055         int err;
3056
3057         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
3058         ops = mlxsw_sp->router->rif_ops_arr[type];
3059
3060         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3061         if (IS_ERR(vr))
3062                 return ERR_CAST(vr);
3063
3064         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
3065         if (err)
3066                 goto err_rif_index_alloc;
3067
3068         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
3069         if (!rif) {
3070                 err = -ENOMEM;
3071                 goto err_rif_alloc;
3072         }
3073         rif->mlxsw_sp = mlxsw_sp;
3074         rif->ops = ops;
3075
3076         fid = ops->fid_get(rif);
3077         if (IS_ERR(fid)) {
3078                 err = PTR_ERR(fid);
3079                 goto err_fid_get;
3080         }
3081         rif->fid = fid;
3082
3083         if (ops->setup)
3084                 ops->setup(rif, params);
3085
3086         err = ops->configure(rif);
3087         if (err)
3088                 goto err_configure;
3089
3090         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
3091                                   mlxsw_sp_fid_index(fid), true);
3092         if (err)
3093                 goto err_rif_fdb_op;
3094
3095         mlxsw_sp_rif_counters_alloc(rif);
3096         mlxsw_sp_fid_rif_set(fid, rif);
3097         mlxsw_sp->router->rifs[rif_index] = rif;
3098         vr->rif_count++;
3099
3100         return rif;
3101
3102 err_rif_fdb_op:
3103         ops->deconfigure(rif);
3104 err_configure:
3105         mlxsw_sp_fid_put(fid);
3106 err_fid_get:
3107         kfree(rif);
3108 err_rif_alloc:
3109 err_rif_index_alloc:
3110         mlxsw_sp_vr_put(vr);
3111         return ERR_PTR(err);
3112 }
3113
3114 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
3115 {
3116         const struct mlxsw_sp_rif_ops *ops = rif->ops;
3117         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3118         struct mlxsw_sp_fid *fid = rif->fid;
3119         struct mlxsw_sp_vr *vr;
3120
3121         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3122         vr = &mlxsw_sp->router->vrs[rif->vr_id];
3123
3124         vr->rif_count--;
3125         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
3126         mlxsw_sp_fid_rif_set(fid, NULL);
3127         mlxsw_sp_rif_counters_free(rif);
3128         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
3129                             mlxsw_sp_fid_index(fid), false);
3130         ops->deconfigure(rif);
3131         mlxsw_sp_fid_put(fid);
3132         kfree(rif);
3133         mlxsw_sp_vr_put(vr);
3134 }
3135
3136 static void
3137 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
3138                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3139 {
3140         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3141
3142         params->vid = mlxsw_sp_port_vlan->vid;
3143         params->lag = mlxsw_sp_port->lagged;
3144         if (params->lag)
3145                 params->lag_id = mlxsw_sp_port->lag_id;
3146         else
3147                 params->system_port = mlxsw_sp_port->local_port;
3148 }
3149
3150 static int
3151 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
3152                                struct net_device *l3_dev)
3153 {
3154         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3155         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
3156         u16 vid = mlxsw_sp_port_vlan->vid;
3157         struct mlxsw_sp_rif *rif;
3158         struct mlxsw_sp_fid *fid;
3159         int err;
3160
3161         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3162         if (!rif) {
3163                 struct mlxsw_sp_rif_params params = {
3164                         .dev = l3_dev,
3165                 };
3166
3167                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
3168                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3169                 if (IS_ERR(rif))
3170                         return PTR_ERR(rif);
3171         }
3172
3173         /* FID was already created, just take a reference */
3174         fid = rif->ops->fid_get(rif);
3175         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
3176         if (err)
3177                 goto err_fid_port_vid_map;
3178
3179         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
3180         if (err)
3181                 goto err_port_vid_learning_set;
3182
3183         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
3184                                         BR_STATE_FORWARDING);
3185         if (err)
3186                 goto err_port_vid_stp_set;
3187
3188         mlxsw_sp_port_vlan->fid = fid;
3189
3190         return 0;
3191
3192 err_port_vid_stp_set:
3193         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3194 err_port_vid_learning_set:
3195         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3196 err_fid_port_vid_map:
3197         mlxsw_sp_fid_put(fid);
3198         return err;
3199 }
3200
3201 void
3202 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3203 {
3204         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3205         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
3206         u16 vid = mlxsw_sp_port_vlan->vid;
3207
3208         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
3209                 return;
3210
3211         mlxsw_sp_port_vlan->fid = NULL;
3212         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
3213         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3214         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3215         /* If router port holds the last reference on the rFID, then the
3216          * associated Sub-port RIF will be destroyed.
3217          */
3218         mlxsw_sp_fid_put(fid);
3219 }
3220
3221 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
3222                                              struct net_device *port_dev,
3223                                              unsigned long event, u16 vid)
3224 {
3225         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
3226         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
3227
3228         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
3229         if (WARN_ON(!mlxsw_sp_port_vlan))
3230                 return -EINVAL;
3231
3232         switch (event) {
3233         case NETDEV_UP:
3234                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
3235                                                       l3_dev);
3236         case NETDEV_DOWN:
3237                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
3238                 break;
3239         }
3240
3241         return 0;
3242 }
3243
3244 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
3245                                         unsigned long event)
3246 {
3247         if (netif_is_bridge_port(port_dev) ||
3248             netif_is_lag_port(port_dev) ||
3249             netif_is_ovs_port(port_dev))
3250                 return 0;
3251
3252         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
3253 }
3254
3255 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
3256                                          struct net_device *lag_dev,
3257                                          unsigned long event, u16 vid)
3258 {
3259         struct net_device *port_dev;
3260         struct list_head *iter;
3261         int err;
3262
3263         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
3264                 if (mlxsw_sp_port_dev_check(port_dev)) {
3265                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
3266                                                                 port_dev,
3267                                                                 event, vid);
3268                         if (err)
3269                                 return err;
3270                 }
3271         }
3272
3273         return 0;
3274 }
3275
3276 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
3277                                        unsigned long event)
3278 {
3279         if (netif_is_bridge_port(lag_dev))
3280                 return 0;
3281
3282         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
3283 }
3284
3285 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3286                                           unsigned long event)
3287 {
3288         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3289         struct mlxsw_sp_rif_params params = {
3290                 .dev = l3_dev,
3291         };
3292         struct mlxsw_sp_rif *rif;
3293
3294         switch (event) {
3295         case NETDEV_UP:
3296                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3297                 if (IS_ERR(rif))
3298                         return PTR_ERR(rif);
3299                 break;
3300         case NETDEV_DOWN:
3301                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3302                 mlxsw_sp_rif_destroy(rif);
3303                 break;
3304         }
3305
3306         return 0;
3307 }
3308
3309 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3310                                         unsigned long event)
3311 {
3312         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3313         u16 vid = vlan_dev_vlan_id(vlan_dev);
3314
3315         if (netif_is_bridge_port(vlan_dev))
3316                 return 0;
3317
3318         if (mlxsw_sp_port_dev_check(real_dev))
3319                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
3320                                                          event, vid);
3321         else if (netif_is_lag_master(real_dev))
3322                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3323                                                      vid);
3324         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
3325                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
3326
3327         return 0;
3328 }
3329
3330 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
3331                                      unsigned long event)
3332 {
3333         if (mlxsw_sp_port_dev_check(dev))
3334                 return mlxsw_sp_inetaddr_port_event(dev, event);
3335         else if (netif_is_lag_master(dev))
3336                 return mlxsw_sp_inetaddr_lag_event(dev, event);
3337         else if (netif_is_bridge_master(dev))
3338                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
3339         else if (is_vlan_dev(dev))
3340                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
3341         else
3342                 return 0;
3343 }
3344
3345 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3346                             unsigned long event, void *ptr)
3347 {
3348         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3349         struct net_device *dev = ifa->ifa_dev->dev;
3350         struct mlxsw_sp *mlxsw_sp;
3351         struct mlxsw_sp_rif *rif;
3352         int err = 0;
3353
3354         mlxsw_sp = mlxsw_sp_lower_get(dev);
3355         if (!mlxsw_sp)
3356                 goto out;
3357
3358         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3359         if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3360                 goto out;
3361
3362         err = __mlxsw_sp_inetaddr_event(dev, event);
3363 out:
3364         return notifier_from_errno(err);
3365 }
3366
3367 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3368                              const char *mac, int mtu)
3369 {
3370         char ritr_pl[MLXSW_REG_RITR_LEN];
3371         int err;
3372
3373         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3374         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3375         if (err)
3376                 return err;
3377
3378         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3379         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3380         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3381         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3382 }
3383
3384 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3385 {
3386         struct mlxsw_sp *mlxsw_sp;
3387         struct mlxsw_sp_rif *rif;
3388         u16 fid_index;
3389         int err;
3390
3391         mlxsw_sp = mlxsw_sp_lower_get(dev);
3392         if (!mlxsw_sp)
3393                 return 0;
3394
3395         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3396         if (!rif)
3397                 return 0;
3398         fid_index = mlxsw_sp_fid_index(rif->fid);
3399
3400         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
3401         if (err)
3402                 return err;
3403
3404         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3405                                 dev->mtu);
3406         if (err)
3407                 goto err_rif_edit;
3408
3409         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
3410         if (err)
3411                 goto err_rif_fdb_op;
3412
3413         ether_addr_copy(rif->addr, dev->dev_addr);
3414         rif->mtu = dev->mtu;
3415
3416         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3417
3418         return 0;
3419
3420 err_rif_fdb_op:
3421         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3422 err_rif_edit:
3423         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
3424         return err;
3425 }
3426
3427 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
3428                                   struct net_device *l3_dev)
3429 {
3430         struct mlxsw_sp_rif *rif;
3431
3432         /* If netdev is already associated with a RIF, then we need to
3433          * destroy it and create a new one with the new virtual router ID.
3434          */
3435         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3436         if (rif)
3437                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3438
3439         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
3440 }
3441
3442 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3443                                     struct net_device *l3_dev)
3444 {
3445         struct mlxsw_sp_rif *rif;
3446
3447         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3448         if (!rif)
3449                 return;
3450         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3451 }
3452
3453 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
3454                                  struct netdev_notifier_changeupper_info *info)
3455 {
3456         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3457         int err = 0;
3458
3459         if (!mlxsw_sp)
3460                 return 0;
3461
3462         switch (event) {
3463         case NETDEV_PRECHANGEUPPER:
3464                 return 0;
3465         case NETDEV_CHANGEUPPER:
3466                 if (info->linking)
3467                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
3468                 else
3469                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
3470                 break;
3471         }
3472
3473         return err;
3474 }
3475
3476 static struct mlxsw_sp_rif_subport *
3477 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
3478 {
3479         return container_of(rif, struct mlxsw_sp_rif_subport, common);
3480 }
3481
3482 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
3483                                        const struct mlxsw_sp_rif_params *params)
3484 {
3485         struct mlxsw_sp_rif_subport *rif_subport;
3486
3487         rif_subport = mlxsw_sp_rif_subport_rif(rif);
3488         rif_subport->vid = params->vid;
3489         rif_subport->lag = params->lag;
3490         if (params->lag)
3491                 rif_subport->lag_id = params->lag_id;
3492         else
3493                 rif_subport->system_port = params->system_port;
3494 }
3495
3496 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
3497 {
3498         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3499         struct mlxsw_sp_rif_subport *rif_subport;
3500         char ritr_pl[MLXSW_REG_RITR_LEN];
3501
3502         rif_subport = mlxsw_sp_rif_subport_rif(rif);
3503         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
3504                             rif->rif_index, rif->vr_id, rif->dev->mtu,
3505                             rif->dev->dev_addr);
3506         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
3507                                   rif_subport->lag ? rif_subport->lag_id :
3508                                                      rif_subport->system_port,
3509                                   rif_subport->vid);
3510
3511         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3512 }
3513
3514 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
3515 {
3516         return mlxsw_sp_rif_subport_op(rif, true);
3517 }
3518
3519 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
3520 {
3521         mlxsw_sp_rif_subport_op(rif, false);
3522 }
3523
3524 static struct mlxsw_sp_fid *
3525 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
3526 {
3527         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
3528 }
3529
3530 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
3531         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
3532         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
3533         .setup                  = mlxsw_sp_rif_subport_setup,
3534         .configure              = mlxsw_sp_rif_subport_configure,
3535         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
3536         .fid_get                = mlxsw_sp_rif_subport_fid_get,
3537 };
3538
3539 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
3540                                     enum mlxsw_reg_ritr_if_type type,
3541                                     u16 vid_fid, bool enable)
3542 {
3543         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3544         char ritr_pl[MLXSW_REG_RITR_LEN];
3545
3546         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
3547                             rif->dev->mtu, rif->dev->dev_addr);
3548         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
3549
3550         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3551 }
3552
3553 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
3554 {
3555         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
3556 }
3557
3558 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
3559 {
3560         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3561         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3562         int err;
3563
3564         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
3565         if (err)
3566                 return err;
3567
3568         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3569                                      mlxsw_sp_router_port(mlxsw_sp), true);
3570         if (err)
3571                 goto err_fid_bc_flood_set;
3572
3573         return 0;
3574
3575 err_fid_bc_flood_set:
3576         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3577         return err;
3578 }
3579
3580 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
3581 {
3582         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3583         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3584
3585         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3586                                mlxsw_sp_router_port(mlxsw_sp), false);
3587         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3588 }
3589
3590 static struct mlxsw_sp_fid *
3591 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
3592 {
3593         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
3594
3595         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
3596 }
3597
3598 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
3599         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
3600         .rif_size               = sizeof(struct mlxsw_sp_rif),
3601         .configure              = mlxsw_sp_rif_vlan_configure,
3602         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
3603         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
3604 };
3605
3606 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
3607 {
3608         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3609         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3610         int err;
3611
3612         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
3613                                        true);
3614         if (err)
3615                 return err;
3616
3617         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3618                                      mlxsw_sp_router_port(mlxsw_sp), true);
3619         if (err)
3620                 goto err_fid_bc_flood_set;
3621
3622         return 0;
3623
3624 err_fid_bc_flood_set:
3625         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3626         return err;
3627 }
3628
3629 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
3630 {
3631         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3632         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3633
3634         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3635                                mlxsw_sp_router_port(mlxsw_sp), false);
3636         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3637 }
3638
3639 static struct mlxsw_sp_fid *
3640 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
3641 {
3642         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
3643 }
3644
3645 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
3646         .type                   = MLXSW_SP_RIF_TYPE_FID,
3647         .rif_size               = sizeof(struct mlxsw_sp_rif),
3648         .configure              = mlxsw_sp_rif_fid_configure,
3649         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
3650         .fid_get                = mlxsw_sp_rif_fid_fid_get,
3651 };
3652
3653 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
3654         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
3655         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
3656         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
3657 };
3658
3659 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
3660 {
3661         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3662
3663         mlxsw_sp->router->rifs = kcalloc(max_rifs,
3664                                          sizeof(struct mlxsw_sp_rif *),
3665                                          GFP_KERNEL);
3666         if (!mlxsw_sp->router->rifs)
3667                 return -ENOMEM;
3668
3669         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
3670
3671         return 0;
3672 }
3673
3674 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
3675 {
3676         int i;
3677
3678         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3679                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
3680
3681         kfree(mlxsw_sp->router->rifs);
3682 }
3683
3684 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3685 {
3686         struct mlxsw_sp_router *router;
3687
3688         /* Flush pending FIB notifications and then flush the device's
3689          * table before requesting another dump. The FIB notification
3690          * block is unregistered, so no need to take RTNL.
3691          */
3692         mlxsw_core_flush_owq();
3693         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3694         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
3695 }
3696
3697 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3698 {
3699         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3700         u64 max_rifs;
3701         int err;
3702
3703         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3704                 return -EIO;
3705         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3706
3707         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3708         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3709         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3710         if (err)
3711                 return err;
3712         return 0;
3713 }
3714
3715 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3716 {
3717         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3718
3719         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3720         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3721 }
3722
3723 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3724 {
3725         struct mlxsw_sp_router *router;
3726         int err;
3727
3728         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
3729         if (!router)
3730                 return -ENOMEM;
3731         mlxsw_sp->router = router;
3732         router->mlxsw_sp = mlxsw_sp;
3733
3734         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
3735         err = __mlxsw_sp_router_init(mlxsw_sp);
3736         if (err)
3737                 goto err_router_init;
3738
3739         err = mlxsw_sp_rifs_init(mlxsw_sp);
3740         if (err)
3741                 goto err_rifs_init;
3742
3743         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
3744                               &mlxsw_sp_nexthop_ht_params);
3745         if (err)
3746                 goto err_nexthop_ht_init;
3747
3748         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
3749                               &mlxsw_sp_nexthop_group_ht_params);
3750         if (err)
3751                 goto err_nexthop_group_ht_init;
3752
3753         err = mlxsw_sp_lpm_init(mlxsw_sp);
3754         if (err)
3755                 goto err_lpm_init;
3756
3757         err = mlxsw_sp_vrs_init(mlxsw_sp);
3758         if (err)
3759                 goto err_vrs_init;
3760
3761         err = mlxsw_sp_neigh_init(mlxsw_sp);
3762         if (err)
3763                 goto err_neigh_init;
3764
3765         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3766         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
3767                                     mlxsw_sp_router_fib_dump_flush);
3768         if (err)
3769                 goto err_register_fib_notifier;
3770
3771         return 0;
3772
3773 err_register_fib_notifier:
3774         mlxsw_sp_neigh_fini(mlxsw_sp);
3775 err_neigh_init:
3776         mlxsw_sp_vrs_fini(mlxsw_sp);
3777 err_vrs_init:
3778         mlxsw_sp_lpm_fini(mlxsw_sp);
3779 err_lpm_init:
3780         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3781 err_nexthop_group_ht_init:
3782         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3783 err_nexthop_ht_init:
3784         mlxsw_sp_rifs_fini(mlxsw_sp);
3785 err_rifs_init:
3786         __mlxsw_sp_router_fini(mlxsw_sp);
3787 err_router_init:
3788         kfree(mlxsw_sp->router);
3789         return err;
3790 }
3791
3792 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3793 {
3794         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
3795         mlxsw_sp_neigh_fini(mlxsw_sp);
3796         mlxsw_sp_vrs_fini(mlxsw_sp);
3797         mlxsw_sp_lpm_fini(mlxsw_sp);
3798         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3799         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3800         mlxsw_sp_rifs_fini(mlxsw_sp);
3801         __mlxsw_sp_router_fini(mlxsw_sp);
3802         kfree(mlxsw_sp->router);
3803 }