drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <net/netevent.h>
  46 #include <net/neighbour.h>
  47 #include <net/arp.h>
  48 #include <net/ip_fib.h>
  49 #include <net/fib_rules.h>
  50 #include <net/l3mdev.h>
  51
  52 #include "spectrum.h"
  53 #include "core.h"
  54 #include "reg.h"
  55
  56 struct mlxsw_sp_rif {
  57         struct list_head nexthop_list;
  58         struct list_head neigh_list;
  59         struct net_device *dev;
  60         struct mlxsw_sp_fid *f;
  61         unsigned char addr[ETH_ALEN];
  62         int mtu;
  63         u16 rif;
  64         u16 vr_id;
  65 };
  66
  67 static struct mlxsw_sp_rif *
  68 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
  69                          const struct net_device *dev);
  70
  71 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  72         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  73
  74 static bool
  75 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  76                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  77 {
  78         unsigned char prefix;
  79
  80         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  81                 if (!test_bit(prefix, prefix_usage2->b))
  82                         return false;
  83         }
  84         return true;
  85 }
  86
  87 static bool
  88 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  89                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  90 {
  91         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  92 }
  93
  94 static bool
  95 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  96 {
  97         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  98
  99         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 100 }
 101
 102 static void
 103 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 104                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 105 {
 106         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 107 }
 108
 109 static void
 110 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 111                           unsigned char prefix_len)
 112 {
 113         set_bit(prefix_len, prefix_usage->b);
 114 }
 115
 116 static void
 117 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 118                             unsigned char prefix_len)
 119 {
 120         clear_bit(prefix_len, prefix_usage->b);
 121 }
 122
 123 struct mlxsw_sp_fib_key {
 124         unsigned char addr[sizeof(struct in6_addr)];
 125         unsigned char prefix_len;
 126 };
 127
 128 enum mlxsw_sp_fib_entry_type {
 129         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 130         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 131         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 132 };
 133
 134 struct mlxsw_sp_nexthop_group;
 135
 136 struct mlxsw_sp_fib_node {
 137         struct list_head entry_list;
 138         struct list_head list;
 139         struct rhash_head ht_node;
 140         struct mlxsw_sp_fib *fib;
 141         struct mlxsw_sp_fib_key key;
 142 };
 143
 144 struct mlxsw_sp_fib_entry_params {
 145         u32 tb_id;
 146         u32 prio;
 147         u8 tos;
 148         u8 type;
 149 };
 150
 151 struct mlxsw_sp_fib_entry {
 152         struct list_head list;
 153         struct mlxsw_sp_fib_node *fib_node;
 154         enum mlxsw_sp_fib_entry_type type;
 155         struct list_head nexthop_group_node;
 156         struct mlxsw_sp_nexthop_group *nh_group;
 157         struct mlxsw_sp_fib_entry_params params;
 158         bool offloaded;
 159 };
 160
 161 struct mlxsw_sp_fib {
 162         struct rhashtable ht;
 163         struct list_head node_list;
 164         struct mlxsw_sp_vr *vr;
 165         struct mlxsw_sp_lpm_tree *lpm_tree;
 166         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 167         struct mlxsw_sp_prefix_usage prefix_usage;
 168         enum mlxsw_sp_l3proto proto;
 169 };
 170
 171 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 172
 173 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 174                                                 enum mlxsw_sp_l3proto proto)
 175 {
 176         struct mlxsw_sp_fib *fib;
 177         int err;
 178
 179         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 180         if (!fib)
 181                 return ERR_PTR(-ENOMEM);
 182         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 183         if (err)
 184                 goto err_rhashtable_init;
 185         INIT_LIST_HEAD(&fib->node_list);
 186         fib->proto = proto;
 187         fib->vr = vr;
 188         return fib;
 189
 190 err_rhashtable_init:
 191         kfree(fib);
 192         return ERR_PTR(err);
 193 }
 194
 195 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 196 {
 197         WARN_ON(!list_empty(&fib->node_list));
 198         WARN_ON(fib->lpm_tree);
 199         rhashtable_destroy(&fib->ht);
 200         kfree(fib);
 201 }
 202
 203 static struct mlxsw_sp_lpm_tree *
 204 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 205 {
 206         static struct mlxsw_sp_lpm_tree *lpm_tree;
 207         int i;
 208
 209         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 210                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 211                 if (lpm_tree->ref_count == 0)
 212                         return lpm_tree;
 213         }
 214         return NULL;
 215 }
 216
 217 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 218                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 219 {
 220         char ralta_pl[MLXSW_REG_RALTA_LEN];
 221
 222         mlxsw_reg_ralta_pack(ralta_pl, true,
 223                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 224                              lpm_tree->id);
 225         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 226 }
 227
 228 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 229                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 230 {
 231         char ralta_pl[MLXSW_REG_RALTA_LEN];
 232
 233         mlxsw_reg_ralta_pack(ralta_pl, false,
 234                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 235                              lpm_tree->id);
 236         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 237 }
 238
 239 static int
 240 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 241                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 242                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 243 {
 244         char ralst_pl[MLXSW_REG_RALST_LEN];
 245         u8 root_bin = 0;
 246         u8 prefix;
 247         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 248
 249         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 250                 root_bin = prefix;
 251
 252         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 253         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 254                 if (prefix == 0)
 255                         continue;
 256                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 257                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 258                 last_prefix = prefix;
 259         }
 260         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 261 }
 262
 263 static struct mlxsw_sp_lpm_tree *
 264 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 265                          struct mlxsw_sp_prefix_usage *prefix_usage,
 266                          enum mlxsw_sp_l3proto proto)
 267 {
 268         struct mlxsw_sp_lpm_tree *lpm_tree;
 269         int err;
 270
 271         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 272         if (!lpm_tree)
 273                 return ERR_PTR(-EBUSY);
 274         lpm_tree->proto = proto;
 275         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 276         if (err)
 277                 return ERR_PTR(err);
 278
 279         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 280                                                 lpm_tree);
 281         if (err)
 282                 goto err_left_struct_set;
 283         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 284                sizeof(lpm_tree->prefix_usage));
 285         return lpm_tree;
 286
 287 err_left_struct_set:
 288         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 289         return ERR_PTR(err);
 290 }
 291
 292 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 293                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 294 {
 295         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 296 }
 297
 298 static struct mlxsw_sp_lpm_tree *
 299 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 300                       struct mlxsw_sp_prefix_usage *prefix_usage,
 301                       enum mlxsw_sp_l3proto proto)
 302 {
 303         struct mlxsw_sp_lpm_tree *lpm_tree;
 304         int i;
 305
 306         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 307                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 308                 if (lpm_tree->ref_count != 0 &&
 309                     lpm_tree->proto == proto &&
 310                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 311                                              prefix_usage))
 312                         goto inc_ref_count;
 313         }
 314         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 315                                             proto);
 316         if (IS_ERR(lpm_tree))
 317                 return lpm_tree;
 318
 319 inc_ref_count:
 320         lpm_tree->ref_count++;
 321         return lpm_tree;
 322 }
 323
 324 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 325                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 326 {
 327         if (--lpm_tree->ref_count == 0)
 328                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 329         return 0;
 330 }
 331
 332 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 333 {
 334         struct mlxsw_sp_lpm_tree *lpm_tree;
 335         int i;
 336
 337         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 338                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 339                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 340         }
 341 }
 342
 343 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 344 {
 345         return !!vr->fib4;
 346 }
 347
 348 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 349 {
 350         struct mlxsw_sp_vr *vr;
 351         int i;
 352
 353         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 354                 vr = &mlxsw_sp->router.vrs[i];
 355                 if (!mlxsw_sp_vr_is_used(vr))
 356                         return vr;
 357         }
 358         return NULL;
 359 }
 360
 361 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 362                                      const struct mlxsw_sp_fib *fib)
 363 {
 364         char raltb_pl[MLXSW_REG_RALTB_LEN];
 365
 366         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 367                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 368                              fib->lpm_tree->id);
 369         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 370 }
 371
 372 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 373                                        const struct mlxsw_sp_fib *fib)
 374 {
 375         char raltb_pl[MLXSW_REG_RALTB_LEN];
 376
 377         /* Bind to tree 0 which is default */
 378         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 379                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 380         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 381 }
 382
 383 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 384 {
 385         /* For our purpose, squash main and local table into one */
 386         if (tb_id == RT_TABLE_LOCAL)
 387                 tb_id = RT_TABLE_MAIN;
 388         return tb_id;
 389 }
 390
 391 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 392                                             u32 tb_id)
 393 {
 394         struct mlxsw_sp_vr *vr;
 395         int i;
 396
 397         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 398
 399         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 400                 vr = &mlxsw_sp->router.vrs[i];
 401                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 402                         return vr;
 403         }
 404         return NULL;
 405 }
 406
 407 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 408                                             enum mlxsw_sp_l3proto proto)
 409 {
 410         switch (proto) {
 411         case MLXSW_SP_L3_PROTO_IPV4:
 412                 return vr->fib4;
 413         case MLXSW_SP_L3_PROTO_IPV6:
 414                 BUG_ON(1);
 415         }
 416         return NULL;
 417 }
 418
 419 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 420                                               u32 tb_id)
 421 {
 422         struct mlxsw_sp_vr *vr;
 423
 424         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 425         if (!vr)
 426                 return ERR_PTR(-EBUSY);
 427         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 428         if (IS_ERR(vr->fib4))
 429                 return ERR_CAST(vr->fib4);
 430         vr->tb_id = tb_id;
 431         return vr;
 432 }
 433
 434 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 435 {
 436         mlxsw_sp_fib_destroy(vr->fib4);
 437         vr->fib4 = NULL;
 438 }
 439
 440 static int
 441 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 442                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 443 {
 444         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 445         struct mlxsw_sp_lpm_tree *new_tree;
 446         int err;
 447
 448         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 449                 return 0;
 450
 451         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 452                                          fib->proto);
 453         if (IS_ERR(new_tree)) {
 454                 /* We failed to get a tree according to the required
 455                  * prefix usage. However, the current tree might be still good
 456                  * for us if our requirement is subset of the prefixes used
 457                  * in the tree.
 458                  */
 459                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 460                                                  &lpm_tree->prefix_usage))
 461                         return 0;
 462                 return PTR_ERR(new_tree);
 463         }
 464
 465         /* Prevent packet loss by overwriting existing binding */
 466         fib->lpm_tree = new_tree;
 467         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 468         if (err)
 469                 goto err_tree_bind;
 470         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 471
 472         return 0;
 473
 474 err_tree_bind:
 475         fib->lpm_tree = lpm_tree;
 476         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 477         return err;
 478 }
 479
 480 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 481 {
 482         struct mlxsw_sp_vr *vr;
 483
 484         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 485         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 486         if (!vr)
 487                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 488         return vr;
 489 }
 490
 491 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 492 {
 493         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 494                 mlxsw_sp_vr_destroy(vr);
 495 }
 496
 497 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 498 {
 499         struct mlxsw_sp_vr *vr;
 500         u64 max_vrs;
 501         int i;
 502
 503         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 504                 return -EIO;
 505
 506         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 507         mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 508                                        GFP_KERNEL);
 509         if (!mlxsw_sp->router.vrs)
 510                 return -ENOMEM;
 511
 512         for (i = 0; i < max_vrs; i++) {
 513                 vr = &mlxsw_sp->router.vrs[i];
 514                 vr->id = i;
 515         }
 516
 517         return 0;
 518 }
 519
 520 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 521
 522 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 523 {
 524         /* At this stage we're guaranteed not to have new incoming
 525          * FIB notifications and the work queue is free from FIBs
 526          * sitting on top of mlxsw netdevs. However, we can still
 527          * have other FIBs queued. Flush the queue before flushing
 528          * the device's tables. No need for locks, as we're the only
 529          * writer.
 530          */
 531         mlxsw_core_flush_owq();
 532         mlxsw_sp_router_fib_flush(mlxsw_sp);
 533         kfree(mlxsw_sp->router.vrs);
 534 }
 535
 536 struct mlxsw_sp_neigh_key {
 537         struct neighbour *n;
 538 };
 539
 540 struct mlxsw_sp_neigh_entry {
 541         struct list_head rif_list_node;
 542         struct rhash_head ht_node;
 543         struct mlxsw_sp_neigh_key key;
 544         u16 rif;
 545         bool connected;
 546         unsigned char ha[ETH_ALEN];
 547         struct list_head nexthop_list; /* list of nexthops using
 548                                         * this neigh entry
 549                                         */
 550         struct list_head nexthop_neighs_list_node;
 551 };
 552
 553 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 554         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 555         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 556         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 557 };
 558
 559 static struct mlxsw_sp_neigh_entry *
 560 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 561                            u16 rif)
 562 {
 563         struct mlxsw_sp_neigh_entry *neigh_entry;
 564
 565         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 566         if (!neigh_entry)
 567                 return NULL;
 568
 569         neigh_entry->key.n = n;
 570         neigh_entry->rif = rif;
 571         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 572
 573         return neigh_entry;
 574 }
 575
 576 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 577 {
 578         kfree(neigh_entry);
 579 }
 580
 581 static int
 582 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 583                             struct mlxsw_sp_neigh_entry *neigh_entry)
 584 {
 585         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 586                                       &neigh_entry->ht_node,
 587                                       mlxsw_sp_neigh_ht_params);
 588 }
 589
 590 static void
 591 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 592                             struct mlxsw_sp_neigh_entry *neigh_entry)
 593 {
 594         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 595                                &neigh_entry->ht_node,
 596                                mlxsw_sp_neigh_ht_params);
 597 }
 598
 599 static struct mlxsw_sp_neigh_entry *
 600 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 601 {
 602         struct mlxsw_sp_neigh_entry *neigh_entry;
 603         struct mlxsw_sp_rif *r;
 604         int err;
 605
 606         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 607         if (!r)
 608                 return ERR_PTR(-EINVAL);
 609
 610         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, r->rif);
 611         if (!neigh_entry)
 612                 return ERR_PTR(-ENOMEM);
 613
 614         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 615         if (err)
 616                 goto err_neigh_entry_insert;
 617
 618         list_add(&neigh_entry->rif_list_node, &r->neigh_list);
 619
 620         return neigh_entry;
 621
 622 err_neigh_entry_insert:
 623         mlxsw_sp_neigh_entry_free(neigh_entry);
 624         return ERR_PTR(err);
 625 }
 626
 627 static void
 628 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 629                              struct mlxsw_sp_neigh_entry *neigh_entry)
 630 {
 631         list_del(&neigh_entry->rif_list_node);
 632         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 633         mlxsw_sp_neigh_entry_free(neigh_entry);
 634 }
 635
 636 static struct mlxsw_sp_neigh_entry *
 637 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 638 {
 639         struct mlxsw_sp_neigh_key key;
 640
 641         key.n = n;
 642         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 643                                       &key, mlxsw_sp_neigh_ht_params);
 644 }
 645
 646 static void
 647 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 648 {
 649         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 650
 651         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 652 }
 653
 654 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 655                                                    char *rauhtd_pl,
 656                                                    int ent_index)
 657 {
 658         struct net_device *dev;
 659         struct neighbour *n;
 660         __be32 dipn;
 661         u32 dip;
 662         u16 rif;
 663
 664         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 665
 666         if (!mlxsw_sp->rifs[rif]) {
 667                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 668                 return;
 669         }
 670
 671         dipn = htonl(dip);
 672         dev = mlxsw_sp->rifs[rif]->dev;
 673         n = neigh_lookup(&arp_tbl, &dipn, dev);
 674         if (!n) {
 675                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 676                            &dip);
 677                 return;
 678         }
 679
 680         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 681         neigh_event_send(n, NULL);
 682         neigh_release(n);
 683 }
 684
 685 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 686                                                    char *rauhtd_pl,
 687                                                    int rec_index)
 688 {
 689         u8 num_entries;
 690         int i;
 691
 692         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 693                                                                 rec_index);
 694         /* Hardware starts counting at 0, so add 1. */
 695         num_entries++;
 696
 697         /* Each record consists of several neighbour entries. */
 698         for (i = 0; i < num_entries; i++) {
 699                 int ent_index;
 700
 701                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 702                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 703                                                        ent_index);
 704         }
 705
 706 }
 707
 708 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 709                                               char *rauhtd_pl, int rec_index)
 710 {
 711         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 712         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 713                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 714                                                        rec_index);
 715                 break;
 716         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 717                 WARN_ON_ONCE(1);
 718                 break;
 719         }
 720 }
 721
 722 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 723 {
 724         u8 num_rec, last_rec_index, num_entries;
 725
 726         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 727         last_rec_index = num_rec - 1;
 728
 729         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 730                 return false;
 731         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 732             MLXSW_REG_RAUHTD_TYPE_IPV6)
 733                 return true;
 734
 735         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 736                                                                 last_rec_index);
 737         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 738                 return true;
 739         return false;
 740 }
 741
 742 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 743 {
 744         char *rauhtd_pl;
 745         u8 num_rec;
 746         int i, err;
 747
 748         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 749         if (!rauhtd_pl)
 750                 return -ENOMEM;
 751
 752         /* Make sure the neighbour's netdev isn't removed in the
 753          * process.
 754          */
 755         rtnl_lock();
 756         do {
 757                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 758                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 759                                       rauhtd_pl);
 760                 if (err) {
 761                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 762                         break;
 763                 }
 764                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 765                 for (i = 0; i < num_rec; i++)
 766                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 767                                                           i);
 768         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 769         rtnl_unlock();
 770
 771         kfree(rauhtd_pl);
 772         return err;
 773 }
 774
 775 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 776 {
 777         struct mlxsw_sp_neigh_entry *neigh_entry;
 778
 779         /* Take RTNL mutex here to prevent lists from changes */
 780         rtnl_lock();
 781         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 782                             nexthop_neighs_list_node)
 783                 /* If this neigh have nexthops, make the kernel think this neigh
 784                  * is active regardless of the traffic.
 785                  */
 786                 neigh_event_send(neigh_entry->key.n, NULL);
 787         rtnl_unlock();
 788 }
 789
 790 static void
 791 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 792 {
 793         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 794
 795         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 796                                msecs_to_jiffies(interval));
 797 }
 798
 799 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 800 {
 801         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 802                                                  router.neighs_update.dw.work);
 803         int err;
 804
 805         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 806         if (err)
 807                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 808
 809         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 810
 811         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 812 }
 813
 814 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 815 {
 816         struct mlxsw_sp_neigh_entry *neigh_entry;
 817         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 818                                                  router.nexthop_probe_dw.work);
 819
 820         /* Iterate over nexthop neighbours, find those who are unresolved and
 821          * send arp on them. This solves the chicken-egg problem when
 822          * the nexthop wouldn't get offloaded until the neighbor is resolved
 823          * but it wouldn't get resolved ever in case traffic is flowing in HW
 824          * using different nexthop.
 825          *
 826          * Take RTNL mutex here to prevent lists from changes.
 827          */
 828         rtnl_lock();
 829         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 830                             nexthop_neighs_list_node)
 831                 if (!neigh_entry->connected)
 832                         neigh_event_send(neigh_entry->key.n, NULL);
 833         rtnl_unlock();
 834
 835         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 836                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 837 }
 838
 839 static void
 840 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 841                               struct mlxsw_sp_neigh_entry *neigh_entry,
 842                               bool removing);
 843
 844 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
 845 {
 846         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
 847                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
 848 }
 849
 850 static void
 851 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
 852                                 struct mlxsw_sp_neigh_entry *neigh_entry,
 853                                 enum mlxsw_reg_rauht_op op)
 854 {
 855         struct neighbour *n = neigh_entry->key.n;
 856         u32 dip = ntohl(*((__be32 *) n->primary_key));
 857         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 858
 859         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
 860                               dip);
 861         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
 862 }
 863
 864 static void
 865 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 866                             struct mlxsw_sp_neigh_entry *neigh_entry,
 867                             bool adding)
 868 {
 869         if (!adding && !neigh_entry->connected)
 870                 return;
 871         neigh_entry->connected = adding;
 872         if (neigh_entry->key.n->tbl == &arp_tbl)
 873                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
 874                                                 mlxsw_sp_rauht_op(adding));
 875         else
 876                 WARN_ON_ONCE(1);
 877 }
 878
 879 struct mlxsw_sp_neigh_event_work {
 880         struct work_struct work;
 881         struct mlxsw_sp *mlxsw_sp;
 882         struct neighbour *n;
 883 };
 884
 885 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
 886 {
 887         struct mlxsw_sp_neigh_event_work *neigh_work =
 888                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
 889         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
 890         struct mlxsw_sp_neigh_entry *neigh_entry;
 891         struct neighbour *n = neigh_work->n;
 892         unsigned char ha[ETH_ALEN];
 893         bool entry_connected;
 894         u8 nud_state, dead;
 895
 896         /* If these parameters are changed after we release the lock,
 897          * then we are guaranteed to receive another event letting us
 898          * know about it.
 899          */
 900         read_lock_bh(&n->lock);
 901         memcpy(ha, n->ha, ETH_ALEN);
 902         nud_state = n->nud_state;
 903         dead = n->dead;
 904         read_unlock_bh(&n->lock);
 905
 906         rtnl_lock();
 907         entry_connected = nud_state & NUD_VALID && !dead;
 908         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 909         if (!entry_connected && !neigh_entry)
 910                 goto out;
 911         if (!neigh_entry) {
 912                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
 913                 if (IS_ERR(neigh_entry))
 914                         goto out;
 915         }
 916
 917         memcpy(neigh_entry->ha, ha, ETH_ALEN);
 918         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
 919         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
 920
 921         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
 922                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
 923
 924 out:
 925         rtnl_unlock();
 926         neigh_release(n);
 927         kfree(neigh_work);
 928 }
 929
 930 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 931                                    unsigned long event, void *ptr)
 932 {
 933         struct mlxsw_sp_neigh_event_work *neigh_work;
 934         struct mlxsw_sp_port *mlxsw_sp_port;
 935         struct mlxsw_sp *mlxsw_sp;
 936         unsigned long interval;
 937         struct neigh_parms *p;
 938         struct neighbour *n;
 939
 940         switch (event) {
 941         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
 942                 p = ptr;
 943
 944                 /* We don't care about changes in the default table. */
 945                 if (!p->dev || p->tbl != &arp_tbl)
 946                         return NOTIFY_DONE;
 947
 948                 /* We are in atomic context and can't take RTNL mutex,
 949                  * so use RCU variant to walk the device chain.
 950                  */
 951                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
 952                 if (!mlxsw_sp_port)
 953                         return NOTIFY_DONE;
 954
 955                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 956                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
 957                 mlxsw_sp->router.neighs_update.interval = interval;
 958
 959                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 960                 break;
 961         case NETEVENT_NEIGH_UPDATE:
 962                 n = ptr;
 963
 964                 if (n->tbl != &arp_tbl)
 965                         return NOTIFY_DONE;
 966
 967                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
 968                 if (!mlxsw_sp_port)
 969                         return NOTIFY_DONE;
 970
 971                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
 972                 if (!neigh_work) {
 973                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
 974                         return NOTIFY_BAD;
 975                 }
 976
 977                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
 978                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 979                 neigh_work->n = n;
 980
 981                 /* Take a reference to ensure the neighbour won't be
 982                  * destructed until we drop the reference in delayed
 983                  * work.
 984                  */
 985                 neigh_clone(n);
 986                 mlxsw_core_schedule_work(&neigh_work->work);
 987                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 988                 break;
 989         }
 990
 991         return NOTIFY_DONE;
 992 }
 993
 994 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
 995 {
 996         int err;
 997
 998         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
 999                               &mlxsw_sp_neigh_ht_params);
1000         if (err)
1001                 return err;
1002
1003         /* Initialize the polling interval according to the default
1004          * table.
1005          */
1006         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1007
1008         /* Create the delayed works for the activity_update */
1009         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1010                           mlxsw_sp_router_neighs_update_work);
1011         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1012                           mlxsw_sp_router_probe_unresolved_nexthops);
1013         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1014         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1015         return 0;
1016 }
1017
1018 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1019 {
1020         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1021         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1022         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1023 }
1024
1025 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1026                                     const struct mlxsw_sp_rif *r)
1027 {
1028         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1029
1030         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1031                              r->rif, r->addr);
1032         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1033 }
1034
1035 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1036                                          struct mlxsw_sp_rif *r)
1037 {
1038         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1039
1040         mlxsw_sp_neigh_rif_flush(mlxsw_sp, r);
1041         list_for_each_entry_safe(neigh_entry, tmp, &r->neigh_list,
1042                                  rif_list_node)
1043                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1044 }
1045
1046 struct mlxsw_sp_nexthop_key {
1047         struct fib_nh *fib_nh;
1048 };
1049
1050 struct mlxsw_sp_nexthop {
1051         struct list_head neigh_list_node; /* member of neigh entry list */
1052         struct list_head rif_list_node;
1053         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1054                                                 * this belongs to
1055                                                 */
1056         struct rhash_head ht_node;
1057         struct mlxsw_sp_nexthop_key key;
1058         struct mlxsw_sp_rif *r;
1059         u8 should_offload:1, /* set indicates this neigh is connected and
1060                               * should be put to KVD linear area of this group.
1061                               */
1062            offloaded:1, /* set in case the neigh is actually put into
1063                          * KVD linear area of this group.
1064                          */
1065            update:1; /* set indicates that MAC of this neigh should be
1066                       * updated in HW
1067                       */
1068         struct mlxsw_sp_neigh_entry *neigh_entry;
1069 };
1070
1071 struct mlxsw_sp_nexthop_group_key {
1072         struct fib_info *fi;
1073 };
1074
1075 struct mlxsw_sp_nexthop_group {
1076         struct rhash_head ht_node;
1077         struct list_head fib_list; /* list of fib entries that use this group */
1078         struct mlxsw_sp_nexthop_group_key key;
1079         u8 adj_index_valid:1,
1080            gateway:1; /* routes using the group use a gateway */
1081         u32 adj_index;
1082         u16 ecmp_size;
1083         u16 count;
1084         struct mlxsw_sp_nexthop nexthops[0];
1085 #define nh_rif  nexthops[0].r
1086 };
1087
1088 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1089         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1090         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1091         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1092 };
1093
1094 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1095                                          struct mlxsw_sp_nexthop_group *nh_grp)
1096 {
1097         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1098                                       &nh_grp->ht_node,
1099                                       mlxsw_sp_nexthop_group_ht_params);
1100 }
1101
1102 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1103                                           struct mlxsw_sp_nexthop_group *nh_grp)
1104 {
1105         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1106                                &nh_grp->ht_node,
1107                                mlxsw_sp_nexthop_group_ht_params);
1108 }
1109
1110 static struct mlxsw_sp_nexthop_group *
1111 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1112                               struct mlxsw_sp_nexthop_group_key key)
1113 {
1114         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1115                                       mlxsw_sp_nexthop_group_ht_params);
1116 }
1117
1118 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1119         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1120         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1121         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1122 };
1123
1124 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1125                                    struct mlxsw_sp_nexthop *nh)
1126 {
1127         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1128                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1129 }
1130
1131 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1132                                     struct mlxsw_sp_nexthop *nh)
1133 {
1134         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1135                                mlxsw_sp_nexthop_ht_params);
1136 }
1137
1138 static struct mlxsw_sp_nexthop *
1139 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1140                         struct mlxsw_sp_nexthop_key key)
1141 {
1142         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1143                                       mlxsw_sp_nexthop_ht_params);
1144 }
1145
1146 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1147                                              const struct mlxsw_sp_fib *fib,
1148                                              u32 adj_index, u16 ecmp_size,
1149                                              u32 new_adj_index,
1150                                              u16 new_ecmp_size)
1151 {
1152         char raleu_pl[MLXSW_REG_RALEU_LEN];
1153
1154         mlxsw_reg_raleu_pack(raleu_pl,
1155                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1156                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1157                              new_ecmp_size);
1158         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1159 }
1160
1161 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1162                                           struct mlxsw_sp_nexthop_group *nh_grp,
1163                                           u32 old_adj_index, u16 old_ecmp_size)
1164 {
1165         struct mlxsw_sp_fib_entry *fib_entry;
1166         struct mlxsw_sp_fib *fib = NULL;
1167         int err;
1168
1169         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1170                 if (fib == fib_entry->fib_node->fib)
1171                         continue;
1172                 fib = fib_entry->fib_node->fib;
1173                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1174                                                         old_adj_index,
1175                                                         old_ecmp_size,
1176                                                         nh_grp->adj_index,
1177                                                         nh_grp->ecmp_size);
1178                 if (err)
1179                         return err;
1180         }
1181         return 0;
1182 }
1183
1184 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1185                                        struct mlxsw_sp_nexthop *nh)
1186 {
1187         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1188         char ratr_pl[MLXSW_REG_RATR_LEN];
1189
1190         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1191                             true, adj_index, neigh_entry->rif);
1192         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1193         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1194 }
1195
1196 static int
1197 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1198                                   struct mlxsw_sp_nexthop_group *nh_grp,
1199                                   bool reallocate)
1200 {
1201         u32 adj_index = nh_grp->adj_index; /* base */
1202         struct mlxsw_sp_nexthop *nh;
1203         int i;
1204         int err;
1205
1206         for (i = 0; i < nh_grp->count; i++) {
1207                 nh = &nh_grp->nexthops[i];
1208
1209                 if (!nh->should_offload) {
1210                         nh->offloaded = 0;
1211                         continue;
1212                 }
1213
1214                 if (nh->update || reallocate) {
1215                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1216                                                           adj_index, nh);
1217                         if (err)
1218                                 return err;
1219                         nh->update = 0;
1220                         nh->offloaded = 1;
1221                 }
1222                 adj_index++;
1223         }
1224         return 0;
1225 }
1226
1227 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1228                                      struct mlxsw_sp_fib_entry *fib_entry);
1229
1230 static int
1231 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1232                                     struct mlxsw_sp_nexthop_group *nh_grp)
1233 {
1234         struct mlxsw_sp_fib_entry *fib_entry;
1235         int err;
1236
1237         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1238                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1239                 if (err)
1240                         return err;
1241         }
1242         return 0;
1243 }
1244
1245 static void
1246 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1247                                struct mlxsw_sp_nexthop_group *nh_grp)
1248 {
1249         struct mlxsw_sp_nexthop *nh;
1250         bool offload_change = false;
1251         u32 adj_index;
1252         u16 ecmp_size = 0;
1253         bool old_adj_index_valid;
1254         u32 old_adj_index;
1255         u16 old_ecmp_size;
1256         int ret;
1257         int i;
1258         int err;
1259
1260         if (!nh_grp->gateway) {
1261                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1262                 return;
1263         }
1264
1265         for (i = 0; i < nh_grp->count; i++) {
1266                 nh = &nh_grp->nexthops[i];
1267
1268                 if (nh->should_offload ^ nh->offloaded) {
1269                         offload_change = true;
1270                         if (nh->should_offload)
1271                                 nh->update = 1;
1272                 }
1273                 if (nh->should_offload)
1274                         ecmp_size++;
1275         }
1276         if (!offload_change) {
1277                 /* Nothing was added or removed, so no need to reallocate. Just
1278                  * update MAC on existing adjacency indexes.
1279                  */
1280                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1281                                                         false);
1282                 if (err) {
1283                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1284                         goto set_trap;
1285                 }
1286                 return;
1287         }
1288         if (!ecmp_size)
1289                 /* No neigh of this group is connected so we just set
1290                  * the trap and let everthing flow through kernel.
1291                  */
1292                 goto set_trap;
1293
1294         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1295         if (ret < 0) {
1296                 /* We ran out of KVD linear space, just set the
1297                  * trap and let everything flow through kernel.
1298                  */
1299                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1300                 goto set_trap;
1301         }
1302         adj_index = ret;
1303         old_adj_index_valid = nh_grp->adj_index_valid;
1304         old_adj_index = nh_grp->adj_index;
1305         old_ecmp_size = nh_grp->ecmp_size;
1306         nh_grp->adj_index_valid = 1;
1307         nh_grp->adj_index = adj_index;
1308         nh_grp->ecmp_size = ecmp_size;
1309         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1310         if (err) {
1311                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1312                 goto set_trap;
1313         }
1314
1315         if (!old_adj_index_valid) {
1316                 /* The trap was set for fib entries, so we have to call
1317                  * fib entry update to unset it and use adjacency index.
1318                  */
1319                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1320                 if (err) {
1321                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1322                         goto set_trap;
1323                 }
1324                 return;
1325         }
1326
1327         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1328                                              old_adj_index, old_ecmp_size);
1329         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1330         if (err) {
1331                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1332                 goto set_trap;
1333         }
1334         return;
1335
1336 set_trap:
1337         old_adj_index_valid = nh_grp->adj_index_valid;
1338         nh_grp->adj_index_valid = 0;
1339         for (i = 0; i < nh_grp->count; i++) {
1340                 nh = &nh_grp->nexthops[i];
1341                 nh->offloaded = 0;
1342         }
1343         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1344         if (err)
1345                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1346         if (old_adj_index_valid)
1347                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1348 }
1349
1350 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1351                                             bool removing)
1352 {
1353         if (!removing && !nh->should_offload)
1354                 nh->should_offload = 1;
1355         else if (removing && nh->offloaded)
1356                 nh->should_offload = 0;
1357         nh->update = 1;
1358 }
1359
1360 static void
1361 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1362                               struct mlxsw_sp_neigh_entry *neigh_entry,
1363                               bool removing)
1364 {
1365         struct mlxsw_sp_nexthop *nh;
1366
1367         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1368                             neigh_list_node) {
1369                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1370                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1371         }
1372 }
1373
1374 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1375                                       struct mlxsw_sp_rif *r)
1376 {
1377         if (nh->r)
1378                 return;
1379
1380         nh->r = r;
1381         list_add(&nh->rif_list_node, &r->nexthop_list);
1382 }
1383
1384 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1385 {
1386         if (!nh->r)
1387                 return;
1388
1389         list_del(&nh->rif_list_node);
1390         nh->r = NULL;
1391 }
1392
1393 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1394                                        struct mlxsw_sp_nexthop *nh)
1395 {
1396         struct mlxsw_sp_neigh_entry *neigh_entry;
1397         struct fib_nh *fib_nh = nh->key.fib_nh;
1398         struct neighbour *n;
1399         u8 nud_state, dead;
1400         int err;
1401
1402         if (!nh->nh_grp->gateway || nh->neigh_entry)
1403                 return 0;
1404
1405         /* Take a reference of neigh here ensuring that neigh would
1406          * not be detructed before the nexthop entry is finished.
1407          * The reference is taken either in neigh_lookup() or
1408          * in neigh_create() in case n is not found.
1409          */
1410         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1411         if (!n) {
1412                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1413                 if (IS_ERR(n))
1414                         return PTR_ERR(n);
1415                 neigh_event_send(n, NULL);
1416         }
1417         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1418         if (!neigh_entry) {
1419                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1420                 if (IS_ERR(neigh_entry)) {
1421                         err = -EINVAL;
1422                         goto err_neigh_entry_create;
1423                 }
1424         }
1425
1426         /* If that is the first nexthop connected to that neigh, add to
1427          * nexthop_neighs_list
1428          */
1429         if (list_empty(&neigh_entry->nexthop_list))
1430                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1431                               &mlxsw_sp->router.nexthop_neighs_list);
1432
1433         nh->neigh_entry = neigh_entry;
1434         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1435         read_lock_bh(&n->lock);
1436         nud_state = n->nud_state;
1437         dead = n->dead;
1438         read_unlock_bh(&n->lock);
1439         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1440
1441         return 0;
1442
1443 err_neigh_entry_create:
1444         neigh_release(n);
1445         return err;
1446 }
1447
1448 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1449                                         struct mlxsw_sp_nexthop *nh)
1450 {
1451         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1452         struct neighbour *n;
1453
1454         if (!neigh_entry)
1455                 return;
1456         n = neigh_entry->key.n;
1457
1458         __mlxsw_sp_nexthop_neigh_update(nh, true);
1459         list_del(&nh->neigh_list_node);
1460         nh->neigh_entry = NULL;
1461
1462         /* If that is the last nexthop connected to that neigh, remove from
1463          * nexthop_neighs_list
1464          */
1465         if (list_empty(&neigh_entry->nexthop_list))
1466                 list_del(&neigh_entry->nexthop_neighs_list_node);
1467
1468         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1469                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1470
1471         neigh_release(n);
1472 }
1473
1474 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1475                                  struct mlxsw_sp_nexthop_group *nh_grp,
1476                                  struct mlxsw_sp_nexthop *nh,
1477                                  struct fib_nh *fib_nh)
1478 {
1479         struct net_device *dev = fib_nh->nh_dev;
1480         struct in_device *in_dev;
1481         struct mlxsw_sp_rif *r;
1482         int err;
1483
1484         nh->nh_grp = nh_grp;
1485         nh->key.fib_nh = fib_nh;
1486         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1487         if (err)
1488                 return err;
1489
1490         if (!dev)
1491                 return 0;
1492
1493         in_dev = __in_dev_get_rtnl(dev);
1494         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1495             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1496                 return 0;
1497
1498         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1499         if (!r)
1500                 return 0;
1501         mlxsw_sp_nexthop_rif_init(nh, r);
1502
1503         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1504         if (err)
1505                 goto err_nexthop_neigh_init;
1506
1507         return 0;
1508
1509 err_nexthop_neigh_init:
1510         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1511         return err;
1512 }
1513
1514 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1515                                   struct mlxsw_sp_nexthop *nh)
1516 {
1517         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1518         mlxsw_sp_nexthop_rif_fini(nh);
1519         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1520 }
1521
1522 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1523                                    unsigned long event, struct fib_nh *fib_nh)
1524 {
1525         struct mlxsw_sp_nexthop_key key;
1526         struct mlxsw_sp_nexthop *nh;
1527         struct mlxsw_sp_rif *r;
1528
1529         if (mlxsw_sp->router.aborted)
1530                 return;
1531
1532         key.fib_nh = fib_nh;
1533         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1534         if (WARN_ON_ONCE(!nh))
1535                 return;
1536
1537         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1538         if (!r)
1539                 return;
1540
1541         switch (event) {
1542         case FIB_EVENT_NH_ADD:
1543                 mlxsw_sp_nexthop_rif_init(nh, r);
1544                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1545                 break;
1546         case FIB_EVENT_NH_DEL:
1547                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1548                 mlxsw_sp_nexthop_rif_fini(nh);
1549                 break;
1550         }
1551
1552         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1553 }
1554
1555 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1556                                            struct mlxsw_sp_rif *r)
1557 {
1558         struct mlxsw_sp_nexthop *nh, *tmp;
1559
1560         list_for_each_entry_safe(nh, tmp, &r->nexthop_list, rif_list_node) {
1561                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1562                 mlxsw_sp_nexthop_rif_fini(nh);
1563                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1564         }
1565 }
1566
1567 static struct mlxsw_sp_nexthop_group *
1568 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1569 {
1570         struct mlxsw_sp_nexthop_group *nh_grp;
1571         struct mlxsw_sp_nexthop *nh;
1572         struct fib_nh *fib_nh;
1573         size_t alloc_size;
1574         int i;
1575         int err;
1576
1577         alloc_size = sizeof(*nh_grp) +
1578                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1579         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1580         if (!nh_grp)
1581                 return ERR_PTR(-ENOMEM);
1582         INIT_LIST_HEAD(&nh_grp->fib_list);
1583         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1584         nh_grp->count = fi->fib_nhs;
1585         nh_grp->key.fi = fi;
1586         for (i = 0; i < nh_grp->count; i++) {
1587                 nh = &nh_grp->nexthops[i];
1588                 fib_nh = &fi->fib_nh[i];
1589                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1590                 if (err)
1591                         goto err_nexthop_init;
1592         }
1593         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1594         if (err)
1595                 goto err_nexthop_group_insert;
1596         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1597         return nh_grp;
1598
1599 err_nexthop_group_insert:
1600 err_nexthop_init:
1601         for (i--; i >= 0; i--) {
1602                 nh = &nh_grp->nexthops[i];
1603                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1604         }
1605         kfree(nh_grp);
1606         return ERR_PTR(err);
1607 }
1608
1609 static void
1610 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1611                                struct mlxsw_sp_nexthop_group *nh_grp)
1612 {
1613         struct mlxsw_sp_nexthop *nh;
1614         int i;
1615
1616         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1617         for (i = 0; i < nh_grp->count; i++) {
1618                 nh = &nh_grp->nexthops[i];
1619                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1620         }
1621         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1622         WARN_ON_ONCE(nh_grp->adj_index_valid);
1623         kfree(nh_grp);
1624 }
1625
1626 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1627                                       struct mlxsw_sp_fib_entry *fib_entry,
1628                                       struct fib_info *fi)
1629 {
1630         struct mlxsw_sp_nexthop_group_key key;
1631         struct mlxsw_sp_nexthop_group *nh_grp;
1632
1633         key.fi = fi;
1634         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1635         if (!nh_grp) {
1636                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1637                 if (IS_ERR(nh_grp))
1638                         return PTR_ERR(nh_grp);
1639         }
1640         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1641         fib_entry->nh_group = nh_grp;
1642         return 0;
1643 }
1644
1645 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1646                                        struct mlxsw_sp_fib_entry *fib_entry)
1647 {
1648         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1649
1650         list_del(&fib_entry->nexthop_group_node);
1651         if (!list_empty(&nh_grp->fib_list))
1652                 return;
1653         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1654 }
1655
1656 static bool
1657 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1658 {
1659         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1660
1661         if (fib_entry->params.tos)
1662                 return false;
1663
1664         switch (fib_entry->type) {
1665         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1666                 return !!nh_group->adj_index_valid;
1667         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1668                 return !!nh_group->nh_rif;
1669         default:
1670                 return false;
1671         }
1672 }
1673
1674 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1675 {
1676         fib_entry->offloaded = true;
1677
1678         switch (fib_entry->fib_node->fib->proto) {
1679         case MLXSW_SP_L3_PROTO_IPV4:
1680                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1681                 break;
1682         case MLXSW_SP_L3_PROTO_IPV6:
1683                 WARN_ON_ONCE(1);
1684         }
1685 }
1686
1687 static void
1688 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1689 {
1690         switch (fib_entry->fib_node->fib->proto) {
1691         case MLXSW_SP_L3_PROTO_IPV4:
1692                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1693                 break;
1694         case MLXSW_SP_L3_PROTO_IPV6:
1695                 WARN_ON_ONCE(1);
1696         }
1697
1698         fib_entry->offloaded = false;
1699 }
1700
1701 static void
1702 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1703                                    enum mlxsw_reg_ralue_op op, int err)
1704 {
1705         switch (op) {
1706         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1707                 if (!fib_entry->offloaded)
1708                         return;
1709                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1710         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1711                 if (err)
1712                         return;
1713                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1714                     !fib_entry->offloaded)
1715                         mlxsw_sp_fib_entry_offload_set(fib_entry);
1716                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1717                          fib_entry->offloaded)
1718                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
1719                 return;
1720         default:
1721                 return;
1722         }
1723 }
1724
1725 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1726                                          struct mlxsw_sp_fib_entry *fib_entry,
1727                                          enum mlxsw_reg_ralue_op op)
1728 {
1729         char ralue_pl[MLXSW_REG_RALUE_LEN];
1730         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1731         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1732         enum mlxsw_reg_ralue_trap_action trap_action;
1733         u16 trap_id = 0;
1734         u32 adjacency_index = 0;
1735         u16 ecmp_size = 0;
1736
1737         /* In case the nexthop group adjacency index is valid, use it
1738          * with provided ECMP size. Otherwise, setup trap and pass
1739          * traffic to kernel.
1740          */
1741         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1742                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1743                 adjacency_index = fib_entry->nh_group->adj_index;
1744                 ecmp_size = fib_entry->nh_group->ecmp_size;
1745         } else {
1746                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1747                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1748         }
1749
1750         mlxsw_reg_ralue_pack4(ralue_pl,
1751                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1752                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1753                               *p_dip);
1754         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1755                                         adjacency_index, ecmp_size);
1756         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1757 }
1758
1759 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1760                                         struct mlxsw_sp_fib_entry *fib_entry,
1761                                         enum mlxsw_reg_ralue_op op)
1762 {
1763         struct mlxsw_sp_rif *r = fib_entry->nh_group->nh_rif;
1764         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1765         enum mlxsw_reg_ralue_trap_action trap_action;
1766         char ralue_pl[MLXSW_REG_RALUE_LEN];
1767         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1768         u16 trap_id = 0;
1769         u16 rif = 0;
1770
1771         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1772                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1773                 rif = r->rif;
1774         } else {
1775                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1776                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1777         }
1778
1779         mlxsw_reg_ralue_pack4(ralue_pl,
1780                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1781                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1782                               *p_dip);
1783         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, rif);
1784         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1785 }
1786
1787 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1788                                        struct mlxsw_sp_fib_entry *fib_entry,
1789                                        enum mlxsw_reg_ralue_op op)
1790 {
1791         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1792         char ralue_pl[MLXSW_REG_RALUE_LEN];
1793         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1794
1795         mlxsw_reg_ralue_pack4(ralue_pl,
1796                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1797                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1798                               *p_dip);
1799         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1800         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1801 }
1802
1803 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1804                                   struct mlxsw_sp_fib_entry *fib_entry,
1805                                   enum mlxsw_reg_ralue_op op)
1806 {
1807         switch (fib_entry->type) {
1808         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1809                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1810         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1811                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1812         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1813                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1814         }
1815         return -EINVAL;
1816 }
1817
1818 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1819                                  struct mlxsw_sp_fib_entry *fib_entry,
1820                                  enum mlxsw_reg_ralue_op op)
1821 {
1822         int err = -EINVAL;
1823
1824         switch (fib_entry->fib_node->fib->proto) {
1825         case MLXSW_SP_L3_PROTO_IPV4:
1826                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1827                 break;
1828         case MLXSW_SP_L3_PROTO_IPV6:
1829                 return err;
1830         }
1831         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1832         return err;
1833 }
1834
1835 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1836                                      struct mlxsw_sp_fib_entry *fib_entry)
1837 {
1838         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1839                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1840 }
1841
1842 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1843                                   struct mlxsw_sp_fib_entry *fib_entry)
1844 {
1845         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1846                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1847 }
1848
1849 static int
1850 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1851                              const struct fib_entry_notifier_info *fen_info,
1852                              struct mlxsw_sp_fib_entry *fib_entry)
1853 {
1854         struct fib_info *fi = fen_info->fi;
1855
1856         switch (fen_info->type) {
1857         case RTN_BROADCAST: /* fall through */
1858         case RTN_LOCAL:
1859                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1860                 return 0;
1861         case RTN_UNREACHABLE: /* fall through */
1862         case RTN_BLACKHOLE: /* fall through */
1863         case RTN_PROHIBIT:
1864                 /* Packets hitting these routes need to be trapped, but
1865                  * can do so with a lower priority than packets directed
1866                  * at the host, so use action type local instead of trap.
1867                  */
1868                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1869                 return 0;
1870         case RTN_UNICAST:
1871                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1872                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1873                 else
1874                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1875                 return 0;
1876         default:
1877                 return -EINVAL;
1878         }
1879 }
1880
1881 static struct mlxsw_sp_fib_entry *
1882 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1883                            struct mlxsw_sp_fib_node *fib_node,
1884                            const struct fib_entry_notifier_info *fen_info)
1885 {
1886         struct mlxsw_sp_fib_entry *fib_entry;
1887         int err;
1888
1889         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1890         if (!fib_entry) {
1891                 err = -ENOMEM;
1892                 goto err_fib_entry_alloc;
1893         }
1894
1895         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1896         if (err)
1897                 goto err_fib4_entry_type_set;
1898
1899         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1900         if (err)
1901                 goto err_nexthop_group_get;
1902
1903         fib_entry->params.prio = fen_info->fi->fib_priority;
1904         fib_entry->params.tb_id = fen_info->tb_id;
1905         fib_entry->params.type = fen_info->type;
1906         fib_entry->params.tos = fen_info->tos;
1907
1908         fib_entry->fib_node = fib_node;
1909
1910         return fib_entry;
1911
1912 err_nexthop_group_get:
1913 err_fib4_entry_type_set:
1914         kfree(fib_entry);
1915 err_fib_entry_alloc:
1916         return ERR_PTR(err);
1917 }
1918
1919 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1920                                         struct mlxsw_sp_fib_entry *fib_entry)
1921 {
1922         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1923         kfree(fib_entry);
1924 }
1925
1926 static struct mlxsw_sp_fib_node *
1927 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1928                        const struct fib_entry_notifier_info *fen_info);
1929
1930 static struct mlxsw_sp_fib_entry *
1931 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1932                            const struct fib_entry_notifier_info *fen_info)
1933 {
1934         struct mlxsw_sp_fib_entry *fib_entry;
1935         struct mlxsw_sp_fib_node *fib_node;
1936
1937         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1938         if (IS_ERR(fib_node))
1939                 return NULL;
1940
1941         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1942                 if (fib_entry->params.tb_id == fen_info->tb_id &&
1943                     fib_entry->params.tos == fen_info->tos &&
1944                     fib_entry->params.type == fen_info->type &&
1945                     fib_entry->nh_group->key.fi == fen_info->fi) {
1946                         return fib_entry;
1947                 }
1948         }
1949
1950         return NULL;
1951 }
1952
1953 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1954         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1955         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1956         .key_len = sizeof(struct mlxsw_sp_fib_key),
1957         .automatic_shrinking = true,
1958 };
1959
1960 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1961                                     struct mlxsw_sp_fib_node *fib_node)
1962 {
1963         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1964                                       mlxsw_sp_fib_ht_params);
1965 }
1966
1967 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1968                                      struct mlxsw_sp_fib_node *fib_node)
1969 {
1970         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1971                                mlxsw_sp_fib_ht_params);
1972 }
1973
1974 static struct mlxsw_sp_fib_node *
1975 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1976                          size_t addr_len, unsigned char prefix_len)
1977 {
1978         struct mlxsw_sp_fib_key key;
1979
1980         memset(&key, 0, sizeof(key));
1981         memcpy(key.addr, addr, addr_len);
1982         key.prefix_len = prefix_len;
1983         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
1984 }
1985
1986 static struct mlxsw_sp_fib_node *
1987 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
1988                          size_t addr_len, unsigned char prefix_len)
1989 {
1990         struct mlxsw_sp_fib_node *fib_node;
1991
1992         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
1993         if (!fib_node)
1994                 return NULL;
1995
1996         INIT_LIST_HEAD(&fib_node->entry_list);
1997         list_add(&fib_node->list, &fib->node_list);
1998         memcpy(fib_node->key.addr, addr, addr_len);
1999         fib_node->key.prefix_len = prefix_len;
2000
2001         return fib_node;
2002 }
2003
2004 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2005 {
2006         list_del(&fib_node->list);
2007         WARN_ON(!list_empty(&fib_node->entry_list));
2008         kfree(fib_node);
2009 }
2010
2011 static bool
2012 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2013                                  const struct mlxsw_sp_fib_entry *fib_entry)
2014 {
2015         return list_first_entry(&fib_node->entry_list,
2016                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2017 }
2018
2019 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2020 {
2021         unsigned char prefix_len = fib_node->key.prefix_len;
2022         struct mlxsw_sp_fib *fib = fib_node->fib;
2023
2024         if (fib->prefix_ref_count[prefix_len]++ == 0)
2025                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2026 }
2027
2028 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2029 {
2030         unsigned char prefix_len = fib_node->key.prefix_len;
2031         struct mlxsw_sp_fib *fib = fib_node->fib;
2032
2033         if (--fib->prefix_ref_count[prefix_len] == 0)
2034                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2035 }
2036
2037 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2038                                   struct mlxsw_sp_fib_node *fib_node,
2039                                   struct mlxsw_sp_fib *fib)
2040 {
2041         struct mlxsw_sp_prefix_usage req_prefix_usage;
2042         struct mlxsw_sp_lpm_tree *lpm_tree;
2043         int err;
2044
2045         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2046         if (err)
2047                 return err;
2048         fib_node->fib = fib;
2049
2050         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2051         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2052
2053         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2054                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2055                                                  &req_prefix_usage);
2056                 if (err)
2057                         goto err_tree_check;
2058         } else {
2059                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2060                                                  fib->proto);
2061                 if (IS_ERR(lpm_tree))
2062                         return PTR_ERR(lpm_tree);
2063                 fib->lpm_tree = lpm_tree;
2064                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2065                 if (err)
2066                         goto err_tree_bind;
2067         }
2068
2069         mlxsw_sp_fib_node_prefix_inc(fib_node);
2070
2071         return 0;
2072
2073 err_tree_bind:
2074         fib->lpm_tree = NULL;
2075         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2076 err_tree_check:
2077         fib_node->fib = NULL;
2078         mlxsw_sp_fib_node_remove(fib, fib_node);
2079         return err;
2080 }
2081
2082 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2083                                    struct mlxsw_sp_fib_node *fib_node)
2084 {
2085         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2086         struct mlxsw_sp_fib *fib = fib_node->fib;
2087
2088         mlxsw_sp_fib_node_prefix_dec(fib_node);
2089
2090         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2091                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2092                 fib->lpm_tree = NULL;
2093                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2094         } else {
2095                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2096         }
2097
2098         fib_node->fib = NULL;
2099         mlxsw_sp_fib_node_remove(fib, fib_node);
2100 }
2101
2102 static struct mlxsw_sp_fib_node *
2103 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2104                        const struct fib_entry_notifier_info *fen_info)
2105 {
2106         struct mlxsw_sp_fib_node *fib_node;
2107         struct mlxsw_sp_fib *fib;
2108         struct mlxsw_sp_vr *vr;
2109         int err;
2110
2111         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2112         if (IS_ERR(vr))
2113                 return ERR_CAST(vr);
2114         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2115
2116         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2117                                             sizeof(fen_info->dst),
2118                                             fen_info->dst_len);
2119         if (fib_node)
2120                 return fib_node;
2121
2122         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2123                                             sizeof(fen_info->dst),
2124                                             fen_info->dst_len);
2125         if (!fib_node) {
2126                 err = -ENOMEM;
2127                 goto err_fib_node_create;
2128         }
2129
2130         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2131         if (err)
2132                 goto err_fib_node_init;
2133
2134         return fib_node;
2135
2136 err_fib_node_init:
2137         mlxsw_sp_fib_node_destroy(fib_node);
2138 err_fib_node_create:
2139         mlxsw_sp_vr_put(vr);
2140         return ERR_PTR(err);
2141 }
2142
2143 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2144                                    struct mlxsw_sp_fib_node *fib_node)
2145 {
2146         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2147
2148         if (!list_empty(&fib_node->entry_list))
2149                 return;
2150         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2151         mlxsw_sp_fib_node_destroy(fib_node);
2152         mlxsw_sp_vr_put(vr);
2153 }
2154
2155 static struct mlxsw_sp_fib_entry *
2156 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2157                               const struct mlxsw_sp_fib_entry_params *params)
2158 {
2159         struct mlxsw_sp_fib_entry *fib_entry;
2160
2161         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2162                 if (fib_entry->params.tb_id > params->tb_id)
2163                         continue;
2164                 if (fib_entry->params.tb_id != params->tb_id)
2165                         break;
2166                 if (fib_entry->params.tos > params->tos)
2167                         continue;
2168                 if (fib_entry->params.prio >= params->prio ||
2169                     fib_entry->params.tos < params->tos)
2170                         return fib_entry;
2171         }
2172
2173         return NULL;
2174 }
2175
2176 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2177                                           struct mlxsw_sp_fib_entry *new_entry)
2178 {
2179         struct mlxsw_sp_fib_node *fib_node;
2180
2181         if (WARN_ON(!fib_entry))
2182                 return -EINVAL;
2183
2184         fib_node = fib_entry->fib_node;
2185         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2186                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2187                     fib_entry->params.tos != new_entry->params.tos ||
2188                     fib_entry->params.prio != new_entry->params.prio)
2189                         break;
2190         }
2191
2192         list_add_tail(&new_entry->list, &fib_entry->list);
2193         return 0;
2194 }
2195
2196 static int
2197 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2198                                struct mlxsw_sp_fib_entry *new_entry,
2199                                bool replace, bool append)
2200 {
2201         struct mlxsw_sp_fib_entry *fib_entry;
2202
2203         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2204
2205         if (append)
2206                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2207         if (replace && WARN_ON(!fib_entry))
2208                 return -EINVAL;
2209
2210         /* Insert new entry before replaced one, so that we can later
2211          * remove the second.
2212          */
2213         if (fib_entry) {
2214                 list_add_tail(&new_entry->list, &fib_entry->list);
2215         } else {
2216                 struct mlxsw_sp_fib_entry *last;
2217
2218                 list_for_each_entry(last, &fib_node->entry_list, list) {
2219                         if (new_entry->params.tb_id > last->params.tb_id)
2220                                 break;
2221                         fib_entry = last;
2222                 }
2223
2224                 if (fib_entry)
2225                         list_add(&new_entry->list, &fib_entry->list);
2226                 else
2227                         list_add(&new_entry->list, &fib_node->entry_list);
2228         }
2229
2230         return 0;
2231 }
2232
2233 static void
2234 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2235 {
2236         list_del(&fib_entry->list);
2237 }
2238
2239 static int
2240 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2241                              const struct mlxsw_sp_fib_node *fib_node,
2242                              struct mlxsw_sp_fib_entry *fib_entry)
2243 {
2244         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2245                 return 0;
2246
2247         /* To prevent packet loss, overwrite the previously offloaded
2248          * entry.
2249          */
2250         if (!list_is_singular(&fib_node->entry_list)) {
2251                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2252                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2253
2254                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2255         }
2256
2257         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2258 }
2259
2260 static void
2261 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2262                              const struct mlxsw_sp_fib_node *fib_node,
2263                              struct mlxsw_sp_fib_entry *fib_entry)
2264 {
2265         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2266                 return;
2267
2268         /* Promote the next entry by overwriting the deleted entry */
2269         if (!list_is_singular(&fib_node->entry_list)) {
2270                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2271                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2272
2273                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2274                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2275                 return;
2276         }
2277
2278         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2279 }
2280
2281 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2282                                          struct mlxsw_sp_fib_entry *fib_entry,
2283                                          bool replace, bool append)
2284 {
2285         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2286         int err;
2287
2288         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2289                                              append);
2290         if (err)
2291                 return err;
2292
2293         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2294         if (err)
2295                 goto err_fib4_node_entry_add;
2296
2297         return 0;
2298
2299 err_fib4_node_entry_add:
2300         mlxsw_sp_fib4_node_list_remove(fib_entry);
2301         return err;
2302 }
2303
2304 static void
2305 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2306                                 struct mlxsw_sp_fib_entry *fib_entry)
2307 {
2308         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2309
2310         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2311         mlxsw_sp_fib4_node_list_remove(fib_entry);
2312 }
2313
2314 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2315                                         struct mlxsw_sp_fib_entry *fib_entry,
2316                                         bool replace)
2317 {
2318         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2319         struct mlxsw_sp_fib_entry *replaced;
2320
2321         if (!replace)
2322                 return;
2323
2324         /* We inserted the new entry before replaced one */
2325         replaced = list_next_entry(fib_entry, list);
2326
2327         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2328         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2329         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2330 }
2331
2332 static int
2333 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2334                          const struct fib_entry_notifier_info *fen_info,
2335                          bool replace, bool append)
2336 {
2337         struct mlxsw_sp_fib_entry *fib_entry;
2338         struct mlxsw_sp_fib_node *fib_node;
2339         int err;
2340
2341         if (mlxsw_sp->router.aborted)
2342                 return 0;
2343
2344         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2345         if (IS_ERR(fib_node)) {
2346                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2347                 return PTR_ERR(fib_node);
2348         }
2349
2350         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2351         if (IS_ERR(fib_entry)) {
2352                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2353                 err = PTR_ERR(fib_entry);
2354                 goto err_fib4_entry_create;
2355         }
2356
2357         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2358                                             append);
2359         if (err) {
2360                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2361                 goto err_fib4_node_entry_link;
2362         }
2363
2364         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2365
2366         return 0;
2367
2368 err_fib4_node_entry_link:
2369         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2370 err_fib4_entry_create:
2371         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2372         return err;
2373 }
2374
2375 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2376                                      struct fib_entry_notifier_info *fen_info)
2377 {
2378         struct mlxsw_sp_fib_entry *fib_entry;
2379         struct mlxsw_sp_fib_node *fib_node;
2380
2381         if (mlxsw_sp->router.aborted)
2382                 return;
2383
2384         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2385         if (WARN_ON(!fib_entry))
2386                 return;
2387         fib_node = fib_entry->fib_node;
2388
2389         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2390         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2391         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2392 }
2393
2394 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2395 {
2396         char ralta_pl[MLXSW_REG_RALTA_LEN];
2397         char ralst_pl[MLXSW_REG_RALST_LEN];
2398         int i, err;
2399
2400         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2401                              MLXSW_SP_LPM_TREE_MIN);
2402         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2403         if (err)
2404                 return err;
2405
2406         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2407         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2408         if (err)
2409                 return err;
2410
2411         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2412                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2413                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2414                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2415
2416                 if (!mlxsw_sp_vr_is_used(vr))
2417                         continue;
2418
2419                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2420                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2421                                      MLXSW_SP_LPM_TREE_MIN);
2422                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2423                                       raltb_pl);
2424                 if (err)
2425                         return err;
2426
2427                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2428                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2429                                       0);
2430                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2431                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2432                                       ralue_pl);
2433                 if (err)
2434                         return err;
2435         }
2436
2437         return 0;
2438 }
2439
2440 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2441                                      struct mlxsw_sp_fib_node *fib_node)
2442 {
2443         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2444
2445         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2446                 bool do_break = &tmp->list == &fib_node->entry_list;
2447
2448                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2449                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2450                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2451                 /* Break when entry list is empty and node was freed.
2452                  * Otherwise, we'll access freed memory in the next
2453                  * iteration.
2454                  */
2455                 if (do_break)
2456                         break;
2457         }
2458 }
2459
2460 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2461                                     struct mlxsw_sp_fib_node *fib_node)
2462 {
2463         switch (fib_node->fib->proto) {
2464         case MLXSW_SP_L3_PROTO_IPV4:
2465                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2466                 break;
2467         case MLXSW_SP_L3_PROTO_IPV6:
2468                 WARN_ON_ONCE(1);
2469                 break;
2470         }
2471 }
2472
2473 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2474                                   struct mlxsw_sp_vr *vr,
2475                                   enum mlxsw_sp_l3proto proto)
2476 {
2477         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2478         struct mlxsw_sp_fib_node *fib_node, *tmp;
2479
2480         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2481                 bool do_break = &tmp->list == &fib->node_list;
2482
2483                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2484                 if (do_break)
2485                         break;
2486         }
2487 }
2488
2489 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2490 {
2491         int i;
2492
2493         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2494                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2495
2496                 if (!mlxsw_sp_vr_is_used(vr))
2497                         continue;
2498                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2499         }
2500 }
2501
2502 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2503 {
2504         int err;
2505
2506         if (mlxsw_sp->router.aborted)
2507                 return;
2508         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2509         mlxsw_sp_router_fib_flush(mlxsw_sp);
2510         mlxsw_sp->router.aborted = true;
2511         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2512         if (err)
2513                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2514 }
2515
2516 struct mlxsw_sp_fib_event_work {
2517         struct work_struct work;
2518         union {
2519                 struct fib_entry_notifier_info fen_info;
2520                 struct fib_rule_notifier_info fr_info;
2521                 struct fib_nh_notifier_info fnh_info;
2522         };
2523         struct mlxsw_sp *mlxsw_sp;
2524         unsigned long event;
2525 };
2526
2527 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2528 {
2529         struct mlxsw_sp_fib_event_work *fib_work =
2530                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2531         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2532         struct fib_rule *rule;
2533         bool replace, append;
2534         int err;
2535
2536         /* Protect internal structures from changes */
2537         rtnl_lock();
2538         switch (fib_work->event) {
2539         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2540         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2541         case FIB_EVENT_ENTRY_ADD:
2542                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2543                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2544                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2545                                                replace, append);
2546                 if (err)
2547                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2548                 fib_info_put(fib_work->fen_info.fi);
2549                 break;
2550         case FIB_EVENT_ENTRY_DEL:
2551                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2552                 fib_info_put(fib_work->fen_info.fi);
2553                 break;
2554         case FIB_EVENT_RULE_ADD: /* fall through */
2555         case FIB_EVENT_RULE_DEL:
2556                 rule = fib_work->fr_info.rule;
2557                 if (!fib4_rule_default(rule))
2558                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2559                 fib_rule_put(rule);
2560                 break;
2561         case FIB_EVENT_NH_ADD: /* fall through */
2562         case FIB_EVENT_NH_DEL:
2563                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2564                                        fib_work->fnh_info.fib_nh);
2565                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2566                 break;
2567         }
2568         rtnl_unlock();
2569         kfree(fib_work);
2570 }
2571
2572 /* Called with rcu_read_lock() */
2573 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2574                                      unsigned long event, void *ptr)
2575 {
2576         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2577         struct mlxsw_sp_fib_event_work *fib_work;
2578         struct fib_notifier_info *info = ptr;
2579
2580         if (!net_eq(info->net, &init_net))
2581                 return NOTIFY_DONE;
2582
2583         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2584         if (WARN_ON(!fib_work))
2585                 return NOTIFY_BAD;
2586
2587         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2588         fib_work->mlxsw_sp = mlxsw_sp;
2589         fib_work->event = event;
2590
2591         switch (event) {
2592         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2593         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2594         case FIB_EVENT_ENTRY_ADD: /* fall through */
2595         case FIB_EVENT_ENTRY_DEL:
2596                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2597                 /* Take referece on fib_info to prevent it from being
2598                  * freed while work is queued. Release it afterwards.
2599                  */
2600                 fib_info_hold(fib_work->fen_info.fi);
2601                 break;
2602         case FIB_EVENT_RULE_ADD: /* fall through */
2603         case FIB_EVENT_RULE_DEL:
2604                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2605                 fib_rule_get(fib_work->fr_info.rule);
2606                 break;
2607         case FIB_EVENT_NH_ADD: /* fall through */
2608         case FIB_EVENT_NH_DEL:
2609                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2610                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2611                 break;
2612         }
2613
2614         mlxsw_core_schedule_work(&fib_work->work);
2615
2616         return NOTIFY_DONE;
2617 }
2618
2619 static struct mlxsw_sp_rif *
2620 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2621                          const struct net_device *dev)
2622 {
2623         int i;
2624
2625         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2626                 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
2627                         return mlxsw_sp->rifs[i];
2628
2629         return NULL;
2630 }
2631
2632 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2633 {
2634         char ritr_pl[MLXSW_REG_RITR_LEN];
2635         int err;
2636
2637         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2638         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2639         if (WARN_ON_ONCE(err))
2640                 return err;
2641
2642         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2643         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2644 }
2645
2646 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2647                                           struct mlxsw_sp_rif *r)
2648 {
2649         mlxsw_sp_router_rif_disable(mlxsw_sp, r->rif);
2650         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, r);
2651         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, r);
2652 }
2653
2654 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
2655                                        const struct in_device *in_dev,
2656                                        unsigned long event)
2657 {
2658         switch (event) {
2659         case NETDEV_UP:
2660                 if (!r)
2661                         return true;
2662                 return false;
2663         case NETDEV_DOWN:
2664                 if (r && !in_dev->ifa_list && !netif_is_l3_slave(r->dev))
2665                         return true;
2666                 /* It is possible we already removed the RIF ourselves
2667                  * if it was assigned to a netdev that is now a bridge
2668                  * or LAG slave.
2669                  */
2670                 return false;
2671         }
2672
2673         return false;
2674 }
2675
2676 #define MLXSW_SP_INVALID_RIF 0xffff
2677 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
2678 {
2679         int i;
2680
2681         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2682                 if (!mlxsw_sp->rifs[i])
2683                         return i;
2684
2685         return MLXSW_SP_INVALID_RIF;
2686 }
2687
2688 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
2689                                            bool *p_lagged, u16 *p_system_port)
2690 {
2691         u8 local_port = mlxsw_sp_vport->local_port;
2692
2693         *p_lagged = mlxsw_sp_vport->lagged;
2694         *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
2695 }
2696
2697 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
2698                                     u16 vr_id, struct net_device *l3_dev,
2699                                     u16 rif, bool create)
2700 {
2701         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2702         bool lagged = mlxsw_sp_vport->lagged;
2703         char ritr_pl[MLXSW_REG_RITR_LEN];
2704         u16 system_port;
2705
2706         mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif, vr_id,
2707                             l3_dev->mtu, l3_dev->dev_addr);
2708
2709         mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
2710         mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
2711                                   mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
2712
2713         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2714 }
2715
2716 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
2717
2718 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif)
2719 {
2720         return MLXSW_SP_RFID_BASE + rif;
2721 }
2722
2723 static struct mlxsw_sp_fid *
2724 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
2725 {
2726         struct mlxsw_sp_fid *f;
2727
2728         f = kzalloc(sizeof(*f), GFP_KERNEL);
2729         if (!f)
2730                 return NULL;
2731
2732         f->leave = mlxsw_sp_vport_rif_sp_leave;
2733         f->ref_count = 0;
2734         f->dev = l3_dev;
2735         f->fid = fid;
2736
2737         return f;
2738 }
2739
2740 static struct mlxsw_sp_rif *
2741 mlxsw_sp_rif_alloc(u16 rif, u16 vr_id, struct net_device *l3_dev,
2742                    struct mlxsw_sp_fid *f)
2743 {
2744         struct mlxsw_sp_rif *r;
2745
2746         r = kzalloc(sizeof(*r), GFP_KERNEL);
2747         if (!r)
2748                 return NULL;
2749
2750         INIT_LIST_HEAD(&r->nexthop_list);
2751         INIT_LIST_HEAD(&r->neigh_list);
2752         ether_addr_copy(r->addr, l3_dev->dev_addr);
2753         r->mtu = l3_dev->mtu;
2754         r->vr_id = vr_id;
2755         r->dev = l3_dev;
2756         r->rif = rif;
2757         r->f = f;
2758
2759         return r;
2760 }
2761
2762 static struct mlxsw_sp_rif *
2763 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
2764                              struct net_device *l3_dev)
2765 {
2766         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2767         u32 tb_id = l3mdev_fib_table(l3_dev);
2768         struct mlxsw_sp_vr *vr;
2769         struct mlxsw_sp_fid *f;
2770         struct mlxsw_sp_rif *r;
2771         u16 fid, rif;
2772         int err;
2773
2774         rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
2775         if (rif == MLXSW_SP_INVALID_RIF)
2776                 return ERR_PTR(-ERANGE);
2777
2778         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
2779         if (IS_ERR(vr))
2780                 return ERR_CAST(vr);
2781
2782         err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif,
2783                                        true);
2784         if (err)
2785                 goto err_vport_rif_sp_op;
2786
2787         fid = mlxsw_sp_rif_sp_to_fid(rif);
2788         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
2789         if (err)
2790                 goto err_rif_fdb_op;
2791
2792         f = mlxsw_sp_rfid_alloc(fid, l3_dev);
2793         if (!f) {
2794                 err = -ENOMEM;
2795                 goto err_rfid_alloc;
2796         }
2797
2798         r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
2799         if (!r) {
2800                 err = -ENOMEM;
2801                 goto err_rif_alloc;
2802         }
2803
2804         f->r = r;
2805         mlxsw_sp->rifs[rif] = r;
2806         vr->rif_count++;
2807
2808         return r;
2809
2810 err_rif_alloc:
2811         kfree(f);
2812 err_rfid_alloc:
2813         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2814 err_rif_fdb_op:
2815         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
2816 err_vport_rif_sp_op:
2817         mlxsw_sp_vr_put(vr);
2818         return ERR_PTR(err);
2819 }
2820
2821 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
2822                                           struct mlxsw_sp_rif *r)
2823 {
2824         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2825         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
2826         struct net_device *l3_dev = r->dev;
2827         struct mlxsw_sp_fid *f = r->f;
2828         u16 fid = f->fid;
2829         u16 rif = r->rif;
2830
2831         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
2832
2833         vr->rif_count--;
2834         mlxsw_sp->rifs[rif] = NULL;
2835         f->r = NULL;
2836
2837         kfree(r);
2838
2839         kfree(f);
2840
2841         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2842
2843         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
2844
2845         mlxsw_sp_vr_put(vr);
2846 }
2847
2848 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
2849                                       struct net_device *l3_dev)
2850 {
2851         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2852         struct mlxsw_sp_rif *r;
2853
2854         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
2855         if (!r) {
2856                 r = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
2857                 if (IS_ERR(r))
2858                         return PTR_ERR(r);
2859         }
2860
2861         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, r->f);
2862         r->f->ref_count++;
2863
2864         netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", r->f->fid);
2865
2866         return 0;
2867 }
2868
2869 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
2870 {
2871         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
2872
2873         netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
2874
2875         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
2876         if (--f->ref_count == 0)
2877                 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->r);
2878 }
2879
2880 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
2881                                          struct net_device *port_dev,
2882                                          unsigned long event, u16 vid)
2883 {
2884         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
2885         struct mlxsw_sp_port *mlxsw_sp_vport;
2886
2887         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
2888         if (WARN_ON(!mlxsw_sp_vport))
2889                 return -EINVAL;
2890
2891         switch (event) {
2892         case NETDEV_UP:
2893                 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
2894         case NETDEV_DOWN:
2895                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
2896                 break;
2897         }
2898
2899         return 0;
2900 }
2901
2902 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
2903                                         unsigned long event)
2904 {
2905         if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
2906                 return 0;
2907
2908         return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
2909 }
2910
2911 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
2912                                          struct net_device *lag_dev,
2913                                          unsigned long event, u16 vid)
2914 {
2915         struct net_device *port_dev;
2916         struct list_head *iter;
2917         int err;
2918
2919         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
2920                 if (mlxsw_sp_port_dev_check(port_dev)) {
2921                         err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
2922                                                             event, vid);
2923                         if (err)
2924                                 return err;
2925                 }
2926         }
2927
2928         return 0;
2929 }
2930
2931 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
2932                                        unsigned long event)
2933 {
2934         if (netif_is_bridge_port(lag_dev))
2935                 return 0;
2936
2937         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
2938 }
2939
2940 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
2941                                                     struct net_device *l3_dev)
2942 {
2943         u16 fid;
2944
2945         if (is_vlan_dev(l3_dev))
2946                 fid = vlan_dev_vlan_id(l3_dev);
2947         else if (mlxsw_sp->master_bridge.dev == l3_dev)
2948                 fid = 1;
2949         else
2950                 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
2951
2952         return mlxsw_sp_fid_find(mlxsw_sp, fid);
2953 }
2954
2955 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
2956 {
2957         return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
2958                MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
2959 }
2960
2961 static u16 mlxsw_sp_flood_table_index_get(u16 fid)
2962 {
2963         return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
2964 }
2965
2966 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
2967                                           bool set)
2968 {
2969         enum mlxsw_flood_table_type table_type;
2970         char *sftr_pl;
2971         u16 index;
2972         int err;
2973
2974         sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
2975         if (!sftr_pl)
2976                 return -ENOMEM;
2977
2978         table_type = mlxsw_sp_flood_table_type_get(fid);
2979         index = mlxsw_sp_flood_table_index_get(fid);
2980         mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
2981                             1, MLXSW_PORT_ROUTER_PORT, set);
2982         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
2983
2984         kfree(sftr_pl);
2985         return err;
2986 }
2987
2988 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
2989 {
2990         if (mlxsw_sp_fid_is_vfid(fid))
2991                 return MLXSW_REG_RITR_FID_IF;
2992         else
2993                 return MLXSW_REG_RITR_VLAN_IF;
2994 }
2995
2996 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
2997                                   struct net_device *l3_dev,
2998                                   u16 fid, u16 rif,
2999                                   bool create)
3000 {
3001         enum mlxsw_reg_ritr_if_type rif_type;
3002         char ritr_pl[MLXSW_REG_RITR_LEN];
3003
3004         rif_type = mlxsw_sp_rif_type_get(fid);
3005         mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
3006                             l3_dev->dev_addr);
3007         mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
3008
3009         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3010 }
3011
3012 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
3013                                       struct net_device *l3_dev,
3014                                       struct mlxsw_sp_fid *f)
3015 {
3016         u32 tb_id = l3mdev_fib_table(l3_dev);
3017         struct mlxsw_sp_vr *vr;
3018         struct mlxsw_sp_rif *r;
3019         u16 rif;
3020         int err;
3021
3022         rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
3023         if (rif == MLXSW_SP_INVALID_RIF)
3024                 return -ERANGE;
3025
3026         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3027         if (IS_ERR(vr))
3028                 return PTR_ERR(vr);
3029
3030         err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
3031         if (err)
3032                 goto err_port_flood_set;
3033
3034         err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif,
3035                                      true);
3036         if (err)
3037                 goto err_rif_bridge_op;
3038
3039         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
3040         if (err)
3041                 goto err_rif_fdb_op;
3042
3043         r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
3044         if (!r) {
3045                 err = -ENOMEM;
3046                 goto err_rif_alloc;
3047         }
3048
3049         f->r = r;
3050         mlxsw_sp->rifs[rif] = r;
3051         vr->rif_count++;
3052
3053         netdev_dbg(l3_dev, "RIF=%d created\n", rif);
3054
3055         return 0;
3056
3057 err_rif_alloc:
3058         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3059 err_rif_fdb_op:
3060         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
3061 err_rif_bridge_op:
3062         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3063 err_port_flood_set:
3064         mlxsw_sp_vr_put(vr);
3065         return err;
3066 }
3067
3068 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
3069                                  struct mlxsw_sp_rif *r)
3070 {
3071         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
3072         struct net_device *l3_dev = r->dev;
3073         struct mlxsw_sp_fid *f = r->f;
3074         u16 rif = r->rif;
3075
3076         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
3077
3078         vr->rif_count--;
3079         mlxsw_sp->rifs[rif] = NULL;
3080         f->r = NULL;
3081
3082         kfree(r);
3083
3084         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3085
3086         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
3087
3088         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3089
3090         mlxsw_sp_vr_put(vr);
3091
3092         netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif);
3093 }
3094
3095 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3096                                           struct net_device *br_dev,
3097                                           unsigned long event)
3098 {
3099         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3100         struct mlxsw_sp_fid *f;
3101
3102         /* FID can either be an actual FID if the L3 device is the
3103          * VLAN-aware bridge or a VLAN device on top. Otherwise, the
3104          * L3 device is a VLAN-unaware bridge and we get a vFID.
3105          */
3106         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3107         if (WARN_ON(!f))
3108                 return -EINVAL;
3109
3110         switch (event) {
3111         case NETDEV_UP:
3112                 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3113         case NETDEV_DOWN:
3114                 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->r);
3115                 break;
3116         }
3117
3118         return 0;
3119 }
3120
3121 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3122                                         unsigned long event)
3123 {
3124         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3125         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
3126         u16 vid = vlan_dev_vlan_id(vlan_dev);
3127
3128         if (mlxsw_sp_port_dev_check(real_dev))
3129                 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
3130                                                      vid);
3131         else if (netif_is_lag_master(real_dev))
3132                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3133                                                      vid);
3134         else if (netif_is_bridge_master(real_dev) &&
3135                  mlxsw_sp->master_bridge.dev == real_dev)
3136                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
3137                                                       event);
3138
3139         return 0;
3140 }
3141
3142 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3143                             unsigned long event, void *ptr)
3144 {
3145         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3146         struct net_device *dev = ifa->ifa_dev->dev;
3147         struct mlxsw_sp *mlxsw_sp;
3148         struct mlxsw_sp_rif *r;
3149         int err = 0;
3150
3151         mlxsw_sp = mlxsw_sp_lower_get(dev);
3152         if (!mlxsw_sp)
3153                 goto out;
3154
3155         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3156         if (!mlxsw_sp_rif_should_config(r, ifa->ifa_dev, event))
3157                 goto out;
3158
3159         if (mlxsw_sp_port_dev_check(dev))
3160                 err = mlxsw_sp_inetaddr_port_event(dev, event);
3161         else if (netif_is_lag_master(dev))
3162                 err = mlxsw_sp_inetaddr_lag_event(dev, event);
3163         else if (netif_is_bridge_master(dev))
3164                 err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
3165         else if (is_vlan_dev(dev))
3166                 err = mlxsw_sp_inetaddr_vlan_event(dev, event);
3167
3168 out:
3169         return notifier_from_errno(err);
3170 }
3171
3172 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif,
3173                              const char *mac, int mtu)
3174 {
3175         char ritr_pl[MLXSW_REG_RITR_LEN];
3176         int err;
3177
3178         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
3179         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3180         if (err)
3181                 return err;
3182
3183         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3184         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3185         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3186         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3187 }
3188
3189 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3190 {
3191         struct mlxsw_sp *mlxsw_sp;
3192         struct mlxsw_sp_rif *r;
3193         int err;
3194
3195         mlxsw_sp = mlxsw_sp_lower_get(dev);
3196         if (!mlxsw_sp)
3197                 return 0;
3198
3199         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3200         if (!r)
3201                 return 0;
3202
3203         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, false);
3204         if (err)
3205                 return err;
3206
3207         err = mlxsw_sp_rif_edit(mlxsw_sp, r->rif, dev->dev_addr, dev->mtu);
3208         if (err)
3209                 goto err_rif_edit;
3210
3211         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, r->f->fid, true);
3212         if (err)
3213                 goto err_rif_fdb_op;
3214
3215         ether_addr_copy(r->addr, dev->dev_addr);
3216         r->mtu = dev->mtu;
3217
3218         netdev_dbg(dev, "Updated RIF=%d\n", r->rif);
3219
3220         return 0;
3221
3222 err_rif_fdb_op:
3223         mlxsw_sp_rif_edit(mlxsw_sp, r->rif, r->addr, r->mtu);
3224 err_rif_edit:
3225         mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, true);
3226         return err;
3227 }
3228
3229 int mlxsw_sp_vport_vrf_join(struct mlxsw_sp_port *mlxsw_sp_vport)
3230 {
3231         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
3232         struct net_device *dev = mlxsw_sp_vport->dev;
3233
3234         /* In case vPort already has a RIF, then we need to drop it.
3235          * A new one will be created using the VRF's VR.
3236          */
3237         if (f && f->r)
3238                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3239
3240         return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, dev);
3241 }
3242
3243 void mlxsw_sp_vport_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
3244 {
3245         mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3246 }
3247
3248 int mlxsw_sp_port_vrf_join(struct mlxsw_sp_port *mlxsw_sp_port)
3249 {
3250         struct mlxsw_sp_port *mlxsw_sp_vport;
3251
3252         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3253         if (WARN_ON(!mlxsw_sp_vport))
3254                 return -EINVAL;
3255
3256         return mlxsw_sp_vport_vrf_join(mlxsw_sp_vport);
3257 }
3258
3259 void mlxsw_sp_port_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_port)
3260 {
3261         struct mlxsw_sp_port *mlxsw_sp_vport;
3262
3263         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3264         if (WARN_ON(!mlxsw_sp_vport))
3265                 return;
3266
3267         mlxsw_sp_vport_vrf_leave(mlxsw_sp_vport);
3268 }
3269
3270 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3271 {
3272         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
3273
3274         /* Flush pending FIB notifications and then flush the device's
3275          * table before requesting another dump. The FIB notification
3276          * block is unregistered, so no need to take RTNL.
3277          */
3278         mlxsw_core_flush_owq();
3279         mlxsw_sp_router_fib_flush(mlxsw_sp);
3280 }
3281
3282 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3283 {
3284         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3285         u64 max_rifs;
3286         int err;
3287
3288         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3289                 return -EIO;
3290
3291         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3292         mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
3293                                  GFP_KERNEL);
3294         if (!mlxsw_sp->rifs)
3295                 return -ENOMEM;
3296
3297         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3298         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3299         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3300         if (err)
3301                 goto err_rgcr_fail;
3302
3303         return 0;
3304
3305 err_rgcr_fail:
3306         kfree(mlxsw_sp->rifs);
3307         return err;
3308 }
3309
3310 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3311 {
3312         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3313         int i;
3314
3315         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3316         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3317
3318         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3319                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
3320
3321         kfree(mlxsw_sp->rifs);
3322 }
3323
3324 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3325 {
3326         int err;
3327
3328         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
3329         err = __mlxsw_sp_router_init(mlxsw_sp);
3330         if (err)
3331                 return err;
3332
3333         err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
3334                               &mlxsw_sp_nexthop_ht_params);
3335         if (err)
3336                 goto err_nexthop_ht_init;
3337
3338         err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
3339                               &mlxsw_sp_nexthop_group_ht_params);
3340         if (err)
3341                 goto err_nexthop_group_ht_init;
3342
3343         mlxsw_sp_lpm_init(mlxsw_sp);
3344         err = mlxsw_sp_vrs_init(mlxsw_sp);
3345         if (err)
3346                 goto err_vrs_init;
3347
3348         err = mlxsw_sp_neigh_init(mlxsw_sp);
3349         if (err)
3350                 goto err_neigh_init;
3351
3352         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3353         err = register_fib_notifier(&mlxsw_sp->fib_nb,
3354                                     mlxsw_sp_router_fib_dump_flush);
3355         if (err)
3356                 goto err_register_fib_notifier;
3357
3358         return 0;
3359
3360 err_register_fib_notifier:
3361         mlxsw_sp_neigh_fini(mlxsw_sp);
3362 err_neigh_init:
3363         mlxsw_sp_vrs_fini(mlxsw_sp);
3364 err_vrs_init:
3365         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3366 err_nexthop_group_ht_init:
3367         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3368 err_nexthop_ht_init:
3369         __mlxsw_sp_router_fini(mlxsw_sp);
3370         return err;
3371 }
3372
3373 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3374 {
3375         unregister_fib_notifier(&mlxsw_sp->fib_nb);
3376         mlxsw_sp_neigh_fini(mlxsw_sp);
3377         mlxsw_sp_vrs_fini(mlxsw_sp);
3378         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3379         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3380         __mlxsw_sp_router_fini(mlxsw_sp);
3381 }