drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <linux/if_bridge.h>
  46 #include <net/netevent.h>
  47 #include <net/neighbour.h>
  48 #include <net/arp.h>
  49 #include <net/ip_fib.h>
  50 #include <net/fib_rules.h>
  51 #include <net/l3mdev.h>
  52
  53 #include "spectrum.h"
  54 #include "core.h"
  55 #include "reg.h"
  56 #include "spectrum_cnt.h"
  57 #include "spectrum_dpipe.h"
  58 #include "spectrum_router.h"
  59
  60 struct mlxsw_sp_vr;
  61 struct mlxsw_sp_lpm_tree;
  62 struct mlxsw_sp_rif_ops;
  63
  64 struct mlxsw_sp_router {
  65         struct mlxsw_sp *mlxsw_sp;
  66         struct mlxsw_sp_rif **rifs;
  67         struct mlxsw_sp_vr *vrs;
  68         struct rhashtable neigh_ht;
  69         struct rhashtable nexthop_group_ht;
  70         struct rhashtable nexthop_ht;
  71         struct {
  72                 struct mlxsw_sp_lpm_tree *trees;
  73                 unsigned int tree_count;
  74         } lpm;
  75         struct {
  76                 struct delayed_work dw;
  77                 unsigned long interval; /* ms */
  78         } neighs_update;
  79         struct delayed_work nexthop_probe_dw;
  80 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  81         struct list_head nexthop_neighs_list;
  82         bool aborted;
  83         struct notifier_block fib_nb;
  84         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  85 };
  86
  87 struct mlxsw_sp_rif {
  88         struct list_head nexthop_list;
  89         struct list_head neigh_list;
  90         struct net_device *dev;
  91         struct mlxsw_sp_fid *fid;
  92         unsigned char addr[ETH_ALEN];
  93         int mtu;
  94         u16 rif_index;
  95         u16 vr_id;
  96         const struct mlxsw_sp_rif_ops *ops;
  97         struct mlxsw_sp *mlxsw_sp;
  98
  99         unsigned int counter_ingress;
 100         bool counter_ingress_valid;
 101         unsigned int counter_egress;
 102         bool counter_egress_valid;
 103 };
 104
 105 struct mlxsw_sp_rif_params {
 106         struct net_device *dev;
 107         union {
 108                 u16 system_port;
 109                 u16 lag_id;
 110         };
 111         u16 vid;
 112         bool lag;
 113 };
 114
 115 struct mlxsw_sp_rif_subport {
 116         struct mlxsw_sp_rif common;
 117         union {
 118                 u16 system_port;
 119                 u16 lag_id;
 120         };
 121         u16 vid;
 122         bool lag;
 123 };
 124
 125 struct mlxsw_sp_rif_ops {
 126         enum mlxsw_sp_rif_type type;
 127         size_t rif_size;
 128
 129         void (*setup)(struct mlxsw_sp_rif *rif,
 130                       const struct mlxsw_sp_rif_params *params);
 131         int (*configure)(struct mlxsw_sp_rif *rif);
 132         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 133         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 134 };
 135
 136 static unsigned int *
 137 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 138                            enum mlxsw_sp_rif_counter_dir dir)
 139 {
 140         switch (dir) {
 141         case MLXSW_SP_RIF_COUNTER_EGRESS:
 142                 return &rif->counter_egress;
 143         case MLXSW_SP_RIF_COUNTER_INGRESS:
 144                 return &rif->counter_ingress;
 145         }
 146         return NULL;
 147 }
 148
 149 static bool
 150 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 151                                enum mlxsw_sp_rif_counter_dir dir)
 152 {
 153         switch (dir) {
 154         case MLXSW_SP_RIF_COUNTER_EGRESS:
 155                 return rif->counter_egress_valid;
 156         case MLXSW_SP_RIF_COUNTER_INGRESS:
 157                 return rif->counter_ingress_valid;
 158         }
 159         return false;
 160 }
 161
 162 static void
 163 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 164                                enum mlxsw_sp_rif_counter_dir dir,
 165                                bool valid)
 166 {
 167         switch (dir) {
 168         case MLXSW_SP_RIF_COUNTER_EGRESS:
 169                 rif->counter_egress_valid = valid;
 170                 break;
 171         case MLXSW_SP_RIF_COUNTER_INGRESS:
 172                 rif->counter_ingress_valid = valid;
 173                 break;
 174         }
 175 }
 176
 177 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 178                                      unsigned int counter_index, bool enable,
 179                                      enum mlxsw_sp_rif_counter_dir dir)
 180 {
 181         char ritr_pl[MLXSW_REG_RITR_LEN];
 182         bool is_egress = false;
 183         int err;
 184
 185         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 186                 is_egress = true;
 187         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 188         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 189         if (err)
 190                 return err;
 191
 192         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 193                                     is_egress);
 194         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 195 }
 196
 197 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 198                                    struct mlxsw_sp_rif *rif,
 199                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 200 {
 201         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 202         unsigned int *p_counter_index;
 203         bool valid;
 204         int err;
 205
 206         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 207         if (!valid)
 208                 return -EINVAL;
 209
 210         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 211         if (!p_counter_index)
 212                 return -EINVAL;
 213         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 214                              MLXSW_REG_RICNT_OPCODE_NOP);
 215         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 216         if (err)
 217                 return err;
 218         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 219         return 0;
 220 }
 221
 222 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 223                                       unsigned int counter_index)
 224 {
 225         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 226
 227         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 228                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 229         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 230 }
 231
 232 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 233                                struct mlxsw_sp_rif *rif,
 234                                enum mlxsw_sp_rif_counter_dir dir)
 235 {
 236         unsigned int *p_counter_index;
 237         int err;
 238
 239         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 240         if (!p_counter_index)
 241                 return -EINVAL;
 242         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 243                                      p_counter_index);
 244         if (err)
 245                 return err;
 246
 247         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 248         if (err)
 249                 goto err_counter_clear;
 250
 251         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 252                                         *p_counter_index, true, dir);
 253         if (err)
 254                 goto err_counter_edit;
 255         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 256         return 0;
 257
 258 err_counter_edit:
 259 err_counter_clear:
 260         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 261                               *p_counter_index);
 262         return err;
 263 }
 264
 265 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 266                                struct mlxsw_sp_rif *rif,
 267                                enum mlxsw_sp_rif_counter_dir dir)
 268 {
 269         unsigned int *p_counter_index;
 270
 271         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 272                 return;
 273
 274         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 275         if (WARN_ON(!p_counter_index))
 276                 return;
 277         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 278                                   *p_counter_index, false, dir);
 279         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 280                               *p_counter_index);
 281         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 282 }
 283
 284 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 285 {
 286         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 287         struct devlink *devlink;
 288
 289         devlink = priv_to_devlink(mlxsw_sp->core);
 290         if (!devlink_dpipe_table_counter_enabled(devlink,
 291                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 292                 return;
 293         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 294 }
 295
 296 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 297 {
 298         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 299
 300         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 301 }
 302
 303 static struct mlxsw_sp_rif *
 304 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 305                          const struct net_device *dev);
 306
 307 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
 308
 309 struct mlxsw_sp_prefix_usage {
 310         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 311 };
 312
 313 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 314         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 315
 316 static bool
 317 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
 318                              struct mlxsw_sp_prefix_usage *prefix_usage2)
 319 {
 320         unsigned char prefix;
 321
 322         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
 323                 if (!test_bit(prefix, prefix_usage2->b))
 324                         return false;
 325         }
 326         return true;
 327 }
 328
 329 static bool
 330 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 331                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 332 {
 333         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 334 }
 335
 336 static bool
 337 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 338 {
 339         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 340
 341         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 342 }
 343
 344 static void
 345 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 346                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 347 {
 348         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 349 }
 350
 351 static void
 352 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 353                           unsigned char prefix_len)
 354 {
 355         set_bit(prefix_len, prefix_usage->b);
 356 }
 357
 358 static void
 359 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 360                             unsigned char prefix_len)
 361 {
 362         clear_bit(prefix_len, prefix_usage->b);
 363 }
 364
 365 struct mlxsw_sp_fib_key {
 366         unsigned char addr[sizeof(struct in6_addr)];
 367         unsigned char prefix_len;
 368 };
 369
 370 enum mlxsw_sp_fib_entry_type {
 371         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 372         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 373         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 374 };
 375
 376 struct mlxsw_sp_nexthop_group;
 377 struct mlxsw_sp_fib;
 378
 379 struct mlxsw_sp_fib_node {
 380         struct list_head entry_list;
 381         struct list_head list;
 382         struct rhash_head ht_node;
 383         struct mlxsw_sp_fib *fib;
 384         struct mlxsw_sp_fib_key key;
 385 };
 386
 387 struct mlxsw_sp_fib_entry_params {
 388         u32 tb_id;
 389         u32 prio;
 390         u8 tos;
 391         u8 type;
 392 };
 393
 394 struct mlxsw_sp_fib_entry {
 395         struct list_head list;
 396         struct mlxsw_sp_fib_node *fib_node;
 397         enum mlxsw_sp_fib_entry_type type;
 398         struct list_head nexthop_group_node;
 399         struct mlxsw_sp_nexthop_group *nh_group;
 400         struct mlxsw_sp_fib_entry_params params;
 401         bool offloaded;
 402 };
 403
 404 enum mlxsw_sp_l3proto {
 405         MLXSW_SP_L3_PROTO_IPV4,
 406         MLXSW_SP_L3_PROTO_IPV6,
 407 };
 408
 409 struct mlxsw_sp_lpm_tree {
 410         u8 id; /* tree ID */
 411         unsigned int ref_count;
 412         enum mlxsw_sp_l3proto proto;
 413         struct mlxsw_sp_prefix_usage prefix_usage;
 414 };
 415
 416 struct mlxsw_sp_fib {
 417         struct rhashtable ht;
 418         struct list_head node_list;
 419         struct mlxsw_sp_vr *vr;
 420         struct mlxsw_sp_lpm_tree *lpm_tree;
 421         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 422         struct mlxsw_sp_prefix_usage prefix_usage;
 423         enum mlxsw_sp_l3proto proto;
 424 };
 425
 426 struct mlxsw_sp_vr {
 427         u16 id; /* virtual router ID */
 428         u32 tb_id; /* kernel fib table id */
 429         unsigned int rif_count;
 430         struct mlxsw_sp_fib *fib4;
 431 };
 432
 433 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 434
 435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 436                                                 enum mlxsw_sp_l3proto proto)
 437 {
 438         struct mlxsw_sp_fib *fib;
 439         int err;
 440
 441         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 442         if (!fib)
 443                 return ERR_PTR(-ENOMEM);
 444         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 445         if (err)
 446                 goto err_rhashtable_init;
 447         INIT_LIST_HEAD(&fib->node_list);
 448         fib->proto = proto;
 449         fib->vr = vr;
 450         return fib;
 451
 452 err_rhashtable_init:
 453         kfree(fib);
 454         return ERR_PTR(err);
 455 }
 456
 457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 458 {
 459         WARN_ON(!list_empty(&fib->node_list));
 460         WARN_ON(fib->lpm_tree);
 461         rhashtable_destroy(&fib->ht);
 462         kfree(fib);
 463 }
 464
 465 static struct mlxsw_sp_lpm_tree *
 466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 467 {
 468         static struct mlxsw_sp_lpm_tree *lpm_tree;
 469         int i;
 470
 471         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 472                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 473                 if (lpm_tree->ref_count == 0)
 474                         return lpm_tree;
 475         }
 476         return NULL;
 477 }
 478
 479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 480                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 481 {
 482         char ralta_pl[MLXSW_REG_RALTA_LEN];
 483
 484         mlxsw_reg_ralta_pack(ralta_pl, true,
 485                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 486                              lpm_tree->id);
 487         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 488 }
 489
 490 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 491                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 492 {
 493         char ralta_pl[MLXSW_REG_RALTA_LEN];
 494
 495         mlxsw_reg_ralta_pack(ralta_pl, false,
 496                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 497                              lpm_tree->id);
 498         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 499 }
 500
 501 static int
 502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 503                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 504                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 505 {
 506         char ralst_pl[MLXSW_REG_RALST_LEN];
 507         u8 root_bin = 0;
 508         u8 prefix;
 509         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 510
 511         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 512                 root_bin = prefix;
 513
 514         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 515         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 516                 if (prefix == 0)
 517                         continue;
 518                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 519                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 520                 last_prefix = prefix;
 521         }
 522         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 523 }
 524
 525 static struct mlxsw_sp_lpm_tree *
 526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 527                          struct mlxsw_sp_prefix_usage *prefix_usage,
 528                          enum mlxsw_sp_l3proto proto)
 529 {
 530         struct mlxsw_sp_lpm_tree *lpm_tree;
 531         int err;
 532
 533         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 534         if (!lpm_tree)
 535                 return ERR_PTR(-EBUSY);
 536         lpm_tree->proto = proto;
 537         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 538         if (err)
 539                 return ERR_PTR(err);
 540
 541         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 542                                                 lpm_tree);
 543         if (err)
 544                 goto err_left_struct_set;
 545         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 546                sizeof(lpm_tree->prefix_usage));
 547         return lpm_tree;
 548
 549 err_left_struct_set:
 550         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 551         return ERR_PTR(err);
 552 }
 553
 554 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 555                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 556 {
 557         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 558 }
 559
 560 static struct mlxsw_sp_lpm_tree *
 561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 562                       struct mlxsw_sp_prefix_usage *prefix_usage,
 563                       enum mlxsw_sp_l3proto proto)
 564 {
 565         struct mlxsw_sp_lpm_tree *lpm_tree;
 566         int i;
 567
 568         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 569                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 570                 if (lpm_tree->ref_count != 0 &&
 571                     lpm_tree->proto == proto &&
 572                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 573                                              prefix_usage))
 574                         goto inc_ref_count;
 575         }
 576         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 577                                             proto);
 578         if (IS_ERR(lpm_tree))
 579                 return lpm_tree;
 580
 581 inc_ref_count:
 582         lpm_tree->ref_count++;
 583         return lpm_tree;
 584 }
 585
 586 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 587                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 588 {
 589         if (--lpm_tree->ref_count == 0)
 590                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 591         return 0;
 592 }
 593
 594 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 595
 596 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 597 {
 598         struct mlxsw_sp_lpm_tree *lpm_tree;
 599         u64 max_trees;
 600         int i;
 601
 602         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 603                 return -EIO;
 604
 605         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 606         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 607         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 608                                              sizeof(struct mlxsw_sp_lpm_tree),
 609                                              GFP_KERNEL);
 610         if (!mlxsw_sp->router->lpm.trees)
 611                 return -ENOMEM;
 612
 613         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 614                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 615                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 616         }
 617
 618         return 0;
 619 }
 620
 621 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 622 {
 623         kfree(mlxsw_sp->router->lpm.trees);
 624 }
 625
 626 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 627 {
 628         return !!vr->fib4;
 629 }
 630
 631 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 632 {
 633         struct mlxsw_sp_vr *vr;
 634         int i;
 635
 636         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 637                 vr = &mlxsw_sp->router->vrs[i];
 638                 if (!mlxsw_sp_vr_is_used(vr))
 639                         return vr;
 640         }
 641         return NULL;
 642 }
 643
 644 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 645                                      const struct mlxsw_sp_fib *fib)
 646 {
 647         char raltb_pl[MLXSW_REG_RALTB_LEN];
 648
 649         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 650                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 651                              fib->lpm_tree->id);
 652         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 653 }
 654
 655 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 656                                        const struct mlxsw_sp_fib *fib)
 657 {
 658         char raltb_pl[MLXSW_REG_RALTB_LEN];
 659
 660         /* Bind to tree 0 which is default */
 661         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 662                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 663         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 664 }
 665
 666 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 667 {
 668         /* For our purpose, squash main and local table into one */
 669         if (tb_id == RT_TABLE_LOCAL)
 670                 tb_id = RT_TABLE_MAIN;
 671         return tb_id;
 672 }
 673
 674 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 675                                             u32 tb_id)
 676 {
 677         struct mlxsw_sp_vr *vr;
 678         int i;
 679
 680         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 681
 682         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 683                 vr = &mlxsw_sp->router->vrs[i];
 684                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 685                         return vr;
 686         }
 687         return NULL;
 688 }
 689
 690 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 691                                             enum mlxsw_sp_l3proto proto)
 692 {
 693         switch (proto) {
 694         case MLXSW_SP_L3_PROTO_IPV4:
 695                 return vr->fib4;
 696         case MLXSW_SP_L3_PROTO_IPV6:
 697                 BUG_ON(1);
 698         }
 699         return NULL;
 700 }
 701
 702 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 703                                               u32 tb_id)
 704 {
 705         struct mlxsw_sp_vr *vr;
 706
 707         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 708         if (!vr)
 709                 return ERR_PTR(-EBUSY);
 710         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 711         if (IS_ERR(vr->fib4))
 712                 return ERR_CAST(vr->fib4);
 713         vr->tb_id = tb_id;
 714         return vr;
 715 }
 716
 717 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 718 {
 719         mlxsw_sp_fib_destroy(vr->fib4);
 720         vr->fib4 = NULL;
 721 }
 722
 723 static int
 724 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 725                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 726 {
 727         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 728         struct mlxsw_sp_lpm_tree *new_tree;
 729         int err;
 730
 731         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 732                 return 0;
 733
 734         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 735                                          fib->proto);
 736         if (IS_ERR(new_tree)) {
 737                 /* We failed to get a tree according to the required
 738                  * prefix usage. However, the current tree might be still good
 739                  * for us if our requirement is subset of the prefixes used
 740                  * in the tree.
 741                  */
 742                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 743                                                  &lpm_tree->prefix_usage))
 744                         return 0;
 745                 return PTR_ERR(new_tree);
 746         }
 747
 748         /* Prevent packet loss by overwriting existing binding */
 749         fib->lpm_tree = new_tree;
 750         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 751         if (err)
 752                 goto err_tree_bind;
 753         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 754
 755         return 0;
 756
 757 err_tree_bind:
 758         fib->lpm_tree = lpm_tree;
 759         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 760         return err;
 761 }
 762
 763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 764 {
 765         struct mlxsw_sp_vr *vr;
 766
 767         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 768         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 769         if (!vr)
 770                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 771         return vr;
 772 }
 773
 774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 775 {
 776         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 777                 mlxsw_sp_vr_destroy(vr);
 778 }
 779
 780 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 781 {
 782         struct mlxsw_sp_vr *vr;
 783         u64 max_vrs;
 784         int i;
 785
 786         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 787                 return -EIO;
 788
 789         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 790         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 791                                         GFP_KERNEL);
 792         if (!mlxsw_sp->router->vrs)
 793                 return -ENOMEM;
 794
 795         for (i = 0; i < max_vrs; i++) {
 796                 vr = &mlxsw_sp->router->vrs[i];
 797                 vr->id = i;
 798         }
 799
 800         return 0;
 801 }
 802
 803 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 804
 805 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 806 {
 807         /* At this stage we're guaranteed not to have new incoming
 808          * FIB notifications and the work queue is free from FIBs
 809          * sitting on top of mlxsw netdevs. However, we can still
 810          * have other FIBs queued. Flush the queue before flushing
 811          * the device's tables. No need for locks, as we're the only
 812          * writer.
 813          */
 814         mlxsw_core_flush_owq();
 815         mlxsw_sp_router_fib_flush(mlxsw_sp);
 816         kfree(mlxsw_sp->router->vrs);
 817 }
 818
 819 struct mlxsw_sp_neigh_key {
 820         struct neighbour *n;
 821 };
 822
 823 struct mlxsw_sp_neigh_entry {
 824         struct list_head rif_list_node;
 825         struct rhash_head ht_node;
 826         struct mlxsw_sp_neigh_key key;
 827         u16 rif;
 828         bool connected;
 829         unsigned char ha[ETH_ALEN];
 830         struct list_head nexthop_list; /* list of nexthops using
 831                                         * this neigh entry
 832                                         */
 833         struct list_head nexthop_neighs_list_node;
 834 };
 835
 836 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 837         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 838         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 839         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 840 };
 841
 842 static struct mlxsw_sp_neigh_entry *
 843 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 844                            u16 rif)
 845 {
 846         struct mlxsw_sp_neigh_entry *neigh_entry;
 847
 848         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 849         if (!neigh_entry)
 850                 return NULL;
 851
 852         neigh_entry->key.n = n;
 853         neigh_entry->rif = rif;
 854         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 855
 856         return neigh_entry;
 857 }
 858
 859 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 860 {
 861         kfree(neigh_entry);
 862 }
 863
 864 static int
 865 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 866                             struct mlxsw_sp_neigh_entry *neigh_entry)
 867 {
 868         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
 869                                       &neigh_entry->ht_node,
 870                                       mlxsw_sp_neigh_ht_params);
 871 }
 872
 873 static void
 874 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 875                             struct mlxsw_sp_neigh_entry *neigh_entry)
 876 {
 877         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
 878                                &neigh_entry->ht_node,
 879                                mlxsw_sp_neigh_ht_params);
 880 }
 881
 882 static struct mlxsw_sp_neigh_entry *
 883 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 884 {
 885         struct mlxsw_sp_neigh_entry *neigh_entry;
 886         struct mlxsw_sp_rif *rif;
 887         int err;
 888
 889         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 890         if (!rif)
 891                 return ERR_PTR(-EINVAL);
 892
 893         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
 894         if (!neigh_entry)
 895                 return ERR_PTR(-ENOMEM);
 896
 897         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 898         if (err)
 899                 goto err_neigh_entry_insert;
 900
 901         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 902
 903         return neigh_entry;
 904
 905 err_neigh_entry_insert:
 906         mlxsw_sp_neigh_entry_free(neigh_entry);
 907         return ERR_PTR(err);
 908 }
 909
 910 static void
 911 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 912                              struct mlxsw_sp_neigh_entry *neigh_entry)
 913 {
 914         list_del(&neigh_entry->rif_list_node);
 915         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 916         mlxsw_sp_neigh_entry_free(neigh_entry);
 917 }
 918
 919 static struct mlxsw_sp_neigh_entry *
 920 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 921 {
 922         struct mlxsw_sp_neigh_key key;
 923
 924         key.n = n;
 925         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
 926                                       &key, mlxsw_sp_neigh_ht_params);
 927 }
 928
 929 static void
 930 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 931 {
 932         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 933
 934         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
 935 }
 936
 937 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 938                                                    char *rauhtd_pl,
 939                                                    int ent_index)
 940 {
 941         struct net_device *dev;
 942         struct neighbour *n;
 943         __be32 dipn;
 944         u32 dip;
 945         u16 rif;
 946
 947         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 948
 949         if (!mlxsw_sp->router->rifs[rif]) {
 950                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 951                 return;
 952         }
 953
 954         dipn = htonl(dip);
 955         dev = mlxsw_sp->router->rifs[rif]->dev;
 956         n = neigh_lookup(&arp_tbl, &dipn, dev);
 957         if (!n) {
 958                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 959                            &dip);
 960                 return;
 961         }
 962
 963         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 964         neigh_event_send(n, NULL);
 965         neigh_release(n);
 966 }
 967
 968 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 969                                                    char *rauhtd_pl,
 970                                                    int rec_index)
 971 {
 972         u8 num_entries;
 973         int i;
 974
 975         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 976                                                                 rec_index);
 977         /* Hardware starts counting at 0, so add 1. */
 978         num_entries++;
 979
 980         /* Each record consists of several neighbour entries. */
 981         for (i = 0; i < num_entries; i++) {
 982                 int ent_index;
 983
 984                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 985                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 986                                                        ent_index);
 987         }
 988
 989 }
 990
 991 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 992                                               char *rauhtd_pl, int rec_index)
 993 {
 994         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 995         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 996                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 997                                                        rec_index);
 998                 break;
 999         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1000                 WARN_ON_ONCE(1);
1001                 break;
1002         }
1003 }
1004
1005 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1006 {
1007         u8 num_rec, last_rec_index, num_entries;
1008
1009         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1010         last_rec_index = num_rec - 1;
1011
1012         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1013                 return false;
1014         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1015             MLXSW_REG_RAUHTD_TYPE_IPV6)
1016                 return true;
1017
1018         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1019                                                                 last_rec_index);
1020         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1021                 return true;
1022         return false;
1023 }
1024
1025 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1026 {
1027         char *rauhtd_pl;
1028         u8 num_rec;
1029         int i, err;
1030
1031         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1032         if (!rauhtd_pl)
1033                 return -ENOMEM;
1034
1035         /* Make sure the neighbour's netdev isn't removed in the
1036          * process.
1037          */
1038         rtnl_lock();
1039         do {
1040                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
1041                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1042                                       rauhtd_pl);
1043                 if (err) {
1044                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1045                         break;
1046                 }
1047                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1048                 for (i = 0; i < num_rec; i++)
1049                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1050                                                           i);
1051         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1052         rtnl_unlock();
1053
1054         kfree(rauhtd_pl);
1055         return err;
1056 }
1057
1058 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1059 {
1060         struct mlxsw_sp_neigh_entry *neigh_entry;
1061
1062         /* Take RTNL mutex here to prevent lists from changes */
1063         rtnl_lock();
1064         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1065                             nexthop_neighs_list_node)
1066                 /* If this neigh have nexthops, make the kernel think this neigh
1067                  * is active regardless of the traffic.
1068                  */
1069                 neigh_event_send(neigh_entry->key.n, NULL);
1070         rtnl_unlock();
1071 }
1072
1073 static void
1074 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1075 {
1076         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1077
1078         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1079                                msecs_to_jiffies(interval));
1080 }
1081
1082 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1083 {
1084         struct mlxsw_sp_router *router;
1085         int err;
1086
1087         router = container_of(work, struct mlxsw_sp_router,
1088                               neighs_update.dw.work);
1089         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1090         if (err)
1091                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1092
1093         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1094
1095         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1096 }
1097
1098 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1099 {
1100         struct mlxsw_sp_neigh_entry *neigh_entry;
1101         struct mlxsw_sp_router *router;
1102
1103         router = container_of(work, struct mlxsw_sp_router,
1104                               nexthop_probe_dw.work);
1105         /* Iterate over nexthop neighbours, find those who are unresolved and
1106          * send arp on them. This solves the chicken-egg problem when
1107          * the nexthop wouldn't get offloaded until the neighbor is resolved
1108          * but it wouldn't get resolved ever in case traffic is flowing in HW
1109          * using different nexthop.
1110          *
1111          * Take RTNL mutex here to prevent lists from changes.
1112          */
1113         rtnl_lock();
1114         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1115                             nexthop_neighs_list_node)
1116                 if (!neigh_entry->connected)
1117                         neigh_event_send(neigh_entry->key.n, NULL);
1118         rtnl_unlock();
1119
1120         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1121                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1122 }
1123
1124 static void
1125 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1126                               struct mlxsw_sp_neigh_entry *neigh_entry,
1127                               bool removing);
1128
1129 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1130 {
1131         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1132                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1133 }
1134
1135 static void
1136 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1137                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1138                                 enum mlxsw_reg_rauht_op op)
1139 {
1140         struct neighbour *n = neigh_entry->key.n;
1141         u32 dip = ntohl(*((__be32 *) n->primary_key));
1142         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1143
1144         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1145                               dip);
1146         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1147 }
1148
1149 static void
1150 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1151                             struct mlxsw_sp_neigh_entry *neigh_entry,
1152                             bool adding)
1153 {
1154         if (!adding && !neigh_entry->connected)
1155                 return;
1156         neigh_entry->connected = adding;
1157         if (neigh_entry->key.n->tbl == &arp_tbl)
1158                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1159                                                 mlxsw_sp_rauht_op(adding));
1160         else
1161                 WARN_ON_ONCE(1);
1162 }
1163
1164 struct mlxsw_sp_neigh_event_work {
1165         struct work_struct work;
1166         struct mlxsw_sp *mlxsw_sp;
1167         struct neighbour *n;
1168 };
1169
1170 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1171 {
1172         struct mlxsw_sp_neigh_event_work *neigh_work =
1173                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1174         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1175         struct mlxsw_sp_neigh_entry *neigh_entry;
1176         struct neighbour *n = neigh_work->n;
1177         unsigned char ha[ETH_ALEN];
1178         bool entry_connected;
1179         u8 nud_state, dead;
1180
1181         /* If these parameters are changed after we release the lock,
1182          * then we are guaranteed to receive another event letting us
1183          * know about it.
1184          */
1185         read_lock_bh(&n->lock);
1186         memcpy(ha, n->ha, ETH_ALEN);
1187         nud_state = n->nud_state;
1188         dead = n->dead;
1189         read_unlock_bh(&n->lock);
1190
1191         rtnl_lock();
1192         entry_connected = nud_state & NUD_VALID && !dead;
1193         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1194         if (!entry_connected && !neigh_entry)
1195                 goto out;
1196         if (!neigh_entry) {
1197                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1198                 if (IS_ERR(neigh_entry))
1199                         goto out;
1200         }
1201
1202         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1203         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1204         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1205
1206         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1207                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1208
1209 out:
1210         rtnl_unlock();
1211         neigh_release(n);
1212         kfree(neigh_work);
1213 }
1214
1215 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1216                                    unsigned long event, void *ptr)
1217 {
1218         struct mlxsw_sp_neigh_event_work *neigh_work;
1219         struct mlxsw_sp_port *mlxsw_sp_port;
1220         struct mlxsw_sp *mlxsw_sp;
1221         unsigned long interval;
1222         struct neigh_parms *p;
1223         struct neighbour *n;
1224
1225         switch (event) {
1226         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1227                 p = ptr;
1228
1229                 /* We don't care about changes in the default table. */
1230                 if (!p->dev || p->tbl != &arp_tbl)
1231                         return NOTIFY_DONE;
1232
1233                 /* We are in atomic context and can't take RTNL mutex,
1234                  * so use RCU variant to walk the device chain.
1235                  */
1236                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1237                 if (!mlxsw_sp_port)
1238                         return NOTIFY_DONE;
1239
1240                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1241                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1242                 mlxsw_sp->router->neighs_update.interval = interval;
1243
1244                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1245                 break;
1246         case NETEVENT_NEIGH_UPDATE:
1247                 n = ptr;
1248
1249                 if (n->tbl != &arp_tbl)
1250                         return NOTIFY_DONE;
1251
1252                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1253                 if (!mlxsw_sp_port)
1254                         return NOTIFY_DONE;
1255
1256                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1257                 if (!neigh_work) {
1258                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1259                         return NOTIFY_BAD;
1260                 }
1261
1262                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1263                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1264                 neigh_work->n = n;
1265
1266                 /* Take a reference to ensure the neighbour won't be
1267                  * destructed until we drop the reference in delayed
1268                  * work.
1269                  */
1270                 neigh_clone(n);
1271                 mlxsw_core_schedule_work(&neigh_work->work);
1272                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1273                 break;
1274         }
1275
1276         return NOTIFY_DONE;
1277 }
1278
1279 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1280 {
1281         int err;
1282
1283         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1284                               &mlxsw_sp_neigh_ht_params);
1285         if (err)
1286                 return err;
1287
1288         /* Initialize the polling interval according to the default
1289          * table.
1290          */
1291         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1292
1293         /* Create the delayed works for the activity_update */
1294         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1295                           mlxsw_sp_router_neighs_update_work);
1296         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1297                           mlxsw_sp_router_probe_unresolved_nexthops);
1298         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1299         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1300         return 0;
1301 }
1302
1303 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1304 {
1305         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1306         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1307         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1308 }
1309
1310 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1311                                     const struct mlxsw_sp_rif *rif)
1312 {
1313         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1314
1315         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1316                              rif->rif_index, rif->addr);
1317         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1318 }
1319
1320 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1321                                          struct mlxsw_sp_rif *rif)
1322 {
1323         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1324
1325         mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1326         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1327                                  rif_list_node)
1328                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1329 }
1330
1331 struct mlxsw_sp_nexthop_key {
1332         struct fib_nh *fib_nh;
1333 };
1334
1335 struct mlxsw_sp_nexthop {
1336         struct list_head neigh_list_node; /* member of neigh entry list */
1337         struct list_head rif_list_node;
1338         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1339                                                 * this belongs to
1340                                                 */
1341         struct rhash_head ht_node;
1342         struct mlxsw_sp_nexthop_key key;
1343         struct mlxsw_sp_rif *rif;
1344         u8 should_offload:1, /* set indicates this neigh is connected and
1345                               * should be put to KVD linear area of this group.
1346                               */
1347            offloaded:1, /* set in case the neigh is actually put into
1348                          * KVD linear area of this group.
1349                          */
1350            update:1; /* set indicates that MAC of this neigh should be
1351                       * updated in HW
1352                       */
1353         struct mlxsw_sp_neigh_entry *neigh_entry;
1354 };
1355
1356 struct mlxsw_sp_nexthop_group_key {
1357         struct fib_info *fi;
1358 };
1359
1360 struct mlxsw_sp_nexthop_group {
1361         struct rhash_head ht_node;
1362         struct list_head fib_list; /* list of fib entries that use this group */
1363         struct mlxsw_sp_nexthop_group_key key;
1364         u8 adj_index_valid:1,
1365            gateway:1; /* routes using the group use a gateway */
1366         u32 adj_index;
1367         u16 ecmp_size;
1368         u16 count;
1369         struct mlxsw_sp_nexthop nexthops[0];
1370 #define nh_rif  nexthops[0].rif
1371 };
1372
1373 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1374         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1375         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1376         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1377 };
1378
1379 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1380                                          struct mlxsw_sp_nexthop_group *nh_grp)
1381 {
1382         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1383                                       &nh_grp->ht_node,
1384                                       mlxsw_sp_nexthop_group_ht_params);
1385 }
1386
1387 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1388                                           struct mlxsw_sp_nexthop_group *nh_grp)
1389 {
1390         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1391                                &nh_grp->ht_node,
1392                                mlxsw_sp_nexthop_group_ht_params);
1393 }
1394
1395 static struct mlxsw_sp_nexthop_group *
1396 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1397                               struct mlxsw_sp_nexthop_group_key key)
1398 {
1399         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
1400                                       mlxsw_sp_nexthop_group_ht_params);
1401 }
1402
1403 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1404         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1405         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1406         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1407 };
1408
1409 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1410                                    struct mlxsw_sp_nexthop *nh)
1411 {
1412         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1413                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1414 }
1415
1416 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1417                                     struct mlxsw_sp_nexthop *nh)
1418 {
1419         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1420                                mlxsw_sp_nexthop_ht_params);
1421 }
1422
1423 static struct mlxsw_sp_nexthop *
1424 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1425                         struct mlxsw_sp_nexthop_key key)
1426 {
1427         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1428                                       mlxsw_sp_nexthop_ht_params);
1429 }
1430
1431 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1432                                              const struct mlxsw_sp_fib *fib,
1433                                              u32 adj_index, u16 ecmp_size,
1434                                              u32 new_adj_index,
1435                                              u16 new_ecmp_size)
1436 {
1437         char raleu_pl[MLXSW_REG_RALEU_LEN];
1438
1439         mlxsw_reg_raleu_pack(raleu_pl,
1440                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1441                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1442                              new_ecmp_size);
1443         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1444 }
1445
1446 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1447                                           struct mlxsw_sp_nexthop_group *nh_grp,
1448                                           u32 old_adj_index, u16 old_ecmp_size)
1449 {
1450         struct mlxsw_sp_fib_entry *fib_entry;
1451         struct mlxsw_sp_fib *fib = NULL;
1452         int err;
1453
1454         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1455                 if (fib == fib_entry->fib_node->fib)
1456                         continue;
1457                 fib = fib_entry->fib_node->fib;
1458                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1459                                                         old_adj_index,
1460                                                         old_ecmp_size,
1461                                                         nh_grp->adj_index,
1462                                                         nh_grp->ecmp_size);
1463                 if (err)
1464                         return err;
1465         }
1466         return 0;
1467 }
1468
1469 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1470                                        struct mlxsw_sp_nexthop *nh)
1471 {
1472         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473         char ratr_pl[MLXSW_REG_RATR_LEN];
1474
1475         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1476                             true, adj_index, neigh_entry->rif);
1477         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1478         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1479 }
1480
1481 static int
1482 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1483                                   struct mlxsw_sp_nexthop_group *nh_grp,
1484                                   bool reallocate)
1485 {
1486         u32 adj_index = nh_grp->adj_index; /* base */
1487         struct mlxsw_sp_nexthop *nh;
1488         int i;
1489         int err;
1490
1491         for (i = 0; i < nh_grp->count; i++) {
1492                 nh = &nh_grp->nexthops[i];
1493
1494                 if (!nh->should_offload) {
1495                         nh->offloaded = 0;
1496                         continue;
1497                 }
1498
1499                 if (nh->update || reallocate) {
1500                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1501                                                           adj_index, nh);
1502                         if (err)
1503                                 return err;
1504                         nh->update = 0;
1505                         nh->offloaded = 1;
1506                 }
1507                 adj_index++;
1508         }
1509         return 0;
1510 }
1511
1512 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1513                                      struct mlxsw_sp_fib_entry *fib_entry);
1514
1515 static int
1516 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1517                                     struct mlxsw_sp_nexthop_group *nh_grp)
1518 {
1519         struct mlxsw_sp_fib_entry *fib_entry;
1520         int err;
1521
1522         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1523                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1524                 if (err)
1525                         return err;
1526         }
1527         return 0;
1528 }
1529
1530 static void
1531 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1532                                struct mlxsw_sp_nexthop_group *nh_grp)
1533 {
1534         struct mlxsw_sp_nexthop *nh;
1535         bool offload_change = false;
1536         u32 adj_index;
1537         u16 ecmp_size = 0;
1538         bool old_adj_index_valid;
1539         u32 old_adj_index;
1540         u16 old_ecmp_size;
1541         int i;
1542         int err;
1543
1544         if (!nh_grp->gateway) {
1545                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1546                 return;
1547         }
1548
1549         for (i = 0; i < nh_grp->count; i++) {
1550                 nh = &nh_grp->nexthops[i];
1551
1552                 if (nh->should_offload ^ nh->offloaded) {
1553                         offload_change = true;
1554                         if (nh->should_offload)
1555                                 nh->update = 1;
1556                 }
1557                 if (nh->should_offload)
1558                         ecmp_size++;
1559         }
1560         if (!offload_change) {
1561                 /* Nothing was added or removed, so no need to reallocate. Just
1562                  * update MAC on existing adjacency indexes.
1563                  */
1564                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1565                                                         false);
1566                 if (err) {
1567                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1568                         goto set_trap;
1569                 }
1570                 return;
1571         }
1572         if (!ecmp_size)
1573                 /* No neigh of this group is connected so we just set
1574                  * the trap and let everthing flow through kernel.
1575                  */
1576                 goto set_trap;
1577
1578         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1579         if (err) {
1580                 /* We ran out of KVD linear space, just set the
1581                  * trap and let everything flow through kernel.
1582                  */
1583                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1584                 goto set_trap;
1585         }
1586         old_adj_index_valid = nh_grp->adj_index_valid;
1587         old_adj_index = nh_grp->adj_index;
1588         old_ecmp_size = nh_grp->ecmp_size;
1589         nh_grp->adj_index_valid = 1;
1590         nh_grp->adj_index = adj_index;
1591         nh_grp->ecmp_size = ecmp_size;
1592         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1593         if (err) {
1594                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1595                 goto set_trap;
1596         }
1597
1598         if (!old_adj_index_valid) {
1599                 /* The trap was set for fib entries, so we have to call
1600                  * fib entry update to unset it and use adjacency index.
1601                  */
1602                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1603                 if (err) {
1604                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1605                         goto set_trap;
1606                 }
1607                 return;
1608         }
1609
1610         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1611                                              old_adj_index, old_ecmp_size);
1612         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1613         if (err) {
1614                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1615                 goto set_trap;
1616         }
1617         return;
1618
1619 set_trap:
1620         old_adj_index_valid = nh_grp->adj_index_valid;
1621         nh_grp->adj_index_valid = 0;
1622         for (i = 0; i < nh_grp->count; i++) {
1623                 nh = &nh_grp->nexthops[i];
1624                 nh->offloaded = 0;
1625         }
1626         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1627         if (err)
1628                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1629         if (old_adj_index_valid)
1630                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1631 }
1632
1633 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1634                                             bool removing)
1635 {
1636         if (!removing && !nh->should_offload)
1637                 nh->should_offload = 1;
1638         else if (removing && nh->offloaded)
1639                 nh->should_offload = 0;
1640         nh->update = 1;
1641 }
1642
1643 static void
1644 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1645                               struct mlxsw_sp_neigh_entry *neigh_entry,
1646                               bool removing)
1647 {
1648         struct mlxsw_sp_nexthop *nh;
1649
1650         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1651                             neigh_list_node) {
1652                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1653                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1654         }
1655 }
1656
1657 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1658                                       struct mlxsw_sp_rif *rif)
1659 {
1660         if (nh->rif)
1661                 return;
1662
1663         nh->rif = rif;
1664         list_add(&nh->rif_list_node, &rif->nexthop_list);
1665 }
1666
1667 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1668 {
1669         if (!nh->rif)
1670                 return;
1671
1672         list_del(&nh->rif_list_node);
1673         nh->rif = NULL;
1674 }
1675
1676 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1677                                        struct mlxsw_sp_nexthop *nh)
1678 {
1679         struct mlxsw_sp_neigh_entry *neigh_entry;
1680         struct fib_nh *fib_nh = nh->key.fib_nh;
1681         struct neighbour *n;
1682         u8 nud_state, dead;
1683         int err;
1684
1685         if (!nh->nh_grp->gateway || nh->neigh_entry)
1686                 return 0;
1687
1688         /* Take a reference of neigh here ensuring that neigh would
1689          * not be detructed before the nexthop entry is finished.
1690          * The reference is taken either in neigh_lookup() or
1691          * in neigh_create() in case n is not found.
1692          */
1693         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1694         if (!n) {
1695                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1696                 if (IS_ERR(n))
1697                         return PTR_ERR(n);
1698                 neigh_event_send(n, NULL);
1699         }
1700         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1701         if (!neigh_entry) {
1702                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1703                 if (IS_ERR(neigh_entry)) {
1704                         err = -EINVAL;
1705                         goto err_neigh_entry_create;
1706                 }
1707         }
1708
1709         /* If that is the first nexthop connected to that neigh, add to
1710          * nexthop_neighs_list
1711          */
1712         if (list_empty(&neigh_entry->nexthop_list))
1713                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1714                               &mlxsw_sp->router->nexthop_neighs_list);
1715
1716         nh->neigh_entry = neigh_entry;
1717         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1718         read_lock_bh(&n->lock);
1719         nud_state = n->nud_state;
1720         dead = n->dead;
1721         read_unlock_bh(&n->lock);
1722         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1723
1724         return 0;
1725
1726 err_neigh_entry_create:
1727         neigh_release(n);
1728         return err;
1729 }
1730
1731 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1732                                         struct mlxsw_sp_nexthop *nh)
1733 {
1734         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1735         struct neighbour *n;
1736
1737         if (!neigh_entry)
1738                 return;
1739         n = neigh_entry->key.n;
1740
1741         __mlxsw_sp_nexthop_neigh_update(nh, true);
1742         list_del(&nh->neigh_list_node);
1743         nh->neigh_entry = NULL;
1744
1745         /* If that is the last nexthop connected to that neigh, remove from
1746          * nexthop_neighs_list
1747          */
1748         if (list_empty(&neigh_entry->nexthop_list))
1749                 list_del(&neigh_entry->nexthop_neighs_list_node);
1750
1751         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1752                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1753
1754         neigh_release(n);
1755 }
1756
1757 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1758                                  struct mlxsw_sp_nexthop_group *nh_grp,
1759                                  struct mlxsw_sp_nexthop *nh,
1760                                  struct fib_nh *fib_nh)
1761 {
1762         struct net_device *dev = fib_nh->nh_dev;
1763         struct in_device *in_dev;
1764         struct mlxsw_sp_rif *rif;
1765         int err;
1766
1767         nh->nh_grp = nh_grp;
1768         nh->key.fib_nh = fib_nh;
1769         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1770         if (err)
1771                 return err;
1772
1773         if (!dev)
1774                 return 0;
1775
1776         in_dev = __in_dev_get_rtnl(dev);
1777         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1778             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1779                 return 0;
1780
1781         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1782         if (!rif)
1783                 return 0;
1784         mlxsw_sp_nexthop_rif_init(nh, rif);
1785
1786         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1787         if (err)
1788                 goto err_nexthop_neigh_init;
1789
1790         return 0;
1791
1792 err_nexthop_neigh_init:
1793         mlxsw_sp_nexthop_rif_fini(nh);
1794         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1795         return err;
1796 }
1797
1798 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1799                                   struct mlxsw_sp_nexthop *nh)
1800 {
1801         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1802         mlxsw_sp_nexthop_rif_fini(nh);
1803         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1804 }
1805
1806 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1807                                    unsigned long event, struct fib_nh *fib_nh)
1808 {
1809         struct mlxsw_sp_nexthop_key key;
1810         struct mlxsw_sp_nexthop *nh;
1811         struct mlxsw_sp_rif *rif;
1812
1813         if (mlxsw_sp->router->aborted)
1814                 return;
1815
1816         key.fib_nh = fib_nh;
1817         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1818         if (WARN_ON_ONCE(!nh))
1819                 return;
1820
1821         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1822         if (!rif)
1823                 return;
1824
1825         switch (event) {
1826         case FIB_EVENT_NH_ADD:
1827                 mlxsw_sp_nexthop_rif_init(nh, rif);
1828                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1829                 break;
1830         case FIB_EVENT_NH_DEL:
1831                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1832                 mlxsw_sp_nexthop_rif_fini(nh);
1833                 break;
1834         }
1835
1836         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1837 }
1838
1839 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1840                                            struct mlxsw_sp_rif *rif)
1841 {
1842         struct mlxsw_sp_nexthop *nh, *tmp;
1843
1844         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1845                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1846                 mlxsw_sp_nexthop_rif_fini(nh);
1847                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1848         }
1849 }
1850
1851 static struct mlxsw_sp_nexthop_group *
1852 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1853 {
1854         struct mlxsw_sp_nexthop_group *nh_grp;
1855         struct mlxsw_sp_nexthop *nh;
1856         struct fib_nh *fib_nh;
1857         size_t alloc_size;
1858         int i;
1859         int err;
1860
1861         alloc_size = sizeof(*nh_grp) +
1862                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1863         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1864         if (!nh_grp)
1865                 return ERR_PTR(-ENOMEM);
1866         INIT_LIST_HEAD(&nh_grp->fib_list);
1867         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1868         nh_grp->count = fi->fib_nhs;
1869         nh_grp->key.fi = fi;
1870         fib_info_hold(fi);
1871         for (i = 0; i < nh_grp->count; i++) {
1872                 nh = &nh_grp->nexthops[i];
1873                 fib_nh = &fi->fib_nh[i];
1874                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1875                 if (err)
1876                         goto err_nexthop_init;
1877         }
1878         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1879         if (err)
1880                 goto err_nexthop_group_insert;
1881         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1882         return nh_grp;
1883
1884 err_nexthop_group_insert:
1885 err_nexthop_init:
1886         for (i--; i >= 0; i--) {
1887                 nh = &nh_grp->nexthops[i];
1888                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1889         }
1890         fib_info_put(nh_grp->key.fi);
1891         kfree(nh_grp);
1892         return ERR_PTR(err);
1893 }
1894
1895 static void
1896 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1897                                struct mlxsw_sp_nexthop_group *nh_grp)
1898 {
1899         struct mlxsw_sp_nexthop *nh;
1900         int i;
1901
1902         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1903         for (i = 0; i < nh_grp->count; i++) {
1904                 nh = &nh_grp->nexthops[i];
1905                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1906         }
1907         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1908         WARN_ON_ONCE(nh_grp->adj_index_valid);
1909         fib_info_put(nh_grp->key.fi);
1910         kfree(nh_grp);
1911 }
1912
1913 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1914                                       struct mlxsw_sp_fib_entry *fib_entry,
1915                                       struct fib_info *fi)
1916 {
1917         struct mlxsw_sp_nexthop_group_key key;
1918         struct mlxsw_sp_nexthop_group *nh_grp;
1919
1920         key.fi = fi;
1921         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1922         if (!nh_grp) {
1923                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1924                 if (IS_ERR(nh_grp))
1925                         return PTR_ERR(nh_grp);
1926         }
1927         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1928         fib_entry->nh_group = nh_grp;
1929         return 0;
1930 }
1931
1932 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1933                                        struct mlxsw_sp_fib_entry *fib_entry)
1934 {
1935         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1936
1937         list_del(&fib_entry->nexthop_group_node);
1938         if (!list_empty(&nh_grp->fib_list))
1939                 return;
1940         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1941 }
1942
1943 static bool
1944 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1945 {
1946         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1947
1948         if (fib_entry->params.tos)
1949                 return false;
1950
1951         switch (fib_entry->type) {
1952         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1953                 return !!nh_group->adj_index_valid;
1954         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1955                 return !!nh_group->nh_rif;
1956         default:
1957                 return false;
1958         }
1959 }
1960
1961 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1962 {
1963         fib_entry->offloaded = true;
1964
1965         switch (fib_entry->fib_node->fib->proto) {
1966         case MLXSW_SP_L3_PROTO_IPV4:
1967                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1968                 break;
1969         case MLXSW_SP_L3_PROTO_IPV6:
1970                 WARN_ON_ONCE(1);
1971         }
1972 }
1973
1974 static void
1975 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1976 {
1977         switch (fib_entry->fib_node->fib->proto) {
1978         case MLXSW_SP_L3_PROTO_IPV4:
1979                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1980                 break;
1981         case MLXSW_SP_L3_PROTO_IPV6:
1982                 WARN_ON_ONCE(1);
1983         }
1984
1985         fib_entry->offloaded = false;
1986 }
1987
1988 static void
1989 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1990                                    enum mlxsw_reg_ralue_op op, int err)
1991 {
1992         switch (op) {
1993         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1994                 if (!fib_entry->offloaded)
1995                         return;
1996                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1997         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1998                 if (err)
1999                         return;
2000                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2001                     !fib_entry->offloaded)
2002                         mlxsw_sp_fib_entry_offload_set(fib_entry);
2003                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2004                          fib_entry->offloaded)
2005                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
2006                 return;
2007         default:
2008                 return;
2009         }
2010 }
2011
2012 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
2013                                          struct mlxsw_sp_fib_entry *fib_entry,
2014                                          enum mlxsw_reg_ralue_op op)
2015 {
2016         char ralue_pl[MLXSW_REG_RALUE_LEN];
2017         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2018         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2019         enum mlxsw_reg_ralue_trap_action trap_action;
2020         u16 trap_id = 0;
2021         u32 adjacency_index = 0;
2022         u16 ecmp_size = 0;
2023
2024         /* In case the nexthop group adjacency index is valid, use it
2025          * with provided ECMP size. Otherwise, setup trap and pass
2026          * traffic to kernel.
2027          */
2028         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2029                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2030                 adjacency_index = fib_entry->nh_group->adj_index;
2031                 ecmp_size = fib_entry->nh_group->ecmp_size;
2032         } else {
2033                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2034                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2035         }
2036
2037         mlxsw_reg_ralue_pack4(ralue_pl,
2038                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2039                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2040                               *p_dip);
2041         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2042                                         adjacency_index, ecmp_size);
2043         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2044 }
2045
2046 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
2047                                         struct mlxsw_sp_fib_entry *fib_entry,
2048                                         enum mlxsw_reg_ralue_op op)
2049 {
2050         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2051         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2052         enum mlxsw_reg_ralue_trap_action trap_action;
2053         char ralue_pl[MLXSW_REG_RALUE_LEN];
2054         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2055         u16 trap_id = 0;
2056         u16 rif_index = 0;
2057
2058         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2059                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2060                 rif_index = rif->rif_index;
2061         } else {
2062                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2063                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2064         }
2065
2066         mlxsw_reg_ralue_pack4(ralue_pl,
2067                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2068                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2069                               *p_dip);
2070         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2071                                        rif_index);
2072         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2073 }
2074
2075 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
2076                                        struct mlxsw_sp_fib_entry *fib_entry,
2077                                        enum mlxsw_reg_ralue_op op)
2078 {
2079         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2080         char ralue_pl[MLXSW_REG_RALUE_LEN];
2081         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2082
2083         mlxsw_reg_ralue_pack4(ralue_pl,
2084                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2085                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
2086                               *p_dip);
2087         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2088         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2089 }
2090
2091 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
2092                                   struct mlxsw_sp_fib_entry *fib_entry,
2093                                   enum mlxsw_reg_ralue_op op)
2094 {
2095         switch (fib_entry->type) {
2096         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2097                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
2098         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2099                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
2100         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2101                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
2102         }
2103         return -EINVAL;
2104 }
2105
2106 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2107                                  struct mlxsw_sp_fib_entry *fib_entry,
2108                                  enum mlxsw_reg_ralue_op op)
2109 {
2110         int err = -EINVAL;
2111
2112         switch (fib_entry->fib_node->fib->proto) {
2113         case MLXSW_SP_L3_PROTO_IPV4:
2114                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
2115                 break;
2116         case MLXSW_SP_L3_PROTO_IPV6:
2117                 return err;
2118         }
2119         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2120         return err;
2121 }
2122
2123 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2124                                      struct mlxsw_sp_fib_entry *fib_entry)
2125 {
2126         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2127                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
2128 }
2129
2130 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2131                                   struct mlxsw_sp_fib_entry *fib_entry)
2132 {
2133         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2134                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
2135 }
2136
2137 static int
2138 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2139                              const struct fib_entry_notifier_info *fen_info,
2140                              struct mlxsw_sp_fib_entry *fib_entry)
2141 {
2142         struct fib_info *fi = fen_info->fi;
2143
2144         switch (fen_info->type) {
2145         case RTN_BROADCAST: /* fall through */
2146         case RTN_LOCAL:
2147                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2148                 return 0;
2149         case RTN_UNREACHABLE: /* fall through */
2150         case RTN_BLACKHOLE: /* fall through */
2151         case RTN_PROHIBIT:
2152                 /* Packets hitting these routes need to be trapped, but
2153                  * can do so with a lower priority than packets directed
2154                  * at the host, so use action type local instead of trap.
2155                  */
2156                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2157                 return 0;
2158         case RTN_UNICAST:
2159                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2160                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2161                 else
2162                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2163                 return 0;
2164         default:
2165                 return -EINVAL;
2166         }
2167 }
2168
2169 static struct mlxsw_sp_fib_entry *
2170 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2171                            struct mlxsw_sp_fib_node *fib_node,
2172                            const struct fib_entry_notifier_info *fen_info)
2173 {
2174         struct mlxsw_sp_fib_entry *fib_entry;
2175         int err;
2176
2177         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
2178         if (!fib_entry) {
2179                 err = -ENOMEM;
2180                 goto err_fib_entry_alloc;
2181         }
2182
2183         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2184         if (err)
2185                 goto err_fib4_entry_type_set;
2186
2187         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2188         if (err)
2189                 goto err_nexthop_group_get;
2190
2191         fib_entry->params.prio = fen_info->fi->fib_priority;
2192         fib_entry->params.tb_id = fen_info->tb_id;
2193         fib_entry->params.type = fen_info->type;
2194         fib_entry->params.tos = fen_info->tos;
2195
2196         fib_entry->fib_node = fib_node;
2197
2198         return fib_entry;
2199
2200 err_nexthop_group_get:
2201 err_fib4_entry_type_set:
2202         kfree(fib_entry);
2203 err_fib_entry_alloc:
2204         return ERR_PTR(err);
2205 }
2206
2207 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2208                                         struct mlxsw_sp_fib_entry *fib_entry)
2209 {
2210         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
2211         kfree(fib_entry);
2212 }
2213
2214 static struct mlxsw_sp_fib_node *
2215 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2216                        const struct fib_entry_notifier_info *fen_info);
2217
2218 static struct mlxsw_sp_fib_entry *
2219 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2220                            const struct fib_entry_notifier_info *fen_info)
2221 {
2222         struct mlxsw_sp_fib_entry *fib_entry;
2223         struct mlxsw_sp_fib_node *fib_node;
2224
2225         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2226         if (IS_ERR(fib_node))
2227                 return NULL;
2228
2229         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2230                 if (fib_entry->params.tb_id == fen_info->tb_id &&
2231                     fib_entry->params.tos == fen_info->tos &&
2232                     fib_entry->params.type == fen_info->type &&
2233                     fib_entry->nh_group->key.fi == fen_info->fi) {
2234                         return fib_entry;
2235                 }
2236         }
2237
2238         return NULL;
2239 }
2240
2241 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2242         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2243         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2244         .key_len = sizeof(struct mlxsw_sp_fib_key),
2245         .automatic_shrinking = true,
2246 };
2247
2248 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2249                                     struct mlxsw_sp_fib_node *fib_node)
2250 {
2251         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2252                                       mlxsw_sp_fib_ht_params);
2253 }
2254
2255 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2256                                      struct mlxsw_sp_fib_node *fib_node)
2257 {
2258         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2259                                mlxsw_sp_fib_ht_params);
2260 }
2261
2262 static struct mlxsw_sp_fib_node *
2263 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2264                          size_t addr_len, unsigned char prefix_len)
2265 {
2266         struct mlxsw_sp_fib_key key;
2267
2268         memset(&key, 0, sizeof(key));
2269         memcpy(key.addr, addr, addr_len);
2270         key.prefix_len = prefix_len;
2271         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2272 }
2273
2274 static struct mlxsw_sp_fib_node *
2275 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2276                          size_t addr_len, unsigned char prefix_len)
2277 {
2278         struct mlxsw_sp_fib_node *fib_node;
2279
2280         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2281         if (!fib_node)
2282                 return NULL;
2283
2284         INIT_LIST_HEAD(&fib_node->entry_list);
2285         list_add(&fib_node->list, &fib->node_list);
2286         memcpy(fib_node->key.addr, addr, addr_len);
2287         fib_node->key.prefix_len = prefix_len;
2288
2289         return fib_node;
2290 }
2291
2292 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2293 {
2294         list_del(&fib_node->list);
2295         WARN_ON(!list_empty(&fib_node->entry_list));
2296         kfree(fib_node);
2297 }
2298
2299 static bool
2300 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2301                                  const struct mlxsw_sp_fib_entry *fib_entry)
2302 {
2303         return list_first_entry(&fib_node->entry_list,
2304                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2305 }
2306
2307 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2308 {
2309         unsigned char prefix_len = fib_node->key.prefix_len;
2310         struct mlxsw_sp_fib *fib = fib_node->fib;
2311
2312         if (fib->prefix_ref_count[prefix_len]++ == 0)
2313                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2314 }
2315
2316 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2317 {
2318         unsigned char prefix_len = fib_node->key.prefix_len;
2319         struct mlxsw_sp_fib *fib = fib_node->fib;
2320
2321         if (--fib->prefix_ref_count[prefix_len] == 0)
2322                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2323 }
2324
2325 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2326                                   struct mlxsw_sp_fib_node *fib_node,
2327                                   struct mlxsw_sp_fib *fib)
2328 {
2329         struct mlxsw_sp_prefix_usage req_prefix_usage;
2330         struct mlxsw_sp_lpm_tree *lpm_tree;
2331         int err;
2332
2333         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2334         if (err)
2335                 return err;
2336         fib_node->fib = fib;
2337
2338         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2339         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2340
2341         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2342                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2343                                                  &req_prefix_usage);
2344                 if (err)
2345                         goto err_tree_check;
2346         } else {
2347                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2348                                                  fib->proto);
2349                 if (IS_ERR(lpm_tree))
2350                         return PTR_ERR(lpm_tree);
2351                 fib->lpm_tree = lpm_tree;
2352                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2353                 if (err)
2354                         goto err_tree_bind;
2355         }
2356
2357         mlxsw_sp_fib_node_prefix_inc(fib_node);
2358
2359         return 0;
2360
2361 err_tree_bind:
2362         fib->lpm_tree = NULL;
2363         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2364 err_tree_check:
2365         fib_node->fib = NULL;
2366         mlxsw_sp_fib_node_remove(fib, fib_node);
2367         return err;
2368 }
2369
2370 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2371                                    struct mlxsw_sp_fib_node *fib_node)
2372 {
2373         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2374         struct mlxsw_sp_fib *fib = fib_node->fib;
2375
2376         mlxsw_sp_fib_node_prefix_dec(fib_node);
2377
2378         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2379                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2380                 fib->lpm_tree = NULL;
2381                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2382         } else {
2383                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2384         }
2385
2386         fib_node->fib = NULL;
2387         mlxsw_sp_fib_node_remove(fib, fib_node);
2388 }
2389
2390 static struct mlxsw_sp_fib_node *
2391 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2392                        const struct fib_entry_notifier_info *fen_info)
2393 {
2394         struct mlxsw_sp_fib_node *fib_node;
2395         struct mlxsw_sp_fib *fib;
2396         struct mlxsw_sp_vr *vr;
2397         int err;
2398
2399         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2400         if (IS_ERR(vr))
2401                 return ERR_CAST(vr);
2402         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2403
2404         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2405                                             sizeof(fen_info->dst),
2406                                             fen_info->dst_len);
2407         if (fib_node)
2408                 return fib_node;
2409
2410         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2411                                             sizeof(fen_info->dst),
2412                                             fen_info->dst_len);
2413         if (!fib_node) {
2414                 err = -ENOMEM;
2415                 goto err_fib_node_create;
2416         }
2417
2418         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2419         if (err)
2420                 goto err_fib_node_init;
2421
2422         return fib_node;
2423
2424 err_fib_node_init:
2425         mlxsw_sp_fib_node_destroy(fib_node);
2426 err_fib_node_create:
2427         mlxsw_sp_vr_put(vr);
2428         return ERR_PTR(err);
2429 }
2430
2431 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2432                                    struct mlxsw_sp_fib_node *fib_node)
2433 {
2434         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2435
2436         if (!list_empty(&fib_node->entry_list))
2437                 return;
2438         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2439         mlxsw_sp_fib_node_destroy(fib_node);
2440         mlxsw_sp_vr_put(vr);
2441 }
2442
2443 static struct mlxsw_sp_fib_entry *
2444 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2445                               const struct mlxsw_sp_fib_entry_params *params)
2446 {
2447         struct mlxsw_sp_fib_entry *fib_entry;
2448
2449         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2450                 if (fib_entry->params.tb_id > params->tb_id)
2451                         continue;
2452                 if (fib_entry->params.tb_id != params->tb_id)
2453                         break;
2454                 if (fib_entry->params.tos > params->tos)
2455                         continue;
2456                 if (fib_entry->params.prio >= params->prio ||
2457                     fib_entry->params.tos < params->tos)
2458                         return fib_entry;
2459         }
2460
2461         return NULL;
2462 }
2463
2464 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2465                                           struct mlxsw_sp_fib_entry *new_entry)
2466 {
2467         struct mlxsw_sp_fib_node *fib_node;
2468
2469         if (WARN_ON(!fib_entry))
2470                 return -EINVAL;
2471
2472         fib_node = fib_entry->fib_node;
2473         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2474                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2475                     fib_entry->params.tos != new_entry->params.tos ||
2476                     fib_entry->params.prio != new_entry->params.prio)
2477                         break;
2478         }
2479
2480         list_add_tail(&new_entry->list, &fib_entry->list);
2481         return 0;
2482 }
2483
2484 static int
2485 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2486                                struct mlxsw_sp_fib_entry *new_entry,
2487                                bool replace, bool append)
2488 {
2489         struct mlxsw_sp_fib_entry *fib_entry;
2490
2491         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2492
2493         if (append)
2494                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2495         if (replace && WARN_ON(!fib_entry))
2496                 return -EINVAL;
2497
2498         /* Insert new entry before replaced one, so that we can later
2499          * remove the second.
2500          */
2501         if (fib_entry) {
2502                 list_add_tail(&new_entry->list, &fib_entry->list);
2503         } else {
2504                 struct mlxsw_sp_fib_entry *last;
2505
2506                 list_for_each_entry(last, &fib_node->entry_list, list) {
2507                         if (new_entry->params.tb_id > last->params.tb_id)
2508                                 break;
2509                         fib_entry = last;
2510                 }
2511
2512                 if (fib_entry)
2513                         list_add(&new_entry->list, &fib_entry->list);
2514                 else
2515                         list_add(&new_entry->list, &fib_node->entry_list);
2516         }
2517
2518         return 0;
2519 }
2520
2521 static void
2522 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2523 {
2524         list_del(&fib_entry->list);
2525 }
2526
2527 static int
2528 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2529                              const struct mlxsw_sp_fib_node *fib_node,
2530                              struct mlxsw_sp_fib_entry *fib_entry)
2531 {
2532         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2533                 return 0;
2534
2535         /* To prevent packet loss, overwrite the previously offloaded
2536          * entry.
2537          */
2538         if (!list_is_singular(&fib_node->entry_list)) {
2539                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2540                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2541
2542                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2543         }
2544
2545         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2546 }
2547
2548 static void
2549 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2550                              const struct mlxsw_sp_fib_node *fib_node,
2551                              struct mlxsw_sp_fib_entry *fib_entry)
2552 {
2553         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2554                 return;
2555
2556         /* Promote the next entry by overwriting the deleted entry */
2557         if (!list_is_singular(&fib_node->entry_list)) {
2558                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2559                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2560
2561                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2562                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2563                 return;
2564         }
2565
2566         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2567 }
2568
2569 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2570                                          struct mlxsw_sp_fib_entry *fib_entry,
2571                                          bool replace, bool append)
2572 {
2573         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2574         int err;
2575
2576         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2577                                              append);
2578         if (err)
2579                 return err;
2580
2581         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2582         if (err)
2583                 goto err_fib4_node_entry_add;
2584
2585         return 0;
2586
2587 err_fib4_node_entry_add:
2588         mlxsw_sp_fib4_node_list_remove(fib_entry);
2589         return err;
2590 }
2591
2592 static void
2593 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2594                                 struct mlxsw_sp_fib_entry *fib_entry)
2595 {
2596         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2597
2598         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2599         mlxsw_sp_fib4_node_list_remove(fib_entry);
2600 }
2601
2602 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2603                                         struct mlxsw_sp_fib_entry *fib_entry,
2604                                         bool replace)
2605 {
2606         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2607         struct mlxsw_sp_fib_entry *replaced;
2608
2609         if (!replace)
2610                 return;
2611
2612         /* We inserted the new entry before replaced one */
2613         replaced = list_next_entry(fib_entry, list);
2614
2615         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2616         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2617         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2618 }
2619
2620 static int
2621 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2622                          const struct fib_entry_notifier_info *fen_info,
2623                          bool replace, bool append)
2624 {
2625         struct mlxsw_sp_fib_entry *fib_entry;
2626         struct mlxsw_sp_fib_node *fib_node;
2627         int err;
2628
2629         if (mlxsw_sp->router->aborted)
2630                 return 0;
2631
2632         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2633         if (IS_ERR(fib_node)) {
2634                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2635                 return PTR_ERR(fib_node);
2636         }
2637
2638         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2639         if (IS_ERR(fib_entry)) {
2640                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2641                 err = PTR_ERR(fib_entry);
2642                 goto err_fib4_entry_create;
2643         }
2644
2645         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2646                                             append);
2647         if (err) {
2648                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2649                 goto err_fib4_node_entry_link;
2650         }
2651
2652         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2653
2654         return 0;
2655
2656 err_fib4_node_entry_link:
2657         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2658 err_fib4_entry_create:
2659         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2660         return err;
2661 }
2662
2663 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2664                                      struct fib_entry_notifier_info *fen_info)
2665 {
2666         struct mlxsw_sp_fib_entry *fib_entry;
2667         struct mlxsw_sp_fib_node *fib_node;
2668
2669         if (mlxsw_sp->router->aborted)
2670                 return;
2671
2672         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2673         if (WARN_ON(!fib_entry))
2674                 return;
2675         fib_node = fib_entry->fib_node;
2676
2677         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2678         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2679         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2680 }
2681
2682 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2683 {
2684         char ralta_pl[MLXSW_REG_RALTA_LEN];
2685         char ralst_pl[MLXSW_REG_RALST_LEN];
2686         int i, err;
2687
2688         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2689                              MLXSW_SP_LPM_TREE_MIN);
2690         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2691         if (err)
2692                 return err;
2693
2694         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2695         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2696         if (err)
2697                 return err;
2698
2699         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2700                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2701                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2702                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2703
2704                 if (!mlxsw_sp_vr_is_used(vr))
2705                         continue;
2706
2707                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2708                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2709                                      MLXSW_SP_LPM_TREE_MIN);
2710                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2711                                       raltb_pl);
2712                 if (err)
2713                         return err;
2714
2715                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2716                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2717                                       0);
2718                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2719                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2720                                       ralue_pl);
2721                 if (err)
2722                         return err;
2723         }
2724
2725         return 0;
2726 }
2727
2728 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2729                                      struct mlxsw_sp_fib_node *fib_node)
2730 {
2731         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2732
2733         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2734                 bool do_break = &tmp->list == &fib_node->entry_list;
2735
2736                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2737                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2738                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2739                 /* Break when entry list is empty and node was freed.
2740                  * Otherwise, we'll access freed memory in the next
2741                  * iteration.
2742                  */
2743                 if (do_break)
2744                         break;
2745         }
2746 }
2747
2748 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2749                                     struct mlxsw_sp_fib_node *fib_node)
2750 {
2751         switch (fib_node->fib->proto) {
2752         case MLXSW_SP_L3_PROTO_IPV4:
2753                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2754                 break;
2755         case MLXSW_SP_L3_PROTO_IPV6:
2756                 WARN_ON_ONCE(1);
2757                 break;
2758         }
2759 }
2760
2761 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2762                                   struct mlxsw_sp_vr *vr,
2763                                   enum mlxsw_sp_l3proto proto)
2764 {
2765         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2766         struct mlxsw_sp_fib_node *fib_node, *tmp;
2767
2768         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2769                 bool do_break = &tmp->list == &fib->node_list;
2770
2771                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2772                 if (do_break)
2773                         break;
2774         }
2775 }
2776
2777 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2778 {
2779         int i;
2780
2781         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2782                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2783
2784                 if (!mlxsw_sp_vr_is_used(vr))
2785                         continue;
2786                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2787         }
2788 }
2789
2790 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2791 {
2792         int err;
2793
2794         if (mlxsw_sp->router->aborted)
2795                 return;
2796         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2797         mlxsw_sp_router_fib_flush(mlxsw_sp);
2798         mlxsw_sp->router->aborted = true;
2799         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2800         if (err)
2801                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2802 }
2803
2804 struct mlxsw_sp_fib_event_work {
2805         struct work_struct work;
2806         union {
2807                 struct fib_entry_notifier_info fen_info;
2808                 struct fib_rule_notifier_info fr_info;
2809                 struct fib_nh_notifier_info fnh_info;
2810         };
2811         struct mlxsw_sp *mlxsw_sp;
2812         unsigned long event;
2813 };
2814
2815 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2816 {
2817         struct mlxsw_sp_fib_event_work *fib_work =
2818                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2819         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2820         struct fib_rule *rule;
2821         bool replace, append;
2822         int err;
2823
2824         /* Protect internal structures from changes */
2825         rtnl_lock();
2826         switch (fib_work->event) {
2827         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2828         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2829         case FIB_EVENT_ENTRY_ADD:
2830                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2831                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2832                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2833                                                replace, append);
2834                 if (err)
2835                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2836                 fib_info_put(fib_work->fen_info.fi);
2837                 break;
2838         case FIB_EVENT_ENTRY_DEL:
2839                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2840                 fib_info_put(fib_work->fen_info.fi);
2841                 break;
2842         case FIB_EVENT_RULE_ADD: /* fall through */
2843         case FIB_EVENT_RULE_DEL:
2844                 rule = fib_work->fr_info.rule;
2845                 if (!fib4_rule_default(rule) && !rule->l3mdev)
2846                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2847                 fib_rule_put(rule);
2848                 break;
2849         case FIB_EVENT_NH_ADD: /* fall through */
2850         case FIB_EVENT_NH_DEL:
2851                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2852                                        fib_work->fnh_info.fib_nh);
2853                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2854                 break;
2855         }
2856         rtnl_unlock();
2857         kfree(fib_work);
2858 }
2859
2860 /* Called with rcu_read_lock() */
2861 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2862                                      unsigned long event, void *ptr)
2863 {
2864         struct mlxsw_sp_fib_event_work *fib_work;
2865         struct fib_notifier_info *info = ptr;
2866         struct mlxsw_sp_router *router;
2867
2868         if (!net_eq(info->net, &init_net))
2869                 return NOTIFY_DONE;
2870
2871         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2872         if (WARN_ON(!fib_work))
2873                 return NOTIFY_BAD;
2874
2875         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2876         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
2877         fib_work->mlxsw_sp = router->mlxsw_sp;
2878         fib_work->event = event;
2879
2880         switch (event) {
2881         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2882         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2883         case FIB_EVENT_ENTRY_ADD: /* fall through */
2884         case FIB_EVENT_ENTRY_DEL:
2885                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2886                 /* Take referece on fib_info to prevent it from being
2887                  * freed while work is queued. Release it afterwards.
2888                  */
2889                 fib_info_hold(fib_work->fen_info.fi);
2890                 break;
2891         case FIB_EVENT_RULE_ADD: /* fall through */
2892         case FIB_EVENT_RULE_DEL:
2893                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2894                 fib_rule_get(fib_work->fr_info.rule);
2895                 break;
2896         case FIB_EVENT_NH_ADD: /* fall through */
2897         case FIB_EVENT_NH_DEL:
2898                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2899                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2900                 break;
2901         }
2902
2903         mlxsw_core_schedule_work(&fib_work->work);
2904
2905         return NOTIFY_DONE;
2906 }
2907
2908 static struct mlxsw_sp_rif *
2909 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2910                          const struct net_device *dev)
2911 {
2912         int i;
2913
2914         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2915                 if (mlxsw_sp->router->rifs[i] &&
2916                     mlxsw_sp->router->rifs[i]->dev == dev)
2917                         return mlxsw_sp->router->rifs[i];
2918
2919         return NULL;
2920 }
2921
2922 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2923 {
2924         char ritr_pl[MLXSW_REG_RITR_LEN];
2925         int err;
2926
2927         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2928         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2929         if (WARN_ON_ONCE(err))
2930                 return err;
2931
2932         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2933         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2934 }
2935
2936 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2937                                           struct mlxsw_sp_rif *rif)
2938 {
2939         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2940         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2941         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2942 }
2943
2944 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2945                                        const struct in_device *in_dev,
2946                                        unsigned long event)
2947 {
2948         switch (event) {
2949         case NETDEV_UP:
2950                 if (!rif)
2951                         return true;
2952                 return false;
2953         case NETDEV_DOWN:
2954                 if (rif && !in_dev->ifa_list &&
2955                     !netif_is_l3_slave(rif->dev))
2956                         return true;
2957                 /* It is possible we already removed the RIF ourselves
2958                  * if it was assigned to a netdev that is now a bridge
2959                  * or LAG slave.
2960                  */
2961                 return false;
2962         }
2963
2964         return false;
2965 }
2966
2967 static enum mlxsw_sp_rif_type
2968 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
2969                       const struct net_device *dev)
2970 {
2971         enum mlxsw_sp_fid_type type;
2972
2973         /* RIF type is derived from the type of the underlying FID */
2974         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
2975                 type = MLXSW_SP_FID_TYPE_8021Q;
2976         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
2977                 type = MLXSW_SP_FID_TYPE_8021Q;
2978         else if (netif_is_bridge_master(dev))
2979                 type = MLXSW_SP_FID_TYPE_8021D;
2980         else
2981                 type = MLXSW_SP_FID_TYPE_RFID;
2982
2983         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
2984 }
2985
2986 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
2987 {
2988         int i;
2989
2990         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
2991                 if (!mlxsw_sp->router->rifs[i]) {
2992                         *p_rif_index = i;
2993                         return 0;
2994                 }
2995         }
2996
2997         return -ENOBUFS;
2998 }
2999
3000 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
3001                                                u16 vr_id,
3002                                                struct net_device *l3_dev)
3003 {
3004         struct mlxsw_sp_rif *rif;
3005
3006         rif = kzalloc(rif_size, GFP_KERNEL);
3007         if (!rif)
3008                 return NULL;
3009
3010         INIT_LIST_HEAD(&rif->nexthop_list);
3011         INIT_LIST_HEAD(&rif->neigh_list);
3012         ether_addr_copy(rif->addr, l3_dev->dev_addr);
3013         rif->mtu = l3_dev->mtu;
3014         rif->vr_id = vr_id;
3015         rif->dev = l3_dev;
3016         rif->rif_index = rif_index;
3017
3018         return rif;
3019 }
3020
3021 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
3022                                            u16 rif_index)
3023 {
3024         return mlxsw_sp->router->rifs[rif_index];
3025 }
3026
3027 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
3028 {
3029         return rif->rif_index;
3030 }
3031
3032 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
3033 {
3034         return rif->dev->ifindex;
3035 }
3036
3037 static struct mlxsw_sp_rif *
3038 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
3039                     const struct mlxsw_sp_rif_params *params)
3040 {
3041         u32 tb_id = l3mdev_fib_table(params->dev);
3042         const struct mlxsw_sp_rif_ops *ops;
3043         enum mlxsw_sp_rif_type type;
3044         struct mlxsw_sp_rif *rif;
3045         struct mlxsw_sp_fid *fid;
3046         struct mlxsw_sp_vr *vr;
3047         u16 rif_index;
3048         int err;
3049
3050         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
3051         ops = mlxsw_sp->router->rif_ops_arr[type];
3052
3053         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3054         if (IS_ERR(vr))
3055                 return ERR_CAST(vr);
3056
3057         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
3058         if (err)
3059                 goto err_rif_index_alloc;
3060
3061         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
3062         if (!rif) {
3063                 err = -ENOMEM;
3064                 goto err_rif_alloc;
3065         }
3066         rif->mlxsw_sp = mlxsw_sp;
3067         rif->ops = ops;
3068
3069         fid = ops->fid_get(rif);
3070         if (IS_ERR(fid)) {
3071                 err = PTR_ERR(fid);
3072                 goto err_fid_get;
3073         }
3074         rif->fid = fid;
3075
3076         if (ops->setup)
3077                 ops->setup(rif, params);
3078
3079         err = ops->configure(rif);
3080         if (err)
3081                 goto err_configure;
3082
3083         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
3084                                   mlxsw_sp_fid_index(fid), true);
3085         if (err)
3086                 goto err_rif_fdb_op;
3087
3088         mlxsw_sp_rif_counters_alloc(rif);
3089         mlxsw_sp_fid_rif_set(fid, rif);
3090         mlxsw_sp->router->rifs[rif_index] = rif;
3091         vr->rif_count++;
3092
3093         return rif;
3094
3095 err_rif_fdb_op:
3096         ops->deconfigure(rif);
3097 err_configure:
3098         mlxsw_sp_fid_put(fid);
3099 err_fid_get:
3100         kfree(rif);
3101 err_rif_alloc:
3102 err_rif_index_alloc:
3103         mlxsw_sp_vr_put(vr);
3104         return ERR_PTR(err);
3105 }
3106
3107 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
3108 {
3109         const struct mlxsw_sp_rif_ops *ops = rif->ops;
3110         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3111         struct mlxsw_sp_fid *fid = rif->fid;
3112         struct mlxsw_sp_vr *vr;
3113
3114         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3115         vr = &mlxsw_sp->router->vrs[rif->vr_id];
3116
3117         vr->rif_count--;
3118         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
3119         mlxsw_sp_fid_rif_set(fid, NULL);
3120         mlxsw_sp_rif_counters_free(rif);
3121         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
3122                             mlxsw_sp_fid_index(fid), false);
3123         ops->deconfigure(rif);
3124         mlxsw_sp_fid_put(fid);
3125         kfree(rif);
3126         mlxsw_sp_vr_put(vr);
3127 }
3128
3129 static void
3130 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
3131                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3132 {
3133         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3134
3135         params->vid = mlxsw_sp_port_vlan->vid;
3136         params->lag = mlxsw_sp_port->lagged;
3137         if (params->lag)
3138                 params->lag_id = mlxsw_sp_port->lag_id;
3139         else
3140                 params->system_port = mlxsw_sp_port->local_port;
3141 }
3142
3143 static int
3144 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
3145                                struct net_device *l3_dev)
3146 {
3147         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3148         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
3149         u16 vid = mlxsw_sp_port_vlan->vid;
3150         struct mlxsw_sp_rif *rif;
3151         struct mlxsw_sp_fid *fid;
3152         int err;
3153
3154         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3155         if (!rif) {
3156                 struct mlxsw_sp_rif_params params = {
3157                         .dev = l3_dev,
3158                 };
3159
3160                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
3161                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3162                 if (IS_ERR(rif))
3163                         return PTR_ERR(rif);
3164         }
3165
3166         /* FID was already created, just take a reference */
3167         fid = rif->ops->fid_get(rif);
3168         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
3169         if (err)
3170                 goto err_fid_port_vid_map;
3171
3172         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
3173         if (err)
3174                 goto err_port_vid_learning_set;
3175
3176         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
3177                                         BR_STATE_FORWARDING);
3178         if (err)
3179                 goto err_port_vid_stp_set;
3180
3181         mlxsw_sp_port_vlan->fid = fid;
3182
3183         return 0;
3184
3185 err_port_vid_stp_set:
3186         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3187 err_port_vid_learning_set:
3188         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3189 err_fid_port_vid_map:
3190         mlxsw_sp_fid_put(fid);
3191         return err;
3192 }
3193
3194 void
3195 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3196 {
3197         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3198         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
3199         u16 vid = mlxsw_sp_port_vlan->vid;
3200
3201         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
3202                 return;
3203
3204         mlxsw_sp_port_vlan->fid = NULL;
3205         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
3206         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3207         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3208         /* If router port holds the last reference on the rFID, then the
3209          * associated Sub-port RIF will be destroyed.
3210          */
3211         mlxsw_sp_fid_put(fid);
3212 }
3213
3214 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
3215                                              struct net_device *port_dev,
3216                                              unsigned long event, u16 vid)
3217 {
3218         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
3219         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
3220
3221         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
3222         if (WARN_ON(!mlxsw_sp_port_vlan))
3223                 return -EINVAL;
3224
3225         switch (event) {
3226         case NETDEV_UP:
3227                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
3228                                                       l3_dev);
3229         case NETDEV_DOWN:
3230                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
3231                 break;
3232         }
3233
3234         return 0;
3235 }
3236
3237 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
3238                                         unsigned long event)
3239 {
3240         if (netif_is_bridge_port(port_dev) ||
3241             netif_is_lag_port(port_dev) ||
3242             netif_is_ovs_port(port_dev))
3243                 return 0;
3244
3245         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
3246 }
3247
3248 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
3249                                          struct net_device *lag_dev,
3250                                          unsigned long event, u16 vid)
3251 {
3252         struct net_device *port_dev;
3253         struct list_head *iter;
3254         int err;
3255
3256         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
3257                 if (mlxsw_sp_port_dev_check(port_dev)) {
3258                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
3259                                                                 port_dev,
3260                                                                 event, vid);
3261                         if (err)
3262                                 return err;
3263                 }
3264         }
3265
3266         return 0;
3267 }
3268
3269 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
3270                                        unsigned long event)
3271 {
3272         if (netif_is_bridge_port(lag_dev))
3273                 return 0;
3274
3275         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
3276 }
3277
3278 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3279                                           unsigned long event)
3280 {
3281         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3282         struct mlxsw_sp_rif_params params = {
3283                 .dev = l3_dev,
3284         };
3285         struct mlxsw_sp_rif *rif;
3286
3287         switch (event) {
3288         case NETDEV_UP:
3289                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
3290                 if (IS_ERR(rif))
3291                         return PTR_ERR(rif);
3292                 break;
3293         case NETDEV_DOWN:
3294                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3295                 mlxsw_sp_rif_destroy(rif);
3296                 break;
3297         }
3298
3299         return 0;
3300 }
3301
3302 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3303                                         unsigned long event)
3304 {
3305         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3306         u16 vid = vlan_dev_vlan_id(vlan_dev);
3307
3308         if (netif_is_bridge_port(vlan_dev))
3309                 return 0;
3310
3311         if (mlxsw_sp_port_dev_check(real_dev))
3312                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
3313                                                          event, vid);
3314         else if (netif_is_lag_master(real_dev))
3315                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3316                                                      vid);
3317         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
3318                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
3319
3320         return 0;
3321 }
3322
3323 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
3324                                      unsigned long event)
3325 {
3326         if (mlxsw_sp_port_dev_check(dev))
3327                 return mlxsw_sp_inetaddr_port_event(dev, event);
3328         else if (netif_is_lag_master(dev))
3329                 return mlxsw_sp_inetaddr_lag_event(dev, event);
3330         else if (netif_is_bridge_master(dev))
3331                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
3332         else if (is_vlan_dev(dev))
3333                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
3334         else
3335                 return 0;
3336 }
3337
3338 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3339                             unsigned long event, void *ptr)
3340 {
3341         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3342         struct net_device *dev = ifa->ifa_dev->dev;
3343         struct mlxsw_sp *mlxsw_sp;
3344         struct mlxsw_sp_rif *rif;
3345         int err = 0;
3346
3347         mlxsw_sp = mlxsw_sp_lower_get(dev);
3348         if (!mlxsw_sp)
3349                 goto out;
3350
3351         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3352         if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3353                 goto out;
3354
3355         err = __mlxsw_sp_inetaddr_event(dev, event);
3356 out:
3357         return notifier_from_errno(err);
3358 }
3359
3360 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3361                              const char *mac, int mtu)
3362 {
3363         char ritr_pl[MLXSW_REG_RITR_LEN];
3364         int err;
3365
3366         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3367         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3368         if (err)
3369                 return err;
3370
3371         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3372         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3373         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3374         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3375 }
3376
3377 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3378 {
3379         struct mlxsw_sp *mlxsw_sp;
3380         struct mlxsw_sp_rif *rif;
3381         u16 fid_index;
3382         int err;
3383
3384         mlxsw_sp = mlxsw_sp_lower_get(dev);
3385         if (!mlxsw_sp)
3386                 return 0;
3387
3388         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3389         if (!rif)
3390                 return 0;
3391         fid_index = mlxsw_sp_fid_index(rif->fid);
3392
3393         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
3394         if (err)
3395                 return err;
3396
3397         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3398                                 dev->mtu);
3399         if (err)
3400                 goto err_rif_edit;
3401
3402         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
3403         if (err)
3404                 goto err_rif_fdb_op;
3405
3406         ether_addr_copy(rif->addr, dev->dev_addr);
3407         rif->mtu = dev->mtu;
3408
3409         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3410
3411         return 0;
3412
3413 err_rif_fdb_op:
3414         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3415 err_rif_edit:
3416         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
3417         return err;
3418 }
3419
3420 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
3421                                   struct net_device *l3_dev)
3422 {
3423         struct mlxsw_sp_rif *rif;
3424
3425         /* If netdev is already associated with a RIF, then we need to
3426          * destroy it and create a new one with the new virtual router ID.
3427          */
3428         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3429         if (rif)
3430                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3431
3432         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
3433 }
3434
3435 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3436                                     struct net_device *l3_dev)
3437 {
3438         struct mlxsw_sp_rif *rif;
3439
3440         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3441         if (!rif)
3442                 return;
3443         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3444 }
3445
3446 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
3447                                  struct netdev_notifier_changeupper_info *info)
3448 {
3449         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3450         int err = 0;
3451
3452         if (!mlxsw_sp)
3453                 return 0;
3454
3455         switch (event) {
3456         case NETDEV_PRECHANGEUPPER:
3457                 return 0;
3458         case NETDEV_CHANGEUPPER:
3459                 if (info->linking)
3460                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
3461                 else
3462                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
3463                 break;
3464         }
3465
3466         return err;
3467 }
3468
3469 static struct mlxsw_sp_rif_subport *
3470 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
3471 {
3472         return container_of(rif, struct mlxsw_sp_rif_subport, common);
3473 }
3474
3475 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
3476                                        const struct mlxsw_sp_rif_params *params)
3477 {
3478         struct mlxsw_sp_rif_subport *rif_subport;
3479
3480         rif_subport = mlxsw_sp_rif_subport_rif(rif);
3481         rif_subport->vid = params->vid;
3482         rif_subport->lag = params->lag;
3483         if (params->lag)
3484                 rif_subport->lag_id = params->lag_id;
3485         else
3486                 rif_subport->system_port = params->system_port;
3487 }
3488
3489 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
3490 {
3491         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3492         struct mlxsw_sp_rif_subport *rif_subport;
3493         char ritr_pl[MLXSW_REG_RITR_LEN];
3494
3495         rif_subport = mlxsw_sp_rif_subport_rif(rif);
3496         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
3497                             rif->rif_index, rif->vr_id, rif->dev->mtu,
3498                             rif->dev->dev_addr);
3499         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
3500                                   rif_subport->lag ? rif_subport->lag_id :
3501                                                      rif_subport->system_port,
3502                                   rif_subport->vid);
3503
3504         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3505 }
3506
3507 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
3508 {
3509         return mlxsw_sp_rif_subport_op(rif, true);
3510 }
3511
3512 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
3513 {
3514         mlxsw_sp_rif_subport_op(rif, false);
3515 }
3516
3517 static struct mlxsw_sp_fid *
3518 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
3519 {
3520         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
3521 }
3522
3523 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
3524         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
3525         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
3526         .setup                  = mlxsw_sp_rif_subport_setup,
3527         .configure              = mlxsw_sp_rif_subport_configure,
3528         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
3529         .fid_get                = mlxsw_sp_rif_subport_fid_get,
3530 };
3531
3532 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
3533                                     enum mlxsw_reg_ritr_if_type type,
3534                                     u16 vid_fid, bool enable)
3535 {
3536         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3537         char ritr_pl[MLXSW_REG_RITR_LEN];
3538
3539         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
3540                             rif->dev->mtu, rif->dev->dev_addr);
3541         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
3542
3543         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3544 }
3545
3546 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
3547 {
3548         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
3549 }
3550
3551 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
3552 {
3553         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3554         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3555         int err;
3556
3557         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
3558         if (err)
3559                 return err;
3560
3561         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3562                                      mlxsw_sp_router_port(mlxsw_sp), true);
3563         if (err)
3564                 goto err_fid_bc_flood_set;
3565
3566         return 0;
3567
3568 err_fid_bc_flood_set:
3569         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3570         return err;
3571 }
3572
3573 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
3574 {
3575         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3576         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3577
3578         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3579                                mlxsw_sp_router_port(mlxsw_sp), false);
3580         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3581 }
3582
3583 static struct mlxsw_sp_fid *
3584 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
3585 {
3586         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
3587
3588         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
3589 }
3590
3591 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
3592         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
3593         .rif_size               = sizeof(struct mlxsw_sp_rif),
3594         .configure              = mlxsw_sp_rif_vlan_configure,
3595         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
3596         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
3597 };
3598
3599 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
3600 {
3601         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3602         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3603         int err;
3604
3605         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
3606                                        true);
3607         if (err)
3608                 return err;
3609
3610         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3611                                      mlxsw_sp_router_port(mlxsw_sp), true);
3612         if (err)
3613                 goto err_fid_bc_flood_set;
3614
3615         return 0;
3616
3617 err_fid_bc_flood_set:
3618         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3619         return err;
3620 }
3621
3622 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
3623 {
3624         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3625         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3626
3627         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3628                                mlxsw_sp_router_port(mlxsw_sp), false);
3629         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3630 }
3631
3632 static struct mlxsw_sp_fid *
3633 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
3634 {
3635         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
3636 }
3637
3638 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
3639         .type                   = MLXSW_SP_RIF_TYPE_FID,
3640         .rif_size               = sizeof(struct mlxsw_sp_rif),
3641         .configure              = mlxsw_sp_rif_fid_configure,
3642         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
3643         .fid_get                = mlxsw_sp_rif_fid_fid_get,
3644 };
3645
3646 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
3647         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
3648         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
3649         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
3650 };
3651
3652 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
3653 {
3654         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3655
3656         mlxsw_sp->router->rifs = kcalloc(max_rifs,
3657                                          sizeof(struct mlxsw_sp_rif *),
3658                                          GFP_KERNEL);
3659         if (!mlxsw_sp->router->rifs)
3660                 return -ENOMEM;
3661
3662         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
3663
3664         return 0;
3665 }
3666
3667 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
3668 {
3669         int i;
3670
3671         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3672                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
3673
3674         kfree(mlxsw_sp->router->rifs);
3675 }
3676
3677 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3678 {
3679         struct mlxsw_sp_router *router;
3680
3681         /* Flush pending FIB notifications and then flush the device's
3682          * table before requesting another dump. The FIB notification
3683          * block is unregistered, so no need to take RTNL.
3684          */
3685         mlxsw_core_flush_owq();
3686         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3687         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
3688 }
3689
3690 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3691 {
3692         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3693         u64 max_rifs;
3694         int err;
3695
3696         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3697                 return -EIO;
3698         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3699
3700         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3701         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3702         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3703         if (err)
3704                 return err;
3705         return 0;
3706 }
3707
3708 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3709 {
3710         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3711
3712         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3713         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3714 }
3715
3716 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3717 {
3718         struct mlxsw_sp_router *router;
3719         int err;
3720
3721         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
3722         if (!router)
3723                 return -ENOMEM;
3724         mlxsw_sp->router = router;
3725         router->mlxsw_sp = mlxsw_sp;
3726
3727         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
3728         err = __mlxsw_sp_router_init(mlxsw_sp);
3729         if (err)
3730                 goto err_router_init;
3731
3732         err = mlxsw_sp_rifs_init(mlxsw_sp);
3733         if (err)
3734                 goto err_rifs_init;
3735
3736         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
3737                               &mlxsw_sp_nexthop_ht_params);
3738         if (err)
3739                 goto err_nexthop_ht_init;
3740
3741         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
3742                               &mlxsw_sp_nexthop_group_ht_params);
3743         if (err)
3744                 goto err_nexthop_group_ht_init;
3745
3746         err = mlxsw_sp_lpm_init(mlxsw_sp);
3747         if (err)
3748                 goto err_lpm_init;
3749
3750         err = mlxsw_sp_vrs_init(mlxsw_sp);
3751         if (err)
3752                 goto err_vrs_init;
3753
3754         err = mlxsw_sp_neigh_init(mlxsw_sp);
3755         if (err)
3756                 goto err_neigh_init;
3757
3758         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3759         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
3760                                     mlxsw_sp_router_fib_dump_flush);
3761         if (err)
3762                 goto err_register_fib_notifier;
3763
3764         return 0;
3765
3766 err_register_fib_notifier:
3767         mlxsw_sp_neigh_fini(mlxsw_sp);
3768 err_neigh_init:
3769         mlxsw_sp_vrs_fini(mlxsw_sp);
3770 err_vrs_init:
3771         mlxsw_sp_lpm_fini(mlxsw_sp);
3772 err_lpm_init:
3773         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3774 err_nexthop_group_ht_init:
3775         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3776 err_nexthop_ht_init:
3777         mlxsw_sp_rifs_fini(mlxsw_sp);
3778 err_rifs_init:
3779         __mlxsw_sp_router_fini(mlxsw_sp);
3780 err_router_init:
3781         kfree(mlxsw_sp->router);
3782         return err;
3783 }
3784
3785 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3786 {
3787         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
3788         mlxsw_sp_neigh_fini(mlxsw_sp);
3789         mlxsw_sp_vrs_fini(mlxsw_sp);
3790         mlxsw_sp_lpm_fini(mlxsw_sp);
3791         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3792         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3793         mlxsw_sp_rifs_fini(mlxsw_sp);
3794         __mlxsw_sp_router_fini(mlxsw_sp);
3795         kfree(mlxsw_sp->router);
3796 }