2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/rhashtable.h>
40 #include <linux/bitops.h>
41 #include <linux/in6.h>
42 #include <linux/notifier.h>
43 #include <linux/inetdevice.h>
44 #include <linux/netdevice.h>
45 #include <net/netevent.h>
46 #include <net/neighbour.h>
48 #include <net/ip_fib.h>
49 #include <net/fib_rules.h>
50 #include <net/l3mdev.h>
57 struct list_head nexthop_list;
58 struct list_head neigh_list;
59 struct net_device *dev;
60 struct mlxsw_sp_fid *f;
61 unsigned char addr[ETH_ALEN];
67 static struct mlxsw_sp_rif *
68 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
69 const struct net_device *dev);
71 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
72 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
75 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
76 struct mlxsw_sp_prefix_usage *prefix_usage2)
80 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
81 if (!test_bit(prefix, prefix_usage2->b))
88 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
89 struct mlxsw_sp_prefix_usage *prefix_usage2)
91 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
95 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
97 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
99 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
103 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
104 struct mlxsw_sp_prefix_usage *prefix_usage2)
106 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
110 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
111 unsigned char prefix_len)
113 set_bit(prefix_len, prefix_usage->b);
117 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
118 unsigned char prefix_len)
120 clear_bit(prefix_len, prefix_usage->b);
123 struct mlxsw_sp_fib_key {
124 unsigned char addr[sizeof(struct in6_addr)];
125 unsigned char prefix_len;
128 enum mlxsw_sp_fib_entry_type {
129 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
130 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
131 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
134 struct mlxsw_sp_nexthop_group;
136 struct mlxsw_sp_fib_node {
137 struct list_head entry_list;
138 struct list_head list;
139 struct rhash_head ht_node;
140 struct mlxsw_sp_fib *fib;
141 struct mlxsw_sp_fib_key key;
144 struct mlxsw_sp_fib_entry_params {
151 struct mlxsw_sp_fib_entry {
152 struct list_head list;
153 struct mlxsw_sp_fib_node *fib_node;
154 enum mlxsw_sp_fib_entry_type type;
155 struct list_head nexthop_group_node;
156 struct mlxsw_sp_nexthop_group *nh_group;
157 struct mlxsw_sp_fib_entry_params params;
161 struct mlxsw_sp_fib {
162 struct rhashtable ht;
163 struct list_head node_list;
164 struct mlxsw_sp_vr *vr;
165 struct mlxsw_sp_lpm_tree *lpm_tree;
166 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
167 struct mlxsw_sp_prefix_usage prefix_usage;
168 enum mlxsw_sp_l3proto proto;
171 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
173 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
174 enum mlxsw_sp_l3proto proto)
176 struct mlxsw_sp_fib *fib;
179 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
181 return ERR_PTR(-ENOMEM);
182 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
184 goto err_rhashtable_init;
185 INIT_LIST_HEAD(&fib->node_list);
195 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
197 WARN_ON(!list_empty(&fib->node_list));
198 WARN_ON(fib->lpm_tree);
199 rhashtable_destroy(&fib->ht);
203 static struct mlxsw_sp_lpm_tree *
204 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
206 static struct mlxsw_sp_lpm_tree *lpm_tree;
209 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
210 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
211 if (lpm_tree->ref_count == 0)
217 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
218 struct mlxsw_sp_lpm_tree *lpm_tree)
220 char ralta_pl[MLXSW_REG_RALTA_LEN];
222 mlxsw_reg_ralta_pack(ralta_pl, true,
223 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
225 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
228 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
229 struct mlxsw_sp_lpm_tree *lpm_tree)
231 char ralta_pl[MLXSW_REG_RALTA_LEN];
233 mlxsw_reg_ralta_pack(ralta_pl, false,
234 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
236 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
240 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
241 struct mlxsw_sp_prefix_usage *prefix_usage,
242 struct mlxsw_sp_lpm_tree *lpm_tree)
244 char ralst_pl[MLXSW_REG_RALST_LEN];
247 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
249 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
252 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
253 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
256 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
257 MLXSW_REG_RALST_BIN_NO_CHILD);
258 last_prefix = prefix;
260 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
263 static struct mlxsw_sp_lpm_tree *
264 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
265 struct mlxsw_sp_prefix_usage *prefix_usage,
266 enum mlxsw_sp_l3proto proto)
268 struct mlxsw_sp_lpm_tree *lpm_tree;
271 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
273 return ERR_PTR(-EBUSY);
274 lpm_tree->proto = proto;
275 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
279 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
282 goto err_left_struct_set;
283 memcpy(&lpm_tree->prefix_usage, prefix_usage,
284 sizeof(lpm_tree->prefix_usage));
288 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
292 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
293 struct mlxsw_sp_lpm_tree *lpm_tree)
295 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
298 static struct mlxsw_sp_lpm_tree *
299 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
300 struct mlxsw_sp_prefix_usage *prefix_usage,
301 enum mlxsw_sp_l3proto proto)
303 struct mlxsw_sp_lpm_tree *lpm_tree;
306 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
307 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
308 if (lpm_tree->ref_count != 0 &&
309 lpm_tree->proto == proto &&
310 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
314 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
316 if (IS_ERR(lpm_tree))
320 lpm_tree->ref_count++;
324 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
325 struct mlxsw_sp_lpm_tree *lpm_tree)
327 if (--lpm_tree->ref_count == 0)
328 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
332 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
334 struct mlxsw_sp_lpm_tree *lpm_tree;
337 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
338 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
339 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
343 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
348 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
350 struct mlxsw_sp_vr *vr;
353 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
354 vr = &mlxsw_sp->router.vrs[i];
355 if (!mlxsw_sp_vr_is_used(vr))
361 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
362 const struct mlxsw_sp_fib *fib)
364 char raltb_pl[MLXSW_REG_RALTB_LEN];
366 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
367 (enum mlxsw_reg_ralxx_protocol) fib->proto,
369 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
372 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
373 const struct mlxsw_sp_fib *fib)
375 char raltb_pl[MLXSW_REG_RALTB_LEN];
377 /* Bind to tree 0 which is default */
378 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
379 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
380 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
383 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
385 /* For our purpose, squash main and local table into one */
386 if (tb_id == RT_TABLE_LOCAL)
387 tb_id = RT_TABLE_MAIN;
391 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
394 struct mlxsw_sp_vr *vr;
397 tb_id = mlxsw_sp_fix_tb_id(tb_id);
399 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
400 vr = &mlxsw_sp->router.vrs[i];
401 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
407 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
408 enum mlxsw_sp_l3proto proto)
411 case MLXSW_SP_L3_PROTO_IPV4:
413 case MLXSW_SP_L3_PROTO_IPV6:
419 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
422 struct mlxsw_sp_vr *vr;
424 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
426 return ERR_PTR(-EBUSY);
427 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
428 if (IS_ERR(vr->fib4))
429 return ERR_CAST(vr->fib4);
434 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
436 mlxsw_sp_fib_destroy(vr->fib4);
441 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
442 struct mlxsw_sp_prefix_usage *req_prefix_usage)
444 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
445 struct mlxsw_sp_lpm_tree *new_tree;
448 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
451 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
453 if (IS_ERR(new_tree)) {
454 /* We failed to get a tree according to the required
455 * prefix usage. However, the current tree might be still good
456 * for us if our requirement is subset of the prefixes used
459 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
460 &lpm_tree->prefix_usage))
462 return PTR_ERR(new_tree);
465 /* Prevent packet loss by overwriting existing binding */
466 fib->lpm_tree = new_tree;
467 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
470 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
475 fib->lpm_tree = lpm_tree;
476 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
480 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
482 struct mlxsw_sp_vr *vr;
484 tb_id = mlxsw_sp_fix_tb_id(tb_id);
485 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
487 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
491 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
493 if (!vr->rif_count && list_empty(&vr->fib4->node_list))
494 mlxsw_sp_vr_destroy(vr);
497 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
499 struct mlxsw_sp_vr *vr;
503 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
506 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
507 mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
509 if (!mlxsw_sp->router.vrs)
512 for (i = 0; i < max_vrs; i++) {
513 vr = &mlxsw_sp->router.vrs[i];
520 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
522 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
524 /* At this stage we're guaranteed not to have new incoming
525 * FIB notifications and the work queue is free from FIBs
526 * sitting on top of mlxsw netdevs. However, we can still
527 * have other FIBs queued. Flush the queue before flushing
528 * the device's tables. No need for locks, as we're the only
531 mlxsw_core_flush_owq();
532 mlxsw_sp_router_fib_flush(mlxsw_sp);
533 kfree(mlxsw_sp->router.vrs);
536 struct mlxsw_sp_neigh_key {
540 struct mlxsw_sp_neigh_entry {
541 struct list_head rif_list_node;
542 struct rhash_head ht_node;
543 struct mlxsw_sp_neigh_key key;
546 unsigned char ha[ETH_ALEN];
547 struct list_head nexthop_list; /* list of nexthops using
550 struct list_head nexthop_neighs_list_node;
553 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
554 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
555 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
556 .key_len = sizeof(struct mlxsw_sp_neigh_key),
559 static struct mlxsw_sp_neigh_entry *
560 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
563 struct mlxsw_sp_neigh_entry *neigh_entry;
565 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
569 neigh_entry->key.n = n;
570 neigh_entry->rif = rif;
571 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
576 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
582 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
583 struct mlxsw_sp_neigh_entry *neigh_entry)
585 return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
586 &neigh_entry->ht_node,
587 mlxsw_sp_neigh_ht_params);
591 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
592 struct mlxsw_sp_neigh_entry *neigh_entry)
594 rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
595 &neigh_entry->ht_node,
596 mlxsw_sp_neigh_ht_params);
599 static struct mlxsw_sp_neigh_entry *
600 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
602 struct mlxsw_sp_neigh_entry *neigh_entry;
603 struct mlxsw_sp_rif *r;
606 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
608 return ERR_PTR(-EINVAL);
610 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, r->rif);
612 return ERR_PTR(-ENOMEM);
614 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
616 goto err_neigh_entry_insert;
618 list_add(&neigh_entry->rif_list_node, &r->neigh_list);
622 err_neigh_entry_insert:
623 mlxsw_sp_neigh_entry_free(neigh_entry);
628 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
629 struct mlxsw_sp_neigh_entry *neigh_entry)
631 list_del(&neigh_entry->rif_list_node);
632 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
633 mlxsw_sp_neigh_entry_free(neigh_entry);
636 static struct mlxsw_sp_neigh_entry *
637 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
639 struct mlxsw_sp_neigh_key key;
642 return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
643 &key, mlxsw_sp_neigh_ht_params);
647 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
649 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
651 mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
654 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
658 struct net_device *dev;
664 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
666 if (!mlxsw_sp->rifs[rif]) {
667 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
672 dev = mlxsw_sp->rifs[rif]->dev;
673 n = neigh_lookup(&arp_tbl, &dipn, dev);
675 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
680 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
681 neigh_event_send(n, NULL);
685 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
692 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
694 /* Hardware starts counting at 0, so add 1. */
697 /* Each record consists of several neighbour entries. */
698 for (i = 0; i < num_entries; i++) {
701 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
702 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
708 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
709 char *rauhtd_pl, int rec_index)
711 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
712 case MLXSW_REG_RAUHTD_TYPE_IPV4:
713 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
716 case MLXSW_REG_RAUHTD_TYPE_IPV6:
722 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
724 u8 num_rec, last_rec_index, num_entries;
726 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
727 last_rec_index = num_rec - 1;
729 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
731 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
732 MLXSW_REG_RAUHTD_TYPE_IPV6)
735 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
737 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
742 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
748 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
752 /* Make sure the neighbour's netdev isn't removed in the
757 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
758 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
761 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
764 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
765 for (i = 0; i < num_rec; i++)
766 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
768 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
775 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
777 struct mlxsw_sp_neigh_entry *neigh_entry;
779 /* Take RTNL mutex here to prevent lists from changes */
781 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
782 nexthop_neighs_list_node)
783 /* If this neigh have nexthops, make the kernel think this neigh
784 * is active regardless of the traffic.
786 neigh_event_send(neigh_entry->key.n, NULL);
791 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
793 unsigned long interval = mlxsw_sp->router.neighs_update.interval;
795 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
796 msecs_to_jiffies(interval));
799 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
801 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
802 router.neighs_update.dw.work);
805 err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
807 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
809 mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
811 mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
814 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
816 struct mlxsw_sp_neigh_entry *neigh_entry;
817 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
818 router.nexthop_probe_dw.work);
820 /* Iterate over nexthop neighbours, find those who are unresolved and
821 * send arp on them. This solves the chicken-egg problem when
822 * the nexthop wouldn't get offloaded until the neighbor is resolved
823 * but it wouldn't get resolved ever in case traffic is flowing in HW
824 * using different nexthop.
826 * Take RTNL mutex here to prevent lists from changes.
829 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
830 nexthop_neighs_list_node)
831 if (!neigh_entry->connected)
832 neigh_event_send(neigh_entry->key.n, NULL);
835 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
836 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
840 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
841 struct mlxsw_sp_neigh_entry *neigh_entry,
844 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
846 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
847 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
851 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
852 struct mlxsw_sp_neigh_entry *neigh_entry,
853 enum mlxsw_reg_rauht_op op)
855 struct neighbour *n = neigh_entry->key.n;
856 u32 dip = ntohl(*((__be32 *) n->primary_key));
857 char rauht_pl[MLXSW_REG_RAUHT_LEN];
859 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
861 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
865 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
866 struct mlxsw_sp_neigh_entry *neigh_entry,
869 if (!adding && !neigh_entry->connected)
871 neigh_entry->connected = adding;
872 if (neigh_entry->key.n->tbl == &arp_tbl)
873 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
874 mlxsw_sp_rauht_op(adding));
879 struct mlxsw_sp_neigh_event_work {
880 struct work_struct work;
881 struct mlxsw_sp *mlxsw_sp;
885 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
887 struct mlxsw_sp_neigh_event_work *neigh_work =
888 container_of(work, struct mlxsw_sp_neigh_event_work, work);
889 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
890 struct mlxsw_sp_neigh_entry *neigh_entry;
891 struct neighbour *n = neigh_work->n;
892 unsigned char ha[ETH_ALEN];
893 bool entry_connected;
896 /* If these parameters are changed after we release the lock,
897 * then we are guaranteed to receive another event letting us
900 read_lock_bh(&n->lock);
901 memcpy(ha, n->ha, ETH_ALEN);
902 nud_state = n->nud_state;
904 read_unlock_bh(&n->lock);
907 entry_connected = nud_state & NUD_VALID && !dead;
908 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
909 if (!entry_connected && !neigh_entry)
912 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
913 if (IS_ERR(neigh_entry))
917 memcpy(neigh_entry->ha, ha, ETH_ALEN);
918 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
919 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
921 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
922 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
930 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
931 unsigned long event, void *ptr)
933 struct mlxsw_sp_neigh_event_work *neigh_work;
934 struct mlxsw_sp_port *mlxsw_sp_port;
935 struct mlxsw_sp *mlxsw_sp;
936 unsigned long interval;
937 struct neigh_parms *p;
941 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
944 /* We don't care about changes in the default table. */
945 if (!p->dev || p->tbl != &arp_tbl)
948 /* We are in atomic context and can't take RTNL mutex,
949 * so use RCU variant to walk the device chain.
951 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
955 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
956 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
957 mlxsw_sp->router.neighs_update.interval = interval;
959 mlxsw_sp_port_dev_put(mlxsw_sp_port);
961 case NETEVENT_NEIGH_UPDATE:
964 if (n->tbl != &arp_tbl)
967 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
971 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
973 mlxsw_sp_port_dev_put(mlxsw_sp_port);
977 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
978 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
981 /* Take a reference to ensure the neighbour won't be
982 * destructed until we drop the reference in delayed
986 mlxsw_core_schedule_work(&neigh_work->work);
987 mlxsw_sp_port_dev_put(mlxsw_sp_port);
994 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
998 err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
999 &mlxsw_sp_neigh_ht_params);
1003 /* Initialize the polling interval according to the default
1006 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1008 /* Create the delayed works for the activity_update */
1009 INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1010 mlxsw_sp_router_neighs_update_work);
1011 INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1012 mlxsw_sp_router_probe_unresolved_nexthops);
1013 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1014 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1018 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1020 cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1021 cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1022 rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1025 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1026 const struct mlxsw_sp_rif *r)
1028 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1030 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1032 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1035 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1036 struct mlxsw_sp_rif *r)
1038 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1040 mlxsw_sp_neigh_rif_flush(mlxsw_sp, r);
1041 list_for_each_entry_safe(neigh_entry, tmp, &r->neigh_list,
1043 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1046 struct mlxsw_sp_nexthop_key {
1047 struct fib_nh *fib_nh;
1050 struct mlxsw_sp_nexthop {
1051 struct list_head neigh_list_node; /* member of neigh entry list */
1052 struct list_head rif_list_node;
1053 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1056 struct rhash_head ht_node;
1057 struct mlxsw_sp_nexthop_key key;
1058 struct mlxsw_sp_rif *r;
1059 u8 should_offload:1, /* set indicates this neigh is connected and
1060 * should be put to KVD linear area of this group.
1062 offloaded:1, /* set in case the neigh is actually put into
1063 * KVD linear area of this group.
1065 update:1; /* set indicates that MAC of this neigh should be
1068 struct mlxsw_sp_neigh_entry *neigh_entry;
1071 struct mlxsw_sp_nexthop_group_key {
1072 struct fib_info *fi;
1075 struct mlxsw_sp_nexthop_group {
1076 struct rhash_head ht_node;
1077 struct list_head fib_list; /* list of fib entries that use this group */
1078 struct mlxsw_sp_nexthop_group_key key;
1079 u8 adj_index_valid:1,
1080 gateway:1; /* routes using the group use a gateway */
1084 struct mlxsw_sp_nexthop nexthops[0];
1085 #define nh_rif nexthops[0].r
1088 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1089 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1090 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1091 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1094 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1095 struct mlxsw_sp_nexthop_group *nh_grp)
1097 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1099 mlxsw_sp_nexthop_group_ht_params);
1102 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1103 struct mlxsw_sp_nexthop_group *nh_grp)
1105 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1107 mlxsw_sp_nexthop_group_ht_params);
1110 static struct mlxsw_sp_nexthop_group *
1111 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1112 struct mlxsw_sp_nexthop_group_key key)
1114 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1115 mlxsw_sp_nexthop_group_ht_params);
1118 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1119 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1120 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1121 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1124 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1125 struct mlxsw_sp_nexthop *nh)
1127 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1128 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1131 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1132 struct mlxsw_sp_nexthop *nh)
1134 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1135 mlxsw_sp_nexthop_ht_params);
1138 static struct mlxsw_sp_nexthop *
1139 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1140 struct mlxsw_sp_nexthop_key key)
1142 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1143 mlxsw_sp_nexthop_ht_params);
1146 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1147 const struct mlxsw_sp_fib *fib,
1148 u32 adj_index, u16 ecmp_size,
1152 char raleu_pl[MLXSW_REG_RALEU_LEN];
1154 mlxsw_reg_raleu_pack(raleu_pl,
1155 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1156 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1158 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1161 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1162 struct mlxsw_sp_nexthop_group *nh_grp,
1163 u32 old_adj_index, u16 old_ecmp_size)
1165 struct mlxsw_sp_fib_entry *fib_entry;
1166 struct mlxsw_sp_fib *fib = NULL;
1169 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1170 if (fib == fib_entry->fib_node->fib)
1172 fib = fib_entry->fib_node->fib;
1173 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1184 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1185 struct mlxsw_sp_nexthop *nh)
1187 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1188 char ratr_pl[MLXSW_REG_RATR_LEN];
1190 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1191 true, adj_index, neigh_entry->rif);
1192 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1197 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1198 struct mlxsw_sp_nexthop_group *nh_grp,
1201 u32 adj_index = nh_grp->adj_index; /* base */
1202 struct mlxsw_sp_nexthop *nh;
1206 for (i = 0; i < nh_grp->count; i++) {
1207 nh = &nh_grp->nexthops[i];
1209 if (!nh->should_offload) {
1214 if (nh->update || reallocate) {
1215 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1227 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1228 struct mlxsw_sp_fib_entry *fib_entry);
1231 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1232 struct mlxsw_sp_nexthop_group *nh_grp)
1234 struct mlxsw_sp_fib_entry *fib_entry;
1237 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1238 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1246 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1247 struct mlxsw_sp_nexthop_group *nh_grp)
1249 struct mlxsw_sp_nexthop *nh;
1250 bool offload_change = false;
1253 bool old_adj_index_valid;
1260 if (!nh_grp->gateway) {
1261 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1265 for (i = 0; i < nh_grp->count; i++) {
1266 nh = &nh_grp->nexthops[i];
1268 if (nh->should_offload ^ nh->offloaded) {
1269 offload_change = true;
1270 if (nh->should_offload)
1273 if (nh->should_offload)
1276 if (!offload_change) {
1277 /* Nothing was added or removed, so no need to reallocate. Just
1278 * update MAC on existing adjacency indexes.
1280 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1283 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1289 /* No neigh of this group is connected so we just set
1290 * the trap and let everthing flow through kernel.
1294 ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1296 /* We ran out of KVD linear space, just set the
1297 * trap and let everything flow through kernel.
1299 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1303 old_adj_index_valid = nh_grp->adj_index_valid;
1304 old_adj_index = nh_grp->adj_index;
1305 old_ecmp_size = nh_grp->ecmp_size;
1306 nh_grp->adj_index_valid = 1;
1307 nh_grp->adj_index = adj_index;
1308 nh_grp->ecmp_size = ecmp_size;
1309 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1311 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1315 if (!old_adj_index_valid) {
1316 /* The trap was set for fib entries, so we have to call
1317 * fib entry update to unset it and use adjacency index.
1319 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1321 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1327 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1328 old_adj_index, old_ecmp_size);
1329 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1331 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1337 old_adj_index_valid = nh_grp->adj_index_valid;
1338 nh_grp->adj_index_valid = 0;
1339 for (i = 0; i < nh_grp->count; i++) {
1340 nh = &nh_grp->nexthops[i];
1343 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1345 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1346 if (old_adj_index_valid)
1347 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1350 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1353 if (!removing && !nh->should_offload)
1354 nh->should_offload = 1;
1355 else if (removing && nh->offloaded)
1356 nh->should_offload = 0;
1361 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1362 struct mlxsw_sp_neigh_entry *neigh_entry,
1365 struct mlxsw_sp_nexthop *nh;
1367 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1369 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1370 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1374 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1375 struct mlxsw_sp_rif *r)
1381 list_add(&nh->rif_list_node, &r->nexthop_list);
1384 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1389 list_del(&nh->rif_list_node);
1393 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1394 struct mlxsw_sp_nexthop *nh)
1396 struct mlxsw_sp_neigh_entry *neigh_entry;
1397 struct fib_nh *fib_nh = nh->key.fib_nh;
1398 struct neighbour *n;
1402 if (!nh->nh_grp->gateway || nh->neigh_entry)
1405 /* Take a reference of neigh here ensuring that neigh would
1406 * not be detructed before the nexthop entry is finished.
1407 * The reference is taken either in neigh_lookup() or
1408 * in neigh_create() in case n is not found.
1410 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1412 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1415 neigh_event_send(n, NULL);
1417 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1419 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1420 if (IS_ERR(neigh_entry)) {
1422 goto err_neigh_entry_create;
1426 /* If that is the first nexthop connected to that neigh, add to
1427 * nexthop_neighs_list
1429 if (list_empty(&neigh_entry->nexthop_list))
1430 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1431 &mlxsw_sp->router.nexthop_neighs_list);
1433 nh->neigh_entry = neigh_entry;
1434 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1435 read_lock_bh(&n->lock);
1436 nud_state = n->nud_state;
1438 read_unlock_bh(&n->lock);
1439 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1443 err_neigh_entry_create:
1448 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1449 struct mlxsw_sp_nexthop *nh)
1451 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1452 struct neighbour *n;
1456 n = neigh_entry->key.n;
1458 __mlxsw_sp_nexthop_neigh_update(nh, true);
1459 list_del(&nh->neigh_list_node);
1460 nh->neigh_entry = NULL;
1462 /* If that is the last nexthop connected to that neigh, remove from
1463 * nexthop_neighs_list
1465 if (list_empty(&neigh_entry->nexthop_list))
1466 list_del(&neigh_entry->nexthop_neighs_list_node);
1468 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1469 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1474 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1475 struct mlxsw_sp_nexthop_group *nh_grp,
1476 struct mlxsw_sp_nexthop *nh,
1477 struct fib_nh *fib_nh)
1479 struct net_device *dev = fib_nh->nh_dev;
1480 struct in_device *in_dev;
1481 struct mlxsw_sp_rif *r;
1484 nh->nh_grp = nh_grp;
1485 nh->key.fib_nh = fib_nh;
1486 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1493 in_dev = __in_dev_get_rtnl(dev);
1494 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1495 fib_nh->nh_flags & RTNH_F_LINKDOWN)
1498 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1501 mlxsw_sp_nexthop_rif_init(nh, r);
1503 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1505 goto err_nexthop_neigh_init;
1509 err_nexthop_neigh_init:
1510 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1514 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1515 struct mlxsw_sp_nexthop *nh)
1517 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1518 mlxsw_sp_nexthop_rif_fini(nh);
1519 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1522 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1523 unsigned long event, struct fib_nh *fib_nh)
1525 struct mlxsw_sp_nexthop_key key;
1526 struct mlxsw_sp_nexthop *nh;
1527 struct mlxsw_sp_rif *r;
1529 if (mlxsw_sp->router.aborted)
1532 key.fib_nh = fib_nh;
1533 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1534 if (WARN_ON_ONCE(!nh))
1537 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1542 case FIB_EVENT_NH_ADD:
1543 mlxsw_sp_nexthop_rif_init(nh, r);
1544 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1546 case FIB_EVENT_NH_DEL:
1547 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1548 mlxsw_sp_nexthop_rif_fini(nh);
1552 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1555 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1556 struct mlxsw_sp_rif *r)
1558 struct mlxsw_sp_nexthop *nh, *tmp;
1560 list_for_each_entry_safe(nh, tmp, &r->nexthop_list, rif_list_node) {
1561 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1562 mlxsw_sp_nexthop_rif_fini(nh);
1563 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1567 static struct mlxsw_sp_nexthop_group *
1568 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1570 struct mlxsw_sp_nexthop_group *nh_grp;
1571 struct mlxsw_sp_nexthop *nh;
1572 struct fib_nh *fib_nh;
1577 alloc_size = sizeof(*nh_grp) +
1578 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1579 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1581 return ERR_PTR(-ENOMEM);
1582 INIT_LIST_HEAD(&nh_grp->fib_list);
1583 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1584 nh_grp->count = fi->fib_nhs;
1585 nh_grp->key.fi = fi;
1586 for (i = 0; i < nh_grp->count; i++) {
1587 nh = &nh_grp->nexthops[i];
1588 fib_nh = &fi->fib_nh[i];
1589 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1591 goto err_nexthop_init;
1593 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1595 goto err_nexthop_group_insert;
1596 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1599 err_nexthop_group_insert:
1601 for (i--; i >= 0; i--) {
1602 nh = &nh_grp->nexthops[i];
1603 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1606 return ERR_PTR(err);
1610 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1611 struct mlxsw_sp_nexthop_group *nh_grp)
1613 struct mlxsw_sp_nexthop *nh;
1616 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1617 for (i = 0; i < nh_grp->count; i++) {
1618 nh = &nh_grp->nexthops[i];
1619 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1621 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1622 WARN_ON_ONCE(nh_grp->adj_index_valid);
1626 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1627 struct mlxsw_sp_fib_entry *fib_entry,
1628 struct fib_info *fi)
1630 struct mlxsw_sp_nexthop_group_key key;
1631 struct mlxsw_sp_nexthop_group *nh_grp;
1634 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1636 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1638 return PTR_ERR(nh_grp);
1640 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1641 fib_entry->nh_group = nh_grp;
1645 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1646 struct mlxsw_sp_fib_entry *fib_entry)
1648 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1650 list_del(&fib_entry->nexthop_group_node);
1651 if (!list_empty(&nh_grp->fib_list))
1653 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1657 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1659 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1661 if (fib_entry->params.tos)
1664 switch (fib_entry->type) {
1665 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1666 return !!nh_group->adj_index_valid;
1667 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1668 return !!nh_group->nh_rif;
1674 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1676 fib_entry->offloaded = true;
1678 switch (fib_entry->fib_node->fib->proto) {
1679 case MLXSW_SP_L3_PROTO_IPV4:
1680 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1682 case MLXSW_SP_L3_PROTO_IPV6:
1688 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1690 switch (fib_entry->fib_node->fib->proto) {
1691 case MLXSW_SP_L3_PROTO_IPV4:
1692 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1694 case MLXSW_SP_L3_PROTO_IPV6:
1698 fib_entry->offloaded = false;
1702 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1703 enum mlxsw_reg_ralue_op op, int err)
1706 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1707 if (!fib_entry->offloaded)
1709 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1710 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1713 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1714 !fib_entry->offloaded)
1715 mlxsw_sp_fib_entry_offload_set(fib_entry);
1716 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1717 fib_entry->offloaded)
1718 mlxsw_sp_fib_entry_offload_unset(fib_entry);
1725 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1726 struct mlxsw_sp_fib_entry *fib_entry,
1727 enum mlxsw_reg_ralue_op op)
1729 char ralue_pl[MLXSW_REG_RALUE_LEN];
1730 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1731 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1732 enum mlxsw_reg_ralue_trap_action trap_action;
1734 u32 adjacency_index = 0;
1737 /* In case the nexthop group adjacency index is valid, use it
1738 * with provided ECMP size. Otherwise, setup trap and pass
1739 * traffic to kernel.
1741 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1742 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1743 adjacency_index = fib_entry->nh_group->adj_index;
1744 ecmp_size = fib_entry->nh_group->ecmp_size;
1746 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1747 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1750 mlxsw_reg_ralue_pack4(ralue_pl,
1751 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1752 fib->vr->id, fib_entry->fib_node->key.prefix_len,
1754 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1755 adjacency_index, ecmp_size);
1756 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1759 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1760 struct mlxsw_sp_fib_entry *fib_entry,
1761 enum mlxsw_reg_ralue_op op)
1763 struct mlxsw_sp_rif *r = fib_entry->nh_group->nh_rif;
1764 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1765 enum mlxsw_reg_ralue_trap_action trap_action;
1766 char ralue_pl[MLXSW_REG_RALUE_LEN];
1767 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1771 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1772 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1775 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1776 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1779 mlxsw_reg_ralue_pack4(ralue_pl,
1780 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1781 fib->vr->id, fib_entry->fib_node->key.prefix_len,
1783 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, rif);
1784 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1787 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1788 struct mlxsw_sp_fib_entry *fib_entry,
1789 enum mlxsw_reg_ralue_op op)
1791 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1792 char ralue_pl[MLXSW_REG_RALUE_LEN];
1793 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1795 mlxsw_reg_ralue_pack4(ralue_pl,
1796 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1797 fib->vr->id, fib_entry->fib_node->key.prefix_len,
1799 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1800 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1803 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1804 struct mlxsw_sp_fib_entry *fib_entry,
1805 enum mlxsw_reg_ralue_op op)
1807 switch (fib_entry->type) {
1808 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1809 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1810 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1811 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1812 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1813 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1818 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1819 struct mlxsw_sp_fib_entry *fib_entry,
1820 enum mlxsw_reg_ralue_op op)
1824 switch (fib_entry->fib_node->fib->proto) {
1825 case MLXSW_SP_L3_PROTO_IPV4:
1826 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1828 case MLXSW_SP_L3_PROTO_IPV6:
1831 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1835 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1836 struct mlxsw_sp_fib_entry *fib_entry)
1838 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1839 MLXSW_REG_RALUE_OP_WRITE_WRITE);
1842 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1843 struct mlxsw_sp_fib_entry *fib_entry)
1845 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1846 MLXSW_REG_RALUE_OP_WRITE_DELETE);
1850 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1851 const struct fib_entry_notifier_info *fen_info,
1852 struct mlxsw_sp_fib_entry *fib_entry)
1854 struct fib_info *fi = fen_info->fi;
1856 switch (fen_info->type) {
1857 case RTN_BROADCAST: /* fall through */
1859 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1861 case RTN_UNREACHABLE: /* fall through */
1862 case RTN_BLACKHOLE: /* fall through */
1864 /* Packets hitting these routes need to be trapped, but
1865 * can do so with a lower priority than packets directed
1866 * at the host, so use action type local instead of trap.
1868 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1871 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1872 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1874 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1881 static struct mlxsw_sp_fib_entry *
1882 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1883 struct mlxsw_sp_fib_node *fib_node,
1884 const struct fib_entry_notifier_info *fen_info)
1886 struct mlxsw_sp_fib_entry *fib_entry;
1889 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1892 goto err_fib_entry_alloc;
1895 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1897 goto err_fib4_entry_type_set;
1899 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1901 goto err_nexthop_group_get;
1903 fib_entry->params.prio = fen_info->fi->fib_priority;
1904 fib_entry->params.tb_id = fen_info->tb_id;
1905 fib_entry->params.type = fen_info->type;
1906 fib_entry->params.tos = fen_info->tos;
1908 fib_entry->fib_node = fib_node;
1912 err_nexthop_group_get:
1913 err_fib4_entry_type_set:
1915 err_fib_entry_alloc:
1916 return ERR_PTR(err);
1919 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1920 struct mlxsw_sp_fib_entry *fib_entry)
1922 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1926 static struct mlxsw_sp_fib_node *
1927 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1928 const struct fib_entry_notifier_info *fen_info);
1930 static struct mlxsw_sp_fib_entry *
1931 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1932 const struct fib_entry_notifier_info *fen_info)
1934 struct mlxsw_sp_fib_entry *fib_entry;
1935 struct mlxsw_sp_fib_node *fib_node;
1937 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1938 if (IS_ERR(fib_node))
1941 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1942 if (fib_entry->params.tb_id == fen_info->tb_id &&
1943 fib_entry->params.tos == fen_info->tos &&
1944 fib_entry->params.type == fen_info->type &&
1945 fib_entry->nh_group->key.fi == fen_info->fi) {
1953 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1954 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1955 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1956 .key_len = sizeof(struct mlxsw_sp_fib_key),
1957 .automatic_shrinking = true,
1960 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1961 struct mlxsw_sp_fib_node *fib_node)
1963 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1964 mlxsw_sp_fib_ht_params);
1967 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1968 struct mlxsw_sp_fib_node *fib_node)
1970 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1971 mlxsw_sp_fib_ht_params);
1974 static struct mlxsw_sp_fib_node *
1975 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1976 size_t addr_len, unsigned char prefix_len)
1978 struct mlxsw_sp_fib_key key;
1980 memset(&key, 0, sizeof(key));
1981 memcpy(key.addr, addr, addr_len);
1982 key.prefix_len = prefix_len;
1983 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
1986 static struct mlxsw_sp_fib_node *
1987 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
1988 size_t addr_len, unsigned char prefix_len)
1990 struct mlxsw_sp_fib_node *fib_node;
1992 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
1996 INIT_LIST_HEAD(&fib_node->entry_list);
1997 list_add(&fib_node->list, &fib->node_list);
1998 memcpy(fib_node->key.addr, addr, addr_len);
1999 fib_node->key.prefix_len = prefix_len;
2004 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2006 list_del(&fib_node->list);
2007 WARN_ON(!list_empty(&fib_node->entry_list));
2012 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2013 const struct mlxsw_sp_fib_entry *fib_entry)
2015 return list_first_entry(&fib_node->entry_list,
2016 struct mlxsw_sp_fib_entry, list) == fib_entry;
2019 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2021 unsigned char prefix_len = fib_node->key.prefix_len;
2022 struct mlxsw_sp_fib *fib = fib_node->fib;
2024 if (fib->prefix_ref_count[prefix_len]++ == 0)
2025 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2028 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2030 unsigned char prefix_len = fib_node->key.prefix_len;
2031 struct mlxsw_sp_fib *fib = fib_node->fib;
2033 if (--fib->prefix_ref_count[prefix_len] == 0)
2034 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2037 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2038 struct mlxsw_sp_fib_node *fib_node,
2039 struct mlxsw_sp_fib *fib)
2041 struct mlxsw_sp_prefix_usage req_prefix_usage;
2042 struct mlxsw_sp_lpm_tree *lpm_tree;
2045 err = mlxsw_sp_fib_node_insert(fib, fib_node);
2048 fib_node->fib = fib;
2050 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2051 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2053 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2054 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2057 goto err_tree_check;
2059 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2061 if (IS_ERR(lpm_tree))
2062 return PTR_ERR(lpm_tree);
2063 fib->lpm_tree = lpm_tree;
2064 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2069 mlxsw_sp_fib_node_prefix_inc(fib_node);
2074 fib->lpm_tree = NULL;
2075 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2077 fib_node->fib = NULL;
2078 mlxsw_sp_fib_node_remove(fib, fib_node);
2082 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2083 struct mlxsw_sp_fib_node *fib_node)
2085 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2086 struct mlxsw_sp_fib *fib = fib_node->fib;
2088 mlxsw_sp_fib_node_prefix_dec(fib_node);
2090 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2091 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2092 fib->lpm_tree = NULL;
2093 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2095 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2098 fib_node->fib = NULL;
2099 mlxsw_sp_fib_node_remove(fib, fib_node);
2102 static struct mlxsw_sp_fib_node *
2103 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2104 const struct fib_entry_notifier_info *fen_info)
2106 struct mlxsw_sp_fib_node *fib_node;
2107 struct mlxsw_sp_fib *fib;
2108 struct mlxsw_sp_vr *vr;
2111 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2113 return ERR_CAST(vr);
2114 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2116 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2117 sizeof(fen_info->dst),
2122 fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2123 sizeof(fen_info->dst),
2127 goto err_fib_node_create;
2130 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2132 goto err_fib_node_init;
2137 mlxsw_sp_fib_node_destroy(fib_node);
2138 err_fib_node_create:
2139 mlxsw_sp_vr_put(vr);
2140 return ERR_PTR(err);
2143 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2144 struct mlxsw_sp_fib_node *fib_node)
2146 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2148 if (!list_empty(&fib_node->entry_list))
2150 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2151 mlxsw_sp_fib_node_destroy(fib_node);
2152 mlxsw_sp_vr_put(vr);
2155 static struct mlxsw_sp_fib_entry *
2156 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2157 const struct mlxsw_sp_fib_entry_params *params)
2159 struct mlxsw_sp_fib_entry *fib_entry;
2161 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2162 if (fib_entry->params.tb_id > params->tb_id)
2164 if (fib_entry->params.tb_id != params->tb_id)
2166 if (fib_entry->params.tos > params->tos)
2168 if (fib_entry->params.prio >= params->prio ||
2169 fib_entry->params.tos < params->tos)
2176 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2177 struct mlxsw_sp_fib_entry *new_entry)
2179 struct mlxsw_sp_fib_node *fib_node;
2181 if (WARN_ON(!fib_entry))
2184 fib_node = fib_entry->fib_node;
2185 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2186 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2187 fib_entry->params.tos != new_entry->params.tos ||
2188 fib_entry->params.prio != new_entry->params.prio)
2192 list_add_tail(&new_entry->list, &fib_entry->list);
2197 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2198 struct mlxsw_sp_fib_entry *new_entry,
2199 bool replace, bool append)
2201 struct mlxsw_sp_fib_entry *fib_entry;
2203 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2206 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2207 if (replace && WARN_ON(!fib_entry))
2210 /* Insert new entry before replaced one, so that we can later
2211 * remove the second.
2214 list_add_tail(&new_entry->list, &fib_entry->list);
2216 struct mlxsw_sp_fib_entry *last;
2218 list_for_each_entry(last, &fib_node->entry_list, list) {
2219 if (new_entry->params.tb_id > last->params.tb_id)
2225 list_add(&new_entry->list, &fib_entry->list);
2227 list_add(&new_entry->list, &fib_node->entry_list);
2234 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2236 list_del(&fib_entry->list);
2240 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2241 const struct mlxsw_sp_fib_node *fib_node,
2242 struct mlxsw_sp_fib_entry *fib_entry)
2244 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2247 /* To prevent packet loss, overwrite the previously offloaded
2250 if (!list_is_singular(&fib_node->entry_list)) {
2251 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2252 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2254 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2257 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2261 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2262 const struct mlxsw_sp_fib_node *fib_node,
2263 struct mlxsw_sp_fib_entry *fib_entry)
2265 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2268 /* Promote the next entry by overwriting the deleted entry */
2269 if (!list_is_singular(&fib_node->entry_list)) {
2270 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2271 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2273 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2274 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2278 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2281 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2282 struct mlxsw_sp_fib_entry *fib_entry,
2283 bool replace, bool append)
2285 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2288 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2293 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2295 goto err_fib4_node_entry_add;
2299 err_fib4_node_entry_add:
2300 mlxsw_sp_fib4_node_list_remove(fib_entry);
2305 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2306 struct mlxsw_sp_fib_entry *fib_entry)
2308 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2310 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2311 mlxsw_sp_fib4_node_list_remove(fib_entry);
2314 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2315 struct mlxsw_sp_fib_entry *fib_entry,
2318 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2319 struct mlxsw_sp_fib_entry *replaced;
2324 /* We inserted the new entry before replaced one */
2325 replaced = list_next_entry(fib_entry, list);
2327 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2328 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2329 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2333 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2334 const struct fib_entry_notifier_info *fen_info,
2335 bool replace, bool append)
2337 struct mlxsw_sp_fib_entry *fib_entry;
2338 struct mlxsw_sp_fib_node *fib_node;
2341 if (mlxsw_sp->router.aborted)
2344 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2345 if (IS_ERR(fib_node)) {
2346 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2347 return PTR_ERR(fib_node);
2350 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2351 if (IS_ERR(fib_entry)) {
2352 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2353 err = PTR_ERR(fib_entry);
2354 goto err_fib4_entry_create;
2357 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2360 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2361 goto err_fib4_node_entry_link;
2364 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2368 err_fib4_node_entry_link:
2369 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2370 err_fib4_entry_create:
2371 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2375 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2376 struct fib_entry_notifier_info *fen_info)
2378 struct mlxsw_sp_fib_entry *fib_entry;
2379 struct mlxsw_sp_fib_node *fib_node;
2381 if (mlxsw_sp->router.aborted)
2384 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2385 if (WARN_ON(!fib_entry))
2387 fib_node = fib_entry->fib_node;
2389 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2390 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2391 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2394 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2396 char ralta_pl[MLXSW_REG_RALTA_LEN];
2397 char ralst_pl[MLXSW_REG_RALST_LEN];
2400 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2401 MLXSW_SP_LPM_TREE_MIN);
2402 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2406 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2407 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2411 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2412 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2413 char raltb_pl[MLXSW_REG_RALTB_LEN];
2414 char ralue_pl[MLXSW_REG_RALUE_LEN];
2416 if (!mlxsw_sp_vr_is_used(vr))
2419 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2420 MLXSW_REG_RALXX_PROTOCOL_IPV4,
2421 MLXSW_SP_LPM_TREE_MIN);
2422 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2427 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2428 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2430 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2431 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2440 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2441 struct mlxsw_sp_fib_node *fib_node)
2443 struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2445 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2446 bool do_break = &tmp->list == &fib_node->entry_list;
2448 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2449 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2450 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2451 /* Break when entry list is empty and node was freed.
2452 * Otherwise, we'll access freed memory in the next
2460 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2461 struct mlxsw_sp_fib_node *fib_node)
2463 switch (fib_node->fib->proto) {
2464 case MLXSW_SP_L3_PROTO_IPV4:
2465 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2467 case MLXSW_SP_L3_PROTO_IPV6:
2473 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2474 struct mlxsw_sp_vr *vr,
2475 enum mlxsw_sp_l3proto proto)
2477 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2478 struct mlxsw_sp_fib_node *fib_node, *tmp;
2480 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2481 bool do_break = &tmp->list == &fib->node_list;
2483 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2489 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2493 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2494 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2496 if (!mlxsw_sp_vr_is_used(vr))
2498 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2502 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2506 if (mlxsw_sp->router.aborted)
2508 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2509 mlxsw_sp_router_fib_flush(mlxsw_sp);
2510 mlxsw_sp->router.aborted = true;
2511 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2513 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2516 struct mlxsw_sp_fib_event_work {
2517 struct work_struct work;
2519 struct fib_entry_notifier_info fen_info;
2520 struct fib_rule_notifier_info fr_info;
2521 struct fib_nh_notifier_info fnh_info;
2523 struct mlxsw_sp *mlxsw_sp;
2524 unsigned long event;
2527 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2529 struct mlxsw_sp_fib_event_work *fib_work =
2530 container_of(work, struct mlxsw_sp_fib_event_work, work);
2531 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2532 struct fib_rule *rule;
2533 bool replace, append;
2536 /* Protect internal structures from changes */
2538 switch (fib_work->event) {
2539 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2540 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2541 case FIB_EVENT_ENTRY_ADD:
2542 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2543 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2544 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2547 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2548 fib_info_put(fib_work->fen_info.fi);
2550 case FIB_EVENT_ENTRY_DEL:
2551 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2552 fib_info_put(fib_work->fen_info.fi);
2554 case FIB_EVENT_RULE_ADD: /* fall through */
2555 case FIB_EVENT_RULE_DEL:
2556 rule = fib_work->fr_info.rule;
2557 if (!fib4_rule_default(rule))
2558 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2561 case FIB_EVENT_NH_ADD: /* fall through */
2562 case FIB_EVENT_NH_DEL:
2563 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2564 fib_work->fnh_info.fib_nh);
2565 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2572 /* Called with rcu_read_lock() */
2573 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2574 unsigned long event, void *ptr)
2576 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2577 struct mlxsw_sp_fib_event_work *fib_work;
2578 struct fib_notifier_info *info = ptr;
2580 if (!net_eq(info->net, &init_net))
2583 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2584 if (WARN_ON(!fib_work))
2587 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2588 fib_work->mlxsw_sp = mlxsw_sp;
2589 fib_work->event = event;
2592 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2593 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2594 case FIB_EVENT_ENTRY_ADD: /* fall through */
2595 case FIB_EVENT_ENTRY_DEL:
2596 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2597 /* Take referece on fib_info to prevent it from being
2598 * freed while work is queued. Release it afterwards.
2600 fib_info_hold(fib_work->fen_info.fi);
2602 case FIB_EVENT_RULE_ADD: /* fall through */
2603 case FIB_EVENT_RULE_DEL:
2604 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2605 fib_rule_get(fib_work->fr_info.rule);
2607 case FIB_EVENT_NH_ADD: /* fall through */
2608 case FIB_EVENT_NH_DEL:
2609 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2610 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2614 mlxsw_core_schedule_work(&fib_work->work);
2619 static struct mlxsw_sp_rif *
2620 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2621 const struct net_device *dev)
2625 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2626 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
2627 return mlxsw_sp->rifs[i];
2632 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2634 char ritr_pl[MLXSW_REG_RITR_LEN];
2637 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2638 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2639 if (WARN_ON_ONCE(err))
2642 mlxsw_reg_ritr_enable_set(ritr_pl, false);
2643 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2646 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2647 struct mlxsw_sp_rif *r)
2649 mlxsw_sp_router_rif_disable(mlxsw_sp, r->rif);
2650 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, r);
2651 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, r);
2654 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
2655 const struct in_device *in_dev,
2656 unsigned long event)
2664 if (r && !in_dev->ifa_list && !netif_is_l3_slave(r->dev))
2666 /* It is possible we already removed the RIF ourselves
2667 * if it was assigned to a netdev that is now a bridge
2676 #define MLXSW_SP_INVALID_RIF 0xffff
2677 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
2681 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2682 if (!mlxsw_sp->rifs[i])
2685 return MLXSW_SP_INVALID_RIF;
2688 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
2689 bool *p_lagged, u16 *p_system_port)
2691 u8 local_port = mlxsw_sp_vport->local_port;
2693 *p_lagged = mlxsw_sp_vport->lagged;
2694 *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
2697 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
2698 u16 vr_id, struct net_device *l3_dev,
2699 u16 rif, bool create)
2701 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2702 bool lagged = mlxsw_sp_vport->lagged;
2703 char ritr_pl[MLXSW_REG_RITR_LEN];
2706 mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif, vr_id,
2707 l3_dev->mtu, l3_dev->dev_addr);
2709 mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
2710 mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
2711 mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
2713 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2716 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
2718 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif)
2720 return MLXSW_SP_RFID_BASE + rif;
2723 static struct mlxsw_sp_fid *
2724 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
2726 struct mlxsw_sp_fid *f;
2728 f = kzalloc(sizeof(*f), GFP_KERNEL);
2732 f->leave = mlxsw_sp_vport_rif_sp_leave;
2740 static struct mlxsw_sp_rif *
2741 mlxsw_sp_rif_alloc(u16 rif, u16 vr_id, struct net_device *l3_dev,
2742 struct mlxsw_sp_fid *f)
2744 struct mlxsw_sp_rif *r;
2746 r = kzalloc(sizeof(*r), GFP_KERNEL);
2750 INIT_LIST_HEAD(&r->nexthop_list);
2751 INIT_LIST_HEAD(&r->neigh_list);
2752 ether_addr_copy(r->addr, l3_dev->dev_addr);
2753 r->mtu = l3_dev->mtu;
2762 static struct mlxsw_sp_rif *
2763 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
2764 struct net_device *l3_dev)
2766 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2767 u32 tb_id = l3mdev_fib_table(l3_dev);
2768 struct mlxsw_sp_vr *vr;
2769 struct mlxsw_sp_fid *f;
2770 struct mlxsw_sp_rif *r;
2774 rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
2775 if (rif == MLXSW_SP_INVALID_RIF)
2776 return ERR_PTR(-ERANGE);
2778 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
2780 return ERR_CAST(vr);
2782 err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif,
2785 goto err_vport_rif_sp_op;
2787 fid = mlxsw_sp_rif_sp_to_fid(rif);
2788 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
2790 goto err_rif_fdb_op;
2792 f = mlxsw_sp_rfid_alloc(fid, l3_dev);
2795 goto err_rfid_alloc;
2798 r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
2805 mlxsw_sp->rifs[rif] = r;
2813 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2815 mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
2816 err_vport_rif_sp_op:
2817 mlxsw_sp_vr_put(vr);
2818 return ERR_PTR(err);
2821 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
2822 struct mlxsw_sp_rif *r)
2824 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2825 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
2826 struct net_device *l3_dev = r->dev;
2827 struct mlxsw_sp_fid *f = r->f;
2831 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
2834 mlxsw_sp->rifs[rif] = NULL;
2841 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2843 mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif, false);
2845 mlxsw_sp_vr_put(vr);
2848 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
2849 struct net_device *l3_dev)
2851 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2852 struct mlxsw_sp_rif *r;
2854 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
2856 r = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
2861 mlxsw_sp_vport_fid_set(mlxsw_sp_vport, r->f);
2864 netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", r->f->fid);
2869 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
2871 struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
2873 netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
2875 mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
2876 if (--f->ref_count == 0)
2877 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->r);
2880 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
2881 struct net_device *port_dev,
2882 unsigned long event, u16 vid)
2884 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
2885 struct mlxsw_sp_port *mlxsw_sp_vport;
2887 mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
2888 if (WARN_ON(!mlxsw_sp_vport))
2893 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
2895 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
2902 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
2903 unsigned long event)
2905 if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
2908 return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
2911 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
2912 struct net_device *lag_dev,
2913 unsigned long event, u16 vid)
2915 struct net_device *port_dev;
2916 struct list_head *iter;
2919 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
2920 if (mlxsw_sp_port_dev_check(port_dev)) {
2921 err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
2931 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
2932 unsigned long event)
2934 if (netif_is_bridge_port(lag_dev))
2937 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
2940 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
2941 struct net_device *l3_dev)
2945 if (is_vlan_dev(l3_dev))
2946 fid = vlan_dev_vlan_id(l3_dev);
2947 else if (mlxsw_sp->master_bridge.dev == l3_dev)
2950 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
2952 return mlxsw_sp_fid_find(mlxsw_sp, fid);
2955 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
2957 return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
2958 MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
2961 static u16 mlxsw_sp_flood_table_index_get(u16 fid)
2963 return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
2966 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
2969 enum mlxsw_flood_table_type table_type;
2974 sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
2978 table_type = mlxsw_sp_flood_table_type_get(fid);
2979 index = mlxsw_sp_flood_table_index_get(fid);
2980 mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
2981 1, MLXSW_PORT_ROUTER_PORT, set);
2982 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
2988 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
2990 if (mlxsw_sp_fid_is_vfid(fid))
2991 return MLXSW_REG_RITR_FID_IF;
2993 return MLXSW_REG_RITR_VLAN_IF;
2996 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
2997 struct net_device *l3_dev,
3001 enum mlxsw_reg_ritr_if_type rif_type;
3002 char ritr_pl[MLXSW_REG_RITR_LEN];
3004 rif_type = mlxsw_sp_rif_type_get(fid);
3005 mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
3007 mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
3009 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3012 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
3013 struct net_device *l3_dev,
3014 struct mlxsw_sp_fid *f)
3016 u32 tb_id = l3mdev_fib_table(l3_dev);
3017 struct mlxsw_sp_vr *vr;
3018 struct mlxsw_sp_rif *r;
3022 rif = mlxsw_sp_avail_rif_get(mlxsw_sp);
3023 if (rif == MLXSW_SP_INVALID_RIF)
3026 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3030 err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
3032 goto err_port_flood_set;
3034 err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif,
3037 goto err_rif_bridge_op;
3039 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
3041 goto err_rif_fdb_op;
3043 r = mlxsw_sp_rif_alloc(rif, vr->id, l3_dev, f);
3050 mlxsw_sp->rifs[rif] = r;
3053 netdev_dbg(l3_dev, "RIF=%d created\n", rif);
3058 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3060 mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
3062 mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3064 mlxsw_sp_vr_put(vr);
3068 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
3069 struct mlxsw_sp_rif *r)
3071 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[r->vr_id];
3072 struct net_device *l3_dev = r->dev;
3073 struct mlxsw_sp_fid *f = r->f;
3076 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, r);
3079 mlxsw_sp->rifs[rif] = NULL;
3084 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3086 mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif, false);
3088 mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3090 mlxsw_sp_vr_put(vr);
3092 netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif);
3095 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3096 struct net_device *br_dev,
3097 unsigned long event)
3099 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3100 struct mlxsw_sp_fid *f;
3102 /* FID can either be an actual FID if the L3 device is the
3103 * VLAN-aware bridge or a VLAN device on top. Otherwise, the
3104 * L3 device is a VLAN-unaware bridge and we get a vFID.
3106 f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3112 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3114 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->r);
3121 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3122 unsigned long event)
3124 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3125 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
3126 u16 vid = vlan_dev_vlan_id(vlan_dev);
3128 if (mlxsw_sp_port_dev_check(real_dev))
3129 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
3131 else if (netif_is_lag_master(real_dev))
3132 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3134 else if (netif_is_bridge_master(real_dev) &&
3135 mlxsw_sp->master_bridge.dev == real_dev)
3136 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
3142 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3143 unsigned long event, void *ptr)
3145 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3146 struct net_device *dev = ifa->ifa_dev->dev;
3147 struct mlxsw_sp *mlxsw_sp;
3148 struct mlxsw_sp_rif *r;
3151 mlxsw_sp = mlxsw_sp_lower_get(dev);
3155 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3156 if (!mlxsw_sp_rif_should_config(r, ifa->ifa_dev, event))
3159 if (mlxsw_sp_port_dev_check(dev))
3160 err = mlxsw_sp_inetaddr_port_event(dev, event);
3161 else if (netif_is_lag_master(dev))
3162 err = mlxsw_sp_inetaddr_lag_event(dev, event);
3163 else if (netif_is_bridge_master(dev))
3164 err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
3165 else if (is_vlan_dev(dev))
3166 err = mlxsw_sp_inetaddr_vlan_event(dev, event);
3169 return notifier_from_errno(err);
3172 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif,
3173 const char *mac, int mtu)
3175 char ritr_pl[MLXSW_REG_RITR_LEN];
3178 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
3179 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3183 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3184 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3185 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3186 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3189 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3191 struct mlxsw_sp *mlxsw_sp;
3192 struct mlxsw_sp_rif *r;
3195 mlxsw_sp = mlxsw_sp_lower_get(dev);
3199 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3203 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, false);
3207 err = mlxsw_sp_rif_edit(mlxsw_sp, r->rif, dev->dev_addr, dev->mtu);
3211 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, r->f->fid, true);
3213 goto err_rif_fdb_op;
3215 ether_addr_copy(r->addr, dev->dev_addr);
3218 netdev_dbg(dev, "Updated RIF=%d\n", r->rif);
3223 mlxsw_sp_rif_edit(mlxsw_sp, r->rif, r->addr, r->mtu);
3225 mlxsw_sp_rif_fdb_op(mlxsw_sp, r->addr, r->f->fid, true);
3229 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3231 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
3233 /* Flush pending FIB notifications and then flush the device's
3234 * table before requesting another dump. The FIB notification
3235 * block is unregistered, so no need to take RTNL.
3237 mlxsw_core_flush_owq();
3238 mlxsw_sp_router_fib_flush(mlxsw_sp);
3241 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3243 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3247 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3250 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3251 mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
3253 if (!mlxsw_sp->rifs)
3256 mlxsw_reg_rgcr_pack(rgcr_pl, true);
3257 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3258 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3265 kfree(mlxsw_sp->rifs);
3269 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3271 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3274 mlxsw_reg_rgcr_pack(rgcr_pl, false);
3275 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3277 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3278 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
3280 kfree(mlxsw_sp->rifs);
3283 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3287 INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
3288 err = __mlxsw_sp_router_init(mlxsw_sp);
3292 err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
3293 &mlxsw_sp_nexthop_ht_params);
3295 goto err_nexthop_ht_init;
3297 err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
3298 &mlxsw_sp_nexthop_group_ht_params);
3300 goto err_nexthop_group_ht_init;
3302 mlxsw_sp_lpm_init(mlxsw_sp);
3303 err = mlxsw_sp_vrs_init(mlxsw_sp);
3307 err = mlxsw_sp_neigh_init(mlxsw_sp);
3309 goto err_neigh_init;
3311 mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3312 err = register_fib_notifier(&mlxsw_sp->fib_nb,
3313 mlxsw_sp_router_fib_dump_flush);
3315 goto err_register_fib_notifier;
3319 err_register_fib_notifier:
3320 mlxsw_sp_neigh_fini(mlxsw_sp);
3322 mlxsw_sp_vrs_fini(mlxsw_sp);
3324 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3325 err_nexthop_group_ht_init:
3326 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3327 err_nexthop_ht_init:
3328 __mlxsw_sp_router_fini(mlxsw_sp);
3332 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3334 unregister_fib_notifier(&mlxsw_sp->fib_nb);
3335 mlxsw_sp_neigh_fini(mlxsw_sp);
3336 mlxsw_sp_vrs_fini(mlxsw_sp);
3337 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3338 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3339 __mlxsw_sp_router_fini(mlxsw_sp);