net/xfrm/xfrm_policy.c

   1 /*
   2  * xfrm_policy.c
   3  *
   4  * Changes:
   5  *      Mitsuru KANDA @USAGI
   6  *      Kazunori MIYAZAWA @USAGI
   7  *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
   8  *              IPv6 support
   9  *      Kazunori MIYAZAWA @USAGI
  10  *      YOSHIFUJI Hideaki
  11  *              Split up af-specific portion
  12  *      Derek Atkins <derek@ihtfp.com>          Add the post_input processor
  13  *
  14  */
  15
  16 #include <linux/err.h>
  17 #include <linux/slab.h>
  18 #include <linux/kmod.h>
  19 #include <linux/list.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/notifier.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/netfilter.h>
  25 #include <linux/module.h>
  26 #include <linux/cache.h>
  27 #include <linux/audit.h>
  28 #include <net/dst.h>
  29 #include <net/flow.h>
  30 #include <net/xfrm.h>
  31 #include <net/ip.h>
  32 #ifdef CONFIG_XFRM_STATISTICS
  33 #include <net/snmp.h>
  34 #endif
  35
  36 #include "xfrm_hash.h"
  37
  38 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
  39 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
  40 #define XFRM_MAX_QUEUE_LEN      100
  41
  42 DEFINE_MUTEX(xfrm_cfg_mutex);
  43 EXPORT_SYMBOL(xfrm_cfg_mutex);
  44
  45 static DEFINE_SPINLOCK(xfrm_policy_sk_bundle_lock);
  46 static struct dst_entry *xfrm_policy_sk_bundles;
  47 static DEFINE_RWLOCK(xfrm_policy_lock);
  48
  49 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
  50 static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[NPROTO]
  51                                                 __read_mostly;
  52
  53 static struct kmem_cache *xfrm_dst_cache __read_mostly;
  54
  55 static void xfrm_init_pmtu(struct dst_entry *dst);
  56 static int stale_bundle(struct dst_entry *dst);
  57 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
  58 static void xfrm_policy_queue_process(unsigned long arg);
  59
  60 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
  61                                                 int dir);
  62
  63 static inline bool
  64 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
  65 {
  66         const struct flowi4 *fl4 = &fl->u.ip4;
  67
  68         return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
  69                 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
  70                 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
  71                 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
  72                 (fl4->flowi4_proto == sel->proto || !sel->proto) &&
  73                 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
  74 }
  75
  76 static inline bool
  77 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
  78 {
  79         const struct flowi6 *fl6 = &fl->u.ip6;
  80
  81         return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
  82                 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
  83                 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
  84                 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
  85                 (fl6->flowi6_proto == sel->proto || !sel->proto) &&
  86                 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
  87 }
  88
  89 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
  90                          unsigned short family)
  91 {
  92         switch (family) {
  93         case AF_INET:
  94                 return __xfrm4_selector_match(sel, fl);
  95         case AF_INET6:
  96                 return __xfrm6_selector_match(sel, fl);
  97         }
  98         return false;
  99 }
 100
 101 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
 102 {
 103         struct xfrm_policy_afinfo *afinfo;
 104
 105         if (unlikely(family >= NPROTO))
 106                 return NULL;
 107         rcu_read_lock();
 108         afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
 109         if (unlikely(!afinfo))
 110                 rcu_read_unlock();
 111         return afinfo;
 112 }
 113
 114 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
 115 {
 116         rcu_read_unlock();
 117 }
 118
 119 static inline struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos,
 120                                                   const xfrm_address_t *saddr,
 121                                                   const xfrm_address_t *daddr,
 122                                                   int family)
 123 {
 124         struct xfrm_policy_afinfo *afinfo;
 125         struct dst_entry *dst;
 126
 127         afinfo = xfrm_policy_get_afinfo(family);
 128         if (unlikely(afinfo == NULL))
 129                 return ERR_PTR(-EAFNOSUPPORT);
 130
 131         dst = afinfo->dst_lookup(net, tos, saddr, daddr);
 132
 133         xfrm_policy_put_afinfo(afinfo);
 134
 135         return dst;
 136 }
 137
 138 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, int tos,
 139                                                 xfrm_address_t *prev_saddr,
 140                                                 xfrm_address_t *prev_daddr,
 141                                                 int family)
 142 {
 143         struct net *net = xs_net(x);
 144         xfrm_address_t *saddr = &x->props.saddr;
 145         xfrm_address_t *daddr = &x->id.daddr;
 146         struct dst_entry *dst;
 147
 148         if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
 149                 saddr = x->coaddr;
 150                 daddr = prev_daddr;
 151         }
 152         if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
 153                 saddr = prev_saddr;
 154                 daddr = x->coaddr;
 155         }
 156
 157         dst = __xfrm_dst_lookup(net, tos, saddr, daddr, family);
 158
 159         if (!IS_ERR(dst)) {
 160                 if (prev_saddr != saddr)
 161                         memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
 162                 if (prev_daddr != daddr)
 163                         memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
 164         }
 165
 166         return dst;
 167 }
 168
 169 static inline unsigned long make_jiffies(long secs)
 170 {
 171         if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
 172                 return MAX_SCHEDULE_TIMEOUT-1;
 173         else
 174                 return secs*HZ;
 175 }
 176
 177 static void xfrm_policy_timer(unsigned long data)
 178 {
 179         struct xfrm_policy *xp = (struct xfrm_policy*)data;
 180         unsigned long now = get_seconds();
 181         long next = LONG_MAX;
 182         int warn = 0;
 183         int dir;
 184
 185         read_lock(&xp->lock);
 186
 187         if (unlikely(xp->walk.dead))
 188                 goto out;
 189
 190         dir = xfrm_policy_id2dir(xp->index);
 191
 192         if (xp->lft.hard_add_expires_seconds) {
 193                 long tmo = xp->lft.hard_add_expires_seconds +
 194                         xp->curlft.add_time - now;
 195                 if (tmo <= 0)
 196                         goto expired;
 197                 if (tmo < next)
 198                         next = tmo;
 199         }
 200         if (xp->lft.hard_use_expires_seconds) {
 201                 long tmo = xp->lft.hard_use_expires_seconds +
 202                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
 203                 if (tmo <= 0)
 204                         goto expired;
 205                 if (tmo < next)
 206                         next = tmo;
 207         }
 208         if (xp->lft.soft_add_expires_seconds) {
 209                 long tmo = xp->lft.soft_add_expires_seconds +
 210                         xp->curlft.add_time - now;
 211                 if (tmo <= 0) {
 212                         warn = 1;
 213                         tmo = XFRM_KM_TIMEOUT;
 214                 }
 215                 if (tmo < next)
 216                         next = tmo;
 217         }
 218         if (xp->lft.soft_use_expires_seconds) {
 219                 long tmo = xp->lft.soft_use_expires_seconds +
 220                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
 221                 if (tmo <= 0) {
 222                         warn = 1;
 223                         tmo = XFRM_KM_TIMEOUT;
 224                 }
 225                 if (tmo < next)
 226                         next = tmo;
 227         }
 228
 229         if (warn)
 230                 km_policy_expired(xp, dir, 0, 0);
 231         if (next != LONG_MAX &&
 232             !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
 233                 xfrm_pol_hold(xp);
 234
 235 out:
 236         read_unlock(&xp->lock);
 237         xfrm_pol_put(xp);
 238         return;
 239
 240 expired:
 241         read_unlock(&xp->lock);
 242         if (!xfrm_policy_delete(xp, dir))
 243                 km_policy_expired(xp, dir, 1, 0);
 244         xfrm_pol_put(xp);
 245 }
 246
 247 static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
 248 {
 249         struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
 250
 251         if (unlikely(pol->walk.dead))
 252                 flo = NULL;
 253         else
 254                 xfrm_pol_hold(pol);
 255
 256         return flo;
 257 }
 258
 259 static int xfrm_policy_flo_check(struct flow_cache_object *flo)
 260 {
 261         struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
 262
 263         return !pol->walk.dead;
 264 }
 265
 266 static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
 267 {
 268         xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
 269 }
 270
 271 static const struct flow_cache_ops xfrm_policy_fc_ops = {
 272         .get = xfrm_policy_flo_get,
 273         .check = xfrm_policy_flo_check,
 274         .delete = xfrm_policy_flo_delete,
 275 };
 276
 277 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 278  * SPD calls.
 279  */
 280
 281 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
 282 {
 283         struct xfrm_policy *policy;
 284
 285         policy = kzalloc(sizeof(struct xfrm_policy), gfp);
 286
 287         if (policy) {
 288                 write_pnet(&policy->xp_net, net);
 289                 INIT_LIST_HEAD(&policy->walk.all);
 290                 INIT_HLIST_NODE(&policy->bydst);
 291                 INIT_HLIST_NODE(&policy->byidx);
 292                 rwlock_init(&policy->lock);
 293                 atomic_set(&policy->refcnt, 1);
 294                 skb_queue_head_init(&policy->polq.hold_queue);
 295                 setup_timer(&policy->timer, xfrm_policy_timer,
 296                                 (unsigned long)policy);
 297                 setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
 298                             (unsigned long)policy);
 299                 policy->flo.ops = &xfrm_policy_fc_ops;
 300         }
 301         return policy;
 302 }
 303 EXPORT_SYMBOL(xfrm_policy_alloc);
 304
 305 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
 306
 307 void xfrm_policy_destroy(struct xfrm_policy *policy)
 308 {
 309         BUG_ON(!policy->walk.dead);
 310
 311         if (del_timer(&policy->timer))
 312                 BUG();
 313
 314         security_xfrm_policy_free(policy->security);
 315         kfree(policy);
 316 }
 317 EXPORT_SYMBOL(xfrm_policy_destroy);
 318
 319 static void xfrm_queue_purge(struct sk_buff_head *list)
 320 {
 321         struct sk_buff *skb;
 322
 323         while ((skb = skb_dequeue(list)) != NULL) {
 324                 dev_put(skb->dev);
 325                 kfree_skb(skb);
 326         }
 327 }
 328
 329 /* Rule must be locked. Release descentant resources, announce
 330  * entry dead. The rule must be unlinked from lists to the moment.
 331  */
 332
 333 static void xfrm_policy_kill(struct xfrm_policy *policy)
 334 {
 335         policy->walk.dead = 1;
 336
 337         atomic_inc(&policy->genid);
 338
 339         del_timer(&policy->polq.hold_timer);
 340         xfrm_queue_purge(&policy->polq.hold_queue);
 341
 342         if (del_timer(&policy->timer))
 343                 xfrm_pol_put(policy);
 344
 345         xfrm_pol_put(policy);
 346 }
 347
 348 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
 349
 350 static inline unsigned int idx_hash(struct net *net, u32 index)
 351 {
 352         return __idx_hash(index, net->xfrm.policy_idx_hmask);
 353 }
 354
 355 static struct hlist_head *policy_hash_bysel(struct net *net,
 356                                             const struct xfrm_selector *sel,
 357                                             unsigned short family, int dir)
 358 {
 359         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 360         unsigned int hash = __sel_hash(sel, family, hmask);
 361
 362         return (hash == hmask + 1 ?
 363                 &net->xfrm.policy_inexact[dir] :
 364                 net->xfrm.policy_bydst[dir].table + hash);
 365 }
 366
 367 static struct hlist_head *policy_hash_direct(struct net *net,
 368                                              const xfrm_address_t *daddr,
 369                                              const xfrm_address_t *saddr,
 370                                              unsigned short family, int dir)
 371 {
 372         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 373         unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
 374
 375         return net->xfrm.policy_bydst[dir].table + hash;
 376 }
 377
 378 static void xfrm_dst_hash_transfer(struct hlist_head *list,
 379                                    struct hlist_head *ndsttable,
 380                                    unsigned int nhashmask)
 381 {
 382         struct hlist_node *tmp, *entry0 = NULL;
 383         struct xfrm_policy *pol;
 384         unsigned int h0 = 0;
 385
 386 redo:
 387         hlist_for_each_entry_safe(pol, tmp, list, bydst) {
 388                 unsigned int h;
 389
 390                 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
 391                                 pol->family, nhashmask);
 392                 if (!entry0) {
 393                         hlist_del(&pol->bydst);
 394                         hlist_add_head(&pol->bydst, ndsttable+h);
 395                         h0 = h;
 396                 } else {
 397                         if (h != h0)
 398                                 continue;
 399                         hlist_del(&pol->bydst);
 400                         hlist_add_after(entry0, &pol->bydst);
 401                 }
 402                 entry0 = &pol->bydst;
 403         }
 404         if (!hlist_empty(list)) {
 405                 entry0 = NULL;
 406                 goto redo;
 407         }
 408 }
 409
 410 static void xfrm_idx_hash_transfer(struct hlist_head *list,
 411                                    struct hlist_head *nidxtable,
 412                                    unsigned int nhashmask)
 413 {
 414         struct hlist_node *tmp;
 415         struct xfrm_policy *pol;
 416
 417         hlist_for_each_entry_safe(pol, tmp, list, byidx) {
 418                 unsigned int h;
 419
 420                 h = __idx_hash(pol->index, nhashmask);
 421                 hlist_add_head(&pol->byidx, nidxtable+h);
 422         }
 423 }
 424
 425 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
 426 {
 427         return ((old_hmask + 1) << 1) - 1;
 428 }
 429
 430 static void xfrm_bydst_resize(struct net *net, int dir)
 431 {
 432         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 433         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 434         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 435         struct hlist_head *odst = net->xfrm.policy_bydst[dir].table;
 436         struct hlist_head *ndst = xfrm_hash_alloc(nsize);
 437         int i;
 438
 439         if (!ndst)
 440                 return;
 441
 442         write_lock_bh(&xfrm_policy_lock);
 443
 444         for (i = hmask; i >= 0; i--)
 445                 xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
 446
 447         net->xfrm.policy_bydst[dir].table = ndst;
 448         net->xfrm.policy_bydst[dir].hmask = nhashmask;
 449
 450         write_unlock_bh(&xfrm_policy_lock);
 451
 452         xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
 453 }
 454
 455 static void xfrm_byidx_resize(struct net *net, int total)
 456 {
 457         unsigned int hmask = net->xfrm.policy_idx_hmask;
 458         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 459         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 460         struct hlist_head *oidx = net->xfrm.policy_byidx;
 461         struct hlist_head *nidx = xfrm_hash_alloc(nsize);
 462         int i;
 463
 464         if (!nidx)
 465                 return;
 466
 467         write_lock_bh(&xfrm_policy_lock);
 468
 469         for (i = hmask; i >= 0; i--)
 470                 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
 471
 472         net->xfrm.policy_byidx = nidx;
 473         net->xfrm.policy_idx_hmask = nhashmask;
 474
 475         write_unlock_bh(&xfrm_policy_lock);
 476
 477         xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
 478 }
 479
 480 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
 481 {
 482         unsigned int cnt = net->xfrm.policy_count[dir];
 483         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 484
 485         if (total)
 486                 *total += cnt;
 487
 488         if ((hmask + 1) < xfrm_policy_hashmax &&
 489             cnt > hmask)
 490                 return 1;
 491
 492         return 0;
 493 }
 494
 495 static inline int xfrm_byidx_should_resize(struct net *net, int total)
 496 {
 497         unsigned int hmask = net->xfrm.policy_idx_hmask;
 498
 499         if ((hmask + 1) < xfrm_policy_hashmax &&
 500             total > hmask)
 501                 return 1;
 502
 503         return 0;
 504 }
 505
 506 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
 507 {
 508         read_lock_bh(&xfrm_policy_lock);
 509         si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
 510         si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
 511         si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
 512         si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
 513         si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
 514         si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
 515         si->spdhcnt = net->xfrm.policy_idx_hmask;
 516         si->spdhmcnt = xfrm_policy_hashmax;
 517         read_unlock_bh(&xfrm_policy_lock);
 518 }
 519 EXPORT_SYMBOL(xfrm_spd_getinfo);
 520
 521 static DEFINE_MUTEX(hash_resize_mutex);
 522 static void xfrm_hash_resize(struct work_struct *work)
 523 {
 524         struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
 525         int dir, total;
 526
 527         mutex_lock(&hash_resize_mutex);
 528
 529         total = 0;
 530         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
 531                 if (xfrm_bydst_should_resize(net, dir, &total))
 532                         xfrm_bydst_resize(net, dir);
 533         }
 534         if (xfrm_byidx_should_resize(net, total))
 535                 xfrm_byidx_resize(net, total);
 536
 537         mutex_unlock(&hash_resize_mutex);
 538 }
 539
 540 /* Generate new index... KAME seems to generate them ordered by cost
 541  * of an absolute inpredictability of ordering of rules. This will not pass. */
 542 static u32 xfrm_gen_index(struct net *net, int dir)
 543 {
 544         static u32 idx_generator;
 545
 546         for (;;) {
 547                 struct hlist_head *list;
 548                 struct xfrm_policy *p;
 549                 u32 idx;
 550                 int found;
 551
 552                 idx = (idx_generator | dir);
 553                 idx_generator += 8;
 554                 if (idx == 0)
 555                         idx = 8;
 556                 list = net->xfrm.policy_byidx + idx_hash(net, idx);
 557                 found = 0;
 558                 hlist_for_each_entry(p, list, byidx) {
 559                         if (p->index == idx) {
 560                                 found = 1;
 561                                 break;
 562                         }
 563                 }
 564                 if (!found)
 565                         return idx;
 566         }
 567 }
 568
 569 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
 570 {
 571         u32 *p1 = (u32 *) s1;
 572         u32 *p2 = (u32 *) s2;
 573         int len = sizeof(struct xfrm_selector) / sizeof(u32);
 574         int i;
 575
 576         for (i = 0; i < len; i++) {
 577                 if (p1[i] != p2[i])
 578                         return 1;
 579         }
 580
 581         return 0;
 582 }
 583
 584 static void xfrm_policy_requeue(struct xfrm_policy *old,
 585                                 struct xfrm_policy *new)
 586 {
 587         struct xfrm_policy_queue *pq = &old->polq;
 588         struct sk_buff_head list;
 589
 590         __skb_queue_head_init(&list);
 591
 592         spin_lock_bh(&pq->hold_queue.lock);
 593         skb_queue_splice_init(&pq->hold_queue, &list);
 594         del_timer(&pq->hold_timer);
 595         spin_unlock_bh(&pq->hold_queue.lock);
 596
 597         if (skb_queue_empty(&list))
 598                 return;
 599
 600         pq = &new->polq;
 601
 602         spin_lock_bh(&pq->hold_queue.lock);
 603         skb_queue_splice(&list, &pq->hold_queue);
 604         pq->timeout = XFRM_QUEUE_TMO_MIN;
 605         mod_timer(&pq->hold_timer, jiffies);
 606         spin_unlock_bh(&pq->hold_queue.lock);
 607 }
 608
 609 static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
 610                                    struct xfrm_policy *pol)
 611 {
 612         u32 mark = policy->mark.v & policy->mark.m;
 613
 614         if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
 615                 return true;
 616
 617         if ((mark & pol->mark.m) == pol->mark.v &&
 618             policy->priority == pol->priority)
 619                 return true;
 620
 621         return false;
 622 }
 623
 624 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
 625 {
 626         struct net *net = xp_net(policy);
 627         struct xfrm_policy *pol;
 628         struct xfrm_policy *delpol;
 629         struct hlist_head *chain;
 630         struct hlist_node *newpos;
 631
 632         write_lock_bh(&xfrm_policy_lock);
 633         chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
 634         delpol = NULL;
 635         newpos = NULL;
 636         hlist_for_each_entry(pol, chain, bydst) {
 637                 if (pol->type == policy->type &&
 638                     !selector_cmp(&pol->selector, &policy->selector) &&
 639                     xfrm_policy_mark_match(policy, pol) &&
 640                     xfrm_sec_ctx_match(pol->security, policy->security) &&
 641                     !WARN_ON(delpol)) {
 642                         if (excl) {
 643                                 write_unlock_bh(&xfrm_policy_lock);
 644                                 return -EEXIST;
 645                         }
 646                         delpol = pol;
 647                         if (policy->priority > pol->priority)
 648                                 continue;
 649                 } else if (policy->priority >= pol->priority) {
 650                         newpos = &pol->bydst;
 651                         continue;
 652                 }
 653                 if (delpol)
 654                         break;
 655         }
 656         if (newpos)
 657                 hlist_add_after(newpos, &policy->bydst);
 658         else
 659                 hlist_add_head(&policy->bydst, chain);
 660         xfrm_pol_hold(policy);
 661         net->xfrm.policy_count[dir]++;
 662         atomic_inc(&flow_cache_genid);
 663         rt_genid_bump(net);
 664         if (delpol) {
 665                 xfrm_policy_requeue(delpol, policy);
 666                 __xfrm_policy_unlink(delpol, dir);
 667         }
 668         policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir);
 669         hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
 670         policy->curlft.add_time = get_seconds();
 671         policy->curlft.use_time = 0;
 672         if (!mod_timer(&policy->timer, jiffies + HZ))
 673                 xfrm_pol_hold(policy);
 674         list_add(&policy->walk.all, &net->xfrm.policy_all);
 675         write_unlock_bh(&xfrm_policy_lock);
 676
 677         if (delpol)
 678                 xfrm_policy_kill(delpol);
 679         else if (xfrm_bydst_should_resize(net, dir, NULL))
 680                 schedule_work(&net->xfrm.policy_hash_work);
 681
 682         return 0;
 683 }
 684 EXPORT_SYMBOL(xfrm_policy_insert);
 685
 686 struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type,
 687                                           int dir, struct xfrm_selector *sel,
 688                                           struct xfrm_sec_ctx *ctx, int delete,
 689                                           int *err)
 690 {
 691         struct xfrm_policy *pol, *ret;
 692         struct hlist_head *chain;
 693
 694         *err = 0;
 695         write_lock_bh(&xfrm_policy_lock);
 696         chain = policy_hash_bysel(net, sel, sel->family, dir);
 697         ret = NULL;
 698         hlist_for_each_entry(pol, chain, bydst) {
 699                 if (pol->type == type &&
 700                     (mark & pol->mark.m) == pol->mark.v &&
 701                     !selector_cmp(sel, &pol->selector) &&
 702                     xfrm_sec_ctx_match(ctx, pol->security)) {
 703                         xfrm_pol_hold(pol);
 704                         if (delete) {
 705                                 *err = security_xfrm_policy_delete(
 706                                                                 pol->security);
 707                                 if (*err) {
 708                                         write_unlock_bh(&xfrm_policy_lock);
 709                                         return pol;
 710                                 }
 711                                 __xfrm_policy_unlink(pol, dir);
 712                         }
 713                         ret = pol;
 714                         break;
 715                 }
 716         }
 717         write_unlock_bh(&xfrm_policy_lock);
 718
 719         if (ret && delete)
 720                 xfrm_policy_kill(ret);
 721         return ret;
 722 }
 723 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
 724
 725 struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type,
 726                                      int dir, u32 id, int delete, int *err)
 727 {
 728         struct xfrm_policy *pol, *ret;
 729         struct hlist_head *chain;
 730
 731         *err = -ENOENT;
 732         if (xfrm_policy_id2dir(id) != dir)
 733                 return NULL;
 734
 735         *err = 0;
 736         write_lock_bh(&xfrm_policy_lock);
 737         chain = net->xfrm.policy_byidx + idx_hash(net, id);
 738         ret = NULL;
 739         hlist_for_each_entry(pol, chain, byidx) {
 740                 if (pol->type == type && pol->index == id &&
 741                     (mark & pol->mark.m) == pol->mark.v) {
 742                         xfrm_pol_hold(pol);
 743                         if (delete) {
 744                                 *err = security_xfrm_policy_delete(
 745                                                                 pol->security);
 746                                 if (*err) {
 747                                         write_unlock_bh(&xfrm_policy_lock);
 748                                         return pol;
 749                                 }
 750                                 __xfrm_policy_unlink(pol, dir);
 751                         }
 752                         ret = pol;
 753                         break;
 754                 }
 755         }
 756         write_unlock_bh(&xfrm_policy_lock);
 757
 758         if (ret && delete)
 759                 xfrm_policy_kill(ret);
 760         return ret;
 761 }
 762 EXPORT_SYMBOL(xfrm_policy_byid);
 763
 764 #ifdef CONFIG_SECURITY_NETWORK_XFRM
 765 static inline int
 766 xfrm_policy_flush_secctx_check(struct net *net, u8 type, struct xfrm_audit *audit_info)
 767 {
 768         int dir, err = 0;
 769
 770         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 771                 struct xfrm_policy *pol;
 772                 int i;
 773
 774                 hlist_for_each_entry(pol,
 775                                      &net->xfrm.policy_inexact[dir], bydst) {
 776                         if (pol->type != type)
 777                                 continue;
 778                         err = security_xfrm_policy_delete(pol->security);
 779                         if (err) {
 780                                 xfrm_audit_policy_delete(pol, 0,
 781                                                          audit_info->loginuid,
 782                                                          audit_info->sessionid,
 783                                                          audit_info->secid);
 784                                 return err;
 785                         }
 786                 }
 787                 for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
 788                         hlist_for_each_entry(pol,
 789                                              net->xfrm.policy_bydst[dir].table + i,
 790                                              bydst) {
 791                                 if (pol->type != type)
 792                                         continue;
 793                                 err = security_xfrm_policy_delete(
 794                                                                 pol->security);
 795                                 if (err) {
 796                                         xfrm_audit_policy_delete(pol, 0,
 797                                                         audit_info->loginuid,
 798                                                         audit_info->sessionid,
 799                                                         audit_info->secid);
 800                                         return err;
 801                                 }
 802                         }
 803                 }
 804         }
 805         return err;
 806 }
 807 #else
 808 static inline int
 809 xfrm_policy_flush_secctx_check(struct net *net, u8 type, struct xfrm_audit *audit_info)
 810 {
 811         return 0;
 812 }
 813 #endif
 814
 815 int xfrm_policy_flush(struct net *net, u8 type, struct xfrm_audit *audit_info)
 816 {
 817         int dir, err = 0, cnt = 0;
 818
 819         write_lock_bh(&xfrm_policy_lock);
 820
 821         err = xfrm_policy_flush_secctx_check(net, type, audit_info);
 822         if (err)
 823                 goto out;
 824
 825         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 826                 struct xfrm_policy *pol;
 827                 int i;
 828
 829         again1:
 830                 hlist_for_each_entry(pol,
 831                                      &net->xfrm.policy_inexact[dir], bydst) {
 832                         if (pol->type != type)
 833                                 continue;
 834                         __xfrm_policy_unlink(pol, dir);
 835                         write_unlock_bh(&xfrm_policy_lock);
 836                         cnt++;
 837
 838                         xfrm_audit_policy_delete(pol, 1, audit_info->loginuid,
 839                                                  audit_info->sessionid,
 840                                                  audit_info->secid);
 841
 842                         xfrm_policy_kill(pol);
 843
 844                         write_lock_bh(&xfrm_policy_lock);
 845                         goto again1;
 846                 }
 847
 848                 for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
 849         again2:
 850                         hlist_for_each_entry(pol,
 851                                              net->xfrm.policy_bydst[dir].table + i,
 852                                              bydst) {
 853                                 if (pol->type != type)
 854                                         continue;
 855                                 __xfrm_policy_unlink(pol, dir);
 856                                 write_unlock_bh(&xfrm_policy_lock);
 857                                 cnt++;
 858
 859                                 xfrm_audit_policy_delete(pol, 1,
 860                                                          audit_info->loginuid,
 861                                                          audit_info->sessionid,
 862                                                          audit_info->secid);
 863                                 xfrm_policy_kill(pol);
 864
 865                                 write_lock_bh(&xfrm_policy_lock);
 866                                 goto again2;
 867                         }
 868                 }
 869
 870         }
 871         if (!cnt)
 872                 err = -ESRCH;
 873 out:
 874         write_unlock_bh(&xfrm_policy_lock);
 875         return err;
 876 }
 877 EXPORT_SYMBOL(xfrm_policy_flush);
 878
 879 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
 880                      int (*func)(struct xfrm_policy *, int, int, void*),
 881                      void *data)
 882 {
 883         struct xfrm_policy *pol;
 884         struct xfrm_policy_walk_entry *x;
 885         int error = 0;
 886
 887         if (walk->type >= XFRM_POLICY_TYPE_MAX &&
 888             walk->type != XFRM_POLICY_TYPE_ANY)
 889                 return -EINVAL;
 890
 891         if (list_empty(&walk->walk.all) && walk->seq != 0)
 892                 return 0;
 893
 894         write_lock_bh(&xfrm_policy_lock);
 895         if (list_empty(&walk->walk.all))
 896                 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
 897         else
 898                 x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all);
 899         list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
 900                 if (x->dead)
 901                         continue;
 902                 pol = container_of(x, struct xfrm_policy, walk);
 903                 if (walk->type != XFRM_POLICY_TYPE_ANY &&
 904                     walk->type != pol->type)
 905                         continue;
 906                 error = func(pol, xfrm_policy_id2dir(pol->index),
 907                              walk->seq, data);
 908                 if (error) {
 909                         list_move_tail(&walk->walk.all, &x->all);
 910                         goto out;
 911                 }
 912                 walk->seq++;
 913         }
 914         if (walk->seq == 0) {
 915                 error = -ENOENT;
 916                 goto out;
 917         }
 918         list_del_init(&walk->walk.all);
 919 out:
 920         write_unlock_bh(&xfrm_policy_lock);
 921         return error;
 922 }
 923 EXPORT_SYMBOL(xfrm_policy_walk);
 924
 925 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
 926 {
 927         INIT_LIST_HEAD(&walk->walk.all);
 928         walk->walk.dead = 1;
 929         walk->type = type;
 930         walk->seq = 0;
 931 }
 932 EXPORT_SYMBOL(xfrm_policy_walk_init);
 933
 934 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk)
 935 {
 936         if (list_empty(&walk->walk.all))
 937                 return;
 938
 939         write_lock_bh(&xfrm_policy_lock);
 940         list_del(&walk->walk.all);
 941         write_unlock_bh(&xfrm_policy_lock);
 942 }
 943 EXPORT_SYMBOL(xfrm_policy_walk_done);
 944
 945 /*
 946  * Find policy to apply to this flow.
 947  *
 948  * Returns 0 if policy found, else an -errno.
 949  */
 950 static int xfrm_policy_match(const struct xfrm_policy *pol,
 951                              const struct flowi *fl,
 952                              u8 type, u16 family, int dir)
 953 {
 954         const struct xfrm_selector *sel = &pol->selector;
 955         int ret = -ESRCH;
 956         bool match;
 957
 958         if (pol->family != family ||
 959             (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
 960             pol->type != type)
 961                 return ret;
 962
 963         match = xfrm_selector_match(sel, fl, family);
 964         if (match)
 965                 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
 966                                                   dir);
 967
 968         return ret;
 969 }
 970
 971 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
 972                                                      const struct flowi *fl,
 973                                                      u16 family, u8 dir)
 974 {
 975         int err;
 976         struct xfrm_policy *pol, *ret;
 977         const xfrm_address_t *daddr, *saddr;
 978         struct hlist_head *chain;
 979         u32 priority = ~0U;
 980
 981         daddr = xfrm_flowi_daddr(fl, family);
 982         saddr = xfrm_flowi_saddr(fl, family);
 983         if (unlikely(!daddr || !saddr))
 984                 return NULL;
 985
 986         read_lock_bh(&xfrm_policy_lock);
 987         chain = policy_hash_direct(net, daddr, saddr, family, dir);
 988         ret = NULL;
 989         hlist_for_each_entry(pol, chain, bydst) {
 990                 err = xfrm_policy_match(pol, fl, type, family, dir);
 991                 if (err) {
 992                         if (err == -ESRCH)
 993                                 continue;
 994                         else {
 995                                 ret = ERR_PTR(err);
 996                                 goto fail;
 997                         }
 998                 } else {
 999                         ret = pol;
1000                         priority = ret->priority;
1001                         break;
1002                 }
1003         }
1004         chain = &net->xfrm.policy_inexact[dir];
1005         hlist_for_each_entry(pol, chain, bydst) {
1006                 err = xfrm_policy_match(pol, fl, type, family, dir);
1007                 if (err) {
1008                         if (err == -ESRCH)
1009                                 continue;
1010                         else {
1011                                 ret = ERR_PTR(err);
1012                                 goto fail;
1013                         }
1014                 } else if (pol->priority < priority) {
1015                         ret = pol;
1016                         break;
1017                 }
1018         }
1019         if (ret)
1020                 xfrm_pol_hold(ret);
1021 fail:
1022         read_unlock_bh(&xfrm_policy_lock);
1023
1024         return ret;
1025 }
1026
1027 static struct xfrm_policy *
1028 __xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)
1029 {
1030 #ifdef CONFIG_XFRM_SUB_POLICY
1031         struct xfrm_policy *pol;
1032
1033         pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);
1034         if (pol != NULL)
1035                 return pol;
1036 #endif
1037         return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);
1038 }
1039
1040 static struct flow_cache_object *
1041 xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family,
1042                    u8 dir, struct flow_cache_object *old_obj, void *ctx)
1043 {
1044         struct xfrm_policy *pol;
1045
1046         if (old_obj)
1047                 xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo));
1048
1049         pol = __xfrm_policy_lookup(net, fl, family, dir);
1050         if (IS_ERR_OR_NULL(pol))
1051                 return ERR_CAST(pol);
1052
1053         /* Resolver returns two references:
1054          * one for cache and one for caller of flow_cache_lookup() */
1055         xfrm_pol_hold(pol);
1056
1057         return &pol->flo;
1058 }
1059
1060 static inline int policy_to_flow_dir(int dir)
1061 {
1062         if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1063             XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1064             XFRM_POLICY_FWD == FLOW_DIR_FWD)
1065                 return dir;
1066         switch (dir) {
1067         default:
1068         case XFRM_POLICY_IN:
1069                 return FLOW_DIR_IN;
1070         case XFRM_POLICY_OUT:
1071                 return FLOW_DIR_OUT;
1072         case XFRM_POLICY_FWD:
1073                 return FLOW_DIR_FWD;
1074         }
1075 }
1076
1077 static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir,
1078                                                  const struct flowi *fl)
1079 {
1080         struct xfrm_policy *pol;
1081
1082         read_lock_bh(&xfrm_policy_lock);
1083         if ((pol = sk->sk_policy[dir]) != NULL) {
1084                 bool match = xfrm_selector_match(&pol->selector, fl,
1085                                                  sk->sk_family);
1086                 int err = 0;
1087
1088                 if (match) {
1089                         if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {
1090                                 pol = NULL;
1091                                 goto out;
1092                         }
1093                         err = security_xfrm_policy_lookup(pol->security,
1094                                                       fl->flowi_secid,
1095                                                       policy_to_flow_dir(dir));
1096                         if (!err)
1097                                 xfrm_pol_hold(pol);
1098                         else if (err == -ESRCH)
1099                                 pol = NULL;
1100                         else
1101                                 pol = ERR_PTR(err);
1102                 } else
1103                         pol = NULL;
1104         }
1105 out:
1106         read_unlock_bh(&xfrm_policy_lock);
1107         return pol;
1108 }
1109
1110 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1111 {
1112         struct net *net = xp_net(pol);
1113         struct hlist_head *chain = policy_hash_bysel(net, &pol->selector,
1114                                                      pol->family, dir);
1115
1116         list_add(&pol->walk.all, &net->xfrm.policy_all);
1117         hlist_add_head(&pol->bydst, chain);
1118         hlist_add_head(&pol->byidx, net->xfrm.policy_byidx+idx_hash(net, pol->index));
1119         net->xfrm.policy_count[dir]++;
1120         xfrm_pol_hold(pol);
1121
1122         if (xfrm_bydst_should_resize(net, dir, NULL))
1123                 schedule_work(&net->xfrm.policy_hash_work);
1124 }
1125
1126 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
1127                                                 int dir)
1128 {
1129         struct net *net = xp_net(pol);
1130
1131         if (hlist_unhashed(&pol->bydst))
1132                 return NULL;
1133
1134         hlist_del(&pol->bydst);
1135         hlist_del(&pol->byidx);
1136         list_del(&pol->walk.all);
1137         net->xfrm.policy_count[dir]--;
1138
1139         return pol;
1140 }
1141
1142 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1143 {
1144         write_lock_bh(&xfrm_policy_lock);
1145         pol = __xfrm_policy_unlink(pol, dir);
1146         write_unlock_bh(&xfrm_policy_lock);
1147         if (pol) {
1148                 xfrm_policy_kill(pol);
1149                 return 0;
1150         }
1151         return -ENOENT;
1152 }
1153 EXPORT_SYMBOL(xfrm_policy_delete);
1154
1155 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
1156 {
1157         struct net *net = xp_net(pol);
1158         struct xfrm_policy *old_pol;
1159
1160 #ifdef CONFIG_XFRM_SUB_POLICY
1161         if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1162                 return -EINVAL;
1163 #endif
1164
1165         write_lock_bh(&xfrm_policy_lock);
1166         old_pol = sk->sk_policy[dir];
1167         sk->sk_policy[dir] = pol;
1168         if (pol) {
1169                 pol->curlft.add_time = get_seconds();
1170                 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir);
1171                 __xfrm_policy_link(pol, XFRM_POLICY_MAX+dir);
1172         }
1173         if (old_pol) {
1174                 if (pol)
1175                         xfrm_policy_requeue(old_pol, pol);
1176
1177                 /* Unlinking succeeds always. This is the only function
1178                  * allowed to delete or replace socket policy.
1179                  */
1180                 __xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir);
1181         }
1182         write_unlock_bh(&xfrm_policy_lock);
1183
1184         if (old_pol) {
1185                 xfrm_policy_kill(old_pol);
1186         }
1187         return 0;
1188 }
1189
1190 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
1191 {
1192         struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
1193
1194         if (newp) {
1195                 newp->selector = old->selector;
1196                 if (security_xfrm_policy_clone(old->security,
1197                                                &newp->security)) {
1198                         kfree(newp);
1199                         return NULL;  /* ENOMEM */
1200                 }
1201                 newp->lft = old->lft;
1202                 newp->curlft = old->curlft;
1203                 newp->mark = old->mark;
1204                 newp->action = old->action;
1205                 newp->flags = old->flags;
1206                 newp->xfrm_nr = old->xfrm_nr;
1207                 newp->index = old->index;
1208                 newp->type = old->type;
1209                 memcpy(newp->xfrm_vec, old->xfrm_vec,
1210                        newp->xfrm_nr*sizeof(struct xfrm_tmpl));
1211                 write_lock_bh(&xfrm_policy_lock);
1212                 __xfrm_policy_link(newp, XFRM_POLICY_MAX+dir);
1213                 write_unlock_bh(&xfrm_policy_lock);
1214                 xfrm_pol_put(newp);
1215         }
1216         return newp;
1217 }
1218
1219 int __xfrm_sk_clone_policy(struct sock *sk)
1220 {
1221         struct xfrm_policy *p0 = sk->sk_policy[0],
1222                            *p1 = sk->sk_policy[1];
1223
1224         sk->sk_policy[0] = sk->sk_policy[1] = NULL;
1225         if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
1226                 return -ENOMEM;
1227         if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
1228                 return -ENOMEM;
1229         return 0;
1230 }
1231
1232 static int
1233 xfrm_get_saddr(struct net *net, xfrm_address_t *local, xfrm_address_t *remote,
1234                unsigned short family)
1235 {
1236         int err;
1237         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1238
1239         if (unlikely(afinfo == NULL))
1240                 return -EINVAL;
1241         err = afinfo->get_saddr(net, local, remote);
1242         xfrm_policy_put_afinfo(afinfo);
1243         return err;
1244 }
1245
1246 /* Resolve list of templates for the flow, given policy. */
1247
1248 static int
1249 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
1250                       struct xfrm_state **xfrm, unsigned short family)
1251 {
1252         struct net *net = xp_net(policy);
1253         int nx;
1254         int i, error;
1255         xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1256         xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1257         xfrm_address_t tmp;
1258
1259         for (nx=0, i = 0; i < policy->xfrm_nr; i++) {
1260                 struct xfrm_state *x;
1261                 xfrm_address_t *remote = daddr;
1262                 xfrm_address_t *local  = saddr;
1263                 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1264
1265                 if (tmpl->mode == XFRM_MODE_TUNNEL ||
1266                     tmpl->mode == XFRM_MODE_BEET) {
1267                         remote = &tmpl->id.daddr;
1268                         local = &tmpl->saddr;
1269                         if (xfrm_addr_any(local, tmpl->encap_family)) {
1270                                 error = xfrm_get_saddr(net, &tmp, remote, tmpl->encap_family);
1271                                 if (error)
1272                                         goto fail;
1273                                 local = &tmp;
1274                         }
1275                 }
1276
1277                 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
1278
1279                 if (x && x->km.state == XFRM_STATE_VALID) {
1280                         xfrm[nx++] = x;
1281                         daddr = remote;
1282                         saddr = local;
1283                         continue;
1284                 }
1285                 if (x) {
1286                         error = (x->km.state == XFRM_STATE_ERROR ?
1287                                  -EINVAL : -EAGAIN);
1288                         xfrm_state_put(x);
1289                 }
1290                 else if (error == -ESRCH)
1291                         error = -EAGAIN;
1292
1293                 if (!tmpl->optional)
1294                         goto fail;
1295         }
1296         return nx;
1297
1298 fail:
1299         for (nx--; nx>=0; nx--)
1300                 xfrm_state_put(xfrm[nx]);
1301         return error;
1302 }
1303
1304 static int
1305 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
1306                   struct xfrm_state **xfrm, unsigned short family)
1307 {
1308         struct xfrm_state *tp[XFRM_MAX_DEPTH];
1309         struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1310         int cnx = 0;
1311         int error;
1312         int ret;
1313         int i;
1314
1315         for (i = 0; i < npols; i++) {
1316                 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1317                         error = -ENOBUFS;
1318                         goto fail;
1319                 }
1320
1321                 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1322                 if (ret < 0) {
1323                         error = ret;
1324                         goto fail;
1325                 } else
1326                         cnx += ret;
1327         }
1328
1329         /* found states are sorted for outbound processing */
1330         if (npols > 1)
1331                 xfrm_state_sort(xfrm, tpp, cnx, family);
1332
1333         return cnx;
1334
1335  fail:
1336         for (cnx--; cnx>=0; cnx--)
1337                 xfrm_state_put(tpp[cnx]);
1338         return error;
1339
1340 }
1341
1342 /* Check that the bundle accepts the flow and its components are
1343  * still valid.
1344  */
1345
1346 static inline int xfrm_get_tos(const struct flowi *fl, int family)
1347 {
1348         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1349         int tos;
1350
1351         if (!afinfo)
1352                 return -EINVAL;
1353
1354         tos = afinfo->get_tos(fl);
1355
1356         xfrm_policy_put_afinfo(afinfo);
1357
1358         return tos;
1359 }
1360
1361 static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo)
1362 {
1363         struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1364         struct dst_entry *dst = &xdst->u.dst;
1365
1366         if (xdst->route == NULL) {
1367                 /* Dummy bundle - if it has xfrms we were not
1368                  * able to build bundle as template resolution failed.
1369                  * It means we need to try again resolving. */
1370                 if (xdst->num_xfrms > 0)
1371                         return NULL;
1372         } else if (dst->flags & DST_XFRM_QUEUE) {
1373                 return NULL;
1374         } else {
1375                 /* Real bundle */
1376                 if (stale_bundle(dst))
1377                         return NULL;
1378         }
1379
1380         dst_hold(dst);
1381         return flo;
1382 }
1383
1384 static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
1385 {
1386         struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1387         struct dst_entry *dst = &xdst->u.dst;
1388
1389         if (!xdst->route)
1390                 return 0;
1391         if (stale_bundle(dst))
1392                 return 0;
1393
1394         return 1;
1395 }
1396
1397 static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
1398 {
1399         struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1400         struct dst_entry *dst = &xdst->u.dst;
1401
1402         dst_free(dst);
1403 }
1404
1405 static const struct flow_cache_ops xfrm_bundle_fc_ops = {
1406         .get = xfrm_bundle_flo_get,
1407         .check = xfrm_bundle_flo_check,
1408         .delete = xfrm_bundle_flo_delete,
1409 };
1410
1411 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
1412 {
1413         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1414         struct dst_ops *dst_ops;
1415         struct xfrm_dst *xdst;
1416
1417         if (!afinfo)
1418                 return ERR_PTR(-EINVAL);
1419
1420         switch (family) {
1421         case AF_INET:
1422                 dst_ops = &net->xfrm.xfrm4_dst_ops;
1423                 break;
1424 #if IS_ENABLED(CONFIG_IPV6)
1425         case AF_INET6:
1426                 dst_ops = &net->xfrm.xfrm6_dst_ops;
1427                 break;
1428 #endif
1429         default:
1430                 BUG();
1431         }
1432         xdst = dst_alloc(dst_ops, NULL, 0, DST_OBSOLETE_NONE, 0);
1433
1434         if (likely(xdst)) {
1435                 struct dst_entry *dst = &xdst->u.dst;
1436
1437                 memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
1438                 xdst->flo.ops = &xfrm_bundle_fc_ops;
1439                 if (afinfo->init_dst)
1440                         afinfo->init_dst(net, xdst);
1441         } else
1442                 xdst = ERR_PTR(-ENOBUFS);
1443
1444         xfrm_policy_put_afinfo(afinfo);
1445
1446         return xdst;
1447 }
1448
1449 static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
1450                                  int nfheader_len)
1451 {
1452         struct xfrm_policy_afinfo *afinfo =
1453                 xfrm_policy_get_afinfo(dst->ops->family);
1454         int err;
1455
1456         if (!afinfo)
1457                 return -EINVAL;
1458
1459         err = afinfo->init_path(path, dst, nfheader_len);
1460
1461         xfrm_policy_put_afinfo(afinfo);
1462
1463         return err;
1464 }
1465
1466 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
1467                                 const struct flowi *fl)
1468 {
1469         struct xfrm_policy_afinfo *afinfo =
1470                 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1471         int err;
1472
1473         if (!afinfo)
1474                 return -EINVAL;
1475
1476         err = afinfo->fill_dst(xdst, dev, fl);
1477
1478         xfrm_policy_put_afinfo(afinfo);
1479
1480         return err;
1481 }
1482
1483
1484 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
1485  * all the metrics... Shortly, bundle a bundle.
1486  */
1487
1488 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
1489                                             struct xfrm_state **xfrm, int nx,
1490                                             const struct flowi *fl,
1491                                             struct dst_entry *dst)
1492 {
1493         struct net *net = xp_net(policy);
1494         unsigned long now = jiffies;
1495         struct net_device *dev;
1496         struct xfrm_mode *inner_mode;
1497         struct dst_entry *dst_prev = NULL;
1498         struct dst_entry *dst0 = NULL;
1499         int i = 0;
1500         int err;
1501         int header_len = 0;
1502         int nfheader_len = 0;
1503         int trailer_len = 0;
1504         int tos;
1505         int family = policy->selector.family;
1506         xfrm_address_t saddr, daddr;
1507
1508         xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
1509
1510         tos = xfrm_get_tos(fl, family);
1511         err = tos;
1512         if (tos < 0)
1513                 goto put_states;
1514
1515         dst_hold(dst);
1516
1517         for (; i < nx; i++) {
1518                 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
1519                 struct dst_entry *dst1 = &xdst->u.dst;
1520
1521                 err = PTR_ERR(xdst);
1522                 if (IS_ERR(xdst)) {
1523                         dst_release(dst);
1524                         goto put_states;
1525                 }
1526
1527                 if (xfrm[i]->sel.family == AF_UNSPEC) {
1528                         inner_mode = xfrm_ip2inner_mode(xfrm[i],
1529                                                         xfrm_af2proto(family));
1530                         if (!inner_mode) {
1531                                 err = -EAFNOSUPPORT;
1532                                 dst_release(dst);
1533                                 goto put_states;
1534                         }
1535                 } else
1536                         inner_mode = xfrm[i]->inner_mode;
1537
1538                 if (!dst_prev)
1539                         dst0 = dst1;
1540                 else {
1541                         dst_prev->child = dst_clone(dst1);
1542                         dst1->flags |= DST_NOHASH;
1543                 }
1544
1545                 xdst->route = dst;
1546                 dst_copy_metrics(dst1, dst);
1547
1548                 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1549                         family = xfrm[i]->props.family;
1550                         dst = xfrm_dst_lookup(xfrm[i], tos, &saddr, &daddr,
1551                                               family);
1552                         err = PTR_ERR(dst);
1553                         if (IS_ERR(dst))
1554                                 goto put_states;
1555                 } else
1556                         dst_hold(dst);
1557
1558                 dst1->xfrm = xfrm[i];
1559                 xdst->xfrm_genid = xfrm[i]->genid;
1560
1561                 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1562                 dst1->flags |= DST_HOST;
1563                 dst1->lastuse = now;
1564
1565                 dst1->input = dst_discard;
1566                 dst1->output = inner_mode->afinfo->output;
1567
1568                 dst1->next = dst_prev;
1569                 dst_prev = dst1;
1570
1571                 header_len += xfrm[i]->props.header_len;
1572                 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1573                         nfheader_len += xfrm[i]->props.header_len;
1574                 trailer_len += xfrm[i]->props.trailer_len;
1575         }
1576
1577         dst_prev->child = dst;
1578         dst0->path = dst;
1579
1580         err = -ENODEV;
1581         dev = dst->dev;
1582         if (!dev)
1583                 goto free_dst;
1584
1585         xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len);
1586         xfrm_init_pmtu(dst_prev);
1587
1588         for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) {
1589                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst_prev;
1590
1591                 err = xfrm_fill_dst(xdst, dev, fl);
1592                 if (err)
1593                         goto free_dst;
1594
1595                 dst_prev->header_len = header_len;
1596                 dst_prev->trailer_len = trailer_len;
1597                 header_len -= xdst->u.dst.xfrm->props.header_len;
1598                 trailer_len -= xdst->u.dst.xfrm->props.trailer_len;
1599         }
1600
1601 out:
1602         return dst0;
1603
1604 put_states:
1605         for (; i < nx; i++)
1606                 xfrm_state_put(xfrm[i]);
1607 free_dst:
1608         if (dst0)
1609                 dst_free(dst0);
1610         dst0 = ERR_PTR(err);
1611         goto out;
1612 }
1613
1614 static int inline
1615 xfrm_dst_alloc_copy(void **target, const void *src, int size)
1616 {
1617         if (!*target) {
1618                 *target = kmalloc(size, GFP_ATOMIC);
1619                 if (!*target)
1620                         return -ENOMEM;
1621         }
1622         memcpy(*target, src, size);
1623         return 0;
1624 }
1625
1626 static int inline
1627 xfrm_dst_update_parent(struct dst_entry *dst, const struct xfrm_selector *sel)
1628 {
1629 #ifdef CONFIG_XFRM_SUB_POLICY
1630         struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1631         return xfrm_dst_alloc_copy((void **)&(xdst->partner),
1632                                    sel, sizeof(*sel));
1633 #else
1634         return 0;
1635 #endif
1636 }
1637
1638 static int inline
1639 xfrm_dst_update_origin(struct dst_entry *dst, const struct flowi *fl)
1640 {
1641 #ifdef CONFIG_XFRM_SUB_POLICY
1642         struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1643         return xfrm_dst_alloc_copy((void **)&(xdst->origin), fl, sizeof(*fl));
1644 #else
1645         return 0;
1646 #endif
1647 }
1648
1649 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
1650                                 struct xfrm_policy **pols,
1651                                 int *num_pols, int *num_xfrms)
1652 {
1653         int i;
1654
1655         if (*num_pols == 0 || !pols[0]) {
1656                 *num_pols = 0;
1657                 *num_xfrms = 0;
1658                 return 0;
1659         }
1660         if (IS_ERR(pols[0]))
1661                 return PTR_ERR(pols[0]);
1662
1663         *num_xfrms = pols[0]->xfrm_nr;
1664
1665 #ifdef CONFIG_XFRM_SUB_POLICY
1666         if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
1667             pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1668                 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
1669                                                     XFRM_POLICY_TYPE_MAIN,
1670                                                     fl, family,
1671                                                     XFRM_POLICY_OUT);
1672                 if (pols[1]) {
1673                         if (IS_ERR(pols[1])) {
1674                                 xfrm_pols_put(pols, *num_pols);
1675                                 return PTR_ERR(pols[1]);
1676                         }
1677                         (*num_pols) ++;
1678                         (*num_xfrms) += pols[1]->xfrm_nr;
1679                 }
1680         }
1681 #endif
1682         for (i = 0; i < *num_pols; i++) {
1683                 if (pols[i]->action != XFRM_POLICY_ALLOW) {
1684                         *num_xfrms = -1;
1685                         break;
1686                 }
1687         }
1688
1689         return 0;
1690
1691 }
1692
1693 static struct xfrm_dst *
1694 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
1695                                const struct flowi *fl, u16 family,
1696                                struct dst_entry *dst_orig)
1697 {
1698         struct net *net = xp_net(pols[0]);
1699         struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
1700         struct dst_entry *dst;
1701         struct xfrm_dst *xdst;
1702         int err;
1703
1704         /* Try to instantiate a bundle */
1705         err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
1706         if (err <= 0) {
1707                 if (err != 0 && err != -EAGAIN)
1708                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1709                 return ERR_PTR(err);
1710         }
1711
1712         dst = xfrm_bundle_create(pols[0], xfrm, err, fl, dst_orig);
1713         if (IS_ERR(dst)) {
1714                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
1715                 return ERR_CAST(dst);
1716         }
1717
1718         xdst = (struct xfrm_dst *)dst;
1719         xdst->num_xfrms = err;
1720         if (num_pols > 1)
1721                 err = xfrm_dst_update_parent(dst, &pols[1]->selector);
1722         else
1723                 err = xfrm_dst_update_origin(dst, fl);
1724         if (unlikely(err)) {
1725                 dst_free(dst);
1726                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1727                 return ERR_PTR(err);
1728         }
1729
1730         xdst->num_pols = num_pols;
1731         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy*) * num_pols);
1732         xdst->policy_genid = atomic_read(&pols[0]->genid);
1733
1734         return xdst;
1735 }
1736
1737 static void xfrm_policy_queue_process(unsigned long arg)
1738 {
1739         int err = 0;
1740         struct sk_buff *skb;
1741         struct sock *sk;
1742         struct dst_entry *dst;
1743         struct net_device *dev;
1744         struct xfrm_policy *pol = (struct xfrm_policy *)arg;
1745         struct xfrm_policy_queue *pq = &pol->polq;
1746         struct flowi fl;
1747         struct sk_buff_head list;
1748
1749         spin_lock(&pq->hold_queue.lock);
1750         skb = skb_peek(&pq->hold_queue);
1751         dst = skb_dst(skb);
1752         sk = skb->sk;
1753         xfrm_decode_session(skb, &fl, dst->ops->family);
1754         spin_unlock(&pq->hold_queue.lock);
1755
1756         dst_hold(dst->path);
1757         dst = xfrm_lookup(xp_net(pol), dst->path, &fl,
1758                           sk, 0);
1759         if (IS_ERR(dst))
1760                 goto purge_queue;
1761
1762         if (dst->flags & DST_XFRM_QUEUE) {
1763                 dst_release(dst);
1764
1765                 if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
1766                         goto purge_queue;
1767
1768                 pq->timeout = pq->timeout << 1;
1769                 mod_timer(&pq->hold_timer, jiffies + pq->timeout);
1770                 return;
1771         }
1772
1773         dst_release(dst);
1774
1775         __skb_queue_head_init(&list);
1776
1777         spin_lock(&pq->hold_queue.lock);
1778         pq->timeout = 0;
1779         skb_queue_splice_init(&pq->hold_queue, &list);
1780         spin_unlock(&pq->hold_queue.lock);
1781
1782         while (!skb_queue_empty(&list)) {
1783                 skb = __skb_dequeue(&list);
1784
1785                 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
1786                 dst_hold(skb_dst(skb)->path);
1787                 dst = xfrm_lookup(xp_net(pol), skb_dst(skb)->path,
1788                                   &fl, skb->sk, 0);
1789                 if (IS_ERR(dst)) {
1790                         dev_put(skb->dev);
1791                         kfree_skb(skb);
1792                         continue;
1793                 }
1794
1795                 nf_reset(skb);
1796                 skb_dst_drop(skb);
1797                 skb_dst_set(skb, dst);
1798
1799                 dev = skb->dev;
1800                 err = dst_output(skb);
1801                 dev_put(dev);
1802         }
1803
1804         return;
1805
1806 purge_queue:
1807         pq->timeout = 0;
1808         xfrm_queue_purge(&pq->hold_queue);
1809 }
1810
1811 static int xdst_queue_output(struct sk_buff *skb)
1812 {
1813         unsigned long sched_next;
1814         struct dst_entry *dst = skb_dst(skb);
1815         struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
1816         struct xfrm_policy_queue *pq = &xdst->pols[0]->polq;
1817
1818         if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
1819                 kfree_skb(skb);
1820                 return -EAGAIN;
1821         }
1822
1823         skb_dst_force(skb);
1824         dev_hold(skb->dev);
1825
1826         spin_lock_bh(&pq->hold_queue.lock);
1827
1828         if (!pq->timeout)
1829                 pq->timeout = XFRM_QUEUE_TMO_MIN;
1830
1831         sched_next = jiffies + pq->timeout;
1832
1833         if (del_timer(&pq->hold_timer)) {
1834                 if (time_before(pq->hold_timer.expires, sched_next))
1835                         sched_next = pq->hold_timer.expires;
1836         }
1837
1838         __skb_queue_tail(&pq->hold_queue, skb);
1839         mod_timer(&pq->hold_timer, sched_next);
1840
1841         spin_unlock_bh(&pq->hold_queue.lock);
1842
1843         return 0;
1844 }
1845
1846 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
1847                                                  struct dst_entry *dst,
1848                                                  const struct flowi *fl,
1849                                                  int num_xfrms,
1850                                                  u16 family)
1851 {
1852         int err;
1853         struct net_device *dev;
1854         struct dst_entry *dst1;
1855         struct xfrm_dst *xdst;
1856
1857         xdst = xfrm_alloc_dst(net, family);
1858         if (IS_ERR(xdst))
1859                 return xdst;
1860
1861         if (net->xfrm.sysctl_larval_drop || num_xfrms <= 0 ||
1862             (fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP))
1863                 return xdst;
1864
1865         dst1 = &xdst->u.dst;
1866         dst_hold(dst);
1867         xdst->route = dst;
1868
1869         dst_copy_metrics(dst1, dst);
1870
1871         dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1872         dst1->flags |= DST_HOST | DST_XFRM_QUEUE;
1873         dst1->lastuse = jiffies;
1874
1875         dst1->input = dst_discard;
1876         dst1->output = xdst_queue_output;
1877
1878         dst_hold(dst);
1879         dst1->child = dst;
1880         dst1->path = dst;
1881
1882         xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
1883
1884         err = -ENODEV;
1885         dev = dst->dev;
1886         if (!dev)
1887                 goto free_dst;
1888
1889         err = xfrm_fill_dst(xdst, dev, fl);
1890         if (err)
1891                 goto free_dst;
1892
1893 out:
1894         return xdst;
1895
1896 free_dst:
1897         dst_release(dst1);
1898         xdst = ERR_PTR(err);
1899         goto out;
1900 }
1901
1902 static struct flow_cache_object *
1903 xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir,
1904                    struct flow_cache_object *oldflo, void *ctx)
1905 {
1906         struct dst_entry *dst_orig = (struct dst_entry *)ctx;
1907         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
1908         struct xfrm_dst *xdst, *new_xdst;
1909         int num_pols = 0, num_xfrms = 0, i, err, pol_dead;
1910
1911         /* Check if the policies from old bundle are usable */
1912         xdst = NULL;
1913         if (oldflo) {
1914                 xdst = container_of(oldflo, struct xfrm_dst, flo);
1915                 num_pols = xdst->num_pols;
1916                 num_xfrms = xdst->num_xfrms;
1917                 pol_dead = 0;
1918                 for (i = 0; i < num_pols; i++) {
1919                         pols[i] = xdst->pols[i];
1920                         pol_dead |= pols[i]->walk.dead;
1921                 }
1922                 if (pol_dead) {
1923                         dst_free(&xdst->u.dst);
1924                         xdst = NULL;
1925                         num_pols = 0;
1926                         num_xfrms = 0;
1927                         oldflo = NULL;
1928                 }
1929         }
1930
1931         /* Resolve policies to use if we couldn't get them from
1932          * previous cache entry */
1933         if (xdst == NULL) {
1934                 num_pols = 1;
1935                 pols[0] = __xfrm_policy_lookup(net, fl, family, dir);
1936                 err = xfrm_expand_policies(fl, family, pols,
1937                                            &num_pols, &num_xfrms);
1938                 if (err < 0)
1939                         goto inc_error;
1940                 if (num_pols == 0)
1941                         return NULL;
1942                 if (num_xfrms <= 0)
1943                         goto make_dummy_bundle;
1944         }
1945
1946         new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family, dst_orig);
1947         if (IS_ERR(new_xdst)) {
1948                 err = PTR_ERR(new_xdst);
1949                 if (err != -EAGAIN)
1950                         goto error;
1951                 if (oldflo == NULL)
1952                         goto make_dummy_bundle;
1953                 dst_hold(&xdst->u.dst);
1954                 return oldflo;
1955         } else if (new_xdst == NULL) {
1956                 num_xfrms = 0;
1957                 if (oldflo == NULL)
1958                         goto make_dummy_bundle;
1959                 xdst->num_xfrms = 0;
1960                 dst_hold(&xdst->u.dst);
1961                 return oldflo;
1962         }
1963
1964         /* Kill the previous bundle */
1965         if (xdst) {
1966                 /* The policies were stolen for newly generated bundle */
1967                 xdst->num_pols = 0;
1968                 dst_free(&xdst->u.dst);
1969         }
1970
1971         /* Flow cache does not have reference, it dst_free()'s,
1972          * but we do need to return one reference for original caller */
1973         dst_hold(&new_xdst->u.dst);
1974         return &new_xdst->flo;
1975
1976 make_dummy_bundle:
1977         /* We found policies, but there's no bundles to instantiate:
1978          * either because the policy blocks, has no transformations or
1979          * we could not build template (no xfrm_states).*/
1980         xdst = xfrm_create_dummy_bundle(net, dst_orig, fl, num_xfrms, family);
1981         if (IS_ERR(xdst)) {
1982                 xfrm_pols_put(pols, num_pols);
1983                 return ERR_CAST(xdst);
1984         }
1985         xdst->num_pols = num_pols;
1986         xdst->num_xfrms = num_xfrms;
1987         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy*) * num_pols);
1988
1989         dst_hold(&xdst->u.dst);
1990         return &xdst->flo;
1991
1992 inc_error:
1993         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1994 error:
1995         if (xdst != NULL)
1996                 dst_free(&xdst->u.dst);
1997         else
1998                 xfrm_pols_put(pols, num_pols);
1999         return ERR_PTR(err);
2000 }
2001
2002 static struct dst_entry *make_blackhole(struct net *net, u16 family,
2003                                         struct dst_entry *dst_orig)
2004 {
2005         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2006         struct dst_entry *ret;
2007
2008         if (!afinfo) {
2009                 dst_release(dst_orig);
2010                 return ERR_PTR(-EINVAL);
2011         } else {
2012                 ret = afinfo->blackhole_route(net, dst_orig);
2013         }
2014         xfrm_policy_put_afinfo(afinfo);
2015
2016         return ret;
2017 }
2018
2019 /* Main function: finds/creates a bundle for given flow.
2020  *
2021  * At the moment we eat a raw IP route. Mostly to speed up lookups
2022  * on interfaces with disabled IPsec.
2023  */
2024 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
2025                               const struct flowi *fl,
2026                               struct sock *sk, int flags)
2027 {
2028         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2029         struct flow_cache_object *flo;
2030         struct xfrm_dst *xdst;
2031         struct dst_entry *dst, *route;
2032         u16 family = dst_orig->ops->family;
2033         u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
2034         int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
2035
2036 restart:
2037         dst = NULL;
2038         xdst = NULL;
2039         route = NULL;
2040
2041         if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
2042                 num_pols = 1;
2043                 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
2044                 err = xfrm_expand_policies(fl, family, pols,
2045                                            &num_pols, &num_xfrms);
2046                 if (err < 0)
2047                         goto dropdst;
2048
2049                 if (num_pols) {
2050                         if (num_xfrms <= 0) {
2051                                 drop_pols = num_pols;
2052                                 goto no_transform;
2053                         }
2054
2055                         xdst = xfrm_resolve_and_create_bundle(
2056                                         pols, num_pols, fl,
2057                                         family, dst_orig);
2058                         if (IS_ERR(xdst)) {
2059                                 xfrm_pols_put(pols, num_pols);
2060                                 err = PTR_ERR(xdst);
2061                                 goto dropdst;
2062                         } else if (xdst == NULL) {
2063                                 num_xfrms = 0;
2064                                 drop_pols = num_pols;
2065                                 goto no_transform;
2066                         }
2067
2068                         dst_hold(&xdst->u.dst);
2069
2070                         spin_lock_bh(&xfrm_policy_sk_bundle_lock);
2071                         xdst->u.dst.next = xfrm_policy_sk_bundles;
2072                         xfrm_policy_sk_bundles = &xdst->u.dst;
2073                         spin_unlock_bh(&xfrm_policy_sk_bundle_lock);
2074
2075                         route = xdst->route;
2076                 }
2077         }
2078
2079         if (xdst == NULL) {
2080                 /* To accelerate a bit...  */
2081                 if ((dst_orig->flags & DST_NOXFRM) ||
2082                     !net->xfrm.policy_count[XFRM_POLICY_OUT])
2083                         goto nopol;
2084
2085                 flo = flow_cache_lookup(net, fl, family, dir,
2086                                         xfrm_bundle_lookup, dst_orig);
2087                 if (flo == NULL)
2088                         goto nopol;
2089                 if (IS_ERR(flo)) {
2090                         err = PTR_ERR(flo);
2091                         goto dropdst;
2092                 }
2093                 xdst = container_of(flo, struct xfrm_dst, flo);
2094
2095                 num_pols = xdst->num_pols;
2096                 num_xfrms = xdst->num_xfrms;
2097                 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy*) * num_pols);
2098                 route = xdst->route;
2099         }
2100
2101         dst = &xdst->u.dst;
2102         if (route == NULL && num_xfrms > 0) {
2103                 /* The only case when xfrm_bundle_lookup() returns a
2104                  * bundle with null route, is when the template could
2105                  * not be resolved. It means policies are there, but
2106                  * bundle could not be created, since we don't yet
2107                  * have the xfrm_state's. We need to wait for KM to
2108                  * negotiate new SA's or bail out with error.*/
2109                 if (net->xfrm.sysctl_larval_drop) {
2110                         /* EREMOTE tells the caller to generate
2111                          * a one-shot blackhole route. */
2112                         dst_release(dst);
2113                         xfrm_pols_put(pols, drop_pols);
2114                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2115
2116                         return make_blackhole(net, family, dst_orig);
2117                 }
2118                 if (fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP) {
2119                         DECLARE_WAITQUEUE(wait, current);
2120
2121                         add_wait_queue(&net->xfrm.km_waitq, &wait);
2122                         set_current_state(TASK_INTERRUPTIBLE);
2123                         schedule();
2124                         set_current_state(TASK_RUNNING);
2125                         remove_wait_queue(&net->xfrm.km_waitq, &wait);
2126
2127                         if (!signal_pending(current)) {
2128                                 dst_release(dst);
2129                                 goto restart;
2130                         }
2131
2132                         err = -ERESTART;
2133                 } else
2134                         err = -EAGAIN;
2135
2136                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2137                 goto error;
2138         }
2139
2140 no_transform:
2141         if (num_pols == 0)
2142                 goto nopol;
2143
2144         if ((flags & XFRM_LOOKUP_ICMP) &&
2145             !(pols[0]->flags & XFRM_POLICY_ICMP)) {
2146                 err = -ENOENT;
2147                 goto error;
2148         }
2149
2150         for (i = 0; i < num_pols; i++)
2151                 pols[i]->curlft.use_time = get_seconds();
2152
2153         if (num_xfrms < 0) {
2154                 /* Prohibit the flow */
2155                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
2156                 err = -EPERM;
2157                 goto error;
2158         } else if (num_xfrms > 0) {
2159                 /* Flow transformed */
2160                 dst_release(dst_orig);
2161         } else {
2162                 /* Flow passes untransformed */
2163                 dst_release(dst);
2164                 dst = dst_orig;
2165         }
2166 ok:
2167         xfrm_pols_put(pols, drop_pols);
2168         if (dst && dst->xfrm &&
2169             dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
2170                 dst->flags |= DST_XFRM_TUNNEL;
2171         return dst;
2172
2173 nopol:
2174         if (!(flags & XFRM_LOOKUP_ICMP)) {
2175                 dst = dst_orig;
2176                 goto ok;
2177         }
2178         err = -ENOENT;
2179 error:
2180         dst_release(dst);
2181 dropdst:
2182         dst_release(dst_orig);
2183         xfrm_pols_put(pols, drop_pols);
2184         return ERR_PTR(err);
2185 }
2186 EXPORT_SYMBOL(xfrm_lookup);
2187
2188 static inline int
2189 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
2190 {
2191         struct xfrm_state *x;
2192
2193         if (!skb->sp || idx < 0 || idx >= skb->sp->len)
2194                 return 0;
2195         x = skb->sp->xvec[idx];
2196         if (!x->type->reject)
2197                 return 0;
2198         return x->type->reject(x, skb, fl);
2199 }
2200
2201 /* When skb is transformed back to its "native" form, we have to
2202  * check policy restrictions. At the moment we make this in maximally
2203  * stupid way. Shame on me. :-) Of course, connected sockets must
2204  * have policy cached at them.
2205  */
2206
2207 static inline int
2208 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
2209               unsigned short family)
2210 {
2211         if (xfrm_state_kern(x))
2212                 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
2213         return  x->id.proto == tmpl->id.proto &&
2214                 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
2215                 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
2216                 x->props.mode == tmpl->mode &&
2217                 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
2218                  !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
2219                 !(x->props.mode != XFRM_MODE_TRANSPORT &&
2220                   xfrm_state_addr_cmp(tmpl, x, family));
2221 }
2222
2223 /*
2224  * 0 or more than 0 is returned when validation is succeeded (either bypass
2225  * because of optional transport mode, or next index of the mathced secpath
2226  * state with the template.
2227  * -1 is returned when no matching template is found.
2228  * Otherwise "-2 - errored_index" is returned.
2229  */
2230 static inline int
2231 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
2232                unsigned short family)
2233 {
2234         int idx = start;
2235
2236         if (tmpl->optional) {
2237                 if (tmpl->mode == XFRM_MODE_TRANSPORT)
2238                         return start;
2239         } else
2240                 start = -1;
2241         for (; idx < sp->len; idx++) {
2242                 if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
2243                         return ++idx;
2244                 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
2245                         if (start == -1)
2246                                 start = -2-idx;
2247                         break;
2248                 }
2249         }
2250         return start;
2251 }
2252
2253 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
2254                           unsigned int family, int reverse)
2255 {
2256         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2257         int err;
2258
2259         if (unlikely(afinfo == NULL))
2260                 return -EAFNOSUPPORT;
2261
2262         afinfo->decode_session(skb, fl, reverse);
2263         err = security_xfrm_decode_session(skb, &fl->flowi_secid);
2264         xfrm_policy_put_afinfo(afinfo);
2265         return err;
2266 }
2267 EXPORT_SYMBOL(__xfrm_decode_session);
2268
2269 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
2270 {
2271         for (; k < sp->len; k++) {
2272                 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
2273                         *idxp = k;
2274                         return 1;
2275                 }
2276         }
2277
2278         return 0;
2279 }
2280
2281 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
2282                         unsigned short family)
2283 {
2284         struct net *net = dev_net(skb->dev);
2285         struct xfrm_policy *pol;
2286         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2287         int npols = 0;
2288         int xfrm_nr;
2289         int pi;
2290         int reverse;
2291         struct flowi fl;
2292         u8 fl_dir;
2293         int xerr_idx = -1;
2294
2295         reverse = dir & ~XFRM_POLICY_MASK;
2296         dir &= XFRM_POLICY_MASK;
2297         fl_dir = policy_to_flow_dir(dir);
2298
2299         if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
2300                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
2301                 return 0;
2302         }
2303
2304         nf_nat_decode_session(skb, &fl, family);
2305
2306         /* First, check used SA against their selectors. */
2307         if (skb->sp) {
2308                 int i;
2309
2310                 for (i=skb->sp->len-1; i>=0; i--) {
2311                         struct xfrm_state *x = skb->sp->xvec[i];
2312                         if (!xfrm_selector_match(&x->sel, &fl, family)) {
2313                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
2314                                 return 0;
2315                         }
2316                 }
2317         }
2318
2319         pol = NULL;
2320         if (sk && sk->sk_policy[dir]) {
2321                 pol = xfrm_sk_policy_lookup(sk, dir, &fl);
2322                 if (IS_ERR(pol)) {
2323                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2324                         return 0;
2325                 }
2326         }
2327
2328         if (!pol) {
2329                 struct flow_cache_object *flo;
2330
2331                 flo = flow_cache_lookup(net, &fl, family, fl_dir,
2332                                         xfrm_policy_lookup, NULL);
2333                 if (IS_ERR_OR_NULL(flo))
2334                         pol = ERR_CAST(flo);
2335                 else
2336                         pol = container_of(flo, struct xfrm_policy, flo);
2337         }
2338
2339         if (IS_ERR(pol)) {
2340                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2341                 return 0;
2342         }
2343
2344         if (!pol) {
2345                 if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
2346                         xfrm_secpath_reject(xerr_idx, skb, &fl);
2347                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
2348                         return 0;
2349                 }
2350                 return 1;
2351         }
2352
2353         pol->curlft.use_time = get_seconds();
2354
2355         pols[0] = pol;
2356         npols ++;
2357 #ifdef CONFIG_XFRM_SUB_POLICY
2358         if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2359                 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
2360                                                     &fl, family,
2361                                                     XFRM_POLICY_IN);
2362                 if (pols[1]) {
2363                         if (IS_ERR(pols[1])) {
2364                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2365                                 return 0;
2366                         }
2367                         pols[1]->curlft.use_time = get_seconds();
2368                         npols ++;
2369                 }
2370         }
2371 #endif
2372
2373         if (pol->action == XFRM_POLICY_ALLOW) {
2374                 struct sec_path *sp;
2375                 static struct sec_path dummy;
2376                 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
2377                 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
2378                 struct xfrm_tmpl **tpp = tp;
2379                 int ti = 0;
2380                 int i, k;
2381
2382                 if ((sp = skb->sp) == NULL)
2383                         sp = &dummy;
2384
2385                 for (pi = 0; pi < npols; pi++) {
2386                         if (pols[pi] != pol &&
2387                             pols[pi]->action != XFRM_POLICY_ALLOW) {
2388                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2389                                 goto reject;
2390                         }
2391                         if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
2392                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
2393                                 goto reject_error;
2394                         }
2395                         for (i = 0; i < pols[pi]->xfrm_nr; i++)
2396                                 tpp[ti++] = &pols[pi]->xfrm_vec[i];
2397                 }
2398                 xfrm_nr = ti;
2399                 if (npols > 1) {
2400                         xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
2401                         tpp = stp;
2402                 }
2403
2404                 /* For each tunnel xfrm, find the first matching tmpl.
2405                  * For each tmpl before that, find corresponding xfrm.
2406                  * Order is _important_. Later we will implement
2407                  * some barriers, but at the moment barriers
2408                  * are implied between each two transformations.
2409                  */
2410                 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2411                         k = xfrm_policy_ok(tpp[i], sp, k, family);
2412                         if (k < 0) {
2413                                 if (k < -1)
2414                                         /* "-2 - errored_index" returned */
2415                                         xerr_idx = -(2+k);
2416                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2417                                 goto reject;
2418                         }
2419                 }
2420
2421                 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
2422                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2423                         goto reject;
2424                 }
2425
2426                 xfrm_pols_put(pols, npols);
2427                 return 1;
2428         }
2429         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2430
2431 reject:
2432         xfrm_secpath_reject(xerr_idx, skb, &fl);
2433 reject_error:
2434         xfrm_pols_put(pols, npols);
2435         return 0;
2436 }
2437 EXPORT_SYMBOL(__xfrm_policy_check);
2438
2439 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2440 {
2441         struct net *net = dev_net(skb->dev);
2442         struct flowi fl;
2443         struct dst_entry *dst;
2444         int res = 1;
2445
2446         if (xfrm_decode_session(skb, &fl, family) < 0) {
2447                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
2448                 return 0;
2449         }
2450
2451         skb_dst_force(skb);
2452
2453         dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, 0);
2454         if (IS_ERR(dst)) {
2455                 res = 0;
2456                 dst = NULL;
2457         }
2458         skb_dst_set(skb, dst);
2459         return res;
2460 }
2461 EXPORT_SYMBOL(__xfrm_route_forward);
2462
2463 /* Optimize later using cookies and generation ids. */
2464
2465 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
2466 {
2467         /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
2468          * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
2469          * get validated by dst_ops->check on every use.  We do this
2470          * because when a normal route referenced by an XFRM dst is
2471          * obsoleted we do not go looking around for all parent
2472          * referencing XFRM dsts so that we can invalidate them.  It
2473          * is just too much work.  Instead we make the checks here on
2474          * every use.  For example:
2475          *
2476          *      XFRM dst A --> IPv4 dst X
2477          *
2478          * X is the "xdst->route" of A (X is also the "dst->path" of A
2479          * in this example).  If X is marked obsolete, "A" will not
2480          * notice.  That's what we are validating here via the
2481          * stale_bundle() check.
2482          *
2483          * When a policy's bundle is pruned, we dst_free() the XFRM
2484          * dst which causes it's ->obsolete field to be set to
2485          * DST_OBSOLETE_DEAD.  If an XFRM dst has been pruned like
2486          * this, we want to force a new route lookup.
2487          */
2488         if (dst->obsolete < 0 && !stale_bundle(dst))
2489                 return dst;
2490
2491         return NULL;
2492 }
2493
2494 static int stale_bundle(struct dst_entry *dst)
2495 {
2496         return !xfrm_bundle_ok((struct xfrm_dst *)dst);
2497 }
2498
2499 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
2500 {
2501         while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
2502                 dst->dev = dev_net(dev)->loopback_dev;
2503                 dev_hold(dst->dev);
2504                 dev_put(dev);
2505         }
2506 }
2507 EXPORT_SYMBOL(xfrm_dst_ifdown);
2508
2509 static void xfrm_link_failure(struct sk_buff *skb)
2510 {
2511         /* Impossible. Such dst must be popped before reaches point of failure. */
2512 }
2513
2514 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
2515 {
2516         if (dst) {
2517                 if (dst->obsolete) {
2518                         dst_release(dst);
2519                         dst = NULL;
2520                 }
2521         }
2522         return dst;
2523 }
2524
2525 static void __xfrm_garbage_collect(struct net *net)
2526 {
2527         struct dst_entry *head, *next;
2528
2529         spin_lock_bh(&xfrm_policy_sk_bundle_lock);
2530         head = xfrm_policy_sk_bundles;
2531         xfrm_policy_sk_bundles = NULL;
2532         spin_unlock_bh(&xfrm_policy_sk_bundle_lock);
2533
2534         while (head) {
2535                 next = head->next;
2536                 dst_free(head);
2537                 head = next;
2538         }
2539 }
2540
2541 static void xfrm_garbage_collect(struct net *net)
2542 {
2543         flow_cache_flush();
2544         __xfrm_garbage_collect(net);
2545 }
2546
2547 static void xfrm_garbage_collect_deferred(struct net *net)
2548 {
2549         flow_cache_flush_deferred();
2550         __xfrm_garbage_collect(net);
2551 }
2552
2553 static void xfrm_init_pmtu(struct dst_entry *dst)
2554 {
2555         do {
2556                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2557                 u32 pmtu, route_mtu_cached;
2558
2559                 pmtu = dst_mtu(dst->child);
2560                 xdst->child_mtu_cached = pmtu;
2561
2562                 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2563
2564                 route_mtu_cached = dst_mtu(xdst->route);
2565                 xdst->route_mtu_cached = route_mtu_cached;
2566
2567                 if (pmtu > route_mtu_cached)
2568                         pmtu = route_mtu_cached;
2569
2570                 dst_metric_set(dst, RTAX_MTU, pmtu);
2571         } while ((dst = dst->next));
2572 }
2573
2574 /* Check that the bundle accepts the flow and its components are
2575  * still valid.
2576  */
2577
2578 static int xfrm_bundle_ok(struct xfrm_dst *first)
2579 {
2580         struct dst_entry *dst = &first->u.dst;
2581         struct xfrm_dst *last;
2582         u32 mtu;
2583
2584         if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) ||
2585             (dst->dev && !netif_running(dst->dev)))
2586                 return 0;
2587
2588         if (dst->flags & DST_XFRM_QUEUE)
2589                 return 1;
2590
2591         last = NULL;
2592
2593         do {
2594                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2595
2596                 if (dst->xfrm->km.state != XFRM_STATE_VALID)
2597                         return 0;
2598                 if (xdst->xfrm_genid != dst->xfrm->genid)
2599                         return 0;
2600                 if (xdst->num_pols > 0 &&
2601                     xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
2602                         return 0;
2603
2604                 mtu = dst_mtu(dst->child);
2605                 if (xdst->child_mtu_cached != mtu) {
2606                         last = xdst;
2607                         xdst->child_mtu_cached = mtu;
2608                 }
2609
2610                 if (!dst_check(xdst->route, xdst->route_cookie))
2611                         return 0;
2612                 mtu = dst_mtu(xdst->route);
2613                 if (xdst->route_mtu_cached != mtu) {
2614                         last = xdst;
2615                         xdst->route_mtu_cached = mtu;
2616                 }
2617
2618                 dst = dst->child;
2619         } while (dst->xfrm);
2620
2621         if (likely(!last))
2622                 return 1;
2623
2624         mtu = last->child_mtu_cached;
2625         for (;;) {
2626                 dst = &last->u.dst;
2627
2628                 mtu = xfrm_state_mtu(dst->xfrm, mtu);
2629                 if (mtu > last->route_mtu_cached)
2630                         mtu = last->route_mtu_cached;
2631                 dst_metric_set(dst, RTAX_MTU, mtu);
2632
2633                 if (last == first)
2634                         break;
2635
2636                 last = (struct xfrm_dst *)last->u.dst.next;
2637                 last->child_mtu_cached = mtu;
2638         }
2639
2640         return 1;
2641 }
2642
2643 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
2644 {
2645         return dst_metric_advmss(dst->path);
2646 }
2647
2648 static unsigned int xfrm_mtu(const struct dst_entry *dst)
2649 {
2650         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2651
2652         return mtu ? : dst_mtu(dst->path);
2653 }
2654
2655 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
2656                                            struct sk_buff *skb,
2657                                            const void *daddr)
2658 {
2659         return dst->path->ops->neigh_lookup(dst, skb, daddr);
2660 }
2661
2662 int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
2663 {
2664         struct net *net;
2665         int err = 0;
2666         if (unlikely(afinfo == NULL))
2667                 return -EINVAL;
2668         if (unlikely(afinfo->family >= NPROTO))
2669                 return -EAFNOSUPPORT;
2670         spin_lock(&xfrm_policy_afinfo_lock);
2671         if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
2672                 err = -ENOBUFS;
2673         else {
2674                 struct dst_ops *dst_ops = afinfo->dst_ops;
2675                 if (likely(dst_ops->kmem_cachep == NULL))
2676                         dst_ops->kmem_cachep = xfrm_dst_cache;
2677                 if (likely(dst_ops->check == NULL))
2678                         dst_ops->check = xfrm_dst_check;
2679                 if (likely(dst_ops->default_advmss == NULL))
2680                         dst_ops->default_advmss = xfrm_default_advmss;
2681                 if (likely(dst_ops->mtu == NULL))
2682                         dst_ops->mtu = xfrm_mtu;
2683                 if (likely(dst_ops->negative_advice == NULL))
2684                         dst_ops->negative_advice = xfrm_negative_advice;
2685                 if (likely(dst_ops->link_failure == NULL))
2686                         dst_ops->link_failure = xfrm_link_failure;
2687                 if (likely(dst_ops->neigh_lookup == NULL))
2688                         dst_ops->neigh_lookup = xfrm_neigh_lookup;
2689                 if (likely(afinfo->garbage_collect == NULL))
2690                         afinfo->garbage_collect = xfrm_garbage_collect_deferred;
2691                 rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
2692         }
2693         spin_unlock(&xfrm_policy_afinfo_lock);
2694
2695         rtnl_lock();
2696         for_each_net(net) {
2697                 struct dst_ops *xfrm_dst_ops;
2698
2699                 switch (afinfo->family) {
2700                 case AF_INET:
2701                         xfrm_dst_ops = &net->xfrm.xfrm4_dst_ops;
2702                         break;
2703 #if IS_ENABLED(CONFIG_IPV6)
2704                 case AF_INET6:
2705                         xfrm_dst_ops = &net->xfrm.xfrm6_dst_ops;
2706                         break;
2707 #endif
2708                 default:
2709                         BUG();
2710                 }
2711                 *xfrm_dst_ops = *afinfo->dst_ops;
2712         }
2713         rtnl_unlock();
2714
2715         return err;
2716 }
2717 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
2718
2719 int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
2720 {
2721         int err = 0;
2722         if (unlikely(afinfo == NULL))
2723                 return -EINVAL;
2724         if (unlikely(afinfo->family >= NPROTO))
2725                 return -EAFNOSUPPORT;
2726         spin_lock(&xfrm_policy_afinfo_lock);
2727         if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
2728                 if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
2729                         err = -EINVAL;
2730                 else
2731                         RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
2732                                          NULL);
2733         }
2734         spin_unlock(&xfrm_policy_afinfo_lock);
2735         if (!err) {
2736                 struct dst_ops *dst_ops = afinfo->dst_ops;
2737
2738                 synchronize_rcu();
2739
2740                 dst_ops->kmem_cachep = NULL;
2741                 dst_ops->check = NULL;
2742                 dst_ops->negative_advice = NULL;
2743                 dst_ops->link_failure = NULL;
2744                 afinfo->garbage_collect = NULL;
2745         }
2746         return err;
2747 }
2748 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
2749
2750 static void __net_init xfrm_dst_ops_init(struct net *net)
2751 {
2752         struct xfrm_policy_afinfo *afinfo;
2753
2754         rcu_read_lock();
2755         afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET]);
2756         if (afinfo)
2757                 net->xfrm.xfrm4_dst_ops = *afinfo->dst_ops;
2758 #if IS_ENABLED(CONFIG_IPV6)
2759         afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET6]);
2760         if (afinfo)
2761                 net->xfrm.xfrm6_dst_ops = *afinfo->dst_ops;
2762 #endif
2763         rcu_read_unlock();
2764 }
2765
2766 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
2767 {
2768         struct net_device *dev = ptr;
2769
2770         switch (event) {
2771         case NETDEV_DOWN:
2772                 xfrm_garbage_collect(dev_net(dev));
2773         }
2774         return NOTIFY_DONE;
2775 }
2776
2777 static struct notifier_block xfrm_dev_notifier = {
2778         .notifier_call  = xfrm_dev_event,
2779 };
2780
2781 #ifdef CONFIG_XFRM_STATISTICS
2782 static int __net_init xfrm_statistics_init(struct net *net)
2783 {
2784         int rv;
2785
2786         if (snmp_mib_init((void __percpu **)net->mib.xfrm_statistics,
2787                           sizeof(struct linux_xfrm_mib),
2788                           __alignof__(struct linux_xfrm_mib)) < 0)
2789                 return -ENOMEM;
2790         rv = xfrm_proc_init(net);
2791         if (rv < 0)
2792                 snmp_mib_free((void __percpu **)net->mib.xfrm_statistics);
2793         return rv;
2794 }
2795
2796 static void xfrm_statistics_fini(struct net *net)
2797 {
2798         xfrm_proc_fini(net);
2799         snmp_mib_free((void __percpu **)net->mib.xfrm_statistics);
2800 }
2801 #else
2802 static int __net_init xfrm_statistics_init(struct net *net)
2803 {
2804         return 0;
2805 }
2806
2807 static void xfrm_statistics_fini(struct net *net)
2808 {
2809 }
2810 #endif
2811
2812 static int __net_init xfrm_policy_init(struct net *net)
2813 {
2814         unsigned int hmask, sz;
2815         int dir;
2816
2817         if (net_eq(net, &init_net))
2818                 xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
2819                                            sizeof(struct xfrm_dst),
2820                                            0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2821                                            NULL);
2822
2823         hmask = 8 - 1;
2824         sz = (hmask+1) * sizeof(struct hlist_head);
2825
2826         net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
2827         if (!net->xfrm.policy_byidx)
2828                 goto out_byidx;
2829         net->xfrm.policy_idx_hmask = hmask;
2830
2831         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2832                 struct xfrm_policy_hash *htab;
2833
2834                 net->xfrm.policy_count[dir] = 0;
2835                 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
2836
2837                 htab = &net->xfrm.policy_bydst[dir];
2838                 htab->table = xfrm_hash_alloc(sz);
2839                 if (!htab->table)
2840                         goto out_bydst;
2841                 htab->hmask = hmask;
2842         }
2843
2844         INIT_LIST_HEAD(&net->xfrm.policy_all);
2845         INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
2846         if (net_eq(net, &init_net))
2847                 register_netdevice_notifier(&xfrm_dev_notifier);
2848         return 0;
2849
2850 out_bydst:
2851         for (dir--; dir >= 0; dir--) {
2852                 struct xfrm_policy_hash *htab;
2853
2854                 htab = &net->xfrm.policy_bydst[dir];
2855                 xfrm_hash_free(htab->table, sz);
2856         }
2857         xfrm_hash_free(net->xfrm.policy_byidx, sz);
2858 out_byidx:
2859         return -ENOMEM;
2860 }
2861
2862 static void xfrm_policy_fini(struct net *net)
2863 {
2864         struct xfrm_audit audit_info;
2865         unsigned int sz;
2866         int dir;
2867
2868         flush_work(&net->xfrm.policy_hash_work);
2869 #ifdef CONFIG_XFRM_SUB_POLICY
2870         audit_info.loginuid = INVALID_UID;
2871         audit_info.sessionid = -1;
2872         audit_info.secid = 0;
2873         xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, &audit_info);
2874 #endif
2875         audit_info.loginuid = INVALID_UID;
2876         audit_info.sessionid = -1;
2877         audit_info.secid = 0;
2878         xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, &audit_info);
2879
2880         WARN_ON(!list_empty(&net->xfrm.policy_all));
2881
2882         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2883                 struct xfrm_policy_hash *htab;
2884
2885                 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
2886
2887                 htab = &net->xfrm.policy_bydst[dir];
2888                 sz = (htab->hmask + 1) * sizeof(struct hlist_head);
2889                 WARN_ON(!hlist_empty(htab->table));
2890                 xfrm_hash_free(htab->table, sz);
2891         }
2892
2893         sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
2894         WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
2895         xfrm_hash_free(net->xfrm.policy_byidx, sz);
2896 }
2897
2898 static int __net_init xfrm_net_init(struct net *net)
2899 {
2900         int rv;
2901
2902         rv = xfrm_statistics_init(net);
2903         if (rv < 0)
2904                 goto out_statistics;
2905         rv = xfrm_state_init(net);
2906         if (rv < 0)
2907                 goto out_state;
2908         rv = xfrm_policy_init(net);
2909         if (rv < 0)
2910                 goto out_policy;
2911         xfrm_dst_ops_init(net);
2912         rv = xfrm_sysctl_init(net);
2913         if (rv < 0)
2914                 goto out_sysctl;
2915         return 0;
2916
2917 out_sysctl:
2918         xfrm_policy_fini(net);
2919 out_policy:
2920         xfrm_state_fini(net);
2921 out_state:
2922         xfrm_statistics_fini(net);
2923 out_statistics:
2924         return rv;
2925 }
2926
2927 static void __net_exit xfrm_net_exit(struct net *net)
2928 {
2929         xfrm_sysctl_fini(net);
2930         xfrm_policy_fini(net);
2931         xfrm_state_fini(net);
2932         xfrm_statistics_fini(net);
2933 }
2934
2935 static struct pernet_operations __net_initdata xfrm_net_ops = {
2936         .init = xfrm_net_init,
2937         .exit = xfrm_net_exit,
2938 };
2939
2940 void __init xfrm_init(void)
2941 {
2942         register_pernet_subsys(&xfrm_net_ops);
2943         xfrm_input_init();
2944 }
2945
2946 #ifdef CONFIG_AUDITSYSCALL
2947 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
2948                                          struct audit_buffer *audit_buf)
2949 {
2950         struct xfrm_sec_ctx *ctx = xp->security;
2951         struct xfrm_selector *sel = &xp->selector;
2952
2953         if (ctx)
2954                 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
2955                                  ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2956
2957         switch(sel->family) {
2958         case AF_INET:
2959                 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
2960                 if (sel->prefixlen_s != 32)
2961                         audit_log_format(audit_buf, " src_prefixlen=%d",
2962                                          sel->prefixlen_s);
2963                 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
2964                 if (sel->prefixlen_d != 32)
2965                         audit_log_format(audit_buf, " dst_prefixlen=%d",
2966                                          sel->prefixlen_d);
2967                 break;
2968         case AF_INET6:
2969                 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
2970                 if (sel->prefixlen_s != 128)
2971                         audit_log_format(audit_buf, " src_prefixlen=%d",
2972                                          sel->prefixlen_s);
2973                 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
2974                 if (sel->prefixlen_d != 128)
2975                         audit_log_format(audit_buf, " dst_prefixlen=%d",
2976                                          sel->prefixlen_d);
2977                 break;
2978         }
2979 }
2980
2981 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result,
2982                            kuid_t auid, u32 sessionid, u32 secid)
2983 {
2984         struct audit_buffer *audit_buf;
2985
2986         audit_buf = xfrm_audit_start("SPD-add");
2987         if (audit_buf == NULL)
2988                 return;
2989         xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
2990         audit_log_format(audit_buf, " res=%u", result);
2991         xfrm_audit_common_policyinfo(xp, audit_buf);
2992         audit_log_end(audit_buf);
2993 }
2994 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
2995
2996 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
2997                               kuid_t auid, u32 sessionid, u32 secid)
2998 {
2999         struct audit_buffer *audit_buf;
3000
3001         audit_buf = xfrm_audit_start("SPD-delete");
3002         if (audit_buf == NULL)
3003                 return;
3004         xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
3005         audit_log_format(audit_buf, " res=%u", result);
3006         xfrm_audit_common_policyinfo(xp, audit_buf);
3007         audit_log_end(audit_buf);
3008 }
3009 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
3010 #endif
3011
3012 #ifdef CONFIG_XFRM_MIGRATE
3013 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
3014                                         const struct xfrm_selector *sel_tgt)
3015 {
3016         if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
3017                 if (sel_tgt->family == sel_cmp->family &&
3018                     xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
3019                                     sel_cmp->family) &&
3020                     xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
3021                                     sel_cmp->family) &&
3022                     sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
3023                     sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
3024                         return true;
3025                 }
3026         } else {
3027                 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
3028                         return true;
3029                 }
3030         }
3031         return false;
3032 }
3033
3034 static struct xfrm_policy * xfrm_migrate_policy_find(const struct xfrm_selector *sel,
3035                                                      u8 dir, u8 type)
3036 {
3037         struct xfrm_policy *pol, *ret = NULL;
3038         struct hlist_head *chain;
3039         u32 priority = ~0U;
3040
3041         read_lock_bh(&xfrm_policy_lock);
3042         chain = policy_hash_direct(&init_net, &sel->daddr, &sel->saddr, sel->family, dir);
3043         hlist_for_each_entry(pol, chain, bydst) {
3044                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3045                     pol->type == type) {
3046                         ret = pol;
3047                         priority = ret->priority;
3048                         break;
3049                 }
3050         }
3051         chain = &init_net.xfrm.policy_inexact[dir];
3052         hlist_for_each_entry(pol, chain, bydst) {
3053                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3054                     pol->type == type &&
3055                     pol->priority < priority) {
3056                         ret = pol;
3057                         break;
3058                 }
3059         }
3060
3061         if (ret)
3062                 xfrm_pol_hold(ret);
3063
3064         read_unlock_bh(&xfrm_policy_lock);
3065
3066         return ret;
3067 }
3068
3069 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
3070 {
3071         int match = 0;
3072
3073         if (t->mode == m->mode && t->id.proto == m->proto &&
3074             (m->reqid == 0 || t->reqid == m->reqid)) {
3075                 switch (t->mode) {
3076                 case XFRM_MODE_TUNNEL:
3077                 case XFRM_MODE_BEET:
3078                         if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
3079                                             m->old_family) &&
3080                             xfrm_addr_equal(&t->saddr, &m->old_saddr,
3081                                             m->old_family)) {
3082                                 match = 1;
3083                         }
3084                         break;
3085                 case XFRM_MODE_TRANSPORT:
3086                         /* in case of transport mode, template does not store
3087                            any IP addresses, hence we just compare mode and
3088                            protocol */
3089                         match = 1;
3090                         break;
3091                 default:
3092                         break;
3093                 }
3094         }
3095         return match;
3096 }
3097
3098 /* update endpoint address(es) of template(s) */
3099 static int xfrm_policy_migrate(struct xfrm_policy *pol,
3100                                struct xfrm_migrate *m, int num_migrate)
3101 {
3102         struct xfrm_migrate *mp;
3103         int i, j, n = 0;
3104
3105         write_lock_bh(&pol->lock);
3106         if (unlikely(pol->walk.dead)) {
3107                 /* target policy has been deleted */
3108                 write_unlock_bh(&pol->lock);
3109                 return -ENOENT;
3110         }
3111
3112         for (i = 0; i < pol->xfrm_nr; i++) {
3113                 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
3114                         if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
3115                                 continue;
3116                         n++;
3117                         if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
3118                             pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
3119                                 continue;
3120                         /* update endpoints */
3121                         memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
3122                                sizeof(pol->xfrm_vec[i].id.daddr));
3123                         memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
3124                                sizeof(pol->xfrm_vec[i].saddr));
3125                         pol->xfrm_vec[i].encap_family = mp->new_family;
3126                         /* flush bundles */
3127                         atomic_inc(&pol->genid);
3128                 }
3129         }
3130
3131         write_unlock_bh(&pol->lock);
3132
3133         if (!n)
3134                 return -ENODATA;
3135
3136         return 0;
3137 }
3138
3139 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
3140 {
3141         int i, j;
3142
3143         if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
3144                 return -EINVAL;
3145
3146         for (i = 0; i < num_migrate; i++) {
3147                 if (xfrm_addr_equal(&m[i].old_daddr, &m[i].new_daddr,
3148                                     m[i].old_family) &&
3149                     xfrm_addr_equal(&m[i].old_saddr, &m[i].new_saddr,
3150                                     m[i].old_family))
3151                         return -EINVAL;
3152                 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
3153                     xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
3154                         return -EINVAL;
3155
3156                 /* check if there is any duplicated entry */
3157                 for (j = i + 1; j < num_migrate; j++) {
3158                         if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
3159                                     sizeof(m[i].old_daddr)) &&
3160                             !memcmp(&m[i].old_saddr, &m[j].old_saddr,
3161                                     sizeof(m[i].old_saddr)) &&
3162                             m[i].proto == m[j].proto &&
3163                             m[i].mode == m[j].mode &&
3164                             m[i].reqid == m[j].reqid &&
3165                             m[i].old_family == m[j].old_family)
3166                                 return -EINVAL;
3167                 }
3168         }
3169
3170         return 0;
3171 }
3172
3173 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
3174                  struct xfrm_migrate *m, int num_migrate,
3175                  struct xfrm_kmaddress *k)
3176 {
3177         int i, err, nx_cur = 0, nx_new = 0;
3178         struct xfrm_policy *pol = NULL;
3179         struct xfrm_state *x, *xc;
3180         struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
3181         struct xfrm_state *x_new[XFRM_MAX_DEPTH];
3182         struct xfrm_migrate *mp;
3183
3184         if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
3185                 goto out;
3186
3187         /* Stage 1 - find policy */
3188         if ((pol = xfrm_migrate_policy_find(sel, dir, type)) == NULL) {
3189                 err = -ENOENT;
3190                 goto out;
3191         }
3192
3193         /* Stage 2 - find and update state(s) */
3194         for (i = 0, mp = m; i < num_migrate; i++, mp++) {
3195                 if ((x = xfrm_migrate_state_find(mp))) {
3196                         x_cur[nx_cur] = x;
3197                         nx_cur++;
3198                         if ((xc = xfrm_state_migrate(x, mp))) {
3199                                 x_new[nx_new] = xc;
3200                                 nx_new++;
3201                         } else {
3202                                 err = -ENODATA;
3203                                 goto restore_state;
3204                         }
3205                 }
3206         }
3207
3208         /* Stage 3 - update policy */
3209         if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
3210                 goto restore_state;
3211
3212         /* Stage 4 - delete old state(s) */
3213         if (nx_cur) {
3214                 xfrm_states_put(x_cur, nx_cur);
3215                 xfrm_states_delete(x_cur, nx_cur);
3216         }
3217
3218         /* Stage 5 - announce */
3219         km_migrate(sel, dir, type, m, num_migrate, k);
3220
3221         xfrm_pol_put(pol);
3222
3223         return 0;
3224 out:
3225         return err;
3226
3227 restore_state:
3228         if (pol)
3229                 xfrm_pol_put(pol);
3230         if (nx_cur)
3231                 xfrm_states_put(x_cur, nx_cur);
3232         if (nx_new)
3233                 xfrm_states_delete(x_new, nx_new);
3234
3235         return err;
3236 }
3237 EXPORT_SYMBOL(xfrm_migrate);
3238 #endif