2 * NET3 Protocol independent device support routines.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Derived from the non IP parts of dev.c 1.0.19
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a
26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function
45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close
51 * Rudi Cilibrasi : Pass the right thing to
53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge
67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback
75 #include <asm/uaccess.h>
76 #include <asm/system.h>
77 #include <linux/bitops.h>
78 #include <linux/capability.h>
79 #include <linux/cpu.h>
80 #include <linux/types.h>
81 #include <linux/kernel.h>
82 #include <linux/sched.h>
83 #include <linux/mutex.h>
84 #include <linux/string.h>
86 #include <linux/socket.h>
87 #include <linux/sockios.h>
88 #include <linux/errno.h>
89 #include <linux/interrupt.h>
90 #include <linux/if_ether.h>
91 #include <linux/netdevice.h>
92 #include <linux/etherdevice.h>
93 #include <linux/ethtool.h>
94 #include <linux/notifier.h>
95 #include <linux/skbuff.h>
96 #include <net/net_namespace.h>
98 #include <linux/rtnetlink.h>
99 #include <linux/proc_fs.h>
100 #include <linux/seq_file.h>
101 #include <linux/stat.h>
102 #include <linux/if_bridge.h>
103 #include <linux/if_macvlan.h>
105 #include <net/pkt_sched.h>
106 #include <net/checksum.h>
107 #include <linux/highmem.h>
108 #include <linux/init.h>
109 #include <linux/kmod.h>
110 #include <linux/module.h>
111 #include <linux/netpoll.h>
112 #include <linux/rcupdate.h>
113 #include <linux/delay.h>
114 #include <net/wext.h>
115 #include <net/iw_handler.h>
116 #include <asm/current.h>
117 #include <linux/audit.h>
118 #include <linux/dmaengine.h>
119 #include <linux/err.h>
120 #include <linux/ctype.h>
121 #include <linux/if_arp.h>
122 #include <linux/if_vlan.h>
123 #include <linux/ip.h>
125 #include <linux/ipv6.h>
126 #include <linux/in.h>
127 #include <linux/jhash.h>
128 #include <linux/random.h>
129 #include <trace/napi.h>
131 #include "net-sysfs.h"
133 /* Instead of increasing this, you should create a hash table. */
134 #define MAX_GRO_SKBS 8
136 /* This should be increased if a protocol with a bigger head is added. */
137 #define GRO_MAX_HEAD (MAX_HEADER + 128)
140 * The list of packet types we will receive (as opposed to discard)
141 * and the routines to invoke.
143 * Why 16. Because with 16 the only overlap we get on a hash of the
144 * low nibble of the protocol value is RARP/SNAP/X.25.
146 * NOTE: That is no longer true with the addition of VLAN tags. Not
147 * sure which should go first, but I bet it won't make much
148 * difference if we are running VLANs. The good news is that
149 * this protocol won't be in the list unless compiled in, so
150 * the average user (w/out VLANs) will not be adversely affected.
167 #define PTYPE_HASH_SIZE (16)
168 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
170 static DEFINE_SPINLOCK(ptype_lock);
171 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
172 static struct list_head ptype_all __read_mostly; /* Taps */
175 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
178 * Pure readers hold dev_base_lock for reading.
180 * Writers must hold the rtnl semaphore while they loop through the
181 * dev_base_head list, and hold dev_base_lock for writing when they do the
182 * actual updates. This allows pure readers to access the list even
183 * while a writer is preparing to update it.
185 * To put it another way, dev_base_lock is held for writing only to
186 * protect against pure readers; the rtnl semaphore provides the
187 * protection against other writers.
189 * See, for example usages, register_netdevice() and
190 * unregister_netdevice(), which must be called with the rtnl
193 DEFINE_RWLOCK(dev_base_lock);
195 EXPORT_SYMBOL(dev_base_lock);
197 #define NETDEV_HASHBITS 8
198 #define NETDEV_HASHENTRIES (1 << NETDEV_HASHBITS)
200 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
202 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
203 return &net->dev_name_head[hash & ((1 << NETDEV_HASHBITS) - 1)];
206 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
208 return &net->dev_index_head[ifindex & ((1 << NETDEV_HASHBITS) - 1)];
211 /* Device list insertion */
212 static int list_netdevice(struct net_device *dev)
214 struct net *net = dev_net(dev);
218 write_lock_bh(&dev_base_lock);
219 list_add_tail(&dev->dev_list, &net->dev_base_head);
220 hlist_add_head(&dev->name_hlist, dev_name_hash(net, dev->name));
221 hlist_add_head(&dev->index_hlist, dev_index_hash(net, dev->ifindex));
222 write_unlock_bh(&dev_base_lock);
226 /* Device list removal */
227 static void unlist_netdevice(struct net_device *dev)
231 /* Unlink dev from the device chain */
232 write_lock_bh(&dev_base_lock);
233 list_del(&dev->dev_list);
234 hlist_del(&dev->name_hlist);
235 hlist_del(&dev->index_hlist);
236 write_unlock_bh(&dev_base_lock);
243 static RAW_NOTIFIER_HEAD(netdev_chain);
246 * Device drivers call our routines to queue packets here. We empty the
247 * queue in the local softnet handler.
250 DEFINE_PER_CPU(struct softnet_data, softnet_data);
252 #ifdef CONFIG_LOCKDEP
254 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
255 * according to dev->type
257 static const unsigned short netdev_lock_type[] =
258 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
259 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
260 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
261 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
262 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
263 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
264 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
265 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
266 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
267 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
268 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
269 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
270 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
271 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
272 ARPHRD_PHONET_PIPE, ARPHRD_VOID, ARPHRD_NONE};
274 static const char *netdev_lock_name[] =
275 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
276 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
277 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
278 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
279 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
280 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
281 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
282 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
283 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
284 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
285 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
286 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
287 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
288 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
289 "_xmit_PHONET_PIPE", "_xmit_VOID", "_xmit_NONE"};
291 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
292 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
294 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
298 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
299 if (netdev_lock_type[i] == dev_type)
301 /* the last key is used by default */
302 return ARRAY_SIZE(netdev_lock_type) - 1;
305 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
306 unsigned short dev_type)
310 i = netdev_lock_pos(dev_type);
311 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
312 netdev_lock_name[i]);
315 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
319 i = netdev_lock_pos(dev->type);
320 lockdep_set_class_and_name(&dev->addr_list_lock,
321 &netdev_addr_lock_key[i],
322 netdev_lock_name[i]);
325 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
326 unsigned short dev_type)
329 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
334 /*******************************************************************************
336 Protocol management and registration routines
338 *******************************************************************************/
341 * Add a protocol ID to the list. Now that the input handler is
342 * smarter we can dispense with all the messy stuff that used to be
345 * BEWARE!!! Protocol handlers, mangling input packets,
346 * MUST BE last in hash buckets and checking protocol handlers
347 * MUST start from promiscuous ptype_all chain in net_bh.
348 * It is true now, do not change it.
349 * Explanation follows: if protocol handler, mangling packet, will
350 * be the first on list, it is not able to sense, that packet
351 * is cloned and should be copied-on-write, so that it will
352 * change it and subsequent readers will get broken packet.
357 * dev_add_pack - add packet handler
358 * @pt: packet type declaration
360 * Add a protocol handler to the networking stack. The passed &packet_type
361 * is linked into kernel lists and may not be freed until it has been
362 * removed from the kernel lists.
364 * This call does not sleep therefore it can not
365 * guarantee all CPU's that are in middle of receiving packets
366 * will see the new packet type (until the next received packet).
369 void dev_add_pack(struct packet_type *pt)
373 spin_lock_bh(&ptype_lock);
374 if (pt->type == htons(ETH_P_ALL))
375 list_add_rcu(&pt->list, &ptype_all);
377 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
378 list_add_rcu(&pt->list, &ptype_base[hash]);
380 spin_unlock_bh(&ptype_lock);
384 * __dev_remove_pack - remove packet handler
385 * @pt: packet type declaration
387 * Remove a protocol handler that was previously added to the kernel
388 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
389 * from the kernel lists and can be freed or reused once this function
392 * The packet type might still be in use by receivers
393 * and must not be freed until after all the CPU's have gone
394 * through a quiescent state.
396 void __dev_remove_pack(struct packet_type *pt)
398 struct list_head *head;
399 struct packet_type *pt1;
401 spin_lock_bh(&ptype_lock);
403 if (pt->type == htons(ETH_P_ALL))
406 head = &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
408 list_for_each_entry(pt1, head, list) {
410 list_del_rcu(&pt->list);
415 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
417 spin_unlock_bh(&ptype_lock);
420 * dev_remove_pack - remove packet handler
421 * @pt: packet type declaration
423 * Remove a protocol handler that was previously added to the kernel
424 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
425 * from the kernel lists and can be freed or reused once this function
428 * This call sleeps to guarantee that no CPU is looking at the packet
431 void dev_remove_pack(struct packet_type *pt)
433 __dev_remove_pack(pt);
438 /******************************************************************************
440 Device Boot-time Settings Routines
442 *******************************************************************************/
444 /* Boot time configuration table */
445 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
448 * netdev_boot_setup_add - add new setup entry
449 * @name: name of the device
450 * @map: configured settings for the device
452 * Adds new setup entry to the dev_boot_setup list. The function
453 * returns 0 on error and 1 on success. This is a generic routine to
456 static int netdev_boot_setup_add(char *name, struct ifmap *map)
458 struct netdev_boot_setup *s;
462 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
463 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
464 memset(s[i].name, 0, sizeof(s[i].name));
465 strlcpy(s[i].name, name, IFNAMSIZ);
466 memcpy(&s[i].map, map, sizeof(s[i].map));
471 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
475 * netdev_boot_setup_check - check boot time settings
476 * @dev: the netdevice
478 * Check boot time settings for the device.
479 * The found settings are set for the device to be used
480 * later in the device probing.
481 * Returns 0 if no settings found, 1 if they are.
483 int netdev_boot_setup_check(struct net_device *dev)
485 struct netdev_boot_setup *s = dev_boot_setup;
488 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
489 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
490 !strcmp(dev->name, s[i].name)) {
491 dev->irq = s[i].map.irq;
492 dev->base_addr = s[i].map.base_addr;
493 dev->mem_start = s[i].map.mem_start;
494 dev->mem_end = s[i].map.mem_end;
503 * netdev_boot_base - get address from boot time settings
504 * @prefix: prefix for network device
505 * @unit: id for network device
507 * Check boot time settings for the base address of device.
508 * The found settings are set for the device to be used
509 * later in the device probing.
510 * Returns 0 if no settings found.
512 unsigned long netdev_boot_base(const char *prefix, int unit)
514 const struct netdev_boot_setup *s = dev_boot_setup;
518 sprintf(name, "%s%d", prefix, unit);
521 * If device already registered then return base of 1
522 * to indicate not to probe for this interface
524 if (__dev_get_by_name(&init_net, name))
527 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
528 if (!strcmp(name, s[i].name))
529 return s[i].map.base_addr;
534 * Saves at boot time configured settings for any netdevice.
536 int __init netdev_boot_setup(char *str)
541 str = get_options(str, ARRAY_SIZE(ints), ints);
546 memset(&map, 0, sizeof(map));
550 map.base_addr = ints[2];
552 map.mem_start = ints[3];
554 map.mem_end = ints[4];
556 /* Add new entry to the list */
557 return netdev_boot_setup_add(str, &map);
560 __setup("netdev=", netdev_boot_setup);
562 /*******************************************************************************
564 Device Interface Subroutines
566 *******************************************************************************/
569 * __dev_get_by_name - find a device by its name
570 * @net: the applicable net namespace
571 * @name: name to find
573 * Find an interface by name. Must be called under RTNL semaphore
574 * or @dev_base_lock. If the name is found a pointer to the device
575 * is returned. If the name is not found then %NULL is returned. The
576 * reference counters are not incremented so the caller must be
577 * careful with locks.
580 struct net_device *__dev_get_by_name(struct net *net, const char *name)
582 struct hlist_node *p;
584 hlist_for_each(p, dev_name_hash(net, name)) {
585 struct net_device *dev
586 = hlist_entry(p, struct net_device, name_hlist);
587 if (!strncmp(dev->name, name, IFNAMSIZ))
594 * dev_get_by_name - find a device by its name
595 * @net: the applicable net namespace
596 * @name: name to find
598 * Find an interface by name. This can be called from any
599 * context and does its own locking. The returned handle has
600 * the usage count incremented and the caller must use dev_put() to
601 * release it when it is no longer needed. %NULL is returned if no
602 * matching device is found.
605 struct net_device *dev_get_by_name(struct net *net, const char *name)
607 struct net_device *dev;
609 read_lock(&dev_base_lock);
610 dev = __dev_get_by_name(net, name);
613 read_unlock(&dev_base_lock);
618 * __dev_get_by_index - find a device by its ifindex
619 * @net: the applicable net namespace
620 * @ifindex: index of device
622 * Search for an interface by index. Returns %NULL if the device
623 * is not found or a pointer to the device. The device has not
624 * had its reference counter increased so the caller must be careful
625 * about locking. The caller must hold either the RTNL semaphore
629 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
631 struct hlist_node *p;
633 hlist_for_each(p, dev_index_hash(net, ifindex)) {
634 struct net_device *dev
635 = hlist_entry(p, struct net_device, index_hlist);
636 if (dev->ifindex == ifindex)
644 * dev_get_by_index - find a device by its ifindex
645 * @net: the applicable net namespace
646 * @ifindex: index of device
648 * Search for an interface by index. Returns NULL if the device
649 * is not found or a pointer to the device. The device returned has
650 * had a reference added and the pointer is safe until the user calls
651 * dev_put to indicate they have finished with it.
654 struct net_device *dev_get_by_index(struct net *net, int ifindex)
656 struct net_device *dev;
658 read_lock(&dev_base_lock);
659 dev = __dev_get_by_index(net, ifindex);
662 read_unlock(&dev_base_lock);
667 * dev_getbyhwaddr - find a device by its hardware address
668 * @net: the applicable net namespace
669 * @type: media type of device
670 * @ha: hardware address
672 * Search for an interface by MAC address. Returns NULL if the device
673 * is not found or a pointer to the device. The caller must hold the
674 * rtnl semaphore. The returned device has not had its ref count increased
675 * and the caller must therefore be careful about locking
678 * If the API was consistent this would be __dev_get_by_hwaddr
681 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
683 struct net_device *dev;
687 for_each_netdev(net, dev)
688 if (dev->type == type &&
689 !memcmp(dev->dev_addr, ha, dev->addr_len))
695 EXPORT_SYMBOL(dev_getbyhwaddr);
697 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
699 struct net_device *dev;
702 for_each_netdev(net, dev)
703 if (dev->type == type)
709 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
711 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
713 struct net_device *dev;
716 dev = __dev_getfirstbyhwtype(net, type);
723 EXPORT_SYMBOL(dev_getfirstbyhwtype);
726 * dev_get_by_flags - find any device with given flags
727 * @net: the applicable net namespace
728 * @if_flags: IFF_* values
729 * @mask: bitmask of bits in if_flags to check
731 * Search for any interface with the given flags. Returns NULL if a device
732 * is not found or a pointer to the device. The device returned has
733 * had a reference added and the pointer is safe until the user calls
734 * dev_put to indicate they have finished with it.
737 struct net_device * dev_get_by_flags(struct net *net, unsigned short if_flags, unsigned short mask)
739 struct net_device *dev, *ret;
742 read_lock(&dev_base_lock);
743 for_each_netdev(net, dev) {
744 if (((dev->flags ^ if_flags) & mask) == 0) {
750 read_unlock(&dev_base_lock);
755 * dev_valid_name - check if name is okay for network device
758 * Network device names need to be valid file names to
759 * to allow sysfs to work. We also disallow any kind of
762 int dev_valid_name(const char *name)
766 if (strlen(name) >= IFNAMSIZ)
768 if (!strcmp(name, ".") || !strcmp(name, ".."))
772 if (*name == '/' || isspace(*name))
780 * __dev_alloc_name - allocate a name for a device
781 * @net: network namespace to allocate the device name in
782 * @name: name format string
783 * @buf: scratch buffer and result name string
785 * Passed a format string - eg "lt%d" it will try and find a suitable
786 * id. It scans list of devices to build up a free map, then chooses
787 * the first empty slot. The caller must hold the dev_base or rtnl lock
788 * while allocating the name and adding the device in order to avoid
790 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
791 * Returns the number of the unit assigned or a negative errno code.
794 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
798 const int max_netdevices = 8*PAGE_SIZE;
799 unsigned long *inuse;
800 struct net_device *d;
802 p = strnchr(name, IFNAMSIZ-1, '%');
805 * Verify the string as this thing may have come from
806 * the user. There must be either one "%d" and no other "%"
809 if (p[1] != 'd' || strchr(p + 2, '%'))
812 /* Use one page as a bit array of possible slots */
813 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
817 for_each_netdev(net, d) {
818 if (!sscanf(d->name, name, &i))
820 if (i < 0 || i >= max_netdevices)
823 /* avoid cases where sscanf is not exact inverse of printf */
824 snprintf(buf, IFNAMSIZ, name, i);
825 if (!strncmp(buf, d->name, IFNAMSIZ))
829 i = find_first_zero_bit(inuse, max_netdevices);
830 free_page((unsigned long) inuse);
833 snprintf(buf, IFNAMSIZ, name, i);
834 if (!__dev_get_by_name(net, buf))
837 /* It is possible to run out of possible slots
838 * when the name is long and there isn't enough space left
839 * for the digits, or if all bits are used.
845 * dev_alloc_name - allocate a name for a device
847 * @name: name format string
849 * Passed a format string - eg "lt%d" it will try and find a suitable
850 * id. It scans list of devices to build up a free map, then chooses
851 * the first empty slot. The caller must hold the dev_base or rtnl lock
852 * while allocating the name and adding the device in order to avoid
854 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
855 * Returns the number of the unit assigned or a negative errno code.
858 int dev_alloc_name(struct net_device *dev, const char *name)
864 BUG_ON(!dev_net(dev));
866 ret = __dev_alloc_name(net, name, buf);
868 strlcpy(dev->name, buf, IFNAMSIZ);
874 * dev_change_name - change name of a device
876 * @newname: name (or format string) must be at least IFNAMSIZ
878 * Change name of a device, can pass format strings "eth%d".
881 int dev_change_name(struct net_device *dev, const char *newname)
883 char oldname[IFNAMSIZ];
889 BUG_ON(!dev_net(dev));
892 if (dev->flags & IFF_UP)
895 if (!dev_valid_name(newname))
898 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
901 memcpy(oldname, dev->name, IFNAMSIZ);
903 if (strchr(newname, '%')) {
904 err = dev_alloc_name(dev, newname);
908 else if (__dev_get_by_name(net, newname))
911 strlcpy(dev->name, newname, IFNAMSIZ);
914 /* For now only devices in the initial network namespace
917 if (net == &init_net) {
918 ret = device_rename(&dev->dev, dev->name);
920 memcpy(dev->name, oldname, IFNAMSIZ);
925 write_lock_bh(&dev_base_lock);
926 hlist_del(&dev->name_hlist);
927 hlist_add_head(&dev->name_hlist, dev_name_hash(net, dev->name));
928 write_unlock_bh(&dev_base_lock);
930 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
931 ret = notifier_to_errno(ret);
936 "%s: name change rollback failed: %d.\n",
940 memcpy(dev->name, oldname, IFNAMSIZ);
949 * dev_set_alias - change ifalias of a device
951 * @alias: name up to IFALIASZ
952 * @len: limit of bytes to copy from info
954 * Set ifalias for a device,
956 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
971 dev->ifalias = krealloc(dev->ifalias, len+1, GFP_KERNEL);
975 strlcpy(dev->ifalias, alias, len+1);
981 * netdev_features_change - device changes features
982 * @dev: device to cause notification
984 * Called to indicate a device has changed features.
986 void netdev_features_change(struct net_device *dev)
988 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
990 EXPORT_SYMBOL(netdev_features_change);
993 * netdev_state_change - device changes state
994 * @dev: device to cause notification
996 * Called to indicate a device has changed state. This function calls
997 * the notifier chains for netdev_chain and sends a NEWLINK message
998 * to the routing socket.
1000 void netdev_state_change(struct net_device *dev)
1002 if (dev->flags & IFF_UP) {
1003 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1004 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1008 void netdev_bonding_change(struct net_device *dev)
1010 call_netdevice_notifiers(NETDEV_BONDING_FAILOVER, dev);
1012 EXPORT_SYMBOL(netdev_bonding_change);
1015 * dev_load - load a network module
1016 * @net: the applicable net namespace
1017 * @name: name of interface
1019 * If a network interface is not present and the process has suitable
1020 * privileges this function loads the module. If module loading is not
1021 * available in this kernel then it becomes a nop.
1024 void dev_load(struct net *net, const char *name)
1026 struct net_device *dev;
1028 read_lock(&dev_base_lock);
1029 dev = __dev_get_by_name(net, name);
1030 read_unlock(&dev_base_lock);
1032 if (!dev && capable(CAP_SYS_MODULE))
1033 request_module("%s", name);
1037 * dev_open - prepare an interface for use.
1038 * @dev: device to open
1040 * Takes a device from down to up state. The device's private open
1041 * function is invoked and then the multicast lists are loaded. Finally
1042 * the device is moved into the up state and a %NETDEV_UP message is
1043 * sent to the netdev notifier chain.
1045 * Calling this function on an active interface is a nop. On a failure
1046 * a negative errno code is returned.
1048 int dev_open(struct net_device *dev)
1050 const struct net_device_ops *ops = dev->netdev_ops;
1059 if (dev->flags & IFF_UP)
1063 * Is it even present?
1065 if (!netif_device_present(dev))
1069 * Call device private open method
1071 set_bit(__LINK_STATE_START, &dev->state);
1073 if (ops->ndo_validate_addr)
1074 ret = ops->ndo_validate_addr(dev);
1076 if (!ret && ops->ndo_open)
1077 ret = ops->ndo_open(dev);
1080 * If it went open OK then:
1084 clear_bit(__LINK_STATE_START, &dev->state);
1089 dev->flags |= IFF_UP;
1094 net_dmaengine_get();
1097 * Initialize multicasting status
1099 dev_set_rx_mode(dev);
1102 * Wakeup transmit queue engine
1107 * ... and announce new interface.
1109 call_netdevice_notifiers(NETDEV_UP, dev);
1116 * dev_close - shutdown an interface.
1117 * @dev: device to shutdown
1119 * This function moves an active device into down state. A
1120 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1121 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1124 int dev_close(struct net_device *dev)
1126 const struct net_device_ops *ops = dev->netdev_ops;
1131 if (!(dev->flags & IFF_UP))
1135 * Tell people we are going down, so that they can
1136 * prepare to death, when device is still operating.
1138 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1140 clear_bit(__LINK_STATE_START, &dev->state);
1142 /* Synchronize to scheduled poll. We cannot touch poll list,
1143 * it can be even on different cpu. So just clear netif_running().
1145 * dev->stop() will invoke napi_disable() on all of it's
1146 * napi_struct instances on this device.
1148 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1150 dev_deactivate(dev);
1153 * Call the device specific close. This cannot fail.
1154 * Only if device is UP
1156 * We allow it to be called even after a DETACH hot-plug
1163 * Device is now down.
1166 dev->flags &= ~IFF_UP;
1169 * Tell people we are down
1171 call_netdevice_notifiers(NETDEV_DOWN, dev);
1176 net_dmaengine_put();
1183 * dev_disable_lro - disable Large Receive Offload on a device
1186 * Disable Large Receive Offload (LRO) on a net device. Must be
1187 * called under RTNL. This is needed if received packets may be
1188 * forwarded to another interface.
1190 void dev_disable_lro(struct net_device *dev)
1192 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1193 dev->ethtool_ops->set_flags) {
1194 u32 flags = dev->ethtool_ops->get_flags(dev);
1195 if (flags & ETH_FLAG_LRO) {
1196 flags &= ~ETH_FLAG_LRO;
1197 dev->ethtool_ops->set_flags(dev, flags);
1200 WARN_ON(dev->features & NETIF_F_LRO);
1202 EXPORT_SYMBOL(dev_disable_lro);
1205 static int dev_boot_phase = 1;
1208 * Device change register/unregister. These are not inline or static
1209 * as we export them to the world.
1213 * register_netdevice_notifier - register a network notifier block
1216 * Register a notifier to be called when network device events occur.
1217 * The notifier passed is linked into the kernel structures and must
1218 * not be reused until it has been unregistered. A negative errno code
1219 * is returned on a failure.
1221 * When registered all registration and up events are replayed
1222 * to the new notifier to allow device to have a race free
1223 * view of the network device list.
1226 int register_netdevice_notifier(struct notifier_block *nb)
1228 struct net_device *dev;
1229 struct net_device *last;
1234 err = raw_notifier_chain_register(&netdev_chain, nb);
1240 for_each_netdev(net, dev) {
1241 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1242 err = notifier_to_errno(err);
1246 if (!(dev->flags & IFF_UP))
1249 nb->notifier_call(nb, NETDEV_UP, dev);
1260 for_each_netdev(net, dev) {
1264 if (dev->flags & IFF_UP) {
1265 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1266 nb->notifier_call(nb, NETDEV_DOWN, dev);
1268 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1272 raw_notifier_chain_unregister(&netdev_chain, nb);
1277 * unregister_netdevice_notifier - unregister a network notifier block
1280 * Unregister a notifier previously registered by
1281 * register_netdevice_notifier(). The notifier is unlinked into the
1282 * kernel structures and may then be reused. A negative errno code
1283 * is returned on a failure.
1286 int unregister_netdevice_notifier(struct notifier_block *nb)
1291 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1297 * call_netdevice_notifiers - call all network notifier blocks
1298 * @val: value passed unmodified to notifier function
1299 * @dev: net_device pointer passed unmodified to notifier function
1301 * Call all network notifier blocks. Parameters and return value
1302 * are as for raw_notifier_call_chain().
1305 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1307 return raw_notifier_call_chain(&netdev_chain, val, dev);
1310 /* When > 0 there are consumers of rx skb time stamps */
1311 static atomic_t netstamp_needed = ATOMIC_INIT(0);
1313 void net_enable_timestamp(void)
1315 atomic_inc(&netstamp_needed);
1318 void net_disable_timestamp(void)
1320 atomic_dec(&netstamp_needed);
1323 static inline void net_timestamp(struct sk_buff *skb)
1325 if (atomic_read(&netstamp_needed))
1326 __net_timestamp(skb);
1328 skb->tstamp.tv64 = 0;
1332 * Support routine. Sends outgoing frames to any network
1333 * taps currently in use.
1336 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1338 struct packet_type *ptype;
1340 #ifdef CONFIG_NET_CLS_ACT
1341 if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
1348 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1349 /* Never send packets back to the socket
1350 * they originated from - MvS (miquels@drinkel.ow.org)
1352 if ((ptype->dev == dev || !ptype->dev) &&
1353 (ptype->af_packet_priv == NULL ||
1354 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1355 struct sk_buff *skb2= skb_clone(skb, GFP_ATOMIC);
1359 /* skb->nh should be correctly
1360 set by sender, so that the second statement is
1361 just protection against buggy protocols.
1363 skb_reset_mac_header(skb2);
1365 if (skb_network_header(skb2) < skb2->data ||
1366 skb2->network_header > skb2->tail) {
1367 if (net_ratelimit())
1368 printk(KERN_CRIT "protocol %04x is "
1370 skb2->protocol, dev->name);
1371 skb_reset_network_header(skb2);
1374 skb2->transport_header = skb2->network_header;
1375 skb2->pkt_type = PACKET_OUTGOING;
1376 ptype->func(skb2, skb->dev, ptype, skb->dev);
1383 static inline void __netif_reschedule(struct Qdisc *q)
1385 struct softnet_data *sd;
1386 unsigned long flags;
1388 local_irq_save(flags);
1389 sd = &__get_cpu_var(softnet_data);
1390 q->next_sched = sd->output_queue;
1391 sd->output_queue = q;
1392 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1393 local_irq_restore(flags);
1396 void __netif_schedule(struct Qdisc *q)
1398 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1399 __netif_reschedule(q);
1401 EXPORT_SYMBOL(__netif_schedule);
1403 void dev_kfree_skb_irq(struct sk_buff *skb)
1405 if (atomic_dec_and_test(&skb->users)) {
1406 struct softnet_data *sd;
1407 unsigned long flags;
1409 local_irq_save(flags);
1410 sd = &__get_cpu_var(softnet_data);
1411 skb->next = sd->completion_queue;
1412 sd->completion_queue = skb;
1413 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1414 local_irq_restore(flags);
1417 EXPORT_SYMBOL(dev_kfree_skb_irq);
1419 void dev_kfree_skb_any(struct sk_buff *skb)
1421 if (in_irq() || irqs_disabled())
1422 dev_kfree_skb_irq(skb);
1426 EXPORT_SYMBOL(dev_kfree_skb_any);
1430 * netif_device_detach - mark device as removed
1431 * @dev: network device
1433 * Mark device as removed from system and therefore no longer available.
1435 void netif_device_detach(struct net_device *dev)
1437 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1438 netif_running(dev)) {
1439 netif_tx_stop_all_queues(dev);
1442 EXPORT_SYMBOL(netif_device_detach);
1445 * netif_device_attach - mark device as attached
1446 * @dev: network device
1448 * Mark device as attached from system and restart if needed.
1450 void netif_device_attach(struct net_device *dev)
1452 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1453 netif_running(dev)) {
1454 netif_tx_wake_all_queues(dev);
1455 __netdev_watchdog_up(dev);
1458 EXPORT_SYMBOL(netif_device_attach);
1460 static bool can_checksum_protocol(unsigned long features, __be16 protocol)
1462 return ((features & NETIF_F_GEN_CSUM) ||
1463 ((features & NETIF_F_IP_CSUM) &&
1464 protocol == htons(ETH_P_IP)) ||
1465 ((features & NETIF_F_IPV6_CSUM) &&
1466 protocol == htons(ETH_P_IPV6)) ||
1467 ((features & NETIF_F_FCOE_CRC) &&
1468 protocol == htons(ETH_P_FCOE)));
1471 static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
1473 if (can_checksum_protocol(dev->features, skb->protocol))
1476 if (skb->protocol == htons(ETH_P_8021Q)) {
1477 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1478 if (can_checksum_protocol(dev->features & dev->vlan_features,
1479 veh->h_vlan_encapsulated_proto))
1487 * Invalidate hardware checksum when packet is to be mangled, and
1488 * complete checksum manually on outgoing path.
1490 int skb_checksum_help(struct sk_buff *skb)
1493 int ret = 0, offset;
1495 if (skb->ip_summed == CHECKSUM_COMPLETE)
1496 goto out_set_summed;
1498 if (unlikely(skb_shinfo(skb)->gso_size)) {
1499 /* Let GSO fix up the checksum. */
1500 goto out_set_summed;
1503 offset = skb->csum_start - skb_headroom(skb);
1504 BUG_ON(offset >= skb_headlen(skb));
1505 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1507 offset += skb->csum_offset;
1508 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1510 if (skb_cloned(skb) &&
1511 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
1512 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1517 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
1519 skb->ip_summed = CHECKSUM_NONE;
1525 * skb_gso_segment - Perform segmentation on skb.
1526 * @skb: buffer to segment
1527 * @features: features for the output path (see dev->features)
1529 * This function segments the given skb and returns a list of segments.
1531 * It may return NULL if the skb requires no segmentation. This is
1532 * only possible when GSO is used for verifying header integrity.
1534 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1536 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1537 struct packet_type *ptype;
1538 __be16 type = skb->protocol;
1541 skb_reset_mac_header(skb);
1542 skb->mac_len = skb->network_header - skb->mac_header;
1543 __skb_pull(skb, skb->mac_len);
1545 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1546 struct net_device *dev = skb->dev;
1547 struct ethtool_drvinfo info = {};
1549 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1550 dev->ethtool_ops->get_drvinfo(dev, &info);
1552 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
1554 info.driver, dev ? dev->features : 0L,
1555 skb->sk ? skb->sk->sk_route_caps : 0L,
1556 skb->len, skb->data_len, skb->ip_summed);
1558 if (skb_header_cloned(skb) &&
1559 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1560 return ERR_PTR(err);
1564 list_for_each_entry_rcu(ptype,
1565 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
1566 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
1567 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1568 err = ptype->gso_send_check(skb);
1569 segs = ERR_PTR(err);
1570 if (err || skb_gso_ok(skb, features))
1572 __skb_push(skb, (skb->data -
1573 skb_network_header(skb)));
1575 segs = ptype->gso_segment(skb, features);
1581 __skb_push(skb, skb->data - skb_mac_header(skb));
1586 EXPORT_SYMBOL(skb_gso_segment);
1588 /* Take action when hardware reception checksum errors are detected. */
1590 void netdev_rx_csum_fault(struct net_device *dev)
1592 if (net_ratelimit()) {
1593 printk(KERN_ERR "%s: hw csum failure.\n",
1594 dev ? dev->name : "<unknown>");
1598 EXPORT_SYMBOL(netdev_rx_csum_fault);
1601 /* Actually, we should eliminate this check as soon as we know, that:
1602 * 1. IOMMU is present and allows to map all the memory.
1603 * 2. No high memory really exists on this machine.
1606 static inline int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1608 #ifdef CONFIG_HIGHMEM
1611 if (dev->features & NETIF_F_HIGHDMA)
1614 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1615 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1623 void (*destructor)(struct sk_buff *skb);
1626 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1628 static void dev_gso_skb_destructor(struct sk_buff *skb)
1630 struct dev_gso_cb *cb;
1633 struct sk_buff *nskb = skb->next;
1635 skb->next = nskb->next;
1638 } while (skb->next);
1640 cb = DEV_GSO_CB(skb);
1642 cb->destructor(skb);
1646 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1647 * @skb: buffer to segment
1649 * This function segments the given skb and stores the list of segments
1652 static int dev_gso_segment(struct sk_buff *skb)
1654 struct net_device *dev = skb->dev;
1655 struct sk_buff *segs;
1656 int features = dev->features & ~(illegal_highdma(dev, skb) ?
1659 segs = skb_gso_segment(skb, features);
1661 /* Verifying header integrity only. */
1666 return PTR_ERR(segs);
1669 DEV_GSO_CB(skb)->destructor = skb->destructor;
1670 skb->destructor = dev_gso_skb_destructor;
1675 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
1676 struct netdev_queue *txq)
1678 const struct net_device_ops *ops = dev->netdev_ops;
1681 if (likely(!skb->next)) {
1682 if (!list_empty(&ptype_all))
1683 dev_queue_xmit_nit(skb, dev);
1685 if (netif_needs_gso(dev, skb)) {
1686 if (unlikely(dev_gso_segment(skb)))
1693 * If device doesnt need skb->dst, release it right now while
1694 * its hot in this cpu cache
1696 if ((dev->priv_flags & IFF_XMIT_DST_RELEASE) && skb->dst) {
1697 dst_release(skb->dst);
1700 rc = ops->ndo_start_xmit(skb, dev);
1702 txq_trans_update(txq);
1704 * TODO: if skb_orphan() was called by
1705 * dev->hard_start_xmit() (for example, the unmodified
1706 * igb driver does that; bnx2 doesn't), then
1707 * skb_tx_software_timestamp() will be unable to send
1708 * back the time stamp.
1710 * How can this be prevented? Always create another
1711 * reference to the socket before calling
1712 * dev->hard_start_xmit()? Prevent that skb_orphan()
1713 * does anything in dev->hard_start_xmit() by clearing
1714 * the skb destructor before the call and restoring it
1715 * afterwards, then doing the skb_orphan() ourselves?
1722 struct sk_buff *nskb = skb->next;
1724 skb->next = nskb->next;
1726 rc = ops->ndo_start_xmit(nskb, dev);
1728 nskb->next = skb->next;
1732 txq_trans_update(txq);
1733 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
1734 return NETDEV_TX_BUSY;
1735 } while (skb->next);
1737 skb->destructor = DEV_GSO_CB(skb)->destructor;
1744 static u32 skb_tx_hashrnd;
1746 u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
1750 if (skb_rx_queue_recorded(skb)) {
1751 hash = skb_get_rx_queue(skb);
1752 while (unlikely (hash >= dev->real_num_tx_queues))
1753 hash -= dev->real_num_tx_queues;
1757 if (skb->sk && skb->sk->sk_hash)
1758 hash = skb->sk->sk_hash;
1760 hash = skb->protocol;
1762 hash = jhash_1word(hash, skb_tx_hashrnd);
1764 return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
1766 EXPORT_SYMBOL(skb_tx_hash);
1768 static struct netdev_queue *dev_pick_tx(struct net_device *dev,
1769 struct sk_buff *skb)
1771 const struct net_device_ops *ops = dev->netdev_ops;
1772 u16 queue_index = 0;
1774 if (ops->ndo_select_queue)
1775 queue_index = ops->ndo_select_queue(dev, skb);
1776 else if (dev->real_num_tx_queues > 1)
1777 queue_index = skb_tx_hash(dev, skb);
1779 skb_set_queue_mapping(skb, queue_index);
1780 return netdev_get_tx_queue(dev, queue_index);
1784 * dev_queue_xmit - transmit a buffer
1785 * @skb: buffer to transmit
1787 * Queue a buffer for transmission to a network device. The caller must
1788 * have set the device and priority and built the buffer before calling
1789 * this function. The function can be called from an interrupt.
1791 * A negative errno code is returned on a failure. A success does not
1792 * guarantee the frame will be transmitted as it may be dropped due
1793 * to congestion or traffic shaping.
1795 * -----------------------------------------------------------------------------------
1796 * I notice this method can also return errors from the queue disciplines,
1797 * including NET_XMIT_DROP, which is a positive value. So, errors can also
1800 * Regardless of the return value, the skb is consumed, so it is currently
1801 * difficult to retry a send to this method. (You can bump the ref count
1802 * before sending to hold a reference for retry if you are careful.)
1804 * When calling this method, interrupts MUST be enabled. This is because
1805 * the BH enable code must have IRQs enabled so that it will not deadlock.
1808 int dev_queue_xmit(struct sk_buff *skb)
1810 struct net_device *dev = skb->dev;
1811 struct netdev_queue *txq;
1815 /* GSO will handle the following emulations directly. */
1816 if (netif_needs_gso(dev, skb))
1819 if (skb_shinfo(skb)->frag_list &&
1820 !(dev->features & NETIF_F_FRAGLIST) &&
1821 __skb_linearize(skb))
1824 /* Fragmented skb is linearized if device does not support SG,
1825 * or if at least one of fragments is in highmem and device
1826 * does not support DMA from it.
1828 if (skb_shinfo(skb)->nr_frags &&
1829 (!(dev->features & NETIF_F_SG) || illegal_highdma(dev, skb)) &&
1830 __skb_linearize(skb))
1833 /* If packet is not checksummed and device does not support
1834 * checksumming for this protocol, complete checksumming here.
1836 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1837 skb_set_transport_header(skb, skb->csum_start -
1839 if (!dev_can_checksum(dev, skb) && skb_checksum_help(skb))
1844 /* Disable soft irqs for various locks below. Also
1845 * stops preemption for RCU.
1849 txq = dev_pick_tx(dev, skb);
1850 q = rcu_dereference(txq->qdisc);
1852 #ifdef CONFIG_NET_CLS_ACT
1853 skb->tc_verd = SET_TC_AT(skb->tc_verd,AT_EGRESS);
1856 spinlock_t *root_lock = qdisc_lock(q);
1858 spin_lock(root_lock);
1860 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
1864 rc = qdisc_enqueue_root(skb, q);
1867 spin_unlock(root_lock);
1872 /* The device has no queue. Common case for software devices:
1873 loopback, all the sorts of tunnels...
1875 Really, it is unlikely that netif_tx_lock protection is necessary
1876 here. (f.e. loopback and IP tunnels are clean ignoring statistics
1878 However, it is possible, that they rely on protection
1881 Check this and shot the lock. It is not prone from deadlocks.
1882 Either shot noqueue qdisc, it is even simpler 8)
1884 if (dev->flags & IFF_UP) {
1885 int cpu = smp_processor_id(); /* ok because BHs are off */
1887 if (txq->xmit_lock_owner != cpu) {
1889 HARD_TX_LOCK(dev, txq, cpu);
1891 if (!netif_tx_queue_stopped(txq)) {
1893 if (!dev_hard_start_xmit(skb, dev, txq)) {
1894 HARD_TX_UNLOCK(dev, txq);
1898 HARD_TX_UNLOCK(dev, txq);
1899 if (net_ratelimit())
1900 printk(KERN_CRIT "Virtual device %s asks to "
1901 "queue packet!\n", dev->name);
1903 /* Recursion is detected! It is possible,
1905 if (net_ratelimit())
1906 printk(KERN_CRIT "Dead loop on virtual device "
1907 "%s, fix it urgently!\n", dev->name);
1912 rcu_read_unlock_bh();
1918 rcu_read_unlock_bh();
1923 /*=======================================================================
1925 =======================================================================*/
1927 int netdev_max_backlog __read_mostly = 1000;
1928 int netdev_budget __read_mostly = 300;
1929 int weight_p __read_mostly = 64; /* old backlog weight */
1931 DEFINE_PER_CPU(struct netif_rx_stats, netdev_rx_stat) = { 0, };
1935 * netif_rx - post buffer to the network code
1936 * @skb: buffer to post
1938 * This function receives a packet from a device driver and queues it for
1939 * the upper (protocol) levels to process. It always succeeds. The buffer
1940 * may be dropped during processing for congestion control or by the
1944 * NET_RX_SUCCESS (no congestion)
1945 * NET_RX_DROP (packet was dropped)
1949 int netif_rx(struct sk_buff *skb)
1951 struct softnet_data *queue;
1952 unsigned long flags;
1954 /* if netpoll wants it, pretend we never saw it */
1955 if (netpoll_rx(skb))
1958 if (!skb->tstamp.tv64)
1962 * The code is rearranged so that the path is the most
1963 * short when CPU is congested, but is still operating.
1965 local_irq_save(flags);
1966 queue = &__get_cpu_var(softnet_data);
1968 __get_cpu_var(netdev_rx_stat).total++;
1969 if (queue->input_pkt_queue.qlen <= netdev_max_backlog) {
1970 if (queue->input_pkt_queue.qlen) {
1972 __skb_queue_tail(&queue->input_pkt_queue, skb);
1973 local_irq_restore(flags);
1974 return NET_RX_SUCCESS;
1977 napi_schedule(&queue->backlog);
1981 __get_cpu_var(netdev_rx_stat).dropped++;
1982 local_irq_restore(flags);
1988 int netif_rx_ni(struct sk_buff *skb)
1993 err = netif_rx(skb);
1994 if (local_softirq_pending())
2001 EXPORT_SYMBOL(netif_rx_ni);
2003 static void net_tx_action(struct softirq_action *h)
2005 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2007 if (sd->completion_queue) {
2008 struct sk_buff *clist;
2010 local_irq_disable();
2011 clist = sd->completion_queue;
2012 sd->completion_queue = NULL;
2016 struct sk_buff *skb = clist;
2017 clist = clist->next;
2019 WARN_ON(atomic_read(&skb->users));
2024 if (sd->output_queue) {
2027 local_irq_disable();
2028 head = sd->output_queue;
2029 sd->output_queue = NULL;
2033 struct Qdisc *q = head;
2034 spinlock_t *root_lock;
2036 head = head->next_sched;
2038 root_lock = qdisc_lock(q);
2039 if (spin_trylock(root_lock)) {
2040 smp_mb__before_clear_bit();
2041 clear_bit(__QDISC_STATE_SCHED,
2044 spin_unlock(root_lock);
2046 if (!test_bit(__QDISC_STATE_DEACTIVATED,
2048 __netif_reschedule(q);
2050 smp_mb__before_clear_bit();
2051 clear_bit(__QDISC_STATE_SCHED,
2059 static inline int deliver_skb(struct sk_buff *skb,
2060 struct packet_type *pt_prev,
2061 struct net_device *orig_dev)
2063 atomic_inc(&skb->users);
2064 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2067 #if defined(CONFIG_BRIDGE) || defined (CONFIG_BRIDGE_MODULE)
2068 /* These hooks defined here for ATM */
2070 struct net_bridge_fdb_entry *(*br_fdb_get_hook)(struct net_bridge *br,
2071 unsigned char *addr);
2072 void (*br_fdb_put_hook)(struct net_bridge_fdb_entry *ent) __read_mostly;
2075 * If bridge module is loaded call bridging hook.
2076 * returns NULL if packet was consumed.
2078 struct sk_buff *(*br_handle_frame_hook)(struct net_bridge_port *p,
2079 struct sk_buff *skb) __read_mostly;
2080 static inline struct sk_buff *handle_bridge(struct sk_buff *skb,
2081 struct packet_type **pt_prev, int *ret,
2082 struct net_device *orig_dev)
2084 struct net_bridge_port *port;
2086 if (skb->pkt_type == PACKET_LOOPBACK ||
2087 (port = rcu_dereference(skb->dev->br_port)) == NULL)
2091 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2095 return br_handle_frame_hook(port, skb);
2098 #define handle_bridge(skb, pt_prev, ret, orig_dev) (skb)
2101 #if defined(CONFIG_MACVLAN) || defined(CONFIG_MACVLAN_MODULE)
2102 struct sk_buff *(*macvlan_handle_frame_hook)(struct sk_buff *skb) __read_mostly;
2103 EXPORT_SYMBOL_GPL(macvlan_handle_frame_hook);
2105 static inline struct sk_buff *handle_macvlan(struct sk_buff *skb,
2106 struct packet_type **pt_prev,
2108 struct net_device *orig_dev)
2110 if (skb->dev->macvlan_port == NULL)
2114 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2117 return macvlan_handle_frame_hook(skb);
2120 #define handle_macvlan(skb, pt_prev, ret, orig_dev) (skb)
2123 #ifdef CONFIG_NET_CLS_ACT
2124 /* TODO: Maybe we should just force sch_ingress to be compiled in
2125 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2126 * a compare and 2 stores extra right now if we dont have it on
2127 * but have CONFIG_NET_CLS_ACT
2128 * NOTE: This doesnt stop any functionality; if you dont have
2129 * the ingress scheduler, you just cant add policies on ingress.
2132 static int ing_filter(struct sk_buff *skb)
2134 struct net_device *dev = skb->dev;
2135 u32 ttl = G_TC_RTTL(skb->tc_verd);
2136 struct netdev_queue *rxq;
2137 int result = TC_ACT_OK;
2140 if (MAX_RED_LOOP < ttl++) {
2142 "Redir loop detected Dropping packet (%d->%d)\n",
2143 skb->iif, dev->ifindex);
2147 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2148 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2150 rxq = &dev->rx_queue;
2153 if (q != &noop_qdisc) {
2154 spin_lock(qdisc_lock(q));
2155 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2156 result = qdisc_enqueue_root(skb, q);
2157 spin_unlock(qdisc_lock(q));
2163 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2164 struct packet_type **pt_prev,
2165 int *ret, struct net_device *orig_dev)
2167 if (skb->dev->rx_queue.qdisc == &noop_qdisc)
2171 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2174 /* Huh? Why does turning on AF_PACKET affect this? */
2175 skb->tc_verd = SET_TC_OK2MUNGE(skb->tc_verd);
2178 switch (ing_filter(skb)) {
2192 * netif_nit_deliver - deliver received packets to network taps
2195 * This function is used to deliver incoming packets to network
2196 * taps. It should be used when the normal netif_receive_skb path
2197 * is bypassed, for example because of VLAN acceleration.
2199 void netif_nit_deliver(struct sk_buff *skb)
2201 struct packet_type *ptype;
2203 if (list_empty(&ptype_all))
2206 skb_reset_network_header(skb);
2207 skb_reset_transport_header(skb);
2208 skb->mac_len = skb->network_header - skb->mac_header;
2211 list_for_each_entry_rcu(ptype, &ptype_all, list) {
2212 if (!ptype->dev || ptype->dev == skb->dev)
2213 deliver_skb(skb, ptype, skb->dev);
2219 * netif_receive_skb - process receive buffer from network
2220 * @skb: buffer to process
2222 * netif_receive_skb() is the main receive data processing function.
2223 * It always succeeds. The buffer may be dropped during processing
2224 * for congestion control or by the protocol layers.
2226 * This function may only be called from softirq context and interrupts
2227 * should be enabled.
2229 * Return values (usually ignored):
2230 * NET_RX_SUCCESS: no congestion
2231 * NET_RX_DROP: packet was dropped
2233 int netif_receive_skb(struct sk_buff *skb)
2235 struct packet_type *ptype, *pt_prev;
2236 struct net_device *orig_dev;
2237 struct net_device *null_or_orig;
2238 int ret = NET_RX_DROP;
2241 if (skb->vlan_tci && vlan_hwaccel_do_receive(skb))
2242 return NET_RX_SUCCESS;
2244 /* if we've gotten here through NAPI, check netpoll */
2245 if (netpoll_receive_skb(skb))
2248 if (!skb->tstamp.tv64)
2252 skb->iif = skb->dev->ifindex;
2254 null_or_orig = NULL;
2255 orig_dev = skb->dev;
2256 if (orig_dev->master) {
2257 if (skb_bond_should_drop(skb))
2258 null_or_orig = orig_dev; /* deliver only exact match */
2260 skb->dev = orig_dev->master;
2263 __get_cpu_var(netdev_rx_stat).total++;
2265 skb_reset_network_header(skb);
2266 skb_reset_transport_header(skb);
2267 skb->mac_len = skb->network_header - skb->mac_header;
2273 #ifdef CONFIG_NET_CLS_ACT
2274 if (skb->tc_verd & TC_NCLS) {
2275 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
2280 list_for_each_entry_rcu(ptype, &ptype_all, list) {
2281 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2282 ptype->dev == orig_dev) {
2284 ret = deliver_skb(skb, pt_prev, orig_dev);
2289 #ifdef CONFIG_NET_CLS_ACT
2290 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
2296 skb = handle_bridge(skb, &pt_prev, &ret, orig_dev);
2299 skb = handle_macvlan(skb, &pt_prev, &ret, orig_dev);
2305 type = skb->protocol;
2306 list_for_each_entry_rcu(ptype,
2307 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
2308 if (ptype->type == type &&
2309 (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
2310 ptype->dev == orig_dev)) {
2312 ret = deliver_skb(skb, pt_prev, orig_dev);
2318 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2321 /* Jamal, now you will not able to escape explaining
2322 * me how you were going to use this. :-)
2332 /* Network device is going away, flush any packets still pending */
2333 static void flush_backlog(void *arg)
2335 struct net_device *dev = arg;
2336 struct softnet_data *queue = &__get_cpu_var(softnet_data);
2337 struct sk_buff *skb, *tmp;
2339 skb_queue_walk_safe(&queue->input_pkt_queue, skb, tmp)
2340 if (skb->dev == dev) {
2341 __skb_unlink(skb, &queue->input_pkt_queue);
2346 static int napi_gro_complete(struct sk_buff *skb)
2348 struct packet_type *ptype;
2349 __be16 type = skb->protocol;
2350 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
2353 if (NAPI_GRO_CB(skb)->count == 1) {
2354 skb_shinfo(skb)->gso_size = 0;
2359 list_for_each_entry_rcu(ptype, head, list) {
2360 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
2363 err = ptype->gro_complete(skb);
2369 WARN_ON(&ptype->list == head);
2371 return NET_RX_SUCCESS;
2375 return netif_receive_skb(skb);
2378 void napi_gro_flush(struct napi_struct *napi)
2380 struct sk_buff *skb, *next;
2382 for (skb = napi->gro_list; skb; skb = next) {
2385 napi_gro_complete(skb);
2388 napi->gro_count = 0;
2389 napi->gro_list = NULL;
2391 EXPORT_SYMBOL(napi_gro_flush);
2393 void *skb_gro_header(struct sk_buff *skb, unsigned int hlen)
2395 unsigned int offset = skb_gro_offset(skb);
2398 if (unlikely(skb_headlen(skb) ||
2399 skb_shinfo(skb)->frags[0].size < hlen ||
2400 PageHighMem(skb_shinfo(skb)->frags[0].page)))
2401 return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
2403 return page_address(skb_shinfo(skb)->frags[0].page) +
2404 skb_shinfo(skb)->frags[0].page_offset + offset;
2406 EXPORT_SYMBOL(skb_gro_header);
2408 int dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2410 struct sk_buff **pp = NULL;
2411 struct packet_type *ptype;
2412 __be16 type = skb->protocol;
2413 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
2418 if (!(skb->dev->features & NETIF_F_GRO))
2421 if (skb_is_gso(skb) || skb_shinfo(skb)->frag_list)
2425 list_for_each_entry_rcu(ptype, head, list) {
2426 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
2429 skb_set_network_header(skb, skb_gro_offset(skb));
2430 mac_len = skb->network_header - skb->mac_header;
2431 skb->mac_len = mac_len;
2432 NAPI_GRO_CB(skb)->same_flow = 0;
2433 NAPI_GRO_CB(skb)->flush = 0;
2434 NAPI_GRO_CB(skb)->free = 0;
2436 pp = ptype->gro_receive(&napi->gro_list, skb);
2441 if (&ptype->list == head)
2444 same_flow = NAPI_GRO_CB(skb)->same_flow;
2445 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
2448 struct sk_buff *nskb = *pp;
2452 napi_gro_complete(nskb);
2459 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
2463 NAPI_GRO_CB(skb)->count = 1;
2464 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
2465 skb->next = napi->gro_list;
2466 napi->gro_list = skb;
2470 if (unlikely(!pskb_may_pull(skb, skb_gro_offset(skb)))) {
2471 if (napi->gro_list == skb)
2472 napi->gro_list = skb->next;
2483 EXPORT_SYMBOL(dev_gro_receive);
2485 static int __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2489 if (netpoll_rx_on(skb))
2492 for (p = napi->gro_list; p; p = p->next) {
2493 NAPI_GRO_CB(p)->same_flow = (p->dev == skb->dev)
2494 && !compare_ether_header(skb_mac_header(p),
2495 skb_gro_mac_header(skb));
2496 NAPI_GRO_CB(p)->flush = 0;
2499 return dev_gro_receive(napi, skb);
2502 int napi_skb_finish(int ret, struct sk_buff *skb)
2504 int err = NET_RX_SUCCESS;
2508 return netif_receive_skb(skb);
2514 case GRO_MERGED_FREE:
2521 EXPORT_SYMBOL(napi_skb_finish);
2523 int napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
2525 skb_gro_reset_offset(skb);
2527 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
2529 EXPORT_SYMBOL(napi_gro_receive);
2531 void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
2533 __skb_pull(skb, skb_headlen(skb));
2534 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
2538 EXPORT_SYMBOL(napi_reuse_skb);
2540 struct sk_buff *napi_get_frags(struct napi_struct *napi)
2542 struct net_device *dev = napi->dev;
2543 struct sk_buff *skb = napi->skb;
2546 skb = netdev_alloc_skb(dev, GRO_MAX_HEAD + NET_IP_ALIGN);
2550 skb_reserve(skb, NET_IP_ALIGN);
2558 EXPORT_SYMBOL(napi_get_frags);
2560 int napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, int ret)
2562 int err = NET_RX_SUCCESS;
2567 skb->protocol = eth_type_trans(skb, napi->dev);
2569 if (ret == GRO_NORMAL)
2570 return netif_receive_skb(skb);
2572 skb_gro_pull(skb, -ETH_HLEN);
2579 case GRO_MERGED_FREE:
2580 napi_reuse_skb(napi, skb);
2586 EXPORT_SYMBOL(napi_frags_finish);
2588 struct sk_buff *napi_frags_skb(struct napi_struct *napi)
2590 struct sk_buff *skb = napi->skb;
2595 skb_reset_mac_header(skb);
2596 skb_gro_reset_offset(skb);
2598 eth = skb_gro_header(skb, sizeof(*eth));
2600 napi_reuse_skb(napi, skb);
2605 skb_gro_pull(skb, sizeof(*eth));
2608 * This works because the only protocols we care about don't require
2609 * special handling. We'll fix it up properly at the end.
2611 skb->protocol = eth->h_proto;
2616 EXPORT_SYMBOL(napi_frags_skb);
2618 int napi_gro_frags(struct napi_struct *napi)
2620 struct sk_buff *skb = napi_frags_skb(napi);
2625 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
2627 EXPORT_SYMBOL(napi_gro_frags);
2629 static int process_backlog(struct napi_struct *napi, int quota)
2632 struct softnet_data *queue = &__get_cpu_var(softnet_data);
2633 unsigned long start_time = jiffies;
2635 napi->weight = weight_p;
2637 struct sk_buff *skb;
2639 local_irq_disable();
2640 skb = __skb_dequeue(&queue->input_pkt_queue);
2642 __napi_complete(napi);
2648 netif_receive_skb(skb);
2649 } while (++work < quota && jiffies == start_time);
2655 * __napi_schedule - schedule for receive
2656 * @n: entry to schedule
2658 * The entry's receive function will be scheduled to run
2660 void __napi_schedule(struct napi_struct *n)
2662 unsigned long flags;
2664 local_irq_save(flags);
2665 list_add_tail(&n->poll_list, &__get_cpu_var(softnet_data).poll_list);
2666 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2667 local_irq_restore(flags);
2669 EXPORT_SYMBOL(__napi_schedule);
2671 void __napi_complete(struct napi_struct *n)
2673 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
2674 BUG_ON(n->gro_list);
2676 list_del(&n->poll_list);
2677 smp_mb__before_clear_bit();
2678 clear_bit(NAPI_STATE_SCHED, &n->state);
2680 EXPORT_SYMBOL(__napi_complete);
2682 void napi_complete(struct napi_struct *n)
2684 unsigned long flags;
2687 * don't let napi dequeue from the cpu poll list
2688 * just in case its running on a different cpu
2690 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
2694 local_irq_save(flags);
2696 local_irq_restore(flags);
2698 EXPORT_SYMBOL(napi_complete);
2700 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2701 int (*poll)(struct napi_struct *, int), int weight)
2703 INIT_LIST_HEAD(&napi->poll_list);
2704 napi->gro_count = 0;
2705 napi->gro_list = NULL;
2708 napi->weight = weight;
2709 list_add(&napi->dev_list, &dev->napi_list);
2711 #ifdef CONFIG_NETPOLL
2712 spin_lock_init(&napi->poll_lock);
2713 napi->poll_owner = -1;
2715 set_bit(NAPI_STATE_SCHED, &napi->state);
2717 EXPORT_SYMBOL(netif_napi_add);
2719 void netif_napi_del(struct napi_struct *napi)
2721 struct sk_buff *skb, *next;
2723 list_del_init(&napi->dev_list);
2724 napi_free_frags(napi);
2726 for (skb = napi->gro_list; skb; skb = next) {
2732 napi->gro_list = NULL;
2733 napi->gro_count = 0;
2735 EXPORT_SYMBOL(netif_napi_del);
2738 static void net_rx_action(struct softirq_action *h)
2740 struct list_head *list = &__get_cpu_var(softnet_data).poll_list;
2741 unsigned long time_limit = jiffies + 2;
2742 int budget = netdev_budget;
2745 local_irq_disable();
2747 while (!list_empty(list)) {
2748 struct napi_struct *n;
2751 /* If softirq window is exhuasted then punt.
2752 * Allow this to run for 2 jiffies since which will allow
2753 * an average latency of 1.5/HZ.
2755 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
2760 /* Even though interrupts have been re-enabled, this
2761 * access is safe because interrupts can only add new
2762 * entries to the tail of this list, and only ->poll()
2763 * calls can remove this head entry from the list.
2765 n = list_entry(list->next, struct napi_struct, poll_list);
2767 have = netpoll_poll_lock(n);
2771 /* This NAPI_STATE_SCHED test is for avoiding a race
2772 * with netpoll's poll_napi(). Only the entity which
2773 * obtains the lock and sees NAPI_STATE_SCHED set will
2774 * actually make the ->poll() call. Therefore we avoid
2775 * accidently calling ->poll() when NAPI is not scheduled.
2778 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
2779 work = n->poll(n, weight);
2783 WARN_ON_ONCE(work > weight);
2787 local_irq_disable();
2789 /* Drivers must not modify the NAPI state if they
2790 * consume the entire weight. In such cases this code
2791 * still "owns" the NAPI instance and therefore can
2792 * move the instance around on the list at-will.
2794 if (unlikely(work == weight)) {
2795 if (unlikely(napi_disable_pending(n)))
2798 list_move_tail(&n->poll_list, list);
2801 netpoll_poll_unlock(have);
2806 #ifdef CONFIG_NET_DMA
2808 * There may not be any more sk_buffs coming right now, so push
2809 * any pending DMA copies to hardware
2811 dma_issue_pending_all();
2817 __get_cpu_var(netdev_rx_stat).time_squeeze++;
2818 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2822 static gifconf_func_t * gifconf_list [NPROTO];
2825 * register_gifconf - register a SIOCGIF handler
2826 * @family: Address family
2827 * @gifconf: Function handler
2829 * Register protocol dependent address dumping routines. The handler
2830 * that is passed must not be freed or reused until it has been replaced
2831 * by another handler.
2833 int register_gifconf(unsigned int family, gifconf_func_t * gifconf)
2835 if (family >= NPROTO)
2837 gifconf_list[family] = gifconf;
2843 * Map an interface index to its name (SIOCGIFNAME)
2847 * We need this ioctl for efficient implementation of the
2848 * if_indextoname() function required by the IPv6 API. Without
2849 * it, we would have to search all the interfaces to find a
2853 static int dev_ifname(struct net *net, struct ifreq __user *arg)
2855 struct net_device *dev;
2859 * Fetch the caller's info block.
2862 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
2865 read_lock(&dev_base_lock);
2866 dev = __dev_get_by_index(net, ifr.ifr_ifindex);
2868 read_unlock(&dev_base_lock);
2872 strcpy(ifr.ifr_name, dev->name);
2873 read_unlock(&dev_base_lock);
2875 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
2881 * Perform a SIOCGIFCONF call. This structure will change
2882 * size eventually, and there is nothing I can do about it.
2883 * Thus we will need a 'compatibility mode'.
2886 static int dev_ifconf(struct net *net, char __user *arg)
2889 struct net_device *dev;
2896 * Fetch the caller's info block.
2899 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
2906 * Loop over the interfaces, and write an info block for each.
2910 for_each_netdev(net, dev) {
2911 for (i = 0; i < NPROTO; i++) {
2912 if (gifconf_list[i]) {
2915 done = gifconf_list[i](dev, NULL, 0);
2917 done = gifconf_list[i](dev, pos + total,
2927 * All done. Write the updated control block back to the caller.
2929 ifc.ifc_len = total;
2932 * Both BSD and Solaris return 0 here, so we do too.
2934 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
2937 #ifdef CONFIG_PROC_FS
2939 * This is invoked by the /proc filesystem handler to display a device
2942 void *dev_seq_start(struct seq_file *seq, loff_t *pos)
2943 __acquires(dev_base_lock)
2945 struct net *net = seq_file_net(seq);
2947 struct net_device *dev;
2949 read_lock(&dev_base_lock);
2951 return SEQ_START_TOKEN;
2954 for_each_netdev(net, dev)
2961 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2963 struct net *net = seq_file_net(seq);
2965 return v == SEQ_START_TOKEN ?
2966 first_net_device(net) : next_net_device((struct net_device *)v);
2969 void dev_seq_stop(struct seq_file *seq, void *v)
2970 __releases(dev_base_lock)
2972 read_unlock(&dev_base_lock);
2975 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
2977 const struct net_device_stats *stats = dev_get_stats(dev);
2979 seq_printf(seq, "%6s:%8lu %7lu %4lu %4lu %4lu %5lu %10lu %9lu "
2980 "%8lu %7lu %4lu %4lu %4lu %5lu %7lu %10lu\n",
2981 dev->name, stats->rx_bytes, stats->rx_packets,
2983 stats->rx_dropped + stats->rx_missed_errors,
2984 stats->rx_fifo_errors,
2985 stats->rx_length_errors + stats->rx_over_errors +
2986 stats->rx_crc_errors + stats->rx_frame_errors,
2987 stats->rx_compressed, stats->multicast,
2988 stats->tx_bytes, stats->tx_packets,
2989 stats->tx_errors, stats->tx_dropped,
2990 stats->tx_fifo_errors, stats->collisions,
2991 stats->tx_carrier_errors +
2992 stats->tx_aborted_errors +
2993 stats->tx_window_errors +
2994 stats->tx_heartbeat_errors,
2995 stats->tx_compressed);
2999 * Called from the PROCfs module. This now uses the new arbitrary sized
3000 * /proc/net interface to create /proc/net/dev
3002 static int dev_seq_show(struct seq_file *seq, void *v)
3004 if (v == SEQ_START_TOKEN)
3005 seq_puts(seq, "Inter-| Receive "
3007 " face |bytes packets errs drop fifo frame "
3008 "compressed multicast|bytes packets errs "
3009 "drop fifo colls carrier compressed\n");
3011 dev_seq_printf_stats(seq, v);
3015 static struct netif_rx_stats *softnet_get_online(loff_t *pos)
3017 struct netif_rx_stats *rc = NULL;
3019 while (*pos < nr_cpu_ids)
3020 if (cpu_online(*pos)) {
3021 rc = &per_cpu(netdev_rx_stat, *pos);
3028 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
3030 return softnet_get_online(pos);
3033 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3036 return softnet_get_online(pos);
3039 static void softnet_seq_stop(struct seq_file *seq, void *v)
3043 static int softnet_seq_show(struct seq_file *seq, void *v)
3045 struct netif_rx_stats *s = v;
3047 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
3048 s->total, s->dropped, s->time_squeeze, 0,
3049 0, 0, 0, 0, /* was fastroute */
3054 static const struct seq_operations dev_seq_ops = {
3055 .start = dev_seq_start,
3056 .next = dev_seq_next,
3057 .stop = dev_seq_stop,
3058 .show = dev_seq_show,
3061 static int dev_seq_open(struct inode *inode, struct file *file)
3063 return seq_open_net(inode, file, &dev_seq_ops,
3064 sizeof(struct seq_net_private));
3067 static const struct file_operations dev_seq_fops = {
3068 .owner = THIS_MODULE,
3069 .open = dev_seq_open,
3071 .llseek = seq_lseek,
3072 .release = seq_release_net,
3075 static const struct seq_operations softnet_seq_ops = {
3076 .start = softnet_seq_start,
3077 .next = softnet_seq_next,
3078 .stop = softnet_seq_stop,
3079 .show = softnet_seq_show,
3082 static int softnet_seq_open(struct inode *inode, struct file *file)
3084 return seq_open(file, &softnet_seq_ops);
3087 static const struct file_operations softnet_seq_fops = {
3088 .owner = THIS_MODULE,
3089 .open = softnet_seq_open,
3091 .llseek = seq_lseek,
3092 .release = seq_release,
3095 static void *ptype_get_idx(loff_t pos)
3097 struct packet_type *pt = NULL;
3101 list_for_each_entry_rcu(pt, &ptype_all, list) {
3107 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
3108 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
3117 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
3121 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
3124 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3126 struct packet_type *pt;
3127 struct list_head *nxt;
3131 if (v == SEQ_START_TOKEN)
3132 return ptype_get_idx(0);
3135 nxt = pt->list.next;
3136 if (pt->type == htons(ETH_P_ALL)) {
3137 if (nxt != &ptype_all)
3140 nxt = ptype_base[0].next;
3142 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
3144 while (nxt == &ptype_base[hash]) {
3145 if (++hash >= PTYPE_HASH_SIZE)
3147 nxt = ptype_base[hash].next;
3150 return list_entry(nxt, struct packet_type, list);
3153 static void ptype_seq_stop(struct seq_file *seq, void *v)
3159 static int ptype_seq_show(struct seq_file *seq, void *v)
3161 struct packet_type *pt = v;
3163 if (v == SEQ_START_TOKEN)
3164 seq_puts(seq, "Type Device Function\n");
3165 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
3166 if (pt->type == htons(ETH_P_ALL))
3167 seq_puts(seq, "ALL ");
3169 seq_printf(seq, "%04x", ntohs(pt->type));
3171 seq_printf(seq, " %-8s %pF\n",
3172 pt->dev ? pt->dev->name : "", pt->func);
3178 static const struct seq_operations ptype_seq_ops = {
3179 .start = ptype_seq_start,
3180 .next = ptype_seq_next,
3181 .stop = ptype_seq_stop,
3182 .show = ptype_seq_show,
3185 static int ptype_seq_open(struct inode *inode, struct file *file)
3187 return seq_open_net(inode, file, &ptype_seq_ops,
3188 sizeof(struct seq_net_private));
3191 static const struct file_operations ptype_seq_fops = {
3192 .owner = THIS_MODULE,
3193 .open = ptype_seq_open,
3195 .llseek = seq_lseek,
3196 .release = seq_release_net,
3200 static int __net_init dev_proc_net_init(struct net *net)
3204 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
3206 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
3208 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
3211 if (wext_proc_init(net))
3217 proc_net_remove(net, "ptype");
3219 proc_net_remove(net, "softnet_stat");
3221 proc_net_remove(net, "dev");
3225 static void __net_exit dev_proc_net_exit(struct net *net)
3227 wext_proc_exit(net);
3229 proc_net_remove(net, "ptype");
3230 proc_net_remove(net, "softnet_stat");
3231 proc_net_remove(net, "dev");
3234 static struct pernet_operations __net_initdata dev_proc_ops = {
3235 .init = dev_proc_net_init,
3236 .exit = dev_proc_net_exit,
3239 static int __init dev_proc_init(void)
3241 return register_pernet_subsys(&dev_proc_ops);
3244 #define dev_proc_init() 0
3245 #endif /* CONFIG_PROC_FS */
3249 * netdev_set_master - set up master/slave pair
3250 * @slave: slave device
3251 * @master: new master device
3253 * Changes the master device of the slave. Pass %NULL to break the
3254 * bonding. The caller must hold the RTNL semaphore. On a failure
3255 * a negative errno code is returned. On success the reference counts
3256 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
3257 * function returns zero.
3259 int netdev_set_master(struct net_device *slave, struct net_device *master)
3261 struct net_device *old = slave->master;
3271 slave->master = master;
3279 slave->flags |= IFF_SLAVE;
3281 slave->flags &= ~IFF_SLAVE;
3283 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
3287 static void dev_change_rx_flags(struct net_device *dev, int flags)
3289 const struct net_device_ops *ops = dev->netdev_ops;
3291 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
3292 ops->ndo_change_rx_flags(dev, flags);
3295 static int __dev_set_promiscuity(struct net_device *dev, int inc)
3297 unsigned short old_flags = dev->flags;
3303 dev->flags |= IFF_PROMISC;
3304 dev->promiscuity += inc;
3305 if (dev->promiscuity == 0) {
3308 * If inc causes overflow, untouch promisc and return error.
3311 dev->flags &= ~IFF_PROMISC;
3313 dev->promiscuity -= inc;
3314 printk(KERN_WARNING "%s: promiscuity touches roof, "
3315 "set promiscuity failed, promiscuity feature "
3316 "of device might be broken.\n", dev->name);
3320 if (dev->flags != old_flags) {
3321 printk(KERN_INFO "device %s %s promiscuous mode\n",
3322 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
3324 if (audit_enabled) {
3325 current_uid_gid(&uid, &gid);
3326 audit_log(current->audit_context, GFP_ATOMIC,
3327 AUDIT_ANOM_PROMISCUOUS,
3328 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
3329 dev->name, (dev->flags & IFF_PROMISC),
3330 (old_flags & IFF_PROMISC),
3331 audit_get_loginuid(current),
3333 audit_get_sessionid(current));
3336 dev_change_rx_flags(dev, IFF_PROMISC);
3342 * dev_set_promiscuity - update promiscuity count on a device
3346 * Add or remove promiscuity from a device. While the count in the device
3347 * remains above zero the interface remains promiscuous. Once it hits zero
3348 * the device reverts back to normal filtering operation. A negative inc
3349 * value is used to drop promiscuity on the device.
3350 * Return 0 if successful or a negative errno code on error.
3352 int dev_set_promiscuity(struct net_device *dev, int inc)
3354 unsigned short old_flags = dev->flags;
3357 err = __dev_set_promiscuity(dev, inc);
3360 if (dev->flags != old_flags)
3361 dev_set_rx_mode(dev);
3366 * dev_set_allmulti - update allmulti count on a device
3370 * Add or remove reception of all multicast frames to a device. While the
3371 * count in the device remains above zero the interface remains listening
3372 * to all interfaces. Once it hits zero the device reverts back to normal
3373 * filtering operation. A negative @inc value is used to drop the counter
3374 * when releasing a resource needing all multicasts.
3375 * Return 0 if successful or a negative errno code on error.
3378 int dev_set_allmulti(struct net_device *dev, int inc)
3380 unsigned short old_flags = dev->flags;
3384 dev->flags |= IFF_ALLMULTI;
3385 dev->allmulti += inc;
3386 if (dev->allmulti == 0) {
3389 * If inc causes overflow, untouch allmulti and return error.
3392 dev->flags &= ~IFF_ALLMULTI;
3394 dev->allmulti -= inc;
3395 printk(KERN_WARNING "%s: allmulti touches roof, "
3396 "set allmulti failed, allmulti feature of "
3397 "device might be broken.\n", dev->name);
3401 if (dev->flags ^ old_flags) {
3402 dev_change_rx_flags(dev, IFF_ALLMULTI);
3403 dev_set_rx_mode(dev);
3409 * Upload unicast and multicast address lists to device and
3410 * configure RX filtering. When the device doesn't support unicast
3411 * filtering it is put in promiscuous mode while unicast addresses
3414 void __dev_set_rx_mode(struct net_device *dev)
3416 const struct net_device_ops *ops = dev->netdev_ops;
3418 /* dev_open will call this function so the list will stay sane. */
3419 if (!(dev->flags&IFF_UP))
3422 if (!netif_device_present(dev))
3425 if (ops->ndo_set_rx_mode)
3426 ops->ndo_set_rx_mode(dev);
3428 /* Unicast addresses changes may only happen under the rtnl,
3429 * therefore calling __dev_set_promiscuity here is safe.
3431 if (dev->uc_count > 0 && !dev->uc_promisc) {
3432 __dev_set_promiscuity(dev, 1);
3433 dev->uc_promisc = 1;
3434 } else if (dev->uc_count == 0 && dev->uc_promisc) {
3435 __dev_set_promiscuity(dev, -1);
3436 dev->uc_promisc = 0;
3439 if (ops->ndo_set_multicast_list)
3440 ops->ndo_set_multicast_list(dev);
3444 void dev_set_rx_mode(struct net_device *dev)
3446 netif_addr_lock_bh(dev);
3447 __dev_set_rx_mode(dev);
3448 netif_addr_unlock_bh(dev);
3451 /* hw addresses list handling functions */
3453 static int __hw_addr_add(struct list_head *list, unsigned char *addr,
3454 int addr_len, unsigned char addr_type)
3456 struct netdev_hw_addr *ha;
3459 if (addr_len > MAX_ADDR_LEN)
3462 alloc_size = sizeof(*ha);
3463 if (alloc_size < L1_CACHE_BYTES)
3464 alloc_size = L1_CACHE_BYTES;
3465 ha = kmalloc(alloc_size, GFP_ATOMIC);
3468 memcpy(ha->addr, addr, addr_len);
3469 ha->type = addr_type;
3470 list_add_tail_rcu(&ha->list, list);
3474 static void ha_rcu_free(struct rcu_head *head)
3476 struct netdev_hw_addr *ha;
3478 ha = container_of(head, struct netdev_hw_addr, rcu_head);
3482 static int __hw_addr_del_ii(struct list_head *list, unsigned char *addr,
3483 int addr_len, unsigned char addr_type,
3486 struct netdev_hw_addr *ha;
3489 list_for_each_entry(ha, list, list) {
3490 if (i++ != ignore_index &&
3491 !memcmp(ha->addr, addr, addr_len) &&
3492 (ha->type == addr_type || !addr_type)) {
3493 list_del_rcu(&ha->list);
3494 call_rcu(&ha->rcu_head, ha_rcu_free);
3501 static int __hw_addr_add_multiple_ii(struct list_head *to_list,
3502 struct list_head *from_list,
3503 int addr_len, unsigned char addr_type,
3507 struct netdev_hw_addr *ha, *ha2;
3510 list_for_each_entry(ha, from_list, list) {
3511 type = addr_type ? addr_type : ha->type;
3512 err = __hw_addr_add(to_list, ha->addr, addr_len, type);
3519 list_for_each_entry(ha2, from_list, list) {
3522 type = addr_type ? addr_type : ha2->type;
3523 __hw_addr_del_ii(to_list, ha2->addr, addr_len, type,
3529 static void __hw_addr_del_multiple_ii(struct list_head *to_list,
3530 struct list_head *from_list,
3531 int addr_len, unsigned char addr_type,
3534 struct netdev_hw_addr *ha;
3537 list_for_each_entry(ha, from_list, list) {
3538 type = addr_type ? addr_type : ha->type;
3539 __hw_addr_del_ii(to_list, ha->addr, addr_len, addr_type,
3544 static void __hw_addr_flush(struct list_head *list)
3546 struct netdev_hw_addr *ha, *tmp;
3548 list_for_each_entry_safe(ha, tmp, list, list) {
3549 list_del_rcu(&ha->list);
3550 call_rcu(&ha->rcu_head, ha_rcu_free);
3554 /* Device addresses handling functions */
3556 static void dev_addr_flush(struct net_device *dev)
3558 /* rtnl_mutex must be held here */
3560 __hw_addr_flush(&dev->dev_addr_list);
3561 dev->dev_addr = NULL;
3564 static int dev_addr_init(struct net_device *dev)
3566 unsigned char addr[MAX_ADDR_LEN];
3567 struct netdev_hw_addr *ha;
3570 /* rtnl_mutex must be held here */
3572 INIT_LIST_HEAD(&dev->dev_addr_list);
3573 memset(addr, 0, sizeof(*addr));
3574 err = __hw_addr_add(&dev->dev_addr_list, addr, sizeof(*addr),
3575 NETDEV_HW_ADDR_T_LAN);
3578 * Get the first (previously created) address from the list
3579 * and set dev_addr pointer to this location.
3581 ha = list_first_entry(&dev->dev_addr_list,
3582 struct netdev_hw_addr, list);
3583 dev->dev_addr = ha->addr;
3589 * dev_addr_add - Add a device address
3591 * @addr: address to add
3592 * @addr_type: address type
3594 * Add a device address to the device or increase the reference count if
3595 * it already exists.
3597 * The caller must hold the rtnl_mutex.
3599 int dev_addr_add(struct net_device *dev, unsigned char *addr,
3600 unsigned char addr_type)
3606 err = __hw_addr_add(&dev->dev_addr_list, addr, dev->addr_len,
3609 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
3612 EXPORT_SYMBOL(dev_addr_add);
3615 * dev_addr_del - Release a device address.
3617 * @addr: address to delete
3618 * @addr_type: address type
3620 * Release reference to a device address and remove it from the device
3621 * if the reference count drops to zero.
3623 * The caller must hold the rtnl_mutex.
3625 int dev_addr_del(struct net_device *dev, unsigned char *addr,
3626 unsigned char addr_type)
3632 err = __hw_addr_del_ii(&dev->dev_addr_list, addr, dev->addr_len,
3635 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
3638 EXPORT_SYMBOL(dev_addr_del);
3641 * dev_addr_add_multiple - Add device addresses from another device
3642 * @to_dev: device to which addresses will be added
3643 * @from_dev: device from which addresses will be added
3644 * @addr_type: address type - 0 means type will be used from from_dev
3646 * Add device addresses of the one device to another.
3648 * The caller must hold the rtnl_mutex.
3650 int dev_addr_add_multiple(struct net_device *to_dev,
3651 struct net_device *from_dev,
3652 unsigned char addr_type)
3658 if (from_dev->addr_len != to_dev->addr_len)
3660 err = __hw_addr_add_multiple_ii(&to_dev->dev_addr_list,
3661 &from_dev->dev_addr_list,
3662 to_dev->addr_len, addr_type, 0);
3664 call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev);
3667 EXPORT_SYMBOL(dev_addr_add_multiple);
3670 * dev_addr_del_multiple - Delete device addresses by another device
3671 * @to_dev: device where the addresses will be deleted
3672 * @from_dev: device by which addresses the addresses will be deleted
3673 * @addr_type: address type - 0 means type will used from from_dev
3675 * Deletes addresses in to device by the list of addresses in from device.
3677 * The caller must hold the rtnl_mutex.
3679 int dev_addr_del_multiple(struct net_device *to_dev,
3680 struct net_device *from_dev,
3681 unsigned char addr_type)
3685 if (from_dev->addr_len != to_dev->addr_len)
3687 __hw_addr_del_multiple_ii(&to_dev->dev_addr_list,
3688 &from_dev->dev_addr_list,
3689 to_dev->addr_len, addr_type, 0);
3690 call_netdevice_notifiers(NETDEV_CHANGEADDR, to_dev);
3693 EXPORT_SYMBOL(dev_addr_del_multiple);
3695 /* unicast and multicast addresses handling functions */
3697 int __dev_addr_delete(struct dev_addr_list **list, int *count,
3698 void *addr, int alen, int glbl)
3700 struct dev_addr_list *da;
3702 for (; (da = *list) != NULL; list = &da->next) {
3703 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 &&
3704 alen == da->da_addrlen) {
3706 int old_glbl = da->da_gusers;
3723 int __dev_addr_add(struct dev_addr_list **list, int *count,
3724 void *addr, int alen, int glbl)
3726 struct dev_addr_list *da;
3728 for (da = *list; da != NULL; da = da->next) {
3729 if (memcmp(da->da_addr, addr, da->da_addrlen) == 0 &&
3730 da->da_addrlen == alen) {
3732 int old_glbl = da->da_gusers;
3742 da = kzalloc(sizeof(*da), GFP_ATOMIC);
3745 memcpy(da->da_addr, addr, alen);
3746 da->da_addrlen = alen;
3748 da->da_gusers = glbl ? 1 : 0;
3756 * dev_unicast_delete - Release secondary unicast address.
3758 * @addr: address to delete
3759 * @alen: length of @addr
3761 * Release reference to a secondary unicast address and remove it
3762 * from the device if the reference count drops to zero.
3764 * The caller must hold the rtnl_mutex.
3766 int dev_unicast_delete(struct net_device *dev, void *addr, int alen)
3772 netif_addr_lock_bh(dev);
3773 err = __dev_addr_delete(&dev->uc_list, &dev->uc_count, addr, alen, 0);
3775 __dev_set_rx_mode(dev);
3776 netif_addr_unlock_bh(dev);
3779 EXPORT_SYMBOL(dev_unicast_delete);
3782 * dev_unicast_add - add a secondary unicast address
3784 * @addr: address to add
3785 * @alen: length of @addr
3787 * Add a secondary unicast address to the device or increase
3788 * the reference count if it already exists.
3790 * The caller must hold the rtnl_mutex.
3792 int dev_unicast_add(struct net_device *dev, void *addr, int alen)
3798 netif_addr_lock_bh(dev);
3799 err = __dev_addr_add(&dev->uc_list, &dev->uc_count, addr, alen, 0);
3801 __dev_set_rx_mode(dev);
3802 netif_addr_unlock_bh(dev);
3805 EXPORT_SYMBOL(dev_unicast_add);
3807 int __dev_addr_sync(struct dev_addr_list **to, int *to_count,
3808 struct dev_addr_list **from, int *from_count)
3810 struct dev_addr_list *da, *next;
3814 while (da != NULL) {
3816 if (!da->da_synced) {
3817 err = __dev_addr_add(to, to_count,
3818 da->da_addr, da->da_addrlen, 0);
3823 } else if (da->da_users == 1) {
3824 __dev_addr_delete(to, to_count,
3825 da->da_addr, da->da_addrlen, 0);
3826 __dev_addr_delete(from, from_count,
3827 da->da_addr, da->da_addrlen, 0);
3834 void __dev_addr_unsync(struct dev_addr_list **to, int *to_count,
3835 struct dev_addr_list **from, int *from_count)
3837 struct dev_addr_list *da, *next;
3840 while (da != NULL) {
3842 if (da->da_synced) {
3843 __dev_addr_delete(to, to_count,
3844 da->da_addr, da->da_addrlen, 0);
3846 __dev_addr_delete(from, from_count,
3847 da->da_addr, da->da_addrlen, 0);
3854 * dev_unicast_sync - Synchronize device's unicast list to another device
3855 * @to: destination device
3856 * @from: source device
3858 * Add newly added addresses to the destination device and release
3859 * addresses that have no users left. The source device must be
3860 * locked by netif_tx_lock_bh.
3862 * This function is intended to be called from the dev->set_rx_mode
3863 * function of layered software devices.
3865 int dev_unicast_sync(struct net_device *to, struct net_device *from)
3869 netif_addr_lock_bh(to);
3870 err = __dev_addr_sync(&to->uc_list, &to->uc_count,
3871 &from->uc_list, &from->uc_count);
3873 __dev_set_rx_mode(to);
3874 netif_addr_unlock_bh(to);
3877 EXPORT_SYMBOL(dev_unicast_sync);
3880 * dev_unicast_unsync - Remove synchronized addresses from the destination device
3881 * @to: destination device
3882 * @from: source device
3884 * Remove all addresses that were added to the destination device by
3885 * dev_unicast_sync(). This function is intended to be called from the
3886 * dev->stop function of layered software devices.
3888 void dev_unicast_unsync(struct net_device *to, struct net_device *from)
3890 netif_addr_lock_bh(from);
3891 netif_addr_lock(to);
3893 __dev_addr_unsync(&to->uc_list, &to->uc_count,
3894 &from->uc_list, &from->uc_count);
3895 __dev_set_rx_mode(to);
3897 netif_addr_unlock(to);
3898 netif_addr_unlock_bh(from);
3900 EXPORT_SYMBOL(dev_unicast_unsync);
3902 static void __dev_addr_discard(struct dev_addr_list **list)
3904 struct dev_addr_list *tmp;
3906 while (*list != NULL) {
3909 if (tmp->da_users > tmp->da_gusers)
3910 printk("__dev_addr_discard: address leakage! "
3911 "da_users=%d\n", tmp->da_users);
3916 static void dev_addr_discard(struct net_device *dev)
3918 netif_addr_lock_bh(dev);
3920 __dev_addr_discard(&dev->uc_list);
3923 __dev_addr_discard(&dev->mc_list);
3926 netif_addr_unlock_bh(dev);
3930 * dev_get_flags - get flags reported to userspace
3933 * Get the combination of flag bits exported through APIs to userspace.
3935 unsigned dev_get_flags(const struct net_device *dev)
3939 flags = (dev->flags & ~(IFF_PROMISC |
3944 (dev->gflags & (IFF_PROMISC |
3947 if (netif_running(dev)) {
3948 if (netif_oper_up(dev))
3949 flags |= IFF_RUNNING;
3950 if (netif_carrier_ok(dev))
3951 flags |= IFF_LOWER_UP;
3952 if (netif_dormant(dev))
3953 flags |= IFF_DORMANT;
3960 * dev_change_flags - change device settings
3962 * @flags: device state flags
3964 * Change settings on device based state flags. The flags are
3965 * in the userspace exported format.
3967 int dev_change_flags(struct net_device *dev, unsigned flags)
3970 int old_flags = dev->flags;
3975 * Set the flags on our device.
3978 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
3979 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
3981 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
3985 * Load in the correct multicast list now the flags have changed.
3988 if ((old_flags ^ flags) & IFF_MULTICAST)
3989 dev_change_rx_flags(dev, IFF_MULTICAST);
3991 dev_set_rx_mode(dev);
3994 * Have we downed the interface. We handle IFF_UP ourselves
3995 * according to user attempts to set it, rather than blindly
4000 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
4001 ret = ((old_flags & IFF_UP) ? dev_close : dev_open)(dev);
4004 dev_set_rx_mode(dev);
4007 if (dev->flags & IFF_UP &&
4008 ((old_flags ^ dev->flags) &~ (IFF_UP | IFF_PROMISC | IFF_ALLMULTI |
4010 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4012 if ((flags ^ dev->gflags) & IFF_PROMISC) {
4013 int inc = (flags & IFF_PROMISC) ? +1 : -1;
4014 dev->gflags ^= IFF_PROMISC;
4015 dev_set_promiscuity(dev, inc);
4018 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4019 is important. Some (broken) drivers set IFF_PROMISC, when
4020 IFF_ALLMULTI is requested not asking us and not reporting.
4022 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4023 int inc = (flags & IFF_ALLMULTI) ? +1 : -1;
4024 dev->gflags ^= IFF_ALLMULTI;
4025 dev_set_allmulti(dev, inc);
4028 /* Exclude state transition flags, already notified */
4029 changes = (old_flags ^ dev->flags) & ~(IFF_UP | IFF_RUNNING);
4031 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4037 * dev_set_mtu - Change maximum transfer unit
4039 * @new_mtu: new transfer unit
4041 * Change the maximum transfer size of the network device.
4043 int dev_set_mtu(struct net_device *dev, int new_mtu)
4045 const struct net_device_ops *ops = dev->netdev_ops;
4048 if (new_mtu == dev->mtu)
4051 /* MTU must be positive. */
4055 if (!netif_device_present(dev))
4059 if (ops->ndo_change_mtu)
4060 err = ops->ndo_change_mtu(dev, new_mtu);
4064 if (!err && dev->flags & IFF_UP)
4065 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4070 * dev_set_mac_address - Change Media Access Control Address
4074 * Change the hardware (MAC) address of the device
4076 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4078 const struct net_device_ops *ops = dev->netdev_ops;
4081 if (!ops->ndo_set_mac_address)
4083 if (sa->sa_family != dev->type)
4085 if (!netif_device_present(dev))
4087 err = ops->ndo_set_mac_address(dev, sa);
4089 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4094 * Perform the SIOCxIFxxx calls, inside read_lock(dev_base_lock)
4096 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
4099 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4105 case SIOCGIFFLAGS: /* Get interface flags */
4106 ifr->ifr_flags = dev_get_flags(dev);
4109 case SIOCGIFMETRIC: /* Get the metric on the interface
4110 (currently unused) */
4111 ifr->ifr_metric = 0;
4114 case SIOCGIFMTU: /* Get the MTU of a device */
4115 ifr->ifr_mtu = dev->mtu;
4120 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4122 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4123 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4124 ifr->ifr_hwaddr.sa_family = dev->type;
4132 ifr->ifr_map.mem_start = dev->mem_start;
4133 ifr->ifr_map.mem_end = dev->mem_end;
4134 ifr->ifr_map.base_addr = dev->base_addr;
4135 ifr->ifr_map.irq = dev->irq;
4136 ifr->ifr_map.dma = dev->dma;
4137 ifr->ifr_map.port = dev->if_port;
4141 ifr->ifr_ifindex = dev->ifindex;
4145 ifr->ifr_qlen = dev->tx_queue_len;
4149 /* dev_ioctl() should ensure this case
4161 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4163 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4166 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4167 const struct net_device_ops *ops;
4172 ops = dev->netdev_ops;
4175 case SIOCSIFFLAGS: /* Set interface flags */
4176 return dev_change_flags(dev, ifr->ifr_flags);
4178 case SIOCSIFMETRIC: /* Set the metric on the interface
4179 (currently unused) */
4182 case SIOCSIFMTU: /* Set the MTU of a device */
4183 return dev_set_mtu(dev, ifr->ifr_mtu);
4186 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
4188 case SIOCSIFHWBROADCAST:
4189 if (ifr->ifr_hwaddr.sa_family != dev->type)
4191 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4192 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4193 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4197 if (ops->ndo_set_config) {
4198 if (!netif_device_present(dev))
4200 return ops->ndo_set_config(dev, &ifr->ifr_map);
4205 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4206 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4208 if (!netif_device_present(dev))
4210 return dev_mc_add(dev, ifr->ifr_hwaddr.sa_data,
4214 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4215 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4217 if (!netif_device_present(dev))
4219 return dev_mc_delete(dev, ifr->ifr_hwaddr.sa_data,
4223 if (ifr->ifr_qlen < 0)
4225 dev->tx_queue_len = ifr->ifr_qlen;
4229 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4230 return dev_change_name(dev, ifr->ifr_newname);
4233 * Unknown or private ioctl
4237 if ((cmd >= SIOCDEVPRIVATE &&
4238 cmd <= SIOCDEVPRIVATE + 15) ||
4239 cmd == SIOCBONDENSLAVE ||
4240 cmd == SIOCBONDRELEASE ||
4241 cmd == SIOCBONDSETHWADDR ||
4242 cmd == SIOCBONDSLAVEINFOQUERY ||
4243 cmd == SIOCBONDINFOQUERY ||
4244 cmd == SIOCBONDCHANGEACTIVE ||
4245 cmd == SIOCGMIIPHY ||
4246 cmd == SIOCGMIIREG ||
4247 cmd == SIOCSMIIREG ||
4248 cmd == SIOCBRADDIF ||
4249 cmd == SIOCBRDELIF ||
4250 cmd == SIOCSHWTSTAMP ||
4251 cmd == SIOCWANDEV) {
4253 if (ops->ndo_do_ioctl) {
4254 if (netif_device_present(dev))
4255 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4267 * This function handles all "interface"-type I/O control requests. The actual
4268 * 'doing' part of this is dev_ifsioc above.
4272 * dev_ioctl - network device ioctl
4273 * @net: the applicable net namespace
4274 * @cmd: command to issue
4275 * @arg: pointer to a struct ifreq in user space
4277 * Issue ioctl functions to devices. This is normally called by the
4278 * user space syscall interfaces but can sometimes be useful for
4279 * other purposes. The return value is the return from the syscall if
4280 * positive or a negative errno code on error.
4283 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4289 /* One special case: SIOCGIFCONF takes ifconf argument
4290 and requires shared lock, because it sleeps writing
4294 if (cmd == SIOCGIFCONF) {
4296 ret = dev_ifconf(net, (char __user *) arg);
4300 if (cmd == SIOCGIFNAME)
4301 return dev_ifname(net, (struct ifreq __user *)arg);
4303 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4306 ifr.ifr_name[IFNAMSIZ-1] = 0;
4308 colon = strchr(ifr.ifr_name, ':');
4313 * See which interface the caller is talking about.
4318 * These ioctl calls:
4319 * - can be done by all.
4320 * - atomic and do not require locking.
4331 dev_load(net, ifr.ifr_name);
4332 read_lock(&dev_base_lock);
4333 ret = dev_ifsioc_locked(net, &ifr, cmd);
4334 read_unlock(&dev_base_lock);
4338 if (copy_to_user(arg, &ifr,
4339 sizeof(struct ifreq)))
4345 dev_load(net, ifr.ifr_name);
4347 ret = dev_ethtool(net, &ifr);
4352 if (copy_to_user(arg, &ifr,
4353 sizeof(struct ifreq)))
4359 * These ioctl calls:
4360 * - require superuser power.
4361 * - require strict serialization.
4367 if (!capable(CAP_NET_ADMIN))
4369 dev_load(net, ifr.ifr_name);
4371 ret = dev_ifsioc(net, &ifr, cmd);
4376 if (copy_to_user(arg, &ifr,
4377 sizeof(struct ifreq)))
4383 * These ioctl calls:
4384 * - require superuser power.
4385 * - require strict serialization.
4386 * - do not return a value
4396 case SIOCSIFHWBROADCAST:
4399 case SIOCBONDENSLAVE:
4400 case SIOCBONDRELEASE:
4401 case SIOCBONDSETHWADDR:
4402 case SIOCBONDCHANGEACTIVE:
4406 if (!capable(CAP_NET_ADMIN))
4409 case SIOCBONDSLAVEINFOQUERY:
4410 case SIOCBONDINFOQUERY:
4411 dev_load(net, ifr.ifr_name);
4413 ret = dev_ifsioc(net, &ifr, cmd);
4418 /* Get the per device memory space. We can add this but
4419 * currently do not support it */
4421 /* Set the per device memory buffer space.
4422 * Not applicable in our case */
4427 * Unknown or private ioctl.
4430 if (cmd == SIOCWANDEV ||
4431 (cmd >= SIOCDEVPRIVATE &&
4432 cmd <= SIOCDEVPRIVATE + 15)) {
4433 dev_load(net, ifr.ifr_name);
4435 ret = dev_ifsioc(net, &ifr, cmd);
4437 if (!ret && copy_to_user(arg, &ifr,
4438 sizeof(struct ifreq)))
4442 /* Take care of Wireless Extensions */
4443 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
4444 return wext_handle_ioctl(net, &ifr, cmd, arg);
4451 * dev_new_index - allocate an ifindex
4452 * @net: the applicable net namespace
4454 * Returns a suitable unique value for a new device interface
4455 * number. The caller must hold the rtnl semaphore or the
4456 * dev_base_lock to be sure it remains unique.
4458 static int dev_new_index(struct net *net)
4464 if (!__dev_get_by_index(net, ifindex))
4469 /* Delayed registration/unregisteration */
4470 static LIST_HEAD(net_todo_list);
4472 static void net_set_todo(struct net_device *dev)
4474 list_add_tail(&dev->todo_list, &net_todo_list);
4477 static void rollback_registered(struct net_device *dev)
4479 BUG_ON(dev_boot_phase);
4482 /* Some devices call without registering for initialization unwind. */
4483 if (dev->reg_state == NETREG_UNINITIALIZED) {
4484 printk(KERN_DEBUG "unregister_netdevice: device %s/%p never "
4485 "was registered\n", dev->name, dev);
4491 BUG_ON(dev->reg_state != NETREG_REGISTERED);
4493 /* If device is running, close it first. */
4496 /* And unlink it from device chain. */
4497 unlist_netdevice(dev);
4499 dev->reg_state = NETREG_UNREGISTERING;
4503 /* Shutdown queueing discipline. */
4507 /* Notify protocols, that we are about to destroy
4508 this device. They should clean all the things.
4510 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4513 * Flush the unicast and multicast chains
4515 dev_addr_discard(dev);
4517 if (dev->netdev_ops->ndo_uninit)
4518 dev->netdev_ops->ndo_uninit(dev);
4520 /* Notifier chain MUST detach us from master device. */
4521 WARN_ON(dev->master);
4523 /* Remove entries from kobject tree */
4524 netdev_unregister_kobject(dev);
4531 static void __netdev_init_queue_locks_one(struct net_device *dev,
4532 struct netdev_queue *dev_queue,
4535 spin_lock_init(&dev_queue->_xmit_lock);
4536 netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);
4537 dev_queue->xmit_lock_owner = -1;
4540 static void netdev_init_queue_locks(struct net_device *dev)
4542 netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);
4543 __netdev_init_queue_locks_one(dev, &dev->rx_queue, NULL);
4546 unsigned long netdev_fix_features(unsigned long features, const char *name)
4548 /* Fix illegal SG+CSUM combinations. */
4549 if ((features & NETIF_F_SG) &&
4550 !(features & NETIF_F_ALL_CSUM)) {
4552 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
4553 "checksum feature.\n", name);
4554 features &= ~NETIF_F_SG;
4557 /* TSO requires that SG is present as well. */
4558 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
4560 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
4561 "SG feature.\n", name);
4562 features &= ~NETIF_F_TSO;
4565 if (features & NETIF_F_UFO) {
4566 if (!(features & NETIF_F_GEN_CSUM)) {
4568 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4569 "since no NETIF_F_HW_CSUM feature.\n",
4571 features &= ~NETIF_F_UFO;
4574 if (!(features & NETIF_F_SG)) {
4576 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
4577 "since no NETIF_F_SG feature.\n", name);
4578 features &= ~NETIF_F_UFO;
4584 EXPORT_SYMBOL(netdev_fix_features);
4587 * register_netdevice - register a network device
4588 * @dev: device to register
4590 * Take a completed network device structure and add it to the kernel
4591 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
4592 * chain. 0 is returned on success. A negative errno code is returned
4593 * on a failure to set up the device, or if the name is a duplicate.
4595 * Callers must hold the rtnl semaphore. You may want
4596 * register_netdev() instead of this.
4599 * The locking appears insufficient to guarantee two parallel registers
4600 * will not get the same name.
4603 int register_netdevice(struct net_device *dev)
4605 struct hlist_head *head;
4606 struct hlist_node *p;
4608 struct net *net = dev_net(dev);
4610 BUG_ON(dev_boot_phase);
4615 /* When net_device's are persistent, this will be fatal. */
4616 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
4619 spin_lock_init(&dev->addr_list_lock);
4620 netdev_set_addr_lockdep_class(dev);
4621 netdev_init_queue_locks(dev);
4625 /* Init, if this function is available */
4626 if (dev->netdev_ops->ndo_init) {
4627 ret = dev->netdev_ops->ndo_init(dev);
4635 if (!dev_valid_name(dev->name)) {
4640 dev->ifindex = dev_new_index(net);
4641 if (dev->iflink == -1)
4642 dev->iflink = dev->ifindex;
4644 /* Check for existence of name */
4645 head = dev_name_hash(net, dev->name);
4646 hlist_for_each(p, head) {
4647 struct net_device *d
4648 = hlist_entry(p, struct net_device, name_hlist);
4649 if (!strncmp(d->name, dev->name, IFNAMSIZ)) {
4655 /* Fix illegal checksum combinations */
4656 if ((dev->features & NETIF_F_HW_CSUM) &&
4657 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4658 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
4660 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
4663 if ((dev->features & NETIF_F_NO_CSUM) &&
4664 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
4665 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
4667 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
4670 dev->features = netdev_fix_features(dev->features, dev->name);
4672 /* Enable software GSO if SG is supported. */
4673 if (dev->features & NETIF_F_SG)
4674 dev->features |= NETIF_F_GSO;
4676 netdev_initialize_kobject(dev);
4677 ret = netdev_register_kobject(dev);
4680 dev->reg_state = NETREG_REGISTERED;
4683 * Default initial state at registry is that the
4684 * device is present.
4687 set_bit(__LINK_STATE_PRESENT, &dev->state);
4689 dev_init_scheduler(dev);
4691 list_netdevice(dev);
4693 /* Notify protocols, that a new device appeared. */
4694 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
4695 ret = notifier_to_errno(ret);
4697 rollback_registered(dev);
4698 dev->reg_state = NETREG_UNREGISTERED;
4705 if (dev->netdev_ops->ndo_uninit)
4706 dev->netdev_ops->ndo_uninit(dev);
4711 * init_dummy_netdev - init a dummy network device for NAPI
4712 * @dev: device to init
4714 * This takes a network device structure and initialize the minimum
4715 * amount of fields so it can be used to schedule NAPI polls without
4716 * registering a full blown interface. This is to be used by drivers
4717 * that need to tie several hardware interfaces to a single NAPI
4718 * poll scheduler due to HW limitations.
4720 int init_dummy_netdev(struct net_device *dev)
4722 /* Clear everything. Note we don't initialize spinlocks
4723 * are they aren't supposed to be taken by any of the
4724 * NAPI code and this dummy netdev is supposed to be
4725 * only ever used for NAPI polls
4727 memset(dev, 0, sizeof(struct net_device));
4729 /* make sure we BUG if trying to hit standard
4730 * register/unregister code path
4732 dev->reg_state = NETREG_DUMMY;
4734 /* initialize the ref count */
4735 atomic_set(&dev->refcnt, 1);
4737 /* NAPI wants this */
4738 INIT_LIST_HEAD(&dev->napi_list);
4740 /* a dummy interface is started by default */
4741 set_bit(__LINK_STATE_PRESENT, &dev->state);
4742 set_bit(__LINK_STATE_START, &dev->state);
4746 EXPORT_SYMBOL_GPL(init_dummy_netdev);
4750 * register_netdev - register a network device
4751 * @dev: device to register
4753 * Take a completed network device structure and add it to the kernel
4754 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
4755 * chain. 0 is returned on success. A negative errno code is returned
4756 * on a failure to set up the device, or if the name is a duplicate.
4758 * This is a wrapper around register_netdevice that takes the rtnl semaphore
4759 * and expands the device name if you passed a format string to
4762 int register_netdev(struct net_device *dev)
4769 * If the name is a format string the caller wants us to do a
4772 if (strchr(dev->name, '%')) {
4773 err = dev_alloc_name(dev, dev->name);
4778 err = register_netdevice(dev);
4783 EXPORT_SYMBOL(register_netdev);
4786 * netdev_wait_allrefs - wait until all references are gone.
4788 * This is called when unregistering network devices.
4790 * Any protocol or device that holds a reference should register
4791 * for netdevice notification, and cleanup and put back the
4792 * reference if they receive an UNREGISTER event.
4793 * We can get stuck here if buggy protocols don't correctly
4796 static void netdev_wait_allrefs(struct net_device *dev)
4798 unsigned long rebroadcast_time, warning_time;
4800 rebroadcast_time = warning_time = jiffies;
4801 while (atomic_read(&dev->refcnt) != 0) {
4802 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
4805 /* Rebroadcast unregister notification */
4806 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
4808 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
4810 /* We must not have linkwatch events
4811 * pending on unregister. If this
4812 * happens, we simply run the queue
4813 * unscheduled, resulting in a noop
4816 linkwatch_run_queue();
4821 rebroadcast_time = jiffies;
4826 if (time_after(jiffies, warning_time + 10 * HZ)) {
4827 printk(KERN_EMERG "unregister_netdevice: "
4828 "waiting for %s to become free. Usage "
4830 dev->name, atomic_read(&dev->refcnt));
4831 warning_time = jiffies;
4840 * register_netdevice(x1);
4841 * register_netdevice(x2);
4843 * unregister_netdevice(y1);
4844 * unregister_netdevice(y2);
4850 * We are invoked by rtnl_unlock().
4851 * This allows us to deal with problems:
4852 * 1) We can delete sysfs objects which invoke hotplug
4853 * without deadlocking with linkwatch via keventd.
4854 * 2) Since we run with the RTNL semaphore not held, we can sleep
4855 * safely in order to wait for the netdev refcnt to drop to zero.
4857 * We must not return until all unregister events added during
4858 * the interval the lock was held have been completed.
4860 void netdev_run_todo(void)
4862 struct list_head list;
4864 /* Snapshot list, allow later requests */
4865 list_replace_init(&net_todo_list, &list);
4869 while (!list_empty(&list)) {
4870 struct net_device *dev
4871 = list_entry(list.next, struct net_device, todo_list);
4872 list_del(&dev->todo_list);
4874 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
4875 printk(KERN_ERR "network todo '%s' but state %d\n",
4876 dev->name, dev->reg_state);
4881 dev->reg_state = NETREG_UNREGISTERED;
4883 on_each_cpu(flush_backlog, dev, 1);
4885 netdev_wait_allrefs(dev);
4888 BUG_ON(atomic_read(&dev->refcnt));
4889 WARN_ON(dev->ip_ptr);
4890 WARN_ON(dev->ip6_ptr);
4891 WARN_ON(dev->dn_ptr);
4893 if (dev->destructor)
4894 dev->destructor(dev);
4896 /* Free network device */
4897 kobject_put(&dev->dev.kobj);
4902 * dev_get_stats - get network device statistics
4903 * @dev: device to get statistics from
4905 * Get network statistics from device. The device driver may provide
4906 * its own method by setting dev->netdev_ops->get_stats; otherwise
4907 * the internal statistics structure is used.
4909 const struct net_device_stats *dev_get_stats(struct net_device *dev)
4911 const struct net_device_ops *ops = dev->netdev_ops;
4913 if (ops->ndo_get_stats)
4914 return ops->ndo_get_stats(dev);
4916 unsigned long tx_bytes = 0, tx_packets = 0, tx_dropped = 0;
4917 struct net_device_stats *stats = &dev->stats;
4919 struct netdev_queue *txq;
4921 for (i = 0; i < dev->num_tx_queues; i++) {
4922 txq = netdev_get_tx_queue(dev, i);
4923 tx_bytes += txq->tx_bytes;
4924 tx_packets += txq->tx_packets;
4925 tx_dropped += txq->tx_dropped;
4927 if (tx_bytes || tx_packets || tx_dropped) {
4928 stats->tx_bytes = tx_bytes;
4929 stats->tx_packets = tx_packets;
4930 stats->tx_dropped = tx_dropped;
4935 EXPORT_SYMBOL(dev_get_stats);
4937 static void netdev_init_one_queue(struct net_device *dev,
4938 struct netdev_queue *queue,
4944 static void netdev_init_queues(struct net_device *dev)
4946 netdev_init_one_queue(dev, &dev->rx_queue, NULL);
4947 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
4948 spin_lock_init(&dev->tx_global_lock);
4952 * alloc_netdev_mq - allocate network device
4953 * @sizeof_priv: size of private data to allocate space for
4954 * @name: device name format string
4955 * @setup: callback to initialize device
4956 * @queue_count: the number of subqueues to allocate
4958 * Allocates a struct net_device with private data area for driver use
4959 * and performs basic initialization. Also allocates subquue structs
4960 * for each queue on the device at the end of the netdevice.
4962 struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
4963 void (*setup)(struct net_device *), unsigned int queue_count)
4965 struct netdev_queue *tx;
4966 struct net_device *dev;
4970 BUG_ON(strlen(name) >= sizeof(dev->name));
4972 alloc_size = sizeof(struct net_device);
4974 /* ensure 32-byte alignment of private area */
4975 alloc_size = (alloc_size + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST;
4976 alloc_size += sizeof_priv;
4978 /* ensure 32-byte alignment of whole construct */
4979 alloc_size += NETDEV_ALIGN_CONST;
4981 p = kzalloc(alloc_size, GFP_KERNEL);
4983 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
4987 tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);
4989 printk(KERN_ERR "alloc_netdev: Unable to allocate "
4994 dev = (struct net_device *)
4995 (((long)p + NETDEV_ALIGN_CONST) & ~NETDEV_ALIGN_CONST);
4996 dev->padded = (char *)dev - (char *)p;
4998 if (dev_addr_init(dev))
5001 dev_net_set(dev, &init_net);
5004 dev->num_tx_queues = queue_count;
5005 dev->real_num_tx_queues = queue_count;
5007 dev->gso_max_size = GSO_MAX_SIZE;
5009 netdev_init_queues(dev);
5011 INIT_LIST_HEAD(&dev->napi_list);
5012 dev->priv_flags = IFF_XMIT_DST_RELEASE;
5014 strcpy(dev->name, name);
5024 EXPORT_SYMBOL(alloc_netdev_mq);
5027 * free_netdev - free network device
5030 * This function does the last stage of destroying an allocated device
5031 * interface. The reference to the device object is released.
5032 * If this is the last reference then it will be freed.
5034 void free_netdev(struct net_device *dev)
5036 struct napi_struct *p, *n;
5038 release_net(dev_net(dev));
5042 /* Flush device addresses */
5043 dev_addr_flush(dev);
5045 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5048 /* Compatibility with error handling in drivers */
5049 if (dev->reg_state == NETREG_UNINITIALIZED) {
5050 kfree((char *)dev - dev->padded);
5054 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5055 dev->reg_state = NETREG_RELEASED;
5057 /* will free via device release */
5058 put_device(&dev->dev);
5062 * synchronize_net - Synchronize with packet receive processing
5064 * Wait for packets currently being received to be done.
5065 * Does not block later packets from starting.
5067 void synchronize_net(void)
5074 * unregister_netdevice - remove device from the kernel
5077 * This function shuts down a device interface and removes it
5078 * from the kernel tables.
5080 * Callers must hold the rtnl semaphore. You may want
5081 * unregister_netdev() instead of this.
5084 void unregister_netdevice(struct net_device *dev)
5088 rollback_registered(dev);
5089 /* Finish processing unregister after unlock */
5094 * unregister_netdev - remove device from the kernel
5097 * This function shuts down a device interface and removes it
5098 * from the kernel tables.
5100 * This is just a wrapper for unregister_netdevice that takes
5101 * the rtnl semaphore. In general you want to use this and not
5102 * unregister_netdevice.
5104 void unregister_netdev(struct net_device *dev)
5107 unregister_netdevice(dev);
5111 EXPORT_SYMBOL(unregister_netdev);
5114 * dev_change_net_namespace - move device to different nethost namespace
5116 * @net: network namespace
5117 * @pat: If not NULL name pattern to try if the current device name
5118 * is already taken in the destination network namespace.
5120 * This function shuts down a device interface and moves it
5121 * to a new network namespace. On success 0 is returned, on
5122 * a failure a netagive errno code is returned.
5124 * Callers must hold the rtnl semaphore.
5127 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5130 const char *destname;
5135 /* Don't allow namespace local devices to be moved. */
5137 if (dev->features & NETIF_F_NETNS_LOCAL)
5141 /* Don't allow real devices to be moved when sysfs
5145 if (dev->dev.parent)
5149 /* Ensure the device has been registrered */
5151 if (dev->reg_state != NETREG_REGISTERED)
5154 /* Get out if there is nothing todo */
5156 if (net_eq(dev_net(dev), net))
5159 /* Pick the destination device name, and ensure
5160 * we can use it in the destination network namespace.
5163 destname = dev->name;
5164 if (__dev_get_by_name(net, destname)) {
5165 /* We get here if we can't use the current device name */
5168 if (!dev_valid_name(pat))
5170 if (strchr(pat, '%')) {
5171 if (__dev_alloc_name(net, pat, buf) < 0)
5176 if (__dev_get_by_name(net, destname))
5181 * And now a mini version of register_netdevice unregister_netdevice.
5184 /* If device is running close it first. */
5187 /* And unlink it from device chain */
5189 unlist_netdevice(dev);
5193 /* Shutdown queueing discipline. */
5196 /* Notify protocols, that we are about to destroy
5197 this device. They should clean all the things.
5199 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5202 * Flush the unicast and multicast chains
5204 dev_addr_discard(dev);
5206 netdev_unregister_kobject(dev);
5208 /* Actually switch the network namespace */
5209 dev_net_set(dev, net);
5211 /* Assign the new device name */
5212 if (destname != dev->name)
5213 strcpy(dev->name, destname);
5215 /* If there is an ifindex conflict assign a new one */
5216 if (__dev_get_by_index(net, dev->ifindex)) {
5217 int iflink = (dev->iflink == dev->ifindex);
5218 dev->ifindex = dev_new_index(net);
5220 dev->iflink = dev->ifindex;
5223 /* Fixup kobjects */
5224 err = netdev_register_kobject(dev);
5227 /* Add the device back in the hashes */
5228 list_netdevice(dev);
5230 /* Notify protocols, that a new device appeared. */
5231 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5239 static int dev_cpu_callback(struct notifier_block *nfb,
5240 unsigned long action,
5243 struct sk_buff **list_skb;
5244 struct Qdisc **list_net;
5245 struct sk_buff *skb;
5246 unsigned int cpu, oldcpu = (unsigned long)ocpu;
5247 struct softnet_data *sd, *oldsd;
5249 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
5252 local_irq_disable();
5253 cpu = smp_processor_id();
5254 sd = &per_cpu(softnet_data, cpu);
5255 oldsd = &per_cpu(softnet_data, oldcpu);
5257 /* Find end of our completion_queue. */
5258 list_skb = &sd->completion_queue;
5260 list_skb = &(*list_skb)->next;
5261 /* Append completion queue from offline CPU. */
5262 *list_skb = oldsd->completion_queue;
5263 oldsd->completion_queue = NULL;
5265 /* Find end of our output_queue. */
5266 list_net = &sd->output_queue;
5268 list_net = &(*list_net)->next_sched;
5269 /* Append output queue from offline CPU. */
5270 *list_net = oldsd->output_queue;
5271 oldsd->output_queue = NULL;
5273 raise_softirq_irqoff(NET_TX_SOFTIRQ);
5276 /* Process offline CPU's input_pkt_queue */
5277 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue)))
5285 * netdev_increment_features - increment feature set by one
5286 * @all: current feature set
5287 * @one: new feature set
5288 * @mask: mask feature set
5290 * Computes a new feature set after adding a device with feature set
5291 * @one to the master device with current feature set @all. Will not
5292 * enable anything that is off in @mask. Returns the new feature set.
5294 unsigned long netdev_increment_features(unsigned long all, unsigned long one,
5297 /* If device needs checksumming, downgrade to it. */
5298 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
5299 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
5300 else if (mask & NETIF_F_ALL_CSUM) {
5301 /* If one device supports v4/v6 checksumming, set for all. */
5302 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
5303 !(all & NETIF_F_GEN_CSUM)) {
5304 all &= ~NETIF_F_ALL_CSUM;
5305 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
5308 /* If one device supports hw checksumming, set for all. */
5309 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
5310 all &= ~NETIF_F_ALL_CSUM;
5311 all |= NETIF_F_HW_CSUM;
5315 one |= NETIF_F_ALL_CSUM;
5317 one |= all & NETIF_F_ONE_FOR_ALL;
5318 all &= one | NETIF_F_LLTX | NETIF_F_GSO;
5319 all |= one & mask & NETIF_F_ONE_FOR_ALL;
5323 EXPORT_SYMBOL(netdev_increment_features);
5325 static struct hlist_head *netdev_create_hash(void)
5328 struct hlist_head *hash;
5330 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
5332 for (i = 0; i < NETDEV_HASHENTRIES; i++)
5333 INIT_HLIST_HEAD(&hash[i]);
5338 /* Initialize per network namespace state */
5339 static int __net_init netdev_init(struct net *net)
5341 INIT_LIST_HEAD(&net->dev_base_head);
5343 net->dev_name_head = netdev_create_hash();
5344 if (net->dev_name_head == NULL)
5347 net->dev_index_head = netdev_create_hash();
5348 if (net->dev_index_head == NULL)
5354 kfree(net->dev_name_head);
5360 * netdev_drivername - network driver for the device
5361 * @dev: network device
5362 * @buffer: buffer for resulting name
5363 * @len: size of buffer
5365 * Determine network driver for device.
5367 char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
5369 const struct device_driver *driver;
5370 const struct device *parent;
5372 if (len <= 0 || !buffer)
5376 parent = dev->dev.parent;
5381 driver = parent->driver;
5382 if (driver && driver->name)
5383 strlcpy(buffer, driver->name, len);
5387 static void __net_exit netdev_exit(struct net *net)
5389 kfree(net->dev_name_head);
5390 kfree(net->dev_index_head);
5393 static struct pernet_operations __net_initdata netdev_net_ops = {
5394 .init = netdev_init,
5395 .exit = netdev_exit,
5398 static void __net_exit default_device_exit(struct net *net)
5400 struct net_device *dev;
5402 * Push all migratable of the network devices back to the
5403 * initial network namespace
5407 for_each_netdev(net, dev) {
5409 char fb_name[IFNAMSIZ];
5411 /* Ignore unmoveable devices (i.e. loopback) */
5412 if (dev->features & NETIF_F_NETNS_LOCAL)
5415 /* Delete virtual devices */
5416 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) {
5417 dev->rtnl_link_ops->dellink(dev);
5421 /* Push remaing network devices to init_net */
5422 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
5423 err = dev_change_net_namespace(dev, &init_net, fb_name);
5425 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
5426 __func__, dev->name, err);
5434 static struct pernet_operations __net_initdata default_device_ops = {
5435 .exit = default_device_exit,
5439 * Initialize the DEV module. At boot time this walks the device list and
5440 * unhooks any devices that fail to initialise (normally hardware not
5441 * present) and leaves us with a valid list of present and active devices.
5446 * This is called single threaded during boot, so no need
5447 * to take the rtnl semaphore.
5449 static int __init net_dev_init(void)
5451 int i, rc = -ENOMEM;
5453 BUG_ON(!dev_boot_phase);
5455 if (dev_proc_init())
5458 if (netdev_kobject_init())
5461 INIT_LIST_HEAD(&ptype_all);
5462 for (i = 0; i < PTYPE_HASH_SIZE; i++)
5463 INIT_LIST_HEAD(&ptype_base[i]);
5465 if (register_pernet_subsys(&netdev_net_ops))
5469 * Initialise the packet receive queues.
5472 for_each_possible_cpu(i) {
5473 struct softnet_data *queue;
5475 queue = &per_cpu(softnet_data, i);
5476 skb_queue_head_init(&queue->input_pkt_queue);
5477 queue->completion_queue = NULL;
5478 INIT_LIST_HEAD(&queue->poll_list);
5480 queue->backlog.poll = process_backlog;
5481 queue->backlog.weight = weight_p;
5482 queue->backlog.gro_list = NULL;
5483 queue->backlog.gro_count = 0;
5488 /* The loopback device is special if any other network devices
5489 * is present in a network namespace the loopback device must
5490 * be present. Since we now dynamically allocate and free the
5491 * loopback device ensure this invariant is maintained by
5492 * keeping the loopback device as the first device on the
5493 * list of network devices. Ensuring the loopback devices
5494 * is the first device that appears and the last network device
5497 if (register_pernet_device(&loopback_net_ops))
5500 if (register_pernet_device(&default_device_ops))
5503 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
5504 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
5506 hotcpu_notifier(dev_cpu_callback, 0);
5514 subsys_initcall(net_dev_init);
5516 static int __init initialize_hashrnd(void)
5518 get_random_bytes(&skb_tx_hashrnd, sizeof(skb_tx_hashrnd));
5522 late_initcall_sync(initialize_hashrnd);
5524 EXPORT_SYMBOL(__dev_get_by_index);
5525 EXPORT_SYMBOL(__dev_get_by_name);
5526 EXPORT_SYMBOL(__dev_remove_pack);
5527 EXPORT_SYMBOL(dev_valid_name);
5528 EXPORT_SYMBOL(dev_add_pack);
5529 EXPORT_SYMBOL(dev_alloc_name);
5530 EXPORT_SYMBOL(dev_close);
5531 EXPORT_SYMBOL(dev_get_by_flags);
5532 EXPORT_SYMBOL(dev_get_by_index);
5533 EXPORT_SYMBOL(dev_get_by_name);
5534 EXPORT_SYMBOL(dev_open);
5535 EXPORT_SYMBOL(dev_queue_xmit);
5536 EXPORT_SYMBOL(dev_remove_pack);
5537 EXPORT_SYMBOL(dev_set_allmulti);
5538 EXPORT_SYMBOL(dev_set_promiscuity);
5539 EXPORT_SYMBOL(dev_change_flags);
5540 EXPORT_SYMBOL(dev_set_mtu);
5541 EXPORT_SYMBOL(dev_set_mac_address);
5542 EXPORT_SYMBOL(free_netdev);
5543 EXPORT_SYMBOL(netdev_boot_setup_check);
5544 EXPORT_SYMBOL(netdev_set_master);
5545 EXPORT_SYMBOL(netdev_state_change);
5546 EXPORT_SYMBOL(netif_receive_skb);
5547 EXPORT_SYMBOL(netif_rx);
5548 EXPORT_SYMBOL(register_gifconf);
5549 EXPORT_SYMBOL(register_netdevice);
5550 EXPORT_SYMBOL(register_netdevice_notifier);
5551 EXPORT_SYMBOL(skb_checksum_help);
5552 EXPORT_SYMBOL(synchronize_net);
5553 EXPORT_SYMBOL(unregister_netdevice);
5554 EXPORT_SYMBOL(unregister_netdevice_notifier);
5555 EXPORT_SYMBOL(net_enable_timestamp);
5556 EXPORT_SYMBOL(net_disable_timestamp);
5557 EXPORT_SYMBOL(dev_get_flags);
5559 #if defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)
5560 EXPORT_SYMBOL(br_handle_frame_hook);
5561 EXPORT_SYMBOL(br_fdb_get_hook);
5562 EXPORT_SYMBOL(br_fdb_put_hook);
5565 EXPORT_SYMBOL(dev_load);
5567 EXPORT_PER_CPU_SYMBOL(softnet_data);