mm/memory-hotplug: switch locking to a percpu rwsem

[karo-tx-linux.git] / mm / sparse.c

diff --git a/mm/sparse.c b/mm/sparse.c
index 6903c8fc308502eab4ca3570075cd6a881efa724..7b4be3fd5cac074177b7f17ebdeb12dfa56d22e5 100644 (file)
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -168,6 +168,44 @@ void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
        }
 }
 
+/*
+ * There are a number of times that we loop over NR_MEM_SECTIONS,
+ * looking for section_present() on each.  But, when we have very
+ * large physical address spaces, NR_MEM_SECTIONS can also be
+ * very large which makes the loops quite long.
+ *
+ * Keeping track of this gives us an easy way to break out of
+ * those loops early.
+ */
+int __highest_present_section_nr;
+static void section_mark_present(struct mem_section *ms)
+{
+       int section_nr = __section_nr(ms);
+
+       if (section_nr > __highest_present_section_nr)
+               __highest_present_section_nr = section_nr;
+
+       ms->section_mem_map |= SECTION_MARKED_PRESENT;
+}
+
+static inline int next_present_section_nr(int section_nr)
+{
+       do {
+               section_nr++;
+               if (present_section_nr(section_nr))
+                       return section_nr;
+       } while ((section_nr < NR_MEM_SECTIONS) &&
+                (section_nr <= __highest_present_section_nr));
+
+       return -1;
+}
+#define for_each_present_section_nr(start, section_nr)         \
+       for (section_nr = next_present_section_nr(start-1);     \
+            ((section_nr >= 0) &&                              \
+             (section_nr < NR_MEM_SECTIONS) &&                 \
+             (section_nr <= __highest_present_section_nr));    \
+            section_nr = next_present_section_nr(section_nr))
+
 /* Record a memory area against a node. */
 void __init memory_present(int nid, unsigned long start, unsigned long end)
 {
@@ -183,9 +221,11 @@ void __init memory_present(int nid, unsigned long start, unsigned long end)
                set_section_nid(section, nid);
 
                ms = __nr_to_section(section);
-               if (!ms->section_mem_map)
+               if (!ms->section_mem_map) {
                        ms->section_mem_map = sparse_encode_early_nid(nid) |
-                                                       SECTION_MARKED_PRESENT;
+                                                       SECTION_IS_ONLINE;
+                       section_mark_present(ms);
+               }
        }
 }
 
@@ -476,23 +516,19 @@ static void __init alloc_usemap_and_memmap(void (*alloc_func)
        int nodeid_begin = 0;
        unsigned long pnum_begin = 0;
 
-       for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
+       for_each_present_section_nr(0, pnum) {
                struct mem_section *ms;
 
-               if (!present_section_nr(pnum))
-                       continue;
                ms = __nr_to_section(pnum);
                nodeid_begin = sparse_early_nid(ms);
                pnum_begin = pnum;
                break;
        }
        map_count = 1;
-       for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
+       for_each_present_section_nr(pnum_begin + 1, pnum) {
                struct mem_section *ms;
                int nodeid;
 
-               if (!present_section_nr(pnum))
-                       continue;
                ms = __nr_to_section(pnum);
                nodeid = sparse_early_nid(ms);
                if (nodeid == nodeid_begin) {
@@ -561,10 +597,7 @@ void __init sparse_init(void)
                                                        (void *)map_map);
 #endif
 
-       for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
-               if (!present_section_nr(pnum))
-                       continue;
-
+       for_each_present_section_nr(0, pnum) {
                usemap = usemap_map[pnum];
                if (!usemap)
                        continue;
@@ -590,6 +623,48 @@ void __init sparse_init(void)
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG
+
+/* Mark all memory sections within the pfn range as online */
+void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
+{
+       unsigned long pfn;
+
+       for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
+               unsigned long section_nr = pfn_to_section_nr(start_pfn);
+               struct mem_section *ms;
+
+               /* onlining code should never touch invalid ranges */
+               if (WARN_ON(!valid_section_nr(section_nr)))
+                       continue;
+
+               ms = __nr_to_section(section_nr);
+               ms->section_mem_map |= SECTION_IS_ONLINE;
+       }
+}
+
+#ifdef CONFIG_MEMORY_HOTREMOVE
+/* Mark all memory sections within the pfn range as online */
+void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
+{
+       unsigned long pfn;
+
+       for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
+               unsigned long section_nr = pfn_to_section_nr(start_pfn);
+               struct mem_section *ms;
+
+               /*
+                * TODO this needs some double checking. Offlining code makes
+                * sure to check pfn_valid but those checks might be just bogus
+                */
+               if (WARN_ON(!valid_section_nr(section_nr)))
+                       continue;
+
+               ms = __nr_to_section(section_nr);
+               ms->section_mem_map &= ~SECTION_IS_ONLINE;
+       }
+}
+#endif
+
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid)
 {
@@ -686,10 +761,9 @@ static void free_map_bootmem(struct page *memmap)
  * set.  If this is <=0, then that means that the passed-in
  * map was not consumed and must be freed.
  */
-int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
+int __meminit sparse_add_one_section(struct pglist_data *pgdat, unsigned long start_pfn)
 {
        unsigned long section_nr = pfn_to_section_nr(start_pfn);
-       struct pglist_data *pgdat = zone->zone_pgdat;
        struct mem_section *ms;
        struct page *memmap;
        unsigned long *usemap;
@@ -722,7 +796,7 @@ int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn)
 
        memset(memmap, 0, sizeof(struct page) * PAGES_PER_SECTION);
 
-       ms->section_mem_map |= SECTION_MARKED_PRESENT;
+       section_mark_present(ms);
 
        ret = sparse_init_one_section(ms, section_nr, memmap, usemap);