[PATCH] mm: copy_pte_range progress fix

[karo-tx-linux.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e1d3d77f4aeef89259d9e3b09e500042a42d7a87..60663232fbb23a34874f482f49a52f760f4d3ea6 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -734,7 +734,7 @@ buffered_rmqueue(struct zone *zone, int order, gfp_t gfp_flags)
  * of the allocation.
  */
 int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
-                     int classzone_idx, int can_try_harder, int gfp_high)
+                     int classzone_idx, int can_try_harder, gfp_t gfp_high)
 {
        /* free_pages my go negative - that's OK */
        long min = mark, free_pages = z->free_pages - (1 << order) + 1;
@@ -777,7 +777,7 @@ struct page * fastcall
 __alloc_pages(gfp_t gfp_mask, unsigned int order,
                struct zonelist *zonelist)
 {
-       const int wait = gfp_mask & __GFP_WAIT;
+       const gfp_t wait = gfp_mask & __GFP_WAIT;
        struct zone **zones, *z;
        struct page *page;
        struct reclaim_state reclaim_state;
@@ -996,7 +996,7 @@ fastcall unsigned long get_zeroed_page(gfp_t gfp_mask)
         * get_zeroed_page() returns a 32-bit address, which cannot represent
         * a highmem page
         */
-       BUG_ON(gfp_mask & __GFP_HIGHMEM);
+       BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
 
        page = alloc_pages(gfp_mask | __GFP_ZERO, 0);
        if (page)
@@ -1089,7 +1089,7 @@ static unsigned int nr_free_zone_pages(int offset)
  */
 unsigned int nr_free_buffer_pages(void)
 {
-       return nr_free_zone_pages(GFP_USER & GFP_ZONEMASK);
+       return nr_free_zone_pages(gfp_zone(GFP_USER));
 }
 
 /*
@@ -1097,7 +1097,7 @@ unsigned int nr_free_buffer_pages(void)
  */
 unsigned int nr_free_pagecache_pages(void)
 {
-       return nr_free_zone_pages(GFP_HIGHUSER & GFP_ZONEMASK);
+       return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER));
 }
 
 #ifdef CONFIG_HIGHMEM
@@ -1428,6 +1428,16 @@ static int __init build_zonelists_node(pg_data_t *pgdat, struct zonelist *zoneli
        return j;
 }
 
+static inline int highest_zone(int zone_bits)
+{
+       int res = ZONE_NORMAL;
+       if (zone_bits & (__force int)__GFP_HIGHMEM)
+               res = ZONE_HIGHMEM;
+       if (zone_bits & (__force int)__GFP_DMA)
+               res = ZONE_DMA;
+       return res;
+}
+
 #ifdef CONFIG_NUMA
 #define MAX_NODE_LOAD (num_online_nodes())
 static int __initdata node_load[MAX_NUMNODES];
@@ -1524,11 +1534,7 @@ static void __init build_zonelists(pg_data_t *pgdat)
                        zonelist = pgdat->node_zonelists + i;
                        for (j = 0; zonelist->zones[j] != NULL; j++);
 
-                       k = ZONE_NORMAL;
-                       if (i & __GFP_HIGHMEM)
-                               k = ZONE_HIGHMEM;
-                       if (i & __GFP_DMA)
-                               k = ZONE_DMA;
+                       k = highest_zone(i);
 
                        j = build_zonelists_node(NODE_DATA(node), zonelist, j, k);
                        zonelist->zones[j] = NULL;
@@ -1549,12 +1555,7 @@ static void __init build_zonelists(pg_data_t *pgdat)
                zonelist = pgdat->node_zonelists + i;
 
                j = 0;
-               k = ZONE_NORMAL;
-               if (i & __GFP_HIGHMEM)
-                       k = ZONE_HIGHMEM;
-               if (i & __GFP_DMA)
-                       k = ZONE_DMA;
-
+               k = highest_zone(i);
                j = build_zonelists_node(pgdat, zonelist, j, k);
                /*
                 * Now we build the zonelist so that it contains the zones
@@ -1720,29 +1721,29 @@ static int __devinit zone_batchsize(struct zone *zone)
 
        /*
         * The per-cpu-pages pools are set to around 1000th of the
-        * size of the zone.  But no more than 1/4 of a meg - there's
-        * no point in going beyond the size of L2 cache.
+        * size of the zone.  But no more than 1/2 of a meg.
         *
         * OK, so we don't know how big the cache is.  So guess.
         */
        batch = zone->present_pages / 1024;
-       if (batch * PAGE_SIZE > 256 * 1024)
-               batch = (256 * 1024) / PAGE_SIZE;
+       if (batch * PAGE_SIZE > 512 * 1024)
+               batch = (512 * 1024) / PAGE_SIZE;
        batch /= 4;             /* We effectively *= 4 below */
        if (batch < 1)
                batch = 1;
 
        /*
-        * Clamp the batch to a 2^n - 1 value. Having a power
-        * of 2 value was found to be more likely to have
-        * suboptimal cache aliasing properties in some cases.
+        * We will be trying to allcoate bigger chunks of contiguous
+        * memory of the order of fls(batch).  This should result in
+        * better cache coloring.
         *
-        * For example if 2 tasks are alternately allocating
-        * batches of pages, one task can end up with a lot
-        * of pages of one half of the possible page colors
-        * and the other with pages of the other colors.
+        * A sanity check also to ensure that batch is still in limits.
         */
-       batch = (1 << fls(batch + batch/2)) - 1;
+       batch = (1 << fls(batch + batch/2));
+
+       if (fls(batch) >= (PAGE_SHIFT + MAX_ORDER - 2))
+               batch = PAGE_SHIFT + ((MAX_ORDER - 1 - PAGE_SHIFT)/2);
+
        return batch;
 }
 
@@ -1754,7 +1755,7 @@ inline void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
 
        pcp = &p->pcp[0];               /* hot */
        pcp->count = 0;
-       pcp->low = 2 * batch;
+       pcp->low = 0;
        pcp->high = 6 * batch;
        pcp->batch = max(1UL, 1 * batch);
        INIT_LIST_HEAD(&pcp->list);
@@ -1763,7 +1764,7 @@ inline void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
        pcp->count = 0;
        pcp->low = 0;
        pcp->high = 2 * batch;
-       pcp->batch = max(1UL, 1 * batch);
+       pcp->batch = max(1UL, batch/2);
        INIT_LIST_HEAD(&pcp->list);
 }