[PATCH] Add __GFP_THISNODE to avoid fallback to other nodes and ignore cpuset/memory...

[linux-2.6] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6c7c2dd1b3ed9b2c2442fd43d4708b1f57669e03..ea498788af53d5c74a0f0dbc9b8883f05e664b8d 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -637,7 +637,8 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
  */
 void drain_node_pages(int nodeid)
 {
-       int i, z;
+       int i;
+       enum zone_type z;
        unsigned long flags;
 
        for (z = 0; z < MAX_NR_ZONES; z++) {
@@ -892,6 +893,9 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
         * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
         */
        do {
+               if (unlikely((gfp_mask & __GFP_THISNODE) &&
+                       (*z)->zone_pgdat != zonelist->zones[0]->zone_pgdat))
+                               break;
                if ((alloc_flags & ALLOC_CPUSET) &&
                                !cpuset_zone_allowed(*z, gfp_mask))
                        continue;
@@ -1158,7 +1162,8 @@ EXPORT_SYMBOL(nr_free_pages);
 #ifdef CONFIG_NUMA
 unsigned int nr_free_pages_pgdat(pg_data_t *pgdat)
 {
-       unsigned int i, sum = 0;
+       unsigned int sum = 0;
+       enum zone_type i;
 
        for (i = 0; i < MAX_NR_ZONES; i++)
                sum += pgdat->node_zones[i].free_pages;
@@ -1358,40 +1363,25 @@ void show_free_areas(void)
  * Add all populated zones of a node to the zonelist.
  */
 static int __meminit build_zonelists_node(pg_data_t *pgdat,
-                       struct zonelist *zonelist, int nr_zones, int zone_type)
+                       struct zonelist *zonelist, int nr_zones, enum zone_type zone_type)
 {
        struct zone *zone;
 
        BUG_ON(zone_type >= MAX_NR_ZONES);
+       zone_type++;
 
        do {
+               zone_type--;
                zone = pgdat->node_zones + zone_type;
                if (populated_zone(zone)) {
                        zonelist->zones[nr_zones++] = zone;
                        check_highest_zone(zone_type);
                }
-               zone_type--;
 
-       } while (zone_type >= 0);
+       } while (zone_type);
        return nr_zones;
 }
 
-static inline int highest_zone(int zone_bits)
-{
-       int res = ZONE_NORMAL;
-#ifdef CONFIG_HIGHMEM
-       if (zone_bits & (__force int)__GFP_HIGHMEM)
-               res = ZONE_HIGHMEM;
-#endif
-#ifdef CONFIG_ZONE_DMA32
-       if (zone_bits & (__force int)__GFP_DMA32)
-               res = ZONE_DMA32;
-#endif
-       if (zone_bits & (__force int)__GFP_DMA)
-               res = ZONE_DMA;
-       return res;
-}
-
 #ifdef CONFIG_NUMA
 #define MAX_NODE_LOAD (num_online_nodes())
 static int __meminitdata node_load[MAX_NUMNODES];
@@ -1457,13 +1447,14 @@ static int __meminit find_next_best_node(int node, nodemask_t *used_node_mask)
 
 static void __meminit build_zonelists(pg_data_t *pgdat)
 {
-       int i, j, k, node, local_node;
+       int j, node, local_node;
+       enum zone_type i;
        int prev_node, load;
        struct zonelist *zonelist;
        nodemask_t used_mask;
 
        /* initialize zonelists */
-       for (i = 0; i < GFP_ZONETYPES; i++) {
+       for (i = 0; i < MAX_NR_ZONES; i++) {
                zonelist = pgdat->node_zonelists + i;
                zonelist->zones[0] = NULL;
        }
@@ -1493,13 +1484,11 @@ static void __meminit build_zonelists(pg_data_t *pgdat)
                        node_load[node] += load;
                prev_node = node;
                load--;
-               for (i = 0; i < GFP_ZONETYPES; i++) {
+               for (i = 0; i < MAX_NR_ZONES; i++) {
                        zonelist = pgdat->node_zonelists + i;
                        for (j = 0; zonelist->zones[j] != NULL; j++);
 
-                       k = highest_zone(i);
-
-                       j = build_zonelists_node(NODE_DATA(node), zonelist, j, k);
+                       j = build_zonelists_node(NODE_DATA(node), zonelist, j, i);
                        zonelist->zones[j] = NULL;
                }
        }
@@ -1509,19 +1498,16 @@ static void __meminit build_zonelists(pg_data_t *pgdat)
 
 static void __meminit build_zonelists(pg_data_t *pgdat)
 {
-       int i, node, local_node;
-       enum zone_type k;
-       enum zone_type j;
+       int node, local_node;
+       enum zone_type i,j;
 
        local_node = pgdat->node_id;
-       for (i = 0; i < GFP_ZONETYPES; i++) {
+       for (i = 0; i < MAX_NR_ZONES; i++) {
                struct zonelist *zonelist;
 
                zonelist = pgdat->node_zonelists + i;
 
-               j = 0;
-               k = highest_zone(i);
-               j = build_zonelists_node(pgdat, zonelist, j, k);
+               j = build_zonelists_node(pgdat, zonelist, 0, i);
                /*
                 * Now we build the zonelist so that it contains the zones
                 * of all the other nodes.
@@ -1533,12 +1519,12 @@ static void __meminit build_zonelists(pg_data_t *pgdat)
                for (node = local_node + 1; node < MAX_NUMNODES; node++) {
                        if (!node_online(node))
                                continue;
-                       j = build_zonelists_node(NODE_DATA(node), zonelist, j, k);
+                       j = build_zonelists_node(NODE_DATA(node), zonelist, j, i);
                }
                for (node = 0; node < local_node; node++) {
                        if (!node_online(node))
                                continue;
-                       j = build_zonelists_node(NODE_DATA(node), zonelist, j, k);
+                       j = build_zonelists_node(NODE_DATA(node), zonelist, j, i);
                }
 
                zonelist->zones[j] = NULL;
@@ -1644,7 +1630,7 @@ static void __init calculate_zone_totalpages(struct pglist_data *pgdat,
                unsigned long *zones_size, unsigned long *zholes_size)
 {
        unsigned long realtotalpages, totalpages = 0;
-       int i;
+       enum zone_type i;
 
        for (i = 0; i < MAX_NR_ZONES; i++)
                totalpages += zones_size[i];
@@ -2132,7 +2118,7 @@ static void calculate_totalreserve_pages(void)
 {
        struct pglist_data *pgdat;
        unsigned long reserve_pages = 0;
-       int i, j;
+       enum zone_type i, j;
 
        for_each_online_pgdat(pgdat) {
                for (i = 0; i < MAX_NR_ZONES; i++) {
@@ -2165,7 +2151,7 @@ static void calculate_totalreserve_pages(void)
 static void setup_per_zone_lowmem_reserve(void)
 {
        struct pglist_data *pgdat;
-       int j, idx;
+       enum zone_type j, idx;
 
        for_each_online_pgdat(pgdat) {
                for (j = 0; j < MAX_NR_ZONES; j++) {
@@ -2174,9 +2160,12 @@ static void setup_per_zone_lowmem_reserve(void)
 
                        zone->lowmem_reserve[j] = 0;
 
-                       for (idx = j-1; idx >= 0; idx--) {
+                       idx = j;
+                       while (idx) {
                                struct zone *lower_zone;
 
+                               idx--;
+
                                if (sysctl_lowmem_reserve_ratio[idx] < 1)
                                        sysctl_lowmem_reserve_ratio[idx] = 1;