[PATCH] Add __GFP_THISNODE to avoid fallback to other nodes and ignore cpuset/memory...

Add a new gfp flag __GFP_THISNODE to avoid fallback to other nodes.  This
flag is essential if a kernel component requires memory to be located on a
certain node.  It will be needed for alloc_pages_node() to force allocation
on the indicated node and for alloc_pages() to force allocation on the
current node.

Signed-off-by: Christoph Lameter <clameter@sgi.com>
Cc: Andy Whitcroft <apw@shadowen.org>
Cc: Mel Gorman <mel@csn.ul.ie>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 63ab88a36f395f671d1d67a400ad6e27bf291ee4..0eda5b6dedbd4815ecccad542c26ba5ac904c0d6 100644 (file)
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -45,6 +45,7 @@ struct vm_area_struct;
 #define __GFP_ZERO     ((__force gfp_t)0x8000u)/* Return zeroed page on success */
 #define __GFP_NOMEMALLOC ((__force gfp_t)0x10000u) /* Don't use emergency reserves */
 #define __GFP_HARDWALL   ((__force gfp_t)0x20000u) /* Enforce hardwall cpuset memory allocs */
+#define __GFP_THISNODE ((__force gfp_t)0x40000u)/* No fallback, no policies */
 
 #define __GFP_BITS_SHIFT 20    /* Room for 20 __GFP_FOO bits */
 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
@@ -53,7 +54,7 @@ struct vm_area_struct;
 #define GFP_LEVEL_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS| \
                        __GFP_COLD|__GFP_NOWARN|__GFP_REPEAT| \
                        __GFP_NOFAIL|__GFP_NORETRY|__GFP_NO_GROW|__GFP_COMP| \
-                       __GFP_NOMEMALLOC|__GFP_HARDWALL)
+                       __GFP_NOMEMALLOC|__GFP_HARDWALL|__GFP_THISNODE)
 
 /* This equals 0, but use constants in case they ever change */
 #define GFP_NOWAIT     (GFP_ATOMIC & ~__GFP_HIGH)

diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 4ea6f0dc2fc568b35ab147903de0a8e31d9e8429..76940361273e953081c751e5ffde51552bfcf69e 100644 (file)
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -2316,7 +2316,7 @@ int __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask)
        const struct cpuset *cs;        /* current cpuset ancestors */
        int allowed;                    /* is allocation in zone z allowed? */
 
-       if (in_interrupt())
+       if (in_interrupt() || (gfp_mask & __GFP_THISNODE))
                return 1;
        node = z->zone_pgdat->node_id;
        might_sleep_if(!(gfp_mask & __GFP_HARDWALL));

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index c3429a710ab1868ee5323367780e7ef88fb260da..8002e1faccdafb19df2aa7dbf3c32930324097a2 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1290,7 +1290,7 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 
        if ((gfp & __GFP_WAIT) && !in_interrupt())
                cpuset_update_task_memory_state();
-       if (!pol || in_interrupt())
+       if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
                pol = &default_policy;
        if (pol->policy == MPOL_INTERLEAVE)
                return alloc_page_interleave(gfp, order, interleave_nodes(pol));

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 208a6b03aa78b8765bb0226b56b01f97e3f26983..ea498788af53d5c74a0f0dbc9b8883f05e664b8d 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -893,6 +893,9 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
         * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
         */
        do {
+               if (unlikely((gfp_mask & __GFP_THISNODE) &&
+                       (*z)->zone_pgdat != zonelist->zones[0]->zone_pgdat))
+                               break;
                if ((alloc_flags & ALLOC_CPUSET) &&
                                !cpuset_zone_allowed(*z, gfp_mask))
                        continue;