[PATCH] optimize follow_hugetlb_page

[linux-2.6] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 2cd80203984b27545141b86f253d00fe8c54c25a..ff0ab772f49d13d5fa27374d53bf902540ec8e3b 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -170,12 +170,12 @@
 #if DEBUG
 # define CREATE_MASK   (SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \
                         SLAB_POISON | SLAB_HWCACHE_ALIGN | \
-                        SLAB_NO_REAP | SLAB_CACHE_DMA | \
+                        SLAB_CACHE_DMA | \
                         SLAB_MUST_HWCACHE_ALIGN | SLAB_STORE_USER | \
                         SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                         SLAB_DESTROY_BY_RCU)
 #else
-# define CREATE_MASK   (SLAB_HWCACHE_ALIGN | SLAB_NO_REAP | \
+# define CREATE_MASK   (SLAB_HWCACHE_ALIGN | \
                         SLAB_CACHE_DMA | SLAB_MUST_HWCACHE_ALIGN | \
                         SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                         SLAB_DESTROY_BY_RCU)
@@ -590,6 +590,8 @@ static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
 
 static inline struct kmem_cache *page_get_cache(struct page *page)
 {
+       if (unlikely(PageCompound(page)))
+               page = (struct page *)page_private(page);
        return (struct kmem_cache *)page->lru.next;
 }
 
@@ -600,6 +602,8 @@ static inline void page_set_slab(struct page *page, struct slab *slab)
 
 static inline struct slab *page_get_slab(struct page *page)
 {
+       if (unlikely(PageCompound(page)))
+               page = (struct page *)page_private(page);
        return (struct slab *)page->lru.prev;
 }
 
@@ -662,7 +666,6 @@ static struct kmem_cache cache_cache = {
        .limit = BOOT_CPUCACHE_ENTRIES,
        .shared = 1,
        .buffer_size = sizeof(struct kmem_cache),
-       .flags = SLAB_NO_REAP,
        .name = "kmem_cache",
 #if DEBUG
        .obj_size = sizeof(struct kmem_cache),
@@ -1615,8 +1618,12 @@ static void check_poison_obj(struct kmem_cache *cachep, void *objp)
 
 #if DEBUG
 /**
- * slab_destroy_objs - call the registered destructor for each object in
- *      a slab that is to be destroyed.
+ * slab_destroy_objs - destroy a slab and its objects
+ * @cachep: cache pointer being destroyed
+ * @slabp: slab pointer being destroyed
+ *
+ * Call the registered destructor for each object in a slab that is being
+ * destroyed.
  */
 static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp)
 {
@@ -1661,7 +1668,11 @@ static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp)
 }
 #endif
 
-/*
+/**
+ * slab_destroy - destroy and release all objects in a slab
+ * @cachep: cache pointer being destroyed
+ * @slabp: slab pointer being destroyed
+ *
  * Destroy all the objs in a slab, and release the mem back to the system.
  * Before calling the slab must have been unlinked from the cache.  The
  * cache-lock is not held/needed.
@@ -1840,9 +1851,6 @@ static void setup_cpu_cache(struct kmem_cache *cachep)
  * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
  * for buffer overruns.
  *
- * %SLAB_NO_REAP - Don't automatically reap this cache when we're under
- * memory pressure.
- *
  * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
  * cacheline.  This can be beneficial if you're counting cycles as closely
  * as davem.
@@ -2115,23 +2123,6 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
 #define check_spinlock_acquired_node(x, y) do { } while(0)
 #endif
 
-/*
- * Waits for all CPUs to execute func().
- */
-static void smp_call_function_all_cpus(void (*func)(void *arg), void *arg)
-{
-       check_irq_on();
-       preempt_disable();
-       local_irq_disable();
-       func(arg);
-       local_irq_enable();
-
-       if (smp_call_function(func, arg, 1, 1))
-               BUG();
-
-       preempt_enable();
-}
-
 static void drain_array_locked(struct kmem_cache *cachep,
                        struct array_cache *ac, int force, int node);
 
@@ -2154,7 +2145,7 @@ static void drain_cpu_caches(struct kmem_cache *cachep)
        struct kmem_list3 *l3;
        int node;
 
-       smp_call_function_all_cpus(do_drain, cachep);
+       on_each_cpu(do_drain, cachep, 1, 1);
        check_irq_on();
        for_each_online_node(node) {
                l3 = cachep->nodelists[node];
@@ -2425,8 +2416,11 @@ static void set_slab_attr(struct kmem_cache *cachep, struct slab *slabp,
        struct page *page;
 
        /* Nasty!!!!!! I hope this is OK. */
-       i = 1 << cachep->gfporder;
        page = virt_to_page(objp);
+
+       i = 1;
+       if (likely(!PageCompound(page)))
+               i <<= cachep->gfporder;
        do {
                page_set_cache(page, cachep);
                page_set_slab(page, slabp);
@@ -3170,6 +3164,7 @@ EXPORT_SYMBOL(kmalloc_node);
  * kmalloc - allocate memory
  * @size: how many bytes of memory are required.
  * @flags: the type of memory to allocate.
+ * @caller: function caller for debug tracking of the caller
  *
  * kmalloc is the normal method of allocating memory
  * in the kernel.
@@ -3444,7 +3439,7 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
        }
        new.cachep = cachep;
 
-       smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
+       on_each_cpu(do_ccupdate_local, (void *)&new, 1, 1);
 
        check_irq_on();
        cachep->batchcount = batchcount;
@@ -3575,10 +3570,6 @@ static void cache_reap(void *unused)
                struct slab *slabp;
 
                searchp = list_entry(walk, struct kmem_cache, next);
-
-               if (searchp->flags & SLAB_NO_REAP)
-                       goto next;
-
                check_irq_on();
 
                l3 = searchp->nodelists[numa_node_id()];
@@ -3626,7 +3617,6 @@ static void cache_reap(void *unused)
                } while (--tofree > 0);
 next_unlock:
                spin_unlock_irq(&l3->list_lock);
-next:
                cond_resched();
        }
        check_irq_on();