[PATCH] x86-64: Allocate the NUMA hash function nodemap dynamically

Remove the statically allocated memory to NUMA node hash map in favor of a
dynamically allocated memory to node hash map (it is cache aligned).

This patch has the nice side effect in that it allows the hash map to grow
for systems with large amounts of memory (256GB - 1TB), but suffer from
having small PCI space tacked onto the boot node (which is somewhere
between 192MB to 512MB on the ES7000).

Signed-off-by: Amul Shah <amul.shah@unisys.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Cc: Andi Kleen <ak@suse.de>
Cc: Rohit Seth <rohitseth@google.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>

diff --git a/arch/x86_64/kernel/e820.c b/arch/x86_64/kernel/e820.c
index 6fe191c5808465ddc7c8673c8996d52a67858d98..9d67955bbc317fe321b368cc941e7a689ba86fb4 100644 (file)
--- a/arch/x86_64/kernel/e820.c
+++ b/arch/x86_64/kernel/e820.c
@@ -83,6 +83,13 @@ static inline int bad_addr(unsigned long *addrp, unsigned long size)
                return 1;
        }
 
+#ifdef CONFIG_NUMA
+       /* NUMA memory to node map */
+       if (last >= nodemap_addr && addr < nodemap_addr + nodemap_size) {
+               *addrp = nodemap_addr + nodemap_size;
+               return 1;
+       }
+#endif
        /* XXX ramdisk image here? */ 
        return 0;
 } 

diff --git a/arch/x86_64/kernel/setup.c b/arch/x86_64/kernel/setup.c
index 60477244d1a371847ad69ac1c05237f0d0d06722..f330f82854992e13da99b03887f4b307f2f27e13 100644 (file)
--- a/arch/x86_64/kernel/setup.c
+++ b/arch/x86_64/kernel/setup.c
@@ -444,6 +444,11 @@ void __init setup_arch(char **cmdline_p)
        /* reserve ebda region */
        if (ebda_addr)
                reserve_bootmem_generic(ebda_addr, ebda_size);
+#ifdef CONFIG_NUMA
+       /* reserve nodemap region */
+       if (nodemap_addr)
+               reserve_bootmem_generic(nodemap_addr, nodemap_size);
+#endif
 
 #ifdef CONFIG_SMP
        /*

diff --git a/arch/x86_64/mm/numa.c b/arch/x86_64/mm/numa.c
index 2ee2e003606cad9cc727042af6ed435dadc45a53..7d9c428f409438ed4d2d80c15e4393bcb7027586 100644 (file)
--- a/arch/x86_64/mm/numa.c
+++ b/arch/x86_64/mm/numa.c
@@ -36,6 +36,8 @@ unsigned char apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
 cpumask_t node_to_cpumask[MAX_NUMNODES] __read_mostly;
 
 int numa_off __initdata;
+unsigned long __initdata nodemap_addr;
+unsigned long __initdata nodemap_size;
 
 
 /*
@@ -52,34 +54,87 @@ populate_memnodemap(const struct bootnode *nodes, int numnodes, int shift)
        int res = -1;
        unsigned long addr, end;
 
-       if (shift >= 64)
-               return -1;
-       memset(memnodemap, 0xff, sizeof(memnodemap));
+       memset(memnodemap, 0xff, memnodemapsize);
        for (i = 0; i < numnodes; i++) {
                addr = nodes[i].start;
                end = nodes[i].end;
                if (addr >= end)
                        continue;
-               if ((end >> shift) >= NODEMAPSIZE)
+               if ((end >> shift) >= memnodemapsize)
                        return 0;
                do {
                        if (memnodemap[addr >> shift] != 0xff)
                                return -1;
                        memnodemap[addr >> shift] = i;
-                       addr += (1UL << shift);
+                       addr += (1UL << shift);
                } while (addr < end);
                res = 1;
        } 
        return res;
 }
 
-int __init compute_hash_shift(struct bootnode *nodes, int numnodes)
+static int __init allocate_cachealigned_memnodemap(void)
+{
+       unsigned long pad, pad_addr;
+
+       memnodemap = memnode.embedded_map;
+       if (memnodemapsize <= 48) {
+               printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
+                      nodemap_addr, nodemap_addr + nodemap_size);
+               return 0;
+       }
+
+       pad = L1_CACHE_BYTES - 1;
+       pad_addr = 0x8000;
+       nodemap_size = pad + memnodemapsize;
+       nodemap_addr = find_e820_area(pad_addr, end_pfn<<PAGE_SHIFT,
+                                     nodemap_size);
+       if (nodemap_addr == -1UL) {
+               printk(KERN_ERR
+                      "NUMA: Unable to allocate Memory to Node hash map\n");
+               nodemap_addr = nodemap_size = 0;
+               return -1;
+       }
+       pad_addr = (nodemap_addr + pad) & ~pad;
+       memnodemap = phys_to_virt(pad_addr);
+
+       printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
+              nodemap_addr, nodemap_addr + nodemap_size);
+       return 0;
+}
+
+/*
+ * The LSB of all start and end addresses in the node map is the value of the
+ * maximum possible shift.
+ */
+static int __init
+extract_lsb_from_nodes (const struct bootnode *nodes, int numnodes)
 {
-       int shift = 20;
+       int i;
+       unsigned long start, end;
+       unsigned long bitfield = 0, memtop = 0;
 
-       while (populate_memnodemap(nodes, numnodes, shift + 1) >= 0)
-               shift++;
+       for (i = 0; i < numnodes; i++) {
+               start = nodes[i].start;
+               end = nodes[i].end;
+               if (start >= end)
+                       continue;
+               bitfield |= start | end;
+               if (end > memtop)
+                       memtop = end;
+       }
+       i = find_first_bit(&bitfield, sizeof(unsigned long)*8);
+       memnodemapsize = (memtop >> i)+1;
+       return i;
+}
+
+int __init compute_hash_shift(struct bootnode *nodes, int numnodes)
+{
+       int shift;
 
+       shift = extract_lsb_from_nodes(nodes, numnodes);
+       if (allocate_cachealigned_memnodemap())
+               return -1;
        printk(KERN_DEBUG "NUMA: Using %d for the hash shift.\n",
                shift);
 
@@ -290,6 +345,7 @@ void __init numa_initmem_init(unsigned long start_pfn, unsigned long end_pfn)
               end_pfn << PAGE_SHIFT); 
                /* setup dummy node covering all memory */ 
        memnode_shift = 63; 
+       memnodemap = memnode.embedded_map;
        memnodemap[0] = 0;
        nodes_clear(node_online_map);
        node_set_online(0);

diff --git a/include/asm-x86_64/e820.h b/include/asm-x86_64/e820.h
index fa2086774105500f5ac2d7a5f8e1a6bbef219d38..855fb4a454b6da51881e1a0bc90bf197627c8a63 100644 (file)
--- a/include/asm-x86_64/e820.h
+++ b/include/asm-x86_64/e820.h
@@ -56,6 +56,7 @@ extern void finish_e820_parsing(void);
 extern struct e820map e820;
 
 extern unsigned ebda_addr, ebda_size;
+extern unsigned long nodemap_addr, nodemap_size;
 #endif/*!__ASSEMBLY__*/
 
 #endif/*__E820_HEADER*/

diff --git a/include/asm-x86_64/mmzone.h b/include/asm-x86_64/mmzone.h
index c38ebdf6f426ad41ddfc19fc8df42553eae4b5a6..39ef106986eb78829e0aab1542aef151807c6139 100644 (file)
--- a/include/asm-x86_64/mmzone.h
+++ b/include/asm-x86_64/mmzone.h
@@ -11,24 +11,25 @@
 
 #include <asm/smp.h>
 
-/* Should really switch to dynamic allocation at some point */
-#define NODEMAPSIZE 0x4fff
-
 /* Simple perfect hash to map physical addresses to node numbers */
 struct memnode {
        int shift;
-       u8 map[NODEMAPSIZE];
-} ____cacheline_aligned;
+       unsigned int mapsize;
+       u8 *map;
+       u8 embedded_map[64-16];
+} ____cacheline_aligned; /* total size = 64 bytes */
 extern struct memnode memnode;
 #define memnode_shift memnode.shift
 #define memnodemap memnode.map
+#define memnodemapsize memnode.mapsize
 
 extern struct pglist_data *node_data[];
 
 static inline __attribute__((pure)) int phys_to_nid(unsigned long addr) 
 { 
        unsigned nid; 
-       VIRTUAL_BUG_ON((addr >> memnode_shift) >= NODEMAPSIZE);
+       VIRTUAL_BUG_ON(!memnodemap);
+       VIRTUAL_BUG_ON((addr >> memnode_shift) >= memnodemapsize);
        nid = memnodemap[addr >> memnode_shift]; 
        VIRTUAL_BUG_ON(nid >= MAX_NUMNODES || !node_data[nid]); 
        return nid;