2 * ACPI 3.0 based NUMA setup
3 * Copyright 2004 Andi Kleen, SuSE Labs.
5 * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
7 * Called from acpi_numa_init while reading the SRAT and SLIT tables.
8 * Assumes all memory regions belonging to a single proximity domain
9 * are in one chunk. Holes between them will be included in the node.
12 #include <linux/kernel.h>
13 #include <linux/acpi.h>
14 #include <linux/mmzone.h>
15 #include <linux/bitmap.h>
16 #include <linux/module.h>
17 #include <linux/topology.h>
18 #include <linux/bootmem.h>
20 #include <asm/proto.h>
23 #include <asm/genapic.h>
25 int acpi_numa __initdata;
27 static struct acpi_table_slit *acpi_slit;
29 static nodemask_t nodes_parsed __initdata;
30 static struct bootnode nodes[MAX_NUMNODES] __initdata;
31 static struct bootnode nodes_add[MAX_NUMNODES];
32 static int found_add_area __initdata;
33 int hotadd_percent __initdata = 0;
35 /* Too small nodes confuse the VM badly. Usually they result
37 #define NODE_MIN_SIZE (4*1024*1024)
39 static __init int setup_node(int pxm)
41 return acpi_map_pxm_to_node(pxm);
44 static __init int conflicting_nodes(unsigned long start, unsigned long end)
47 for_each_node_mask(i, nodes_parsed) {
48 struct bootnode *nd = &nodes[i];
49 if (nd->start == nd->end)
51 if (nd->end > start && nd->start < end)
53 if (nd->end == end && nd->start == start)
59 static __init void cutoff_node(int i, unsigned long start, unsigned long end)
61 struct bootnode *nd = &nodes[i];
66 if (nd->start < start) {
68 if (nd->end < nd->start)
73 if (nd->start > nd->end)
78 static __init void bad_srat(void)
81 printk(KERN_ERR "SRAT: SRAT not used.\n");
84 for (i = 0; i < MAX_LOCAL_APIC; i++)
85 apicid_to_node[i] = NUMA_NO_NODE;
86 for (i = 0; i < MAX_NUMNODES; i++)
87 nodes_add[i].start = nodes[i].end = 0;
88 remove_all_active_ranges();
91 static __init inline int srat_disabled(void)
93 return numa_off || acpi_numa < 0;
97 * A lot of BIOS fill in 10 (= no distance) everywhere. This messes
98 * up the NUMA heuristics which wants the local node to have a smaller
99 * distance than the others.
100 * Do some quick checks here and only use the SLIT if it passes.
102 static __init int slit_valid(struct acpi_table_slit *slit)
105 int d = slit->locality_count;
106 for (i = 0; i < d; i++) {
107 for (j = 0; j < d; j++) {
108 u8 val = slit->entry[d*i + j];
110 if (val != LOCAL_DISTANCE)
112 } else if (val <= LOCAL_DISTANCE)
119 /* Callback for SLIT parsing */
120 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
122 if (!slit_valid(slit)) {
123 printk(KERN_INFO "ACPI: SLIT table looks invalid. Not used.\n");
129 /* Callback for Proximity Domain -> LAPIC mapping */
131 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
138 if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
142 if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
144 pxm = pa->proximity_domain_lo;
145 node = setup_node(pxm);
147 printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
153 apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
155 apic_id = pa->apic_id;
156 apicid_to_node[apic_id] = node;
158 printk(KERN_INFO "SRAT: PXM %u -> APIC %u -> Node %u\n",
162 int update_end_of_memory(unsigned long end) {return -1;}
163 static int hotadd_enough_memory(struct bootnode *nd) {return 1;}
164 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
165 static inline int save_add_info(void) {return 1;}
167 static inline int save_add_info(void) {return 0;}
170 * Update nodes_add and decide if to include add are in the zone.
171 * Both SPARSE and RESERVE need nodes_add information.
172 * This code supports one contiguous hot add area per node.
175 reserve_hotadd(int node, unsigned long start, unsigned long end)
177 unsigned long s_pfn = start >> PAGE_SHIFT;
178 unsigned long e_pfn = end >> PAGE_SHIFT;
179 int ret = 0, changed = 0;
180 struct bootnode *nd = &nodes_add[node];
182 /* I had some trouble with strange memory hotadd regions breaking
183 the boot. Be very strict here and reject anything unexpected.
184 If you want working memory hotadd write correct SRATs.
186 The node size check is a basic sanity check to guard against
188 if ((signed long)(end - start) < NODE_MIN_SIZE) {
189 printk(KERN_ERR "SRAT: Hotplug area too small\n");
193 /* This check might be a bit too strict, but I'm keeping it for now. */
194 if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
196 "SRAT: Hotplug area %lu -> %lu has existing memory\n",
201 if (!hotadd_enough_memory(&nodes_add[node])) {
202 printk(KERN_ERR "SRAT: Hotplug area too large\n");
208 if (nd->start == nd->end) {
213 if (nd->start == end) {
217 if (nd->end == start) {
222 printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
225 ret = update_end_of_memory(nd->end);
228 printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n", nd->start, nd->end);
232 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
234 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
236 struct bootnode *nd, oldnode;
237 unsigned long start, end;
243 if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
247 if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
250 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
252 start = ma->base_address;
253 end = start + ma->length;
254 pxm = ma->proximity_domain;
255 node = setup_node(pxm);
257 printk(KERN_ERR "SRAT: Too many proximity domains.\n");
261 i = conflicting_nodes(start, end);
264 "SRAT: Warning: PXM %d (%lx-%lx) overlaps with itself (%Lx-%Lx)\n",
265 pxm, start, end, nodes[i].start, nodes[i].end);
268 "SRAT: PXM %d (%lx-%lx) overlaps with PXM %d (%Lx-%Lx)\n",
269 pxm, start, end, node_to_pxm(i),
270 nodes[i].start, nodes[i].end);
276 if (!node_test_and_set(node, nodes_parsed)) {
280 if (start < nd->start)
286 printk(KERN_INFO "SRAT: Node %u PXM %u %Lx-%Lx\n", node, pxm,
288 e820_register_active_regions(node, nd->start >> PAGE_SHIFT,
289 nd->end >> PAGE_SHIFT);
290 push_node_boundaries(node, nd->start >> PAGE_SHIFT,
291 nd->end >> PAGE_SHIFT);
293 if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) &&
294 (reserve_hotadd(node, start, end) < 0)) {
295 /* Ignore hotadd region. Undo damage */
296 printk(KERN_NOTICE "SRAT: Hotplug region ignored\n");
298 if ((nd->start | nd->end) == 0)
299 node_clear(node, nodes_parsed);
303 /* Sanity check to catch more bad SRATs (they are amazingly common).
304 Make sure the PXMs cover all memory. */
305 static int __init nodes_cover_memory(const struct bootnode *nodes)
308 unsigned long pxmram, e820ram;
311 for_each_node_mask(i, nodes_parsed) {
312 unsigned long s = nodes[i].start >> PAGE_SHIFT;
313 unsigned long e = nodes[i].end >> PAGE_SHIFT;
315 pxmram -= absent_pages_in_range(s, e);
316 if ((long)pxmram < 0)
320 e820ram = end_pfn - absent_pages_in_range(0, end_pfn);
321 /* We seem to lose 3 pages somewhere. Allow a bit of slack. */
322 if ((long)(e820ram - pxmram) >= 1*1024*1024) {
324 "SRAT: PXMs only cover %luMB of your %luMB e820 RAM. Not used.\n",
325 (pxmram << PAGE_SHIFT) >> 20,
326 (e820ram << PAGE_SHIFT) >> 20);
332 static void __init unparse_node(int node)
335 node_clear(node, nodes_parsed);
336 for (i = 0; i < MAX_LOCAL_APIC; i++) {
337 if (apicid_to_node[i] == node)
338 apicid_to_node[i] = NUMA_NO_NODE;
342 void __init acpi_numa_arch_fixup(void) {}
344 /* Use the information discovered above to actually set up the nodes. */
345 int __init acpi_scan_nodes(unsigned long start, unsigned long end)
352 /* First clean up the node list */
353 for (i = 0; i < MAX_NUMNODES; i++) {
354 cutoff_node(i, start, end);
356 * don't confuse VM with a node that doesn't have the
360 (nodes[i].end - nodes[i].start) < NODE_MIN_SIZE) {
366 if (!nodes_cover_memory(nodes)) {
371 memnode_shift = compute_hash_shift(nodes, MAX_NUMNODES);
372 if (memnode_shift < 0) {
374 "SRAT: No NUMA node hash function found. Contact maintainer\n");
379 node_possible_map = nodes_parsed;
381 /* Finally register nodes */
382 for_each_node_mask(i, node_possible_map)
383 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
384 /* Try again in case setup_node_bootmem missed one due
385 to missing bootmem */
386 for_each_node_mask(i, node_possible_map)
388 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
390 for (i = 0; i < NR_CPUS; i++) {
391 int node = early_cpu_to_node(i);
393 if (node == NUMA_NO_NODE)
395 if (!node_isset(node, node_possible_map))
396 numa_set_node(i, NUMA_NO_NODE);
402 #ifdef CONFIG_NUMA_EMU
403 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
404 [0 ... MAX_NUMNODES-1] = PXM_INVAL
406 static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
407 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
409 static int __init find_node_by_addr(unsigned long addr)
411 int ret = NUMA_NO_NODE;
414 for_each_node_mask(i, nodes_parsed) {
416 * Find the real node that this emulated node appears on. For
417 * the sake of simplicity, we only use a real node's starting
418 * address to determine which emulated node it appears on.
420 if (addr >= nodes[i].start && addr < nodes[i].end) {
429 * In NUMA emulation, we need to setup proximity domain (_PXM) to node ID
430 * mappings that respect the real ACPI topology but reflect our emulated
431 * environment. For each emulated node, we find which real node it appears on
432 * and create PXM to NID mappings for those fake nodes which mirror that
433 * locality. SLIT will now represent the correct distances between emulated
434 * nodes as a result of the real topology.
436 void __init acpi_fake_nodes(const struct bootnode *fake_nodes, int num_nodes)
440 printk(KERN_INFO "Faking PXM affinity for fake nodes on real "
442 for (i = 0; i < num_nodes; i++) {
445 nid = find_node_by_addr(fake_nodes[i].start);
446 if (nid == NUMA_NO_NODE)
448 pxm = node_to_pxm(nid);
449 if (pxm == PXM_INVAL)
451 fake_node_to_pxm_map[i] = pxm;
453 * For each apicid_to_node mapping that exists for this real
454 * node, it must now point to the fake node ID.
456 for (j = 0; j < MAX_LOCAL_APIC; j++)
457 if (apicid_to_node[j] == nid)
458 fake_apicid_to_node[j] = i;
460 for (i = 0; i < num_nodes; i++)
461 __acpi_map_pxm_to_node(fake_node_to_pxm_map[i], i);
462 memcpy(apicid_to_node, fake_apicid_to_node, sizeof(apicid_to_node));
464 nodes_clear(nodes_parsed);
465 for (i = 0; i < num_nodes; i++)
466 if (fake_nodes[i].start != fake_nodes[i].end)
467 node_set(i, nodes_parsed);
468 WARN_ON(!nodes_cover_memory(fake_nodes));
471 static int null_slit_node_compare(int a, int b)
473 return node_to_pxm(a) == node_to_pxm(b);
476 static int null_slit_node_compare(int a, int b)
480 #endif /* CONFIG_NUMA_EMU */
482 void __init srat_reserve_add_area(int nodeid)
484 if (found_add_area && nodes_add[nodeid].end) {
487 printk(KERN_INFO "SRAT: Reserving hot-add memory space "
488 "for node %d at %Lx-%Lx\n",
489 nodeid, nodes_add[nodeid].start, nodes_add[nodeid].end);
490 total_mb = (nodes_add[nodeid].end - nodes_add[nodeid].start)
492 total_mb *= sizeof(struct page);
494 printk(KERN_INFO "SRAT: This will cost you %Lu MB of "
495 "pre-allocated memory.\n", (unsigned long long)total_mb);
496 reserve_bootmem_node(NODE_DATA(nodeid), nodes_add[nodeid].start,
497 nodes_add[nodeid].end - nodes_add[nodeid].start,
502 int __node_distance(int a, int b)
507 return null_slit_node_compare(a, b) ? LOCAL_DISTANCE :
509 index = acpi_slit->locality_count * node_to_pxm(a);
510 return acpi_slit->entry[index + node_to_pxm(b)];
513 EXPORT_SYMBOL(__node_distance);
515 int memory_add_physaddr_to_nid(u64 start)
520 if (nodes_add[i].start <= start && nodes_add[i].end > start)
525 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);