#include <linux/threads.h>
#include <linux/numa.h>
#include <linux/init.h>
+#include <linux/seqlock.h>
#include <asm/atomic.h>
/* Free memory management - zoned buddy allocator. */
* free areas of different sizes
*/
spinlock_t lock;
+#ifdef CONFIG_MEMORY_HOTPLUG
+ /* see spanned/present_pages for more description */
+ seqlock_t span_seqlock;
+#endif
struct free_area free_area[MAX_ORDER];
/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
unsigned long zone_start_pfn;
+ /*
+ * zone_start_pfn, spanned_pages and present_pages are all
+ * protected by span_seqlock. It is a seqlock because it has
+ * to be read outside of zone->lock, and it is done in the main
+ * allocator path. But, it is written quite infrequently.
+ *
+ * The lock is declared along with zone->lock because it is
+ * frequently read in proximity to zone->lock. It's good to
+ * give them a chance of being in the same cacheline.
+ */
unsigned long spanned_pages; /* total size, including holes */
unsigned long present_pages; /* amount of memory (excluding holes) */
struct page *node_mem_map;
#endif
struct bootmem_data *bdata;
+#ifdef CONFIG_MEMORY_HOTPLUG
+ /*
+ * Must be held any time you expect node_start_pfn, node_present_pages
+ * or node_spanned_pages stay constant. Holding this will also
+ * guarantee that any pfn_valid() stays that way.
+ *
+ * Nests above zone->lock and zone->size_seqlock.
+ */
+ spinlock_t node_size_lock;
+#endif
unsigned long node_start_pfn;
unsigned long node_present_pages; /* total number of physical pages */
unsigned long node_spanned_pages; /* total size of physical page
#endif
#define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr))
+#include <linux/memory_hotplug.h>
+
extern struct pglist_data *pgdat_list;
void __get_zone_counts(unsigned long *active, unsigned long *inactive,
void build_all_zonelists(void);
void wakeup_kswapd(struct zone *zone, int order);
int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
- int alloc_type, int can_try_harder, int gfp_high);
+ int alloc_type, int can_try_harder, gfp_t gfp_high);
#ifdef CONFIG_HAVE_MEMORY_PRESENT
void memory_present(int nid, unsigned long start, unsigned long end);
unsigned long section_mem_map;
};
-#ifdef CONFIG_ARCH_SPARSEMEM_EXTREME
-/*
- * Should we ever require GCC 4 or later then the flat array scheme
- * can be eliminated and a uniform solution for EXTREME and !EXTREME can
- * be arrived at.
- */
-#define SECTION_ROOT_SHIFT (PAGE_SHIFT-3)
-#define SECTION_ROOT_MASK ((1UL<<SECTION_ROOT_SHIFT) - 1)
-#define SECTION_TO_ROOT(_sec) ((_sec) >> SECTION_ROOT_SHIFT)
-#define NR_SECTION_ROOTS (NR_MEM_SECTIONS >> SECTION_ROOT_SHIFT)
-
-extern struct mem_section *mem_section[NR_SECTION_ROOTS];
+#ifdef CONFIG_SPARSEMEM_EXTREME
+#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
+#else
+#define SECTIONS_PER_ROOT 1
+#endif
-static inline struct mem_section *__nr_to_section(unsigned long nr)
-{
- if (!mem_section[SECTION_TO_ROOT(nr)])
- return NULL;
- return &mem_section[SECTION_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
-}
+#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
+#define NR_SECTION_ROOTS (NR_MEM_SECTIONS / SECTIONS_PER_ROOT)
+#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
+#ifdef CONFIG_SPARSEMEM_EXTREME
+extern struct mem_section *mem_section[NR_SECTION_ROOTS];
#else
-
-extern struct mem_section mem_section[NR_MEM_SECTIONS];
+extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
+#endif
static inline struct mem_section *__nr_to_section(unsigned long nr)
{
- return &mem_section[nr];
+ if (!mem_section[SECTION_NR_TO_ROOT(nr)])
+ return NULL;
+ return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
}
-
-#define sparse_index_init(_sec, _nid) do {} while (0)
-
-#endif
+extern int __section_nr(struct mem_section* ms);
/*
* We use the lower bits of the mem_map pointer to store
void sparse_init(void);
#else
#define sparse_init() do {} while (0)
+#define sparse_index_init(_sec, _nid) do {} while (0)
#endif /* CONFIG_SPARSEMEM */
#ifdef CONFIG_NODES_SPAN_OTHER_NODES