1 #ifndef __LINUX_PERCPU_H
2 #define __LINUX_PERCPU_H
4 #include <linux/preempt.h>
5 #include <linux/slab.h> /* For kmalloc() */
7 #include <linux/string.h> /* For memset() */
8 #include <linux/cpumask.h>
10 #include <asm/percpu.h>
12 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
13 #ifndef PERCPU_ENOUGH_ROOM
15 #define PERCPU_MODULE_RESERVE 8192
17 #define PERCPU_MODULE_RESERVE 0
20 #define PERCPU_ENOUGH_ROOM \
21 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
22 #endif /* PERCPU_ENOUGH_ROOM */
25 * Must be an lvalue. Since @var must be a simple identifier,
26 * we force a syntax error here if it isn't.
28 #define get_cpu_var(var) (*({ \
29 extern int simple_identifier_##var(void); \
31 &__get_cpu_var(var); }))
32 #define put_cpu_var(var) preempt_enable()
40 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
42 * Use this to get to a cpu's version of the per-cpu object dynamically
43 * allocated. Non-atomic access to the current CPU's version should
44 * probably be combined with get_cpu()/put_cpu().
46 #define percpu_ptr(ptr, cpu) \
48 struct percpu_data *__p = __percpu_disguise(ptr); \
49 (__typeof__(ptr))__p->ptrs[(cpu)]; \
52 extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
53 extern void percpu_depopulate(void *__pdata, int cpu);
54 extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
56 extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
57 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
58 extern void percpu_free(void *__pdata);
60 #else /* CONFIG_SMP */
62 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
64 static inline void percpu_depopulate(void *__pdata, int cpu)
68 static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
72 static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
75 return percpu_ptr(__pdata, cpu);
78 static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
84 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
86 return kzalloc(size, gfp);
89 static inline void percpu_free(void *__pdata)
94 #endif /* CONFIG_SMP */
96 #define percpu_populate_mask(__pdata, size, gfp, mask) \
97 __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
98 #define percpu_depopulate_mask(__pdata, mask) \
99 __percpu_depopulate_mask((__pdata), &(mask))
100 #define percpu_alloc_mask(size, gfp, mask) \
101 __percpu_alloc_mask((size), (gfp), &(mask))
103 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
105 /* (legacy) interface for use without CPU hotplug handling */
107 #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \
109 #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type))
110 #define free_percpu(ptr) percpu_free((ptr))
111 #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu))
113 #endif /* __LINUX_PERCPU_H */