X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=include%2Fasm-x86%2Fparavirt.h;h=3d419398499b4a6fe14de3eec92761f2e80ba19a;hb=1de87bd40e119d26533b5135677901990390bfa9;hp=eac25b5323a256e51a1c279e8569c0b57d4bd541;hpb=ef38503e034f82ad16a75a9dfea40ed7adaf00a8;p=linux-2.6 diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index eac25b5323..3d41939849 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -101,6 +101,11 @@ struct pv_cpu_ops { unsigned long (*read_cr4)(void); void (*write_cr4)(unsigned long); +#ifdef CONFIG_X86_64 + unsigned long (*read_cr8)(void); + void (*write_cr8)(unsigned long); +#endif + /* Segment descriptor handling */ void (*load_tr_desc)(void); void (*load_gdt)(const struct desc_ptr *); @@ -216,7 +221,7 @@ struct pv_mmu_ops { /* Hooks for allocating/releasing pagetable pages */ void (*alloc_pt)(struct mm_struct *mm, u32 pfn); - void (*alloc_pd)(u32 pfn); + void (*alloc_pd)(struct mm_struct *mm, u32 pfn); void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count); void (*release_pt)(u32 pfn); void (*release_pd)(u32 pfn); @@ -226,7 +231,8 @@ struct pv_mmu_ops { void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval); void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); - void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + void (*pte_update)(struct mm_struct *mm, unsigned long addr, + pte_t *ptep); void (*pte_update_defer)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); @@ -241,7 +247,8 @@ struct pv_mmu_ops { void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); - void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + void (*pte_clear)(struct mm_struct *mm, unsigned long addr, + pte_t *ptep); void (*pmd_clear)(pmd_t *pmdp); #endif /* CONFIG_X86_PAE */ @@ -254,6 +261,8 @@ struct pv_mmu_ops { #if PAGETABLE_LEVELS == 4 pudval_t (*pud_val)(pud_t); pud_t (*make_pud)(pudval_t pud); + + void (*set_pgd)(pgd_t *pudp, pgd_t pgdval); #endif /* PAGETABLE_LEVELS == 4 */ #endif /* PAGETABLE_LEVELS >= 3 */ @@ -267,8 +276,7 @@ struct pv_mmu_ops { /* This contains all the paravirt structures: we get a convenient * number for each function using the offset which we use to indicate * what to patch. */ -struct paravirt_patch_template -{ +struct paravirt_patch_template { struct pv_init_ops pv_init_ops; struct pv_time_ops pv_time_ops; struct pv_cpu_ops pv_cpu_ops; @@ -614,6 +622,18 @@ static inline void write_cr4(unsigned long x) PVOP_VCALL1(pv_cpu_ops.write_cr4, x); } +#ifdef CONFIG_X86_64 +static inline unsigned long read_cr8(void) +{ + return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8); +} + +static inline void write_cr8(unsigned long x) +{ + PVOP_VCALL1(pv_cpu_ops.write_cr8, x); +} +#endif + static inline void raw_safe_halt(void) { PVOP_VCALL0(pv_irq_ops.safe_halt); @@ -641,43 +661,56 @@ static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) } /* These should all do BUG_ON(_err), but our headers are too tangled. */ -#define rdmsr(msr,val1,val2) do { \ +#define rdmsr(msr, val1, val2) \ +do { \ int _err; \ u64 _l = paravirt_read_msr(msr, &_err); \ val1 = (u32)_l; \ val2 = _l >> 32; \ -} while(0) +} while (0) -#define wrmsr(msr,val1,val2) do { \ +#define wrmsr(msr, val1, val2) \ +do { \ paravirt_write_msr(msr, val1, val2); \ -} while(0) +} while (0) -#define rdmsrl(msr,val) do { \ +#define rdmsrl(msr, val) \ +do { \ int _err; \ val = paravirt_read_msr(msr, &_err); \ -} while(0) +} while (0) -#define wrmsrl(msr,val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) -#define wrmsr_safe(msr,a,b) paravirt_write_msr(msr, a, b) +#define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) +#define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b) /* rdmsr with exception handling */ -#define rdmsr_safe(msr,a,b) ({ \ +#define rdmsr_safe(msr, a, b) \ +({ \ int _err; \ u64 _l = paravirt_read_msr(msr, &_err); \ (*a) = (u32)_l; \ (*b) = _l >> 32; \ - _err; }) + _err; \ +}) +static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) +{ + int err; + + *p = paravirt_read_msr(msr, &err); + return err; +} static inline u64 paravirt_read_tsc(void) { return PVOP_CALL0(u64, pv_cpu_ops.read_tsc); } -#define rdtscl(low) do { \ +#define rdtscl(low) \ +do { \ u64 _l = paravirt_read_tsc(); \ low = (int)_l; \ -} while(0) +} while (0) #define rdtscll(val) (val = paravirt_read_tsc()) @@ -692,11 +725,12 @@ static inline unsigned long long paravirt_read_pmc(int counter) return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); } -#define rdpmc(counter,low,high) do { \ +#define rdpmc(counter, low, high) \ +do { \ u64 _l = paravirt_read_pmc(counter); \ low = (u32)_l; \ high = _l >> 32; \ -} while(0) +} while (0) static inline unsigned long long paravirt_rdtscp(unsigned int *aux) { @@ -775,7 +809,8 @@ static inline void set_iopl_mask(unsigned mask) } /* The paravirtualized I/O functions */ -static inline void slow_down_io(void) { +static inline void slow_down_io(void) +{ pv_cpu_ops.io_delay(); #ifdef REALLY_SLOW_IO pv_cpu_ops.io_delay(); @@ -884,9 +919,9 @@ static inline void paravirt_release_pt(unsigned pfn) PVOP_VCALL1(pv_mmu_ops.release_pt, pfn); } -static inline void paravirt_alloc_pd(unsigned pfn) +static inline void paravirt_alloc_pd(struct mm_struct *mm, unsigned pfn) { - PVOP_VCALL1(pv_mmu_ops.alloc_pd, pfn); + PVOP_VCALL2(pv_mmu_ops.alloc_pd, mm, pfn); } static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, @@ -978,94 +1013,158 @@ static inline pgdval_t pgd_val(pgd_t pgd) return ret; } -#ifdef CONFIG_X86_PAE -static inline pmd_t __pmd(unsigned long long val) +static inline void set_pte(pte_t *ptep, pte_t pte) { - return (pmd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pmd, - val, val >> 32) }; + if (sizeof(pteval_t) > sizeof(long)) + PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, + pte.pte, (u64)pte.pte >> 32); + else + PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, + pte.pte); } -static inline unsigned long long pmd_val(pmd_t x) +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) { - return PVOP_CALL2(unsigned long long, pv_mmu_ops.pmd_val, - x.pmd, x.pmd >> 32); + if (sizeof(pteval_t) > sizeof(long)) + /* 5 arg words */ + pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); + else + PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); } -static inline void set_pte(pte_t *ptep, pte_t pteval) +static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { - PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, pteval.pte_low, pteval.pte_high); + pmdval_t val = native_pmd_val(pmd); + + if (sizeof(pmdval_t) > sizeof(long)) + PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); + else + PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); } -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval) +#if PAGETABLE_LEVELS >= 3 +static inline pmd_t __pmd(pmdval_t val) { - /* 5 arg words */ - pv_mmu_ops.set_pte_at(mm, addr, ptep, pteval); + pmdval_t ret; + + if (sizeof(pmdval_t) > sizeof(long)) + ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.make_pmd, + val, (u64)val >> 32); + else + ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.make_pmd, + val); + + return (pmd_t) { ret }; } -static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) +static inline pmdval_t pmd_val(pmd_t pmd) { - PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, - pteval.pte_low, pteval.pte_high); + pmdval_t ret; + + if (sizeof(pmdval_t) > sizeof(long)) + ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.pmd_val, + pmd.pmd, (u64)pmd.pmd >> 32); + else + ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.pmd_val, + pmd.pmd); + + return ret; } -static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte) +static inline void set_pud(pud_t *pudp, pud_t pud) { - /* 5 arg words */ - pv_mmu_ops.set_pte_present(mm, addr, ptep, pte); -} + pudval_t val = native_pud_val(pud); -static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) + if (sizeof(pudval_t) > sizeof(long)) + PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, + val, (u64)val >> 32); + else + PVOP_VCALL2(pv_mmu_ops.set_pud, pudp, + val); +} +#if PAGETABLE_LEVELS == 4 +static inline pud_t __pud(pudval_t val) { - PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, - pmdval.pmd, pmdval.pmd >> 32); + pudval_t ret; + + if (sizeof(pudval_t) > sizeof(long)) + ret = PVOP_CALL2(pudval_t, pv_mmu_ops.make_pud, + val, (u64)val >> 32); + else + ret = PVOP_CALL1(pudval_t, pv_mmu_ops.make_pud, + val); + + return (pud_t) { ret }; } -static inline void set_pud(pud_t *pudp, pud_t pudval) +static inline pudval_t pud_val(pud_t pud) { - PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, - pudval.pgd.pgd, pudval.pgd.pgd >> 32); + pudval_t ret; + + if (sizeof(pudval_t) > sizeof(long)) + ret = PVOP_CALL2(pudval_t, pv_mmu_ops.pud_val, + pud.pud, (u64)pud.pud >> 32); + else + ret = PVOP_CALL1(pudval_t, pv_mmu_ops.pud_val, + pud.pud); + + return ret; } -static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { - PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); + pgdval_t val = native_pgd_val(pgd); + + if (sizeof(pgdval_t) > sizeof(long)) + PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp, + val, (u64)val >> 32); + else + PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, + val); } -static inline void pmd_clear(pmd_t *pmdp) +static inline void pgd_clear(pgd_t *pgdp) { - PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); + set_pgd(pgdp, __pgd(0)); } -#else /* !CONFIG_X86_PAE */ - -static inline void set_pte(pte_t *ptep, pte_t pteval) +static inline void pud_clear(pud_t *pudp) { - PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, pteval.pte_low); + set_pud(pudp, __pud(0)); } -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval) +#endif /* PAGETABLE_LEVELS == 4 */ + +#endif /* PAGETABLE_LEVELS >= 3 */ + +#ifdef CONFIG_X86_PAE +/* Special-case pte-setting operations for PAE, which can't update a + 64-bit pte atomically */ +static inline void set_pte_atomic(pte_t *ptep, pte_t pte) { - PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pteval.pte_low); + PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, + pte.pte, pte.pte >> 32); } -static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) +static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) { - PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, pmdval.pud.pgd.pgd); + /* 5 arg words */ + pv_mmu_ops.set_pte_present(mm, addr, ptep, pte); } -static inline void pmd_clear(pmd_t *pmdp) +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { - set_pmd(pmdp, __pmd(0)); + PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); } -static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +static inline void pmd_clear(pmd_t *pmdp) { - set_pte_at(mm, addr, ptep, __pte(0)); + PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); } - +#else /* !CONFIG_X86_PAE */ static inline void set_pte_atomic(pte_t *ptep, pte_t pte) { set_pte(ptep, pte); @@ -1076,6 +1175,17 @@ static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, { set_pte(ptep, pte); } + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) +{ + set_pte_at(mm, addr, ptep, __pte(0)); +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(0)); +} #endif /* CONFIG_X86_PAE */ /* Lazy mode for batching updates / context switch */