x86: add gbpages support to lookup_address

[linux-2.6] / arch / x86 / mm / pageattr.c

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 40b7ac58e671ff8f39f70a2e051671f1564d4de5..143fbafc948ae25c9c8added4a1a7534c0521d9f 100644 (file)
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -119,7 +119,7 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache)
                /*
                 * Only flush present addresses:
                 */
-               if (pte && pte_present(*pte))
+               if (pte && (pte_val(*pte) & _PAGE_PRESENT))
                        clflush_cache_range((void *) addr, PAGE_SIZE);
        }
 }
@@ -188,6 +188,14 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long address)
        return prot;
 }
 
+/*
+ * Lookup the page table entry for a virtual address. Return a pointer
+ * to the entry and the level of the mapping.
+ *
+ * Note: We return pud and pmd either when the entry is marked large
+ * or when the present bit is not set. Otherwise we would return a
+ * pointer to a nonexisting mapping.
+ */
 pte_t *lookup_address(unsigned long address, int *level)
 {
        pgd_t *pgd = pgd_offset_k(address);
@@ -201,12 +209,17 @@ pte_t *lookup_address(unsigned long address, int *level)
        pud = pud_offset(pgd, address);
        if (pud_none(*pud))
                return NULL;
+
+       *level = PG_LEVEL_1G;
+       if (pud_large(*pud) || !pud_present(*pud))
+               return (pte_t *)pud;
+
        pmd = pmd_offset(pud, address);
        if (pmd_none(*pmd))
                return NULL;
 
        *level = PG_LEVEL_2M;
-       if (pmd_large(*pmd))
+       if (pmd_large(*pmd) || !pmd_present(*pmd))
                return (pte_t *)pmd;
 
        *level = PG_LEVEL_4K;
@@ -243,6 +256,17 @@ static int try_preserve_large_page(pte_t *kpte, unsigned long address,
        pgprot_t old_prot, new_prot;
        int level, res = CPA_SPLIT;
 
+       /*
+        * An Athlon 64 X2 showed hard hangs if we tried to preserve
+        * largepages and changed the PSE entry from RW to RO.
+        *
+        * As AMD CPUs have a long series of erratas in this area,
+        * (and none of the known ones seem to explain this hang),
+        * disable this code until the hang can be debugged:
+        */
+       if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
+               return res;
+
        spin_lock_irqsave(&pgd_lock, flags);
        /*
         * Check for races, another CPU might have split this page
@@ -254,8 +278,8 @@ static int try_preserve_large_page(pte_t *kpte, unsigned long address,
 
        switch (level) {
        case PG_LEVEL_2M:
-               psize = LARGE_PAGE_SIZE;
-               pmask = LARGE_PAGE_MASK;
+               psize = PMD_PAGE_SIZE;
+               pmask = PMD_PAGE_MASK;
                break;
        case PG_LEVEL_1G:
        default:
@@ -317,7 +341,7 @@ out_unlock:
 
 static int split_large_page(pte_t *kpte, unsigned long address)
 {
-       pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
+       pgprot_t ref_prot;
        gfp_t gfp_flags = GFP_KERNEL;
        unsigned long flags, addr, pfn;
        pte_t *pbase, *tmp;
@@ -325,7 +349,6 @@ static int split_large_page(pte_t *kpte, unsigned long address)
        unsigned int i, level;
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
-       gfp_flags = __GFP_HIGH | __GFP_NOFAIL | __GFP_NOWARN;
        gfp_flags = GFP_ATOMIC | __GFP_NOWARN;
 #endif
        base = alloc_pages(gfp_flags, 0);
@@ -338,17 +361,16 @@ static int split_large_page(pte_t *kpte, unsigned long address)
         * up for us already:
         */
        tmp = lookup_address(address, &level);
-       if (tmp != kpte) {
-               WARN_ON_ONCE(1);
+       if (tmp != kpte)
                goto out_unlock;
-       }
 
        address = __pa(address);
-       addr = address & LARGE_PAGE_MASK;
+       addr = address & PMD_PAGE_MASK;
        pbase = (pte_t *)page_address(base);
 #ifdef CONFIG_X86_32
        paravirt_alloc_pt(&init_mm, page_to_pfn(base));
 #endif
+       ref_prot = pte_pgprot(pte_clrhuge(*kpte));
 
        /*
         * Get the target pfn from the original entry:
@@ -358,13 +380,17 @@ static int split_large_page(pte_t *kpte, unsigned long address)
                set_pte(&pbase[i], pfn_pte(pfn, ref_prot));
 
        /*
-        * Install the new, split up pagetable. Important detail here:
+        * Install the new, split up pagetable. Important details here:
         *
         * On Intel the NX bit of all levels must be cleared to make a
         * page executable. See section 4.13.2 of Intel 64 and IA-32
         * Architectures Software Developer's Manual).
+        *
+        * Mark the entry present. The current mapping might be
+        * set to not present, which we preserved above.
         */
        ref_prot = pte_pgprot(pte_mkexec(pte_clrhuge(*kpte)));
+       pgprot_val(ref_prot) |= _PAGE_PRESENT;
        __set_pmd_pte(kpte, address, mk_pte(base, ref_prot));
        base = NULL;