-
-#include <asm/pgtable.h>
-#include <asm/tlbflush.h>
-
-/*
- * Called with mm->page_table_lock held to protect against other
- * threads/the swapper from ripping pte's out from under us.
- */
-
-static void sync_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
- unsigned long addr, unsigned long end)
-{
- pte_t *pte;
-
- pte = pte_offset_map(pmd, addr);
- do {
- unsigned long pfn;
- struct page *page;
-
- if (!pte_present(*pte))
- continue;
- if (!pte_maybe_dirty(*pte))
- continue;
- pfn = pte_pfn(*pte);
- if (!pfn_valid(pfn))
- continue;
- page = pfn_to_page(pfn);
- if (PageReserved(page))
- continue;
-
- if (ptep_clear_flush_dirty(vma, addr, pte) ||
- page_test_and_clear_dirty(page))
- set_page_dirty(page);
- } while (pte++, addr += PAGE_SIZE, addr != end);
- pte_unmap(pte - 1);
-}
-
-static inline void sync_pmd_range(struct vm_area_struct *vma, pud_t *pud,
- unsigned long addr, unsigned long end)
-{
- pmd_t *pmd;
- unsigned long next;
-
- pmd = pmd_offset(pud, addr);
- do {
- next = pmd_addr_end(addr, end);
- if (pmd_none_or_clear_bad(pmd))
- continue;
- sync_pte_range(vma, pmd, addr, next);
- } while (pmd++, addr = next, addr != end);
-}
-
-static inline void sync_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
- unsigned long addr, unsigned long end)
-{
- pud_t *pud;
- unsigned long next;
-
- pud = pud_offset(pgd, addr);
- do {
- next = pud_addr_end(addr, end);
- if (pud_none_or_clear_bad(pud))
- continue;
- sync_pmd_range(vma, pud, addr, next);
- } while (pud++, addr = next, addr != end);
-}
-
-static void sync_page_range(struct vm_area_struct *vma,
- unsigned long addr, unsigned long end)
-{
- struct mm_struct *mm = vma->vm_mm;
- pgd_t *pgd;
- unsigned long next;
-
- /* For hugepages we can't go walking the page table normally,
- * but that's ok, hugetlbfs is memory based, so we don't need
- * to do anything more on an msync() */
- if (is_vm_hugetlb_page(vma))
- return;
-
- BUG_ON(addr >= end);
- pgd = pgd_offset(mm, addr);
- flush_cache_range(vma, addr, end);
- spin_lock(&mm->page_table_lock);
- do {
- next = pgd_addr_end(addr, end);
- if (pgd_none_or_clear_bad(pgd))
- continue;
- sync_pud_range(vma, pgd, addr, next);
- } while (pgd++, addr = next, addr != end);
- spin_unlock(&mm->page_table_lock);
-}
-
-#ifdef CONFIG_PREEMPT
-static inline void filemap_sync(struct vm_area_struct *vma,
- unsigned long addr, unsigned long end)
-{
- const size_t chunk = 64 * 1024; /* bytes */
- unsigned long next;
-
- do {
- next = addr + chunk;
- if (next > end || next < addr)
- next = end;
- sync_page_range(vma, addr, next);
- cond_resched();
- } while (addr = next, addr != end);
-}
-#else
-static inline void filemap_sync(struct vm_area_struct *vma,
- unsigned long addr, unsigned long end)
-{
- sync_page_range(vma, addr, end);
-}
-#endif