2 * Re-map IO memory to kernel address space so that we can access it.
3 * This is needed for high PCI addresses that aren't mapped in the
4 * 640k-1MB IO memory area on PC's
6 * (C) Copyright 1995 1996 Linus Torvalds
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
16 #include <asm/cacheflush.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
26 unsigned long __phys_addr(unsigned long x)
28 if (x >= __START_KERNEL_map)
29 return x - __START_KERNEL_map + phys_base;
30 return x - PAGE_OFFSET;
32 EXPORT_SYMBOL(__phys_addr);
34 static inline int phys_addr_valid(unsigned long addr)
36 return addr < (1UL << boot_cpu_data.x86_phys_bits);
41 static inline int phys_addr_valid(unsigned long addr)
48 int page_is_ram(unsigned long pagenr)
50 unsigned long addr, end;
54 * A special case is the first 4Kb of memory;
55 * This is a BIOS owned area, not kernel ram, but generally
56 * not listed as such in the E820 table.
62 * Second special case: Some BIOSen report the PC BIOS
63 * area (640->1Mb) as ram even though it is not.
65 if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
66 pagenr < (BIOS_END >> PAGE_SHIFT))
69 for (i = 0; i < e820.nr_map; i++) {
73 if (e820.map[i].type != E820_RAM)
75 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
76 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
79 if ((pagenr >= addr) && (pagenr < end))
86 * Fix up the linear direct mapping of the kernel to avoid cache attribute
89 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
90 unsigned long prot_val)
92 unsigned long nrpages = size >> PAGE_SHIFT;
98 err = set_memory_uc(vaddr, nrpages);
101 err = set_memory_wb(vaddr, nrpages);
109 * Remap an arbitrary physical address space into the kernel virtual
110 * address space. Needed when the kernel wants to access high addresses
113 * NOTE! We need to allow non-page-aligned mappings too: we will obviously
114 * have to convert them into an offset in a page-aligned mapping, but the
115 * caller shouldn't need to know that small detail.
117 static void __iomem *__ioremap(resource_size_t phys_addr, unsigned long size,
118 unsigned long prot_val)
120 unsigned long pfn, offset, last_addr, vaddr;
121 struct vm_struct *area;
122 unsigned long new_prot_val;
125 /* Don't allow wraparound or zero size */
126 last_addr = phys_addr + size - 1;
127 if (!size || last_addr < phys_addr)
130 if (!phys_addr_valid(phys_addr)) {
131 printk(KERN_WARNING "ioremap: invalid physical address %lx\n",
138 * Don't remap the low PCI/ISA area, it's always mapped..
140 if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
141 return (__force void __iomem *)phys_to_virt(phys_addr);
144 * Don't allow anybody to remap normal RAM that we're using..
146 for (pfn = phys_addr >> PAGE_SHIFT;
147 (pfn << PAGE_SHIFT) < last_addr; pfn++) {
149 int is_ram = page_is_ram(pfn);
151 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
153 WARN_ON_ONCE(is_ram);
157 * Mappings have to be page-aligned
159 offset = phys_addr & ~PAGE_MASK;
160 phys_addr &= PAGE_MASK;
161 size = PAGE_ALIGN(last_addr+1) - phys_addr;
163 if (reserve_memtype(phys_addr, phys_addr + size,
164 prot_val, &new_prot_val)) {
166 * Do not fallback to certain memory types with certain
168 * - request is uncached, return cannot be write-back
170 if ((prot_val == _PAGE_CACHE_UC &&
171 new_prot_val == _PAGE_CACHE_WB)) {
172 free_memtype(phys_addr, phys_addr + size);
175 prot_val = new_prot_val;
181 prot = PAGE_KERNEL_NOCACHE;
191 area = get_vm_area(size, VM_IOREMAP);
194 area->phys_addr = phys_addr;
195 vaddr = (unsigned long) area->addr;
196 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
197 free_memtype(phys_addr, phys_addr + size);
202 if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
203 free_memtype(phys_addr, phys_addr + size);
208 return (void __iomem *) (vaddr + offset);
212 * ioremap_nocache - map bus memory into CPU space
213 * @offset: bus address of the memory
214 * @size: size of the resource to map
216 * ioremap_nocache performs a platform specific sequence of operations to
217 * make bus memory CPU accessible via the readb/readw/readl/writeb/
218 * writew/writel functions and the other mmio helpers. The returned
219 * address is not guaranteed to be usable directly as a virtual
222 * This version of ioremap ensures that the memory is marked uncachable
223 * on the CPU as well as honouring existing caching rules from things like
224 * the PCI bus. Note that there are other caches and buffers on many
225 * busses. In particular driver authors should read up on PCI writes
227 * It's useful if some control registers are in such an area and
228 * write combining or read caching is not desirable:
230 * Must be freed with iounmap.
232 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
234 return __ioremap(phys_addr, size, _PAGE_CACHE_UC);
236 EXPORT_SYMBOL(ioremap_nocache);
238 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
240 return __ioremap(phys_addr, size, _PAGE_CACHE_WB);
242 EXPORT_SYMBOL(ioremap_cache);
245 * iounmap - Free a IO remapping
246 * @addr: virtual address from ioremap_*
248 * Caller must ensure there is only one unmapping for the same pointer.
250 void iounmap(volatile void __iomem *addr)
252 struct vm_struct *p, *o;
254 if ((void __force *)addr <= high_memory)
258 * __ioremap special-cases the PCI/ISA range by not instantiating a
259 * vm_area and by simply returning an address into the kernel mapping
260 * of ISA space. So handle that here.
262 if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
263 addr < phys_to_virt(ISA_END_ADDRESS))
266 addr = (volatile void __iomem *)
267 (PAGE_MASK & (unsigned long __force)addr);
269 /* Use the vm area unlocked, assuming the caller
270 ensures there isn't another iounmap for the same address
271 in parallel. Reuse of the virtual address is prevented by
272 leaving it in the global lists until we're done with it.
273 cpa takes care of the direct mappings. */
274 read_lock(&vmlist_lock);
275 for (p = vmlist; p; p = p->next) {
279 read_unlock(&vmlist_lock);
282 printk(KERN_ERR "iounmap: bad address %p\n", addr);
287 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
289 /* Finally remove it */
290 o = remove_vm_area((void *)addr);
291 BUG_ON(p != o || o == NULL);
294 EXPORT_SYMBOL(iounmap);
298 int __initdata early_ioremap_debug;
300 static int __init early_ioremap_debug_setup(char *str)
302 early_ioremap_debug = 1;
306 early_param("early_ioremap_debug", early_ioremap_debug_setup);
308 static __initdata int after_paging_init;
309 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
310 __section(.bss.page_aligned);
312 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
314 /* Don't assume we're using swapper_pg_dir at this point */
315 pgd_t *base = __va(read_cr3());
316 pgd_t *pgd = &base[pgd_index(addr)];
317 pud_t *pud = pud_offset(pgd, addr);
318 pmd_t *pmd = pmd_offset(pud, addr);
323 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
325 return &bm_pte[pte_index(addr)];
328 void __init early_ioremap_init(void)
332 if (early_ioremap_debug)
333 printk(KERN_INFO "early_ioremap_init()\n");
335 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
336 memset(bm_pte, 0, sizeof(bm_pte));
337 pmd_populate_kernel(&init_mm, pmd, bm_pte);
340 * The boot-ioremap range spans multiple pmds, for which
341 * we are not prepared:
343 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
345 printk(KERN_WARNING "pmd %p != %p\n",
346 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
347 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
348 fix_to_virt(FIX_BTMAP_BEGIN));
349 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
350 fix_to_virt(FIX_BTMAP_END));
352 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
353 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
358 void __init early_ioremap_clear(void)
362 if (early_ioremap_debug)
363 printk(KERN_INFO "early_ioremap_clear()\n");
365 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
367 paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
371 void __init early_ioremap_reset(void)
373 enum fixed_addresses idx;
374 unsigned long addr, phys;
377 after_paging_init = 1;
378 for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
379 addr = fix_to_virt(idx);
380 pte = early_ioremap_pte(addr);
381 if (pte_present(*pte)) {
382 phys = pte_val(*pte) & PAGE_MASK;
383 set_fixmap(idx, phys);
388 static void __init __early_set_fixmap(enum fixed_addresses idx,
389 unsigned long phys, pgprot_t flags)
391 unsigned long addr = __fix_to_virt(idx);
394 if (idx >= __end_of_fixed_addresses) {
398 pte = early_ioremap_pte(addr);
399 if (pgprot_val(flags))
400 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
402 pte_clear(NULL, addr, pte);
403 __flush_tlb_one(addr);
406 static inline void __init early_set_fixmap(enum fixed_addresses idx,
409 if (after_paging_init)
410 set_fixmap(idx, phys);
412 __early_set_fixmap(idx, phys, PAGE_KERNEL);
415 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
417 if (after_paging_init)
420 __early_set_fixmap(idx, 0, __pgprot(0));
424 int __initdata early_ioremap_nested;
426 static int __init check_early_ioremap_leak(void)
428 if (!early_ioremap_nested)
432 "Debug warning: early ioremap leak of %d areas detected.\n",
433 early_ioremap_nested);
435 "please boot with early_ioremap_debug and report the dmesg.\n");
440 late_initcall(check_early_ioremap_leak);
442 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
444 unsigned long offset, last_addr;
445 unsigned int nrpages, nesting;
446 enum fixed_addresses idx0, idx;
448 WARN_ON(system_state != SYSTEM_BOOTING);
450 nesting = early_ioremap_nested;
451 if (early_ioremap_debug) {
452 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
453 phys_addr, size, nesting);
457 /* Don't allow wraparound or zero size */
458 last_addr = phys_addr + size - 1;
459 if (!size || last_addr < phys_addr) {
464 if (nesting >= FIX_BTMAPS_NESTING) {
468 early_ioremap_nested++;
470 * Mappings have to be page-aligned
472 offset = phys_addr & ~PAGE_MASK;
473 phys_addr &= PAGE_MASK;
474 size = PAGE_ALIGN(last_addr) - phys_addr;
477 * Mappings have to fit in the FIX_BTMAP area.
479 nrpages = size >> PAGE_SHIFT;
480 if (nrpages > NR_FIX_BTMAPS) {
488 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
490 while (nrpages > 0) {
491 early_set_fixmap(idx, phys_addr);
492 phys_addr += PAGE_SIZE;
496 if (early_ioremap_debug)
497 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
499 return (void *) (offset + fix_to_virt(idx0));
502 void __init early_iounmap(void *addr, unsigned long size)
504 unsigned long virt_addr;
505 unsigned long offset;
506 unsigned int nrpages;
507 enum fixed_addresses idx;
508 unsigned int nesting;
510 nesting = --early_ioremap_nested;
511 WARN_ON(nesting < 0);
513 if (early_ioremap_debug) {
514 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
519 virt_addr = (unsigned long)addr;
520 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
524 offset = virt_addr & ~PAGE_MASK;
525 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
527 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
528 while (nrpages > 0) {
529 early_clear_fixmap(idx);
535 void __this_fixmap_does_not_exist(void)
540 #endif /* CONFIG_X86_32 */