x86: add PAT related debug prints
[linux-2.6] / arch / x86 / mm / ioremap.c
1 /*
2  * Re-map IO memory to kernel address space so that we can access it.
3  * This is needed for high PCI addresses that aren't mapped in the
4  * 640k-1MB IO memory area on PC's
5  *
6  * (C) Copyright 1995 1996 Linus Torvalds
7  */
8
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
11 #include <linux/io.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
15
16 #include <asm/cacheflush.h>
17 #include <asm/e820.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
22 #include <asm/pat.h>
23
24 #ifdef CONFIG_X86_64
25
26 unsigned long __phys_addr(unsigned long x)
27 {
28         if (x >= __START_KERNEL_map)
29                 return x - __START_KERNEL_map + phys_base;
30         return x - PAGE_OFFSET;
31 }
32 EXPORT_SYMBOL(__phys_addr);
33
34 static inline int phys_addr_valid(unsigned long addr)
35 {
36         return addr < (1UL << boot_cpu_data.x86_phys_bits);
37 }
38
39 #else
40
41 static inline int phys_addr_valid(unsigned long addr)
42 {
43         return 1;
44 }
45
46 #endif
47
48 int page_is_ram(unsigned long pagenr)
49 {
50         unsigned long addr, end;
51         int i;
52
53         /*
54          * A special case is the first 4Kb of memory;
55          * This is a BIOS owned area, not kernel ram, but generally
56          * not listed as such in the E820 table.
57          */
58         if (pagenr == 0)
59                 return 0;
60
61         /*
62          * Second special case: Some BIOSen report the PC BIOS
63          * area (640->1Mb) as ram even though it is not.
64          */
65         if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
66                     pagenr < (BIOS_END >> PAGE_SHIFT))
67                 return 0;
68
69         for (i = 0; i < e820.nr_map; i++) {
70                 /*
71                  * Not usable memory:
72                  */
73                 if (e820.map[i].type != E820_RAM)
74                         continue;
75                 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
76                 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
77
78
79                 if ((pagenr >= addr) && (pagenr < end))
80                         return 1;
81         }
82         return 0;
83 }
84
85 /*
86  * Fix up the linear direct mapping of the kernel to avoid cache attribute
87  * conflicts.
88  */
89 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
90                                unsigned long prot_val)
91 {
92         unsigned long nrpages = size >> PAGE_SHIFT;
93         int err;
94
95         switch (prot_val) {
96         case _PAGE_CACHE_UC:
97         default:
98                 err = _set_memory_uc(vaddr, nrpages);
99                 break;
100         case _PAGE_CACHE_WC:
101                 err = _set_memory_wc(vaddr, nrpages);
102                 break;
103         case _PAGE_CACHE_WB:
104                 err = _set_memory_wb(vaddr, nrpages);
105                 break;
106         }
107
108         return err;
109 }
110
111 /*
112  * Remap an arbitrary physical address space into the kernel virtual
113  * address space. Needed when the kernel wants to access high addresses
114  * directly.
115  *
116  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
117  * have to convert them into an offset in a page-aligned mapping, but the
118  * caller shouldn't need to know that small detail.
119  */
120 static void __iomem *__ioremap(resource_size_t phys_addr, unsigned long size,
121                                unsigned long prot_val)
122 {
123         unsigned long pfn, offset, last_addr, vaddr;
124         struct vm_struct *area;
125         unsigned long new_prot_val;
126         pgprot_t prot;
127
128         /* Don't allow wraparound or zero size */
129         last_addr = phys_addr + size - 1;
130         if (!size || last_addr < phys_addr)
131                 return NULL;
132
133         if (!phys_addr_valid(phys_addr)) {
134                 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
135                        phys_addr);
136                 WARN_ON_ONCE(1);
137                 return NULL;
138         }
139
140         /*
141          * Don't remap the low PCI/ISA area, it's always mapped..
142          */
143         if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
144                 return (__force void __iomem *)phys_to_virt(phys_addr);
145
146         /*
147          * Don't allow anybody to remap normal RAM that we're using..
148          */
149         for (pfn = phys_addr >> PAGE_SHIFT;
150                                 (pfn << PAGE_SHIFT) < last_addr; pfn++) {
151
152                 int is_ram = page_is_ram(pfn);
153
154                 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
155                         return NULL;
156                 WARN_ON_ONCE(is_ram);
157         }
158
159         /*
160          * Mappings have to be page-aligned
161          */
162         offset = phys_addr & ~PAGE_MASK;
163         phys_addr &= PAGE_MASK;
164         size = PAGE_ALIGN(last_addr+1) - phys_addr;
165
166         if (reserve_memtype(phys_addr, phys_addr + size,
167                             prot_val, &new_prot_val)) {
168                 /*
169                  * Do not fallback to certain memory types with certain
170                  * requested type:
171                  * - request is uncached, return cannot be write-back
172                  * - request is uncached, return cannot be write-combine
173                  * - request is write-combine, return cannot be write-back
174                  */
175                 if ((prot_val == _PAGE_CACHE_UC &&
176                      (new_prot_val == _PAGE_CACHE_WB ||
177                       new_prot_val == _PAGE_CACHE_WC)) ||
178                     (prot_val == _PAGE_CACHE_WC &&
179                      new_prot_val == _PAGE_CACHE_WB)) {
180                         printk(
181                 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
182                                 phys_addr, phys_addr + size,
183                                 prot_val, new_prot_val);
184                         free_memtype(phys_addr, phys_addr + size);
185                         return NULL;
186                 }
187                 prot_val = new_prot_val;
188         }
189
190         switch (prot_val) {
191         case _PAGE_CACHE_UC:
192         default:
193                 prot = PAGE_KERNEL_NOCACHE;
194                 break;
195         case _PAGE_CACHE_WC:
196                 prot = PAGE_KERNEL_WC;
197                 break;
198         case _PAGE_CACHE_WB:
199                 prot = PAGE_KERNEL;
200                 break;
201         }
202
203         /*
204          * Ok, go for it..
205          */
206         area = get_vm_area(size, VM_IOREMAP);
207         if (!area)
208                 return NULL;
209         area->phys_addr = phys_addr;
210         vaddr = (unsigned long) area->addr;
211         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
212                 free_memtype(phys_addr, phys_addr + size);
213                 free_vm_area(area);
214                 return NULL;
215         }
216
217         if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
218                 free_memtype(phys_addr, phys_addr + size);
219                 vunmap(area->addr);
220                 return NULL;
221         }
222
223         return (void __iomem *) (vaddr + offset);
224 }
225
226 /**
227  * ioremap_nocache     -   map bus memory into CPU space
228  * @offset:    bus address of the memory
229  * @size:      size of the resource to map
230  *
231  * ioremap_nocache performs a platform specific sequence of operations to
232  * make bus memory CPU accessible via the readb/readw/readl/writeb/
233  * writew/writel functions and the other mmio helpers. The returned
234  * address is not guaranteed to be usable directly as a virtual
235  * address.
236  *
237  * This version of ioremap ensures that the memory is marked uncachable
238  * on the CPU as well as honouring existing caching rules from things like
239  * the PCI bus. Note that there are other caches and buffers on many
240  * busses. In particular driver authors should read up on PCI writes
241  *
242  * It's useful if some control registers are in such an area and
243  * write combining or read caching is not desirable:
244  *
245  * Must be freed with iounmap.
246  */
247 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
248 {
249         return __ioremap(phys_addr, size, _PAGE_CACHE_UC);
250 }
251 EXPORT_SYMBOL(ioremap_nocache);
252
253 /**
254  * ioremap_wc   -       map memory into CPU space write combined
255  * @offset:     bus address of the memory
256  * @size:       size of the resource to map
257  *
258  * This version of ioremap ensures that the memory is marked write combining.
259  * Write combining allows faster writes to some hardware devices.
260  *
261  * Must be freed with iounmap.
262  */
263 void __iomem *ioremap_wc(unsigned long phys_addr, unsigned long size)
264 {
265         if (pat_wc_enabled)
266                 return __ioremap(phys_addr, size, _PAGE_CACHE_WC);
267         else
268                 return ioremap_nocache(phys_addr, size);
269 }
270 EXPORT_SYMBOL(ioremap_wc);
271
272 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
273 {
274         return __ioremap(phys_addr, size, _PAGE_CACHE_WB);
275 }
276 EXPORT_SYMBOL(ioremap_cache);
277
278 /**
279  * iounmap - Free a IO remapping
280  * @addr: virtual address from ioremap_*
281  *
282  * Caller must ensure there is only one unmapping for the same pointer.
283  */
284 void iounmap(volatile void __iomem *addr)
285 {
286         struct vm_struct *p, *o;
287
288         if ((void __force *)addr <= high_memory)
289                 return;
290
291         /*
292          * __ioremap special-cases the PCI/ISA range by not instantiating a
293          * vm_area and by simply returning an address into the kernel mapping
294          * of ISA space.   So handle that here.
295          */
296         if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
297             addr < phys_to_virt(ISA_END_ADDRESS))
298                 return;
299
300         addr = (volatile void __iomem *)
301                 (PAGE_MASK & (unsigned long __force)addr);
302
303         /* Use the vm area unlocked, assuming the caller
304            ensures there isn't another iounmap for the same address
305            in parallel. Reuse of the virtual address is prevented by
306            leaving it in the global lists until we're done with it.
307            cpa takes care of the direct mappings. */
308         read_lock(&vmlist_lock);
309         for (p = vmlist; p; p = p->next) {
310                 if (p->addr == addr)
311                         break;
312         }
313         read_unlock(&vmlist_lock);
314
315         if (!p) {
316                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
317                 dump_stack();
318                 return;
319         }
320
321         free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
322
323         /* Finally remove it */
324         o = remove_vm_area((void *)addr);
325         BUG_ON(p != o || o == NULL);
326         kfree(p);
327 }
328 EXPORT_SYMBOL(iounmap);
329
330 #ifdef CONFIG_X86_32
331
332 int __initdata early_ioremap_debug;
333
334 static int __init early_ioremap_debug_setup(char *str)
335 {
336         early_ioremap_debug = 1;
337
338         return 0;
339 }
340 early_param("early_ioremap_debug", early_ioremap_debug_setup);
341
342 static __initdata int after_paging_init;
343 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
344                 __section(.bss.page_aligned);
345
346 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
347 {
348         /* Don't assume we're using swapper_pg_dir at this point */
349         pgd_t *base = __va(read_cr3());
350         pgd_t *pgd = &base[pgd_index(addr)];
351         pud_t *pud = pud_offset(pgd, addr);
352         pmd_t *pmd = pmd_offset(pud, addr);
353
354         return pmd;
355 }
356
357 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
358 {
359         return &bm_pte[pte_index(addr)];
360 }
361
362 void __init early_ioremap_init(void)
363 {
364         pmd_t *pmd;
365
366         if (early_ioremap_debug)
367                 printk(KERN_INFO "early_ioremap_init()\n");
368
369         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
370         memset(bm_pte, 0, sizeof(bm_pte));
371         pmd_populate_kernel(&init_mm, pmd, bm_pte);
372
373         /*
374          * The boot-ioremap range spans multiple pmds, for which
375          * we are not prepared:
376          */
377         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
378                 WARN_ON(1);
379                 printk(KERN_WARNING "pmd %p != %p\n",
380                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
381                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
382                         fix_to_virt(FIX_BTMAP_BEGIN));
383                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
384                         fix_to_virt(FIX_BTMAP_END));
385
386                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
387                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
388                        FIX_BTMAP_BEGIN);
389         }
390 }
391
392 void __init early_ioremap_clear(void)
393 {
394         pmd_t *pmd;
395
396         if (early_ioremap_debug)
397                 printk(KERN_INFO "early_ioremap_clear()\n");
398
399         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
400         pmd_clear(pmd);
401         paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
402         __flush_tlb_all();
403 }
404
405 void __init early_ioremap_reset(void)
406 {
407         enum fixed_addresses idx;
408         unsigned long addr, phys;
409         pte_t *pte;
410
411         after_paging_init = 1;
412         for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
413                 addr = fix_to_virt(idx);
414                 pte = early_ioremap_pte(addr);
415                 if (pte_present(*pte)) {
416                         phys = pte_val(*pte) & PAGE_MASK;
417                         set_fixmap(idx, phys);
418                 }
419         }
420 }
421
422 static void __init __early_set_fixmap(enum fixed_addresses idx,
423                                    unsigned long phys, pgprot_t flags)
424 {
425         unsigned long addr = __fix_to_virt(idx);
426         pte_t *pte;
427
428         if (idx >= __end_of_fixed_addresses) {
429                 BUG();
430                 return;
431         }
432         pte = early_ioremap_pte(addr);
433         if (pgprot_val(flags))
434                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
435         else
436                 pte_clear(NULL, addr, pte);
437         __flush_tlb_one(addr);
438 }
439
440 static inline void __init early_set_fixmap(enum fixed_addresses idx,
441                                         unsigned long phys)
442 {
443         if (after_paging_init)
444                 set_fixmap(idx, phys);
445         else
446                 __early_set_fixmap(idx, phys, PAGE_KERNEL);
447 }
448
449 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
450 {
451         if (after_paging_init)
452                 clear_fixmap(idx);
453         else
454                 __early_set_fixmap(idx, 0, __pgprot(0));
455 }
456
457
458 int __initdata early_ioremap_nested;
459
460 static int __init check_early_ioremap_leak(void)
461 {
462         if (!early_ioremap_nested)
463                 return 0;
464
465         printk(KERN_WARNING
466                "Debug warning: early ioremap leak of %d areas detected.\n",
467                early_ioremap_nested);
468         printk(KERN_WARNING
469                "please boot with early_ioremap_debug and report the dmesg.\n");
470         WARN_ON(1);
471
472         return 1;
473 }
474 late_initcall(check_early_ioremap_leak);
475
476 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
477 {
478         unsigned long offset, last_addr;
479         unsigned int nrpages, nesting;
480         enum fixed_addresses idx0, idx;
481
482         WARN_ON(system_state != SYSTEM_BOOTING);
483
484         nesting = early_ioremap_nested;
485         if (early_ioremap_debug) {
486                 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
487                        phys_addr, size, nesting);
488                 dump_stack();
489         }
490
491         /* Don't allow wraparound or zero size */
492         last_addr = phys_addr + size - 1;
493         if (!size || last_addr < phys_addr) {
494                 WARN_ON(1);
495                 return NULL;
496         }
497
498         if (nesting >= FIX_BTMAPS_NESTING) {
499                 WARN_ON(1);
500                 return NULL;
501         }
502         early_ioremap_nested++;
503         /*
504          * Mappings have to be page-aligned
505          */
506         offset = phys_addr & ~PAGE_MASK;
507         phys_addr &= PAGE_MASK;
508         size = PAGE_ALIGN(last_addr) - phys_addr;
509
510         /*
511          * Mappings have to fit in the FIX_BTMAP area.
512          */
513         nrpages = size >> PAGE_SHIFT;
514         if (nrpages > NR_FIX_BTMAPS) {
515                 WARN_ON(1);
516                 return NULL;
517         }
518
519         /*
520          * Ok, go for it..
521          */
522         idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
523         idx = idx0;
524         while (nrpages > 0) {
525                 early_set_fixmap(idx, phys_addr);
526                 phys_addr += PAGE_SIZE;
527                 --idx;
528                 --nrpages;
529         }
530         if (early_ioremap_debug)
531                 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
532
533         return (void *) (offset + fix_to_virt(idx0));
534 }
535
536 void __init early_iounmap(void *addr, unsigned long size)
537 {
538         unsigned long virt_addr;
539         unsigned long offset;
540         unsigned int nrpages;
541         enum fixed_addresses idx;
542         unsigned int nesting;
543
544         nesting = --early_ioremap_nested;
545         WARN_ON(nesting < 0);
546
547         if (early_ioremap_debug) {
548                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
549                        size, nesting);
550                 dump_stack();
551         }
552
553         virt_addr = (unsigned long)addr;
554         if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
555                 WARN_ON(1);
556                 return;
557         }
558         offset = virt_addr & ~PAGE_MASK;
559         nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
560
561         idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
562         while (nrpages > 0) {
563                 early_clear_fixmap(idx);
564                 --idx;
565                 --nrpages;
566         }
567 }
568
569 void __this_fixmap_does_not_exist(void)
570 {
571         WARN_ON(1);
572 }
573
574 #endif /* CONFIG_X86_32 */