memory hotplug x86_64: fix section mismatch in init_memory_mapping()
[linux-2.6] / arch / x86 / mm / init_64.c
1 /*
2  *  linux/arch/x86_64/mm/init.c
3  *
4  *  Copyright (C) 1995  Linus Torvalds
5  *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
6  *  Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
7  */
8
9 #include <linux/signal.h>
10 #include <linux/sched.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/types.h>
15 #include <linux/ptrace.h>
16 #include <linux/mman.h>
17 #include <linux/mm.h>
18 #include <linux/swap.h>
19 #include <linux/smp.h>
20 #include <linux/init.h>
21 #include <linux/pagemap.h>
22 #include <linux/bootmem.h>
23 #include <linux/proc_fs.h>
24 #include <linux/pci.h>
25 #include <linux/pfn.h>
26 #include <linux/poison.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/memory_hotplug.h>
30 #include <linux/nmi.h>
31
32 #include <asm/processor.h>
33 #include <asm/system.h>
34 #include <asm/uaccess.h>
35 #include <asm/pgtable.h>
36 #include <asm/pgalloc.h>
37 #include <asm/dma.h>
38 #include <asm/fixmap.h>
39 #include <asm/e820.h>
40 #include <asm/apic.h>
41 #include <asm/tlb.h>
42 #include <asm/mmu_context.h>
43 #include <asm/proto.h>
44 #include <asm/smp.h>
45 #include <asm/sections.h>
46
47 #ifndef Dprintk
48 #define Dprintk(x...)
49 #endif
50
51 const struct dma_mapping_ops* dma_ops;
52 EXPORT_SYMBOL(dma_ops);
53
54 static unsigned long dma_reserve __initdata;
55
56 DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
57
58 /*
59  * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
60  * physical space so we can cache the place of the first one and move
61  * around without checking the pgd every time.
62  */
63
64 void show_mem(void)
65 {
66         long i, total = 0, reserved = 0;
67         long shared = 0, cached = 0;
68         pg_data_t *pgdat;
69         struct page *page;
70
71         printk(KERN_INFO "Mem-info:\n");
72         show_free_areas();
73         printk(KERN_INFO "Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
74
75         for_each_online_pgdat(pgdat) {
76                for (i = 0; i < pgdat->node_spanned_pages; ++i) {
77                         /* this loop can take a while with 256 GB and 4k pages
78                            so update the NMI watchdog */
79                         if (unlikely(i % MAX_ORDER_NR_PAGES == 0)) {
80                                 touch_nmi_watchdog();
81                         }
82                         if (!pfn_valid(pgdat->node_start_pfn + i))
83                                 continue;
84                         page = pfn_to_page(pgdat->node_start_pfn + i);
85                         total++;
86                         if (PageReserved(page))
87                                 reserved++;
88                         else if (PageSwapCache(page))
89                                 cached++;
90                         else if (page_count(page))
91                                 shared += page_count(page) - 1;
92                }
93         }
94         printk(KERN_INFO "%lu pages of RAM\n", total);
95         printk(KERN_INFO "%lu reserved pages\n",reserved);
96         printk(KERN_INFO "%lu pages shared\n",shared);
97         printk(KERN_INFO "%lu pages swap cached\n",cached);
98 }
99
100 int after_bootmem;
101
102 static __init void *spp_getpage(void)
103
104         void *ptr;
105         if (after_bootmem)
106                 ptr = (void *) get_zeroed_page(GFP_ATOMIC); 
107         else
108                 ptr = alloc_bootmem_pages(PAGE_SIZE);
109         if (!ptr || ((unsigned long)ptr & ~PAGE_MASK))
110                 panic("set_pte_phys: cannot allocate page data %s\n", after_bootmem?"after bootmem":"");
111
112         Dprintk("spp_getpage %p\n", ptr);
113         return ptr;
114
115
116 static __init void set_pte_phys(unsigned long vaddr,
117                          unsigned long phys, pgprot_t prot)
118 {
119         pgd_t *pgd;
120         pud_t *pud;
121         pmd_t *pmd;
122         pte_t *pte, new_pte;
123
124         Dprintk("set_pte_phys %lx to %lx\n", vaddr, phys);
125
126         pgd = pgd_offset_k(vaddr);
127         if (pgd_none(*pgd)) {
128                 printk("PGD FIXMAP MISSING, it should be setup in head.S!\n");
129                 return;
130         }
131         pud = pud_offset(pgd, vaddr);
132         if (pud_none(*pud)) {
133                 pmd = (pmd_t *) spp_getpage(); 
134                 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
135                 if (pmd != pmd_offset(pud, 0)) {
136                         printk("PAGETABLE BUG #01! %p <-> %p\n", pmd, pmd_offset(pud,0));
137                         return;
138                 }
139         }
140         pmd = pmd_offset(pud, vaddr);
141         if (pmd_none(*pmd)) {
142                 pte = (pte_t *) spp_getpage();
143                 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
144                 if (pte != pte_offset_kernel(pmd, 0)) {
145                         printk("PAGETABLE BUG #02!\n");
146                         return;
147                 }
148         }
149         new_pte = pfn_pte(phys >> PAGE_SHIFT, prot);
150
151         pte = pte_offset_kernel(pmd, vaddr);
152         if (!pte_none(*pte) &&
153             pte_val(*pte) != (pte_val(new_pte) & __supported_pte_mask))
154                 pte_ERROR(*pte);
155         set_pte(pte, new_pte);
156
157         /*
158          * It's enough to flush this one mapping.
159          * (PGE mappings get flushed as well)
160          */
161         __flush_tlb_one(vaddr);
162 }
163
164 /* NOTE: this is meant to be run only at boot */
165 void __init 
166 __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
167 {
168         unsigned long address = __fix_to_virt(idx);
169
170         if (idx >= __end_of_fixed_addresses) {
171                 printk("Invalid __set_fixmap\n");
172                 return;
173         }
174         set_pte_phys(address, phys, prot);
175 }
176
177 unsigned long __meminitdata table_start, table_end;
178
179 static __meminit void *alloc_low_page(unsigned long *phys)
180
181         unsigned long pfn = table_end++;
182         void *adr;
183
184         if (after_bootmem) {
185                 adr = (void *)get_zeroed_page(GFP_ATOMIC);
186                 *phys = __pa(adr);
187                 return adr;
188         }
189
190         if (pfn >= end_pfn) 
191                 panic("alloc_low_page: ran out of memory"); 
192
193         adr = early_ioremap(pfn * PAGE_SIZE, PAGE_SIZE);
194         memset(adr, 0, PAGE_SIZE);
195         *phys  = pfn * PAGE_SIZE;
196         return adr;
197 }
198
199 static __meminit void unmap_low_page(void *adr)
200
201
202         if (after_bootmem)
203                 return;
204
205         early_iounmap(adr, PAGE_SIZE);
206
207
208 /* Must run before zap_low_mappings */
209 __meminit void *early_ioremap(unsigned long addr, unsigned long size)
210 {
211         unsigned long vaddr;
212         pmd_t *pmd, *last_pmd;
213         int i, pmds;
214
215         pmds = ((addr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
216         vaddr = __START_KERNEL_map;
217         pmd = level2_kernel_pgt;
218         last_pmd = level2_kernel_pgt + PTRS_PER_PMD - 1;
219         for (; pmd <= last_pmd; pmd++, vaddr += PMD_SIZE) {
220                 for (i = 0; i < pmds; i++) {
221                         if (pmd_present(pmd[i]))
222                                 goto next;
223                 }
224                 vaddr += addr & ~PMD_MASK;
225                 addr &= PMD_MASK;
226                 for (i = 0; i < pmds; i++, addr += PMD_SIZE)
227                         set_pmd(pmd + i,__pmd(addr | _KERNPG_TABLE | _PAGE_PSE));
228                 __flush_tlb();
229                 return (void *)vaddr;
230         next:
231                 ;
232         }
233         printk("early_ioremap(0x%lx, %lu) failed\n", addr, size);
234         return NULL;
235 }
236
237 /* To avoid virtual aliases later */
238 __meminit void early_iounmap(void *addr, unsigned long size)
239 {
240         unsigned long vaddr;
241         pmd_t *pmd;
242         int i, pmds;
243
244         vaddr = (unsigned long)addr;
245         pmds = ((vaddr & ~PMD_MASK) + size + ~PMD_MASK) / PMD_SIZE;
246         pmd = level2_kernel_pgt + pmd_index(vaddr);
247         for (i = 0; i < pmds; i++)
248                 pmd_clear(pmd + i);
249         __flush_tlb();
250 }
251
252 static void __meminit
253 phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end)
254 {
255         int i = pmd_index(address);
256
257         for (; i < PTRS_PER_PMD; i++, address += PMD_SIZE) {
258                 unsigned long entry;
259                 pmd_t *pmd = pmd_page + pmd_index(address);
260
261                 if (address >= end) {
262                         if (!after_bootmem)
263                                 for (; i < PTRS_PER_PMD; i++, pmd++)
264                                         set_pmd(pmd, __pmd(0));
265                         break;
266                 }
267
268                 if (pmd_val(*pmd))
269                         continue;
270
271                 entry = _PAGE_NX|_PAGE_PSE|_KERNPG_TABLE|_PAGE_GLOBAL|address;
272                 entry &= __supported_pte_mask;
273                 set_pmd(pmd, __pmd(entry));
274         }
275 }
276
277 static void __meminit
278 phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end)
279 {
280         pmd_t *pmd = pmd_offset(pud,0);
281         spin_lock(&init_mm.page_table_lock);
282         phys_pmd_init(pmd, address, end);
283         spin_unlock(&init_mm.page_table_lock);
284         __flush_tlb_all();
285 }
286
287 static void __meminit phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end)
288
289         int i = pud_index(addr);
290
291
292         for (; i < PTRS_PER_PUD; i++, addr = (addr & PUD_MASK) + PUD_SIZE ) {
293                 unsigned long pmd_phys;
294                 pud_t *pud = pud_page + pud_index(addr);
295                 pmd_t *pmd;
296
297                 if (addr >= end)
298                         break;
299
300                 if (!after_bootmem && !e820_any_mapped(addr,addr+PUD_SIZE,0)) {
301                         set_pud(pud, __pud(0)); 
302                         continue;
303                 } 
304
305                 if (pud_val(*pud)) {
306                         phys_pmd_update(pud, addr, end);
307                         continue;
308                 }
309
310                 pmd = alloc_low_page(&pmd_phys);
311                 spin_lock(&init_mm.page_table_lock);
312                 set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
313                 phys_pmd_init(pmd, addr, end);
314                 spin_unlock(&init_mm.page_table_lock);
315                 unmap_low_page(pmd);
316         }
317         __flush_tlb();
318
319
320 static void __init find_early_table_space(unsigned long end)
321 {
322         unsigned long puds, pmds, tables, start;
323
324         puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
325         pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
326         tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
327                  round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
328
329         /* RED-PEN putting page tables only on node 0 could
330            cause a hotspot and fill up ZONE_DMA. The page tables
331            need roughly 0.5KB per GB. */
332         start = 0x8000;
333         table_start = find_e820_area(start, end, tables);
334         if (table_start == -1UL)
335                 panic("Cannot find space for the kernel page tables");
336
337         table_start >>= PAGE_SHIFT;
338         table_end = table_start;
339
340         early_printk("kernel direct mapping tables up to %lx @ %lx-%lx\n",
341                 end, table_start << PAGE_SHIFT,
342                 (table_start << PAGE_SHIFT) + tables);
343 }
344
345 /* Setup the direct mapping of the physical memory at PAGE_OFFSET.
346    This runs before bootmem is initialized and gets pages directly from the 
347    physical memory. To access them they are temporarily mapped. */
348 void __init_refok init_memory_mapping(unsigned long start, unsigned long end)
349
350         unsigned long next; 
351
352         Dprintk("init_memory_mapping\n");
353
354         /* 
355          * Find space for the kernel direct mapping tables.
356          * Later we should allocate these tables in the local node of the memory
357          * mapped.  Unfortunately this is done currently before the nodes are 
358          * discovered.
359          */
360         if (!after_bootmem)
361                 find_early_table_space(end);
362
363         start = (unsigned long)__va(start);
364         end = (unsigned long)__va(end);
365
366         for (; start < end; start = next) {
367                 unsigned long pud_phys; 
368                 pgd_t *pgd = pgd_offset_k(start);
369                 pud_t *pud;
370
371                 if (after_bootmem)
372                         pud = pud_offset(pgd, start & PGDIR_MASK);
373                 else
374                         pud = alloc_low_page(&pud_phys);
375
376                 next = start + PGDIR_SIZE;
377                 if (next > end) 
378                         next = end; 
379                 phys_pud_init(pud, __pa(start), __pa(next));
380                 if (!after_bootmem)
381                         set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
382                 unmap_low_page(pud);
383         } 
384
385         if (!after_bootmem)
386                 mmu_cr4_features = read_cr4();
387         __flush_tlb_all();
388 }
389
390 #ifndef CONFIG_NUMA
391 void __init paging_init(void)
392 {
393         unsigned long max_zone_pfns[MAX_NR_ZONES];
394         memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
395         max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
396         max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
397         max_zone_pfns[ZONE_NORMAL] = end_pfn;
398
399         memory_present(0, 0, end_pfn);
400         sparse_init();
401         free_area_init_nodes(max_zone_pfns);
402 }
403 #endif
404
405 /* Unmap a kernel mapping if it exists. This is useful to avoid prefetches
406    from the CPU leading to inconsistent cache lines. address and size
407    must be aligned to 2MB boundaries. 
408    Does nothing when the mapping doesn't exist. */
409 void __init clear_kernel_mapping(unsigned long address, unsigned long size) 
410 {
411         unsigned long end = address + size;
412
413         BUG_ON(address & ~LARGE_PAGE_MASK);
414         BUG_ON(size & ~LARGE_PAGE_MASK); 
415         
416         for (; address < end; address += LARGE_PAGE_SIZE) { 
417                 pgd_t *pgd = pgd_offset_k(address);
418                 pud_t *pud;
419                 pmd_t *pmd;
420                 if (pgd_none(*pgd))
421                         continue;
422                 pud = pud_offset(pgd, address);
423                 if (pud_none(*pud))
424                         continue; 
425                 pmd = pmd_offset(pud, address);
426                 if (!pmd || pmd_none(*pmd))
427                         continue; 
428                 if (0 == (pmd_val(*pmd) & _PAGE_PSE)) { 
429                         /* Could handle this, but it should not happen currently. */
430                         printk(KERN_ERR 
431                "clear_kernel_mapping: mapping has been split. will leak memory\n"); 
432                         pmd_ERROR(*pmd); 
433                 }
434                 set_pmd(pmd, __pmd(0));                 
435         }
436         __flush_tlb_all();
437
438
439 /*
440  * Memory hotplug specific functions
441  */
442 void online_page(struct page *page)
443 {
444         ClearPageReserved(page);
445         init_page_count(page);
446         __free_page(page);
447         totalram_pages++;
448         num_physpages++;
449 }
450
451 #ifdef CONFIG_MEMORY_HOTPLUG
452 /*
453  * Memory is added always to NORMAL zone. This means you will never get
454  * additional DMA/DMA32 memory.
455  */
456 int arch_add_memory(int nid, u64 start, u64 size)
457 {
458         struct pglist_data *pgdat = NODE_DATA(nid);
459         struct zone *zone = pgdat->node_zones + ZONE_NORMAL;
460         unsigned long start_pfn = start >> PAGE_SHIFT;
461         unsigned long nr_pages = size >> PAGE_SHIFT;
462         int ret;
463
464         init_memory_mapping(start, (start + size -1));
465
466         ret = __add_pages(zone, start_pfn, nr_pages);
467         if (ret)
468                 goto error;
469
470         return ret;
471 error:
472         printk("%s: Problem encountered in __add_pages!\n", __func__);
473         return ret;
474 }
475 EXPORT_SYMBOL_GPL(arch_add_memory);
476
477 #if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA)
478 int memory_add_physaddr_to_nid(u64 start)
479 {
480         return 0;
481 }
482 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
483 #endif
484
485 #endif /* CONFIG_MEMORY_HOTPLUG */
486
487 #ifdef CONFIG_MEMORY_HOTPLUG_RESERVE
488 /*
489  * Memory Hotadd without sparsemem. The mem_maps have been allocated in advance,
490  * just online the pages.
491  */
492 int __add_pages(struct zone *z, unsigned long start_pfn, unsigned long nr_pages)
493 {
494         int err = -EIO;
495         unsigned long pfn;
496         unsigned long total = 0, mem = 0;
497         for (pfn = start_pfn; pfn < start_pfn + nr_pages; pfn++) {
498                 if (pfn_valid(pfn)) {
499                         online_page(pfn_to_page(pfn));
500                         err = 0;
501                         mem++;
502                 }
503                 total++;
504         }
505         if (!err) {
506                 z->spanned_pages += total;
507                 z->present_pages += mem;
508                 z->zone_pgdat->node_spanned_pages += total;
509                 z->zone_pgdat->node_present_pages += mem;
510         }
511         return err;
512 }
513 #endif
514
515 static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
516                          kcore_vsyscall;
517
518 void __init mem_init(void)
519 {
520         long codesize, reservedpages, datasize, initsize;
521
522         pci_iommu_alloc();
523
524         /* clear the zero-page */
525         memset(empty_zero_page, 0, PAGE_SIZE);
526
527         reservedpages = 0;
528
529         /* this will put all low memory onto the freelists */
530 #ifdef CONFIG_NUMA
531         totalram_pages = numa_free_all_bootmem();
532 #else
533         totalram_pages = free_all_bootmem();
534 #endif
535         reservedpages = end_pfn - totalram_pages -
536                                         absent_pages_in_range(0, end_pfn);
537
538         after_bootmem = 1;
539
540         codesize =  (unsigned long) &_etext - (unsigned long) &_text;
541         datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
542         initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;
543
544         /* Register memory areas for /proc/kcore */
545         kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT); 
546         kclist_add(&kcore_vmalloc, (void *)VMALLOC_START, 
547                    VMALLOC_END-VMALLOC_START);
548         kclist_add(&kcore_kernel, &_stext, _end - _stext);
549         kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
550         kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START, 
551                                  VSYSCALL_END - VSYSCALL_START);
552
553         printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, %ldk data, %ldk init)\n",
554                 (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
555                 end_pfn << (PAGE_SHIFT-10),
556                 codesize >> 10,
557                 reservedpages << (PAGE_SHIFT-10),
558                 datasize >> 10,
559                 initsize >> 10);
560 }
561
562 void free_init_pages(char *what, unsigned long begin, unsigned long end)
563 {
564         unsigned long addr;
565
566         if (begin >= end)
567                 return;
568
569         printk(KERN_INFO "Freeing %s: %luk freed\n", what, (end - begin) >> 10);
570         for (addr = begin; addr < end; addr += PAGE_SIZE) {
571                 ClearPageReserved(virt_to_page(addr));
572                 init_page_count(virt_to_page(addr));
573                 memset((void *)(addr & ~(PAGE_SIZE-1)),
574                         POISON_FREE_INITMEM, PAGE_SIZE);
575                 if (addr >= __START_KERNEL_map)
576                         change_page_attr_addr(addr, 1, __pgprot(0));
577                 free_page(addr);
578                 totalram_pages++;
579         }
580         if (addr > __START_KERNEL_map)
581                 global_flush_tlb();
582 }
583
584 void free_initmem(void)
585 {
586         free_init_pages("unused kernel memory",
587                         (unsigned long)(&__init_begin),
588                         (unsigned long)(&__init_end));
589 }
590
591 #ifdef CONFIG_DEBUG_RODATA
592
593 void mark_rodata_ro(void)
594 {
595         unsigned long start = (unsigned long)_stext, end;
596
597 #ifdef CONFIG_HOTPLUG_CPU
598         /* It must still be possible to apply SMP alternatives. */
599         if (num_possible_cpus() > 1)
600                 start = (unsigned long)_etext;
601 #endif
602
603 #ifdef CONFIG_KPROBES
604         start = (unsigned long)__start_rodata;
605 #endif
606         
607         end = (unsigned long)__end_rodata;
608         start = (start + PAGE_SIZE - 1) & PAGE_MASK;
609         end &= PAGE_MASK;
610         if (end <= start)
611                 return;
612
613         change_page_attr_addr(start, (end - start) >> PAGE_SHIFT, PAGE_KERNEL_RO);
614
615         printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
616                (end - start) >> 10);
617
618         /*
619          * change_page_attr_addr() requires a global_flush_tlb() call after it.
620          * We do this after the printk so that if something went wrong in the
621          * change, the printk gets out at least to give a better debug hint
622          * of who is the culprit.
623          */
624         global_flush_tlb();
625 }
626 #endif
627
628 #ifdef CONFIG_BLK_DEV_INITRD
629 void free_initrd_mem(unsigned long start, unsigned long end)
630 {
631         free_init_pages("initrd memory", start, end);
632 }
633 #endif
634
635 void __init reserve_bootmem_generic(unsigned long phys, unsigned len) 
636
637 #ifdef CONFIG_NUMA
638         int nid = phys_to_nid(phys);
639 #endif
640         unsigned long pfn = phys >> PAGE_SHIFT;
641         if (pfn >= end_pfn) {
642                 /* This can happen with kdump kernels when accessing firmware
643                    tables. */
644                 if (pfn < end_pfn_map)
645                         return;
646                 printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
647                                 phys, len);
648                 return;
649         }
650
651         /* Should check here against the e820 map to avoid double free */
652 #ifdef CONFIG_NUMA
653         reserve_bootmem_node(NODE_DATA(nid), phys, len);
654 #else                   
655         reserve_bootmem(phys, len);    
656 #endif
657         if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
658                 dma_reserve += len / PAGE_SIZE;
659                 set_dma_reserve(dma_reserve);
660         }
661 }
662
663 int kern_addr_valid(unsigned long addr) 
664
665         unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
666        pgd_t *pgd;
667        pud_t *pud;
668        pmd_t *pmd;
669        pte_t *pte;
670
671         if (above != 0 && above != -1UL)
672                 return 0; 
673         
674         pgd = pgd_offset_k(addr);
675         if (pgd_none(*pgd))
676                 return 0;
677
678         pud = pud_offset(pgd, addr);
679         if (pud_none(*pud))
680                 return 0; 
681
682         pmd = pmd_offset(pud, addr);
683         if (pmd_none(*pmd))
684                 return 0;
685         if (pmd_large(*pmd))
686                 return pfn_valid(pmd_pfn(*pmd));
687
688         pte = pte_offset_kernel(pmd, addr);
689         if (pte_none(*pte))
690                 return 0;
691         return pfn_valid(pte_pfn(*pte));
692 }
693
694 /* A pseudo VMA to allow ptrace access for the vsyscall page.  This only
695    covers the 64bit vsyscall page now. 32bit has a real VMA now and does
696    not need special handling anymore. */
697
698 static struct vm_area_struct gate_vma = {
699         .vm_start = VSYSCALL_START,
700         .vm_end = VSYSCALL_START + (VSYSCALL_MAPPED_PAGES << PAGE_SHIFT),
701         .vm_page_prot = PAGE_READONLY_EXEC,
702         .vm_flags = VM_READ | VM_EXEC
703 };
704
705 struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
706 {
707 #ifdef CONFIG_IA32_EMULATION
708         if (test_tsk_thread_flag(tsk, TIF_IA32))
709                 return NULL;
710 #endif
711         return &gate_vma;
712 }
713
714 int in_gate_area(struct task_struct *task, unsigned long addr)
715 {
716         struct vm_area_struct *vma = get_gate_vma(task);
717         if (!vma)
718                 return 0;
719         return (addr >= vma->vm_start) && (addr < vma->vm_end);
720 }
721
722 /* Use this when you have no reliable task/vma, typically from interrupt
723  * context.  It is less reliable than using the task's vma and may give
724  * false positives.
725  */
726 int in_gate_area_no_task(unsigned long addr)
727 {
728         return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
729 }
730
731 const char *arch_vma_name(struct vm_area_struct *vma)
732 {
733         if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
734                 return "[vdso]";
735         if (vma == &gate_vma)
736                 return "[vsyscall]";
737         return NULL;
738 }
739
740 #ifdef CONFIG_SPARSEMEM_VMEMMAP
741 /*
742  * Initialise the sparsemem vmemmap using huge-pages at the PMD level.
743  */
744 int __meminit vmemmap_populate(struct page *start_page,
745                                                 unsigned long size, int node)
746 {
747         unsigned long addr = (unsigned long)start_page;
748         unsigned long end = (unsigned long)(start_page + size);
749         unsigned long next;
750         pgd_t *pgd;
751         pud_t *pud;
752         pmd_t *pmd;
753
754         for (; addr < end; addr = next) {
755                 next = pmd_addr_end(addr, end);
756
757                 pgd = vmemmap_pgd_populate(addr, node);
758                 if (!pgd)
759                         return -ENOMEM;
760                 pud = vmemmap_pud_populate(pgd, addr, node);
761                 if (!pud)
762                         return -ENOMEM;
763
764                 pmd = pmd_offset(pud, addr);
765                 if (pmd_none(*pmd)) {
766                         pte_t entry;
767                         void *p = vmemmap_alloc_block(PMD_SIZE, node);
768                         if (!p)
769                                 return -ENOMEM;
770
771                         entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
772                         mk_pte_huge(entry);
773                         set_pmd(pmd, __pmd(pte_val(entry)));
774
775                         printk(KERN_DEBUG " [%lx-%lx] PMD ->%p on node %d\n",
776                                 addr, addr + PMD_SIZE - 1, p, node);
777                 } else
778                         vmemmap_verify((pte_t *)pmd, node, addr, next);
779         }
780
781         return 0;
782 }
783 #endif