Merge with /pub/scm/linux/kernel/git/torvalds/linux-2.6.git
[linux-2.6] / arch / powerpc / mm / hash_utils_64.c
1 /*
2  * PowerPC64 port by Mike Corrigan and Dave Engebretsen
3  *   {mikejc|engebret}@us.ibm.com
4  *
5  *    Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
6  *
7  * SMP scalability work:
8  *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
9  * 
10  *    Module name: htab.c
11  *
12  *    Description:
13  *      PowerPC Hashed Page Table functions
14  *
15  * This program is free software; you can redistribute it and/or
16  * modify it under the terms of the GNU General Public License
17  * as published by the Free Software Foundation; either version
18  * 2 of the License, or (at your option) any later version.
19  */
20
21 #undef DEBUG
22 #undef DEBUG_LOW
23
24 #include <linux/config.h>
25 #include <linux/spinlock.h>
26 #include <linux/errno.h>
27 #include <linux/sched.h>
28 #include <linux/proc_fs.h>
29 #include <linux/stat.h>
30 #include <linux/sysctl.h>
31 #include <linux/ctype.h>
32 #include <linux/cache.h>
33 #include <linux/init.h>
34 #include <linux/signal.h>
35
36 #include <asm/processor.h>
37 #include <asm/pgtable.h>
38 #include <asm/mmu.h>
39 #include <asm/mmu_context.h>
40 #include <asm/page.h>
41 #include <asm/types.h>
42 #include <asm/system.h>
43 #include <asm/uaccess.h>
44 #include <asm/machdep.h>
45 #include <asm/lmb.h>
46 #include <asm/abs_addr.h>
47 #include <asm/tlbflush.h>
48 #include <asm/io.h>
49 #include <asm/eeh.h>
50 #include <asm/tlb.h>
51 #include <asm/cacheflush.h>
52 #include <asm/cputable.h>
53 #include <asm/abs_addr.h>
54 #include <asm/sections.h>
55
56 #ifdef DEBUG
57 #define DBG(fmt...) udbg_printf(fmt)
58 #else
59 #define DBG(fmt...)
60 #endif
61
62 #ifdef DEBUG_LOW
63 #define DBG_LOW(fmt...) udbg_printf(fmt)
64 #else
65 #define DBG_LOW(fmt...)
66 #endif
67
68 #define KB (1024)
69 #define MB (1024*KB)
70
71 /*
72  * Note:  pte   --> Linux PTE
73  *        HPTE  --> PowerPC Hashed Page Table Entry
74  *
75  * Execution context:
76  *   htab_initialize is called with the MMU off (of course), but
77  *   the kernel has been copied down to zero so it can directly
78  *   reference global data.  At this point it is very difficult
79  *   to print debug info.
80  *
81  */
82
83 #ifdef CONFIG_U3_DART
84 extern unsigned long dart_tablebase;
85 #endif /* CONFIG_U3_DART */
86
87 static unsigned long _SDR1;
88 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
89
90 hpte_t *htab_address;
91 unsigned long htab_size_bytes;
92 unsigned long htab_hash_mask;
93 int mmu_linear_psize = MMU_PAGE_4K;
94 int mmu_virtual_psize = MMU_PAGE_4K;
95 int mmu_vmalloc_psize = MMU_PAGE_4K;
96 int mmu_io_psize = MMU_PAGE_4K;
97 #ifdef CONFIG_HUGETLB_PAGE
98 int mmu_huge_psize = MMU_PAGE_16M;
99 unsigned int HPAGE_SHIFT;
100 #endif
101 #ifdef CONFIG_PPC_64K_PAGES
102 int mmu_ci_restrictions;
103 #endif
104
105 /* There are definitions of page sizes arrays to be used when none
106  * is provided by the firmware.
107  */
108
109 /* Pre-POWER4 CPUs (4k pages only)
110  */
111 struct mmu_psize_def mmu_psize_defaults_old[] = {
112         [MMU_PAGE_4K] = {
113                 .shift  = 12,
114                 .sllp   = 0,
115                 .penc   = 0,
116                 .avpnm  = 0,
117                 .tlbiel = 0,
118         },
119 };
120
121 /* POWER4, GPUL, POWER5
122  *
123  * Support for 16Mb large pages
124  */
125 struct mmu_psize_def mmu_psize_defaults_gp[] = {
126         [MMU_PAGE_4K] = {
127                 .shift  = 12,
128                 .sllp   = 0,
129                 .penc   = 0,
130                 .avpnm  = 0,
131                 .tlbiel = 1,
132         },
133         [MMU_PAGE_16M] = {
134                 .shift  = 24,
135                 .sllp   = SLB_VSID_L,
136                 .penc   = 0,
137                 .avpnm  = 0x1UL,
138                 .tlbiel = 0,
139         },
140 };
141
142
143 int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
144                       unsigned long pstart, unsigned long mode, int psize)
145 {
146         unsigned long vaddr, paddr;
147         unsigned int step, shift;
148         unsigned long tmp_mode;
149         int ret = 0;
150
151         shift = mmu_psize_defs[psize].shift;
152         step = 1 << shift;
153
154         for (vaddr = vstart, paddr = pstart; vaddr < vend;
155              vaddr += step, paddr += step) {
156                 unsigned long vpn, hash, hpteg;
157                 unsigned long vsid = get_kernel_vsid(vaddr);
158                 unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff);
159
160                 vpn = va >> shift;
161                 tmp_mode = mode;
162                 
163                 /* Make non-kernel text non-executable */
164                 if (!in_kernel_text(vaddr))
165                         tmp_mode = mode | HPTE_R_N;
166
167                 hash = hpt_hash(va, shift);
168                 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
169
170                 /* The crap below can be cleaned once ppd_md.probe() can
171                  * set up the hash callbacks, thus we can just used the
172                  * normal insert callback here.
173                  */
174 #ifdef CONFIG_PPC_ISERIES
175                 if (machine_is(iseries))
176                         ret = iSeries_hpte_insert(hpteg, va,
177                                                   paddr,
178                                                   tmp_mode,
179                                                   HPTE_V_BOLTED,
180                                                   psize);
181                 else
182 #endif
183 #ifdef CONFIG_PPC_PSERIES
184                 if (machine_is(pseries) && firmware_has_feature(FW_FEATURE_LPAR))
185                         ret = pSeries_lpar_hpte_insert(hpteg, va,
186                                                        paddr,
187                                                        tmp_mode,
188                                                        HPTE_V_BOLTED,
189                                                        psize);
190                 else
191 #endif
192 #ifdef CONFIG_PPC_MULTIPLATFORM
193                         ret = native_hpte_insert(hpteg, va,
194                                                  paddr,
195                                                  tmp_mode, HPTE_V_BOLTED,
196                                                  psize);
197 #endif
198                 if (ret < 0)
199                         break;
200         }
201         return ret < 0 ? ret : 0;
202 }
203
204 static int __init htab_dt_scan_page_sizes(unsigned long node,
205                                           const char *uname, int depth,
206                                           void *data)
207 {
208         char *type = of_get_flat_dt_prop(node, "device_type", NULL);
209         u32 *prop;
210         unsigned long size = 0;
211
212         /* We are scanning "cpu" nodes only */
213         if (type == NULL || strcmp(type, "cpu") != 0)
214                 return 0;
215
216         prop = (u32 *)of_get_flat_dt_prop(node,
217                                           "ibm,segment-page-sizes", &size);
218         if (prop != NULL) {
219                 DBG("Page sizes from device-tree:\n");
220                 size /= 4;
221                 cur_cpu_spec->cpu_features &= ~(CPU_FTR_16M_PAGE);
222                 while(size > 0) {
223                         unsigned int shift = prop[0];
224                         unsigned int slbenc = prop[1];
225                         unsigned int lpnum = prop[2];
226                         unsigned int lpenc = 0;
227                         struct mmu_psize_def *def;
228                         int idx = -1;
229
230                         size -= 3; prop += 3;
231                         while(size > 0 && lpnum) {
232                                 if (prop[0] == shift)
233                                         lpenc = prop[1];
234                                 prop += 2; size -= 2;
235                                 lpnum--;
236                         }
237                         switch(shift) {
238                         case 0xc:
239                                 idx = MMU_PAGE_4K;
240                                 break;
241                         case 0x10:
242                                 idx = MMU_PAGE_64K;
243                                 break;
244                         case 0x14:
245                                 idx = MMU_PAGE_1M;
246                                 break;
247                         case 0x18:
248                                 idx = MMU_PAGE_16M;
249                                 cur_cpu_spec->cpu_features |= CPU_FTR_16M_PAGE;
250                                 break;
251                         case 0x22:
252                                 idx = MMU_PAGE_16G;
253                                 break;
254                         }
255                         if (idx < 0)
256                                 continue;
257                         def = &mmu_psize_defs[idx];
258                         def->shift = shift;
259                         if (shift <= 23)
260                                 def->avpnm = 0;
261                         else
262                                 def->avpnm = (1 << (shift - 23)) - 1;
263                         def->sllp = slbenc;
264                         def->penc = lpenc;
265                         /* We don't know for sure what's up with tlbiel, so
266                          * for now we only set it for 4K and 64K pages
267                          */
268                         if (idx == MMU_PAGE_4K || idx == MMU_PAGE_64K)
269                                 def->tlbiel = 1;
270                         else
271                                 def->tlbiel = 0;
272
273                         DBG(" %d: shift=%02x, sllp=%04x, avpnm=%08x, "
274                             "tlbiel=%d, penc=%d\n",
275                             idx, shift, def->sllp, def->avpnm, def->tlbiel,
276                             def->penc);
277                 }
278                 return 1;
279         }
280         return 0;
281 }
282
283
284 static void __init htab_init_page_sizes(void)
285 {
286         int rc;
287
288         /* Default to 4K pages only */
289         memcpy(mmu_psize_defs, mmu_psize_defaults_old,
290                sizeof(mmu_psize_defaults_old));
291
292         /*
293          * Try to find the available page sizes in the device-tree
294          */
295         rc = of_scan_flat_dt(htab_dt_scan_page_sizes, NULL);
296         if (rc != 0)  /* Found */
297                 goto found;
298
299         /*
300          * Not in the device-tree, let's fallback on known size
301          * list for 16M capable GP & GR
302          */
303         if (cpu_has_feature(CPU_FTR_16M_PAGE) && !machine_is(iseries))
304                 memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
305                        sizeof(mmu_psize_defaults_gp));
306  found:
307         /*
308          * Pick a size for the linear mapping. Currently, we only support
309          * 16M, 1M and 4K which is the default
310          */
311         if (mmu_psize_defs[MMU_PAGE_16M].shift)
312                 mmu_linear_psize = MMU_PAGE_16M;
313         else if (mmu_psize_defs[MMU_PAGE_1M].shift)
314                 mmu_linear_psize = MMU_PAGE_1M;
315
316 #ifdef CONFIG_PPC_64K_PAGES
317         /*
318          * Pick a size for the ordinary pages. Default is 4K, we support
319          * 64K for user mappings and vmalloc if supported by the processor.
320          * We only use 64k for ioremap if the processor
321          * (and firmware) support cache-inhibited large pages.
322          * If not, we use 4k and set mmu_ci_restrictions so that
323          * hash_page knows to switch processes that use cache-inhibited
324          * mappings to 4k pages.
325          */
326         if (mmu_psize_defs[MMU_PAGE_64K].shift) {
327                 mmu_virtual_psize = MMU_PAGE_64K;
328                 mmu_vmalloc_psize = MMU_PAGE_64K;
329                 if (cpu_has_feature(CPU_FTR_CI_LARGE_PAGE))
330                         mmu_io_psize = MMU_PAGE_64K;
331                 else
332                         mmu_ci_restrictions = 1;
333         }
334 #endif
335
336         printk(KERN_DEBUG "Page orders: linear mapping = %d, "
337                "virtual = %d, io = %d\n",
338                mmu_psize_defs[mmu_linear_psize].shift,
339                mmu_psize_defs[mmu_virtual_psize].shift,
340                mmu_psize_defs[mmu_io_psize].shift);
341
342 #ifdef CONFIG_HUGETLB_PAGE
343         /* Init large page size. Currently, we pick 16M or 1M depending
344          * on what is available
345          */
346         if (mmu_psize_defs[MMU_PAGE_16M].shift)
347                 mmu_huge_psize = MMU_PAGE_16M;
348         /* With 4k/4level pagetables, we can't (for now) cope with a
349          * huge page size < PMD_SIZE */
350         else if (mmu_psize_defs[MMU_PAGE_1M].shift)
351                 mmu_huge_psize = MMU_PAGE_1M;
352
353         /* Calculate HPAGE_SHIFT and sanity check it */
354         if (mmu_psize_defs[mmu_huge_psize].shift > MIN_HUGEPTE_SHIFT &&
355             mmu_psize_defs[mmu_huge_psize].shift < SID_SHIFT)
356                 HPAGE_SHIFT = mmu_psize_defs[mmu_huge_psize].shift;
357         else
358                 HPAGE_SHIFT = 0; /* No huge pages dude ! */
359 #endif /* CONFIG_HUGETLB_PAGE */
360 }
361
362 static int __init htab_dt_scan_pftsize(unsigned long node,
363                                        const char *uname, int depth,
364                                        void *data)
365 {
366         char *type = of_get_flat_dt_prop(node, "device_type", NULL);
367         u32 *prop;
368
369         /* We are scanning "cpu" nodes only */
370         if (type == NULL || strcmp(type, "cpu") != 0)
371                 return 0;
372
373         prop = (u32 *)of_get_flat_dt_prop(node, "ibm,pft-size", NULL);
374         if (prop != NULL) {
375                 /* pft_size[0] is the NUMA CEC cookie */
376                 ppc64_pft_size = prop[1];
377                 return 1;
378         }
379         return 0;
380 }
381
382 static unsigned long __init htab_get_table_size(void)
383 {
384         unsigned long mem_size, rnd_mem_size, pteg_count;
385
386         /* If hash size isn't already provided by the platform, we try to
387          * retrieve it from the device-tree. If it's not there neither, we
388          * calculate it now based on the total RAM size
389          */
390         if (ppc64_pft_size == 0)
391                 of_scan_flat_dt(htab_dt_scan_pftsize, NULL);
392         if (ppc64_pft_size)
393                 return 1UL << ppc64_pft_size;
394
395         /* round mem_size up to next power of 2 */
396         mem_size = lmb_phys_mem_size();
397         rnd_mem_size = 1UL << __ilog2(mem_size);
398         if (rnd_mem_size < mem_size)
399                 rnd_mem_size <<= 1;
400
401         /* # pages / 2 */
402         pteg_count = max(rnd_mem_size >> (12 + 1), 1UL << 11);
403
404         return pteg_count << 7;
405 }
406
407 #ifdef CONFIG_MEMORY_HOTPLUG
408 void create_section_mapping(unsigned long start, unsigned long end)
409 {
410                 BUG_ON(htab_bolt_mapping(start, end, __pa(start),
411                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX,
412                         mmu_linear_psize));
413 }
414 #endif /* CONFIG_MEMORY_HOTPLUG */
415
416 void __init htab_initialize(void)
417 {
418         unsigned long table;
419         unsigned long pteg_count;
420         unsigned long mode_rw;
421         unsigned long base = 0, size = 0;
422         int i;
423
424         extern unsigned long tce_alloc_start, tce_alloc_end;
425
426         DBG(" -> htab_initialize()\n");
427
428         /* Initialize page sizes */
429         htab_init_page_sizes();
430
431         /*
432          * Calculate the required size of the htab.  We want the number of
433          * PTEGs to equal one half the number of real pages.
434          */ 
435         htab_size_bytes = htab_get_table_size();
436         pteg_count = htab_size_bytes >> 7;
437
438         htab_hash_mask = pteg_count - 1;
439
440         if (firmware_has_feature(FW_FEATURE_LPAR)) {
441                 /* Using a hypervisor which owns the htab */
442                 htab_address = NULL;
443                 _SDR1 = 0; 
444         } else {
445                 /* Find storage for the HPT.  Must be contiguous in
446                  * the absolute address space.
447                  */
448                 table = lmb_alloc(htab_size_bytes, htab_size_bytes);
449
450                 DBG("Hash table allocated at %lx, size: %lx\n", table,
451                     htab_size_bytes);
452
453                 htab_address = abs_to_virt(table);
454
455                 /* htab absolute addr + encoded htabsize */
456                 _SDR1 = table + __ilog2(pteg_count) - 11;
457
458                 /* Initialize the HPT with no entries */
459                 memset((void *)table, 0, htab_size_bytes);
460
461                 /* Set SDR1 */
462                 mtspr(SPRN_SDR1, _SDR1);
463         }
464
465         mode_rw = _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX;
466
467         /* On U3 based machines, we need to reserve the DART area and
468          * _NOT_ map it to avoid cache paradoxes as it's remapped non
469          * cacheable later on
470          */
471
472         /* create bolted the linear mapping in the hash table */
473         for (i=0; i < lmb.memory.cnt; i++) {
474                 base = (unsigned long)__va(lmb.memory.region[i].base);
475                 size = lmb.memory.region[i].size;
476
477                 DBG("creating mapping for region: %lx : %lx\n", base, size);
478
479 #ifdef CONFIG_U3_DART
480                 /* Do not map the DART space. Fortunately, it will be aligned
481                  * in such a way that it will not cross two lmb regions and
482                  * will fit within a single 16Mb page.
483                  * The DART space is assumed to be a full 16Mb region even if
484                  * we only use 2Mb of that space. We will use more of it later
485                  * for AGP GART. We have to use a full 16Mb large page.
486                  */
487                 DBG("DART base: %lx\n", dart_tablebase);
488
489                 if (dart_tablebase != 0 && dart_tablebase >= base
490                     && dart_tablebase < (base + size)) {
491                         unsigned long dart_table_end = dart_tablebase + 16 * MB;
492                         if (base != dart_tablebase)
493                                 BUG_ON(htab_bolt_mapping(base, dart_tablebase,
494                                                         __pa(base), mode_rw,
495                                                         mmu_linear_psize));
496                         if ((base + size) > dart_table_end)
497                                 BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
498                                                         base + size,
499                                                         __pa(dart_table_end),
500                                                          mode_rw,
501                                                          mmu_linear_psize));
502                         continue;
503                 }
504 #endif /* CONFIG_U3_DART */
505                 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
506                                         mode_rw, mmu_linear_psize));
507        }
508
509         /*
510          * If we have a memory_limit and we've allocated TCEs then we need to
511          * explicitly map the TCE area at the top of RAM. We also cope with the
512          * case that the TCEs start below memory_limit.
513          * tce_alloc_start/end are 16MB aligned so the mapping should work
514          * for either 4K or 16MB pages.
515          */
516         if (tce_alloc_start) {
517                 tce_alloc_start = (unsigned long)__va(tce_alloc_start);
518                 tce_alloc_end = (unsigned long)__va(tce_alloc_end);
519
520                 if (base + size >= tce_alloc_start)
521                         tce_alloc_start = base + size + 1;
522
523                 BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end,
524                                          __pa(tce_alloc_start), mode_rw,
525                                          mmu_linear_psize));
526         }
527
528         DBG(" <- htab_initialize()\n");
529 }
530 #undef KB
531 #undef MB
532
533 void htab_initialize_secondary(void)
534 {
535         if (!firmware_has_feature(FW_FEATURE_LPAR))
536                 mtspr(SPRN_SDR1, _SDR1);
537 }
538
539 /*
540  * Called by asm hashtable.S for doing lazy icache flush
541  */
542 unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
543 {
544         struct page *page;
545
546         if (!pfn_valid(pte_pfn(pte)))
547                 return pp;
548
549         page = pte_page(pte);
550
551         /* page is dirty */
552         if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
553                 if (trap == 0x400) {
554                         __flush_dcache_icache(page_address(page));
555                         set_bit(PG_arch_1, &page->flags);
556                 } else
557                         pp |= HPTE_R_N;
558         }
559         return pp;
560 }
561
562 /* Result code is:
563  *  0 - handled
564  *  1 - normal page fault
565  * -1 - critical hash insertion error
566  */
567 int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
568 {
569         void *pgdir;
570         unsigned long vsid;
571         struct mm_struct *mm;
572         pte_t *ptep;
573         cpumask_t tmp;
574         int rc, user_region = 0, local = 0;
575         int psize;
576
577         DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
578                 ea, access, trap);
579
580         if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) {
581                 DBG_LOW(" out of pgtable range !\n");
582                 return 1;
583         }
584
585         /* Get region & vsid */
586         switch (REGION_ID(ea)) {
587         case USER_REGION_ID:
588                 user_region = 1;
589                 mm = current->mm;
590                 if (! mm) {
591                         DBG_LOW(" user region with no mm !\n");
592                         return 1;
593                 }
594                 vsid = get_vsid(mm->context.id, ea);
595                 psize = mm->context.user_psize;
596                 break;
597         case VMALLOC_REGION_ID:
598                 mm = &init_mm;
599                 vsid = get_kernel_vsid(ea);
600                 if (ea < VMALLOC_END)
601                         psize = mmu_vmalloc_psize;
602                 else
603                         psize = mmu_io_psize;
604                 break;
605         default:
606                 /* Not a valid range
607                  * Send the problem up to do_page_fault 
608                  */
609                 return 1;
610         }
611         DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
612
613         /* Get pgdir */
614         pgdir = mm->pgd;
615         if (pgdir == NULL)
616                 return 1;
617
618         /* Check CPU locality */
619         tmp = cpumask_of_cpu(smp_processor_id());
620         if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
621                 local = 1;
622
623         /* Handle hugepage regions */
624         if (unlikely(in_hugepage_area(mm->context, ea))) {
625                 DBG_LOW(" -> huge page !\n");
626                 return hash_huge_page(mm, access, ea, vsid, local, trap);
627         }
628
629         /* Get PTE and page size from page tables */
630         ptep = find_linux_pte(pgdir, ea);
631         if (ptep == NULL || !pte_present(*ptep)) {
632                 DBG_LOW(" no PTE !\n");
633                 return 1;
634         }
635
636 #ifndef CONFIG_PPC_64K_PAGES
637         DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
638 #else
639         DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
640                 pte_val(*(ptep + PTRS_PER_PTE)));
641 #endif
642         /* Pre-check access permissions (will be re-checked atomically
643          * in __hash_page_XX but this pre-check is a fast path
644          */
645         if (access & ~pte_val(*ptep)) {
646                 DBG_LOW(" no access !\n");
647                 return 1;
648         }
649
650         /* Do actual hashing */
651 #ifndef CONFIG_PPC_64K_PAGES
652         rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
653 #else
654         if (mmu_ci_restrictions) {
655                 /* If this PTE is non-cacheable, switch to 4k */
656                 if (psize == MMU_PAGE_64K &&
657                     (pte_val(*ptep) & _PAGE_NO_CACHE)) {
658                         if (user_region) {
659                                 psize = MMU_PAGE_4K;
660                                 mm->context.user_psize = MMU_PAGE_4K;
661                                 mm->context.sllp = SLB_VSID_USER |
662                                         mmu_psize_defs[MMU_PAGE_4K].sllp;
663                         } else if (ea < VMALLOC_END) {
664                                 /*
665                                  * some driver did a non-cacheable mapping
666                                  * in vmalloc space, so switch vmalloc
667                                  * to 4k pages
668                                  */
669                                 printk(KERN_ALERT "Reducing vmalloc segment "
670                                        "to 4kB pages because of "
671                                        "non-cacheable mapping\n");
672                                 psize = mmu_vmalloc_psize = MMU_PAGE_4K;
673                         }
674                 }
675                 if (user_region) {
676                         if (psize != get_paca()->context.user_psize) {
677                                 get_paca()->context = mm->context;
678                                 slb_flush_and_rebolt();
679                         }
680                 } else if (get_paca()->vmalloc_sllp !=
681                            mmu_psize_defs[mmu_vmalloc_psize].sllp) {
682                         get_paca()->vmalloc_sllp =
683                                 mmu_psize_defs[mmu_vmalloc_psize].sllp;
684                         slb_flush_and_rebolt();
685                 }
686         }
687         if (psize == MMU_PAGE_64K)
688                 rc = __hash_page_64K(ea, access, vsid, ptep, trap, local);
689         else
690                 rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
691 #endif /* CONFIG_PPC_64K_PAGES */
692
693 #ifndef CONFIG_PPC_64K_PAGES
694         DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
695 #else
696         DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
697                 pte_val(*(ptep + PTRS_PER_PTE)));
698 #endif
699         DBG_LOW(" -> rc=%d\n", rc);
700         return rc;
701 }
702 EXPORT_SYMBOL_GPL(hash_page);
703
704 void hash_preload(struct mm_struct *mm, unsigned long ea,
705                   unsigned long access, unsigned long trap)
706 {
707         unsigned long vsid;
708         void *pgdir;
709         pte_t *ptep;
710         cpumask_t mask;
711         unsigned long flags;
712         int local = 0;
713
714         /* We don't want huge pages prefaulted for now
715          */
716         if (unlikely(in_hugepage_area(mm->context, ea)))
717                 return;
718
719         DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
720                 " trap=%lx\n", mm, mm->pgd, ea, access, trap);
721
722         /* Get PTE, VSID, access mask */
723         pgdir = mm->pgd;
724         if (pgdir == NULL)
725                 return;
726         ptep = find_linux_pte(pgdir, ea);
727         if (!ptep)
728                 return;
729         vsid = get_vsid(mm->context.id, ea);
730
731         /* Hash it in */
732         local_irq_save(flags);
733         mask = cpumask_of_cpu(smp_processor_id());
734         if (cpus_equal(mm->cpu_vm_mask, mask))
735                 local = 1;
736 #ifndef CONFIG_PPC_64K_PAGES
737         __hash_page_4K(ea, access, vsid, ptep, trap, local);
738 #else
739         if (mmu_ci_restrictions) {
740                 /* If this PTE is non-cacheable, switch to 4k */
741                 if (mm->context.user_psize == MMU_PAGE_64K &&
742                     (pte_val(*ptep) & _PAGE_NO_CACHE)) {
743                         mm->context.user_psize = MMU_PAGE_4K;
744                         mm->context.sllp = SLB_VSID_USER |
745                                 mmu_psize_defs[MMU_PAGE_4K].sllp;
746                         get_paca()->context = mm->context;
747                         slb_flush_and_rebolt();
748                 }
749         }
750         if (mm->context.user_psize == MMU_PAGE_64K)
751                 __hash_page_64K(ea, access, vsid, ptep, trap, local);
752         else
753                 __hash_page_4K(ea, access, vsid, ptep, trap, local);
754 #endif /* CONFIG_PPC_64K_PAGES */
755         local_irq_restore(flags);
756 }
757
758 void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int local)
759 {
760         unsigned long hash, index, shift, hidx, slot;
761
762         DBG_LOW("flush_hash_page(va=%016x)\n", va);
763         pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
764                 hash = hpt_hash(va, shift);
765                 hidx = __rpte_to_hidx(pte, index);
766                 if (hidx & _PTEIDX_SECONDARY)
767                         hash = ~hash;
768                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
769                 slot += hidx & _PTEIDX_GROUP_IX;
770                 DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx);
771                 ppc_md.hpte_invalidate(slot, va, psize, local);
772         } pte_iterate_hashed_end();
773 }
774
775 void flush_hash_range(unsigned long number, int local)
776 {
777         if (ppc_md.flush_hash_range)
778                 ppc_md.flush_hash_range(number, local);
779         else {
780                 int i;
781                 struct ppc64_tlb_batch *batch =
782                         &__get_cpu_var(ppc64_tlb_batch);
783
784                 for (i = 0; i < number; i++)
785                         flush_hash_page(batch->vaddr[i], batch->pte[i],
786                                         batch->psize, local);
787         }
788 }
789
790 static inline void make_bl(unsigned int *insn_addr, void *func)
791 {
792         unsigned long funcp = *((unsigned long *)func);
793         int offset = funcp - (unsigned long)insn_addr;
794
795         *insn_addr = (unsigned int)(0x48000001 | (offset & 0x03fffffc));
796         flush_icache_range((unsigned long)insn_addr, 4+
797                            (unsigned long)insn_addr);
798 }
799
800 /*
801  * low_hash_fault is called when we the low level hash code failed
802  * to instert a PTE due to an hypervisor error
803  */
804 void low_hash_fault(struct pt_regs *regs, unsigned long address)
805 {
806         if (user_mode(regs)) {
807                 siginfo_t info;
808
809                 info.si_signo = SIGBUS;
810                 info.si_errno = 0;
811                 info.si_code = BUS_ADRERR;
812                 info.si_addr = (void __user *)address;
813                 force_sig_info(SIGBUS, &info, current);
814                 return;
815         }
816         bad_page_fault(regs, address, SIGBUS);
817 }
818
819 void __init htab_finish_init(void)
820 {
821         extern unsigned int *htab_call_hpte_insert1;
822         extern unsigned int *htab_call_hpte_insert2;
823         extern unsigned int *htab_call_hpte_remove;
824         extern unsigned int *htab_call_hpte_updatepp;
825
826 #ifdef CONFIG_PPC_64K_PAGES
827         extern unsigned int *ht64_call_hpte_insert1;
828         extern unsigned int *ht64_call_hpte_insert2;
829         extern unsigned int *ht64_call_hpte_remove;
830         extern unsigned int *ht64_call_hpte_updatepp;
831
832         make_bl(ht64_call_hpte_insert1, ppc_md.hpte_insert);
833         make_bl(ht64_call_hpte_insert2, ppc_md.hpte_insert);
834         make_bl(ht64_call_hpte_remove, ppc_md.hpte_remove);
835         make_bl(ht64_call_hpte_updatepp, ppc_md.hpte_updatepp);
836 #endif /* CONFIG_PPC_64K_PAGES */
837
838         make_bl(htab_call_hpte_insert1, ppc_md.hpte_insert);
839         make_bl(htab_call_hpte_insert2, ppc_md.hpte_insert);
840         make_bl(htab_call_hpte_remove, ppc_md.hpte_remove);
841         make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp);
842 }