Merge branches 'release' and 'autoload' into release
[linux-2.6] / arch / powerpc / mm / hash_low_64.S
1 /*
2  * ppc64 MMU hashtable management routines
3  *
4  * (c) Copyright IBM Corp. 2003, 2005
5  *
6  * Maintained by: Benjamin Herrenschmidt
7  *                <benh@kernel.crashing.org>
8  *
9  * This file is covered by the GNU Public Licence v2 as
10  * described in the kernel's COPYING file.
11  */
12
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
21
22         .text
23
24 /*
25  * Stackframe:
26  *              
27  *         +-> Back chain                       (SP + 256)
28  *         |   General register save area       (SP + 112)
29  *         |   Parameter save area              (SP + 48)
30  *         |   TOC save area                    (SP + 40)
31  *         |   link editor doubleword           (SP + 32)
32  *         |   compiler doubleword              (SP + 24)
33  *         |   LR save area                     (SP + 16)
34  *         |   CR save area                     (SP + 8)
35  * SP ---> +-- Back chain                       (SP + 0)
36  */
37 #define STACKFRAMESIZE  256
38
39 /* Save parameters offsets */
40 #define STK_PARM(i)     (STACKFRAMESIZE + 48 + ((i)-3)*8)
41
42 /* Save non-volatile offsets */
43 #define STK_REG(i)      (112 + ((i)-14)*8)
44
45
46 #ifndef CONFIG_PPC_64K_PAGES
47
48 /*****************************************************************************
49  *                                                                           *
50  *           4K SW & 4K HW pages implementation                              *
51  *                                                                           *
52  *****************************************************************************/
53
54
55 /*
56  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57  *               pte_t *ptep, unsigned long trap, int local, int ssize)
58  *
59  * Adds a 4K page to the hash table in a segment of 4K pages only
60  */
61
62 _GLOBAL(__hash_page_4K)
63         mflr    r0
64         std     r0,16(r1)
65         stdu    r1,-STACKFRAMESIZE(r1)
66         /* Save all params that we need after a function call */
67         std     r6,STK_PARM(r6)(r1)
68         std     r8,STK_PARM(r8)(r1)
69         std     r9,STK_PARM(r9)(r1)
70         
71         /* Add _PAGE_PRESENT to access */
72         ori     r4,r4,_PAGE_PRESENT
73
74         /* Save non-volatile registers.
75          * r31 will hold "old PTE"
76          * r30 is "new PTE"
77          * r29 is "va"
78          * r28 is a hash value
79          * r27 is hashtab mask (maybe dynamic patched instead ?)
80          */
81         std     r27,STK_REG(r27)(r1)
82         std     r28,STK_REG(r28)(r1)
83         std     r29,STK_REG(r29)(r1)
84         std     r30,STK_REG(r30)(r1)
85         std     r31,STK_REG(r31)(r1)
86         
87         /* Step 1:
88          *
89          * Check permissions, atomically mark the linux PTE busy
90          * and hashed.
91          */ 
92 1:
93         ldarx   r31,0,r6
94         /* Check access rights (access & ~(pte_val(*ptep))) */
95         andc.   r0,r4,r31
96         bne-    htab_wrong_access
97         /* Check if PTE is busy */
98         andi.   r0,r31,_PAGE_BUSY
99         /* If so, just bail out and refault if needed. Someone else
100          * is changing this PTE anyway and might hash it.
101          */
102         bne-    htab_bail_ok
103
104         /* Prepare new PTE value (turn access RW into DIRTY, then
105          * add BUSY,HASHPTE and ACCESSED)
106          */
107         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108         or      r30,r30,r31
109         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110         /* Write the linux PTE atomically (setting busy) */
111         stdcx.  r30,0,r6
112         bne-    1b
113         isync
114
115         /* Step 2:
116          *
117          * Insert/Update the HPTE in the hash table. At this point,
118          * r4 (access) is re-useable, we use it for the new HPTE flags
119          */
120
121 BEGIN_FTR_SECTION
122         cmpdi   r9,0                    /* check segment size */
123         bne     3f
124 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
125         /* Calc va and put it in r29 */
126         rldicr  r29,r5,28,63-28
127         rldicl  r3,r3,0,36
128         or      r29,r3,r29
129
130         /* Calculate hash value for primary slot and store it in r28 */
131         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
132         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
133         xor     r28,r5,r0
134         b       4f
135
136 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
137         sldi    r29,r5,40               /* vsid << 40 */
138         clrldi  r3,r3,24                /* ea & 0xffffffffff */
139         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
140         clrldi  r5,r5,40                /* vsid & 0xffffff */
141         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
142         xor     r28,r28,r5
143         or      r29,r3,r29              /* VA */
144         xor     r28,r28,r0              /* hash */
145
146         /* Convert linux PTE bits into HW equivalents */
147 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
148         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
149         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
150         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
151         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
152         andc    r0,r30,r0               /* r0 = pte & ~r0 */
153         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
154         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
155
156         /* We eventually do the icache sync here (maybe inline that
157          * code rather than call a C function...) 
158          */
159 BEGIN_FTR_SECTION
160         mr      r4,r30
161         mr      r5,r7
162         bl      .hash_page_do_lazy_icache
163 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
164
165         /* At this point, r3 contains new PP bits, save them in
166          * place of "access" in the param area (sic)
167          */
168         std     r3,STK_PARM(r4)(r1)
169
170         /* Get htab_hash_mask */
171         ld      r4,htab_hash_mask@got(2)
172         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
173
174         /* Check if we may already be in the hashtable, in this case, we
175          * go to out-of-line code to try to modify the HPTE
176          */
177         andi.   r0,r31,_PAGE_HASHPTE
178         bne     htab_modify_pte
179
180 htab_insert_pte:
181         /* Clear hpte bits in new pte (we also clear BUSY btw) and
182          * add _PAGE_HASHPTE
183          */
184         lis     r0,_PAGE_HPTEFLAGS@h
185         ori     r0,r0,_PAGE_HPTEFLAGS@l
186         andc    r30,r30,r0
187         ori     r30,r30,_PAGE_HASHPTE
188
189         /* physical address r5 */
190         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
191         sldi    r5,r5,PAGE_SHIFT
192
193         /* Calculate primary group hash */
194         and     r0,r28,r27
195         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
196
197         /* Call ppc_md.hpte_insert */
198         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
199         mr      r4,r29                  /* Retreive va */
200         li      r7,0                    /* !bolted, !secondary */
201         li      r8,MMU_PAGE_4K          /* page size */
202         ld      r9,STK_PARM(r9)(r1)     /* segment size */
203 _GLOBAL(htab_call_hpte_insert1)
204         bl      .                       /* Patched by htab_finish_init() */
205         cmpdi   0,r3,0
206         bge     htab_pte_insert_ok      /* Insertion successful */
207         cmpdi   0,r3,-2                 /* Critical failure */
208         beq-    htab_pte_insert_failure
209
210         /* Now try secondary slot */
211         
212         /* physical address r5 */
213         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
214         sldi    r5,r5,PAGE_SHIFT
215
216         /* Calculate secondary group hash */
217         andc    r0,r27,r28
218         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
219         
220         /* Call ppc_md.hpte_insert */
221         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
222         mr      r4,r29                  /* Retreive va */
223         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
224         li      r8,MMU_PAGE_4K          /* page size */
225         ld      r9,STK_PARM(r9)(r1)     /* segment size */
226 _GLOBAL(htab_call_hpte_insert2)
227         bl      .                       /* Patched by htab_finish_init() */
228         cmpdi   0,r3,0
229         bge+    htab_pte_insert_ok      /* Insertion successful */
230         cmpdi   0,r3,-2                 /* Critical failure */
231         beq-    htab_pte_insert_failure
232
233         /* Both are full, we need to evict something */
234         mftb    r0
235         /* Pick a random group based on TB */
236         andi.   r0,r0,1
237         mr      r5,r28
238         bne     2f
239         not     r5,r5
240 2:      and     r0,r5,r27
241         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */   
242         /* Call ppc_md.hpte_remove */
243 _GLOBAL(htab_call_hpte_remove)
244         bl      .                       /* Patched by htab_finish_init() */
245
246         /* Try all again */
247         b       htab_insert_pte 
248
249 htab_bail_ok:
250         li      r3,0
251         b       htab_bail
252
253 htab_pte_insert_ok:
254         /* Insert slot number & secondary bit in PTE */
255         rldimi  r30,r3,12,63-15
256                 
257         /* Write out the PTE with a normal write
258          * (maybe add eieio may be good still ?)
259          */
260 htab_write_out_pte:
261         ld      r6,STK_PARM(r6)(r1)
262         std     r30,0(r6)
263         li      r3, 0
264 htab_bail:
265         ld      r27,STK_REG(r27)(r1)
266         ld      r28,STK_REG(r28)(r1)
267         ld      r29,STK_REG(r29)(r1)
268         ld      r30,STK_REG(r30)(r1)
269         ld      r31,STK_REG(r31)(r1)
270         addi    r1,r1,STACKFRAMESIZE
271         ld      r0,16(r1)
272         mtlr    r0
273         blr
274
275 htab_modify_pte:
276         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
277         mr      r4,r3
278         rlwinm  r3,r31,32-12,29,31
279
280         /* Secondary group ? if yes, get a inverted hash value */
281         mr      r5,r28
282         andi.   r0,r31,_PAGE_SECONDARY
283         beq     1f
284         not     r5,r5
285 1:
286         /* Calculate proper slot value for ppc_md.hpte_updatepp */
287         and     r0,r5,r27
288         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
289         add     r3,r0,r3        /* add slot idx */
290
291         /* Call ppc_md.hpte_updatepp */
292         mr      r5,r29                  /* va */
293         li      r6,MMU_PAGE_4K          /* page size */
294         ld      r7,STK_PARM(r9)(r1)     /* segment size */
295         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
296 _GLOBAL(htab_call_hpte_updatepp)
297         bl      .                       /* Patched by htab_finish_init() */
298
299         /* if we failed because typically the HPTE wasn't really here
300          * we try an insertion. 
301          */
302         cmpdi   0,r3,-1
303         beq-    htab_insert_pte
304
305         /* Clear the BUSY bit and Write out the PTE */
306         li      r0,_PAGE_BUSY
307         andc    r30,r30,r0
308         b       htab_write_out_pte
309
310 htab_wrong_access:
311         /* Bail out clearing reservation */
312         stdcx.  r31,0,r6
313         li      r3,1
314         b       htab_bail
315
316 htab_pte_insert_failure:
317         /* Bail out restoring old PTE */
318         ld      r6,STK_PARM(r6)(r1)
319         std     r31,0(r6)
320         li      r3,-1
321         b       htab_bail
322
323
324 #else /* CONFIG_PPC_64K_PAGES */
325
326
327 /*****************************************************************************
328  *                                                                           *
329  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
330  *                                                                           *
331  *****************************************************************************/
332
333 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
334  *               pte_t *ptep, unsigned long trap, int local, int ssize,
335  *               int subpg_prot)
336  */
337
338 /*
339  * For now, we do NOT implement Admixed pages
340  */
341 _GLOBAL(__hash_page_4K)
342         mflr    r0
343         std     r0,16(r1)
344         stdu    r1,-STACKFRAMESIZE(r1)
345         /* Save all params that we need after a function call */
346         std     r6,STK_PARM(r6)(r1)
347         std     r8,STK_PARM(r8)(r1)
348         std     r9,STK_PARM(r9)(r1)
349
350         /* Add _PAGE_PRESENT to access */
351         ori     r4,r4,_PAGE_PRESENT
352
353         /* Save non-volatile registers.
354          * r31 will hold "old PTE"
355          * r30 is "new PTE"
356          * r29 is "va"
357          * r28 is a hash value
358          * r27 is hashtab mask (maybe dynamic patched instead ?)
359          * r26 is the hidx mask
360          * r25 is the index in combo page
361          */
362         std     r25,STK_REG(r25)(r1)
363         std     r26,STK_REG(r26)(r1)
364         std     r27,STK_REG(r27)(r1)
365         std     r28,STK_REG(r28)(r1)
366         std     r29,STK_REG(r29)(r1)
367         std     r30,STK_REG(r30)(r1)
368         std     r31,STK_REG(r31)(r1)
369
370         /* Step 1:
371          *
372          * Check permissions, atomically mark the linux PTE busy
373          * and hashed.
374          */
375 1:
376         ldarx   r31,0,r6
377         /* Check access rights (access & ~(pte_val(*ptep))) */
378         andc.   r0,r4,r31
379         bne-    htab_wrong_access
380         /* Check if PTE is busy */
381         andi.   r0,r31,_PAGE_BUSY
382         /* If so, just bail out and refault if needed. Someone else
383          * is changing this PTE anyway and might hash it.
384          */
385         bne-    htab_bail_ok
386         /* Prepare new PTE value (turn access RW into DIRTY, then
387          * add BUSY and ACCESSED)
388          */
389         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
390         or      r30,r30,r31
391         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
392         oris    r30,r30,_PAGE_COMBO@h
393         /* Write the linux PTE atomically (setting busy) */
394         stdcx.  r30,0,r6
395         bne-    1b
396         isync
397
398         /* Step 2:
399          *
400          * Insert/Update the HPTE in the hash table. At this point,
401          * r4 (access) is re-useable, we use it for the new HPTE flags
402          */
403
404         /* Load the hidx index */
405         rldicl  r25,r3,64-12,60
406
407 BEGIN_FTR_SECTION
408         cmpdi   r9,0                    /* check segment size */
409         bne     3f
410 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
411         /* Calc va and put it in r29 */
412         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
413         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
414         or      r29,r3,r29              /* r29 = va */
415
416         /* Calculate hash value for primary slot and store it in r28 */
417         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
418         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
419         xor     r28,r5,r0
420         b       4f
421
422 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
423         sldi    r29,r5,40               /* vsid << 40 */
424         clrldi  r3,r3,24                /* ea & 0xffffffffff */
425         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
426         clrldi  r5,r5,40                /* vsid & 0xffffff */
427         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
428         xor     r28,r28,r5
429         or      r29,r3,r29              /* VA */
430         xor     r28,r28,r0              /* hash */
431
432         /* Convert linux PTE bits into HW equivalents */
433 4:
434 #ifdef CONFIG_PPC_SUBPAGE_PROT
435         andc    r10,r30,r10
436         andi.   r3,r10,0x1fe            /* Get basic set of flags */
437         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
438 #else
439         andi.   r3,r30,0x1fe            /* Get basic set of flags */
440         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
441 #endif
442         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
443         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
444         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
445         andc    r0,r3,r0                /* r0 = pte & ~r0 */
446         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
447         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
448
449         /* We eventually do the icache sync here (maybe inline that
450          * code rather than call a C function...)
451          */
452 BEGIN_FTR_SECTION
453         mr      r4,r30
454         mr      r5,r7
455         bl      .hash_page_do_lazy_icache
456 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
457
458         /* At this point, r3 contains new PP bits, save them in
459          * place of "access" in the param area (sic)
460          */
461         std     r3,STK_PARM(r4)(r1)
462
463         /* Get htab_hash_mask */
464         ld      r4,htab_hash_mask@got(2)
465         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
466
467         /* Check if we may already be in the hashtable, in this case, we
468          * go to out-of-line code to try to modify the HPTE. We look for
469          * the bit at (1 >> (index + 32))
470          */
471         andi.   r0,r31,_PAGE_HASHPTE
472         li      r26,0                   /* Default hidx */
473         beq     htab_insert_pte
474
475         /*
476          * Check if the pte was already inserted into the hash table
477          * as a 64k HW page, and invalidate the 64k HPTE if so.
478          */
479         andis.  r0,r31,_PAGE_COMBO@h
480         beq     htab_inval_old_hpte
481
482         ld      r6,STK_PARM(r6)(r1)
483         ori     r26,r6,0x8000           /* Load the hidx mask */
484         ld      r26,0(r26)
485         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
486         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
487         bne     htab_modify_pte
488
489 htab_insert_pte:
490         /* real page number in r5, PTE RPN value + index */
491         andis.  r0,r31,_PAGE_4K_PFN@h
492         srdi    r5,r31,PTE_RPN_SHIFT
493         bne-    htab_special_pfn
494         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
495         add     r5,r5,r25
496 htab_special_pfn:
497         sldi    r5,r5,HW_PAGE_SHIFT
498
499         /* Calculate primary group hash */
500         and     r0,r28,r27
501         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
502
503         /* Call ppc_md.hpte_insert */
504         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
505         mr      r4,r29                  /* Retreive va */
506         li      r7,0                    /* !bolted, !secondary */
507         li      r8,MMU_PAGE_4K          /* page size */
508         ld      r9,STK_PARM(r9)(r1)     /* segment size */
509 _GLOBAL(htab_call_hpte_insert1)
510         bl      .                       /* patched by htab_finish_init() */
511         cmpdi   0,r3,0
512         bge     htab_pte_insert_ok      /* Insertion successful */
513         cmpdi   0,r3,-2                 /* Critical failure */
514         beq-    htab_pte_insert_failure
515
516         /* Now try secondary slot */
517
518         /* real page number in r5, PTE RPN value + index */
519         andis.  r0,r31,_PAGE_4K_PFN@h
520         srdi    r5,r31,PTE_RPN_SHIFT
521         bne-    3f
522         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
523         add     r5,r5,r25
524 3:      sldi    r5,r5,HW_PAGE_SHIFT
525
526         /* Calculate secondary group hash */
527         andc    r0,r27,r28
528         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
529
530         /* Call ppc_md.hpte_insert */
531         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
532         mr      r4,r29                  /* Retreive va */
533         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
534         li      r8,MMU_PAGE_4K          /* page size */
535         ld      r9,STK_PARM(r9)(r1)     /* segment size */
536 _GLOBAL(htab_call_hpte_insert2)
537         bl      .                       /* patched by htab_finish_init() */
538         cmpdi   0,r3,0
539         bge+    htab_pte_insert_ok      /* Insertion successful */
540         cmpdi   0,r3,-2                 /* Critical failure */
541         beq-    htab_pte_insert_failure
542
543         /* Both are full, we need to evict something */
544         mftb    r0
545         /* Pick a random group based on TB */
546         andi.   r0,r0,1
547         mr      r5,r28
548         bne     2f
549         not     r5,r5
550 2:      and     r0,r5,r27
551         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
552         /* Call ppc_md.hpte_remove */
553 _GLOBAL(htab_call_hpte_remove)
554         bl      .                       /* patched by htab_finish_init() */
555
556         /* Try all again */
557         b       htab_insert_pte
558
559         /*
560          * Call out to C code to invalidate an 64k HW HPTE that is
561          * useless now that the segment has been switched to 4k pages.
562          */
563 htab_inval_old_hpte:
564         mr      r3,r29                  /* virtual addr */
565         mr      r4,r31                  /* PTE.pte */
566         li      r5,0                    /* PTE.hidx */
567         li      r6,MMU_PAGE_64K         /* psize */
568         ld      r7,STK_PARM(r9)(r1)     /* ssize */
569         ld      r8,STK_PARM(r8)(r1)     /* local */
570         bl      .flush_hash_page
571         b       htab_insert_pte
572         
573 htab_bail_ok:
574         li      r3,0
575         b       htab_bail
576
577 htab_pte_insert_ok:
578         /* Insert slot number & secondary bit in PTE second half,
579          * clear _PAGE_BUSY and set approriate HPTE slot bit
580          */
581         ld      r6,STK_PARM(r6)(r1)
582         li      r0,_PAGE_BUSY
583         andc    r30,r30,r0
584         /* HPTE SUB bit */
585         li      r0,1
586         subfic  r5,r25,27               /* Must match bit position in */
587         sld     r0,r0,r5                /* pgtable.h */
588         or      r30,r30,r0
589         /* hindx */
590         sldi    r5,r25,2
591         sld     r3,r3,r5
592         li      r4,0xf
593         sld     r4,r4,r5
594         andc    r26,r26,r4
595         or      r26,r26,r3
596         ori     r5,r6,0x8000
597         std     r26,0(r5)
598         lwsync
599         std     r30,0(r6)
600         li      r3, 0
601 htab_bail:
602         ld      r25,STK_REG(r25)(r1)
603         ld      r26,STK_REG(r26)(r1)
604         ld      r27,STK_REG(r27)(r1)
605         ld      r28,STK_REG(r28)(r1)
606         ld      r29,STK_REG(r29)(r1)
607         ld      r30,STK_REG(r30)(r1)
608         ld      r31,STK_REG(r31)(r1)
609         addi    r1,r1,STACKFRAMESIZE
610         ld      r0,16(r1)
611         mtlr    r0
612         blr
613
614 htab_modify_pte:
615         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
616         mr      r4,r3
617         sldi    r5,r25,2
618         srd     r3,r26,r5
619
620         /* Secondary group ? if yes, get a inverted hash value */
621         mr      r5,r28
622         andi.   r0,r3,0x8 /* page secondary ? */
623         beq     1f
624         not     r5,r5
625 1:      andi.   r3,r3,0x7 /* extract idx alone */
626
627         /* Calculate proper slot value for ppc_md.hpte_updatepp */
628         and     r0,r5,r27
629         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
630         add     r3,r0,r3        /* add slot idx */
631
632         /* Call ppc_md.hpte_updatepp */
633         mr      r5,r29                  /* va */
634         li      r6,MMU_PAGE_4K          /* page size */
635         ld      r7,STK_PARM(r9)(r1)     /* segment size */
636         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
637 _GLOBAL(htab_call_hpte_updatepp)
638         bl      .                       /* patched by htab_finish_init() */
639
640         /* if we failed because typically the HPTE wasn't really here
641          * we try an insertion.
642          */
643         cmpdi   0,r3,-1
644         beq-    htab_insert_pte
645
646         /* Clear the BUSY bit and Write out the PTE */
647         li      r0,_PAGE_BUSY
648         andc    r30,r30,r0
649         ld      r6,STK_PARM(r6)(r1)
650         std     r30,0(r6)
651         li      r3,0
652         b       htab_bail
653
654 htab_wrong_access:
655         /* Bail out clearing reservation */
656         stdcx.  r31,0,r6
657         li      r3,1
658         b       htab_bail
659
660 htab_pte_insert_failure:
661         /* Bail out restoring old PTE */
662         ld      r6,STK_PARM(r6)(r1)
663         std     r31,0(r6)
664         li      r3,-1
665         b       htab_bail
666
667 #endif /* CONFIG_PPC_64K_PAGES */
668
669 #ifdef CONFIG_PPC_HAS_HASH_64K
670
671 /*****************************************************************************
672  *                                                                           *
673  *           64K SW & 64K HW in a 64K segment pages implementation           *
674  *                                                                           *
675  *****************************************************************************/
676
677 _GLOBAL(__hash_page_64K)
678         mflr    r0
679         std     r0,16(r1)
680         stdu    r1,-STACKFRAMESIZE(r1)
681         /* Save all params that we need after a function call */
682         std     r6,STK_PARM(r6)(r1)
683         std     r8,STK_PARM(r8)(r1)
684         std     r9,STK_PARM(r9)(r1)
685
686         /* Add _PAGE_PRESENT to access */
687         ori     r4,r4,_PAGE_PRESENT
688
689         /* Save non-volatile registers.
690          * r31 will hold "old PTE"
691          * r30 is "new PTE"
692          * r29 is "va"
693          * r28 is a hash value
694          * r27 is hashtab mask (maybe dynamic patched instead ?)
695          */
696         std     r27,STK_REG(r27)(r1)
697         std     r28,STK_REG(r28)(r1)
698         std     r29,STK_REG(r29)(r1)
699         std     r30,STK_REG(r30)(r1)
700         std     r31,STK_REG(r31)(r1)
701
702         /* Step 1:
703          *
704          * Check permissions, atomically mark the linux PTE busy
705          * and hashed.
706          */
707 1:
708         ldarx   r31,0,r6
709         /* Check access rights (access & ~(pte_val(*ptep))) */
710         andc.   r0,r4,r31
711         bne-    ht64_wrong_access
712         /* Check if PTE is busy */
713         andi.   r0,r31,_PAGE_BUSY
714         /* If so, just bail out and refault if needed. Someone else
715          * is changing this PTE anyway and might hash it.
716          */
717         bne-    ht64_bail_ok
718 BEGIN_FTR_SECTION
719         /* Check if PTE has the cache-inhibit bit set */
720         andi.   r0,r31,_PAGE_NO_CACHE
721         /* If so, bail out and refault as a 4k page */
722         bne-    ht64_bail_ok
723 END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
724         /* Prepare new PTE value (turn access RW into DIRTY, then
725          * add BUSY,HASHPTE and ACCESSED)
726          */
727         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
728         or      r30,r30,r31
729         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
730         /* Write the linux PTE atomically (setting busy) */
731         stdcx.  r30,0,r6
732         bne-    1b
733         isync
734
735         /* Step 2:
736          *
737          * Insert/Update the HPTE in the hash table. At this point,
738          * r4 (access) is re-useable, we use it for the new HPTE flags
739          */
740
741 BEGIN_FTR_SECTION
742         cmpdi   r9,0                    /* check segment size */
743         bne     3f
744 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
745         /* Calc va and put it in r29 */
746         rldicr  r29,r5,28,63-28
747         rldicl  r3,r3,0,36
748         or      r29,r3,r29
749
750         /* Calculate hash value for primary slot and store it in r28 */
751         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
752         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
753         xor     r28,r5,r0
754         b       4f
755
756 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
757         sldi    r29,r5,40               /* vsid << 40 */
758         clrldi  r3,r3,24                /* ea & 0xffffffffff */
759         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
760         clrldi  r5,r5,40                /* vsid & 0xffffff */
761         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
762         xor     r28,r28,r5
763         or      r29,r3,r29              /* VA */
764         xor     r28,r28,r0              /* hash */
765
766         /* Convert linux PTE bits into HW equivalents */
767 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
768         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
769         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
770         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
771         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
772         andc    r0,r30,r0               /* r0 = pte & ~r0 */
773         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
774         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
775
776         /* We eventually do the icache sync here (maybe inline that
777          * code rather than call a C function...)
778          */
779 BEGIN_FTR_SECTION
780         mr      r4,r30
781         mr      r5,r7
782         bl      .hash_page_do_lazy_icache
783 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
784
785         /* At this point, r3 contains new PP bits, save them in
786          * place of "access" in the param area (sic)
787          */
788         std     r3,STK_PARM(r4)(r1)
789
790         /* Get htab_hash_mask */
791         ld      r4,htab_hash_mask@got(2)
792         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
793
794         /* Check if we may already be in the hashtable, in this case, we
795          * go to out-of-line code to try to modify the HPTE
796          */
797         andi.   r0,r31,_PAGE_HASHPTE
798         bne     ht64_modify_pte
799
800 ht64_insert_pte:
801         /* Clear hpte bits in new pte (we also clear BUSY btw) and
802          * add _PAGE_HASHPTE
803          */
804         lis     r0,_PAGE_HPTEFLAGS@h
805         ori     r0,r0,_PAGE_HPTEFLAGS@l
806         andc    r30,r30,r0
807         ori     r30,r30,_PAGE_HASHPTE
808
809         /* Phyical address in r5 */
810         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
811         sldi    r5,r5,PAGE_SHIFT
812
813         /* Calculate primary group hash */
814         and     r0,r28,r27
815         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
816
817         /* Call ppc_md.hpte_insert */
818         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
819         mr      r4,r29                  /* Retreive va */
820         li      r7,0                    /* !bolted, !secondary */
821         li      r8,MMU_PAGE_64K
822         ld      r9,STK_PARM(r9)(r1)     /* segment size */
823 _GLOBAL(ht64_call_hpte_insert1)
824         bl      .                       /* patched by htab_finish_init() */
825         cmpdi   0,r3,0
826         bge     ht64_pte_insert_ok      /* Insertion successful */
827         cmpdi   0,r3,-2                 /* Critical failure */
828         beq-    ht64_pte_insert_failure
829
830         /* Now try secondary slot */
831
832         /* Phyical address in r5 */
833         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
834         sldi    r5,r5,PAGE_SHIFT
835
836         /* Calculate secondary group hash */
837         andc    r0,r27,r28
838         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
839
840         /* Call ppc_md.hpte_insert */
841         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
842         mr      r4,r29                  /* Retreive va */
843         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
844         li      r8,MMU_PAGE_64K
845         ld      r9,STK_PARM(r9)(r1)     /* segment size */
846 _GLOBAL(ht64_call_hpte_insert2)
847         bl      .                       /* patched by htab_finish_init() */
848         cmpdi   0,r3,0
849         bge+    ht64_pte_insert_ok      /* Insertion successful */
850         cmpdi   0,r3,-2                 /* Critical failure */
851         beq-    ht64_pte_insert_failure
852
853         /* Both are full, we need to evict something */
854         mftb    r0
855         /* Pick a random group based on TB */
856         andi.   r0,r0,1
857         mr      r5,r28
858         bne     2f
859         not     r5,r5
860 2:      and     r0,r5,r27
861         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
862         /* Call ppc_md.hpte_remove */
863 _GLOBAL(ht64_call_hpte_remove)
864         bl      .                       /* patched by htab_finish_init() */
865
866         /* Try all again */
867         b       ht64_insert_pte
868
869 ht64_bail_ok:
870         li      r3,0
871         b       ht64_bail
872
873 ht64_pte_insert_ok:
874         /* Insert slot number & secondary bit in PTE */
875         rldimi  r30,r3,12,63-15
876
877         /* Write out the PTE with a normal write
878          * (maybe add eieio may be good still ?)
879          */
880 ht64_write_out_pte:
881         ld      r6,STK_PARM(r6)(r1)
882         std     r30,0(r6)
883         li      r3, 0
884 ht64_bail:
885         ld      r27,STK_REG(r27)(r1)
886         ld      r28,STK_REG(r28)(r1)
887         ld      r29,STK_REG(r29)(r1)
888         ld      r30,STK_REG(r30)(r1)
889         ld      r31,STK_REG(r31)(r1)
890         addi    r1,r1,STACKFRAMESIZE
891         ld      r0,16(r1)
892         mtlr    r0
893         blr
894
895 ht64_modify_pte:
896         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
897         mr      r4,r3
898         rlwinm  r3,r31,32-12,29,31
899
900         /* Secondary group ? if yes, get a inverted hash value */
901         mr      r5,r28
902         andi.   r0,r31,_PAGE_F_SECOND
903         beq     1f
904         not     r5,r5
905 1:
906         /* Calculate proper slot value for ppc_md.hpte_updatepp */
907         and     r0,r5,r27
908         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
909         add     r3,r0,r3        /* add slot idx */
910
911         /* Call ppc_md.hpte_updatepp */
912         mr      r5,r29                  /* va */
913         li      r6,MMU_PAGE_64K
914         ld      r7,STK_PARM(r9)(r1)     /* segment size */
915         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
916 _GLOBAL(ht64_call_hpte_updatepp)
917         bl      .                       /* patched by htab_finish_init() */
918
919         /* if we failed because typically the HPTE wasn't really here
920          * we try an insertion.
921          */
922         cmpdi   0,r3,-1
923         beq-    ht64_insert_pte
924
925         /* Clear the BUSY bit and Write out the PTE */
926         li      r0,_PAGE_BUSY
927         andc    r30,r30,r0
928         b       ht64_write_out_pte
929
930 ht64_wrong_access:
931         /* Bail out clearing reservation */
932         stdcx.  r31,0,r6
933         li      r3,1
934         b       ht64_bail
935
936 ht64_pte_insert_failure:
937         /* Bail out restoring old PTE */
938         ld      r6,STK_PARM(r6)(r1)
939         std     r31,0(r6)
940         li      r3,-1
941         b       ht64_bail
942
943
944 #endif /* CONFIG_PPC_HAS_HASH_64K */
945
946
947 /*****************************************************************************
948  *                                                                           *
949  *           Huge pages implementation is in hugetlbpage.c                   *
950  *                                                                           *
951  *****************************************************************************/