Merge git://git.infradead.org/battery-2.6
[linux-2.6] / arch / powerpc / mm / hash_low_64.S
1 /*
2  * ppc64 MMU hashtable management routines
3  *
4  * (c) Copyright IBM Corp. 2003, 2005
5  *
6  * Maintained by: Benjamin Herrenschmidt
7  *                <benh@kernel.crashing.org>
8  *
9  * This file is covered by the GNU Public Licence v2 as
10  * described in the kernel's COPYING file.
11  */
12
13 #include <asm/reg.h>
14 #include <asm/pgtable.h>
15 #include <asm/mmu.h>
16 #include <asm/page.h>
17 #include <asm/types.h>
18 #include <asm/ppc_asm.h>
19 #include <asm/asm-offsets.h>
20 #include <asm/cputable.h>
21
22         .text
23
24 /*
25  * Stackframe:
26  *              
27  *         +-> Back chain                       (SP + 256)
28  *         |   General register save area       (SP + 112)
29  *         |   Parameter save area              (SP + 48)
30  *         |   TOC save area                    (SP + 40)
31  *         |   link editor doubleword           (SP + 32)
32  *         |   compiler doubleword              (SP + 24)
33  *         |   LR save area                     (SP + 16)
34  *         |   CR save area                     (SP + 8)
35  * SP ---> +-- Back chain                       (SP + 0)
36  */
37 #define STACKFRAMESIZE  256
38
39 /* Save parameters offsets */
40 #define STK_PARM(i)     (STACKFRAMESIZE + 48 + ((i)-3)*8)
41
42 /* Save non-volatile offsets */
43 #define STK_REG(i)      (112 + ((i)-14)*8)
44
45
46 #ifndef CONFIG_PPC_64K_PAGES
47
48 /*****************************************************************************
49  *                                                                           *
50  *           4K SW & 4K HW pages implementation                              *
51  *                                                                           *
52  *****************************************************************************/
53
54
55 /*
56  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
57  *               pte_t *ptep, unsigned long trap, int local, int ssize)
58  *
59  * Adds a 4K page to the hash table in a segment of 4K pages only
60  */
61
62 _GLOBAL(__hash_page_4K)
63         mflr    r0
64         std     r0,16(r1)
65         stdu    r1,-STACKFRAMESIZE(r1)
66         /* Save all params that we need after a function call */
67         std     r6,STK_PARM(r6)(r1)
68         std     r8,STK_PARM(r8)(r1)
69         std     r9,STK_PARM(r9)(r1)
70         
71         /* Add _PAGE_PRESENT to access */
72         ori     r4,r4,_PAGE_PRESENT
73
74         /* Save non-volatile registers.
75          * r31 will hold "old PTE"
76          * r30 is "new PTE"
77          * r29 is "va"
78          * r28 is a hash value
79          * r27 is hashtab mask (maybe dynamic patched instead ?)
80          */
81         std     r27,STK_REG(r27)(r1)
82         std     r28,STK_REG(r28)(r1)
83         std     r29,STK_REG(r29)(r1)
84         std     r30,STK_REG(r30)(r1)
85         std     r31,STK_REG(r31)(r1)
86         
87         /* Step 1:
88          *
89          * Check permissions, atomically mark the linux PTE busy
90          * and hashed.
91          */ 
92 1:
93         ldarx   r31,0,r6
94         /* Check access rights (access & ~(pte_val(*ptep))) */
95         andc.   r0,r4,r31
96         bne-    htab_wrong_access
97         /* Check if PTE is busy */
98         andi.   r0,r31,_PAGE_BUSY
99         /* If so, just bail out and refault if needed. Someone else
100          * is changing this PTE anyway and might hash it.
101          */
102         bne-    htab_bail_ok
103
104         /* Prepare new PTE value (turn access RW into DIRTY, then
105          * add BUSY,HASHPTE and ACCESSED)
106          */
107         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108         or      r30,r30,r31
109         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110         /* Write the linux PTE atomically (setting busy) */
111         stdcx.  r30,0,r6
112         bne-    1b
113         isync
114
115         /* Step 2:
116          *
117          * Insert/Update the HPTE in the hash table. At this point,
118          * r4 (access) is re-useable, we use it for the new HPTE flags
119          */
120
121 BEGIN_FTR_SECTION
122         cmpdi   r9,0                    /* check segment size */
123         bne     3f
124 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
125         /* Calc va and put it in r29 */
126         rldicr  r29,r5,28,63-28
127         rldicl  r3,r3,0,36
128         or      r29,r3,r29
129
130         /* Calculate hash value for primary slot and store it in r28 */
131         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
132         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
133         xor     r28,r5,r0
134         b       4f
135
136 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
137         sldi    r29,r5,40               /* vsid << 40 */
138         clrldi  r3,r3,24                /* ea & 0xffffffffff */
139         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
140         clrldi  r5,r5,40                /* vsid & 0xffffff */
141         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
142         xor     r28,r28,r5
143         or      r29,r3,r29              /* VA */
144         xor     r28,r28,r0              /* hash */
145
146         /* Convert linux PTE bits into HW equivalents */
147 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
148         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
149         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
150         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
151         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
152         andc    r0,r30,r0               /* r0 = pte & ~r0 */
153         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
154         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
155
156         /* We eventually do the icache sync here (maybe inline that
157          * code rather than call a C function...) 
158          */
159 BEGIN_FTR_SECTION
160         mr      r4,r30
161         mr      r5,r7
162         bl      .hash_page_do_lazy_icache
163 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
164
165         /* At this point, r3 contains new PP bits, save them in
166          * place of "access" in the param area (sic)
167          */
168         std     r3,STK_PARM(r4)(r1)
169
170         /* Get htab_hash_mask */
171         ld      r4,htab_hash_mask@got(2)
172         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
173
174         /* Check if we may already be in the hashtable, in this case, we
175          * go to out-of-line code to try to modify the HPTE
176          */
177         andi.   r0,r31,_PAGE_HASHPTE
178         bne     htab_modify_pte
179
180 htab_insert_pte:
181         /* Clear hpte bits in new pte (we also clear BUSY btw) and
182          * add _PAGE_HASHPTE
183          */
184         lis     r0,_PAGE_HPTEFLAGS@h
185         ori     r0,r0,_PAGE_HPTEFLAGS@l
186         andc    r30,r30,r0
187         ori     r30,r30,_PAGE_HASHPTE
188
189         /* physical address r5 */
190         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
191         sldi    r5,r5,PAGE_SHIFT
192
193         /* Calculate primary group hash */
194         and     r0,r28,r27
195         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
196
197         /* Call ppc_md.hpte_insert */
198         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
199         mr      r4,r29                  /* Retreive va */
200         li      r7,0                    /* !bolted, !secondary */
201         li      r8,MMU_PAGE_4K          /* page size */
202         ld      r9,STK_PARM(r9)(r1)     /* segment size */
203 _GLOBAL(htab_call_hpte_insert1)
204         bl      .                       /* Patched by htab_finish_init() */
205         cmpdi   0,r3,0
206         bge     htab_pte_insert_ok      /* Insertion successful */
207         cmpdi   0,r3,-2                 /* Critical failure */
208         beq-    htab_pte_insert_failure
209
210         /* Now try secondary slot */
211         
212         /* physical address r5 */
213         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
214         sldi    r5,r5,PAGE_SHIFT
215
216         /* Calculate secondary group hash */
217         andc    r0,r27,r28
218         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
219         
220         /* Call ppc_md.hpte_insert */
221         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
222         mr      r4,r29                  /* Retreive va */
223         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
224         li      r8,MMU_PAGE_4K          /* page size */
225         ld      r9,STK_PARM(r9)(r1)     /* segment size */
226 _GLOBAL(htab_call_hpte_insert2)
227         bl      .                       /* Patched by htab_finish_init() */
228         cmpdi   0,r3,0
229         bge+    htab_pte_insert_ok      /* Insertion successful */
230         cmpdi   0,r3,-2                 /* Critical failure */
231         beq-    htab_pte_insert_failure
232
233         /* Both are full, we need to evict something */
234         mftb    r0
235         /* Pick a random group based on TB */
236         andi.   r0,r0,1
237         mr      r5,r28
238         bne     2f
239         not     r5,r5
240 2:      and     r0,r5,r27
241         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */   
242         /* Call ppc_md.hpte_remove */
243 _GLOBAL(htab_call_hpte_remove)
244         bl      .                       /* Patched by htab_finish_init() */
245
246         /* Try all again */
247         b       htab_insert_pte 
248
249 htab_bail_ok:
250         li      r3,0
251         b       htab_bail
252
253 htab_pte_insert_ok:
254         /* Insert slot number & secondary bit in PTE */
255         rldimi  r30,r3,12,63-15
256                 
257         /* Write out the PTE with a normal write
258          * (maybe add eieio may be good still ?)
259          */
260 htab_write_out_pte:
261         ld      r6,STK_PARM(r6)(r1)
262         std     r30,0(r6)
263         li      r3, 0
264 htab_bail:
265         ld      r27,STK_REG(r27)(r1)
266         ld      r28,STK_REG(r28)(r1)
267         ld      r29,STK_REG(r29)(r1)
268         ld      r30,STK_REG(r30)(r1)
269         ld      r31,STK_REG(r31)(r1)
270         addi    r1,r1,STACKFRAMESIZE
271         ld      r0,16(r1)
272         mtlr    r0
273         blr
274
275 htab_modify_pte:
276         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
277         mr      r4,r3
278         rlwinm  r3,r31,32-12,29,31
279
280         /* Secondary group ? if yes, get a inverted hash value */
281         mr      r5,r28
282         andi.   r0,r31,_PAGE_SECONDARY
283         beq     1f
284         not     r5,r5
285 1:
286         /* Calculate proper slot value for ppc_md.hpte_updatepp */
287         and     r0,r5,r27
288         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
289         add     r3,r0,r3        /* add slot idx */
290
291         /* Call ppc_md.hpte_updatepp */
292         mr      r5,r29                  /* va */
293         li      r6,MMU_PAGE_4K          /* page size */
294         ld      r7,STK_PARM(r9)(r1)     /* segment size */
295         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
296 _GLOBAL(htab_call_hpte_updatepp)
297         bl      .                       /* Patched by htab_finish_init() */
298
299         /* if we failed because typically the HPTE wasn't really here
300          * we try an insertion. 
301          */
302         cmpdi   0,r3,-1
303         beq-    htab_insert_pte
304
305         /* Clear the BUSY bit and Write out the PTE */
306         li      r0,_PAGE_BUSY
307         andc    r30,r30,r0
308         b       htab_write_out_pte
309
310 htab_wrong_access:
311         /* Bail out clearing reservation */
312         stdcx.  r31,0,r6
313         li      r3,1
314         b       htab_bail
315
316 htab_pte_insert_failure:
317         /* Bail out restoring old PTE */
318         ld      r6,STK_PARM(r6)(r1)
319         std     r31,0(r6)
320         li      r3,-1
321         b       htab_bail
322
323
324 #else /* CONFIG_PPC_64K_PAGES */
325
326
327 /*****************************************************************************
328  *                                                                           *
329  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
330  *                                                                           *
331  *****************************************************************************/
332
333 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
334  *               pte_t *ptep, unsigned long trap, int local)
335  */
336
337 /*
338  * For now, we do NOT implement Admixed pages
339  */
340 _GLOBAL(__hash_page_4K)
341         mflr    r0
342         std     r0,16(r1)
343         stdu    r1,-STACKFRAMESIZE(r1)
344         /* Save all params that we need after a function call */
345         std     r6,STK_PARM(r6)(r1)
346         std     r8,STK_PARM(r8)(r1)
347         std     r9,STK_PARM(r9)(r1)
348
349         /* Add _PAGE_PRESENT to access */
350         ori     r4,r4,_PAGE_PRESENT
351
352         /* Save non-volatile registers.
353          * r31 will hold "old PTE"
354          * r30 is "new PTE"
355          * r29 is "va"
356          * r28 is a hash value
357          * r27 is hashtab mask (maybe dynamic patched instead ?)
358          * r26 is the hidx mask
359          * r25 is the index in combo page
360          */
361         std     r25,STK_REG(r25)(r1)
362         std     r26,STK_REG(r26)(r1)
363         std     r27,STK_REG(r27)(r1)
364         std     r28,STK_REG(r28)(r1)
365         std     r29,STK_REG(r29)(r1)
366         std     r30,STK_REG(r30)(r1)
367         std     r31,STK_REG(r31)(r1)
368
369         /* Step 1:
370          *
371          * Check permissions, atomically mark the linux PTE busy
372          * and hashed.
373          */
374 1:
375         ldarx   r31,0,r6
376         /* Check access rights (access & ~(pte_val(*ptep))) */
377         andc.   r0,r4,r31
378         bne-    htab_wrong_access
379         /* Check if PTE is busy */
380         andi.   r0,r31,_PAGE_BUSY
381         /* If so, just bail out and refault if needed. Someone else
382          * is changing this PTE anyway and might hash it.
383          */
384         bne-    htab_bail_ok
385         /* Prepare new PTE value (turn access RW into DIRTY, then
386          * add BUSY and ACCESSED)
387          */
388         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
389         or      r30,r30,r31
390         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
391         oris    r30,r30,_PAGE_COMBO@h
392         /* Write the linux PTE atomically (setting busy) */
393         stdcx.  r30,0,r6
394         bne-    1b
395         isync
396
397         /* Step 2:
398          *
399          * Insert/Update the HPTE in the hash table. At this point,
400          * r4 (access) is re-useable, we use it for the new HPTE flags
401          */
402
403         /* Load the hidx index */
404         rldicl  r25,r3,64-12,60
405
406 BEGIN_FTR_SECTION
407         cmpdi   r9,0                    /* check segment size */
408         bne     3f
409 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
410         /* Calc va and put it in r29 */
411         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
412         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
413         or      r29,r3,r29              /* r29 = va */
414
415         /* Calculate hash value for primary slot and store it in r28 */
416         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
417         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
418         xor     r28,r5,r0
419         b       4f
420
421 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
422         sldi    r29,r5,40               /* vsid << 40 */
423         clrldi  r3,r3,24                /* ea & 0xffffffffff */
424         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
425         clrldi  r5,r5,40                /* vsid & 0xffffff */
426         rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
427         xor     r28,r28,r5
428         or      r29,r3,r29              /* VA */
429         xor     r28,r28,r0              /* hash */
430
431         /* Convert linux PTE bits into HW equivalents */
432 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
433         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
434         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
435         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
436         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
437         andc    r0,r30,r0               /* r0 = pte & ~r0 */
438         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
439         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
440
441         /* We eventually do the icache sync here (maybe inline that
442          * code rather than call a C function...)
443          */
444 BEGIN_FTR_SECTION
445         mr      r4,r30
446         mr      r5,r7
447         bl      .hash_page_do_lazy_icache
448 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
449
450         /* At this point, r3 contains new PP bits, save them in
451          * place of "access" in the param area (sic)
452          */
453         std     r3,STK_PARM(r4)(r1)
454
455         /* Get htab_hash_mask */
456         ld      r4,htab_hash_mask@got(2)
457         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
458
459         /* Check if we may already be in the hashtable, in this case, we
460          * go to out-of-line code to try to modify the HPTE. We look for
461          * the bit at (1 >> (index + 32))
462          */
463         andi.   r0,r31,_PAGE_HASHPTE
464         li      r26,0                   /* Default hidx */
465         beq     htab_insert_pte
466
467         /*
468          * Check if the pte was already inserted into the hash table
469          * as a 64k HW page, and invalidate the 64k HPTE if so.
470          */
471         andis.  r0,r31,_PAGE_COMBO@h
472         beq     htab_inval_old_hpte
473
474         ld      r6,STK_PARM(r6)(r1)
475         ori     r26,r6,0x8000           /* Load the hidx mask */
476         ld      r26,0(r26)
477         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
478         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
479         bne     htab_modify_pte
480
481 htab_insert_pte:
482         /* real page number in r5, PTE RPN value + index */
483         andis.  r0,r31,_PAGE_4K_PFN@h
484         srdi    r5,r31,PTE_RPN_SHIFT
485         bne-    htab_special_pfn
486         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
487         add     r5,r5,r25
488 htab_special_pfn:
489         sldi    r5,r5,HW_PAGE_SHIFT
490
491         /* Calculate primary group hash */
492         and     r0,r28,r27
493         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
494
495         /* Call ppc_md.hpte_insert */
496         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
497         mr      r4,r29                  /* Retreive va */
498         li      r7,0                    /* !bolted, !secondary */
499         li      r8,MMU_PAGE_4K          /* page size */
500         ld      r9,STK_PARM(r9)(r1)     /* segment size */
501 _GLOBAL(htab_call_hpte_insert1)
502         bl      .                       /* patched by htab_finish_init() */
503         cmpdi   0,r3,0
504         bge     htab_pte_insert_ok      /* Insertion successful */
505         cmpdi   0,r3,-2                 /* Critical failure */
506         beq-    htab_pte_insert_failure
507
508         /* Now try secondary slot */
509
510         /* real page number in r5, PTE RPN value + index */
511         andis.  r0,r31,_PAGE_4K_PFN@h
512         srdi    r5,r31,PTE_RPN_SHIFT
513         bne-    3f
514         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
515         add     r5,r5,r25
516 3:      sldi    r5,r5,HW_PAGE_SHIFT
517
518         /* Calculate secondary group hash */
519         andc    r0,r27,r28
520         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
521
522         /* Call ppc_md.hpte_insert */
523         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
524         mr      r4,r29                  /* Retreive va */
525         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
526         li      r8,MMU_PAGE_4K          /* page size */
527         ld      r9,STK_PARM(r9)(r1)     /* segment size */
528 _GLOBAL(htab_call_hpte_insert2)
529         bl      .                       /* patched by htab_finish_init() */
530         cmpdi   0,r3,0
531         bge+    htab_pte_insert_ok      /* Insertion successful */
532         cmpdi   0,r3,-2                 /* Critical failure */
533         beq-    htab_pte_insert_failure
534
535         /* Both are full, we need to evict something */
536         mftb    r0
537         /* Pick a random group based on TB */
538         andi.   r0,r0,1
539         mr      r5,r28
540         bne     2f
541         not     r5,r5
542 2:      and     r0,r5,r27
543         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
544         /* Call ppc_md.hpte_remove */
545 _GLOBAL(htab_call_hpte_remove)
546         bl      .                       /* patched by htab_finish_init() */
547
548         /* Try all again */
549         b       htab_insert_pte
550
551         /*
552          * Call out to C code to invalidate an 64k HW HPTE that is
553          * useless now that the segment has been switched to 4k pages.
554          */
555 htab_inval_old_hpte:
556         mr      r3,r29                  /* virtual addr */
557         mr      r4,r31                  /* PTE.pte */
558         li      r5,0                    /* PTE.hidx */
559         li      r6,MMU_PAGE_64K         /* psize */
560         ld      r7,STK_PARM(r8)(r1)     /* local */
561         bl      .flush_hash_page
562         b       htab_insert_pte
563         
564 htab_bail_ok:
565         li      r3,0
566         b       htab_bail
567
568 htab_pte_insert_ok:
569         /* Insert slot number & secondary bit in PTE second half,
570          * clear _PAGE_BUSY and set approriate HPTE slot bit
571          */
572         ld      r6,STK_PARM(r6)(r1)
573         li      r0,_PAGE_BUSY
574         andc    r30,r30,r0
575         /* HPTE SUB bit */
576         li      r0,1
577         subfic  r5,r25,27               /* Must match bit position in */
578         sld     r0,r0,r5                /* pgtable.h */
579         or      r30,r30,r0
580         /* hindx */
581         sldi    r5,r25,2
582         sld     r3,r3,r5
583         li      r4,0xf
584         sld     r4,r4,r5
585         andc    r26,r26,r4
586         or      r26,r26,r3
587         ori     r5,r6,0x8000
588         std     r26,0(r5)
589         lwsync
590         std     r30,0(r6)
591         li      r3, 0
592 htab_bail:
593         ld      r25,STK_REG(r25)(r1)
594         ld      r26,STK_REG(r26)(r1)
595         ld      r27,STK_REG(r27)(r1)
596         ld      r28,STK_REG(r28)(r1)
597         ld      r29,STK_REG(r29)(r1)
598         ld      r30,STK_REG(r30)(r1)
599         ld      r31,STK_REG(r31)(r1)
600         addi    r1,r1,STACKFRAMESIZE
601         ld      r0,16(r1)
602         mtlr    r0
603         blr
604
605 htab_modify_pte:
606         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
607         mr      r4,r3
608         sldi    r5,r25,2
609         srd     r3,r26,r5
610
611         /* Secondary group ? if yes, get a inverted hash value */
612         mr      r5,r28
613         andi.   r0,r3,0x8 /* page secondary ? */
614         beq     1f
615         not     r5,r5
616 1:      andi.   r3,r3,0x7 /* extract idx alone */
617
618         /* Calculate proper slot value for ppc_md.hpte_updatepp */
619         and     r0,r5,r27
620         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
621         add     r3,r0,r3        /* add slot idx */
622
623         /* Call ppc_md.hpte_updatepp */
624         mr      r5,r29                  /* va */
625         li      r6,MMU_PAGE_4K          /* page size */
626         ld      r7,STK_PARM(r9)(r1)     /* segment size */
627         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
628 _GLOBAL(htab_call_hpte_updatepp)
629         bl      .                       /* patched by htab_finish_init() */
630
631         /* if we failed because typically the HPTE wasn't really here
632          * we try an insertion.
633          */
634         cmpdi   0,r3,-1
635         beq-    htab_insert_pte
636
637         /* Clear the BUSY bit and Write out the PTE */
638         li      r0,_PAGE_BUSY
639         andc    r30,r30,r0
640         ld      r6,STK_PARM(r6)(r1)
641         std     r30,0(r6)
642         li      r3,0
643         b       htab_bail
644
645 htab_wrong_access:
646         /* Bail out clearing reservation */
647         stdcx.  r31,0,r6
648         li      r3,1
649         b       htab_bail
650
651 htab_pte_insert_failure:
652         /* Bail out restoring old PTE */
653         ld      r6,STK_PARM(r6)(r1)
654         std     r31,0(r6)
655         li      r3,-1
656         b       htab_bail
657
658 #endif /* CONFIG_PPC_64K_PAGES */
659
660 #ifdef CONFIG_PPC_HAS_HASH_64K
661
662 /*****************************************************************************
663  *                                                                           *
664  *           64K SW & 64K HW in a 64K segment pages implementation           *
665  *                                                                           *
666  *****************************************************************************/
667
668 _GLOBAL(__hash_page_64K)
669         mflr    r0
670         std     r0,16(r1)
671         stdu    r1,-STACKFRAMESIZE(r1)
672         /* Save all params that we need after a function call */
673         std     r6,STK_PARM(r6)(r1)
674         std     r8,STK_PARM(r8)(r1)
675         std     r9,STK_PARM(r9)(r1)
676
677         /* Add _PAGE_PRESENT to access */
678         ori     r4,r4,_PAGE_PRESENT
679
680         /* Save non-volatile registers.
681          * r31 will hold "old PTE"
682          * r30 is "new PTE"
683          * r29 is "va"
684          * r28 is a hash value
685          * r27 is hashtab mask (maybe dynamic patched instead ?)
686          */
687         std     r27,STK_REG(r27)(r1)
688         std     r28,STK_REG(r28)(r1)
689         std     r29,STK_REG(r29)(r1)
690         std     r30,STK_REG(r30)(r1)
691         std     r31,STK_REG(r31)(r1)
692
693         /* Step 1:
694          *
695          * Check permissions, atomically mark the linux PTE busy
696          * and hashed.
697          */
698 1:
699         ldarx   r31,0,r6
700         /* Check access rights (access & ~(pte_val(*ptep))) */
701         andc.   r0,r4,r31
702         bne-    ht64_wrong_access
703         /* Check if PTE is busy */
704         andi.   r0,r31,_PAGE_BUSY
705         /* If so, just bail out and refault if needed. Someone else
706          * is changing this PTE anyway and might hash it.
707          */
708         bne-    ht64_bail_ok
709 BEGIN_FTR_SECTION
710         /* Check if PTE has the cache-inhibit bit set */
711         andi.   r0,r31,_PAGE_NO_CACHE
712         /* If so, bail out and refault as a 4k page */
713         bne-    ht64_bail_ok
714 END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
715         /* Prepare new PTE value (turn access RW into DIRTY, then
716          * add BUSY,HASHPTE and ACCESSED)
717          */
718         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
719         or      r30,r30,r31
720         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
721         /* Write the linux PTE atomically (setting busy) */
722         stdcx.  r30,0,r6
723         bne-    1b
724         isync
725
726         /* Step 2:
727          *
728          * Insert/Update the HPTE in the hash table. At this point,
729          * r4 (access) is re-useable, we use it for the new HPTE flags
730          */
731
732 BEGIN_FTR_SECTION
733         cmpdi   r9,0                    /* check segment size */
734         bne     3f
735 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
736         /* Calc va and put it in r29 */
737         rldicr  r29,r5,28,63-28
738         rldicl  r3,r3,0,36
739         or      r29,r3,r29
740
741         /* Calculate hash value for primary slot and store it in r28 */
742         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
743         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
744         xor     r28,r5,r0
745         b       4f
746
747 3:      /* Calc VA and hash in r29 and r28 for 1T segment */
748         sldi    r29,r5,40               /* vsid << 40 */
749         clrldi  r3,r3,24                /* ea & 0xffffffffff */
750         rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
751         clrldi  r5,r5,40                /* vsid & 0xffffff */
752         rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
753         xor     r28,r28,r5
754         or      r29,r3,r29              /* VA */
755         xor     r28,r28,r0              /* hash */
756
757         /* Convert linux PTE bits into HW equivalents */
758 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
759         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
760         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
761         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
762         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
763         andc    r0,r30,r0               /* r0 = pte & ~r0 */
764         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
765         ori     r3,r3,HPTE_R_C          /* Always add "C" bit for perf. */
766
767         /* We eventually do the icache sync here (maybe inline that
768          * code rather than call a C function...)
769          */
770 BEGIN_FTR_SECTION
771         mr      r4,r30
772         mr      r5,r7
773         bl      .hash_page_do_lazy_icache
774 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
775
776         /* At this point, r3 contains new PP bits, save them in
777          * place of "access" in the param area (sic)
778          */
779         std     r3,STK_PARM(r4)(r1)
780
781         /* Get htab_hash_mask */
782         ld      r4,htab_hash_mask@got(2)
783         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
784
785         /* Check if we may already be in the hashtable, in this case, we
786          * go to out-of-line code to try to modify the HPTE
787          */
788         andi.   r0,r31,_PAGE_HASHPTE
789         bne     ht64_modify_pte
790
791 ht64_insert_pte:
792         /* Clear hpte bits in new pte (we also clear BUSY btw) and
793          * add _PAGE_HASHPTE
794          */
795         lis     r0,_PAGE_HPTEFLAGS@h
796         ori     r0,r0,_PAGE_HPTEFLAGS@l
797         andc    r30,r30,r0
798         ori     r30,r30,_PAGE_HASHPTE
799
800         /* Phyical address in r5 */
801         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
802         sldi    r5,r5,PAGE_SHIFT
803
804         /* Calculate primary group hash */
805         and     r0,r28,r27
806         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
807
808         /* Call ppc_md.hpte_insert */
809         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
810         mr      r4,r29                  /* Retreive va */
811         li      r7,0                    /* !bolted, !secondary */
812         li      r8,MMU_PAGE_64K
813         ld      r9,STK_PARM(r9)(r1)     /* segment size */
814 _GLOBAL(ht64_call_hpte_insert1)
815         bl      .                       /* patched by htab_finish_init() */
816         cmpdi   0,r3,0
817         bge     ht64_pte_insert_ok      /* Insertion successful */
818         cmpdi   0,r3,-2                 /* Critical failure */
819         beq-    ht64_pte_insert_failure
820
821         /* Now try secondary slot */
822
823         /* Phyical address in r5 */
824         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
825         sldi    r5,r5,PAGE_SHIFT
826
827         /* Calculate secondary group hash */
828         andc    r0,r27,r28
829         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
830
831         /* Call ppc_md.hpte_insert */
832         ld      r6,STK_PARM(r4)(r1)     /* Retreive new pp bits */
833         mr      r4,r29                  /* Retreive va */
834         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
835         li      r8,MMU_PAGE_64K
836         ld      r9,STK_PARM(r9)(r1)     /* segment size */
837 _GLOBAL(ht64_call_hpte_insert2)
838         bl      .                       /* patched by htab_finish_init() */
839         cmpdi   0,r3,0
840         bge+    ht64_pte_insert_ok      /* Insertion successful */
841         cmpdi   0,r3,-2                 /* Critical failure */
842         beq-    ht64_pte_insert_failure
843
844         /* Both are full, we need to evict something */
845         mftb    r0
846         /* Pick a random group based on TB */
847         andi.   r0,r0,1
848         mr      r5,r28
849         bne     2f
850         not     r5,r5
851 2:      and     r0,r5,r27
852         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
853         /* Call ppc_md.hpte_remove */
854 _GLOBAL(ht64_call_hpte_remove)
855         bl      .                       /* patched by htab_finish_init() */
856
857         /* Try all again */
858         b       ht64_insert_pte
859
860 ht64_bail_ok:
861         li      r3,0
862         b       ht64_bail
863
864 ht64_pte_insert_ok:
865         /* Insert slot number & secondary bit in PTE */
866         rldimi  r30,r3,12,63-15
867
868         /* Write out the PTE with a normal write
869          * (maybe add eieio may be good still ?)
870          */
871 ht64_write_out_pte:
872         ld      r6,STK_PARM(r6)(r1)
873         std     r30,0(r6)
874         li      r3, 0
875 ht64_bail:
876         ld      r27,STK_REG(r27)(r1)
877         ld      r28,STK_REG(r28)(r1)
878         ld      r29,STK_REG(r29)(r1)
879         ld      r30,STK_REG(r30)(r1)
880         ld      r31,STK_REG(r31)(r1)
881         addi    r1,r1,STACKFRAMESIZE
882         ld      r0,16(r1)
883         mtlr    r0
884         blr
885
886 ht64_modify_pte:
887         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
888         mr      r4,r3
889         rlwinm  r3,r31,32-12,29,31
890
891         /* Secondary group ? if yes, get a inverted hash value */
892         mr      r5,r28
893         andi.   r0,r31,_PAGE_F_SECOND
894         beq     1f
895         not     r5,r5
896 1:
897         /* Calculate proper slot value for ppc_md.hpte_updatepp */
898         and     r0,r5,r27
899         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
900         add     r3,r0,r3        /* add slot idx */
901
902         /* Call ppc_md.hpte_updatepp */
903         mr      r5,r29                  /* va */
904         li      r6,MMU_PAGE_64K
905         ld      r7,STK_PARM(r9)(r1)     /* segment size */
906         ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
907 _GLOBAL(ht64_call_hpte_updatepp)
908         bl      .                       /* patched by htab_finish_init() */
909
910         /* if we failed because typically the HPTE wasn't really here
911          * we try an insertion.
912          */
913         cmpdi   0,r3,-1
914         beq-    ht64_insert_pte
915
916         /* Clear the BUSY bit and Write out the PTE */
917         li      r0,_PAGE_BUSY
918         andc    r30,r30,r0
919         b       ht64_write_out_pte
920
921 ht64_wrong_access:
922         /* Bail out clearing reservation */
923         stdcx.  r31,0,r6
924         li      r3,1
925         b       ht64_bail
926
927 ht64_pte_insert_failure:
928         /* Bail out restoring old PTE */
929         ld      r6,STK_PARM(r6)(r1)
930         std     r31,0(r6)
931         li      r3,-1
932         b       ht64_bail
933
934
935 #endif /* CONFIG_PPC_HAS_HASH_64K */
936
937
938 /*****************************************************************************
939  *                                                                           *
940  *           Huge pages implementation is in hugetlbpage.c                   *
941  *                                                                           *
942  *****************************************************************************/