[SPARC64]: Access TSB with physical addresses when possible.
[linux-2.6] / arch / sparc64 / kernel / ktlb.S
1 /* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2  *
3  * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
4  * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
5  * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
6  * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
7  */
8
9 #include <linux/config.h>
10 #include <asm/head.h>
11 #include <asm/asi.h>
12 #include <asm/page.h>
13 #include <asm/pgtable.h>
14 #include <asm/tsb.h>
15
16         .text
17         .align          32
18
19         .globl          kvmap_itlb
20 kvmap_itlb:
21         /* g6: TAG TARGET */
22         mov             TLB_TAG_ACCESS, %g4
23         ldxa            [%g4] ASI_IMMU, %g4
24
25 kvmap_itlb_nonlinear:
26         /* Catch kernel NULL pointer calls.  */
27         sethi           %hi(PAGE_SIZE), %g5
28         cmp             %g4, %g5
29         bleu,pn         %xcc, kvmap_dtlb_longpath
30          nop
31
32         KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
33
34 kvmap_itlb_tsb_miss:
35         sethi           %hi(LOW_OBP_ADDRESS), %g5
36         cmp             %g4, %g5
37         blu,pn          %xcc, kvmap_itlb_vmalloc_addr
38          mov            0x1, %g5
39         sllx            %g5, 32, %g5
40         cmp             %g4, %g5
41         blu,pn          %xcc, kvmap_itlb_obp
42          nop
43
44 kvmap_itlb_vmalloc_addr:
45         KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
46
47         KTSB_LOCK_TAG(%g1, %g2, %g4)
48
49         /* Load and check PTE.  */
50         ldxa            [%g5] ASI_PHYS_USE_EC, %g5
51         brgez,a,pn      %g5, kvmap_itlb_longpath
52          KTSB_STORE(%g1, %g0)
53
54         KTSB_WRITE(%g1, %g5, %g6)
55
56         /* fallthrough to TLB load */
57
58 kvmap_itlb_load:
59         stxa            %g5, [%g0] ASI_ITLB_DATA_IN     ! Reload TLB
60         retry
61
62 kvmap_itlb_longpath:
63         rdpr    %pstate, %g5
64         wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
65         rdpr    %tpc, %g5
66         ba,pt   %xcc, sparc64_realfault_common
67          mov    FAULT_CODE_ITLB, %g4
68
69 kvmap_itlb_obp:
70         OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
71
72         KTSB_LOCK_TAG(%g1, %g2, %g4)
73
74         KTSB_WRITE(%g1, %g5, %g6)
75
76         ba,pt           %xcc, kvmap_itlb_load
77          nop
78
79 kvmap_dtlb_obp:
80         OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
81
82         KTSB_LOCK_TAG(%g1, %g2, %g4)
83
84         KTSB_WRITE(%g1, %g5, %g6)
85
86         ba,pt           %xcc, kvmap_dtlb_load
87          nop
88
89         .align          32
90         .globl          kvmap_dtlb
91 kvmap_dtlb:
92         /* %g6: TAG TARGET */
93         mov             TLB_TAG_ACCESS, %g4
94         ldxa            [%g4] ASI_DMMU, %g4
95         brgez,pn        %g4, kvmap_dtlb_nonlinear
96          nop
97
98 #define KERN_HIGHBITS   ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
99 #define KERN_LOWBITS    (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
100
101         sethi           %uhi(KERN_HIGHBITS), %g2
102         or              %g2, %ulo(KERN_HIGHBITS), %g2
103         sllx            %g2, 32, %g2
104         or              %g2, KERN_LOWBITS, %g2
105
106 #undef KERN_HIGHBITS
107 #undef KERN_LOWBITS
108
109         .globl          kvmap_linear_patch
110 kvmap_linear_patch:
111         ba,pt           %xcc, kvmap_dtlb_load
112          xor            %g2, %g4, %g5
113
114 kvmap_dtlb_vmalloc_addr:
115         KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
116
117         KTSB_LOCK_TAG(%g1, %g2, %g4)
118
119         /* Load and check PTE.  */
120         ldxa            [%g5] ASI_PHYS_USE_EC, %g5
121         brgez,a,pn      %g5, kvmap_dtlb_longpath
122          KTSB_STORE(%g1, %g0)
123
124         KTSB_WRITE(%g1, %g5, %g6)
125
126         /* fallthrough to TLB load */
127
128 kvmap_dtlb_load:
129         stxa            %g5, [%g0] ASI_DTLB_DATA_IN     ! Reload TLB
130         retry
131
132 kvmap_dtlb_nonlinear:
133         /* Catch kernel NULL pointer derefs.  */
134         sethi           %hi(PAGE_SIZE), %g5
135         cmp             %g4, %g5
136         bleu,pn         %xcc, kvmap_dtlb_longpath
137          nop
138
139         KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
140
141 kvmap_dtlb_tsbmiss:
142         sethi           %hi(MODULES_VADDR), %g5
143         cmp             %g4, %g5
144         blu,pn          %xcc, kvmap_dtlb_longpath
145          mov            (VMALLOC_END >> 24), %g5
146         sllx            %g5, 24, %g5
147         cmp             %g4, %g5
148         bgeu,pn         %xcc, kvmap_dtlb_longpath
149          nop
150
151 kvmap_check_obp:
152         sethi           %hi(LOW_OBP_ADDRESS), %g5
153         cmp             %g4, %g5
154         blu,pn          %xcc, kvmap_dtlb_vmalloc_addr
155          mov            0x1, %g5
156         sllx            %g5, 32, %g5
157         cmp             %g4, %g5
158         blu,pn          %xcc, kvmap_dtlb_obp
159          nop
160         ba,pt           %xcc, kvmap_dtlb_vmalloc_addr
161          nop
162
163 kvmap_dtlb_longpath:
164         rdpr    %pstate, %g5
165         wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
166         rdpr    %tl, %g4
167         cmp     %g4, 1
168         mov     TLB_TAG_ACCESS, %g4
169         ldxa    [%g4] ASI_DMMU, %g5
170         be,pt   %xcc, sparc64_realfault_common
171          mov    FAULT_CODE_DTLB, %g4
172         ba,pt   %xcc, winfix_trampoline
173          nop