Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6
[linux-2.6] / arch / x86 / mm / pat.c
1 /*
2  * Handle caching attributes in page tables (PAT)
3  *
4  * Authors: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
5  *          Suresh B Siddha <suresh.b.siddha@intel.com>
6  *
7  * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
8  */
9
10 #include <linux/mm.h>
11 #include <linux/kernel.h>
12 #include <linux/gfp.h>
13 #include <linux/fs.h>
14
15 #include <asm/msr.h>
16 #include <asm/tlbflush.h>
17 #include <asm/processor.h>
18 #include <asm/pgtable.h>
19 #include <asm/pat.h>
20 #include <asm/e820.h>
21 #include <asm/cacheflush.h>
22 #include <asm/fcntl.h>
23 #include <asm/mtrr.h>
24
25 int pat_wc_enabled = 1;
26
27 static u64 __read_mostly boot_pat_state;
28
29 static int nopat(char *str)
30 {
31         pat_wc_enabled = 0;
32         printk(KERN_INFO "x86: PAT support disabled.\n");
33
34         return 0;
35 }
36 early_param("nopat", nopat);
37
38 static int pat_known_cpu(void)
39 {
40         if (!pat_wc_enabled)
41                 return 0;
42
43         if (cpu_has_pat)
44                 return 1;
45
46         pat_wc_enabled = 0;
47         printk(KERN_INFO "CPU and/or kernel does not support PAT.\n");
48         return 0;
49 }
50
51 enum {
52         PAT_UC = 0,             /* uncached */
53         PAT_WC = 1,             /* Write combining */
54         PAT_WT = 4,             /* Write Through */
55         PAT_WP = 5,             /* Write Protected */
56         PAT_WB = 6,             /* Write Back (default) */
57         PAT_UC_MINUS = 7,       /* UC, but can be overriden by MTRR */
58 };
59
60 #define PAT(x,y)        ((u64)PAT_ ## y << ((x)*8))
61
62 void pat_init(void)
63 {
64         u64 pat;
65
66 #ifndef CONFIG_X86_PAT
67         nopat(NULL);
68 #endif
69
70         /* Boot CPU enables PAT based on CPU feature */
71         if (!smp_processor_id() && !pat_known_cpu())
72                 return;
73
74         /* APs enable PAT iff boot CPU has enabled it before */
75         if (smp_processor_id() && !pat_wc_enabled)
76                 return;
77
78         /* Set PWT to Write-Combining. All other bits stay the same */
79         /*
80          * PTE encoding used in Linux:
81          *      PAT
82          *      |PCD
83          *      ||PWT
84          *      |||
85          *      000 WB          _PAGE_CACHE_WB
86          *      001 WC          _PAGE_CACHE_WC
87          *      010 UC-         _PAGE_CACHE_UC_MINUS
88          *      011 UC          _PAGE_CACHE_UC
89          * PAT bit unused
90          */
91         pat = PAT(0,WB) | PAT(1,WC) | PAT(2,UC_MINUS) | PAT(3,UC) |
92               PAT(4,WB) | PAT(5,WC) | PAT(6,UC_MINUS) | PAT(7,UC);
93
94         /* Boot CPU check */
95         if (!smp_processor_id()) {
96                 rdmsrl(MSR_IA32_CR_PAT, boot_pat_state);
97         }
98
99         wrmsrl(MSR_IA32_CR_PAT, pat);
100         printk(KERN_INFO "x86 PAT enabled: cpu %d, old 0x%Lx, new 0x%Lx\n",
101                smp_processor_id(), boot_pat_state, pat);
102 }
103
104 #undef PAT
105
106 static char *cattr_name(unsigned long flags)
107 {
108         switch (flags & _PAGE_CACHE_MASK) {
109                 case _PAGE_CACHE_UC:            return "uncached";
110                 case _PAGE_CACHE_UC_MINUS:      return "uncached-minus";
111                 case _PAGE_CACHE_WB:            return "write-back";
112                 case _PAGE_CACHE_WC:            return "write-combining";
113                 default:                        return "broken";
114         }
115 }
116
117 /*
118  * The global memtype list keeps track of memory type for specific
119  * physical memory areas. Conflicting memory types in different
120  * mappings can cause CPU cache corruption. To avoid this we keep track.
121  *
122  * The list is sorted based on starting address and can contain multiple
123  * entries for each address (this allows reference counting for overlapping
124  * areas). All the aliases have the same cache attributes of course.
125  * Zero attributes are represented as holes.
126  *
127  * Currently the data structure is a list because the number of mappings
128  * are expected to be relatively small. If this should be a problem
129  * it could be changed to a rbtree or similar.
130  *
131  * memtype_lock protects the whole list.
132  */
133
134 struct memtype {
135         u64 start;
136         u64 end;
137         unsigned long type;
138         struct list_head nd;
139 };
140
141 static LIST_HEAD(memtype_list);
142 static DEFINE_SPINLOCK(memtype_lock);   /* protects memtype list */
143
144 /*
145  * Does intersection of PAT memory type and MTRR memory type and returns
146  * the resulting memory type as PAT understands it.
147  * (Type in pat and mtrr will not have same value)
148  * The intersection is based on "Effective Memory Type" tables in IA-32
149  * SDM vol 3a
150  */
151 static int pat_x_mtrr_type(u64 start, u64 end, unsigned long prot,
152                                 unsigned long *ret_prot)
153 {
154         unsigned long pat_type;
155         u8 mtrr_type;
156
157         mtrr_type = mtrr_type_lookup(start, end);
158         if (mtrr_type == 0xFF) {                /* MTRR not enabled */
159                 *ret_prot = prot;
160                 return 0;
161         }
162         if (mtrr_type == 0xFE) {                /* MTRR match error */
163                 *ret_prot = _PAGE_CACHE_UC;
164                 return -1;
165         }
166         if (mtrr_type != MTRR_TYPE_UNCACHABLE &&
167             mtrr_type != MTRR_TYPE_WRBACK &&
168             mtrr_type != MTRR_TYPE_WRCOMB) {    /* MTRR type unhandled */
169                 *ret_prot = _PAGE_CACHE_UC;
170                 return -1;
171         }
172
173         pat_type = prot & _PAGE_CACHE_MASK;
174         prot &= (~_PAGE_CACHE_MASK);
175
176         /* Currently doing intersection by hand. Optimize it later. */
177         if (pat_type == _PAGE_CACHE_WC) {
178                 *ret_prot = prot | _PAGE_CACHE_WC;
179         } else if (pat_type == _PAGE_CACHE_UC_MINUS) {
180                 *ret_prot = prot | _PAGE_CACHE_UC_MINUS;
181         } else if (pat_type == _PAGE_CACHE_UC ||
182                    mtrr_type == MTRR_TYPE_UNCACHABLE) {
183                 *ret_prot = prot | _PAGE_CACHE_UC;
184         } else if (mtrr_type == MTRR_TYPE_WRCOMB) {
185                 *ret_prot = prot | _PAGE_CACHE_WC;
186         } else {
187                 *ret_prot = prot | _PAGE_CACHE_WB;
188         }
189
190         return 0;
191 }
192
193 int reserve_memtype(u64 start, u64 end, unsigned long req_type,
194                         unsigned long *ret_type)
195 {
196         struct memtype *new_entry = NULL;
197         struct memtype *parse;
198         unsigned long actual_type;
199         int err = 0;
200
201         /* Only track when pat_wc_enabled */
202         if (!pat_wc_enabled) {
203                 if (ret_type)
204                         *ret_type = req_type;
205
206                 return 0;
207         }
208
209         /* Low ISA region is always mapped WB in page table. No need to track */
210         if (start >= ISA_START_ADDRESS && (end - 1) <= ISA_END_ADDRESS) {
211                 if (ret_type)
212                         *ret_type = _PAGE_CACHE_WB;
213
214                 return 0;
215         }
216
217         req_type &= _PAGE_CACHE_MASK;
218         err = pat_x_mtrr_type(start, end, req_type, &actual_type);
219         if (err) {
220                 if (ret_type)
221                         *ret_type = actual_type;
222
223                 return -EINVAL;
224         }
225
226         new_entry  = kmalloc(sizeof(struct memtype), GFP_KERNEL);
227         if (!new_entry)
228                 return -ENOMEM;
229
230         new_entry->start = start;
231         new_entry->end = end;
232         new_entry->type = actual_type;
233
234         if (ret_type)
235                 *ret_type = actual_type;
236
237         spin_lock(&memtype_lock);
238
239         /* Search for existing mapping that overlaps the current range */
240         list_for_each_entry(parse, &memtype_list, nd) {
241                 struct memtype *saved_ptr;
242
243                 if (parse->start >= end) {
244                         printk("New Entry\n");
245                         list_add(&new_entry->nd, parse->nd.prev);
246                         new_entry = NULL;
247                         break;
248                 }
249
250                 if (start <= parse->start && end >= parse->start) {
251                         if (actual_type != parse->type && ret_type) {
252                                 actual_type = parse->type;
253                                 *ret_type = actual_type;
254                                 new_entry->type = actual_type;
255                         }
256
257                         if (actual_type != parse->type) {
258                                 printk(
259                 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
260                                         current->comm, current->pid,
261                                         start, end,
262                                         cattr_name(actual_type),
263                                         cattr_name(parse->type));
264                                 err = -EBUSY;
265                                 break;
266                         }
267
268                         saved_ptr = parse;
269                         /*
270                          * Check to see whether the request overlaps more
271                          * than one entry in the list
272                          */
273                         list_for_each_entry_continue(parse, &memtype_list, nd) {
274                                 if (end <= parse->start) {
275                                         break;
276                                 }
277
278                                 if (actual_type != parse->type) {
279                                         printk(
280                 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
281                                                 current->comm, current->pid,
282                                                 start, end,
283                                                 cattr_name(actual_type),
284                                                 cattr_name(parse->type));
285                                         err = -EBUSY;
286                                         break;
287                                 }
288                         }
289
290                         if (err) {
291                                 break;
292                         }
293
294                         printk("Overlap at 0x%Lx-0x%Lx\n",
295                                saved_ptr->start, saved_ptr->end);
296                         /* No conflict. Go ahead and add this new entry */
297                         list_add(&new_entry->nd, saved_ptr->nd.prev);
298                         new_entry = NULL;
299                         break;
300                 }
301
302                 if (start < parse->end) {
303                         if (actual_type != parse->type && ret_type) {
304                                 actual_type = parse->type;
305                                 *ret_type = actual_type;
306                                 new_entry->type = actual_type;
307                         }
308
309                         if (actual_type != parse->type) {
310                                 printk(
311                 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
312                                         current->comm, current->pid,
313                                         start, end,
314                                         cattr_name(actual_type),
315                                         cattr_name(parse->type));
316                                 err = -EBUSY;
317                                 break;
318                         }
319
320                         saved_ptr = parse;
321                         /*
322                          * Check to see whether the request overlaps more
323                          * than one entry in the list
324                          */
325                         list_for_each_entry_continue(parse, &memtype_list, nd) {
326                                 if (end <= parse->start) {
327                                         break;
328                                 }
329
330                                 if (actual_type != parse->type) {
331                                         printk(
332                 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
333                                                 current->comm, current->pid,
334                                                 start, end,
335                                                 cattr_name(actual_type),
336                                                 cattr_name(parse->type));
337                                         err = -EBUSY;
338                                         break;
339                                 }
340                         }
341
342                         if (err) {
343                                 break;
344                         }
345
346                         printk("Overlap at 0x%Lx-0x%Lx\n",
347                                saved_ptr->start, saved_ptr->end);
348                         /* No conflict. Go ahead and add this new entry */
349                         list_add(&new_entry->nd, &saved_ptr->nd);
350                         new_entry = NULL;
351                         break;
352                 }
353         }
354
355         if (err) {
356                 printk(
357         "reserve_memtype failed 0x%Lx-0x%Lx, track %s, req %s\n",
358                         start, end, cattr_name(new_entry->type),
359                         cattr_name(req_type));
360                 kfree(new_entry);
361                 spin_unlock(&memtype_lock);
362                 return err;
363         }
364
365         if (new_entry) {
366                 /* No conflict. Not yet added to the list. Add to the tail */
367                 list_add_tail(&new_entry->nd, &memtype_list);
368                 printk("New Entry\n");
369         }
370
371         if (ret_type) {
372                 printk(
373         "reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s, ret %s\n",
374                         start, end, cattr_name(actual_type),
375                         cattr_name(req_type), cattr_name(*ret_type));
376         } else {
377                 printk(
378         "reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s\n",
379                         start, end, cattr_name(actual_type),
380                         cattr_name(req_type));
381         }
382
383         spin_unlock(&memtype_lock);
384         return err;
385 }
386
387 int free_memtype(u64 start, u64 end)
388 {
389         struct memtype *ml;
390         int err = -EINVAL;
391
392         /* Only track when pat_wc_enabled */
393         if (!pat_wc_enabled) {
394                 return 0;
395         }
396
397         /* Low ISA region is always mapped WB. No need to track */
398         if (start >= ISA_START_ADDRESS && end <= ISA_END_ADDRESS) {
399                 return 0;
400         }
401
402         spin_lock(&memtype_lock);
403         list_for_each_entry(ml, &memtype_list, nd) {
404                 if (ml->start == start && ml->end == end) {
405                         list_del(&ml->nd);
406                         kfree(ml);
407                         err = 0;
408                         break;
409                 }
410         }
411         spin_unlock(&memtype_lock);
412
413         if (err) {
414                 printk(KERN_DEBUG "%s:%d freeing invalid memtype %Lx-%Lx\n",
415                         current->comm, current->pid, start, end);
416         }
417
418         printk( "free_memtype request 0x%Lx-0x%Lx\n", start, end);
419         return err;
420 }
421