Merge branch 'for-2.6.28' of git://linux-nfs.org/~bfields/linux
[linux-2.6] / mm / highmem.c
1 /*
2  * High memory handling common code and variables.
3  *
4  * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
5  *          Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
6  *
7  *
8  * Redesigned the x86 32-bit VM architecture to deal with
9  * 64-bit physical space. With current x86 CPUs this
10  * means up to 64 Gigabytes physical RAM.
11  *
12  * Rewrote high memory support to move the page cache into
13  * high memory. Implemented permanent (schedulable) kmaps
14  * based on Linus' idea.
15  *
16  * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
17  */
18
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 #include <linux/swap.h>
22 #include <linux/bio.h>
23 #include <linux/pagemap.h>
24 #include <linux/mempool.h>
25 #include <linux/blkdev.h>
26 #include <linux/init.h>
27 #include <linux/hash.h>
28 #include <linux/highmem.h>
29 #include <linux/blktrace_api.h>
30 #include <asm/tlbflush.h>
31
32 /*
33  * Virtual_count is not a pure "count".
34  *  0 means that it is not mapped, and has not been mapped
35  *    since a TLB flush - it is usable.
36  *  1 means that there are no users, but it has been mapped
37  *    since the last TLB flush - so we can't use it.
38  *  n means that there are (n-1) current users of it.
39  */
40 #ifdef CONFIG_HIGHMEM
41
42 unsigned long totalhigh_pages __read_mostly;
43 EXPORT_SYMBOL(totalhigh_pages);
44
45 unsigned int nr_free_highpages (void)
46 {
47         pg_data_t *pgdat;
48         unsigned int pages = 0;
49
50         for_each_online_pgdat(pgdat) {
51                 pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
52                         NR_FREE_PAGES);
53                 if (zone_movable_is_highmem())
54                         pages += zone_page_state(
55                                         &pgdat->node_zones[ZONE_MOVABLE],
56                                         NR_FREE_PAGES);
57         }
58
59         return pages;
60 }
61
62 static int pkmap_count[LAST_PKMAP];
63 static unsigned int last_pkmap_nr;
64 static  __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock);
65
66 pte_t * pkmap_page_table;
67
68 static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
69
70 static void flush_all_zero_pkmaps(void)
71 {
72         int i;
73         int need_flush = 0;
74
75         flush_cache_kmaps();
76
77         for (i = 0; i < LAST_PKMAP; i++) {
78                 struct page *page;
79
80                 /*
81                  * zero means we don't have anything to do,
82                  * >1 means that it is still in use. Only
83                  * a count of 1 means that it is free but
84                  * needs to be unmapped
85                  */
86                 if (pkmap_count[i] != 1)
87                         continue;
88                 pkmap_count[i] = 0;
89
90                 /* sanity check */
91                 BUG_ON(pte_none(pkmap_page_table[i]));
92
93                 /*
94                  * Don't need an atomic fetch-and-clear op here;
95                  * no-one has the page mapped, and cannot get at
96                  * its virtual address (and hence PTE) without first
97                  * getting the kmap_lock (which is held here).
98                  * So no dangers, even with speculative execution.
99                  */
100                 page = pte_page(pkmap_page_table[i]);
101                 pte_clear(&init_mm, (unsigned long)page_address(page),
102                           &pkmap_page_table[i]);
103
104                 set_page_address(page, NULL);
105                 need_flush = 1;
106         }
107         if (need_flush)
108                 flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
109 }
110
111 /**
112  * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings
113  */
114 void kmap_flush_unused(void)
115 {
116         spin_lock(&kmap_lock);
117         flush_all_zero_pkmaps();
118         spin_unlock(&kmap_lock);
119 }
120
121 static inline unsigned long map_new_virtual(struct page *page)
122 {
123         unsigned long vaddr;
124         int count;
125
126 start:
127         count = LAST_PKMAP;
128         /* Find an empty entry */
129         for (;;) {
130                 last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
131                 if (!last_pkmap_nr) {
132                         flush_all_zero_pkmaps();
133                         count = LAST_PKMAP;
134                 }
135                 if (!pkmap_count[last_pkmap_nr])
136                         break;  /* Found a usable entry */
137                 if (--count)
138                         continue;
139
140                 /*
141                  * Sleep for somebody else to unmap their entries
142                  */
143                 {
144                         DECLARE_WAITQUEUE(wait, current);
145
146                         __set_current_state(TASK_UNINTERRUPTIBLE);
147                         add_wait_queue(&pkmap_map_wait, &wait);
148                         spin_unlock(&kmap_lock);
149                         schedule();
150                         remove_wait_queue(&pkmap_map_wait, &wait);
151                         spin_lock(&kmap_lock);
152
153                         /* Somebody else might have mapped it while we slept */
154                         if (page_address(page))
155                                 return (unsigned long)page_address(page);
156
157                         /* Re-start */
158                         goto start;
159                 }
160         }
161         vaddr = PKMAP_ADDR(last_pkmap_nr);
162         set_pte_at(&init_mm, vaddr,
163                    &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
164
165         pkmap_count[last_pkmap_nr] = 1;
166         set_page_address(page, (void *)vaddr);
167
168         return vaddr;
169 }
170
171 /**
172  * kmap_high - map a highmem page into memory
173  * @page: &struct page to map
174  *
175  * Returns the page's virtual memory address.
176  *
177  * We cannot call this from interrupts, as it may block.
178  */
179 void *kmap_high(struct page *page)
180 {
181         unsigned long vaddr;
182
183         /*
184          * For highmem pages, we can't trust "virtual" until
185          * after we have the lock.
186          */
187         spin_lock(&kmap_lock);
188         vaddr = (unsigned long)page_address(page);
189         if (!vaddr)
190                 vaddr = map_new_virtual(page);
191         pkmap_count[PKMAP_NR(vaddr)]++;
192         BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2);
193         spin_unlock(&kmap_lock);
194         return (void*) vaddr;
195 }
196
197 EXPORT_SYMBOL(kmap_high);
198
199 /**
200  * kunmap_high - map a highmem page into memory
201  * @page: &struct page to unmap
202  */
203 void kunmap_high(struct page *page)
204 {
205         unsigned long vaddr;
206         unsigned long nr;
207         int need_wakeup;
208
209         spin_lock(&kmap_lock);
210         vaddr = (unsigned long)page_address(page);
211         BUG_ON(!vaddr);
212         nr = PKMAP_NR(vaddr);
213
214         /*
215          * A count must never go down to zero
216          * without a TLB flush!
217          */
218         need_wakeup = 0;
219         switch (--pkmap_count[nr]) {
220         case 0:
221                 BUG();
222         case 1:
223                 /*
224                  * Avoid an unnecessary wake_up() function call.
225                  * The common case is pkmap_count[] == 1, but
226                  * no waiters.
227                  * The tasks queued in the wait-queue are guarded
228                  * by both the lock in the wait-queue-head and by
229                  * the kmap_lock.  As the kmap_lock is held here,
230                  * no need for the wait-queue-head's lock.  Simply
231                  * test if the queue is empty.
232                  */
233                 need_wakeup = waitqueue_active(&pkmap_map_wait);
234         }
235         spin_unlock(&kmap_lock);
236
237         /* do wake-up, if needed, race-free outside of the spin lock */
238         if (need_wakeup)
239                 wake_up(&pkmap_map_wait);
240 }
241
242 EXPORT_SYMBOL(kunmap_high);
243 #endif
244
245 #if defined(HASHED_PAGE_VIRTUAL)
246
247 #define PA_HASH_ORDER   7
248
249 /*
250  * Describes one page->virtual association
251  */
252 struct page_address_map {
253         struct page *page;
254         void *virtual;
255         struct list_head list;
256 };
257
258 /*
259  * page_address_map freelist, allocated from page_address_maps.
260  */
261 static struct list_head page_address_pool;      /* freelist */
262 static spinlock_t pool_lock;                    /* protects page_address_pool */
263
264 /*
265  * Hash table bucket
266  */
267 static struct page_address_slot {
268         struct list_head lh;                    /* List of page_address_maps */
269         spinlock_t lock;                        /* Protect this bucket's list */
270 } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER];
271
272 static struct page_address_slot *page_slot(struct page *page)
273 {
274         return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)];
275 }
276
277 /**
278  * page_address - get the mapped virtual address of a page
279  * @page: &struct page to get the virtual address of
280  *
281  * Returns the page's virtual address.
282  */
283 void *page_address(struct page *page)
284 {
285         unsigned long flags;
286         void *ret;
287         struct page_address_slot *pas;
288
289         if (!PageHighMem(page))
290                 return lowmem_page_address(page);
291
292         pas = page_slot(page);
293         ret = NULL;
294         spin_lock_irqsave(&pas->lock, flags);
295         if (!list_empty(&pas->lh)) {
296                 struct page_address_map *pam;
297
298                 list_for_each_entry(pam, &pas->lh, list) {
299                         if (pam->page == page) {
300                                 ret = pam->virtual;
301                                 goto done;
302                         }
303                 }
304         }
305 done:
306         spin_unlock_irqrestore(&pas->lock, flags);
307         return ret;
308 }
309
310 EXPORT_SYMBOL(page_address);
311
312 /**
313  * set_page_address - set a page's virtual address
314  * @page: &struct page to set
315  * @virtual: virtual address to use
316  */
317 void set_page_address(struct page *page, void *virtual)
318 {
319         unsigned long flags;
320         struct page_address_slot *pas;
321         struct page_address_map *pam;
322
323         BUG_ON(!PageHighMem(page));
324
325         pas = page_slot(page);
326         if (virtual) {          /* Add */
327                 BUG_ON(list_empty(&page_address_pool));
328
329                 spin_lock_irqsave(&pool_lock, flags);
330                 pam = list_entry(page_address_pool.next,
331                                 struct page_address_map, list);
332                 list_del(&pam->list);
333                 spin_unlock_irqrestore(&pool_lock, flags);
334
335                 pam->page = page;
336                 pam->virtual = virtual;
337
338                 spin_lock_irqsave(&pas->lock, flags);
339                 list_add_tail(&pam->list, &pas->lh);
340                 spin_unlock_irqrestore(&pas->lock, flags);
341         } else {                /* Remove */
342                 spin_lock_irqsave(&pas->lock, flags);
343                 list_for_each_entry(pam, &pas->lh, list) {
344                         if (pam->page == page) {
345                                 list_del(&pam->list);
346                                 spin_unlock_irqrestore(&pas->lock, flags);
347                                 spin_lock_irqsave(&pool_lock, flags);
348                                 list_add_tail(&pam->list, &page_address_pool);
349                                 spin_unlock_irqrestore(&pool_lock, flags);
350                                 goto done;
351                         }
352                 }
353                 spin_unlock_irqrestore(&pas->lock, flags);
354         }
355 done:
356         return;
357 }
358
359 static struct page_address_map page_address_maps[LAST_PKMAP];
360
361 void __init page_address_init(void)
362 {
363         int i;
364
365         INIT_LIST_HEAD(&page_address_pool);
366         for (i = 0; i < ARRAY_SIZE(page_address_maps); i++)
367                 list_add(&page_address_maps[i].list, &page_address_pool);
368         for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) {
369                 INIT_LIST_HEAD(&page_address_htable[i].lh);
370                 spin_lock_init(&page_address_htable[i].lock);
371         }
372         spin_lock_init(&pool_lock);
373 }
374
375 #endif  /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */