2 * IA-32 Huge TLB Page Support for Kernel.
4 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
7 #include <linux/config.h>
8 #include <linux/init.h>
11 #include <linux/hugetlb.h>
12 #include <linux/pagemap.h>
13 #include <linux/smp_lock.h>
14 #include <linux/slab.h>
15 #include <linux/err.h>
16 #include <linux/sysctl.h>
19 #include <asm/tlbflush.h>
21 pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
27 pgd = pgd_offset(mm, addr);
28 pud = pud_alloc(mm, pgd, addr);
29 pmd = pmd_alloc(mm, pud, addr);
33 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
39 pgd = pgd_offset(mm, addr);
40 pud = pud_offset(pgd, addr);
41 pmd = pmd_offset(pud, addr);
46 * This function checks for proper alignment of input addr and len parameters.
48 int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
50 if (len & ~HPAGE_MASK)
52 if (addr & ~HPAGE_MASK)
57 #if 0 /* This is just for testing */
59 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
61 unsigned long start = address;
65 struct vm_area_struct *vma;
67 vma = find_vma(mm, addr);
68 if (!vma || !is_vm_hugetlb_page(vma))
69 return ERR_PTR(-EINVAL);
71 pte = huge_pte_offset(mm, address);
73 /* hugetlb should be locked, and hence, prefaulted */
74 WARN_ON(!pte || pte_none(*pte));
76 page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];
78 WARN_ON(!PageCompound(page));
83 int pmd_huge(pmd_t pmd)
89 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
90 pmd_t *pmd, int write)
98 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
100 return ERR_PTR(-EINVAL);
103 int pmd_huge(pmd_t pmd)
105 return !!(pmd_val(pmd) & _PAGE_PSE);
109 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
110 pmd_t *pmd, int write)
114 page = pte_page(*(pte_t *)pmd);
116 page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
121 void hugetlb_clean_stale_pgtable(pte_t *pte)
123 pmd_t *pmd = (pmd_t *) pte;
126 page = pmd_page(*pmd);
128 dec_page_state(nr_page_table_pages);
129 page_cache_release(page);
132 /* x86_64 also uses this file */
134 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
135 static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
136 unsigned long addr, unsigned long len,
137 unsigned long pgoff, unsigned long flags)
139 struct mm_struct *mm = current->mm;
140 struct vm_area_struct *vma;
141 unsigned long start_addr;
143 if (len > mm->cached_hole_size) {
144 start_addr = mm->free_area_cache;
146 start_addr = TASK_UNMAPPED_BASE;
147 mm->cached_hole_size = 0;
151 addr = ALIGN(start_addr, HPAGE_SIZE);
153 for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
154 /* At this point: (!vma || addr < vma->vm_end). */
155 if (TASK_SIZE - len < addr) {
157 * Start a new search - just in case we missed
160 if (start_addr != TASK_UNMAPPED_BASE) {
161 start_addr = TASK_UNMAPPED_BASE;
162 mm->cached_hole_size = 0;
167 if (!vma || addr + len <= vma->vm_start) {
168 mm->free_area_cache = addr + len;
171 if (addr + mm->cached_hole_size < vma->vm_start)
172 mm->cached_hole_size = vma->vm_start - addr;
173 addr = ALIGN(vma->vm_end, HPAGE_SIZE);
177 static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
178 unsigned long addr0, unsigned long len,
179 unsigned long pgoff, unsigned long flags)
181 struct mm_struct *mm = current->mm;
182 struct vm_area_struct *vma, *prev_vma;
183 unsigned long base = mm->mmap_base, addr = addr0;
184 unsigned long largest_hole = mm->cached_hole_size;
187 /* don't allow allocations above current base */
188 if (mm->free_area_cache > base)
189 mm->free_area_cache = base;
191 if (len <= largest_hole) {
193 mm->free_area_cache = base;
196 /* make sure it can fit in the remaining address space */
197 if (mm->free_area_cache < len)
200 /* either no address requested or cant fit in requested address hole */
201 addr = (mm->free_area_cache - len) & HPAGE_MASK;
204 * Lookup failure means no vma is above this address,
205 * i.e. return with success:
207 if (!(vma = find_vma_prev(mm, addr, &prev_vma)))
211 * new region fits between prev_vma->vm_end and
212 * vma->vm_start, use it:
214 if (addr + len <= vma->vm_start &&
215 (!prev_vma || (addr >= prev_vma->vm_end))) {
216 /* remember the address as a hint for next time */
217 mm->cached_hole_size = largest_hole;
218 return (mm->free_area_cache = addr);
220 /* pull free_area_cache down to the first hole */
221 if (mm->free_area_cache == vma->vm_end) {
222 mm->free_area_cache = vma->vm_start;
223 mm->cached_hole_size = largest_hole;
227 /* remember the largest hole we saw so far */
228 if (addr + largest_hole < vma->vm_start)
229 largest_hole = vma->vm_start - addr;
231 /* try just below the current vma->vm_start */
232 addr = (vma->vm_start - len) & HPAGE_MASK;
233 } while (len <= vma->vm_start);
237 * if hint left us with no space for the requested
238 * mapping then try again:
241 mm->free_area_cache = base;
247 * A failed mmap() very likely causes application failure,
248 * so fall back to the bottom-up function here. This scenario
249 * can happen with large stack limits and large mmap()
252 mm->free_area_cache = TASK_UNMAPPED_BASE;
253 mm->cached_hole_size = ~0UL;
254 addr = hugetlb_get_unmapped_area_bottomup(file, addr0,
258 * Restore the topdown base:
260 mm->free_area_cache = base;
261 mm->cached_hole_size = ~0UL;
267 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
268 unsigned long len, unsigned long pgoff, unsigned long flags)
270 struct mm_struct *mm = current->mm;
271 struct vm_area_struct *vma;
273 if (len & ~HPAGE_MASK)
279 addr = ALIGN(addr, HPAGE_SIZE);
280 vma = find_vma(mm, addr);
281 if (TASK_SIZE - len >= addr &&
282 (!vma || addr + len <= vma->vm_start))
285 if (mm->get_unmapped_area == arch_get_unmapped_area)
286 return hugetlb_get_unmapped_area_bottomup(file, addr, len,
289 return hugetlb_get_unmapped_area_topdown(file, addr, len,
293 #endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/