2 * linux/mm/percpu.c - percpu memory allocator
4 * Copyright (C) 2009 SUSE Linux Products GmbH
5 * Copyright (C) 2009 Tejun Heo <tj@kernel.org>
7 * This file is released under the GPLv2.
9 * This is percpu allocator which can handle both static and dynamic
10 * areas. Percpu areas are allocated in chunks in vmalloc area. Each
11 * chunk is consisted of num_possible_cpus() units and the first chunk
12 * is used for static percpu variables in the kernel image (special
13 * boot time alloc/init handling necessary as these areas need to be
14 * brought up before allocation services are running). Unit grows as
15 * necessary and all units grow or shrink in unison. When a chunk is
16 * filled up, another chunk is allocated. ie. in vmalloc area
19 * ------------------- ------------------- ------------
20 * | u0 | u1 | u2 | u3 | | u0 | u1 | u2 | u3 | | u0 | u1 | u
21 * ------------------- ...... ------------------- .... ------------
23 * Allocation is done in offset-size areas of single unit space. Ie,
24 * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
25 * c1:u1, c1:u2 and c1:u3. Percpu access can be done by configuring
26 * percpu base registers UNIT_SIZE apart.
28 * There are usually many small percpu allocations many of them as
29 * small as 4 bytes. The allocator organizes chunks into lists
30 * according to free size and tries to allocate from the fullest one.
31 * Each chunk keeps the maximum contiguous area size hint which is
32 * guaranteed to be eqaul to or larger than the maximum contiguous
33 * area in the chunk. This helps the allocator not to iterate the
34 * chunk maps unnecessarily.
36 * Allocation state in each chunk is kept using an array of integers
37 * on chunk->map. A positive value in the map represents a free
38 * region and negative allocated. Allocation inside a chunk is done
39 * by scanning this map sequentially and serving the first matching
40 * entry. This is mostly copied from the percpu_modalloc() allocator.
41 * Chunks are also linked into a rb tree to ease address to chunk
42 * mapping during free.
44 * To use this allocator, arch code should do the followings.
46 * - define CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
48 * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
49 * regular address to percpu pointer and back
51 * - use pcpu_setup_static() during percpu area initialization to
52 * setup kernel static percpu area
55 #include <linux/bitmap.h>
56 #include <linux/bootmem.h>
57 #include <linux/list.h>
59 #include <linux/module.h>
60 #include <linux/mutex.h>
61 #include <linux/percpu.h>
62 #include <linux/pfn.h>
63 #include <linux/rbtree.h>
64 #include <linux/slab.h>
65 #include <linux/vmalloc.h>
67 #include <asm/cacheflush.h>
68 #include <asm/tlbflush.h>
70 #define PCPU_MIN_UNIT_PAGES 16 /* max alloc size in pages */
71 #define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */
72 #define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */
75 struct list_head list; /* linked to pcpu_slot lists */
76 struct rb_node rb_node; /* key is chunk->vm->addr */
77 int free_size; /* free bytes in the chunk */
78 int contig_hint; /* max contiguous size hint */
79 struct vm_struct *vm; /* mapped vmalloc region */
80 int map_used; /* # of map entries used */
81 int map_alloc; /* # of map entries allocated */
82 int *map; /* allocation map */
83 struct page *page[]; /* #cpus * UNIT_PAGES */
86 static int pcpu_unit_pages;
87 static int pcpu_unit_size;
88 static int pcpu_chunk_size;
89 static int pcpu_nr_slots;
90 static size_t pcpu_chunk_struct_size;
92 /* the address of the first chunk which starts with the kernel static area */
94 EXPORT_SYMBOL_GPL(pcpu_base_addr);
96 /* the size of kernel static area */
97 static int pcpu_static_size;
100 * One mutex to rule them all.
102 * The following mutex is grabbed in the outermost public alloc/free
103 * interface functions and released only when the operation is
104 * complete. As such, every function in this file other than the
105 * outermost functions are called under pcpu_mutex.
107 * It can easily be switched to use spinlock such that only the area
108 * allocation and page population commit are protected with it doing
109 * actual [de]allocation without holding any lock. However, given
110 * what this allocator does, I think it's better to let them run
113 static DEFINE_MUTEX(pcpu_mutex);
115 static struct list_head *pcpu_slot; /* chunk list slots */
116 static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */
118 static int __pcpu_size_to_slot(int size)
120 int highbit = fls(size); /* size is in bytes */
121 return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1);
124 static int pcpu_size_to_slot(int size)
126 if (size == pcpu_unit_size)
127 return pcpu_nr_slots - 1;
128 return __pcpu_size_to_slot(size);
131 static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
133 if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int))
136 return pcpu_size_to_slot(chunk->free_size);
139 static int pcpu_page_idx(unsigned int cpu, int page_idx)
141 return cpu * pcpu_unit_pages + page_idx;
144 static struct page **pcpu_chunk_pagep(struct pcpu_chunk *chunk,
145 unsigned int cpu, int page_idx)
147 return &chunk->page[pcpu_page_idx(cpu, page_idx)];
150 static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
151 unsigned int cpu, int page_idx)
153 return (unsigned long)chunk->vm->addr +
154 (pcpu_page_idx(cpu, page_idx) << PAGE_SHIFT);
157 static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk,
160 return *pcpu_chunk_pagep(chunk, 0, page_idx) != NULL;
164 * pcpu_realloc - versatile realloc
165 * @p: the current pointer (can be NULL for new allocations)
166 * @size: the current size in bytes (can be 0 for new allocations)
167 * @new_size: the wanted new size in bytes (can be 0 for free)
169 * More robust realloc which can be used to allocate, resize or free a
170 * memory area of arbitrary size. If the needed size goes over
171 * PAGE_SIZE, kernel VM is used.
174 * The new pointer on success, NULL on failure.
176 static void *pcpu_realloc(void *p, size_t size, size_t new_size)
180 if (new_size <= PAGE_SIZE)
181 new = kmalloc(new_size, GFP_KERNEL);
183 new = vmalloc(new_size);
184 if (new_size && !new)
187 memcpy(new, p, min(size, new_size));
189 memset(new + size, 0, new_size - size);
191 if (size <= PAGE_SIZE)
200 * pcpu_chunk_relocate - put chunk in the appropriate chunk slot
201 * @chunk: chunk of interest
202 * @oslot: the previous slot it was on
204 * This function is called after an allocation or free changed @chunk.
205 * New slot according to the changed state is determined and @chunk is
208 static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
210 int nslot = pcpu_chunk_slot(chunk);
212 if (oslot != nslot) {
214 list_move(&chunk->list, &pcpu_slot[nslot]);
216 list_move_tail(&chunk->list, &pcpu_slot[nslot]);
220 static struct rb_node **pcpu_chunk_rb_search(void *addr,
221 struct rb_node **parentp)
223 struct rb_node **p = &pcpu_addr_root.rb_node;
224 struct rb_node *parent = NULL;
225 struct pcpu_chunk *chunk;
229 chunk = rb_entry(parent, struct pcpu_chunk, rb_node);
231 if (addr < chunk->vm->addr)
233 else if (addr > chunk->vm->addr)
245 * pcpu_chunk_addr_search - search for chunk containing specified address
246 * @addr: address to search for
248 * Look for chunk which might contain @addr. More specifically, it
249 * searchs for the chunk with the highest start address which isn't
253 * The address of the found chunk.
255 static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
257 struct rb_node *n, *parent;
258 struct pcpu_chunk *chunk;
260 n = *pcpu_chunk_rb_search(addr, &parent);
262 /* no exactly matching chunk, the parent is the closest */
266 chunk = rb_entry(n, struct pcpu_chunk, rb_node);
268 if (addr < chunk->vm->addr) {
269 /* the parent was the next one, look for the previous one */
272 chunk = rb_entry(n, struct pcpu_chunk, rb_node);
279 * pcpu_chunk_addr_insert - insert chunk into address rb tree
280 * @new: chunk to insert
282 * Insert @new into address rb tree.
284 static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
286 struct rb_node **p, *parent;
288 p = pcpu_chunk_rb_search(new->vm->addr, &parent);
290 rb_link_node(&new->rb_node, parent, p);
291 rb_insert_color(&new->rb_node, &pcpu_addr_root);
295 * pcpu_split_block - split a map block
296 * @chunk: chunk of interest
297 * @i: index of map block to split
298 * @head: head size in bytes (can be 0)
299 * @tail: tail size in bytes (can be 0)
301 * Split the @i'th map block into two or three blocks. If @head is
302 * non-zero, @head bytes block is inserted before block @i moving it
303 * to @i+1 and reducing its size by @head bytes.
305 * If @tail is non-zero, the target block, which can be @i or @i+1
306 * depending on @head, is reduced by @tail bytes and @tail byte block
307 * is inserted after the target block.
310 * 0 on success, -errno on failure.
312 static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
314 int nr_extra = !!head + !!tail;
315 int target = chunk->map_used + nr_extra;
317 /* reallocation required? */
318 if (chunk->map_alloc < target) {
319 int new_alloc = chunk->map_alloc;
322 while (new_alloc < target)
325 new = pcpu_realloc(chunk->map,
326 chunk->map_alloc * sizeof(new[0]),
327 new_alloc * sizeof(new[0]));
331 chunk->map_alloc = new_alloc;
335 /* insert a new subblock */
336 memmove(&chunk->map[i + nr_extra], &chunk->map[i],
337 sizeof(chunk->map[0]) * (chunk->map_used - i));
338 chunk->map_used += nr_extra;
341 chunk->map[i + 1] = chunk->map[i] - head;
342 chunk->map[i++] = head;
345 chunk->map[i++] -= tail;
346 chunk->map[i] = tail;
352 * pcpu_alloc_area - allocate area from a pcpu_chunk
353 * @chunk: chunk of interest
354 * @size: wanted size in bytes
355 * @align: wanted align
357 * Try to allocate @size bytes area aligned at @align from @chunk.
358 * Note that this function only allocates the offset. It doesn't
359 * populate or map the area.
362 * Allocated offset in @chunk on success, -errno on failure.
364 static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
366 int oslot = pcpu_chunk_slot(chunk);
371 * The static chunk initially doesn't have map attached
372 * because kmalloc wasn't available during init. Give it one.
374 if (unlikely(!chunk->map)) {
375 chunk->map = pcpu_realloc(NULL, 0,
376 PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
380 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
381 chunk->map[chunk->map_used++] = -pcpu_static_size;
382 if (chunk->free_size)
383 chunk->map[chunk->map_used++] = chunk->free_size;
386 for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {
387 bool is_last = i + 1 == chunk->map_used;
390 /* extra for alignment requirement */
391 head = ALIGN(off, align) - off;
392 BUG_ON(i == 0 && head != 0);
394 if (chunk->map[i] < 0)
396 if (chunk->map[i] < head + size) {
397 max_contig = max(chunk->map[i], max_contig);
402 * If head is small or the previous block is free,
403 * merge'em. Note that 'small' is defined as smaller
404 * than sizeof(int), which is very small but isn't too
405 * uncommon for percpu allocations.
407 if (head && (head < sizeof(int) || chunk->map[i - 1] > 0)) {
408 if (chunk->map[i - 1] > 0)
409 chunk->map[i - 1] += head;
411 chunk->map[i - 1] -= head;
412 chunk->free_size -= head;
414 chunk->map[i] -= head;
419 /* if tail is small, just keep it around */
420 tail = chunk->map[i] - head - size;
421 if (tail < sizeof(int))
424 /* split if warranted */
426 if (pcpu_split_block(chunk, i, head, tail))
431 max_contig = max(chunk->map[i - 1], max_contig);
434 max_contig = max(chunk->map[i + 1], max_contig);
437 /* update hint and mark allocated */
439 chunk->contig_hint = max_contig; /* fully scanned */
441 chunk->contig_hint = max(chunk->contig_hint,
444 chunk->free_size -= chunk->map[i];
445 chunk->map[i] = -chunk->map[i];
447 pcpu_chunk_relocate(chunk, oslot);
451 chunk->contig_hint = max_contig; /* fully scanned */
452 pcpu_chunk_relocate(chunk, oslot);
455 * Tell the upper layer that this chunk has no area left.
456 * Note that this is not an error condition but a notification
457 * to upper layer that it needs to look at other chunks.
458 * -ENOSPC is chosen as it isn't used in memory subsystem and
459 * matches the meaning in a way.
465 * pcpu_free_area - free area to a pcpu_chunk
466 * @chunk: chunk of interest
467 * @freeme: offset of area to free
469 * Free area starting from @freeme to @chunk. Note that this function
470 * only modifies the allocation map. It doesn't depopulate or unmap
473 static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
475 int oslot = pcpu_chunk_slot(chunk);
478 for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++]))
481 BUG_ON(off != freeme);
482 BUG_ON(chunk->map[i] > 0);
484 chunk->map[i] = -chunk->map[i];
485 chunk->free_size += chunk->map[i];
487 /* merge with previous? */
488 if (i > 0 && chunk->map[i - 1] >= 0) {
489 chunk->map[i - 1] += chunk->map[i];
491 memmove(&chunk->map[i], &chunk->map[i + 1],
492 (chunk->map_used - i) * sizeof(chunk->map[0]));
495 /* merge with next? */
496 if (i + 1 < chunk->map_used && chunk->map[i + 1] >= 0) {
497 chunk->map[i] += chunk->map[i + 1];
499 memmove(&chunk->map[i + 1], &chunk->map[i + 2],
500 (chunk->map_used - (i + 1)) * sizeof(chunk->map[0]));
503 chunk->contig_hint = max(chunk->map[i], chunk->contig_hint);
504 pcpu_chunk_relocate(chunk, oslot);
508 * pcpu_unmap - unmap pages out of a pcpu_chunk
509 * @chunk: chunk of interest
510 * @page_start: page index of the first page to unmap
511 * @page_end: page index of the last page to unmap + 1
512 * @flush: whether to flush cache and tlb or not
514 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
515 * If @flush is true, vcache is flushed before unmapping and tlb
518 static void pcpu_unmap(struct pcpu_chunk *chunk, int page_start, int page_end,
521 unsigned int last = num_possible_cpus() - 1;
525 * Each flushing trial can be very expensive, issue flush on
526 * the whole region at once rather than doing it for each cpu.
527 * This could be an overkill but is more scalable.
530 flush_cache_vunmap(pcpu_chunk_addr(chunk, 0, page_start),
531 pcpu_chunk_addr(chunk, last, page_end));
533 for_each_possible_cpu(cpu)
534 unmap_kernel_range_noflush(
535 pcpu_chunk_addr(chunk, cpu, page_start),
536 (page_end - page_start) << PAGE_SHIFT);
538 /* ditto as flush_cache_vunmap() */
540 flush_tlb_kernel_range(pcpu_chunk_addr(chunk, 0, page_start),
541 pcpu_chunk_addr(chunk, last, page_end));
545 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
546 * @chunk: chunk to depopulate
547 * @off: offset to the area to depopulate
548 * @size: size of the area to depopulate in bytes
549 * @flush: whether to flush cache and tlb or not
551 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
552 * from @chunk. If @flush is true, vcache is flushed before unmapping
555 static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size,
558 int page_start = PFN_DOWN(off);
559 int page_end = PFN_UP(off + size);
560 int unmap_start = -1;
561 int uninitialized_var(unmap_end);
565 for (i = page_start; i < page_end; i++) {
566 for_each_possible_cpu(cpu) {
567 struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i);
575 * If it's partial depopulation, it might get
576 * populated or depopulated again. Mark the
581 unmap_start = unmap_start < 0 ? i : unmap_start;
586 if (unmap_start >= 0)
587 pcpu_unmap(chunk, unmap_start, unmap_end, flush);
591 * pcpu_map - map pages into a pcpu_chunk
592 * @chunk: chunk of interest
593 * @page_start: page index of the first page to map
594 * @page_end: page index of the last page to map + 1
596 * For each cpu, map pages [@page_start,@page_end) into @chunk.
597 * vcache is flushed afterwards.
599 static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end)
601 unsigned int last = num_possible_cpus() - 1;
605 for_each_possible_cpu(cpu) {
606 err = map_kernel_range_noflush(
607 pcpu_chunk_addr(chunk, cpu, page_start),
608 (page_end - page_start) << PAGE_SHIFT,
610 pcpu_chunk_pagep(chunk, cpu, page_start));
615 /* flush at once, please read comments in pcpu_unmap() */
616 flush_cache_vmap(pcpu_chunk_addr(chunk, 0, page_start),
617 pcpu_chunk_addr(chunk, last, page_end));
622 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
623 * @chunk: chunk of interest
624 * @off: offset to the area to populate
625 * @size: size of the area to populate in bytes
627 * For each cpu, populate and map pages [@page_start,@page_end) into
628 * @chunk. The area is cleared on return.
630 static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
632 const gfp_t alloc_mask = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
633 int page_start = PFN_DOWN(off);
634 int page_end = PFN_UP(off + size);
640 for (i = page_start; i < page_end; i++) {
641 if (pcpu_chunk_page_occupied(chunk, i)) {
642 if (map_start >= 0) {
643 if (pcpu_map(chunk, map_start, map_end))
650 map_start = map_start < 0 ? i : map_start;
653 for_each_possible_cpu(cpu) {
654 struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i);
656 *pagep = alloc_pages_node(cpu_to_node(cpu),
663 if (map_start >= 0 && pcpu_map(chunk, map_start, map_end))
666 for_each_possible_cpu(cpu)
667 memset(chunk->vm->addr + cpu * pcpu_unit_size + off, 0,
672 /* likely under heavy memory pressure, give memory back */
673 pcpu_depopulate_chunk(chunk, off, size, true);
677 static void free_pcpu_chunk(struct pcpu_chunk *chunk)
682 free_vm_area(chunk->vm);
683 pcpu_realloc(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]), 0);
687 static struct pcpu_chunk *alloc_pcpu_chunk(void)
689 struct pcpu_chunk *chunk;
691 chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL);
695 chunk->map = pcpu_realloc(NULL, 0,
696 PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
697 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
698 chunk->map[chunk->map_used++] = pcpu_unit_size;
700 chunk->vm = get_vm_area(pcpu_chunk_size, GFP_KERNEL);
702 free_pcpu_chunk(chunk);
706 INIT_LIST_HEAD(&chunk->list);
707 chunk->free_size = pcpu_unit_size;
708 chunk->contig_hint = pcpu_unit_size;
714 * __alloc_percpu - allocate percpu area
715 * @size: size of area to allocate in bytes
716 * @align: alignment of area (max PAGE_SIZE)
718 * Allocate percpu area of @size bytes aligned at @align. Might
719 * sleep. Might trigger writeouts.
722 * Percpu pointer to the allocated area on success, NULL on failure.
724 void *__alloc_percpu(size_t size, size_t align)
727 struct pcpu_chunk *chunk;
730 if (unlikely(!size || size > PCPU_MIN_UNIT_PAGES * PAGE_SIZE ||
731 align > PAGE_SIZE)) {
732 WARN(true, "illegal size (%zu) or align (%zu) for "
733 "percpu allocation\n", size, align);
737 mutex_lock(&pcpu_mutex);
740 for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) {
741 list_for_each_entry(chunk, &pcpu_slot[slot], list) {
742 if (size > chunk->contig_hint)
744 off = pcpu_alloc_area(chunk, size, align);
752 /* hmmm... no space left, create a new chunk */
753 chunk = alloc_pcpu_chunk();
756 pcpu_chunk_relocate(chunk, -1);
757 pcpu_chunk_addr_insert(chunk);
759 off = pcpu_alloc_area(chunk, size, align);
764 /* populate, map and clear the area */
765 if (pcpu_populate_chunk(chunk, off, size)) {
766 pcpu_free_area(chunk, off);
770 ptr = __addr_to_pcpu_ptr(chunk->vm->addr + off);
772 mutex_unlock(&pcpu_mutex);
775 EXPORT_SYMBOL_GPL(__alloc_percpu);
777 static void pcpu_kill_chunk(struct pcpu_chunk *chunk)
779 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false);
780 list_del(&chunk->list);
781 rb_erase(&chunk->rb_node, &pcpu_addr_root);
782 free_pcpu_chunk(chunk);
786 * free_percpu - free percpu area
787 * @ptr: pointer to area to free
789 * Free percpu area @ptr. Might sleep.
791 void free_percpu(void *ptr)
793 void *addr = __pcpu_ptr_to_addr(ptr);
794 struct pcpu_chunk *chunk;
800 mutex_lock(&pcpu_mutex);
802 chunk = pcpu_chunk_addr_search(addr);
803 off = addr - chunk->vm->addr;
805 pcpu_free_area(chunk, off);
807 /* the chunk became fully free, kill one if there are other free ones */
808 if (chunk->free_size == pcpu_unit_size) {
809 struct pcpu_chunk *pos;
811 list_for_each_entry(pos,
812 &pcpu_slot[pcpu_chunk_slot(chunk)], list)
814 pcpu_kill_chunk(pos);
819 mutex_unlock(&pcpu_mutex);
821 EXPORT_SYMBOL_GPL(free_percpu);
824 * pcpu_setup_static - initialize kernel static percpu area
825 * @populate_pte_fn: callback to allocate pagetable
826 * @pages: num_possible_cpus() * PFN_UP(cpu_size) pages
827 * @cpu_size: the size of static percpu area in bytes
829 * Initialize kernel static percpu area. The caller should allocate
830 * all the necessary pages and pass them in @pages.
831 * @populate_pte_fn() is called on each page to be used for percpu
832 * mapping and is responsible for making sure all the necessary page
833 * tables for the page is allocated.
836 * The determined pcpu_unit_size which can be used to initialize
839 size_t __init pcpu_setup_static(pcpu_populate_pte_fn_t populate_pte_fn,
840 struct page **pages, size_t cpu_size)
842 static struct vm_struct static_vm;
843 struct pcpu_chunk *static_chunk;
844 int nr_cpu_pages = DIV_ROUND_UP(cpu_size, PAGE_SIZE);
848 pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_PAGES, PFN_UP(cpu_size));
850 pcpu_static_size = cpu_size;
851 pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
852 pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;
853 pcpu_chunk_struct_size = sizeof(struct pcpu_chunk)
854 + num_possible_cpus() * pcpu_unit_pages * sizeof(struct page *);
857 * Allocate chunk slots. The additional last slot is for
860 pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2;
861 pcpu_slot = alloc_bootmem(pcpu_nr_slots * sizeof(pcpu_slot[0]));
862 for (i = 0; i < pcpu_nr_slots; i++)
863 INIT_LIST_HEAD(&pcpu_slot[i]);
865 /* init and register vm area */
866 static_vm.flags = VM_ALLOC;
867 static_vm.size = pcpu_chunk_size;
868 vm_area_register_early(&static_vm, PAGE_SIZE);
870 /* init static_chunk */
871 static_chunk = alloc_bootmem(pcpu_chunk_struct_size);
872 INIT_LIST_HEAD(&static_chunk->list);
873 static_chunk->vm = &static_vm;
874 static_chunk->free_size = pcpu_unit_size - pcpu_static_size;
875 static_chunk->contig_hint = static_chunk->free_size;
877 /* assign pages and map them */
878 for_each_possible_cpu(cpu) {
879 for (i = 0; i < nr_cpu_pages; i++) {
880 *pcpu_chunk_pagep(static_chunk, cpu, i) = *pages++;
881 populate_pte_fn(pcpu_chunk_addr(static_chunk, cpu, i));
885 err = pcpu_map(static_chunk, 0, nr_cpu_pages);
887 panic("failed to setup static percpu area, err=%d\n", err);
889 /* link static_chunk in */
890 pcpu_chunk_relocate(static_chunk, -1);
891 pcpu_chunk_addr_insert(static_chunk);
894 pcpu_base_addr = (void *)pcpu_chunk_addr(static_chunk, 0, 0);
895 return pcpu_unit_size;