mirror of
https://github.com/torvalds/linux.git
synced 2024-11-22 12:11:40 +00:00
percpu: introduce start_offset to pcpu_chunk
The reserved chunk arithmetic uses a global variable pcpu_reserved_chunk_limit that is set in the first chunk init code to hide a portion of the area map. The bitmap allocator to come will eventually move the base_addr up and require both the reserved chunk and static chunk to maintain this offset. pcpu_reserved_chunk_limit is removed and start_offset is added. The first chunk that is circulated and is pcpu_first_chunk serves the dynamic region, the region following the reserved region. The reserved chunk address check will temporarily use the first chunk to identify its address range. A following patch will increase the base_addr and remove this. If there is no reserved chunk, this will check the static region and return false because those values should never be passed into the allocator. Lastly, when linking in the first chunk, make sure to count the right free region for the number of empty populated pages. Signed-off-by: Dennis Zhou <dennisszhou@gmail.com> Reviewed-by: Josef Bacik <jbacik@fb.com> Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
parent
fb29a2cc6b
commit
e226670566
@ -28,6 +28,9 @@ struct pcpu_chunk {
|
||||
contain reservation for static chunk.
|
||||
Dynamic chunk will contain reservation
|
||||
for static and reserved chunks. */
|
||||
int start_offset; /* the overlap with the previous
|
||||
region to have a page aligned
|
||||
base_addr */
|
||||
int nr_populated; /* # of populated pages */
|
||||
unsigned long populated[]; /* populated bitmap */
|
||||
};
|
||||
|
21
mm/percpu.c
21
mm/percpu.c
@ -145,13 +145,10 @@ struct pcpu_chunk *pcpu_first_chunk __ro_after_init;
|
||||
|
||||
/*
|
||||
* Optional reserved chunk. This chunk reserves part of the first
|
||||
* chunk and serves it for reserved allocations. The amount of
|
||||
* reserved offset is in pcpu_reserved_chunk_limit. When reserved
|
||||
* area doesn't exist, the following variables contain NULL and 0
|
||||
* respectively.
|
||||
* chunk and serves it for reserved allocations. When the reserved
|
||||
* region doesn't exist, the following variable is NULL.
|
||||
*/
|
||||
struct pcpu_chunk *pcpu_reserved_chunk __ro_after_init;
|
||||
static int pcpu_reserved_chunk_limit __ro_after_init;
|
||||
|
||||
DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */
|
||||
static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop, map ext */
|
||||
@ -196,7 +193,7 @@ static bool pcpu_addr_in_reserved_chunk(void *addr)
|
||||
void *first_start = pcpu_first_chunk->base_addr;
|
||||
|
||||
return addr >= first_start &&
|
||||
addr < first_start + pcpu_reserved_chunk_limit;
|
||||
addr < first_start + pcpu_first_chunk->start_offset;
|
||||
}
|
||||
|
||||
static int __pcpu_size_to_slot(int size)
|
||||
@ -1687,6 +1684,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
||||
INIT_LIST_HEAD(&schunk->list);
|
||||
INIT_LIST_HEAD(&schunk->map_extend_list);
|
||||
schunk->base_addr = base_addr;
|
||||
schunk->start_offset = ai->static_size;
|
||||
schunk->map = smap;
|
||||
schunk->map_alloc = ARRAY_SIZE(smap);
|
||||
schunk->immutable = true;
|
||||
@ -1696,7 +1694,6 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
||||
if (ai->reserved_size) {
|
||||
schunk->free_size = ai->reserved_size;
|
||||
pcpu_reserved_chunk = schunk;
|
||||
pcpu_reserved_chunk_limit = ai->static_size + ai->reserved_size;
|
||||
} else {
|
||||
schunk->free_size = dyn_size;
|
||||
dyn_size = 0; /* dynamic area covered */
|
||||
@ -1704,7 +1701,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
||||
|
||||
schunk->contig_hint = schunk->free_size;
|
||||
schunk->map[0] = 1;
|
||||
schunk->map[1] = ai->static_size;
|
||||
schunk->map[1] = schunk->start_offset;
|
||||
schunk->map[2] = (ai->static_size + schunk->free_size) | 1;
|
||||
schunk->map_used = 2;
|
||||
schunk->has_reserved = true;
|
||||
@ -1715,6 +1712,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
||||
INIT_LIST_HEAD(&dchunk->list);
|
||||
INIT_LIST_HEAD(&dchunk->map_extend_list);
|
||||
dchunk->base_addr = base_addr;
|
||||
dchunk->start_offset = ai->static_size + ai->reserved_size;
|
||||
dchunk->map = dmap;
|
||||
dchunk->map_alloc = ARRAY_SIZE(dmap);
|
||||
dchunk->immutable = true;
|
||||
@ -1723,16 +1721,17 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
|
||||
|
||||
dchunk->contig_hint = dchunk->free_size = dyn_size;
|
||||
dchunk->map[0] = 1;
|
||||
dchunk->map[1] = pcpu_reserved_chunk_limit;
|
||||
dchunk->map[2] = (pcpu_reserved_chunk_limit + dchunk->free_size) | 1;
|
||||
dchunk->map[1] = dchunk->start_offset;
|
||||
dchunk->map[2] = (dchunk->start_offset + dchunk->free_size) | 1;
|
||||
dchunk->map_used = 2;
|
||||
dchunk->has_reserved = true;
|
||||
}
|
||||
|
||||
/* link the first chunk in */
|
||||
pcpu_first_chunk = dchunk ?: schunk;
|
||||
i = (pcpu_first_chunk->start_offset) ? 1 : 0;
|
||||
pcpu_nr_empty_pop_pages +=
|
||||
pcpu_count_occupied_pages(pcpu_first_chunk, 1);
|
||||
pcpu_count_occupied_pages(pcpu_first_chunk, i);
|
||||
pcpu_chunk_relocate(pcpu_first_chunk, -1);
|
||||
|
||||
pcpu_stats_chunk_alloc();
|
||||
|
Loading…
Reference in New Issue
Block a user