2009-03-13 04:19:52 +00:00
|
|
|
/* Common code for 32 and 64-bit NUMA */
|
2016-05-24 22:35:39 +00:00
|
|
|
#include <linux/acpi.h>
|
2011-05-02 12:18:53 +00:00
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/init.h>
|
2009-03-13 04:19:52 +00:00
|
|
|
#include <linux/bootmem.h>
|
2011-05-02 12:18:53 +00:00
|
|
|
#include <linux/memblock.h>
|
|
|
|
#include <linux/mmzone.h>
|
|
|
|
#include <linux/ctype.h>
|
|
|
|
#include <linux/nodemask.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/topology.h>
|
|
|
|
|
2017-01-27 09:27:10 +00:00
|
|
|
#include <asm/e820/api.h>
|
2011-05-02 12:18:53 +00:00
|
|
|
#include <asm/proto.h>
|
|
|
|
#include <asm/dma.h>
|
|
|
|
#include <asm/amd_nb.h>
|
|
|
|
|
|
|
|
#include "numa_internal.h"
|
2011-01-19 08:57:21 +00:00
|
|
|
|
2016-12-13 04:18:29 +00:00
|
|
|
int numa_off;
|
2011-05-02 12:18:53 +00:00
|
|
|
nodemask_t numa_nodes_parsed __initdata;
|
2011-01-19 08:57:21 +00:00
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
|
|
|
|
EXPORT_SYMBOL(node_data);
|
|
|
|
|
|
|
|
static struct numa_meminfo numa_meminfo
|
|
|
|
#ifndef CONFIG_MEMORY_HOTPLUG
|
|
|
|
__initdata
|
|
|
|
#endif
|
|
|
|
;
|
|
|
|
|
|
|
|
static int numa_distance_cnt;
|
|
|
|
static u8 *numa_distance;
|
|
|
|
|
2011-01-19 08:57:21 +00:00
|
|
|
static __init int numa_setup(char *opt)
|
|
|
|
{
|
|
|
|
if (!opt)
|
|
|
|
return -EINVAL;
|
|
|
|
if (!strncmp(opt, "off", 3))
|
|
|
|
numa_off = 1;
|
|
|
|
#ifdef CONFIG_NUMA_EMU
|
|
|
|
if (!strncmp(opt, "fake=", 5))
|
|
|
|
numa_emu_cmdline(opt + 5);
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
|
|
if (!strncmp(opt, "noacpi", 6))
|
|
|
|
acpi_numa = -1;
|
|
|
|
#endif
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("numa", numa_setup);
|
2009-03-13 04:19:52 +00:00
|
|
|
|
|
|
|
/*
|
2011-01-23 13:37:39 +00:00
|
|
|
* apicid, cpu, node mappings
|
2009-03-13 04:19:52 +00:00
|
|
|
*/
|
2013-02-23 00:33:24 +00:00
|
|
|
s16 __apicid_to_node[MAX_LOCAL_APIC] = {
|
2011-01-23 13:37:39 +00:00
|
|
|
[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
|
|
|
|
};
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
int numa_cpu_node(int cpu)
|
2011-05-02 12:18:52 +00:00
|
|
|
{
|
|
|
|
int apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
|
|
|
|
|
|
|
|
if (apicid != BAD_APICID)
|
|
|
|
return __apicid_to_node[apicid];
|
|
|
|
return NUMA_NO_NODE;
|
|
|
|
}
|
|
|
|
|
2009-03-13 04:19:53 +00:00
|
|
|
cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
2009-03-13 04:19:52 +00:00
|
|
|
EXPORT_SYMBOL(node_to_cpumask_map);
|
|
|
|
|
2011-01-23 13:37:40 +00:00
|
|
|
/*
|
|
|
|
* Map cpu index to node index
|
|
|
|
*/
|
|
|
|
DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE);
|
|
|
|
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map);
|
|
|
|
|
2013-02-23 00:33:31 +00:00
|
|
|
void numa_set_node(int cpu, int node)
|
2011-01-23 13:37:40 +00:00
|
|
|
{
|
|
|
|
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
|
|
|
|
|
|
|
|
/* early setting, no percpu area yet */
|
|
|
|
if (cpu_to_node_map) {
|
|
|
|
cpu_to_node_map[cpu] = node;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
|
|
|
if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
|
|
|
|
printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
|
|
|
|
dump_stack();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
per_cpu(x86_cpu_to_node_map, cpu) = node;
|
|
|
|
|
2013-02-22 23:11:47 +00:00
|
|
|
set_cpu_numa_node(cpu, node);
|
2011-01-23 13:37:40 +00:00
|
|
|
}
|
|
|
|
|
2013-02-23 00:33:31 +00:00
|
|
|
void numa_clear_node(int cpu)
|
2011-01-23 13:37:40 +00:00
|
|
|
{
|
|
|
|
numa_set_node(cpu, NUMA_NO_NODE);
|
|
|
|
}
|
|
|
|
|
2009-03-13 04:19:52 +00:00
|
|
|
/*
|
|
|
|
* Allocate node_to_cpumask_map based on number of available nodes
|
|
|
|
* Requires node_possible_map to be valid.
|
|
|
|
*
|
2012-01-13 01:20:09 +00:00
|
|
|
* Note: cpumask_of_node() is not valid until after this is done.
|
2009-03-13 04:19:52 +00:00
|
|
|
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
|
|
|
|
*/
|
|
|
|
void __init setup_node_to_cpumask_map(void)
|
|
|
|
{
|
2013-04-29 22:08:02 +00:00
|
|
|
unsigned int node;
|
2009-03-13 04:19:52 +00:00
|
|
|
|
|
|
|
/* setup nr_node_ids if not done yet */
|
2013-04-29 22:08:02 +00:00
|
|
|
if (nr_node_ids == MAX_NUMNODES)
|
|
|
|
setup_nr_node_ids();
|
2009-03-13 04:19:52 +00:00
|
|
|
|
|
|
|
/* allocate the map */
|
2009-03-13 04:19:53 +00:00
|
|
|
for (node = 0; node < nr_node_ids; node++)
|
|
|
|
alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
|
2009-03-13 04:19:52 +00:00
|
|
|
|
2009-03-13 04:19:53 +00:00
|
|
|
/* cpumask_of_node() will now work */
|
|
|
|
pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
|
2009-03-13 04:19:52 +00:00
|
|
|
}
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
static int __init numa_add_memblk_to(int nid, u64 start, u64 end,
|
|
|
|
struct numa_meminfo *mi)
|
|
|
|
{
|
|
|
|
/* ignore zero length blks */
|
|
|
|
if (start == end)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* whine about and ignore invalid blks */
|
|
|
|
if (start > end || nid < 0 || nid >= MAX_NUMNODES) {
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n",
|
|
|
|
nid, start, end - 1);
|
2011-05-02 12:18:53 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (mi->nr_blks >= NR_NODE_MEMBLKS) {
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_err("too many memblk ranges\n");
|
2011-05-02 12:18:53 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
mi->blk[mi->nr_blks].start = start;
|
|
|
|
mi->blk[mi->nr_blks].end = end;
|
|
|
|
mi->blk[mi->nr_blks].nid = nid;
|
|
|
|
mi->nr_blks++;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo
|
|
|
|
* @idx: Index of memblk to remove
|
|
|
|
* @mi: numa_meminfo to remove memblk from
|
|
|
|
*
|
|
|
|
* Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and
|
|
|
|
* decrementing @mi->nr_blks.
|
|
|
|
*/
|
|
|
|
void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi)
|
|
|
|
{
|
|
|
|
mi->nr_blks--;
|
|
|
|
memmove(&mi->blk[idx], &mi->blk[idx + 1],
|
|
|
|
(mi->nr_blks - idx) * sizeof(mi->blk[0]));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* numa_add_memblk - Add one numa_memblk to numa_meminfo
|
|
|
|
* @nid: NUMA node ID of the new memblk
|
|
|
|
* @start: Start address of the new memblk
|
|
|
|
* @end: End address of the new memblk
|
|
|
|
*
|
|
|
|
* Add a new memblk to the default numa_meminfo.
|
|
|
|
*
|
|
|
|
* RETURNS:
|
|
|
|
* 0 on success, -errno on failure.
|
|
|
|
*/
|
|
|
|
int __init numa_add_memblk(int nid, u64 start, u64 end)
|
|
|
|
{
|
|
|
|
return numa_add_memblk_to(nid, start, end, &numa_meminfo);
|
|
|
|
}
|
|
|
|
|
x86/mm/numa: Drop dead code and rename setup_node_data() to setup_alloc_data()
The setup_node_data() function allocates a pg_data_t object,
inserts it into the node_data[] array and initializes the
following fields: node_id, node_start_pfn and
node_spanned_pages.
However, a few function calls later during the kernel boot,
free_area_init_node() re-initializes those fields, possibly with
setup_node_data() is not used.
This causes a small glitch when running Linux as a hyperv numa
guest:
SRAT: PXM 0 -> APIC 0x00 -> Node 0
SRAT: PXM 0 -> APIC 0x01 -> Node 0
SRAT: PXM 1 -> APIC 0x02 -> Node 1
SRAT: PXM 1 -> APIC 0x03 -> Node 1
SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
SRAT: Node 1 PXM 1 [mem 0x80200000-0xf7ffffff]
SRAT: Node 1 PXM 1 [mem 0x100000000-0x1081fffff]
NUMA: Node 1 [mem 0x80200000-0xf7ffffff] + [mem 0x100000000-0x1081fffff] -> [mem 0x80200000-0x1081fffff]
Initmem setup node 0 [mem 0x00000000-0x7fffffff]
NODE_DATA [mem 0x7ffdc000-0x7ffeffff]
Initmem setup node 1 [mem 0x80800000-0x1081fffff]
NODE_DATA [mem 0x1081ea000-0x1081fdfff]
crashkernel: memory value expected
[ffffea0000000000-ffffea0001ffffff] PMD -> [ffff88007de00000-ffff88007fdfffff] on node 0
[ffffea0002000000-ffffea00043fffff] PMD -> [ffff880105600000-ffff8801077fffff] on node 1
Zone ranges:
DMA [mem 0x00001000-0x00ffffff]
DMA32 [mem 0x01000000-0xffffffff]
Normal [mem 0x100000000-0x1081fffff]
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x00001000-0x0009efff]
node 0: [mem 0x00100000-0x7ffeffff]
node 1: [mem 0x80200000-0xf7ffffff]
node 1: [mem 0x100000000-0x1081fffff]
On node 0 totalpages: 524174
DMA zone: 64 pages used for memmap
DMA zone: 21 pages reserved
DMA zone: 3998 pages, LIFO batch:0
DMA32 zone: 8128 pages used for memmap
DMA32 zone: 520176 pages, LIFO batch:31
On node 1 totalpages: 524288
DMA32 zone: 7672 pages used for memmap
DMA32 zone: 491008 pages, LIFO batch:31
Normal zone: 520 pages used for memmap
Normal zone: 33280 pages, LIFO batch:7
In this dmesg, the SRAT table reports that the memory range for
node 1 starts at 0x80200000. However, the line starting with
"Initmem" reports that node 1 memory range starts at 0x80800000.
The "Initmem" line is reported by setup_node_data() and is
wrong, because the kernel ends up using the range as reported in
the SRAT table.
This commit drops all that dead code from setup_node_data(),
renames it to alloc_node_data() and adds a printk() to
free_area_init_node() so that we report a node's memory range
accurately.
Here's the same dmesg section with this patch applied:
SRAT: PXM 0 -> APIC 0x00 -> Node 0
SRAT: PXM 0 -> APIC 0x01 -> Node 0
SRAT: PXM 1 -> APIC 0x02 -> Node 1
SRAT: PXM 1 -> APIC 0x03 -> Node 1
SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
SRAT: Node 1 PXM 1 [mem 0x80200000-0xf7ffffff]
SRAT: Node 1 PXM 1 [mem 0x100000000-0x1081fffff]
NUMA: Node 1 [mem 0x80200000-0xf7ffffff] + [mem 0x100000000-0x1081fffff] -> [mem 0x80200000-0x1081fffff]
NODE_DATA(0) allocated [mem 0x7ffdc000-0x7ffeffff]
NODE_DATA(1) allocated [mem 0x1081ea000-0x1081fdfff]
crashkernel: memory value expected
[ffffea0000000000-ffffea0001ffffff] PMD -> [ffff88007de00000-ffff88007fdfffff] on node 0
[ffffea0002000000-ffffea00043fffff] PMD -> [ffff880105600000-ffff8801077fffff] on node 1
Zone ranges:
DMA [mem 0x00001000-0x00ffffff]
DMA32 [mem 0x01000000-0xffffffff]
Normal [mem 0x100000000-0x1081fffff]
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x00001000-0x0009efff]
node 0: [mem 0x00100000-0x7ffeffff]
node 1: [mem 0x80200000-0xf7ffffff]
node 1: [mem 0x100000000-0x1081fffff]
Initmem setup node 0 [mem 0x00001000-0x7ffeffff]
On node 0 totalpages: 524174
DMA zone: 64 pages used for memmap
DMA zone: 21 pages reserved
DMA zone: 3998 pages, LIFO batch:0
DMA32 zone: 8128 pages used for memmap
DMA32 zone: 520176 pages, LIFO batch:31
Initmem setup node 1 [mem 0x80200000-0x1081fffff]
On node 1 totalpages: 524288
DMA32 zone: 7672 pages used for memmap
DMA32 zone: 491008 pages, LIFO batch:31
Normal zone: 520 pages used for memmap
Normal zone: 33280 pages, LIFO batch:7
This commit was tested on a two node bare-metal NUMA machine and
Linux as a numa guest on hyperv and qemu/kvm.
PS: The wrong memory range reported by setup_node_data() seems to be
harmless in the current kernel because it's just not used. However,
that bad range is used in kernel 2.6.32 to initialize the old boot
memory allocator, which causes a crash during boot.
Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-08-22 20:27:36 +00:00
|
|
|
/* Allocate NODE_DATA for a node on the local memory */
|
|
|
|
static void __init alloc_node_data(int nid)
|
2011-05-02 12:18:53 +00:00
|
|
|
{
|
|
|
|
const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
|
2011-05-02 12:18:53 +00:00
|
|
|
u64 nd_pa;
|
2011-05-02 12:18:54 +00:00
|
|
|
void *nd;
|
2011-05-02 12:18:53 +00:00
|
|
|
int tnid;
|
|
|
|
|
|
|
|
/*
|
2013-01-31 22:00:48 +00:00
|
|
|
* Allocate node data. Try node-local memory and then any node.
|
|
|
|
* Never allocate in DMA zone.
|
2011-05-02 12:18:53 +00:00
|
|
|
*/
|
x86, ACPI, mm: Revert movablemem_map support
Tim found:
WARNING: at arch/x86/kernel/smpboot.c:324 topology_sane.isra.2+0x6f/0x80()
Hardware name: S2600CP
sched: CPU #1's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency.
smpboot: Booting Node 1, Processors #1
Modules linked in:
Pid: 0, comm: swapper/1 Not tainted 3.9.0-0-generic #1
Call Trace:
set_cpu_sibling_map+0x279/0x449
start_secondary+0x11d/0x1e5
Don Morris reproduced on a HP z620 workstation, and bisected it to
commit e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock
is ready")
It turns out movable_map has some problems, and it breaks several things
1. numa_init is called several times, NOT just for srat. so those
nodes_clear(numa_nodes_parsed)
memset(&numa_meminfo, 0, sizeof(numa_meminfo))
can not be just removed. Need to consider sequence is: numaq, srat, amd, dummy.
and make fall back path working.
2. simply split acpi_numa_init to early_parse_srat.
a. that early_parse_srat is NOT called for ia64, so you break ia64.
b. for (i = 0; i < MAX_LOCAL_APIC; i++)
set_apicid_to_node(i, NUMA_NO_NODE)
still left in numa_init. So it will just clear result from early_parse_srat.
it should be moved before that....
c. it breaks ACPI_TABLE_OVERIDE...as the acpi table scan is moved
early before override from INITRD is settled.
3. that patch TITLE is total misleading, there is NO x86 in the title,
but it changes critical x86 code. It caused x86 guys did not
pay attention to find the problem early. Those patches really should
be routed via tip/x86/mm.
4. after that commit, following range can not use movable ram:
a. real_mode code.... well..funny, legacy Node0 [0,1M) could be hot-removed?
b. initrd... it will be freed after booting, so it could be on movable...
c. crashkernel for kdump...: looks like we can not put kdump kernel above 4G
anymore.
d. init_mem_mapping: can not put page table high anymore.
e. initmem_init: vmemmap can not be high local node anymore. That is
not good.
If node is hotplugable, the mem related range like page table and
vmemmap could be on the that node without problem and should be on that
node.
We have workaround patch that could fix some problems, but some can not
be fixed.
So just remove that offending commit and related ones including:
f7210e6c4ac7 ("mm/memblock.c: use CONFIG_HAVE_MEMBLOCK_NODE_MAP to
protect movablecore_map in memblock_overlaps_region().")
01a178a94e8e ("acpi, memory-hotplug: support getting hotplug info from
SRAT")
27168d38fa20 ("acpi, memory-hotplug: extend movablemem_map ranges to
the end of node")
e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock is
ready")
fb06bc8e5f42 ("page_alloc: bootmem limit with movablecore_map")
42f47e27e761 ("page_alloc: make movablemem_map have higher priority")
6981ec31146c ("page_alloc: introduce zone_movable_limit[] to keep
movable limit for nodes")
34b71f1e04fc ("page_alloc: add movable_memmap kernel parameter")
4d59a75125d5 ("x86: get pg_data_t's memory from other node")
Later we should have patches that will make sure kernel put page table
and vmemmap on local node ram instead of push them down to node0. Also
need to find way to put other kernel used ram to local node ram.
Reported-by: Tim Gardner <tim.gardner@canonical.com>
Reported-by: Don Morris <don.morris@hp.com>
Bisected-by: Don Morris <don.morris@hp.com>
Tested-by: Don Morris <don.morris@hp.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-03-01 22:51:27 +00:00
|
|
|
nd_pa = memblock_alloc_nid(nd_size, SMP_CACHE_BYTES, nid);
|
2013-01-31 22:00:48 +00:00
|
|
|
if (!nd_pa) {
|
2013-12-06 04:18:30 +00:00
|
|
|
nd_pa = __memblock_alloc_base(nd_size, SMP_CACHE_BYTES,
|
|
|
|
MEMBLOCK_ALLOC_ACCESSIBLE);
|
|
|
|
if (!nd_pa) {
|
2017-03-14 03:08:00 +00:00
|
|
|
pr_err("Cannot find %zu bytes in any node (initial node: %d)\n",
|
2013-12-06 04:18:30 +00:00
|
|
|
nd_size, nid);
|
|
|
|
return;
|
|
|
|
}
|
2011-05-02 12:18:53 +00:00
|
|
|
}
|
2013-01-31 22:00:48 +00:00
|
|
|
nd = __va(nd_pa);
|
2011-05-02 12:18:53 +00:00
|
|
|
|
|
|
|
/* report and initialize */
|
x86/mm/numa: Drop dead code and rename setup_node_data() to setup_alloc_data()
The setup_node_data() function allocates a pg_data_t object,
inserts it into the node_data[] array and initializes the
following fields: node_id, node_start_pfn and
node_spanned_pages.
However, a few function calls later during the kernel boot,
free_area_init_node() re-initializes those fields, possibly with
setup_node_data() is not used.
This causes a small glitch when running Linux as a hyperv numa
guest:
SRAT: PXM 0 -> APIC 0x00 -> Node 0
SRAT: PXM 0 -> APIC 0x01 -> Node 0
SRAT: PXM 1 -> APIC 0x02 -> Node 1
SRAT: PXM 1 -> APIC 0x03 -> Node 1
SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
SRAT: Node 1 PXM 1 [mem 0x80200000-0xf7ffffff]
SRAT: Node 1 PXM 1 [mem 0x100000000-0x1081fffff]
NUMA: Node 1 [mem 0x80200000-0xf7ffffff] + [mem 0x100000000-0x1081fffff] -> [mem 0x80200000-0x1081fffff]
Initmem setup node 0 [mem 0x00000000-0x7fffffff]
NODE_DATA [mem 0x7ffdc000-0x7ffeffff]
Initmem setup node 1 [mem 0x80800000-0x1081fffff]
NODE_DATA [mem 0x1081ea000-0x1081fdfff]
crashkernel: memory value expected
[ffffea0000000000-ffffea0001ffffff] PMD -> [ffff88007de00000-ffff88007fdfffff] on node 0
[ffffea0002000000-ffffea00043fffff] PMD -> [ffff880105600000-ffff8801077fffff] on node 1
Zone ranges:
DMA [mem 0x00001000-0x00ffffff]
DMA32 [mem 0x01000000-0xffffffff]
Normal [mem 0x100000000-0x1081fffff]
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x00001000-0x0009efff]
node 0: [mem 0x00100000-0x7ffeffff]
node 1: [mem 0x80200000-0xf7ffffff]
node 1: [mem 0x100000000-0x1081fffff]
On node 0 totalpages: 524174
DMA zone: 64 pages used for memmap
DMA zone: 21 pages reserved
DMA zone: 3998 pages, LIFO batch:0
DMA32 zone: 8128 pages used for memmap
DMA32 zone: 520176 pages, LIFO batch:31
On node 1 totalpages: 524288
DMA32 zone: 7672 pages used for memmap
DMA32 zone: 491008 pages, LIFO batch:31
Normal zone: 520 pages used for memmap
Normal zone: 33280 pages, LIFO batch:7
In this dmesg, the SRAT table reports that the memory range for
node 1 starts at 0x80200000. However, the line starting with
"Initmem" reports that node 1 memory range starts at 0x80800000.
The "Initmem" line is reported by setup_node_data() and is
wrong, because the kernel ends up using the range as reported in
the SRAT table.
This commit drops all that dead code from setup_node_data(),
renames it to alloc_node_data() and adds a printk() to
free_area_init_node() so that we report a node's memory range
accurately.
Here's the same dmesg section with this patch applied:
SRAT: PXM 0 -> APIC 0x00 -> Node 0
SRAT: PXM 0 -> APIC 0x01 -> Node 0
SRAT: PXM 1 -> APIC 0x02 -> Node 1
SRAT: PXM 1 -> APIC 0x03 -> Node 1
SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
SRAT: Node 1 PXM 1 [mem 0x80200000-0xf7ffffff]
SRAT: Node 1 PXM 1 [mem 0x100000000-0x1081fffff]
NUMA: Node 1 [mem 0x80200000-0xf7ffffff] + [mem 0x100000000-0x1081fffff] -> [mem 0x80200000-0x1081fffff]
NODE_DATA(0) allocated [mem 0x7ffdc000-0x7ffeffff]
NODE_DATA(1) allocated [mem 0x1081ea000-0x1081fdfff]
crashkernel: memory value expected
[ffffea0000000000-ffffea0001ffffff] PMD -> [ffff88007de00000-ffff88007fdfffff] on node 0
[ffffea0002000000-ffffea00043fffff] PMD -> [ffff880105600000-ffff8801077fffff] on node 1
Zone ranges:
DMA [mem 0x00001000-0x00ffffff]
DMA32 [mem 0x01000000-0xffffffff]
Normal [mem 0x100000000-0x1081fffff]
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x00001000-0x0009efff]
node 0: [mem 0x00100000-0x7ffeffff]
node 1: [mem 0x80200000-0xf7ffffff]
node 1: [mem 0x100000000-0x1081fffff]
Initmem setup node 0 [mem 0x00001000-0x7ffeffff]
On node 0 totalpages: 524174
DMA zone: 64 pages used for memmap
DMA zone: 21 pages reserved
DMA zone: 3998 pages, LIFO batch:0
DMA32 zone: 8128 pages used for memmap
DMA32 zone: 520176 pages, LIFO batch:31
Initmem setup node 1 [mem 0x80200000-0x1081fffff]
On node 1 totalpages: 524288
DMA32 zone: 7672 pages used for memmap
DMA32 zone: 491008 pages, LIFO batch:31
Normal zone: 520 pages used for memmap
Normal zone: 33280 pages, LIFO batch:7
This commit was tested on a two node bare-metal NUMA machine and
Linux as a numa guest on hyperv and qemu/kvm.
PS: The wrong memory range reported by setup_node_data() seems to be
harmless in the current kernel because it's just not used. However,
that bad range is used in kernel 2.6.32 to initialize the old boot
memory allocator, which causes a crash during boot.
Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-08-22 20:27:36 +00:00
|
|
|
printk(KERN_INFO "NODE_DATA(%d) allocated [mem %#010Lx-%#010Lx]\n", nid,
|
2013-01-31 22:00:48 +00:00
|
|
|
nd_pa, nd_pa + nd_size - 1);
|
2011-05-02 12:18:53 +00:00
|
|
|
tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
|
2013-01-31 22:00:48 +00:00
|
|
|
if (tnid != nid)
|
2011-05-02 12:18:53 +00:00
|
|
|
printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nid, tnid);
|
|
|
|
|
2011-05-02 12:18:54 +00:00
|
|
|
node_data[nid] = nd;
|
2011-05-02 12:18:53 +00:00
|
|
|
memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
|
|
|
|
|
|
|
|
node_set_online(nid);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* numa_cleanup_meminfo - Cleanup a numa_meminfo
|
|
|
|
* @mi: numa_meminfo to clean up
|
|
|
|
*
|
2017-03-14 03:08:00 +00:00
|
|
|
* Sanitize @mi by merging and removing unnecessary memblks. Also check for
|
2011-05-02 12:18:53 +00:00
|
|
|
* conflicts and clear unused memblks.
|
|
|
|
*
|
|
|
|
* RETURNS:
|
|
|
|
* 0 on success, -errno on failure.
|
|
|
|
*/
|
|
|
|
int __init numa_cleanup_meminfo(struct numa_meminfo *mi)
|
|
|
|
{
|
|
|
|
const u64 low = 0;
|
2011-05-02 12:18:53 +00:00
|
|
|
const u64 high = PFN_PHYS(max_pfn);
|
2011-05-02 12:18:53 +00:00
|
|
|
int i, j, k;
|
|
|
|
|
x86, NUMA: Trim numa meminfo with max_pfn in a separate loop
During testing 32bit numa unifying code from tj, found one system with
more than 64g fails to use numa. It turns out we do not trim numa
meminfo correctly against max_pfn in case start address of a node is
higher than 64GiB. Bug fix made it to tip tree.
This patch moves the checking and trimming to a separate loop. So we
don't need to compare low/high in following merge loops. It makes the
code more readable.
Also it makes the node merge printouts less strange. On a 512GiB numa
system with 32bit,
before:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1080000000) -> [0,1000000000)
after:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1000000000) -> [0,1000000000)
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
[Updated patch description and comment slightly.]
Signed-off-by: Tejun Heo <tj@kernel.org>
2011-05-02 15:24:49 +00:00
|
|
|
/* first, trim all entries */
|
2011-05-02 12:18:53 +00:00
|
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
|
|
struct numa_memblk *bi = &mi->blk[i];
|
|
|
|
|
|
|
|
/* make sure all blocks are inside the limits */
|
|
|
|
bi->start = max(bi->start, low);
|
|
|
|
bi->end = min(bi->end, high);
|
|
|
|
|
mem-hotplug: handle node hole when initializing numa_meminfo.
When parsing SRAT, all memory ranges are added into numa_meminfo. In
numa_init(), before entering numa_cleanup_meminfo(), all possible memory
ranges are in numa_meminfo. And numa_cleanup_meminfo() removes all
ranges over max_pfn or empty.
But, this only works if the nodes are continuous. Let's have a look at
the following example:
We have an SRAT like this:
SRAT: Node 0 PXM 0 [mem 0x00000000-0x5fffffff]
SRAT: Node 0 PXM 0 [mem 0x100000000-0x1ffffffffff]
SRAT: Node 1 PXM 1 [mem 0x20000000000-0x3ffffffffff]
SRAT: Node 4 PXM 2 [mem 0x40000000000-0x5ffffffffff] hotplug
SRAT: Node 5 PXM 3 [mem 0x60000000000-0x7ffffffffff] hotplug
SRAT: Node 2 PXM 4 [mem 0x80000000000-0x9ffffffffff] hotplug
SRAT: Node 3 PXM 5 [mem 0xa0000000000-0xbffffffffff] hotplug
SRAT: Node 6 PXM 6 [mem 0xc0000000000-0xdffffffffff] hotplug
SRAT: Node 7 PXM 7 [mem 0xe0000000000-0xfffffffffff] hotplug
On boot, only node 0,1,2,3 exist.
And the numa_meminfo will look like this:
numa_meminfo.nr_blks = 9
1. on node 0: [0, 60000000]
2. on node 0: [100000000, 20000000000]
3. on node 1: [20000000000, 40000000000]
4. on node 4: [40000000000, 60000000000]
5. on node 5: [60000000000, 80000000000]
6. on node 2: [80000000000, a0000000000]
7. on node 3: [a0000000000, a0800000000]
8. on node 6: [c0000000000, a0800000000]
9. on node 7: [e0000000000, a0800000000]
And numa_cleanup_meminfo() will merge 1 and 2, and remove 8,9 because the
end address is over max_pfn, which is a0800000000. But 4 and 5 are not
removed because their end addresses are less then max_pfn. But in fact,
node 4 and 5 don't exist.
In a word, numa_cleanup_meminfo() is not able to handle holes between nodes.
Since memory ranges in node 4 and 5 are in numa_meminfo, in
numa_register_memblks(), node 4 and 5 will be mistakenly set to online.
If you run lscpu, it will show:
NUMA node0 CPU(s): 0-14,128-142
NUMA node1 CPU(s): 15-29,143-157
NUMA node2 CPU(s):
NUMA node3 CPU(s):
NUMA node4 CPU(s): 62-76,190-204
NUMA node5 CPU(s): 78-92,206-220
In this patch, we use memblock_overlaps_region() to check if ranges in
numa_meminfo overlap with ranges in memory_block. Since memory_block
contains all available memory at boot time, if they overlap, it means the
ranges exist. If not, then remove them from numa_meminfo.
After this patch, lscpu will show:
NUMA node0 CPU(s): 0-14,128-142
NUMA node1 CPU(s): 15-29,143-157
NUMA node4 CPU(s): 62-76,190-204
NUMA node5 CPU(s): 78-92,206-220
Signed-off-by: Tang Chen <tangchen@cn.fujitsu.com>
Reviewed-by: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tejun Heo <tj@kernel.org>
Cc: Luiz Capitulino <lcapitulino@redhat.com>
Cc: Xishi Qiu <qiuxishi@huawei.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Fabian Frederick <fabf@skynet.be>
Cc: Alexander Kuleshov <kuleshovmail@gmail.com>
Cc: Baoquan He <bhe@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 22:02:03 +00:00
|
|
|
/* and there's no empty or non-exist block */
|
|
|
|
if (bi->start >= bi->end ||
|
|
|
|
!memblock_overlaps_region(&memblock.memory,
|
|
|
|
bi->start, bi->end - bi->start))
|
2011-05-02 12:18:53 +00:00
|
|
|
numa_remove_memblk_from(i--, mi);
|
x86, NUMA: Trim numa meminfo with max_pfn in a separate loop
During testing 32bit numa unifying code from tj, found one system with
more than 64g fails to use numa. It turns out we do not trim numa
meminfo correctly against max_pfn in case start address of a node is
higher than 64GiB. Bug fix made it to tip tree.
This patch moves the checking and trimming to a separate loop. So we
don't need to compare low/high in following merge loops. It makes the
code more readable.
Also it makes the node merge printouts less strange. On a 512GiB numa
system with 32bit,
before:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1080000000) -> [0,1000000000)
after:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1000000000) -> [0,1000000000)
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
[Updated patch description and comment slightly.]
Signed-off-by: Tejun Heo <tj@kernel.org>
2011-05-02 15:24:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* merge neighboring / overlapping entries */
|
|
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
|
|
struct numa_memblk *bi = &mi->blk[i];
|
2011-05-02 12:18:53 +00:00
|
|
|
|
|
|
|
for (j = i + 1; j < mi->nr_blks; j++) {
|
|
|
|
struct numa_memblk *bj = &mi->blk[j];
|
2011-05-02 12:18:53 +00:00
|
|
|
u64 start, end;
|
2011-05-02 12:18:53 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* See whether there are overlapping blocks. Whine
|
|
|
|
* about but allow overlaps of the same nid. They
|
|
|
|
* will be merged below.
|
|
|
|
*/
|
|
|
|
if (bi->end > bj->start && bi->start < bj->end) {
|
|
|
|
if (bi->nid != bj->nid) {
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_err("node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n",
|
2012-05-29 22:06:29 +00:00
|
|
|
bi->nid, bi->start, bi->end - 1,
|
|
|
|
bj->nid, bj->start, bj->end - 1);
|
2011-05-02 12:18:53 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_warn("Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n",
|
|
|
|
bi->nid, bi->start, bi->end - 1,
|
|
|
|
bj->start, bj->end - 1);
|
2011-05-02 12:18:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Join together blocks on the same node, holes
|
|
|
|
* between which don't overlap with memory on other
|
|
|
|
* nodes.
|
|
|
|
*/
|
|
|
|
if (bi->nid != bj->nid)
|
|
|
|
continue;
|
x86, NUMA: Trim numa meminfo with max_pfn in a separate loop
During testing 32bit numa unifying code from tj, found one system with
more than 64g fails to use numa. It turns out we do not trim numa
meminfo correctly against max_pfn in case start address of a node is
higher than 64GiB. Bug fix made it to tip tree.
This patch moves the checking and trimming to a separate loop. So we
don't need to compare low/high in following merge loops. It makes the
code more readable.
Also it makes the node merge printouts less strange. On a 512GiB numa
system with 32bit,
before:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1080000000) -> [0,1000000000)
after:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1000000000) -> [0,1000000000)
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
[Updated patch description and comment slightly.]
Signed-off-by: Tejun Heo <tj@kernel.org>
2011-05-02 15:24:49 +00:00
|
|
|
start = min(bi->start, bj->start);
|
|
|
|
end = max(bi->end, bj->end);
|
2011-05-02 12:18:53 +00:00
|
|
|
for (k = 0; k < mi->nr_blks; k++) {
|
|
|
|
struct numa_memblk *bk = &mi->blk[k];
|
|
|
|
|
|
|
|
if (bi->nid == bk->nid)
|
|
|
|
continue;
|
|
|
|
if (start < bk->end && end > bk->start)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (k < mi->nr_blks)
|
|
|
|
continue;
|
2012-05-29 22:06:29 +00:00
|
|
|
printk(KERN_INFO "NUMA: Node %d [mem %#010Lx-%#010Lx] + [mem %#010Lx-%#010Lx] -> [mem %#010Lx-%#010Lx]\n",
|
|
|
|
bi->nid, bi->start, bi->end - 1, bj->start,
|
|
|
|
bj->end - 1, start, end - 1);
|
2011-05-02 12:18:53 +00:00
|
|
|
bi->start = start;
|
|
|
|
bi->end = end;
|
|
|
|
numa_remove_memblk_from(j--, mi);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
x86, NUMA: Trim numa meminfo with max_pfn in a separate loop
During testing 32bit numa unifying code from tj, found one system with
more than 64g fails to use numa. It turns out we do not trim numa
meminfo correctly against max_pfn in case start address of a node is
higher than 64GiB. Bug fix made it to tip tree.
This patch moves the checking and trimming to a separate loop. So we
don't need to compare low/high in following merge loops. It makes the
code more readable.
Also it makes the node merge printouts less strange. On a 512GiB numa
system with 32bit,
before:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1080000000) -> [0,1000000000)
after:
> NUMA: Node 0 [0,a0000) + [100000,80000000) -> [0,80000000)
> NUMA: Node 0 [0,80000000) + [100000000,1000000000) -> [0,1000000000)
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
[Updated patch description and comment slightly.]
Signed-off-by: Tejun Heo <tj@kernel.org>
2011-05-02 15:24:49 +00:00
|
|
|
/* clear unused ones */
|
2011-05-02 12:18:53 +00:00
|
|
|
for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) {
|
|
|
|
mi->blk[i].start = mi->blk[i].end = 0;
|
|
|
|
mi->blk[i].nid = NUMA_NO_NODE;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-04-07 22:00:53 +00:00
|
|
|
/*
|
|
|
|
* Set nodes, which have memory in @mi, in *@nodemask.
|
|
|
|
*/
|
|
|
|
static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask,
|
|
|
|
const struct numa_meminfo *mi)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(mi->blk); i++)
|
|
|
|
if (mi->blk[i].start != mi->blk[i].end &&
|
|
|
|
mi->blk[i].nid != NUMA_NO_NODE)
|
|
|
|
node_set(mi->blk[i].nid, *nodemask);
|
|
|
|
}
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
/**
|
|
|
|
* numa_reset_distance - Reset NUMA distance table
|
|
|
|
*
|
|
|
|
* The current table is freed. The next numa_set_distance() call will
|
|
|
|
* create a new one.
|
|
|
|
*/
|
|
|
|
void __init numa_reset_distance(void)
|
|
|
|
{
|
|
|
|
size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]);
|
|
|
|
|
|
|
|
/* numa_distance could be 1LU marking allocation failure, test cnt */
|
|
|
|
if (numa_distance_cnt)
|
2011-07-12 09:16:06 +00:00
|
|
|
memblock_free(__pa(numa_distance), size);
|
2011-05-02 12:18:53 +00:00
|
|
|
numa_distance_cnt = 0;
|
|
|
|
numa_distance = NULL; /* enable table creation */
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init numa_alloc_distance(void)
|
|
|
|
{
|
2017-04-07 22:00:53 +00:00
|
|
|
nodemask_t nodes_parsed;
|
2011-05-02 12:18:53 +00:00
|
|
|
size_t size;
|
|
|
|
int i, j, cnt = 0;
|
|
|
|
u64 phys;
|
|
|
|
|
|
|
|
/* size the new table and allocate it */
|
2017-04-07 22:00:53 +00:00
|
|
|
nodes_parsed = numa_nodes_parsed;
|
|
|
|
numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo);
|
|
|
|
|
|
|
|
for_each_node_mask(i, nodes_parsed)
|
2011-05-02 12:18:53 +00:00
|
|
|
cnt = i;
|
|
|
|
cnt++;
|
|
|
|
size = cnt * cnt * sizeof(numa_distance[0]);
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped),
|
2011-05-02 12:18:53 +00:00
|
|
|
size, PAGE_SIZE);
|
2011-07-12 07:58:09 +00:00
|
|
|
if (!phys) {
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_warn("Warning: can't allocate distance table!\n");
|
2011-05-02 12:18:53 +00:00
|
|
|
/* don't retry until explicitly reset */
|
|
|
|
numa_distance = (void *)1LU;
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2011-07-12 09:16:06 +00:00
|
|
|
memblock_reserve(phys, size);
|
2011-05-02 12:18:53 +00:00
|
|
|
|
|
|
|
numa_distance = __va(phys);
|
|
|
|
numa_distance_cnt = cnt;
|
|
|
|
|
|
|
|
/* fill with the default distances */
|
|
|
|
for (i = 0; i < cnt; i++)
|
|
|
|
for (j = 0; j < cnt; j++)
|
|
|
|
numa_distance[i * cnt + j] = i == j ?
|
|
|
|
LOCAL_DISTANCE : REMOTE_DISTANCE;
|
|
|
|
printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* numa_set_distance - Set NUMA distance from one NUMA to another
|
|
|
|
* @from: the 'from' node to set distance
|
|
|
|
* @to: the 'to' node to set distance
|
|
|
|
* @distance: NUMA distance
|
|
|
|
*
|
|
|
|
* Set the distance from node @from to @to to @distance. If distance table
|
|
|
|
* doesn't exist, one which is large enough to accommodate all the currently
|
|
|
|
* known nodes will be created.
|
|
|
|
*
|
|
|
|
* If such table cannot be allocated, a warning is printed and further
|
|
|
|
* calls are ignored until the distance table is reset with
|
|
|
|
* numa_reset_distance().
|
|
|
|
*
|
2011-12-08 12:16:41 +00:00
|
|
|
* If @from or @to is higher than the highest known node or lower than zero
|
|
|
|
* at the time of table creation or @distance doesn't make sense, the call
|
|
|
|
* is ignored.
|
2011-05-02 12:18:53 +00:00
|
|
|
* This is to allow simplification of specific NUMA config implementations.
|
|
|
|
*/
|
|
|
|
void __init numa_set_distance(int from, int to, int distance)
|
|
|
|
{
|
|
|
|
if (!numa_distance && numa_alloc_distance() < 0)
|
|
|
|
return;
|
|
|
|
|
2011-12-08 12:16:41 +00:00
|
|
|
if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
|
|
|
|
from < 0 || to < 0) {
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_warn_once("Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
|
|
|
|
from, to, distance);
|
2011-05-02 12:18:53 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((u8)distance != distance ||
|
|
|
|
(from == to && distance != LOCAL_DISTANCE)) {
|
2018-05-10 15:45:30 +00:00
|
|
|
pr_warn_once("Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
|
2011-05-02 12:18:53 +00:00
|
|
|
from, to, distance);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
numa_distance[from * numa_distance_cnt + to] = distance;
|
|
|
|
}
|
|
|
|
|
|
|
|
int __node_distance(int from, int to)
|
|
|
|
{
|
|
|
|
if (from >= numa_distance_cnt || to >= numa_distance_cnt)
|
|
|
|
return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
|
|
|
|
return numa_distance[from * numa_distance_cnt + to];
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__node_distance);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Sanity check to catch more bad NUMA configurations (they are amazingly
|
|
|
|
* common). Make sure the nodes cover all memory.
|
|
|
|
*/
|
|
|
|
static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
|
|
|
|
{
|
2011-05-02 12:18:53 +00:00
|
|
|
u64 numaram, e820ram;
|
2011-05-02 12:18:53 +00:00
|
|
|
int i;
|
|
|
|
|
|
|
|
numaram = 0;
|
|
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
2011-05-02 12:18:53 +00:00
|
|
|
u64 s = mi->blk[i].start >> PAGE_SHIFT;
|
|
|
|
u64 e = mi->blk[i].end >> PAGE_SHIFT;
|
2011-05-02 12:18:53 +00:00
|
|
|
numaram += e - s;
|
|
|
|
numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
|
2011-05-02 12:18:53 +00:00
|
|
|
if ((s64)numaram < 0)
|
2011-05-02 12:18:53 +00:00
|
|
|
numaram = 0;
|
|
|
|
}
|
|
|
|
|
2011-07-12 09:16:04 +00:00
|
|
|
e820ram = max_pfn - absent_pages_in_range(0, max_pfn);
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
|
2011-05-02 12:18:53 +00:00
|
|
|
if ((s64)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
|
|
|
|
printk(KERN_ERR "NUMA: nodes only cover %LuMB of your %LuMB e820 RAM. Not used.\n",
|
2011-05-02 12:18:53 +00:00
|
|
|
(numaram << PAGE_SHIFT) >> 20,
|
|
|
|
(e820ram << PAGE_SHIFT) >> 20);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-02-08 08:57:34 +00:00
|
|
|
/*
|
|
|
|
* Mark all currently memblock-reserved physical memory (which covers the
|
|
|
|
* kernel's own memory ranges) as hot-unswappable.
|
|
|
|
*/
|
2014-10-13 22:55:07 +00:00
|
|
|
static void __init numa_clear_kernel_node_hotplug(void)
|
|
|
|
{
|
2016-02-08 08:57:34 +00:00
|
|
|
nodemask_t reserved_nodemask = NODE_MASK_NONE;
|
|
|
|
struct memblock_region *mb_region;
|
|
|
|
int i;
|
2014-10-13 22:55:07 +00:00
|
|
|
|
|
|
|
/*
|
2016-02-08 08:57:34 +00:00
|
|
|
* We have to do some preprocessing of memblock regions, to
|
|
|
|
* make them suitable for reservation.
|
|
|
|
*
|
2014-10-13 22:55:07 +00:00
|
|
|
* At this time, all memory regions reserved by memblock are
|
2016-02-08 08:57:34 +00:00
|
|
|
* used by the kernel, but those regions are not split up
|
|
|
|
* along node boundaries yet, and don't necessarily have their
|
|
|
|
* node ID set yet either.
|
|
|
|
*
|
|
|
|
* So iterate over all memory known to the x86 architecture,
|
|
|
|
* and use those ranges to set the nid in memblock.reserved.
|
|
|
|
* This will split up the memblock regions along node
|
|
|
|
* boundaries and will set the node IDs as well.
|
2014-10-13 22:55:07 +00:00
|
|
|
*/
|
|
|
|
for (i = 0; i < numa_meminfo.nr_blks; i++) {
|
2016-02-08 08:57:34 +00:00
|
|
|
struct numa_memblk *mb = numa_meminfo.blk + i;
|
2016-02-08 11:14:55 +00:00
|
|
|
int ret;
|
2014-10-13 22:55:07 +00:00
|
|
|
|
2016-02-08 11:14:55 +00:00
|
|
|
ret = memblock_set_node(mb->start, mb->end - mb->start, &memblock.reserved, mb->nid);
|
|
|
|
WARN_ON_ONCE(ret);
|
2014-10-13 22:55:07 +00:00
|
|
|
}
|
|
|
|
|
2015-04-07 13:41:32 +00:00
|
|
|
/*
|
2016-02-08 08:57:34 +00:00
|
|
|
* Now go over all reserved memblock regions, to construct a
|
|
|
|
* node mask of all kernel reserved memory areas.
|
2015-04-07 13:41:32 +00:00
|
|
|
*
|
2016-02-08 08:57:34 +00:00
|
|
|
* [ Note, when booting with mem=nn[kMG] or in a kdump kernel,
|
|
|
|
* numa_meminfo might not include all memblock.reserved
|
|
|
|
* memory ranges, because quirks such as trim_snb_memory()
|
|
|
|
* reserve specific pages for Sandy Bridge graphics. ]
|
2015-04-07 13:41:32 +00:00
|
|
|
*/
|
2016-02-08 08:57:34 +00:00
|
|
|
for_each_memblock(reserved, mb_region) {
|
|
|
|
if (mb_region->nid != MAX_NUMNODES)
|
|
|
|
node_set(mb_region->nid, reserved_nodemask);
|
|
|
|
}
|
2014-10-13 22:55:07 +00:00
|
|
|
|
2016-02-08 08:57:34 +00:00
|
|
|
/*
|
|
|
|
* Finally, clear the MEMBLOCK_HOTPLUG flag for all memory
|
|
|
|
* belonging to the reserved node mask.
|
|
|
|
*
|
|
|
|
* Note that this will include memory regions that reside
|
|
|
|
* on nodes that contain kernel memory - entire nodes
|
|
|
|
* become hot-unpluggable:
|
|
|
|
*/
|
2014-10-13 22:55:07 +00:00
|
|
|
for (i = 0; i < numa_meminfo.nr_blks; i++) {
|
2016-02-08 08:57:34 +00:00
|
|
|
struct numa_memblk *mb = numa_meminfo.blk + i;
|
2014-10-13 22:55:07 +00:00
|
|
|
|
2016-02-08 08:57:34 +00:00
|
|
|
if (!node_isset(mb->nid, reserved_nodemask))
|
|
|
|
continue;
|
2014-10-13 22:55:07 +00:00
|
|
|
|
2016-02-08 08:57:34 +00:00
|
|
|
memblock_clear_hotplug(mb->start, mb->end - mb->start);
|
2014-10-13 22:55:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
static int __init numa_register_memblks(struct numa_meminfo *mi)
|
|
|
|
{
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 07:45:34 +00:00
|
|
|
unsigned long uninitialized_var(pfn_align);
|
2011-05-02 12:18:53 +00:00
|
|
|
int i, nid;
|
|
|
|
|
|
|
|
/* Account for nodes with cpus and no memory */
|
|
|
|
node_possible_map = numa_nodes_parsed;
|
2017-04-07 22:00:53 +00:00
|
|
|
numa_nodemask_from_meminfo(&node_possible_map, mi);
|
2011-05-02 12:18:53 +00:00
|
|
|
if (WARN_ON(nodes_empty(node_possible_map)))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2011-07-14 09:44:23 +00:00
|
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
|
|
struct numa_memblk *mb = &mi->blk[i];
|
2014-01-21 23:49:26 +00:00
|
|
|
memblock_set_node(mb->start, mb->end - mb->start,
|
|
|
|
&memblock.memory, mb->nid);
|
2011-07-14 09:44:23 +00:00
|
|
|
}
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 07:45:34 +00:00
|
|
|
|
2014-10-13 22:55:07 +00:00
|
|
|
/*
|
|
|
|
* At very early time, the kernel have to use some memory such as
|
|
|
|
* loading the kernel image. We cannot prevent this anyway. So any
|
|
|
|
* node the kernel resides in should be un-hotpluggable.
|
|
|
|
*
|
|
|
|
* And when we come here, alloc node data won't fail.
|
|
|
|
*/
|
|
|
|
numa_clear_kernel_node_hotplug();
|
|
|
|
|
x86, numa: Implement pfn -> nid mapping granularity check
SPARSEMEM w/o VMEMMAP and DISCONTIGMEM, both used only on 32bit, use
sections array to map pfn to nid which is limited in granularity. If
NUMA nodes are laid out such that the mapping cannot be accurate, boot
will fail triggering BUG_ON() in mminit_verify_page_links().
On 32bit, it's 512MiB w/ PAE and SPARSEMEM. This seems to have been
granular enough until commit 2706a0bf7b (x86, NUMA: Enable
CONFIG_AMD_NUMA on 32bit too). Apparently, there is a machine which
aligns NUMA nodes to 128MiB and has only AMD NUMA but not SRAT. This
led to the following BUG_ON().
On node 0 totalpages: 2096615
DMA zone: 32 pages used for memmap
DMA zone: 0 pages reserved
DMA zone: 3927 pages, LIFO batch:0
Normal zone: 1740 pages used for memmap
Normal zone: 220978 pages, LIFO batch:31
HighMem zone: 16405 pages used for memmap
HighMem zone: 1853533 pages, LIFO batch:31
BUG: Int 6: CR2 (null)
EDI (null) ESI 00000002 EBP 00000002 ESP c1543ecc
EBX f2400000 EDX 00000006 ECX (null) EAX 00000001
err (null) EIP c16209aa CS 00000060 flg 00010002
Stack: f2400000 00220000 f7200800 c1620613 00220000 01000000 04400000 00238000
(null) f7200000 00000002 f7200b58 f7200800 c1620929 000375fe (null)
f7200b80 c16395f0 00200a02 f7200a80 (null) 000375fe 00000002 (null)
Pid: 0, comm: swapper Not tainted 2.6.39-rc5-00181-g2706a0b #17
Call Trace:
[<c136b1e5>] ? early_fault+0x2e/0x2e
[<c16209aa>] ? mminit_verify_page_links+0x12/0x42
[<c1620613>] ? memmap_init_zone+0xaf/0x10c
[<c1620929>] ? free_area_init_node+0x2b9/0x2e3
[<c1607e99>] ? free_area_init_nodes+0x3f2/0x451
[<c1601d80>] ? paging_init+0x112/0x118
[<c15f578d>] ? setup_arch+0x791/0x82f
[<c15f43d9>] ? start_kernel+0x6a/0x257
This patch implements node_map_pfn_alignment() which determines
maximum internode alignment and update numa_register_memblks() to
reject NUMA configuration if alignment exceeds the pfn -> nid mapping
granularity of the memory model as determined by PAGES_PER_SECTION.
This makes the problematic machine boot w/ flatmem by rejecting the
NUMA config and provides protection against crazy NUMA configurations.
Signed-off-by: Tejun Heo <tj@kernel.org>
Link: http://lkml.kernel.org/r/20110712074534.GB2872@htj.dyndns.org
LKML-Reference: <20110628174613.GP478@escobedo.osrc.amd.com>
Reported-and-Tested-by: Hans Rosenfeld <hans.rosenfeld@amd.com>
Cc: Conny Seidel <conny.seidel@amd.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
2011-07-12 07:45:34 +00:00
|
|
|
/*
|
|
|
|
* If sections array is gonna be used for pfn -> nid mapping, check
|
|
|
|
* whether its granularity is fine enough.
|
|
|
|
*/
|
|
|
|
#ifdef NODE_NOT_IN_PAGE_FLAGS
|
|
|
|
pfn_align = node_map_pfn_alignment();
|
|
|
|
if (pfn_align && pfn_align < PAGES_PER_SECTION) {
|
|
|
|
printk(KERN_WARNING "Node alignment %LuMB < min %LuMB, rejecting NUMA config\n",
|
|
|
|
PFN_PHYS(pfn_align) >> 20,
|
|
|
|
PFN_PHYS(PAGES_PER_SECTION) >> 20);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
#endif
|
2011-05-02 12:18:53 +00:00
|
|
|
if (!numa_meminfo_cover_memory(mi))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Finally register nodes. */
|
|
|
|
for_each_node_mask(nid, node_possible_map) {
|
2011-05-02 12:18:53 +00:00
|
|
|
u64 start = PFN_PHYS(max_pfn);
|
2011-05-02 12:18:53 +00:00
|
|
|
u64 end = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < mi->nr_blks; i++) {
|
|
|
|
if (nid != mi->blk[i].nid)
|
|
|
|
continue;
|
|
|
|
start = min(mi->blk[i].start, start);
|
|
|
|
end = max(mi->blk[i].end, end);
|
|
|
|
}
|
|
|
|
|
x86/mm/numa: Drop dead code and rename setup_node_data() to setup_alloc_data()
The setup_node_data() function allocates a pg_data_t object,
inserts it into the node_data[] array and initializes the
following fields: node_id, node_start_pfn and
node_spanned_pages.
However, a few function calls later during the kernel boot,
free_area_init_node() re-initializes those fields, possibly with
setup_node_data() is not used.
This causes a small glitch when running Linux as a hyperv numa
guest:
SRAT: PXM 0 -> APIC 0x00 -> Node 0
SRAT: PXM 0 -> APIC 0x01 -> Node 0
SRAT: PXM 1 -> APIC 0x02 -> Node 1
SRAT: PXM 1 -> APIC 0x03 -> Node 1
SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
SRAT: Node 1 PXM 1 [mem 0x80200000-0xf7ffffff]
SRAT: Node 1 PXM 1 [mem 0x100000000-0x1081fffff]
NUMA: Node 1 [mem 0x80200000-0xf7ffffff] + [mem 0x100000000-0x1081fffff] -> [mem 0x80200000-0x1081fffff]
Initmem setup node 0 [mem 0x00000000-0x7fffffff]
NODE_DATA [mem 0x7ffdc000-0x7ffeffff]
Initmem setup node 1 [mem 0x80800000-0x1081fffff]
NODE_DATA [mem 0x1081ea000-0x1081fdfff]
crashkernel: memory value expected
[ffffea0000000000-ffffea0001ffffff] PMD -> [ffff88007de00000-ffff88007fdfffff] on node 0
[ffffea0002000000-ffffea00043fffff] PMD -> [ffff880105600000-ffff8801077fffff] on node 1
Zone ranges:
DMA [mem 0x00001000-0x00ffffff]
DMA32 [mem 0x01000000-0xffffffff]
Normal [mem 0x100000000-0x1081fffff]
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x00001000-0x0009efff]
node 0: [mem 0x00100000-0x7ffeffff]
node 1: [mem 0x80200000-0xf7ffffff]
node 1: [mem 0x100000000-0x1081fffff]
On node 0 totalpages: 524174
DMA zone: 64 pages used for memmap
DMA zone: 21 pages reserved
DMA zone: 3998 pages, LIFO batch:0
DMA32 zone: 8128 pages used for memmap
DMA32 zone: 520176 pages, LIFO batch:31
On node 1 totalpages: 524288
DMA32 zone: 7672 pages used for memmap
DMA32 zone: 491008 pages, LIFO batch:31
Normal zone: 520 pages used for memmap
Normal zone: 33280 pages, LIFO batch:7
In this dmesg, the SRAT table reports that the memory range for
node 1 starts at 0x80200000. However, the line starting with
"Initmem" reports that node 1 memory range starts at 0x80800000.
The "Initmem" line is reported by setup_node_data() and is
wrong, because the kernel ends up using the range as reported in
the SRAT table.
This commit drops all that dead code from setup_node_data(),
renames it to alloc_node_data() and adds a printk() to
free_area_init_node() so that we report a node's memory range
accurately.
Here's the same dmesg section with this patch applied:
SRAT: PXM 0 -> APIC 0x00 -> Node 0
SRAT: PXM 0 -> APIC 0x01 -> Node 0
SRAT: PXM 1 -> APIC 0x02 -> Node 1
SRAT: PXM 1 -> APIC 0x03 -> Node 1
SRAT: Node 0 PXM 0 [mem 0x00000000-0x7fffffff]
SRAT: Node 1 PXM 1 [mem 0x80200000-0xf7ffffff]
SRAT: Node 1 PXM 1 [mem 0x100000000-0x1081fffff]
NUMA: Node 1 [mem 0x80200000-0xf7ffffff] + [mem 0x100000000-0x1081fffff] -> [mem 0x80200000-0x1081fffff]
NODE_DATA(0) allocated [mem 0x7ffdc000-0x7ffeffff]
NODE_DATA(1) allocated [mem 0x1081ea000-0x1081fdfff]
crashkernel: memory value expected
[ffffea0000000000-ffffea0001ffffff] PMD -> [ffff88007de00000-ffff88007fdfffff] on node 0
[ffffea0002000000-ffffea00043fffff] PMD -> [ffff880105600000-ffff8801077fffff] on node 1
Zone ranges:
DMA [mem 0x00001000-0x00ffffff]
DMA32 [mem 0x01000000-0xffffffff]
Normal [mem 0x100000000-0x1081fffff]
Movable zone start for each node
Early memory node ranges
node 0: [mem 0x00001000-0x0009efff]
node 0: [mem 0x00100000-0x7ffeffff]
node 1: [mem 0x80200000-0xf7ffffff]
node 1: [mem 0x100000000-0x1081fffff]
Initmem setup node 0 [mem 0x00001000-0x7ffeffff]
On node 0 totalpages: 524174
DMA zone: 64 pages used for memmap
DMA zone: 21 pages reserved
DMA zone: 3998 pages, LIFO batch:0
DMA32 zone: 8128 pages used for memmap
DMA32 zone: 520176 pages, LIFO batch:31
Initmem setup node 1 [mem 0x80200000-0x1081fffff]
On node 1 totalpages: 524288
DMA32 zone: 7672 pages used for memmap
DMA32 zone: 491008 pages, LIFO batch:31
Normal zone: 520 pages used for memmap
Normal zone: 33280 pages, LIFO batch:7
This commit was tested on a two node bare-metal NUMA machine and
Linux as a numa guest on hyperv and qemu/kvm.
PS: The wrong memory range reported by setup_node_data() seems to be
harmless in the current kernel because it's just not used. However,
that bad range is used in kernel 2.6.32 to initialize the old boot
memory allocator, which causes a crash during boot.
Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2014-08-22 20:27:36 +00:00
|
|
|
if (start >= end)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't confuse VM with a node that doesn't have the
|
|
|
|
* minimum amount of memory:
|
|
|
|
*/
|
|
|
|
if (end && (end - start) < NODE_MIN_SIZE)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
alloc_node_data(nid);
|
2011-05-02 12:18:53 +00:00
|
|
|
}
|
|
|
|
|
2011-07-14 09:44:23 +00:00
|
|
|
/* Dump memblock with node info and return. */
|
|
|
|
memblock_dump_all();
|
2011-05-02 12:18:53 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-01-23 13:37:42 +00:00
|
|
|
/*
|
|
|
|
* There are unfortunately some poorly designed mainboards around that
|
|
|
|
* only connect memory to a single CPU. This breaks the 1:1 cpu->node
|
|
|
|
* mapping. To avoid this fill in the mapping for all possible CPUs,
|
|
|
|
* as the number of CPUs is not known yet. We round robin the existing
|
|
|
|
* nodes.
|
|
|
|
*/
|
2011-05-02 15:24:48 +00:00
|
|
|
static void __init numa_init_array(void)
|
2011-01-23 13:37:42 +00:00
|
|
|
{
|
|
|
|
int rr, i;
|
|
|
|
|
|
|
|
rr = first_node(node_online_map);
|
|
|
|
for (i = 0; i < nr_cpu_ids; i++) {
|
|
|
|
if (early_cpu_to_node(i) != NUMA_NO_NODE)
|
|
|
|
continue;
|
|
|
|
numa_set_node(i, rr);
|
2016-05-20 00:10:58 +00:00
|
|
|
rr = next_node_in(rr, node_online_map);
|
2011-01-23 13:37:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
static int __init numa_init(int (*init_func)(void))
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
for (i = 0; i < MAX_LOCAL_APIC; i++)
|
|
|
|
set_apicid_to_node(i, NUMA_NO_NODE);
|
|
|
|
|
x86, ACPI, mm: Revert movablemem_map support
Tim found:
WARNING: at arch/x86/kernel/smpboot.c:324 topology_sane.isra.2+0x6f/0x80()
Hardware name: S2600CP
sched: CPU #1's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency.
smpboot: Booting Node 1, Processors #1
Modules linked in:
Pid: 0, comm: swapper/1 Not tainted 3.9.0-0-generic #1
Call Trace:
set_cpu_sibling_map+0x279/0x449
start_secondary+0x11d/0x1e5
Don Morris reproduced on a HP z620 workstation, and bisected it to
commit e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock
is ready")
It turns out movable_map has some problems, and it breaks several things
1. numa_init is called several times, NOT just for srat. so those
nodes_clear(numa_nodes_parsed)
memset(&numa_meminfo, 0, sizeof(numa_meminfo))
can not be just removed. Need to consider sequence is: numaq, srat, amd, dummy.
and make fall back path working.
2. simply split acpi_numa_init to early_parse_srat.
a. that early_parse_srat is NOT called for ia64, so you break ia64.
b. for (i = 0; i < MAX_LOCAL_APIC; i++)
set_apicid_to_node(i, NUMA_NO_NODE)
still left in numa_init. So it will just clear result from early_parse_srat.
it should be moved before that....
c. it breaks ACPI_TABLE_OVERIDE...as the acpi table scan is moved
early before override from INITRD is settled.
3. that patch TITLE is total misleading, there is NO x86 in the title,
but it changes critical x86 code. It caused x86 guys did not
pay attention to find the problem early. Those patches really should
be routed via tip/x86/mm.
4. after that commit, following range can not use movable ram:
a. real_mode code.... well..funny, legacy Node0 [0,1M) could be hot-removed?
b. initrd... it will be freed after booting, so it could be on movable...
c. crashkernel for kdump...: looks like we can not put kdump kernel above 4G
anymore.
d. init_mem_mapping: can not put page table high anymore.
e. initmem_init: vmemmap can not be high local node anymore. That is
not good.
If node is hotplugable, the mem related range like page table and
vmemmap could be on the that node without problem and should be on that
node.
We have workaround patch that could fix some problems, but some can not
be fixed.
So just remove that offending commit and related ones including:
f7210e6c4ac7 ("mm/memblock.c: use CONFIG_HAVE_MEMBLOCK_NODE_MAP to
protect movablecore_map in memblock_overlaps_region().")
01a178a94e8e ("acpi, memory-hotplug: support getting hotplug info from
SRAT")
27168d38fa20 ("acpi, memory-hotplug: extend movablemem_map ranges to
the end of node")
e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock is
ready")
fb06bc8e5f42 ("page_alloc: bootmem limit with movablecore_map")
42f47e27e761 ("page_alloc: make movablemem_map have higher priority")
6981ec31146c ("page_alloc: introduce zone_movable_limit[] to keep
movable limit for nodes")
34b71f1e04fc ("page_alloc: add movable_memmap kernel parameter")
4d59a75125d5 ("x86: get pg_data_t's memory from other node")
Later we should have patches that will make sure kernel put page table
and vmemmap on local node ram instead of push them down to node0. Also
need to find way to put other kernel used ram to local node ram.
Reported-by: Tim Gardner <tim.gardner@canonical.com>
Reported-by: Don Morris <don.morris@hp.com>
Bisected-by: Don Morris <don.morris@hp.com>
Tested-by: Don Morris <don.morris@hp.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-03-01 22:51:27 +00:00
|
|
|
nodes_clear(numa_nodes_parsed);
|
2011-05-02 12:18:53 +00:00
|
|
|
nodes_clear(node_possible_map);
|
|
|
|
nodes_clear(node_online_map);
|
x86, ACPI, mm: Revert movablemem_map support
Tim found:
WARNING: at arch/x86/kernel/smpboot.c:324 topology_sane.isra.2+0x6f/0x80()
Hardware name: S2600CP
sched: CPU #1's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency.
smpboot: Booting Node 1, Processors #1
Modules linked in:
Pid: 0, comm: swapper/1 Not tainted 3.9.0-0-generic #1
Call Trace:
set_cpu_sibling_map+0x279/0x449
start_secondary+0x11d/0x1e5
Don Morris reproduced on a HP z620 workstation, and bisected it to
commit e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock
is ready")
It turns out movable_map has some problems, and it breaks several things
1. numa_init is called several times, NOT just for srat. so those
nodes_clear(numa_nodes_parsed)
memset(&numa_meminfo, 0, sizeof(numa_meminfo))
can not be just removed. Need to consider sequence is: numaq, srat, amd, dummy.
and make fall back path working.
2. simply split acpi_numa_init to early_parse_srat.
a. that early_parse_srat is NOT called for ia64, so you break ia64.
b. for (i = 0; i < MAX_LOCAL_APIC; i++)
set_apicid_to_node(i, NUMA_NO_NODE)
still left in numa_init. So it will just clear result from early_parse_srat.
it should be moved before that....
c. it breaks ACPI_TABLE_OVERIDE...as the acpi table scan is moved
early before override from INITRD is settled.
3. that patch TITLE is total misleading, there is NO x86 in the title,
but it changes critical x86 code. It caused x86 guys did not
pay attention to find the problem early. Those patches really should
be routed via tip/x86/mm.
4. after that commit, following range can not use movable ram:
a. real_mode code.... well..funny, legacy Node0 [0,1M) could be hot-removed?
b. initrd... it will be freed after booting, so it could be on movable...
c. crashkernel for kdump...: looks like we can not put kdump kernel above 4G
anymore.
d. init_mem_mapping: can not put page table high anymore.
e. initmem_init: vmemmap can not be high local node anymore. That is
not good.
If node is hotplugable, the mem related range like page table and
vmemmap could be on the that node without problem and should be on that
node.
We have workaround patch that could fix some problems, but some can not
be fixed.
So just remove that offending commit and related ones including:
f7210e6c4ac7 ("mm/memblock.c: use CONFIG_HAVE_MEMBLOCK_NODE_MAP to
protect movablecore_map in memblock_overlaps_region().")
01a178a94e8e ("acpi, memory-hotplug: support getting hotplug info from
SRAT")
27168d38fa20 ("acpi, memory-hotplug: extend movablemem_map ranges to
the end of node")
e8d195525809 ("acpi, memory-hotplug: parse SRAT before memblock is
ready")
fb06bc8e5f42 ("page_alloc: bootmem limit with movablecore_map")
42f47e27e761 ("page_alloc: make movablemem_map have higher priority")
6981ec31146c ("page_alloc: introduce zone_movable_limit[] to keep
movable limit for nodes")
34b71f1e04fc ("page_alloc: add movable_memmap kernel parameter")
4d59a75125d5 ("x86: get pg_data_t's memory from other node")
Later we should have patches that will make sure kernel put page table
and vmemmap on local node ram instead of push them down to node0. Also
need to find way to put other kernel used ram to local node ram.
Reported-by: Tim Gardner <tim.gardner@canonical.com>
Reported-by: Don Morris <don.morris@hp.com>
Bisected-by: Don Morris <don.morris@hp.com>
Tested-by: Don Morris <don.morris@hp.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Tejun Heo <tj@kernel.org>
Cc: Tang Chen <tangchen@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-03-01 22:51:27 +00:00
|
|
|
memset(&numa_meminfo, 0, sizeof(numa_meminfo));
|
2014-01-21 23:49:26 +00:00
|
|
|
WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.memory,
|
|
|
|
MAX_NUMNODES));
|
2014-01-21 23:49:32 +00:00
|
|
|
WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.reserved,
|
|
|
|
MAX_NUMNODES));
|
2014-01-21 23:49:29 +00:00
|
|
|
/* In case that parsing SRAT failed. */
|
|
|
|
WARN_ON(memblock_clear_hotplug(0, ULLONG_MAX));
|
2011-05-02 12:18:53 +00:00
|
|
|
numa_reset_distance();
|
|
|
|
|
|
|
|
ret = init_func();
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
mem-hotplug: introduce movable_node boot option
The hot-Pluggable field in SRAT specifies which memory is hotpluggable.
As we mentioned before, if hotpluggable memory is used by the kernel, it
cannot be hot-removed. So memory hotplug users may want to set all
hotpluggable memory in ZONE_MOVABLE so that the kernel won't use it.
Memory hotplug users may also set a node as movable node, which has
ZONE_MOVABLE only, so that the whole node can be hot-removed.
But the kernel cannot use memory in ZONE_MOVABLE. By doing this, the
kernel cannot use memory in movable nodes. This will cause NUMA
performance down. And other users may be unhappy.
So we need a way to allow users to enable and disable this functionality.
In this patch, we introduce movable_node boot option to allow users to
choose to not to consume hotpluggable memory at early boot time and later
we can set it as ZONE_MOVABLE.
To achieve this, the movable_node boot option will control the memblock
allocation direction. That said, after memblock is ready, before SRAT is
parsed, we should allocate memory near the kernel image as we explained in
the previous patches. So if movable_node boot option is set, the kernel
does the following:
1. After memblock is ready, make memblock allocate memory bottom up.
2. After SRAT is parsed, make memblock behave as default, allocate memory
top down.
Users can specify "movable_node" in kernel commandline to enable this
functionality. For those who don't use memory hotplug or who don't want
to lose their NUMA performance, just don't specify anything. The kernel
will work as before.
Signed-off-by: Tang Chen <tangchen@cn.fujitsu.com>
Signed-off-by: Zhang Yanfei <zhangyanfei@cn.fujitsu.com>
Suggested-by: Kamezawa Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Suggested-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Tejun Heo <tj@kernel.org>
Acked-by: Toshi Kani <toshi.kani@hp.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Wanpeng Li <liwanp@linux.vnet.ibm.com>
Cc: Thomas Renninger <trenn@suse.de>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Jiang Liu <jiang.liu@huawei.com>
Cc: Wen Congyang <wency@cn.fujitsu.com>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Cc: Yasuaki Ishimatsu <isimatu.yasuaki@jp.fujitsu.com>
Cc: Taku Izumi <izumi.taku@jp.fujitsu.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Michal Nazarewicz <mina86@mina86.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 23:08:10 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We reset memblock back to the top-down direction
|
|
|
|
* here because if we configured ACPI_NUMA, we have
|
|
|
|
* parsed SRAT in init_func(). It is ok to have the
|
|
|
|
* reset here even if we did't configure ACPI_NUMA
|
|
|
|
* or acpi numa init fails and fallbacks to dummy
|
|
|
|
* numa init.
|
|
|
|
*/
|
|
|
|
memblock_set_bottom_up(false);
|
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
ret = numa_cleanup_meminfo(&numa_meminfo);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
numa_emulation(&numa_meminfo, numa_distance_cnt);
|
|
|
|
|
|
|
|
ret = numa_register_memblks(&numa_meminfo);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_cpu_ids; i++) {
|
|
|
|
int nid = early_cpu_to_node(i);
|
|
|
|
|
|
|
|
if (nid == NUMA_NO_NODE)
|
|
|
|
continue;
|
|
|
|
if (!node_online(nid))
|
|
|
|
numa_clear_node(i);
|
|
|
|
}
|
|
|
|
numa_init_array();
|
2014-01-21 23:49:32 +00:00
|
|
|
|
2011-05-02 12:18:53 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dummy_numa_init - Fallback dummy NUMA init
|
|
|
|
*
|
|
|
|
* Used if there's no underlying NUMA architecture, NUMA initialization
|
|
|
|
* fails, or NUMA is disabled on the command line.
|
|
|
|
*
|
|
|
|
* Must online at least one node and add memory blocks that cover all
|
|
|
|
* allowed memory. This function must not fail.
|
|
|
|
*/
|
|
|
|
static int __init dummy_numa_init(void)
|
|
|
|
{
|
|
|
|
printk(KERN_INFO "%s\n",
|
|
|
|
numa_off ? "NUMA turned off" : "No NUMA configuration found");
|
2012-05-29 22:06:29 +00:00
|
|
|
printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n",
|
|
|
|
0LLU, PFN_PHYS(max_pfn) - 1);
|
2011-05-02 12:18:53 +00:00
|
|
|
|
|
|
|
node_set(0, numa_nodes_parsed);
|
2011-05-02 12:18:53 +00:00
|
|
|
numa_add_memblk(0, 0, PFN_PHYS(max_pfn));
|
2011-05-02 12:18:53 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* x86_numa_init - Initialize NUMA
|
|
|
|
*
|
|
|
|
* Try each configured NUMA initialization method until one succeeds. The
|
|
|
|
* last fallback is dummy single node config encomapssing whole memory and
|
|
|
|
* never fails.
|
|
|
|
*/
|
|
|
|
void __init x86_numa_init(void)
|
|
|
|
{
|
|
|
|
if (!numa_off) {
|
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
|
|
|
if (!numa_init(x86_acpi_numa_init))
|
|
|
|
return;
|
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_AMD_NUMA
|
|
|
|
if (!numa_init(amd_numa_init))
|
|
|
|
return;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
numa_init(dummy_numa_init);
|
|
|
|
}
|
|
|
|
|
2016-08-25 08:35:14 +00:00
|
|
|
static void __init init_memory_less_node(int nid)
|
2011-01-23 13:37:42 +00:00
|
|
|
{
|
2016-08-25 08:35:14 +00:00
|
|
|
unsigned long zones_size[MAX_NR_ZONES] = {0};
|
|
|
|
unsigned long zholes_size[MAX_NR_ZONES] = {0};
|
2011-01-23 13:37:42 +00:00
|
|
|
|
2016-08-25 08:35:14 +00:00
|
|
|
/* Allocate and initialize node data. Memory-less node is now online.*/
|
|
|
|
alloc_node_data(nid);
|
|
|
|
free_area_init_node(nid, zones_size, 0, zholes_size);
|
2011-01-23 13:37:42 +00:00
|
|
|
|
2016-08-25 08:35:14 +00:00
|
|
|
/*
|
|
|
|
* All zonelists will be built later in start_kernel() after per cpu
|
|
|
|
* areas are initialized.
|
|
|
|
*/
|
2011-01-23 13:37:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Setup early cpu_to_node.
|
|
|
|
*
|
|
|
|
* Populate cpu_to_node[] only if x86_cpu_to_apicid[],
|
|
|
|
* and apicid_to_node[] tables have valid entries for a CPU.
|
|
|
|
* This means we skip cpu_to_node[] initialisation for NUMA
|
|
|
|
* emulation and faking node case (when running a kernel compiled
|
|
|
|
* for NUMA on a non NUMA box), which is OK as cpu_to_node[]
|
|
|
|
* is already initialized in a round robin manner at numa_init_array,
|
|
|
|
* prior to this call, and this initialization is good enough
|
|
|
|
* for the fake NUMA cases.
|
|
|
|
*
|
|
|
|
* Called before the per_cpu areas are setup.
|
|
|
|
*/
|
|
|
|
void __init init_cpu_to_node(void)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
|
|
|
|
|
|
|
|
BUG_ON(cpu_to_apicid == NULL);
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
int node = numa_cpu_node(cpu);
|
|
|
|
|
|
|
|
if (node == NUMA_NO_NODE)
|
|
|
|
continue;
|
2016-08-25 08:35:14 +00:00
|
|
|
|
2011-01-23 13:37:42 +00:00
|
|
|
if (!node_online(node))
|
2016-08-25 08:35:14 +00:00
|
|
|
init_memory_less_node(node);
|
|
|
|
|
2011-01-23 13:37:42 +00:00
|
|
|
numa_set_node(cpu, node);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-01-23 13:37:41 +00:00
|
|
|
#ifndef CONFIG_DEBUG_PER_CPU_MAPS
|
|
|
|
|
|
|
|
# ifndef CONFIG_NUMA_EMU
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
void numa_add_cpu(int cpu)
|
2011-01-23 13:37:41 +00:00
|
|
|
{
|
|
|
|
cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
void numa_remove_cpu(int cpu)
|
2011-01-23 13:37:41 +00:00
|
|
|
{
|
|
|
|
cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]);
|
|
|
|
}
|
|
|
|
# endif /* !CONFIG_NUMA_EMU */
|
|
|
|
|
|
|
|
#else /* !CONFIG_DEBUG_PER_CPU_MAPS */
|
2011-01-23 13:37:40 +00:00
|
|
|
|
|
|
|
int __cpu_to_node(int cpu)
|
|
|
|
{
|
|
|
|
if (early_per_cpu_ptr(x86_cpu_to_node_map)) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"cpu_to_node(%d): usage too early!\n", cpu);
|
|
|
|
dump_stack();
|
|
|
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
|
|
}
|
|
|
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__cpu_to_node);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Same function as cpu_to_node() but used if called before the
|
|
|
|
* per_cpu areas are setup.
|
|
|
|
*/
|
|
|
|
int early_cpu_to_node(int cpu)
|
|
|
|
{
|
|
|
|
if (early_per_cpu_ptr(x86_cpu_to_node_map))
|
|
|
|
return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
|
|
|
|
|
|
|
|
if (!cpu_possible(cpu)) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
|
|
|
|
dump_stack();
|
|
|
|
return NUMA_NO_NODE;
|
|
|
|
}
|
|
|
|
return per_cpu(x86_cpu_to_node_map, cpu);
|
|
|
|
}
|
|
|
|
|
2011-04-21 02:19:13 +00:00
|
|
|
void debug_cpumask_set_cpu(int cpu, int node, bool enable)
|
2011-01-23 13:37:41 +00:00
|
|
|
{
|
|
|
|
struct cpumask *mask;
|
|
|
|
|
2011-02-07 22:08:53 +00:00
|
|
|
if (node == NUMA_NO_NODE) {
|
|
|
|
/* early_cpu_to_node() already emits a warning and trace */
|
2011-04-21 02:19:13 +00:00
|
|
|
return;
|
2011-02-07 22:08:53 +00:00
|
|
|
}
|
2011-01-23 13:37:41 +00:00
|
|
|
mask = node_to_cpumask_map[node];
|
|
|
|
if (!mask) {
|
|
|
|
pr_err("node_to_cpumask_map[%i] NULL\n", node);
|
|
|
|
dump_stack();
|
2011-04-21 02:19:13 +00:00
|
|
|
return;
|
2011-01-23 13:37:41 +00:00
|
|
|
}
|
|
|
|
|
2011-04-21 02:19:13 +00:00
|
|
|
if (enable)
|
|
|
|
cpumask_set_cpu(cpu, mask);
|
|
|
|
else
|
|
|
|
cpumask_clear_cpu(cpu, mask);
|
|
|
|
|
2015-02-13 22:37:12 +00:00
|
|
|
printk(KERN_DEBUG "%s cpu %d node %d: mask now %*pbl\n",
|
2011-01-23 13:37:41 +00:00
|
|
|
enable ? "numa_add_cpu" : "numa_remove_cpu",
|
2015-02-13 22:37:12 +00:00
|
|
|
cpu, node, cpumask_pr_args(mask));
|
2011-04-21 02:19:13 +00:00
|
|
|
return;
|
2011-01-23 13:37:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
# ifndef CONFIG_NUMA_EMU
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
static void numa_set_cpumask(int cpu, bool enable)
|
2011-01-23 13:37:41 +00:00
|
|
|
{
|
2011-04-21 02:19:13 +00:00
|
|
|
debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable);
|
2011-01-23 13:37:41 +00:00
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
void numa_add_cpu(int cpu)
|
2011-01-23 13:37:41 +00:00
|
|
|
{
|
2011-04-21 02:19:13 +00:00
|
|
|
numa_set_cpumask(cpu, true);
|
2011-01-23 13:37:41 +00:00
|
|
|
}
|
|
|
|
|
x86: delete __cpuinit usage from all x86 files
The __cpuinit type of throwaway sections might have made sense
some time ago when RAM was more constrained, but now the savings
do not offset the cost and complications. For example, the fix in
commit 5e427ec2d0 ("x86: Fix bit corruption at CPU resume time")
is a good example of the nasty type of bugs that can be created
with improper use of the various __init prefixes.
After a discussion on LKML[1] it was decided that cpuinit should go
the way of devinit and be phased out. Once all the users are gone,
we can then finally remove the macros themselves from linux/init.h.
Note that some harmless section mismatch warnings may result, since
notify_cpu_starting() and cpu_up() are arch independent (kernel/cpu.c)
are flagged as __cpuinit -- so if we remove the __cpuinit from
arch specific callers, we will also get section mismatch warnings.
As an intermediate step, we intend to turn the linux/init.h cpuinit
content into no-ops as early as possible, since that will get rid
of these warnings. In any case, they are temporary and harmless.
This removes all the arch/x86 uses of the __cpuinit macros from
all C files. x86 only had the one __CPUINIT used in assembly files,
and it wasn't paired off with a .previous or a __FINIT, so we can
delete it directly w/o any corresponding additional change there.
[1] https://lkml.org/lkml/2013/5/20/589
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Acked-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
2013-06-18 22:23:59 +00:00
|
|
|
void numa_remove_cpu(int cpu)
|
2011-01-23 13:37:41 +00:00
|
|
|
{
|
2011-04-21 02:19:13 +00:00
|
|
|
numa_set_cpumask(cpu, false);
|
2011-01-23 13:37:41 +00:00
|
|
|
}
|
|
|
|
# endif /* !CONFIG_NUMA_EMU */
|
|
|
|
|
2009-03-13 04:19:52 +00:00
|
|
|
/*
|
|
|
|
* Returns a pointer to the bitmask of CPUs on Node 'node'.
|
|
|
|
*/
|
2009-03-13 04:19:57 +00:00
|
|
|
const struct cpumask *cpumask_of_node(int node)
|
2009-03-13 04:19:52 +00:00
|
|
|
{
|
|
|
|
if (node >= nr_node_ids) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"cpumask_of_node(%d): node > nr_node_ids(%d)\n",
|
|
|
|
node, nr_node_ids);
|
|
|
|
dump_stack();
|
|
|
|
return cpu_none_mask;
|
|
|
|
}
|
2009-03-13 04:19:53 +00:00
|
|
|
if (node_to_cpumask_map[node] == NULL) {
|
|
|
|
printk(KERN_WARNING
|
|
|
|
"cpumask_of_node(%d): no node_to_cpumask_map!\n",
|
|
|
|
node);
|
|
|
|
dump_stack();
|
|
|
|
return cpu_online_mask;
|
|
|
|
}
|
2009-03-13 13:12:42 +00:00
|
|
|
return node_to_cpumask_map[node];
|
2009-03-13 04:19:52 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(cpumask_of_node);
|
2011-01-23 13:37:40 +00:00
|
|
|
|
2011-01-23 13:37:41 +00:00
|
|
|
#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
|
2011-05-02 12:18:53 +00:00
|
|
|
|
2011-05-02 15:24:48 +00:00
|
|
|
#ifdef CONFIG_MEMORY_HOTPLUG
|
2011-05-02 12:18:53 +00:00
|
|
|
int memory_add_physaddr_to_nid(u64 start)
|
|
|
|
{
|
|
|
|
struct numa_meminfo *mi = &numa_meminfo;
|
|
|
|
int nid = mi->blk[0].nid;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < mi->nr_blks; i++)
|
|
|
|
if (mi->blk[i].start <= start && mi->blk[i].end > start)
|
|
|
|
nid = mi->blk[i].nid;
|
|
|
|
return nid;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
|
|
|
|
#endif
|