2012-08-02 08:16:29 +00:00
|
|
|
/*
|
2014-11-13 09:39:00 +00:00
|
|
|
* Coherency fabric (Aurora) support for Armada 370, 375, 38x and XP
|
|
|
|
* platforms.
|
2012-08-02 08:16:29 +00:00
|
|
|
*
|
|
|
|
* Copyright (C) 2012 Marvell
|
|
|
|
*
|
|
|
|
* Yehuda Yitschak <yehuday@marvell.com>
|
|
|
|
* Gregory Clement <gregory.clement@free-electrons.com>
|
|
|
|
* Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
|
|
|
|
*
|
|
|
|
* This file is licensed under the terms of the GNU General Public
|
|
|
|
* License version 2. This program is licensed "as is" without any
|
|
|
|
* warranty of any kind, whether express or implied.
|
|
|
|
*
|
2014-11-13 09:39:00 +00:00
|
|
|
* The Armada 370, 375, 38x and XP SOCs have a coherency fabric which is
|
2012-08-02 08:16:29 +00:00
|
|
|
* responsible for ensuring hardware coherency between all CPUs and between
|
|
|
|
* CPUs and I/O masters. This file initializes the coherency fabric and
|
|
|
|
* supplies basic routines for configuring and controlling hardware coherency
|
|
|
|
*/
|
|
|
|
|
2014-04-14 13:47:05 +00:00
|
|
|
#define pr_fmt(fmt) "mvebu-coherency: " fmt
|
|
|
|
|
2012-08-02 08:16:29 +00:00
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/of_address.h>
|
|
|
|
#include <linux/io.h>
|
|
|
|
#include <linux/smp.h>
|
2012-10-12 17:20:36 +00:00
|
|
|
#include <linux/dma-mapping.h>
|
|
|
|
#include <linux/platform_device.h>
|
2014-04-14 13:47:05 +00:00
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/mbus.h>
|
2014-05-13 16:04:30 +00:00
|
|
|
#include <linux/pci.h>
|
2012-08-02 08:16:29 +00:00
|
|
|
#include <asm/smp_plat.h>
|
2013-06-06 10:24:28 +00:00
|
|
|
#include <asm/cacheflush.h>
|
2014-05-15 14:59:34 +00:00
|
|
|
#include <asm/mach/map.h>
|
2015-01-16 16:11:29 +00:00
|
|
|
#include <asm/dma-mapping.h>
|
2013-11-07 09:02:38 +00:00
|
|
|
#include "coherency.h"
|
2014-05-05 15:05:26 +00:00
|
|
|
#include "mvebu-soc-id.h"
|
2012-08-02 08:16:29 +00:00
|
|
|
|
2013-06-17 19:43:14 +00:00
|
|
|
unsigned long coherency_phys_base;
|
2014-04-14 15:10:05 +00:00
|
|
|
void __iomem *coherency_base;
|
2012-10-12 17:20:36 +00:00
|
|
|
static void __iomem *coherency_cpu_base;
|
2015-07-08 14:09:20 +00:00
|
|
|
static void __iomem *cpu_config_base;
|
2012-08-02 08:16:29 +00:00
|
|
|
|
|
|
|
/* Coherency fabric registers */
|
2012-10-12 17:20:36 +00:00
|
|
|
#define IO_SYNC_BARRIER_CTL_OFFSET 0x0
|
|
|
|
|
2014-04-14 13:46:59 +00:00
|
|
|
enum {
|
2014-04-14 13:47:00 +00:00
|
|
|
COHERENCY_FABRIC_TYPE_NONE,
|
2014-04-14 13:46:59 +00:00
|
|
|
COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
|
ARM: mvebu: add Armada 375 support to the coherency code
The Armada 375, like the Armada 370 and Armada XP, has a coherency
unit. However, unlike the coherency unit of 370/XP which does both CPU
and I/O coherency, the one on Armada 735 only does I/O
coherency. Therefore, instead of having two sets of registers (the
first one being used mainly to register each CPU in the coherency
fabric, the second one being used for the I/O coherency barrier), it
has only one set of register (for the I/O coherency barrier).
This commit adds a new "marvell,armada-375-coherency-fabric"
compatible string for this variant of the coherency fabric. The custom
DMA operations, and the way of triggering an I/O barrier is the same
as Armada 370/XP, so the code changes are minimal. However, the
set_cpu_coherent() function is not needed on Armada 375 and will not
work.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Link: https://lkml.kernel.org/r/1397483228-25625-7-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-04-14 13:47:04 +00:00
|
|
|
COHERENCY_FABRIC_TYPE_ARMADA_375,
|
2014-04-14 13:47:06 +00:00
|
|
|
COHERENCY_FABRIC_TYPE_ARMADA_380,
|
2014-04-14 13:46:59 +00:00
|
|
|
};
|
|
|
|
|
2015-02-18 20:19:56 +00:00
|
|
|
static const struct of_device_id of_coherency_table[] = {
|
2014-04-14 13:46:59 +00:00
|
|
|
{.compatible = "marvell,coherency-fabric",
|
|
|
|
.data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
|
ARM: mvebu: add Armada 375 support to the coherency code
The Armada 375, like the Armada 370 and Armada XP, has a coherency
unit. However, unlike the coherency unit of 370/XP which does both CPU
and I/O coherency, the one on Armada 735 only does I/O
coherency. Therefore, instead of having two sets of registers (the
first one being used mainly to register each CPU in the coherency
fabric, the second one being used for the I/O coherency barrier), it
has only one set of register (for the I/O coherency barrier).
This commit adds a new "marvell,armada-375-coherency-fabric"
compatible string for this variant of the coherency fabric. The custom
DMA operations, and the way of triggering an I/O barrier is the same
as Armada 370/XP, so the code changes are minimal. However, the
set_cpu_coherent() function is not needed on Armada 375 and will not
work.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Link: https://lkml.kernel.org/r/1397483228-25625-7-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-04-14 13:47:04 +00:00
|
|
|
{.compatible = "marvell,armada-375-coherency-fabric",
|
|
|
|
.data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
|
2014-04-14 13:47:06 +00:00
|
|
|
{.compatible = "marvell,armada-380-coherency-fabric",
|
|
|
|
.data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
|
2012-08-02 08:16:29 +00:00
|
|
|
{ /* end of list */ },
|
|
|
|
};
|
|
|
|
|
2014-04-14 15:10:08 +00:00
|
|
|
/* Functions defined in coherency_ll.S */
|
|
|
|
int ll_enable_coherency(void);
|
|
|
|
void ll_add_cpu_to_smp_group(void);
|
2012-08-02 08:16:29 +00:00
|
|
|
|
2015-07-08 14:09:20 +00:00
|
|
|
#define CPU_CONFIG_SHARED_L2 BIT(16)
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable the "Shared L2 Present" bit in CPU Configuration register
|
|
|
|
* on Armada XP.
|
|
|
|
*
|
|
|
|
* The "Shared L2 Present" bit affects the "level of coherence" value
|
|
|
|
* in the clidr CP15 register. Cache operation functions such as
|
|
|
|
* "flush all" and "invalidate all" operate on all the cache levels
|
|
|
|
* that included in the defined level of coherence. When HW I/O
|
|
|
|
* coherency is used, this bit causes unnecessary flushes of the L2
|
|
|
|
* cache.
|
|
|
|
*/
|
|
|
|
static void armada_xp_clear_shared_l2(void)
|
|
|
|
{
|
|
|
|
u32 reg;
|
|
|
|
|
|
|
|
if (!cpu_config_base)
|
|
|
|
return;
|
|
|
|
|
|
|
|
reg = readl(cpu_config_base);
|
|
|
|
reg &= ~CPU_CONFIG_SHARED_L2;
|
|
|
|
writel(reg, cpu_config_base);
|
|
|
|
}
|
|
|
|
|
2014-05-13 16:04:30 +00:00
|
|
|
static int mvebu_hwcc_notifier(struct notifier_block *nb,
|
|
|
|
unsigned long event, void *__dev)
|
2012-10-12 17:20:36 +00:00
|
|
|
{
|
|
|
|
struct device *dev = __dev;
|
|
|
|
|
|
|
|
if (event != BUS_NOTIFY_ADD_DEVICE)
|
|
|
|
return NOTIFY_DONE;
|
2015-01-16 16:11:29 +00:00
|
|
|
set_dma_ops(dev, &arm_coherent_dma_ops);
|
2012-10-12 17:20:36 +00:00
|
|
|
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
2014-05-13 16:04:30 +00:00
|
|
|
static struct notifier_block mvebu_hwcc_nb = {
|
|
|
|
.notifier_call = mvebu_hwcc_notifier,
|
2012-10-12 17:20:36 +00:00
|
|
|
};
|
|
|
|
|
2016-02-23 14:06:39 +00:00
|
|
|
static struct notifier_block mvebu_hwcc_pci_nb __maybe_unused = {
|
2014-07-08 13:37:37 +00:00
|
|
|
.notifier_call = mvebu_hwcc_notifier,
|
|
|
|
};
|
|
|
|
|
2015-07-08 14:09:20 +00:00
|
|
|
static int armada_xp_clear_shared_l2_notifier_func(struct notifier_block *nfb,
|
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
|
|
|
if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
|
|
|
|
armada_xp_clear_shared_l2();
|
|
|
|
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct notifier_block armada_xp_clear_shared_l2_notifier = {
|
|
|
|
.notifier_call = armada_xp_clear_shared_l2_notifier_func,
|
|
|
|
.priority = 100,
|
|
|
|
};
|
|
|
|
|
2014-04-14 13:46:59 +00:00
|
|
|
static void __init armada_370_coherency_init(struct device_node *np)
|
|
|
|
{
|
|
|
|
struct resource res;
|
2015-07-08 14:09:20 +00:00
|
|
|
struct device_node *cpu_config_np;
|
2014-04-14 13:46:59 +00:00
|
|
|
|
|
|
|
of_address_to_resource(np, 0, &res);
|
|
|
|
coherency_phys_base = res.start;
|
|
|
|
/*
|
|
|
|
* Ensure secondary CPUs will see the updated value,
|
|
|
|
* which they read before they join the coherency
|
|
|
|
* fabric, and therefore before they are coherent with
|
|
|
|
* the boot CPU cache.
|
|
|
|
*/
|
|
|
|
sync_cache_w(&coherency_phys_base);
|
|
|
|
coherency_base = of_iomap(np, 0);
|
|
|
|
coherency_cpu_base = of_iomap(np, 1);
|
2015-07-08 14:09:20 +00:00
|
|
|
|
|
|
|
cpu_config_np = of_find_compatible_node(NULL, NULL,
|
|
|
|
"marvell,armada-xp-cpu-config");
|
|
|
|
if (!cpu_config_np)
|
|
|
|
goto exit;
|
|
|
|
|
|
|
|
cpu_config_base = of_iomap(cpu_config_np, 0);
|
|
|
|
if (!cpu_config_base) {
|
|
|
|
of_node_put(cpu_config_np);
|
|
|
|
goto exit;
|
|
|
|
}
|
|
|
|
|
|
|
|
of_node_put(cpu_config_np);
|
|
|
|
|
|
|
|
register_cpu_notifier(&armada_xp_clear_shared_l2_notifier);
|
|
|
|
|
|
|
|
exit:
|
2014-04-14 15:10:07 +00:00
|
|
|
set_cpu_coherent();
|
2014-04-14 13:46:59 +00:00
|
|
|
}
|
|
|
|
|
2014-05-15 14:59:34 +00:00
|
|
|
/*
|
2016-06-16 13:42:25 +00:00
|
|
|
* This ioremap hook is used on Armada 375/38x to ensure that all MMIO
|
|
|
|
* areas are mapped as MT_UNCACHED instead of MT_DEVICE. This is
|
|
|
|
* needed for the HW I/O coherency mechanism to work properly without
|
|
|
|
* deadlock.
|
2014-05-15 14:59:34 +00:00
|
|
|
*/
|
|
|
|
static void __iomem *
|
2016-06-16 13:42:25 +00:00
|
|
|
armada_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
|
|
|
|
unsigned int mtype, void *caller)
|
2014-05-15 14:59:34 +00:00
|
|
|
{
|
2016-06-16 13:42:25 +00:00
|
|
|
mtype = MT_UNCACHED;
|
2014-05-15 14:59:34 +00:00
|
|
|
return __arm_ioremap_caller(phys_addr, size, mtype, caller);
|
|
|
|
}
|
|
|
|
|
2014-04-14 13:47:06 +00:00
|
|
|
static void __init armada_375_380_coherency_init(struct device_node *np)
|
ARM: mvebu: add Armada 375 support to the coherency code
The Armada 375, like the Armada 370 and Armada XP, has a coherency
unit. However, unlike the coherency unit of 370/XP which does both CPU
and I/O coherency, the one on Armada 735 only does I/O
coherency. Therefore, instead of having two sets of registers (the
first one being used mainly to register each CPU in the coherency
fabric, the second one being used for the I/O coherency barrier), it
has only one set of register (for the I/O coherency barrier).
This commit adds a new "marvell,armada-375-coherency-fabric"
compatible string for this variant of the coherency fabric. The custom
DMA operations, and the way of triggering an I/O barrier is the same
as Armada 370/XP, so the code changes are minimal. However, the
set_cpu_coherent() function is not needed on Armada 375 and will not
work.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Link: https://lkml.kernel.org/r/1397483228-25625-7-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-04-14 13:47:04 +00:00
|
|
|
{
|
2014-05-15 14:59:34 +00:00
|
|
|
struct device_node *cache_dn;
|
|
|
|
|
ARM: mvebu: add Armada 375 support to the coherency code
The Armada 375, like the Armada 370 and Armada XP, has a coherency
unit. However, unlike the coherency unit of 370/XP which does both CPU
and I/O coherency, the one on Armada 735 only does I/O
coherency. Therefore, instead of having two sets of registers (the
first one being used mainly to register each CPU in the coherency
fabric, the second one being used for the I/O coherency barrier), it
has only one set of register (for the I/O coherency barrier).
This commit adds a new "marvell,armada-375-coherency-fabric"
compatible string for this variant of the coherency fabric. The custom
DMA operations, and the way of triggering an I/O barrier is the same
as Armada 370/XP, so the code changes are minimal. However, the
set_cpu_coherent() function is not needed on Armada 375 and will not
work.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Link: https://lkml.kernel.org/r/1397483228-25625-7-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-04-14 13:47:04 +00:00
|
|
|
coherency_cpu_base = of_iomap(np, 0);
|
2016-06-16 13:42:25 +00:00
|
|
|
arch_ioremap_caller = armada_wa_ioremap_caller;
|
2014-05-15 14:59:34 +00:00
|
|
|
|
ARM: mvebu: don't set the PL310 in I/O coherency mode when I/O coherency is disabled
Since commit f2c3c67f00 (merge commit that adds commit "ARM: mvebu:
completely disable hardware I/O coherency"), we disable I/O coherency
on Armada EBU platforms.
However, we continue to initialize the coherency fabric, because this
coherency fabric is needed on Armada XP for inter-CPU
coherency. Unfortunately, due to this, we also continued to execute
the coherency fabric initialization code for Armada 375/38x, which
switched the PL310 into I/O coherent mode. This has the effect of
disabling the outer cache sync operation: this is needed when I/O
coherency is enabled to work around a PCIe/L2 deadlock. But obviously,
when I/O coherency is disabled, having the outer cache sync operation
is crucial.
Therefore, this commit fixes the armada_375_380_coherency_init() so
that the PL310 is switched to I/O coherent mode only if I/O coherency
is enabled.
Without this fix, all devices using DMA are broken on Armada 375/38x.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Tested-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Signed-off-by: Andrew Lunn <andrew@lunn.ch>
Cc: <stable@vger.kernel.org> # v3.8+
2015-01-28 11:55:45 +00:00
|
|
|
/*
|
|
|
|
* We should switch the PL310 to I/O coherency mode only if
|
|
|
|
* I/O coherency is actually enabled.
|
|
|
|
*/
|
|
|
|
if (!coherency_available())
|
|
|
|
return;
|
|
|
|
|
2014-05-15 14:59:34 +00:00
|
|
|
/*
|
|
|
|
* Add the PL310 property "arm,io-coherent". This makes sure the
|
|
|
|
* outer sync operation is not used, which allows to
|
|
|
|
* workaround the system erratum that causes deadlocks when
|
|
|
|
* doing PCIe in an SMP situation on Armada 375 and Armada
|
|
|
|
* 38x.
|
|
|
|
*/
|
|
|
|
for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
|
|
|
|
struct property *p;
|
|
|
|
|
|
|
|
p = kzalloc(sizeof(*p), GFP_KERNEL);
|
|
|
|
p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
|
|
|
|
of_add_property(cache_dn, p);
|
|
|
|
}
|
ARM: mvebu: add Armada 375 support to the coherency code
The Armada 375, like the Armada 370 and Armada XP, has a coherency
unit. However, unlike the coherency unit of 370/XP which does both CPU
and I/O coherency, the one on Armada 735 only does I/O
coherency. Therefore, instead of having two sets of registers (the
first one being used mainly to register each CPU in the coherency
fabric, the second one being used for the I/O coherency barrier), it
has only one set of register (for the I/O coherency barrier).
This commit adds a new "marvell,armada-375-coherency-fabric"
compatible string for this variant of the coherency fabric. The custom
DMA operations, and the way of triggering an I/O barrier is the same
as Armada 370/XP, so the code changes are minimal. However, the
set_cpu_coherent() function is not needed on Armada 375 and will not
work.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Link: https://lkml.kernel.org/r/1397483228-25625-7-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-04-14 13:47:04 +00:00
|
|
|
}
|
|
|
|
|
2014-04-14 13:47:00 +00:00
|
|
|
static int coherency_type(void)
|
2012-08-02 08:16:29 +00:00
|
|
|
{
|
|
|
|
struct device_node *np;
|
2014-04-14 13:47:02 +00:00
|
|
|
const struct of_device_id *match;
|
ARM: mvebu: disable I/O coherency on non-SMP situations on Armada 370/375/38x/XP
Enabling the hardware I/O coherency on Armada 370, Armada 375, Armada
38x and Armada XP requires a certain number of conditions:
- On Armada 370, the cache policy must be set to write-allocate.
- On Armada 375, 38x and XP, the cache policy must be set to
write-allocate, the pages must be mapped with the shareable
attribute, and the SMP bit must be set
Currently, on Armada XP, when CONFIG_SMP is enabled, those conditions
are met. However, when Armada XP is used in a !CONFIG_SMP kernel, none
of these conditions are met. With Armada 370, the situation is worse:
since the processor is single core, regardless of whether CONFIG_SMP
or !CONFIG_SMP is used, the cache policy will be set to write-back by
the kernel and not write-allocate.
Since solving this problem turns out to be quite complicated, and we
don't want to let users with a mainline kernel known to have
infrequent but existing data corruptions, this commit proposes to
simply disable hardware I/O coherency in situations where it is known
not to work.
And basically, the is_smp() function of the kernel tells us whether it
is OK to enable hardware I/O coherency or not, so this commit slightly
refactors the coherency_type() function to return
COHERENCY_FABRIC_TYPE_NONE when is_smp() is false, or the appropriate
type of the coherency fabric in the other case.
Thanks to this, the I/O coherency fabric will no longer be used at all
in !CONFIG_SMP configurations. It will continue to be used in
CONFIG_SMP configurations on Armada XP, Armada 375 and Armada 38x
(which are multiple cores processors), but will no longer be used on
Armada 370 (which is a single core processor).
In the process, it simplifies the implementation of the
coherency_type() function, and adds a missing call to of_node_put().
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Fixes: e60304f8cb7bb545e79fe62d9b9762460c254ec2 ("arm: mvebu: Add hardware I/O Coherency support")
Cc: <stable@vger.kernel.org> # v3.8+
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Link: https://lkml.kernel.org/r/1415871540-20302-3-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-11-13 09:38:57 +00:00
|
|
|
int type;
|
2012-08-02 08:16:29 +00:00
|
|
|
|
ARM: mvebu: disable I/O coherency on non-SMP situations on Armada 370/375/38x/XP
Enabling the hardware I/O coherency on Armada 370, Armada 375, Armada
38x and Armada XP requires a certain number of conditions:
- On Armada 370, the cache policy must be set to write-allocate.
- On Armada 375, 38x and XP, the cache policy must be set to
write-allocate, the pages must be mapped with the shareable
attribute, and the SMP bit must be set
Currently, on Armada XP, when CONFIG_SMP is enabled, those conditions
are met. However, when Armada XP is used in a !CONFIG_SMP kernel, none
of these conditions are met. With Armada 370, the situation is worse:
since the processor is single core, regardless of whether CONFIG_SMP
or !CONFIG_SMP is used, the cache policy will be set to write-back by
the kernel and not write-allocate.
Since solving this problem turns out to be quite complicated, and we
don't want to let users with a mainline kernel known to have
infrequent but existing data corruptions, this commit proposes to
simply disable hardware I/O coherency in situations where it is known
not to work.
And basically, the is_smp() function of the kernel tells us whether it
is OK to enable hardware I/O coherency or not, so this commit slightly
refactors the coherency_type() function to return
COHERENCY_FABRIC_TYPE_NONE when is_smp() is false, or the appropriate
type of the coherency fabric in the other case.
Thanks to this, the I/O coherency fabric will no longer be used at all
in !CONFIG_SMP configurations. It will continue to be used in
CONFIG_SMP configurations on Armada XP, Armada 375 and Armada 38x
(which are multiple cores processors), but will no longer be used on
Armada 370 (which is a single core processor).
In the process, it simplifies the implementation of the
coherency_type() function, and adds a missing call to of_node_put().
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Fixes: e60304f8cb7bb545e79fe62d9b9762460c254ec2 ("arm: mvebu: Add hardware I/O Coherency support")
Cc: <stable@vger.kernel.org> # v3.8+
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Link: https://lkml.kernel.org/r/1415871540-20302-3-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-11-13 09:38:57 +00:00
|
|
|
/*
|
|
|
|
* The coherency fabric is needed:
|
|
|
|
* - For coherency between processors on Armada XP, so only
|
|
|
|
* when SMP is enabled.
|
|
|
|
* - For coherency between the processor and I/O devices, but
|
|
|
|
* this coherency requires many pre-requisites (write
|
|
|
|
* allocate cache policy, shareable pages, SMP bit set) that
|
|
|
|
* are only meant in SMP situations.
|
|
|
|
*
|
|
|
|
* Note that this means that on Armada 370, there is currently
|
|
|
|
* no way to use hardware I/O coherency, because even when
|
|
|
|
* CONFIG_SMP is enabled, is_smp() returns false due to the
|
|
|
|
* Armada 370 being a single-core processor. To lift this
|
|
|
|
* limitation, we would have to find a way to make the cache
|
|
|
|
* policy set to write-allocate (on all Armada SoCs), and to
|
|
|
|
* set the shareable attribute in page tables (on all Armada
|
|
|
|
* SoCs except the Armada 370). Unfortunately, such decisions
|
|
|
|
* are taken very early in the kernel boot process, at a point
|
|
|
|
* where we don't know yet on which SoC we are running.
|
|
|
|
|
|
|
|
*/
|
|
|
|
if (!is_smp())
|
|
|
|
return COHERENCY_FABRIC_TYPE_NONE;
|
2014-04-14 13:46:59 +00:00
|
|
|
|
ARM: mvebu: disable I/O coherency on non-SMP situations on Armada 370/375/38x/XP
Enabling the hardware I/O coherency on Armada 370, Armada 375, Armada
38x and Armada XP requires a certain number of conditions:
- On Armada 370, the cache policy must be set to write-allocate.
- On Armada 375, 38x and XP, the cache policy must be set to
write-allocate, the pages must be mapped with the shareable
attribute, and the SMP bit must be set
Currently, on Armada XP, when CONFIG_SMP is enabled, those conditions
are met. However, when Armada XP is used in a !CONFIG_SMP kernel, none
of these conditions are met. With Armada 370, the situation is worse:
since the processor is single core, regardless of whether CONFIG_SMP
or !CONFIG_SMP is used, the cache policy will be set to write-back by
the kernel and not write-allocate.
Since solving this problem turns out to be quite complicated, and we
don't want to let users with a mainline kernel known to have
infrequent but existing data corruptions, this commit proposes to
simply disable hardware I/O coherency in situations where it is known
not to work.
And basically, the is_smp() function of the kernel tells us whether it
is OK to enable hardware I/O coherency or not, so this commit slightly
refactors the coherency_type() function to return
COHERENCY_FABRIC_TYPE_NONE when is_smp() is false, or the appropriate
type of the coherency fabric in the other case.
Thanks to this, the I/O coherency fabric will no longer be used at all
in !CONFIG_SMP configurations. It will continue to be used in
CONFIG_SMP configurations on Armada XP, Armada 375 and Armada 38x
(which are multiple cores processors), but will no longer be used on
Armada 370 (which is a single core processor).
In the process, it simplifies the implementation of the
coherency_type() function, and adds a missing call to of_node_put().
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Fixes: e60304f8cb7bb545e79fe62d9b9762460c254ec2 ("arm: mvebu: Add hardware I/O Coherency support")
Cc: <stable@vger.kernel.org> # v3.8+
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Link: https://lkml.kernel.org/r/1415871540-20302-3-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-11-13 09:38:57 +00:00
|
|
|
np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
|
|
|
|
if (!np)
|
|
|
|
return COHERENCY_FABRIC_TYPE_NONE;
|
2014-04-14 13:46:59 +00:00
|
|
|
|
ARM: mvebu: disable I/O coherency on non-SMP situations on Armada 370/375/38x/XP
Enabling the hardware I/O coherency on Armada 370, Armada 375, Armada
38x and Armada XP requires a certain number of conditions:
- On Armada 370, the cache policy must be set to write-allocate.
- On Armada 375, 38x and XP, the cache policy must be set to
write-allocate, the pages must be mapped with the shareable
attribute, and the SMP bit must be set
Currently, on Armada XP, when CONFIG_SMP is enabled, those conditions
are met. However, when Armada XP is used in a !CONFIG_SMP kernel, none
of these conditions are met. With Armada 370, the situation is worse:
since the processor is single core, regardless of whether CONFIG_SMP
or !CONFIG_SMP is used, the cache policy will be set to write-back by
the kernel and not write-allocate.
Since solving this problem turns out to be quite complicated, and we
don't want to let users with a mainline kernel known to have
infrequent but existing data corruptions, this commit proposes to
simply disable hardware I/O coherency in situations where it is known
not to work.
And basically, the is_smp() function of the kernel tells us whether it
is OK to enable hardware I/O coherency or not, so this commit slightly
refactors the coherency_type() function to return
COHERENCY_FABRIC_TYPE_NONE when is_smp() is false, or the appropriate
type of the coherency fabric in the other case.
Thanks to this, the I/O coherency fabric will no longer be used at all
in !CONFIG_SMP configurations. It will continue to be used in
CONFIG_SMP configurations on Armada XP, Armada 375 and Armada 38x
(which are multiple cores processors), but will no longer be used on
Armada 370 (which is a single core processor).
In the process, it simplifies the implementation of the
coherency_type() function, and adds a missing call to of_node_put().
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Fixes: e60304f8cb7bb545e79fe62d9b9762460c254ec2 ("arm: mvebu: Add hardware I/O Coherency support")
Cc: <stable@vger.kernel.org> # v3.8+
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Link: https://lkml.kernel.org/r/1415871540-20302-3-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-11-13 09:38:57 +00:00
|
|
|
type = (int) match->data;
|
ARM: mvebu: add Armada 375 support to the coherency code
The Armada 375, like the Armada 370 and Armada XP, has a coherency
unit. However, unlike the coherency unit of 370/XP which does both CPU
and I/O coherency, the one on Armada 735 only does I/O
coherency. Therefore, instead of having two sets of registers (the
first one being used mainly to register each CPU in the coherency
fabric, the second one being used for the I/O coherency barrier), it
has only one set of register (for the I/O coherency barrier).
This commit adds a new "marvell,armada-375-coherency-fabric"
compatible string for this variant of the coherency fabric. The custom
DMA operations, and the way of triggering an I/O barrier is the same
as Armada 370/XP, so the code changes are minimal. However, the
set_cpu_coherent() function is not needed on Armada 375 and will not
work.
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Link: https://lkml.kernel.org/r/1397483228-25625-7-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-04-14 13:47:04 +00:00
|
|
|
|
ARM: mvebu: disable I/O coherency on non-SMP situations on Armada 370/375/38x/XP
Enabling the hardware I/O coherency on Armada 370, Armada 375, Armada
38x and Armada XP requires a certain number of conditions:
- On Armada 370, the cache policy must be set to write-allocate.
- On Armada 375, 38x and XP, the cache policy must be set to
write-allocate, the pages must be mapped with the shareable
attribute, and the SMP bit must be set
Currently, on Armada XP, when CONFIG_SMP is enabled, those conditions
are met. However, when Armada XP is used in a !CONFIG_SMP kernel, none
of these conditions are met. With Armada 370, the situation is worse:
since the processor is single core, regardless of whether CONFIG_SMP
or !CONFIG_SMP is used, the cache policy will be set to write-back by
the kernel and not write-allocate.
Since solving this problem turns out to be quite complicated, and we
don't want to let users with a mainline kernel known to have
infrequent but existing data corruptions, this commit proposes to
simply disable hardware I/O coherency in situations where it is known
not to work.
And basically, the is_smp() function of the kernel tells us whether it
is OK to enable hardware I/O coherency or not, so this commit slightly
refactors the coherency_type() function to return
COHERENCY_FABRIC_TYPE_NONE when is_smp() is false, or the appropriate
type of the coherency fabric in the other case.
Thanks to this, the I/O coherency fabric will no longer be used at all
in !CONFIG_SMP configurations. It will continue to be used in
CONFIG_SMP configurations on Armada XP, Armada 375 and Armada 38x
(which are multiple cores processors), but will no longer be used on
Armada 370 (which is a single core processor).
In the process, it simplifies the implementation of the
coherency_type() function, and adds a missing call to of_node_put().
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Fixes: e60304f8cb7bb545e79fe62d9b9762460c254ec2 ("arm: mvebu: Add hardware I/O Coherency support")
Cc: <stable@vger.kernel.org> # v3.8+
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Link: https://lkml.kernel.org/r/1415871540-20302-3-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-11-13 09:38:57 +00:00
|
|
|
of_node_put(np);
|
2012-08-02 08:16:29 +00:00
|
|
|
|
ARM: mvebu: disable I/O coherency on non-SMP situations on Armada 370/375/38x/XP
Enabling the hardware I/O coherency on Armada 370, Armada 375, Armada
38x and Armada XP requires a certain number of conditions:
- On Armada 370, the cache policy must be set to write-allocate.
- On Armada 375, 38x and XP, the cache policy must be set to
write-allocate, the pages must be mapped with the shareable
attribute, and the SMP bit must be set
Currently, on Armada XP, when CONFIG_SMP is enabled, those conditions
are met. However, when Armada XP is used in a !CONFIG_SMP kernel, none
of these conditions are met. With Armada 370, the situation is worse:
since the processor is single core, regardless of whether CONFIG_SMP
or !CONFIG_SMP is used, the cache policy will be set to write-back by
the kernel and not write-allocate.
Since solving this problem turns out to be quite complicated, and we
don't want to let users with a mainline kernel known to have
infrequent but existing data corruptions, this commit proposes to
simply disable hardware I/O coherency in situations where it is known
not to work.
And basically, the is_smp() function of the kernel tells us whether it
is OK to enable hardware I/O coherency or not, so this commit slightly
refactors the coherency_type() function to return
COHERENCY_FABRIC_TYPE_NONE when is_smp() is false, or the appropriate
type of the coherency fabric in the other case.
Thanks to this, the I/O coherency fabric will no longer be used at all
in !CONFIG_SMP configurations. It will continue to be used in
CONFIG_SMP configurations on Armada XP, Armada 375 and Armada 38x
(which are multiple cores processors), but will no longer be used on
Armada 370 (which is a single core processor).
In the process, it simplifies the implementation of the
coherency_type() function, and adds a missing call to of_node_put().
Signed-off-by: Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
Fixes: e60304f8cb7bb545e79fe62d9b9762460c254ec2 ("arm: mvebu: Add hardware I/O Coherency support")
Cc: <stable@vger.kernel.org> # v3.8+
Acked-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
Link: https://lkml.kernel.org/r/1415871540-20302-3-git-send-email-thomas.petazzoni@free-electrons.com
Signed-off-by: Jason Cooper <jason@lakedaemon.net>
2014-11-13 09:38:57 +00:00
|
|
|
return type;
|
2012-08-02 08:16:29 +00:00
|
|
|
}
|
2013-06-05 07:04:55 +00:00
|
|
|
|
2015-07-08 15:02:30 +00:00
|
|
|
int set_cpu_coherent(void)
|
|
|
|
{
|
|
|
|
int type = coherency_type();
|
|
|
|
|
|
|
|
if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP) {
|
|
|
|
if (!coherency_base) {
|
|
|
|
pr_warn("Can't make current CPU cache coherent.\n");
|
|
|
|
pr_warn("Coherency fabric is not initialized\n");
|
|
|
|
return 1;
|
|
|
|
}
|
2015-07-08 14:09:20 +00:00
|
|
|
|
|
|
|
armada_xp_clear_shared_l2();
|
2015-07-08 15:02:30 +00:00
|
|
|
ll_add_cpu_to_smp_group();
|
|
|
|
return ll_enable_coherency();
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-04-14 13:47:00 +00:00
|
|
|
int coherency_available(void)
|
2013-06-05 07:04:55 +00:00
|
|
|
{
|
2015-01-16 16:11:29 +00:00
|
|
|
return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
|
2014-04-14 13:47:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int __init coherency_init(void)
|
|
|
|
{
|
|
|
|
int type = coherency_type();
|
2013-08-27 04:41:14 +00:00
|
|
|
struct device_node *np;
|
|
|
|
|
|
|
|
np = of_find_matching_node(NULL, of_coherency_table);
|
2014-04-14 13:47:00 +00:00
|
|
|
|
|
|
|
if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
|
|
|
|
armada_370_coherency_init(np);
|
2014-04-14 13:47:06 +00:00
|
|
|
else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
|
|
|
|
type == COHERENCY_FABRIC_TYPE_ARMADA_380)
|
|
|
|
armada_375_380_coherency_init(np);
|
2014-04-14 13:47:00 +00:00
|
|
|
|
2014-10-27 15:32:35 +00:00
|
|
|
of_node_put(np);
|
|
|
|
|
2014-04-14 13:47:00 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init coherency_late_init(void)
|
|
|
|
{
|
2014-11-13 09:38:59 +00:00
|
|
|
if (coherency_available())
|
|
|
|
bus_register_notifier(&platform_bus_type,
|
|
|
|
&mvebu_hwcc_nb);
|
2013-06-05 07:04:55 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
postcore_initcall(coherency_late_init);
|
2014-05-13 16:04:30 +00:00
|
|
|
|
2014-05-20 15:13:03 +00:00
|
|
|
#if IS_ENABLED(CONFIG_PCI)
|
2014-05-13 16:04:30 +00:00
|
|
|
static int __init coherency_pci_init(void)
|
|
|
|
{
|
|
|
|
if (coherency_available())
|
|
|
|
bus_register_notifier(&pci_bus_type,
|
2014-07-08 13:37:37 +00:00
|
|
|
&mvebu_hwcc_pci_nb);
|
2014-05-13 16:04:30 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
arch_initcall(coherency_pci_init);
|
2014-05-20 15:13:03 +00:00
|
|
|
#endif
|