Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-26 22:21:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branches 'arm/exynos', 'arm/renesas', 'arm/rockchip', 'arm/omap', 'arm/mediatek', 'arm/tegra', 'arm/qcom', 'arm/smmu', 'ppc/pamu', 'x86/vt-d', 'x86/amd', 's390' and 'core' into next

Browse Source
This commit is contained in:
Joerg Roedel 2017-09-01 11:31:42 +02:00
commit 47b59d8e40
41 changed files with 2883 additions and 1235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
Documentation/devicetree/bindings/iommu/qcom,iommu.txt Normal file
View File

@ -0,0 +1,121 @@
* QCOM IOMMU v1 Implementation
Qualcomm "B" family devices which are not compatible with arm-smmu have
a similar looking IOMMU but without access to the global register space,
and optionally requiring additional configuration to route context irqs
to non-secure vs secure interrupt line.
** Required properties:
- compatible : Should be one of:
"qcom,msm8916-iommu"
Followed by "qcom,msm-iommu-v1".
- clock-names : Should be a pair of "iface" (required for IOMMUs
register group access) and "bus" (required for
the IOMMUs underlying bus access).
- clocks : Phandles for respective clocks described by
clock-names.
- #address-cells : must be 1.
- #size-cells : must be 1.
- #iommu-cells : Must be 1. Index identifies the context-bank #.
- ranges : Base address and size of the iommu context banks.
- qcom,iommu-secure-id : secure-id.
- List of sub-nodes, one per translation context bank. Each sub-node
has the following required properties:
- compatible : Should be one of:
- "qcom,msm-iommu-v1-ns" : non-secure context bank
- "qcom,msm-iommu-v1-sec" : secure context bank
- reg : Base address and size of context bank within the iommu
- interrupts : The context fault irq.
** Optional properties:
- reg : Base address and size of the SMMU local base, should
be only specified if the iommu requires configuration
for routing of context bank irq's to secure vs non-
secure lines. (Ie. if the iommu contains secure
context banks)
** Examples:
apps_iommu: iommu@1e20000 {
#address-cells = <1>;
#size-cells = <1>;
#iommu-cells = <1>;
compatible = "qcom,msm8916-iommu", "qcom,msm-iommu-v1";
ranges = <0 0x1e20000 0x40000>;
reg = <0x1ef0000 0x3000>;
clocks = <&gcc GCC_SMMU_CFG_CLK>,
<&gcc GCC_APSS_TCU_CLK>;
clock-names = "iface", "bus";
qcom,iommu-secure-id = <17>;
// mdp_0:
iommu-ctx@4000 {
compatible = "qcom,msm-iommu-v1-ns";
reg = <0x4000 0x1000>;
interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
};
// venus_ns:
iommu-ctx@5000 {
compatible = "qcom,msm-iommu-v1-sec";
reg = <0x5000 0x1000>;
interrupts = <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
};
};
gpu_iommu: iommu@1f08000 {
#address-cells = <1>;
#size-cells = <1>;
#iommu-cells = <1>;
compatible = "qcom,msm8916-iommu", "qcom,msm-iommu-v1";
ranges = <0 0x1f08000 0x10000>;
clocks = <&gcc GCC_SMMU_CFG_CLK>,
<&gcc GCC_GFX_TCU_CLK>;
clock-names = "iface", "bus";
qcom,iommu-secure-id = <18>;
// gfx3d_user:
iommu-ctx@1000 {
compatible = "qcom,msm-iommu-v1-ns";
reg = <0x1000 0x1000>;
interrupts = <GIC_SPI 241 IRQ_TYPE_LEVEL_HIGH>;
};
// gfx3d_priv:
iommu-ctx@2000 {
compatible = "qcom,msm-iommu-v1-ns";
reg = <0x2000 0x1000>;
interrupts = <GIC_SPI 242 IRQ_TYPE_LEVEL_HIGH>;
};
};
...
venus: video-codec@1d00000 {
...
iommus = <&apps_iommu 5>;
};
mdp: mdp@1a01000 {
...
iommus = <&apps_iommu 4>;
};
gpu@01c00000 {
...
iommus = <&gpu_iommu 1>, <&gpu_iommu 2>;
};

5
Documentation/devicetree/bindings/iommu/rockchip,iommu.txt
View File

@ -15,6 +15,11 @@ Required properties:
to associate with its master device. See:
Documentation/devicetree/bindings/iommu/iommu.txt
Optional properties:
- rockchip,disable-mmu-reset : Don't use the mmu reset operation.
Some mmu instances may produce unexpected results
when the reset operation is used.
Example:
vopl_mmu: iommu@ff940300 {

15
Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt
View File

@ -15,6 +15,9 @@ Required properties:
the register.
- "smi" : It's the clock for transfer data and command.
Required property for mt2701:
- mediatek,larb-id :the hardware id of this larb.
Example:
larb1: larb@16010000 {
compatible = "mediatek,mt8173-smi-larb";
@ -25,3 +28,15 @@ Example:
<&vdecsys CLK_VDEC_LARB_CKEN>;
clock-names = "apb", "smi";
};
Example for mt2701:
larb0: larb@14010000 {
compatible = "mediatek,mt2701-smi-larb";
reg = <0 0x14010000 0 0x1000>;
mediatek,smi = <&smi_common>;
mediatek,larb-id = <0>;
clocks = <&mmsys CLK_MM_SMI_LARB0>,
<&mmsys CLK_MM_SMI_LARB0>;
clock-names = "apb", "smi";
power-domains = <&scpsys MT2701_POWER_DOMAIN_DISP>;
};

7
MAINTAINERS
View File

@ -10940,6 +10940,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rkuo/linux-hexagon-kernel.g
S: Supported
F: arch/hexagon/
QUALCOMM IOMMU
M: Rob Clark <robdclark@gmail.com>
L: iommu@lists.linux-foundation.org
L: linux-arm-msm@vger.kernel.org
S: Maintained
F: drivers/iommu/qcom_iommu.c
QUALCOMM VENUS VIDEO ACCELERATOR DRIVER
M: Stanimir Varbanov <stanimir.varbanov@linaro.org>
L: linux-media@vger.kernel.org

7
arch/s390/include/asm/pci.h
View File

@ -8,6 +8,7 @@
#include <linux/pci.h>
#include <linux/mutex.h>
#include <linux/iommu.h>
#include <asm-generic/pci.h>
#include <asm/pci_clp.h>
#include <asm/pci_debug.h>
@ -122,6 +123,8 @@ struct zpci_dev {
unsigned long iommu_pages;
unsigned int next_bit;
struct iommu_device iommu_dev; /* IOMMU core handle */
char res_name[16];
struct zpci_bar_struct bars[PCI_BAR_COUNT];
@ -174,6 +177,10 @@ int clp_enable_fh(struct zpci_dev *, u8);
int clp_disable_fh(struct zpci_dev *);
int clp_get_state(u32 fid, enum zpci_state *state);
/* IOMMU Interface */
int zpci_init_iommu(struct zpci_dev *zdev);
void zpci_destroy_iommu(struct zpci_dev *zdev);
#ifdef CONFIG_PCI
/* Error handling and recovery */
void zpci_event_error(void *);

9
arch/s390/pci/pci.c
View File

@ -776,6 +776,7 @@ void pcibios_remove_bus(struct pci_bus *bus)
zpci_exit_slot(zdev);
zpci_cleanup_bus_resources(zdev);
zpci_destroy_iommu(zdev);
zpci_free_domain(zdev);
spin_lock(&zpci_list_lock);
@ -848,11 +849,15 @@ int zpci_create_device(struct zpci_dev *zdev)
if (rc)
goto out;
rc = zpci_init_iommu(zdev);
if (rc)
goto out_free;
mutex_init(&zdev->lock);
if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
rc = zpci_enable_device(zdev);
if (rc)
goto out_free;
goto out_destroy_iommu;
}
rc = zpci_scan_bus(zdev);
if (rc)
@ -869,6 +874,8 @@ int zpci_create_device(struct zpci_dev *zdev)
out_disable:
if (zdev->state == ZPCI_FN_STATE_ONLINE)
zpci_disable_device(zdev);
out_destroy_iommu:
zpci_destroy_iommu(zdev);
out_free:
zpci_free_domain(zdev);
out:

13
drivers/iommu/Kconfig
View File

@ -76,6 +76,8 @@ config IOMMU_DMA
config FSL_PAMU
bool "Freescale IOMMU support"
depends on PCI
depends on PHYS_64BIT
depends on PPC_E500MC || (COMPILE_TEST && PPC)
select IOMMU_API
select GENERIC_ALLOCATOR
@ -253,6 +255,7 @@ config TEGRA_IOMMU_SMMU
config EXYNOS_IOMMU
bool "Exynos IOMMU Support"
depends on ARCH_EXYNOS && MMU
depends on !CPU_BIG_ENDIAN # revisit driver if we can enable big-endian ptes
select IOMMU_API
select ARM_DMA_USE_IOMMU
help
@ -367,4 +370,14 @@ config MTK_IOMMU_V1
if unsure, say N here.
config QCOM_IOMMU
# Note: iommu drivers cannot (yet?) be built as modules
bool "Qualcomm IOMMU Support"
depends on ARCH_QCOM || COMPILE_TEST
select IOMMU_API
select IOMMU_IO_PGTABLE_LPAE
select ARM_DMA_USE_IOMMU
help
Support for IOMMU on certain Qualcomm SoCs.
endif # IOMMU_SUPPORT

1
drivers/iommu/Makefile
View File

@ -27,3 +27,4 @@ obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o

313
drivers/iommu/amd_iommu.c
View File

@ -102,29 +102,6 @@ int amd_iommu_max_glx_val = -1;
static const struct dma_map_ops amd_iommu_dma_ops;
/*
* This struct contains device specific data for the IOMMU
*/
struct iommu_dev_data {
struct list_head list; /* For domain->dev_list */
struct list_head dev_data_list; /* For global dev_data_list */
struct protection_domain *domain; /* Domain the device is bound to */
u16 devid; /* PCI Device ID */
u16 alias; /* Alias Device ID */
bool iommu_v2; /* Device can make use of IOMMUv2 */
bool passthrough; /* Device is identity mapped */
struct {
bool enabled;
int qdep;
} ats; /* ATS state */
bool pri_tlp; /* PASID TLB required for
PPR completions */
u32 errata; /* Bitmap for errata to apply */
bool use_vapic; /* Enable device to use vapic mode */
struct ratelimit_state rs; /* Ratelimit IOPF messages */
};
/*
* general struct to manage commands send to an IOMMU
*/
@ -137,20 +114,7 @@ struct kmem_cache *amd_iommu_irq_cache;
static void update_domain(struct protection_domain *domain);
static int protection_domain_init(struct protection_domain *domain);
static void detach_device(struct device *dev);
#define FLUSH_QUEUE_SIZE 256
struct flush_queue_entry {
unsigned long iova_pfn;
unsigned long pages;
u64 counter; /* Flush counter when this entry was added to the queue */
};
struct flush_queue {
struct flush_queue_entry *entries;
unsigned head, tail;
spinlock_t lock;
};
static void iova_domain_flush_tlb(struct iova_domain *iovad);
/*
* Data container for a dma_ops specific protection domain
@ -161,36 +125,6 @@ struct dma_ops_domain {
/* IOVA RB-Tree */
struct iova_domain iovad;
struct flush_queue __percpu *flush_queue;
/*
* We need two counter here to be race-free wrt. IOTLB flushing and
* adding entries to the flush queue.
*
* The flush_start_cnt is incremented _before_ the IOTLB flush starts.
* New entries added to the flush ring-buffer get their 'counter' value
* from here. This way we can make sure that entries added to the queue
* (or other per-cpu queues of the same domain) while the TLB is about
* to be flushed are not considered to be flushed already.
*/
atomic64_t flush_start_cnt;
/*
* The flush_finish_cnt is incremented when an IOTLB flush is complete.
* This value is always smaller than flush_start_cnt. The queue_add
* function frees all IOVAs that have a counter value smaller than
* flush_finish_cnt. This makes sure that we only free IOVAs that are
* flushed out of the IOTLB of the domain.
*/
atomic64_t flush_finish_cnt;
/*
* Timer to make sure we don't keep IOVAs around unflushed
* for too long
*/
struct timer_list flush_timer;
atomic_t flush_timer_on;
};
static struct iova_domain reserved_iova_ranges;
@ -371,19 +305,25 @@ static u16 get_alias(struct device *dev)
static struct iommu_dev_data *find_dev_data(u16 devid)
{
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
dev_data = search_dev_data(devid);
if (dev_data == NULL)
if (dev_data == NULL) {
dev_data = alloc_dev_data(devid);
if (translation_pre_enabled(iommu))
dev_data->defer_attach = true;
}
return dev_data;
}
static struct iommu_dev_data *get_dev_data(struct device *dev)
struct iommu_dev_data *get_dev_data(struct device *dev)
{
return dev->archdata.iommu;
}
EXPORT_SYMBOL(get_dev_data);
/*
* Find or create an IOMMU group for a acpihid device.
@ -1165,7 +1105,7 @@ static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
return iommu_queue_command(iommu, &cmd);
}
static void iommu_flush_dte_all(struct amd_iommu *iommu)
static void amd_iommu_flush_dte_all(struct amd_iommu *iommu)
{
u32 devid;
@ -1179,7 +1119,7 @@ static void iommu_flush_dte_all(struct amd_iommu *iommu)
* This function uses heavy locking and may disable irqs for some time. But
* this is no issue because it is only called during resume.
*/
static void iommu_flush_tlb_all(struct amd_iommu *iommu)
static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu)
{
u32 dom_id;
@ -1193,7 +1133,7 @@ static void iommu_flush_tlb_all(struct amd_iommu *iommu)
iommu_completion_wait(iommu);
}
static void iommu_flush_all(struct amd_iommu *iommu)
static void amd_iommu_flush_all(struct amd_iommu *iommu)
{
struct iommu_cmd cmd;
@ -1212,7 +1152,7 @@ static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
iommu_queue_command(iommu, &cmd);
}
static void iommu_flush_irt_all(struct amd_iommu *iommu)
static void amd_iommu_flush_irt_all(struct amd_iommu *iommu)
{
u32 devid;
@ -1225,11 +1165,11 @@ static void iommu_flush_irt_all(struct amd_iommu *iommu)
void iommu_flush_all_caches(struct amd_iommu *iommu)
{
if (iommu_feature(iommu, FEATURE_IA)) {
iommu_flush_all(iommu);
amd_iommu_flush_all(iommu);
} else {
iommu_flush_dte_all(iommu);
iommu_flush_irt_all(iommu);
iommu_flush_tlb_all(iommu);
amd_iommu_flush_dte_all(iommu);
amd_iommu_flush_irt_all(iommu);
amd_iommu_flush_tlb_all(iommu);
}
}
@ -1537,9 +1477,9 @@ static int iommu_map_page(struct protection_domain *dom,
if (count > 1) {
__pte = PAGE_SIZE_PTE(phys_addr, page_size);
__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC;
} else
__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
__pte = phys_addr | IOMMU_PTE_PR | IOMMU_PTE_FC;
if (prot & IOMMU_PROT_IR)
__pte |= IOMMU_PTE_IR;
@ -1788,178 +1728,19 @@ static void free_gcr3_table(struct protection_domain *domain)
free_page((unsigned long)domain->gcr3_tbl);
}
static void dma_ops_domain_free_flush_queue(struct dma_ops_domain *dom)
{
int cpu;
for_each_possible_cpu(cpu) {
struct flush_queue *queue;
queue = per_cpu_ptr(dom->flush_queue, cpu);
kfree(queue->entries);
}
free_percpu(dom->flush_queue);
dom->flush_queue = NULL;
}
static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
{
int cpu;
atomic64_set(&dom->flush_start_cnt, 0);
atomic64_set(&dom->flush_finish_cnt, 0);
dom->flush_queue = alloc_percpu(struct flush_queue);
if (!dom->flush_queue)
return -ENOMEM;
/* First make sure everything is cleared */
for_each_possible_cpu(cpu) {
struct flush_queue *queue;
queue = per_cpu_ptr(dom->flush_queue, cpu);
queue->head = 0;
queue->tail = 0;
queue->entries = NULL;
}
/* Now start doing the allocation */
for_each_possible_cpu(cpu) {
struct flush_queue *queue;
queue = per_cpu_ptr(dom->flush_queue, cpu);
queue->entries = kzalloc(FLUSH_QUEUE_SIZE * sizeof(*queue->entries),
GFP_KERNEL);
if (!queue->entries) {
dma_ops_domain_free_flush_queue(dom);
return -ENOMEM;
}
spin_lock_init(&queue->lock);
}
return 0;
}
static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
{
atomic64_inc(&dom->flush_start_cnt);
domain_flush_tlb(&dom->domain);
domain_flush_complete(&dom->domain);
atomic64_inc(&dom->flush_finish_cnt);
}
static inline bool queue_ring_full(struct flush_queue *queue)
static void iova_domain_flush_tlb(struct iova_domain *iovad)
{
assert_spin_locked(&queue->lock);
struct dma_ops_domain *dom;
return (((queue->tail + 1) % FLUSH_QUEUE_SIZE) == queue->head);
}
#define queue_ring_for_each(i, q) \
for (i = (q)->head; i != (q)->tail; i = (i + 1) % FLUSH_QUEUE_SIZE)
static inline unsigned queue_ring_add(struct flush_queue *queue)
{
unsigned idx = queue->tail;
assert_spin_locked(&queue->lock);
queue->tail = (idx + 1) % FLUSH_QUEUE_SIZE;
return idx;
}
static inline void queue_ring_remove_head(struct flush_queue *queue)
{
assert_spin_locked(&queue->lock);
queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
}
static void queue_ring_free_flushed(struct dma_ops_domain *dom,
struct flush_queue *queue)
{
u64 counter = atomic64_read(&dom->flush_finish_cnt);
int idx;
queue_ring_for_each(idx, queue) {
/*
* This assumes that counter values in the ring-buffer are
* monotonously rising.
*/
if (queue->entries[idx].counter >= counter)
break;
free_iova_fast(&dom->iovad,
queue->entries[idx].iova_pfn,
queue->entries[idx].pages);
queue_ring_remove_head(queue);
}
}
static void queue_add(struct dma_ops_domain *dom,
unsigned long address, unsigned long pages)
{
struct flush_queue *queue;
unsigned long flags;
int idx;
pages = __roundup_pow_of_two(pages);
address >>= PAGE_SHIFT;
queue = get_cpu_ptr(dom->flush_queue);
spin_lock_irqsave(&queue->lock, flags);
/*
* First remove the enries from the ring-buffer that are already
* flushed to make the below queue_ring_full() check less likely
*/
queue_ring_free_flushed(dom, queue);
/*
* When ring-queue is full, flush the entries from the IOTLB so
* that we can free all entries with queue_ring_free_flushed()
* below.
*/
if (queue_ring_full(queue)) {
dma_ops_domain_flush_tlb(dom);
queue_ring_free_flushed(dom, queue);
}
idx = queue_ring_add(queue);
queue->entries[idx].iova_pfn = address;
queue->entries[idx].pages = pages;
queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
spin_unlock_irqrestore(&queue->lock, flags);
if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
put_cpu_ptr(dom->flush_queue);
}
static void queue_flush_timeout(unsigned long data)
{
struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
int cpu;
atomic_set(&dom->flush_timer_on, 0);
dom = container_of(iovad, struct dma_ops_domain, iovad);
dma_ops_domain_flush_tlb(dom);
for_each_possible_cpu(cpu) {
struct flush_queue *queue;
unsigned long flags;
queue = per_cpu_ptr(dom->flush_queue, cpu);
spin_lock_irqsave(&queue->lock, flags);
queue_ring_free_flushed(dom, queue);
spin_unlock_irqrestore(&queue->lock, flags);
}
}
/*
@ -1973,11 +1754,6 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
del_domain_from_list(&dom->domain);
if (timer_pending(&dom->flush_timer))
del_timer(&dom->flush_timer);
dma_ops_domain_free_flush_queue(dom);
put_iova_domain(&dom->iovad);
free_pagetable(&dom->domain);
@ -2013,16 +1789,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
init_iova_domain(&dma_dom->iovad, PAGE_SIZE,
IOVA_START_PFN, DMA_32BIT_PFN);
/* Initialize reserved ranges */
copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
if (dma_ops_domain_alloc_flush_queue(dma_dom))
if (init_iova_flush_queue(&dma_dom->iovad, iova_domain_flush_tlb, NULL))
goto free_dma_dom;
setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
(unsigned long)dma_dom);
atomic_set(&dma_dom->flush_timer_on, 0);
/* Initialize reserved ranges */
copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
add_domain_to_list(&dma_dom->domain);
@ -2053,7 +1824,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;
pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV;
flags = amd_iommu_dev_table[devid].data[1];
@ -2086,8 +1857,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
flags |= tmp;
}
flags &= ~(DTE_FLAG_SA | 0xffffULL);
flags &= ~DEV_DOMID_MASK;
flags |= domain->id;
amd_iommu_dev_table[devid].data[1] = flags;
@ -2097,7 +1867,7 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
static void clear_dte_entry(u16 devid)
{
/* remove entry from the device table seen by the hardware */
amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
amd_iommu_dev_table[devid].data[0] = DTE_FLAG_V | DTE_FLAG_TV;
amd_iommu_dev_table[devid].data[1] &= DTE_FLAG_MASK;
amd_iommu_apply_erratum_63(devid);
@ -2478,11 +2248,21 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev)
static struct protection_domain *get_domain(struct device *dev)
{
struct protection_domain *domain;
struct iommu_domain *io_domain;
if (!check_device(dev))
return ERR_PTR(-EINVAL);
domain = get_dev_data(dev)->domain;
if (domain == NULL && get_dev_data(dev)->defer_attach) {
get_dev_data(dev)->defer_attach = false;
io_domain = iommu_get_domain_for_dev(dev);
domain = to_pdomain(io_domain);
attach_device(dev, domain);
}
if (domain == NULL)
return ERR_PTR(-EBUSY);
if (!dma_ops_domain(domain))
return ERR_PTR(-EBUSY);
@ -2528,6 +2308,7 @@ static int dir2prot(enum dma_data_direction direction)
else
return 0;
}
/*
* This function contains common code for mapping of a physically
* contiguous memory region into DMA address space. It is used by all
@ -2619,7 +2400,8 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
domain_flush_tlb(&dma_dom->domain);
domain_flush_complete(&dma_dom->domain);
} else {
queue_add(dma_dom, dma_addr, pages);
pages = __roundup_pow_of_two(pages);
queue_iova(&dma_dom->iovad, dma_addr >> PAGE_SHIFT, pages, 0);
}
}
@ -3373,6 +3155,13 @@ static void amd_iommu_apply_resv_region(struct device *dev,
WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL);
}
static bool amd_iommu_is_attach_deferred(struct iommu_domain *domain,
struct device *dev)
{
struct iommu_dev_data *dev_data = dev->archdata.iommu;
return dev_data->defer_attach;
}
const struct iommu_ops amd_iommu_ops = {
.capable = amd_iommu_capable,
.domain_alloc = amd_iommu_domain_alloc,
@ -3389,6 +3178,7 @@ const struct iommu_ops amd_iommu_ops = {
.get_resv_regions = amd_iommu_get_resv_regions,
.put_resv_regions = amd_iommu_put_resv_regions,
.apply_resv_region = amd_iommu_apply_resv_region,
.is_attach_deferred = amd_iommu_is_attach_deferred,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
};
@ -3777,11 +3567,6 @@ EXPORT_SYMBOL(amd_iommu_device_info);
static struct irq_chip amd_ir_chip;
#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
#define DTE_IRQ_TABLE_LEN (8ULL << 1)
#define DTE_IRQ_REMAP_ENABLE 1ULL
static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
{
u64 dte;

223
drivers/iommu/amd_iommu_init.c
View File

@ -29,7 +29,6 @@
#include <linux/export.h>
#include <linux/iommu.h>
#include <linux/kmemleak.h>
#include <linux/crash_dump.h>
#include <asm/pci-direct.h>
#include <asm/iommu.h>
#include <asm/gart.h>
@ -38,6 +37,7 @@
#include <asm/io_apic.h>
#include <asm/irq_remapping.h>
#include <linux/crash_dump.h>
#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
#include "irq_remapping.h"
@ -196,6 +196,11 @@ spinlock_t amd_iommu_pd_lock;
* page table root pointer.
*/
struct dev_table_entry *amd_iommu_dev_table;
/*
* Pointer to a device table which the content of old device table
* will be copied to. It's only be used in kdump kernel.
*/
static struct dev_table_entry *old_dev_tbl_cpy;
/*
* The alias table is a driver specific data structure which contains the
@ -209,6 +214,7 @@ u16 *amd_iommu_alias_table;
* for a specific device. It is also indexed by the PCI device id.
*/
struct amd_iommu **amd_iommu_rlookup_table;
EXPORT_SYMBOL(amd_iommu_rlookup_table);
/*
* This table is used to find the irq remapping table for a given device id
@ -258,6 +264,28 @@ static int amd_iommu_enable_interrupts(void);
static int __init iommu_go_to_state(enum iommu_init_state state);
static void init_device_table_dma(void);
static bool amd_iommu_pre_enabled = true;
bool translation_pre_enabled(struct amd_iommu *iommu)
{
return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
}
EXPORT_SYMBOL(translation_pre_enabled);
static void clear_translation_pre_enabled(struct amd_iommu *iommu)
{
iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
}
static void init_translation_status(struct amd_iommu *iommu)
{
u32 ctrl;
ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
if (ctrl & (1<<CONTROL_IOMMU_EN))
iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
}
static inline void update_last_devid(u16 devid)
{
if (devid > amd_iommu_last_bdf)
@ -615,6 +643,14 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)
amd_iommu_reset_cmd_buffer(iommu);
}
/*
* This function disables the command buffer
*/
static void iommu_disable_command_buffer(struct amd_iommu *iommu)
{
iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
}
static void __init free_command_buffer(struct amd_iommu *iommu)
{
free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
@ -647,6 +683,14 @@ static void iommu_enable_event_buffer(struct amd_iommu *iommu)
iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
}
/*
* This function disables the event log buffer
*/
static void iommu_disable_event_buffer(struct amd_iommu *iommu)
{
iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
}
static void __init free_event_buffer(struct amd_iommu *iommu)
{
free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
@ -808,6 +852,96 @@ static int get_dev_entry_bit(u16 devid, u8 bit)
}
static bool copy_device_table(void)
{
u64 int_ctl, int_tab_len, entry = 0, last_entry = 0;
struct dev_table_entry *old_devtb = NULL;
u32 lo, hi, devid, old_devtb_size;
phys_addr_t old_devtb_phys;
struct amd_iommu *iommu;
u16 dom_id, dte_v, irq_v;
gfp_t gfp_flag;
u64 tmp;
if (!amd_iommu_pre_enabled)
return false;
pr_warn("Translation is already enabled - trying to copy translation structures\n");
for_each_iommu(iommu) {
/* All IOMMUs should use the same device table with the same size */
lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
entry = (((u64) hi) << 32) + lo;
if (last_entry && last_entry != entry) {
pr_err("IOMMU:%d should use the same dev table as others!/n",
iommu->index);
return false;
}
last_entry = entry;
old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
if (old_devtb_size != dev_table_size) {
pr_err("The device table size of IOMMU:%d is not expected!/n",
iommu->index);
return false;
}
}
old_devtb_phys = entry & PAGE_MASK;
if (old_devtb_phys >= 0x100000000ULL) {
pr_err("The address of old device table is above 4G, not trustworthy!/n");
return false;
}
old_devtb = memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB);
if (!old_devtb)
return false;
gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32;
old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,
get_order(dev_table_size));
if (old_dev_tbl_cpy == NULL) {
pr_err("Failed to allocate memory for copying old device table!/n");
return false;
}
for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
old_dev_tbl_cpy[devid] = old_devtb[devid];
dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
if (dte_v && dom_id) {
old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
__set_bit(dom_id, amd_iommu_pd_alloc_bitmap);
/* If gcr3 table existed, mask it out */
if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
old_dev_tbl_cpy[devid].data[1] &= ~tmp;
tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
tmp |= DTE_FLAG_GV;
old_dev_tbl_cpy[devid].data[0] &= ~tmp;
}
}
irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
int_tab_len = old_devtb[devid].data[2] & DTE_IRQ_TABLE_LEN_MASK;
if (irq_v && (int_ctl || int_tab_len)) {
if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
(int_tab_len != DTE_IRQ_TABLE_LEN)) {
pr_err("Wrong old irq remapping flag: %#x\n", devid);
return false;
}
old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
}
}
memunmap(old_devtb);
return true;
}
void amd_iommu_apply_erratum_63(u16 devid)
{
int sysmgt;
@ -1399,6 +1533,16 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
iommu->int_enabled = false;
init_translation_status(iommu);
if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
iommu_disable(iommu);
clear_translation_pre_enabled(iommu);
pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
iommu->index);
}
if (amd_iommu_pre_enabled)
amd_iommu_pre_enabled = translation_pre_enabled(iommu);
ret = init_iommu_from_acpi(iommu, h);
if (ret)
return ret;
@ -1892,8 +2036,7 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
}
/*
* Init the device table to not allow DMA access for devices and
* suppress all page faults
* Init the device table to not allow DMA access for devices
*/
static void init_device_table_dma(void)
{
@ -1902,14 +2045,6 @@ static void init_device_table_dma(void)
for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
set_dev_entry_bit(devid, DEV_ENTRY_VALID);
set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
/*
* In kdump kernels in-flight DMA from the old kernel might
* cause IO_PAGE_FAULTs. There are no reports that a kdump
* actually failed because of that, so just disable fault
* reporting in the hardware to get rid of the messages
*/
if (is_kdump_kernel())
set_dev_entry_bit(devid, DEV_ENTRY_NO_PAGE_FAULT);
}
}
@ -2022,24 +2157,62 @@ static void iommu_enable_ga(struct amd_iommu *iommu)
#endif
}
static void early_enable_iommu(struct amd_iommu *iommu)
{
iommu_disable(iommu);
iommu_init_flags(iommu);
iommu_set_device_table(iommu);
iommu_enable_command_buffer(iommu);
iommu_enable_event_buffer(iommu);
iommu_set_exclusion_range(iommu);
iommu_enable_ga(iommu);
iommu_enable(iommu);
iommu_flush_all_caches(iommu);
}
/*
* This function finally enables all IOMMUs found in the system after
* they have been initialized
* they have been initialized.
*
* Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
* the old content of device table entries. Not this case or copy failed,
* just continue as normal kernel does.
*/
static void early_enable_iommus(void)
{
struct amd_iommu *iommu;
for_each_iommu(iommu) {
iommu_disable(iommu);
iommu_init_flags(iommu);
iommu_set_device_table(iommu);
iommu_enable_command_buffer(iommu);
iommu_enable_event_buffer(iommu);
iommu_set_exclusion_range(iommu);
iommu_enable_ga(iommu);
iommu_enable(iommu);
iommu_flush_all_caches(iommu);
if (!copy_device_table()) {
/*
* If come here because of failure in copying device table from old
* kernel with all IOMMUs enabled, print error message and try to
* free allocated old_dev_tbl_cpy.
*/
if (amd_iommu_pre_enabled)
pr_err("Failed to copy DEV table from previous kernel.\n");
if (old_dev_tbl_cpy != NULL)
free_pages((unsigned long)old_dev_tbl_cpy,
get_order(dev_table_size));
for_each_iommu(iommu) {
clear_translation_pre_enabled(iommu);
early_enable_iommu(iommu);
}
} else {
pr_info("Copied DEV table from previous kernel.\n");
free_pages((unsigned long)amd_iommu_dev_table,
get_order(dev_table_size));
amd_iommu_dev_table = old_dev_tbl_cpy;
for_each_iommu(iommu) {
iommu_disable_command_buffer(iommu);
iommu_disable_event_buffer(iommu);
iommu_enable_command_buffer(iommu);
iommu_enable_event_buffer(iommu);
iommu_enable_ga(iommu);
iommu_set_device_table(iommu);
iommu_flush_all_caches(iommu);
}
}
#ifdef CONFIG_IRQ_REMAP
@ -2275,7 +2448,8 @@ static int __init early_amd_iommu_init(void)
/* Device table - directly used by all IOMMUs */
ret = -ENOMEM;
amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
amd_iommu_dev_table = (void *)__get_free_pages(
GFP_KERNEL | __GFP_ZERO | GFP_DMA32,
get_order(dev_table_size));
if (amd_iommu_dev_table == NULL)
goto out;
@ -2325,7 +2499,8 @@ static int __init early_amd_iommu_init(void)
goto out;
/* Disable any previously enabled IOMMUs */
disable_iommus();
if (!is_kdump_kernel() || amd_iommu_disabled)
disable_iommus();
if (amd_iommu_irq_remap)
amd_iommu_irq_remap = check_ioapic_information();

2
drivers/iommu/amd_iommu_proto.h
View File

@ -87,4 +87,6 @@ static inline bool iommu_feature(struct amd_iommu *iommu, u64 f)
return !!(iommu->features & f);
}
extern bool translation_pre_enabled(struct amd_iommu *iommu);
extern struct iommu_dev_data *get_dev_data(struct device *dev);
#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */

55
drivers/iommu/amd_iommu_types.h
View File

@ -250,6 +250,14 @@
#define GA_GUEST_NR 0x1
/* Bit value definition for dte irq remapping fields*/
#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
#define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
#define DTE_IRQ_TABLE_LEN_MASK (0xfULL << 1)
#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
#define DTE_IRQ_TABLE_LEN (8ULL << 1)
#define DTE_IRQ_REMAP_ENABLE 1ULL
#define PAGE_MODE_NONE 0x00
#define PAGE_MODE_1_LEVEL 0x01
#define PAGE_MODE_2_LEVEL 0x02
@ -265,7 +273,7 @@
#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL)
#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL)
#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \
IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
IOMMU_PTE_PR | IOMMU_PTE_IR | IOMMU_PTE_IW)
#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL)
#define PM_MAP_4k 0
@ -314,19 +322,29 @@
#define PTE_LEVEL_PAGE_SIZE(level) \
(1ULL << (12 + (9 * (level))))
#define IOMMU_PTE_P (1ULL << 0)
#define IOMMU_PTE_TV (1ULL << 1)
/*
* Bit value definition for I/O PTE fields
*/
#define IOMMU_PTE_PR (1ULL << 0)
#define IOMMU_PTE_U (1ULL << 59)
#define IOMMU_PTE_FC (1ULL << 60)
#define IOMMU_PTE_IR (1ULL << 61)
#define IOMMU_PTE_IW (1ULL << 62)
/*
* Bit value definition for DTE fields
*/
#define DTE_FLAG_V (1ULL << 0)
#define DTE_FLAG_TV (1ULL << 1)
#define DTE_FLAG_IR (1ULL << 61)
#define DTE_FLAG_IW (1ULL << 62)
#define DTE_FLAG_IOTLB (1ULL << 32)
#define DTE_FLAG_SA (1ULL << 34)
#define DTE_FLAG_GV (1ULL << 55)
#define DTE_FLAG_MASK (0x3ffULL << 32)
#define DTE_GLX_SHIFT (56)
#define DTE_GLX_MASK (3)
#define DEV_DOMID_MASK 0xffffULL
#define DTE_GCR3_VAL_A(x) (((x) >> 12) & 0x00007ULL)
#define DTE_GCR3_VAL_B(x) (((x) >> 15) & 0x0ffffULL)
@ -343,7 +361,7 @@
#define GCR3_VALID 0x01ULL
#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_PR)
#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
#define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07)
@ -435,6 +453,8 @@ struct iommu_domain;
struct irq_domain;
struct amd_irte_ops;
#define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
/*
* This structure contains generic data for IOMMU protection domains
* independent of their use.
@ -569,6 +589,7 @@ struct amd_iommu {
struct amd_irte_ops *irte_ops;
#endif
u32 flags;
volatile u64 __aligned(8) cmd_sem;
};
@ -599,6 +620,30 @@ struct devid_map {
bool cmd_line;
};
/*
* This struct contains device specific data for the IOMMU
*/
struct iommu_dev_data {
struct list_head list; /* For domain->dev_list */
struct list_head dev_data_list; /* For global dev_data_list */
struct protection_domain *domain; /* Domain the device is bound to */
u16 devid; /* PCI Device ID */
u16 alias; /* Alias Device ID */
bool iommu_v2; /* Device can make use of IOMMUv2 */
bool passthrough; /* Device is identity mapped */
struct {
bool enabled;
int qdep;
} ats; /* ATS state */
bool pri_tlp; /* PASID TLB required for
PPR completions */
u32 errata; /* Bitmap for errata to apply */
bool use_vapic; /* Enable device to use vapic mode */
bool defer_attach;
struct ratelimit_state rs; /* Ratelimit IOPF messages */
};
/* Map HPET and IOAPIC ids to the devid used by the IOMMU */
extern struct list_head ioapic_map;
extern struct list_head hpet_map;

18
drivers/iommu/amd_iommu_v2.c
View File

@ -562,14 +562,30 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
unsigned long flags;
struct fault *fault;
bool finish;
u16 tag;
u16 tag, devid;
int ret;
struct iommu_dev_data *dev_data;
struct pci_dev *pdev = NULL;
iommu_fault = data;
tag = iommu_fault->tag & 0x1ff;
finish = (iommu_fault->tag >> 9) & 1;
devid = iommu_fault->device_id;
pdev = pci_get_bus_and_slot(PCI_BUS_NUM(devid), devid & 0xff);
if (!pdev)
return -ENODEV;
dev_data = get_dev_data(&pdev->dev);
/* In kdump kernel pci dev is not initialized yet -> send INVALID */
ret = NOTIFY_DONE;
if (translation_pre_enabled(amd_iommu_rlookup_table[devid])
&& dev_data->defer_attach) {
amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
PPR_INVALID, tag);
goto out;
}
dev_state = get_device_state(iommu_fault->device_id);
if (dev_state == NULL)
goto out;

220
drivers/iommu/arm-smmu-regs.h Normal file
View File

@ -0,0 +1,220 @@
/*
* IOMMU API for ARM architected SMMU implementations.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* Copyright (C) 2013 ARM Limited
*
* Author: Will Deacon <will.deacon@arm.com>
*/
#ifndef _ARM_SMMU_REGS_H
#define _ARM_SMMU_REGS_H
/* Configuration registers */
#define ARM_SMMU_GR0_sCR0 0x0
#define sCR0_CLIENTPD (1 << 0)
#define sCR0_GFRE (1 << 1)
#define sCR0_GFIE (1 << 2)
#define sCR0_EXIDENABLE (1 << 3)
#define sCR0_GCFGFRE (1 << 4)
#define sCR0_GCFGFIE (1 << 5)
#define sCR0_USFCFG (1 << 10)
#define sCR0_VMIDPNE (1 << 11)
#define sCR0_PTM (1 << 12)
#define sCR0_FB (1 << 13)
#define sCR0_VMID16EN (1 << 31)
#define sCR0_BSU_SHIFT 14
#define sCR0_BSU_MASK 0x3
/* Auxiliary Configuration register */
#define ARM_SMMU_GR0_sACR 0x10
/* Identification registers */
#define ARM_SMMU_GR0_ID0 0x20
#define ARM_SMMU_GR0_ID1 0x24
#define ARM_SMMU_GR0_ID2 0x28
#define ARM_SMMU_GR0_ID3 0x2c
#define ARM_SMMU_GR0_ID4 0x30
#define ARM_SMMU_GR0_ID5 0x34
#define ARM_SMMU_GR0_ID6 0x38
#define ARM_SMMU_GR0_ID7 0x3c
#define ARM_SMMU_GR0_sGFSR 0x48
#define ARM_SMMU_GR0_sGFSYNR0 0x50
#define ARM_SMMU_GR0_sGFSYNR1 0x54
#define ARM_SMMU_GR0_sGFSYNR2 0x58
#define ID0_S1TS (1 << 30)
#define ID0_S2TS (1 << 29)
#define ID0_NTS (1 << 28)
#define ID0_SMS (1 << 27)
#define ID0_ATOSNS (1 << 26)
#define ID0_PTFS_NO_AARCH32 (1 << 25)
#define ID0_PTFS_NO_AARCH32S (1 << 24)
#define ID0_CTTW (1 << 14)
#define ID0_NUMIRPT_SHIFT 16
#define ID0_NUMIRPT_MASK 0xff
#define ID0_NUMSIDB_SHIFT 9
#define ID0_NUMSIDB_MASK 0xf
#define ID0_EXIDS (1 << 8)
#define ID0_NUMSMRG_SHIFT 0
#define ID0_NUMSMRG_MASK 0xff
#define ID1_PAGESIZE (1 << 31)
#define ID1_NUMPAGENDXB_SHIFT 28
#define ID1_NUMPAGENDXB_MASK 7
#define ID1_NUMS2CB_SHIFT 16
#define ID1_NUMS2CB_MASK 0xff
#define ID1_NUMCB_SHIFT 0
#define ID1_NUMCB_MASK 0xff
#define ID2_OAS_SHIFT 4
#define ID2_OAS_MASK 0xf
#define ID2_IAS_SHIFT 0
#define ID2_IAS_MASK 0xf
#define ID2_UBS_SHIFT 8
#define ID2_UBS_MASK 0xf
#define ID2_PTFS_4K (1 << 12)
#define ID2_PTFS_16K (1 << 13)
#define ID2_PTFS_64K (1 << 14)
#define ID2_VMID16 (1 << 15)
#define ID7_MAJOR_SHIFT 4
#define ID7_MAJOR_MASK 0xf
/* Global TLB invalidation */
#define ARM_SMMU_GR0_TLBIVMID 0x64
#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
#define ARM_SMMU_GR0_TLBIALLH 0x6c
#define ARM_SMMU_GR0_sTLBGSYNC 0x70
#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
#define sTLBGSTATUS_GSACTIVE (1 << 0)
/* Stream mapping registers */
#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
#define SMR_VALID (1 << 31)
#define SMR_MASK_SHIFT 16
#define SMR_ID_SHIFT 0
#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
#define S2CR_CBNDX_SHIFT 0
#define S2CR_CBNDX_MASK 0xff
#define S2CR_EXIDVALID (1 << 10)
#define S2CR_TYPE_SHIFT 16
#define S2CR_TYPE_MASK 0x3
enum arm_smmu_s2cr_type {
S2CR_TYPE_TRANS,
S2CR_TYPE_BYPASS,
S2CR_TYPE_FAULT,
};
#define S2CR_PRIVCFG_SHIFT 24
#define S2CR_PRIVCFG_MASK 0x3
enum arm_smmu_s2cr_privcfg {
S2CR_PRIVCFG_DEFAULT,
S2CR_PRIVCFG_DIPAN,
S2CR_PRIVCFG_UNPRIV,
S2CR_PRIVCFG_PRIV,
};
/* Context bank attribute registers */
#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
#define CBAR_VMID_SHIFT 0
#define CBAR_VMID_MASK 0xff
#define CBAR_S1_BPSHCFG_SHIFT 8
#define CBAR_S1_BPSHCFG_MASK 3
#define CBAR_S1_BPSHCFG_NSH 3
#define CBAR_S1_MEMATTR_SHIFT 12
#define CBAR_S1_MEMATTR_MASK 0xf
#define CBAR_S1_MEMATTR_WB 0xf
#define CBAR_TYPE_SHIFT 16
#define CBAR_TYPE_MASK 0x3
#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
#define CBAR_IRPTNDX_SHIFT 24
#define CBAR_IRPTNDX_MASK 0xff
#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
#define CBA2R_RW64_32BIT (0 << 0)
#define CBA2R_RW64_64BIT (1 << 0)
#define CBA2R_VMID_SHIFT 16
#define CBA2R_VMID_MASK 0xffff
#define ARM_SMMU_CB_SCTLR 0x0
#define ARM_SMMU_CB_ACTLR 0x4
#define ARM_SMMU_CB_RESUME 0x8
#define ARM_SMMU_CB_TTBCR2 0x10
#define ARM_SMMU_CB_TTBR0 0x20
#define ARM_SMMU_CB_TTBR1 0x28
#define ARM_SMMU_CB_TTBCR 0x30
#define ARM_SMMU_CB_CONTEXTIDR 0x34
#define ARM_SMMU_CB_S1_MAIR0 0x38
#define ARM_SMMU_CB_S1_MAIR1 0x3c
#define ARM_SMMU_CB_PAR 0x50
#define ARM_SMMU_CB_FSR 0x58
#define ARM_SMMU_CB_FAR 0x60
#define ARM_SMMU_CB_FSYNR0 0x68
#define ARM_SMMU_CB_S1_TLBIVA 0x600
#define ARM_SMMU_CB_S1_TLBIASID 0x610
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
#define ARM_SMMU_CB_TLBSYNC 0x7f0
#define ARM_SMMU_CB_TLBSTATUS 0x7f4
#define ARM_SMMU_CB_ATS1PR 0x800
#define ARM_SMMU_CB_ATSR 0x8f0
#define SCTLR_S1_ASIDPNE (1 << 12)
#define SCTLR_CFCFG (1 << 7)
#define SCTLR_CFIE (1 << 6)
#define SCTLR_CFRE (1 << 5)
#define SCTLR_E (1 << 4)
#define SCTLR_AFE (1 << 2)
#define SCTLR_TRE (1 << 1)
#define SCTLR_M (1 << 0)
#define CB_PAR_F (1 << 0)
#define ATSR_ACTIVE (1 << 0)
#define RESUME_RETRY (0 << 0)
#define RESUME_TERMINATE (1 << 0)
#define TTBCR2_SEP_SHIFT 15
#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
#define TTBCR2_AS (1 << 4)
#define TTBRn_ASID_SHIFT 48
#define FSR_MULTI (1 << 31)
#define FSR_SS (1 << 30)
#define FSR_UUT (1 << 8)
#define FSR_ASF (1 << 7)
#define FSR_TLBLKF (1 << 6)
#define FSR_TLBMCF (1 << 5)
#define FSR_EF (1 << 4)
#define FSR_PF (1 << 3)
#define FSR_AFF (1 << 2)
#define FSR_TF (1 << 1)
#define FSR_IGN (FSR_AFF | FSR_ASF | \
FSR_TLBMCF | FSR_TLBLKF)
#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
#define FSYNR0_WNR (1 << 4)
#endif /* _ARM_SMMU_REGS_H */

7
drivers/iommu/arm-smmu-v3.c
View File

@ -2852,9 +2852,15 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
arm_smmu_device_disable(smmu);
return 0;
}
static void arm_smmu_device_shutdown(struct platform_device *pdev)
{
arm_smmu_device_remove(pdev);
}
static const struct of_device_id arm_smmu_of_match[] = {
{ .compatible = "arm,smmu-v3", },
{ },
@ -2868,6 +2874,7 @@ static struct platform_driver arm_smmu_driver = {
},
.probe = arm_smmu_device_probe,
.remove = arm_smmu_device_remove,
.shutdown = arm_smmu_device_shutdown,
};
module_platform_driver(arm_smmu_driver);

384
drivers/iommu/arm-smmu.c
View File

@ -54,6 +54,15 @@
#include <linux/amba/bus.h>
#include "io-pgtable.h"
#include "arm-smmu-regs.h"
#define ARM_MMU500_ACTLR_CPRE (1 << 1)
#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
#define TLB_SPIN_COUNT 10
/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS 128
@ -83,211 +92,9 @@
#define smmu_write_atomic_lq writel_relaxed
#endif
/* Configuration registers */
#define ARM_SMMU_GR0_sCR0 0x0
#define sCR0_CLIENTPD (1 << 0)
#define sCR0_GFRE (1 << 1)
#define sCR0_GFIE (1 << 2)
#define sCR0_EXIDENABLE (1 << 3)
#define sCR0_GCFGFRE (1 << 4)
#define sCR0_GCFGFIE (1 << 5)
#define sCR0_USFCFG (1 << 10)
#define sCR0_VMIDPNE (1 << 11)
#define sCR0_PTM (1 << 12)
#define sCR0_FB (1 << 13)
#define sCR0_VMID16EN (1 << 31)
#define sCR0_BSU_SHIFT 14
#define sCR0_BSU_MASK 0x3
/* Auxiliary Configuration register */
#define ARM_SMMU_GR0_sACR 0x10
/* Identification registers */
#define ARM_SMMU_GR0_ID0 0x20
#define ARM_SMMU_GR0_ID1 0x24
#define ARM_SMMU_GR0_ID2 0x28
#define ARM_SMMU_GR0_ID3 0x2c
#define ARM_SMMU_GR0_ID4 0x30
#define ARM_SMMU_GR0_ID5 0x34
#define ARM_SMMU_GR0_ID6 0x38
#define ARM_SMMU_GR0_ID7 0x3c
#define ARM_SMMU_GR0_sGFSR 0x48
#define ARM_SMMU_GR0_sGFSYNR0 0x50
#define ARM_SMMU_GR0_sGFSYNR1 0x54
#define ARM_SMMU_GR0_sGFSYNR2 0x58
#define ID0_S1TS (1 << 30)
#define ID0_S2TS (1 << 29)
#define ID0_NTS (1 << 28)
#define ID0_SMS (1 << 27)
#define ID0_ATOSNS (1 << 26)
#define ID0_PTFS_NO_AARCH32 (1 << 25)
#define ID0_PTFS_NO_AARCH32S (1 << 24)
#define ID0_CTTW (1 << 14)
#define ID0_NUMIRPT_SHIFT 16
#define ID0_NUMIRPT_MASK 0xff
#define ID0_NUMSIDB_SHIFT 9
#define ID0_NUMSIDB_MASK 0xf
#define ID0_EXIDS (1 << 8)
#define ID0_NUMSMRG_SHIFT 0
#define ID0_NUMSMRG_MASK 0xff
#define ID1_PAGESIZE (1 << 31)
#define ID1_NUMPAGENDXB_SHIFT 28
#define ID1_NUMPAGENDXB_MASK 7
#define ID1_NUMS2CB_SHIFT 16
#define ID1_NUMS2CB_MASK 0xff
#define ID1_NUMCB_SHIFT 0
#define ID1_NUMCB_MASK 0xff
#define ID2_OAS_SHIFT 4
#define ID2_OAS_MASK 0xf
#define ID2_IAS_SHIFT 0
#define ID2_IAS_MASK 0xf
#define ID2_UBS_SHIFT 8
#define ID2_UBS_MASK 0xf
#define ID2_PTFS_4K (1 << 12)
#define ID2_PTFS_16K (1 << 13)
#define ID2_PTFS_64K (1 << 14)
#define ID2_VMID16 (1 << 15)
#define ID7_MAJOR_SHIFT 4
#define ID7_MAJOR_MASK 0xf
/* Global TLB invalidation */
#define ARM_SMMU_GR0_TLBIVMID 0x64
#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
#define ARM_SMMU_GR0_TLBIALLH 0x6c
#define ARM_SMMU_GR0_sTLBGSYNC 0x70
#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
#define sTLBGSTATUS_GSACTIVE (1 << 0)
#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
#define TLB_SPIN_COUNT 10
/* Stream mapping registers */
#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
#define SMR_VALID (1 << 31)
#define SMR_MASK_SHIFT 16
#define SMR_ID_SHIFT 0
#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
#define S2CR_CBNDX_SHIFT 0
#define S2CR_CBNDX_MASK 0xff
#define S2CR_EXIDVALID (1 << 10)
#define S2CR_TYPE_SHIFT 16
#define S2CR_TYPE_MASK 0x3
enum arm_smmu_s2cr_type {
S2CR_TYPE_TRANS,
S2CR_TYPE_BYPASS,
S2CR_TYPE_FAULT,
};
#define S2CR_PRIVCFG_SHIFT 24
#define S2CR_PRIVCFG_MASK 0x3
enum arm_smmu_s2cr_privcfg {
S2CR_PRIVCFG_DEFAULT,
S2CR_PRIVCFG_DIPAN,
S2CR_PRIVCFG_UNPRIV,
S2CR_PRIVCFG_PRIV,
};
/* Context bank attribute registers */
#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
#define CBAR_VMID_SHIFT 0
#define CBAR_VMID_MASK 0xff
#define CBAR_S1_BPSHCFG_SHIFT 8
#define CBAR_S1_BPSHCFG_MASK 3
#define CBAR_S1_BPSHCFG_NSH 3
#define CBAR_S1_MEMATTR_SHIFT 12
#define CBAR_S1_MEMATTR_MASK 0xf
#define CBAR_S1_MEMATTR_WB 0xf
#define CBAR_TYPE_SHIFT 16
#define CBAR_TYPE_MASK 0x3
#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
#define CBAR_IRPTNDX_SHIFT 24
#define CBAR_IRPTNDX_MASK 0xff
#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
#define CBA2R_RW64_32BIT (0 << 0)
#define CBA2R_RW64_64BIT (1 << 0)
#define CBA2R_VMID_SHIFT 16
#define CBA2R_VMID_MASK 0xffff
/* Translation context bank */
#define ARM_SMMU_CB(smmu, n) ((smmu)->cb_base + ((n) << (smmu)->pgshift))
#define ARM_SMMU_CB_SCTLR 0x0
#define ARM_SMMU_CB_ACTLR 0x4
#define ARM_SMMU_CB_RESUME 0x8
#define ARM_SMMU_CB_TTBCR2 0x10
#define ARM_SMMU_CB_TTBR0 0x20
#define ARM_SMMU_CB_TTBR1 0x28
#define ARM_SMMU_CB_TTBCR 0x30
#define ARM_SMMU_CB_CONTEXTIDR 0x34
#define ARM_SMMU_CB_S1_MAIR0 0x38
#define ARM_SMMU_CB_S1_MAIR1 0x3c
#define ARM_SMMU_CB_PAR 0x50
#define ARM_SMMU_CB_FSR 0x58
#define ARM_SMMU_CB_FAR 0x60
#define ARM_SMMU_CB_FSYNR0 0x68
#define ARM_SMMU_CB_S1_TLBIVA 0x600
#define ARM_SMMU_CB_S1_TLBIASID 0x610
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
#define ARM_SMMU_CB_TLBSYNC 0x7f0
#define ARM_SMMU_CB_TLBSTATUS 0x7f4
#define ARM_SMMU_CB_ATS1PR 0x800
#define ARM_SMMU_CB_ATSR 0x8f0
#define SCTLR_S1_ASIDPNE (1 << 12)
#define SCTLR_CFCFG (1 << 7)
#define SCTLR_CFIE (1 << 6)
#define SCTLR_CFRE (1 << 5)
#define SCTLR_E (1 << 4)
#define SCTLR_AFE (1 << 2)
#define SCTLR_TRE (1 << 1)
#define SCTLR_M (1 << 0)
#define ARM_MMU500_ACTLR_CPRE (1 << 1)
#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
#define CB_PAR_F (1 << 0)
#define ATSR_ACTIVE (1 << 0)
#define RESUME_RETRY (0 << 0)
#define RESUME_TERMINATE (1 << 0)
#define TTBCR2_SEP_SHIFT 15
#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
#define TTBCR2_AS (1 << 4)
#define TTBRn_ASID_SHIFT 48
#define FSR_MULTI (1 << 31)
#define FSR_SS (1 << 30)
#define FSR_UUT (1 << 8)
#define FSR_ASF (1 << 7)
#define FSR_TLBLKF (1 << 6)
#define FSR_TLBMCF (1 << 5)
#define FSR_EF (1 << 4)
#define FSR_PF (1 << 3)
#define FSR_AFF (1 << 2)
#define FSR_TF (1 << 1)
#define FSR_IGN (FSR_AFF | FSR_ASF | \
FSR_TLBMCF | FSR_TLBLKF)
#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
#define FSYNR0_WNR (1 << 4)
#define MSI_IOVA_BASE 0x8000000
#define MSI_IOVA_LENGTH 0x100000
@ -338,6 +145,13 @@ struct arm_smmu_smr {
bool valid;
};
struct arm_smmu_cb {
u64 ttbr[2];
u32 tcr[2];
u32 mair[2];
struct arm_smmu_cfg *cfg;
};
struct arm_smmu_master_cfg {
struct arm_smmu_device *smmu;
s16 smendx[];
@ -380,6 +194,7 @@ struct arm_smmu_device {
u32 num_context_banks;
u32 num_s2_context_banks;
DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
struct arm_smmu_cb *cbs;
atomic_t irptndx;
u32 num_mapping_groups;
@ -776,17 +591,74 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
struct io_pgtable_cfg *pgtbl_cfg)
{
u32 reg, reg2;
u64 reg64;
bool stage1;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
cb->cfg = cfg;
/* TTBCR */
if (stage1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
} else {
cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
cb->tcr[1] |= TTBCR2_SEP_UPSTREAM;
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
cb->tcr[1] |= TTBCR2_AS;
}
} else {
cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
}
/* TTBRs */
if (stage1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
} else {
cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
}
} else {
cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
}
/* MAIRs (stage-1 only) */
if (stage1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
} else {
cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
}
}
}
static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
{
u32 reg;
bool stage1;
struct arm_smmu_cb *cb = &smmu->cbs[idx];
struct arm_smmu_cfg *cfg = cb->cfg;
void __iomem *cb_base, *gr1_base;
cb_base = ARM_SMMU_CB(smmu, idx);
/* Unassigned context banks only need disabling */
if (!cfg) {
writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
return;
}
gr1_base = ARM_SMMU_GR1(smmu);
stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
/* CBA2R */
if (smmu->version > ARM_SMMU_V1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
reg = CBA2R_RW64_64BIT;
@ -796,7 +668,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
if (smmu->features & ARM_SMMU_FEAT_VMID16)
reg |= cfg->vmid << CBA2R_VMID_SHIFT;
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
}
/* CBAR */
@ -815,72 +687,41 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
/* 8-bit VMIDs live in CBAR */
reg |= cfg->vmid << CBAR_VMID_SHIFT;
}
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
/*
* TTBCR
* We must write this before the TTBRs, since it determines the
* access behaviour of some fields (in particular, ASID[15:8]).
*/
if (stage1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
reg = pgtbl_cfg->arm_v7s_cfg.tcr;
reg2 = 0;
} else {
reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
reg2 |= TTBCR2_SEP_UPSTREAM;
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
reg2 |= TTBCR2_AS;
}
if (smmu->version > ARM_SMMU_V1)
writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
} else {
reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
}
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
if (stage1 && smmu->version > ARM_SMMU_V1)
writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TTBCR2);
writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TTBCR);
/* TTBRs */
if (stage1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
} else {
reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
}
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
} else {
reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
if (stage1)
writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
}
/* MAIRs (stage-1 only) */
if (stage1) {
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
reg = pgtbl_cfg->arm_v7s_cfg.prrr;
reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
} else {
reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
}
writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
}
/* SCTLR */
reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
if (stage1)
reg |= SCTLR_S1_ASIDPNE;
#ifdef __BIG_ENDIAN
reg |= SCTLR_E;
#endif
if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
reg |= SCTLR_E;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
}
@ -1043,6 +884,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
/* Initialise the context bank with our page table cfg */
arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
arm_smmu_write_context_bank(smmu, cfg->cbndx);
/*
* Request context fault interrupt. Do this last to avoid the
@ -1075,7 +917,6 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
struct arm_smmu_device *smmu = smmu_domain->smmu;
struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
void __iomem *cb_base;
int irq;
if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
@ -1085,8 +926,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
* Disable the context bank and free the page tables before freeing
* it.
*/
cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
smmu->cbs[cfg->cbndx].cfg = NULL;
arm_smmu_write_context_bank(smmu, cfg->cbndx);
if (cfg->irptndx != INVALID_IRPTNDX) {
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
@ -1736,7 +1577,6 @@ static struct iommu_ops arm_smmu_ops = {
static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
{
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
void __iomem *cb_base;
int i;
u32 reg, major;
@ -1772,8 +1612,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Make sure all context banks are disabled and clear CB_FSR */
for (i = 0; i < smmu->num_context_banks; ++i) {
cb_base = ARM_SMMU_CB(smmu, i);
writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
arm_smmu_write_context_bank(smmu, i);
writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
/*
* Disable MMU-500's not-particularly-beneficial next-page
@ -1979,6 +1820,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
smmu->cavium_id_base -= smmu->num_context_banks;
dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
}
smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
sizeof(*smmu->cbs), GFP_KERNEL);
if (!smmu->cbs)
return -ENOMEM;
/* ID2 */
id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
@ -2336,13 +2181,30 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
return 0;
}
static void arm_smmu_device_shutdown(struct platform_device *pdev)
{
arm_smmu_device_remove(pdev);
}
static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
{
struct arm_smmu_device *smmu = dev_get_drvdata(dev);
arm_smmu_device_reset(smmu);
return 0;
}
static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume);
static struct platform_driver arm_smmu_driver = {
.driver = {
.name = "arm-smmu",
.of_match_table = of_match_ptr(arm_smmu_of_match),
.pm = &arm_smmu_pm_ops,
},
.probe = arm_smmu_device_probe,
.remove = arm_smmu_device_remove,
.shutdown = arm_smmu_device_shutdown,
};
module_platform_driver(arm_smmu_driver);

2
drivers/iommu/dmar.c
View File

@ -1343,7 +1343,7 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
if (mask) {
BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
} else
desc.high = QI_DEV_IOTLB_ADDR(addr);

44
drivers/iommu/exynos-iommu.c
View File

@ -54,10 +54,6 @@ typedef u32 sysmmu_pte_t;
#define lv2ent_small(pent) ((*(pent) & 2) == 2)
#define lv2ent_large(pent) ((*(pent) & 3) == 1)
#ifdef CONFIG_BIG_ENDIAN
#warning "revisit driver if we can enable big-endian ptes"
#endif
/*
* v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
* v5.0 introduced support for 36bit physical address space by shifting
@ -569,7 +565,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
spin_unlock_irqrestore(&data->lock, flags);
}
static struct iommu_ops exynos_iommu_ops;
static const struct iommu_ops exynos_iommu_ops;
static int __init exynos_sysmmu_probe(struct platform_device *pdev)
{
@ -659,6 +655,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
}
}
/*
* use the first registered sysmmu device for performing
* dma mapping operations on iommu page tables (cpu cache flush)
*/
if (!dma_dev)
dma_dev = &pdev->dev;
pm_runtime_enable(dev);
return 0;
@ -1323,7 +1326,7 @@ static int exynos_iommu_of_xlate(struct device *dev,
return 0;
}
static struct iommu_ops exynos_iommu_ops = {
static const struct iommu_ops exynos_iommu_ops = {
.domain_alloc = exynos_iommu_domain_alloc,
.domain_free = exynos_iommu_domain_free,
.attach_dev = exynos_iommu_attach_device,
@ -1339,8 +1342,6 @@ static struct iommu_ops exynos_iommu_ops = {
.of_xlate = exynos_iommu_of_xlate,
};
static bool init_done;
static int __init exynos_iommu_init(void)
{
int ret;
@ -1373,8 +1374,6 @@ static int __init exynos_iommu_init(void)
goto err_set_iommu;
}
init_done = true;
return 0;
err_set_iommu:
kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
@ -1384,27 +1383,6 @@ err_reg_driver:
kmem_cache_destroy(lv2table_kmem_cache);
return ret;
}
core_initcall(exynos_iommu_init);
static int __init exynos_iommu_of_setup(struct device_node *np)
{
struct platform_device *pdev;
if (!init_done)
exynos_iommu_init();
pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
if (!pdev)
return -ENODEV;
/*
* use the first registered sysmmu device for performing
* dma mapping operations on iommu page tables (cpu cache flush)
*/
if (!dma_dev)
dma_dev = &pdev->dev;
return 0;
}
IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu",
exynos_iommu_of_setup);
IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", NULL);

27
drivers/iommu/fsl_pamu.c
View File

@ -42,6 +42,8 @@ struct pamu_isr_data {
static struct paace *ppaact;
static struct paace *spaact;
static bool probed; /* Has PAMU been probed? */
/*
* Table for matching compatible strings, for device tree
* guts node, for QorIQ SOCs.
@ -530,8 +532,8 @@ u32 get_stash_id(u32 stash_dest_hint, u32 vcpu)
if (node) {
prop = of_get_property(node, "cache-stash-id", NULL);
if (!prop) {
pr_debug("missing cache-stash-id at %s\n",
node->full_name);
pr_debug("missing cache-stash-id at %pOF\n",
node);
of_node_put(node);
return ~(u32)0;
}
@ -557,8 +559,8 @@ found_cpu_node:
if (stash_dest_hint == cache_level) {
prop = of_get_property(node, "cache-stash-id", NULL);
if (!prop) {
pr_debug("missing cache-stash-id at %s\n",
node->full_name);
pr_debug("missing cache-stash-id at %pOF\n",
node);
of_node_put(node);
return ~(u32)0;
}
@ -568,8 +570,7 @@ found_cpu_node:
prop = of_get_property(node, "next-level-cache", NULL);
if (!prop) {
pr_debug("can't find next-level-cache at %s\n",
node->full_name);
pr_debug("can't find next-level-cache at %pOF\n", node);
of_node_put(node);
return ~(u32)0; /* can't traverse any further */
}
@ -1033,6 +1034,9 @@ static int fsl_pamu_probe(struct platform_device *pdev)
* NOTE : All PAMUs share the same LIODN tables.
*/
if (WARN_ON(probed))
return -EBUSY;
pamu_regs = of_iomap(dev->of_node, 0);
if (!pamu_regs) {
dev_err(dev, "ioremap of PAMU node failed\n");
@ -1063,8 +1067,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
guts_node = of_find_matching_node(NULL, guts_device_ids);
if (!guts_node) {
dev_err(dev, "could not find GUTS node %s\n",
dev->of_node->full_name);
dev_err(dev, "could not find GUTS node %pOF\n", dev->of_node);
ret = -ENODEV;
goto error;
}
@ -1172,6 +1175,8 @@ static int fsl_pamu_probe(struct platform_device *pdev)
setup_liodns();
probed = true;
return 0;
error_genpool:
@ -1246,8 +1251,7 @@ static __init int fsl_pamu_init(void)
pdev = platform_device_alloc("fsl-of-pamu", 0);
if (!pdev) {
pr_err("could not allocate device %s\n",
np->full_name);
pr_err("could not allocate device %pOF\n", np);
ret = -ENOMEM;
goto error_device_alloc;
}
@ -1259,8 +1263,7 @@ static __init int fsl_pamu_init(void)
ret = platform_device_add(pdev);
if (ret) {
pr_err("could not add device %s (err=%i)\n",
np->full_name, ret);
pr_err("could not add device %pOF (err=%i)\n", np, ret);
goto error_device_add;
}

28
drivers/iommu/fsl_pamu_domain.c
View File

@ -33,6 +33,8 @@ static struct kmem_cache *fsl_pamu_domain_cache;
static struct kmem_cache *iommu_devinfo_cache;
static DEFINE_SPINLOCK(device_domain_lock);
struct iommu_device pamu_iommu; /* IOMMU core code handle */
static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
{
return container_of(dom, struct fsl_dma_domain, iommu_domain);
@ -619,8 +621,8 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain,
for (i = 0; i < num; i++) {
/* Ensure that LIODN value is valid */
if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
pr_debug("Invalid liodn %d, attach device failed for %s\n",
liodn[i], dev->of_node->full_name);
pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
liodn[i], dev->of_node);
ret = -EINVAL;
break;
}
@ -684,8 +686,7 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain,
liodn_cnt = len / sizeof(u32);
ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
} else {
pr_debug("missing fsl,liodn property at %s\n",
dev->of_node->full_name);
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
ret = -EINVAL;
}
@ -720,8 +721,7 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain,
if (prop)
detach_device(dev, dma_domain);
else
pr_debug("missing fsl,liodn property at %s\n",
dev->of_node->full_name);
pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
}
static int configure_domain_geometry(struct iommu_domain *domain, void *data)
@ -983,11 +983,14 @@ static int fsl_pamu_add_device(struct device *dev)
iommu_group_put(group);
iommu_device_link(&pamu_iommu, dev);
return 0;
}
static void fsl_pamu_remove_device(struct device *dev)
{
iommu_device_unlink(&pamu_iommu, dev);
iommu_group_remove_device(dev);
}
@ -1073,6 +1076,19 @@ int __init pamu_domain_init(void)
if (ret)
return ret;
ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
if (ret)
return ret;
iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops);
ret = iommu_device_register(&pamu_iommu);
if (ret) {
iommu_device_sysfs_remove(&pamu_iommu);
pr_err("Can't register iommu device\n");
return ret;
}
bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);

280
drivers/iommu/intel-iommu.c
View File

@ -458,31 +458,6 @@ static LIST_HEAD(dmar_rmrr_units);
#define for_each_rmrr_units(rmrr) \
list_for_each_entry(rmrr, &dmar_rmrr_units, list)
static void flush_unmaps_timeout(unsigned long data);
struct deferred_flush_entry {
unsigned long iova_pfn;
unsigned long nrpages;
struct dmar_domain *domain;
struct page *freelist;
};
#define HIGH_WATER_MARK 250
struct deferred_flush_table {
int next;
struct deferred_flush_entry entries[HIGH_WATER_MARK];
};
struct deferred_flush_data {
spinlock_t lock;
int timer_on;
struct timer_list timer;
long size;
struct deferred_flush_table *tables;
};
static DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
/* bitmap for indexing intel_iommus */
static int g_num_of_iommus;
@ -974,20 +949,6 @@ static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
return ret;
}
static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
struct context_entry *context;
unsigned long flags;
spin_lock_irqsave(&iommu->lock, flags);
context = iommu_context_addr(iommu, bus, devfn, 0);
if (context) {
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
}
spin_unlock_irqrestore(&iommu->lock, flags);
}
static void free_context_table(struct intel_iommu *iommu)
{
int i;
@ -1137,8 +1098,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain,
}
static void dma_pte_free_level(struct dmar_domain *domain, int level,
struct dma_pte *pte, unsigned long pfn,
unsigned long start_pfn, unsigned long last_pfn)
int retain_level, struct dma_pte *pte,
unsigned long pfn, unsigned long start_pfn,
unsigned long last_pfn)
{
pfn = max(start_pfn, pfn);
pte = &pte[pfn_level_offset(pfn, level)];
@ -1153,12 +1115,17 @@ static void dma_pte_free_level(struct dmar_domain *domain, int level,
level_pfn = pfn & level_mask(level);
level_pte = phys_to_virt(dma_pte_addr(pte));
if (level > 2)
dma_pte_free_level(domain, level - 1, level_pte,
level_pfn, start_pfn, last_pfn);
if (level > 2) {
dma_pte_free_level(domain, level - 1, retain_level,
level_pte, level_pfn, start_pfn,
last_pfn);
}
/* If range covers entire pagetable, free it */
if (!(start_pfn > level_pfn ||
/*
* Free the page table if we're below the level we want to
* retain and the range covers the entire table.
*/
if (level < retain_level && !(start_pfn > level_pfn ||
last_pfn < level_pfn + level_size(level) - 1)) {
dma_clear_pte(pte);
domain_flush_cache(domain, pte, sizeof(*pte));
@ -1169,10 +1136,14 @@ next:
} while (!first_pte_in_page(++pte) && pfn <= last_pfn);
}
/* clear last level (leaf) ptes and free page table pages. */
/*
* clear last level (leaf) ptes and free page table pages below the
* level we wish to keep intact.
*/
static void dma_pte_free_pagetable(struct dmar_domain *domain,
unsigned long start_pfn,
unsigned long last_pfn)
unsigned long last_pfn,
int retain_level)
{
BUG_ON(!domain_pfn_supported(domain, start_pfn));
BUG_ON(!domain_pfn_supported(domain, last_pfn));
@ -1181,7 +1152,7 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
dma_pte_clear_range(domain, start_pfn, last_pfn);
/* We don't need lock here; nobody else touches the iova range */
dma_pte_free_level(domain, agaw_to_level(domain->agaw),
dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
domain->pgd, 0, start_pfn, last_pfn);
/* free pgd */
@ -1309,6 +1280,13 @@ static void dma_free_pagelist(struct page *freelist)
}
}
static void iova_entry_free(unsigned long data)
{
struct page *freelist = (struct page *)data;
dma_free_pagelist(freelist);
}
/* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{
@ -1622,6 +1600,25 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
addr, mask);
}
static void iommu_flush_iova(struct iova_domain *iovad)
{
struct dmar_domain *domain;
int idx;
domain = container_of(iovad, struct dmar_domain, iovad);
for_each_domain_iommu(idx, domain) {
struct intel_iommu *iommu = g_iommus[idx];
u16 did = domain->iommu_did[iommu->seq_id];
iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
if (!cap_caching_mode(iommu->cap))
iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
0, MAX_AGAW_PFN_WIDTH);
}
}
static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
{
u32 pmen;
@ -1932,9 +1929,16 @@ static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
{
int adjust_width, agaw;
unsigned long sagaw;
int err;
init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
DMA_32BIT_PFN);
err = init_iova_flush_queue(&domain->iovad,
iommu_flush_iova, iova_entry_free);
if (err)
return err;
domain_reserve_special_ranges(domain);
/* calculate AGAW */
@ -1986,14 +1990,6 @@ static void domain_exit(struct dmar_domain *domain)
if (!domain)
return;
/* Flush any lazy unmaps that may reference this domain */
if (!intel_iommu_strict) {
int cpu;
for_each_possible_cpu(cpu)
flush_unmaps_timeout(cpu);
}
/* Remove associated devices and clear attached or cached domains */
rcu_read_lock();
domain_remove_dev_info(domain);
@ -2277,8 +2273,11 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
/*
* Ensure that old small page tables are
* removed to make room for superpage(s).
* We're adding new large pages, so make sure
* we don't remove their parent tables.
*/
dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
largepage_lvl + 1);
} else {
pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
}
@ -2351,13 +2350,33 @@ static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long i
static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
unsigned long flags;
struct context_entry *context;
u16 did_old;
if (!iommu)
return;
clear_context_table(iommu, bus, devfn);
iommu->flush.flush_context(iommu, 0, 0, 0,
DMA_CCMD_GLOBAL_INVL);
iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
spin_lock_irqsave(&iommu->lock, flags);
context = iommu_context_addr(iommu, bus, devfn, 0);
if (!context) {
spin_unlock_irqrestore(&iommu->lock, flags);
return;
}
did_old = context_domain_id(context);
context_clear_entry(context);
__iommu_flush_cache(iommu, context, sizeof(*context));
spin_unlock_irqrestore(&iommu->lock, flags);
iommu->flush.flush_context(iommu,
did_old,
(((u16)bus) << 8) | devfn,
DMA_CCMD_MASK_NOBIT,
DMA_CCMD_DEVICE_INVL);
iommu->flush.flush_iotlb(iommu,
did_old,
0,
0,
DMA_TLB_DSI_FLUSH);
}
static inline void unlink_domain_info(struct device_domain_info *info)
@ -3206,7 +3225,7 @@ static int __init init_dmars(void)
bool copied_tables = false;
struct device *dev;
struct intel_iommu *iommu;
int i, ret, cpu;
int i, ret;
/*
* for each drhd
@ -3239,22 +3258,6 @@ static int __init init_dmars(void)
goto error;
}
for_each_possible_cpu(cpu) {
struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
cpu);
dfd->tables = kzalloc(g_num_of_iommus *
sizeof(struct deferred_flush_table),
GFP_KERNEL);
if (!dfd->tables) {
ret = -ENOMEM;
goto free_g_iommus;
}
spin_lock_init(&dfd->lock);
setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
}
for_each_active_iommu(iommu, drhd) {
g_iommus[iommu->seq_id] = iommu;
@ -3437,10 +3440,9 @@ free_iommu:
disable_dmar_iommu(iommu);
free_dmar_iommu(iommu);
}
free_g_iommus:
for_each_possible_cpu(cpu)
kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
kfree(g_iommus);
error:
return ret;
}
@ -3645,110 +3647,6 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
dir, *dev->dma_mask);
}
static void flush_unmaps(struct deferred_flush_data *flush_data)
{
int i, j;
flush_data->timer_on = 0;
/* just flush them all */
for (i = 0; i < g_num_of_iommus; i++) {
struct intel_iommu *iommu = g_iommus[i];
struct deferred_flush_table *flush_table =
&flush_data->tables[i];
if (!iommu)
continue;
if (!flush_table->next)
continue;
/* In caching mode, global flushes turn emulation expensive */
if (!cap_caching_mode(iommu->cap))
iommu->flush.flush_iotlb(iommu, 0, 0, 0,
DMA_TLB_GLOBAL_FLUSH);
for (j = 0; j < flush_table->next; j++) {
unsigned long mask;
struct deferred_flush_entry *entry =
&flush_table->entries[j];
unsigned long iova_pfn = entry->iova_pfn;
unsigned long nrpages = entry->nrpages;
struct dmar_domain *domain = entry->domain;
struct page *freelist = entry->freelist;
/* On real hardware multiple invalidations are expensive */
if (cap_caching_mode(iommu->cap))
iommu_flush_iotlb_psi(iommu, domain,
mm_to_dma_pfn(iova_pfn),
nrpages, !freelist, 0);
else {
mask = ilog2(nrpages);
iommu_flush_dev_iotlb(domain,
(uint64_t)iova_pfn << PAGE_SHIFT, mask);
}
free_iova_fast(&domain->iovad, iova_pfn, nrpages);
if (freelist)
dma_free_pagelist(freelist);
}
flush_table->next = 0;
}
flush_data->size = 0;
}
static void flush_unmaps_timeout(unsigned long cpuid)
{
struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
unsigned long flags;
spin_lock_irqsave(&flush_data->lock, flags);
flush_unmaps(flush_data);
spin_unlock_irqrestore(&flush_data->lock, flags);
}
static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
unsigned long nrpages, struct page *freelist)
{
unsigned long flags;
int entry_id, iommu_id;
struct intel_iommu *iommu;
struct deferred_flush_entry *entry;
struct deferred_flush_data *flush_data;
flush_data = raw_cpu_ptr(&deferred_flush);
/* Flush all CPUs' entries to avoid deferring too much. If
* this becomes a bottleneck, can just flush us, and rely on
* flush timer for the rest.
*/
if (flush_data->size == HIGH_WATER_MARK) {
int cpu;
for_each_online_cpu(cpu)
flush_unmaps_timeout(cpu);
}
spin_lock_irqsave(&flush_data->lock, flags);
iommu = domain_get_iommu(dom);
iommu_id = iommu->seq_id;
entry_id = flush_data->tables[iommu_id].next;
++(flush_data->tables[iommu_id].next);
entry = &flush_data->tables[iommu_id].entries[entry_id];
entry->domain = dom;
entry->iova_pfn = iova_pfn;
entry->nrpages = nrpages;
entry->freelist = freelist;
if (!flush_data->timer_on) {
mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
flush_data->timer_on = 1;
}
flush_data->size++;
spin_unlock_irqrestore(&flush_data->lock, flags);
}
static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
{
struct dmar_domain *domain;
@ -3784,7 +3682,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
dma_free_pagelist(freelist);
} else {
add_unmap(domain, iova_pfn, nrpages, freelist);
queue_iova(&domain->iovad, iova_pfn, nrpages,
(unsigned long)freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
@ -3938,7 +3837,8 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
if (unlikely(ret)) {
dma_pte_free_pagetable(domain, start_vpfn,
start_vpfn + size - 1);
start_vpfn + size - 1,
agaw_to_level(domain->agaw) + 1);
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
return 0;
}
@ -4721,7 +4621,6 @@ static void free_all_cpu_cached_iovas(unsigned int cpu)
static int intel_iommu_cpu_dead(unsigned int cpu)
{
free_all_cpu_cached_iovas(cpu);
flush_unmaps_timeout(cpu);
return 0;
}
@ -5343,7 +5242,8 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd
sdev->sid = PCI_DEVID(info->bus, info->devfn);
if (!(ctx_lo & CONTEXT_PASIDE)) {
context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
if (iommu->pasid_state_table)
context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
intel_iommu_get_pts(iommu);

14
drivers/iommu/intel-svm.c
View File

@ -24,6 +24,7 @@
#include <linux/pci-ats.h>
#include <linux/dmar.h>
#include <linux/interrupt.h>
#include <asm/page.h>
static irqreturn_t prq_event_thread(int irq, void *d);
@ -555,6 +556,14 @@ static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req)
return (requested & ~vma->vm_flags) != 0;
}
static bool is_canonical_address(u64 addr)
{
int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1);
long saddr = (long) addr;
return (((saddr << shift) >> shift) == saddr);
}
static irqreturn_t prq_event_thread(int irq, void *d)
{
struct intel_iommu *iommu = d;
@ -612,6 +621,11 @@ static irqreturn_t prq_event_thread(int irq, void *d)
/* If the mm is already defunct, don't handle faults. */
if (!mmget_not_zero(svm->mm))
goto bad_req;
/* If address is not canonical, return invalid response */
if (!is_canonical_address(address))
goto bad_req;
down_read(&svm->mm->mmap_sem);
vma = find_extend_vma(svm->mm, address);
if (!vma || address < vma->vm_start)

59
drivers/iommu/iommu.c
View File

@ -527,6 +527,8 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group,
}
iommu_flush_tlb_all(domain);
out:
iommu_put_resv_regions(dev, &mappings);
@ -1005,11 +1007,10 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
if (group)
return group;
group = ERR_PTR(-EINVAL);
if (ops && ops->device_group)
group = ops->device_group(dev);
if (!ops)
return ERR_PTR(-EINVAL);
group = ops->device_group(dev);
if (WARN_ON_ONCE(group == NULL))
return ERR_PTR(-EINVAL);
@ -1283,6 +1284,10 @@ static int __iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
int ret;
if ((domain->ops->is_attach_deferred != NULL) &&
domain->ops->is_attach_deferred(domain, dev))
return 0;
if (unlikely(domain->ops->attach_dev == NULL))
return -ENODEV;
@ -1298,12 +1303,8 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
int ret;
group = iommu_group_get(dev);
/* FIXME: Remove this when groups a mandatory for iommu drivers */
if (group == NULL)
return __iommu_attach_device(domain, dev);
/*
* We have a group - lock it to make sure the device-count doesn't
* Lock the group to make sure the device-count doesn't
* change while we are attaching
*/
mutex_lock(&group->mutex);
@ -1324,6 +1325,10 @@ EXPORT_SYMBOL_GPL(iommu_attach_device);
static void __iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
if ((domain->ops->is_attach_deferred != NULL) &&
domain->ops->is_attach_deferred(domain, dev))
return;
if (unlikely(domain->ops->detach_dev == NULL))
return;
@ -1336,9 +1341,6 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
struct iommu_group *group;
group = iommu_group_get(dev);
/* FIXME: Remove this when groups a mandatory for iommu drivers */
if (group == NULL)
return __iommu_detach_device(domain, dev);
mutex_lock(&group->mutex);
if (iommu_group_device_count(group) != 1) {
@ -1360,8 +1362,7 @@ struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
struct iommu_group *group;
group = iommu_group_get(dev);
/* FIXME: Remove this when groups a mandatory for iommu drivers */
if (group == NULL)
if (!group)
return NULL;
domain = group->domain;
@ -1556,13 +1557,16 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
}
EXPORT_SYMBOL_GPL(iommu_map);
size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
static size_t __iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
bool sync)
{
const struct iommu_ops *ops = domain->ops;
size_t unmapped_page, unmapped = 0;
unsigned int min_pagesz;
unsigned long orig_iova = iova;
unsigned int min_pagesz;
if (unlikely(domain->ops->unmap == NULL ||
if (unlikely(ops->unmap == NULL ||
domain->pgsize_bitmap == 0UL))
return -ENODEV;
@ -1592,10 +1596,13 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
while (unmapped < size) {
size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
unmapped_page = domain->ops->unmap(domain, iova, pgsize);
unmapped_page = ops->unmap(domain, iova, pgsize);
if (!unmapped_page)
break;
if (sync && ops->iotlb_range_add)
ops->iotlb_range_add(domain, iova, pgsize);
pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
iova, unmapped_page);
@ -1603,11 +1610,27 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
unmapped += unmapped_page;
}
if (sync && ops->iotlb_sync)
ops->iotlb_sync(domain);
trace_unmap(orig_iova, size, unmapped);
return unmapped;
}
size_t iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
return __iommu_unmap(domain, iova, size, true);
}
EXPORT_SYMBOL_GPL(iommu_unmap);
size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
return __iommu_unmap(domain, iova, size, false);
}
EXPORT_SYMBOL_GPL(iommu_unmap_fast);
size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
struct scatterlist *sg, unsigned int nents, int prot)
{

183
drivers/iommu/iova.c
View File

@ -32,6 +32,8 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,
unsigned long limit_pfn);
static void init_iova_rcaches(struct iova_domain *iovad);
static void free_iova_rcaches(struct iova_domain *iovad);
static void fq_destroy_all_entries(struct iova_domain *iovad);
static void fq_flush_timeout(unsigned long data);
void
init_iova_domain(struct iova_domain *iovad, unsigned long granule,
@ -50,10 +52,61 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
iovad->granule = granule;
iovad->start_pfn = start_pfn;
iovad->dma_32bit_pfn = pfn_32bit + 1;
iovad->flush_cb = NULL;
iovad->fq = NULL;
init_iova_rcaches(iovad);
}
EXPORT_SYMBOL_GPL(init_iova_domain);
static void free_iova_flush_queue(struct iova_domain *iovad)
{
if (!iovad->fq)
return;
if (timer_pending(&iovad->fq_timer))
del_timer(&iovad->fq_timer);
fq_destroy_all_entries(iovad);
free_percpu(iovad->fq);
iovad->fq = NULL;
iovad->flush_cb = NULL;
iovad->entry_dtor = NULL;
}
int init_iova_flush_queue(struct iova_domain *iovad,
iova_flush_cb flush_cb, iova_entry_dtor entry_dtor)
{
int cpu;
atomic64_set(&iovad->fq_flush_start_cnt, 0);
atomic64_set(&iovad->fq_flush_finish_cnt, 0);
iovad->fq = alloc_percpu(struct iova_fq);
if (!iovad->fq)
return -ENOMEM;
iovad->flush_cb = flush_cb;
iovad->entry_dtor = entry_dtor;
for_each_possible_cpu(cpu) {
struct iova_fq *fq;
fq = per_cpu_ptr(iovad->fq, cpu);
fq->head = 0;
fq->tail = 0;
spin_lock_init(&fq->lock);
}
setup_timer(&iovad->fq_timer, fq_flush_timeout, (unsigned long)iovad);
atomic_set(&iovad->fq_timer_on, 0);
return 0;
}
EXPORT_SYMBOL_GPL(init_iova_flush_queue);
static struct rb_node *
__get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
{
@ -423,6 +476,135 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
}
EXPORT_SYMBOL_GPL(free_iova_fast);
#define fq_ring_for_each(i, fq) \
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
static inline bool fq_full(struct iova_fq *fq)
{
assert_spin_locked(&fq->lock);
return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
}
static inline unsigned fq_ring_add(struct iova_fq *fq)
{
unsigned idx = fq->tail;
assert_spin_locked(&fq->lock);
fq->tail = (idx + 1) % IOVA_FQ_SIZE;
return idx;
}
static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
{
u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
unsigned idx;
assert_spin_locked(&fq->lock);
fq_ring_for_each(idx, fq) {
if (fq->entries[idx].counter >= counter)
break;
if (iovad->entry_dtor)
iovad->entry_dtor(fq->entries[idx].data);
free_iova_fast(iovad,
fq->entries[idx].iova_pfn,
fq->entries[idx].pages);
fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
}
}
static void iova_domain_flush(struct iova_domain *iovad)
{
atomic64_inc(&iovad->fq_flush_start_cnt);
iovad->flush_cb(iovad);
atomic64_inc(&iovad->fq_flush_finish_cnt);
}
static void fq_destroy_all_entries(struct iova_domain *iovad)
{
int cpu;
/*
* This code runs when the iova_domain is being detroyed, so don't
* bother to free iovas, just call the entry_dtor on all remaining
* entries.
*/
if (!iovad->entry_dtor)
return;
for_each_possible_cpu(cpu) {
struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
int idx;
fq_ring_for_each(idx, fq)
iovad->entry_dtor(fq->entries[idx].data);
}
}
static void fq_flush_timeout(unsigned long data)
{
struct iova_domain *iovad = (struct iova_domain *)data;
int cpu;
atomic_set(&iovad->fq_timer_on, 0);
iova_domain_flush(iovad);
for_each_possible_cpu(cpu) {
unsigned long flags;
struct iova_fq *fq;
fq = per_cpu_ptr(iovad->fq, cpu);
spin_lock_irqsave(&fq->lock, flags);
fq_ring_free(iovad, fq);
spin_unlock_irqrestore(&fq->lock, flags);
}
}
void queue_iova(struct iova_domain *iovad,
unsigned long pfn, unsigned long pages,
unsigned long data)
{
struct iova_fq *fq = get_cpu_ptr(iovad->fq);
unsigned long flags;
unsigned idx;
spin_lock_irqsave(&fq->lock, flags);
/*
* First remove all entries from the flush queue that have already been
* flushed out on another CPU. This makes the fq_full() check below less
* likely to be true.
*/
fq_ring_free(iovad, fq);
if (fq_full(fq)) {
iova_domain_flush(iovad);
fq_ring_free(iovad, fq);
}
idx = fq_ring_add(fq);
fq->entries[idx].iova_pfn = pfn;
fq->entries[idx].pages = pages;
fq->entries[idx].data = data;
fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
spin_unlock_irqrestore(&fq->lock, flags);
if (atomic_cmpxchg(&iovad->fq_timer_on, 0, 1) == 0)
mod_timer(&iovad->fq_timer,
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
put_cpu_ptr(iovad->fq);
}
EXPORT_SYMBOL_GPL(queue_iova);
/**
* put_iova_domain - destroys the iova doamin
* @iovad: - iova domain in question.
@ -433,6 +615,7 @@ void put_iova_domain(struct iova_domain *iovad)
struct rb_node *node;
unsigned long flags;
free_iova_flush_queue(iovad);
free_iova_rcaches(iovad);
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
node = rb_first(&iovad->rbroot);

242
drivers/iommu/ipmmu-vmsa.c
View File

@ -19,6 +19,7 @@
#include <linux/iommu.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/sizes.h>
#include <linux/slab.h>
@ -35,7 +36,7 @@
struct ipmmu_vmsa_device {
struct device *dev;
void __iomem *base;
struct list_head list;
struct iommu_device iommu;
unsigned int num_utlbs;
spinlock_t lock; /* Protects ctx and domains[] */
@ -58,36 +59,18 @@ struct ipmmu_vmsa_domain {
struct ipmmu_vmsa_iommu_priv {
struct ipmmu_vmsa_device *mmu;
unsigned int *utlbs;
unsigned int num_utlbs;
struct device *dev;
struct list_head list;
};
static DEFINE_SPINLOCK(ipmmu_devices_lock);
static LIST_HEAD(ipmmu_devices);
static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
{
return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
}
static struct ipmmu_vmsa_iommu_priv *to_priv(struct device *dev)
{
#if defined(CONFIG_ARM)
return dev->archdata.iommu;
#else
return dev->iommu_fwspec->iommu_priv;
#endif
}
static void set_priv(struct device *dev, struct ipmmu_vmsa_iommu_priv *p)
{
#if defined(CONFIG_ARM)
dev->archdata.iommu = p;
#else
dev->iommu_fwspec->iommu_priv = p;
#endif
return dev->iommu_fwspec ? dev->iommu_fwspec->iommu_priv : NULL;
}
#define TLB_LOOP_TIMEOUT 100 /* 100us */
@ -312,7 +295,7 @@ static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
/* The hardware doesn't support selective TLB flush. */
}
static struct iommu_gather_ops ipmmu_gather_ops = {
static const struct iommu_gather_ops ipmmu_gather_ops = {
.tlb_flush_all = ipmmu_tlb_flush_all,
.tlb_add_flush = ipmmu_tlb_add_flush,
.tlb_sync = ipmmu_tlb_flush_all,
@ -341,6 +324,19 @@ static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,
return ret;
}
static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
unsigned int context_id)
{
unsigned long flags;
spin_lock_irqsave(&mmu->lock, flags);
clear_bit(context_id, mmu->ctx);
mmu->domains[context_id] = NULL;
spin_unlock_irqrestore(&mmu->lock, flags);
}
static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
{
u64 ttbr;
@ -370,22 +366,22 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
*/
domain->cfg.iommu_dev = domain->mmu->dev;
domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
domain);
if (!domain->iop)
return -EINVAL;
/*
* Find an unused context.
*/
ret = ipmmu_domain_allocate_context(domain->mmu, domain);
if (ret == IPMMU_CTX_MAX) {
free_io_pgtable_ops(domain->iop);
if (ret == IPMMU_CTX_MAX)
return -EBUSY;
}
domain->context_id = ret;
domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
domain);
if (!domain->iop) {
ipmmu_domain_free_context(domain->mmu, domain->context_id);
return -EINVAL;
}
/* TTBR0 */
ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];
ipmmu_ctx_write(domain, IMTTLBR0, ttbr);
@ -426,19 +422,6 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
return 0;
}
static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
unsigned int context_id)
{
unsigned long flags;
spin_lock_irqsave(&mmu->lock, flags);
clear_bit(context_id, mmu->ctx);
mmu->domains[context_id] = NULL;
spin_unlock_irqrestore(&mmu->lock, flags);
}
static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
{
/*
@ -562,13 +545,14 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
struct device *dev)
{
struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct ipmmu_vmsa_device *mmu = priv->mmu;
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
unsigned long flags;
unsigned int i;
int ret = 0;
if (!mmu) {
if (!priv || !priv->mmu) {
dev_err(dev, "Cannot attach to IPMMU\n");
return -ENXIO;
}
@ -595,8 +579,8 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
if (ret < 0)
return ret;
for (i = 0; i < priv->num_utlbs; ++i)
ipmmu_utlb_enable(domain, priv->utlbs[i]);
for (i = 0; i < fwspec->num_ids; ++i)
ipmmu_utlb_enable(domain, fwspec->ids[i]);
return 0;
}
@ -604,12 +588,12 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain,
static void ipmmu_detach_device(struct iommu_domain *io_domain,
struct device *dev)
{
struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
unsigned int i;
for (i = 0; i < priv->num_utlbs; ++i)
ipmmu_utlb_disable(domain, priv->utlbs[i]);
for (i = 0; i < fwspec->num_ids; ++i)
ipmmu_utlb_disable(domain, fwspec->ids[i]);
/*
* TODO: Optimize by disabling the context when no device is attached.
@ -645,92 +629,36 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
return domain->iop->iova_to_phys(domain->iop, iova);
}
static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev,
unsigned int *utlbs, unsigned int num_utlbs)
static int ipmmu_init_platform_device(struct device *dev,
struct of_phandle_args *args)
{
unsigned int i;
struct platform_device *ipmmu_pdev;
struct ipmmu_vmsa_iommu_priv *priv;
for (i = 0; i < num_utlbs; ++i) {
struct of_phandle_args args;
int ret;
ipmmu_pdev = of_find_device_by_node(args->np);
if (!ipmmu_pdev)
return -ENODEV;
ret = of_parse_phandle_with_args(dev->of_node, "iommus",
"#iommu-cells", i, &args);
if (ret < 0)
return ret;
of_node_put(args.np);
if (args.np != mmu->dev->of_node || args.args_count != 1)
return -EINVAL;
utlbs[i] = args.args[0];
}
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
priv->mmu = platform_get_drvdata(ipmmu_pdev);
priv->dev = dev;
dev->iommu_fwspec->iommu_priv = priv;
return 0;
}
static int ipmmu_init_platform_device(struct device *dev)
static int ipmmu_of_xlate(struct device *dev,
struct of_phandle_args *spec)
{
struct ipmmu_vmsa_iommu_priv *priv;
struct ipmmu_vmsa_device *mmu;
unsigned int *utlbs;
unsigned int i;
int num_utlbs;
int ret = -ENODEV;
iommu_fwspec_add_ids(dev, spec->args, 1);
/* Find the master corresponding to the device. */
/* Initialize once - xlate() will call multiple times */
if (to_priv(dev))
return 0;
num_utlbs = of_count_phandle_with_args(dev->of_node, "iommus",
"#iommu-cells");
if (num_utlbs < 0)
return -ENODEV;
utlbs = kcalloc(num_utlbs, sizeof(*utlbs), GFP_KERNEL);
if (!utlbs)
return -ENOMEM;
spin_lock(&ipmmu_devices_lock);
list_for_each_entry(mmu, &ipmmu_devices, list) {
ret = ipmmu_find_utlbs(mmu, dev, utlbs, num_utlbs);
if (!ret) {
/*
* TODO Take a reference to the MMU to protect
* against device removal.
*/
break;
}
}
spin_unlock(&ipmmu_devices_lock);
if (ret < 0)
goto error;
for (i = 0; i < num_utlbs; ++i) {
if (utlbs[i] >= mmu->num_utlbs) {
ret = -EINVAL;
goto error;
}
}
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv) {
ret = -ENOMEM;
goto error;
}
priv->mmu = mmu;
priv->utlbs = utlbs;
priv->num_utlbs = num_utlbs;
priv->dev = dev;
set_priv(dev, priv);
return 0;
error:
kfree(utlbs);
return ret;
return ipmmu_init_platform_device(dev, spec);
}
#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
@ -749,11 +677,11 @@ static int ipmmu_add_device(struct device *dev)
struct iommu_group *group;
int ret;
if (to_priv(dev)) {
dev_warn(dev, "IOMMU driver already assigned to device %s\n",
dev_name(dev));
return -EINVAL;
}
/*
* Only let through devices that have been verified in xlate()
*/
if (!to_priv(dev))
return -ENODEV;
/* Create a device group and add the device to it. */
group = iommu_group_alloc();
@ -772,10 +700,6 @@ static int ipmmu_add_device(struct device *dev)
goto error;
}
ret = ipmmu_init_platform_device(dev);
if (ret < 0)
goto error;
/*
* Create the ARM mapping, used by the ARM DMA mapping core to allocate
* VAs. This will allocate a corresponding IOMMU domain.
@ -816,24 +740,13 @@ error:
if (!IS_ERR_OR_NULL(group))
iommu_group_remove_device(dev);
kfree(to_priv(dev)->utlbs);
kfree(to_priv(dev));
set_priv(dev, NULL);
return ret;
}
static void ipmmu_remove_device(struct device *dev)
{
struct ipmmu_vmsa_iommu_priv *priv = to_priv(dev);
arm_iommu_detach_device(dev);
iommu_group_remove_device(dev);
kfree(priv->utlbs);
kfree(priv);
set_priv(dev, NULL);
}
static const struct iommu_ops ipmmu_ops = {
@ -848,6 +761,7 @@ static const struct iommu_ops ipmmu_ops = {
.add_device = ipmmu_add_device,
.remove_device = ipmmu_remove_device,
.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
.of_xlate = ipmmu_of_xlate,
};
#endif /* !CONFIG_ARM && CONFIG_IOMMU_DMA */
@ -890,14 +804,12 @@ static void ipmmu_domain_free_dma(struct iommu_domain *io_domain)
static int ipmmu_add_device_dma(struct device *dev)
{
struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct iommu_group *group;
/*
* Only let through devices that have been verified in xlate()
* We may get called with dev->iommu_fwspec set to NULL.
*/
if (!fwspec || !fwspec->iommu_priv)
if (!to_priv(dev))
return -ENODEV;
group = iommu_group_get_for_dev(dev);
@ -957,19 +869,6 @@ static struct iommu_group *ipmmu_find_group_dma(struct device *dev)
return group;
}
static int ipmmu_of_xlate_dma(struct device *dev,
struct of_phandle_args *spec)
{
/* If the IPMMU device is disabled in DT then return error
* to make sure the of_iommu code does not install ops
* even though the iommu device is disabled
*/
if (!of_device_is_available(spec->np))
return -ENODEV;
return ipmmu_init_platform_device(dev);
}
static const struct iommu_ops ipmmu_ops = {
.domain_alloc = ipmmu_domain_alloc_dma,
.domain_free = ipmmu_domain_free_dma,
@ -983,7 +882,7 @@ static const struct iommu_ops ipmmu_ops = {
.remove_device = ipmmu_remove_device_dma,
.device_group = ipmmu_find_group_dma,
.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
.of_xlate = ipmmu_of_xlate_dma,
.of_xlate = ipmmu_of_xlate,
};
#endif /* CONFIG_IOMMU_DMA */
@ -1054,16 +953,24 @@ static int ipmmu_probe(struct platform_device *pdev)
ipmmu_device_reset(mmu);
ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,
dev_name(&pdev->dev));
if (ret)
return ret;
iommu_device_set_ops(&mmu->iommu, &ipmmu_ops);
iommu_device_set_fwnode(&mmu->iommu, &pdev->dev.of_node->fwnode);
ret = iommu_device_register(&mmu->iommu);
if (ret)
return ret;
/*
* We can't create the ARM mapping here as it requires the bus to have
* an IOMMU, which only happens when bus_set_iommu() is called in
* ipmmu_init() after the probe function returns.
*/
spin_lock(&ipmmu_devices_lock);
list_add(&mmu->list, &ipmmu_devices);
spin_unlock(&ipmmu_devices_lock);
platform_set_drvdata(pdev, mmu);
return 0;
@ -1073,9 +980,8 @@ static int ipmmu_remove(struct platform_device *pdev)
{
struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
spin_lock(&ipmmu_devices_lock);
list_del(&mmu->list);
spin_unlock(&ipmmu_devices_lock);
iommu_device_sysfs_remove(&mmu->iommu);
iommu_device_unregister(&mmu->iommu);
#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
arm_iommu_release_mapping(mmu->mapping);

15
drivers/iommu/msm_iommu.c
View File

@ -393,6 +393,7 @@ static struct msm_iommu_dev *find_iommu_for_dev(struct device *dev)
static int msm_iommu_add_device(struct device *dev)
{
struct msm_iommu_dev *iommu;
struct iommu_group *group;
unsigned long flags;
int ret = 0;
@ -406,7 +407,16 @@ static int msm_iommu_add_device(struct device *dev)
spin_unlock_irqrestore(&msm_iommu_lock, flags);
return ret;
if (ret)
return ret;
group = iommu_group_get_for_dev(dev);
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
return 0;
}
static void msm_iommu_remove_device(struct device *dev)
@ -421,6 +431,8 @@ static void msm_iommu_remove_device(struct device *dev)
iommu_device_unlink(&iommu->iommu, dev);
spin_unlock_irqrestore(&msm_iommu_lock, flags);
iommu_group_remove_device(dev);
}
static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
@ -700,6 +712,7 @@ static struct iommu_ops msm_iommu_ops = {
.iova_to_phys = msm_iommu_iova_to_phys,
.add_device = msm_iommu_add_device,
.remove_device = msm_iommu_remove_device,
.device_group = generic_device_group,
.pgsize_bitmap = MSM_IOMMU_PGSIZES,
.of_xlate = qcom_iommu_of_xlate,
};

214
drivers/iommu/mtk_iommu.c
View File

@ -31,7 +31,6 @@
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <asm/barrier.h>
#include <dt-bindings/memory/mt8173-larb-port.h>
#include <soc/mediatek/smi.h>
#include "mtk_iommu.h"
@ -54,10 +53,16 @@
#define REG_MMU_CTRL_REG 0x110
#define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5)
#define F_MMU_TF_PROTECT_SEL_SHIFT(data) \
((data)->m4u_plat == M4U_MT2712 ? 4 : 5)
/* It's named by F_MMU_TF_PROT_SEL in mt2712. */
#define F_MMU_TF_PROTECT_SEL(prot, data) \
(((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data))
#define REG_MMU_IVRP_PADDR 0x114
#define F_MMU_IVRP_PA_SET(pa, ext) (((pa) >> 1) | ((!!(ext)) << 31))
#define REG_MMU_VLD_PA_RNG 0x118
#define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA))
#define REG_MMU_INT_CONTROL0 0x120
#define F_L2_MULIT_HIT_EN BIT(0)
@ -82,7 +87,6 @@
#define REG_MMU_FAULT_ST1 0x134
#define REG_MMU_FAULT_VA 0x13c
#define F_MMU_FAULT_VA_MSK 0xfffff000
#define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
#define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
@ -93,6 +97,13 @@
#define MTK_PROTECT_PA_ALIGN 128
/*
* Get the local arbiter ID and the portid within the larb arbiter
* from mtk_m4u_id which is defined by MTK_M4U_ID.
*/
#define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf)
#define MTK_M4U_TO_PORT(id) ((id) & 0x1f)
struct mtk_iommu_domain {
spinlock_t pgtlock; /* lock for page table */
@ -104,6 +115,27 @@ struct mtk_iommu_domain {
static struct iommu_ops mtk_iommu_ops;
static LIST_HEAD(m4ulist); /* List all the M4U HWs */
#define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list)
/*
* There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
* for the performance.
*
* Here always return the mtk_iommu_data of the first probed M4U where the
* iommu domain information is recorded.
*/
static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void)
{
struct mtk_iommu_data *data;
for_each_m4u(data)
return data;
return NULL;
}
static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
{
return container_of(dom, struct mtk_iommu_domain, domain);
@ -113,9 +145,12 @@ static void mtk_iommu_tlb_flush_all(void *cookie)
{
struct mtk_iommu_data *data = cookie;
writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL);
writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
wmb(); /* Make sure the tlb flush all done */
for_each_m4u(data) {
writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
data->base + REG_MMU_INV_SEL);
writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
wmb(); /* Make sure the tlb flush all done */
}
}
static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
@ -124,12 +159,17 @@ static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
{
struct mtk_iommu_data *data = cookie;
writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL);
for_each_m4u(data) {
writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
data->base + REG_MMU_INV_SEL);
writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A);
writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE);
data->tlb_flush_active = true;
writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
writel_relaxed(iova + size - 1,
data->base + REG_MMU_INVLD_END_A);
writel_relaxed(F_MMU_INV_RANGE,
data->base + REG_MMU_INVALIDATE);
data->tlb_flush_active = true;
}
}
static void mtk_iommu_tlb_sync(void *cookie)
@ -138,20 +178,22 @@ static void mtk_iommu_tlb_sync(void *cookie)
int ret;
u32 tmp;
/* Avoid timing out if there's nothing to wait for */
if (!data->tlb_flush_active)
return;
for_each_m4u(data) {
/* Avoid timing out if there's nothing to wait for */
if (!data->tlb_flush_active)
return;
ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp,
tmp != 0, 10, 100000);
if (ret) {
dev_warn(data->dev,
"Partial TLB flush timed out, falling back to full flush\n");
mtk_iommu_tlb_flush_all(cookie);
ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
tmp, tmp != 0, 10, 100000);
if (ret) {
dev_warn(data->dev,
"Partial TLB flush timed out, falling back to full flush\n");
mtk_iommu_tlb_flush_all(cookie);
}
/* Clear the CPE status */
writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
data->tlb_flush_active = false;
}
/* Clear the CPE status */
writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
data->tlb_flush_active = false;
}
static const struct iommu_gather_ops mtk_iommu_gather_ops = {
@ -173,7 +215,6 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
fault_iova &= F_MMU_FAULT_VA_MSK;
fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
regval = readl_relaxed(data->base + REG_MMU_INT_ID);
fault_larb = F_MMU0_INT_ID_LARB_ID(regval);
@ -221,9 +262,9 @@ static void mtk_iommu_config(struct mtk_iommu_data *data,
}
}
static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
{
struct mtk_iommu_domain *dom = data->m4u_dom;
struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
spin_lock_init(&dom->pgtlock);
@ -249,9 +290,6 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
/* Update our support page sizes bitmap */
dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
data->base + REG_MMU_PT_BASE_ADDR);
return 0;
}
@ -266,20 +304,30 @@ static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
if (!dom)
return NULL;
if (iommu_get_dma_cookie(&dom->domain)) {
kfree(dom);
return NULL;
}
if (iommu_get_dma_cookie(&dom->domain))
goto free_dom;
if (mtk_iommu_domain_finalise(dom))
goto put_dma_cookie;
dom->domain.geometry.aperture_start = 0;
dom->domain.geometry.aperture_end = DMA_BIT_MASK(32);
dom->domain.geometry.force_aperture = true;
return &dom->domain;
put_dma_cookie:
iommu_put_dma_cookie(&dom->domain);
free_dom:
kfree(dom);
return NULL;
}
static void mtk_iommu_domain_free(struct iommu_domain *domain)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
free_io_pgtable_ops(dom->iop);
iommu_put_dma_cookie(domain);
kfree(to_mtk_domain(domain));
}
@ -289,22 +337,15 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
int ret;
if (!data)
return -ENODEV;
/* Update the pgtable base address register of the M4U HW */
if (!data->m4u_dom) {
data->m4u_dom = dom;
ret = mtk_iommu_domain_finalise(data);
if (ret) {
data->m4u_dom = NULL;
return ret;
}
} else if (data->m4u_dom != dom) {
/* All the client devices should be in the same m4u domain */
dev_err(dev, "try to attach into the error iommu domain\n");
return -EPERM;
writel(dom->cfg.arm_v7s_cfg.ttbr[0],
data->base + REG_MMU_PT_BASE_ADDR);
}
mtk_iommu_config(data, dev, true);
@ -354,6 +395,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
unsigned long flags;
phys_addr_t pa;
@ -361,6 +403,9 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
pa = dom->iop->iova_to_phys(dom->iop, iova);
spin_unlock_irqrestore(&dom->pgtlock, flags);
if (data->enable_4GB)
pa |= BIT_ULL(32);
return pa;
}
@ -399,7 +444,7 @@ static void mtk_iommu_remove_device(struct device *dev)
static struct iommu_group *mtk_iommu_device_group(struct device *dev)
{
struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv;
struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
if (!data)
return ERR_PTR(-ENODEV);
@ -464,8 +509,9 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
return ret;
}
regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
F_MMU_TF_PROTECT_SEL(2);
regval = F_MMU_TF_PROTECT_SEL(2, data);
if (data->m4u_plat == M4U_MT8173)
regval |= F_MMU_PREFETCH_RT_REPLACE_MOD;
writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
regval = F_L2_MULIT_HIT_EN |
@ -487,9 +533,19 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB),
data->base + REG_MMU_IVRP_PADDR);
if (data->enable_4GB && data->m4u_plat != M4U_MT8173) {
/*
* If 4GB mode is enabled, the validate PA range is from
* 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
*/
regval = F_MMU_VLD_PA_RNG(7, 4);
writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG);
}
writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
/* It's MISC control register whose default value is ok except mt8173.*/
if (data->m4u_plat == M4U_MT8173)
writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
dev_name(data->dev), (void *)data)) {
@ -521,6 +577,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (!data)
return -ENOMEM;
data->dev = dev;
data->m4u_plat = (enum mtk_iommu_plat)of_device_get_match_data(dev);
/* Protect memory. HW will access here while translation fault.*/
protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
@ -529,7 +586,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
/* Whether the current dram is over 4GB */
data->enable_4GB = !!(max_pfn > (0xffffffffUL >> PAGE_SHIFT));
data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
data->base = devm_ioremap_resource(dev, res);
@ -554,6 +611,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
for (i = 0; i < larb_nr; i++) {
struct device_node *larbnode;
struct platform_device *plarbdev;
u32 id;
larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
if (!larbnode)
@ -562,17 +620,14 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (!of_device_is_available(larbnode))
continue;
ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
if (ret)/* The id is consecutive if there is no this property */
id = i;
plarbdev = of_find_device_by_node(larbnode);
if (!plarbdev) {
plarbdev = of_platform_device_create(
larbnode, NULL,
platform_bus_type.dev_root);
if (!plarbdev) {
of_node_put(larbnode);
return -EPROBE_DEFER;
}
}
data->smi_imu.larb_imu[i].dev = &plarbdev->dev;
if (!plarbdev)
return -EPROBE_DEFER;
data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
component_match_add_release(dev, &match, release_of,
compare_of, larbnode);
@ -596,6 +651,8 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (ret)
return ret;
list_add_tail(&data->list, &m4ulist);
if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
@ -612,7 +669,6 @@ static int mtk_iommu_remove(struct platform_device *pdev)
if (iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, NULL);
free_io_pgtable_ops(data->m4u_dom->iop);
clk_disable_unprepare(data->bclk);
devm_free_irq(&pdev->dev, data->irq, data);
component_master_del(&pdev->dev, &mtk_iommu_com_ops);
@ -631,6 +687,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev)
reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
clk_disable_unprepare(data->bclk);
return 0;
}
@ -639,9 +696,13 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
struct mtk_iommu_data *data = dev_get_drvdata(dev);
struct mtk_iommu_suspend_reg *reg = &data->reg;
void __iomem *base = data->base;
int ret;
writel_relaxed(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
base + REG_MMU_PT_BASE_ADDR);
ret = clk_prepare_enable(data->bclk);
if (ret) {
dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
return ret;
}
writel_relaxed(reg->standard_axi_mode,
base + REG_MMU_STANDARD_AXI_MODE);
writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
@ -650,15 +711,19 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base, data->enable_4GB),
base + REG_MMU_IVRP_PADDR);
if (data->m4u_dom)
writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
base + REG_MMU_PT_BASE_ADDR);
return 0;
}
const struct dev_pm_ops mtk_iommu_pm_ops = {
SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
static const struct dev_pm_ops mtk_iommu_pm_ops = {
SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
};
static const struct of_device_id mtk_iommu_of_ids[] = {
{ .compatible = "mediatek,mt8173-m4u", },
{ .compatible = "mediatek,mt2712-m4u", .data = (void *)M4U_MT2712},
{ .compatible = "mediatek,mt8173-m4u", .data = (void *)M4U_MT8173},
{}
};
@ -667,27 +732,20 @@ static struct platform_driver mtk_iommu_driver = {
.remove = mtk_iommu_remove,
.driver = {
.name = "mtk-iommu",
.of_match_table = mtk_iommu_of_ids,
.of_match_table = of_match_ptr(mtk_iommu_of_ids),
.pm = &mtk_iommu_pm_ops,
}
};
static int mtk_iommu_init_fn(struct device_node *np)
static int __init mtk_iommu_init(void)
{
int ret;
struct platform_device *pdev;
pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
if (!pdev)
return -ENOMEM;
ret = platform_driver_register(&mtk_iommu_driver);
if (ret) {
pr_err("%s: Failed to register driver\n", __func__);
return ret;
}
if (ret != 0)
pr_err("Failed to register MTK IOMMU driver\n");
return 0;
return ret;
}
IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn);
subsys_initcall(mtk_iommu_init)

9
drivers/iommu/mtk_iommu.h
View File

@ -34,6 +34,12 @@ struct mtk_iommu_suspend_reg {
u32 int_main_control;
};
enum mtk_iommu_plat {
M4U_MT2701,
M4U_MT2712,
M4U_MT8173,
};
struct mtk_iommu_domain;
struct mtk_iommu_data {
@ -50,6 +56,9 @@ struct mtk_iommu_data {
bool tlb_flush_active;
struct iommu_device iommu;
enum mtk_iommu_plat m4u_plat;
struct list_head list;
};
static inline int compare_of(struct device *dev, void *data)

144
drivers/iommu/of_iommu.c
View File

@ -25,6 +25,8 @@
#include <linux/of_pci.h>
#include <linux/slab.h>
#define NO_IOMMU 1
static const struct of_device_id __iommu_of_table_sentinel
__used __section(__iommu_of_table_end);
@ -109,8 +111,8 @@ static bool of_iommu_driver_present(struct device_node *np)
return of_match_node(&__iommu_of_table, np);
}
static const struct iommu_ops
*of_iommu_xlate(struct device *dev, struct of_phandle_args *iommu_spec)
static int of_iommu_xlate(struct device *dev,
struct of_phandle_args *iommu_spec)
{
const struct iommu_ops *ops;
struct fwnode_handle *fwnode = &iommu_spec->np->fwnode;
@ -120,95 +122,53 @@ static const struct iommu_ops
if ((ops && !ops->of_xlate) ||
!of_device_is_available(iommu_spec->np) ||
(!ops && !of_iommu_driver_present(iommu_spec->np)))
return NULL;
return NO_IOMMU;
err = iommu_fwspec_init(dev, &iommu_spec->np->fwnode, ops);
if (err)
return ERR_PTR(err);
return err;
/*
* The otherwise-empty fwspec handily serves to indicate the specific
* IOMMU device we're waiting for, which will be useful if we ever get
* a proper probe-ordering dependency mechanism in future.
*/
if (!ops)
return ERR_PTR(-EPROBE_DEFER);
return -EPROBE_DEFER;
err = ops->of_xlate(dev, iommu_spec);
if (err)
return ERR_PTR(err);
return ops;
return ops->of_xlate(dev, iommu_spec);
}
static int __get_pci_rid(struct pci_dev *pdev, u16 alias, void *data)
{
struct of_phandle_args *iommu_spec = data;
struct of_pci_iommu_alias_info {
struct device *dev;
struct device_node *np;
};
iommu_spec->args[0] = alias;
return iommu_spec->np == pdev->bus->dev.of_node;
}
static const struct iommu_ops
*of_pci_iommu_init(struct pci_dev *pdev, struct device_node *bridge_np)
static int of_pci_iommu_init(struct pci_dev *pdev, u16 alias, void *data)
{
const struct iommu_ops *ops;
struct of_phandle_args iommu_spec;
struct of_pci_iommu_alias_info *info = data;
struct of_phandle_args iommu_spec = { .args_count = 1 };
int err;
/*
* Start by tracing the RID alias down the PCI topology as
* far as the host bridge whose OF node we have...
* (we're not even attempting to handle multi-alias devices yet)
*/
iommu_spec.args_count = 1;
iommu_spec.np = bridge_np;
pci_for_each_dma_alias(pdev, __get_pci_rid, &iommu_spec);
/*
* ...then find out what that becomes once it escapes the PCI
* bus into the system beyond, and which IOMMU it ends up at.
*/
iommu_spec.np = NULL;
err = of_pci_map_rid(bridge_np, iommu_spec.args[0], "iommu-map",
err = of_pci_map_rid(info->np, alias, "iommu-map",
"iommu-map-mask", &iommu_spec.np,
iommu_spec.args);
if (err)
return err == -ENODEV ? NULL : ERR_PTR(err);
ops = of_iommu_xlate(&pdev->dev, &iommu_spec);
return err == -ENODEV ? NO_IOMMU : err;
err = of_iommu_xlate(info->dev, &iommu_spec);
of_node_put(iommu_spec.np);
return ops;
}
if (err)
return err;
static const struct iommu_ops
*of_platform_iommu_init(struct device *dev, struct device_node *np)
{
struct of_phandle_args iommu_spec;
const struct iommu_ops *ops = NULL;
int idx = 0;
/*
* We don't currently walk up the tree looking for a parent IOMMU.
* See the `Notes:' section of
* Documentation/devicetree/bindings/iommu/iommu.txt
*/
while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells",
idx, &iommu_spec)) {
ops = of_iommu_xlate(dev, &iommu_spec);
of_node_put(iommu_spec.np);
idx++;
if (IS_ERR_OR_NULL(ops))
break;
}
return ops;
return info->np == pdev->bus->dev.of_node;
}
const struct iommu_ops *of_iommu_configure(struct device *dev,
struct device_node *master_np)
{
const struct iommu_ops *ops;
const struct iommu_ops *ops = NULL;
struct iommu_fwspec *fwspec = dev->iommu_fwspec;
int err = NO_IOMMU;
if (!master_np)
return NULL;
@ -221,25 +181,54 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
iommu_fwspec_free(dev);
}
if (dev_is_pci(dev))
ops = of_pci_iommu_init(to_pci_dev(dev), master_np);
else
ops = of_platform_iommu_init(dev, master_np);
/*
* We don't currently walk up the tree looking for a parent IOMMU.
* See the `Notes:' section of
* Documentation/devicetree/bindings/iommu/iommu.txt
*/
if (dev_is_pci(dev)) {
struct of_pci_iommu_alias_info info = {
.dev = dev,
.np = master_np,
};
err = pci_for_each_dma_alias(to_pci_dev(dev),
of_pci_iommu_init, &info);
} else {
struct of_phandle_args iommu_spec;
int idx = 0;
while (!of_parse_phandle_with_args(master_np, "iommus",
"#iommu-cells",
idx, &iommu_spec)) {
err = of_iommu_xlate(dev, &iommu_spec);
of_node_put(iommu_spec.np);
idx++;
if (err)
break;
}
}
/*
* Two success conditions can be represented by non-negative err here:
* >0 : there is no IOMMU, or one was unavailable for non-fatal reasons
* 0 : we found an IOMMU, and dev->fwspec is initialised appropriately
* <0 : any actual error
*/
if (!err)
ops = dev->iommu_fwspec->ops;
/*
* If we have reason to believe the IOMMU driver missed the initial
* add_device callback for dev, replay it to get things in order.
*/
if (!IS_ERR_OR_NULL(ops) && ops->add_device &&
dev->bus && !dev->iommu_group) {
int err = ops->add_device(dev);
if (err)
ops = ERR_PTR(err);
}
if (ops && ops->add_device && dev->bus && !dev->iommu_group)
err = ops->add_device(dev);
/* Ignore all other errors apart from EPROBE_DEFER */
if (IS_ERR(ops) && (PTR_ERR(ops) != -EPROBE_DEFER)) {
dev_dbg(dev, "Adding to IOMMU failed: %ld\n", PTR_ERR(ops));
if (err == -EPROBE_DEFER) {
ops = ERR_PTR(err);
} else if (err < 0) {
dev_dbg(dev, "Adding to IOMMU failed: %d\n", err);
ops = NULL;
}
@ -255,8 +244,7 @@ static int __init of_iommu_init(void)
const of_iommu_init_fn init_fn = match->data;
if (init_fn && init_fn(np))
pr_err("Failed to initialise IOMMU %s\n",
of_node_full_name(np));
pr_err("Failed to initialise IOMMU %pOF\n", np);
}
return 0;

127
drivers/iommu/omap-iommu.c
View File

@ -11,6 +11,7 @@
* published by the Free Software Foundation.
*/
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
@ -29,8 +30,6 @@
#include <linux/regmap.h>
#include <linux/mfd/syscon.h>
#include <asm/cacheflush.h>
#include <linux/platform_data/iommu-omap.h>
#include "omap-iopgtable.h"
@ -454,36 +453,35 @@ static void flush_iotlb_all(struct omap_iommu *obj)
/*
* H/W pagetable operations
*/
static void flush_iopgd_range(u32 *first, u32 *last)
static void flush_iopte_range(struct device *dev, dma_addr_t dma,
unsigned long offset, int num_entries)
{
/* FIXME: L2 cache should be taken care of if it exists */
do {
asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
: : "r" (first));
first += L1_CACHE_BYTES / sizeof(*first);
} while (first <= last);
size_t size = num_entries * sizeof(u32);
dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE);
}
static void flush_iopte_range(u32 *first, u32 *last)
static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid)
{
/* FIXME: L2 cache should be taken care of if it exists */
do {
asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
: : "r" (first));
first += L1_CACHE_BYTES / sizeof(*first);
} while (first <= last);
}
dma_addr_t pt_dma;
static void iopte_free(u32 *iopte)
{
/* Note: freed iopte's must be clean ready for re-use */
if (iopte)
if (iopte) {
if (dma_valid) {
pt_dma = virt_to_phys(iopte);
dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE,
DMA_TO_DEVICE);
}
kmem_cache_free(iopte_cachep, iopte);
}
}
static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd,
dma_addr_t *pt_dma, u32 da)
{
u32 *iopte;
unsigned long offset = iopgd_index(da) * sizeof(da);
/* a table has already existed */
if (*iopgd)
@ -500,18 +498,38 @@ static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
if (!iopte)
return ERR_PTR(-ENOMEM);
*iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
flush_iopgd_range(iopgd, iopgd);
*pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE,
DMA_TO_DEVICE);
if (dma_mapping_error(obj->dev, *pt_dma)) {
dev_err(obj->dev, "DMA map error for L2 table\n");
iopte_free(obj, iopte, false);
return ERR_PTR(-ENOMEM);
}
/*
* we rely on dma address and the physical address to be
* the same for mapping the L2 table
*/
if (WARN_ON(*pt_dma != virt_to_phys(iopte))) {
dev_err(obj->dev, "DMA translation error for L2 table\n");
dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE,
DMA_TO_DEVICE);
iopte_free(obj, iopte, false);
return ERR_PTR(-ENOMEM);
}
*iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
} else {
/* We raced, free the reduniovant table */
iopte_free(iopte);
iopte_free(obj, iopte, false);
}
pte_ready:
iopte = iopte_offset(iopgd, da);
*pt_dma = virt_to_phys(iopte);
dev_vdbg(obj->dev,
"%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
__func__, da, iopgd, *iopgd, iopte, *iopte);
@ -522,6 +540,7 @@ pte_ready:
static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
unsigned long offset = iopgd_index(da) * sizeof(da);
if ((da | pa) & ~IOSECTION_MASK) {
dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
@ -530,13 +549,14 @@ static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
}
*iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
flush_iopgd_range(iopgd, iopgd);
flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
return 0;
}
static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
unsigned long offset = iopgd_index(da) * sizeof(da);
int i;
if ((da | pa) & ~IOSUPER_MASK) {
@ -547,20 +567,22 @@ static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
for (i = 0; i < 16; i++)
*(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
flush_iopgd_range(iopgd, iopgd + 15);
flush_iopte_range(obj->dev, obj->pd_dma, offset, 16);
return 0;
}
static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
u32 *iopte = iopte_alloc(obj, iopgd, da);
dma_addr_t pt_dma;
u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
unsigned long offset = iopte_index(da) * sizeof(da);
if (IS_ERR(iopte))
return PTR_ERR(iopte);
*iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
flush_iopte_range(iopte, iopte);
flush_iopte_range(obj->dev, pt_dma, offset, 1);
dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
__func__, da, pa, iopte, *iopte);
@ -571,7 +593,9 @@ static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
{
u32 *iopgd = iopgd_offset(obj, da);
u32 *iopte = iopte_alloc(obj, iopgd, da);
dma_addr_t pt_dma;
u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
unsigned long offset = iopte_index(da) * sizeof(da);
int i;
if ((da | pa) & ~IOLARGE_MASK) {
@ -585,7 +609,7 @@ static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
for (i = 0; i < 16; i++)
*(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
flush_iopte_range(iopte, iopte + 15);
flush_iopte_range(obj->dev, pt_dma, offset, 16);
return 0;
}
@ -674,6 +698,9 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
size_t bytes;
u32 *iopgd = iopgd_offset(obj, da);
int nent = 1;
dma_addr_t pt_dma;
unsigned long pd_offset = iopgd_index(da) * sizeof(da);
unsigned long pt_offset = iopte_index(da) * sizeof(da);
if (!*iopgd)
return 0;
@ -690,7 +717,8 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
}
bytes *= nent;
memset(iopte, 0, nent * sizeof(*iopte));
flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
pt_dma = virt_to_phys(iopte);
flush_iopte_range(obj->dev, pt_dma, pt_offset, nent);
/*
* do table walk to check if this table is necessary or not
@ -700,7 +728,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
if (iopte[i])
goto out;
iopte_free(iopte);
iopte_free(obj, iopte, true);
nent = 1; /* for the next L1 entry */
} else {
bytes = IOPGD_SIZE;
@ -712,7 +740,7 @@ static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
bytes *= nent;
}
memset(iopgd, 0, nent * sizeof(*iopgd));
flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent);
out:
return bytes;
}
@ -738,6 +766,7 @@ static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
static void iopgtable_clear_entry_all(struct omap_iommu *obj)
{
unsigned long offset;
int i;
spin_lock(&obj->page_table_lock);
@ -748,15 +777,16 @@ static void iopgtable_clear_entry_all(struct omap_iommu *obj)
da = i << IOPGD_SHIFT;
iopgd = iopgd_offset(obj, da);
offset = iopgd_index(da) * sizeof(da);
if (!*iopgd)
continue;
if (iopgd_is_table(*iopgd))
iopte_free(iopte_offset(iopgd, 0));
iopte_free(obj, iopte_offset(iopgd, 0), true);
*iopgd = 0;
flush_iopgd_range(iopgd, iopgd);
flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
}
flush_iotlb_all(obj);
@ -786,7 +816,7 @@ static irqreturn_t iommu_fault_handler(int irq, void *data)
if (!report_iommu_fault(domain, obj->dev, da, 0))
return IRQ_HANDLED;
iommu_disable(obj);
iommu_write_reg(obj, 0, MMU_IRQENABLE);
iopgd = iopgd_offset(obj, da);
@ -815,10 +845,18 @@ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
spin_lock(&obj->iommu_lock);
obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE,
DMA_TO_DEVICE);
if (dma_mapping_error(obj->dev, obj->pd_dma)) {
dev_err(obj->dev, "DMA map error for L1 table\n");
err = -ENOMEM;
goto out_err;
}
obj->iopgd = iopgd;
err = iommu_enable(obj);
if (err)
goto err_enable;
goto out_err;
flush_iotlb_all(obj);
spin_unlock(&obj->iommu_lock);
@ -827,7 +865,7 @@ static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
return 0;
err_enable:
out_err:
spin_unlock(&obj->iommu_lock);
return err;
@ -844,7 +882,10 @@ static void omap_iommu_detach(struct omap_iommu *obj)
spin_lock(&obj->iommu_lock);
dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
DMA_TO_DEVICE);
iommu_disable(obj);
obj->pd_dma = 0;
obj->iopgd = NULL;
spin_unlock(&obj->iommu_lock);
@ -1008,11 +1049,6 @@ static struct platform_driver omap_iommu_driver = {
},
};
static void iopte_cachep_ctor(void *iopte)
{
clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
}
static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
{
memset(e, 0, sizeof(*e));
@ -1159,7 +1195,6 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
if (WARN_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE)))
goto fail_align;
clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
spin_lock_init(&omap_domain->lock);
omap_domain->domain.geometry.aperture_start = 0;
@ -1347,7 +1382,7 @@ static int __init omap_iommu_init(void)
of_node_put(np);
p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
iopte_cachep_ctor);
NULL);
if (!p)
return -ENOMEM;
iopte_cachep = p;

1
drivers/iommu/omap-iommu.h
View File

@ -61,6 +61,7 @@ struct omap_iommu {
*/
u32 *iopgd;
spinlock_t page_table_lock; /* protect iopgd */
dma_addr_t pd_dma;
int nr_tlb_entries;

930
drivers/iommu/qcom_iommu.c Normal file
View File

@ -0,0 +1,930 @@
/*
* IOMMU API for QCOM secure IOMMUs. Somewhat based on arm-smmu.c
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* Copyright (C) 2013 ARM Limited
* Copyright (C) 2017 Red Hat
*/
#include <linux/atomic.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/dma-iommu.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/iommu.h>
#include <linux/iopoll.h>
#include <linux/kconfig.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_device.h>
#include <linux/of_iommu.h>
#include <linux/platform_device.h>
#include <linux/pm.h>
#include <linux/pm_runtime.h>
#include <linux/qcom_scm.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "io-pgtable.h"
#include "arm-smmu-regs.h"
#define SMMU_INTR_SEL_NS 0x2000
struct qcom_iommu_ctx;
struct qcom_iommu_dev {
/* IOMMU core code handle */
struct iommu_device iommu;
struct device *dev;
struct clk *iface_clk;
struct clk *bus_clk;
void __iomem *local_base;
u32 sec_id;
u8 num_ctxs;
struct qcom_iommu_ctx *ctxs[0]; /* indexed by asid-1 */
};
struct qcom_iommu_ctx {
struct device *dev;
void __iomem *base;
bool secure_init;
u8 asid; /* asid and ctx bank # are 1:1 */
};
struct qcom_iommu_domain {
struct io_pgtable_ops *pgtbl_ops;
spinlock_t pgtbl_lock;
struct mutex init_mutex; /* Protects iommu pointer */
struct iommu_domain domain;
struct qcom_iommu_dev *iommu;
};
static struct qcom_iommu_domain *to_qcom_iommu_domain(struct iommu_domain *dom)
{
return container_of(dom, struct qcom_iommu_domain, domain);
}
static const struct iommu_ops qcom_iommu_ops;
static struct qcom_iommu_dev * to_iommu(struct iommu_fwspec *fwspec)
{
if (!fwspec || fwspec->ops != &qcom_iommu_ops)
return NULL;
return fwspec->iommu_priv;
}
static struct qcom_iommu_ctx * to_ctx(struct iommu_fwspec *fwspec, unsigned asid)
{
struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
if (!qcom_iommu)
return NULL;
return qcom_iommu->ctxs[asid - 1];
}
static inline void
iommu_writel(struct qcom_iommu_ctx *ctx, unsigned reg, u32 val)
{
writel_relaxed(val, ctx->base + reg);
}
static inline void
iommu_writeq(struct qcom_iommu_ctx *ctx, unsigned reg, u64 val)
{
writeq_relaxed(val, ctx->base + reg);
}
static inline u32
iommu_readl(struct qcom_iommu_ctx *ctx, unsigned reg)
{
return readl_relaxed(ctx->base + reg);
}
static inline u64
iommu_readq(struct qcom_iommu_ctx *ctx, unsigned reg)
{
return readq_relaxed(ctx->base + reg);
}
static void qcom_iommu_tlb_sync(void *cookie)
{
struct iommu_fwspec *fwspec = cookie;
unsigned i;
for (i = 0; i < fwspec->num_ids; i++) {
struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
unsigned int val, ret;
iommu_writel(ctx, ARM_SMMU_CB_TLBSYNC, 0);
ret = readl_poll_timeout(ctx->base + ARM_SMMU_CB_TLBSTATUS, val,
(val & 0x1) == 0, 0, 5000000);
if (ret)
dev_err(ctx->dev, "timeout waiting for TLB SYNC\n");
}
}
static void qcom_iommu_tlb_inv_context(void *cookie)
{
struct iommu_fwspec *fwspec = cookie;
unsigned i;
for (i = 0; i < fwspec->num_ids; i++) {
struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
iommu_writel(ctx, ARM_SMMU_CB_S1_TLBIASID, ctx->asid);
}
qcom_iommu_tlb_sync(cookie);
}
static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
size_t granule, bool leaf, void *cookie)
{
struct iommu_fwspec *fwspec = cookie;
unsigned i, reg;
reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
for (i = 0; i < fwspec->num_ids; i++) {
struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
size_t s = size;
iova &= ~12UL;
iova |= ctx->asid;
do {
iommu_writel(ctx, reg, iova);
iova += granule;
} while (s -= granule);
}
}
static const struct iommu_gather_ops qcom_gather_ops = {
.tlb_flush_all = qcom_iommu_tlb_inv_context,
.tlb_add_flush = qcom_iommu_tlb_inv_range_nosync,
.tlb_sync = qcom_iommu_tlb_sync,
};
static irqreturn_t qcom_iommu_fault(int irq, void *dev)
{
struct qcom_iommu_ctx *ctx = dev;
u32 fsr, fsynr;
u64 iova;
fsr = iommu_readl(ctx, ARM_SMMU_CB_FSR);
if (!(fsr & FSR_FAULT))
return IRQ_NONE;
fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0);
iova = iommu_readq(ctx, ARM_SMMU_CB_FAR);
dev_err_ratelimited(ctx->dev,
"Unhandled context fault: fsr=0x%x, "
"iova=0x%016llx, fsynr=0x%x, cb=%d\n",
fsr, iova, fsynr, ctx->asid);
iommu_writel(ctx, ARM_SMMU_CB_FSR, fsr);
return IRQ_HANDLED;
}
static int qcom_iommu_init_domain(struct iommu_domain *domain,
struct qcom_iommu_dev *qcom_iommu,
struct iommu_fwspec *fwspec)
{
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
struct io_pgtable_ops *pgtbl_ops;
struct io_pgtable_cfg pgtbl_cfg;
int i, ret = 0;
u32 reg;
mutex_lock(&qcom_domain->init_mutex);
if (qcom_domain->iommu)
goto out_unlock;
pgtbl_cfg = (struct io_pgtable_cfg) {
.pgsize_bitmap = qcom_iommu_ops.pgsize_bitmap,
.ias = 32,
.oas = 40,
.tlb = &qcom_gather_ops,
.iommu_dev = qcom_iommu->dev,
};
qcom_domain->iommu = qcom_iommu;
pgtbl_ops = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &pgtbl_cfg, fwspec);
if (!pgtbl_ops) {
dev_err(qcom_iommu->dev, "failed to allocate pagetable ops\n");
ret = -ENOMEM;
goto out_clear_iommu;
}
/* Update the domain's page sizes to reflect the page table format */
domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
domain->geometry.aperture_end = (1ULL << pgtbl_cfg.ias) - 1;
domain->geometry.force_aperture = true;
for (i = 0; i < fwspec->num_ids; i++) {
struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
if (!ctx->secure_init) {
ret = qcom_scm_restore_sec_cfg(qcom_iommu->sec_id, ctx->asid);
if (ret) {
dev_err(qcom_iommu->dev, "secure init failed: %d\n", ret);
goto out_clear_iommu;
}
ctx->secure_init = true;
}
/* TTBRs */
iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
((u64)ctx->asid << TTBRn_ASID_SHIFT));
iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
((u64)ctx->asid << TTBRn_ASID_SHIFT));
/* TTBCR */
iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
(pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
TTBCR2_SEP_UPSTREAM);
iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
pgtbl_cfg.arm_lpae_s1_cfg.tcr);
/* MAIRs (stage-1 only) */
iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR0,
pgtbl_cfg.arm_lpae_s1_cfg.mair[0]);
iommu_writel(ctx, ARM_SMMU_CB_S1_MAIR1,
pgtbl_cfg.arm_lpae_s1_cfg.mair[1]);
/* SCTLR */
reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE |
SCTLR_M | SCTLR_S1_ASIDPNE;
if (IS_ENABLED(CONFIG_BIG_ENDIAN))
reg |= SCTLR_E;
iommu_writel(ctx, ARM_SMMU_CB_SCTLR, reg);
}
mutex_unlock(&qcom_domain->init_mutex);
/* Publish page table ops for map/unmap */
qcom_domain->pgtbl_ops = pgtbl_ops;
return 0;
out_clear_iommu:
qcom_domain->iommu = NULL;
out_unlock:
mutex_unlock(&qcom_domain->init_mutex);
return ret;
}
static struct iommu_domain *qcom_iommu_domain_alloc(unsigned type)
{
struct qcom_iommu_domain *qcom_domain;
if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
return NULL;
/*
* Allocate the domain and initialise some of its data structures.
* We can't really do anything meaningful until we've added a
* master.
*/
qcom_domain = kzalloc(sizeof(*qcom_domain), GFP_KERNEL);
if (!qcom_domain)
return NULL;
if (type == IOMMU_DOMAIN_DMA &&
iommu_get_dma_cookie(&qcom_domain->domain)) {
kfree(qcom_domain);
return NULL;
}
mutex_init(&qcom_domain->init_mutex);
spin_lock_init(&qcom_domain->pgtbl_lock);
return &qcom_domain->domain;
}
static void qcom_iommu_domain_free(struct iommu_domain *domain)
{
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
if (WARN_ON(qcom_domain->iommu)) /* forgot to detach? */
return;
iommu_put_dma_cookie(domain);
/* NOTE: unmap can be called after client device is powered off,
* for example, with GPUs or anything involving dma-buf. So we
* cannot rely on the device_link. Make sure the IOMMU is on to
* avoid unclocked accesses in the TLB inv path:
*/
pm_runtime_get_sync(qcom_domain->iommu->dev);
free_io_pgtable_ops(qcom_domain->pgtbl_ops);
pm_runtime_put_sync(qcom_domain->iommu->dev);
kfree(qcom_domain);
}
static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
{
struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
int ret;
if (!qcom_iommu) {
dev_err(dev, "cannot attach to IOMMU, is it on the same bus?\n");
return -ENXIO;
}
/* Ensure that the domain is finalized */
pm_runtime_get_sync(qcom_iommu->dev);
ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec);
pm_runtime_put_sync(qcom_iommu->dev);
if (ret < 0)
return ret;
/*
* Sanity check the domain. We don't support domains across
* different IOMMUs.
*/
if (qcom_domain->iommu != qcom_iommu) {
dev_err(dev, "cannot attach to IOMMU %s while already "
"attached to domain on IOMMU %s\n",
dev_name(qcom_domain->iommu->dev),
dev_name(qcom_iommu->dev));
return -EINVAL;
}
return 0;
}
static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev)
{
struct iommu_fwspec *fwspec = dev->iommu_fwspec;
struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec);
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
unsigned i;
if (!qcom_domain->iommu)
return;
pm_runtime_get_sync(qcom_iommu->dev);
for (i = 0; i < fwspec->num_ids; i++) {
struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
/* Disable the context bank: */
iommu_writel(ctx, ARM_SMMU_CB_SCTLR, 0);
}
pm_runtime_put_sync(qcom_iommu->dev);
qcom_domain->iommu = NULL;
}
static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
int ret;
unsigned long flags;
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
if (!ops)
return -ENODEV;
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->map(ops, iova, paddr, size, prot);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
return ret;
}
static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size)
{
size_t ret;
unsigned long flags;
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
if (!ops)
return 0;
/* NOTE: unmap can be called after client device is powered off,
* for example, with GPUs or anything involving dma-buf. So we
* cannot rely on the device_link. Make sure the IOMMU is on to
* avoid unclocked accesses in the TLB inv path:
*/
pm_runtime_get_sync(qcom_domain->iommu->dev);
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->unmap(ops, iova, size);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
pm_runtime_put_sync(qcom_domain->iommu->dev);
return ret;
}
static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
phys_addr_t ret;
unsigned long flags;
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
struct io_pgtable_ops *ops = qcom_domain->pgtbl_ops;
if (!ops)
return 0;
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->iova_to_phys(ops, iova);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
return ret;
}
static bool qcom_iommu_capable(enum iommu_cap cap)
{
switch (cap) {
case IOMMU_CAP_CACHE_COHERENCY:
/*
* Return true here as the SMMU can always send out coherent
* requests.
*/
return true;
case IOMMU_CAP_NOEXEC:
return true;
default:
return false;
}
}
static int qcom_iommu_add_device(struct device *dev)
{
struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
struct iommu_group *group;
struct device_link *link;
if (!qcom_iommu)
return -ENODEV;
/*
* Establish the link between iommu and master, so that the
* iommu gets runtime enabled/disabled as per the master's
* needs.
*/
link = device_link_add(dev, qcom_iommu->dev, DL_FLAG_PM_RUNTIME);
if (!link) {
dev_err(qcom_iommu->dev, "Unable to create device link between %s and %s\n",
dev_name(qcom_iommu->dev), dev_name(dev));
return -ENODEV;
}
group = iommu_group_get_for_dev(dev);
if (IS_ERR_OR_NULL(group))
return PTR_ERR_OR_ZERO(group);
iommu_group_put(group);
iommu_device_link(&qcom_iommu->iommu, dev);
return 0;
}
static void qcom_iommu_remove_device(struct device *dev)
{
struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec);
if (!qcom_iommu)
return;
iommu_device_unlink(&qcom_iommu->iommu, dev);
iommu_group_remove_device(dev);
iommu_fwspec_free(dev);
}
static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
{
struct qcom_iommu_dev *qcom_iommu;
struct platform_device *iommu_pdev;
unsigned asid = args->args[0];
if (args->args_count != 1) {
dev_err(dev, "incorrect number of iommu params found for %s "
"(found %d, expected 1)\n",
args->np->full_name, args->args_count);
return -EINVAL;
}
iommu_pdev = of_find_device_by_node(args->np);
if (WARN_ON(!iommu_pdev))
return -EINVAL;
qcom_iommu = platform_get_drvdata(iommu_pdev);
/* make sure the asid specified in dt is valid, so we don't have
* to sanity check this elsewhere, since 'asid - 1' is used to
* index into qcom_iommu->ctxs:
*/
if (WARN_ON(asid < 1) ||
WARN_ON(asid > qcom_iommu->num_ctxs))
return -EINVAL;
if (!dev->iommu_fwspec->iommu_priv) {
dev->iommu_fwspec->iommu_priv = qcom_iommu;
} else {
/* make sure devices iommus dt node isn't referring to
* multiple different iommu devices. Multiple context
* banks are ok, but multiple devices are not:
*/
if (WARN_ON(qcom_iommu != dev->iommu_fwspec->iommu_priv))
return -EINVAL;
}
return iommu_fwspec_add_ids(dev, &asid, 1);
}
static const struct iommu_ops qcom_iommu_ops = {
.capable = qcom_iommu_capable,
.domain_alloc = qcom_iommu_domain_alloc,
.domain_free = qcom_iommu_domain_free,
.attach_dev = qcom_iommu_attach_dev,
.detach_dev = qcom_iommu_detach_dev,
.map = qcom_iommu_map,
.unmap = qcom_iommu_unmap,
.map_sg = default_iommu_map_sg,
.iova_to_phys = qcom_iommu_iova_to_phys,
.add_device = qcom_iommu_add_device,
.remove_device = qcom_iommu_remove_device,
.device_group = generic_device_group,
.of_xlate = qcom_iommu_of_xlate,
.pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
};
static int qcom_iommu_enable_clocks(struct qcom_iommu_dev *qcom_iommu)
{
int ret;
ret = clk_prepare_enable(qcom_iommu->iface_clk);
if (ret) {
dev_err(qcom_iommu->dev, "Couldn't enable iface_clk\n");
return ret;
}
ret = clk_prepare_enable(qcom_iommu->bus_clk);
if (ret) {
dev_err(qcom_iommu->dev, "Couldn't enable bus_clk\n");
clk_disable_unprepare(qcom_iommu->iface_clk);
return ret;
}
return 0;
}
static void qcom_iommu_disable_clocks(struct qcom_iommu_dev *qcom_iommu)
{
clk_disable_unprepare(qcom_iommu->bus_clk);
clk_disable_unprepare(qcom_iommu->iface_clk);
}
static int qcom_iommu_sec_ptbl_init(struct device *dev)
{
size_t psize = 0;
unsigned int spare = 0;
void *cpu_addr;
dma_addr_t paddr;
unsigned long attrs;
static bool allocated = false;
int ret;
if (allocated)
return 0;
ret = qcom_scm_iommu_secure_ptbl_size(spare, &psize);
if (ret) {
dev_err(dev, "failed to get iommu secure pgtable size (%d)\n",
ret);
return ret;
}
dev_info(dev, "iommu sec: pgtable size: %zu\n", psize);
attrs = DMA_ATTR_NO_KERNEL_MAPPING;
cpu_addr = dma_alloc_attrs(dev, psize, &paddr, GFP_KERNEL, attrs);
if (!cpu_addr) {
dev_err(dev, "failed to allocate %zu bytes for pgtable\n",
psize);
return -ENOMEM;
}
ret = qcom_scm_iommu_secure_ptbl_init(paddr, psize, spare);
if (ret) {
dev_err(dev, "failed to init iommu pgtable (%d)\n", ret);
goto free_mem;
}
allocated = true;
return 0;
free_mem:
dma_free_attrs(dev, psize, cpu_addr, paddr, attrs);
return ret;
}
static int get_asid(const struct device_node *np)
{
u32 reg;
/* read the "reg" property directly to get the relative address
* of the context bank, and calculate the asid from that:
*/
if (of_property_read_u32_index(np, "reg", 0, &reg))
return -ENODEV;
return reg / 0x1000; /* context banks are 0x1000 apart */
}
static int qcom_iommu_ctx_probe(struct platform_device *pdev)
{
struct qcom_iommu_ctx *ctx;
struct device *dev = &pdev->dev;
struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(dev->parent);
struct resource *res;
int ret, irq;
ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return -ENOMEM;
ctx->dev = dev;
platform_set_drvdata(pdev, ctx);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
ctx->base = devm_ioremap_resource(dev, res);
if (IS_ERR(ctx->base))
return PTR_ERR(ctx->base);
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
dev_err(dev, "failed to get irq\n");
return -ENODEV;
}
/* clear IRQs before registering fault handler, just in case the
* boot-loader left us a surprise:
*/
iommu_writel(ctx, ARM_SMMU_CB_FSR, iommu_readl(ctx, ARM_SMMU_CB_FSR));
ret = devm_request_irq(dev, irq,
qcom_iommu_fault,
IRQF_SHARED,
"qcom-iommu-fault",
ctx);
if (ret) {
dev_err(dev, "failed to request IRQ %u\n", irq);
return ret;
}
ret = get_asid(dev->of_node);
if (ret < 0) {
dev_err(dev, "missing reg property\n");
return ret;
}
ctx->asid = ret;
dev_dbg(dev, "found asid %u\n", ctx->asid);
qcom_iommu->ctxs[ctx->asid - 1] = ctx;
return 0;
}
static int qcom_iommu_ctx_remove(struct platform_device *pdev)
{
struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent);
struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev);
platform_set_drvdata(pdev, NULL);
qcom_iommu->ctxs[ctx->asid - 1] = NULL;
return 0;
}
static const struct of_device_id ctx_of_match[] = {
{ .compatible = "qcom,msm-iommu-v1-ns" },
{ .compatible = "qcom,msm-iommu-v1-sec" },
{ /* sentinel */ }
};
static struct platform_driver qcom_iommu_ctx_driver = {
.driver = {
.name = "qcom-iommu-ctx",
.of_match_table = of_match_ptr(ctx_of_match),
},
.probe = qcom_iommu_ctx_probe,
.remove = qcom_iommu_ctx_remove,
};
static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu)
{
struct device_node *child;
for_each_child_of_node(qcom_iommu->dev->of_node, child)
if (of_device_is_compatible(child, "qcom,msm-iommu-v1-sec"))
return true;
return false;
}
static int qcom_iommu_device_probe(struct platform_device *pdev)
{
struct device_node *child;
struct qcom_iommu_dev *qcom_iommu;
struct device *dev = &pdev->dev;
struct resource *res;
int ret, sz, max_asid = 0;
/* find the max asid (which is 1:1 to ctx bank idx), so we know how
* many child ctx devices we have:
*/
for_each_child_of_node(dev->of_node, child)
max_asid = max(max_asid, get_asid(child));
sz = sizeof(*qcom_iommu) + (max_asid * sizeof(qcom_iommu->ctxs[0]));
qcom_iommu = devm_kzalloc(dev, sz, GFP_KERNEL);
if (!qcom_iommu)
return -ENOMEM;
qcom_iommu->num_ctxs = max_asid;
qcom_iommu->dev = dev;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (res)
qcom_iommu->local_base = devm_ioremap_resource(dev, res);
qcom_iommu->iface_clk = devm_clk_get(dev, "iface");
if (IS_ERR(qcom_iommu->iface_clk)) {
dev_err(dev, "failed to get iface clock\n");
return PTR_ERR(qcom_iommu->iface_clk);
}
qcom_iommu->bus_clk = devm_clk_get(dev, "bus");
if (IS_ERR(qcom_iommu->bus_clk)) {
dev_err(dev, "failed to get bus clock\n");
return PTR_ERR(qcom_iommu->bus_clk);
}
if (of_property_read_u32(dev->of_node, "qcom,iommu-secure-id",
&qcom_iommu->sec_id)) {
dev_err(dev, "missing qcom,iommu-secure-id property\n");
return -ENODEV;
}
if (qcom_iommu_has_secure_context(qcom_iommu)) {
ret = qcom_iommu_sec_ptbl_init(dev);
if (ret) {
dev_err(dev, "cannot init secure pg table(%d)\n", ret);
return ret;
}
}
platform_set_drvdata(pdev, qcom_iommu);
pm_runtime_enable(dev);
/* register context bank devices, which are child nodes: */
ret = devm_of_platform_populate(dev);
if (ret) {
dev_err(dev, "Failed to populate iommu contexts\n");
return ret;
}
ret = iommu_device_sysfs_add(&qcom_iommu->iommu, dev, NULL,
dev_name(dev));
if (ret) {
dev_err(dev, "Failed to register iommu in sysfs\n");
return ret;
}
iommu_device_set_ops(&qcom_iommu->iommu, &qcom_iommu_ops);
iommu_device_set_fwnode(&qcom_iommu->iommu, dev->fwnode);
ret = iommu_device_register(&qcom_iommu->iommu);
if (ret) {
dev_err(dev, "Failed to register iommu\n");
return ret;
}
bus_set_iommu(&platform_bus_type, &qcom_iommu_ops);
if (qcom_iommu->local_base) {
pm_runtime_get_sync(dev);
writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS);
pm_runtime_put_sync(dev);
}
return 0;
}
static int qcom_iommu_device_remove(struct platform_device *pdev)
{
struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
bus_set_iommu(&platform_bus_type, NULL);
pm_runtime_force_suspend(&pdev->dev);
platform_set_drvdata(pdev, NULL);
iommu_device_sysfs_remove(&qcom_iommu->iommu);
iommu_device_unregister(&qcom_iommu->iommu);
return 0;
}
static int __maybe_unused qcom_iommu_resume(struct device *dev)
{
struct platform_device *pdev = to_platform_device(dev);
struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
return qcom_iommu_enable_clocks(qcom_iommu);
}
static int __maybe_unused qcom_iommu_suspend(struct device *dev)
{
struct platform_device *pdev = to_platform_device(dev);
struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev);
qcom_iommu_disable_clocks(qcom_iommu);
return 0;
}
static const struct dev_pm_ops qcom_iommu_pm_ops = {
SET_RUNTIME_PM_OPS(qcom_iommu_suspend, qcom_iommu_resume, NULL)
SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
};
static const struct of_device_id qcom_iommu_of_match[] = {
{ .compatible = "qcom,msm-iommu-v1" },
{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, qcom_iommu_of_match);
static struct platform_driver qcom_iommu_driver = {
.driver = {
.name = "qcom-iommu",
.of_match_table = of_match_ptr(qcom_iommu_of_match),
.pm = &qcom_iommu_pm_ops,
},
.probe = qcom_iommu_device_probe,
.remove = qcom_iommu_device_remove,
};
static int __init qcom_iommu_init(void)
{
int ret;
ret = platform_driver_register(&qcom_iommu_ctx_driver);
if (ret)
return ret;
ret = platform_driver_register(&qcom_iommu_driver);
if (ret)
platform_driver_unregister(&qcom_iommu_ctx_driver);
return ret;
}
static void __exit qcom_iommu_exit(void)
{
platform_driver_unregister(&qcom_iommu_driver);
platform_driver_unregister(&qcom_iommu_ctx_driver);
}
module_init(qcom_iommu_init);
module_exit(qcom_iommu_exit);
IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
MODULE_LICENSE("GPL v2");

52
drivers/iommu/rockchip-iommu.c
View File

@ -90,7 +90,9 @@ struct rk_iommu {
struct device *dev;
void __iomem **bases;
int num_mmu;
int irq;
int *irq;
int num_irq;
bool reset_disabled;
struct iommu_device iommu;
struct list_head node; /* entry in rk_iommu_domain.iommus */
struct iommu_domain *domain; /* domain to which iommu is attached */
@ -414,6 +416,9 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu)
int ret, i;
u32 dte_addr;
if (iommu->reset_disabled)
return 0;
/*
* Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY
* and verifying that upper 5 nybbles are read back.
@ -825,10 +830,12 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
iommu->domain = domain;
ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq,
IRQF_SHARED, dev_name(dev), iommu);
if (ret)
return ret;
for (i = 0; i < iommu->num_irq; i++) {
ret = devm_request_irq(iommu->dev, iommu->irq[i], rk_iommu_irq,
IRQF_SHARED, dev_name(dev), iommu);
if (ret)
return ret;
}
for (i = 0; i < iommu->num_mmu; i++) {
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
@ -878,7 +885,8 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
}
rk_iommu_disable_stall(iommu);
devm_free_irq(iommu->dev, iommu->irq, iommu);
for (i = 0; i < iommu->num_irq; i++)
devm_free_irq(iommu->dev, iommu->irq[i], iommu);
iommu->domain = NULL;
@ -1008,20 +1016,20 @@ static int rk_iommu_group_set_iommudata(struct iommu_group *group,
ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0,
&args);
if (ret) {
dev_err(dev, "of_parse_phandle_with_args(%s) => %d\n",
np->full_name, ret);
dev_err(dev, "of_parse_phandle_with_args(%pOF) => %d\n",
np, ret);
return ret;
}
if (args.args_count != 0) {
dev_err(dev, "incorrect number of iommu params found for %s (found %d, expected 0)\n",
args.np->full_name, args.args_count);
dev_err(dev, "incorrect number of iommu params found for %pOF (found %d, expected 0)\n",
args.np, args.args_count);
return -EINVAL;
}
pd = of_find_device_by_node(args.np);
of_node_put(args.np);
if (!pd) {
dev_err(dev, "iommu %s not found\n", args.np->full_name);
dev_err(dev, "iommu %pOF not found\n", args.np);
return -EPROBE_DEFER;
}
@ -1157,12 +1165,28 @@ static int rk_iommu_probe(struct platform_device *pdev)
if (iommu->num_mmu == 0)
return PTR_ERR(iommu->bases[0]);
iommu->irq = platform_get_irq(pdev, 0);
if (iommu->irq < 0) {
dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq);
iommu->num_irq = platform_irq_count(pdev);
if (iommu->num_irq < 0)
return iommu->num_irq;
if (iommu->num_irq == 0)
return -ENXIO;
iommu->irq = devm_kcalloc(dev, iommu->num_irq, sizeof(*iommu->irq),
GFP_KERNEL);
if (!iommu->irq)
return -ENOMEM;
for (i = 0; i < iommu->num_irq; i++) {
iommu->irq[i] = platform_get_irq(pdev, i);
if (iommu->irq[i] < 0) {
dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq[i]);
return -ENXIO;
}
}
iommu->reset_disabled = device_property_read_bool(dev,
"rockchip,disable-mmu-reset");
err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
if (err)
return err;

37
drivers/iommu/s390-iommu.c
View File

@ -18,6 +18,8 @@
*/
#define S390_IOMMU_PGSIZES (~0xFFFUL)
static const struct iommu_ops s390_iommu_ops;
struct s390_domain {
struct iommu_domain domain;
struct list_head devices;
@ -166,11 +168,13 @@ static void s390_iommu_detach_device(struct iommu_domain *domain,
static int s390_iommu_add_device(struct device *dev)
{
struct iommu_group *group = iommu_group_get_for_dev(dev);
struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
iommu_device_link(&zdev->iommu_dev, dev);
return 0;
}
@ -197,6 +201,7 @@ static void s390_iommu_remove_device(struct device *dev)
s390_iommu_detach_device(domain, dev);
}
iommu_device_unlink(&zdev->iommu_dev, dev);
iommu_group_remove_device(dev);
}
@ -327,7 +332,37 @@ static size_t s390_iommu_unmap(struct iommu_domain *domain,
return size;
}
static struct iommu_ops s390_iommu_ops = {
int zpci_init_iommu(struct zpci_dev *zdev)
{
int rc = 0;
rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
"s390-iommu.%08x", zdev->fid);
if (rc)
goto out_err;
iommu_device_set_ops(&zdev->iommu_dev, &s390_iommu_ops);
rc = iommu_device_register(&zdev->iommu_dev);
if (rc)
goto out_sysfs;
return 0;
out_sysfs:
iommu_device_sysfs_remove(&zdev->iommu_dev);
out_err:
return rc;
}
void zpci_destroy_iommu(struct zpci_dev *zdev)
{
iommu_device_unregister(&zdev->iommu_dev);
iommu_device_sysfs_remove(&zdev->iommu_dev);
}
static const struct iommu_ops s390_iommu_ops = {
.capable = s390_iommu_capable,
.domain_alloc = s390_domain_alloc,
.domain_free = s390_domain_free,

45
drivers/iommu/tegra-gart.c
View File

@ -61,6 +61,8 @@ struct gart_device {
struct list_head client;
spinlock_t client_lock; /* for client list */
struct device *dev;
struct iommu_device iommu; /* IOMMU Core handle */
};
struct gart_domain {
@ -334,12 +336,35 @@ static bool gart_iommu_capable(enum iommu_cap cap)
return false;
}
static int gart_iommu_add_device(struct device *dev)
{
struct iommu_group *group = iommu_group_get_for_dev(dev);
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
iommu_device_link(&gart_handle->iommu, dev);
return 0;
}
static void gart_iommu_remove_device(struct device *dev)
{
iommu_group_remove_device(dev);
iommu_device_unlink(&gart_handle->iommu, dev);
}
static const struct iommu_ops gart_iommu_ops = {
.capable = gart_iommu_capable,
.domain_alloc = gart_iommu_domain_alloc,
.domain_free = gart_iommu_domain_free,
.attach_dev = gart_iommu_attach_dev,
.detach_dev = gart_iommu_detach_dev,
.add_device = gart_iommu_add_device,
.remove_device = gart_iommu_remove_device,
.device_group = generic_device_group,
.map = gart_iommu_map,
.map_sg = default_iommu_map_sg,
.unmap = gart_iommu_unmap,
@ -378,6 +403,7 @@ static int tegra_gart_probe(struct platform_device *pdev)
struct resource *res, *res_remap;
void __iomem *gart_regs;
struct device *dev = &pdev->dev;
int ret;
if (gart_handle)
return -EIO;
@ -404,6 +430,22 @@ static int tegra_gart_probe(struct platform_device *pdev)
return -ENXIO;
}
ret = iommu_device_sysfs_add(&gart->iommu, &pdev->dev, NULL,
dev_name(&pdev->dev));
if (ret) {
dev_err(dev, "Failed to register IOMMU in sysfs\n");
return ret;
}
iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
ret = iommu_device_register(&gart->iommu);
if (ret) {
dev_err(dev, "Failed to register IOMMU\n");
iommu_device_sysfs_remove(&gart->iommu);
return ret;
}
gart->dev = &pdev->dev;
spin_lock_init(&gart->pte_lock);
spin_lock_init(&gart->client_lock);
@ -430,6 +472,9 @@ static int tegra_gart_remove(struct platform_device *pdev)
{
struct gart_device *gart = platform_get_drvdata(pdev);
iommu_device_unregister(&gart->iommu);
iommu_device_sysfs_remove(&gart->iommu);
writel(0, gart->regs + GART_CONFIG);
if (gart->savedata)
vfree(gart->savedata);

41
drivers/iommu/tegra-smmu.c
View File

@ -36,6 +36,8 @@ struct tegra_smmu {
struct list_head list;
struct dentry *debugfs;
struct iommu_device iommu; /* IOMMU Core code handle */
};
struct tegra_smmu_as {
@ -704,6 +706,7 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
static int tegra_smmu_add_device(struct device *dev)
{
struct device_node *np = dev->of_node;
struct iommu_group *group;
struct of_phandle_args args;
unsigned int index = 0;
@ -719,18 +722,33 @@ static int tegra_smmu_add_device(struct device *dev)
* first match.
*/
dev->archdata.iommu = smmu;
iommu_device_link(&smmu->iommu, dev);
break;
}
index++;
}
group = iommu_group_get_for_dev(dev);
if (IS_ERR(group))
return PTR_ERR(group);
iommu_group_put(group);
return 0;
}
static void tegra_smmu_remove_device(struct device *dev)
{
struct tegra_smmu *smmu = dev->archdata.iommu;
if (smmu)
iommu_device_unlink(&smmu->iommu, dev);
dev->archdata.iommu = NULL;
iommu_group_remove_device(dev);
}
static const struct iommu_ops tegra_smmu_ops = {
@ -741,6 +759,7 @@ static const struct iommu_ops tegra_smmu_ops = {
.detach_dev = tegra_smmu_detach_dev,
.add_device = tegra_smmu_add_device,
.remove_device = tegra_smmu_remove_device,
.device_group = generic_device_group,
.map = tegra_smmu_map,
.unmap = tegra_smmu_unmap,
.map_sg = default_iommu_map_sg,
@ -930,10 +949,25 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
tegra_smmu_ahb_enable();
err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
if (err < 0)
err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));
if (err)
return ERR_PTR(err);
iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
err = iommu_device_register(&smmu->iommu);
if (err) {
iommu_device_sysfs_remove(&smmu->iommu);
return ERR_PTR(err);
}
err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
if (err < 0) {
iommu_device_unregister(&smmu->iommu);
iommu_device_sysfs_remove(&smmu->iommu);
return ERR_PTR(err);
}
if (IS_ENABLED(CONFIG_DEBUG_FS))
tegra_smmu_debugfs_init(smmu);
@ -942,6 +976,9 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
void tegra_smmu_remove(struct tegra_smmu *smmu)
{
iommu_device_unregister(&smmu->iommu);
iommu_device_sysfs_remove(&smmu->iommu);
if (IS_ENABLED(CONFIG_DEBUG_FS))
tegra_smmu_debugfs_exit(smmu);
}

96
drivers/memory/mtk-smi.c
View File

@ -16,6 +16,7 @@
#include <linux/device.h>
#include <linux/err.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
@ -23,7 +24,10 @@
#include <soc/mediatek/smi.h>
#include <dt-bindings/memory/mt2701-larb-port.h>
/* mt8173 */
#define SMI_LARB_MMU_EN 0xf00
/* mt2701 */
#define REG_SMI_SECUR_CON_BASE 0x5c0
/* every register control 8 port, register offset 0x4 */
@ -41,7 +45,12 @@
/* mt2701 domain should be set to 3 */
#define SMI_SECUR_CON_VAL_DOMAIN(id) (0x3 << ((((id) & 0x7) << 2) + 1))
/* mt2712 */
#define SMI_LARB_NONSEC_CON(id) (0x380 + ((id) * 4))
#define F_MMU_EN BIT(0)
struct mtk_smi_larb_gen {
bool need_larbid;
int port_in_larb[MTK_LARB_NR_MAX + 1];
void (*config_port)(struct device *);
};
@ -148,6 +157,15 @@ mtk_smi_larb_bind(struct device *dev, struct device *master, void *data)
struct mtk_smi_iommu *smi_iommu = data;
unsigned int i;
if (larb->larb_gen->need_larbid) {
larb->mmu = &smi_iommu->larb_imu[larb->larbid].mmu;
return 0;
}
/*
* If there is no larbid property, Loop to find the corresponding
* iommu information.
*/
for (i = 0; i < smi_iommu->larb_nr; i++) {
if (dev == smi_iommu->larb_imu[i].dev) {
/* The 'mmu' may be updated in iommu-attach/detach. */
@ -158,14 +176,33 @@ mtk_smi_larb_bind(struct device *dev, struct device *master, void *data)
return -ENODEV;
}
static void mtk_smi_larb_config_port(struct device *dev)
static void mtk_smi_larb_config_port_mt2712(struct device *dev)
{
struct mtk_smi_larb *larb = dev_get_drvdata(dev);
u32 reg;
int i;
/*
* larb 8/9 is the bdpsys larb, the iommu_en is enabled defaultly.
* Don't need to set it again.
*/
if (larb->larbid == 8 || larb->larbid == 9)
return;
for_each_set_bit(i, (unsigned long *)larb->mmu, 32) {
reg = readl_relaxed(larb->base + SMI_LARB_NONSEC_CON(i));
reg |= F_MMU_EN;
writel(reg, larb->base + SMI_LARB_NONSEC_CON(i));
}
}
static void mtk_smi_larb_config_port_mt8173(struct device *dev)
{
struct mtk_smi_larb *larb = dev_get_drvdata(dev);
writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN);
}
static void mtk_smi_larb_config_port_gen1(struct device *dev)
{
struct mtk_smi_larb *larb = dev_get_drvdata(dev);
@ -210,10 +247,11 @@ static const struct component_ops mtk_smi_larb_component_ops = {
static const struct mtk_smi_larb_gen mtk_smi_larb_mt8173 = {
/* mt8173 do not need the port in larb */
.config_port = mtk_smi_larb_config_port,
.config_port = mtk_smi_larb_config_port_mt8173,
};
static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = {
.need_larbid = true,
.port_in_larb = {
LARB0_PORT_OFFSET, LARB1_PORT_OFFSET,
LARB2_PORT_OFFSET, LARB3_PORT_OFFSET
@ -221,6 +259,11 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = {
.config_port = mtk_smi_larb_config_port_gen1,
};
static const struct mtk_smi_larb_gen mtk_smi_larb_mt2712 = {
.need_larbid = true,
.config_port = mtk_smi_larb_config_port_mt2712,
};
static const struct of_device_id mtk_smi_larb_of_ids[] = {
{
.compatible = "mediatek,mt8173-smi-larb",
@ -230,6 +273,10 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = {
.compatible = "mediatek,mt2701-smi-larb",
.data = &mtk_smi_larb_mt2701
},
{
.compatible = "mediatek,mt2712-smi-larb",
.data = &mtk_smi_larb_mt2712
},
{}
};
@ -240,20 +287,13 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct device_node *smi_node;
struct platform_device *smi_pdev;
const struct of_device_id *of_id;
if (!dev->pm_domain)
return -EPROBE_DEFER;
of_id = of_match_node(mtk_smi_larb_of_ids, pdev->dev.of_node);
if (!of_id)
return -EINVAL;
int err;
larb = devm_kzalloc(dev, sizeof(*larb), GFP_KERNEL);
if (!larb)
return -ENOMEM;
larb->larb_gen = of_id->data;
larb->larb_gen = of_device_get_match_data(dev);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
larb->base = devm_ioremap_resource(dev, res);
if (IS_ERR(larb->base))
@ -268,6 +308,15 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
return PTR_ERR(larb->smi.clk_smi);
larb->smi.dev = dev;
if (larb->larb_gen->need_larbid) {
err = of_property_read_u32(dev->of_node, "mediatek,larb-id",
&larb->larbid);
if (err) {
dev_err(dev, "missing larbid property\n");
return err;
}
}
smi_node = of_parse_phandle(dev->of_node, "mediatek,smi", 0);
if (!smi_node)
return -EINVAL;
@ -275,6 +324,8 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
smi_pdev = of_find_device_by_node(smi_node);
of_node_put(smi_node);
if (smi_pdev) {
if (!platform_get_drvdata(smi_pdev))
return -EPROBE_DEFER;
larb->smi_common_dev = &smi_pdev->dev;
} else {
dev_err(dev, "Failed to get the smi_common device\n");
@ -311,6 +362,10 @@ static const struct of_device_id mtk_smi_common_of_ids[] = {
.compatible = "mediatek,mt2701-smi-common",
.data = (void *)MTK_SMI_GEN1
},
{
.compatible = "mediatek,mt2712-smi-common",
.data = (void *)MTK_SMI_GEN2
},
{}
};
@ -319,11 +374,8 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
struct device *dev = &pdev->dev;
struct mtk_smi *common;
struct resource *res;
const struct of_device_id *of_id;
enum mtk_smi_gen smi_gen;
if (!dev->pm_domain)
return -EPROBE_DEFER;
int ret;
common = devm_kzalloc(dev, sizeof(*common), GFP_KERNEL);
if (!common)
@ -338,17 +390,13 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
if (IS_ERR(common->clk_smi))
return PTR_ERR(common->clk_smi);
of_id = of_match_node(mtk_smi_common_of_ids, pdev->dev.of_node);
if (!of_id)
return -EINVAL;
/*
* for mtk smi gen 1, we need to get the ao(always on) base to config
* m4u port, and we need to enable the aync clock for transform the smi
* clock into emi clock domain, but for mtk smi gen2, there's no smi ao
* base.
*/
smi_gen = (enum mtk_smi_gen)of_id->data;
smi_gen = (enum mtk_smi_gen)of_device_get_match_data(dev);
if (smi_gen == MTK_SMI_GEN1) {
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
common->smi_ao_base = devm_ioremap_resource(dev, res);
@ -359,7 +407,9 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
if (IS_ERR(common->clk_async))
return PTR_ERR(common->clk_async);
clk_prepare_enable(common->clk_async);
ret = clk_prepare_enable(common->clk_async);
if (ret)
return ret;
}
pm_runtime_enable(dev);
platform_set_drvdata(pdev, common);
@ -403,4 +453,4 @@ err_unreg_smi:
return ret;
}
subsys_initcall(mtk_smi_init);
module_init(mtk_smi_init);

4
include/dt-bindings/memory/mt8173-larb-port.h
View File

@ -15,10 +15,6 @@
#define __DTS_IOMMU_PORT_MT8173_H
#define MTK_M4U_ID(larb, port) (((larb) << 5) | (port))
/* Local arbiter ID */
#define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0x7)
/* PortID within the local arbiter */
#define MTK_M4U_TO_PORT(id) ((id) & 0x1f)
#define M4U_LARB0_ID 0
#define M4U_LARB1_ID 1

55
include/linux/iommu.h
View File

@ -167,6 +167,10 @@ struct iommu_resv_region {
* @map: map a physically contiguous memory region to an iommu domain
* @unmap: unmap a physically contiguous memory region from an iommu domain
* @map_sg: map a scatter-gather list of physically contiguous memory chunks
* @flush_tlb_all: Synchronously flush all hardware TLBs for this domain
* @tlb_range_add: Add a given iova range to the flush queue for this domain
* @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
* queue
* to an iommu domain
* @iova_to_phys: translate iova to physical address
* @add_device: add device to iommu grouping
@ -199,6 +203,10 @@ struct iommu_ops {
size_t size);
size_t (*map_sg)(struct iommu_domain *domain, unsigned long iova,
struct scatterlist *sg, unsigned int nents, int prot);
void (*flush_iotlb_all)(struct iommu_domain *domain);
void (*iotlb_range_add)(struct iommu_domain *domain,
unsigned long iova, size_t size);
void (*iotlb_sync)(struct iommu_domain *domain);
phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);
int (*add_device)(struct device *dev);
void (*remove_device)(struct device *dev);
@ -225,6 +233,7 @@ struct iommu_ops {
u32 (*domain_get_windows)(struct iommu_domain *domain);
int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
bool (*is_attach_deferred)(struct iommu_domain *domain, struct device *dev);
unsigned long pgsize_bitmap;
};
@ -291,7 +300,9 @@ extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot);
extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size);
size_t size);
extern size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, size_t size);
extern size_t default_iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
struct scatterlist *sg,unsigned int nents,
int prot);
@ -348,6 +359,25 @@ extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr)
extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
unsigned long iova, int flags);
static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
{
if (domain->ops->flush_iotlb_all)
domain->ops->flush_iotlb_all(domain);
}
static inline void iommu_tlb_range_add(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
if (domain->ops->iotlb_range_add)
domain->ops->iotlb_range_add(domain, iova, size);
}
static inline void iommu_tlb_sync(struct iommu_domain *domain)
{
if (domain->ops->iotlb_sync)
domain->ops->iotlb_sync(domain);
}
static inline size_t iommu_map_sg(struct iommu_domain *domain,
unsigned long iova, struct scatterlist *sg,
unsigned int nents, int prot)
@ -430,13 +460,19 @@ static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
}
static inline int iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, int gfp_order, int prot)
phys_addr_t paddr, size_t size, int prot)
{
return -ENODEV;
}
static inline int iommu_unmap(struct iommu_domain *domain, unsigned long iova,
int gfp_order)
size_t size)
{
return -ENODEV;
}
static inline int iommu_unmap_fast(struct iommu_domain *domain, unsigned long iova,
int gfp_order)
{
return -ENODEV;
}
@ -448,6 +484,19 @@ static inline size_t iommu_map_sg(struct iommu_domain *domain,
return -ENODEV;
}
static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
{
}
static inline void iommu_tlb_range_add(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
}
static inline void iommu_tlb_sync(struct iommu_domain *domain)
{
}
static inline int iommu_domain_window_enable(struct iommu_domain *domain,
u32 wnd_nr, phys_addr_t paddr,
u64 size, int prot)

67
include/linux/iova.h
View File

@ -14,6 +14,7 @@
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/rbtree.h>
#include <linux/atomic.h>
#include <linux/dma-mapping.h>
/* iova structure */
@ -36,6 +37,35 @@ struct iova_rcache {
struct iova_cpu_rcache __percpu *cpu_rcaches;
};
struct iova_domain;
/* Call-Back from IOVA code into IOMMU drivers */
typedef void (* iova_flush_cb)(struct iova_domain *domain);
/* Destructor for per-entry data */
typedef void (* iova_entry_dtor)(unsigned long data);
/* Number of entries per Flush Queue */
#define IOVA_FQ_SIZE 256
/* Timeout (in ms) after which entries are flushed from the Flush-Queue */
#define IOVA_FQ_TIMEOUT 10
/* Flush Queue entry for defered flushing */
struct iova_fq_entry {
unsigned long iova_pfn;
unsigned long pages;
unsigned long data;
u64 counter; /* Flush counter when this entrie was added */
};
/* Per-CPU Flush Queue structure */
struct iova_fq {
struct iova_fq_entry entries[IOVA_FQ_SIZE];
unsigned head, tail;
spinlock_t lock;
};
/* holds all the iova translations for a domain */
struct iova_domain {
spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */
@ -45,6 +75,25 @@ struct iova_domain {
unsigned long start_pfn; /* Lower limit for this domain */
unsigned long dma_32bit_pfn;
struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU
TLBs */
iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for
iova entry */
struct iova_fq __percpu *fq; /* Flush Queue */
atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that
have been started */
atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that
have been finished */
struct timer_list fq_timer; /* Timer to regularily empty the
flush-queues */
atomic_t fq_timer_on; /* 1 when timer is active, 0
when not */
};
static inline unsigned long iova_size(struct iova *iova)
@ -95,6 +144,9 @@ struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
bool size_aligned);
void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
unsigned long size);
void queue_iova(struct iova_domain *iovad,
unsigned long pfn, unsigned long pages,
unsigned long data);
unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
unsigned long limit_pfn);
struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
@ -102,6 +154,8 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
unsigned long start_pfn, unsigned long pfn_32bit);
int init_iova_flush_queue(struct iova_domain *iovad,
iova_flush_cb flush_cb, iova_entry_dtor entry_dtor);
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
void put_iova_domain(struct iova_domain *iovad);
struct iova *split_and_remove_iova(struct iova_domain *iovad,
@ -148,6 +202,12 @@ static inline void free_iova_fast(struct iova_domain *iovad,
{
}
static inline void queue_iova(struct iova_domain *iovad,
unsigned long pfn, unsigned long pages,
unsigned long data)
{
}
static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
unsigned long size,
unsigned long limit_pfn)
@ -174,6 +234,13 @@ static inline void init_iova_domain(struct iova_domain *iovad,
{
}
static inline int init_iova_flush_queue(struct iova_domain *iovad,
iova_flush_cb flush_cb,
iova_entry_dtor entry_dtor)
{
return -ENODEV;
}
static inline struct iova *find_iova(struct iova_domain *iovad,
unsigned long pfn)
{

2
include/soc/mediatek/smi.h
View File

@ -19,7 +19,7 @@
#ifdef CONFIG_MTK_SMI
#define MTK_LARB_NR_MAX 8
#define MTK_LARB_NR_MAX 16
#define MTK_SMI_MMU_EN(port) BIT(port)