From 5c8e9a47b5e6e5a203e3316ba3071f060f89525b Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Tue, 7 Sep 2021 17:30:19 +0900 Subject: [PATCH 01/35] dt-bindings: iommu: renesas,ipmmu-vmsa: add r8a779a0 support Add support for r8a779a0 (R-Car V3U). Signed-off-by: Yoshihiro Shimoda Acked-by: Rob Herring Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20210907083020.907648-2-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml index 02c69a95c332..ce0c715205c6 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml @@ -43,6 +43,7 @@ properties: - renesas,ipmmu-r8a77980 # R-Car V3H - renesas,ipmmu-r8a77990 # R-Car E3 - renesas,ipmmu-r8a77995 # R-Car D3 + - renesas,ipmmu-r8a779a0 # R-Car V3U reg: maxItems: 1 From 7a62ced8ebd0e1b692c9dc4781a8d4ddb0f74792 Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Tue, 7 Sep 2021 17:30:20 +0900 Subject: [PATCH 02/35] iommu/ipmmu-vmsa: Add support for r8a779a0 Add support for r8a779a0 (R-Car V3U). The IPMMU hardware design of this SoC differs than others. So, add a new ipmmu_features for it. Signed-off-by: Yoshihiro Shimoda Reviewed-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20210907083020.907648-3-yoshihiro.shimoda.uh@renesas.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index d38ff29a76e8..d8fe5ba0cb23 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -33,10 +33,10 @@ #define arm_iommu_detach_device(...) do {} while (0) #endif -#define IPMMU_CTX_MAX 8U +#define IPMMU_CTX_MAX 16U #define IPMMU_CTX_INVALID -1 -#define IPMMU_UTLB_MAX 48U +#define IPMMU_UTLB_MAX 64U struct ipmmu_features { bool use_ns_alias_offset; @@ -189,8 +189,12 @@ static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu, unsigned int context_id, unsigned int reg) { - return mmu->features->ctx_offset_base + - context_id * mmu->features->ctx_offset_stride + reg; + unsigned int base = mmu->features->ctx_offset_base; + + if (context_id > 7) + base += 0x800 - 8 * 0x40; + + return base + context_id * mmu->features->ctx_offset_stride + reg; } static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu, @@ -922,6 +926,20 @@ static const struct ipmmu_features ipmmu_features_rcar_gen3 = { .utlb_offset_base = 0, }; +static const struct ipmmu_features ipmmu_features_r8a779a0 = { + .use_ns_alias_offset = false, + .has_cache_leaf_nodes = true, + .number_of_contexts = 16, + .num_utlbs = 64, + .setup_imbuscr = false, + .twobit_imttbcr_sl0 = true, + .reserved_context = true, + .cache_snoop = false, + .ctx_offset_base = 0x10000, + .ctx_offset_stride = 0x1040, + .utlb_offset_base = 0x3000, +}; + static const struct of_device_id ipmmu_of_ids[] = { { .compatible = "renesas,ipmmu-vmsa", @@ -959,6 +977,9 @@ static const struct of_device_id ipmmu_of_ids[] = { }, { .compatible = "renesas,ipmmu-r8a77995", .data = &ipmmu_features_rcar_gen3, + }, { + .compatible = "renesas,ipmmu-r8a779a0", + .data = &ipmmu_features_r8a779a0, }, { /* Terminator */ }, From b2b2781a9755bf18987647bb8cdf6a7d5b5d34c2 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 13 Sep 2021 13:40:05 +0100 Subject: [PATCH 03/35] iommu/dart: Clean up IOVA cookie crumbs The addition of the DART driver crossed over with moving IOVA cookie management into core code; clean up the now-unnecessary remnants here. Acked-by: Sven Peter Tested-by: Sven Peter Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/edb72b4965baa8999dd70b8294785d6deebd4183.1631531973.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/apple-dart.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 559db9259e65..cdc2e83b2186 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -596,7 +595,6 @@ static struct iommu_domain *apple_dart_domain_alloc(unsigned int type) if (!dart_domain) return NULL; - iommu_get_dma_cookie(&dart_domain->domain); mutex_init(&dart_domain->init_lock); /* no need to allocate pgtbl_ops or do any other finalization steps */ From 3d31d4e7a3ef6e5951d185cb5924b53cf9cbd0e1 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 13 Sep 2021 13:40:06 +0100 Subject: [PATCH 04/35] iommu/dma: Unexport IOVA cookie management IOVA cookies are now got and put by core code, so we no longer need to export these to modular drivers. The export for getting MSI cookies stays, since VFIO can still be a module, but it was already relying on someone else putting them, so that aspect is unaffected. Reviewed-by: Lu Baolu Reviewed-by: Jean-Philippe Brucker Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/ef89db54a27df7d8bc0af094c7d7b204fd61774c.1631531973.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 7 ------- drivers/iommu/iommu.c | 3 +-- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 896bea04c347..26cb95d3830a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -98,9 +98,6 @@ static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type) /** * iommu_get_dma_cookie - Acquire DMA-API resources for a domain * @domain: IOMMU domain to prepare for DMA-API usage - * - * IOMMU drivers should normally call this from their domain_alloc - * callback when domain->type == IOMMU_DOMAIN_DMA. */ int iommu_get_dma_cookie(struct iommu_domain *domain) { @@ -113,7 +110,6 @@ int iommu_get_dma_cookie(struct iommu_domain *domain) return 0; } -EXPORT_SYMBOL(iommu_get_dma_cookie); /** * iommu_get_msi_cookie - Acquire just MSI remapping resources @@ -151,8 +147,6 @@ EXPORT_SYMBOL(iommu_get_msi_cookie); * iommu_put_dma_cookie - Release a domain's DMA mapping resources * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or * iommu_get_msi_cookie() - * - * IOMMU drivers should normally call this from their domain_free callback. */ void iommu_put_dma_cookie(struct iommu_domain *domain) { @@ -172,7 +166,6 @@ void iommu_put_dma_cookie(struct iommu_domain *domain) kfree(cookie); domain->iova_cookie = NULL; } -EXPORT_SYMBOL(iommu_put_dma_cookie); /** * iommu_dma_get_resv_regions - Reserved region driver helper diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3303d707bab4..a80f13867bed 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1953,8 +1953,7 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, /* Assume all sizes by default; the driver may override this later */ domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; - /* Temporarily avoid -EEXIST while drivers still get their own cookies */ - if (iommu_is_dma_domain(domain) && !domain->iova_cookie && iommu_get_dma_cookie(domain)) { + if (iommu_is_dma_domain(domain) && iommu_get_dma_cookie(domain)) { iommu_domain_free(domain); domain = NULL; } From 1cdeb52e5c245bd3712f54bc38d199f98e90419a Mon Sep 17 00:00:00 2001 From: Nikita Yushchenko Date: Thu, 23 Sep 2021 22:11:16 +0300 Subject: [PATCH 05/35] iommu/ipmmu-vmsa: Hook up r8a77980 DT matching code Add r8a77980 (R-Car V3H) to the list of supported devices. The hardware is the same as on already-supportred V3M and other R-Car Gen3 chips. Signed-off-by: Nikita Yushchenko Link: https://lore.kernel.org/r/20210923191115.22864-1-nikita.yoush@cogentembedded.com Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index d8fe5ba0cb23..ca752bdc710f 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -971,6 +971,9 @@ static const struct of_device_id ipmmu_of_ids[] = { }, { .compatible = "renesas,ipmmu-r8a77970", .data = &ipmmu_features_rcar_gen3, + }, { + .compatible = "renesas,ipmmu-r8a77980", + .data = &ipmmu_features_rcar_gen3, }, { .compatible = "renesas,ipmmu-r8a77990", .data = &ipmmu_features_rcar_gen3, From f13efafc1a2cf30d4a74c00f40210d6de36db2d0 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 27 Sep 2021 14:18:44 +0200 Subject: [PATCH 06/35] iommu/mediatek: Fix out-of-range warning with clang clang-14 notices that a comparison is never true when CONFIG_PHYS_ADDR_T_64BIT is disabled: drivers/iommu/mtk_iommu.c:553:34: error: result of comparison of constant 5368709120 with expression of type 'phys_addr_t' (aka 'unsigned int') is always false [-Werror,-Wtautological-constant-out-of-range-compare] if (dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) ~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Add an explicit check for the type of the variable to skip the check and the warning in that case. Fixes: b4dad40e4f35 ("iommu/mediatek: Adjust the PA for the 4GB Mode") Signed-off-by: Arnd Bergmann Reviewed-by: Yong Wu Link: https://lore.kernel.org/r/20210927121857.941160-1-arnd@kernel.org Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d837adfd1da5..25b834104790 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -550,7 +550,9 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, phys_addr_t pa; pa = dom->iop->iova_to_phys(dom->iop, iova); - if (dom->data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) + if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) && + dom->data->enable_4GB && + pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE) pa &= ~BIT_ULL(32); return pa; From 08ae5d4a1ae96b72222e7b02d072bb997ff29dac Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:54 +0900 Subject: [PATCH 07/35] iommu/dma: Fix sync_sg with swiotlb The is_swiotlb_buffer function takes the physical address of the swiotlb buffer, not the physical address of the original buffer. The sglist contains the physical addresses of the original buffer, so for the sync_sg functions to work properly when a bounce buffer might have been used, we need to use iommu_iova_to_phys to look up the physical address. This is what sync_single does, so call that function on each sglist segment. The previous code mostly worked because swiotlb does the transfer on map and unmap. However, any callers which use DMA_ATTR_SKIP_CPU_SYNC with sglists or which call sync_sg would not have had anything copied to the bounce buffer. Fixes: 82612d66d51d ("iommu: Allow the iommu/dma api to use bounce buffers") Signed-off-by: David Stevens Reviewed-by: Robin Murphy Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210929023300.335969-2-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 896bea04c347..c4d205b63c58 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -828,17 +828,13 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg; int i; - if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) - return; - - for_each_sg(sgl, sg, nelems, i) { - if (!dev_is_dma_coherent(dev)) + if (dev_is_untrusted(dev)) + for_each_sg(sgl, sg, nelems, i) + iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg), + sg->length, dir); + else if (!dev_is_dma_coherent(dev)) + for_each_sg(sgl, sg, nelems, i) arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); - - if (is_swiotlb_buffer(dev, sg_phys(sg))) - swiotlb_sync_single_for_cpu(dev, sg_phys(sg), - sg->length, dir); - } } static void iommu_dma_sync_sg_for_device(struct device *dev, @@ -848,17 +844,14 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg; int i; - if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) - return; - - for_each_sg(sgl, sg, nelems, i) { - if (is_swiotlb_buffer(dev, sg_phys(sg))) - swiotlb_sync_single_for_device(dev, sg_phys(sg), - sg->length, dir); - - if (!dev_is_dma_coherent(dev)) + if (dev_is_untrusted(dev)) + for_each_sg(sgl, sg, nelems, i) + iommu_dma_sync_single_for_device(dev, + sg_dma_address(sg), + sg->length, dir); + else if (!dev_is_dma_coherent(dev)) + for_each_sg(sgl, sg, nelems, i) arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); - } } static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, From 06e620345d544e559b2961cb5a676ec9c80c8950 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:55 +0900 Subject: [PATCH 08/35] iommu/dma: Fix arch_sync_dma for map When calling arch_sync_dma, we need to pass it the memory that's actually being used for dma. When using swiotlb bounce buffers, this is the bounce buffer. Move arch_sync_dma into the __iommu_dma_map_swiotlb helper, so it can use the bounce buffer address if necessary. Now that iommu_dma_map_sg delegates to a function which takes care of architectural syncing in the untrusted device case, the call to iommu_dma_sync_sg_for_device can be moved so it only occurs for trusted devices. Doing the sync for untrusted devices before mapping never really worked, since it needs to be able to target swiotlb buffers. This also moves the architectural sync to before the call to __iommu_dma_map, to guarantee that untrusted devices can't see stale data they shouldn't see. Fixes: 82612d66d51d ("iommu: Allow the iommu/dma api to use bounce buffers") Signed-off-by: David Stevens Reviewed-by: Christoph Hellwig Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20210929023300.335969-3-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c4d205b63c58..19bebacbf178 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -593,6 +593,9 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys, memset(padding_start, 0, padding_size); } + if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + arch_sync_dma_for_device(phys, org_size, dir); + iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys)) swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs); @@ -860,14 +863,9 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, { phys_addr_t phys = page_to_phys(page) + offset; bool coherent = dev_is_dma_coherent(dev); - dma_addr_t dma_handle; - dma_handle = __iommu_dma_map_swiotlb(dev, phys, size, dma_get_mask(dev), + return __iommu_dma_map_swiotlb(dev, phys, size, dma_get_mask(dev), coherent, dir, attrs); - if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - dma_handle != DMA_MAPPING_ERROR) - arch_sync_dma_for_device(phys, size, dir); - return dma_handle; } static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, @@ -1012,12 +1010,12 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, goto out; } - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - iommu_dma_sync_sg_for_device(dev, sg, nents, dir); - if (dev_is_untrusted(dev)) return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs); + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_sg_for_device(dev, sg, nents, dir); + /* * Work out how much IOVA space we need, and align the segments to * IOVA granules for the IOMMU driver to handle. With some clever From ee9d4097cc145dcaebedf6b113d17c91c21333a0 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:56 +0900 Subject: [PATCH 09/35] iommu/dma: Skip extra sync during unmap w/swiotlb Calling the iommu_dma_sync_*_for_cpu functions during unmap can cause two copies out of the swiotlb buffer. Do the arch sync directly in __iommu_dma_unmap_swiotlb instead to avoid this. This makes the call to iommu_dma_sync_sg_for_cpu for untrusted devices in iommu_dma_unmap_sg no longer necessary, so move that invocation later in the function. Signed-off-by: David Stevens Reviewed-by: Christoph Hellwig Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20210929023300.335969-4-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 19bebacbf178..027b489714b7 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -521,6 +521,9 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, if (WARN_ON(!phys)) return; + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev)) + arch_sync_dma_for_cpu(phys, size, dir); + __iommu_dma_unmap(dev, dma_addr, size); if (unlikely(is_swiotlb_buffer(dev, phys))) @@ -871,8 +874,6 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir); __iommu_dma_unmap_swiotlb(dev, dma_handle, size, dir, attrs); } @@ -1088,14 +1089,14 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, struct scatterlist *tmp; int i; - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); - if (dev_is_untrusted(dev)) { iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs); return; } + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir); + /* * The scatterlist segments are mapped into a single * contiguous IOVA allocation, so this is incredibly easy. From 9b49bbc2c4dfd0431bf7ff4e862171189cf94b7e Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:57 +0900 Subject: [PATCH 10/35] iommu/dma: Fold _swiotlb helpers into callers Fold the _swiotlb helper functions into the respective _page functions, since recent fixes have moved all logic from the _page functions to the _swiotlb functions. Signed-off-by: David Stevens Reviewed-by: Christoph Hellwig Reviewed-by: Robin Murphy Link: https://lore.kernel.org/r/20210929023300.335969-5-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 135 +++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 76 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 027b489714b7..4f77c44eaf14 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -510,26 +510,6 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr, iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather); } -static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - struct iommu_domain *domain = iommu_get_dma_domain(dev); - phys_addr_t phys; - - phys = iommu_iova_to_phys(domain, dma_addr); - if (WARN_ON(!phys)) - return; - - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev)) - arch_sync_dma_for_cpu(phys, size, dir); - - __iommu_dma_unmap(dev, dma_addr, size); - - if (unlikely(is_swiotlb_buffer(dev, phys))) - swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); -} - static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, size_t size, int prot, u64 dma_mask) { @@ -556,55 +536,6 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, return iova + iova_off; } -static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys, - size_t org_size, dma_addr_t dma_mask, bool coherent, - enum dma_data_direction dir, unsigned long attrs) -{ - int prot = dma_info_to_prot(dir, coherent, attrs); - struct iommu_domain *domain = iommu_get_dma_domain(dev); - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; - size_t aligned_size = org_size; - void *padding_start; - size_t padding_size; - dma_addr_t iova; - - /* - * If both the physical buffer start address and size are - * page aligned, we don't need to use a bounce page. - */ - if (IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev) && - iova_offset(iovad, phys | org_size)) { - aligned_size = iova_align(iovad, org_size); - phys = swiotlb_tbl_map_single(dev, phys, org_size, - aligned_size, dir, attrs); - - if (phys == DMA_MAPPING_ERROR) - return DMA_MAPPING_ERROR; - - /* Cleanup the padding area. */ - padding_start = phys_to_virt(phys); - padding_size = aligned_size; - - if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && - (dir == DMA_TO_DEVICE || - dir == DMA_BIDIRECTIONAL)) { - padding_start += org_size; - padding_size -= org_size; - } - - memset(padding_start, 0, padding_size); - } - - if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - arch_sync_dma_for_device(phys, org_size, dir); - - iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); - if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys)) - swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs); - return iova; -} - static void __iommu_dma_free_pages(struct page **pages, int count) { while (count--) @@ -866,15 +797,68 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, { phys_addr_t phys = page_to_phys(page) + offset; bool coherent = dev_is_dma_coherent(dev); + int prot = dma_info_to_prot(dir, coherent, attrs); + struct iommu_domain *domain = iommu_get_dma_domain(dev); + struct iommu_dma_cookie *cookie = domain->iova_cookie; + struct iova_domain *iovad = &cookie->iovad; + size_t aligned_size = size; + dma_addr_t iova, dma_mask = dma_get_mask(dev); - return __iommu_dma_map_swiotlb(dev, phys, size, dma_get_mask(dev), - coherent, dir, attrs); + /* + * If both the physical buffer start address and size are + * page aligned, we don't need to use a bounce page. + */ + if (IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev) && + iova_offset(iovad, phys | size)) { + void *padding_start; + size_t padding_size; + + aligned_size = iova_align(iovad, size); + phys = swiotlb_tbl_map_single(dev, phys, size, + aligned_size, dir, attrs); + + if (phys == DMA_MAPPING_ERROR) + return DMA_MAPPING_ERROR; + + /* Cleanup the padding area. */ + padding_start = phys_to_virt(phys); + padding_size = aligned_size; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) { + padding_start += size; + padding_size -= size; + } + + memset(padding_start, 0, padding_size); + } + + if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + arch_sync_dma_for_device(phys, size, dir); + + iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); + if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys)) + swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); + return iova; } static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { - __iommu_dma_unmap_swiotlb(dev, dma_handle, size, dir, attrs); + struct iommu_domain *domain = iommu_get_dma_domain(dev); + phys_addr_t phys; + + phys = iommu_iova_to_phys(domain, dma_handle); + if (WARN_ON(!phys)) + return; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev)) + arch_sync_dma_for_cpu(phys, size, dir); + + __iommu_dma_unmap(dev, dma_handle, size); + + if (unlikely(is_swiotlb_buffer(dev, phys))) + swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); } /* @@ -959,7 +943,7 @@ static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *s int i; for_each_sg(sg, s, nents, i) - __iommu_dma_unmap_swiotlb(dev, sg_dma_address(s), + iommu_dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); } @@ -970,9 +954,8 @@ static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg, int i; for_each_sg(sg, s, nents, i) { - sg_dma_address(s) = __iommu_dma_map_swiotlb(dev, sg_phys(s), - s->length, dma_get_mask(dev), - dev_is_dma_coherent(dev), dir, attrs); + sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s), + s->offset, s->length, dir, attrs); if (sg_dma_address(s) == DMA_MAPPING_ERROR) goto out_unmap; sg_dma_len(s) = s->length; From 2e727bffbe93750a13d2414f3ce43de2f21600d2 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:58 +0900 Subject: [PATCH 11/35] iommu/dma: Check CONFIG_SWIOTLB more broadly Introduce a new dev_use_swiotlb function to guard swiotlb code, instead of overloading dev_is_untrusted. This allows CONFIG_SWIOTLB to be checked more broadly, so the swiotlb related code can be removed more aggressively. Signed-off-by: David Stevens Reviewed-by: Robin Murphy Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210929023300.335969-6-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 4f77c44eaf14..85a005b268f6 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -317,6 +317,11 @@ static bool dev_is_untrusted(struct device *dev) return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; } +static bool dev_use_swiotlb(struct device *dev) +{ + return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev); +} + /* sysfs updates are serialised by the mutex of the group owning @domain */ int iommu_dma_init_fq(struct iommu_domain *domain) { @@ -731,7 +736,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev, { phys_addr_t phys; - if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) + if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev)) return; phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); @@ -747,7 +752,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev, { phys_addr_t phys; - if (dev_is_dma_coherent(dev) && !dev_is_untrusted(dev)) + if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev)) return; phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); @@ -765,7 +770,7 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg; int i; - if (dev_is_untrusted(dev)) + if (dev_use_swiotlb(dev)) for_each_sg(sgl, sg, nelems, i) iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg), sg->length, dir); @@ -781,7 +786,7 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg; int i; - if (dev_is_untrusted(dev)) + if (dev_use_swiotlb(dev)) for_each_sg(sgl, sg, nelems, i) iommu_dma_sync_single_for_device(dev, sg_dma_address(sg), @@ -808,8 +813,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, * If both the physical buffer start address and size are * page aligned, we don't need to use a bounce page. */ - if (IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev) && - iova_offset(iovad, phys | size)) { + if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) { void *padding_start; size_t padding_size; @@ -994,7 +998,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, goto out; } - if (dev_is_untrusted(dev)) + if (dev_use_swiotlb(dev)) return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs); if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) @@ -1072,7 +1076,7 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, struct scatterlist *tmp; int i; - if (dev_is_untrusted(dev)) { + if (dev_use_swiotlb(dev)) { iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs); return; } From e81e99bacc9f9347bda7808a949c1ce9fcc2bbf4 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:32:59 +0900 Subject: [PATCH 12/35] swiotlb: Support aligned swiotlb buffers Add an argument to swiotlb_tbl_map_single that specifies the desired alignment of the allocated buffer. This is used by dma-iommu to ensure the buffer is aligned to the iova granule size when using swiotlb with untrusted sub-granule mappings. This addresses an issue where adjacent slots could be exposed to the untrusted device if IO_TLB_SIZE < iova granule < PAGE_SIZE. Signed-off-by: David Stevens Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20210929023300.335969-7-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 4 ++-- drivers/xen/swiotlb-xen.c | 2 +- include/linux/swiotlb.h | 3 ++- kernel/dma/swiotlb.c | 13 ++++++++----- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 85a005b268f6..289c49ead01a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -818,8 +818,8 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, size_t padding_size; aligned_size = iova_align(iovad, size); - phys = swiotlb_tbl_map_single(dev, phys, size, - aligned_size, dir, attrs); + phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size, + iova_mask(iovad), dir, attrs); if (phys == DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index e56a5faac395..cbdff8979980 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -380,7 +380,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, */ trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); - map = swiotlb_tbl_map_single(dev, phys, size, size, dir, attrs); + map = swiotlb_tbl_map_single(dev, phys, size, size, 0, dir, attrs); if (map == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index b0cb2a9973f4..569272871375 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -45,7 +45,8 @@ extern void __init swiotlb_update_mem_attributes(void); phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, size_t mapping_size, size_t alloc_size, - enum dma_data_direction dir, unsigned long attrs); + unsigned int alloc_aligned_mask, enum dma_data_direction dir, + unsigned long attrs); extern void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 87c40517e822..019672b3da1d 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -459,7 +459,7 @@ static unsigned int wrap_index(struct io_tlb_mem *mem, unsigned int index) * allocate a buffer from that IO TLB pool. */ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, - size_t alloc_size) + size_t alloc_size, unsigned int alloc_align_mask) { struct io_tlb_mem *mem = dev->dma_io_tlb_mem; unsigned long boundary_mask = dma_get_seg_boundary(dev); @@ -483,6 +483,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr, stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1; if (alloc_size >= PAGE_SIZE) stride = max(stride, stride << (PAGE_SHIFT - IO_TLB_SHIFT)); + stride = max(stride, (alloc_align_mask >> IO_TLB_SHIFT) + 1); spin_lock_irqsave(&mem->lock, flags); if (unlikely(nslots > mem->nslabs - mem->used)) @@ -541,7 +542,8 @@ found: phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, size_t mapping_size, size_t alloc_size, - enum dma_data_direction dir, unsigned long attrs) + unsigned int alloc_align_mask, enum dma_data_direction dir, + unsigned long attrs) { struct io_tlb_mem *mem = dev->dma_io_tlb_mem; unsigned int offset = swiotlb_align_offset(dev, orig_addr); @@ -561,7 +563,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, return (phys_addr_t)DMA_MAPPING_ERROR; } - index = swiotlb_find_slots(dev, orig_addr, alloc_size + offset); + index = swiotlb_find_slots(dev, orig_addr, + alloc_size + offset, alloc_align_mask); if (index == -1) { if (!(attrs & DMA_ATTR_NO_WARN)) dev_warn_ratelimited(dev, @@ -675,7 +678,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size, trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size, swiotlb_force); - swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, dir, + swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, 0, dir, attrs); if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; @@ -759,7 +762,7 @@ struct page *swiotlb_alloc(struct device *dev, size_t size) if (!mem) return NULL; - index = swiotlb_find_slots(dev, 0, size); + index = swiotlb_find_slots(dev, 0, size, 0); if (index == -1) return NULL; From 2cbc61a1b1665c84282dbf2b1747ffa0b6248639 Mon Sep 17 00:00:00 2001 From: David Stevens Date: Wed, 29 Sep 2021 11:33:00 +0900 Subject: [PATCH 13/35] iommu/dma: Account for min_align_mask w/swiotlb Pass the non-aligned size to __iommu_dma_map when using swiotlb bounce buffers in iommu_dma_map_page, to account for min_align_mask. To deal with granule alignment, __iommu_dma_map maps iova_align(size + iova_off) bytes starting at phys - iova_off. If iommu_dma_map_page passes aligned size when using swiotlb, then this becomes iova_align(iova_align(orig_size) + iova_off). Normally iova_off will be zero when using swiotlb. However, this is not the case for devices that set min_align_mask. When iova_off is non-zero, __iommu_dma_map ends up mapping an extra page at the end of the buffer. Beyond just being a security issue, the extra page is not cleaned up by __iommu_dma_unmap. This causes problems when the IOVA is reused, due to collisions in the iommu driver. Just passing the original size is sufficient, since __iommu_dma_map will take care of granule alignment. Fixes: 1f221a0d0dbf ("swiotlb: respect min_align_mask") Signed-off-by: David Stevens Link: https://lore.kernel.org/r/20210929023300.335969-8-stevensd@google.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 289c49ead01a..342359727a59 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -806,7 +806,6 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, struct iommu_domain *domain = iommu_get_dma_domain(dev); struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; - size_t aligned_size = size; dma_addr_t iova, dma_mask = dma_get_mask(dev); /* @@ -815,7 +814,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, */ if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) { void *padding_start; - size_t padding_size; + size_t padding_size, aligned_size; aligned_size = iova_align(iovad, size); phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size, @@ -840,7 +839,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page, if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) arch_sync_dma_for_device(phys, size, dir); - iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); + iova = __iommu_dma_map(dev, phys, size, prot, dma_mask); if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys)) swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); return iova; From 9f78e446bde812d18f228976f2c6b8f25b93f08b Mon Sep 17 00:00:00 2001 From: Lennert Buytenhek Date: Tue, 28 Sep 2021 13:44:21 +0300 Subject: [PATCH 14/35] iommu/amd: Use report_iommu_fault() This patch makes iommu/amd call report_iommu_fault() when an I/O page fault occurs, which has two effects: 1) It allows device drivers to register a callback to be notified of I/O page faults, via the iommu_set_fault_handler() API. 2) It triggers the io_page_fault tracepoint in report_iommu_fault() when an I/O page fault occurs. The latter point is the main aim of this patch, as it allows rasdaemon-like daemons to be notified of I/O page faults, and to possibly initiate corrective action in response. A number of other IOMMU drivers already use report_iommu_fault(), and I/O page faults on those IOMMUs therefore already trigger this tracepoint -- but this isn't yet the case for AMD-Vi and Intel DMAR. The AMD IOMMU specification suggests that the bit in an I/O page fault event log entry that signals whether an I/O page fault was for a read request or for a write request is only meaningful when the faulting access was to a present page, but some testing on a Ryzen 3700X suggests that this bit encodes the correct value even for I/O page faults to non-present pages, and therefore, this patch passes the R/W information up the stack even for I/O page faults to non-present pages. Signed-off-by: Lennert Buytenhek Link: https://lore.kernel.org/r/YVLyBW97vZLpOaAp@wantstofly.org Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 2 ++ drivers/iommu/amd/iommu.c | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 8dbe61e2b3c1..867535eb0ce9 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -138,6 +138,8 @@ #define EVENT_DOMID_MASK_HI 0xf0000 #define EVENT_FLAGS_MASK 0xfff #define EVENT_FLAGS_SHIFT 0x10 +#define EVENT_FLAG_RW 0x020 +#define EVENT_FLAG_I 0x008 /* feature control bits */ #define CONTROL_IOMMU_EN 0x00ULL diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 1722bb161841..beadcffcc223 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -473,6 +473,12 @@ static void amd_iommu_report_rmp_fault(volatile u32 *event) pci_dev_put(pdev); } +#define IS_IOMMU_MEM_TRANSACTION(flags) \ + (((flags) & EVENT_FLAG_I) == 0) + +#define IS_WRITE_REQUEST(flags) \ + ((flags) & EVENT_FLAG_RW) + static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, u64 address, int flags) { @@ -485,6 +491,20 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, dev_data = dev_iommu_priv_get(&pdev->dev); if (dev_data) { + /* + * If this is a DMA fault (for which the I(nterrupt) + * bit will be unset), allow report_iommu_fault() to + * prevent logging it. + */ + if (IS_IOMMU_MEM_TRANSACTION(flags)) { + if (!report_iommu_fault(&dev_data->domain->domain, + &pdev->dev, address, + IS_WRITE_REQUEST(flags) ? + IOMMU_FAULT_WRITE : + IOMMU_FAULT_READ)) + goto out; + } + if (__ratelimit(&dev_data->rs)) { pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", domain_id, address, flags); @@ -495,6 +515,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, domain_id, address, flags); } +out: if (pdev) pci_dev_put(pdev); } From 93f9f7958f12a1703799442011dd5f18db893c13 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Tue, 17 Aug 2021 19:34:11 +0800 Subject: [PATCH 15/35] iommu/arm-smmu-v3: Stop pre-zeroing batch commands in arm_smmu_atc_inv_master() Pre-zeroing the batched commands structure is inefficient, as individual commands are zeroed later in arm_smmu_cmdq_build_cmd(). Therefore, only the member 'num' needs to be initialized to 0. Signed-off-by: Zhen Lei Reviewed-by: John Garry Link: https://lore.kernel.org/r/20210817113411.1962-1-thunder.leizhen@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index a388e318f86e..6340a88a1755 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1764,10 +1764,11 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master) { int i; struct arm_smmu_cmdq_ent cmd; - struct arm_smmu_cmdq_batch cmds = {}; + struct arm_smmu_cmdq_batch cmds; arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd); + cmds.num = 0; for (i = 0; i < master->num_streams; i++) { cmd.atc.sid = master->streams[i].id; arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd); From 59d9bd727495b13292dedee2c6c8c5d80de4d16f Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 18 Aug 2021 16:04:51 +0800 Subject: [PATCH 16/35] iommu/arm-smmu-v3: Properly handle the return value of arm_smmu_cmdq_build_cmd() 1. Build command CMD_SYNC cannot fail. So the return value can be ignored. 2. The arm_smmu_cmdq_build_cmd() almost never fails, the addition of "unlikely()" can optimize the instruction pipeline. 3. Check the return value in arm_smmu_cmdq_batch_add(). Signed-off-by: Zhen Lei Link: https://lore.kernel.org/r/20210818080452.2079-2-thunder.leizhen@huawei.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 6340a88a1755..f5848b351b19 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -409,10 +409,7 @@ static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); /* Convert the erroneous command into a CMD_SYNC */ - if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) { - dev_err(smmu->dev, "failed to convert to CMD_SYNC\n"); - return; - } + arm_smmu_cmdq_build_cmd(cmd, &cmd_sync); queue_write(Q_ENT(q, cons), cmd, q->ent_dwords); } @@ -860,7 +857,7 @@ static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, { u64 cmd[CMDQ_ENT_DWORDS]; - if (arm_smmu_cmdq_build_cmd(cmd, ent)) { + if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) { dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", ent->opcode); return -EINVAL; @@ -885,11 +882,20 @@ static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, struct arm_smmu_cmdq_batch *cmds, struct arm_smmu_cmdq_ent *cmd) { + int index; + if (cmds->num == CMDQ_BATCH_ENTRIES) { arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false); cmds->num = 0; } - arm_smmu_cmdq_build_cmd(&cmds->cmds[cmds->num * CMDQ_ENT_DWORDS], cmd); + + index = cmds->num * CMDQ_ENT_DWORDS; + if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) { + dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", + cmd->opcode); + return; + } + cmds->num++; } From e4a40f15b03154d81704b0b3f69e0a408cbb5d38 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Fri, 20 Aug 2021 22:29:04 +0200 Subject: [PATCH 17/35] dt-bindings: arm-smmu: Add compatible for SM6350 SoC Add the SoC specific compatible for SM6350 implementing arm,mmu-500. Signed-off-by: Konrad Dybcio Acked-by: Rob Herring Link: https://lore.kernel.org/r/20210820202906.229292-1-konrad.dybcio@somainline.org Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 03f2b2d4db30..87b93ab3f34e 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -37,6 +37,7 @@ properties: - qcom,sc7280-smmu-500 - qcom,sc8180x-smmu-500 - qcom,sdm845-smmu-500 + - qcom,sm6350-smmu-500 - qcom,sm8150-smmu-500 - qcom,sm8250-smmu-500 - qcom,sm8350-smmu-500 From bc53c8b8b087c8dec3c1bd7501bbef46a4edadf3 Mon Sep 17 00:00:00 2001 From: Konrad Dybcio Date: Fri, 20 Aug 2021 22:29:05 +0200 Subject: [PATCH 18/35] iommu/arm-smmu-qcom: Add SM6350 SMMU compatible Add compatible for SM6350 SMMU to use the Qualcomm Technologies, Inc. specific implementation. Signed-off-by: Konrad Dybcio Link: https://lore.kernel.org/r/20210820202906.229292-2-konrad.dybcio@somainline.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 55690af1b25d..e9827842856d 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -409,6 +409,7 @@ static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { { .compatible = "qcom,sdm630-smmu-v2" }, { .compatible = "qcom,sdm845-smmu-500" }, { .compatible = "qcom,sm6125-smmu-500" }, + { .compatible = "qcom,sm6350-smmu-500" }, { .compatible = "qcom,sm8150-smmu-500" }, { .compatible = "qcom,sm8250-smmu-500" }, { .compatible = "qcom,sm8350-smmu-500" }, From f1edce3db54312571580695b641835e57c990d10 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Fri, 1 Oct 2021 16:00:32 +0200 Subject: [PATCH 19/35] dt-bindings: arm-smmu: Add compatible for QCM2290 SoC Add the SoC specific compatible for QCM2290 implementing arm,mmu-500. Signed-off-by: Loic Poulain Link: https://lore.kernel.org/r/1633096832-7762-2-git-send-email-loic.poulain@linaro.org [will: Sort by alphabetical order, rewrite commit message] Signed-off-by: Will Deacon --- Documentation/devicetree/bindings/iommu/arm,smmu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 87b93ab3f34e..f66a3effba73 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -33,6 +33,7 @@ properties: - description: Qcom SoCs implementing "arm,mmu-500" items: - enum: + - qcom,qcm2290-smmu-500 - qcom,sc7180-smmu-500 - qcom,sc7280-smmu-500 - qcom,sc8180x-smmu-500 From 756a622c8f061ef7cc9938562f4ce67bedcc949c Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Fri, 1 Oct 2021 16:00:31 +0200 Subject: [PATCH 20/35] iommu: arm-smmu-qcom: Add compatible for QCM2290 Add compatible for QCM2290 SMMU to use the Qualcomm Technologies, Inc. specific implementation. Reviewed-by: Bjorn Andersson Signed-off-by: Loic Poulain Link: https://lore.kernel.org/r/1633096832-7762-1-git-send-email-loic.poulain@linaro.org [will: Sort by alphabetical order, add commit message] Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index e9827842856d..503f687b508d 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -403,6 +403,7 @@ static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu, static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { { .compatible = "qcom,msm8998-smmu-v2" }, + { .compatible = "qcom,qcm2290-smmu-500" }, { .compatible = "qcom,sc7180-smmu-500" }, { .compatible = "qcom,sc7280-smmu-500" }, { .compatible = "qcom,sc8180x-smmu-500" }, From e37f1fe4332491bf2f7b7849d5c3adba0d2a77b3 Mon Sep 17 00:00:00 2001 From: Sibi Sankar Date: Fri, 17 Sep 2021 19:25:29 +0530 Subject: [PATCH 21/35] iommu/arm-smmu-qcom: Request direct mapping for modem device The SID configuration requirement for Modem on SC7280 is similar to the ones found on SC7180/SDM845 SoCs. So, add the SC7280 modem compatible to the list to defer the programming of the modem SIDs to the kernel. Signed-off-by: Sibi Sankar Reviewed-by: Sai Prakash Ranjan Reviewed-by: Stephen Boyd Link: https://lore.kernel.org/r/1631886935-14691-5-git-send-email-sibis@codeaurora.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 503f687b508d..ca736b065dd0 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -231,6 +231,7 @@ static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = { { .compatible = "qcom,sc7180-mdss" }, { .compatible = "qcom,sc7180-mss-pil" }, { .compatible = "qcom,sc7280-mdss" }, + { .compatible = "qcom,sc7280-mss-pil" }, { .compatible = "qcom,sc8180x-mdss" }, { .compatible = "qcom,sdm845-mdss" }, { .compatible = "qcom,sdm845-mss-pil" }, From 5240aed2cd2594fb392239f11b9681e5e1591619 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Thu, 14 Oct 2021 13:38:31 +0800 Subject: [PATCH 22/35] iommu/vt-d: Do not falsely log intel_iommu is unsupported kernel option Handling of intel_iommu kernel command line option should return "true" to indicate option is valid and so avoid logging it as unknown by the core parsing code. Also log unknown sub-options at the notice level to let user know of potential typos or similar. Reported-by: Eero Tamminen Signed-off-by: Tvrtko Ursulin Link: https://lore.kernel.org/r/20210831112947.310080-1-tvrtko.ursulin@linux.intel.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index d75f59ae28e6..9a356075d345 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -412,6 +412,7 @@ static int __init intel_iommu_setup(char *str) { if (!str) return -EINVAL; + while (*str) { if (!strncmp(str, "on", 2)) { dmar_disabled = 0; @@ -441,13 +442,16 @@ static int __init intel_iommu_setup(char *str) } else if (!strncmp(str, "tboot_noforce", 13)) { pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n"); intel_iommu_tboot_noforce = 1; + } else { + pr_notice("Unknown option - '%s'\n", str); } str += strcspn(str, ","); while (*str == ',') str++; } - return 0; + + return 1; } __setup("intel_iommu=", intel_iommu_setup); From 914ff7719e8adaf87131dd7cd0a3520afcf89516 Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Thu, 14 Oct 2021 13:38:32 +0800 Subject: [PATCH 23/35] iommu/vt-d: Dump DMAR translation structure when DMA fault occurs When the dmar translation fault happens, the kernel prints a single line fault reason with corresponding hexadecimal code defined in the Intel VT-d specification. Currently, when user wants to debug the translation fault in detail, debugfs is used for dumping the dmar_translation_struct, which is not available when the kernel failed to boot. Dump the DMAR translation structure, pagewalk the IO page table and print the page table entry when the fault happens. This takes effect only when CONFIG_DMAR_DEBUG is enabled. Signed-off-by: Kyung Min Park Link: https://lore.kernel.org/r/20210815203845.31287-1-kyung.min.park@intel.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/Kconfig | 4 ++ drivers/iommu/intel/dmar.c | 10 +++- drivers/iommu/intel/iommu.c | 113 ++++++++++++++++++++++++++++++++++++ include/linux/dmar.h | 8 +++ 4 files changed, 133 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 0ddb77115be7..247d0f2d5fdf 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -6,6 +6,9 @@ config DMAR_TABLE config DMAR_PERF bool +config DMAR_DEBUG + bool + config INTEL_IOMMU bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64) @@ -31,6 +34,7 @@ config INTEL_IOMMU_DEBUGFS bool "Export Intel IOMMU internals in Debugfs" depends on IOMMU_DEBUGFS select DMAR_PERF + select DMAR_DEBUG help !!!WARNING!!! diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index b7708b93f3fa..915bff76fe96 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1941,12 +1941,16 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, reason = dmar_get_fault_reason(fault_reason, &fault_type); - if (fault_type == INTR_REMAP) + if (fault_type == INTR_REMAP) { pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n", source_id >> 8, PCI_SLOT(source_id & 0xFF), PCI_FUNC(source_id & 0xFF), addr >> 48, fault_reason, reason); - else if (pasid == INVALID_IOASID) + + return 0; + } + + if (pasid == INVALID_IOASID) pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n", type ? "DMA Read" : "DMA Write", source_id >> 8, PCI_SLOT(source_id & 0xFF), @@ -1959,6 +1963,8 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type, PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason); + dmar_fault_dump_ptes(iommu, source_id, addr, pasid); + return 0; } diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 9a356075d345..abb2599998b1 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -156,6 +156,8 @@ static struct intel_iommu **g_iommus; static void __init check_tylersburg_isoch(void); static int rwbf_quirk; +static inline struct device_domain_info * +dmar_search_domain_by_dev_info(int segment, int bus, int devfn); /* * set to 1 to panic kernel if can't successfully enable VT-d @@ -996,6 +998,117 @@ out: spin_unlock_irqrestore(&iommu->lock, flags); } +#ifdef CONFIG_DMAR_DEBUG +static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn, u8 bus, u8 devfn) +{ + struct device_domain_info *info; + struct dma_pte *parent, *pte; + struct dmar_domain *domain; + int offset, level; + + info = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn); + if (!info || !info->domain) { + pr_info("device [%02x:%02x.%d] not probed\n", + bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + return; + } + + domain = info->domain; + level = agaw_to_level(domain->agaw); + parent = domain->pgd; + if (!parent) { + pr_info("no page table setup\n"); + return; + } + + while (1) { + offset = pfn_level_offset(pfn, level); + pte = &parent[offset]; + if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) { + pr_info("PTE not present at level %d\n", level); + break; + } + + pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val); + + if (level == 1) + break; + + parent = phys_to_virt(dma_pte_addr(pte)); + level--; + } +} + +void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, + unsigned long long addr, u32 pasid) +{ + struct pasid_dir_entry *dir, *pde; + struct pasid_entry *entries, *pte; + struct context_entry *ctx_entry; + struct root_entry *rt_entry; + u8 devfn = source_id & 0xff; + u8 bus = source_id >> 8; + int i, dir_index, index; + + pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr); + + /* root entry dump */ + rt_entry = &iommu->root_entry[bus]; + if (!rt_entry) { + pr_info("root table entry is not present\n"); + return; + } + + if (sm_supported(iommu)) + pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n", + rt_entry->hi, rt_entry->lo); + else + pr_info("root entry: 0x%016llx", rt_entry->lo); + + /* context entry dump */ + ctx_entry = iommu_context_addr(iommu, bus, devfn, 0); + if (!ctx_entry) { + pr_info("context table entry is not present\n"); + return; + } + + pr_info("context entry: hi 0x%016llx, low 0x%016llx\n", + ctx_entry->hi, ctx_entry->lo); + + /* legacy mode does not require PASID entries */ + if (!sm_supported(iommu)) + goto pgtable_walk; + + /* get the pointer to pasid directory entry */ + dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK); + if (!dir) { + pr_info("pasid directory entry is not present\n"); + return; + } + /* For request-without-pasid, get the pasid from context entry */ + if (intel_iommu_sm && pasid == INVALID_IOASID) + pasid = PASID_RID2PASID; + + dir_index = pasid >> PASID_PDE_SHIFT; + pde = &dir[dir_index]; + pr_info("pasid dir entry: 0x%016llx\n", pde->val); + + /* get the pointer to the pasid table entry */ + entries = get_pasid_table_from_pde(pde); + if (!entries) { + pr_info("pasid table entry is not present\n"); + return; + } + index = pasid & PASID_PTE_MASK; + pte = &entries[index]; + for (i = 0; i < ARRAY_SIZE(pte->val); i++) + pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]); + +pgtable_walk: + pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn); +} +#endif + static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, unsigned long pfn, int *target_level) { diff --git a/include/linux/dmar.h b/include/linux/dmar.h index e04436a7ff27..45e903d84733 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -131,6 +131,14 @@ static inline int dmar_res_noop(struct acpi_dmar_header *hdr, void *arg) return 0; } +#ifdef CONFIG_DMAR_DEBUG +void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, + unsigned long long addr, u32 pasid); +#else +static inline void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, + unsigned long long addr, u32 pasid) {} +#endif + #ifdef CONFIG_INTEL_IOMMU extern int iommu_detected, no_iommu; extern int intel_iommu_init(void); From b34380a6d767c54480a937951e6189a7f9699443 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Oct 2021 13:38:33 +0800 Subject: [PATCH 24/35] iommu/vt-d: Remove duplicate identity domain flag The iommu_domain data structure already has the "type" field to keep the type of a domain. It's unnecessary to have the DOMAIN_FLAG_STATIC_IDENTITY flag in the vt-d implementation. This cleans it up with no functionality change. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20210926114535.923263-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20211014053839.727419-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 9 ++++----- include/linux/intel-iommu.h | 3 --- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index abb2599998b1..ec9ceb2dcf57 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -528,7 +528,7 @@ static inline void free_devinfo_mem(void *vaddr) static inline int domain_type_is_si(struct dmar_domain *domain) { - return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY; + return domain->domain.type == IOMMU_DOMAIN_IDENTITY; } static inline bool domain_use_first_level(struct dmar_domain *domain) @@ -1996,7 +1996,7 @@ static bool first_level_by_default(void) return scalable_mode_support() && intel_cap_flts_sanity(); } -static struct dmar_domain *alloc_domain(int flags) +static struct dmar_domain *alloc_domain(unsigned int type) { struct dmar_domain *domain; @@ -2006,7 +2006,6 @@ static struct dmar_domain *alloc_domain(int flags) memset(domain, 0, sizeof(*domain)); domain->nid = NUMA_NO_NODE; - domain->flags = flags; if (first_level_by_default()) domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL; domain->has_iotlb_device = false; @@ -2825,7 +2824,7 @@ static int __init si_domain_init(int hw) struct device *dev; int i, nid, ret; - si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY); + si_domain = alloc_domain(IOMMU_DOMAIN_IDENTITY); if (!si_domain) return -EFAULT; @@ -4634,7 +4633,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) case IOMMU_DOMAIN_DMA: case IOMMU_DOMAIN_DMA_FQ: case IOMMU_DOMAIN_UNMANAGED: - dmar_domain = alloc_domain(0); + dmar_domain = alloc_domain(type); if (!dmar_domain) { pr_err("Can't allocate dmar_domain\n"); return NULL; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 05a65eb155f7..65b15af3cf1a 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -517,9 +517,6 @@ struct context_entry { u64 hi; }; -/* si_domain contains mulitple devices */ -#define DOMAIN_FLAG_STATIC_IDENTITY BIT(0) - /* * When VT-d works in the scalable mode, it allows DMA translation to * happen through either first level or second level page table. This From 7afd7f6aa21a2929aff3a059b741933ee1819c6b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Oct 2021 13:38:34 +0800 Subject: [PATCH 25/35] iommu/vt-d: Check FL and SL capability sanity in scalable mode An iommu domain could be allocated and mapped before it's attached to any device. This requires that in scalable mode, when the domain is allocated, the format (FL or SL) of the page table must be determined. In order to achieve this, the platform should support consistent SL or FL capabilities on all IOMMU's. This adds a check for this and aborts IOMMU probing if it doesn't meet this requirement. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20210926114535.923263-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20211014053839.727419-5-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/cap_audit.c | 13 +++++++++++++ drivers/iommu/intel/cap_audit.h | 1 + 2 files changed, 14 insertions(+) diff --git a/drivers/iommu/intel/cap_audit.c b/drivers/iommu/intel/cap_audit.c index b12e421a2f1a..b39d223926a4 100644 --- a/drivers/iommu/intel/cap_audit.c +++ b/drivers/iommu/intel/cap_audit.c @@ -163,6 +163,14 @@ static int cap_audit_static(struct intel_iommu *iommu, enum cap_audit_type type) check_irq_capabilities(iommu, i); } + /* + * If the system is sane to support scalable mode, either SL or FL + * should be sane. + */ + if (intel_cap_smts_sanity() && + !intel_cap_flts_sanity() && !intel_cap_slts_sanity()) + return -EOPNOTSUPP; + out: rcu_read_unlock(); return 0; @@ -203,3 +211,8 @@ bool intel_cap_flts_sanity(void) { return ecap_flts(intel_iommu_ecap_sanity); } + +bool intel_cap_slts_sanity(void) +{ + return ecap_slts(intel_iommu_ecap_sanity); +} diff --git a/drivers/iommu/intel/cap_audit.h b/drivers/iommu/intel/cap_audit.h index 74cfccae0e81..d07b75938961 100644 --- a/drivers/iommu/intel/cap_audit.h +++ b/drivers/iommu/intel/cap_audit.h @@ -111,6 +111,7 @@ bool intel_cap_smts_sanity(void); bool intel_cap_pasid_sanity(void); bool intel_cap_nest_sanity(void); bool intel_cap_flts_sanity(void); +bool intel_cap_slts_sanity(void); static inline bool scalable_mode_support(void) { From 032c5ee40e9fc68ed650a3f86f23259376ec93fc Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Oct 2021 13:38:35 +0800 Subject: [PATCH 26/35] iommu/vt-d: Use second level for GPA->HPA translation The IOMMU VT-d implementation uses the first level for GPA->HPA translation by default. Although both the first level and the second level could handle the DMA translation, they're different in some way. For example, the second level translation has separate controls for the Access/Dirty page tracking. With the first level translation, there's no such control. On the other hand, the second level translation has the page-level control for forcing snoop, but the first level only has global control with pasid granularity. This uses the second level for GPA->HPA translation so that we can provide a consistent hardware interface for use cases like dirty page tracking for live migration. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20210926114535.923263-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20211014053839.727419-6-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ec9ceb2dcf57..1c26a7a012ce 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1991,9 +1991,18 @@ static void free_dmar_iommu(struct intel_iommu *iommu) * Check and return whether first level is used by default for * DMA translation. */ -static bool first_level_by_default(void) +static bool first_level_by_default(unsigned int type) { - return scalable_mode_support() && intel_cap_flts_sanity(); + /* Only SL is available in legacy mode */ + if (!scalable_mode_support()) + return false; + + /* Only level (either FL or SL) is available, just use it */ + if (intel_cap_flts_sanity() ^ intel_cap_slts_sanity()) + return intel_cap_flts_sanity(); + + /* Both levels are available, decide it based on domain type */ + return type != IOMMU_DOMAIN_UNMANAGED; } static struct dmar_domain *alloc_domain(unsigned int type) @@ -2006,7 +2015,7 @@ static struct dmar_domain *alloc_domain(unsigned int type) memset(domain, 0, sizeof(*domain)); domain->nid = NUMA_NO_NODE; - if (first_level_by_default()) + if (first_level_by_default(type)) domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL; domain->has_iotlb_device = false; INIT_LIST_HEAD(&domain->devices); From 94f797ad61d337945d90911e6ad09edf2a82b92a Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Thu, 14 Oct 2021 13:38:36 +0800 Subject: [PATCH 27/35] iommu/vt-d: Delete dev_has_feat callback The commit 262948f8ba573 ("iommu: Delete iommu_dev_has_feature()") has deleted the iommu_dev_has_feature() interface. Remove the dev_has_feat callback to avoid dead code. Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20210929072030.1330225-1-baolu.lu@linux.intel.com Link: https://lore.kernel.org/r/20211014053839.727419-7-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 59 ++++--------------------------------- 1 file changed, 5 insertions(+), 54 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 1c26a7a012ce..16a35669a9d0 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -5511,62 +5511,14 @@ static int intel_iommu_disable_sva(struct device *dev) return ret; } -/* - * A PCI express designated vendor specific extended capability is defined - * in the section 3.7 of Intel scalable I/O virtualization technical spec - * for system software and tools to detect endpoint devices supporting the - * Intel scalable IO virtualization without host driver dependency. - * - * Returns the address of the matching extended capability structure within - * the device's PCI configuration space or 0 if the device does not support - * it. - */ -static int siov_find_pci_dvsec(struct pci_dev *pdev) -{ - int pos; - u16 vendor, id; - - pos = pci_find_next_ext_capability(pdev, 0, 0x23); - while (pos) { - pci_read_config_word(pdev, pos + 4, &vendor); - pci_read_config_word(pdev, pos + 8, &id); - if (vendor == PCI_VENDOR_ID_INTEL && id == 5) - return pos; - - pos = pci_find_next_ext_capability(pdev, pos, 0x23); - } - - return 0; -} - -static bool -intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat) +static int intel_iommu_enable_iopf(struct device *dev) { struct device_domain_info *info = get_domain_info(dev); - if (feat == IOMMU_DEV_FEAT_AUX) { - int ret; + if (info && info->pri_supported) + return 0; - if (!dev_is_pci(dev) || dmar_disabled || - !scalable_mode_support() || !pasid_mode_support()) - return false; - - ret = pci_pasid_features(to_pci_dev(dev)); - if (ret < 0) - return false; - - return !!siov_find_pci_dvsec(to_pci_dev(dev)); - } - - if (feat == IOMMU_DEV_FEAT_IOPF) - return info && info->pri_supported; - - if (feat == IOMMU_DEV_FEAT_SVA) - return info && (info->iommu->flags & VTD_FLAG_SVM_CAPABLE) && - info->pasid_supported && info->pri_supported && - info->ats_supported; - - return false; + return -ENODEV; } static int @@ -5577,7 +5529,7 @@ intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat) return intel_iommu_enable_auxd(dev); case IOMMU_DEV_FEAT_IOPF: - return intel_iommu_dev_has_feat(dev, feat) ? 0 : -ENODEV; + return intel_iommu_enable_iopf(dev); case IOMMU_DEV_FEAT_SVA: return intel_iommu_enable_sva(dev); @@ -5703,7 +5655,6 @@ const struct iommu_ops intel_iommu_ops = { .get_resv_regions = intel_iommu_get_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions, .device_group = intel_iommu_device_group, - .dev_has_feat = intel_iommu_dev_has_feat, .dev_feat_enabled = intel_iommu_dev_feat_enabled, .dev_enable_feat = intel_iommu_dev_enable_feat, .dev_disable_feat = intel_iommu_dev_disable_feat, From 00ecd5401349a5e23d2ad768a23b110030165fac Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Thu, 14 Oct 2021 13:38:37 +0800 Subject: [PATCH 28/35] iommu/vt-d: Clean up unused PASID updating functions update_pasid() and its call chain are currently unused in the tree because Thomas disabled the ENQCMD feature. The feature will be re-enabled shortly using a different approach and update_pasid() and its call chain will not be used in the new approach. Remove the useless functions. Signed-off-by: Fenghua Yu Reviewed-by: Tony Luck Link: https://lore.kernel.org/r/20210920192349.2602141-1-fenghua.yu@intel.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-8-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- arch/x86/include/asm/fpu/api.h | 2 -- drivers/iommu/intel/svm.c | 24 +----------------------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index 23bef08a8388..ca4d0dee1ecd 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -106,6 +106,4 @@ extern int cpu_has_xfeatures(u64 xfeatures_mask, const char **feature_name); */ #define PASID_DISABLED 0 -static inline void update_pasid(void) { } - #endif /* _ASM_X86_FPU_API_H */ diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 0c228787704f..5b5d69b04fcc 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -505,21 +505,6 @@ out: return ret; } -static void _load_pasid(void *unused) -{ - update_pasid(); -} - -static void load_pasid(struct mm_struct *mm, u32 pasid) -{ - mutex_lock(&mm->context.lock); - - /* Update PASID MSR on all CPUs running the mm's tasks. */ - on_each_cpu_mask(mm_cpumask(mm), _load_pasid, NULL, true); - - mutex_unlock(&mm->context.lock); -} - static int intel_svm_alloc_pasid(struct device *dev, struct mm_struct *mm, unsigned int flags) { @@ -614,10 +599,6 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu, if (ret) goto free_sdev; - /* The newly allocated pasid is loaded to the mm. */ - if (!(flags & SVM_FLAG_SUPERVISOR_MODE) && list_empty(&svm->devs)) - load_pasid(mm, svm->pasid); - list_add_rcu(&sdev->list, &svm->devs); success: return &sdev->sva; @@ -670,11 +651,8 @@ static int intel_svm_unbind_mm(struct device *dev, u32 pasid) kfree_rcu(sdev, rcu); if (list_empty(&svm->devs)) { - if (svm->notifier.ops) { + if (svm->notifier.ops) mmu_notifier_unregister(&svm->notifier, mm); - /* Clear mm's pasid. */ - load_pasid(mm, PASID_DISABLED); - } pasid_private_remove(svm->pasid); /* We mandate that no page faults may be outstanding * for the PASID when intel_svm_unbind_mm() is called. From 37c8041a818dec2cea22ea48a3f90d512a9e1fcd Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Thu, 14 Oct 2021 13:38:38 +0800 Subject: [PATCH 29/35] iommu/vt-d: Convert the return type of first_pte_in_page to bool The first_pte_in_page() returns true or false, so let's convert its return type to bool. In addition, use 'IS_ALIGNED' to make the code more readable and neater. Signed-off-by: Longpeng(Mike) Link: https://lore.kernel.org/r/20211008000433.1115-1-longpeng2@huawei.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-9-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- include/linux/intel-iommu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 65b15af3cf1a..52481625838c 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -705,9 +705,9 @@ static inline bool dma_pte_superpage(struct dma_pte *pte) return (pte->val & DMA_PTE_LARGE_PAGE); } -static inline int first_pte_in_page(struct dma_pte *pte) +static inline bool first_pte_in_page(struct dma_pte *pte) { - return !((unsigned long)pte & ~VTD_PAGE_MASK); + return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); } extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); From 9906b9352a35427508d974db3a496eb3c49e2010 Mon Sep 17 00:00:00 2001 From: "Longpeng(Mike)" Date: Thu, 14 Oct 2021 13:38:39 +0800 Subject: [PATCH 30/35] iommu/vt-d: Avoid duplicate removing in __domain_mapping() The __domain_mapping() always removes the pages in the range from 'iov_pfn' to 'end_pfn', but the 'end_pfn' is always the last pfn of the range that the caller wants to map. This would introduce too many duplicated removing and leads the map operation take too long, for example: Map iova=0x100000,nr_pages=0x7d61800 iov_pfn: 0x100000, end_pfn: 0x7e617ff iov_pfn: 0x140000, end_pfn: 0x7e617ff iov_pfn: 0x180000, end_pfn: 0x7e617ff iov_pfn: 0x1c0000, end_pfn: 0x7e617ff iov_pfn: 0x200000, end_pfn: 0x7e617ff ... it takes about 50ms in total. We can reduce the cost by recalculate the 'end_pfn' and limit it to the boundary of the end of this pte page. Map iova=0x100000,nr_pages=0x7d61800 iov_pfn: 0x100000, end_pfn: 0x13ffff iov_pfn: 0x140000, end_pfn: 0x17ffff iov_pfn: 0x180000, end_pfn: 0x1bffff iov_pfn: 0x1c0000, end_pfn: 0x1fffff iov_pfn: 0x200000, end_pfn: 0x23ffff ... it only need 9ms now. This also removes a meaningless BUG_ON() in __domain_mapping(). Signed-off-by: Longpeng(Mike) Tested-by: Liujunjie Link: https://lore.kernel.org/r/20211008000433.1115-1-longpeng2@huawei.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211014053839.727419-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 11 ++++++----- include/linux/intel-iommu.h | 6 ++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 16a35669a9d0..0bde0c8b4126 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2479,12 +2479,17 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, return -ENOMEM; first_pte = pte; + lvl_pages = lvl_to_nr_pages(largepage_lvl); + /* It is large page*/ if (largepage_lvl > 1) { unsigned long end_pfn; + unsigned long pages_to_remove; pteval |= DMA_PTE_LARGE_PAGE; - end_pfn = ((iov_pfn + nr_pages) & level_mask(largepage_lvl)) - 1; + pages_to_remove = min_t(unsigned long, nr_pages, + nr_pte_to_next_page(pte) * lvl_pages); + end_pfn = iov_pfn + pages_to_remove - 1; switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl); } else { pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE; @@ -2506,10 +2511,6 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, WARN_ON(1); } - lvl_pages = lvl_to_nr_pages(largepage_lvl); - - BUG_ON(nr_pages < lvl_pages); - nr_pages -= lvl_pages; iov_pfn += lvl_pages; phys_pfn += lvl_pages; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 52481625838c..69230fd695ea 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -710,6 +710,12 @@ static inline bool first_pte_in_page(struct dma_pte *pte) return IS_ALIGNED((unsigned long)pte, VTD_PAGE_SIZE); } +static inline int nr_pte_to_next_page(struct dma_pte *pte) +{ + return first_pte_in_page(pte) ? BIT_ULL(VTD_STRIDE_SHIFT) : + (struct dma_pte *)ALIGN((unsigned long)pte, VTD_PAGE_SIZE) - pte; +} + extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); extern int dmar_find_matched_atsr_unit(struct pci_dev *dev); From 260aecd643fc129c1771cefdd2aaaedf0485fa59 Mon Sep 17 00:00:00 2001 From: Wan Jiabing Date: Wed, 13 Oct 2021 02:34:41 -0400 Subject: [PATCH 31/35] iommu/dart: Use kmemdup instead of kzalloc and memcpy Fix following coccicheck warning: drivers/iommu/apple-dart.c:704:20-27: WARNING opportunity for kmemdup Signed-off-by: Wan Jiabing Acked-by: Sven Peter Reviewed-by: Alyssa Rosenzweig Link: https://lore.kernel.org/r/20211013063441.29888-1-wanjiabing@vivo.com Signed-off-by: Joerg Roedel --- drivers/iommu/apple-dart.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index fdfa39ec2a4d..13ba258b7413 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -702,13 +702,12 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) if (!group) goto out; - group_master_cfg = kzalloc(sizeof(*group_master_cfg), GFP_KERNEL); + group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg), GFP_KERNEL); if (!group_master_cfg) { iommu_group_put(group); goto out; } - memcpy(group_master_cfg, cfg, sizeof(*group_master_cfg)); iommu_group_set_iommudata(group, group_master_cfg, apple_dart_release_group); From 89374244a43e51137ade6d22849ef73876f94a2b Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 26 Sep 2021 15:07:18 +0200 Subject: [PATCH 32/35] iommu/tegra-smmu: Use devm_bitmap_zalloc when applicable 'smmu->asids' is a bitmap. So use 'devm_kzalloc()' to simplify code, improve the semantic of the code and avoid some open-coded arithmetic in allocator arguments. Signed-off-by: Christophe JAILLET Acked-by: Thierry Reding Link: https://lore.kernel.org/r/2c0f4da80c3b5ef83299c651f69a563034c1c6cb.1632661557.git.christophe.jaillet@wanadoo.fr Signed-off-by: Joerg Roedel --- drivers/iommu/tegra-smmu.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 0a281833f611..e900e3c46903 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -1079,7 +1079,6 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, struct tegra_mc *mc) { struct tegra_smmu *smmu; - size_t size; u32 value; int err; @@ -1097,9 +1096,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, */ mc->smmu = smmu; - size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); - - smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); + smmu->asids = devm_bitmap_zalloc(dev, soc->num_asids, GFP_KERNEL); if (!smmu->asids) return ERR_PTR(-ENOMEM); From ab6f4b001c8c726b5e0bb01429710dc61f13e24d Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Tue, 28 Sep 2021 17:22:29 -0500 Subject: [PATCH 33/35] iommu/dma: Use kvcalloc() instead of kvzalloc() Use 2-factor argument form kvcalloc() instead of kvzalloc(). Link: https://github.com/KSPP/linux/issues/162 Signed-off-by: Gustavo A. R. Silva Acked-by: Robin Murphy Reviewed-by: Kees Cook Link: https://lore.kernel.org/r/20210928222229.GA280355@embeddedor Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 342359727a59..fffa8721a8f0 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -558,7 +558,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, if (!order_mask) return NULL; - pages = kvzalloc(count * sizeof(*pages), GFP_KERNEL); + pages = kvcalloc(count, sizeof(*pages), GFP_KERNEL); if (!pages) return NULL; From 5a009fc1364170b240a4d351b345e69bb3728b3e Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Tue, 19 Oct 2021 18:22:53 +0200 Subject: [PATCH 34/35] iommu/dart: Initialize DART_STREAMS_ENABLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DART has an additional global register to control which streams are isolated. This register is a bit redundant since DART_TCR can already be used to control isolation and is usually initialized to DART_STREAM_ALL by the time we get control. Some DARTs (namely the one used for the audio controller) however have some streams disabled initially. Make sure those work by initializing DART_STREAMS_ENABLE during reset. Reported-by: Martin PoviĊĦer Signed-off-by: Sven Peter Reviewed-by: Hector Martin Link: https://lore.kernel.org/r/20211019162253.45919-1-sven@svenpeter.dev Signed-off-by: Joerg Roedel --- drivers/iommu/apple-dart.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 13ba258b7413..f0f4d1f74f92 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -70,6 +70,8 @@ #define DART_ERROR_ADDR_HI 0x54 #define DART_ERROR_ADDR_LO 0x50 +#define DART_STREAMS_ENABLE 0xfc + #define DART_TCR(sid) (0x100 + 4 * (sid)) #define DART_TCR_TRANSLATE_ENABLE BIT(7) #define DART_TCR_BYPASS0_ENABLE BIT(8) @@ -301,6 +303,9 @@ static int apple_dart_hw_reset(struct apple_dart *dart) apple_dart_hw_disable_dma(&stream_map); apple_dart_hw_clear_all_ttbrs(&stream_map); + /* enable all streams globally since TCR is used to control isolation */ + writel(DART_STREAM_ALL, dart->regs + DART_STREAMS_ENABLE); + /* clear any pending errors before the interrupt is unmasked */ writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR); From ac315f96b3bd6f6b8f18f387816c7c2cc6b32e02 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Wed, 27 Oct 2021 11:47:57 -0600 Subject: [PATCH 35/35] iommu/dma: Fix incorrect error return on iommu deferred attach scsi_dma_map() was reporting a failure during boot on an AMD machine with the IOMMU enabled. scsi_dma_map failed: request for 36 bytes! The issue was tracked down to a mistake in logic: should not return an error if iommu_deferred_attach() returns zero. Reported-by: Marshall Midden Fixes: dabb16f67215 ("iommu/dma: return error code from iommu_dma_map_sg()") Link: https://lore.kernel.org/all/CAD2CkAWjS8=kKwEEN4cgVNjyFORUibzEiCUA-X+SMtbo0JoMmA@mail.gmail.com Signed-off-by: Logan Gunthorpe Cc: Joerg Roedel Cc: Will Deacon Link: https://lore.kernel.org/r/20211027174757.119755-1-logang@deltatee.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 896bea04c347..3e5a21b0bb24 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1016,7 +1016,8 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, if (static_branch_unlikely(&iommu_deferred_attach_enabled)) { ret = iommu_deferred_attach(dev, domain); - goto out; + if (ret) + goto out; } if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))