leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge tag 'dma-mapping-5.15' of git://git.infradead.org/users/hch/dma-mapping

Browse Source 
Pull dma-mapping updates from Christoph Hellwig:

 - fix debugfs initialization order (Anthony Iliopoulos)

 - use memory_intersects() directly (Kefeng Wang)

 - allow to return specific errors from ->map_sg (Logan Gunthorpe,
   Martin Oliveira)

 - turn the dma_map_sg return value into an unsigned int (me)

 - provide a common global coherent pool іmplementation (me)

* tag 'dma-mapping-5.15' of git://git.infradead.org/users/hch/dma-mapping: (31 commits)
  hexagon: use the generic global coherent pool
  dma-mapping: make the global coherent pool conditional
  dma-mapping: add a dma_init_global_coherent helper
  dma-mapping: simplify dma_init_coherent_memory
  dma-mapping: allow using the global coherent pool for !ARM
  ARM/nommu: use the generic dma-direct code for non-coherent devices
  dma-direct: add support for dma_coherent_default_memory
  dma-mapping: return an unsigned int from dma_map_sg{,_attrs}
  dma-mapping: disallow .map_sg operations from returning zero on error
  dma-mapping: return error code from dma_dummy_map_sg()
  x86/amd_gart: don't set failed sg dma_address to DMA_MAPPING_ERROR
  x86/amd_gart: return error code from gart_map_sg()
  xen: swiotlb: return error code from xen_swiotlb_map_sg()
  parisc: return error code from .map_sg() ops
  sparc/iommu: don't set failed sg dma_address to DMA_MAPPING_ERROR
  sparc/iommu: return error codes from .map_sg() ops
  s390/pci: don't set failed sg dma_address to DMA_MAPPING_ERROR
  s390/pci: return error code from s390_dma_map_sg()
  powerpc/iommu: don't set failed sg dma_address to DMA_MAPPING_ERROR
  powerpc/iommu: return error code from .map_sg() ops
  ...
This commit is contained in:
Linus Torvalds
2021-09-02 10:32:06 -07:00
parent eceae1e7ac c1dec343d7
commit 4a3bb4200a
30 changed files with 309 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
kernel/dma/Kconfig
View File

@@ -93,6 +93,10 @@ config DMA_COHERENT_POOL
select GENERIC_ALLOCATOR
bool
config DMA_GLOBAL_POOL
select DMA_DECLARE_COHERENT
bool
config DMA_REMAP
bool
depends on MMU

159
kernel/dma/coherent.c
View File

@@ -20,8 +20,6 @@ struct dma_coherent_mem {
bool use_dev_dma_pfn_offset;
};
static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev)
{
if (dev && dev->dma_mem)
@@ -37,51 +35,44 @@ static inline dma_addr_t dma_get_device_base(struct device *dev,
return mem->device_base;
}
static int dma_init_coherent_memory(phys_addr_t phys_addr,
dma_addr_t device_addr, size_t size,
struct dma_coherent_mem **mem)
static struct dma_coherent_mem *dma_init_coherent_memory(phys_addr_t phys_addr,
dma_addr_t device_addr, size_t size, bool use_dma_pfn_offset)
{
struct dma_coherent_mem *dma_mem = NULL;
void *mem_base = NULL;
struct dma_coherent_mem *dma_mem;
int pages = size >> PAGE_SHIFT;
int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
int ret;
void *mem_base;
if (!size) {
ret = -EINVAL;
goto out;
}
if (!size)
return ERR_PTR(-EINVAL);
mem_base = memremap(phys_addr, size, MEMREMAP_WC);
if (!mem_base) {
ret = -EINVAL;
goto out;
}
if (!mem_base)
return ERR_PTR(-EINVAL);
dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
if (!dma_mem) {
ret = -ENOMEM;
goto out;
}
if (!dma_mem)
goto out_unmap_membase;
dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
if (!dma_mem->bitmap) {
ret = -ENOMEM;
goto out;
}
if (!dma_mem->bitmap)
goto out_free_dma_mem;
dma_mem->virt_base = mem_base;
dma_mem->device_base = device_addr;
dma_mem->pfn_base = PFN_DOWN(phys_addr);
dma_mem->size = pages;
dma_mem->use_dev_dma_pfn_offset = use_dma_pfn_offset;
spin_lock_init(&dma_mem->spinlock);
*mem = dma_mem;
return 0;
return dma_mem;
out:
out_free_dma_mem:
kfree(dma_mem);
if (mem_base)
memunmap(mem_base);
return ret;
out_unmap_membase:
memunmap(mem_base);
pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %zd MiB\n",
&phys_addr, size / SZ_1M);
return ERR_PTR(-ENOMEM);
}
static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
@@ -130,9 +121,9 @@ int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
struct dma_coherent_mem *mem;
int ret;
ret = dma_init_coherent_memory(phys_addr, device_addr, size, &mem);
if (ret)
return ret;
mem = dma_init_coherent_memory(phys_addr, device_addr, size, false);
if (IS_ERR(mem))
return PTR_ERR(mem);
ret = dma_assign_coherent_memory(dev, mem);
if (ret)
@@ -198,16 +189,6 @@ int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
return 1;
}
void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
dma_addr_t *dma_handle)
{
if (!dma_coherent_default_memory)
return NULL;
return __dma_alloc_from_coherent(dev, dma_coherent_default_memory, size,
dma_handle);
}
static int __dma_release_from_coherent(struct dma_coherent_mem *mem,
int order, void *vaddr)
{
@@ -243,15 +224,6 @@ int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr)
return __dma_release_from_coherent(mem, order, vaddr);
}
int dma_release_from_global_coherent(int order, void *vaddr)
{
if (!dma_coherent_default_memory)
return 0;
return __dma_release_from_coherent(dma_coherent_default_memory, order,
vaddr);
}
static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,
struct vm_area_struct *vma, void *vaddr, size_t size, int *ret)
{
@@ -297,6 +269,28 @@ int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);
}
#ifdef CONFIG_DMA_GLOBAL_POOL
static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
dma_addr_t *dma_handle)
{
if (!dma_coherent_default_memory)
return NULL;
return __dma_alloc_from_coherent(dev, dma_coherent_default_memory, size,
dma_handle);
}
int dma_release_from_global_coherent(int order, void *vaddr)
{
if (!dma_coherent_default_memory)
return 0;
return __dma_release_from_coherent(dma_coherent_default_memory, order,
vaddr);
}
int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
size_t size, int *ret)
{
@@ -307,6 +301,19 @@ int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
vaddr, size, ret);
}
int dma_init_global_coherent(phys_addr_t phys_addr, size_t size)
{
struct dma_coherent_mem *mem;
mem = dma_init_coherent_memory(phys_addr, phys_addr, size, true);
if (IS_ERR(mem))
return PTR_ERR(mem);
dma_coherent_default_memory = mem;
pr_info("DMA: default coherent area is set\n");
return 0;
}
#endif /* CONFIG_DMA_GLOBAL_POOL */
/*
* Support for reserved memory regions defined in device tree
*/
@@ -315,25 +322,22 @@ int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
#include <linux/of_fdt.h>
#include <linux/of_reserved_mem.h>
#ifdef CONFIG_DMA_GLOBAL_POOL
static struct reserved_mem *dma_reserved_default_memory __initdata;
#endif
static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
{
struct dma_coherent_mem *mem = rmem->priv;
int ret;
if (!rmem->priv) {
struct dma_coherent_mem *mem;
if (!mem) {
ret = dma_init_coherent_memory(rmem->base, rmem->base,
rmem->size, &mem);
if (ret) {
pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
&rmem->base, (unsigned long)rmem->size / SZ_1M);
return ret;
}
mem = dma_init_coherent_memory(rmem->base, rmem->base,
rmem->size, true);
if (IS_ERR(mem))
return PTR_ERR(mem);
rmem->priv = mem;
}
mem->use_dev_dma_pfn_offset = true;
rmem->priv = mem;
dma_assign_coherent_memory(dev, mem);
dma_assign_coherent_memory(dev, rmem->priv);
return 0;
}
@@ -361,7 +365,9 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
pr_err("Reserved memory: regions without no-map are not yet supported\n");
return -EINVAL;
}
#endif
#ifdef CONFIG_DMA_GLOBAL_POOL
if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
WARN(dma_reserved_default_memory,
"Reserved memory: region for default DMA coherent area is redefined\n");
@@ -375,31 +381,16 @@ static int __init rmem_dma_setup(struct reserved_mem *rmem)
return 0;
}
#ifdef CONFIG_DMA_GLOBAL_POOL
static int __init dma_init_reserved_memory(void)
{
const struct reserved_mem_ops *ops;
int ret;
if (!dma_reserved_default_memory)
return -ENOMEM;
ops = dma_reserved_default_memory->ops;
/*
* We rely on rmem_dma_device_init() does not propagate error of
* dma_assign_coherent_memory() for "NULL" device.
*/
ret = ops->device_init(dma_reserved_default_memory, NULL);
if (!ret) {
dma_coherent_default_memory = dma_reserved_default_memory->priv;
pr_info("DMA: default coherent area is set\n");
}
return ret;
return dma_init_global_coherent(dma_reserved_default_memory->base,
dma_reserved_default_memory->size);
}
core_initcall(dma_init_reserved_memory);
#endif /* CONFIG_DMA_GLOBAL_POOL */
RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);
#endif

21
kernel/dma/debug.c
View File

@@ -792,7 +792,7 @@ static int dump_show(struct seq_file *seq, void *v)
}
DEFINE_SHOW_ATTRIBUTE(dump);
static void dma_debug_fs_init(void)
static int __init dma_debug_fs_init(void)
{
struct dentry *dentry = debugfs_create_dir("dma-api", NULL);
@@ -805,7 +805,10 @@ static void dma_debug_fs_init(void)
debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries);
debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops);
debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops);
return 0;
}
core_initcall_sync(dma_debug_fs_init);
static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
{
@@ -890,8 +893,6 @@ static int dma_debug_init(void)
spin_lock_init(&dma_entry_hash[i].lock);
}
dma_debug_fs_init();
nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);
for (i = 0; i < nr_pages; ++i)
dma_debug_create_entries(GFP_KERNEL);
@@ -1064,20 +1065,10 @@ static void check_for_stack(struct device *dev,
}
}
static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
{
unsigned long a1 = (unsigned long)addr;
unsigned long b1 = a1 + len;
unsigned long a2 = (unsigned long)start;
unsigned long b2 = (unsigned long)end;
return !(b1 <= a2 || a1 >= b2);
}
static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
{
if (overlap(addr, len, _stext, _etext) ||
overlap(addr, len, __start_rodata, __end_rodata))
if (memory_intersects(_stext, _etext, addr, len) ||
memory_intersects(__start_rodata, __end_rodata, addr, len))
err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
}

17
kernel/dma/direct.c
View File

@@ -156,9 +156,14 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev))
return dma_alloc_from_global_coherent(dev, size, dma_handle);
/*
* Remapping or decrypting memory may block. If either is required and
* we can't block, allocate the memory from the atomic pools.
@@ -255,11 +260,19 @@ void dma_direct_free(struct device *dev, size_t size,
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev)) {
arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
return;
}
if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev)) {
if (!dma_release_from_global_coherent(page_order, cpu_addr))
WARN_ON_ONCE(1);
return;
}
/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
@@ -411,7 +424,7 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
out_unmap:
dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
return 0;
return -EIO;
}
dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
@@ -462,6 +475,8 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
return ret;
if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff)
return -ENXIO;

2
kernel/dma/dummy.c
View File

@@ -22,7 +22,7 @@ static int dma_dummy_map_sg(struct device *dev, struct scatterlist *sgl,
int nelems, enum dma_data_direction dir,
unsigned long attrs)
{
return 0;
return -EINVAL;
}
static int dma_dummy_supported(struct device *hwdev, u64 mask)

80
kernel/dma/mapping.c
View File

@@ -177,12 +177,8 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
}
EXPORT_SYMBOL(dma_unmap_page_attrs);
/*
* dma_maps_sg_attrs returns 0 on error and > 0 on success.
* It should never return a value < 0.
*/
int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
enum dma_data_direction dir, unsigned long attrs)
static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
const struct dma_map_ops *ops = get_dma_ops(dev);
int ents;
@@ -197,13 +193,81 @@ int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
else
ents = ops->map_sg(dev, sg, nents, dir, attrs);
BUG_ON(ents < 0);
debug_dma_map_sg(dev, sg, nents, ents, dir);
if (ents > 0)
debug_dma_map_sg(dev, sg, nents, ents, dir);
else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
ents != -EIO))
return -EIO;
return ents;
}
/**
* dma_map_sg_attrs - Map the given buffer for DMA
* @dev: The device for which to perform the DMA operation
* @sg: The sg_table object describing the buffer
* @dir: DMA direction
* @attrs: Optional DMA attributes for the map operation
*
* Maps a buffer described by a scatterlist passed in the sg argument with
* nents segments for the @dir DMA operation by the @dev device.
*
* Returns the number of mapped entries (which can be less than nents)
* on success. Zero is returned for any error.
*
* dma_unmap_sg_attrs() should be used to unmap the buffer with the
* original sg and original nents (not the value returned by this funciton).
*/
unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir, unsigned long attrs)
{
int ret;
ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs);
if (ret < 0)
return 0;
return ret;
}
EXPORT_SYMBOL(dma_map_sg_attrs);
/**
* dma_map_sgtable - Map the given buffer for DMA
* @dev: The device for which to perform the DMA operation
* @sgt: The sg_table object describing the buffer
* @dir: DMA direction
* @attrs: Optional DMA attributes for the map operation
*
* Maps a buffer described by a scatterlist stored in the given sg_table
* object for the @dir DMA operation by the @dev device. After success, the
* ownership for the buffer is transferred to the DMA domain. One has to
* call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the
* ownership of the buffer back to the CPU domain before touching the
* buffer by the CPU.
*
* Returns 0 on success or a negative error code on error. The following
* error codes are supported with the given meaning:
*
* -EINVAL - An invalid argument, unaligned access or other error
* in usage. Will not succeed if retried.
* -ENOMEM - Insufficient resources (like memory or IOVA space) to
* complete the mapping. Should succeed if retried later.
* -EIO - Legacy error code with an unknown meaning. eg. this is
* returned if a lower level call returned DMA_MAPPING_ERROR.
*/
int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
enum dma_data_direction dir, unsigned long attrs)
{
int nents;
nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);
if (nents < 0)
return nents;
sgt->nents = nents;
return 0;
}
EXPORT_SYMBOL_GPL(dma_map_sgtable);
void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
int nents, enum dma_data_direction dir,
unsigned long attrs)