leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

dma-mapping updates for Linux 5.17

Browse Source 
- refactor the dma-direct coherent allocator
  - turn an macro into an inline in scatterlist.h (Logan Gunthorpe)
 -----BEGIN PGP SIGNATURE-----
 
 iQI/BAABCgApFiEEgdbnc3r/njty3Iq9D55TZVIEUYMFAmHcfpILHGhjaEBsc3Qu
 ZGUACgkQD55TZVIEUYP0MA/+N8Vs2ZgemWunnTyZxCOlia+bq9QPXOMlM+GLLYwQ
 pMizhYXKmiM6HHkLDpVOtzOQTBztqBmWzt7a5z/xlwtS3SIT4ktS1MkrPwW56l3F
 Ml8u1OLv3rTXF/6K9PHU1XvXBIdkFkAnoVLPn1k6V9C8MiEUzo43Nx01p5ooNq/u
 +cptbN119BKQ19EAyOWM5QC9hhQoUNRTeWYd9AQjxTMC6mLPF1VKD1KjUM7gwNB5
 TxqGReYTJNNXbEsN+s6JvkOPId8Ps9JpwSys6Kx9lcXYqNchTCpo+HApvV1hcQrm
 hWBY2u94TPRuBIIp6xe3JBAPhbRTwF5gLEOwJxS+UTj4vcioKUdYnRxEt+JfD5AA
 XjlFyAbCwOPw4qXe00+EefQpuSim1nkUb0FXUJC/zRRJjnjmPxxsnZEnLaxlmyOd
 FlYwyY1x/TqiFr6ZfVr67msnPFy4tRRyXa/W9zcXKwPgI6/3nbYjkpbnK/RocZx+
 4vo8LK0zaHEstrCcCbAwRZk5YgOHRFzrnDhRYAifqJqzlvKqEeF9S8hIRSypcIyG
 GD4ViVK4RJQwtKG3XfG8GPhKLTKoL1NQgbNjszSgc1wIt5E+CJ5PUK2TuBVc9I85
 xScxWeGVfgV7a8hDZ+fyc07VV+1El7Bk3qN88Uassu1HhMbcOJeVJ2dV7u/VJRf5
 vIQ=
 =YkF5
 -----END PGP SIGNATURE-----

Merge tag 'dma-mapping-5.17' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping updates from Christoph Hellwig:

 - refactor the dma-direct coherent allocator

 - turn an macro into an inline in scatterlist.h (Logan Gunthorpe)

* tag 'dma-mapping-5.17' of git://git.infradead.org/users/hch/dma-mapping:
  lib/scatterlist: cleanup macros into static inline functions
  dma-direct: add a dma_direct_use_pool helper
  dma-direct: factor the swiotlb code out of __dma_direct_alloc_pages
  dma-direct: drop two CONFIG_DMA_RESTRICTED_POOL conditionals
  dma-direct: warn if there is no pool for force unencrypted allocations
  dma-direct: fail allocations that can't be made coherent
  dma-direct: refactor the !coherent checks in dma_direct_alloc
  dma-direct: factor out a helper for DMA_ATTR_NO_KERNEL_MAPPING allocations
  dma-direct: clean up the remapping checks in dma_direct_alloc
  dma-direct: always leak memory that can't be re-encrypted
  dma-direct: don't call dma_set_decrypted for remapped allocations
  dma-direct: factor out dma_set_{de,en}crypted helpers
This commit is contained in:
Linus Torvalds 2022-01-12 10:08:11 -08:00
commit 7e7b696547
2 changed files with 164 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
include/linux/scatterlist.h
View File

@ -69,10 +69,27 @@ struct sg_append_table {
* a valid sg entry, or whether it points to the start of a new scatterlist.
* Those low bits are there for everyone! (thanks mason :-)
*/
#define sg_is_chain(sg) ((sg)->page_link & SG_CHAIN)
#define sg_is_last(sg) ((sg)->page_link & SG_END)
#define sg_chain_ptr(sg) \
((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END)))
#define SG_PAGE_LINK_MASK (SG_CHAIN | SG_END)
static inline unsigned int __sg_flags(struct scatterlist *sg)
{
return sg->page_link & SG_PAGE_LINK_MASK;
}
static inline struct scatterlist *sg_chain_ptr(struct scatterlist *sg)
{
return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);
}
static inline bool sg_is_chain(struct scatterlist *sg)
{
return __sg_flags(sg) & SG_CHAIN;
}
static inline bool sg_is_last(struct scatterlist *sg)
{
return __sg_flags(sg) & SG_END;
}
/**
* sg_assign_page - Assign a given page to an SG entry
@ -92,7 +109,7 @@ static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
* In order for the low bit stealing approach to work, pages
* must be aligned at a 32-bit boundary as a minimum.
*/
BUG_ON((unsigned long) page & (SG_CHAIN | SG_END));
BUG_ON((unsigned long)page & SG_PAGE_LINK_MASK);
#ifdef CONFIG_DEBUG_SG
BUG_ON(sg_is_chain(sg));
#endif
@ -126,7 +143,7 @@ static inline struct page *sg_page(struct scatterlist *sg)
#ifdef CONFIG_DEBUG_SG
BUG_ON(sg_is_chain(sg));
#endif
return (struct page *)((sg)->page_link & ~(SG_CHAIN | SG_END));
return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);
}
/**

242
kernel/dma/direct.c
View File

@ -75,15 +75,45 @@ static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);
}
static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size)
{
if (!force_dma_unencrypted(dev))
return 0;
return set_memory_decrypted((unsigned long)vaddr, 1 << get_order(size));
}
static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
{
int ret;
if (!force_dma_unencrypted(dev))
return 0;
ret = set_memory_encrypted((unsigned long)vaddr, 1 << get_order(size));
if (ret)
pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");
return ret;
}
static void __dma_direct_free_pages(struct device *dev, struct page *page,
size_t size)
{
if (IS_ENABLED(CONFIG_DMA_RESTRICTED_POOL) &&
swiotlb_free(dev, page, size))
if (swiotlb_free(dev, page, size))
return;
dma_free_contiguous(dev, page, size);
}
static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
{
struct page *page = swiotlb_alloc(dev, size);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
swiotlb_free(dev, page, size);
return NULL;
}
return page;
}
static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
gfp_t gfp)
{
@ -93,18 +123,11 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
WARN_ON_ONCE(!PAGE_ALIGNED(size));
if (is_swiotlb_for_alloc(dev))
return dma_direct_alloc_swiotlb(dev, size);
gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
&phys_limit);
if (IS_ENABLED(CONFIG_DMA_RESTRICTED_POOL) &&
is_swiotlb_for_alloc(dev)) {
page = swiotlb_alloc(dev, size);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
__dma_direct_free_pages(dev, page, size);
return NULL;
}
return page;
}
page = dma_alloc_contiguous(dev, size, gfp);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
dma_free_contiguous(dev, page, size);
@ -133,6 +156,15 @@ again:
return page;
}
/*
* Check if a potentially blocking operations needs to dip into the atomic
* pools for the given device/gfp.
*/
static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
{
return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev);
}
static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp)
{
@ -140,6 +172,9 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
u64 phys_mask;
void *ret;
if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)))
return NULL;
gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,
&phys_mask);
page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
@ -149,64 +184,103 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
return ret;
}
static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp)
{
struct page *page;
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);
if (!page)
return NULL;
/* remove any dirty cache lines on the kernel alias */
if (!PageHighMem(page))
arch_dma_prep_coherent(page, size);
/* return the page pointer as the opaque cookie */
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return page;
}
void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
struct page *page;
void *ret;
int err;
size = PAGE_ALIGN(size);
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
!force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);
if (!page)
return NULL;
/* remove any dirty cache lines on the kernel alias */
if (!PageHighMem(page))
arch_dma_prep_coherent(page, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
/* return the page pointer as the opaque cookie */
return page;
!force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
if (!dev_is_dma_coherent(dev)) {
/*
* Fallback to the arch handler if it exists. This should
* eventually go away.
*/
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!is_swiotlb_for_alloc(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp,
attrs);
/*
* If there is a global pool, always allocate from it for
* non-coherent devices.
*/
if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL))
return dma_alloc_from_global_coherent(dev, size,
dma_handle);
/*
* Otherwise remap if the architecture is asking for it. But
* given that remapping memory is a blocking operation we'll
* instead have to dip into the atomic pools.
*/
remap = IS_ENABLED(CONFIG_DMA_DIRECT_REMAP);
if (remap) {
if (dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size,
dma_handle, gfp);
} else {
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED))
return NULL;
set_uncached = true;
}
}
if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev) &&
!is_swiotlb_for_alloc(dev))
return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) &&
!dev_is_dma_coherent(dev))
return dma_alloc_from_global_coherent(dev, size, dma_handle);
/*
* Remapping or decrypting memory may block. If either is required and
* we can't block, allocate the memory from the atomic pools.
* If restricted DMA (i.e., is_swiotlb_for_alloc) is required, one must
* set up another device coherent pool by shared-dma-pool and use
* dma_alloc_from_dev_coherent instead.
* Decrypting memory may block, so allocate the memory from the atomic
* pools if we can't block.
*/
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
!gfpflags_allow_blocking(gfp) &&
(force_dma_unencrypted(dev) ||
(IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!dev_is_dma_coherent(dev))) &&
!is_swiotlb_for_alloc(dev))
if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
/* we always manually zero the memory once we are done */
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);
if (!page)
return NULL;
if (PageHighMem(page)) {
/*
* Depending on the cma= arguments and per-arch setup,
* dma_alloc_contiguous could return highmem pages.
* Without remapping there is no way to return them here, so
* log an error and fail.
*/
if (!IS_ENABLED(CONFIG_DMA_REMAP)) {
dev_info(dev, "Rejecting highmem page from CMA.\n");
goto out_free_pages;
}
remap = true;
set_uncached = false;
}
if ((IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!dev_is_dma_coherent(dev)) ||
(IS_ENABLED(CONFIG_DMA_REMAP) && PageHighMem(page))) {
if (remap) {
/* remove any dirty cache lines on the kernel alias */
arch_dma_prep_coherent(page, size);
@ -216,56 +290,27 @@ void *dma_direct_alloc(struct device *dev, size_t size,
__builtin_return_address(0));
if (!ret)
goto out_free_pages;
if (force_dma_unencrypted(dev)) {
err = set_memory_decrypted((unsigned long)ret,
1 << get_order(size));
if (err)
goto out_free_pages;
}
memset(ret, 0, size);
goto done;
}
if (PageHighMem(page)) {
/*
* Depending on the cma= arguments and per-arch setup
* dma_alloc_contiguous could return highmem pages.
* Without remapping there is no way to return them here,
* so log an error and fail.
*/
dev_info(dev, "Rejecting highmem page from CMA.\n");
goto out_free_pages;
}
ret = page_address(page);
if (force_dma_unencrypted(dev)) {
err = set_memory_decrypted((unsigned long)ret,
1 << get_order(size));
if (err)
} else {
ret = page_address(page);
if (dma_set_decrypted(dev, ret, size))
goto out_free_pages;
}
memset(ret, 0, size);
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) &&
!dev_is_dma_coherent(dev)) {
if (set_uncached) {
arch_dma_prep_coherent(page, size);
ret = arch_dma_set_uncached(ret, size);
if (IS_ERR(ret))
goto out_encrypt_pages;
}
done:
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return ret;
out_encrypt_pages:
if (force_dma_unencrypted(dev)) {
err = set_memory_encrypted((unsigned long)page_address(page),
1 << get_order(size));
/* If memory cannot be re-encrypted, it must be leaked */
if (err)
return NULL;
}
if (dma_set_encrypted(dev, page_address(page), size))
return NULL;
out_free_pages:
__dma_direct_free_pages(dev, page, size);
return NULL;
@ -304,13 +349,14 @@ void dma_direct_free(struct device *dev, size_t size,
dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
return;
if (force_dma_unencrypted(dev))
set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr))
if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
vunmap(cpu_addr);
else if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
if (dma_set_encrypted(dev, cpu_addr, 1 << page_order))
return;
}
__dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);
}
@ -321,9 +367,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
struct page *page;
void *ret;
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
force_dma_unencrypted(dev) && !gfpflags_allow_blocking(gfp) &&
!is_swiotlb_for_alloc(dev))
if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
page = __dma_direct_alloc_pages(dev, size, gfp);
@ -341,11 +385,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
}
ret = page_address(page);
if (force_dma_unencrypted(dev)) {
if (set_memory_decrypted((unsigned long)ret,
1 << get_order(size)))
goto out_free_pages;
}
if (dma_set_decrypted(dev, ret, size))
goto out_free_pages;
memset(ret, 0, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return page;
@ -366,9 +407,8 @@ void dma_direct_free_pages(struct device *dev, size_t size,
dma_free_from_pool(dev, vaddr, size))
return;
if (force_dma_unencrypted(dev))
set_memory_encrypted((unsigned long)vaddr, 1 << page_order);
if (dma_set_encrypted(dev, vaddr, 1 << page_order))
return;
__dma_direct_free_pages(dev, page, size);
}