mirror of
https://github.com/torvalds/linux.git
synced 2024-11-05 11:32:04 +00:00
ac2cbab21f
Normal boot path on system with iommu support:
swiotlb buffer will be allocated early at first and then try to initialize
iommu, if iommu for intel or AMD could setup properly, swiotlb buffer
will be freed.
The early allocating is with bootmem, and could panic when we try to use
kdump with buffer above 4G only, or with memmap to limit mem under 4G.
for example: memmap=4095M$1M to remove memory under 4G.
According to Eric, add _nopanic version and no_iotlb_memory to fail
map single later if swiotlb is still needed.
-v2: don't pass nopanic, and use -ENOMEM return value according to Eric.
panic early instead of using swiotlb_full to panic...according to Eric/Konrad.
-v3: make swiotlb_init to be notpanic, but will affect:
arm64, ia64, powerpc, tile, unicore32, x86.
-v4: cleanup swiotlb_init by removing swiotlb_init_with_default_size.
Suggested-by: Eric W. Biederman <ebiederm@xmission.com>
Signed-off-by: Yinghai Lu <yinghai@kernel.org>
Link: http://lkml.kernel.org/r/1359058816-7615-36-git-send-email-yinghai@kernel.org
Reviewed-and-tested-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Kyungmin Park <kyungmin.park@samsung.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Andrzej Pietrasiewicz <andrzej.p@samsung.com>
Cc: linux-mips@linux-mips.org
Cc: xen-devel@lists.xensource.com
Cc: virtualization@lists.linux-foundation.org
Cc: Shuah Khan <shuahkhan@gmail.com>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
371 lines
9.9 KiB
C
371 lines
9.9 KiB
C
/*
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
* License. See the file "COPYING" in the main directory of this archive
|
|
* for more details.
|
|
*
|
|
* Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
|
|
* Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
|
|
* Copyright (C) 2005 Ilya A. Volynets-Evenbakh <ilya@total-knowledge.com>
|
|
* swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
|
|
* IP32 changes by Ilya.
|
|
* Copyright (C) 2010 Cavium Networks, Inc.
|
|
*/
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/scatterlist.h>
|
|
#include <linux/bootmem.h>
|
|
#include <linux/export.h>
|
|
#include <linux/swiotlb.h>
|
|
#include <linux/types.h>
|
|
#include <linux/init.h>
|
|
#include <linux/mm.h>
|
|
|
|
#include <asm/bootinfo.h>
|
|
|
|
#include <asm/octeon/octeon.h>
|
|
|
|
#ifdef CONFIG_PCI
|
|
#include <asm/octeon/pci-octeon.h>
|
|
#include <asm/octeon/cvmx-npi-defs.h>
|
|
#include <asm/octeon/cvmx-pci-defs.h>
|
|
|
|
static dma_addr_t octeon_hole_phys_to_dma(phys_addr_t paddr)
|
|
{
|
|
if (paddr >= CVMX_PCIE_BAR1_PHYS_BASE && paddr < (CVMX_PCIE_BAR1_PHYS_BASE + CVMX_PCIE_BAR1_PHYS_SIZE))
|
|
return paddr - CVMX_PCIE_BAR1_PHYS_BASE + CVMX_PCIE_BAR1_RC_BASE;
|
|
else
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_hole_dma_to_phys(dma_addr_t daddr)
|
|
{
|
|
if (daddr >= CVMX_PCIE_BAR1_RC_BASE)
|
|
return daddr + CVMX_PCIE_BAR1_PHYS_BASE - CVMX_PCIE_BAR1_RC_BASE;
|
|
else
|
|
return daddr;
|
|
}
|
|
|
|
static dma_addr_t octeon_gen1_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
if (paddr >= 0x410000000ull && paddr < 0x420000000ull)
|
|
paddr -= 0x400000000ull;
|
|
return octeon_hole_phys_to_dma(paddr);
|
|
}
|
|
|
|
static phys_addr_t octeon_gen1_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
daddr = octeon_hole_dma_to_phys(daddr);
|
|
|
|
if (daddr >= 0x10000000ull && daddr < 0x20000000ull)
|
|
daddr += 0x400000000ull;
|
|
|
|
return daddr;
|
|
}
|
|
|
|
static dma_addr_t octeon_gen2_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
return octeon_hole_phys_to_dma(paddr);
|
|
}
|
|
|
|
static phys_addr_t octeon_gen2_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
return octeon_hole_dma_to_phys(daddr);
|
|
}
|
|
|
|
static dma_addr_t octeon_big_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
if (paddr >= 0x410000000ull && paddr < 0x420000000ull)
|
|
paddr -= 0x400000000ull;
|
|
|
|
/* Anything in the BAR1 hole or above goes via BAR2 */
|
|
if (paddr >= 0xf0000000ull)
|
|
paddr = OCTEON_BAR2_PCI_ADDRESS + paddr;
|
|
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_big_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
if (daddr >= OCTEON_BAR2_PCI_ADDRESS)
|
|
daddr -= OCTEON_BAR2_PCI_ADDRESS;
|
|
|
|
if (daddr >= 0x10000000ull && daddr < 0x20000000ull)
|
|
daddr += 0x400000000ull;
|
|
return daddr;
|
|
}
|
|
|
|
static dma_addr_t octeon_small_phys_to_dma(struct device *dev,
|
|
phys_addr_t paddr)
|
|
{
|
|
if (paddr >= 0x410000000ull && paddr < 0x420000000ull)
|
|
paddr -= 0x400000000ull;
|
|
|
|
/* Anything not in the BAR1 range goes via BAR2 */
|
|
if (paddr >= octeon_bar1_pci_phys && paddr < octeon_bar1_pci_phys + 0x8000000ull)
|
|
paddr = paddr - octeon_bar1_pci_phys;
|
|
else
|
|
paddr = OCTEON_BAR2_PCI_ADDRESS + paddr;
|
|
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_small_dma_to_phys(struct device *dev,
|
|
dma_addr_t daddr)
|
|
{
|
|
if (daddr >= OCTEON_BAR2_PCI_ADDRESS)
|
|
daddr -= OCTEON_BAR2_PCI_ADDRESS;
|
|
else
|
|
daddr += octeon_bar1_pci_phys;
|
|
|
|
if (daddr >= 0x10000000ull && daddr < 0x20000000ull)
|
|
daddr += 0x400000000ull;
|
|
return daddr;
|
|
}
|
|
|
|
#endif /* CONFIG_PCI */
|
|
|
|
static dma_addr_t octeon_dma_map_page(struct device *dev, struct page *page,
|
|
unsigned long offset, size_t size, enum dma_data_direction direction,
|
|
struct dma_attrs *attrs)
|
|
{
|
|
dma_addr_t daddr = swiotlb_map_page(dev, page, offset, size,
|
|
direction, attrs);
|
|
mb();
|
|
|
|
return daddr;
|
|
}
|
|
|
|
static int octeon_dma_map_sg(struct device *dev, struct scatterlist *sg,
|
|
int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
|
|
{
|
|
int r = swiotlb_map_sg_attrs(dev, sg, nents, direction, attrs);
|
|
mb();
|
|
return r;
|
|
}
|
|
|
|
static void octeon_dma_sync_single_for_device(struct device *dev,
|
|
dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
|
|
{
|
|
swiotlb_sync_single_for_device(dev, dma_handle, size, direction);
|
|
mb();
|
|
}
|
|
|
|
static void octeon_dma_sync_sg_for_device(struct device *dev,
|
|
struct scatterlist *sg, int nelems, enum dma_data_direction direction)
|
|
{
|
|
swiotlb_sync_sg_for_device(dev, sg, nelems, direction);
|
|
mb();
|
|
}
|
|
|
|
static void *octeon_dma_alloc_coherent(struct device *dev, size_t size,
|
|
dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs)
|
|
{
|
|
void *ret;
|
|
|
|
if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
|
|
return ret;
|
|
|
|
/* ignore region specifiers */
|
|
gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
|
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
if (dev == NULL)
|
|
gfp |= __GFP_DMA;
|
|
else if (dev->coherent_dma_mask <= DMA_BIT_MASK(24))
|
|
gfp |= __GFP_DMA;
|
|
else
|
|
#endif
|
|
#ifdef CONFIG_ZONE_DMA32
|
|
if (dev->coherent_dma_mask <= DMA_BIT_MASK(32))
|
|
gfp |= __GFP_DMA32;
|
|
else
|
|
#endif
|
|
;
|
|
|
|
/* Don't invoke OOM killer */
|
|
gfp |= __GFP_NORETRY;
|
|
|
|
ret = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
|
|
|
|
mb();
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void octeon_dma_free_coherent(struct device *dev, size_t size,
|
|
void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs)
|
|
{
|
|
int order = get_order(size);
|
|
|
|
if (dma_release_from_coherent(dev, order, vaddr))
|
|
return;
|
|
|
|
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
|
|
}
|
|
|
|
static dma_addr_t octeon_unity_phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
return paddr;
|
|
}
|
|
|
|
static phys_addr_t octeon_unity_dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
return daddr;
|
|
}
|
|
|
|
struct octeon_dma_map_ops {
|
|
struct dma_map_ops dma_map_ops;
|
|
dma_addr_t (*phys_to_dma)(struct device *dev, phys_addr_t paddr);
|
|
phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr);
|
|
};
|
|
|
|
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
|
struct octeon_dma_map_ops,
|
|
dma_map_ops);
|
|
|
|
return ops->phys_to_dma(dev, paddr);
|
|
}
|
|
EXPORT_SYMBOL(phys_to_dma);
|
|
|
|
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
|
|
{
|
|
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
|
|
struct octeon_dma_map_ops,
|
|
dma_map_ops);
|
|
|
|
return ops->dma_to_phys(dev, daddr);
|
|
}
|
|
EXPORT_SYMBOL(dma_to_phys);
|
|
|
|
static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
|
|
.dma_map_ops = {
|
|
.alloc = octeon_dma_alloc_coherent,
|
|
.free = octeon_dma_free_coherent,
|
|
.map_page = octeon_dma_map_page,
|
|
.unmap_page = swiotlb_unmap_page,
|
|
.map_sg = octeon_dma_map_sg,
|
|
.unmap_sg = swiotlb_unmap_sg_attrs,
|
|
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
|
|
.sync_single_for_device = octeon_dma_sync_single_for_device,
|
|
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
|
|
.sync_sg_for_device = octeon_dma_sync_sg_for_device,
|
|
.mapping_error = swiotlb_dma_mapping_error,
|
|
.dma_supported = swiotlb_dma_supported
|
|
},
|
|
.phys_to_dma = octeon_unity_phys_to_dma,
|
|
.dma_to_phys = octeon_unity_dma_to_phys
|
|
};
|
|
|
|
char *octeon_swiotlb;
|
|
|
|
void __init plat_swiotlb_setup(void)
|
|
{
|
|
int i;
|
|
phys_t max_addr;
|
|
phys_t addr_size;
|
|
size_t swiotlbsize;
|
|
unsigned long swiotlb_nslabs;
|
|
|
|
max_addr = 0;
|
|
addr_size = 0;
|
|
|
|
for (i = 0 ; i < boot_mem_map.nr_map; i++) {
|
|
struct boot_mem_map_entry *e = &boot_mem_map.map[i];
|
|
if (e->type != BOOT_MEM_RAM && e->type != BOOT_MEM_INIT_RAM)
|
|
continue;
|
|
|
|
/* These addresses map low for PCI. */
|
|
if (e->addr > 0x410000000ull && !OCTEON_IS_MODEL(OCTEON_CN6XXX))
|
|
continue;
|
|
|
|
addr_size += e->size;
|
|
|
|
if (max_addr < e->addr + e->size)
|
|
max_addr = e->addr + e->size;
|
|
|
|
}
|
|
|
|
swiotlbsize = PAGE_SIZE;
|
|
|
|
#ifdef CONFIG_PCI
|
|
/*
|
|
* For OCTEON_DMA_BAR_TYPE_SMALL, size the iotlb at 1/4 memory
|
|
* size to a maximum of 64MB
|
|
*/
|
|
if (OCTEON_IS_MODEL(OCTEON_CN31XX)
|
|
|| OCTEON_IS_MODEL(OCTEON_CN38XX_PASS2)) {
|
|
swiotlbsize = addr_size / 4;
|
|
if (swiotlbsize > 64 * (1<<20))
|
|
swiotlbsize = 64 * (1<<20);
|
|
} else if (max_addr > 0xf0000000ul) {
|
|
/*
|
|
* Otherwise only allocate a big iotlb if there is
|
|
* memory past the BAR1 hole.
|
|
*/
|
|
swiotlbsize = 64 * (1<<20);
|
|
}
|
|
#endif
|
|
#ifdef CONFIG_USB_OCTEON_OHCI
|
|
/* OCTEON II ohci is only 32-bit. */
|
|
if (OCTEON_IS_MODEL(OCTEON_CN6XXX) && max_addr >= 0x100000000ul)
|
|
swiotlbsize = 64 * (1<<20);
|
|
#endif
|
|
swiotlb_nslabs = swiotlbsize >> IO_TLB_SHIFT;
|
|
swiotlb_nslabs = ALIGN(swiotlb_nslabs, IO_TLB_SEGSIZE);
|
|
swiotlbsize = swiotlb_nslabs << IO_TLB_SHIFT;
|
|
|
|
octeon_swiotlb = alloc_bootmem_low_pages(swiotlbsize);
|
|
|
|
if (swiotlb_init_with_tbl(octeon_swiotlb, swiotlb_nslabs, 1) == -ENOMEM)
|
|
panic("Cannot allocate SWIOTLB buffer");
|
|
|
|
mips_dma_map_ops = &octeon_linear_dma_map_ops.dma_map_ops;
|
|
}
|
|
|
|
#ifdef CONFIG_PCI
|
|
static struct octeon_dma_map_ops _octeon_pci_dma_map_ops = {
|
|
.dma_map_ops = {
|
|
.alloc = octeon_dma_alloc_coherent,
|
|
.free = octeon_dma_free_coherent,
|
|
.map_page = octeon_dma_map_page,
|
|
.unmap_page = swiotlb_unmap_page,
|
|
.map_sg = octeon_dma_map_sg,
|
|
.unmap_sg = swiotlb_unmap_sg_attrs,
|
|
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
|
|
.sync_single_for_device = octeon_dma_sync_single_for_device,
|
|
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
|
|
.sync_sg_for_device = octeon_dma_sync_sg_for_device,
|
|
.mapping_error = swiotlb_dma_mapping_error,
|
|
.dma_supported = swiotlb_dma_supported
|
|
},
|
|
};
|
|
|
|
struct dma_map_ops *octeon_pci_dma_map_ops;
|
|
|
|
void __init octeon_pci_dma_init(void)
|
|
{
|
|
switch (octeon_dma_bar_type) {
|
|
case OCTEON_DMA_BAR_TYPE_PCIE2:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_gen2_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_gen2_dma_to_phys;
|
|
break;
|
|
case OCTEON_DMA_BAR_TYPE_PCIE:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_gen1_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_gen1_dma_to_phys;
|
|
break;
|
|
case OCTEON_DMA_BAR_TYPE_BIG:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_big_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_big_dma_to_phys;
|
|
break;
|
|
case OCTEON_DMA_BAR_TYPE_SMALL:
|
|
_octeon_pci_dma_map_ops.phys_to_dma = octeon_small_phys_to_dma;
|
|
_octeon_pci_dma_map_ops.dma_to_phys = octeon_small_dma_to_phys;
|
|
break;
|
|
default:
|
|
BUG();
|
|
}
|
|
octeon_pci_dma_map_ops = &_octeon_pci_dma_map_ops.dma_map_ops;
|
|
}
|
|
#endif /* CONFIG_PCI */
|