Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-22 04:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

IOMMU Updates for Linux v6.2

Browse Source 
Including:
 
 	- Core code:
 	  - map/unmap_pages() cleanup
 	  - SVA and IOPF refactoring
 	  - Clean up and document return codes from device/domain
 	    attachment code
 
 	- AMD driver:
 	  - Rework and extend parsing code for ivrs_ioapic, ivrs_hpet
 	    and ivrs_acpihid command line options
 	  - Some smaller cleanups
 
 	- Intel driver:
 	  - Blocking domain support
 	  - Cleanups
 
 	- S390 driver:
 	  - Fixes and improvements for attach and aperture handling
 
 	- PAMU driver:
 	  - Resource leak fix and cleanup
 
 	- Rockchip driver:
 	  - Page table permission bit fix
 
 	- Mediatek driver:
 	  - Improve safety from invalid dts input
 	  - Smaller fixes and improvements
 
 	- Exynos driver:
 	  - Fix driver initialization sequence
 
 	- Sun50i driver:
 	  - Remove IOMMU_DOMAIN_IDENTITY as it has not been working
 	    forever
 	  - Various other fixes
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEr9jSbILcajRFYWYyK/BELZcBGuMFAmOd1PQACgkQK/BELZcB
 GuO7NxAAiwJUO99pTwvqnByzcC783AuE/fqKHDb9DZaN6Cr0VXSbKEwm8Lc2PC00
 2CTwK/zGhy8BKBQnPiooJ+YOMPjE4yhFIF9jr5ASH5AVWv8EEFpo8zIFKAcF5rh/
 c2Y5RIUwsGXuhR7U3lMTw84r39TZG2eHPwTEU6KvEJ1LCOMyD8IBYrZK2rvpGpem
 3swXUfF5bQGAT8LlIFN7p+qsVs6ZtuD40qre3kerjrBtCPUMlxIIV5TJ8oQTecsk
 vKpD51mEVW+rjUKvqui8NDYuPfT76F2FPS37dfA1F36p8dmsMGSrtWngNm73r546
 AmY8Gui6wKsv4Qn7Mxv49f/WZIXzdRTXOKx/zhYvvGxu7keqQIRIWYcLSxqfaGku
 cqJT401Ws1NHmRpx/t90lMH/anY5+kUMRTQG9Iq5ruLhExskd0SJcffa1i7YIGIe
 lPCTDf7MOXfDudR0Dtp87pGZQBaSkrSzZvb7qZY3Bj83WGZnLPpl6Z3N8KbkGzEO
 zNNvv1CtxZnIPrdOaKvfxQlAKiWKxkPRHuqk1TE8hkoNOe5ZgdOSJP5SeCrZ5tEf
 qljPXvDVF9f8CYw7QlfEDnbLnqDMGZpPAGqKPItbaijQLPZx4Jm4dw6+7i9hETIa
 wJ+1R9iAf+qiR0rlqueALKRaI4DjE8RU8yYSDpn2kn0BUOhWmb8=
 =ZM/m
 -----END PGP SIGNATURE-----

Merge tag 'iommu-updates-v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu updates from Joerg Roedel:
 "Core code:
   - map/unmap_pages() cleanup
   - SVA and IOPF refactoring
   - Clean up and document return codes from device/domain attachment

  AMD driver:
   - Rework and extend parsing code for ivrs_ioapic, ivrs_hpet and
     ivrs_acpihid command line options
   - Some smaller cleanups

  Intel driver:
   - Blocking domain support
   - Cleanups

  S390 driver:
   - Fixes and improvements for attach and aperture handling

  PAMU driver:
   - Resource leak fix and cleanup

  Rockchip driver:
   - Page table permission bit fix

  Mediatek driver:
   - Improve safety from invalid dts input
   - Smaller fixes and improvements

  Exynos driver:
   - Fix driver initialization sequence

  Sun50i driver:
   - Remove IOMMU_DOMAIN_IDENTITY as it has not been working forever
   - Various other fixes"

* tag 'iommu-updates-v6.2' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (74 commits)
  iommu/mediatek: Fix forever loop in error handling
  iommu/mediatek: Fix crash on isr after kexec()
  iommu/sun50i: Remove IOMMU_DOMAIN_IDENTITY
  iommu/amd: Fix typo in macro parameter name
  iommu/mediatek: Remove unused "mapping" member from mtk_iommu_data
  iommu/mediatek: Improve safety for mediatek,smi property in larb nodes
  iommu/mediatek: Validate number of phandles associated with "mediatek,larbs"
  iommu/mediatek: Add error path for loop of mm_dts_parse
  iommu/mediatek: Use component_match_add
  iommu/mediatek: Add platform_device_put for recovering the device refcnt
  iommu/fsl_pamu: Fix resource leak in fsl_pamu_probe()
  iommu/vt-d: Use real field for indication of first level
  iommu/vt-d: Remove unnecessary domain_context_mapped()
  iommu/vt-d: Rename domain_add_dev_info()
  iommu/vt-d: Rename iommu_disable_dev_iotlb()
  iommu/vt-d: Add blocking domain support
  iommu/vt-d: Add device_block_translation() helper
  iommu/vt-d: Allocate pasid table in device probe path
  iommu/amd: Check return value of mmu_notifier_register()
  iommu/amd: Fix pci device refcount leak in ppr_notifier()
  ...
This commit is contained in:
Linus Torvalds 2022-12-19 08:34:39 -06:00
commit b8fd76f418
32 changed files with 1178 additions and 663 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Documentation/admin-guide/kernel-parameters.txt
View File

@ -2313,7 +2313,13 @@
Provide an override to the IOAPIC-ID<->DEVICE-ID
mapping provided in the IVRS ACPI table.
By default, PCI segment is 0, and can be omitted.
For example:
For example, to map IOAPIC-ID decimal 10 to
PCI segment 0x1 and PCI device 00:14.0,
write the parameter as:
ivrs_ioapic=10@0001:00:14.0
Deprecated formats:
* To map IOAPIC-ID decimal 10 to PCI device 00:14.0
write the parameter as:
ivrs_ioapic[10]=00:14.0
@ -2325,7 +2331,13 @@
Provide an override to the HPET-ID<->DEVICE-ID
mapping provided in the IVRS ACPI table.
By default, PCI segment is 0, and can be omitted.
For example:
For example, to map HPET-ID decimal 10 to
PCI segment 0x1 and PCI device 00:14.0,
write the parameter as:
ivrs_hpet=10@0001:00:14.0
Deprecated formats:
* To map HPET-ID decimal 0 to PCI device 00:14.0
write the parameter as:
ivrs_hpet[0]=00:14.0
@ -2336,15 +2348,20 @@
ivrs_acpihid [HW,X86-64]
Provide an override to the ACPI-HID:UID<->DEVICE-ID
mapping provided in the IVRS ACPI table.
By default, PCI segment is 0, and can be omitted.
For example, to map UART-HID:UID AMD0020:0 to
PCI segment 0x1 and PCI device ID 00:14.5,
write the parameter as:
ivrs_acpihid[0001:00:14.5]=AMD0020:0
ivrs_acpihid=AMD0020:0@0001:00:14.5
By default, PCI segment is 0, and can be omitted.
For example, PCI device 00:14.5 write the parameter as:
Deprecated formats:
* To map UART-HID:UID AMD0020:0 to PCI segment is 0,
PCI device ID 00:14.5, write the parameter as:
ivrs_acpihid[00:14.5]=AMD0020:0
* To map UART-HID:UID AMD0020:0 to PCI segment 0x1 and
PCI device ID 00:14.5, write the parameter as:
ivrs_acpihid[0001:00:14.5]=AMD0020:0
js= [HW,JOY] Analog joystick
See Documentation/input/joydev/joystick.rst.

182
Documentation/devicetree/bindings/iommu/arm,smmu.yaml
View File

@ -28,10 +28,42 @@ properties:
- enum:
- qcom,msm8996-smmu-v2
- qcom,msm8998-smmu-v2
- qcom,sdm630-smmu-v2
- const: qcom,smmu-v2
- description: Qcom SoCs implementing "arm,mmu-500"
- description: Qcom SoCs implementing "qcom,smmu-500" and "arm,mmu-500"
items:
- enum:
- qcom,qcm2290-smmu-500
- qcom,qdu1000-smmu-500
- qcom,sc7180-smmu-500
- qcom,sc7280-smmu-500
- qcom,sc8180x-smmu-500
- qcom,sc8280xp-smmu-500
- qcom,sdm670-smmu-500
- qcom,sdm845-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6350-smmu-500
- qcom,sm6375-smmu-500
- qcom,sm8150-smmu-500
- qcom,sm8250-smmu-500
- qcom,sm8350-smmu-500
- qcom,sm8450-smmu-500
- const: qcom,smmu-500
- const: arm,mmu-500
- description: Qcom SoCs implementing "arm,mmu-500" (non-qcom implementation)
deprecated: true
items:
- enum:
- qcom,sdx55-smmu-500
- qcom,sdx65-smmu-500
- const: arm,mmu-500
- description: Qcom SoCs implementing "arm,mmu-500" (legacy binding)
deprecated: true
items:
# Do not add additional SoC to this list. Instead use two previous lists.
- enum:
- qcom,qcm2290-smmu-500
- qcom,sc7180-smmu-500
@ -39,8 +71,7 @@ properties:
- qcom,sc8180x-smmu-500
- qcom,sc8280xp-smmu-500
- qcom,sdm845-smmu-500
- qcom,sdx55-smmu-500
- qcom,sdx65-smmu-500
- qcom,sm6115-smmu-500
- qcom,sm6350-smmu-500
- qcom,sm6375-smmu-500
- qcom,sm8150-smmu-500
@ -48,13 +79,28 @@ properties:
- qcom,sm8350-smmu-500
- qcom,sm8450-smmu-500
- const: arm,mmu-500
- description: Qcom Adreno GPUs implementing "arm,smmu-500"
items:
- enum:
- qcom,sc7280-smmu-500
- qcom,sm8250-smmu-500
- const: qcom,adreno-smmu
- const: arm,mmu-500
- description: Qcom Adreno GPUs implementing "arm,smmu-v2"
items:
- enum:
- qcom,msm8996-smmu-v2
- qcom,sc7180-smmu-v2
- qcom,sdm630-smmu-v2
- qcom,sdm845-smmu-v2
- qcom,sm6350-smmu-v2
- const: qcom,adreno-smmu
- const: qcom,smmu-v2
- description: Qcom Adreno GPUs on Google Cheza platform
items:
- const: qcom,sdm845-smmu-v2
- const: qcom,smmu-v2
- description: Marvell SoCs implementing "arm,mmu-500"
items:
- const: marvell,ap806-smmu-500
@ -147,16 +193,12 @@ properties:
present in such cases.
clock-names:
items:
- const: bus
- const: iface
minItems: 1
maxItems: 7
clocks:
items:
- description: bus clock required for downstream bus access and for the
smmu ptw
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
minItems: 1
maxItems: 7
power-domains:
maxItems: 1
@ -206,6 +248,124 @@ allOf:
reg:
maxItems: 1
- if:
properties:
compatible:
contains:
enum:
- qcom,msm8998-smmu-v2
- qcom,sdm630-smmu-v2
then:
anyOf:
- properties:
clock-names:
items:
- const: bus
clocks:
items:
- description: bus clock required for downstream bus access and for
the smmu ptw
- properties:
clock-names:
items:
- const: iface
- const: mem
- const: mem_iface
clocks:
items:
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
- description: bus clock required for memory access
- description: bus clock required for GPU memory access
- properties:
clock-names:
items:
- const: iface-mm
- const: iface-smmu
- const: bus-mm
- const: bus-smmu
clocks:
items:
- description: interface clock required to access mnoc's registers
through the TCU's programming interface.
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
- description: bus clock required for downstream bus access
- description: bus clock required for the smmu ptw
- if:
properties:
compatible:
contains:
enum:
- qcom,msm8996-smmu-v2
- qcom,sc7180-smmu-v2
- qcom,sdm845-smmu-v2
then:
properties:
clock-names:
items:
- const: bus
- const: iface
clocks:
items:
- description: bus clock required for downstream bus access and for
the smmu ptw
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
- if:
properties:
compatible:
contains:
const: qcom,sc7280-smmu-500
then:
properties:
clock-names:
items:
- const: gcc_gpu_memnoc_gfx_clk
- const: gcc_gpu_snoc_dvm_gfx_clk
- const: gpu_cc_ahb_clk
- const: gpu_cc_hlos1_vote_gpu_smmu_clk
- const: gpu_cc_cx_gmu_clk
- const: gpu_cc_hub_cx_int_clk
- const: gpu_cc_hub_aon_clk
clocks:
items:
- description: GPU memnoc_gfx clock
- description: GPU snoc_dvm_gfx clock
- description: GPU ahb clock
- description: GPU hlos1_vote_GPU smmu clock
- description: GPU cx_gmu clock
- description: GPU hub_cx_int clock
- description: GPU hub_aon clock
- if:
properties:
compatible:
contains:
enum:
- qcom,sm6350-smmu-v2
- qcom,sm8150-smmu-500
- qcom,sm8250-smmu-500
then:
properties:
clock-names:
items:
- const: ahb
- const: bus
- const: iface
clocks:
items:
- description: bus clock required for AHB bus access
- description: bus clock required for downstream bus access and for
the smmu ptw
- description: interface clock required to access smmu's registers
through the TCU's programming interface.
examples:
- |+
/* SMMU with stream matching or stream indexing */

2
Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml
View File

@ -82,6 +82,7 @@ properties:
- mediatek,mt8195-iommu-vdo # generation two
- mediatek,mt8195-iommu-vpp # generation two
- mediatek,mt8195-iommu-infra # generation two
- mediatek,mt8365-m4u # generation two
- description: mt7623 generation one
items:
@ -132,6 +133,7 @@ properties:
dt-binding/memory/mt8186-memory-port.h for mt8186,
dt-binding/memory/mt8192-larb-port.h for mt8192.
dt-binding/memory/mt8195-memory-port.h for mt8195.
dt-binding/memory/mediatek,mt8365-larb-port.h for mt8365.
power-domains:
maxItems: 1

5
arch/s390/include/asm/pci.h
View File

@ -117,7 +117,9 @@ struct zpci_bus {
struct zpci_dev {
struct zpci_bus *zbus;
struct list_head entry; /* list of all zpci_devices, needed for hotplug, etc. */
struct list_head iommu_list;
struct kref kref;
struct rcu_head rcu;
struct hotplug_slot hotplug_slot;
enum zpci_state state;
@ -155,7 +157,6 @@ struct zpci_dev {
/* DMA stuff */
unsigned long *dma_table;
spinlock_t dma_table_lock;
int tlb_refresh;
spinlock_t iommu_bitmap_lock;
@ -220,7 +221,7 @@ void zpci_device_reserved(struct zpci_dev *zdev);
bool zpci_is_device_configured(struct zpci_dev *zdev);
int zpci_hot_reset_device(struct zpci_dev *zdev);
int zpci_register_ioat(struct zpci_dev *, u8, u64, u64, u64);
int zpci_register_ioat(struct zpci_dev *, u8, u64, u64, u64, u8 *);
int zpci_unregister_ioat(struct zpci_dev *, u8);
void zpci_remove_reserved_devices(void);
void zpci_update_fh(struct zpci_dev *zdev, u32 fh);

6
arch/s390/kvm/pci.c
View File

@ -434,6 +434,7 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev)
static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
{
struct zpci_dev *zdev = opaque;
u8 status;
int rc;
if (!zdev)
@ -486,7 +487,7 @@ static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm)
/* Re-register the IOMMU that was already created */
rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
virt_to_phys(zdev->dma_table), &status);
if (rc)
goto clear_gisa;
@ -516,6 +517,7 @@ static void kvm_s390_pci_unregister_kvm(void *opaque)
{
struct zpci_dev *zdev = opaque;
struct kvm *kvm;
u8 status;
if (!zdev)
return;
@ -554,7 +556,7 @@ static void kvm_s390_pci_unregister_kvm(void *opaque)
/* Re-register the IOMMU that was already created */
zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
virt_to_phys(zdev->dma_table), &status);
out:
spin_lock(&kvm->arch.kzdev_list_lock);

13
arch/s390/pci/pci.c
View File

@ -116,20 +116,20 @@ EXPORT_SYMBOL_GPL(pci_proc_domain);
/* Modify PCI: Register I/O address translation parameters */
int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
u64 base, u64 limit, u64 iota)
u64 base, u64 limit, u64 iota, u8 *status)
{
u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
struct zpci_fib fib = {0};
u8 cc, status;
u8 cc;
WARN_ON_ONCE(iota & 0x3fff);
fib.pba = base;
fib.pal = limit;
fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
fib.gd = zdev->gisa;
cc = zpci_mod_fc(req, &fib, &status);
cc = zpci_mod_fc(req, &fib, status);
if (cc)
zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);
zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status);
return cc;
}
EXPORT_SYMBOL_GPL(zpci_register_ioat);
@ -764,6 +764,7 @@ EXPORT_SYMBOL_GPL(zpci_disable_device);
*/
int zpci_hot_reset_device(struct zpci_dev *zdev)
{
u8 status;
int rc;
zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);
@ -787,7 +788,7 @@ int zpci_hot_reset_device(struct zpci_dev *zdev)
if (zdev->dma_table)
rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
virt_to_phys(zdev->dma_table), &status);
else
rc = zpci_dma_init_device(zdev);
if (rc) {
@ -995,7 +996,7 @@ void zpci_release_device(struct kref *kref)
break;
}
zpci_dbg(3, "rem fid:%x\n", zdev->fid);
kfree(zdev);
kfree_rcu(zdev, rcu);
}
int zpci_report_error(struct pci_dev *pdev,

77
arch/s390/pci/pci_dma.c
View File

@ -63,37 +63,55 @@ static void dma_free_page_table(void *table)
kmem_cache_free(dma_page_table_cache, table);
}
static unsigned long *dma_get_seg_table_origin(unsigned long *entry)
static unsigned long *dma_get_seg_table_origin(unsigned long *rtep)
{
unsigned long old_rte, rte;
unsigned long *sto;
if (reg_entry_isvalid(*entry))
sto = get_rt_sto(*entry);
else {
rte = READ_ONCE(*rtep);
if (reg_entry_isvalid(rte)) {
sto = get_rt_sto(rte);
} else {
sto = dma_alloc_cpu_table();
if (!sto)
return NULL;
set_rt_sto(entry, virt_to_phys(sto));
validate_rt_entry(entry);
entry_clr_protected(entry);
set_rt_sto(&rte, virt_to_phys(sto));
validate_rt_entry(&rte);
entry_clr_protected(&rte);
old_rte = cmpxchg(rtep, ZPCI_TABLE_INVALID, rte);
if (old_rte != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_cpu_table(sto);
sto = get_rt_sto(old_rte);
}
}
return sto;
}
static unsigned long *dma_get_page_table_origin(unsigned long *entry)
static unsigned long *dma_get_page_table_origin(unsigned long *step)
{
unsigned long old_ste, ste;
unsigned long *pto;
if (reg_entry_isvalid(*entry))
pto = get_st_pto(*entry);
else {
ste = READ_ONCE(*step);
if (reg_entry_isvalid(ste)) {
pto = get_st_pto(ste);
} else {
pto = dma_alloc_page_table();
if (!pto)
return NULL;
set_st_pto(entry, virt_to_phys(pto));
validate_st_entry(entry);
entry_clr_protected(entry);
set_st_pto(&ste, virt_to_phys(pto));
validate_st_entry(&ste);
entry_clr_protected(&ste);
old_ste = cmpxchg(step, ZPCI_TABLE_INVALID, ste);
if (old_ste != ZPCI_TABLE_INVALID) {
/* Somone else was faster, use theirs */
dma_free_page_table(pto);
pto = get_st_pto(old_ste);
}
}
return pto;
}
@ -117,19 +135,24 @@ unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr)
return &pto[px];
}
void dma_update_cpu_trans(unsigned long *entry, phys_addr_t page_addr, int flags)
void dma_update_cpu_trans(unsigned long *ptep, phys_addr_t page_addr, int flags)
{
unsigned long pte;
pte = READ_ONCE(*ptep);
if (flags & ZPCI_PTE_INVALID) {
invalidate_pt_entry(entry);
invalidate_pt_entry(&pte);
} else {
set_pt_pfaa(entry, page_addr);
validate_pt_entry(entry);
set_pt_pfaa(&pte, page_addr);
validate_pt_entry(&pte);
}
if (flags & ZPCI_TABLE_PROTECTED)
entry_set_protected(entry);
entry_set_protected(&pte);
else
entry_clr_protected(entry);
entry_clr_protected(&pte);
xchg(ptep, pte);
}
static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
@ -137,18 +160,14 @@ static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
{
unsigned int nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
phys_addr_t page_addr = (pa & PAGE_MASK);
unsigned long irq_flags;
unsigned long *entry;
int i, rc = 0;
if (!nr_pages)
return -EINVAL;
spin_lock_irqsave(&zdev->dma_table_lock, irq_flags);
if (!zdev->dma_table) {
rc = -EINVAL;
goto out_unlock;
}
if (!zdev->dma_table)
return -EINVAL;
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
@ -173,8 +192,6 @@ undo_cpu_trans:
dma_update_cpu_trans(entry, page_addr, flags);
}
}
out_unlock:
spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags);
return rc;
}
@ -547,6 +564,7 @@ static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
int zpci_dma_init_device(struct zpci_dev *zdev)
{
u8 status;
int rc;
/*
@ -557,7 +575,6 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
WARN_ON(zdev->s390_domain);
spin_lock_init(&zdev->iommu_bitmap_lock);
spin_lock_init(&zdev->dma_table_lock);
zdev->dma_table = dma_alloc_cpu_table();
if (!zdev->dma_table) {
@ -598,7 +615,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
}
if (zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table))) {
virt_to_phys(zdev->dma_table), &status)) {
rc = -EIO;
goto free_bitmap;
}

86
drivers/iommu/amd/init.c
View File

@ -85,7 +85,7 @@
#define LOOP_TIMEOUT 2000000
#define IVRS_GET_SBDF_ID(seg, bus, dev, fd) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
#define IVRS_GET_SBDF_ID(seg, bus, dev, fn) (((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
| ((dev & 0x1f) << 3) | (fn & 0x7))
/*
@ -3402,18 +3402,24 @@ static int __init parse_amd_iommu_options(char *str)
static int __init parse_ivrs_ioapic(char *str)
{
u32 seg = 0, bus, dev, fn;
int ret, id, i;
int id, i;
u32 devid;
ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
if (ret != 4) {
ret = sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn);
if (ret != 5) {
pr_err("Invalid command line: ivrs_ioapic%s\n", str);
return 1;
}
if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
goto found;
if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
str, id, seg, bus, dev, fn);
goto found;
}
pr_err("Invalid command line: ivrs_ioapic%s\n", str);
return 1;
found:
if (early_ioapic_map_size == EARLY_MAP_SIZE) {
pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
str);
@ -3434,18 +3440,24 @@ static int __init parse_ivrs_ioapic(char *str)
static int __init parse_ivrs_hpet(char *str)
{
u32 seg = 0, bus, dev, fn;
int ret, id, i;
int id, i;
u32 devid;
ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
if (ret != 4) {
ret = sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn);
if (ret != 5) {
pr_err("Invalid command line: ivrs_hpet%s\n", str);
return 1;
}
if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
goto found;
if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
str, id, seg, bus, dev, fn);
goto found;
}
pr_err("Invalid command line: ivrs_hpet%s\n", str);
return 1;
found:
if (early_hpet_map_size == EARLY_MAP_SIZE) {
pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
str);
@ -3466,19 +3478,36 @@ static int __init parse_ivrs_hpet(char *str)
static int __init parse_ivrs_acpihid(char *str)
{
u32 seg = 0, bus, dev, fn;
char *hid, *uid, *p;
char *hid, *uid, *p, *addr;
char acpiid[ACPIHID_UID_LEN + ACPIHID_HID_LEN] = {0};
int ret, i;
int i;
ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid);
if (ret != 4) {
ret = sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid);
if (ret != 5) {
pr_err("Invalid command line: ivrs_acpihid(%s)\n", str);
return 1;
addr = strchr(str, '@');
if (!addr) {
if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
str, acpiid, seg, bus, dev, fn);
goto found;
}
goto not_found;
}
/* We have the '@', make it the terminator to get just the acpiid */
*addr++ = 0;
if (sscanf(str, "=%s", acpiid) != 1)
goto not_found;
if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
goto found;
not_found:
pr_err("Invalid command line: ivrs_acpihid%s\n", str);
return 1;
found:
p = acpiid;
hid = strsep(&p, ":");
uid = p;
@ -3488,6 +3517,13 @@ static int __init parse_ivrs_acpihid(char *str)
return 1;
}
/*
* Ignore leading zeroes after ':', so e.g., AMDI0095:00
* will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
*/
while (*uid == '0' && *(uid + 1))
uid++;
i = early_acpihid_map_size++;
memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
memcpy(early_acpihid_map[i].uid, uid, strlen(uid));

3
drivers/iommu/amd/iommu.c
View File

@ -767,7 +767,7 @@ EXPORT_SYMBOL(amd_iommu_register_ga_log_notifier);
static void iommu_poll_ga_log(struct amd_iommu *iommu)
{
u32 head, tail, cnt = 0;
u32 head, tail;
if (iommu->ga_log == NULL)
return;
@ -780,7 +780,6 @@ static void iommu_poll_ga_log(struct amd_iommu *iommu)
u64 log_entry;
raw = (u64 *)(iommu->ga_log + head);
cnt++;
/* Avoid memcpy function-call overhead */
log_entry = *raw;

5
drivers/iommu/amd/iommu_v2.c
View File

@ -587,6 +587,7 @@ out_drop_state:
put_device_state(dev_state);
out:
pci_dev_put(pdev);
return ret;
}
@ -639,7 +640,9 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
if (pasid_state->mm == NULL)
goto out_free;
mmu_notifier_register(&pasid_state->mn, mm);
ret = mmu_notifier_register(&pasid_state->mn, mm);
if (ret)
goto out_free;
ret = set_pasid_state(dev_state, pasid_state, pasid);
if (ret)

3
drivers/iommu/arm/arm-smmu/arm-smmu-impl.c
View File

@ -136,6 +136,9 @@ int arm_mmu500_reset(struct arm_smmu_device *smmu)
reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
reg &= ~ARM_MMU500_ACTLR_CPRE;
arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
if (reg & ARM_MMU500_ACTLR_CPRE)
dev_warn_once(smmu->dev, "Failed to disable prefetcher [errata #841119 and #826419], check ACR.CACHE_LOCK\n");
}
return 0;

91
drivers/iommu/arm/arm-smmu/arm-smmu-qcom-debug.c
View File

@ -10,16 +10,6 @@
#include "arm-smmu.h"
#include "arm-smmu-qcom.h"
enum qcom_smmu_impl_reg_offset {
QCOM_SMMU_TBU_PWR_STATUS,
QCOM_SMMU_STATS_SYNC_INV_TBU_ACK,
QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR,
};
struct qcom_smmu_config {
const u32 *reg_offset;
};
void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu)
{
int ret;
@ -59,84 +49,3 @@ void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu)
tbu_pwr_status, sync_inv_ack, sync_inv_progress);
}
}
/* Implementation Defined Register Space 0 register offsets */
static const u32 qcom_smmu_impl0_reg_offset[] = {
[QCOM_SMMU_TBU_PWR_STATUS] = 0x2204,
[QCOM_SMMU_STATS_SYNC_INV_TBU_ACK] = 0x25dc,
[QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR] = 0x2670,
};
static const struct qcom_smmu_config qcm2290_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc7180_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc7280_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc8180x_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sc8280xp_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm6125_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm6350_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8150_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8250_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8350_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct qcom_smmu_config sm8450_smmu_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
static const struct of_device_id __maybe_unused qcom_smmu_impl_debug_match[] = {
{ .compatible = "qcom,msm8998-smmu-v2" },
{ .compatible = "qcom,qcm2290-smmu-500", .data = &qcm2290_smmu_cfg },
{ .compatible = "qcom,sc7180-smmu-500", .data = &sc7180_smmu_cfg },
{ .compatible = "qcom,sc7280-smmu-500", .data = &sc7280_smmu_cfg},
{ .compatible = "qcom,sc8180x-smmu-500", .data = &sc8180x_smmu_cfg },
{ .compatible = "qcom,sc8280xp-smmu-500", .data = &sc8280xp_smmu_cfg },
{ .compatible = "qcom,sdm630-smmu-v2" },
{ .compatible = "qcom,sdm845-smmu-500" },
{ .compatible = "qcom,sm6125-smmu-500", .data = &sm6125_smmu_cfg},
{ .compatible = "qcom,sm6350-smmu-500", .data = &sm6350_smmu_cfg},
{ .compatible = "qcom,sm8150-smmu-500", .data = &sm8150_smmu_cfg },
{ .compatible = "qcom,sm8250-smmu-500", .data = &sm8250_smmu_cfg },
{ .compatible = "qcom,sm8350-smmu-500", .data = &sm8350_smmu_cfg },
{ .compatible = "qcom,sm8450-smmu-500", .data = &sm8450_smmu_cfg },
{ }
};
const void *qcom_smmu_impl_data(struct arm_smmu_device *smmu)
{
const struct of_device_id *match;
const struct device_node *np = smmu->dev->of_node;
match = of_match_node(qcom_smmu_impl_debug_match, np);
if (!match)
return NULL;
return match->data;
}

161
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c
View File

@ -361,6 +361,8 @@ static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
{
int ret;
arm_mmu500_reset(smmu);
/*
* To address performance degradation in non-real time clients,
* such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
@ -374,41 +376,67 @@ static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
return ret;
}
static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
{
const struct device_node *np = smmu->dev->of_node;
arm_mmu500_reset(smmu);
if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
return qcom_sdm845_smmu500_reset(smmu);
return 0;
}
static const struct arm_smmu_impl qcom_smmu_impl = {
static const struct arm_smmu_impl qcom_smmu_v2_impl = {
.init_context = qcom_smmu_init_context,
.cfg_probe = qcom_smmu_cfg_probe,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = qcom_smmu500_reset,
.write_s2cr = qcom_smmu_write_s2cr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
static const struct arm_smmu_impl qcom_smmu_500_impl = {
.init_context = qcom_smmu_init_context,
.cfg_probe = qcom_smmu_cfg_probe,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = arm_mmu500_reset,
.write_s2cr = qcom_smmu_write_s2cr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl sdm845_smmu_500_impl = {
.init_context = qcom_smmu_init_context,
.cfg_probe = qcom_smmu_cfg_probe,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = qcom_sdm845_smmu500_reset,
.write_s2cr = qcom_smmu_write_s2cr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl qcom_adreno_smmu_v2_impl = {
.init_context = qcom_adreno_smmu_init_context,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = qcom_smmu500_reset,
.alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
.write_sctlr = qcom_adreno_smmu_write_sctlr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static const struct arm_smmu_impl qcom_adreno_smmu_500_impl = {
.init_context = qcom_adreno_smmu_init_context,
.def_domain_type = qcom_smmu_def_domain_type,
.reset = arm_mmu500_reset,
.alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
.write_sctlr = qcom_adreno_smmu_write_sctlr,
.tlb_sync = qcom_smmu_tlb_sync,
};
static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
const struct arm_smmu_impl *impl)
const struct qcom_smmu_match_data *data)
{
const struct device_node *np = smmu->dev->of_node;
const struct arm_smmu_impl *impl;
struct qcom_smmu *qsmmu;
if (!data)
return ERR_PTR(-EINVAL);
if (np && of_device_is_compatible(np, "qcom,adreno-smmu"))
impl = data->adreno_impl;
else
impl = data->impl;
if (!impl)
return smmu;
/* Check to make sure qcom_scm has finished probing */
if (!qcom_scm_is_available())
return ERR_PTR(-EPROBE_DEFER);
@ -418,27 +446,77 @@ static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
return ERR_PTR(-ENOMEM);
qsmmu->smmu.impl = impl;
qsmmu->cfg = qcom_smmu_impl_data(smmu);
qsmmu->cfg = data->cfg;
return &qsmmu->smmu;
}
/* Implementation Defined Register Space 0 register offsets */
static const u32 qcom_smmu_impl0_reg_offset[] = {
[QCOM_SMMU_TBU_PWR_STATUS] = 0x2204,
[QCOM_SMMU_STATS_SYNC_INV_TBU_ACK] = 0x25dc,
[QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR] = 0x2670,
};
static const struct qcom_smmu_config qcom_smmu_impl0_cfg = {
.reg_offset = qcom_smmu_impl0_reg_offset,
};
/*
* It is not yet possible to use MDP SMMU with the bypass quirk on the msm8996,
* there are not enough context banks.
*/
static const struct qcom_smmu_match_data msm8996_smmu_data = {
.impl = NULL,
.adreno_impl = &qcom_adreno_smmu_v2_impl,
};
static const struct qcom_smmu_match_data qcom_smmu_v2_data = {
.impl = &qcom_smmu_v2_impl,
.adreno_impl = &qcom_adreno_smmu_v2_impl,
};
static const struct qcom_smmu_match_data sdm845_smmu_500_data = {
.impl = &sdm845_smmu_500_impl,
/*
* No need for adreno impl here. On sdm845 the Adreno SMMU is handled
* by the separate sdm845-smmu-v2 device.
*/
/* Also no debug configuration. */
};
static const struct qcom_smmu_match_data qcom_smmu_500_impl0_data = {
.impl = &qcom_smmu_500_impl,
.adreno_impl = &qcom_adreno_smmu_500_impl,
.cfg = &qcom_smmu_impl0_cfg,
};
/*
* Do not add any more qcom,SOC-smmu-500 entries to this list, unless they need
* special handling and can not be covered by the qcom,smmu-500 entry.
*/
static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
{ .compatible = "qcom,msm8998-smmu-v2" },
{ .compatible = "qcom,qcm2290-smmu-500" },
{ .compatible = "qcom,sc7180-smmu-500" },
{ .compatible = "qcom,sc7280-smmu-500" },
{ .compatible = "qcom,sc8180x-smmu-500" },
{ .compatible = "qcom,sc8280xp-smmu-500" },
{ .compatible = "qcom,sdm630-smmu-v2" },
{ .compatible = "qcom,sdm845-smmu-500" },
{ .compatible = "qcom,sm6125-smmu-500" },
{ .compatible = "qcom,sm6350-smmu-500" },
{ .compatible = "qcom,sm6375-smmu-500" },
{ .compatible = "qcom,sm8150-smmu-500" },
{ .compatible = "qcom,sm8250-smmu-500" },
{ .compatible = "qcom,sm8350-smmu-500" },
{ .compatible = "qcom,sm8450-smmu-500" },
{ .compatible = "qcom,msm8996-smmu-v2", .data = &msm8996_smmu_data },
{ .compatible = "qcom,msm8998-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,qcm2290-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,qdu1000-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc7180-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc7280-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc8180x-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sc8280xp-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sdm630-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sdm845-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sdm845-smmu-500", .data = &sdm845_smmu_500_data },
{ .compatible = "qcom,sm6115-smmu-500", .data = &qcom_smmu_500_impl0_data},
{ .compatible = "qcom,sm6125-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm6350-smmu-v2", .data = &qcom_smmu_v2_data },
{ .compatible = "qcom,sm6350-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm6375-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8150-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8250-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8350-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,sm8450-smmu-500", .data = &qcom_smmu_500_impl0_data },
{ .compatible = "qcom,smmu-500", .data = &qcom_smmu_500_impl0_data },
{ }
};
@ -453,26 +531,19 @@ static struct acpi_platform_list qcom_acpi_platlist[] = {
struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
{
const struct device_node *np = smmu->dev->of_node;
const struct of_device_id *match;
#ifdef CONFIG_ACPI
if (np == NULL) {
/* Match platform for ACPI boot */
if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
return qcom_smmu_create(smmu, &qcom_smmu_impl);
return qcom_smmu_create(smmu, &qcom_smmu_500_impl0_data);
}
#endif
/*
* Do not change this order of implementation, i.e., first adreno
* smmu impl and then apss smmu since we can have both implementing
* arm,mmu-500 in which case we will miss setting adreno smmu specific
* features if the order is changed.
*/
if (of_device_is_compatible(np, "qcom,adreno-smmu"))
return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
if (of_match_node(qcom_smmu_impl_of_match, np))
return qcom_smmu_create(smmu, &qcom_smmu_impl);
match = of_match_node(qcom_smmu_impl_of_match, np);
if (match)
return qcom_smmu_create(smmu, match->data);
return smmu;
}

21
drivers/iommu/arm/arm-smmu/arm-smmu-qcom.h
View File

@ -14,15 +14,26 @@ struct qcom_smmu {
u32 stall_enabled;
};
enum qcom_smmu_impl_reg_offset {
QCOM_SMMU_TBU_PWR_STATUS,
QCOM_SMMU_STATS_SYNC_INV_TBU_ACK,
QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR,
};
struct qcom_smmu_config {
const u32 *reg_offset;
};
struct qcom_smmu_match_data {
const struct qcom_smmu_config *cfg;
const struct arm_smmu_impl *impl;
const struct arm_smmu_impl *adreno_impl;
};
#ifdef CONFIG_ARM_SMMU_QCOM_DEBUG
void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu);
const void *qcom_smmu_impl_data(struct arm_smmu_device *smmu);
#else
static inline void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu) { }
static inline const void *qcom_smmu_impl_data(struct arm_smmu_device *smmu)
{
return NULL;
}
#endif
#endif /* _ARM_SMMU_QCOM_H */

14
drivers/iommu/arm/arm-smmu/qcom_iommu.c
View File

@ -410,7 +410,8 @@ static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *de
}
static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
int ret;
unsigned long flags;
@ -421,13 +422,14 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
return -ENODEV;
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->map(ops, iova, paddr, size, prot, GFP_ATOMIC);
ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, GFP_ATOMIC, mapped);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
return ret;
}
static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
size_t ret;
unsigned long flags;
@ -444,7 +446,7 @@ static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
*/
pm_runtime_get_sync(qcom_domain->iommu->dev);
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->unmap(ops, iova, size, gather);
ret = ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
pm_runtime_put_sync(qcom_domain->iommu->dev);
@ -582,8 +584,8 @@ static const struct iommu_ops qcom_iommu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = qcom_iommu_attach_dev,
.detach_dev = qcom_iommu_detach_dev,
.map = qcom_iommu_map,
.unmap = qcom_iommu_unmap,
.map_pages = qcom_iommu_map,
.unmap_pages = qcom_iommu_unmap,
.flush_iotlb_all = qcom_iommu_flush_iotlb_all,
.iotlb_sync = qcom_iommu_iotlb_sync,
.iova_to_phys = qcom_iommu_iova_to_phys,

26
drivers/iommu/exynos-iommu.c
View File

@ -708,10 +708,6 @@ static int exynos_sysmmu_probe(struct platform_device *pdev)
if (ret)
return ret;
ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev);
if (ret)
goto err_iommu_register;
platform_set_drvdata(pdev, data);
if (PG_ENT_SHIFT < 0) {
@ -743,11 +739,13 @@ static int exynos_sysmmu_probe(struct platform_device *pdev)
pm_runtime_enable(dev);
ret = iommu_device_register(&data->iommu, &exynos_iommu_ops, dev);
if (ret)
goto err_dma_set_mask;
return 0;
err_dma_set_mask:
iommu_device_unregister(&data->iommu);
err_iommu_register:
iommu_device_sysfs_remove(&data->iommu);
return ret;
}
@ -1432,12 +1430,6 @@ static int __init exynos_iommu_init(void)
return -ENOMEM;
}
ret = platform_driver_register(&exynos_sysmmu_driver);
if (ret) {
pr_err("%s: Failed to register driver\n", __func__);
goto err_reg_driver;
}
zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL);
if (zero_lv2_table == NULL) {
pr_err("%s: Failed to allocate zero level2 page table\n",
@ -1446,10 +1438,16 @@ static int __init exynos_iommu_init(void)
goto err_zero_lv2;
}
ret = platform_driver_register(&exynos_sysmmu_driver);
if (ret) {
pr_err("%s: Failed to register driver\n", __func__);
goto err_reg_driver;
}
return 0;
err_zero_lv2:
platform_driver_unregister(&exynos_sysmmu_driver);
err_reg_driver:
platform_driver_unregister(&exynos_sysmmu_driver);
err_zero_lv2:
kmem_cache_destroy(lv2table_kmem_cache);
return ret;
}

6
drivers/iommu/fsl_pamu.c
View File

@ -779,7 +779,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
of_get_address(dev->of_node, 0, &size, NULL);
irq = irq_of_parse_and_map(dev->of_node, 0);
if (irq == NO_IRQ) {
if (!irq) {
dev_warn(dev, "no interrupts listed in PAMU node\n");
goto error;
}
@ -868,7 +868,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
ret = create_csd(ppaact_phys, mem_size, csd_port_id);
if (ret) {
dev_err(dev, "could not create coherence subdomain\n");
return ret;
goto error;
}
}
@ -903,7 +903,7 @@ static int fsl_pamu_probe(struct platform_device *pdev)
return 0;
error:
if (irq != NO_IRQ)
if (irq)
free_irq(irq, data);
kfree_sensitive(data);

169
drivers/iommu/intel/iommu.c
View File

@ -277,7 +277,8 @@ static LIST_HEAD(dmar_satc_units);
#define for_each_rmrr_units(rmrr) \
list_for_each_entry(rmrr, &dmar_rmrr_units, list)
static void dmar_remove_one_dev_info(struct device *dev);
static void device_block_translation(struct device *dev);
static void intel_iommu_domain_free(struct iommu_domain *domain);
int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
@ -382,11 +383,6 @@ static inline int domain_type_is_si(struct dmar_domain *domain)
return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
}
static inline bool domain_use_first_level(struct dmar_domain *domain)
{
return domain->flags & DOMAIN_FLAG_USE_FIRST_LEVEL;
}
static inline int domain_pfn_supported(struct dmar_domain *domain,
unsigned long pfn)
{
@ -500,7 +496,7 @@ static int domain_update_iommu_superpage(struct dmar_domain *domain,
rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
if (iommu != skip) {
if (domain && domain_use_first_level(domain)) {
if (domain && domain->use_first_level) {
if (!cap_fl1gp_support(iommu->cap))
mask = 0x1;
} else {
@ -578,7 +574,7 @@ static void domain_update_iommu_cap(struct dmar_domain *domain)
* paging and 57-bits with 5-level paging). Hence, skip bit
* [N-1].
*/
if (domain_use_first_level(domain))
if (domain->use_first_level)
domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
else
domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
@ -779,19 +775,6 @@ static void domain_flush_cache(struct dmar_domain *domain,
clflush_cache_range(addr, size);
}
static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
{
struct context_entry *context;
int ret = 0;
spin_lock(&iommu->lock);
context = iommu_context_addr(iommu, bus, devfn, 0);
if (context)
ret = context_present(context);
spin_unlock(&iommu->lock);
return ret;
}
static void free_context_table(struct intel_iommu *iommu)
{
struct context_entry *context;
@ -959,7 +942,7 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
if (domain_use_first_level(domain))
if (domain->use_first_level)
pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
if (cmpxchg64(&pte->val, 0ULL, pteval))
@ -1418,7 +1401,7 @@ static void iommu_enable_pci_caps(struct device_domain_info *info)
{
struct pci_dev *pdev;
if (!info || !dev_is_pci(info->dev))
if (!dev_is_pci(info->dev))
return;
pdev = to_pci_dev(info->dev);
@ -1458,7 +1441,7 @@ static void iommu_enable_pci_caps(struct device_domain_info *info)
}
}
static void iommu_disable_dev_iotlb(struct device_domain_info *info)
static void iommu_disable_pci_caps(struct device_domain_info *info)
{
struct pci_dev *pdev;
@ -1529,7 +1512,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
if (ih)
ih = 1 << 6;
if (domain_use_first_level(domain)) {
if (domain->use_first_level) {
qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih);
} else {
unsigned long bitmask = aligned_pages - 1;
@ -1583,7 +1566,7 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu,
* It's a non-present to present mapping. Only flush if caching mode
* and second level.
*/
if (cap_caching_mode(iommu->cap) && !domain_use_first_level(domain))
if (cap_caching_mode(iommu->cap) && !domain->use_first_level)
iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
else
iommu_flush_write_buffer(iommu);
@ -1599,7 +1582,7 @@ static void intel_flush_iotlb_all(struct iommu_domain *domain)
struct intel_iommu *iommu = info->iommu;
u16 did = domain_id_iommu(dmar_domain, iommu);
if (domain_use_first_level(dmar_domain))
if (dmar_domain->use_first_level)
qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0);
else
iommu->flush.flush_iotlb(iommu, did, 0, 0,
@ -1772,7 +1755,7 @@ static struct dmar_domain *alloc_domain(unsigned int type)
domain->nid = NUMA_NO_NODE;
if (first_level_by_default(type))
domain->flags |= DOMAIN_FLAG_USE_FIRST_LEVEL;
domain->use_first_level = true;
domain->has_iotlb_device = false;
INIT_LIST_HEAD(&domain->devices);
spin_lock_init(&domain->lock);
@ -2064,7 +2047,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
} else {
iommu_flush_write_buffer(iommu);
}
iommu_enable_pci_caps(info);
ret = 0;
@ -2116,30 +2098,6 @@ domain_context_mapping(struct dmar_domain *domain, struct device *dev)
&domain_context_mapping_cb, &data);
}
static int domain_context_mapped_cb(struct pci_dev *pdev,
u16 alias, void *opaque)
{
struct intel_iommu *iommu = opaque;
return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
}
static int domain_context_mapped(struct device *dev)
{
struct intel_iommu *iommu;
u8 bus, devfn;
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
if (!dev_is_pci(dev))
return device_context_mapped(iommu, bus, devfn);
return !pci_for_each_dma_alias(to_pci_dev(dev),
domain_context_mapped_cb, iommu);
}
/* Returns a number of VTD pages, but aligned to MM page size */
static inline unsigned long aligned_nrpages(unsigned long host_addr,
size_t size)
@ -2229,7 +2187,7 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
attr |= DMA_FL_PTE_PRESENT;
if (domain_use_first_level(domain)) {
if (domain->use_first_level) {
attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
if (prot & DMA_PTE_WRITE)
attr |= DMA_FL_PTE_DIRTY;
@ -2472,7 +2430,8 @@ static int __init si_domain_init(int hw)
return 0;
}
static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
static int dmar_domain_attach_device(struct dmar_domain *domain,
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu;
@ -2494,18 +2453,11 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
/* PASID table is mandatory for a PCI device in scalable mode. */
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
ret = intel_pasid_alloc_table(dev);
if (ret) {
dev_err(dev, "PASID table allocation failed\n");
dmar_remove_one_dev_info(dev);
return ret;
}
/* Setup the PASID entry for requests without PASID: */
if (hw_pass_through && domain_type_is_si(domain))
ret = intel_pasid_setup_pass_through(iommu, domain,
dev, PASID_RID2PASID);
else if (domain_use_first_level(domain))
else if (domain->use_first_level)
ret = domain_setup_first_level(iommu, domain, dev,
PASID_RID2PASID);
else
@ -2513,7 +2465,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
dev, PASID_RID2PASID);
if (ret) {
dev_err(dev, "Setup RID2PASID failed\n");
dmar_remove_one_dev_info(dev);
device_block_translation(dev);
return ret;
}
}
@ -2521,10 +2473,12 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
ret = domain_context_mapping(domain, dev);
if (ret) {
dev_err(dev, "Domain context map failed\n");
dmar_remove_one_dev_info(dev);
device_block_translation(dev);
return ret;
}
iommu_enable_pci_caps(info);
return 0;
}
@ -4125,9 +4079,8 @@ static void dmar_remove_one_dev_info(struct device *dev)
intel_pasid_tear_down_entry(iommu, info->dev,
PASID_RID2PASID, false);
iommu_disable_dev_iotlb(info);
iommu_disable_pci_caps(info);
domain_context_clear(info);
intel_pasid_free_table(info->dev);
}
spin_lock_irqsave(&domain->lock, flags);
@ -4138,6 +4091,37 @@ static void dmar_remove_one_dev_info(struct device *dev)
info->domain = NULL;
}
/*
* Clear the page table pointer in context or pasid table entries so that
* all DMA requests without PASID from the device are blocked. If the page
* table has been set, clean up the data structures.
*/
static void device_block_translation(struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
struct intel_iommu *iommu = info->iommu;
unsigned long flags;
iommu_disable_pci_caps(info);
if (!dev_is_real_dma_subdevice(dev)) {
if (sm_supported(iommu))
intel_pasid_tear_down_entry(iommu, dev,
PASID_RID2PASID, false);
else
domain_context_clear(info);
}
if (!info->domain)
return;
spin_lock_irqsave(&info->domain->lock, flags);
list_del(&info->link);
spin_unlock_irqrestore(&info->domain->lock, flags);
domain_detach_iommu(info->domain, iommu);
info->domain = NULL;
}
static int md_domain_init(struct dmar_domain *domain, int guest_width)
{
int adjust_width;
@ -4159,12 +4143,28 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
return 0;
}
static int blocking_domain_attach_dev(struct iommu_domain *domain,
struct device *dev)
{
device_block_translation(dev);
return 0;
}
static struct iommu_domain blocking_domain = {
.ops = &(const struct iommu_domain_ops) {
.attach_dev = blocking_domain_attach_dev,
.free = intel_iommu_domain_free
}
};
static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
{
struct dmar_domain *dmar_domain;
struct iommu_domain *domain;
switch (type) {
case IOMMU_DOMAIN_BLOCKED:
return &blocking_domain;
case IOMMU_DOMAIN_DMA:
case IOMMU_DOMAIN_DMA_FQ:
case IOMMU_DOMAIN_UNMANAGED:
@ -4199,7 +4199,7 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
static void intel_iommu_domain_free(struct iommu_domain *domain)
{
if (domain != &si_domain->domain)
if (domain != &si_domain->domain && domain != &blocking_domain)
domain_exit(to_dmar_domain(domain));
}
@ -4246,6 +4246,7 @@ static int prepare_domain_attach_device(struct iommu_domain *domain,
static int intel_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
struct device_domain_info *info = dev_iommu_priv_get(dev);
int ret;
if (domain->type == IOMMU_DOMAIN_UNMANAGED &&
@ -4254,25 +4255,14 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
return -EPERM;
}
/* normally dev is not mapped */
if (unlikely(domain_context_mapped(dev))) {
struct device_domain_info *info = dev_iommu_priv_get(dev);
if (info->domain)
dmar_remove_one_dev_info(dev);
}
if (info->domain)
device_block_translation(dev);
ret = prepare_domain_attach_device(domain, dev);
if (ret)
return ret;
return domain_add_dev_info(to_dmar_domain(domain), dev);
}
static void intel_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
dmar_remove_one_dev_info(dev);
return dmar_domain_attach_device(to_dmar_domain(domain), dev);
}
static int intel_iommu_map(struct iommu_domain *domain,
@ -4436,7 +4426,7 @@ static void domain_set_force_snooping(struct dmar_domain *domain)
* Second level page table supports per-PTE snoop control. The
* iommu_map() interface will handle this by setting SNP bit.
*/
if (!domain_use_first_level(domain)) {
if (!domain->use_first_level) {
domain->set_pte_snp = true;
return;
}
@ -4491,6 +4481,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
struct device_domain_info *info;
struct intel_iommu *iommu;
u8 bus, devfn;
int ret;
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu || !iommu->iommu.ops)
@ -4535,6 +4526,16 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
dev_iommu_priv_set(dev, info);
if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
ret = intel_pasid_alloc_table(dev);
if (ret) {
dev_err(dev, "PASID table allocation failed\n");
dev_iommu_priv_set(dev, NULL);
kfree(info);
return ERR_PTR(ret);
}
}
return &iommu->iommu;
}
@ -4543,6 +4544,7 @@ static void intel_iommu_release_device(struct device *dev)
struct device_domain_info *info = dev_iommu_priv_get(dev);
dmar_remove_one_dev_info(dev);
intel_pasid_free_table(dev);
dev_iommu_priv_set(dev, NULL);
kfree(info);
set_dma_ops(dev, NULL);
@ -4777,7 +4779,6 @@ const struct iommu_ops intel_iommu_ops = {
#endif
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = intel_iommu_attach_device,
.detach_dev = intel_iommu_detach_device,
.map_pages = intel_iommu_map_pages,
.unmap_pages = intel_iommu_unmap_pages,
.iotlb_sync_map = intel_iommu_iotlb_sync_map,

15
drivers/iommu/intel/iommu.h
View File

@ -515,14 +515,6 @@ struct context_entry {
u64 hi;
};
/*
* When VT-d works in the scalable mode, it allows DMA translation to
* happen through either first level or second level page table. This
* bit marks that the DMA translation for the domain goes through the
* first level page table, otherwise, it goes through the second level.
*/
#define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(1)
struct iommu_domain_info {
struct intel_iommu *iommu;
unsigned int refcnt; /* Refcount of devices per iommu */
@ -539,6 +531,11 @@ struct dmar_domain {
u8 iommu_coherency: 1; /* indicate coherency of iommu access */
u8 force_snooping : 1; /* Create IOPTEs with snoop control */
u8 set_pte_snp:1;
u8 use_first_level:1; /* DMA translation for the domain goes
* through the first level page table,
* otherwise, goes through the second
* level.
*/
spinlock_t lock; /* Protect device tracking lists */
struct list_head devices; /* all devices' list */
@ -548,8 +545,6 @@ struct dmar_domain {
/* adjusted guest address width, 0 is level 2 30-bit */
int agaw;
int flags; /* flags to find out type of domain */
int iommu_superpage;/* Level of superpages supported:
0 == 4KiB (no superpages), 1 == 2MiB,
2 == 1GiB, 3 == 512GiB, 4 == 1TiB */

41
drivers/iommu/io-pgtable-arm-v7s.c
View File

@ -564,8 +564,7 @@ static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
iova += pgsize;
paddr += pgsize;
if (mapped)
*mapped += pgsize;
*mapped += pgsize;
}
/*
* Synchronise all PTE updates for the new mapping before there's
@ -576,12 +575,6 @@ static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
return ret;
}
static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
return arm_v7s_map_pages(ops, iova, paddr, size, 1, prot, gfp, NULL);
}
static void arm_v7s_free_pgtable(struct io_pgtable *iop)
{
struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop);
@ -764,12 +757,6 @@ static size_t arm_v7s_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova
return unmapped;
}
static size_t arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
return arm_v7s_unmap_pages(ops, iova, size, 1, gather);
}
static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops,
unsigned long iova)
{
@ -842,9 +829,7 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
goto out_free_data;
data->iop.ops = (struct io_pgtable_ops) {
.map = arm_v7s_map,
.map_pages = arm_v7s_map_pages,
.unmap = arm_v7s_unmap,
.unmap_pages = arm_v7s_unmap_pages,
.iova_to_phys = arm_v7s_iova_to_phys,
};
@ -954,6 +939,7 @@ static int __init arm_v7s_do_selftests(void)
};
unsigned int iova, size, iova_start;
unsigned int i, loopnr = 0;
size_t mapped;
selftest_running = true;
@ -984,15 +970,16 @@ static int __init arm_v7s_do_selftests(void)
iova = 0;
for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << i;
if (ops->map(ops, iova, iova, size, IOMMU_READ |
IOMMU_WRITE |
IOMMU_NOEXEC |
IOMMU_CACHE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1,
IOMMU_READ | IOMMU_WRITE |
IOMMU_NOEXEC | IOMMU_CACHE,
GFP_KERNEL, &mapped))
return __FAIL(ops);
/* Overlapping mappings */
if (!ops->map(ops, iova, iova + size, size,
IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL))
if (!ops->map_pages(ops, iova, iova + size, size, 1,
IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL,
&mapped))
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
@ -1007,11 +994,12 @@ static int __init arm_v7s_do_selftests(void)
size = 1UL << __ffs(cfg.pgsize_bitmap);
while (i < loopnr) {
iova_start = i * SZ_16M;
if (ops->unmap(ops, iova_start + size, size, NULL) != size)
if (ops->unmap_pages(ops, iova_start + size, size, 1, NULL) != size)
return __FAIL(ops);
/* Remap of partial unmap */
if (ops->map(ops, iova_start + size, size, size, IOMMU_READ, GFP_KERNEL))
if (ops->map_pages(ops, iova_start + size, size, size, 1,
IOMMU_READ, GFP_KERNEL, &mapped))
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova_start + size + 42)
@ -1025,14 +1013,15 @@ static int __init arm_v7s_do_selftests(void)
for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << i;
if (ops->unmap(ops, iova, size, NULL) != size)
if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42))
return __FAIL(ops);
/* Remap full block */
if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1, IOMMU_WRITE,
GFP_KERNEL, &mapped))
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))

42
drivers/iommu/io-pgtable-arm.c
View File

@ -360,7 +360,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start;
num_entries = min_t(int, pgcount, max_entries);
ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
if (!ret && mapped)
if (!ret)
*mapped += num_entries * size;
return ret;
@ -496,13 +496,6 @@ static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
return ret;
}
static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int iommu_prot, gfp_t gfp)
{
return arm_lpae_map_pages(ops, iova, paddr, size, 1, iommu_prot, gfp,
NULL);
}
static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
arm_lpae_iopte *ptep)
{
@ -682,12 +675,6 @@ static size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iov
data->start_level, ptep);
}
static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
{
return arm_lpae_unmap_pages(ops, iova, size, 1, gather);
}
static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
unsigned long iova)
{
@ -799,9 +786,7 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg)
data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
data->iop.ops = (struct io_pgtable_ops) {
.map = arm_lpae_map,
.map_pages = arm_lpae_map_pages,
.unmap = arm_lpae_unmap,
.unmap_pages = arm_lpae_unmap_pages,
.iova_to_phys = arm_lpae_iova_to_phys,
};
@ -1176,7 +1161,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
int i, j;
unsigned long iova;
size_t size;
size_t size, mapped;
struct io_pgtable_ops *ops;
selftest_running = true;
@ -1209,15 +1194,16 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << j;
if (ops->map(ops, iova, iova, size, IOMMU_READ |
IOMMU_WRITE |
IOMMU_NOEXEC |
IOMMU_CACHE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1,
IOMMU_READ | IOMMU_WRITE |
IOMMU_NOEXEC | IOMMU_CACHE,
GFP_KERNEL, &mapped))
return __FAIL(ops, i);
/* Overlapping mappings */
if (!ops->map(ops, iova, iova + size, size,
IOMMU_READ | IOMMU_NOEXEC, GFP_KERNEL))
if (!ops->map_pages(ops, iova, iova + size, size, 1,
IOMMU_READ | IOMMU_NOEXEC,
GFP_KERNEL, &mapped))
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))
@ -1228,11 +1214,12 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
/* Partial unmap */
size = 1UL << __ffs(cfg->pgsize_bitmap);
if (ops->unmap(ops, SZ_1G + size, size, NULL) != size)
if (ops->unmap_pages(ops, SZ_1G + size, size, 1, NULL) != size)
return __FAIL(ops, i);
/* Remap of partial unmap */
if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ, GFP_KERNEL))
if (ops->map_pages(ops, SZ_1G + size, size, size, 1,
IOMMU_READ, GFP_KERNEL, &mapped))
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42))
@ -1243,14 +1230,15 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << j;
if (ops->unmap(ops, iova, size, NULL) != size)
if (ops->unmap_pages(ops, iova, size, 1, NULL) != size)
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42))
return __FAIL(ops, i);
/* Remap full block */
if (ops->map(ops, iova, iova, size, IOMMU_WRITE, GFP_KERNEL))
if (ops->map_pages(ops, iova, iova, size, 1,
IOMMU_WRITE, GFP_KERNEL, &mapped))
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42) != (iova + 42))

28
drivers/iommu/iommu.c
View File

@ -306,13 +306,23 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
const struct iommu_ops *ops = dev->bus->iommu_ops;
struct iommu_device *iommu_dev;
struct iommu_group *group;
static DEFINE_MUTEX(iommu_probe_device_lock);
int ret;
if (!ops)
return -ENODEV;
if (!dev_iommu_get(dev))
return -ENOMEM;
/*
* Serialise to avoid races between IOMMU drivers registering in
* parallel and/or the "replay" calls from ACPI/OF code via client
* driver probe. Once the latter have been cleaned up we should
* probably be able to use device_lock() here to minimise the scope,
* but for now enforcing a simple global ordering is fine.
*/
mutex_lock(&iommu_probe_device_lock);
if (!dev_iommu_get(dev)) {
ret = -ENOMEM;
goto err_unlock;
}
if (!try_module_get(ops->owner)) {
ret = -EINVAL;
@ -333,11 +343,14 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list
ret = PTR_ERR(group);
goto out_release;
}
iommu_group_put(group);
mutex_lock(&group->mutex);
if (group_list && !group->default_domain && list_empty(&group->entry))
list_add_tail(&group->entry, group_list);
mutex_unlock(&group->mutex);
iommu_group_put(group);
mutex_unlock(&iommu_probe_device_lock);
iommu_device_link(iommu_dev, dev);
return 0;
@ -352,6 +365,9 @@ out_module_put:
err_free:
dev_iommu_free(dev);
err_unlock:
mutex_unlock(&iommu_probe_device_lock);
return ret;
}
@ -1824,11 +1840,11 @@ int bus_iommu_probe(struct bus_type *bus)
return ret;
list_for_each_entry_safe(group, next, &group_list, entry) {
mutex_lock(&group->mutex);
/* Remove item from the list */
list_del_init(&group->entry);
mutex_lock(&group->mutex);
/* Try to allocate default domain */
probe_alloc_default_domain(bus, group);

18
drivers/iommu/ipmmu-vmsa.c
View File

@ -659,22 +659,22 @@ static void ipmmu_detach_device(struct iommu_domain *io_domain,
}
static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
if (!domain)
return -ENODEV;
return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp);
return domain->iop->map_pages(domain->iop, iova, paddr, pgsize, pgcount,
prot, gfp, mapped);
}
static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
return domain->iop->unmap(domain->iop, iova, size, gather);
return domain->iop->unmap_pages(domain->iop, iova, pgsize, pgcount, gather);
}
static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
@ -877,8 +877,8 @@ static const struct iommu_ops ipmmu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = ipmmu_attach_device,
.detach_dev = ipmmu_detach_device,
.map = ipmmu_map,
.unmap = ipmmu_unmap,
.map_pages = ipmmu_map,
.unmap_pages = ipmmu_unmap,
.flush_iotlb_all = ipmmu_flush_iotlb_all,
.iotlb_sync = ipmmu_iotlb_sync,
.iova_to_phys = ipmmu_iova_to_phys,

18
drivers/iommu/msm_iommu.c
View File

@ -471,14 +471,16 @@ fail:
}
static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t pa, size_t len, int prot, gfp_t gfp)
phys_addr_t pa, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct msm_priv *priv = to_msm_priv(domain);
unsigned long flags;
int ret;
spin_lock_irqsave(&priv->pgtlock, flags);
ret = priv->iop->map(priv->iop, iova, pa, len, prot, GFP_ATOMIC);
ret = priv->iop->map_pages(priv->iop, iova, pa, pgsize, pgcount, prot,
GFP_ATOMIC, mapped);
spin_unlock_irqrestore(&priv->pgtlock, flags);
return ret;
@ -493,16 +495,18 @@ static void msm_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t len, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct msm_priv *priv = to_msm_priv(domain);
unsigned long flags;
size_t ret;
spin_lock_irqsave(&priv->pgtlock, flags);
len = priv->iop->unmap(priv->iop, iova, len, gather);
ret = priv->iop->unmap_pages(priv->iop, iova, pgsize, pgcount, gather);
spin_unlock_irqrestore(&priv->pgtlock, flags);
return len;
return ret;
}
static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
@ -679,8 +683,8 @@ static struct iommu_ops msm_iommu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = msm_iommu_attach_dev,
.detach_dev = msm_iommu_detach_dev,
.map = msm_iommu_map,
.unmap = msm_iommu_unmap,
.map_pages = msm_iommu_map,
.unmap_pages = msm_iommu_unmap,
/*
* Nothing is needed here, the barrier to guarantee
* completion of the tlb sync operation is implicitly

149
drivers/iommu/mtk_iommu.c
View File

@ -108,8 +108,12 @@
#define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3)
#define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7)
#define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7)
/* Macro for 5 bits length port ID field (default) */
#define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
#define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
/* Macro for 6 bits length port ID field */
#define F_MMU_INT_ID_LARB_ID_WID_6(a) (((a) >> 8) & 0x7)
#define F_MMU_INT_ID_PORT_ID_WID_6(a) (((a) >> 2) & 0x3f)
#define MTK_PROTECT_PA_ALIGN 256
#define MTK_IOMMU_BANK_SZ 0x1000
@ -139,6 +143,7 @@
#define IFA_IOMMU_PCIE_SUPPORT BIT(16)
#define PGTABLE_PA_35_EN BIT(17)
#define TF_PORT_TO_ADDR_MT8173 BIT(18)
#define INT_ID_PORT_WIDTH_6 BIT(19)
#define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \
((((pdata)->flags) & (mask)) == (_x))
@ -165,6 +170,7 @@ enum mtk_iommu_plat {
M4U_MT8186,
M4U_MT8192,
M4U_MT8195,
M4U_MT8365,
};
struct mtk_iommu_iova_region {
@ -223,10 +229,7 @@ struct mtk_iommu_data {
struct device *smicomm_dev;
struct mtk_iommu_bank_data *bank;
struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */
struct regmap *pericfg;
struct mutex mutex; /* Protect m4u_group/m4u_dom above */
/*
@ -441,20 +444,25 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
fault_pa |= (u64)pa34_32 << 32;
if (MTK_IOMMU_IS_TYPE(plat_data, MTK_IOMMU_TYPE_MM)) {
fault_port = F_MMU_INT_ID_PORT_ID(regval);
if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_2BITS)) {
fault_larb = F_MMU_INT_ID_COMM_ID(regval);
sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval);
fault_port = F_MMU_INT_ID_PORT_ID(regval);
} else if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_3BITS)) {
fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval);
sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval);
fault_port = F_MMU_INT_ID_PORT_ID(regval);
} else if (MTK_IOMMU_HAS_FLAG(plat_data, INT_ID_PORT_WIDTH_6)) {
fault_port = F_MMU_INT_ID_PORT_ID_WID_6(regval);
fault_larb = F_MMU_INT_ID_LARB_ID_WID_6(regval);
} else {
fault_port = F_MMU_INT_ID_PORT_ID(regval);
fault_larb = F_MMU_INT_ID_LARB_ID(regval);
}
fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm];
}
if (report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova,
if (!dom || report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova,
write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
dev_err_ratelimited(
bank->parent_dev,
@ -711,7 +719,8 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain,
}
static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
@ -720,17 +729,17 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
paddr |= BIT_ULL(32);
/* Synchronize with the tlb_lock */
return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp);
return dom->iop->map_pages(dom->iop, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
}
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
unsigned long iova, size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
iommu_iotlb_gather_add_range(gather, iova, size);
return dom->iop->unmap(dom->iop, iova, size, gather);
iommu_iotlb_gather_add_range(gather, iova, pgsize * pgcount);
return dom->iop->unmap_pages(dom->iop, iova, pgsize, pgcount, gather);
}
static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
@ -938,8 +947,8 @@ static const struct iommu_ops mtk_iommu_ops = {
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = mtk_iommu_attach_device,
.detach_dev = mtk_iommu_detach_device,
.map = mtk_iommu_map,
.unmap = mtk_iommu_unmap,
.map_pages = mtk_iommu_map,
.unmap_pages = mtk_iommu_unmap,
.flush_iotlb_all = mtk_iommu_flush_iotlb_all,
.iotlb_sync = mtk_iommu_iotlb_sync,
.iotlb_sync_map = mtk_iommu_sync_map,
@ -1043,21 +1052,26 @@ static const struct component_master_ops mtk_iommu_com_ops = {
static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match,
struct mtk_iommu_data *data)
{
struct device_node *larbnode, *smicomm_node, *smi_subcomm_node;
struct platform_device *plarbdev;
struct device_node *larbnode, *frst_avail_smicomm_node = NULL;
struct platform_device *plarbdev, *pcommdev;
struct device_link *link;
int i, larb_nr, ret;
larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL);
if (larb_nr < 0)
return larb_nr;
if (larb_nr == 0 || larb_nr > MTK_LARB_NR_MAX)
return -EINVAL;
for (i = 0; i < larb_nr; i++) {
struct device_node *smicomm_node, *smi_subcomm_node;
u32 id;
larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
if (!larbnode)
return -EINVAL;
if (!larbnode) {
ret = -EINVAL;
goto err_larbdev_put;
}
if (!of_device_is_available(larbnode)) {
of_node_put(larbnode);
@ -1067,48 +1081,91 @@ static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **m
ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
if (ret)/* The id is consecutive if there is no this property */
id = i;
if (id >= MTK_LARB_NR_MAX) {
of_node_put(larbnode);
ret = -EINVAL;
goto err_larbdev_put;
}
plarbdev = of_find_device_by_node(larbnode);
of_node_put(larbnode);
if (!plarbdev) {
of_node_put(larbnode);
return -ENODEV;
ret = -ENODEV;
goto err_larbdev_put;
}
if (!plarbdev->dev.driver) {
of_node_put(larbnode);
return -EPROBE_DEFER;
if (data->larb_imu[id].dev) {
platform_device_put(plarbdev);
ret = -EEXIST;
goto err_larbdev_put;
}
data->larb_imu[id].dev = &plarbdev->dev;
component_match_add_release(dev, match, component_release_of,
component_compare_of, larbnode);
if (!plarbdev->dev.driver) {
ret = -EPROBE_DEFER;
goto err_larbdev_put;
}
/* Get smi-(sub)-common dev from the last larb. */
smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0);
if (!smi_subcomm_node) {
ret = -EINVAL;
goto err_larbdev_put;
}
/*
* It may have two level smi-common. the node is smi-sub-common if it
* has a new mediatek,smi property. otherwise it is smi-commmon.
*/
smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0);
if (smicomm_node)
of_node_put(smi_subcomm_node);
else
smicomm_node = smi_subcomm_node;
/*
* All the larbs that connect to one IOMMU must connect with the same
* smi-common.
*/
if (!frst_avail_smicomm_node) {
frst_avail_smicomm_node = smicomm_node;
} else if (frst_avail_smicomm_node != smicomm_node) {
dev_err(dev, "mediatek,smi property is not right @larb%d.", id);
of_node_put(smicomm_node);
ret = -EINVAL;
goto err_larbdev_put;
} else {
of_node_put(smicomm_node);
}
component_match_add(dev, match, component_compare_dev, &plarbdev->dev);
platform_device_put(plarbdev);
}
/* Get smi-(sub)-common dev from the last larb. */
smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0);
if (!smi_subcomm_node)
if (!frst_avail_smicomm_node)
return -EINVAL;
/*
* It may have two level smi-common. the node is smi-sub-common if it
* has a new mediatek,smi property. otherwise it is smi-commmon.
*/
smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0);
if (smicomm_node)
of_node_put(smi_subcomm_node);
else
smicomm_node = smi_subcomm_node;
plarbdev = of_find_device_by_node(smicomm_node);
of_node_put(smicomm_node);
data->smicomm_dev = &plarbdev->dev;
pcommdev = of_find_device_by_node(frst_avail_smicomm_node);
of_node_put(frst_avail_smicomm_node);
if (!pcommdev)
return -ENODEV;
data->smicomm_dev = &pcommdev->dev;
link = device_link_add(data->smicomm_dev, dev,
DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
platform_device_put(pcommdev);
if (!link) {
dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev));
return -EINVAL;
}
return 0;
err_larbdev_put:
for (i = MTK_LARB_NR_MAX - 1; i >= 0; i--) {
if (!data->larb_imu[i].dev)
continue;
put_device(data->larb_imu[i].dev);
}
return ret;
}
static int mtk_iommu_probe(struct platform_device *pdev)
@ -1173,6 +1230,8 @@ static int mtk_iommu_probe(struct platform_device *pdev)
banks_num = data->plat_data->banks_num;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res)
return -EINVAL;
if (resource_size(res) < banks_num * MTK_IOMMU_BANK_SZ) {
dev_err(dev, "banknr %d. res %pR is not enough.\n", banks_num, res);
return -EINVAL;
@ -1516,6 +1575,17 @@ static const struct mtk_iommu_plat_data mt8195_data_vpp = {
{4, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 6}},
};
static const struct mtk_iommu_plat_data mt8365_data = {
.m4u_plat = M4U_MT8365,
.flags = RESET_AXI | INT_ID_PORT_WIDTH_6,
.inv_sel_reg = REG_MMU_INV_SEL_GEN1,
.banks_num = 1,
.banks_enable = {true},
.iova_region = single_domain,
.iova_region_nr = ARRAY_SIZE(single_domain),
.larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */
};
static const struct of_device_id mtk_iommu_of_ids[] = {
{ .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data},
{ .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data},
@ -1528,6 +1598,7 @@ static const struct of_device_id mtk_iommu_of_ids[] = {
{ .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra},
{ .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo},
{ .compatible = "mediatek,mt8195-iommu-vpp", .data = &mt8195_data_vpp},
{ .compatible = "mediatek,mt8365-m4u", .data = &mt8365_data},
{}
};

30
drivers/iommu/mtk_iommu_v1.c
View File

@ -327,44 +327,42 @@ static void mtk_iommu_v1_detach_device(struct iommu_domain *domain, struct devic
}
static int mtk_iommu_v1_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain);
unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;
unsigned long flags;
unsigned int i;
u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT);
u32 pabase = (u32)paddr;
int map_size = 0;
spin_lock_irqsave(&dom->pgtlock, flags);
for (i = 0; i < page_num; i++) {
if (pgt_base_iova[i]) {
memset(pgt_base_iova, 0, i * sizeof(u32));
for (i = 0; i < pgcount; i++) {
if (pgt_base_iova[i])
break;
}
pgt_base_iova[i] = pabase | F_DESC_VALID | F_DESC_NONSEC;
pabase += MT2701_IOMMU_PAGE_SIZE;
map_size += MT2701_IOMMU_PAGE_SIZE;
}
spin_unlock_irqrestore(&dom->pgtlock, flags);
mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);
*mapped = i * MT2701_IOMMU_PAGE_SIZE;
mtk_iommu_v1_tlb_flush_range(dom->data, iova, *mapped);
return map_size == size ? 0 : -EEXIST;
return i == pgcount ? 0 : -EEXIST;
}
static size_t mtk_iommu_v1_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_v1_domain *dom = to_mtk_domain(domain);
unsigned long flags;
u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT);
unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;
size_t size = pgcount * MT2701_IOMMU_PAGE_SIZE;
spin_lock_irqsave(&dom->pgtlock, flags);
memset(pgt_base_iova, 0, page_num * sizeof(u32));
memset(pgt_base_iova, 0, pgcount * sizeof(u32));
spin_unlock_irqrestore(&dom->pgtlock, flags);
mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);
@ -586,13 +584,13 @@ static const struct iommu_ops mtk_iommu_v1_ops = {
.release_device = mtk_iommu_v1_release_device,
.def_domain_type = mtk_iommu_v1_def_domain_type,
.device_group = generic_device_group,
.pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT,
.pgsize_bitmap = MT2701_IOMMU_PAGE_SIZE,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = mtk_iommu_v1_attach_device,
.detach_dev = mtk_iommu_v1_detach_device,
.map = mtk_iommu_v1_map,
.unmap = mtk_iommu_v1_unmap,
.map_pages = mtk_iommu_v1_map,
.unmap_pages = mtk_iommu_v1_unmap,
.iova_to_phys = mtk_iommu_v1_iova_to_phys,
.free = mtk_iommu_v1_domain_free,
}

10
drivers/iommu/rockchip-iommu.c
View File

@ -280,19 +280,17 @@ static u32 rk_mk_pte(phys_addr_t page, int prot)
* 11:9 - Page address bit 34:32
* 8:4 - Page address bit 39:35
* 3 - Security
* 2 - Readable
* 1 - Writable
* 2 - Writable
* 1 - Readable
* 0 - 1 if Page @ Page address is valid
*/
#define RK_PTE_PAGE_READABLE_V2 BIT(2)
#define RK_PTE_PAGE_WRITABLE_V2 BIT(1)
static u32 rk_mk_pte_v2(phys_addr_t page, int prot)
{
u32 flags = 0;
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE_V2 : 0;
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE_V2 : 0;
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0;
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0;
return rk_mk_dte_v2(page) | flags;
}

383
drivers/iommu/s390-iommu.c
View File

@ -10,28 +10,18 @@
#include <linux/iommu.h>
#include <linux/iommu-helper.h>
#include <linux/sizes.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <asm/pci_dma.h>
/*
* Physically contiguous memory regions can be mapped with 4 KiB alignment,
* we allow all page sizes that are an order of 4KiB (no special large page
* support so far).
*/
#define S390_IOMMU_PGSIZES (~0xFFFUL)
static const struct iommu_ops s390_iommu_ops;
struct s390_domain {
struct iommu_domain domain;
struct list_head devices;
unsigned long *dma_table;
spinlock_t dma_table_lock;
spinlock_t list_lock;
};
struct s390_domain_device {
struct list_head list;
struct zpci_dev *zdev;
struct rcu_head rcu;
};
static struct s390_domain *to_s390_domain(struct iommu_domain *dom)
@ -67,20 +57,50 @@ static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
kfree(s390_domain);
return NULL;
}
s390_domain->domain.geometry.force_aperture = true;
s390_domain->domain.geometry.aperture_start = 0;
s390_domain->domain.geometry.aperture_end = ZPCI_TABLE_SIZE_RT - 1;
spin_lock_init(&s390_domain->dma_table_lock);
spin_lock_init(&s390_domain->list_lock);
INIT_LIST_HEAD(&s390_domain->devices);
INIT_LIST_HEAD_RCU(&s390_domain->devices);
return &s390_domain->domain;
}
static void s390_iommu_rcu_free_domain(struct rcu_head *head)
{
struct s390_domain *s390_domain = container_of(head, struct s390_domain, rcu);
dma_cleanup_tables(s390_domain->dma_table);
kfree(s390_domain);
}
static void s390_domain_free(struct iommu_domain *domain)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
dma_cleanup_tables(s390_domain->dma_table);
kfree(s390_domain);
rcu_read_lock();
WARN_ON(!list_empty(&s390_domain->devices));
rcu_read_unlock();
call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain);
}
static void __s390_iommu_detach_device(struct zpci_dev *zdev)
{
struct s390_domain *s390_domain = zdev->s390_domain;
unsigned long flags;
if (!s390_domain)
return;
spin_lock_irqsave(&s390_domain->list_lock, flags);
list_del_rcu(&zdev->iommu_list);
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
zpci_unregister_ioat(zdev, 0);
zdev->s390_domain = NULL;
zdev->dma_table = NULL;
}
static int s390_iommu_attach_device(struct iommu_domain *domain,
@ -88,98 +108,74 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev = to_zpci_dev(dev);
struct s390_domain_device *domain_device;
unsigned long flags;
int cc, rc;
u8 status;
int cc;
if (!zdev)
return -ENODEV;
domain_device = kzalloc(sizeof(*domain_device), GFP_KERNEL);
if (!domain_device)
return -ENOMEM;
if (zdev->dma_table && !zdev->s390_domain) {
cc = zpci_dma_exit_device(zdev);
if (cc) {
rc = -EIO;
goto out_free;
}
}
if (WARN_ON(domain->geometry.aperture_start > zdev->end_dma ||
domain->geometry.aperture_end < zdev->start_dma))
return -EINVAL;
if (zdev->s390_domain)
zpci_unregister_ioat(zdev, 0);
__s390_iommu_detach_device(zdev);
else if (zdev->dma_table)
zpci_dma_exit_device(zdev);
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(s390_domain->dma_table), &status);
/*
* If the device is undergoing error recovery the reset code
* will re-establish the new domain.
*/
if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL)
return -EIO;
zdev->dma_table = s390_domain->dma_table;
zdev->dma_table = s390_domain->dma_table;
cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
if (cc) {
rc = -EIO;
goto out_restore;
}
zdev->s390_domain = s390_domain;
spin_lock_irqsave(&s390_domain->list_lock, flags);
/* First device defines the DMA range limits */
if (list_empty(&s390_domain->devices)) {
domain->geometry.aperture_start = zdev->start_dma;
domain->geometry.aperture_end = zdev->end_dma;
domain->geometry.force_aperture = true;
/* Allow only devices with identical DMA range limits */
} else if (domain->geometry.aperture_start != zdev->start_dma ||
domain->geometry.aperture_end != zdev->end_dma) {
rc = -EINVAL;
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
goto out_restore;
}
domain_device->zdev = zdev;
zdev->s390_domain = s390_domain;
list_add(&domain_device->list, &s390_domain->devices);
list_add_rcu(&zdev->iommu_list, &s390_domain->devices);
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
return 0;
out_restore:
if (!zdev->s390_domain) {
zpci_dma_init_device(zdev);
} else {
zdev->dma_table = zdev->s390_domain->dma_table;
zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
virt_to_phys(zdev->dma_table));
}
out_free:
kfree(domain_device);
return rc;
}
static void s390_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev = to_zpci_dev(dev);
struct s390_domain_device *domain_device, *tmp;
unsigned long flags;
int found = 0;
if (!zdev)
return;
WARN_ON(zdev->s390_domain != to_s390_domain(domain));
spin_lock_irqsave(&s390_domain->list_lock, flags);
list_for_each_entry_safe(domain_device, tmp, &s390_domain->devices,
list) {
if (domain_device->zdev == zdev) {
list_del(&domain_device->list);
kfree(domain_device);
found = 1;
break;
}
__s390_iommu_detach_device(zdev);
zpci_dma_init_device(zdev);
}
static void s390_iommu_get_resv_regions(struct device *dev,
struct list_head *list)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
struct iommu_resv_region *region;
if (zdev->start_dma) {
region = iommu_alloc_resv_region(0, zdev->start_dma, 0,
IOMMU_RESV_RESERVED, GFP_KERNEL);
if (!region)
return;
list_add_tail(&region->list, list);
}
spin_unlock_irqrestore(&s390_domain->list_lock, flags);
if (found && (zdev->s390_domain == s390_domain)) {
zdev->s390_domain = NULL;
zpci_unregister_ioat(zdev, 0);
zpci_dma_init_device(zdev);
if (zdev->end_dma < ZPCI_TABLE_SIZE_RT - 1) {
region = iommu_alloc_resv_region(zdev->end_dma + 1,
ZPCI_TABLE_SIZE_RT - zdev->end_dma - 1,
0, IOMMU_RESV_RESERVED, GFP_KERNEL);
if (!region)
return;
list_add_tail(&region->list, list);
}
}
@ -192,55 +188,88 @@ static struct iommu_device *s390_iommu_probe_device(struct device *dev)
zdev = to_zpci_dev(dev);
if (zdev->start_dma > zdev->end_dma ||
zdev->start_dma > ZPCI_TABLE_SIZE_RT - 1)
return ERR_PTR(-EINVAL);
if (zdev->end_dma > ZPCI_TABLE_SIZE_RT - 1)
zdev->end_dma = ZPCI_TABLE_SIZE_RT - 1;
return &zdev->iommu_dev;
}
static void s390_iommu_release_device(struct device *dev)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
struct iommu_domain *domain;
/*
* This is a workaround for a scenario where the IOMMU API common code
* "forgets" to call the detach_dev callback: After binding a device
* to vfio-pci and completing the VFIO_SET_IOMMU ioctl (which triggers
* the attach_dev), removing the device via
* "echo 1 > /sys/bus/pci/devices/.../remove" won't trigger detach_dev,
* only release_device will be called via the BUS_NOTIFY_REMOVED_DEVICE
* notifier.
*
* So let's call detach_dev from here if it hasn't been called before.
* release_device is expected to detach any domain currently attached
* to the device, but keep it attached to other devices in the group.
*/
if (zdev && zdev->s390_domain) {
domain = iommu_get_domain_for_dev(dev);
if (domain)
s390_iommu_detach_device(domain, dev);
}
if (zdev)
__s390_iommu_detach_device(zdev);
}
static int s390_iommu_update_trans(struct s390_domain *s390_domain,
phys_addr_t pa, dma_addr_t dma_addr,
size_t size, int flags)
static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma,
zdev->end_dma - zdev->start_dma + 1);
}
rcu_read_unlock();
}
static void s390_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
size_t size = gather->end - gather->start + 1;
struct zpci_dev *zdev;
/* If gather was never added to there is nothing to flush */
if (!gather->end)
return;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
zpci_refresh_trans((u64)zdev->fh << 32, gather->start,
size);
}
rcu_read_unlock();
}
static void s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
struct zpci_dev *zdev;
rcu_read_lock();
list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) {
if (!zdev->tlb_refresh)
continue;
zpci_refresh_trans((u64)zdev->fh << 32,
iova, size);
}
rcu_read_unlock();
}
static int s390_iommu_validate_trans(struct s390_domain *s390_domain,
phys_addr_t pa, dma_addr_t dma_addr,
unsigned long nr_pages, int flags)
{
struct s390_domain_device *domain_device;
phys_addr_t page_addr = pa & PAGE_MASK;
dma_addr_t start_dma_addr = dma_addr;
unsigned long irq_flags, nr_pages, i;
unsigned long *entry;
int rc = 0;
unsigned long i;
int rc;
if (dma_addr < s390_domain->domain.geometry.aperture_start ||
dma_addr + size > s390_domain->domain.geometry.aperture_end)
return -EINVAL;
nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
if (!nr_pages)
return 0;
spin_lock_irqsave(&s390_domain->dma_table_lock, irq_flags);
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
if (!entry) {
if (unlikely(!entry)) {
rc = -ENOMEM;
goto undo_cpu_trans;
}
@ -249,47 +278,70 @@ static int s390_iommu_update_trans(struct s390_domain *s390_domain,
dma_addr += PAGE_SIZE;
}
spin_lock(&s390_domain->list_lock);
list_for_each_entry(domain_device, &s390_domain->devices, list) {
rc = zpci_refresh_trans((u64) domain_device->zdev->fh << 32,
start_dma_addr, nr_pages * PAGE_SIZE);
if (rc)
break;
}
spin_unlock(&s390_domain->list_lock);
return 0;
undo_cpu_trans:
if (rc && ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID)) {
flags = ZPCI_PTE_INVALID;
while (i-- > 0) {
page_addr -= PAGE_SIZE;
dma_addr -= PAGE_SIZE;
entry = dma_walk_cpu_trans(s390_domain->dma_table,
dma_addr);
if (!entry)
break;
dma_update_cpu_trans(entry, page_addr, flags);
}
while (i-- > 0) {
dma_addr -= PAGE_SIZE;
entry = dma_walk_cpu_trans(s390_domain->dma_table,
dma_addr);
if (!entry)
break;
dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID);
}
spin_unlock_irqrestore(&s390_domain->dma_table_lock, irq_flags);
return rc;
}
static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
static int s390_iommu_invalidate_trans(struct s390_domain *s390_domain,
dma_addr_t dma_addr, unsigned long nr_pages)
{
unsigned long *entry;
unsigned long i;
int rc = 0;
for (i = 0; i < nr_pages; i++) {
entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
if (unlikely(!entry)) {
rc = -EINVAL;
break;
}
dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID);
dma_addr += PAGE_SIZE;
}
return rc;
}
static int s390_iommu_map_pages(struct iommu_domain *domain,
unsigned long iova, phys_addr_t paddr,
size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
size_t size = pgcount << __ffs(pgsize);
int flags = ZPCI_PTE_VALID, rc = 0;
if (pgsize != SZ_4K)
return -EINVAL;
if (iova < s390_domain->domain.geometry.aperture_start ||
(iova + size - 1) > s390_domain->domain.geometry.aperture_end)
return -EINVAL;
if (!IS_ALIGNED(iova | paddr, pgsize))
return -EINVAL;
if (!(prot & IOMMU_READ))
return -EINVAL;
if (!(prot & IOMMU_WRITE))
flags |= ZPCI_TABLE_PROTECTED;
rc = s390_iommu_update_trans(s390_domain, paddr, iova,
size, flags);
rc = s390_iommu_validate_trans(s390_domain, paddr, iova,
pgcount, flags);
if (!rc)
*mapped = size;
return rc;
}
@ -298,7 +350,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
unsigned long *sto, *pto, *rto, flags;
unsigned long *rto, *sto, *pto;
unsigned long ste, pte, rte;
unsigned int rtx, sx, px;
phys_addr_t phys = 0;
@ -311,38 +364,40 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
px = calc_px(iova);
rto = s390_domain->dma_table;
spin_lock_irqsave(&s390_domain->dma_table_lock, flags);
if (rto && reg_entry_isvalid(rto[rtx])) {
sto = get_rt_sto(rto[rtx]);
if (sto && reg_entry_isvalid(sto[sx])) {
pto = get_st_pto(sto[sx]);
if (pto && pt_entry_isvalid(pto[px]))
phys = pto[px] & ZPCI_PTE_ADDR_MASK;
rte = READ_ONCE(rto[rtx]);
if (reg_entry_isvalid(rte)) {
sto = get_rt_sto(rte);
ste = READ_ONCE(sto[sx]);
if (reg_entry_isvalid(ste)) {
pto = get_st_pto(ste);
pte = READ_ONCE(pto[px]);
if (pt_entry_isvalid(pte))
phys = pte & ZPCI_PTE_ADDR_MASK;
}
}
spin_unlock_irqrestore(&s390_domain->dma_table_lock, flags);
return phys;
}
static size_t s390_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
struct iommu_iotlb_gather *gather)
static size_t s390_iommu_unmap_pages(struct iommu_domain *domain,
unsigned long iova,
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
int flags = ZPCI_PTE_INVALID;
phys_addr_t paddr;
size_t size = pgcount << __ffs(pgsize);
int rc;
paddr = s390_iommu_iova_to_phys(domain, iova);
if (!paddr)
if (WARN_ON(iova < s390_domain->domain.geometry.aperture_start ||
(iova + size - 1) > s390_domain->domain.geometry.aperture_end))
return 0;
rc = s390_iommu_update_trans(s390_domain, paddr, iova,
size, flags);
rc = s390_iommu_invalidate_trans(s390_domain, iova, pgcount);
if (rc)
return 0;
iommu_iotlb_gather_add_range(gather, iova, size);
return size;
}
@ -380,12 +435,16 @@ static const struct iommu_ops s390_iommu_ops = {
.probe_device = s390_iommu_probe_device,
.release_device = s390_iommu_release_device,
.device_group = generic_device_group,
.pgsize_bitmap = S390_IOMMU_PGSIZES,
.pgsize_bitmap = SZ_4K,
.get_resv_regions = s390_iommu_get_resv_regions,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = s390_iommu_attach_device,
.detach_dev = s390_iommu_detach_device,
.map = s390_iommu_map,
.unmap = s390_iommu_unmap,
.map_pages = s390_iommu_map_pages,
.unmap_pages = s390_iommu_unmap_pages,
.flush_iotlb_all = s390_iommu_flush_iotlb_all,
.iotlb_sync = s390_iommu_iotlb_sync,
.iotlb_sync_map = s390_iommu_iotlb_sync_map,
.iova_to_phys = s390_iommu_iova_to_phys,
.free = s390_domain_free,
}

25
drivers/iommu/sprd-iommu.c
View File

@ -271,10 +271,11 @@ static void sprd_iommu_detach_device(struct iommu_domain *domain,
}
static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped)
{
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
unsigned int page_num = size >> SPRD_IOMMU_PAGE_SHIFT;
size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE;
unsigned long flags;
unsigned int i;
u32 *pgt_base_iova;
@ -296,35 +297,37 @@ static int sprd_iommu_map(struct iommu_domain *domain, unsigned long iova,
pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT);
spin_lock_irqsave(&dom->pgtlock, flags);
for (i = 0; i < page_num; i++) {
for (i = 0; i < pgcount; i++) {
pgt_base_iova[i] = pabase >> SPRD_IOMMU_PAGE_SHIFT;
pabase += SPRD_IOMMU_PAGE_SIZE;
}
spin_unlock_irqrestore(&dom->pgtlock, flags);
*mapped = size;
return 0;
}
static size_t sprd_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size, struct iommu_iotlb_gather *iotlb_gather)
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *iotlb_gather)
{
struct sprd_iommu_domain *dom = to_sprd_domain(domain);
unsigned long flags;
u32 *pgt_base_iova;
unsigned int page_num = size >> SPRD_IOMMU_PAGE_SHIFT;
size_t size = pgcount * SPRD_IOMMU_PAGE_SIZE;
unsigned long start = domain->geometry.aperture_start;
unsigned long end = domain->geometry.aperture_end;
if (iova < start || (iova + size) > (end + 1))
return -EINVAL;
return 0;
pgt_base_iova = dom->pgt_va + ((iova - start) >> SPRD_IOMMU_PAGE_SHIFT);
spin_lock_irqsave(&dom->pgtlock, flags);
memset(pgt_base_iova, 0, page_num * sizeof(u32));
memset(pgt_base_iova, 0, pgcount * sizeof(u32));
spin_unlock_irqrestore(&dom->pgtlock, flags);
return 0;
return size;
}
static void sprd_iommu_sync_map(struct iommu_domain *domain,
@ -407,13 +410,13 @@ static const struct iommu_ops sprd_iommu_ops = {
.probe_device = sprd_iommu_probe_device,
.device_group = sprd_iommu_device_group,
.of_xlate = sprd_iommu_of_xlate,
.pgsize_bitmap = ~0UL << SPRD_IOMMU_PAGE_SHIFT,
.pgsize_bitmap = SPRD_IOMMU_PAGE_SIZE,
.owner = THIS_MODULE,
.default_domain_ops = &(const struct iommu_domain_ops) {
.attach_dev = sprd_iommu_attach_device,
.detach_dev = sprd_iommu_detach_device,
.map = sprd_iommu_map,
.unmap = sprd_iommu_unmap,
.map_pages = sprd_iommu_map,
.unmap_pages = sprd_iommu_unmap,
.iotlb_sync_map = sprd_iommu_sync_map,
.iotlb_sync = sprd_iommu_sync,
.iova_to_phys = sprd_iommu_iova_to_phys,

89
drivers/iommu/sun50i-iommu.c
View File

@ -27,6 +27,7 @@
#include <linux/types.h>
#define IOMMU_RESET_REG 0x010
#define IOMMU_RESET_RELEASE_ALL 0xffffffff
#define IOMMU_ENABLE_REG 0x020
#define IOMMU_ENABLE_ENABLE BIT(0)
@ -92,6 +93,8 @@
#define NUM_PT_ENTRIES 256
#define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
#define SPAGE_SIZE 4096
struct sun50i_iommu {
struct iommu_device iommu;
@ -270,7 +273,7 @@ static u32 sun50i_mk_pte(phys_addr_t page, int prot)
enum sun50i_iommu_aci aci;
u32 flags = 0;
if (prot & (IOMMU_READ | IOMMU_WRITE))
if ((prot & (IOMMU_READ | IOMMU_WRITE)) == (IOMMU_READ | IOMMU_WRITE))
aci = SUN50I_IOMMU_ACI_RD_WR;
else if (prot & IOMMU_READ)
aci = SUN50I_IOMMU_ACI_RD;
@ -294,6 +297,62 @@ static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
}
static void sun50i_iommu_zap_iova(struct sun50i_iommu *iommu,
unsigned long iova)
{
u32 reg;
int ret;
iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova);
iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_MASK_REG, GENMASK(31, 12));
iommu_write(iommu, IOMMU_TLB_IVLD_ENABLE_REG,
IOMMU_TLB_IVLD_ENABLE_ENABLE);
ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_IVLD_ENABLE_REG,
reg, !reg, 1, 2000);
if (ret)
dev_warn(iommu->dev, "TLB invalidation timed out!\n");
}
static void sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu,
unsigned long iova)
{
u32 reg;
int ret;
iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova);
iommu_write(iommu, IOMMU_PC_IVLD_ENABLE_REG,
IOMMU_PC_IVLD_ENABLE_ENABLE);
ret = readl_poll_timeout_atomic(iommu->base + IOMMU_PC_IVLD_ENABLE_REG,
reg, !reg, 1, 2000);
if (ret)
dev_warn(iommu->dev, "PTW cache invalidation timed out!\n");
}
static void sun50i_iommu_zap_range(struct sun50i_iommu *iommu,
unsigned long iova, size_t size)
{
assert_spin_locked(&iommu->iommu_lock);
iommu_write(iommu, IOMMU_AUTO_GATING_REG, 0);
sun50i_iommu_zap_iova(iommu, iova);
sun50i_iommu_zap_iova(iommu, iova + SPAGE_SIZE);
if (size > SPAGE_SIZE) {
sun50i_iommu_zap_iova(iommu, iova + size);
sun50i_iommu_zap_iova(iommu, iova + size + SPAGE_SIZE);
}
sun50i_iommu_zap_ptw_cache(iommu, iova);
sun50i_iommu_zap_ptw_cache(iommu, iova + SZ_1M);
if (size > SZ_1M) {
sun50i_iommu_zap_ptw_cache(iommu, iova + size);
sun50i_iommu_zap_ptw_cache(iommu, iova + size + SZ_1M);
}
iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
}
static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
{
u32 reg;
@ -343,6 +402,18 @@ static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}
static void sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
struct sun50i_iommu *iommu = sun50i_domain->iommu;
unsigned long flags;
spin_lock_irqsave(&iommu->iommu_lock, flags);
sun50i_iommu_zap_range(iommu, iova, size);
spin_unlock_irqrestore(&iommu->iommu_lock, flags);
}
static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
@ -511,7 +582,7 @@ static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
sun50i_iommu_free_page_table(iommu, drop_pt);
}
sun50i_table_flush(sun50i_domain, page_table, PT_SIZE);
sun50i_table_flush(sun50i_domain, page_table, NUM_PT_ENTRIES);
sun50i_table_flush(sun50i_domain, dte_addr, 1);
return page_table;
@ -601,7 +672,6 @@ static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
struct sun50i_iommu_domain *sun50i_domain;
if (type != IOMMU_DOMAIN_DMA &&
type != IOMMU_DOMAIN_IDENTITY &&
type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
@ -766,6 +836,7 @@ static const struct iommu_ops sun50i_iommu_ops = {
.attach_dev = sun50i_iommu_attach_device,
.detach_dev = sun50i_iommu_detach_device,
.flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
.iotlb_sync_map = sun50i_iommu_iotlb_sync_map,
.iotlb_sync = sun50i_iommu_iotlb_sync,
.iova_to_phys = sun50i_iommu_iova_to_phys,
.map = sun50i_iommu_map,
@ -785,6 +856,8 @@ static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
else
dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
sun50i_iommu_zap_range(iommu, iova, SPAGE_SIZE);
}
static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
@ -868,8 +941,8 @@ static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
{
u32 status, l1_status, l2_status, resets;
struct sun50i_iommu *iommu = dev_id;
u32 status;
spin_lock(&iommu->iommu_lock);
@ -879,6 +952,9 @@ static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
return IRQ_NONE;
}
l1_status = iommu_read(iommu, IOMMU_L1PG_INT_REG);
l2_status = iommu_read(iommu, IOMMU_L2PG_INT_REG);
if (status & IOMMU_INT_INVALID_L2PG)
sun50i_iommu_handle_pt_irq(iommu,
IOMMU_INT_ERR_ADDR_L2_REG,
@ -892,8 +968,9 @@ static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
iommu_write(iommu, IOMMU_INT_CLR_REG, status);
iommu_write(iommu, IOMMU_RESET_REG, ~status);
iommu_write(iommu, IOMMU_RESET_REG, status);
resets = (status | l1_status | l2_status) & IOMMU_INT_MASTER_MASK;
iommu_write(iommu, IOMMU_RESET_REG, ~resets);
iommu_write(iommu, IOMMU_RESET_REG, IOMMU_RESET_RELEASE_ALL);
spin_unlock(&iommu->iommu_lock);

90
include/dt-bindings/memory/mediatek,mt8365-larb-port.h Normal file
View File

@ -0,0 +1,90 @@
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/*
* Copyright (c) 2022 MediaTek Inc.
* Author: Yong Wu <yong.wu@mediatek.com>
*/
#ifndef _DT_BINDINGS_MEMORY_MT8365_LARB_PORT_H_
#define _DT_BINDINGS_MEMORY_MT8365_LARB_PORT_H_
#include <dt-bindings/memory/mtk-memory-port.h>
#define M4U_LARB0_ID 0
#define M4U_LARB1_ID 1
#define M4U_LARB2_ID 2
#define M4U_LARB3_ID 3
/* larb0 */
#define M4U_PORT_DISP_OVL0 MTK_M4U_ID(M4U_LARB0_ID, 0)
#define M4U_PORT_DISP_OVL0_2L MTK_M4U_ID(M4U_LARB0_ID, 1)
#define M4U_PORT_DISP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 2)
#define M4U_PORT_DISP_WDMA0 MTK_M4U_ID(M4U_LARB0_ID, 3)
#define M4U_PORT_DISP_RDMA1 MTK_M4U_ID(M4U_LARB0_ID, 4)
#define M4U_PORT_MDP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 5)
#define M4U_PORT_MDP_WROT1 MTK_M4U_ID(M4U_LARB0_ID, 6)
#define M4U_PORT_MDP_WROT0 MTK_M4U_ID(M4U_LARB0_ID, 7)
#define M4U_PORT_MDP_RDMA1 MTK_M4U_ID(M4U_LARB0_ID, 8)
#define M4U_PORT_DISP_FAKE0 MTK_M4U_ID(M4U_LARB0_ID, 9)
#define M4U_PORT_APU_READ MTK_M4U_ID(M4U_LARB0_ID, 10)
#define M4U_PORT_APU_WRITE MTK_M4U_ID(M4U_LARB0_ID, 11)
/* larb1 */
#define M4U_PORT_VENC_RCPU MTK_M4U_ID(M4U_LARB1_ID, 0)
#define M4U_PORT_VENC_REC MTK_M4U_ID(M4U_LARB1_ID, 1)
#define M4U_PORT_VENC_BSDMA MTK_M4U_ID(M4U_LARB1_ID, 2)
#define M4U_PORT_VENC_SV_COMV MTK_M4U_ID(M4U_LARB1_ID, 3)
#define M4U_PORT_VENC_RD_COMV MTK_M4U_ID(M4U_LARB1_ID, 4)
#define M4U_PORT_VENC_NBM_RDMA MTK_M4U_ID(M4U_LARB1_ID, 5)
#define M4U_PORT_VENC_NBM_RDMA_LITE MTK_M4U_ID(M4U_LARB1_ID, 6)
#define M4U_PORT_JPGENC_Y_RDMA MTK_M4U_ID(M4U_LARB1_ID, 7)
#define M4U_PORT_JPGENC_C_RDMA MTK_M4U_ID(M4U_LARB1_ID, 8)
#define M4U_PORT_JPGENC_Q_TABLE MTK_M4U_ID(M4U_LARB1_ID, 9)
#define M4U_PORT_JPGENC_BSDMA MTK_M4U_ID(M4U_LARB1_ID, 10)
#define M4U_PORT_JPGDEC_WDMA MTK_M4U_ID(M4U_LARB1_ID, 11)
#define M4U_PORT_JPGDEC_BSDMA MTK_M4U_ID(M4U_LARB1_ID, 12)
#define M4U_PORT_VENC_NBM_WDMA MTK_M4U_ID(M4U_LARB1_ID, 13)
#define M4U_PORT_VENC_NBM_WDMA_LITE MTK_M4U_ID(M4U_LARB1_ID, 14)
#define M4U_PORT_VENC_CUR_LUMA MTK_M4U_ID(M4U_LARB1_ID, 15)
#define M4U_PORT_VENC_CUR_CHROMA MTK_M4U_ID(M4U_LARB1_ID, 16)
#define M4U_PORT_VENC_REF_LUMA MTK_M4U_ID(M4U_LARB1_ID, 17)
#define M4U_PORT_VENC_REF_CHROMA MTK_M4U_ID(M4U_LARB1_ID, 18)
/* larb2 */
#define M4U_PORT_CAM_IMGO MTK_M4U_ID(M4U_LARB2_ID, 0)
#define M4U_PORT_CAM_RRZO MTK_M4U_ID(M4U_LARB2_ID, 1)
#define M4U_PORT_CAM_AAO MTK_M4U_ID(M4U_LARB2_ID, 2)
#define M4U_PORT_CAM_LCS MTK_M4U_ID(M4U_LARB2_ID, 3)
#define M4U_PORT_CAM_ESFKO MTK_M4U_ID(M4U_LARB2_ID, 4)
#define M4U_PORT_CAM_CAM_SV0 MTK_M4U_ID(M4U_LARB2_ID, 5)
#define M4U_PORT_CAM_CAM_SV1 MTK_M4U_ID(M4U_LARB2_ID, 6)
#define M4U_PORT_CAM_LSCI MTK_M4U_ID(M4U_LARB2_ID, 7)
#define M4U_PORT_CAM_LSCI_D MTK_M4U_ID(M4U_LARB2_ID, 8)
#define M4U_PORT_CAM_AFO MTK_M4U_ID(M4U_LARB2_ID, 9)
#define M4U_PORT_CAM_SPARE MTK_M4U_ID(M4U_LARB2_ID, 10)
#define M4U_PORT_CAM_BPCI MTK_M4U_ID(M4U_LARB2_ID, 11)
#define M4U_PORT_CAM_BPCI_D MTK_M4U_ID(M4U_LARB2_ID, 12)
#define M4U_PORT_CAM_UFDI MTK_M4U_ID(M4U_LARB2_ID, 13)
#define M4U_PORT_CAM_IMGI MTK_M4U_ID(M4U_LARB2_ID, 14)
#define M4U_PORT_CAM_IMG2O MTK_M4U_ID(M4U_LARB2_ID, 15)
#define M4U_PORT_CAM_IMG3O MTK_M4U_ID(M4U_LARB2_ID, 16)
#define M4U_PORT_CAM_WPE0_I MTK_M4U_ID(M4U_LARB2_ID, 17)
#define M4U_PORT_CAM_WPE1_I MTK_M4U_ID(M4U_LARB2_ID, 18)
#define M4U_PORT_CAM_WPE_O MTK_M4U_ID(M4U_LARB2_ID, 19)
#define M4U_PORT_CAM_FD0_I MTK_M4U_ID(M4U_LARB2_ID, 20)
#define M4U_PORT_CAM_FD1_I MTK_M4U_ID(M4U_LARB2_ID, 21)
#define M4U_PORT_CAM_FD0_O MTK_M4U_ID(M4U_LARB2_ID, 22)
#define M4U_PORT_CAM_FD1_O MTK_M4U_ID(M4U_LARB2_ID, 23)
/* larb3 */
#define M4U_PORT_HW_VDEC_MC_EXT MTK_M4U_ID(M4U_LARB3_ID, 0)
#define M4U_PORT_HW_VDEC_UFO_EXT MTK_M4U_ID(M4U_LARB3_ID, 1)
#define M4U_PORT_HW_VDEC_PP_EXT MTK_M4U_ID(M4U_LARB3_ID, 2)
#define M4U_PORT_HW_VDEC_PRED_RD_EXT MTK_M4U_ID(M4U_LARB3_ID, 3)
#define M4U_PORT_HW_VDEC_PRED_WR_EXT MTK_M4U_ID(M4U_LARB3_ID, 4)
#define M4U_PORT_HW_VDEC_PPWRAP_EXT MTK_M4U_ID(M4U_LARB3_ID, 5)
#define M4U_PORT_HW_VDEC_TILE_EXT MTK_M4U_ID(M4U_LARB3_ID, 6)
#define M4U_PORT_HW_VDEC_VLD_EXT MTK_M4U_ID(M4U_LARB3_ID, 7)
#define M4U_PORT_HW_VDEC_VLD2_EXT MTK_M4U_ID(M4U_LARB3_ID, 8)
#define M4U_PORT_HW_VDEC_AVC_MV_EXT MTK_M4U_ID(M4U_LARB3_ID, 9)
#define M4U_PORT_HW_VDEC_RG_CTRL_DMA_EXT MTK_M4U_ID(M4U_LARB3_ID, 10)
#endif

6
include/linux/io-pgtable.h
View File

@ -150,9 +150,7 @@ struct io_pgtable_cfg {
/**
* struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
*
* @map: Map a physically contiguous memory region.
* @map_pages: Map a physically contiguous range of pages of the same size.
* @unmap: Unmap a physically contiguous memory region.
* @unmap_pages: Unmap a range of virtually contiguous pages of the same size.
* @iova_to_phys: Translate iova to physical address.
*
@ -160,13 +158,9 @@ struct io_pgtable_cfg {
* the same names.
*/
struct io_pgtable_ops {
int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
int (*map_pages)(struct io_pgtable_ops *ops, unsigned long iova,
phys_addr_t paddr, size_t pgsize, size_t pgcount,
int prot, gfp_t gfp, size_t *mapped);
size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
size_t size, struct iommu_iotlb_gather *gather);
size_t (*unmap_pages)(struct io_pgtable_ops *ops, unsigned long iova,
size_t pgsize, size_t pgcount,
struct iommu_iotlb_gather *gather);