2008-09-14 00:48:28 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2006, Intel Corporation.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
|
|
* under the terms and conditions of the GNU General Public License,
|
|
|
|
* version 2, as published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope it will be useful, but WITHOUT
|
|
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
* more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License along with
|
|
|
|
* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
|
|
|
|
* Place - Suite 330, Boston, MA 02111-1307 USA.
|
|
|
|
*
|
|
|
|
* Copyright (C) 2006-2008 Intel Corporation
|
|
|
|
* Copyright IBM Corporation, 2008
|
2010-05-23 15:37:00 +00:00
|
|
|
* Copyright 2010 Red Hat, Inc. and/or its affiliates.
|
|
|
|
*
|
2008-09-14 00:48:28 +00:00
|
|
|
* Author: Allen M. Kay <allen.m.kay@intel.com>
|
|
|
|
* Author: Weidong Han <weidong.han@intel.com>
|
|
|
|
* Author: Ben-Ami Yassour <benami@il.ibm.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/list.h>
|
|
|
|
#include <linux/kvm_host.h>
|
2011-07-28 01:25:05 +00:00
|
|
|
#include <linux/module.h>
|
2008-09-14 00:48:28 +00:00
|
|
|
#include <linux/pci.h>
|
2011-07-28 01:17:59 +00:00
|
|
|
#include <linux/stat.h>
|
2008-09-14 00:48:28 +00:00
|
|
|
#include <linux/dmar.h>
|
2008-12-03 13:43:34 +00:00
|
|
|
#include <linux/iommu.h>
|
2008-09-14 00:48:28 +00:00
|
|
|
#include <linux/intel-iommu.h>
|
2014-11-24 14:27:17 +00:00
|
|
|
#include "assigned-dev.h"
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2012-01-12 23:02:20 +00:00
|
|
|
static bool allow_unsafe_assigned_interrupts;
|
2011-07-14 19:27:03 +00:00
|
|
|
module_param_named(allow_unsafe_assigned_interrupts,
|
|
|
|
allow_unsafe_assigned_interrupts, bool, S_IRUGO | S_IWUSR);
|
|
|
|
MODULE_PARM_DESC(allow_unsafe_assigned_interrupts,
|
|
|
|
"Enable device assignment on platforms without interrupt remapping support.");
|
|
|
|
|
2008-09-14 00:48:28 +00:00
|
|
|
static int kvm_iommu_unmap_memslots(struct kvm *kvm);
|
|
|
|
static void kvm_iommu_put_pages(struct kvm *kvm,
|
|
|
|
gfn_t base_gfn, unsigned long npages);
|
|
|
|
|
2012-07-17 13:56:16 +00:00
|
|
|
static pfn_t kvm_pin_pages(struct kvm_memory_slot *slot, gfn_t gfn,
|
2014-10-17 20:55:59 +00:00
|
|
|
unsigned long npages)
|
2010-01-11 15:38:18 +00:00
|
|
|
{
|
|
|
|
gfn_t end_gfn;
|
|
|
|
pfn_t pfn;
|
|
|
|
|
2012-07-17 13:56:16 +00:00
|
|
|
pfn = gfn_to_pfn_memslot(slot, gfn);
|
2014-10-17 20:55:59 +00:00
|
|
|
end_gfn = gfn + npages;
|
2010-01-11 15:38:18 +00:00
|
|
|
gfn += 1;
|
|
|
|
|
2012-10-16 12:10:59 +00:00
|
|
|
if (is_error_noslot_pfn(pfn))
|
2010-01-11 15:38:18 +00:00
|
|
|
return pfn;
|
|
|
|
|
|
|
|
while (gfn < end_gfn)
|
2012-07-17 13:56:16 +00:00
|
|
|
gfn_to_pfn_memslot(slot, gfn++);
|
2010-01-11 15:38:18 +00:00
|
|
|
|
|
|
|
return pfn;
|
|
|
|
}
|
|
|
|
|
kvm: iommu: fix the third parameter of kvm_iommu_put_pages (CVE-2014-3601)
The third parameter of kvm_iommu_put_pages is wrong,
It should be 'gfn - slot->base_gfn'.
By making gfn very large, malicious guest or userspace can cause kvm to
go to this error path, and subsequently to pass a huge value as size.
Alternatively if gfn is small, then pages would be pinned but never
unpinned, causing host memory leak and local DOS.
Passing a reasonable but large value could be the most dangerous case,
because it would unpin a page that should have stayed pinned, and thus
allow the device to DMA into arbitrary memory. However, this cannot
happen because of the condition that can trigger the error:
- out of memory (where you can't allocate even a single page)
should not be possible for the attacker to trigger
- when exceeding the iommu's address space, guest pages after gfn
will also exceed the iommu's address space, and inside
kvm_iommu_put_pages() the iommu_iova_to_phys() will fail. The
page thus would not be unpinned at all.
Reported-by: Jack Morgenstein <jackm@mellanox.com>
Cc: stable@vger.kernel.org
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2014-08-19 11:14:50 +00:00
|
|
|
static void kvm_unpin_pages(struct kvm *kvm, pfn_t pfn, unsigned long npages)
|
|
|
|
{
|
|
|
|
unsigned long i;
|
|
|
|
|
|
|
|
for (i = 0; i < npages; ++i)
|
|
|
|
kvm_release_pfn_clean(pfn + i);
|
|
|
|
}
|
|
|
|
|
2009-12-23 16:35:20 +00:00
|
|
|
int kvm_iommu_map_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
|
2008-09-14 00:48:28 +00:00
|
|
|
{
|
2010-01-11 15:38:18 +00:00
|
|
|
gfn_t gfn, end_gfn;
|
2008-09-14 00:48:28 +00:00
|
|
|
pfn_t pfn;
|
2010-01-11 15:38:18 +00:00
|
|
|
int r = 0;
|
2008-12-03 13:43:34 +00:00
|
|
|
struct iommu_domain *domain = kvm->arch.iommu_domain;
|
2009-04-27 12:35:43 +00:00
|
|
|
int flags;
|
2008-09-14 00:48:28 +00:00
|
|
|
|
|
|
|
/* check if iommu exists and in use */
|
|
|
|
if (!domain)
|
|
|
|
return 0;
|
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
gfn = slot->base_gfn;
|
|
|
|
end_gfn = gfn + slot->npages;
|
|
|
|
|
2013-01-24 22:04:09 +00:00
|
|
|
flags = IOMMU_READ;
|
|
|
|
if (!(slot->flags & KVM_MEM_READONLY))
|
|
|
|
flags |= IOMMU_WRITE;
|
2013-10-30 17:02:23 +00:00
|
|
|
if (!kvm->arch.iommu_noncoherent)
|
2009-04-27 12:35:43 +00:00
|
|
|
flags |= IOMMU_CACHE;
|
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
|
|
|
|
while (gfn < end_gfn) {
|
|
|
|
unsigned long page_size;
|
|
|
|
|
|
|
|
/* Check if already mapped */
|
|
|
|
if (iommu_iova_to_phys(domain, gfn_to_gpa(gfn))) {
|
|
|
|
gfn += 1;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Get the page size we could use to map */
|
|
|
|
page_size = kvm_host_page_size(kvm, gfn);
|
|
|
|
|
|
|
|
/* Make sure the page_size does not exceed the memslot */
|
|
|
|
while ((gfn + (page_size >> PAGE_SHIFT)) > end_gfn)
|
|
|
|
page_size >>= 1;
|
|
|
|
|
|
|
|
/* Make sure gfn is aligned to the page size we want to map */
|
|
|
|
while ((gfn << PAGE_SHIFT) & (page_size - 1))
|
|
|
|
page_size >>= 1;
|
|
|
|
|
KVM: IOMMU: hva align mapping page size
When determining the page size we could use to map with the IOMMU, the
page size should also be aligned with the hva, not just the gfn. The
gfn may not reflect the real alignment within the hugetlbfs file.
Most of the time, this works fine. However, if the hugetlbfs file is
backed by non-contiguous huge pages, a multi-huge page memslot starts at
an unaligned offset within the hugetlbfs file, and the gfn is aligned
with respect to the huge page size, kvm_host_page_size() will return the
huge page size and we will use that to map with the IOMMU.
When we later unpin that same memslot, the IOMMU returns the unmap size
as the huge page size, and we happily unpin that many pfns in
monotonically increasing order, not realizing we are spanning
non-contiguous huge pages and partially unpin the wrong huge page.
Ensure the IOMMU mapping page size is aligned with the hva corresponding
to the gfn, which does reflect the alignment within the hugetlbfs file.
Reviewed-by: Marcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: Greg Edwards <gedwards@ddn.com>
Cc: stable@vger.kernel.org
Signed-off-by: Gleb Natapov <gleb@redhat.com>
2013-11-04 16:08:12 +00:00
|
|
|
/* Make sure hva is aligned to the page size we want to map */
|
|
|
|
while (__gfn_to_hva_memslot(slot, gfn) & (page_size - 1))
|
|
|
|
page_size >>= 1;
|
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
/*
|
|
|
|
* Pin all pages we are about to map in memory. This is
|
|
|
|
* important because we unmap and unpin in 4kb steps later.
|
|
|
|
*/
|
2014-10-17 20:55:59 +00:00
|
|
|
pfn = kvm_pin_pages(slot, gfn, page_size >> PAGE_SHIFT);
|
2012-10-16 12:10:59 +00:00
|
|
|
if (is_error_noslot_pfn(pfn)) {
|
2010-01-11 15:38:18 +00:00
|
|
|
gfn += 1;
|
2008-09-14 00:48:28 +00:00
|
|
|
continue;
|
2010-01-11 15:38:18 +00:00
|
|
|
}
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
/* Map into IO address space */
|
|
|
|
r = iommu_map(domain, gfn_to_gpa(gfn), pfn_to_hpa(pfn),
|
iommu/core: split mapping to page sizes as supported by the hardware
When mapping a memory region, split it to page sizes as supported
by the iommu hardware. Always prefer bigger pages, when possible,
in order to reduce the TLB pressure.
The logic to do that is now added to the IOMMU core, so neither the iommu
drivers themselves nor users of the IOMMU API have to duplicate it.
This allows a more lenient granularity of mappings; traditionally the
IOMMU API took 'order' (of a page) as a mapping size, and directly let
the low level iommu drivers handle the mapping, but now that the IOMMU
core can split arbitrary memory regions into pages, we can remove this
limitation, so users don't have to split those regions by themselves.
Currently the supported page sizes are advertised once and they then
remain static. That works well for OMAP and MSM but it would probably
not fly well with intel's hardware, where the page size capabilities
seem to have the potential to be different between several DMA
remapping devices.
register_iommu() currently sets a default pgsize behavior, so we can convert
the IOMMU drivers in subsequent patches. After all the drivers
are converted, the temporary default settings will be removed.
Mainline users of the IOMMU API (kvm and omap-iovmm) are adopted
to deal with bytes instead of page order.
Many thanks to Joerg Roedel <Joerg.Roedel@amd.com> for significant review!
Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
Cc: David Brown <davidb@codeaurora.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Cc: Joerg Roedel <Joerg.Roedel@amd.com>
Cc: Stepan Moskovchenko <stepanm@codeaurora.org>
Cc: KyongHo Cho <pullip.cho@samsung.com>
Cc: Hiroshi DOYU <hdoyu@nvidia.com>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
2011-11-10 09:32:26 +00:00
|
|
|
page_size, flags);
|
2008-09-25 15:32:02 +00:00
|
|
|
if (r) {
|
2008-12-02 13:03:39 +00:00
|
|
|
printk(KERN_ERR "kvm_iommu_map_address:"
|
2010-07-01 14:00:12 +00:00
|
|
|
"iommu failed to map pfn=%llx\n", pfn);
|
2014-10-17 20:55:59 +00:00
|
|
|
kvm_unpin_pages(kvm, pfn, page_size >> PAGE_SHIFT);
|
2008-09-14 00:48:28 +00:00
|
|
|
goto unmap_pages;
|
|
|
|
}
|
2010-01-11 15:38:18 +00:00
|
|
|
|
|
|
|
gfn += page_size >> PAGE_SHIFT;
|
|
|
|
|
2015-01-27 10:57:19 +00:00
|
|
|
cond_resched();
|
2008-09-14 00:48:28 +00:00
|
|
|
}
|
2010-01-11 15:38:18 +00:00
|
|
|
|
2008-09-14 00:48:28 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
unmap_pages:
|
kvm: iommu: fix the third parameter of kvm_iommu_put_pages (CVE-2014-3601)
The third parameter of kvm_iommu_put_pages is wrong,
It should be 'gfn - slot->base_gfn'.
By making gfn very large, malicious guest or userspace can cause kvm to
go to this error path, and subsequently to pass a huge value as size.
Alternatively if gfn is small, then pages would be pinned but never
unpinned, causing host memory leak and local DOS.
Passing a reasonable but large value could be the most dangerous case,
because it would unpin a page that should have stayed pinned, and thus
allow the device to DMA into arbitrary memory. However, this cannot
happen because of the condition that can trigger the error:
- out of memory (where you can't allocate even a single page)
should not be possible for the attacker to trigger
- when exceeding the iommu's address space, guest pages after gfn
will also exceed the iommu's address space, and inside
kvm_iommu_put_pages() the iommu_iova_to_phys() will fail. The
page thus would not be unpinned at all.
Reported-by: Jack Morgenstein <jackm@mellanox.com>
Cc: stable@vger.kernel.org
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2014-08-19 11:14:50 +00:00
|
|
|
kvm_iommu_put_pages(kvm, slot->base_gfn, gfn - slot->base_gfn);
|
2008-09-14 00:48:28 +00:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int kvm_iommu_map_memslots(struct kvm *kvm)
|
|
|
|
{
|
2011-11-24 09:39:18 +00:00
|
|
|
int idx, r = 0;
|
2009-12-23 16:35:16 +00:00
|
|
|
struct kvm_memslots *slots;
|
2011-11-24 09:39:18 +00:00
|
|
|
struct kvm_memory_slot *memslot;
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2013-10-30 17:02:30 +00:00
|
|
|
if (kvm->arch.iommu_noncoherent)
|
|
|
|
kvm_arch_register_noncoherent_dma(kvm);
|
|
|
|
|
2010-07-01 07:00:50 +00:00
|
|
|
idx = srcu_read_lock(&kvm->srcu);
|
2010-04-19 09:41:23 +00:00
|
|
|
slots = kvm_memslots(kvm);
|
2009-12-23 16:35:16 +00:00
|
|
|
|
2011-11-24 09:39:18 +00:00
|
|
|
kvm_for_each_memslot(memslot, slots) {
|
|
|
|
r = kvm_iommu_map_pages(kvm, memslot);
|
2008-09-14 00:48:28 +00:00
|
|
|
if (r)
|
|
|
|
break;
|
|
|
|
}
|
2010-07-01 07:00:50 +00:00
|
|
|
srcu_read_unlock(&kvm->srcu, idx);
|
2009-02-05 18:23:46 +00:00
|
|
|
|
2008-09-14 00:48:28 +00:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:27:17 +00:00
|
|
|
int kvm_assign_device(struct kvm *kvm, struct pci_dev *pdev)
|
2008-09-14 00:48:28 +00:00
|
|
|
{
|
2008-12-03 13:43:34 +00:00
|
|
|
struct iommu_domain *domain = kvm->arch.iommu_domain;
|
2013-10-30 17:02:23 +00:00
|
|
|
int r;
|
|
|
|
bool noncoherent;
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2008-12-02 13:03:39 +00:00
|
|
|
/* check if iommu exists and in use */
|
|
|
|
if (!domain)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (pdev == NULL)
|
2008-09-14 00:48:28 +00:00
|
|
|
return -ENODEV;
|
2008-12-02 13:03:39 +00:00
|
|
|
|
2008-12-03 13:43:34 +00:00
|
|
|
r = iommu_attach_device(domain, &pdev->dev);
|
2008-12-02 13:03:39 +00:00
|
|
|
if (r) {
|
2012-10-09 00:36:11 +00:00
|
|
|
dev_err(&pdev->dev, "kvm assign device failed ret %d", r);
|
2008-12-02 13:03:39 +00:00
|
|
|
return r;
|
2008-09-14 00:48:28 +00:00
|
|
|
}
|
|
|
|
|
2014-09-05 08:54:09 +00:00
|
|
|
noncoherent = !iommu_capable(&pci_bus_type, IOMMU_CAP_CACHE_COHERENCY);
|
2009-04-27 12:35:43 +00:00
|
|
|
|
|
|
|
/* Check if need to update IOMMU page table for guest memory */
|
2013-10-30 17:02:23 +00:00
|
|
|
if (noncoherent != kvm->arch.iommu_noncoherent) {
|
2009-04-27 12:35:43 +00:00
|
|
|
kvm_iommu_unmap_memslots(kvm);
|
2013-10-30 17:02:23 +00:00
|
|
|
kvm->arch.iommu_noncoherent = noncoherent;
|
2009-04-27 12:35:43 +00:00
|
|
|
r = kvm_iommu_map_memslots(kvm);
|
|
|
|
if (r)
|
|
|
|
goto out_unmap;
|
|
|
|
}
|
|
|
|
|
2015-07-07 13:41:58 +00:00
|
|
|
kvm_arch_start_assignment(kvm);
|
2014-09-09 02:21:26 +00:00
|
|
|
pci_set_dev_assigned(pdev);
|
2011-07-22 05:46:07 +00:00
|
|
|
|
2013-10-02 10:23:26 +00:00
|
|
|
dev_info(&pdev->dev, "kvm assign device\n");
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2008-12-02 13:03:39 +00:00
|
|
|
return 0;
|
2009-04-27 12:35:43 +00:00
|
|
|
out_unmap:
|
|
|
|
kvm_iommu_unmap_memslots(kvm);
|
|
|
|
return r;
|
2008-12-02 13:03:39 +00:00
|
|
|
}
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2014-11-24 14:27:17 +00:00
|
|
|
int kvm_deassign_device(struct kvm *kvm, struct pci_dev *pdev)
|
2008-12-02 13:24:23 +00:00
|
|
|
{
|
2008-12-03 13:43:34 +00:00
|
|
|
struct iommu_domain *domain = kvm->arch.iommu_domain;
|
2008-12-02 13:24:23 +00:00
|
|
|
|
|
|
|
/* check if iommu exists and in use */
|
|
|
|
if (!domain)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (pdev == NULL)
|
|
|
|
return -ENODEV;
|
|
|
|
|
2008-12-03 13:43:34 +00:00
|
|
|
iommu_detach_device(domain, &pdev->dev);
|
2008-12-02 13:24:23 +00:00
|
|
|
|
2014-09-09 02:21:26 +00:00
|
|
|
pci_clear_dev_assigned(pdev);
|
2015-07-07 13:41:58 +00:00
|
|
|
kvm_arch_end_assignment(kvm);
|
2011-07-22 05:46:07 +00:00
|
|
|
|
2013-10-02 10:23:26 +00:00
|
|
|
dev_info(&pdev->dev, "kvm deassign device\n");
|
2008-12-02 13:24:23 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-12-02 13:03:39 +00:00
|
|
|
int kvm_iommu_map_guest(struct kvm *kvm)
|
|
|
|
{
|
|
|
|
int r;
|
|
|
|
|
2011-09-06 16:46:34 +00:00
|
|
|
if (!iommu_present(&pci_bus_type)) {
|
2008-12-03 13:43:34 +00:00
|
|
|
printk(KERN_ERR "%s: iommu not found\n", __func__);
|
2008-09-14 00:48:28 +00:00
|
|
|
return -ENODEV;
|
|
|
|
}
|
|
|
|
|
2012-04-18 03:46:44 +00:00
|
|
|
mutex_lock(&kvm->slots_lock);
|
|
|
|
|
2011-09-06 14:03:26 +00:00
|
|
|
kvm->arch.iommu_domain = iommu_domain_alloc(&pci_bus_type);
|
2012-04-18 03:46:44 +00:00
|
|
|
if (!kvm->arch.iommu_domain) {
|
|
|
|
r = -ENOMEM;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2011-07-14 19:27:03 +00:00
|
|
|
if (!allow_unsafe_assigned_interrupts &&
|
2014-09-05 08:54:09 +00:00
|
|
|
!iommu_capable(&pci_bus_type, IOMMU_CAP_INTR_REMAP)) {
|
2011-07-14 19:27:03 +00:00
|
|
|
printk(KERN_WARNING "%s: No interrupt remapping support,"
|
|
|
|
" disallowing device assignment."
|
|
|
|
" Re-enble with \"allow_unsafe_assigned_interrupts=1\""
|
|
|
|
" module option.\n", __func__);
|
|
|
|
iommu_domain_free(kvm->arch.iommu_domain);
|
|
|
|
kvm->arch.iommu_domain = NULL;
|
2012-04-18 03:46:44 +00:00
|
|
|
r = -EPERM;
|
|
|
|
goto out_unlock;
|
2011-07-14 19:27:03 +00:00
|
|
|
}
|
|
|
|
|
2008-09-14 00:48:28 +00:00
|
|
|
r = kvm_iommu_map_memslots(kvm);
|
|
|
|
if (r)
|
2012-04-18 03:46:44 +00:00
|
|
|
kvm_iommu_unmap_memslots(kvm);
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2012-04-18 03:46:44 +00:00
|
|
|
out_unlock:
|
|
|
|
mutex_unlock(&kvm->slots_lock);
|
2008-09-14 00:48:28 +00:00
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void kvm_iommu_put_pages(struct kvm *kvm,
|
2008-12-02 13:03:39 +00:00
|
|
|
gfn_t base_gfn, unsigned long npages)
|
2008-09-14 00:48:28 +00:00
|
|
|
{
|
2010-01-11 15:38:18 +00:00
|
|
|
struct iommu_domain *domain;
|
|
|
|
gfn_t end_gfn, gfn;
|
2008-09-14 00:48:28 +00:00
|
|
|
pfn_t pfn;
|
2008-12-02 13:03:39 +00:00
|
|
|
u64 phys;
|
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
domain = kvm->arch.iommu_domain;
|
|
|
|
end_gfn = base_gfn + npages;
|
|
|
|
gfn = base_gfn;
|
|
|
|
|
2008-12-02 13:03:39 +00:00
|
|
|
/* check if iommu exists and in use */
|
|
|
|
if (!domain)
|
|
|
|
return;
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
while (gfn < end_gfn) {
|
|
|
|
unsigned long unmap_pages;
|
iommu/core: split mapping to page sizes as supported by the hardware
When mapping a memory region, split it to page sizes as supported
by the iommu hardware. Always prefer bigger pages, when possible,
in order to reduce the TLB pressure.
The logic to do that is now added to the IOMMU core, so neither the iommu
drivers themselves nor users of the IOMMU API have to duplicate it.
This allows a more lenient granularity of mappings; traditionally the
IOMMU API took 'order' (of a page) as a mapping size, and directly let
the low level iommu drivers handle the mapping, but now that the IOMMU
core can split arbitrary memory regions into pages, we can remove this
limitation, so users don't have to split those regions by themselves.
Currently the supported page sizes are advertised once and they then
remain static. That works well for OMAP and MSM but it would probably
not fly well with intel's hardware, where the page size capabilities
seem to have the potential to be different between several DMA
remapping devices.
register_iommu() currently sets a default pgsize behavior, so we can convert
the IOMMU drivers in subsequent patches. After all the drivers
are converted, the temporary default settings will be removed.
Mainline users of the IOMMU API (kvm and omap-iovmm) are adopted
to deal with bytes instead of page order.
Many thanks to Joerg Roedel <Joerg.Roedel@amd.com> for significant review!
Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
Cc: David Brown <davidb@codeaurora.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Cc: Joerg Roedel <Joerg.Roedel@amd.com>
Cc: Stepan Moskovchenko <stepanm@codeaurora.org>
Cc: KyongHo Cho <pullip.cho@samsung.com>
Cc: Hiroshi DOYU <hdoyu@nvidia.com>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
2011-11-10 09:32:26 +00:00
|
|
|
size_t size;
|
2010-01-11 15:38:18 +00:00
|
|
|
|
|
|
|
/* Get physical address */
|
2008-12-03 13:43:34 +00:00
|
|
|
phys = iommu_iova_to_phys(domain, gfn_to_gpa(gfn));
|
2012-08-03 07:36:52 +00:00
|
|
|
|
|
|
|
if (!phys) {
|
|
|
|
gfn++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
pfn = phys >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
/* Unmap address from IO address space */
|
iommu/core: split mapping to page sizes as supported by the hardware
When mapping a memory region, split it to page sizes as supported
by the iommu hardware. Always prefer bigger pages, when possible,
in order to reduce the TLB pressure.
The logic to do that is now added to the IOMMU core, so neither the iommu
drivers themselves nor users of the IOMMU API have to duplicate it.
This allows a more lenient granularity of mappings; traditionally the
IOMMU API took 'order' (of a page) as a mapping size, and directly let
the low level iommu drivers handle the mapping, but now that the IOMMU
core can split arbitrary memory regions into pages, we can remove this
limitation, so users don't have to split those regions by themselves.
Currently the supported page sizes are advertised once and they then
remain static. That works well for OMAP and MSM but it would probably
not fly well with intel's hardware, where the page size capabilities
seem to have the potential to be different between several DMA
remapping devices.
register_iommu() currently sets a default pgsize behavior, so we can convert
the IOMMU drivers in subsequent patches. After all the drivers
are converted, the temporary default settings will be removed.
Mainline users of the IOMMU API (kvm and omap-iovmm) are adopted
to deal with bytes instead of page order.
Many thanks to Joerg Roedel <Joerg.Roedel@amd.com> for significant review!
Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>
Cc: David Brown <davidb@codeaurora.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Cc: Joerg Roedel <Joerg.Roedel@amd.com>
Cc: Stepan Moskovchenko <stepanm@codeaurora.org>
Cc: KyongHo Cho <pullip.cho@samsung.com>
Cc: Hiroshi DOYU <hdoyu@nvidia.com>
Cc: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
2011-11-10 09:32:26 +00:00
|
|
|
size = iommu_unmap(domain, gfn_to_gpa(gfn), PAGE_SIZE);
|
|
|
|
unmap_pages = 1ULL << get_order(size);
|
2008-12-02 13:03:39 +00:00
|
|
|
|
2010-01-11 15:38:18 +00:00
|
|
|
/* Unpin all pages we just unmapped to not leak any memory */
|
|
|
|
kvm_unpin_pages(kvm, pfn, unmap_pages);
|
|
|
|
|
|
|
|
gfn += unmap_pages;
|
2015-01-27 10:57:19 +00:00
|
|
|
|
|
|
|
cond_resched();
|
2010-01-11 15:38:18 +00:00
|
|
|
}
|
2008-09-14 00:48:28 +00:00
|
|
|
}
|
|
|
|
|
2012-04-11 15:51:49 +00:00
|
|
|
void kvm_iommu_unmap_pages(struct kvm *kvm, struct kvm_memory_slot *slot)
|
|
|
|
{
|
|
|
|
kvm_iommu_put_pages(kvm, slot->base_gfn, slot->npages);
|
|
|
|
}
|
|
|
|
|
2008-09-14 00:48:28 +00:00
|
|
|
static int kvm_iommu_unmap_memslots(struct kvm *kvm)
|
|
|
|
{
|
2011-11-24 09:39:18 +00:00
|
|
|
int idx;
|
2009-12-23 16:35:16 +00:00
|
|
|
struct kvm_memslots *slots;
|
2011-11-24 09:39:18 +00:00
|
|
|
struct kvm_memory_slot *memslot;
|
2009-12-23 16:35:16 +00:00
|
|
|
|
2010-07-01 07:00:50 +00:00
|
|
|
idx = srcu_read_lock(&kvm->srcu);
|
2010-04-19 09:41:23 +00:00
|
|
|
slots = kvm_memslots(kvm);
|
2009-02-05 18:23:46 +00:00
|
|
|
|
2011-11-24 09:39:18 +00:00
|
|
|
kvm_for_each_memslot(memslot, slots)
|
2012-04-11 15:51:49 +00:00
|
|
|
kvm_iommu_unmap_pages(kvm, memslot);
|
2011-11-24 09:39:18 +00:00
|
|
|
|
2010-07-01 07:00:50 +00:00
|
|
|
srcu_read_unlock(&kvm->srcu, idx);
|
2008-09-14 00:48:28 +00:00
|
|
|
|
2013-10-30 17:02:30 +00:00
|
|
|
if (kvm->arch.iommu_noncoherent)
|
|
|
|
kvm_arch_unregister_noncoherent_dma(kvm);
|
|
|
|
|
2008-09-14 00:48:28 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int kvm_iommu_unmap_guest(struct kvm *kvm)
|
|
|
|
{
|
2008-12-03 13:43:34 +00:00
|
|
|
struct iommu_domain *domain = kvm->arch.iommu_domain;
|
2008-09-14 00:48:28 +00:00
|
|
|
|
|
|
|
/* check if iommu exists and in use */
|
|
|
|
if (!domain)
|
|
|
|
return 0;
|
|
|
|
|
2012-04-18 03:46:44 +00:00
|
|
|
mutex_lock(&kvm->slots_lock);
|
2008-09-14 00:48:28 +00:00
|
|
|
kvm_iommu_unmap_memslots(kvm);
|
2012-04-18 03:46:44 +00:00
|
|
|
kvm->arch.iommu_domain = NULL;
|
2013-10-30 17:02:23 +00:00
|
|
|
kvm->arch.iommu_noncoherent = false;
|
2012-04-18 03:46:44 +00:00
|
|
|
mutex_unlock(&kvm->slots_lock);
|
|
|
|
|
2008-12-03 13:43:34 +00:00
|
|
|
iommu_domain_free(domain);
|
2008-09-14 00:48:28 +00:00
|
|
|
return 0;
|
|
|
|
}
|