2018-01-26 17:45:16 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2018-03-09 22:36:33 +00:00
|
|
|
* PCI Message Signaled Interrupt (MSI)
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* Copyright (C) 2003-2004 Intel
|
|
|
|
* Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
|
2016-07-12 09:20:17 +00:00
|
|
|
* Copyright (C) 2016 Christoph Hellwig.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2006-10-04 09:16:41 +00:00
|
|
|
#include <linux/err.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/irq.h>
|
|
|
|
#include <linux/interrupt.h>
|
2011-05-27 13:37:25 +00:00
|
|
|
#include <linux/export.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/ioport.h>
|
|
|
|
#include <linux/pci.h>
|
|
|
|
#include <linux/proc_fs.h>
|
2006-10-04 09:16:59 +00:00
|
|
|
#include <linux/msi.h>
|
2007-04-27 01:21:38 +00:00
|
|
|
#include <linux/smp.h>
|
2009-08-10 01:14:15 +00:00
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/io.h>
|
2016-09-12 18:32:22 +00:00
|
|
|
#include <linux/acpi_iort.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
2014-11-11 13:02:18 +00:00
|
|
|
#include <linux/irqdomain.h>
|
2015-10-08 22:10:49 +00:00
|
|
|
#include <linux/of_irq.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#include "pci.h"
|
|
|
|
|
2020-12-03 18:51:08 +00:00
|
|
|
#ifdef CONFIG_PCI_MSI
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static int pci_msi_enable = 1;
|
2014-10-27 02:44:36 +00:00
|
|
|
int pci_msi_ignore_mask;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-04-17 23:44:48 +00:00
|
|
|
#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
|
|
|
|
|
2014-11-15 14:24:07 +00:00
|
|
|
#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
|
|
|
|
static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
|
|
|
|
{
|
|
|
|
struct irq_domain *domain;
|
|
|
|
|
2017-02-08 17:17:43 +00:00
|
|
|
domain = dev_get_msi_domain(&dev->dev);
|
2015-12-04 16:28:14 +00:00
|
|
|
if (domain && irq_domain_is_hierarchy(domain))
|
2017-02-08 17:17:44 +00:00
|
|
|
return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
|
2014-11-15 14:24:07 +00:00
|
|
|
|
|
|
|
return arch_setup_msi_irqs(dev, nvec, type);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
struct irq_domain *domain;
|
|
|
|
|
2017-02-08 17:17:43 +00:00
|
|
|
domain = dev_get_msi_domain(&dev->dev);
|
2015-12-04 16:28:14 +00:00
|
|
|
if (domain && irq_domain_is_hierarchy(domain))
|
2017-02-08 17:17:44 +00:00
|
|
|
msi_domain_free_irqs(domain, &dev->dev);
|
2014-11-15 14:24:07 +00:00
|
|
|
else
|
|
|
|
arch_teardown_msi_irqs(dev);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#define pci_msi_setup_msi_irqs arch_setup_msi_irqs
|
|
|
|
#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
|
|
|
|
#endif
|
2013-04-17 23:44:48 +00:00
|
|
|
|
2020-08-26 11:17:02 +00:00
|
|
|
#ifdef CONFIG_PCI_MSI_ARCH_FALLBACKS
|
2007-12-11 22:19:41 +00:00
|
|
|
/* Arch hooks */
|
2013-08-09 20:27:06 +00:00
|
|
|
int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
|
|
|
|
{
|
2021-03-30 15:11:38 +00:00
|
|
|
return -EINVAL;
|
2013-08-09 20:27:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void __weak arch_teardown_msi_irq(unsigned int irq)
|
2007-12-11 22:19:41 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2013-08-09 20:27:06 +00:00
|
|
|
int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
|
2007-12-11 22:19:41 +00:00
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
|
|
|
int ret;
|
|
|
|
|
2009-03-17 12:54:10 +00:00
|
|
|
/*
|
|
|
|
* If an architecture wants to support multiple MSI, it needs to
|
|
|
|
* override arch_setup_msi_irqs()
|
|
|
|
*/
|
|
|
|
if (type == PCI_CAP_ID_MSI && nvec > 1)
|
|
|
|
return 1;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
2007-12-11 22:19:41 +00:00
|
|
|
ret = arch_setup_msi_irq(dev, entry);
|
2009-02-11 11:27:02 +00:00
|
|
|
if (ret < 0)
|
2007-12-11 22:19:41 +00:00
|
|
|
return ret;
|
2009-02-11 11:27:02 +00:00
|
|
|
if (ret > 0)
|
|
|
|
return -ENOSPC;
|
2007-12-11 22:19:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2010-10-06 20:05:35 +00:00
|
|
|
|
2021-03-30 15:11:40 +00:00
|
|
|
void __weak arch_teardown_msi_irqs(struct pci_dev *dev)
|
2007-12-11 22:19:41 +00:00
|
|
|
{
|
2014-11-06 14:20:32 +00:00
|
|
|
int i;
|
2007-12-11 22:19:41 +00:00
|
|
|
struct msi_desc *entry;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev)
|
2014-11-06 14:20:32 +00:00
|
|
|
if (entry->irq)
|
|
|
|
for (i = 0; i < entry->nvec_used; i++)
|
|
|
|
arch_teardown_msi_irq(entry->irq + i);
|
2007-12-11 22:19:41 +00:00
|
|
|
}
|
2020-08-26 11:17:02 +00:00
|
|
|
#endif /* CONFIG_PCI_MSI_ARCH_FALLBACKS */
|
2011-12-16 22:38:18 +00:00
|
|
|
|
2013-12-04 05:09:16 +00:00
|
|
|
static void default_restore_msi_irq(struct pci_dev *dev, int irq)
|
2011-12-16 22:38:18 +00:00
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
|
|
|
|
|
|
|
entry = NULL;
|
|
|
|
if (dev->msix_enabled) {
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
2011-12-16 22:38:18 +00:00
|
|
|
if (irq == entry->irq)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else if (dev->msi_enabled) {
|
|
|
|
entry = irq_get_msi_desc(irq);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (entry)
|
2014-11-09 15:10:34 +00:00
|
|
|
__pci_write_msi_msg(entry, &entry->msg);
|
2011-12-16 22:38:18 +00:00
|
|
|
}
|
2013-08-09 20:27:06 +00:00
|
|
|
|
2013-12-04 05:09:16 +00:00
|
|
|
void __weak arch_restore_msi_irqs(struct pci_dev *dev)
|
2013-08-09 20:27:06 +00:00
|
|
|
{
|
2013-12-04 05:09:16 +00:00
|
|
|
return default_restore_msi_irqs(dev);
|
2013-08-09 20:27:06 +00:00
|
|
|
}
|
2011-12-16 22:38:18 +00:00
|
|
|
|
2009-01-22 00:19:19 +00:00
|
|
|
static inline __attribute_const__ u32 msi_mask(unsigned x)
|
|
|
|
{
|
2009-02-09 03:27:47 +00:00
|
|
|
/* Don't shift by >= width of type */
|
|
|
|
if (x >= 5)
|
|
|
|
return 0xffffffff;
|
|
|
|
return (1 << (1 << x)) - 1;
|
2009-01-22 00:19:19 +00:00
|
|
|
}
|
|
|
|
|
2008-07-25 21:42:58 +00:00
|
|
|
/*
|
|
|
|
* PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to
|
|
|
|
* mask all MSI interrupts by clearing the MSI enable bit does not work
|
|
|
|
* reliably as devices without an INTx disable bit will then generate a
|
|
|
|
* level IRQ which will never be cleared.
|
|
|
|
*/
|
2014-11-23 10:55:58 +00:00
|
|
|
u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-03-17 12:54:09 +00:00
|
|
|
u32 mask_bits = desc->masked;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-10-27 02:44:36 +00:00
|
|
|
if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit)
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
return 0;
|
2009-03-17 12:54:09 +00:00
|
|
|
|
|
|
|
mask_bits &= ~mask;
|
|
|
|
mask_bits |= flag;
|
2015-07-09 08:00:43 +00:00
|
|
|
pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos,
|
|
|
|
mask_bits);
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
|
|
|
|
return mask_bits;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
|
|
|
|
{
|
2014-11-23 10:55:58 +00:00
|
|
|
desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag);
|
2009-03-17 12:54:09 +00:00
|
|
|
}
|
|
|
|
|
2016-07-12 09:20:14 +00:00
|
|
|
static void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
|
|
|
|
{
|
2019-05-23 22:30:51 +00:00
|
|
|
if (desc->msi_attrib.is_virtual)
|
|
|
|
return NULL;
|
|
|
|
|
2016-07-12 09:20:14 +00:00
|
|
|
return desc->mask_base +
|
|
|
|
desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
|
|
|
|
}
|
|
|
|
|
2009-03-17 12:54:09 +00:00
|
|
|
/*
|
|
|
|
* This internal function does not flush PCI writes to the device.
|
|
|
|
* All users must ensure that they read from the device before either
|
|
|
|
* assuming that the device state is up to date, or returning out of this
|
|
|
|
* file. This saves a few milliseconds when initialising devices with lots
|
|
|
|
* of MSI-X interrupts.
|
|
|
|
*/
|
2014-11-23 10:55:58 +00:00
|
|
|
u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
|
2009-03-17 12:54:09 +00:00
|
|
|
{
|
|
|
|
u32 mask_bits = desc->masked;
|
2019-05-23 22:30:51 +00:00
|
|
|
void __iomem *desc_addr;
|
2014-10-27 02:44:36 +00:00
|
|
|
|
|
|
|
if (pci_msi_ignore_mask)
|
|
|
|
return 0;
|
PCI/MSI: Fix incorrect MSI-X masking on resume
When a driver enables MSI-X, msix_program_entries() reads the MSI-X Vector
Control register for each vector and saves it in desc->masked. Each
register is 32 bits and bit 0 is the actual Mask bit.
When we restored these registers during resume, we previously set the Mask
bit if *any* bit in desc->masked was set instead of when the Mask bit
itself was set:
pci_restore_state
pci_restore_msi_state
__pci_restore_msix_state
for_each_pci_msi_entry
msix_mask_irq(entry, entry->masked) <-- entire u32 word
__pci_msix_desc_mask_irq(desc, flag)
mask_bits = desc->masked & ~PCI_MSIX_ENTRY_CTRL_MASKBIT
if (flag) <-- testing entire u32, not just bit 0
mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT
writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL)
This means that after resume, MSI-X vectors were masked when they shouldn't
be, which leads to timeouts like this:
nvme nvme0: I/O 978 QID 3 timeout, completion polled
On resume, set the Mask bit only when the saved Mask bit from suspend was
set.
This should remove the need for 19ea025e1d28 ("nvme: Add quirk for Kingston
NVME SSD running FW E8FK11.T").
[bhelgaas: commit log, move fix to __pci_msix_desc_mask_irq()]
Link: https://bugzilla.kernel.org/show_bug.cgi?id=204887
Link: https://lore.kernel.org/r/20191008034238.2503-1-jian-hong@endlessm.com
Fixes: f2440d9acbe8 ("PCI MSI: Refactor interrupt masking code")
Signed-off-by: Jian-Hong Pan <jian-hong@endlessm.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Cc: stable@vger.kernel.org
2019-10-08 03:42:39 +00:00
|
|
|
|
2019-05-23 22:30:51 +00:00
|
|
|
desc_addr = pci_msix_desc_addr(desc);
|
|
|
|
if (!desc_addr)
|
|
|
|
return 0;
|
2014-10-27 02:44:36 +00:00
|
|
|
|
2010-11-11 07:46:55 +00:00
|
|
|
mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
|
PCI/MSI: Fix incorrect MSI-X masking on resume
When a driver enables MSI-X, msix_program_entries() reads the MSI-X Vector
Control register for each vector and saves it in desc->masked. Each
register is 32 bits and bit 0 is the actual Mask bit.
When we restored these registers during resume, we previously set the Mask
bit if *any* bit in desc->masked was set instead of when the Mask bit
itself was set:
pci_restore_state
pci_restore_msi_state
__pci_restore_msix_state
for_each_pci_msi_entry
msix_mask_irq(entry, entry->masked) <-- entire u32 word
__pci_msix_desc_mask_irq(desc, flag)
mask_bits = desc->masked & ~PCI_MSIX_ENTRY_CTRL_MASKBIT
if (flag) <-- testing entire u32, not just bit 0
mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT
writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL)
This means that after resume, MSI-X vectors were masked when they shouldn't
be, which leads to timeouts like this:
nvme nvme0: I/O 978 QID 3 timeout, completion polled
On resume, set the Mask bit only when the saved Mask bit from suspend was
set.
This should remove the need for 19ea025e1d28 ("nvme: Add quirk for Kingston
NVME SSD running FW E8FK11.T").
[bhelgaas: commit log, move fix to __pci_msix_desc_mask_irq()]
Link: https://bugzilla.kernel.org/show_bug.cgi?id=204887
Link: https://lore.kernel.org/r/20191008034238.2503-1-jian-hong@endlessm.com
Fixes: f2440d9acbe8 ("PCI MSI: Refactor interrupt masking code")
Signed-off-by: Jian-Hong Pan <jian-hong@endlessm.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Cc: stable@vger.kernel.org
2019-10-08 03:42:39 +00:00
|
|
|
if (flag & PCI_MSIX_ENTRY_CTRL_MASKBIT)
|
2010-11-11 07:46:55 +00:00
|
|
|
mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
|
2019-05-23 22:30:51 +00:00
|
|
|
|
|
|
|
writel(mask_bits, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL);
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
|
|
|
|
return mask_bits;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void msix_mask_irq(struct msi_desc *desc, u32 flag)
|
|
|
|
{
|
2014-11-23 10:55:58 +00:00
|
|
|
desc->masked = __pci_msix_desc_mask_irq(desc, flag);
|
2009-03-17 12:54:09 +00:00
|
|
|
}
|
2009-03-17 12:54:06 +00:00
|
|
|
|
2010-09-28 14:46:51 +00:00
|
|
|
static void msi_set_mask_bit(struct irq_data *data, u32 flag)
|
2009-03-17 12:54:09 +00:00
|
|
|
{
|
2015-06-01 08:05:41 +00:00
|
|
|
struct msi_desc *desc = irq_data_get_msi_desc(data);
|
2009-03-17 12:54:06 +00:00
|
|
|
|
2009-03-17 12:54:09 +00:00
|
|
|
if (desc->msi_attrib.is_msix) {
|
|
|
|
msix_mask_irq(desc, flag);
|
|
|
|
readl(desc->mask_base); /* Flush write to device */
|
|
|
|
} else {
|
2014-07-08 02:08:55 +00:00
|
|
|
unsigned offset = data->irq - desc->irq;
|
2009-03-17 12:54:10 +00:00
|
|
|
msi_mask_irq(desc, 1 << offset, flag << offset);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2009-03-17 12:54:09 +00:00
|
|
|
}
|
|
|
|
|
2014-11-23 10:55:58 +00:00
|
|
|
/**
|
2019-05-30 13:05:58 +00:00
|
|
|
* pci_msi_mask_irq - Generic IRQ chip callback to mask PCI/MSI interrupts
|
2014-11-23 10:55:58 +00:00
|
|
|
* @data: pointer to irqdata associated to that interrupt
|
|
|
|
*/
|
|
|
|
void pci_msi_mask_irq(struct irq_data *data)
|
2009-03-17 12:54:09 +00:00
|
|
|
{
|
2010-09-28 14:46:51 +00:00
|
|
|
msi_set_mask_bit(data, 1);
|
2009-03-17 12:54:09 +00:00
|
|
|
}
|
2015-12-10 17:52:59 +00:00
|
|
|
EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
|
2009-03-17 12:54:09 +00:00
|
|
|
|
2014-11-23 10:55:58 +00:00
|
|
|
/**
|
2019-05-30 13:05:58 +00:00
|
|
|
* pci_msi_unmask_irq - Generic IRQ chip callback to unmask PCI/MSI interrupts
|
2014-11-23 10:55:58 +00:00
|
|
|
* @data: pointer to irqdata associated to that interrupt
|
|
|
|
*/
|
|
|
|
void pci_msi_unmask_irq(struct irq_data *data)
|
2009-03-17 12:54:09 +00:00
|
|
|
{
|
2010-09-28 14:46:51 +00:00
|
|
|
msi_set_mask_bit(data, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2015-12-10 17:52:59 +00:00
|
|
|
EXPORT_SYMBOL_GPL(pci_msi_unmask_irq);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-12-04 05:09:16 +00:00
|
|
|
void default_restore_msi_irqs(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev)
|
2013-12-04 05:09:16 +00:00
|
|
|
default_restore_msi_irq(dev, entry->irq);
|
|
|
|
}
|
|
|
|
|
2014-11-09 15:10:33 +00:00
|
|
|
void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-07-09 08:00:43 +00:00
|
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(entry);
|
|
|
|
|
|
|
|
BUG_ON(dev->current_state != PCI_D0);
|
2010-07-23 13:56:28 +00:00
|
|
|
|
|
|
|
if (entry->msi_attrib.is_msix) {
|
2016-07-12 09:20:14 +00:00
|
|
|
void __iomem *base = pci_msix_desc_addr(entry);
|
2010-07-23 13:56:28 +00:00
|
|
|
|
2019-05-23 22:30:51 +00:00
|
|
|
if (!base) {
|
|
|
|
WARN_ON(1);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2010-07-23 13:56:28 +00:00
|
|
|
msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
|
|
|
|
msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
|
|
|
|
msg->data = readl(base + PCI_MSIX_ENTRY_DATA);
|
|
|
|
} else {
|
2013-04-17 23:34:36 +00:00
|
|
|
int pos = dev->msi_cap;
|
2010-07-23 13:56:28 +00:00
|
|
|
u16 data;
|
|
|
|
|
2013-04-17 23:39:57 +00:00
|
|
|
pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
|
|
|
|
&msg->address_lo);
|
2010-07-23 13:56:28 +00:00
|
|
|
if (entry->msi_attrib.is_64) {
|
2013-04-17 23:39:57 +00:00
|
|
|
pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
|
|
|
|
&msg->address_hi);
|
2013-04-17 23:41:13 +00:00
|
|
|
pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data);
|
2010-07-23 13:56:28 +00:00
|
|
|
} else {
|
|
|
|
msg->address_hi = 0;
|
2013-04-17 23:41:13 +00:00
|
|
|
pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data);
|
2010-07-23 13:56:28 +00:00
|
|
|
}
|
|
|
|
msg->data = data;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-11-09 15:10:34 +00:00
|
|
|
void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
|
2008-12-06 02:58:34 +00:00
|
|
|
{
|
2015-07-09 08:00:43 +00:00
|
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(entry);
|
|
|
|
|
2017-03-30 03:49:11 +00:00
|
|
|
if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) {
|
2010-06-17 19:16:36 +00:00
|
|
|
/* Don't touch the hardware now */
|
|
|
|
} else if (entry->msi_attrib.is_msix) {
|
2016-07-12 09:20:14 +00:00
|
|
|
void __iomem *base = pci_msix_desc_addr(entry);
|
2009-03-17 12:54:06 +00:00
|
|
|
|
2019-05-23 22:30:51 +00:00
|
|
|
if (!base)
|
|
|
|
goto skip;
|
|
|
|
|
2009-06-23 08:40:04 +00:00
|
|
|
writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
|
|
|
|
writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
|
|
|
|
writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
|
2009-03-17 12:54:06 +00:00
|
|
|
} else {
|
2013-04-17 23:34:36 +00:00
|
|
|
int pos = dev->msi_cap;
|
2009-03-17 12:54:10 +00:00
|
|
|
u16 msgctl;
|
|
|
|
|
2013-04-17 23:38:32 +00:00
|
|
|
pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
|
2009-03-17 12:54:10 +00:00
|
|
|
msgctl &= ~PCI_MSI_FLAGS_QSIZE;
|
|
|
|
msgctl |= entry->msi_attrib.multiple << 4;
|
2013-04-17 23:38:32 +00:00
|
|
|
pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
|
2006-10-04 09:16:33 +00:00
|
|
|
|
2013-04-17 23:39:57 +00:00
|
|
|
pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
|
|
|
|
msg->address_lo);
|
2006-10-04 09:16:33 +00:00
|
|
|
if (entry->msi_attrib.is_64) {
|
2013-04-17 23:39:57 +00:00
|
|
|
pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
|
|
|
|
msg->address_hi);
|
2013-04-17 23:41:13 +00:00
|
|
|
pci_write_config_word(dev, pos + PCI_MSI_DATA_64,
|
|
|
|
msg->data);
|
2006-10-04 09:16:33 +00:00
|
|
|
} else {
|
2013-04-17 23:41:13 +00:00
|
|
|
pci_write_config_word(dev, pos + PCI_MSI_DATA_32,
|
|
|
|
msg->data);
|
2006-10-04 09:16:33 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2019-05-23 22:30:51 +00:00
|
|
|
|
|
|
|
skip:
|
2007-03-08 20:04:57 +00:00
|
|
|
entry->msg = *msg;
|
2019-05-23 22:30:51 +00:00
|
|
|
|
|
|
|
if (entry->write_msi_msg)
|
|
|
|
entry->write_msi_msg(entry, entry->write_msi_msg_data);
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2006-10-04 09:16:33 +00:00
|
|
|
|
2014-11-09 15:10:34 +00:00
|
|
|
void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
|
2008-12-06 02:58:34 +00:00
|
|
|
{
|
2011-03-28 15:49:12 +00:00
|
|
|
struct msi_desc *entry = irq_get_msi_desc(irq);
|
2008-12-06 02:58:34 +00:00
|
|
|
|
2014-11-09 15:10:34 +00:00
|
|
|
__pci_write_msi_msg(entry, msg);
|
2008-12-06 02:58:34 +00:00
|
|
|
}
|
2014-11-09 15:10:34 +00:00
|
|
|
EXPORT_SYMBOL_GPL(pci_write_msi_msg);
|
2008-12-06 02:58:34 +00:00
|
|
|
|
2009-08-06 02:32:51 +00:00
|
|
|
static void free_msi_irqs(struct pci_dev *dev)
|
|
|
|
{
|
2015-07-09 08:00:41 +00:00
|
|
|
struct list_head *msi_list = dev_to_msi_list(&dev->dev);
|
2009-08-06 02:32:51 +00:00
|
|
|
struct msi_desc *entry, *tmp;
|
2013-12-19 20:30:17 +00:00
|
|
|
struct attribute **msi_attrs;
|
|
|
|
struct device_attribute *dev_attr;
|
2014-11-06 14:20:32 +00:00
|
|
|
int i, count = 0;
|
2009-08-06 02:32:51 +00:00
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev)
|
2014-11-06 14:20:32 +00:00
|
|
|
if (entry->irq)
|
|
|
|
for (i = 0; i < entry->nvec_used; i++)
|
|
|
|
BUG_ON(irq_has_action(entry->irq + i));
|
2009-08-06 02:32:51 +00:00
|
|
|
|
2014-11-15 14:24:07 +00:00
|
|
|
pci_msi_teardown_msi_irqs(dev);
|
2009-08-06 02:32:51 +00:00
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
list_for_each_entry_safe(entry, tmp, msi_list, list) {
|
2009-08-06 02:32:51 +00:00
|
|
|
if (entry->msi_attrib.is_msix) {
|
2015-07-09 08:00:41 +00:00
|
|
|
if (list_is_last(&entry->list, msi_list))
|
2009-08-06 02:32:51 +00:00
|
|
|
iounmap(entry->mask_base);
|
|
|
|
}
|
2012-01-03 15:29:54 +00:00
|
|
|
|
2009-08-06 02:32:51 +00:00
|
|
|
list_del(&entry->list);
|
2017-02-15 16:53:08 +00:00
|
|
|
free_msi_entry(entry);
|
2009-08-06 02:32:51 +00:00
|
|
|
}
|
2013-12-19 20:30:17 +00:00
|
|
|
|
|
|
|
if (dev->msi_irq_groups) {
|
|
|
|
sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
|
|
|
|
msi_attrs = dev->msi_irq_groups[0]->attrs;
|
2014-06-04 22:49:50 +00:00
|
|
|
while (msi_attrs[count]) {
|
2013-12-19 20:30:17 +00:00
|
|
|
dev_attr = container_of(msi_attrs[count],
|
|
|
|
struct device_attribute, attr);
|
|
|
|
kfree(dev_attr->attr.name);
|
|
|
|
kfree(dev_attr);
|
|
|
|
++count;
|
|
|
|
}
|
|
|
|
kfree(msi_attrs);
|
|
|
|
kfree(dev->msi_irq_groups[0]);
|
|
|
|
kfree(dev->msi_irq_groups);
|
|
|
|
dev->msi_irq_groups = NULL;
|
|
|
|
}
|
2009-08-06 02:32:51 +00:00
|
|
|
}
|
2007-01-18 04:50:05 +00:00
|
|
|
|
2007-10-25 08:16:30 +00:00
|
|
|
static void pci_intx_for_msi(struct pci_dev *dev, int enable)
|
|
|
|
{
|
|
|
|
if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))
|
|
|
|
pci_intx(dev, enable);
|
|
|
|
}
|
|
|
|
|
2020-12-03 18:51:09 +00:00
|
|
|
static void pci_msi_set_enable(struct pci_dev *dev, int enable)
|
|
|
|
{
|
|
|
|
u16 control;
|
|
|
|
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
|
|
|
control &= ~PCI_MSI_FLAGS_ENABLE;
|
|
|
|
if (enable)
|
|
|
|
control |= PCI_MSI_FLAGS_ENABLE;
|
|
|
|
pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
|
|
|
|
}
|
|
|
|
|
2007-01-25 08:34:08 +00:00
|
|
|
static void __pci_restore_msi_state(struct pci_dev *dev)
|
2006-02-08 09:11:38 +00:00
|
|
|
{
|
|
|
|
u16 control;
|
2007-03-08 20:04:57 +00:00
|
|
|
struct msi_desc *entry;
|
2006-02-08 09:11:38 +00:00
|
|
|
|
2007-03-05 08:30:10 +00:00
|
|
|
if (!dev->msi_enabled)
|
|
|
|
return;
|
|
|
|
|
2011-03-28 15:49:12 +00:00
|
|
|
entry = irq_get_msi_desc(dev->irq);
|
2006-02-08 09:11:38 +00:00
|
|
|
|
2007-10-25 08:16:30 +00:00
|
|
|
pci_intx_for_msi(dev, 0);
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msi_set_enable(dev, 0);
|
2013-12-04 05:09:16 +00:00
|
|
|
arch_restore_msi_irqs(dev);
|
2007-03-08 20:04:57 +00:00
|
|
|
|
2013-04-17 23:34:36 +00:00
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
2014-06-19 08:30:30 +00:00
|
|
|
msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap),
|
|
|
|
entry->masked);
|
2008-08-07 15:52:37 +00:00
|
|
|
control &= ~PCI_MSI_FLAGS_QSIZE;
|
2009-03-17 12:54:10 +00:00
|
|
|
control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
|
2013-04-17 23:34:36 +00:00
|
|
|
pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
|
2007-01-25 08:34:08 +00:00
|
|
|
}
|
|
|
|
|
2020-12-03 18:51:09 +00:00
|
|
|
static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set)
|
|
|
|
{
|
|
|
|
u16 ctrl;
|
|
|
|
|
|
|
|
pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
|
|
|
|
ctrl &= ~clear;
|
|
|
|
ctrl |= set;
|
|
|
|
pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl);
|
|
|
|
}
|
|
|
|
|
2007-01-25 08:34:08 +00:00
|
|
|
static void __pci_restore_msix_state(struct pci_dev *dev)
|
2006-02-08 09:11:38 +00:00
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
|
|
|
|
2007-01-28 19:42:52 +00:00
|
|
|
if (!dev->msix_enabled)
|
|
|
|
return;
|
2015-07-09 08:00:41 +00:00
|
|
|
BUG_ON(list_empty(dev_to_msi_list(&dev->dev)));
|
2007-01-28 19:42:52 +00:00
|
|
|
|
2006-02-08 09:11:38 +00:00
|
|
|
/* route the table */
|
2007-10-25 08:16:30 +00:00
|
|
|
pci_intx_for_msi(dev, 0);
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msix_clear_and_set_ctrl(dev, 0,
|
2014-06-19 08:29:53 +00:00
|
|
|
PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
|
2006-02-08 09:11:38 +00:00
|
|
|
|
2013-12-04 05:09:16 +00:00
|
|
|
arch_restore_msi_irqs(dev);
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev)
|
2009-03-17 12:54:09 +00:00
|
|
|
msix_mask_irq(entry, entry->masked);
|
2006-02-08 09:11:38 +00:00
|
|
|
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
|
2006-02-08 09:11:38 +00:00
|
|
|
}
|
2007-01-25 08:34:08 +00:00
|
|
|
|
|
|
|
void pci_restore_msi_state(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
__pci_restore_msi_state(dev);
|
|
|
|
__pci_restore_msix_state(dev);
|
|
|
|
}
|
2007-11-07 21:43:59 +00:00
|
|
|
EXPORT_SYMBOL_GPL(pci_restore_msi_state);
|
2006-02-08 09:11:38 +00:00
|
|
|
|
2013-12-19 20:30:17 +00:00
|
|
|
static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
|
2011-10-06 18:08:18 +00:00
|
|
|
char *buf)
|
|
|
|
{
|
2013-12-19 20:30:17 +00:00
|
|
|
struct msi_desc *entry;
|
|
|
|
unsigned long irq;
|
|
|
|
int retval;
|
2011-10-06 18:08:18 +00:00
|
|
|
|
2013-12-19 20:30:17 +00:00
|
|
|
retval = kstrtoul(attr->attr.name, 10, &irq);
|
|
|
|
if (retval)
|
|
|
|
return retval;
|
2011-10-06 18:08:18 +00:00
|
|
|
|
2014-07-08 02:09:19 +00:00
|
|
|
entry = irq_get_msi_desc(irq);
|
2021-06-03 00:01:07 +00:00
|
|
|
if (!entry)
|
|
|
|
return -ENODEV;
|
2014-07-08 02:09:19 +00:00
|
|
|
|
2021-06-03 00:01:07 +00:00
|
|
|
return sysfs_emit(buf, "%s\n",
|
|
|
|
entry->msi_attrib.is_msix ? "msix" : "msi");
|
2011-10-06 18:08:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int populate_msi_sysfs(struct pci_dev *pdev)
|
|
|
|
{
|
2013-12-19 20:30:17 +00:00
|
|
|
struct attribute **msi_attrs;
|
|
|
|
struct attribute *msi_attr;
|
|
|
|
struct device_attribute *msi_dev_attr;
|
|
|
|
struct attribute_group *msi_irq_group;
|
|
|
|
const struct attribute_group **msi_irq_groups;
|
2011-10-06 18:08:18 +00:00
|
|
|
struct msi_desc *entry;
|
2013-12-19 20:30:17 +00:00
|
|
|
int ret = -ENOMEM;
|
|
|
|
int num_msi = 0;
|
2011-10-06 18:08:18 +00:00
|
|
|
int count = 0;
|
2015-09-23 23:31:16 +00:00
|
|
|
int i;
|
2011-10-06 18:08:18 +00:00
|
|
|
|
2013-12-19 20:30:17 +00:00
|
|
|
/* Determine how many msi entries we have */
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, pdev)
|
2015-09-23 23:31:16 +00:00
|
|
|
num_msi += entry->nvec_used;
|
2013-12-19 20:30:17 +00:00
|
|
|
if (!num_msi)
|
|
|
|
return 0;
|
2011-10-06 18:08:18 +00:00
|
|
|
|
2013-12-19 20:30:17 +00:00
|
|
|
/* Dynamically create the MSI attributes for the PCI device */
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
|
2013-12-19 20:30:17 +00:00
|
|
|
if (!msi_attrs)
|
|
|
|
return -ENOMEM;
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, pdev) {
|
2015-09-23 23:31:16 +00:00
|
|
|
for (i = 0; i < entry->nvec_used; i++) {
|
|
|
|
msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
|
|
|
|
if (!msi_dev_attr)
|
|
|
|
goto error_attrs;
|
|
|
|
msi_attrs[count] = &msi_dev_attr->attr;
|
|
|
|
|
|
|
|
sysfs_attr_init(&msi_dev_attr->attr);
|
|
|
|
msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
|
|
|
|
entry->irq + i);
|
|
|
|
if (!msi_dev_attr->attr.name)
|
|
|
|
goto error_attrs;
|
|
|
|
msi_dev_attr->attr.mode = S_IRUGO;
|
|
|
|
msi_dev_attr->show = msi_mode_show;
|
|
|
|
++count;
|
|
|
|
}
|
2011-10-06 18:08:18 +00:00
|
|
|
}
|
|
|
|
|
2013-12-19 20:30:17 +00:00
|
|
|
msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
|
|
|
|
if (!msi_irq_group)
|
|
|
|
goto error_attrs;
|
|
|
|
msi_irq_group->name = "msi_irqs";
|
|
|
|
msi_irq_group->attrs = msi_attrs;
|
|
|
|
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
|
2013-12-19 20:30:17 +00:00
|
|
|
if (!msi_irq_groups)
|
|
|
|
goto error_irq_group;
|
|
|
|
msi_irq_groups[0] = msi_irq_group;
|
|
|
|
|
|
|
|
ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
|
|
|
|
if (ret)
|
|
|
|
goto error_irq_groups;
|
|
|
|
pdev->msi_irq_groups = msi_irq_groups;
|
|
|
|
|
2011-10-06 18:08:18 +00:00
|
|
|
return 0;
|
|
|
|
|
2013-12-19 20:30:17 +00:00
|
|
|
error_irq_groups:
|
|
|
|
kfree(msi_irq_groups);
|
|
|
|
error_irq_group:
|
|
|
|
kfree(msi_irq_group);
|
|
|
|
error_attrs:
|
|
|
|
count = 0;
|
|
|
|
msi_attr = msi_attrs[count];
|
|
|
|
while (msi_attr) {
|
|
|
|
msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
|
|
|
|
kfree(msi_attr->name);
|
|
|
|
kfree(msi_dev_attr);
|
|
|
|
++count;
|
|
|
|
msi_attr = msi_attrs[count];
|
2011-10-06 18:08:18 +00:00
|
|
|
}
|
2014-02-13 17:47:35 +00:00
|
|
|
kfree(msi_attrs);
|
2011-10-06 18:08:18 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-09-14 14:18:49 +00:00
|
|
|
static struct msi_desc *
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
|
2014-07-08 02:07:23 +00:00
|
|
|
{
|
2018-12-04 15:51:20 +00:00
|
|
|
struct irq_affinity_desc *masks = NULL;
|
2014-07-08 02:07:23 +00:00
|
|
|
struct msi_desc *entry;
|
2016-09-14 14:18:49 +00:00
|
|
|
u16 control;
|
|
|
|
|
2017-08-25 23:58:42 +00:00
|
|
|
if (affd)
|
2016-11-09 01:15:04 +00:00
|
|
|
masks = irq_create_affinity_masks(nvec, affd);
|
2017-08-25 23:58:42 +00:00
|
|
|
|
2014-07-08 02:07:23 +00:00
|
|
|
/* MSI Entry Initialization */
|
2016-09-14 14:18:49 +00:00
|
|
|
entry = alloc_msi_entry(&dev->dev, nvec, masks);
|
2014-07-08 02:07:23 +00:00
|
|
|
if (!entry)
|
2016-09-14 14:18:49 +00:00
|
|
|
goto out;
|
2014-07-08 02:07:23 +00:00
|
|
|
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
|
|
|
|
|
|
|
entry->msi_attrib.is_msix = 0;
|
|
|
|
entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT);
|
2019-05-23 22:30:51 +00:00
|
|
|
entry->msi_attrib.is_virtual = 0;
|
2014-07-08 02:07:23 +00:00
|
|
|
entry->msi_attrib.entry_nr = 0;
|
|
|
|
entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT);
|
|
|
|
entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
|
|
|
|
entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1;
|
2014-11-06 14:20:32 +00:00
|
|
|
entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
|
2014-07-08 02:07:23 +00:00
|
|
|
|
|
|
|
if (control & PCI_MSI_FLAGS_64BIT)
|
|
|
|
entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
|
|
|
|
else
|
|
|
|
entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32;
|
|
|
|
|
|
|
|
/* Save the initial mask status */
|
|
|
|
if (entry->msi_attrib.maskbit)
|
|
|
|
pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
|
|
|
|
|
2016-09-14 14:18:49 +00:00
|
|
|
out:
|
|
|
|
kfree(masks);
|
2014-07-08 02:07:23 +00:00
|
|
|
return entry;
|
|
|
|
}
|
|
|
|
|
2014-10-03 05:13:24 +00:00
|
|
|
static int msi_verify_entries(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
2020-12-03 18:51:10 +00:00
|
|
|
if (entry->msg.address_hi && dev->no_64bit_msi) {
|
|
|
|
pci_err(dev, "arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n",
|
|
|
|
entry->msg.address_hi, entry->msg.address_lo);
|
|
|
|
return -EIO;
|
|
|
|
}
|
2014-10-03 05:13:24 +00:00
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
|
|
|
* msi_capability_init - configure device's MSI capability structure
|
|
|
|
* @dev: pointer to the pci_dev data structure of MSI device function
|
2009-03-17 12:54:10 +00:00
|
|
|
* @nvec: number of interrupts to allocate
|
2019-05-30 13:05:58 +00:00
|
|
|
* @affd: description of automatic IRQ affinity assignments (may be %NULL)
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2009-03-17 12:54:10 +00:00
|
|
|
* Setup the MSI capability structure of the device with the requested
|
|
|
|
* number of interrupts. A return value of zero indicates the successful
|
2019-05-30 13:05:58 +00:00
|
|
|
* setup of an entry with the new MSI IRQ. A negative return value indicates
|
2009-03-17 12:54:10 +00:00
|
|
|
* an error, and a positive return value indicates the number of interrupts
|
|
|
|
* which could have been allocated.
|
|
|
|
*/
|
2016-11-09 01:15:04 +00:00
|
|
|
static int msi_capability_init(struct pci_dev *dev, int nvec,
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
struct irq_affinity *affd)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
2013-04-04 16:54:32 +00:00
|
|
|
int ret;
|
2009-03-17 12:54:09 +00:00
|
|
|
unsigned mask;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msi_set_enable(dev, 0); /* Disable MSI during set up */
|
2009-06-16 12:31:45 +00:00
|
|
|
|
2016-11-09 01:15:04 +00:00
|
|
|
entry = msi_setup_entry(dev, nvec, affd);
|
2007-01-28 19:56:37 +00:00
|
|
|
if (!entry)
|
|
|
|
return -ENOMEM;
|
2006-10-04 09:16:41 +00:00
|
|
|
|
2019-05-30 13:05:58 +00:00
|
|
|
/* All MSIs are unmasked by default; mask them all */
|
2014-06-19 08:30:30 +00:00
|
|
|
mask = msi_mask(entry->msi_attrib.multi_cap);
|
2009-03-17 12:54:09 +00:00
|
|
|
msi_mask_irq(entry, mask, mask);
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
|
MSI: Give archs the option to allocate all MSI/Xs at once.
This patch introduces an optional function, arch_setup_msi_irqs(),
(note the plural) which gives an arch the opportunity to do per-device
setup for MSI/X and then allocate all the requested MSI/Xs at once.
If that's not required by the arch, the default version simply calls
arch_setup_msi_irq() for each MSI irq required.
arch_setup_msi_irqs() is passed a pdev, attached to the pdev is a list
of msi_descs with irq == 0, it is up to the arch to connect these up to
an irq (via set_irq_msi()) or return an error. For convenience the number
of vectors and the type are passed also.
All msi_descs with irq != 0 are considered allocated, and the arch
teardown routine will be called on them when necessary.
The existing semantics of pci_enable_msix() are that if the requested
number of irqs can not be allocated, the maximum number that _could_ be
allocated is returned. To support that, we define that in case of an
error from arch_setup_msi_irqs(), the number of msi_descs with irq != 0
are considered allocated, and are counted toward the "max that could be
allocated".
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
2007-04-18 09:39:21 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Configure MSI capability structure */
|
2014-11-15 14:24:07 +00:00
|
|
|
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
|
2007-04-18 09:39:21 +00:00
|
|
|
if (ret) {
|
2009-06-23 08:39:27 +00:00
|
|
|
msi_mask_irq(entry, mask, ~mask);
|
2009-08-06 02:32:51 +00:00
|
|
|
free_msi_irqs(dev);
|
2007-04-18 09:39:21 +00:00
|
|
|
return ret;
|
2006-04-11 02:17:48 +00:00
|
|
|
}
|
2007-01-28 19:56:37 +00:00
|
|
|
|
2014-10-03 05:13:24 +00:00
|
|
|
ret = msi_verify_entries(dev);
|
|
|
|
if (ret) {
|
|
|
|
msi_mask_irq(entry, mask, ~mask);
|
|
|
|
free_msi_irqs(dev);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2011-10-06 18:08:18 +00:00
|
|
|
ret = populate_msi_sysfs(dev);
|
|
|
|
if (ret) {
|
|
|
|
msi_mask_irq(entry, mask, ~mask);
|
|
|
|
free_msi_irqs(dev);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2019-05-30 13:05:58 +00:00
|
|
|
/* Set MSI enabled bits */
|
2007-10-25 08:16:30 +00:00
|
|
|
pci_intx_for_msi(dev, 0);
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msi_set_enable(dev, 1);
|
2007-03-05 08:30:10 +00:00
|
|
|
dev->msi_enabled = 1;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-07-30 19:00:08 +00:00
|
|
|
pcibios_free_irq(dev);
|
2007-04-18 09:39:21 +00:00
|
|
|
dev->irq = entry->irq;
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-04-04 16:54:33 +00:00
|
|
|
static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)
|
2009-08-06 02:34:34 +00:00
|
|
|
{
|
2010-06-17 01:42:44 +00:00
|
|
|
resource_size_t phys_addr;
|
2009-08-06 02:34:34 +00:00
|
|
|
u32 table_offset;
|
PCI: Fail MSI-X mappings if there's no space assigned to MSI-X BAR
Unlike MSI, which is configured via registers in the MSI capability in
Configuration Space, MSI-X is configured via tables in Memory Space.
These MSI-X tables are mapped by a device BAR, and if no Memory Space
has been assigned to the BAR, MSI-X cannot be used.
Fail MSI-X setup if no space has been assigned for the BAR.
Previously, we ioremapped the MSI-X table even if the resource hadn't been
assigned. In this case, the resource address is undefined (and is often
zero), which may lead to warnings or oopses in this path:
pci_enable_msix
msix_capability_init
msix_map_region
ioremap_nocache
The PCI core sets resource flags to zero when it can't assign space for the
resource (see reset_resource()). There are also some cases where it sets
the IORESOURCE_UNSET flag, e.g., pci_reassigndev_resource_alignment(),
pci_assign_resource(), etc. So we must check for both cases.
[bhelgaas: changelog]
Reported-by: Zhang Jukuo <zhangjukuo@huawei.com>
Tested-by: Zhang Jukuo <zhangjukuo@huawei.com>
Signed-off-by: Yijing Wang <wangyijing@huawei.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
2015-01-28 01:52:17 +00:00
|
|
|
unsigned long flags;
|
2009-08-06 02:34:34 +00:00
|
|
|
u8 bir;
|
|
|
|
|
2013-04-17 23:43:40 +00:00
|
|
|
pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE,
|
|
|
|
&table_offset);
|
2013-04-18 00:10:07 +00:00
|
|
|
bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR);
|
PCI: Fail MSI-X mappings if there's no space assigned to MSI-X BAR
Unlike MSI, which is configured via registers in the MSI capability in
Configuration Space, MSI-X is configured via tables in Memory Space.
These MSI-X tables are mapped by a device BAR, and if no Memory Space
has been assigned to the BAR, MSI-X cannot be used.
Fail MSI-X setup if no space has been assigned for the BAR.
Previously, we ioremapped the MSI-X table even if the resource hadn't been
assigned. In this case, the resource address is undefined (and is often
zero), which may lead to warnings or oopses in this path:
pci_enable_msix
msix_capability_init
msix_map_region
ioremap_nocache
The PCI core sets resource flags to zero when it can't assign space for the
resource (see reset_resource()). There are also some cases where it sets
the IORESOURCE_UNSET flag, e.g., pci_reassigndev_resource_alignment(),
pci_assign_resource(), etc. So we must check for both cases.
[bhelgaas: changelog]
Reported-by: Zhang Jukuo <zhangjukuo@huawei.com>
Tested-by: Zhang Jukuo <zhangjukuo@huawei.com>
Signed-off-by: Yijing Wang <wangyijing@huawei.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
2015-01-28 01:52:17 +00:00
|
|
|
flags = pci_resource_flags(dev, bir);
|
|
|
|
if (!flags || (flags & IORESOURCE_UNSET))
|
|
|
|
return NULL;
|
|
|
|
|
2013-04-18 00:10:07 +00:00
|
|
|
table_offset &= PCI_MSIX_TABLE_OFFSET;
|
2009-08-06 02:34:34 +00:00
|
|
|
phys_addr = pci_resource_start(dev, bir) + table_offset;
|
|
|
|
|
2020-01-06 08:43:50 +00:00
|
|
|
return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
|
2009-08-06 02:34:34 +00:00
|
|
|
}
|
|
|
|
|
2013-04-04 16:54:33 +00:00
|
|
|
static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
|
2016-09-14 14:18:49 +00:00
|
|
|
struct msix_entry *entries, int nvec,
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
struct irq_affinity *affd)
|
2009-08-06 02:35:48 +00:00
|
|
|
{
|
2018-12-04 15:51:20 +00:00
|
|
|
struct irq_affinity_desc *curmsk, *masks = NULL;
|
2009-08-06 02:35:48 +00:00
|
|
|
struct msi_desc *entry;
|
2016-09-14 14:18:49 +00:00
|
|
|
int ret, i;
|
2019-05-23 22:30:51 +00:00
|
|
|
int vec_count = pci_msix_vec_count(dev);
|
2016-07-12 09:20:18 +00:00
|
|
|
|
2017-08-25 23:58:42 +00:00
|
|
|
if (affd)
|
2016-11-09 01:15:04 +00:00
|
|
|
masks = irq_create_affinity_masks(nvec, affd);
|
2016-07-12 09:20:18 +00:00
|
|
|
|
2016-09-14 14:18:49 +00:00
|
|
|
for (i = 0, curmsk = masks; i < nvec; i++) {
|
|
|
|
entry = alloc_msi_entry(&dev->dev, 1, curmsk);
|
2009-08-06 02:35:48 +00:00
|
|
|
if (!entry) {
|
|
|
|
if (!i)
|
|
|
|
iounmap(base);
|
|
|
|
else
|
|
|
|
free_msi_irqs(dev);
|
|
|
|
/* No enough memory. Don't try again */
|
2016-09-14 14:18:49 +00:00
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out;
|
2009-08-06 02:35:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
entry->msi_attrib.is_msix = 1;
|
|
|
|
entry->msi_attrib.is_64 = 1;
|
2016-07-12 09:20:16 +00:00
|
|
|
if (entries)
|
|
|
|
entry->msi_attrib.entry_nr = entries[i].entry;
|
|
|
|
else
|
|
|
|
entry->msi_attrib.entry_nr = i;
|
2019-05-23 22:30:51 +00:00
|
|
|
|
|
|
|
entry->msi_attrib.is_virtual =
|
|
|
|
entry->msi_attrib.entry_nr >= vec_count;
|
|
|
|
|
2009-08-06 02:35:48 +00:00
|
|
|
entry->msi_attrib.default_irq = dev->irq;
|
|
|
|
entry->mask_base = base;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
|
2016-09-14 14:18:49 +00:00
|
|
|
if (masks)
|
|
|
|
curmsk++;
|
2009-08-06 02:35:48 +00:00
|
|
|
}
|
2016-09-14 14:18:49 +00:00
|
|
|
ret = 0;
|
|
|
|
out:
|
|
|
|
kfree(masks);
|
2017-01-27 15:14:53 +00:00
|
|
|
return ret;
|
2009-08-06 02:35:48 +00:00
|
|
|
}
|
|
|
|
|
2009-08-06 02:35:10 +00:00
|
|
|
static void msix_program_entries(struct pci_dev *dev,
|
2013-04-04 16:54:33 +00:00
|
|
|
struct msix_entry *entries)
|
2009-08-06 02:35:10 +00:00
|
|
|
{
|
|
|
|
struct msi_desc *entry;
|
|
|
|
int i = 0;
|
2019-05-23 22:30:51 +00:00
|
|
|
void __iomem *desc_addr;
|
2009-08-06 02:35:10 +00:00
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
2016-07-12 09:20:16 +00:00
|
|
|
if (entries)
|
|
|
|
entries[i++].vector = entry->irq;
|
2019-05-23 22:30:51 +00:00
|
|
|
|
|
|
|
desc_addr = pci_msix_desc_addr(entry);
|
|
|
|
if (desc_addr)
|
|
|
|
entry->masked = readl(desc_addr +
|
|
|
|
PCI_MSIX_ENTRY_VECTOR_CTRL);
|
|
|
|
else
|
|
|
|
entry->masked = 0;
|
|
|
|
|
2009-08-06 02:35:10 +00:00
|
|
|
msix_mask_irq(entry, 1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
|
|
|
* msix_capability_init - configure device's MSI-X capability
|
|
|
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
2005-10-23 18:57:38 +00:00
|
|
|
* @entries: pointer to an array of struct msix_entry entries
|
|
|
|
* @nvec: number of @entries
|
2019-05-30 13:05:58 +00:00
|
|
|
* @affd: Optional pointer to enable automatic affinity assignment
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2005-05-04 00:38:30 +00:00
|
|
|
* Setup the MSI-X capability structure of device function with a
|
2019-05-30 13:05:58 +00:00
|
|
|
* single MSI-X IRQ. A return of zero indicates the successful setup of
|
|
|
|
* requested MSI-X entries with allocated IRQs or non-zero for otherwise.
|
2005-04-16 22:20:36 +00:00
|
|
|
**/
|
2016-09-14 14:18:49 +00:00
|
|
|
static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
int nvec, struct irq_affinity *affd)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-04-04 16:54:33 +00:00
|
|
|
int ret;
|
2009-08-06 02:34:34 +00:00
|
|
|
u16 control;
|
2005-04-16 22:20:36 +00:00
|
|
|
void __iomem *base;
|
|
|
|
|
2009-06-19 02:15:59 +00:00
|
|
|
/* Ensure MSI-X is disabled while it is set up */
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
|
2009-06-19 02:15:59 +00:00
|
|
|
|
2014-06-19 08:29:53 +00:00
|
|
|
pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Request & Map MSI-X table region */
|
2013-04-17 23:44:48 +00:00
|
|
|
base = msix_map_region(dev, msix_table_size(control));
|
2009-08-06 02:34:34 +00:00
|
|
|
if (!base)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2016-11-09 01:15:04 +00:00
|
|
|
ret = msix_setup_entries(dev, base, entries, nvec, affd);
|
2009-08-06 02:35:48 +00:00
|
|
|
if (ret)
|
|
|
|
return ret;
|
MSI: Give archs the option to allocate all MSI/Xs at once.
This patch introduces an optional function, arch_setup_msi_irqs(),
(note the plural) which gives an arch the opportunity to do per-device
setup for MSI/X and then allocate all the requested MSI/Xs at once.
If that's not required by the arch, the default version simply calls
arch_setup_msi_irq() for each MSI irq required.
arch_setup_msi_irqs() is passed a pdev, attached to the pdev is a list
of msi_descs with irq == 0, it is up to the arch to connect these up to
an irq (via set_irq_msi()) or return an error. For convenience the number
of vectors and the type are passed also.
All msi_descs with irq != 0 are considered allocated, and the arch
teardown routine will be called on them when necessary.
The existing semantics of pci_enable_msix() are that if the requested
number of irqs can not be allocated, the maximum number that _could_ be
allocated is returned. To support that, we define that in case of an
error from arch_setup_msi_irqs(), the number of msi_descs with irq != 0
are considered allocated, and are counted toward the "max that could be
allocated".
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
2007-04-18 09:39:21 +00:00
|
|
|
|
2014-11-15 14:24:07 +00:00
|
|
|
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
|
2009-08-06 02:33:39 +00:00
|
|
|
if (ret)
|
2013-12-16 08:34:56 +00:00
|
|
|
goto out_avail;
|
MSI: Give archs the option to allocate all MSI/Xs at once.
This patch introduces an optional function, arch_setup_msi_irqs(),
(note the plural) which gives an arch the opportunity to do per-device
setup for MSI/X and then allocate all the requested MSI/Xs at once.
If that's not required by the arch, the default version simply calls
arch_setup_msi_irq() for each MSI irq required.
arch_setup_msi_irqs() is passed a pdev, attached to the pdev is a list
of msi_descs with irq == 0, it is up to the arch to connect these up to
an irq (via set_irq_msi()) or return an error. For convenience the number
of vectors and the type are passed also.
All msi_descs with irq != 0 are considered allocated, and the arch
teardown routine will be called on them when necessary.
The existing semantics of pci_enable_msix() are that if the requested
number of irqs can not be allocated, the maximum number that _could_ be
allocated is returned. To support that, we define that in case of an
error from arch_setup_msi_irqs(), the number of msi_descs with irq != 0
are considered allocated, and are counted toward the "max that could be
allocated".
Signed-off-by: Michael Ellerman <michael@ellerman.id.au>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>
2007-04-18 09:39:21 +00:00
|
|
|
|
2014-10-03 05:13:24 +00:00
|
|
|
/* Check if all MSI entries honor device restrictions */
|
|
|
|
ret = msi_verify_entries(dev);
|
|
|
|
if (ret)
|
|
|
|
goto out_free;
|
|
|
|
|
2009-06-19 02:15:59 +00:00
|
|
|
/*
|
|
|
|
* Some devices require MSI-X to be enabled before we can touch the
|
|
|
|
* MSI-X registers. We need to mask all the vectors to prevent
|
|
|
|
* interrupts coming in before they're fully set up.
|
|
|
|
*/
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msix_clear_and_set_ctrl(dev, 0,
|
2014-06-19 08:29:53 +00:00
|
|
|
PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE);
|
2009-06-19 02:15:59 +00:00
|
|
|
|
2009-08-06 02:35:10 +00:00
|
|
|
msix_program_entries(dev, entries);
|
2009-06-19 02:15:59 +00:00
|
|
|
|
2011-10-06 18:08:18 +00:00
|
|
|
ret = populate_msi_sysfs(dev);
|
2013-12-16 08:34:56 +00:00
|
|
|
if (ret)
|
|
|
|
goto out_free;
|
2011-10-06 18:08:18 +00:00
|
|
|
|
2009-06-19 02:15:59 +00:00
|
|
|
/* Set MSI-X enabled bits and unmask the function */
|
2007-10-25 08:16:30 +00:00
|
|
|
pci_intx_for_msi(dev, 0);
|
2007-03-05 08:30:10 +00:00
|
|
|
dev->msix_enabled = 1;
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
|
2009-05-08 13:13:33 +00:00
|
|
|
|
2015-07-30 19:00:08 +00:00
|
|
|
pcibios_free_irq(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
2009-08-06 02:33:39 +00:00
|
|
|
|
2013-12-16 08:34:56 +00:00
|
|
|
out_avail:
|
2009-08-06 02:33:39 +00:00
|
|
|
if (ret < 0) {
|
|
|
|
/*
|
2019-05-30 13:05:58 +00:00
|
|
|
* If we had some success, report the number of IRQs
|
2009-08-06 02:33:39 +00:00
|
|
|
* we succeeded in setting up.
|
|
|
|
*/
|
2009-08-06 02:35:48 +00:00
|
|
|
struct msi_desc *entry;
|
2009-08-06 02:33:39 +00:00
|
|
|
int avail = 0;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
2009-08-06 02:33:39 +00:00
|
|
|
if (entry->irq != 0)
|
|
|
|
avail++;
|
|
|
|
}
|
|
|
|
if (avail != 0)
|
|
|
|
ret = avail;
|
|
|
|
}
|
|
|
|
|
2013-12-16 08:34:56 +00:00
|
|
|
out_free:
|
2009-08-06 02:33:39 +00:00
|
|
|
free_msi_irqs(dev);
|
|
|
|
|
|
|
|
return ret;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2006-08-31 05:55:07 +00:00
|
|
|
/**
|
2014-09-23 18:45:58 +00:00
|
|
|
* pci_msi_supported - check whether MSI may be enabled on a device
|
2006-08-31 05:55:07 +00:00
|
|
|
* @dev: pointer to the pci_dev data structure of MSI device function
|
2019-05-30 13:05:58 +00:00
|
|
|
* @nvec: how many MSIs have been requested?
|
2006-08-31 05:55:07 +00:00
|
|
|
*
|
2013-11-14 18:28:18 +00:00
|
|
|
* Look at global flags, the device itself, and its parent buses
|
2007-04-05 07:19:07 +00:00
|
|
|
* to determine if MSI/-X are supported for the device. If MSI/-X is
|
2014-09-23 18:45:58 +00:00
|
|
|
* supported return 1, else return 0.
|
2006-08-31 05:55:07 +00:00
|
|
|
**/
|
2014-09-23 18:45:58 +00:00
|
|
|
static int pci_msi_supported(struct pci_dev *dev, int nvec)
|
2006-08-31 05:55:07 +00:00
|
|
|
{
|
|
|
|
struct pci_bus *bus;
|
|
|
|
|
2006-10-05 08:24:31 +00:00
|
|
|
/* MSI must be globally enabled and supported by the device */
|
2014-09-23 20:25:11 +00:00
|
|
|
if (!pci_msi_enable)
|
2014-09-23 18:45:58 +00:00
|
|
|
return 0;
|
2014-09-23 20:25:11 +00:00
|
|
|
|
2019-10-14 21:17:05 +00:00
|
|
|
if (!dev || dev->no_msi)
|
2014-09-23 18:45:58 +00:00
|
|
|
return 0;
|
2006-08-31 05:55:07 +00:00
|
|
|
|
2007-04-05 07:19:12 +00:00
|
|
|
/*
|
|
|
|
* You can't ask to have 0 or less MSIs configured.
|
|
|
|
* a) it's stupid ..
|
|
|
|
* b) the list manipulation code assumes nvec >= 1.
|
|
|
|
*/
|
|
|
|
if (nvec < 1)
|
2014-09-23 18:45:58 +00:00
|
|
|
return 0;
|
2007-04-05 07:19:12 +00:00
|
|
|
|
2009-08-10 01:14:15 +00:00
|
|
|
/*
|
|
|
|
* Any bridge which does NOT route MSI transactions from its
|
|
|
|
* secondary bus to its primary bus must set NO_MSI flag on
|
2006-10-05 08:24:31 +00:00
|
|
|
* the secondary pci_bus.
|
2021-03-30 15:11:44 +00:00
|
|
|
*
|
|
|
|
* The NO_MSI flag can either be set directly by:
|
|
|
|
* - arch-specific PCI host bus controller drivers (deprecated)
|
|
|
|
* - quirks for specific PCI bridges
|
|
|
|
*
|
|
|
|
* or indirectly by platform-specific PCI host bridge drivers by
|
|
|
|
* advertising the 'msi_domain' property, which results in
|
|
|
|
* the NO_MSI flag when no MSI domain is found for this bridge
|
|
|
|
* at probe time.
|
2006-10-05 08:24:31 +00:00
|
|
|
*/
|
2006-08-31 05:55:07 +00:00
|
|
|
for (bus = dev->bus; bus; bus = bus->parent)
|
|
|
|
if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
|
2014-09-23 18:45:58 +00:00
|
|
|
return 0;
|
2006-08-31 05:55:07 +00:00
|
|
|
|
2014-09-23 18:45:58 +00:00
|
|
|
return 1;
|
2006-08-31 05:55:07 +00:00
|
|
|
}
|
|
|
|
|
2013-12-30 07:28:13 +00:00
|
|
|
/**
|
|
|
|
* pci_msi_vec_count - Return the number of MSI vectors a device can send
|
|
|
|
* @dev: device to report about
|
|
|
|
*
|
|
|
|
* This function returns the number of MSI vectors a device requested via
|
|
|
|
* Multiple Message Capable register. It returns a negative errno if the
|
|
|
|
* device is not capable sending MSI interrupts. Otherwise, the call succeeds
|
|
|
|
* and returns a power of two, up to a maximum of 2^5 (32), according to the
|
|
|
|
* MSI specification.
|
|
|
|
**/
|
|
|
|
int pci_msi_vec_count(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
u16 msgctl;
|
|
|
|
|
|
|
|
if (!dev->msi_cap)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl);
|
|
|
|
ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(pci_msi_vec_count);
|
|
|
|
|
2017-03-09 21:45:14 +00:00
|
|
|
static void pci_msi_shutdown(struct pci_dev *dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-03-17 12:54:09 +00:00
|
|
|
struct msi_desc *desc;
|
|
|
|
u32 mask;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-03-22 10:51:39 +00:00
|
|
|
if (!pci_msi_enable || !dev || !dev->msi_enabled)
|
2007-01-28 19:42:52 +00:00
|
|
|
return;
|
|
|
|
|
2015-07-09 08:00:41 +00:00
|
|
|
BUG_ON(list_empty(dev_to_msi_list(&dev->dev)));
|
2015-07-09 08:00:44 +00:00
|
|
|
desc = first_pci_msi_entry(dev);
|
2009-06-16 12:31:45 +00:00
|
|
|
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msi_set_enable(dev, 0);
|
2007-10-25 08:16:30 +00:00
|
|
|
pci_intx_for_msi(dev, 1);
|
2007-03-05 08:30:10 +00:00
|
|
|
dev->msi_enabled = 0;
|
2006-10-04 09:16:31 +00:00
|
|
|
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
/* Return the device with MSI unmasked as initial states */
|
2014-06-19 08:30:30 +00:00
|
|
|
mask = msi_mask(desc->msi_attrib.multi_cap);
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
/* Keep cached state to be restored */
|
2014-11-23 10:55:58 +00:00
|
|
|
__pci_msi_desc_mask_irq(desc, mask, ~mask);
|
2007-03-22 10:51:27 +00:00
|
|
|
|
2019-05-30 13:05:58 +00:00
|
|
|
/* Restore dev->irq to its default pin-assertion IRQ */
|
2009-03-17 12:54:09 +00:00
|
|
|
dev->irq = desc->msi_attrib.default_irq;
|
2015-07-30 19:00:08 +00:00
|
|
|
pcibios_alloc_irq(dev);
|
2008-04-23 21:58:09 +00:00
|
|
|
}
|
2009-03-17 12:54:06 +00:00
|
|
|
|
2009-08-10 01:14:15 +00:00
|
|
|
void pci_disable_msi(struct pci_dev *dev)
|
2008-04-23 21:58:09 +00:00
|
|
|
{
|
|
|
|
if (!pci_msi_enable || !dev || !dev->msi_enabled)
|
|
|
|
return;
|
|
|
|
|
|
|
|
pci_msi_shutdown(dev);
|
2009-08-06 02:32:51 +00:00
|
|
|
free_msi_irqs(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2007-03-22 10:51:34 +00:00
|
|
|
EXPORT_SYMBOL(pci_disable_msi);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-01-23 23:21:14 +00:00
|
|
|
/**
|
2013-12-30 07:28:15 +00:00
|
|
|
* pci_msix_vec_count - return the number of device's MSI-X table entries
|
2009-01-23 23:21:14 +00:00
|
|
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
2013-12-30 07:28:15 +00:00
|
|
|
* This function returns the number of device's MSI-X table entries and
|
|
|
|
* therefore the number of MSI-X vectors device is capable of sending.
|
|
|
|
* It returns a negative errno if the device is not capable of sending MSI-X
|
|
|
|
* interrupts.
|
|
|
|
**/
|
|
|
|
int pci_msix_vec_count(struct pci_dev *dev)
|
2009-01-23 23:21:14 +00:00
|
|
|
{
|
|
|
|
u16 control;
|
|
|
|
|
2013-04-04 16:54:33 +00:00
|
|
|
if (!dev->msix_cap)
|
2013-12-30 07:28:15 +00:00
|
|
|
return -EINVAL;
|
2009-01-23 23:21:14 +00:00
|
|
|
|
2013-04-17 23:38:32 +00:00
|
|
|
pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
|
2013-04-17 23:44:48 +00:00
|
|
|
return msix_table_size(control);
|
2009-01-23 23:21:14 +00:00
|
|
|
}
|
2013-12-30 07:28:15 +00:00
|
|
|
EXPORT_SYMBOL(pci_msix_vec_count);
|
2009-01-23 23:21:14 +00:00
|
|
|
|
2016-09-14 14:18:49 +00:00
|
|
|
static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
|
2019-05-23 22:30:51 +00:00
|
|
|
int nvec, struct irq_affinity *affd, int flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-09-23 20:38:28 +00:00
|
|
|
int nr_entries;
|
2007-01-28 19:42:52 +00:00
|
|
|
int i, j;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-10-14 21:17:05 +00:00
|
|
|
if (!pci_msi_supported(dev, nvec) || dev->current_state != PCI_D0)
|
2014-09-23 18:45:58 +00:00
|
|
|
return -EINVAL;
|
2007-04-05 07:19:08 +00:00
|
|
|
|
2013-12-30 07:28:15 +00:00
|
|
|
nr_entries = pci_msix_vec_count(dev);
|
|
|
|
if (nr_entries < 0)
|
|
|
|
return nr_entries;
|
2019-05-23 22:30:51 +00:00
|
|
|
if (nvec > nr_entries && !(flags & PCI_IRQ_VIRTUAL))
|
2009-05-07 08:28:41 +00:00
|
|
|
return nr_entries;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-07-12 09:20:16 +00:00
|
|
|
if (entries) {
|
|
|
|
/* Check for any invalid entries */
|
|
|
|
for (i = 0; i < nvec; i++) {
|
|
|
|
if (entries[i].entry >= nr_entries)
|
|
|
|
return -EINVAL; /* invalid entry */
|
|
|
|
for (j = i + 1; j < nvec; j++) {
|
|
|
|
if (entries[i].entry == entries[j].entry)
|
|
|
|
return -EINVAL; /* duplicate entry */
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
2006-10-04 09:16:31 +00:00
|
|
|
|
2019-05-30 13:05:58 +00:00
|
|
|
/* Check whether driver already requested for MSI IRQ */
|
2009-08-10 01:14:15 +00:00
|
|
|
if (dev->msi_enabled) {
|
2018-01-18 18:55:24 +00:00
|
|
|
pci_info(dev, "can't enable MSI-X (MSI IRQ already assigned)\n");
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
2016-11-09 01:15:04 +00:00
|
|
|
return msix_capability_init(dev, entries, nvec, affd);
|
2016-09-14 14:18:49 +00:00
|
|
|
}
|
|
|
|
|
2017-03-09 21:45:14 +00:00
|
|
|
static void pci_msix_shutdown(struct pci_dev *dev)
|
2007-03-22 10:51:33 +00:00
|
|
|
{
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
struct msi_desc *entry;
|
|
|
|
|
2007-03-22 10:51:39 +00:00
|
|
|
if (!pci_msi_enable || !dev || !dev->msix_enabled)
|
2007-01-28 19:42:52 +00:00
|
|
|
return;
|
|
|
|
|
2017-03-30 03:49:11 +00:00
|
|
|
if (pci_dev_is_disconnected(dev)) {
|
|
|
|
dev->msix_enabled = 0;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
/* Return the device with MSI-X masked as initial states */
|
2015-07-09 08:00:41 +00:00
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
/* Keep cached states to be restored */
|
2014-11-23 10:55:58 +00:00
|
|
|
__pci_msix_desc_mask_irq(entry, 1);
|
PCI MSI: Fix restoration of MSI/MSI-X mask states in suspend/resume
There are 2 problems on mask states in suspend/resume.
[1]:
It is better to restore the mask states of MSI/MSI-X to initial states
(MSI is unmasked, MSI-X is masked) when we release the device.
The pci_msi_shutdown() does the restoration of mask states for MSI,
while the msi_free_irqs() does it for MSI-X. In other words, in the
"disable" path both of MSI and MSI-X are handled, but in the "shutdown"
path only MSI is handled.
MSI:
pci_disable_msi()
=> pci_msi_shutdown()
[ mask states for MSI restored ]
=> msi_set_enable(dev, pos, 0);
=> msi_free_irqs()
MSI-X:
pci_disable_msix()
=> pci_msix_shutdown()
=> msix_set_enable(dev, 0);
=> msix_free_all_irqs
=> msi_free_irqs()
[ mask states for MSI-X restored ]
This patch moves the masking for MSI-X from msi_free_irqs() to
pci_msix_shutdown().
This change has some positive side effects:
- It prevents OS from touching mask states before reading preserved
bits in the register, which can be happen if msi_free_irqs() is
called from error path in msix_capability_init().
- It also prevents touching the register after turning off MSI-X in
"disable" path, which can be a problem on some devices.
[2]:
We have cache of the mask state in msi_desc, which is automatically
updated when msi/msix_mask_irq() is called. This cached states are
used for the resume.
But since what need to be restored in the resume is the states before
the shutdown on the suspend, calling msi/msix_mask_irq() from
pci_msi/msix_shutdown() is not appropriate.
This patch introduces __msi/msix_mask_irq() that do mask as same
as msi/msix_mask_irq() but does not update cached state, for use
in pci_msi/msix_shutdown().
[updated: get rid of msi/msix_mask_irq_nocache() (proposed by Matthew Wilcox)]
Reviewed-by: Matthew Wilcox <willy@linux.intel.com>
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Signed-off-by: Jesse Barnes <jbarnes@virtuousgeek.org>
2009-06-24 03:08:09 +00:00
|
|
|
}
|
|
|
|
|
2015-05-07 14:52:21 +00:00
|
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
|
2007-10-25 08:16:30 +00:00
|
|
|
pci_intx_for_msi(dev, 1);
|
2007-03-05 08:30:10 +00:00
|
|
|
dev->msix_enabled = 0;
|
2015-07-30 19:00:08 +00:00
|
|
|
pcibios_alloc_irq(dev);
|
2008-04-23 21:58:09 +00:00
|
|
|
}
|
2009-08-06 02:31:27 +00:00
|
|
|
|
2009-08-10 01:14:15 +00:00
|
|
|
void pci_disable_msix(struct pci_dev *dev)
|
2008-04-23 21:58:09 +00:00
|
|
|
{
|
|
|
|
if (!pci_msi_enable || !dev || !dev->msix_enabled)
|
|
|
|
return;
|
|
|
|
|
|
|
|
pci_msix_shutdown(dev);
|
2009-08-06 02:32:51 +00:00
|
|
|
free_msi_irqs(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2007-03-22 10:51:34 +00:00
|
|
|
EXPORT_SYMBOL(pci_disable_msix);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-03-06 05:33:34 +00:00
|
|
|
void pci_no_msi(void)
|
|
|
|
{
|
|
|
|
pci_msi_enable = 0;
|
|
|
|
}
|
2007-04-05 07:19:08 +00:00
|
|
|
|
2008-11-10 22:31:05 +00:00
|
|
|
/**
|
|
|
|
* pci_msi_enabled - is MSI enabled?
|
|
|
|
*
|
|
|
|
* Returns true if MSI has not been disabled by the command-line option
|
|
|
|
* pci=nomsi.
|
|
|
|
**/
|
|
|
|
int pci_msi_enabled(void)
|
2008-10-17 04:52:51 +00:00
|
|
|
{
|
2008-11-10 22:31:05 +00:00
|
|
|
return pci_msi_enable;
|
2008-10-17 04:52:51 +00:00
|
|
|
}
|
2008-11-10 22:31:05 +00:00
|
|
|
EXPORT_SYMBOL(pci_msi_enabled);
|
2008-10-17 04:52:51 +00:00
|
|
|
|
2016-07-12 09:20:18 +00:00
|
|
|
static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
struct irq_affinity *affd)
|
2013-12-30 07:28:16 +00:00
|
|
|
{
|
2014-04-14 13:28:35 +00:00
|
|
|
int nvec;
|
2013-12-30 07:28:16 +00:00
|
|
|
int rc;
|
|
|
|
|
2019-10-14 21:17:05 +00:00
|
|
|
if (!pci_msi_supported(dev, minvec) || dev->current_state != PCI_D0)
|
2014-09-23 18:45:58 +00:00
|
|
|
return -EINVAL;
|
2014-04-14 13:28:35 +00:00
|
|
|
|
2019-05-30 13:05:58 +00:00
|
|
|
/* Check whether driver already requested MSI-X IRQs */
|
2014-04-14 13:28:35 +00:00
|
|
|
if (dev->msix_enabled) {
|
2018-01-18 18:55:24 +00:00
|
|
|
pci_info(dev, "can't enable MSI (MSI-X already enabled)\n");
|
2014-04-14 13:28:35 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2013-12-30 07:28:16 +00:00
|
|
|
if (maxvec < minvec)
|
|
|
|
return -ERANGE;
|
|
|
|
|
2018-09-24 14:00:41 +00:00
|
|
|
if (WARN_ON_ONCE(dev->msi_enabled))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2014-04-14 13:28:35 +00:00
|
|
|
nvec = pci_msi_vec_count(dev);
|
|
|
|
if (nvec < 0)
|
|
|
|
return nvec;
|
2016-07-12 09:20:18 +00:00
|
|
|
if (nvec < minvec)
|
2016-12-01 02:15:04 +00:00
|
|
|
return -ENOSPC;
|
2016-07-12 09:20:18 +00:00
|
|
|
|
|
|
|
if (nvec > maxvec)
|
2014-04-14 13:28:35 +00:00
|
|
|
nvec = maxvec;
|
|
|
|
|
2016-07-12 09:20:18 +00:00
|
|
|
for (;;) {
|
2016-11-09 01:15:04 +00:00
|
|
|
if (affd) {
|
2017-05-18 17:47:47 +00:00
|
|
|
nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
|
2016-07-12 09:20:18 +00:00
|
|
|
if (nvec < minvec)
|
|
|
|
return -ENOSPC;
|
|
|
|
}
|
|
|
|
|
2016-11-09 01:15:04 +00:00
|
|
|
rc = msi_capability_init(dev, nvec, affd);
|
2016-07-12 09:20:18 +00:00
|
|
|
if (rc == 0)
|
|
|
|
return nvec;
|
|
|
|
|
|
|
|
if (rc < 0)
|
2013-12-30 07:28:16 +00:00
|
|
|
return rc;
|
2016-07-12 09:20:18 +00:00
|
|
|
if (rc < minvec)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
nvec = rc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-09 20:37:40 +00:00
|
|
|
/* deprecated, don't use */
|
|
|
|
int pci_enable_msi(struct pci_dev *dev)
|
2016-07-12 09:20:18 +00:00
|
|
|
{
|
2017-01-09 20:37:40 +00:00
|
|
|
int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
|
|
|
|
if (rc < 0)
|
|
|
|
return rc;
|
|
|
|
return 0;
|
2016-07-12 09:20:18 +00:00
|
|
|
}
|
2017-01-09 20:37:40 +00:00
|
|
|
EXPORT_SYMBOL(pci_enable_msi);
|
2016-07-12 09:20:18 +00:00
|
|
|
|
|
|
|
static int __pci_enable_msix_range(struct pci_dev *dev,
|
2016-11-09 01:15:04 +00:00
|
|
|
struct msix_entry *entries, int minvec,
|
2019-05-23 22:30:51 +00:00
|
|
|
int maxvec, struct irq_affinity *affd,
|
|
|
|
int flags)
|
2016-07-12 09:20:18 +00:00
|
|
|
{
|
2016-09-14 14:18:49 +00:00
|
|
|
int rc, nvec = maxvec;
|
2016-07-12 09:20:18 +00:00
|
|
|
|
|
|
|
if (maxvec < minvec)
|
|
|
|
return -ERANGE;
|
|
|
|
|
2018-09-24 14:00:41 +00:00
|
|
|
if (WARN_ON_ONCE(dev->msix_enabled))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2016-07-12 09:20:18 +00:00
|
|
|
for (;;) {
|
2016-11-09 01:15:04 +00:00
|
|
|
if (affd) {
|
2017-05-18 17:47:47 +00:00
|
|
|
nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
|
2016-07-12 09:20:18 +00:00
|
|
|
if (nvec < minvec)
|
2013-12-30 07:28:16 +00:00
|
|
|
return -ENOSPC;
|
|
|
|
}
|
|
|
|
|
2019-05-23 22:30:51 +00:00
|
|
|
rc = __pci_enable_msix(dev, entries, nvec, affd, flags);
|
2016-07-12 09:20:18 +00:00
|
|
|
if (rc == 0)
|
|
|
|
return nvec;
|
|
|
|
|
|
|
|
if (rc < 0)
|
|
|
|
return rc;
|
|
|
|
if (rc < minvec)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
nvec = rc;
|
|
|
|
}
|
2013-12-30 07:28:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* pci_enable_msix_range - configure device's MSI-X capability structure
|
|
|
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
|
|
|
* @entries: pointer to an array of MSI-X entries
|
2019-05-30 13:05:58 +00:00
|
|
|
* @minvec: minimum number of MSI-X IRQs requested
|
|
|
|
* @maxvec: maximum number of MSI-X IRQs requested
|
2013-12-30 07:28:16 +00:00
|
|
|
*
|
|
|
|
* Setup the MSI-X capability structure of device function with a maximum
|
|
|
|
* possible number of interrupts in the range between @minvec and @maxvec
|
|
|
|
* upon its software driver call to request for MSI-X mode enabled on its
|
|
|
|
* hardware device function. It returns a negative errno if an error occurs.
|
|
|
|
* If it succeeds, it returns the actual number of interrupts allocated and
|
|
|
|
* indicates the successful configuration of MSI-X capability structure
|
|
|
|
* with new allocated MSI-X interrupts.
|
|
|
|
**/
|
|
|
|
int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
|
2016-07-12 09:20:18 +00:00
|
|
|
int minvec, int maxvec)
|
2013-12-30 07:28:16 +00:00
|
|
|
{
|
2019-05-23 22:30:51 +00:00
|
|
|
return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
|
2013-12-30 07:28:16 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(pci_enable_msix_range);
|
2014-11-11 13:02:18 +00:00
|
|
|
|
2016-07-12 09:20:17 +00:00
|
|
|
/**
|
2016-11-09 01:15:05 +00:00
|
|
|
* pci_alloc_irq_vectors_affinity - allocate multiple IRQs for a device
|
2016-07-12 09:20:17 +00:00
|
|
|
* @dev: PCI device to operate on
|
|
|
|
* @min_vecs: minimum number of vectors required (must be >= 1)
|
|
|
|
* @max_vecs: maximum (desired) number of vectors
|
|
|
|
* @flags: flags or quirks for the allocation
|
2016-11-09 01:15:05 +00:00
|
|
|
* @affd: optional description of the affinity requirements
|
2016-07-12 09:20:17 +00:00
|
|
|
*
|
|
|
|
* Allocate up to @max_vecs interrupt vectors for @dev, using MSI-X or MSI
|
|
|
|
* vectors if available, and fall back to a single legacy vector
|
|
|
|
* if neither is available. Return the number of vectors allocated,
|
|
|
|
* (which might be smaller than @max_vecs) if successful, or a negative
|
|
|
|
* error code on error. If less than @min_vecs interrupt vectors are
|
|
|
|
* available for @dev the function will fail with -ENOSPC.
|
|
|
|
*
|
|
|
|
* To get the Linux IRQ number used for a vector that can be passed to
|
|
|
|
* request_irq() use the pci_irq_vector() helper.
|
|
|
|
*/
|
2016-11-09 01:15:05 +00:00
|
|
|
int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
|
|
|
|
unsigned int max_vecs, unsigned int flags,
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
struct irq_affinity *affd)
|
2016-07-12 09:20:17 +00:00
|
|
|
{
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
struct irq_affinity msi_default_affd = {0};
|
2020-06-16 07:33:16 +00:00
|
|
|
int nvecs = -ENOSPC;
|
2016-07-12 09:20:17 +00:00
|
|
|
|
2016-11-09 01:15:05 +00:00
|
|
|
if (flags & PCI_IRQ_AFFINITY) {
|
|
|
|
if (!affd)
|
|
|
|
affd = &msi_default_affd;
|
|
|
|
} else {
|
|
|
|
if (WARN_ON(affd))
|
|
|
|
affd = NULL;
|
|
|
|
}
|
2016-11-09 01:15:04 +00:00
|
|
|
|
2016-08-11 14:11:04 +00:00
|
|
|
if (flags & PCI_IRQ_MSIX) {
|
2020-06-16 07:33:16 +00:00
|
|
|
nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
|
|
|
|
affd, flags);
|
|
|
|
if (nvecs > 0)
|
|
|
|
return nvecs;
|
2016-07-12 09:20:17 +00:00
|
|
|
}
|
|
|
|
|
2016-08-11 14:11:04 +00:00
|
|
|
if (flags & PCI_IRQ_MSI) {
|
2020-06-16 07:33:16 +00:00
|
|
|
nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
|
|
|
|
if (nvecs > 0)
|
|
|
|
return nvecs;
|
2016-07-12 09:20:17 +00:00
|
|
|
}
|
|
|
|
|
2019-05-30 13:05:58 +00:00
|
|
|
/* use legacy IRQ if allowed */
|
2017-02-01 13:41:42 +00:00
|
|
|
if (flags & PCI_IRQ_LEGACY) {
|
|
|
|
if (min_vecs == 1 && dev->irq) {
|
genirq/affinity: Add new callback for (re)calculating interrupt sets
The interrupt affinity spreading mechanism supports to spread out
affinities for one or more interrupt sets. A interrupt set contains one or
more interrupts. Each set is mapped to a specific functionality of a
device, e.g. general I/O queues and read I/O queus of multiqueue block
devices.
The number of interrupts per set is defined by the driver. It depends on
the total number of available interrupts for the device, which is
determined by the PCI capabilites and the availability of underlying CPU
resources, and the number of queues which the device provides and the
driver wants to instantiate.
The driver passes initial configuration for the interrupt allocation via a
pointer to struct irq_affinity.
Right now the allocation mechanism is complex as it requires to have a loop
in the driver to determine the maximum number of interrupts which are
provided by the PCI capabilities and the underlying CPU resources. This
loop would have to be replicated in every driver which wants to utilize
this mechanism. That's unwanted code duplication and error prone.
In order to move this into generic facilities it is required to have a
mechanism, which allows the recalculation of the interrupt sets and their
size, in the core code. As the core code does not have any knowledge about the
underlying device, a driver specific callback is required in struct
irq_affinity, which can be invoked by the core code. The callback gets the
number of available interupts as an argument, so the driver can calculate the
corresponding number and size of interrupt sets.
At the moment the struct irq_affinity pointer which is handed in from the
driver and passed through to several core functions is marked 'const', but for
the callback to be able to modify the data in the struct it's required to
remove the 'const' qualifier.
Add the optional callback to struct irq_affinity, which allows drivers to
recalculate the number and size of interrupt sets and remove the 'const'
qualifier.
For simple invocations, which do not supply a callback, a default callback
is installed, which just sets nr_sets to 1 and transfers the number of
spreadable vectors to the set_size array at index 0.
This is for now guarded by a check for nr_sets != 0 to keep the NVME driver
working until it is converted to the callback mechanism.
To make sure that the driver configuration is correct under all circumstances
the callback is invoked even when there are no interrupts for queues left,
i.e. the pre/post requirements already exhaust the numner of available
interrupts.
At the PCI layer irq_create_affinity_masks() has to be invoked even for the
case where the legacy interrupt is used. That ensures that the callback is
invoked and the device driver can adjust to that situation.
[ tglx: Fixed the simple case (no sets required). Moved the sanity check
for nr_sets after the invocation of the callback so it catches
broken drivers. Fixed the kernel doc comments for struct
irq_affinity and de-'This patch'-ed the changelog ]
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bjorn Helgaas <helgaas@kernel.org>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: linux-block@vger.kernel.org
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: linux-nvme@lists.infradead.org
Cc: linux-pci@vger.kernel.org
Cc: Keith Busch <keith.busch@intel.com>
Cc: Sumit Saxena <sumit.saxena@broadcom.com>
Cc: Kashyap Desai <kashyap.desai@broadcom.com>
Cc: Shivasharan Srikanteshwara <shivasharan.srikanteshwara@broadcom.com>
Link: https://lkml.kernel.org/r/20190216172228.512444498@linutronix.de
2019-02-16 17:13:09 +00:00
|
|
|
/*
|
|
|
|
* Invoke the affinity spreading logic to ensure that
|
|
|
|
* the device driver can adjust queue configuration
|
|
|
|
* for the single interrupt case.
|
|
|
|
*/
|
|
|
|
if (affd)
|
|
|
|
irq_create_affinity_masks(1, affd);
|
2017-02-01 13:41:42 +00:00
|
|
|
pci_intx(dev, 1);
|
|
|
|
return 1;
|
|
|
|
}
|
2016-08-11 14:11:05 +00:00
|
|
|
}
|
|
|
|
|
2020-06-16 07:33:16 +00:00
|
|
|
return nvecs;
|
2016-07-12 09:20:17 +00:00
|
|
|
}
|
2016-11-09 01:15:05 +00:00
|
|
|
EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
|
2016-07-12 09:20:17 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* pci_free_irq_vectors - free previously allocated IRQs for a device
|
|
|
|
* @dev: PCI device to operate on
|
|
|
|
*
|
|
|
|
* Undoes the allocations and enabling in pci_alloc_irq_vectors().
|
|
|
|
*/
|
|
|
|
void pci_free_irq_vectors(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
pci_disable_msix(dev);
|
|
|
|
pci_disable_msi(dev);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(pci_free_irq_vectors);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* pci_irq_vector - return Linux IRQ number of a device vector
|
|
|
|
* @dev: PCI device to operate on
|
|
|
|
* @nr: device-relative interrupt vector index (0-based).
|
|
|
|
*/
|
|
|
|
int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
|
|
|
|
{
|
|
|
|
if (dev->msix_enabled) {
|
|
|
|
struct msi_desc *entry;
|
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
|
|
if (i == nr)
|
|
|
|
return entry->irq;
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dev->msi_enabled) {
|
|
|
|
struct msi_desc *entry = first_pci_msi_entry(dev);
|
|
|
|
|
|
|
|
if (WARN_ON_ONCE(nr >= entry->nvec_used))
|
|
|
|
return -EINVAL;
|
|
|
|
} else {
|
|
|
|
if (WARN_ON_ONCE(nr > 0))
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return dev->irq + nr;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(pci_irq_vector);
|
|
|
|
|
2016-09-14 14:18:51 +00:00
|
|
|
/**
|
2019-05-30 13:05:58 +00:00
|
|
|
* pci_irq_get_affinity - return the affinity of a particular MSI vector
|
2016-09-14 14:18:51 +00:00
|
|
|
* @dev: PCI device to operate on
|
|
|
|
* @nr: device-relative interrupt vector index (0-based).
|
|
|
|
*/
|
|
|
|
const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
|
|
|
|
{
|
|
|
|
if (dev->msix_enabled) {
|
|
|
|
struct msi_desc *entry;
|
|
|
|
int i = 0;
|
|
|
|
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
|
|
if (i == nr)
|
2018-12-04 15:51:20 +00:00
|
|
|
return &entry->affinity->mask;
|
2016-09-14 14:18:51 +00:00
|
|
|
i++;
|
|
|
|
}
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return NULL;
|
|
|
|
} else if (dev->msi_enabled) {
|
|
|
|
struct msi_desc *entry = first_pci_msi_entry(dev);
|
|
|
|
|
2016-11-08 07:43:54 +00:00
|
|
|
if (WARN_ON_ONCE(!entry || !entry->affinity ||
|
|
|
|
nr >= entry->nvec_used))
|
2016-09-14 14:18:51 +00:00
|
|
|
return NULL;
|
|
|
|
|
2018-12-04 15:51:20 +00:00
|
|
|
return &entry->affinity[nr].mask;
|
2016-09-14 14:18:51 +00:00
|
|
|
} else {
|
|
|
|
return cpu_possible_mask;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(pci_irq_get_affinity);
|
|
|
|
|
2015-07-09 08:00:45 +00:00
|
|
|
struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
|
|
|
|
{
|
|
|
|
return to_pci_dev(desc->dev);
|
|
|
|
}
|
2015-12-10 17:52:59 +00:00
|
|
|
EXPORT_SYMBOL(msi_desc_to_pci_dev);
|
2015-07-09 08:00:45 +00:00
|
|
|
|
2015-07-09 08:00:36 +00:00
|
|
|
void *msi_desc_to_pci_sysdata(struct msi_desc *desc)
|
|
|
|
{
|
|
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(desc);
|
|
|
|
|
|
|
|
return dev->bus->sysdata;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata);
|
|
|
|
|
2014-11-11 13:02:18 +00:00
|
|
|
#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
|
|
|
|
/**
|
|
|
|
* pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
|
|
|
|
* @irq_data: Pointer to interrupt data of the MSI interrupt
|
|
|
|
* @msg: Pointer to the message
|
|
|
|
*/
|
|
|
|
void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
|
|
|
|
{
|
2015-06-01 08:05:42 +00:00
|
|
|
struct msi_desc *desc = irq_data_get_msi_desc(irq_data);
|
2014-11-11 13:02:18 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* For MSI-X desc->irq is always equal to irq_data->irq. For
|
|
|
|
* MSI only the first interrupt of MULTI MSI passes the test.
|
|
|
|
*/
|
|
|
|
if (desc->irq == irq_data->irq)
|
|
|
|
__pci_write_msi_msg(desc, msg);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
|
2019-05-30 13:05:58 +00:00
|
|
|
* @desc: Pointer to the MSI descriptor
|
2014-11-11 13:02:18 +00:00
|
|
|
*
|
|
|
|
* The ID number is only used within the irqdomain.
|
|
|
|
*/
|
2020-08-26 11:16:47 +00:00
|
|
|
static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc)
|
2014-11-11 13:02:18 +00:00
|
|
|
{
|
2020-08-26 11:16:45 +00:00
|
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(desc);
|
|
|
|
|
2014-11-11 13:02:18 +00:00
|
|
|
return (irq_hw_number_t)desc->msi_attrib.entry_nr |
|
2019-04-24 19:11:58 +00:00
|
|
|
pci_dev_id(dev) << 11 |
|
2014-11-11 13:02:18 +00:00
|
|
|
(pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc)
|
|
|
|
{
|
|
|
|
return !desc->msi_attrib.is_msix && desc->nvec_used > 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2019-05-30 13:05:58 +00:00
|
|
|
* pci_msi_domain_check_cap - Verify that @domain supports the capabilities
|
|
|
|
* for @dev
|
2014-11-11 13:02:18 +00:00
|
|
|
* @domain: The interrupt domain to check
|
|
|
|
* @info: The domain info for verification
|
|
|
|
* @dev: The device to check
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 if the functionality is supported
|
|
|
|
* 1 if Multi MSI is requested, but the domain does not support it
|
|
|
|
* -ENOTSUPP otherwise
|
|
|
|
*/
|
|
|
|
int pci_msi_domain_check_cap(struct irq_domain *domain,
|
|
|
|
struct msi_domain_info *info, struct device *dev)
|
|
|
|
{
|
|
|
|
struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev));
|
|
|
|
|
2017-01-09 20:37:40 +00:00
|
|
|
/* Special handling to support __pci_enable_msi_range() */
|
2014-11-11 13:02:18 +00:00
|
|
|
if (pci_msi_desc_is_multi_msi(desc) &&
|
|
|
|
!(info->flags & MSI_FLAG_MULTI_PCI_MSI))
|
|
|
|
return 1;
|
|
|
|
else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX))
|
|
|
|
return -ENOTSUPP;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int pci_msi_domain_handle_error(struct irq_domain *domain,
|
|
|
|
struct msi_desc *desc, int error)
|
|
|
|
{
|
2017-01-09 20:37:40 +00:00
|
|
|
/* Special handling to support __pci_enable_msi_range() */
|
2014-11-11 13:02:18 +00:00
|
|
|
if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return error;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
|
|
|
|
struct msi_desc *desc)
|
|
|
|
{
|
|
|
|
arg->desc = desc;
|
2020-08-26 11:16:45 +00:00
|
|
|
arg->hwirq = pci_msi_domain_calc_hwirq(desc);
|
2014-11-11 13:02:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct msi_domain_ops pci_msi_domain_ops_default = {
|
|
|
|
.set_desc = pci_msi_domain_set_desc,
|
|
|
|
.msi_check = pci_msi_domain_check_cap,
|
|
|
|
.handle_error = pci_msi_domain_handle_error,
|
|
|
|
};
|
|
|
|
|
|
|
|
static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)
|
|
|
|
{
|
|
|
|
struct msi_domain_ops *ops = info->ops;
|
|
|
|
|
|
|
|
if (ops == NULL) {
|
|
|
|
info->ops = &pci_msi_domain_ops_default;
|
|
|
|
} else {
|
|
|
|
if (ops->set_desc == NULL)
|
|
|
|
ops->set_desc = pci_msi_domain_set_desc;
|
|
|
|
if (ops->msi_check == NULL)
|
|
|
|
ops->msi_check = pci_msi_domain_check_cap;
|
|
|
|
if (ops->handle_error == NULL)
|
|
|
|
ops->handle_error = pci_msi_domain_handle_error;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)
|
|
|
|
{
|
|
|
|
struct irq_chip *chip = info->chip;
|
|
|
|
|
|
|
|
BUG_ON(!chip);
|
|
|
|
if (!chip->irq_write_msi_msg)
|
|
|
|
chip->irq_write_msi_msg = pci_msi_domain_write_msg;
|
2015-10-13 18:14:45 +00:00
|
|
|
if (!chip->irq_mask)
|
|
|
|
chip->irq_mask = pci_msi_mask_irq;
|
|
|
|
if (!chip->irq_unmask)
|
|
|
|
chip->irq_unmask = pci_msi_unmask_irq;
|
2014-11-11 13:02:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2015-10-13 11:51:44 +00:00
|
|
|
* pci_msi_create_irq_domain - Create a MSI interrupt domain
|
|
|
|
* @fwnode: Optional fwnode of the interrupt controller
|
2014-11-11 13:02:18 +00:00
|
|
|
* @info: MSI domain info
|
|
|
|
* @parent: Parent irq domain
|
|
|
|
*
|
|
|
|
* Updates the domain and chip ops and creates a MSI interrupt domain.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* A domain pointer or NULL in case of failure.
|
|
|
|
*/
|
2015-10-13 11:51:44 +00:00
|
|
|
struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
|
2014-11-11 13:02:18 +00:00
|
|
|
struct msi_domain_info *info,
|
|
|
|
struct irq_domain *parent)
|
|
|
|
{
|
2015-07-28 13:46:09 +00:00
|
|
|
struct irq_domain *domain;
|
|
|
|
|
2018-05-08 12:14:31 +00:00
|
|
|
if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE))
|
|
|
|
info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
|
|
|
|
|
2014-11-11 13:02:18 +00:00
|
|
|
if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
|
|
|
|
pci_msi_domain_update_dom_ops(info);
|
|
|
|
if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
|
|
|
|
pci_msi_domain_update_chip_ops(info);
|
|
|
|
|
2016-07-13 16:18:33 +00:00
|
|
|
info->flags |= MSI_FLAG_ACTIVATE_EARLY;
|
2017-10-17 07:54:58 +00:00
|
|
|
if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
|
|
|
|
info->flags |= MSI_FLAG_MUST_REACTIVATE;
|
2016-07-13 16:18:33 +00:00
|
|
|
|
2018-08-05 20:31:03 +00:00
|
|
|
/* PCI-MSI is oneshot-safe */
|
|
|
|
info->chip->flags |= IRQCHIP_ONESHOT_SAFE;
|
|
|
|
|
2015-10-13 11:51:44 +00:00
|
|
|
domain = msi_create_irq_domain(fwnode, info, parent);
|
2015-07-28 13:46:09 +00:00
|
|
|
if (!domain)
|
|
|
|
return NULL;
|
|
|
|
|
2017-06-22 10:42:50 +00:00
|
|
|
irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI);
|
2015-07-28 13:46:09 +00:00
|
|
|
return domain;
|
2014-11-11 13:02:18 +00:00
|
|
|
}
|
2015-12-10 17:52:59 +00:00
|
|
|
EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
|
2014-11-11 13:02:18 +00:00
|
|
|
|
2017-08-01 17:59:08 +00:00
|
|
|
/*
|
|
|
|
* Users of the generic MSI infrastructure expect a device to have a single ID,
|
|
|
|
* so with DMA aliases we have to pick the least-worst compromise. Devices with
|
|
|
|
* DMA phantom functions tend to still emit MSIs from the real function number,
|
|
|
|
* so we ignore those and only consider topological aliases where either the
|
|
|
|
* alias device or RID appears on a different bus number. We also make the
|
|
|
|
* reasonable assumption that bridges are walked in an upstream direction (so
|
|
|
|
* the last one seen wins), and the much braver assumption that the most likely
|
|
|
|
* case is that of PCI->PCIe so we should always use the alias RID. This echoes
|
|
|
|
* the logic from intel_irq_remapping's set_msi_sid(), which presumably works
|
|
|
|
* well enough in practice; in the face of the horrible PCIe<->PCI-X conditions
|
|
|
|
* for taking ownership all we can really do is close our eyes and hope...
|
|
|
|
*/
|
2015-10-08 22:10:49 +00:00
|
|
|
static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
|
|
|
|
{
|
|
|
|
u32 *pa = data;
|
2017-08-01 17:59:08 +00:00
|
|
|
u8 bus = PCI_BUS_NUM(*pa);
|
|
|
|
|
|
|
|
if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)
|
|
|
|
*pa = alias;
|
2015-10-08 22:10:49 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2017-08-01 17:59:08 +00:00
|
|
|
|
2015-10-08 22:10:49 +00:00
|
|
|
/**
|
|
|
|
* pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)
|
|
|
|
* @domain: The interrupt domain
|
|
|
|
* @pdev: The PCI device.
|
|
|
|
*
|
|
|
|
* The RID for a device is formed from the alias, with a firmware
|
|
|
|
* supplied mapping applied
|
|
|
|
*
|
|
|
|
* Returns: The RID.
|
|
|
|
*/
|
|
|
|
u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
|
|
|
|
{
|
|
|
|
struct device_node *of_node;
|
2019-04-24 19:11:58 +00:00
|
|
|
u32 rid = pci_dev_id(pdev);
|
2015-10-08 22:10:49 +00:00
|
|
|
|
|
|
|
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
|
|
|
|
|
|
|
|
of_node = irq_domain_get_of_node(domain);
|
2020-06-19 08:20:11 +00:00
|
|
|
rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) :
|
2020-06-19 08:20:04 +00:00
|
|
|
iort_msi_map_id(&pdev->dev, rid);
|
2015-10-08 22:10:49 +00:00
|
|
|
|
|
|
|
return rid;
|
|
|
|
}
|
2015-10-02 13:43:06 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* pci_msi_get_device_domain - Get the MSI domain for a given PCI device
|
|
|
|
* @pdev: The PCI device
|
|
|
|
*
|
|
|
|
* Use the firmware data to find a device-specific MSI domain
|
2017-08-01 17:59:08 +00:00
|
|
|
* (i.e. not one that is set as a default).
|
2015-10-02 13:43:06 +00:00
|
|
|
*
|
2017-08-01 17:59:08 +00:00
|
|
|
* Returns: The corresponding MSI domain or NULL if none has been found.
|
2015-10-02 13:43:06 +00:00
|
|
|
*/
|
|
|
|
struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
|
|
|
|
{
|
2016-09-12 18:32:22 +00:00
|
|
|
struct irq_domain *dom;
|
2019-04-24 19:11:58 +00:00
|
|
|
u32 rid = pci_dev_id(pdev);
|
2015-10-02 13:43:06 +00:00
|
|
|
|
|
|
|
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
|
2020-06-19 08:20:10 +00:00
|
|
|
dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI);
|
2016-09-12 18:32:22 +00:00
|
|
|
if (!dom)
|
2020-06-19 08:20:03 +00:00
|
|
|
dom = iort_get_device_domain(&pdev->dev, rid,
|
|
|
|
DOMAIN_BUS_PCI_MSI);
|
2016-09-12 18:32:22 +00:00
|
|
|
return dom;
|
2015-10-02 13:43:06 +00:00
|
|
|
}
|
2020-08-26 11:16:53 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* pci_dev_has_special_msi_domain - Check whether the device is handled by
|
|
|
|
* a non-standard PCI-MSI domain
|
|
|
|
* @pdev: The PCI device to check.
|
|
|
|
*
|
|
|
|
* Returns: True if the device irqdomain or the bus irqdomain is
|
|
|
|
* non-standard PCI/MSI.
|
|
|
|
*/
|
|
|
|
bool pci_dev_has_special_msi_domain(struct pci_dev *pdev)
|
|
|
|
{
|
|
|
|
struct irq_domain *dom = dev_get_msi_domain(&pdev->dev);
|
|
|
|
|
|
|
|
if (!dom)
|
|
|
|
dom = dev_get_msi_domain(&pdev->bus->dev);
|
|
|
|
|
|
|
|
if (!dom)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return dom->bus_token != DOMAIN_BUS_PCI_MSI;
|
|
|
|
}
|
|
|
|
|
2014-11-11 13:02:18 +00:00
|
|
|
#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
|
2020-12-03 18:51:08 +00:00
|
|
|
#endif /* CONFIG_PCI_MSI */
|
|
|
|
|
|
|
|
void pci_msi_init(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
u16 ctrl;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Disable the MSI hardware to avoid screaming interrupts
|
|
|
|
* during boot. This is the power on reset default so
|
|
|
|
* usually this should be a noop.
|
|
|
|
*/
|
|
|
|
dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI);
|
|
|
|
if (!dev->msi_cap)
|
|
|
|
return;
|
|
|
|
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &ctrl);
|
|
|
|
if (ctrl & PCI_MSI_FLAGS_ENABLE)
|
|
|
|
pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS,
|
|
|
|
ctrl & ~PCI_MSI_FLAGS_ENABLE);
|
2020-12-03 18:51:10 +00:00
|
|
|
|
|
|
|
if (!(ctrl & PCI_MSI_FLAGS_64BIT))
|
|
|
|
dev->no_64bit_msi = 1;
|
2020-12-03 18:51:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void pci_msix_init(struct pci_dev *dev)
|
|
|
|
{
|
|
|
|
u16 ctrl;
|
|
|
|
|
|
|
|
dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX);
|
|
|
|
if (!dev->msix_cap)
|
|
|
|
return;
|
|
|
|
|
|
|
|
pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
|
|
|
|
if (ctrl & PCI_MSIX_FLAGS_ENABLE)
|
|
|
|
pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS,
|
|
|
|
ctrl & ~PCI_MSIX_FLAGS_ENABLE);
|
|
|
|
}
|