linux/drivers/pci/msi.c
Eric W. Biederman 92db6d10bc [PATCH] genirq: msi: simplify the msi irq limit policy
Currently we attempt to predict how many irqs we will be able to allocate with
msi using pci_vector_resources and some complicated accounting, and then we
only allow each device as many irqs as we think are available on average.

Only the s2io driver even takes advantage of this feature all other drivers
have a fixed number of irqs they need and bail if they can't get them.

pci_vector_resources is inaccurate if anyone ever frees an irq.  The whole
implmentation is racy.  The current irq limit policy does not appear to make
sense with current drivers.  So I have simplified things.  We can revisit this
we we need a more sophisticated policy.

Signed-off-by: Eric W. Biederman <ebiederm@xmission.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Rajesh Shah <rajesh.shah@intel.com>
Cc: Andi Kleen <ak@muc.de>
Cc: "Protasevich, Natalie" <Natalie.Protasevich@UNISYS.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-10-04 07:55:27 -07:00

1228 lines
31 KiB
C

/*
* File: msi.c
* Purpose: PCI Message Signaled Interrupt (MSI)
*
* Copyright (C) 2003-2004 Intel
* Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
*/
#include <linux/mm.h>
#include <linux/irq.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include <linux/smp_lock.h>
#include <linux/pci.h>
#include <linux/proc_fs.h>
#include <asm/errno.h>
#include <asm/io.h>
#include <asm/smp.h>
#include "pci.h"
#include "msi.h"
static DEFINE_SPINLOCK(msi_lock);
static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL };
static kmem_cache_t* msi_cachep;
static int pci_msi_enable = 1;
static int last_alloc_vector;
static int nr_released_vectors;
#ifndef CONFIG_X86_IO_APIC
int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1};
#endif
static struct msi_ops *msi_ops;
int
msi_register(struct msi_ops *ops)
{
msi_ops = ops;
return 0;
}
static int msi_cache_init(void)
{
msi_cachep = kmem_cache_create("msi_cache", sizeof(struct msi_desc),
0, SLAB_HWCACHE_ALIGN, NULL, NULL);
if (!msi_cachep)
return -ENOMEM;
return 0;
}
static void msi_set_mask_bit(unsigned int vector, int flag)
{
struct msi_desc *entry;
entry = (struct msi_desc *)msi_desc[vector];
if (!entry || !entry->dev || !entry->mask_base)
return;
switch (entry->msi_attrib.type) {
case PCI_CAP_ID_MSI:
{
int pos;
u32 mask_bits;
pos = (long)entry->mask_base;
pci_read_config_dword(entry->dev, pos, &mask_bits);
mask_bits &= ~(1);
mask_bits |= flag;
pci_write_config_dword(entry->dev, pos, mask_bits);
break;
}
case PCI_CAP_ID_MSIX:
{
int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET;
writel(flag, entry->mask_base + offset);
break;
}
default:
break;
}
}
static void read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
{
switch(entry->msi_attrib.type) {
case PCI_CAP_ID_MSI:
{
struct pci_dev *dev = entry->dev;
int pos = entry->msi_attrib.pos;
u16 data;
pci_read_config_dword(dev, msi_lower_address_reg(pos),
&msg->address_lo);
if (entry->msi_attrib.is_64) {
pci_read_config_dword(dev, msi_upper_address_reg(pos),
&msg->address_hi);
pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
} else {
msg->address_hi = 0;
pci_read_config_word(dev, msi_data_reg(pos, 1), &data);
}
msg->data = data;
break;
}
case PCI_CAP_ID_MSIX:
{
void __iomem *base;
base = entry->mask_base +
entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
msg->data = readl(base + PCI_MSIX_ENTRY_DATA_OFFSET);
break;
}
default:
BUG();
}
}
static void write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
{
switch (entry->msi_attrib.type) {
case PCI_CAP_ID_MSI:
{
struct pci_dev *dev = entry->dev;
int pos = entry->msi_attrib.pos;
pci_write_config_dword(dev, msi_lower_address_reg(pos),
msg->address_lo);
if (entry->msi_attrib.is_64) {
pci_write_config_dword(dev, msi_upper_address_reg(pos),
msg->address_hi);
pci_write_config_word(dev, msi_data_reg(pos, 1),
msg->data);
} else {
pci_write_config_word(dev, msi_data_reg(pos, 0),
msg->data);
}
break;
}
case PCI_CAP_ID_MSIX:
{
void __iomem *base;
base = entry->mask_base +
entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
writel(msg->address_lo,
base + PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET);
writel(msg->address_hi,
base + PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET);
writel(msg->data, base + PCI_MSIX_ENTRY_DATA_OFFSET);
break;
}
default:
BUG();
}
}
#ifdef CONFIG_SMP
static void set_msi_affinity(unsigned int irq, cpumask_t cpu_mask)
{
struct msi_desc *entry;
struct msi_msg msg;
entry = msi_desc[irq];
if (!entry || !entry->dev)
return;
read_msi_msg(entry, &msg);
msi_ops->target(irq, cpu_mask, &msg);
write_msi_msg(entry, &msg);
set_native_irq_info(irq, cpu_mask);
}
#else
#define set_msi_affinity NULL
#endif /* CONFIG_SMP */
static void mask_MSI_irq(unsigned int vector)
{
msi_set_mask_bit(vector, 1);
}
static void unmask_MSI_irq(unsigned int vector)
{
msi_set_mask_bit(vector, 0);
}
static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector)
{
struct msi_desc *entry;
unsigned long flags;
spin_lock_irqsave(&msi_lock, flags);
entry = msi_desc[vector];
if (!entry || !entry->dev) {
spin_unlock_irqrestore(&msi_lock, flags);
return 0;
}
entry->msi_attrib.state = 1; /* Mark it active */
spin_unlock_irqrestore(&msi_lock, flags);
return 0; /* never anything pending */
}
static unsigned int startup_msi_irq_w_maskbit(unsigned int vector)
{
startup_msi_irq_wo_maskbit(vector);
unmask_MSI_irq(vector);
return 0; /* never anything pending */
}
static void shutdown_msi_irq(unsigned int vector)
{
struct msi_desc *entry;
unsigned long flags;
spin_lock_irqsave(&msi_lock, flags);
entry = msi_desc[vector];
if (entry && entry->dev)
entry->msi_attrib.state = 0; /* Mark it not active */
spin_unlock_irqrestore(&msi_lock, flags);
}
static void end_msi_irq_wo_maskbit(unsigned int vector)
{
move_native_irq(vector);
ack_APIC_irq();
}
static void end_msi_irq_w_maskbit(unsigned int vector)
{
move_native_irq(vector);
unmask_MSI_irq(vector);
ack_APIC_irq();
}
static void do_nothing(unsigned int vector)
{
}
/*
* Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices,
* which implement the MSI-X Capability Structure.
*/
static struct hw_interrupt_type msix_irq_type = {
.typename = "PCI-MSI-X",
.startup = startup_msi_irq_w_maskbit,
.shutdown = shutdown_msi_irq,
.enable = unmask_MSI_irq,
.disable = mask_MSI_irq,
.ack = mask_MSI_irq,
.end = end_msi_irq_w_maskbit,
.set_affinity = set_msi_affinity
};
/*
* Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
* which implement the MSI Capability Structure with
* Mask-and-Pending Bits.
*/
static struct hw_interrupt_type msi_irq_w_maskbit_type = {
.typename = "PCI-MSI",
.startup = startup_msi_irq_w_maskbit,
.shutdown = shutdown_msi_irq,
.enable = unmask_MSI_irq,
.disable = mask_MSI_irq,
.ack = mask_MSI_irq,
.end = end_msi_irq_w_maskbit,
.set_affinity = set_msi_affinity
};
/*
* Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices,
* which implement the MSI Capability Structure without
* Mask-and-Pending Bits.
*/
static struct hw_interrupt_type msi_irq_wo_maskbit_type = {
.typename = "PCI-MSI",
.startup = startup_msi_irq_wo_maskbit,
.shutdown = shutdown_msi_irq,
.enable = do_nothing,
.disable = do_nothing,
.ack = do_nothing,
.end = end_msi_irq_wo_maskbit,
.set_affinity = set_msi_affinity
};
static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);
static int assign_msi_vector(void)
{
static int new_vector_avail = 1;
int vector;
unsigned long flags;
/*
* msi_lock is provided to ensure that successful allocation of MSI
* vector is assigned unique among drivers.
*/
spin_lock_irqsave(&msi_lock, flags);
if (!new_vector_avail) {
int free_vector = 0;
/*
* vector_irq[] = -1 indicates that this specific vector is:
* - assigned for MSI (since MSI have no associated IRQ) or
* - assigned for legacy if less than 16, or
* - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping
* vector_irq[] = 0 indicates that this vector, previously
* assigned for MSI, is freed by hotplug removed operations.
* This vector will be reused for any subsequent hotplug added
* operations.
* vector_irq[] > 0 indicates that this vector is assigned for
* IOxAPIC IRQs. This vector and its value provides a 1-to-1
* vector-to-IOxAPIC IRQ mapping.
*/
for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
if (vector_irq[vector] != 0)
continue;
free_vector = vector;
if (!msi_desc[vector])
break;
else
continue;
}
if (!free_vector) {
spin_unlock_irqrestore(&msi_lock, flags);
return -EBUSY;
}
vector_irq[free_vector] = -1;
nr_released_vectors--;
spin_unlock_irqrestore(&msi_lock, flags);
if (msi_desc[free_vector] != NULL) {
struct pci_dev *dev;
int tail;
/* free all linked vectors before re-assign */
do {
spin_lock_irqsave(&msi_lock, flags);
dev = msi_desc[free_vector]->dev;
tail = msi_desc[free_vector]->link.tail;
spin_unlock_irqrestore(&msi_lock, flags);
msi_free_vector(dev, tail, 1);
} while (free_vector != tail);
}
return free_vector;
}
vector = assign_irq_vector(AUTO_ASSIGN);
last_alloc_vector = vector;
if (vector == LAST_DEVICE_VECTOR)
new_vector_avail = 0;
spin_unlock_irqrestore(&msi_lock, flags);
return vector;
}
static int get_new_vector(void)
{
int vector = assign_msi_vector();
if (vector > 0)
set_intr_gate(vector, interrupt[vector]);
return vector;
}
static int msi_init(void)
{
static int status = -ENOMEM;
if (!status)
return status;
if (pci_msi_quirk) {
pci_msi_enable = 0;
printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
status = -EINVAL;
return status;
}
status = msi_arch_init();
if (status < 0) {
pci_msi_enable = 0;
printk(KERN_WARNING
"PCI: MSI arch init failed. MSI disabled.\n");
return status;
}
if (! msi_ops) {
printk(KERN_WARNING
"PCI: MSI ops not registered. MSI disabled.\n");
status = -EINVAL;
return status;
}
last_alloc_vector = assign_irq_vector(AUTO_ASSIGN);
status = msi_cache_init();
if (status < 0) {
pci_msi_enable = 0;
printk(KERN_WARNING "PCI: MSI cache init failed\n");
return status;
}
if (last_alloc_vector < 0) {
pci_msi_enable = 0;
printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n");
status = -EBUSY;
return status;
}
vector_irq[last_alloc_vector] = 0;
nr_released_vectors++;
return status;
}
static int get_msi_vector(struct pci_dev *dev)
{
return get_new_vector();
}
static struct msi_desc* alloc_msi_entry(void)
{
struct msi_desc *entry;
entry = kmem_cache_zalloc(msi_cachep, GFP_KERNEL);
if (!entry)
return NULL;
entry->link.tail = entry->link.head = 0; /* single message */
entry->dev = NULL;
return entry;
}
static void attach_msi_entry(struct msi_desc *entry, int vector)
{
unsigned long flags;
spin_lock_irqsave(&msi_lock, flags);
msi_desc[vector] = entry;
spin_unlock_irqrestore(&msi_lock, flags);
}
static void irq_handler_init(int cap_id, int pos, int mask)
{
unsigned long flags;
spin_lock_irqsave(&irq_desc[pos].lock, flags);
if (cap_id == PCI_CAP_ID_MSIX)
irq_desc[pos].chip = &msix_irq_type;
else {
if (!mask)
irq_desc[pos].chip = &msi_irq_wo_maskbit_type;
else
irq_desc[pos].chip = &msi_irq_w_maskbit_type;
}
spin_unlock_irqrestore(&irq_desc[pos].lock, flags);
}
static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
{
u16 control;
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (type == PCI_CAP_ID_MSI) {
/* Set enabled bits to single MSI & enable MSI_enable bit */
msi_enable(control, 1);
pci_write_config_word(dev, msi_control_reg(pos), control);
dev->msi_enabled = 1;
} else {
msix_enable(control);
pci_write_config_word(dev, msi_control_reg(pos), control);
dev->msix_enabled = 1;
}
if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
/* PCI Express Endpoint device detected */
pci_intx(dev, 0); /* disable intx */
}
}
void disable_msi_mode(struct pci_dev *dev, int pos, int type)
{
u16 control;
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (type == PCI_CAP_ID_MSI) {
/* Set enabled bits to single MSI & enable MSI_enable bit */
msi_disable(control);
pci_write_config_word(dev, msi_control_reg(pos), control);
dev->msi_enabled = 0;
} else {
msix_disable(control);
pci_write_config_word(dev, msi_control_reg(pos), control);
dev->msix_enabled = 0;
}
if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
/* PCI Express Endpoint device detected */
pci_intx(dev, 1); /* enable intx */
}
}
static int msi_lookup_vector(struct pci_dev *dev, int type)
{
int vector;
unsigned long flags;
spin_lock_irqsave(&msi_lock, flags);
for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) {
if (!msi_desc[vector] || msi_desc[vector]->dev != dev ||
msi_desc[vector]->msi_attrib.type != type ||
msi_desc[vector]->msi_attrib.default_vector != dev->irq)
continue;
spin_unlock_irqrestore(&msi_lock, flags);
/* This pre-assigned MSI vector for this device
already exits. Override dev->irq with this vector */
dev->irq = vector;
return 0;
}
spin_unlock_irqrestore(&msi_lock, flags);
return -EACCES;
}
void pci_scan_msi_device(struct pci_dev *dev)
{
if (!dev)
return;
}
#ifdef CONFIG_PM
int pci_save_msi_state(struct pci_dev *dev)
{
int pos, i = 0;
u16 control;
struct pci_cap_saved_state *save_state;
u32 *cap;
pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
if (pos <= 0 || dev->no_msi)
return 0;
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (!(control & PCI_MSI_FLAGS_ENABLE))
return 0;
save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u32) * 5,
GFP_KERNEL);
if (!save_state) {
printk(KERN_ERR "Out of memory in pci_save_msi_state\n");
return -ENOMEM;
}
cap = &save_state->data[0];
pci_read_config_dword(dev, pos, &cap[i++]);
control = cap[0] >> 16;
pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, &cap[i++]);
if (control & PCI_MSI_FLAGS_64BIT) {
pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, &cap[i++]);
pci_read_config_dword(dev, pos + PCI_MSI_DATA_64, &cap[i++]);
} else
pci_read_config_dword(dev, pos + PCI_MSI_DATA_32, &cap[i++]);
if (control & PCI_MSI_FLAGS_MASKBIT)
pci_read_config_dword(dev, pos + PCI_MSI_MASK_BIT, &cap[i++]);
save_state->cap_nr = PCI_CAP_ID_MSI;
pci_add_saved_cap(dev, save_state);
return 0;
}
void pci_restore_msi_state(struct pci_dev *dev)
{
int i = 0, pos;
u16 control;
struct pci_cap_saved_state *save_state;
u32 *cap;
save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSI);
pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
if (!save_state || pos <= 0)
return;
cap = &save_state->data[0];
control = cap[i++] >> 16;
pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, cap[i++]);
if (control & PCI_MSI_FLAGS_64BIT) {
pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, cap[i++]);
pci_write_config_dword(dev, pos + PCI_MSI_DATA_64, cap[i++]);
} else
pci_write_config_dword(dev, pos + PCI_MSI_DATA_32, cap[i++]);
if (control & PCI_MSI_FLAGS_MASKBIT)
pci_write_config_dword(dev, pos + PCI_MSI_MASK_BIT, cap[i++]);
pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
pci_remove_saved_cap(save_state);
kfree(save_state);
}
int pci_save_msix_state(struct pci_dev *dev)
{
int pos;
int temp;
int vector, head, tail = 0;
u16 control;
struct pci_cap_saved_state *save_state;
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
if (pos <= 0 || dev->no_msi)
return 0;
/* save the capability */
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (!(control & PCI_MSIX_FLAGS_ENABLE))
return 0;
save_state = kzalloc(sizeof(struct pci_cap_saved_state) + sizeof(u16),
GFP_KERNEL);
if (!save_state) {
printk(KERN_ERR "Out of memory in pci_save_msix_state\n");
return -ENOMEM;
}
*((u16 *)&save_state->data[0]) = control;
/* save the table */
temp = dev->irq;
if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
kfree(save_state);
return -EINVAL;
}
vector = head = dev->irq;
while (head != tail) {
struct msi_desc *entry;
entry = msi_desc[vector];
read_msi_msg(entry, &entry->msg_save);
tail = msi_desc[vector]->link.tail;
vector = tail;
}
dev->irq = temp;
save_state->cap_nr = PCI_CAP_ID_MSIX;
pci_add_saved_cap(dev, save_state);
return 0;
}
void pci_restore_msix_state(struct pci_dev *dev)
{
u16 save;
int pos;
int vector, head, tail = 0;
struct msi_desc *entry;
int temp;
struct pci_cap_saved_state *save_state;
save_state = pci_find_saved_cap(dev, PCI_CAP_ID_MSIX);
if (!save_state)
return;
save = *((u16 *)&save_state->data[0]);
pci_remove_saved_cap(save_state);
kfree(save_state);
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
if (pos <= 0)
return;
/* route the table */
temp = dev->irq;
if (msi_lookup_vector(dev, PCI_CAP_ID_MSIX))
return;
vector = head = dev->irq;
while (head != tail) {
entry = msi_desc[vector];
write_msi_msg(entry, &entry->msg_save);
tail = msi_desc[vector]->link.tail;
vector = tail;
}
dev->irq = temp;
pci_write_config_word(dev, msi_control_reg(pos), save);
enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
}
#endif
static int msi_register_init(struct pci_dev *dev, struct msi_desc *entry)
{
int status;
struct msi_msg msg;
int pos;
u16 control;
pos = entry->msi_attrib.pos;
pci_read_config_word(dev, msi_control_reg(pos), &control);
/* Configure MSI capability structure */
status = msi_ops->setup(dev, dev->irq, &msg);
if (status < 0)
return status;
write_msi_msg(entry, &msg);
if (entry->msi_attrib.maskbit) {
unsigned int maskbits, temp;
/* All MSIs are unmasked by default, Mask them all */
pci_read_config_dword(dev,
msi_mask_bits_reg(pos, is_64bit_address(control)),
&maskbits);
temp = (1 << multi_msi_capable(control));
temp = ((temp - 1) & ~temp);
maskbits |= temp;
pci_write_config_dword(dev,
msi_mask_bits_reg(pos, is_64bit_address(control)),
maskbits);
}
return 0;
}
/**
* msi_capability_init - configure device's MSI capability structure
* @dev: pointer to the pci_dev data structure of MSI device function
*
* Setup the MSI capability structure of device function with a single
* MSI vector, regardless of device function is capable of handling
* multiple messages. A return of zero indicates the successful setup
* of an entry zero with the new MSI vector or non-zero for otherwise.
**/
static int msi_capability_init(struct pci_dev *dev)
{
int status;
struct msi_desc *entry;
int pos, vector;
u16 control;
pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
pci_read_config_word(dev, msi_control_reg(pos), &control);
/* MSI Entry Initialization */
entry = alloc_msi_entry();
if (!entry)
return -ENOMEM;
vector = get_msi_vector(dev);
if (vector < 0) {
kmem_cache_free(msi_cachep, entry);
return -EBUSY;
}
entry->link.head = vector;
entry->link.tail = vector;
entry->msi_attrib.type = PCI_CAP_ID_MSI;
entry->msi_attrib.state = 0; /* Mark it not active */
entry->msi_attrib.is_64 = is_64bit_address(control);
entry->msi_attrib.entry_nr = 0;
entry->msi_attrib.maskbit = is_mask_bit_support(control);
entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */
entry->msi_attrib.pos = pos;
dev->irq = vector;
entry->dev = dev;
if (is_mask_bit_support(control)) {
entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos,
is_64bit_address(control));
}
/* Replace with MSI handler */
irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit);
/* Configure MSI capability structure */
status = msi_register_init(dev, entry);
if (status != 0) {
dev->irq = entry->msi_attrib.default_vector;
kmem_cache_free(msi_cachep, entry);
return status;
}
attach_msi_entry(entry, vector);
/* Set MSI enabled bits */
enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
return 0;
}
/**
* msix_capability_init - configure device's MSI-X capability
* @dev: pointer to the pci_dev data structure of MSI-X device function
* @entries: pointer to an array of struct msix_entry entries
* @nvec: number of @entries
*
* Setup the MSI-X capability structure of device function with a
* single MSI-X vector. A return of zero indicates the successful setup of
* requested MSI-X entries with allocated vectors or non-zero for otherwise.
**/
static int msix_capability_init(struct pci_dev *dev,
struct msix_entry *entries, int nvec)
{
struct msi_desc *head = NULL, *tail = NULL, *entry = NULL;
struct msi_msg msg;
int status;
int vector, pos, i, j, nr_entries, temp = 0;
unsigned long phys_addr;
u32 table_offset;
u16 control;
u8 bir;
void __iomem *base;
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
/* Request & Map MSI-X table region */
pci_read_config_word(dev, msi_control_reg(pos), &control);
nr_entries = multi_msix_capable(control);
pci_read_config_dword(dev, msix_table_offset_reg(pos), &table_offset);
bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK);
table_offset &= ~PCI_MSIX_FLAGS_BIRMASK;
phys_addr = pci_resource_start (dev, bir) + table_offset;
base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
if (base == NULL)
return -ENOMEM;
/* MSI-X Table Initialization */
for (i = 0; i < nvec; i++) {
entry = alloc_msi_entry();
if (!entry)
break;
vector = get_msi_vector(dev);
if (vector < 0) {
kmem_cache_free(msi_cachep, entry);
break;
}
j = entries[i].entry;
entries[i].vector = vector;
entry->msi_attrib.type = PCI_CAP_ID_MSIX;
entry->msi_attrib.state = 0; /* Mark it not active */
entry->msi_attrib.is_64 = 1;
entry->msi_attrib.entry_nr = j;
entry->msi_attrib.maskbit = 1;
entry->msi_attrib.default_vector = dev->irq;
entry->msi_attrib.pos = pos;
entry->dev = dev;
entry->mask_base = base;
if (!head) {
entry->link.head = vector;
entry->link.tail = vector;
head = entry;
} else {
entry->link.head = temp;
entry->link.tail = tail->link.tail;
tail->link.tail = vector;
head->link.head = vector;
}
temp = vector;
tail = entry;
/* Replace with MSI-X handler */
irq_handler_init(PCI_CAP_ID_MSIX, vector, 1);
/* Configure MSI-X capability structure */
status = msi_ops->setup(dev, vector, &msg);
if (status < 0)
break;
write_msi_msg(entry, &msg);
attach_msi_entry(entry, vector);
}
if (i != nvec) {
int avail = i - 1;
i--;
for (; i >= 0; i--) {
vector = (entries + i)->vector;
msi_free_vector(dev, vector, 0);
(entries + i)->vector = 0;
}
/* If we had some success report the number of irqs
* we succeeded in setting up.
*/
if (avail <= 0)
avail = -EBUSY;
return avail;
}
/* Set MSI-X enabled bits */
enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
return 0;
}
/**
* pci_msi_supported - check whether MSI may be enabled on device
* @dev: pointer to the pci_dev data structure of MSI device function
*
* MSI must be globally enabled and supported by the device and its root
* bus. But, the root bus is not easy to find since some architectures
* have virtual busses on top of the PCI hierarchy (for instance the
* hypertransport bus), while the actual bus where MSI must be supported
* is below. So we test the MSI flag on all parent busses and assume
* that no quirk will ever set the NO_MSI flag on a non-root bus.
**/
static
int pci_msi_supported(struct pci_dev * dev)
{
struct pci_bus *bus;
if (!pci_msi_enable || !dev || dev->no_msi)
return -EINVAL;
/* check MSI flags of all parent busses */
for (bus = dev->bus; bus; bus = bus->parent)
if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
return -EINVAL;
return 0;
}
/**
* pci_enable_msi - configure device's MSI capability structure
* @dev: pointer to the pci_dev data structure of MSI device function
*
* Setup the MSI capability structure of device function with
* a single MSI vector upon its software driver call to request for
* MSI mode enabled on its hardware device function. A return of zero
* indicates the successful setup of an entry zero with the new MSI
* vector or non-zero for otherwise.
**/
int pci_enable_msi(struct pci_dev* dev)
{
int pos, temp, status;
u16 control;
if (pci_msi_supported(dev) < 0)
return -EINVAL;
temp = dev->irq;
status = msi_init();
if (status < 0)
return status;
pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
if (!pos)
return -EINVAL;
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (!is_64bit_address(control) && msi_ops->needs_64bit_address)
return -EINVAL;
WARN_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSI));
/* Check whether driver already requested for MSI-X vectors */
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
printk(KERN_INFO "PCI: %s: Can't enable MSI. "
"Device already has MSI-X vectors assigned\n",
pci_name(dev));
dev->irq = temp;
return -EINVAL;
}
status = msi_capability_init(dev);
return status;
}
void pci_disable_msi(struct pci_dev* dev)
{
struct msi_desc *entry;
int pos, default_vector;
u16 control;
unsigned long flags;
if (!pci_msi_enable)
return;
if (!dev)
return;
pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
if (!pos)
return;
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (!(control & PCI_MSI_FLAGS_ENABLE))
return;
disable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
spin_lock_irqsave(&msi_lock, flags);
entry = msi_desc[dev->irq];
if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) {
spin_unlock_irqrestore(&msi_lock, flags);
return;
}
if (entry->msi_attrib.state) {
spin_unlock_irqrestore(&msi_lock, flags);
printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without "
"free_irq() on MSI vector %d\n",
pci_name(dev), dev->irq);
BUG_ON(entry->msi_attrib.state > 0);
} else {
default_vector = entry->msi_attrib.default_vector;
spin_unlock_irqrestore(&msi_lock, flags);
msi_free_vector(dev, dev->irq, 0);
/* Restore dev->irq to its default pin-assertion vector */
dev->irq = default_vector;
}
}
static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
{
struct msi_desc *entry;
int head, entry_nr, type;
void __iomem *base;
unsigned long flags;
msi_ops->teardown(vector);
spin_lock_irqsave(&msi_lock, flags);
entry = msi_desc[vector];
if (!entry || entry->dev != dev) {
spin_unlock_irqrestore(&msi_lock, flags);
return -EINVAL;
}
type = entry->msi_attrib.type;
entry_nr = entry->msi_attrib.entry_nr;
head = entry->link.head;
base = entry->mask_base;
msi_desc[entry->link.head]->link.tail = entry->link.tail;
msi_desc[entry->link.tail]->link.head = entry->link.head;
entry->dev = NULL;
if (!reassign) {
vector_irq[vector] = 0;
nr_released_vectors++;
}
msi_desc[vector] = NULL;
spin_unlock_irqrestore(&msi_lock, flags);
kmem_cache_free(msi_cachep, entry);
if (type == PCI_CAP_ID_MSIX) {
if (!reassign)
writel(1, base +
entry_nr * PCI_MSIX_ENTRY_SIZE +
PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET);
if (head == vector)
iounmap(base);
}
return 0;
}
/**
* pci_enable_msix - configure device's MSI-X capability structure
* @dev: pointer to the pci_dev data structure of MSI-X device function
* @entries: pointer to an array of MSI-X entries
* @nvec: number of MSI-X vectors requested for allocation by device driver
*
* Setup the MSI-X capability structure of device function with the number
* of requested vectors upon its software driver call to request for
* MSI-X mode enabled on its hardware device function. A return of zero
* indicates the successful configuration of MSI-X capability structure
* with new allocated MSI-X vectors. A return of < 0 indicates a failure.
* Or a return of > 0 indicates that driver request is exceeding the number
* of vectors available. Driver should use the returned value to re-send
* its request.
**/
int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
{
int status, pos, nr_entries;
int i, j, temp;
u16 control;
if (!entries || pci_msi_supported(dev) < 0)
return -EINVAL;
status = msi_init();
if (status < 0)
return status;
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
if (!pos)
return -EINVAL;
pci_read_config_word(dev, msi_control_reg(pos), &control);
nr_entries = multi_msix_capable(control);
if (nvec > nr_entries)
return -EINVAL;
/* Check for any invalid entries */
for (i = 0; i < nvec; i++) {
if (entries[i].entry >= nr_entries)
return -EINVAL; /* invalid entry */
for (j = i + 1; j < nvec; j++) {
if (entries[i].entry == entries[j].entry)
return -EINVAL; /* duplicate entry */
}
}
temp = dev->irq;
WARN_ON(!msi_lookup_vector(dev, PCI_CAP_ID_MSIX));
/* Check whether driver already requested for MSI vector */
if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 &&
!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
printk(KERN_INFO "PCI: %s: Can't enable MSI-X. "
"Device already has an MSI vector assigned\n",
pci_name(dev));
dev->irq = temp;
return -EINVAL;
}
status = msix_capability_init(dev, entries, nvec);
return status;
}
void pci_disable_msix(struct pci_dev* dev)
{
int pos, temp;
u16 control;
if (!pci_msi_enable)
return;
if (!dev)
return;
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
if (!pos)
return;
pci_read_config_word(dev, msi_control_reg(pos), &control);
if (!(control & PCI_MSIX_FLAGS_ENABLE))
return;
disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
temp = dev->irq;
if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
int state, vector, head, tail = 0, warning = 0;
unsigned long flags;
vector = head = dev->irq;
dev->irq = temp; /* Restore pin IRQ */
while (head != tail) {
spin_lock_irqsave(&msi_lock, flags);
state = msi_desc[vector]->msi_attrib.state;
tail = msi_desc[vector]->link.tail;
spin_unlock_irqrestore(&msi_lock, flags);
if (state)
warning = 1;
else if (vector != head) /* Release MSI-X vector */
msi_free_vector(dev, vector, 0);
vector = tail;
}
msi_free_vector(dev, vector, 0);
if (warning) {
printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without "
"free_irq() on all MSI-X vectors\n",
pci_name(dev));
BUG_ON(warning > 0);
}
}
}
/**
* msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
* @dev: pointer to the pci_dev data structure of MSI(X) device function
*
* Being called during hotplug remove, from which the device function
* is hot-removed. All previous assigned MSI/MSI-X vectors, if
* allocated for this device function, are reclaimed to unused state,
* which may be used later on.
**/
void msi_remove_pci_irq_vectors(struct pci_dev* dev)
{
int state, pos, temp;
unsigned long flags;
if (!pci_msi_enable || !dev)
return;
temp = dev->irq; /* Save IOAPIC IRQ */
pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) {
spin_lock_irqsave(&msi_lock, flags);
state = msi_desc[dev->irq]->msi_attrib.state;
spin_unlock_irqrestore(&msi_lock, flags);
if (state) {
printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
"called without free_irq() on MSI vector %d\n",
pci_name(dev), dev->irq);
BUG_ON(state > 0);
} else /* Release MSI vector assigned to this device */
msi_free_vector(dev, dev->irq, 0);
dev->irq = temp; /* Restore IOAPIC IRQ */
}
pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
if (pos > 0 && !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) {
int vector, head, tail = 0, warning = 0;
void __iomem *base = NULL;
vector = head = dev->irq;
while (head != tail) {
spin_lock_irqsave(&msi_lock, flags);
state = msi_desc[vector]->msi_attrib.state;
tail = msi_desc[vector]->link.tail;
base = msi_desc[vector]->mask_base;
spin_unlock_irqrestore(&msi_lock, flags);
if (state)
warning = 1;
else if (vector != head) /* Release MSI-X vector */
msi_free_vector(dev, vector, 0);
vector = tail;
}
msi_free_vector(dev, vector, 0);
if (warning) {
iounmap(base);
printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() "
"called without free_irq() on all MSI-X vectors\n",
pci_name(dev));
BUG_ON(warning > 0);
}
dev->irq = temp; /* Restore IOAPIC IRQ */
}
}
void pci_no_msi(void)
{
pci_msi_enable = 0;
}
EXPORT_SYMBOL(pci_enable_msi);
EXPORT_SYMBOL(pci_disable_msi);
EXPORT_SYMBOL(pci_enable_msix);
EXPORT_SYMBOL(pci_disable_msix);