Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-06 03:51:48 +00:00
Code Issues Packages Projects Releases Wiki Activity
1517a3f21a
linux/drivers/ntb/ntb_hw.c
Jon Mason 1517a3f21a NTB: Correct debugfs to work with more than 1 NTB Device 
Debugfs was setup in NTB to only have a single debugfs directory.  This
resulted in the leaking of debugfs directories and files when multiple
NTB devices were present, due to each device stomping on the variables
containing the previous device's values (thus preventing them from being
freed on cleanup).  Correct this by creating a secondary directory of
the PCI BDF for each device present, and nesting the previously existing
information in those directories.

Signed-off-by: Jon Mason <jon.mason@intel.com>
2013-09-03 11:14:54 -07:00

1175 lines
30 KiB
C
Raw Blame History

/*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright(c) 2012 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* BSD LICENSE
*
* Copyright(c) 2012 Intel Corporation. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copy
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* Intel PCIe NTB Linux driver
*
* Contact Information:
* Jon Mason <jon.mason@intel.com>
*/
#include <linux/debugfs.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include "ntb_hw.h"
#include "ntb_regs.h"
#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver"
#define NTB_VER "0.25"
MODULE_DESCRIPTION(NTB_NAME);
MODULE_VERSION(NTB_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");
enum {
NTB_CONN_CLASSIC = 0,
NTB_CONN_B2B,
NTB_CONN_RP,
};
enum {
NTB_DEV_USD = 0,
NTB_DEV_DSD,
};
enum {
SNB_HW = 0,
BWD_HW,
};
static struct dentry *debugfs_dir;
/* Translate memory window 0,1 to BAR 2,4 */
#define MW_TO_BAR(mw) (mw * 2 + 2)
static DEFINE_PCI_DEVICE_TABLE(ntb_pci_tbl) = {
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_CLASSIC_JSF)},
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_RP_JSF)},
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_RP_SNB)},
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_CLASSIC_SNB)},
{0}
};
MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
/**
* ntb_register_event_callback() - register event callback
* @ndev: pointer to ntb_device instance
* @func: callback function to register
*
* This function registers a callback for any HW driver events such as link
* up/down, power management notices and etc.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int ntb_register_event_callback(struct ntb_device *ndev,
void (*func)(void *handle, enum ntb_hw_event event))
{
if (ndev->event_cb)
return -EINVAL;
ndev->event_cb = func;
return 0;
}
/**
* ntb_unregister_event_callback() - unregisters the event callback
* @ndev: pointer to ntb_device instance
*
* This function unregisters the existing callback from transport
*/
void ntb_unregister_event_callback(struct ntb_device *ndev)
{
ndev->event_cb = NULL;
}
/**
* ntb_register_db_callback() - register a callback for doorbell interrupt
* @ndev: pointer to ntb_device instance
* @idx: doorbell index to register callback, zero based
* @func: callback function to register
*
* This function registers a callback function for the doorbell interrupt
* on the primary side. The function will unmask the doorbell as well to
* allow interrupt.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
void *data, void (*func)(void *data, int db_num))
{
unsigned long mask;
if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
return -EINVAL;
}
ndev->db_cb[idx].callback = func;
ndev->db_cb[idx].data = data;
/* unmask interrupt */
mask = readw(ndev->reg_ofs.pdb_mask);
clear_bit(idx * ndev->bits_per_vector, &mask);
writew(mask, ndev->reg_ofs.pdb_mask);
return 0;
}
/**
* ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
* @ndev: pointer to ntb_device instance
* @idx: doorbell index to register callback, zero based
*
* This function unregisters a callback function for the doorbell interrupt
* on the primary side. The function will also mask the said doorbell.
*/
void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
{
unsigned long mask;
if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
return;
mask = readw(ndev->reg_ofs.pdb_mask);
set_bit(idx * ndev->bits_per_vector, &mask);
writew(mask, ndev->reg_ofs.pdb_mask);
ndev->db_cb[idx].callback = NULL;
}
/**
* ntb_find_transport() - find the transport pointer
* @transport: pointer to pci device
*
* Given the pci device pointer, return the transport pointer passed in when
* the transport attached when it was inited.
*
* RETURNS: pointer to transport.
*/
void *ntb_find_transport(struct pci_dev *pdev)
{
struct ntb_device *ndev = pci_get_drvdata(pdev);
return ndev->ntb_transport;
}
/**
* ntb_register_transport() - Register NTB transport with NTB HW driver
* @transport: transport identifier
*
* This function allows a transport to reserve the hardware driver for
* NTB usage.
*
* RETURNS: pointer to ntb_device, NULL on error.
*/
struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
{
struct ntb_device *ndev = pci_get_drvdata(pdev);
if (ndev->ntb_transport)
return NULL;
ndev->ntb_transport = transport;
return ndev;
}
/**
* ntb_unregister_transport() - Unregister the transport with the NTB HW driver
* @ndev - ntb_device of the transport to be freed
*
* This function unregisters the transport from the HW driver and performs any
* necessary cleanups.
*/
void ntb_unregister_transport(struct ntb_device *ndev)
{
int i;
if (!ndev->ntb_transport)
return;
for (i = 0; i < ndev->max_cbs; i++)
ntb_unregister_db_callback(ndev, i);
ntb_unregister_event_callback(ndev);
ndev->ntb_transport = NULL;
}
/**
* ntb_write_local_spad() - write to the secondary scratchpad register
* @ndev: pointer to ntb_device instance
* @idx: index to the scratchpad register, 0 based
* @val: the data value to put into the register
*
* This function allows writing of a 32bit value to the indexed scratchpad
* register. This writes over the data mirrored to the local scratchpad register
* by the remote system.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
{
if (idx >= ndev->limits.max_spads)
return -EINVAL;
dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
val, idx);
writel(val, ndev->reg_ofs.spad_read + idx * 4);
return 0;
}
/**
* ntb_read_local_spad() - read from the primary scratchpad register
* @ndev: pointer to ntb_device instance
* @idx: index to scratchpad register, 0 based
* @val: pointer to 32bit integer for storing the register value
*
* This function allows reading of the 32bit scratchpad register on
* the primary (internal) side. This allows the local system to read data
* written and mirrored to the scratchpad register by the remote system.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
{
if (idx >= ndev->limits.max_spads)
return -EINVAL;
*val = readl(ndev->reg_ofs.spad_write + idx * 4);
dev_dbg(&ndev->pdev->dev,
"Reading %x from local scratch pad index %d\n", *val, idx);
return 0;
}
/**
* ntb_write_remote_spad() - write to the secondary scratchpad register
* @ndev: pointer to ntb_device instance
* @idx: index to the scratchpad register, 0 based
* @val: the data value to put into the register
*
* This function allows writing of a 32bit value to the indexed scratchpad
* register. The register resides on the secondary (external) side. This allows
* the local system to write data to be mirrored to the remote systems
* scratchpad register.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
{
if (idx >= ndev->limits.max_spads)
return -EINVAL;
dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
val, idx);
writel(val, ndev->reg_ofs.spad_write + idx * 4);
return 0;
}
/**
* ntb_read_remote_spad() - read from the primary scratchpad register
* @ndev: pointer to ntb_device instance
* @idx: index to scratchpad register, 0 based
* @val: pointer to 32bit integer for storing the register value
*
* This function allows reading of the 32bit scratchpad register on
* the primary (internal) side. This alloows the local system to read the data
* it wrote to be mirrored on the remote system.
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
{
if (idx >= ndev->limits.max_spads)
return -EINVAL;
*val = readl(ndev->reg_ofs.spad_read + idx * 4);
dev_dbg(&ndev->pdev->dev,
"Reading %x from remote scratch pad index %d\n", *val, idx);
return 0;
}
/**
* ntb_get_mw_vbase() - get virtual addr for the NTB memory window
* @ndev: pointer to ntb_device instance
* @mw: memory window number
*
* This function provides the base virtual address of the memory window
* specified.
*
* RETURNS: pointer to virtual address, or NULL on error.
*/
void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
{
if (mw >= NTB_NUM_MW)
return NULL;
return ndev->mw[mw].vbase;
}
/**
* ntb_get_mw_size() - return size of NTB memory window
* @ndev: pointer to ntb_device instance
* @mw: memory window number
*
* This function provides the physical size of the memory window specified
*
* RETURNS: the size of the memory window or zero on error
*/
resource_size_t ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
{
if (mw >= NTB_NUM_MW)
return 0;
return ndev->mw[mw].bar_sz;
}
/**
* ntb_set_mw_addr - set the memory window address
* @ndev: pointer to ntb_device instance
* @mw: memory window number
* @addr: base address for data
*
* This function sets the base physical address of the memory window. This
* memory address is where data from the remote system will be transfered into
* or out of depending on how the transport is configured.
*/
void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
{
if (mw >= NTB_NUM_MW)
return;
dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
MW_TO_BAR(mw));
ndev->mw[mw].phys_addr = addr;
switch (MW_TO_BAR(mw)) {
case NTB_BAR_23:
writeq(addr, ndev->reg_ofs.sbar2_xlat);
break;
case NTB_BAR_45:
writeq(addr, ndev->reg_ofs.sbar4_xlat);
break;
}
}
/**
* ntb_ring_sdb() - Set the doorbell on the secondary/external side
* @ndev: pointer to ntb_device instance
* @db: doorbell to ring
*
* This function allows triggering of a doorbell on the secondary/external
* side that will initiate an interrupt on the remote host
*
* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
*/
void ntb_ring_sdb(struct ntb_device *ndev, unsigned int db)
{
dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
if (ndev->hw_type == BWD_HW)
writeq((u64) 1 << db, ndev->reg_ofs.sdb);
else
writew(((1 << ndev->bits_per_vector) - 1) <<
(db * ndev->bits_per_vector), ndev->reg_ofs.sdb);
}
static void ntb_link_event(struct ntb_device *ndev, int link_state)
{
unsigned int event;
if (ndev->link_status == link_state)
return;
if (link_state == NTB_LINK_UP) {
u16 status;
dev_info(&ndev->pdev->dev, "Link Up\n");
ndev->link_status = NTB_LINK_UP;
event = NTB_EVENT_HW_LINK_UP;
if (ndev->hw_type == BWD_HW)
status = readw(ndev->reg_ofs.lnk_stat);
else {
int rc = pci_read_config_word(ndev->pdev,
SNB_LINK_STATUS_OFFSET,
&status);
if (rc)
return;
}
dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
(status & NTB_LINK_WIDTH_MASK) >> 4,
(status & NTB_LINK_SPEED_MASK));
} else {
dev_info(&ndev->pdev->dev, "Link Down\n");
ndev->link_status = NTB_LINK_DOWN;
event = NTB_EVENT_HW_LINK_DOWN;
}
/* notify the upper layer if we have an event change */
if (ndev->event_cb)
ndev->event_cb(ndev->ntb_transport, event);
}
static int ntb_link_status(struct ntb_device *ndev)
{
int link_state;
if (ndev->hw_type == BWD_HW) {
u32 ntb_cntl;
ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
if (ntb_cntl & BWD_CNTL_LINK_DOWN)
link_state = NTB_LINK_DOWN;
else
link_state = NTB_LINK_UP;
} else {
u16 status;
int rc;
rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
&status);
if (rc)
return rc;
if (status & NTB_LINK_STATUS_ACTIVE)
link_state = NTB_LINK_UP;
else
link_state = NTB_LINK_DOWN;
}
ntb_link_event(ndev, link_state);
return 0;
}
/* BWD doesn't have link status interrupt, poll on that platform */
static void bwd_link_poll(struct work_struct *work)
{
struct ntb_device *ndev = container_of(work, struct ntb_device,
hb_timer.work);
unsigned long ts = jiffies;
/* If we haven't gotten an interrupt in a while, check the BWD link
* status bit
*/
if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
int rc = ntb_link_status(ndev);
if (rc)
dev_err(&ndev->pdev->dev,
"Error determining link status\n");
}
schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
}
static int ntb_xeon_setup(struct ntb_device *ndev)
{
int rc;
u8 val;
ndev->hw_type = SNB_HW;
rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val);
if (rc)
return rc;
switch (val & SNB_PPD_CONN_TYPE) {
case NTB_CONN_B2B:
ndev->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_CLASSIC:
case NTB_CONN_RP:
default:
dev_err(&ndev->pdev->dev, "Only B2B supported at this time\n");
return -EINVAL;
}
if (val & SNB_PPD_DEV_TYPE)
ndev->dev_type = NTB_DEV_USD;
else
ndev->dev_type = NTB_DEV_DSD;
ndev->reg_ofs.pdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
ndev->reg_ofs.pdb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
ndev->reg_ofs.sbar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
ndev->reg_ofs.sbar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_LINK_STATUS_OFFSET;
ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
if (ndev->conn_type == NTB_CONN_B2B) {
ndev->reg_ofs.sdb = ndev->reg_base + SNB_B2B_DOORBELL_OFFSET;
ndev->reg_ofs.spad_write = ndev->reg_base + SNB_B2B_SPAD_OFFSET;
ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
} else {
ndev->reg_ofs.sdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS;
}
ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
ndev->limits.msix_cnt = SNB_MSIX_CNT;
ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
return 0;
}
static int ntb_bwd_setup(struct ntb_device *ndev)
{
int rc;
u32 val;
ndev->hw_type = BWD_HW;
rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
if (rc)
return rc;
switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
case NTB_CONN_B2B:
ndev->conn_type = NTB_CONN_B2B;
break;
case NTB_CONN_RP:
default:
dev_err(&ndev->pdev->dev, "Only B2B supported at this time\n");
return -EINVAL;
}
if (val & BWD_PPD_DEV_TYPE)
ndev->dev_type = NTB_DEV_DSD;
else
ndev->dev_type = NTB_DEV_USD;
/* Initiate PCI-E link training */
rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
val | BWD_PPD_INIT_LINK);
if (rc)
return rc;
ndev->reg_ofs.pdb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
ndev->reg_ofs.pdb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
ndev->reg_ofs.sbar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
ndev->reg_ofs.sbar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
if (ndev->conn_type == NTB_CONN_B2B) {
ndev->reg_ofs.sdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
ndev->limits.max_spads = BWD_MAX_SPADS;
} else {
ndev->reg_ofs.sdb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
ndev->reg_ofs.spad_write = ndev->reg_base + BWD_SPAD_OFFSET;
ndev->limits.max_spads = BWD_MAX_COMPAT_SPADS;
}
ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
ndev->limits.msix_cnt = BWD_MSIX_CNT;
ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
/* Since bwd doesn't have a link interrupt, setup a poll timer */
INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
return 0;
}
static int ntb_device_setup(struct ntb_device *ndev)
{
int rc;
switch (ndev->pdev->device) {
case PCI_DEVICE_ID_INTEL_NTB_2ND_SNB:
case PCI_DEVICE_ID_INTEL_NTB_RP_JSF:
case PCI_DEVICE_ID_INTEL_NTB_RP_SNB:
case PCI_DEVICE_ID_INTEL_NTB_CLASSIC_JSF:
case PCI_DEVICE_ID_INTEL_NTB_CLASSIC_SNB:
case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
rc = ntb_xeon_setup(ndev);
break;
case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
rc = ntb_bwd_setup(ndev);
break;
default:
rc = -ENODEV;
}
if (rc)
return rc;
dev_info(&ndev->pdev->dev, "Device Type = %s\n",
ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
/* Enable Bus Master and Memory Space on the secondary side */
writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, ndev->reg_ofs.spci_cmd);
return 0;
}
static void ntb_device_free(struct ntb_device *ndev)
{
if (ndev->hw_type == BWD_HW)
cancel_delayed_work_sync(&ndev->hb_timer);
}
static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
{
struct ntb_db_cb *db_cb = data;
struct ntb_device *ndev = db_cb->ndev;
dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
db_cb->db_num);
if (db_cb->callback)
db_cb->callback(db_cb->data, db_cb->db_num);
/* No need to check for the specific HB irq, any interrupt means
* we're connected.
*/
ndev->last_ts = jiffies;
writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.pdb);
return IRQ_HANDLED;
}
static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
{
struct ntb_db_cb *db_cb = data;
struct ntb_device *ndev = db_cb->ndev;
dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
db_cb->db_num);
if (db_cb->callback)
db_cb->callback(db_cb->data, db_cb->db_num);
/* On Sandybridge, there are 16 bits in the interrupt register
* but only 4 vectors. So, 5 bits are assigned to the first 3
* vectors, with the 4th having a single bit for link
* interrupts.
*/
writew(((1 << ndev->bits_per_vector) - 1) <<
(db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.pdb);
return IRQ_HANDLED;
}
/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
{
struct ntb_device *ndev = dev;
int rc;
dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
rc = ntb_link_status(ndev);
if (rc)
dev_err(&ndev->pdev->dev, "Error determining link status\n");
/* bit 15 is always the link bit */
writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.pdb);
return IRQ_HANDLED;
}
static irqreturn_t ntb_interrupt(int irq, void *dev)
{
struct ntb_device *ndev = dev;
unsigned int i = 0;
if (ndev->hw_type == BWD_HW) {
u64 pdb = readq(ndev->reg_ofs.pdb);
dev_dbg(&ndev->pdev->dev, "irq %d - pdb = %Lx\n", irq, pdb);
while (pdb) {
i = __ffs(pdb);
pdb &= pdb - 1;
bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
}
} else {
u16 pdb = readw(ndev->reg_ofs.pdb);
dev_dbg(&ndev->pdev->dev, "irq %d - pdb = %x sdb %x\n", irq,
pdb, readw(ndev->reg_ofs.sdb));
if (pdb & SNB_DB_HW_LINK) {
xeon_event_msix_irq(irq, dev);
pdb &= ~SNB_DB_HW_LINK;
}
while (pdb) {
i = __ffs(pdb);
pdb &= pdb - 1;
xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
}
}
return IRQ_HANDLED;
}
static int ntb_setup_msix(struct ntb_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
struct msix_entry *msix;
int msix_entries;
int rc, i, pos;
u16 val;
pos = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
if (!pos) {
rc = -EIO;
goto err;
}
rc = pci_read_config_word(pdev, pos + PCI_MSIX_FLAGS, &val);
if (rc)
goto err;
msix_entries = msix_table_size(val);
if (msix_entries > ndev->limits.msix_cnt) {
rc = -EINVAL;
goto err;
}
ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
GFP_KERNEL);
if (!ndev->msix_entries) {
rc = -ENOMEM;
goto err;
}
for (i = 0; i < msix_entries; i++)
ndev->msix_entries[i].entry = i;
rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
if (rc < 0)
goto err1;
if (rc > 0) {
/* On SNB, the link interrupt is always tied to 4th vector. If
* we can't get all 4, then we can't use MSI-X.
*/
if (ndev->hw_type != BWD_HW) {
rc = -EIO;
goto err1;
}
dev_warn(&pdev->dev,
"Only %d MSI-X vectors. Limiting the number of queues to that number.\n",
rc);
msix_entries = rc;
}
for (i = 0; i < msix_entries; i++) {
msix = &ndev->msix_entries[i];
WARN_ON(!msix->vector);
/* Use the last MSI-X vector for Link status */
if (ndev->hw_type == BWD_HW) {
rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
"ntb-callback-msix", &ndev->db_cb[i]);
if (rc)
goto err2;
} else {
if (i == msix_entries - 1) {
rc = request_irq(msix->vector,
xeon_event_msix_irq, 0,
"ntb-event-msix", ndev);
if (rc)
goto err2;
} else {
rc = request_irq(msix->vector,
xeon_callback_msix_irq, 0,
"ntb-callback-msix",
&ndev->db_cb[i]);
if (rc)
goto err2;
}
}
}
ndev->num_msix = msix_entries;
if (ndev->hw_type == BWD_HW)
ndev->max_cbs = msix_entries;
else
ndev->max_cbs = msix_entries - 1;
return 0;
err2:
while (--i >= 0) {
msix = &ndev->msix_entries[i];
if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
free_irq(msix->vector, ndev);
else
free_irq(msix->vector, &ndev->db_cb[i]);
}
pci_disable_msix(pdev);
err1:
kfree(ndev->msix_entries);
dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
err:
ndev->num_msix = 0;
return rc;
}
static int ntb_setup_msi(struct ntb_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
int rc;
rc = pci_enable_msi(pdev);
if (rc)
return rc;
rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
if (rc) {
pci_disable_msi(pdev);
dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
return rc;
}
return 0;
}
static int ntb_setup_intx(struct ntb_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
int rc;
pci_msi_off(pdev);
/* Verify intx is enabled */
pci_intx(pdev, 1);
rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
ndev);
if (rc)
return rc;
return 0;
}
static int ntb_setup_interrupts(struct ntb_device *ndev)
{
int rc;
/* On BWD, disable all interrupts. On SNB, disable all but Link
* Interrupt. The rest will be unmasked as callbacks are registered.
*/
if (ndev->hw_type == BWD_HW)
writeq(~0, ndev->reg_ofs.pdb_mask);
else
writew(~(1 << ndev->limits.max_db_bits),
ndev->reg_ofs.pdb_mask);
rc = ntb_setup_msix(ndev);
if (!rc)
goto done;
ndev->bits_per_vector = 1;
ndev->max_cbs = ndev->limits.max_db_bits;
rc = ntb_setup_msi(ndev);
if (!rc)
goto done;
rc = ntb_setup_intx(ndev);
if (rc) {
dev_err(&ndev->pdev->dev, "no usable interrupts\n");
return rc;
}
done:
return 0;
}
static void ntb_free_interrupts(struct ntb_device *ndev)
{
struct pci_dev *pdev = ndev->pdev;
/* mask interrupts */
if (ndev->hw_type == BWD_HW)
writeq(~0, ndev->reg_ofs.pdb_mask);
else
writew(~0, ndev->reg_ofs.pdb_mask);
if (ndev->num_msix) {
struct msix_entry *msix;
u32 i;
for (i = 0; i < ndev->num_msix; i++) {
msix = &ndev->msix_entries[i];
if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
free_irq(msix->vector, ndev);
else
free_irq(msix->vector, &ndev->db_cb[i]);
}
pci_disable_msix(pdev);
} else {
free_irq(pdev->irq, ndev);
if (pci_dev_msi_enabled(pdev))
pci_disable_msi(pdev);
}
}
static int ntb_create_callbacks(struct ntb_device *ndev)
{
int i;
/* Checken-egg issue. We won't know how many callbacks are necessary
* until we see how many MSI-X vectors we get, but these pointers need
* to be passed into the MSI-X register fucntion. So, we allocate the
* max, knowing that they might not all be used, to work around this.
*/
ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
sizeof(struct ntb_db_cb),
GFP_KERNEL);
if (!ndev->db_cb)
return -ENOMEM;
for (i = 0; i < ndev->limits.max_db_bits; i++) {
ndev->db_cb[i].db_num = i;
ndev->db_cb[i].ndev = ndev;
}
return 0;
}
static void ntb_free_callbacks(struct ntb_device *ndev)
{
int i;
for (i = 0; i < ndev->limits.max_db_bits; i++)
ntb_unregister_db_callback(ndev, i);
kfree(ndev->db_cb);
}
static void ntb_setup_debugfs(struct ntb_device *ndev)
{
if (!debugfs_initialized())
return;
if (!debugfs_dir)
debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
debugfs_dir);
}
static void ntb_free_debugfs(struct ntb_device *ndev)
{
debugfs_remove_recursive(ndev->debugfs_dir);
if (debugfs_dir && simple_empty(debugfs_dir)) {
debugfs_remove_recursive(debugfs_dir);
debugfs_dir = NULL;
}
}
static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct ntb_device *ndev;
int rc, i;
ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
if (!ndev)
return -ENOMEM;
ndev->pdev = pdev;
ndev->link_status = NTB_LINK_DOWN;
pci_set_drvdata(pdev, ndev);
ntb_setup_debugfs(ndev);
rc = pci_enable_device(pdev);
if (rc)
goto err;
pci_set_master(ndev->pdev);
rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
if (rc)
goto err1;
ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
if (!ndev->reg_base) {
dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
rc = -EIO;
goto err2;
}
for (i = 0; i < NTB_NUM_MW; i++) {
ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
ndev->mw[i].vbase =
ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
ndev->mw[i].bar_sz);
dev_info(&pdev->dev, "MW %d size %llu\n", i,
pci_resource_len(pdev, MW_TO_BAR(i)));
if (!ndev->mw[i].vbase) {
dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
MW_TO_BAR(i));
rc = -EIO;
goto err3;
}
}
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc) {
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
goto err3;
dev_warn(&pdev->dev, "Cannot DMA highmem\n");
}
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc) {
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
goto err3;
dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
}
rc = ntb_device_setup(ndev);
if (rc)
goto err3;
rc = ntb_create_callbacks(ndev);
if (rc)
goto err4;
rc = ntb_setup_interrupts(ndev);
if (rc)
goto err5;
/* The scratchpad registers keep the values between rmmod/insmod,
* blast them now
*/
for (i = 0; i < ndev->limits.max_spads; i++) {
ntb_write_local_spad(ndev, i, 0);
ntb_write_remote_spad(ndev, i, 0);
}
rc = ntb_transport_init(pdev);
if (rc)
goto err6;
/* Let's bring the NTB link up */
writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,
ndev->reg_ofs.lnk_cntl);
return 0;
err6:
ntb_free_interrupts(ndev);
err5:
ntb_free_callbacks(ndev);
err4:
ntb_device_free(ndev);
err3:
for (i--; i >= 0; i--)
iounmap(ndev->mw[i].vbase);
iounmap(ndev->reg_base);
err2:
pci_release_selected_regions(pdev, NTB_BAR_MASK);
err1:
pci_disable_device(pdev);
err:
ntb_free_debugfs(ndev);
kfree(ndev);
dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
return rc;
}
static void ntb_pci_remove(struct pci_dev *pdev)
{
struct ntb_device *ndev = pci_get_drvdata(pdev);
int i;
u32 ntb_cntl;
/* Bring NTB link down */
ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
ntb_cntl |= NTB_LINK_DISABLE;
writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
ntb_transport_free(ndev->ntb_transport);
ntb_free_interrupts(ndev);
ntb_free_callbacks(ndev);
ntb_device_free(ndev);
for (i = 0; i < NTB_NUM_MW; i++)
iounmap(ndev->mw[i].vbase);
iounmap(ndev->reg_base);
pci_release_selected_regions(pdev, NTB_BAR_MASK);
pci_disable_device(pdev);
ntb_free_debugfs(ndev);
kfree(ndev);
}
static struct pci_driver ntb_pci_driver = {
.name = KBUILD_MODNAME,
.id_table = ntb_pci_tbl,
.probe = ntb_pci_probe,
.remove = ntb_pci_remove,
};
module_pci_driver(ntb_pci_driver);
Reference in New Issue View Git Blame Copy Permalink