Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-25 13:41:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

dmaengine updates for v5.13-rc1

Browse Source 
New drivers/devices
  - Support for QCOM SM8150 GPI DMA
 
 Updates:
  - Big pile of idxd updates including support for performance monitoring
  - Support in dw-edma for interleaved dma
  - Support for synchronize() in Xilinx driver
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+vs47OPLdNbVcHzyfBQHDyUjg0cFAmCRd8UACgkQfBQHDyUj
 g0da8w/+L0o/qmwIYr2WHLIX8fXNSJkVu001p+eqN7UcSy4DBym4YEeo66jYzMHu
 lJV9Wa0LC1Yzi0CTwP/bDMMVeT2NpTquHyat4SB8deTI9H4RiAoEX2hogjPYYZ14
 ZCCjSpyHjF6VomFu7yyluXPe3s1cCopeiSDMDrHBfTYWhH0SSya6ObGcCdqEV1SO
 p2MwW+5mTLjYVMcWTV8tuRS67MVf2tUPT+gvmX8KY0bEeqL+hpzTKDEAHOSW8p9D
 PiyKX0bPwfXupXiYmbkQlSEH8+qwarrLNPFU/uxXAym5vsTxP2D3eoeKp/9U/H6H
 nOuueFod+7LDgI5fe+BpOXW98G0mnzX/anPLMUInCbkc4JPLdHvnakQ7kxM7EPn3
 hMi8DCPv2Ft/cc14KLT1mgnL2+SawVHigyVcSK1YFq2vlzy+m7tbEHXpiGUDlY8h
 bwG6gCafN7D8U33vtipQtMmwgRGBXgytUPFq8J73tw+DuHTZqP2eZUQuqNfRPXa6
 4cmWAbIn4JLVBxlwADfhMJNdeBEgHqkl2aWZPcoQmKOiBtnOd+tAL5Hb7EQWqyhB
 J1cVkYyCGASVxrTTiK3s1gcqJ0lsjFqon+OA4V03GO0yHqkK+LTd3RsubKdp7y7R
 db3ab0C0uIH9oc9NmqShN6j9aaQIiEWtQTBlJju/ObLaMfV3mGk=
 =Jjb4
 -----END PGP SIGNATURE-----

Merge tag 'dmaengine-5.13-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine

Pull dmaengine updates from Vinod Koul:
 "New drivers/devices:

   - Support for QCOM SM8150 GPI DMA

  Updates:

   - Big pile of idxd updates including support for performance
     monitoring

   - Support in dw-edma for interleaved dma

   - Support for synchronize() in Xilinx driver"

* tag 'dmaengine-5.13-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (42 commits)
  dmaengine: idxd: Enable IDXD performance monitor support
  dmaengine: idxd: Add IDXD performance monitor support
  dmaengine: idxd: remove MSIX masking for interrupt handlers
  dmaengine: idxd: device cmd should use dedicated lock
  dmaengine: idxd: support reporting of halt interrupt
  dmaengine: idxd: enable SVA feature for IOMMU
  dmaengine: idxd: convert sprintf() to sysfs_emit() for all usages
  dmaengine: idxd: add interrupt handle request and release support
  dmaengine: idxd: add support for readonly config mode
  dmaengine: idxd: add percpu_ref to descriptor submission path
  dmaengine: idxd: remove detection of device type
  dmaengine: idxd: iax bus removal
  dmaengine: idxd: fix cdev setup and free device lifetime issues
  dmaengine: idxd: fix group conf_dev lifetime
  dmaengine: idxd: fix engine conf_dev lifetime
  dmaengine: idxd: fix wq conf_dev 'struct device' lifetime
  dmaengine: idxd: fix idxd conf_dev 'struct device' lifetime
  dmaengine: idxd: use ida for device instance enumeration
  dmaengine: idxd: removal of pcim managed mmio mapping
  dmaengine: idxd: cleanup pci interrupt vector allocation management
  ...
This commit is contained in:
Linus Torvalds 2021-05-04 11:24:46 -07:00
commit e4adffb8da
31 changed files with 3057 additions and 1163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
Documentation/ABI/testing/sysfs-bus-event_source-devices-dsa Normal file
View File

@ -0,0 +1,30 @@
What: /sys/bus/event_source/devices/dsa*/format
Date: April 2021
KernelVersion: 5.13
Contact: Tom Zanussi <tom.zanussi@linux.intel.com>
Description: Read-only. Attribute group to describe the magic bits
that go into perf_event_attr.config or
perf_event_attr.config1 for the IDXD DSA pmu. (See also
ABI/testing/sysfs-bus-event_source-devices-format).
Each attribute in this group defines a bit range in
perf_event_attr.config or perf_event_attr.config1.
All supported attributes are listed below (See the
IDXD DSA Spec for possible attribute values)::
event_category = "config:0-3" - event category
event = "config:4-31" - event ID
filter_wq = "config1:0-31" - workqueue filter
filter_tc = "config1:32-39" - traffic class filter
filter_pgsz = "config1:40-43" - page size filter
filter_sz = "config1:44-51" - transfer size filter
filter_eng = "config1:52-59" - engine filter
What: /sys/bus/event_source/devices/dsa*/cpumask
Date: April 2021
KernelVersion: 5.13
Contact: Tom Zanussi <tom.zanussi@linux.intel.com>
Description: Read-only. This file always returns the cpu to which the
IDXD DSA pmu is bound for access to all dsa pmu
performance monitoring events.

1
Documentation/devicetree/bindings/dma/qcom,gpi.yaml
View File

@ -20,6 +20,7 @@ properties:
compatible:
enum:
- qcom,sdm845-gpi-dma
- qcom,sm8150-gpi-dma
reg:
maxItems: 1

12
drivers/dma/Kconfig
View File

@ -300,6 +300,18 @@ config INTEL_IDXD_SVM
depends on PCI_PASID
depends on PCI_IOV
config INTEL_IDXD_PERFMON
bool "Intel Data Accelerators performance monitor support"
depends on INTEL_IDXD
help
Enable performance monitor (pmu) support for the Intel(R)
data accelerators present in Intel Xeon CPU. With this
enabled, perf can be used to monitor the DSA (Intel Data
Streaming Accelerator) events described in the Intel DSA
spec.
If unsure, say N.
config INTEL_IOATDMA
tristate "Intel I/OAT DMA support"
depends on PCI && X86_64

11
drivers/dma/at_xdmac.c
View File

@ -344,17 +344,6 @@ static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)
return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
}
static inline int at_xdmac_csize(u32 maxburst)
{
int csize;
csize = ffs(maxburst) - 1;
if (csize > 4)
csize = -EINVAL;
return csize;
};
static inline bool at_xdmac_chan_is_peripheral_xfer(u32 cfg)
{
return cfg & AT_XDMAC_CC_TYPE_PER_TRAN;

192
drivers/dma/dw-edma/dw-edma-core.c
View File

@ -81,8 +81,13 @@ static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
* - Even chunks originate CB equal to 1
*/
chunk->cb = !(desc->chunks_alloc % 2);
chunk->ll_region.paddr = dw->ll_region.paddr + chan->ll_off;
chunk->ll_region.vaddr = dw->ll_region.vaddr + chan->ll_off;
if (chan->dir == EDMA_DIR_WRITE) {
chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr;
chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr;
} else {
chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr;
chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr;
}
if (desc->chunk) {
/* Create and add new element into the linked list */
@ -329,22 +334,22 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
struct dw_edma_chunk *chunk;
struct dw_edma_burst *burst;
struct dw_edma_desc *desc;
u32 cnt;
u32 cnt = 0;
int i;
if (!chan->configured)
return NULL;
switch (chan->config.direction) {
case DMA_DEV_TO_MEM: /* local dma */
case DMA_DEV_TO_MEM: /* local DMA */
if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ)
break;
return NULL;
case DMA_MEM_TO_DEV: /* local dma */
case DMA_MEM_TO_DEV: /* local DMA */
if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE)
break;
return NULL;
default: /* remote dma */
default: /* remote DMA */
if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ)
break;
if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE)
@ -352,12 +357,19 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
return NULL;
}
if (xfer->cyclic) {
if (xfer->type == EDMA_XFER_CYCLIC) {
if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
return NULL;
} else {
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
if (xfer->xfer.sg.len < 1)
return NULL;
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
if (!xfer->xfer.il->numf)
return NULL;
if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0)
return NULL;
} else {
return NULL;
}
desc = dw_edma_alloc_desc(chan);
@ -368,18 +380,28 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
if (unlikely(!chunk))
goto err_alloc;
src_addr = chan->config.src_addr;
dst_addr = chan->config.dst_addr;
if (xfer->cyclic) {
cnt = xfer->xfer.cyclic.cnt;
if (xfer->type == EDMA_XFER_INTERLEAVED) {
src_addr = xfer->xfer.il->src_start;
dst_addr = xfer->xfer.il->dst_start;
} else {
src_addr = chan->config.src_addr;
dst_addr = chan->config.dst_addr;
}
if (xfer->type == EDMA_XFER_CYCLIC) {
cnt = xfer->xfer.cyclic.cnt;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
cnt = xfer->xfer.sg.len;
sg = xfer->xfer.sg.sgl;
} else if (xfer->type == EDMA_XFER_INTERLEAVED) {
if (xfer->xfer.il->numf > 0)
cnt = xfer->xfer.il->numf;
else
cnt = xfer->xfer.il->frame_size;
}
for (i = 0; i < cnt; i++) {
if (!xfer->cyclic && !sg)
if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
break;
if (chunk->bursts_alloc == chan->ll_max) {
@ -392,20 +414,23 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
if (unlikely(!burst))
goto err_alloc;
if (xfer->cyclic)
if (xfer->type == EDMA_XFER_CYCLIC)
burst->sz = xfer->xfer.cyclic.len;
else
else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
burst->sz = sg_dma_len(sg);
else if (xfer->type == EDMA_XFER_INTERLEAVED)
burst->sz = xfer->xfer.il->sgl[i].size;
chunk->ll_region.sz += burst->sz;
desc->alloc_sz += burst->sz;
if (chan->dir == EDMA_DIR_WRITE) {
burst->sar = src_addr;
if (xfer->cyclic) {
if (xfer->type == EDMA_XFER_CYCLIC) {
burst->dar = xfer->xfer.cyclic.paddr;
} else {
burst->dar = dst_addr;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
src_addr += sg_dma_len(sg);
burst->dar = sg_dma_address(sg);
/* Unlike the typical assumption by other
* drivers/IPs the peripheral memory isn't
* a FIFO memory, in this case, it's a
@ -416,10 +441,11 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
}
} else {
burst->dar = dst_addr;
if (xfer->cyclic) {
if (xfer->type == EDMA_XFER_CYCLIC) {
burst->sar = xfer->xfer.cyclic.paddr;
} else {
burst->sar = src_addr;
} else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
dst_addr += sg_dma_len(sg);
burst->sar = sg_dma_address(sg);
/* Unlike the typical assumption by other
* drivers/IPs the peripheral memory isn't
* a FIFO memory, in this case, it's a
@ -430,10 +456,22 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
}
}
if (!xfer->cyclic) {
src_addr += sg_dma_len(sg);
dst_addr += sg_dma_len(sg);
if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
sg = sg_next(sg);
} else if (xfer->type == EDMA_XFER_INTERLEAVED &&
xfer->xfer.il->frame_size > 0) {
struct dma_interleaved_template *il = xfer->xfer.il;
struct data_chunk *dc = &il->sgl[i];
if (il->src_sgl) {
src_addr += burst->sz;
src_addr += dmaengine_get_src_icg(il, dc);
}
if (il->dst_sgl) {
dst_addr += burst->sz;
dst_addr += dmaengine_get_dst_icg(il, dc);
}
}
}
@ -459,7 +497,7 @@ dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
xfer.xfer.sg.sgl = sgl;
xfer.xfer.sg.len = len;
xfer.flags = flags;
xfer.cyclic = false;
xfer.type = EDMA_XFER_SCATTER_GATHER;
return dw_edma_device_transfer(&xfer);
}
@ -478,7 +516,23 @@ dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
xfer.xfer.cyclic.len = len;
xfer.xfer.cyclic.cnt = count;
xfer.flags = flags;
xfer.cyclic = true;
xfer.type = EDMA_XFER_CYCLIC;
return dw_edma_device_transfer(&xfer);
}
static struct dma_async_tx_descriptor *
dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
struct dma_interleaved_template *ilt,
unsigned long flags)
{
struct dw_edma_transfer xfer;
xfer.dchan = dchan;
xfer.direction = ilt->dir;
xfer.xfer.il = ilt;
xfer.flags = flags;
xfer.type = EDMA_XFER_INTERLEAVED;
return dw_edma_device_transfer(&xfer);
}
@ -642,24 +696,13 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
struct device *dev = chip->dev;
struct dw_edma *dw = chip->dw;
struct dw_edma_chan *chan;
size_t ll_chunk, dt_chunk;
struct dw_edma_irq *irq;
struct dma_device *dma;
u32 i, j, cnt, ch_cnt;
u32 alloc, off_alloc;
u32 i, j, cnt;
int err = 0;
u32 pos;
ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
ll_chunk = dw->ll_region.sz;
dt_chunk = dw->dt_region.sz;
/* Calculate linked list chunk for each channel */
ll_chunk /= roundup_pow_of_two(ch_cnt);
/* Calculate linked list chunk for each channel */
dt_chunk /= roundup_pow_of_two(ch_cnt);
if (write) {
i = 0;
cnt = dw->wr_ch_cnt;
@ -691,14 +734,14 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
chan->ll_off = (ll_chunk * i);
chan->ll_max = (ll_chunk / EDMA_LL_SZ) - 1;
if (write)
chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ);
else
chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ);
chan->ll_max -= 1;
chan->dt_off = (dt_chunk * i);
dev_vdbg(dev, "L. List:\tChannel %s[%u] off=0x%.8lx, max_cnt=%u\n",
write ? "write" : "read", j,
chan->ll_off, chan->ll_max);
dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
write ? "write" : "read", j, chan->ll_max);
if (dw->nr_irqs == 1)
pos = 0;
@ -723,12 +766,15 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
chan->vc.desc_free = vchan_free_desc;
vchan_init(&chan->vc, dma);
dt_region->paddr = dw->dt_region.paddr + chan->dt_off;
dt_region->vaddr = dw->dt_region.vaddr + chan->dt_off;
dt_region->sz = dt_chunk;
dev_vdbg(dev, "Data:\tChannel %s[%u] off=0x%.8lx\n",
write ? "write" : "read", j, chan->dt_off);
if (write) {
dt_region->paddr = dw->dt_region_wr[j].paddr;
dt_region->vaddr = dw->dt_region_wr[j].vaddr;
dt_region->sz = dw->dt_region_wr[j].sz;
} else {
dt_region->paddr = dw->dt_region_rd[j].paddr;
dt_region->vaddr = dw->dt_region_rd[j].vaddr;
dt_region->sz = dw->dt_region_rd[j].sz;
}
dw_edma_v0_core_device_config(chan);
}
@ -738,6 +784,7 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
dma_cap_set(DMA_SLAVE, dma->cap_mask);
dma_cap_set(DMA_CYCLIC, dma->cap_mask);
dma_cap_set(DMA_PRIVATE, dma->cap_mask);
dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV);
dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
@ -756,6 +803,7 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write,
dma->device_tx_status = dw_edma_device_tx_status;
dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
dma_set_max_seg_size(dma->dev, U32_MAX);
@ -863,14 +911,15 @@ int dw_edma_probe(struct dw_edma_chip *chip)
raw_spin_lock_init(&dw->lock);
/* Find out how many write channels are supported by hardware */
dw->wr_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE);
if (!dw->wr_ch_cnt)
return -EINVAL;
dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt,
dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE));
dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
/* Find out how many read channels are supported by hardware */
dw->rd_ch_cnt = dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ);
if (!dw->rd_ch_cnt)
dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt,
dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ));
dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
return -EINVAL;
dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
@ -937,24 +986,23 @@ int dw_edma_remove(struct dw_edma_chip *chip)
/* Power management */
pm_runtime_disable(dev);
list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
vc.chan.device_node) {
list_del(&chan->vc.chan.device_node);
tasklet_kill(&chan->vc.task);
}
list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
vc.chan.device_node) {
list_del(&chan->vc.chan.device_node);
tasklet_kill(&chan->vc.task);
}
/* Deregister eDMA device */
dma_async_device_unregister(&dw->wr_edma);
list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels,
vc.chan.device_node) {
tasklet_kill(&chan->vc.task);
list_del(&chan->vc.chan.device_node);
}
dma_async_device_unregister(&dw->rd_edma);
list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels,
vc.chan.device_node) {
tasklet_kill(&chan->vc.task);
list_del(&chan->vc.chan.device_node);
}
/* Turn debugfs off */
dw_edma_v0_core_debugfs_off();
dw_edma_v0_core_debugfs_off(chip);
return 0;
}

37
drivers/dma/dw-edma/dw-edma-core.h
View File

@ -15,15 +15,18 @@
#include "../virt-dma.h"
#define EDMA_LL_SZ 24
#define EDMA_MAX_WR_CH 8
#define EDMA_MAX_RD_CH 8
enum dw_edma_dir {
EDMA_DIR_WRITE = 0,
EDMA_DIR_READ
};
enum dw_edma_mode {
EDMA_MODE_LEGACY = 0,
EDMA_MODE_UNROLL
enum dw_edma_map_format {
EDMA_MF_EDMA_LEGACY = 0x0,
EDMA_MF_EDMA_UNROLL = 0x1,
EDMA_MF_HDMA_COMPAT = 0x5
};
enum dw_edma_request {
@ -38,6 +41,12 @@ enum dw_edma_status {
EDMA_ST_BUSY
};
enum dw_edma_xfer_type {
EDMA_XFER_SCATTER_GATHER = 0,
EDMA_XFER_CYCLIC,
EDMA_XFER_INTERLEAVED
};
struct dw_edma_chan;
struct dw_edma_chunk;
@ -82,11 +91,8 @@ struct dw_edma_chan {
int id;
enum dw_edma_dir dir;
off_t ll_off;
u32 ll_max;
off_t dt_off;
struct msi_msg msi;
enum dw_edma_request request;
@ -117,19 +123,23 @@ struct dw_edma {
u16 rd_ch_cnt;
struct dw_edma_region rg_region; /* Registers */
struct dw_edma_region ll_region; /* Linked list */
struct dw_edma_region dt_region; /* Data */
struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH];
struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH];
struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH];
struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH];
struct dw_edma_irq *irq;
int nr_irqs;
u32 version;
enum dw_edma_mode mode;
enum dw_edma_map_format mf;
struct dw_edma_chan *chan;
const struct dw_edma_core_ops *ops;
raw_spinlock_t lock; /* Only for legacy */
#ifdef CONFIG_DEBUG_FS
struct dentry *debugfs;
#endif /* CONFIG_DEBUG_FS */
};
struct dw_edma_sg {
@ -146,12 +156,13 @@ struct dw_edma_cyclic {
struct dw_edma_transfer {
struct dma_chan *dchan;
union dw_edma_xfer {
struct dw_edma_sg sg;
struct dw_edma_cyclic cyclic;
struct dw_edma_sg sg;
struct dw_edma_cyclic cyclic;
struct dma_interleaved_template *il;
} xfer;
enum dma_transfer_direction direction;
unsigned long flags;
bool cyclic;
enum dw_edma_xfer_type type;
};
static inline

277
drivers/dma/dw-edma/dw-edma-pcie.c
View File

@ -13,45 +13,81 @@
#include <linux/dma/edma.h>
#include <linux/pci-epf.h>
#include <linux/msi.h>
#include <linux/bitfield.h>
#include "dw-edma-core.h"
#define DW_PCIE_VSEC_DMA_ID 0x6
#define DW_PCIE_VSEC_DMA_BAR GENMASK(10, 8)
#define DW_PCIE_VSEC_DMA_MAP GENMASK(2, 0)
#define DW_PCIE_VSEC_DMA_WR_CH GENMASK(9, 0)
#define DW_PCIE_VSEC_DMA_RD_CH GENMASK(25, 16)
#define DW_BLOCK(a, b, c) \
{ \
.bar = a, \
.off = b, \
.sz = c, \
},
struct dw_edma_block {
enum pci_barno bar;
off_t off;
size_t sz;
};
struct dw_edma_pcie_data {
/* eDMA registers location */
enum pci_barno rg_bar;
off_t rg_off;
size_t rg_sz;
struct dw_edma_block rg;
/* eDMA memory linked list location */
enum pci_barno ll_bar;
off_t ll_off;
size_t ll_sz;
struct dw_edma_block ll_wr[EDMA_MAX_WR_CH];
struct dw_edma_block ll_rd[EDMA_MAX_RD_CH];
/* eDMA memory data location */
enum pci_barno dt_bar;
off_t dt_off;
size_t dt_sz;
struct dw_edma_block dt_wr[EDMA_MAX_WR_CH];
struct dw_edma_block dt_rd[EDMA_MAX_RD_CH];
/* Other */
u32 version;
enum dw_edma_mode mode;
enum dw_edma_map_format mf;
u8 irqs;
u16 wr_ch_cnt;
u16 rd_ch_cnt;
};
static const struct dw_edma_pcie_data snps_edda_data = {
/* eDMA registers location */
.rg_bar = BAR_0,
.rg_off = 0x00001000, /* 4 Kbytes */
.rg_sz = 0x00002000, /* 8 Kbytes */
.rg.bar = BAR_0,
.rg.off = 0x00001000, /* 4 Kbytes */
.rg.sz = 0x00002000, /* 8 Kbytes */
/* eDMA memory linked list location */
.ll_bar = BAR_2,
.ll_off = 0x00000000, /* 0 Kbytes */
.ll_sz = 0x00800000, /* 8 Mbytes */
.ll_wr = {
/* Channel 0 - BAR 2, offset 0 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00000000, 0x00000800)
/* Channel 1 - BAR 2, offset 2 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00200000, 0x00000800)
},
.ll_rd = {
/* Channel 0 - BAR 2, offset 4 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00400000, 0x00000800)
/* Channel 1 - BAR 2, offset 6 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00600000, 0x00000800)
},
/* eDMA memory data location */
.dt_bar = BAR_2,
.dt_off = 0x00800000, /* 8 Mbytes */
.dt_sz = 0x03800000, /* 56 Mbytes */
.dt_wr = {
/* Channel 0 - BAR 2, offset 8 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00800000, 0x00000800)
/* Channel 1 - BAR 2, offset 9 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00900000, 0x00000800)
},
.dt_rd = {
/* Channel 0 - BAR 2, offset 10 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00a00000, 0x00000800)
/* Channel 1 - BAR 2, offset 11 Mbytes, size 2 Kbytes */
DW_BLOCK(BAR_2, 0x00b00000, 0x00000800)
},
/* Other */
.version = 0,
.mode = EDMA_MODE_UNROLL,
.mf = EDMA_MF_EDMA_UNROLL,
.irqs = 1,
.wr_ch_cnt = 2,
.rd_ch_cnt = 2,
};
static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr)
@ -63,14 +99,58 @@ static const struct dw_edma_core_ops dw_edma_pcie_core_ops = {
.irq_vector = dw_edma_pcie_irq_vector,
};
static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev,
struct dw_edma_pcie_data *pdata)
{
u32 val, map;
u16 vsec;
u64 off;
vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_SYNOPSYS,
DW_PCIE_VSEC_DMA_ID);
if (!vsec)
return;
pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val);
if (PCI_VNDR_HEADER_REV(val) != 0x00 ||
PCI_VNDR_HEADER_LEN(val) != 0x18)
return;
pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability DMA\n");
pci_read_config_dword(pdev, vsec + 0x8, &val);
map = FIELD_GET(DW_PCIE_VSEC_DMA_MAP, val);
if (map != EDMA_MF_EDMA_LEGACY &&
map != EDMA_MF_EDMA_UNROLL &&
map != EDMA_MF_HDMA_COMPAT)
return;
pdata->mf = map;
pdata->rg.bar = FIELD_GET(DW_PCIE_VSEC_DMA_BAR, val);
pci_read_config_dword(pdev, vsec + 0xc, &val);
pdata->wr_ch_cnt = min_t(u16, pdata->wr_ch_cnt,
FIELD_GET(DW_PCIE_VSEC_DMA_WR_CH, val));
pdata->rd_ch_cnt = min_t(u16, pdata->rd_ch_cnt,
FIELD_GET(DW_PCIE_VSEC_DMA_RD_CH, val));
pci_read_config_dword(pdev, vsec + 0x14, &val);
off = val;
pci_read_config_dword(pdev, vsec + 0x10, &val);
off <<= 32;
off |= val;
pdata->rg.off = off;
}
static int dw_edma_pcie_probe(struct pci_dev *pdev,
const struct pci_device_id *pid)
{
const struct dw_edma_pcie_data *pdata = (void *)pid->driver_data;
struct dw_edma_pcie_data *pdata = (void *)pid->driver_data;
struct dw_edma_pcie_data vsec_data;
struct device *dev = &pdev->dev;
struct dw_edma_chip *chip;
int err, nr_irqs;
struct dw_edma *dw;
int err, nr_irqs;
int i, mask;
/* Enable PCI device */
err = pcim_enable_device(pdev);
@ -79,11 +159,25 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
return err;
}
memcpy(&vsec_data, pdata, sizeof(struct dw_edma_pcie_data));
/*
* Tries to find if exists a PCIe Vendor-Specific Extended Capability
* for the DMA, if one exists, then reconfigures it.
*/
dw_edma_pcie_get_vsec_dma_data(pdev, &vsec_data);
/* Mapping PCI BAR regions */
err = pcim_iomap_regions(pdev, BIT(pdata->rg_bar) |
BIT(pdata->ll_bar) |
BIT(pdata->dt_bar),
pci_name(pdev));
mask = BIT(vsec_data.rg.bar);
for (i = 0; i < vsec_data.wr_ch_cnt; i++) {
mask |= BIT(vsec_data.ll_wr[i].bar);
mask |= BIT(vsec_data.dt_wr[i].bar);
}
for (i = 0; i < vsec_data.rd_ch_cnt; i++) {
mask |= BIT(vsec_data.ll_rd[i].bar);
mask |= BIT(vsec_data.dt_rd[i].bar);
}
err = pcim_iomap_regions(pdev, mask, pci_name(pdev));
if (err) {
pci_err(pdev, "eDMA BAR I/O remapping failed\n");
return err;
@ -125,7 +219,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
return -ENOMEM;
/* IRQs allocation */
nr_irqs = pci_alloc_irq_vectors(pdev, 1, pdata->irqs,
nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data.irqs,
PCI_IRQ_MSI | PCI_IRQ_MSIX);
if (nr_irqs < 1) {
pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n",
@ -139,46 +233,109 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
chip->id = pdev->devfn;
chip->irq = pdev->irq;
dw->rg_region.vaddr = pcim_iomap_table(pdev)[pdata->rg_bar];
dw->rg_region.vaddr += pdata->rg_off;
dw->rg_region.paddr = pdev->resource[pdata->rg_bar].start;
dw->rg_region.paddr += pdata->rg_off;
dw->rg_region.sz = pdata->rg_sz;
dw->ll_region.vaddr = pcim_iomap_table(pdev)[pdata->ll_bar];
dw->ll_region.vaddr += pdata->ll_off;
dw->ll_region.paddr = pdev->resource[pdata->ll_bar].start;
dw->ll_region.paddr += pdata->ll_off;
dw->ll_region.sz = pdata->ll_sz;
dw->dt_region.vaddr = pcim_iomap_table(pdev)[pdata->dt_bar];
dw->dt_region.vaddr += pdata->dt_off;
dw->dt_region.paddr = pdev->resource[pdata->dt_bar].start;
dw->dt_region.paddr += pdata->dt_off;
dw->dt_region.sz = pdata->dt_sz;
dw->version = pdata->version;
dw->mode = pdata->mode;
dw->mf = vsec_data.mf;
dw->nr_irqs = nr_irqs;
dw->ops = &dw_edma_pcie_core_ops;
dw->wr_ch_cnt = vsec_data.wr_ch_cnt;
dw->rd_ch_cnt = vsec_data.rd_ch_cnt;
dw->rg_region.vaddr = pcim_iomap_table(pdev)[vsec_data.rg.bar];
if (!dw->rg_region.vaddr)
return -ENOMEM;
dw->rg_region.vaddr += vsec_data.rg.off;
dw->rg_region.paddr = pdev->resource[vsec_data.rg.bar].start;
dw->rg_region.paddr += vsec_data.rg.off;
dw->rg_region.sz = vsec_data.rg.sz;
for (i = 0; i < dw->wr_ch_cnt; i++) {
struct dw_edma_region *ll_region = &dw->ll_region_wr[i];
struct dw_edma_region *dt_region = &dw->dt_region_wr[i];
struct dw_edma_block *ll_block = &vsec_data.ll_wr[i];
struct dw_edma_block *dt_block = &vsec_data.dt_wr[i];
ll_region->vaddr = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr)
return -ENOMEM;
ll_region->vaddr += ll_block->off;
ll_region->paddr = pdev->resource[ll_block->bar].start;
ll_region->paddr += ll_block->off;
ll_region->sz = ll_block->sz;
dt_region->vaddr = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr)
return -ENOMEM;
dt_region->vaddr += dt_block->off;
dt_region->paddr = pdev->resource[dt_block->bar].start;
dt_region->paddr += dt_block->off;
dt_region->sz = dt_block->sz;
}
for (i = 0; i < dw->rd_ch_cnt; i++) {
struct dw_edma_region *ll_region = &dw->ll_region_rd[i];
struct dw_edma_region *dt_region = &dw->dt_region_rd[i];
struct dw_edma_block *ll_block = &vsec_data.ll_rd[i];
struct dw_edma_block *dt_block = &vsec_data.dt_rd[i];
ll_region->vaddr = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr)
return -ENOMEM;
ll_region->vaddr += ll_block->off;
ll_region->paddr = pdev->resource[ll_block->bar].start;
ll_region->paddr += ll_block->off;
ll_region->sz = ll_block->sz;
dt_region->vaddr = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr)
return -ENOMEM;
dt_region->vaddr += dt_block->off;
dt_region->paddr = pdev->resource[dt_block->bar].start;
dt_region->paddr += dt_block->off;
dt_region->sz = dt_block->sz;
}
/* Debug info */
pci_dbg(pdev, "Version:\t%u\n", dw->version);
pci_dbg(pdev, "Mode:\t%s\n",
dw->mode == EDMA_MODE_LEGACY ? "Legacy" : "Unroll");
if (dw->mf == EDMA_MF_EDMA_LEGACY)
pci_dbg(pdev, "Version:\teDMA Port Logic (0x%x)\n", dw->mf);
else if (dw->mf == EDMA_MF_EDMA_UNROLL)
pci_dbg(pdev, "Version:\teDMA Unroll (0x%x)\n", dw->mf);
else if (dw->mf == EDMA_MF_HDMA_COMPAT)
pci_dbg(pdev, "Version:\tHDMA Compatible (0x%x)\n", dw->mf);
else
pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", dw->mf);
pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
pdata->rg_bar, pdata->rg_off, pdata->rg_sz,
vsec_data.rg.bar, vsec_data.rg.off, vsec_data.rg.sz,
dw->rg_region.vaddr, &dw->rg_region.paddr);
pci_dbg(pdev, "L. List:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
pdata->ll_bar, pdata->ll_off, pdata->ll_sz,
dw->ll_region.vaddr, &dw->ll_region.paddr);
pci_dbg(pdev, "Data:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
pdata->dt_bar, pdata->dt_off, pdata->dt_sz,
dw->dt_region.vaddr, &dw->dt_region.paddr);
for (i = 0; i < dw->wr_ch_cnt; i++) {
pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.ll_wr[i].bar,
vsec_data.ll_wr[i].off, dw->ll_region_wr[i].sz,
dw->ll_region_wr[i].vaddr, &dw->ll_region_wr[i].paddr);
pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.dt_wr[i].bar,
vsec_data.dt_wr[i].off, dw->dt_region_wr[i].sz,
dw->dt_region_wr[i].vaddr, &dw->dt_region_wr[i].paddr);
}
for (i = 0; i < dw->rd_ch_cnt; i++) {
pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.ll_rd[i].bar,
vsec_data.ll_rd[i].off, dw->ll_region_rd[i].sz,
dw->ll_region_rd[i].vaddr, &dw->ll_region_rd[i].paddr);
pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data.dt_rd[i].bar,
vsec_data.dt_rd[i].off, dw->dt_region_rd[i].sz,
dw->dt_region_rd[i].vaddr, &dw->dt_region_rd[i].paddr);
}
pci_dbg(pdev, "Nr. IRQs:\t%u\n", dw->nr_irqs);

300
drivers/dma/dw-edma/dw-edma-v0-core.c
View File

@ -28,35 +28,75 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
return dw->rg_region.vaddr;
}
#define SET(dw, name, value) \
#define SET_32(dw, name, value) \
writel(value, &(__dw_regs(dw)->name))
#define GET(dw, name) \
#define GET_32(dw, name) \
readl(&(__dw_regs(dw)->name))
#define SET_RW(dw, dir, name, value) \
#define SET_RW_32(dw, dir, name, value) \
do { \
if ((dir) == EDMA_DIR_WRITE) \
SET(dw, wr_##name, value); \
SET_32(dw, wr_##name, value); \
else \
SET(dw, rd_##name, value); \
SET_32(dw, rd_##name, value); \
} while (0)
#define GET_RW(dw, dir, name) \
#define GET_RW_32(dw, dir, name) \
((dir) == EDMA_DIR_WRITE \
? GET(dw, wr_##name) \
: GET(dw, rd_##name))
? GET_32(dw, wr_##name) \
: GET_32(dw, rd_##name))
#define SET_BOTH(dw, name, value) \
#define SET_BOTH_32(dw, name, value) \
do { \
SET(dw, wr_##name, value); \
SET(dw, rd_##name, value); \
SET_32(dw, wr_##name, value); \
SET_32(dw, rd_##name, value); \
} while (0)
#ifdef CONFIG_64BIT
#define SET_64(dw, name, value) \
writeq(value, &(__dw_regs(dw)->name))
#define GET_64(dw, name) \
readq(&(__dw_regs(dw)->name))
#define SET_RW_64(dw, dir, name, value) \
do { \
if ((dir) == EDMA_DIR_WRITE) \
SET_64(dw, wr_##name, value); \
else \
SET_64(dw, rd_##name, value); \
} while (0)
#define GET_RW_64(dw, dir, name) \
((dir) == EDMA_DIR_WRITE \
? GET_64(dw, wr_##name) \
: GET_64(dw, rd_##name))
#define SET_BOTH_64(dw, name, value) \
do { \
SET_64(dw, wr_##name, value); \
SET_64(dw, rd_##name, value); \
} while (0)
#endif /* CONFIG_64BIT */
#define SET_COMPAT(dw, name, value) \
writel(value, &(__dw_regs(dw)->type.unroll.name))
#define SET_RW_COMPAT(dw, dir, name, value) \
do { \
if ((dir) == EDMA_DIR_WRITE) \
SET_COMPAT(dw, wr_##name, value); \
else \
SET_COMPAT(dw, rd_##name, value); \
} while (0)
static inline struct dw_edma_v0_ch_regs __iomem *
__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
{
if (dw->mode == EDMA_MODE_LEGACY)
if (dw->mf == EDMA_MF_EDMA_LEGACY)
return &(__dw_regs(dw)->type.legacy.ch);
if (dir == EDMA_DIR_WRITE)
@ -68,7 +108,7 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
u32 value, void __iomem *addr)
{
if (dw->mode == EDMA_MODE_LEGACY) {
if (dw->mf == EDMA_MF_EDMA_LEGACY) {
u32 viewport_sel;
unsigned long flags;
@ -93,7 +133,7 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
{
u32 value;
if (dw->mode == EDMA_MODE_LEGACY) {
if (dw->mf == EDMA_MF_EDMA_LEGACY) {
u32 viewport_sel;
unsigned long flags;
@ -115,21 +155,86 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
return value;
}
#define SET_CH(dw, dir, ch, name, value) \
#define SET_CH_32(dw, dir, ch, name, value) \
writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
#define GET_CH(dw, dir, ch, name) \
#define GET_CH_32(dw, dir, ch, name) \
readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
#define SET_LL(ll, value) \
#define SET_LL_32(ll, value) \
writel(value, ll)
#ifdef CONFIG_64BIT
static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
u64 value, void __iomem *addr)
{
if (dw->mf == EDMA_MF_EDMA_LEGACY) {
u32 viewport_sel;
unsigned long flags;
raw_spin_lock_irqsave(&dw->lock, flags);
viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
if (dir == EDMA_DIR_READ)
viewport_sel |= BIT(31);
writel(viewport_sel,
&(__dw_regs(dw)->type.legacy.viewport_sel));
writeq(value, addr);
raw_spin_unlock_irqrestore(&dw->lock, flags);
} else {
writeq(value, addr);
}
}
static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
const void __iomem *addr)
{
u32 value;
if (dw->mf == EDMA_MF_EDMA_LEGACY) {
u32 viewport_sel;
unsigned long flags;
raw_spin_lock_irqsave(&dw->lock, flags);
viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
if (dir == EDMA_DIR_READ)
viewport_sel |= BIT(31);
writel(viewport_sel,
&(__dw_regs(dw)->type.legacy.viewport_sel));
value = readq(addr);
raw_spin_unlock_irqrestore(&dw->lock, flags);
} else {
value = readq(addr);
}
return value;
}
#define SET_CH_64(dw, dir, ch, name, value) \
writeq_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
#define GET_CH_64(dw, dir, ch, name) \
readq_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
#define SET_LL_64(ll, value) \
writeq(value, ll)
#endif /* CONFIG_64BIT */
/* eDMA management callbacks */
void dw_edma_v0_core_off(struct dw_edma *dw)
{
SET_BOTH(dw, int_mask, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
SET_BOTH(dw, int_clear, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
SET_BOTH(dw, engine_en, 0);
SET_BOTH_32(dw, int_mask,
EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
SET_BOTH_32(dw, int_clear,
EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
SET_BOTH_32(dw, engine_en, 0);
}
u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
@ -137,9 +242,11 @@ u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
u32 num_ch;
if (dir == EDMA_DIR_WRITE)
num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK, GET(dw, ctrl));
num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK,
GET_32(dw, ctrl));
else
num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK, GET(dw, ctrl));
num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK,
GET_32(dw, ctrl));
if (num_ch > EDMA_V0_MAX_NR_CH)
num_ch = EDMA_V0_MAX_NR_CH;
@ -153,7 +260,7 @@ enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
u32 tmp;
tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,
GET_CH(dw, chan->dir, chan->id, ch_control1));
GET_CH_32(dw, chan->dir, chan->id, ch_control1));
if (tmp == 1)
return DMA_IN_PROGRESS;
@ -167,26 +274,28 @@ void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->chip->dw;
SET_RW(dw, chan->dir, int_clear,
FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
SET_RW_32(dw, chan->dir, int_clear,
FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
}
void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->chip->dw;
SET_RW(dw, chan->dir, int_clear,
FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
SET_RW_32(dw, chan->dir, int_clear,
FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
}
u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
{
return FIELD_GET(EDMA_V0_DONE_INT_MASK, GET_RW(dw, dir, int_status));
return FIELD_GET(EDMA_V0_DONE_INT_MASK,
GET_RW_32(dw, dir, int_status));
}
u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
{
return FIELD_GET(EDMA_V0_ABORT_INT_MASK, GET_RW(dw, dir, int_status));
return FIELD_GET(EDMA_V0_ABORT_INT_MASK,
GET_RW_32(dw, dir, int_status));
}
static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
@ -209,15 +318,23 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
control |= (DW_EDMA_V0_LIE | DW_EDMA_V0_RIE);
/* Channel control */
SET_LL(&lli[i].control, control);
SET_LL_32(&lli[i].control, control);
/* Transfer size */
SET_LL(&lli[i].transfer_size, child->sz);
/* SAR - low, high */
SET_LL(&lli[i].sar_low, lower_32_bits(child->sar));
SET_LL(&lli[i].sar_high, upper_32_bits(child->sar));
/* DAR - low, high */
SET_LL(&lli[i].dar_low, lower_32_bits(child->dar));
SET_LL(&lli[i].dar_high, upper_32_bits(child->dar));
SET_LL_32(&lli[i].transfer_size, child->sz);
/* SAR */
#ifdef CONFIG_64BIT
SET_LL_64(&lli[i].sar.reg, child->sar);
#else /* CONFIG_64BIT */
SET_LL_32(&lli[i].sar.lsb, lower_32_bits(child->sar));
SET_LL_32(&lli[i].sar.msb, upper_32_bits(child->sar));
#endif /* CONFIG_64BIT */
/* DAR */
#ifdef CONFIG_64BIT
SET_LL_64(&lli[i].dar.reg, child->dar);
#else /* CONFIG_64BIT */
SET_LL_32(&lli[i].dar.lsb, lower_32_bits(child->dar));
SET_LL_32(&lli[i].dar.msb, upper_32_bits(child->dar));
#endif /* CONFIG_64BIT */
i++;
}
@ -227,10 +344,14 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
control |= DW_EDMA_V0_CB;
/* Channel control */
SET_LL(&llp->control, control);
/* Linked list - low, high */
SET_LL(&llp->llp_low, lower_32_bits(chunk->ll_region.paddr));
SET_LL(&llp->llp_high, upper_32_bits(chunk->ll_region.paddr));
SET_LL_32(&llp->control, control);
/* Linked list */
#ifdef CONFIG_64BIT
SET_LL_64(&llp->llp.reg, chunk->ll_region.paddr);
#else /* CONFIG_64BIT */
SET_LL_32(&llp->llp.lsb, lower_32_bits(chunk->ll_region.paddr));
SET_LL_32(&llp->llp.msb, upper_32_bits(chunk->ll_region.paddr));
#endif /* CONFIG_64BIT */
}
void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
@ -243,28 +364,69 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
if (first) {
/* Enable engine */
SET_RW(dw, chan->dir, engine_en, BIT(0));
SET_RW_32(dw, chan->dir, engine_en, BIT(0));
if (dw->mf == EDMA_MF_HDMA_COMPAT) {
switch (chan->id) {
case 0:
SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
BIT(0));
break;
case 1:
SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
BIT(0));
break;
case 2:
SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
BIT(0));
break;
case 3:
SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
BIT(0));
break;
case 4:
SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
BIT(0));
break;
case 5:
SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
BIT(0));
break;
case 6:
SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
BIT(0));
break;
case 7:
SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
BIT(0));
break;
}
}
/* Interrupt unmask - done, abort */
tmp = GET_RW(dw, chan->dir, int_mask);
tmp = GET_RW_32(dw, chan->dir, int_mask);
tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
SET_RW(dw, chan->dir, int_mask, tmp);
SET_RW_32(dw, chan->dir, int_mask, tmp);
/* Linked list error */
tmp = GET_RW(dw, chan->dir, linked_list_err_en);
tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
SET_RW(dw, chan->dir, linked_list_err_en, tmp);
SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
/* Channel control */
SET_CH(dw, chan->dir, chan->id, ch_control1,
(DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
/* Linked list - low, high */
SET_CH(dw, chan->dir, chan->id, llp_low,
lower_32_bits(chunk->ll_region.paddr));
SET_CH(dw, chan->dir, chan->id, llp_high,
upper_32_bits(chunk->ll_region.paddr));
SET_CH_32(dw, chan->dir, chan->id, ch_control1,
(DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
/* Linked list */
#ifdef CONFIG_64BIT
SET_CH_64(dw, chan->dir, chan->id, llp.reg,
chunk->ll_region.paddr);
#else /* CONFIG_64BIT */
SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
lower_32_bits(chunk->ll_region.paddr));
SET_CH_32(dw, chan->dir, chan->id, llp.msb,
upper_32_bits(chunk->ll_region.paddr));
#endif /* CONFIG_64BIT */
}
/* Doorbell */
SET_RW(dw, chan->dir, doorbell,
FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
SET_RW_32(dw, chan->dir, doorbell,
FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
}
int dw_edma_v0_core_device_config(struct dw_edma_chan *chan)
@ -273,31 +435,31 @@ int dw_edma_v0_core_device_config(struct dw_edma_chan *chan)
u32 tmp = 0;
/* MSI done addr - low, high */
SET_RW(dw, chan->dir, done_imwr_low, chan->msi.address_lo);
SET_RW(dw, chan->dir, done_imwr_high, chan->msi.address_hi);
SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo);
SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi);
/* MSI abort addr - low, high */
SET_RW(dw, chan->dir, abort_imwr_low, chan->msi.address_lo);
SET_RW(dw, chan->dir, abort_imwr_high, chan->msi.address_hi);
SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo);
SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi);
/* MSI data - low, high */
switch (chan->id) {
case 0:
case 1:
tmp = GET_RW(dw, chan->dir, ch01_imwr_data);
tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data);
break;
case 2:
case 3:
tmp = GET_RW(dw, chan->dir, ch23_imwr_data);
tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data);
break;
case 4:
case 5:
tmp = GET_RW(dw, chan->dir, ch45_imwr_data);
tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data);
break;
case 6:
case 7:
tmp = GET_RW(dw, chan->dir, ch67_imwr_data);
tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data);
break;
}
@ -316,22 +478,22 @@ int dw_edma_v0_core_device_config(struct dw_edma_chan *chan)
switch (chan->id) {
case 0:
case 1:
SET_RW(dw, chan->dir, ch01_imwr_data, tmp);
SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp);
break;
case 2:
case 3:
SET_RW(dw, chan->dir, ch23_imwr_data, tmp);
SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp);
break;
case 4:
case 5:
SET_RW(dw, chan->dir, ch45_imwr_data, tmp);
SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp);
break;
case 6:
case 7:
SET_RW(dw, chan->dir, ch67_imwr_data, tmp);
SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp);
break;
}
@ -344,7 +506,7 @@ void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip)
dw_edma_v0_debugfs_on(chip);
}
void dw_edma_v0_core_debugfs_off(void)
void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip)
{
dw_edma_v0_debugfs_off();
dw_edma_v0_debugfs_off(chip);
}

2
drivers/dma/dw-edma/dw-edma-v0-core.h
View File

@ -23,6 +23,6 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first);
int dw_edma_v0_core_device_config(struct dw_edma_chan *chan);
/* eDMA debug fs callbacks */
void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip);
void dw_edma_v0_core_debugfs_off(void);
void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip);
#endif /* _DW_EDMA_V0_CORE_H */

77
drivers/dma/dw-edma/dw-edma-v0-debugfs.c
View File

@ -38,7 +38,6 @@
#define CHANNEL_STR "channel"
#define REGISTERS_STR "registers"
static struct dentry *base_dir;
static struct dw_edma *dw;
static struct dw_edma_v0_regs __iomem *regs;
@ -55,7 +54,7 @@ struct debugfs_entries {
static int dw_edma_debugfs_u32_get(void *data, u64 *val)
{
void __iomem *reg = (void __force __iomem *)data;
if (dw->mode == EDMA_MODE_LEGACY &&
if (dw->mf == EDMA_MF_EDMA_LEGACY &&
reg >= (void __iomem *)&regs->type.legacy.ch) {
void __iomem *ptr = &regs->type.legacy.ch;
u32 viewport_sel = 0;
@ -114,12 +113,12 @@ static void dw_edma_debugfs_regs_ch(struct dw_edma_v0_ch_regs __iomem *regs,
REGISTER(ch_control1),
REGISTER(ch_control2),
REGISTER(transfer_size),
REGISTER(sar_low),
REGISTER(sar_high),
REGISTER(dar_low),
REGISTER(dar_high),
REGISTER(llp_low),
REGISTER(llp_high),
REGISTER(sar.lsb),
REGISTER(sar.msb),
REGISTER(dar.lsb),
REGISTER(dar.msb),
REGISTER(llp.lsb),
REGISTER(llp.msb),
};
nr_entries = ARRAY_SIZE(debugfs_regs);
@ -132,17 +131,17 @@ static void dw_edma_debugfs_regs_wr(struct dentry *dir)
/* eDMA global registers */
WR_REGISTER(engine_en),
WR_REGISTER(doorbell),
WR_REGISTER(ch_arb_weight_low),
WR_REGISTER(ch_arb_weight_high),
WR_REGISTER(ch_arb_weight.lsb),
WR_REGISTER(ch_arb_weight.msb),
/* eDMA interrupts registers */
WR_REGISTER(int_status),
WR_REGISTER(int_mask),
WR_REGISTER(int_clear),
WR_REGISTER(err_status),
WR_REGISTER(done_imwr_low),
WR_REGISTER(done_imwr_high),
WR_REGISTER(abort_imwr_low),
WR_REGISTER(abort_imwr_high),
WR_REGISTER(done_imwr.lsb),
WR_REGISTER(done_imwr.msb),
WR_REGISTER(abort_imwr.lsb),
WR_REGISTER(abort_imwr.msb),
WR_REGISTER(ch01_imwr_data),
WR_REGISTER(ch23_imwr_data),
WR_REGISTER(ch45_imwr_data),
@ -152,8 +151,8 @@ static void dw_edma_debugfs_regs_wr(struct dentry *dir)
const struct debugfs_entries debugfs_unroll_regs[] = {
/* eDMA channel context grouping */
WR_REGISTER_UNROLL(engine_chgroup),
WR_REGISTER_UNROLL(engine_hshake_cnt_low),
WR_REGISTER_UNROLL(engine_hshake_cnt_high),
WR_REGISTER_UNROLL(engine_hshake_cnt.lsb),
WR_REGISTER_UNROLL(engine_hshake_cnt.msb),
WR_REGISTER_UNROLL(ch0_pwr_en),
WR_REGISTER_UNROLL(ch1_pwr_en),
WR_REGISTER_UNROLL(ch2_pwr_en),
@ -174,7 +173,7 @@ static void dw_edma_debugfs_regs_wr(struct dentry *dir)
nr_entries = ARRAY_SIZE(debugfs_regs);
dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
if (dw->mode == EDMA_MODE_UNROLL) {
if (dw->mf == EDMA_MF_HDMA_COMPAT) {
nr_entries = ARRAY_SIZE(debugfs_unroll_regs);
dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries,
regs_dir);
@ -200,19 +199,19 @@ static void dw_edma_debugfs_regs_rd(struct dentry *dir)
/* eDMA global registers */
RD_REGISTER(engine_en),
RD_REGISTER(doorbell),
RD_REGISTER(ch_arb_weight_low),
RD_REGISTER(ch_arb_weight_high),
RD_REGISTER(ch_arb_weight.lsb),
RD_REGISTER(ch_arb_weight.msb),
/* eDMA interrupts registers */
RD_REGISTER(int_status),
RD_REGISTER(int_mask),
RD_REGISTER(int_clear),
RD_REGISTER(err_status_low),
RD_REGISTER(err_status_high),
RD_REGISTER(err_status.lsb),
RD_REGISTER(err_status.msb),
RD_REGISTER(linked_list_err_en),
RD_REGISTER(done_imwr_low),
RD_REGISTER(done_imwr_high),
RD_REGISTER(abort_imwr_low),
RD_REGISTER(abort_imwr_high),
RD_REGISTER(done_imwr.lsb),
RD_REGISTER(done_imwr.msb),
RD_REGISTER(abort_imwr.lsb),
RD_REGISTER(abort_imwr.msb),
RD_REGISTER(ch01_imwr_data),
RD_REGISTER(ch23_imwr_data),
RD_REGISTER(ch45_imwr_data),
@ -221,8 +220,8 @@ static void dw_edma_debugfs_regs_rd(struct dentry *dir)
const struct debugfs_entries debugfs_unroll_regs[] = {
/* eDMA channel context grouping */
RD_REGISTER_UNROLL(engine_chgroup),
RD_REGISTER_UNROLL(engine_hshake_cnt_low),
RD_REGISTER_UNROLL(engine_hshake_cnt_high),
RD_REGISTER_UNROLL(engine_hshake_cnt.lsb),
RD_REGISTER_UNROLL(engine_hshake_cnt.msb),
RD_REGISTER_UNROLL(ch0_pwr_en),
RD_REGISTER_UNROLL(ch1_pwr_en),
RD_REGISTER_UNROLL(ch2_pwr_en),
@ -243,7 +242,7 @@ static void dw_edma_debugfs_regs_rd(struct dentry *dir)
nr_entries = ARRAY_SIZE(debugfs_regs);
dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir);
if (dw->mode == EDMA_MODE_UNROLL) {
if (dw->mf == EDMA_MF_HDMA_COMPAT) {
nr_entries = ARRAY_SIZE(debugfs_unroll_regs);
dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries,
regs_dir);
@ -272,7 +271,7 @@ static void dw_edma_debugfs_regs(void)
struct dentry *regs_dir;
int nr_entries;
regs_dir = debugfs_create_dir(REGISTERS_STR, base_dir);
regs_dir = debugfs_create_dir(REGISTERS_STR, dw->debugfs);
if (!regs_dir)
return;
@ -293,19 +292,23 @@ void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip)
if (!regs)
return;
base_dir = debugfs_create_dir(dw->name, NULL);
if (!base_dir)
dw->debugfs = debugfs_create_dir(dw->name, NULL);
if (!dw->debugfs)
return;
debugfs_create_u32("version", 0444, base_dir, &dw->version);
debugfs_create_u32("mode", 0444, base_dir, &dw->mode);
debugfs_create_u16("wr_ch_cnt", 0444, base_dir, &dw->wr_ch_cnt);
debugfs_create_u16("rd_ch_cnt", 0444, base_dir, &dw->rd_ch_cnt);
debugfs_create_u32("mf", 0444, dw->debugfs, &dw->mf);
debugfs_create_u16("wr_ch_cnt", 0444, dw->debugfs, &dw->wr_ch_cnt);
debugfs_create_u16("rd_ch_cnt", 0444, dw->debugfs, &dw->rd_ch_cnt);
dw_edma_debugfs_regs();
}
void dw_edma_v0_debugfs_off(void)
void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip)
{
debugfs_remove_recursive(base_dir);
dw = chip->dw;
if (!dw)
return;
debugfs_remove_recursive(dw->debugfs);
dw->debugfs = NULL;
}

4
drivers/dma/dw-edma/dw-edma-v0-debugfs.h
View File

@ -13,13 +13,13 @@
#ifdef CONFIG_DEBUG_FS
void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip);
void dw_edma_v0_debugfs_off(void);
void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip);
#else
static inline void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip)
{
}
static inline void dw_edma_v0_debugfs_off(void)
static inline void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip)
{
}
#endif /* CONFIG_DEBUG_FS */

291
drivers/dma/dw-edma/dw-edma-v0-regs.h
View File

@ -25,134 +25,209 @@
#define EDMA_V0_CH_EVEN_MSI_DATA_MASK GENMASK(15, 0)
struct dw_edma_v0_ch_regs {
u32 ch_control1; /* 0x000 */
u32 ch_control2; /* 0x004 */
u32 transfer_size; /* 0x008 */
u32 sar_low; /* 0x00c */
u32 sar_high; /* 0x010 */
u32 dar_low; /* 0x014 */
u32 dar_high; /* 0x018 */
u32 llp_low; /* 0x01c */
u32 llp_high; /* 0x020 */
};
u32 ch_control1; /* 0x0000 */
u32 ch_control2; /* 0x0004 */
u32 transfer_size; /* 0x0008 */
union {
u64 reg; /* 0x000c..0x0010 */
struct {
u32 lsb; /* 0x000c */
u32 msb; /* 0x0010 */
};
} sar;
union {
u64 reg; /* 0x0014..0x0018 */
struct {
u32 lsb; /* 0x0014 */
u32 msb; /* 0x0018 */
};
} dar;
union {
u64 reg; /* 0x001c..0x0020 */
struct {
u32 lsb; /* 0x001c */
u32 msb; /* 0x0020 */
};
} llp;
} __packed;
struct dw_edma_v0_ch {
struct dw_edma_v0_ch_regs wr; /* 0x200 */
u32 padding_1[55]; /* [0x224..0x2fc] */
struct dw_edma_v0_ch_regs rd; /* 0x300 */
u32 padding_2[55]; /* [0x324..0x3fc] */
};
struct dw_edma_v0_ch_regs wr; /* 0x0200 */
u32 padding_1[55]; /* 0x0224..0x02fc */
struct dw_edma_v0_ch_regs rd; /* 0x0300 */
u32 padding_2[55]; /* 0x0324..0x03fc */
} __packed;
struct dw_edma_v0_unroll {
u32 padding_1; /* 0x0f8 */
u32 wr_engine_chgroup; /* 0x100 */
u32 rd_engine_chgroup; /* 0x104 */
u32 wr_engine_hshake_cnt_low; /* 0x108 */
u32 wr_engine_hshake_cnt_high; /* 0x10c */
u32 padding_2[2]; /* [0x110..0x114] */
u32 rd_engine_hshake_cnt_low; /* 0x118 */
u32 rd_engine_hshake_cnt_high; /* 0x11c */
u32 padding_3[2]; /* [0x120..0x124] */
u32 wr_ch0_pwr_en; /* 0x128 */
u32 wr_ch1_pwr_en; /* 0x12c */
u32 wr_ch2_pwr_en; /* 0x130 */
u32 wr_ch3_pwr_en; /* 0x134 */
u32 wr_ch4_pwr_en; /* 0x138 */
u32 wr_ch5_pwr_en; /* 0x13c */
u32 wr_ch6_pwr_en; /* 0x140 */
u32 wr_ch7_pwr_en; /* 0x144 */
u32 padding_4[8]; /* [0x148..0x164] */
u32 rd_ch0_pwr_en; /* 0x168 */
u32 rd_ch1_pwr_en; /* 0x16c */
u32 rd_ch2_pwr_en; /* 0x170 */
u32 rd_ch3_pwr_en; /* 0x174 */
u32 rd_ch4_pwr_en; /* 0x178 */
u32 rd_ch5_pwr_en; /* 0x18c */
u32 rd_ch6_pwr_en; /* 0x180 */
u32 rd_ch7_pwr_en; /* 0x184 */
u32 padding_5[30]; /* [0x188..0x1fc] */
struct dw_edma_v0_ch ch[EDMA_V0_MAX_NR_CH]; /* [0x200..0x1120] */
};
u32 padding_1; /* 0x00f8 */
u32 wr_engine_chgroup; /* 0x0100 */
u32 rd_engine_chgroup; /* 0x0104 */
union {
u64 reg; /* 0x0108..0x010c */
struct {
u32 lsb; /* 0x0108 */
u32 msb; /* 0x010c */
};
} wr_engine_hshake_cnt;
u32 padding_2[2]; /* 0x0110..0x0114 */
union {
u64 reg; /* 0x0120..0x0124 */
struct {
u32 lsb; /* 0x0120 */
u32 msb; /* 0x0124 */
};
} rd_engine_hshake_cnt;
u32 padding_3[2]; /* 0x0120..0x0124 */
u32 wr_ch0_pwr_en; /* 0x0128 */
u32 wr_ch1_pwr_en; /* 0x012c */
u32 wr_ch2_pwr_en; /* 0x0130 */
u32 wr_ch3_pwr_en; /* 0x0134 */
u32 wr_ch4_pwr_en; /* 0x0138 */
u32 wr_ch5_pwr_en; /* 0x013c */
u32 wr_ch6_pwr_en; /* 0x0140 */
u32 wr_ch7_pwr_en; /* 0x0144 */
u32 padding_4[8]; /* 0x0148..0x0164 */
u32 rd_ch0_pwr_en; /* 0x0168 */
u32 rd_ch1_pwr_en; /* 0x016c */
u32 rd_ch2_pwr_en; /* 0x0170 */
u32 rd_ch3_pwr_en; /* 0x0174 */
u32 rd_ch4_pwr_en; /* 0x0178 */
u32 rd_ch5_pwr_en; /* 0x018c */
u32 rd_ch6_pwr_en; /* 0x0180 */
u32 rd_ch7_pwr_en; /* 0x0184 */
u32 padding_5[30]; /* 0x0188..0x01fc */
struct dw_edma_v0_ch ch[EDMA_V0_MAX_NR_CH]; /* 0x0200..0x1120 */
} __packed;
struct dw_edma_v0_legacy {
u32 viewport_sel; /* 0x0f8 */
struct dw_edma_v0_ch_regs ch; /* [0x100..0x120] */
};
u32 viewport_sel; /* 0x00f8 */
struct dw_edma_v0_ch_regs ch; /* 0x0100..0x0120 */
} __packed;
struct dw_edma_v0_regs {
/* eDMA global registers */
u32 ctrl_data_arb_prior; /* 0x000 */
u32 padding_1; /* 0x004 */
u32 ctrl; /* 0x008 */
u32 wr_engine_en; /* 0x00c */
u32 wr_doorbell; /* 0x010 */
u32 padding_2; /* 0x014 */
u32 wr_ch_arb_weight_low; /* 0x018 */
u32 wr_ch_arb_weight_high; /* 0x01c */
u32 padding_3[3]; /* [0x020..0x028] */
u32 rd_engine_en; /* 0x02c */
u32 rd_doorbell; /* 0x030 */
u32 padding_4; /* 0x034 */
u32 rd_ch_arb_weight_low; /* 0x038 */
u32 rd_ch_arb_weight_high; /* 0x03c */
u32 padding_5[3]; /* [0x040..0x048] */
u32 ctrl_data_arb_prior; /* 0x0000 */
u32 padding_1; /* 0x0004 */
u32 ctrl; /* 0x0008 */
u32 wr_engine_en; /* 0x000c */
u32 wr_doorbell; /* 0x0010 */
u32 padding_2; /* 0x0014 */
union {
u64 reg; /* 0x0018..0x001c */
struct {
u32 lsb; /* 0x0018 */
u32 msb; /* 0x001c */
};
} wr_ch_arb_weight;
u32 padding_3[3]; /* 0x0020..0x0028 */
u32 rd_engine_en; /* 0x002c */
u32 rd_doorbell; /* 0x0030 */
u32 padding_4; /* 0x0034 */
union {
u64 reg; /* 0x0038..0x003c */
struct {
u32 lsb; /* 0x0038 */
u32 msb; /* 0x003c */
};
} rd_ch_arb_weight;
u32 padding_5[3]; /* 0x0040..0x0048 */
/* eDMA interrupts registers */
u32 wr_int_status; /* 0x04c */
u32 padding_6; /* 0x050 */
u32 wr_int_mask; /* 0x054 */
u32 wr_int_clear; /* 0x058 */
u32 wr_err_status; /* 0x05c */
u32 wr_done_imwr_low; /* 0x060 */
u32 wr_done_imwr_high; /* 0x064 */
u32 wr_abort_imwr_low; /* 0x068 */
u32 wr_abort_imwr_high; /* 0x06c */
u32 wr_ch01_imwr_data; /* 0x070 */
u32 wr_ch23_imwr_data; /* 0x074 */
u32 wr_ch45_imwr_data; /* 0x078 */
u32 wr_ch67_imwr_data; /* 0x07c */
u32 padding_7[4]; /* [0x080..0x08c] */
u32 wr_linked_list_err_en; /* 0x090 */
u32 padding_8[3]; /* [0x094..0x09c] */
u32 rd_int_status; /* 0x0a0 */
u32 padding_9; /* 0x0a4 */
u32 rd_int_mask; /* 0x0a8 */
u32 rd_int_clear; /* 0x0ac */
u32 padding_10; /* 0x0b0 */
u32 rd_err_status_low; /* 0x0b4 */
u32 rd_err_status_high; /* 0x0b8 */
u32 padding_11[2]; /* [0x0bc..0x0c0] */
u32 rd_linked_list_err_en; /* 0x0c4 */
u32 padding_12; /* 0x0c8 */
u32 rd_done_imwr_low; /* 0x0cc */
u32 rd_done_imwr_high; /* 0x0d0 */
u32 rd_abort_imwr_low; /* 0x0d4 */
u32 rd_abort_imwr_high; /* 0x0d8 */
u32 rd_ch01_imwr_data; /* 0x0dc */
u32 rd_ch23_imwr_data; /* 0x0e0 */
u32 rd_ch45_imwr_data; /* 0x0e4 */
u32 rd_ch67_imwr_data; /* 0x0e8 */
u32 padding_13[4]; /* [0x0ec..0x0f8] */
u32 wr_int_status; /* 0x004c */
u32 padding_6; /* 0x0050 */
u32 wr_int_mask; /* 0x0054 */
u32 wr_int_clear; /* 0x0058 */
u32 wr_err_status; /* 0x005c */
union {
u64 reg; /* 0x0060..0x0064 */
struct {
u32 lsb; /* 0x0060 */
u32 msb; /* 0x0064 */
};
} wr_done_imwr;
union {
u64 reg; /* 0x0068..0x006c */
struct {
u32 lsb; /* 0x0068 */
u32 msb; /* 0x006c */
};
} wr_abort_imwr;
u32 wr_ch01_imwr_data; /* 0x0070 */
u32 wr_ch23_imwr_data; /* 0x0074 */
u32 wr_ch45_imwr_data; /* 0x0078 */
u32 wr_ch67_imwr_data; /* 0x007c */
u32 padding_7[4]; /* 0x0080..0x008c */
u32 wr_linked_list_err_en; /* 0x0090 */
u32 padding_8[3]; /* 0x0094..0x009c */
u32 rd_int_status; /* 0x00a0 */
u32 padding_9; /* 0x00a4 */
u32 rd_int_mask; /* 0x00a8 */
u32 rd_int_clear; /* 0x00ac */
u32 padding_10; /* 0x00b0 */
union {
u64 reg; /* 0x00b4..0x00b8 */
struct {
u32 lsb; /* 0x00b4 */
u32 msb; /* 0x00b8 */
};
} rd_err_status;
u32 padding_11[2]; /* 0x00bc..0x00c0 */
u32 rd_linked_list_err_en; /* 0x00c4 */
u32 padding_12; /* 0x00c8 */
union {
u64 reg; /* 0x00cc..0x00d0 */
struct {
u32 lsb; /* 0x00cc */
u32 msb; /* 0x00d0 */
};
} rd_done_imwr;
union {
u64 reg; /* 0x00d4..0x00d8 */
struct {
u32 lsb; /* 0x00d4 */
u32 msb; /* 0x00d8 */
};
} rd_abort_imwr;
u32 rd_ch01_imwr_data; /* 0x00dc */
u32 rd_ch23_imwr_data; /* 0x00e0 */
u32 rd_ch45_imwr_data; /* 0x00e4 */
u32 rd_ch67_imwr_data; /* 0x00e8 */
u32 padding_13[4]; /* 0x00ec..0x00f8 */
/* eDMA channel context grouping */
union dw_edma_v0_type {
struct dw_edma_v0_legacy legacy; /* [0x0f8..0x120] */
struct dw_edma_v0_unroll unroll; /* [0x0f8..0x1120] */
struct dw_edma_v0_legacy legacy; /* 0x00f8..0x0120 */
struct dw_edma_v0_unroll unroll; /* 0x00f8..0x1120 */
} type;
};
} __packed;
struct dw_edma_v0_lli {
u32 control;
u32 transfer_size;
u32 sar_low;
u32 sar_high;
u32 dar_low;
u32 dar_high;
};
union {
u64 reg;
struct {
u32 lsb;
u32 msb;
};
} sar;
union {
u64 reg;
struct {
u32 lsb;
u32 msb;
};
} dar;
} __packed;
struct dw_edma_v0_llp {
u32 control;
u32 reserved;
u32 llp_low;
u32 llp_high;
};
union {
u64 reg;
struct {
u32 lsb;
u32 msb;
};
} llp;
} __packed;
#endif /* _DW_EDMA_V0_REGS_H */

2
drivers/dma/idxd/Makefile
View File

@ -1,2 +1,4 @@
obj-$(CONFIG_INTEL_IDXD) += idxd.o
idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o
idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o

148
drivers/dma/idxd/cdev.c
View File

@ -39,15 +39,15 @@ struct idxd_user_context {
struct iommu_sva *sva;
};
enum idxd_cdev_cleanup {
CDEV_NORMAL = 0,
CDEV_FAILED,
};
static void idxd_cdev_dev_release(struct device *dev)
{
dev_dbg(dev, "releasing cdev device\n");
kfree(dev);
struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev);
struct idxd_cdev_context *cdev_ctx;
struct idxd_wq *wq = idxd_cdev->wq;
cdev_ctx = &ictx[wq->idxd->data->type];
ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
kfree(idxd_cdev);
}
static struct device_type idxd_cdev_device_type = {
@ -62,14 +62,11 @@ static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
return container_of(cdev, struct idxd_cdev, cdev);
}
static inline struct idxd_wq *idxd_cdev_wq(struct idxd_cdev *idxd_cdev)
{
return container_of(idxd_cdev, struct idxd_wq, idxd_cdev);
}
static inline struct idxd_wq *inode_wq(struct inode *inode)
{
return idxd_cdev_wq(inode_idxd_cdev(inode));
struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
return idxd_cdev->wq;
}
static int idxd_cdev_open(struct inode *inode, struct file *filp)
@ -220,11 +217,10 @@ static __poll_t idxd_cdev_poll(struct file *filp,
struct idxd_user_context *ctx = filp->private_data;
struct idxd_wq *wq = ctx->wq;
struct idxd_device *idxd = wq->idxd;
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
unsigned long flags;
__poll_t out = 0;
poll_wait(filp, &idxd_cdev->err_queue, wait);
poll_wait(filp, &wq->err_queue, wait);
spin_lock_irqsave(&idxd->dev_lock, flags);
if (idxd->sw_err.valid)
out = EPOLLIN | EPOLLRDNORM;
@ -243,101 +239,69 @@ static const struct file_operations idxd_cdev_fops = {
int idxd_cdev_get_major(struct idxd_device *idxd)
{
return MAJOR(ictx[idxd->type].devt);
}
static int idxd_wq_cdev_dev_setup(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
struct idxd_cdev_context *cdev_ctx;
struct device *dev;
int minor, rc;
idxd_cdev->dev = kzalloc(sizeof(*idxd_cdev->dev), GFP_KERNEL);
if (!idxd_cdev->dev)
return -ENOMEM;
dev = idxd_cdev->dev;
dev->parent = &idxd->pdev->dev;
dev_set_name(dev, "%s/wq%u.%u", idxd_get_dev_name(idxd),
idxd->id, wq->id);
dev->bus = idxd_get_bus_type(idxd);
cdev_ctx = &ictx[wq->idxd->type];
minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
if (minor < 0) {
rc = minor;
kfree(dev);
goto ida_err;
}
dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
dev->type = &idxd_cdev_device_type;
rc = device_register(dev);
if (rc < 0) {
dev_err(&idxd->pdev->dev, "device register failed\n");
goto dev_reg_err;
}
idxd_cdev->minor = minor;
return 0;
dev_reg_err:
ida_simple_remove(&cdev_ctx->minor_ida, MINOR(dev->devt));
put_device(dev);
ida_err:
idxd_cdev->dev = NULL;
return rc;
}
static void idxd_wq_cdev_cleanup(struct idxd_wq *wq,
enum idxd_cdev_cleanup cdev_state)
{
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
struct idxd_cdev_context *cdev_ctx;
cdev_ctx = &ictx[wq->idxd->type];
if (cdev_state == CDEV_NORMAL)
cdev_del(&idxd_cdev->cdev);
device_unregister(idxd_cdev->dev);
/*
* The device_type->release() will be called on the device and free
* the allocated struct device. We can just forget it.
*/
ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
idxd_cdev->dev = NULL;
idxd_cdev->minor = -1;
return MAJOR(ictx[idxd->data->type].devt);
}
int idxd_wq_add_cdev(struct idxd_wq *wq)
{
struct idxd_cdev *idxd_cdev = &wq->idxd_cdev;
struct cdev *cdev = &idxd_cdev->cdev;
struct idxd_device *idxd = wq->idxd;
struct idxd_cdev *idxd_cdev;
struct cdev *cdev;
struct device *dev;
int rc;
struct idxd_cdev_context *cdev_ctx;
int rc, minor;
rc = idxd_wq_cdev_dev_setup(wq);
idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
if (!idxd_cdev)
return -ENOMEM;
idxd_cdev->wq = wq;
cdev = &idxd_cdev->cdev;
dev = &idxd_cdev->dev;
cdev_ctx = &ictx[wq->idxd->data->type];
minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
if (minor < 0) {
kfree(idxd_cdev);
return minor;
}
idxd_cdev->minor = minor;
device_initialize(dev);
dev->parent = &wq->conf_dev;
dev->bus = &dsa_bus_type;
dev->type = &idxd_cdev_device_type;
dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
if (rc < 0)
return rc;
goto err;
dev = idxd_cdev->dev;
wq->idxd_cdev = idxd_cdev;
cdev_init(cdev, &idxd_cdev_fops);
cdev_set_parent(cdev, &dev->kobj);
rc = cdev_add(cdev, dev->devt, 1);
rc = cdev_device_add(cdev, dev);
if (rc) {
dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
idxd_wq_cdev_cleanup(wq, CDEV_FAILED);
return rc;
goto err;
}
init_waitqueue_head(&idxd_cdev->err_queue);
return 0;
err:
put_device(dev);
wq->idxd_cdev = NULL;
return rc;
}
void idxd_wq_del_cdev(struct idxd_wq *wq)
{
idxd_wq_cdev_cleanup(wq, CDEV_NORMAL);
struct idxd_cdev *idxd_cdev;
struct idxd_cdev_context *cdev_ctx;
cdev_ctx = &ictx[wq->idxd->data->type];
idxd_cdev = wq->idxd_cdev;
wq->idxd_cdev = NULL;
cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev);
put_device(&idxd_cdev->dev);
}
int idxd_cdev_register(void)

283
drivers/dma/idxd/device.c
View File

@ -19,7 +19,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
/* Interrupt control bits */
void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
{
struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);
struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector);
pci_msi_mask_irq(data);
}
@ -36,7 +36,7 @@ void idxd_mask_msix_vectors(struct idxd_device *idxd)
void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
{
struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);
struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector);
pci_msi_unmask_irq(data);
}
@ -47,6 +47,7 @@ void idxd_unmask_error_interrupts(struct idxd_device *idxd)
genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
genctrl.softerr_int_en = 1;
genctrl.halt_int_en = 1;
iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
}
@ -56,6 +57,7 @@ void idxd_mask_error_interrupts(struct idxd_device *idxd)
genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
genctrl.softerr_int_en = 0;
genctrl.halt_int_en = 0;
iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
}
@ -144,14 +146,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
if (rc < 0)
return rc;
if (idxd->type == IDXD_TYPE_DSA)
align = 32;
else if (idxd->type == IDXD_TYPE_IAX)
align = 64;
else
return -ENODEV;
wq->compls_size = num_descs * idxd->compl_size + align;
align = idxd->data->align;
wq->compls_size = num_descs * idxd->data->compl_size + align;
wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size,
&wq->compls_addr_raw, GFP_KERNEL);
if (!wq->compls_raw) {
@ -178,16 +174,14 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
struct idxd_desc *desc = wq->descs[i];
desc->hw = wq->hw_descs[i];
if (idxd->type == IDXD_TYPE_DSA)
if (idxd->data->type == IDXD_TYPE_DSA)
desc->completion = &wq->compls[i];
else if (idxd->type == IDXD_TYPE_IAX)
else if (idxd->data->type == IDXD_TYPE_IAX)
desc->iax_completion = &wq->iax_compls[i];
desc->compl_dma = wq->compls_addr + idxd->compl_size * i;
desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;
desc->id = i;
desc->wq = wq;
desc->cpu = -1;
dma_async_tx_descriptor_init(&desc->txd, &wq->dma_chan);
desc->txd.tx_submit = idxd_dma_tx_submit;
}
return 0;
@ -320,6 +314,19 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq)
struct device *dev = &wq->idxd->pdev->dev;
devm_iounmap(dev, wq->portal);
wq->portal = NULL;
}
void idxd_wqs_unmap_portal(struct idxd_device *idxd)
{
int i;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = idxd->wqs[i];
if (wq->portal)
idxd_wq_unmap_portal(wq);
}
}
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
@ -392,6 +399,32 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq)
memset(wq->name, 0, WQ_NAME_SIZE);
}
static void idxd_wq_ref_release(struct percpu_ref *ref)
{
struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);
complete(&wq->wq_dead);
}
int idxd_wq_init_percpu_ref(struct idxd_wq *wq)
{
int rc;
memset(&wq->wq_active, 0, sizeof(wq->wq_active));
rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, 0, GFP_KERNEL);
if (rc < 0)
return rc;
reinit_completion(&wq->wq_dead);
return 0;
}
void idxd_wq_quiesce(struct idxd_wq *wq)
{
percpu_ref_kill(&wq->wq_active);
wait_for_completion(&wq->wq_dead);
percpu_ref_exit(&wq->wq_active);
}
/* Device control bits */
static inline bool idxd_is_enabled(struct idxd_device *idxd)
{
@ -432,13 +465,13 @@ int idxd_device_init_reset(struct idxd_device *idxd)
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = IDXD_CMD_RESET_DEVICE;
dev_dbg(dev, "%s: sending reset for init.\n", __func__);
spin_lock_irqsave(&idxd->dev_lock, flags);
spin_lock_irqsave(&idxd->cmd_lock, flags);
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
IDXD_CMDSTS_ACTIVE)
cpu_relax();
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
return 0;
}
@ -451,7 +484,8 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
if (idxd_device_is_halted(idxd)) {
dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
*status = IDXD_CMDSTS_HW_ERR;
if (status)
*status = IDXD_CMDSTS_HW_ERR;
return;
}
@ -460,10 +494,10 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
cmd.operand = operand;
cmd.int_req = 1;
spin_lock_irqsave(&idxd->dev_lock, flags);
spin_lock_irqsave(&idxd->cmd_lock, flags);
wait_event_lock_irq(idxd->cmd_waitq,
!test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
idxd->dev_lock);
idxd->cmd_lock);
dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
__func__, cmd_code, operand);
@ -477,9 +511,9 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
* After command submitted, release lock and go to sleep until
* the command completes via interrupt.
*/
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
wait_for_completion(&done);
spin_lock_irqsave(&idxd->dev_lock, flags);
spin_lock_irqsave(&idxd->cmd_lock, flags);
if (status) {
*status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
idxd->cmd_status = *status & GENMASK(7, 0);
@ -488,7 +522,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
__clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
/* Wake up other pending commands */
wake_up(&idxd->cmd_waitq);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
}
int idxd_device_enable(struct idxd_device *idxd)
@ -521,7 +555,7 @@ void idxd_device_wqs_clear_state(struct idxd_device *idxd)
lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
struct idxd_wq *wq = idxd->wqs[i];
if (wq->state == IDXD_WQ_ENABLED) {
idxd_wq_disable_cleanup(wq);
@ -579,6 +613,77 @@ void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
dev_dbg(dev, "pasid %d drained\n", pasid);
}
int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
enum idxd_interrupt_type irq_type)
{
struct device *dev = &idxd->pdev->dev;
u32 operand, status;
if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)))
return -EOPNOTSUPP;
dev_dbg(dev, "get int handle, idx %d\n", idx);
operand = idx & GENMASK(15, 0);
if (irq_type == IDXD_IRQ_IMS)
operand |= CMD_INT_HANDLE_IMS;
dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_REQUEST_INT_HANDLE, operand);
idxd_cmd_exec(idxd, IDXD_CMD_REQUEST_INT_HANDLE, operand, &status);
if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
dev_dbg(dev, "request int handle failed: %#x\n", status);
return -ENXIO;
}
*handle = (status >> IDXD_CMDSTS_RES_SHIFT) & GENMASK(15, 0);
dev_dbg(dev, "int handle acquired: %u\n", *handle);
return 0;
}
int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
enum idxd_interrupt_type irq_type)
{
struct device *dev = &idxd->pdev->dev;
u32 operand, status;
union idxd_command_reg cmd;
unsigned long flags;
if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)))
return -EOPNOTSUPP;
dev_dbg(dev, "release int handle, handle %d\n", handle);
memset(&cmd, 0, sizeof(cmd));
operand = handle & GENMASK(15, 0);
if (irq_type == IDXD_IRQ_IMS)
operand |= CMD_INT_HANDLE_IMS;
cmd.cmd = IDXD_CMD_RELEASE_INT_HANDLE;
cmd.operand = operand;
dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand);
spin_lock_irqsave(&idxd->cmd_lock, flags);
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE)
cpu_relax();
status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
dev_dbg(dev, "release int handle failed: %#x\n", status);
return -ENXIO;
}
dev_dbg(dev, "int handle released.\n");
return 0;
}
/* Device configuration bits */
void idxd_msix_perm_setup(struct idxd_device *idxd)
{
@ -660,7 +765,7 @@ static int idxd_groups_config_write(struct idxd_device *idxd)
ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
for (i = 0; i < idxd->max_groups; i++) {
struct idxd_group *group = &idxd->groups[i];
struct idxd_group *group = idxd->groups[i];
idxd_group_config_write(group);
}
@ -739,7 +844,7 @@ static int idxd_wqs_config_write(struct idxd_device *idxd)
int i, rc;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
struct idxd_wq *wq = idxd->wqs[i];
rc = idxd_wq_config_write(wq);
if (rc < 0)
@ -755,7 +860,7 @@ static void idxd_group_flags_setup(struct idxd_device *idxd)
/* TC-A 0 and TC-B 1 should be defaults */
for (i = 0; i < idxd->max_groups; i++) {
struct idxd_group *group = &idxd->groups[i];
struct idxd_group *group = idxd->groups[i];
if (group->tc_a == -1)
group->tc_a = group->grpcfg.flags.tc_a = 0;
@ -782,12 +887,12 @@ static int idxd_engines_setup(struct idxd_device *idxd)
struct idxd_group *group;
for (i = 0; i < idxd->max_groups; i++) {
group = &idxd->groups[i];
group = idxd->groups[i];
group->grpcfg.engines = 0;
}
for (i = 0; i < idxd->max_engines; i++) {
eng = &idxd->engines[i];
eng = idxd->engines[i];
group = eng->group;
if (!group)
@ -811,13 +916,13 @@ static int idxd_wqs_setup(struct idxd_device *idxd)
struct device *dev = &idxd->pdev->dev;
for (i = 0; i < idxd->max_groups; i++) {
group = &idxd->groups[i];
group = idxd->groups[i];
for (j = 0; j < 4; j++)
group->grpcfg.wqs[j] = 0;
}
for (i = 0; i < idxd->max_wqs; i++) {
wq = &idxd->wqs[i];
wq = idxd->wqs[i];
group = wq->group;
if (!wq->group)
@ -865,3 +970,119 @@ int idxd_device_config(struct idxd_device *idxd)
return 0;
}
static int idxd_wq_load_config(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
int wqcfg_offset;
int i;
wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);
memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);
wq->size = wq->wqcfg->wq_size;
wq->threshold = wq->wqcfg->wq_thresh;
if (wq->wqcfg->priv)
wq->type = IDXD_WQT_KERNEL;
/* The driver does not support shared WQ mode in read-only config yet */
if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)
return -EOPNOTSUPP;
set_bit(WQ_FLAG_DEDICATED, &wq->flags);
wq->priority = wq->wqcfg->priority;
for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
}
return 0;
}
static void idxd_group_load_config(struct idxd_group *group)
{
struct idxd_device *idxd = group->idxd;
struct device *dev = &idxd->pdev->dev;
int i, j, grpcfg_offset;
/*
* Load WQS bit fields
* Iterate through all 256 bits 64 bits at a time
*/
for (i = 0; i < GRPWQCFG_STRIDES; i++) {
struct idxd_wq *wq;
grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
group->grpcfg.wqs[i] = ioread64(idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
group->id, i, grpcfg_offset, group->grpcfg.wqs[i]);
if (i * 64 >= idxd->max_wqs)
break;
/* Iterate through all 64 bits and check for wq set */
for (j = 0; j < 64; j++) {
int id = i * 64 + j;
/* No need to check beyond max wqs */
if (id >= idxd->max_wqs)
break;
/* Set group assignment for wq if wq bit is set */
if (group->grpcfg.wqs[i] & BIT(j)) {
wq = idxd->wqs[id];
wq->group = group;
}
}
}
grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
group->grpcfg.engines = ioread64(idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
grpcfg_offset, group->grpcfg.engines);
/* Iterate through all 64 bits to check engines set */
for (i = 0; i < 64; i++) {
if (i >= idxd->max_engines)
break;
if (group->grpcfg.engines & BIT(i)) {
struct idxd_engine *engine = idxd->engines[i];
engine->group = group;
}
}
grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset);
dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
group->id, grpcfg_offset, group->grpcfg.flags.bits);
}
int idxd_device_load_config(struct idxd_device *idxd)
{
union gencfg_reg reg;
int i, rc;
reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
idxd->token_limit = reg.token_limit;
for (i = 0; i < idxd->max_groups; i++) {
struct idxd_group *group = idxd->groups[i];
idxd_group_load_config(group);
}
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = idxd->wqs[i];
rc = idxd_wq_load_config(wq);
if (rc < 0)
return rc;
}
return 0;
}

77
drivers/dma/idxd/dma.c
View File

@ -14,7 +14,10 @@
static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
{
return container_of(c, struct idxd_wq, dma_chan);
struct idxd_dma_chan *idxd_chan;
idxd_chan = container_of(c, struct idxd_dma_chan, chan);
return idxd_chan->wq;
}
void idxd_dma_complete_txd(struct idxd_desc *desc,
@ -135,7 +138,7 @@ static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
{
}
dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct dma_chan *c = tx->chan;
struct idxd_wq *wq = to_idxd_wq(c);
@ -156,14 +159,25 @@ dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
static void idxd_dma_release(struct dma_device *device)
{
struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma);
kfree(idxd_dma);
}
int idxd_register_dma_device(struct idxd_device *idxd)
{
struct dma_device *dma = &idxd->dma_dev;
struct idxd_dma_dev *idxd_dma;
struct dma_device *dma;
struct device *dev = &idxd->pdev->dev;
int rc;
idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev));
if (!idxd_dma)
return -ENOMEM;
dma = &idxd_dma->dma;
INIT_LIST_HEAD(&dma->channels);
dma->dev = &idxd->pdev->dev;
dma->dev = dev;
dma_cap_set(DMA_PRIVATE, dma->cap_mask);
dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
@ -179,35 +193,72 @@ int idxd_register_dma_device(struct idxd_device *idxd)
dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
dma->device_free_chan_resources = idxd_dma_free_chan_resources;
return dma_async_device_register(&idxd->dma_dev);
rc = dma_async_device_register(dma);
if (rc < 0) {
kfree(idxd_dma);
return rc;
}
idxd_dma->idxd = idxd;
/*
* This pointer is protected by the refs taken by the dma_chan. It will remain valid
* as long as there are outstanding channels.
*/
idxd->idxd_dma = idxd_dma;
return 0;
}
void idxd_unregister_dma_device(struct idxd_device *idxd)
{
dma_async_device_unregister(&idxd->dma_dev);
dma_async_device_unregister(&idxd->idxd_dma->dma);
}
int idxd_register_dma_channel(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct dma_device *dma = &idxd->dma_dev;
struct dma_chan *chan = &wq->dma_chan;
int rc;
struct dma_device *dma = &idxd->idxd_dma->dma;
struct device *dev = &idxd->pdev->dev;
struct idxd_dma_chan *idxd_chan;
struct dma_chan *chan;
int rc, i;
memset(&wq->dma_chan, 0, sizeof(struct dma_chan));
idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev));
if (!idxd_chan)
return -ENOMEM;
chan = &idxd_chan->chan;
chan->device = dma;
list_add_tail(&chan->device_node, &dma->channels);
for (i = 0; i < wq->num_descs; i++) {
struct idxd_desc *desc = wq->descs[i];
dma_async_tx_descriptor_init(&desc->txd, chan);
desc->txd.tx_submit = idxd_dma_tx_submit;
}
rc = dma_async_device_channel_register(dma, chan);
if (rc < 0)
if (rc < 0) {
kfree(idxd_chan);
return rc;
}
wq->idxd_chan = idxd_chan;
idxd_chan->wq = wq;
get_device(&wq->conf_dev);
return 0;
}
void idxd_unregister_dma_channel(struct idxd_wq *wq)
{
struct dma_chan *chan = &wq->dma_chan;
struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
struct dma_chan *chan = &idxd_chan->chan;
struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma;
dma_async_device_channel_unregister(&wq->idxd->dma_dev, chan);
dma_async_device_channel_unregister(&idxd_dma->dma, chan);
list_del(&chan->device_node);
kfree(wq->idxd_chan);
wq->idxd_chan = NULL;
put_device(&wq->conf_dev);
}

168
drivers/dma/idxd/idxd.h
View File

@ -8,12 +8,18 @@
#include <linux/percpu-rwsem.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include <linux/idr.h>
#include <linux/pci.h>
#include <linux/perf_event.h>
#include "registers.h"
#define IDXD_DRIVER_VERSION "1.00"
extern struct kmem_cache *idxd_desc_pool;
struct idxd_device;
struct idxd_wq;
#define IDXD_REG_TIMEOUT 50
#define IDXD_DRAIN_TIMEOUT 5000
@ -25,6 +31,7 @@ enum idxd_type {
};
#define IDXD_NAME_SIZE 128
#define IDXD_PMU_EVENT_MAX 64
struct idxd_device_driver {
struct device_driver drv;
@ -33,6 +40,7 @@ struct idxd_device_driver {
struct idxd_irq_entry {
struct idxd_device *idxd;
int id;
int vector;
struct llist_head pending_llist;
struct list_head work_list;
/*
@ -56,6 +64,31 @@ struct idxd_group {
int tc_b;
};
struct idxd_pmu {
struct idxd_device *idxd;
struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
int n_events;
DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
struct pmu pmu;
char name[IDXD_NAME_SIZE];
int cpu;
int n_counters;
int counter_width;
int n_event_categories;
bool per_counter_caps_supported;
unsigned long supported_event_categories;
unsigned long supported_filters;
int n_filters;
struct hlist_node cpuhp_node;
};
#define IDXD_MAX_PRIORITY 0xf
enum idxd_wq_state {
@ -75,10 +108,10 @@ enum idxd_wq_type {
};
struct idxd_cdev {
struct idxd_wq *wq;
struct cdev cdev;
struct device *dev;
struct device dev;
int minor;
struct wait_queue_head err_queue;
};
#define IDXD_ALLOCATED_BATCH_SIZE 128U
@ -96,10 +129,18 @@ enum idxd_complete_type {
IDXD_COMPLETE_DEV_FAIL,
};
struct idxd_dma_chan {
struct dma_chan chan;
struct idxd_wq *wq;
};
struct idxd_wq {
void __iomem *portal;
struct percpu_ref wq_active;
struct completion wq_dead;
struct device conf_dev;
struct idxd_cdev idxd_cdev;
struct idxd_cdev *idxd_cdev;
struct wait_queue_head err_queue;
struct idxd_device *idxd;
int id;
enum idxd_wq_type type;
@ -125,7 +166,7 @@ struct idxd_wq {
int compls_size;
struct idxd_desc **descs;
struct sbitmap_queue sbq;
struct dma_chan dma_chan;
struct idxd_dma_chan *idxd_chan;
char name[WQ_NAME_SIZE + 1];
u64 max_xfer_bytes;
u32 max_batch_size;
@ -147,6 +188,7 @@ struct idxd_hw {
union group_cap_reg group_cap;
union engine_cap_reg engine_cap;
struct opcap opcap;
u32 cmd_cap;
};
enum idxd_device_state {
@ -162,9 +204,22 @@ enum idxd_device_flag {
IDXD_FLAG_PASID_ENABLED,
};
struct idxd_device {
struct idxd_dma_dev {
struct idxd_device *idxd;
struct dma_device dma;
};
struct idxd_driver_data {
const char *name_prefix;
enum idxd_type type;
struct device_type *dev_type;
int compl_size;
int align;
};
struct idxd_device {
struct device conf_dev;
struct idxd_driver_data *data;
struct list_head list;
struct idxd_hw hw;
enum idxd_device_state state;
@ -177,10 +232,11 @@ struct idxd_device {
void __iomem *reg_base;
spinlock_t dev_lock; /* spinlock for device */
spinlock_t cmd_lock; /* spinlock for device commands */
struct completion *cmd_done;
struct idxd_group *groups;
struct idxd_wq *wqs;
struct idxd_engine *engines;
struct idxd_group **groups;
struct idxd_wq **wqs;
struct idxd_engine **engines;
struct iommu_sva *sva;
unsigned int pasid;
@ -202,17 +258,19 @@ struct idxd_device {
int token_limit;
int nr_tokens; /* non-reserved tokens */
unsigned int wqcfg_size;
int compl_size;
union sw_err_reg sw_err;
wait_queue_head_t cmd_waitq;
struct msix_entry *msix_entries;
int num_wq_irqs;
struct idxd_irq_entry *irq_entries;
struct dma_device dma_dev;
struct idxd_dma_dev *idxd_dma;
struct workqueue_struct *wq;
struct work_struct work;
int *int_handles;
struct idxd_pmu *idxd_pmu;
};
/* IDXD software descriptor */
@ -232,6 +290,7 @@ struct idxd_desc {
struct list_head list;
int id;
int cpu;
unsigned int vector;
struct idxd_wq *wq;
};
@ -242,6 +301,44 @@ extern struct bus_type dsa_bus_type;
extern struct bus_type iax_bus_type;
extern bool support_enqcmd;
extern struct ida idxd_ida;
extern struct device_type dsa_device_type;
extern struct device_type iax_device_type;
extern struct device_type idxd_wq_device_type;
extern struct device_type idxd_engine_device_type;
extern struct device_type idxd_group_device_type;
static inline bool is_dsa_dev(struct device *dev)
{
return dev->type == &dsa_device_type;
}
static inline bool is_iax_dev(struct device *dev)
{
return dev->type == &iax_device_type;
}
static inline bool is_idxd_dev(struct device *dev)
{
return is_dsa_dev(dev) || is_iax_dev(dev);
}
static inline bool is_idxd_wq_dev(struct device *dev)
{
return dev->type == &idxd_wq_device_type;
}
static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
{
if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
return true;
return false;
}
static inline bool is_idxd_wq_cdev(struct idxd_wq *wq)
{
return wq->type == IDXD_WQT_USER;
}
static inline bool wq_dedicated(struct idxd_wq *wq)
{
@ -268,6 +365,11 @@ enum idxd_portal_prot {
IDXD_PORTAL_LIMITED,
};
enum idxd_interrupt_type {
IDXD_IRQ_MSIX = 0,
IDXD_IRQ_IMS,
};
static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
{
return prot * 0x1000;
@ -279,18 +381,6 @@ static inline int idxd_get_wq_portal_full_offset(int wq_id,
return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
}
static inline void idxd_set_type(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
idxd->type = IDXD_TYPE_DSA;
else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0)
idxd->type = IDXD_TYPE_IAX;
else
idxd->type = IDXD_TYPE_UNKNOWN;
}
static inline void idxd_wq_get(struct idxd_wq *wq)
{
wq->client_count++;
@ -306,19 +396,17 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq)
return wq->client_count;
};
const char *idxd_get_dev_name(struct idxd_device *idxd);
int idxd_register_bus_type(void);
void idxd_unregister_bus_type(void);
int idxd_setup_sysfs(struct idxd_device *idxd);
void idxd_cleanup_sysfs(struct idxd_device *idxd);
int idxd_register_devices(struct idxd_device *idxd);
void idxd_unregister_devices(struct idxd_device *idxd);
int idxd_register_driver(void);
void idxd_unregister_driver(void);
struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
void idxd_wqs_quiesce(struct idxd_device *idxd);
/* device interrupt control */
void idxd_msix_perm_setup(struct idxd_device *idxd);
void idxd_msix_perm_clear(struct idxd_device *idxd);
irqreturn_t idxd_irq_handler(int vec, void *data);
irqreturn_t idxd_misc_thread(int vec, void *data);
irqreturn_t idxd_wq_thread(int irq, void *data);
void idxd_mask_error_interrupts(struct idxd_device *idxd);
@ -336,8 +424,14 @@ void idxd_device_cleanup(struct idxd_device *idxd);
int idxd_device_config(struct idxd_device *idxd);
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
int idxd_device_load_config(struct idxd_device *idxd);
int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
enum idxd_interrupt_type irq_type);
int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
enum idxd_interrupt_type irq_type);
/* work queue control */
void idxd_wqs_unmap_portal(struct idxd_device *idxd);
int idxd_wq_alloc_resources(struct idxd_wq *wq);
void idxd_wq_free_resources(struct idxd_wq *wq);
int idxd_wq_enable(struct idxd_wq *wq);
@ -349,6 +443,8 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq);
void idxd_wq_disable_cleanup(struct idxd_wq *wq);
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
int idxd_wq_disable_pasid(struct idxd_wq *wq);
void idxd_wq_quiesce(struct idxd_wq *wq);
int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
/* submission */
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
@ -363,7 +459,6 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq);
void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
void idxd_dma_complete_txd(struct idxd_desc *desc,
enum idxd_complete_type comp_type);
dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx);
/* cdev */
int idxd_cdev_register(void);
@ -372,4 +467,19 @@ int idxd_cdev_get_major(struct idxd_device *idxd);
int idxd_wq_add_cdev(struct idxd_wq *wq);
void idxd_wq_del_cdev(struct idxd_wq *wq);
/* perfmon */
#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
int perfmon_pmu_init(struct idxd_device *idxd);
void perfmon_pmu_remove(struct idxd_device *idxd);
void perfmon_counter_overflow(struct idxd_device *idxd);
void perfmon_init(void);
void perfmon_exit(void);
#else
static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
static inline void perfmon_init(void) {}
static inline void perfmon_exit(void) {}
#endif
#endif

499
drivers/dma/idxd/init.c
View File

@ -21,6 +21,7 @@
#include "../dmaengine.h"
#include "registers.h"
#include "idxd.h"
#include "perfmon.h"
MODULE_VERSION(IDXD_DRIVER_VERSION);
MODULE_LICENSE("GPL v2");
@ -33,35 +34,39 @@ MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
#define DRV_NAME "idxd"
bool support_enqcmd;
DEFINE_IDA(idxd_ida);
static struct idr idxd_idrs[IDXD_TYPE_MAX];
static DEFINE_MUTEX(idxd_idr_lock);
static struct idxd_driver_data idxd_driver_data[] = {
[IDXD_TYPE_DSA] = {
.name_prefix = "dsa",
.type = IDXD_TYPE_DSA,
.compl_size = sizeof(struct dsa_completion_record),
.align = 32,
.dev_type = &dsa_device_type,
},
[IDXD_TYPE_IAX] = {
.name_prefix = "iax",
.type = IDXD_TYPE_IAX,
.compl_size = sizeof(struct iax_completion_record),
.align = 64,
.dev_type = &iax_device_type,
},
};
static struct pci_device_id idxd_pci_tbl[] = {
/* DSA ver 1.0 platforms */
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) },
{ PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) },
/* IAX ver 1.0 platforms */
{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) },
{ PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) },
{ 0, }
};
MODULE_DEVICE_TABLE(pci, idxd_pci_tbl);
static char *idxd_name[] = {
"dsa",
"iax"
};
const char *idxd_get_dev_name(struct idxd_device *idxd)
{
return idxd_name[idxd->type];
}
static int idxd_setup_interrupts(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
struct device *dev = &pdev->dev;
struct msix_entry *msix;
struct idxd_irq_entry *irq_entry;
int i, msixcnt;
int rc = 0;
@ -69,23 +74,13 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
msixcnt = pci_msix_vec_count(pdev);
if (msixcnt < 0) {
dev_err(dev, "Not MSI-X interrupt capable.\n");
goto err_no_irq;
return -ENOSPC;
}
idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) *
msixcnt, GFP_KERNEL);
if (!idxd->msix_entries) {
rc = -ENOMEM;
goto err_no_irq;
}
for (i = 0; i < msixcnt; i++)
idxd->msix_entries[i].entry = i;
rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt);
if (rc) {
dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt);
goto err_no_irq;
rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX);
if (rc != msixcnt) {
dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc);
return -ENOSPC;
}
dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt);
@ -93,119 +88,266 @@ static int idxd_setup_interrupts(struct idxd_device *idxd)
* We implement 1 completion list per MSI-X entry except for
* entry 0, which is for errors and others.
*/
idxd->irq_entries = devm_kcalloc(dev, msixcnt,
sizeof(struct idxd_irq_entry),
GFP_KERNEL);
idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry),
GFP_KERNEL, dev_to_node(dev));
if (!idxd->irq_entries) {
rc = -ENOMEM;
goto err_no_irq;
goto err_irq_entries;
}
for (i = 0; i < msixcnt; i++) {
idxd->irq_entries[i].id = i;
idxd->irq_entries[i].idxd = idxd;
idxd->irq_entries[i].vector = pci_irq_vector(pdev, i);
spin_lock_init(&idxd->irq_entries[i].list_lock);
}
msix = &idxd->msix_entries[0];
irq_entry = &idxd->irq_entries[0];
rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler,
idxd_misc_thread, 0, "idxd-misc",
irq_entry);
rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread,
0, "idxd-misc", irq_entry);
if (rc < 0) {
dev_err(dev, "Failed to allocate misc interrupt.\n");
goto err_no_irq;
goto err_misc_irq;
}
dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n",
msix->vector);
dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector);
/* first MSI-X entry is not for wq interrupts */
idxd->num_wq_irqs = msixcnt - 1;
for (i = 1; i < msixcnt; i++) {
msix = &idxd->msix_entries[i];
irq_entry = &idxd->irq_entries[i];
init_llist_head(&idxd->irq_entries[i].pending_llist);
INIT_LIST_HEAD(&idxd->irq_entries[i].work_list);
rc = devm_request_threaded_irq(dev, msix->vector,
idxd_irq_handler,
idxd_wq_thread, 0,
"idxd-portal", irq_entry);
rc = request_threaded_irq(irq_entry->vector, NULL,
idxd_wq_thread, 0, "idxd-portal", irq_entry);
if (rc < 0) {
dev_err(dev, "Failed to allocate irq %d.\n",
msix->vector);
goto err_no_irq;
dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector);
goto err_wq_irqs;
}
dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector);
if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) {
/*
* The MSIX vector enumeration starts at 1 with vector 0 being the
* misc interrupt that handles non I/O completion events. The
* interrupt handles are for IMS enumeration on guest. The misc
* interrupt vector does not require a handle and therefore we start
* the int_handles at index 0. Since 'i' starts at 1, the first
* int_handles index will be 0.
*/
rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1],
IDXD_IRQ_MSIX);
if (rc < 0) {
free_irq(irq_entry->vector, irq_entry);
goto err_wq_irqs;
}
dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]);
}
dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n",
i, msix->vector);
}
idxd_unmask_error_interrupts(idxd);
idxd_msix_perm_setup(idxd);
return 0;
err_no_irq:
err_wq_irqs:
while (--i >= 0) {
irq_entry = &idxd->irq_entries[i];
free_irq(irq_entry->vector, irq_entry);
if (i != 0)
idxd_device_release_int_handle(idxd,
idxd->int_handles[i], IDXD_IRQ_MSIX);
}
err_misc_irq:
/* Disable error interrupt generation */
idxd_mask_error_interrupts(idxd);
pci_disable_msix(pdev);
err_irq_entries:
pci_free_irq_vectors(pdev);
dev_err(dev, "No usable interrupts\n");
return rc;
}
static int idxd_setup_wqs(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
struct idxd_wq *wq;
int i, rc;
idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
GFP_KERNEL, dev_to_node(dev));
if (!idxd->wqs)
return -ENOMEM;
for (i = 0; i < idxd->max_wqs; i++) {
wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev));
if (!wq) {
rc = -ENOMEM;
goto err;
}
wq->id = i;
wq->idxd = idxd;
device_initialize(&wq->conf_dev);
wq->conf_dev.parent = &idxd->conf_dev;
wq->conf_dev.bus = &dsa_bus_type;
wq->conf_dev.type = &idxd_wq_device_type;
rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id);
if (rc < 0) {
put_device(&wq->conf_dev);
goto err;
}
mutex_init(&wq->wq_lock);
init_waitqueue_head(&wq->err_queue);
init_completion(&wq->wq_dead);
wq->max_xfer_bytes = idxd->max_xfer_bytes;
wq->max_batch_size = idxd->max_batch_size;
wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
if (!wq->wqcfg) {
put_device(&wq->conf_dev);
rc = -ENOMEM;
goto err;
}
idxd->wqs[i] = wq;
}
return 0;
err:
while (--i >= 0)
put_device(&idxd->wqs[i]->conf_dev);
return rc;
}
static int idxd_setup_engines(struct idxd_device *idxd)
{
struct idxd_engine *engine;
struct device *dev = &idxd->pdev->dev;
int i, rc;
idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
GFP_KERNEL, dev_to_node(dev));
if (!idxd->engines)
return -ENOMEM;
for (i = 0; i < idxd->max_engines; i++) {
engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev));
if (!engine) {
rc = -ENOMEM;
goto err;
}
engine->id = i;
engine->idxd = idxd;
device_initialize(&engine->conf_dev);
engine->conf_dev.parent = &idxd->conf_dev;
engine->conf_dev.type = &idxd_engine_device_type;
rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id);
if (rc < 0) {
put_device(&engine->conf_dev);
goto err;
}
idxd->engines[i] = engine;
}
return 0;
err:
while (--i >= 0)
put_device(&idxd->engines[i]->conf_dev);
return rc;
}
static int idxd_setup_groups(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
struct idxd_group *group;
int i, rc;
idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *),
GFP_KERNEL, dev_to_node(dev));
if (!idxd->groups)
return -ENOMEM;
for (i = 0; i < idxd->max_groups; i++) {
group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev));
if (!group) {
rc = -ENOMEM;
goto err;
}
group->id = i;
group->idxd = idxd;
device_initialize(&group->conf_dev);
group->conf_dev.parent = &idxd->conf_dev;
group->conf_dev.bus = &dsa_bus_type;
group->conf_dev.type = &idxd_group_device_type;
rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id);
if (rc < 0) {
put_device(&group->conf_dev);
goto err;
}
idxd->groups[i] = group;
group->tc_a = -1;
group->tc_b = -1;
}
return 0;
err:
while (--i >= 0)
put_device(&idxd->groups[i]->conf_dev);
return rc;
}
static int idxd_setup_internals(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int i;
int rc, i;
init_waitqueue_head(&idxd->cmd_waitq);
idxd->groups = devm_kcalloc(dev, idxd->max_groups,
sizeof(struct idxd_group), GFP_KERNEL);
if (!idxd->groups)
return -ENOMEM;
for (i = 0; i < idxd->max_groups; i++) {
idxd->groups[i].idxd = idxd;
idxd->groups[i].id = i;
idxd->groups[i].tc_a = -1;
idxd->groups[i].tc_b = -1;
}
idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq),
GFP_KERNEL);
if (!idxd->wqs)
return -ENOMEM;
idxd->engines = devm_kcalloc(dev, idxd->max_engines,
sizeof(struct idxd_engine), GFP_KERNEL);
if (!idxd->engines)
return -ENOMEM;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
wq->id = i;
wq->idxd = idxd;
mutex_init(&wq->wq_lock);
wq->idxd_cdev.minor = -1;
wq->max_xfer_bytes = idxd->max_xfer_bytes;
wq->max_batch_size = idxd->max_batch_size;
wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL);
if (!wq->wqcfg)
if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) {
idxd->int_handles = devm_kcalloc(dev, idxd->max_wqs, sizeof(int), GFP_KERNEL);
if (!idxd->int_handles)
return -ENOMEM;
}
for (i = 0; i < idxd->max_engines; i++) {
idxd->engines[i].idxd = idxd;
idxd->engines[i].id = i;
}
rc = idxd_setup_wqs(idxd);
if (rc < 0)
goto err_wqs;
rc = idxd_setup_engines(idxd);
if (rc < 0)
goto err_engine;
rc = idxd_setup_groups(idxd);
if (rc < 0)
goto err_group;
idxd->wq = create_workqueue(dev_name(dev));
if (!idxd->wq)
return -ENOMEM;
if (!idxd->wq) {
rc = -ENOMEM;
goto err_wkq_create;
}
return 0;
err_wkq_create:
for (i = 0; i < idxd->max_groups; i++)
put_device(&idxd->groups[i]->conf_dev);
err_group:
for (i = 0; i < idxd->max_engines; i++)
put_device(&idxd->engines[i]->conf_dev);
err_engine:
for (i = 0; i < idxd->max_wqs; i++)
put_device(&idxd->wqs[i]->conf_dev);
err_wqs:
kfree(idxd->int_handles);
return rc;
}
static void idxd_read_table_offsets(struct idxd_device *idxd)
@ -233,6 +375,12 @@ static void idxd_read_caps(struct idxd_device *idxd)
/* reading generic capabilities */
idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET);
dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits);
if (idxd->hw.gen_cap.cmd_cap) {
idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET);
dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap);
}
idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift;
dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes);
idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift;
@ -275,17 +423,34 @@ static void idxd_read_caps(struct idxd_device *idxd)
}
}
static struct idxd_device *idxd_alloc(struct pci_dev *pdev)
static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
{
struct device *dev = &pdev->dev;
struct idxd_device *idxd;
int rc;
idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL);
idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev));
if (!idxd)
return NULL;
idxd->pdev = pdev;
idxd->data = data;
idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
if (idxd->id < 0)
return NULL;
device_initialize(&idxd->conf_dev);
idxd->conf_dev.parent = dev;
idxd->conf_dev.bus = &dsa_bus_type;
idxd->conf_dev.type = idxd->data->dev_type;
rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
if (rc < 0) {
put_device(&idxd->conf_dev);
return NULL;
}
spin_lock_init(&idxd->dev_lock);
spin_lock_init(&idxd->cmd_lock);
return idxd;
}
@ -338,11 +503,18 @@ static int idxd_probe(struct idxd_device *idxd)
dev_dbg(dev, "IDXD reset complete\n");
if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
rc = idxd_enable_system_pasid(idxd);
if (rc < 0)
dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
else
set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA);
if (rc == 0) {
rc = idxd_enable_system_pasid(idxd);
if (rc < 0) {
iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
} else {
set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
}
} else {
dev_warn(dev, "Unable to turn on SVA feature.\n");
}
} else if (!sva) {
dev_warn(dev, "User forced SVA off via module param.\n");
}
@ -352,80 +524,75 @@ static int idxd_probe(struct idxd_device *idxd)
rc = idxd_setup_internals(idxd);
if (rc)
goto err_setup;
goto err;
/* If the configs are readonly, then load them from device */
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
dev_dbg(dev, "Loading RO device config\n");
rc = idxd_device_load_config(idxd);
if (rc < 0)
goto err;
}
rc = idxd_setup_interrupts(idxd);
if (rc)
goto err_setup;
goto err;
dev_dbg(dev, "IDXD interrupt setup complete.\n");
mutex_lock(&idxd_idr_lock);
idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL);
mutex_unlock(&idxd_idr_lock);
if (idxd->id < 0) {
rc = -ENOMEM;
goto err_idr_fail;
}
idxd->major = idxd_cdev_get_major(idxd);
rc = perfmon_pmu_init(idxd);
if (rc < 0)
dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc);
dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id);
return 0;
err_idr_fail:
idxd_mask_error_interrupts(idxd);
idxd_mask_msix_vectors(idxd);
err_setup:
err:
if (device_pasid_enabled(idxd))
idxd_disable_system_pasid(idxd);
iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
return rc;
}
static void idxd_type_init(struct idxd_device *idxd)
{
if (idxd->type == IDXD_TYPE_DSA)
idxd->compl_size = sizeof(struct dsa_completion_record);
else if (idxd->type == IDXD_TYPE_IAX)
idxd->compl_size = sizeof(struct iax_completion_record);
}
static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct device *dev = &pdev->dev;
struct idxd_device *idxd;
struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data;
int rc;
rc = pcim_enable_device(pdev);
rc = pci_enable_device(pdev);
if (rc)
return rc;
dev_dbg(dev, "Alloc IDXD context\n");
idxd = idxd_alloc(pdev);
if (!idxd)
return -ENOMEM;
idxd = idxd_alloc(pdev, data);
if (!idxd) {
rc = -ENOMEM;
goto err_idxd_alloc;
}
dev_dbg(dev, "Mapping BARs\n");
idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0);
if (!idxd->reg_base)
return -ENOMEM;
idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0);
if (!idxd->reg_base) {
rc = -ENOMEM;
goto err_iomap;
}
dev_dbg(dev, "Set DMA masks\n");
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc)
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
return rc;
goto err;
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc)
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
return rc;
idxd_set_type(idxd);
idxd_type_init(idxd);
goto err;
dev_dbg(dev, "Set PCI master\n");
pci_set_master(pdev);
@ -435,13 +602,13 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
rc = idxd_probe(idxd);
if (rc) {
dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n");
return -ENODEV;
goto err;
}
rc = idxd_setup_sysfs(idxd);
rc = idxd_register_devices(idxd);
if (rc) {
dev_err(dev, "IDXD sysfs setup failed\n");
return -ENODEV;
goto err;
}
idxd->state = IDXD_DEV_CONF_READY;
@ -450,6 +617,14 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
idxd->hw.version);
return 0;
err:
pci_iounmap(pdev, idxd->reg_base);
err_iomap:
put_device(&idxd->conf_dev);
err_idxd_alloc:
pci_disable_device(pdev);
return rc;
}
static void idxd_flush_pending_llist(struct idxd_irq_entry *ie)
@ -478,6 +653,36 @@ static void idxd_flush_work_list(struct idxd_irq_entry *ie)
}
}
void idxd_wqs_quiesce(struct idxd_device *idxd)
{
struct idxd_wq *wq;
int i;
for (i = 0; i < idxd->max_wqs; i++) {
wq = idxd->wqs[i];
if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL)
idxd_wq_quiesce(wq);
}
}
static void idxd_release_int_handles(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
int i, rc;
for (i = 0; i < idxd->num_wq_irqs; i++) {
if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) {
rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i],
IDXD_IRQ_MSIX);
if (rc < 0)
dev_warn(dev, "irq handle %d release failed\n",
idxd->int_handles[i]);
else
dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]);
}
}
}
static void idxd_shutdown(struct pci_dev *pdev)
{
struct idxd_device *idxd = pci_get_drvdata(pdev);
@ -495,7 +700,8 @@ static void idxd_shutdown(struct pci_dev *pdev)
for (i = 0; i < msixcnt; i++) {
irq_entry = &idxd->irq_entries[i];
synchronize_irq(idxd->msix_entries[i].vector);
synchronize_irq(irq_entry->vector);
free_irq(irq_entry->vector, irq_entry);
if (i == 0)
continue;
idxd_flush_pending_llist(irq_entry);
@ -503,6 +709,10 @@ static void idxd_shutdown(struct pci_dev *pdev)
}
idxd_msix_perm_clear(idxd);
idxd_release_int_handles(idxd);
pci_free_irq_vectors(pdev);
pci_iounmap(pdev, idxd->reg_base);
pci_disable_device(pdev);
destroy_workqueue(idxd->wq);
}
@ -511,13 +721,12 @@ static void idxd_remove(struct pci_dev *pdev)
struct idxd_device *idxd = pci_get_drvdata(pdev);
dev_dbg(&pdev->dev, "%s called\n", __func__);
idxd_cleanup_sysfs(idxd);
idxd_shutdown(pdev);
if (device_pasid_enabled(idxd))
idxd_disable_system_pasid(idxd);
mutex_lock(&idxd_idr_lock);
idr_remove(&idxd_idrs[idxd->type], idxd->id);
mutex_unlock(&idxd_idr_lock);
idxd_unregister_devices(idxd);
perfmon_pmu_remove(idxd);
iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
}
static struct pci_driver idxd_pci_driver = {
@ -530,7 +739,7 @@ static struct pci_driver idxd_pci_driver = {
static int __init idxd_init_module(void)
{
int err, i;
int err;
/*
* If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in
@ -546,8 +755,7 @@ static int __init idxd_init_module(void)
else
support_enqcmd = true;
for (i = 0; i < IDXD_TYPE_MAX; i++)
idr_init(&idxd_idrs[i]);
perfmon_init();
err = idxd_register_bus_type();
if (err < 0)
@ -582,5 +790,6 @@ static void __exit idxd_exit_module(void)
pci_unregister_driver(&idxd_pci_driver);
idxd_cdev_remove();
idxd_unregister_bus_type();
perfmon_exit();
}
module_exit(idxd_exit_module);

29
drivers/dma/idxd/irq.c
View File

@ -45,7 +45,7 @@ static void idxd_device_reinit(struct work_struct *work)
goto out;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
struct idxd_wq *wq = idxd->wqs[i];
if (wq->state == IDXD_WQ_ENABLED) {
rc = idxd_wq_enable(wq);
@ -102,15 +102,6 @@ static int idxd_device_schedule_fault_process(struct idxd_device *idxd,
return 0;
}
irqreturn_t idxd_irq_handler(int vec, void *data)
{
struct idxd_irq_entry *irq_entry = data;
struct idxd_device *idxd = irq_entry->idxd;
idxd_mask_msix_vector(idxd, irq_entry->id);
return IRQ_WAKE_THREAD;
}
static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
{
struct device *dev = &idxd->pdev->dev;
@ -130,18 +121,18 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) {
int id = idxd->sw_err.wq_idx;
struct idxd_wq *wq = &idxd->wqs[id];
struct idxd_wq *wq = idxd->wqs[id];
if (wq->type == IDXD_WQT_USER)
wake_up_interruptible(&wq->idxd_cdev.err_queue);
wake_up_interruptible(&wq->err_queue);
} else {
int i;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = &idxd->wqs[i];
struct idxd_wq *wq = idxd->wqs[i];
if (wq->type == IDXD_WQT_USER)
wake_up_interruptible(&wq->idxd_cdev.err_queue);
wake_up_interruptible(&wq->err_queue);
}
}
@ -165,11 +156,8 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
}
if (cause & IDXD_INTC_PERFMON_OVFL) {
/*
* Driver does not utilize perfmon counter overflow interrupt
* yet.
*/
val |= IDXD_INTC_PERFMON_OVFL;
perfmon_counter_overflow(idxd);
}
val ^= cause;
@ -202,6 +190,8 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
queue_work(idxd->wq, &idxd->work);
} else {
spin_lock_bh(&idxd->dev_lock);
idxd_wqs_quiesce(idxd);
idxd_wqs_unmap_portal(idxd);
idxd_device_wqs_clear_state(idxd);
dev_err(&idxd->pdev->dev,
"idxd halted, need %s.\n",
@ -235,7 +225,6 @@ irqreturn_t idxd_misc_thread(int vec, void *data)
iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
}
idxd_unmask_msix_vector(idxd, irq_entry->id);
return IRQ_HANDLED;
}
@ -392,8 +381,6 @@ irqreturn_t idxd_wq_thread(int irq, void *data)
int processed;
processed = idxd_desc_process(irq_entry);
idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id);
if (processed == 0)
return IRQ_NONE;

662
drivers/dma/idxd/perfmon.c Normal file
View File

@ -0,0 +1,662 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
#include <linux/sched/task.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include "idxd.h"
#include "perfmon.h"
static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
char *buf);
static cpumask_t perfmon_dsa_cpu_mask;
static bool cpuhp_set_up;
static enum cpuhp_state cpuhp_slot;
/*
* perf userspace reads this attribute to determine which cpus to open
* counters on. It's connected to perfmon_dsa_cpu_mask, which is
* maintained by the cpu hotplug handlers.
*/
static DEVICE_ATTR_RO(cpumask);
static struct attribute *perfmon_cpumask_attrs[] = {
&dev_attr_cpumask.attr,
NULL,
};
static struct attribute_group cpumask_attr_group = {
.attrs = perfmon_cpumask_attrs,
};
/*
* These attributes specify the bits in the config word that the perf
* syscall uses to pass the event ids and categories to perfmon.
*/
DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
/*
* These attributes specify the bits in the config1 word that the perf
* syscall uses to pass filter data to perfmon.
*/
DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
#define PERFMON_FILTERS_START 2
#define PERFMON_FILTERS_MAX 5
static struct attribute *perfmon_format_attrs[] = {
&format_attr_idxd_event_category.attr,
&format_attr_idxd_event.attr,
&format_attr_idxd_filter_wq.attr,
&format_attr_idxd_filter_tc.attr,
&format_attr_idxd_filter_pgsz.attr,
&format_attr_idxd_filter_sz.attr,
&format_attr_idxd_filter_eng.attr,
NULL,
};
static struct attribute_group perfmon_format_attr_group = {
.name = "format",
.attrs = perfmon_format_attrs,
};
static const struct attribute_group *perfmon_attr_groups[] = {
&perfmon_format_attr_group,
&cpumask_attr_group,
NULL,
};
static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask);
}
static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
{
return &idxd_pmu->pmu == event->pmu;
}
static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
struct perf_event *leader,
bool do_grp)
{
struct perf_event *event;
int n, max_count;
max_count = idxd_pmu->n_counters;
n = idxd_pmu->n_events;
if (n >= max_count)
return -EINVAL;
if (is_idxd_event(idxd_pmu, leader)) {
idxd_pmu->event_list[n] = leader;
idxd_pmu->event_list[n]->hw.idx = n;
n++;
}
if (!do_grp)
return n;
for_each_sibling_event(event, leader) {
if (!is_idxd_event(idxd_pmu, event) ||
event->state <= PERF_EVENT_STATE_OFF)
continue;
if (n >= max_count)
return -EINVAL;
idxd_pmu->event_list[n] = event;
idxd_pmu->event_list[n]->hw.idx = n;
n++;
}
return n;
}
static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
struct perf_event *event, int idx)
{
struct idxd_device *idxd = idxd_pmu->idxd;
struct hw_perf_event *hwc = &event->hw;
hwc->idx = idx;
hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
}
static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
struct perf_event *event)
{
int i;
for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
if (!test_and_set_bit(i, idxd_pmu->used_mask))
return i;
return -EINVAL;
}
/*
* Check whether there are enough counters to satisfy that all the
* events in the group can actually be scheduled at the same time.
*
* To do this, create a fake idxd_pmu object so the event collection
* and assignment functions can be used without affecting the internal
* state of the real idxd_pmu object.
*/
static int perfmon_validate_group(struct idxd_pmu *pmu,
struct perf_event *event)
{
struct perf_event *leader = event->group_leader;
struct idxd_pmu *fake_pmu;
int i, ret = 0, n, idx;
fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
if (!fake_pmu)
return -ENOMEM;
fake_pmu->pmu.name = pmu->pmu.name;
fake_pmu->n_counters = pmu->n_counters;
n = perfmon_collect_events(fake_pmu, leader, true);
if (n < 0) {
ret = n;
goto out;
}
fake_pmu->n_events = n;
n = perfmon_collect_events(fake_pmu, event, false);
if (n < 0) {
ret = n;
goto out;
}
fake_pmu->n_events = n;
for (i = 0; i < n; i++) {
event = fake_pmu->event_list[i];
idx = perfmon_assign_event(fake_pmu, event);
if (idx < 0) {
ret = idx;
goto out;
}
}
out:
kfree(fake_pmu);
return ret;
}
static int perfmon_pmu_event_init(struct perf_event *event)
{
struct idxd_device *idxd;
int ret = 0;
idxd = event_to_idxd(event);
event->hw.idx = -1;
if (event->attr.type != event->pmu->type)
return -ENOENT;
/* sampling not supported */
if (event->attr.sample_period)
return -EINVAL;
if (event->cpu < 0)
return -EINVAL;
if (event->pmu != &idxd->idxd_pmu->pmu)
return -EINVAL;
event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
event->cpu = idxd->idxd_pmu->cpu;
event->hw.config = event->attr.config;
if (event->group_leader != event)
/* non-group events have themselves as leader */
ret = perfmon_validate_group(idxd->idxd_pmu, event);
return ret;
}
static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
struct idxd_device *idxd;
int cntr = hwc->idx;
idxd = event_to_idxd(event);
return ioread64(CNTRDATA_REG(idxd, cntr));
}
static void perfmon_pmu_event_update(struct perf_event *event)
{
struct idxd_device *idxd = event_to_idxd(event);
u64 prev_raw_count, new_raw_count, delta, p, n;
int shift = 64 - idxd->idxd_pmu->counter_width;
struct hw_perf_event *hwc = &event->hw;
do {
prev_raw_count = local64_read(&hwc->prev_count);
new_raw_count = perfmon_pmu_read_counter(event);
} while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
new_raw_count) != prev_raw_count);
n = (new_raw_count << shift);
p = (prev_raw_count << shift);
delta = ((n - p) >> shift);
local64_add(delta, &event->count);
}
void perfmon_counter_overflow(struct idxd_device *idxd)
{
int i, n_counters, max_loop = OVERFLOW_SIZE;
struct perf_event *event;
unsigned long ovfstatus;
n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
ovfstatus = ioread32(OVFSTATUS_REG(idxd));
/*
* While updating overflowed counters, other counters behind
* them could overflow and be missed in a given pass.
* Normally this could happen at most n_counters times, but in
* theory a tiny counter width could result in continual
* overflows and endless looping. max_loop provides a
* failsafe in that highly unlikely case.
*/
while (ovfstatus && max_loop--) {
/* Figure out which counter(s) overflowed */
for_each_set_bit(i, &ovfstatus, n_counters) {
unsigned long ovfstatus_clear = 0;
/* Update event->count for overflowed counter */
event = idxd->idxd_pmu->event_list[i];
perfmon_pmu_event_update(event);
/* Writing 1 to OVFSTATUS bit clears it */
set_bit(i, &ovfstatus_clear);
iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
}
ovfstatus = ioread32(OVFSTATUS_REG(idxd));
}
/*
* Should never happen. If so, it means a counter(s) looped
* around twice while this handler was running.
*/
WARN_ON_ONCE(ovfstatus);
}
static inline void perfmon_reset_config(struct idxd_device *idxd)
{
iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
iowrite32(0, OVFSTATUS_REG(idxd));
iowrite32(0, PERFFRZ_REG(idxd));
}
static inline void perfmon_reset_counters(struct idxd_device *idxd)
{
iowrite32(CNTR_RESET, PERFRST_REG(idxd));
}
static inline void perfmon_reset(struct idxd_device *idxd)
{
perfmon_reset_config(idxd);
perfmon_reset_counters(idxd);
}
static void perfmon_pmu_event_start(struct perf_event *event, int mode)
{
u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
struct hw_perf_event *hwc = &event->hw;
union filter_cfg flt_cfg;
union event_cfg event_cfg;
struct idxd_device *idxd;
int cntr;
idxd = event_to_idxd(event);
event->hw.idx = hwc->idx;
cntr = hwc->idx;
/* Obtain event category and event value from user space */
event_cfg.val = event->attr.config;
flt_cfg.val = event->attr.config1;
event_cat = event_cfg.event_cat;
event_enc = event_cfg.event_enc;
/* Obtain filter configuration from user space */
flt_wq = flt_cfg.wq;
flt_tc = flt_cfg.tc;
flt_pg_sz = flt_cfg.pg_sz;
flt_xfer_sz = flt_cfg.xfer_sz;
flt_eng = flt_cfg.eng;
if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
/* Read the start value */
cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
local64_set(&event->hw.prev_count, cntrdata);
/* Set counter to event/category */
cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
/* Set interrupt on overflow and counter enable bits */
cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
}
static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
{
struct hw_perf_event *hwc = &event->hw;
struct idxd_device *idxd;
int i, cntr = hwc->idx;
u64 cntr_cfg;
idxd = event_to_idxd(event);
/* remove this event from event list */
for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
if (event != idxd->idxd_pmu->event_list[i])
continue;
for (++i; i < idxd->idxd_pmu->n_events; i++)
idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
--idxd->idxd_pmu->n_events;
break;
}
cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
cntr_cfg &= ~CNTRCFG_ENABLE;
iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
if (mode == PERF_EF_UPDATE)
perfmon_pmu_event_update(event);
event->hw.idx = -1;
clear_bit(cntr, idxd->idxd_pmu->used_mask);
}
static void perfmon_pmu_event_del(struct perf_event *event, int mode)
{
perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
}
static int perfmon_pmu_event_add(struct perf_event *event, int flags)
{
struct idxd_device *idxd = event_to_idxd(event);
struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
struct hw_perf_event *hwc = &event->hw;
int idx, n;
n = perfmon_collect_events(idxd_pmu, event, false);
if (n < 0)
return n;
hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
if (!(flags & PERF_EF_START))
hwc->state |= PERF_HES_ARCH;
idx = perfmon_assign_event(idxd_pmu, event);
if (idx < 0)
return idx;
perfmon_assign_hw_event(idxd_pmu, event, idx);
if (flags & PERF_EF_START)
perfmon_pmu_event_start(event, 0);
idxd_pmu->n_events = n;
return 0;
}
static void enable_perfmon_pmu(struct idxd_device *idxd)
{
iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
}
static void disable_perfmon_pmu(struct idxd_device *idxd)
{
iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
}
static void perfmon_pmu_enable(struct pmu *pmu)
{
struct idxd_device *idxd = pmu_to_idxd(pmu);
enable_perfmon_pmu(idxd);
}
static void perfmon_pmu_disable(struct pmu *pmu)
{
struct idxd_device *idxd = pmu_to_idxd(pmu);
disable_perfmon_pmu(idxd);
}
static void skip_filter(int i)
{
int j;
for (j = i; j < PERFMON_FILTERS_MAX; j++)
perfmon_format_attrs[PERFMON_FILTERS_START + j] =
perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
}
static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
{
int i;
for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
if (!test_bit(i, &idxd_pmu->supported_filters))
skip_filter(i);
}
idxd_pmu->pmu.name = idxd_pmu->name;
idxd_pmu->pmu.attr_groups = perfmon_attr_groups;
idxd_pmu->pmu.task_ctx_nr = perf_invalid_context;
idxd_pmu->pmu.event_init = perfmon_pmu_event_init;
idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable,
idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable,
idxd_pmu->pmu.add = perfmon_pmu_event_add;
idxd_pmu->pmu.del = perfmon_pmu_event_del;
idxd_pmu->pmu.start = perfmon_pmu_event_start;
idxd_pmu->pmu.stop = perfmon_pmu_event_stop;
idxd_pmu->pmu.read = perfmon_pmu_event_update;
idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
idxd_pmu->pmu.module = THIS_MODULE;
}
void perfmon_pmu_remove(struct idxd_device *idxd)
{
if (!idxd->idxd_pmu)
return;
cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node);
perf_pmu_unregister(&idxd->idxd_pmu->pmu);
kfree(idxd->idxd_pmu);
idxd->idxd_pmu = NULL;
}
static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node)
{
struct idxd_pmu *idxd_pmu;
idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
/* select the first online CPU as the designated reader */
if (cpumask_empty(&perfmon_dsa_cpu_mask)) {
cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask);
idxd_pmu->cpu = cpu;
}
return 0;
}
static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node)
{
struct idxd_pmu *idxd_pmu;
unsigned int target;
idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask))
return 0;
target = cpumask_any_but(cpu_online_mask, cpu);
/* migrate events if there is a valid target */
if (target < nr_cpu_ids)
cpumask_set_cpu(target, &perfmon_dsa_cpu_mask);
else
target = -1;
perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target);
return 0;
}
int perfmon_pmu_init(struct idxd_device *idxd)
{
union idxd_perfcap perfcap;
struct idxd_pmu *idxd_pmu;
int rc = -ENODEV;
/*
* perfmon module initialization failed, nothing to do
*/
if (!cpuhp_set_up)
return -ENODEV;
/*
* If perfmon_offset or num_counters is 0, it means perfmon is
* not supported on this hardware.
*/
if (idxd->perfmon_offset == 0)
return -ENODEV;
idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
if (!idxd_pmu)
return -ENOMEM;
idxd_pmu->idxd = idxd;
idxd->idxd_pmu = idxd_pmu;
if (idxd->data->type == IDXD_TYPE_DSA) {
rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);
if (rc < 0)
goto free;
} else if (idxd->data->type == IDXD_TYPE_IAX) {
rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);
if (rc < 0)
goto free;
} else {
goto free;
}
perfmon_reset(idxd);
perfcap.bits = ioread64(PERFCAP_REG(idxd));
/*
* If total perf counter is 0, stop further registration.
* This is necessary in order to support driver running on
* guest which does not have pmon support.
*/
if (perfcap.num_perf_counter == 0)
goto free;
/* A counter width of 0 means it can't count */
if (perfcap.counter_width == 0)
goto free;
/* Overflow interrupt and counter freeze support must be available */
if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
goto free;
/* Number of event categories cannot be 0 */
if (perfcap.num_event_category == 0)
goto free;
/*
* We don't support per-counter capabilities for now.
*/
if (perfcap.cap_per_counter)
goto free;
idxd_pmu->n_event_categories = perfcap.num_event_category;
idxd_pmu->supported_event_categories = perfcap.global_event_category;
idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
/* check filter capability. If 0, then filters are not supported */
idxd_pmu->supported_filters = perfcap.filter;
if (perfcap.filter)
idxd_pmu->n_filters = hweight8(perfcap.filter);
/* Store the total number of counters categories, and counter width */
idxd_pmu->n_counters = perfcap.num_perf_counter;
idxd_pmu->counter_width = perfcap.counter_width;
idxd_pmu_init(idxd_pmu);
rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);
if (rc)
goto free;
rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node);
if (rc) {
perf_pmu_unregister(&idxd->idxd_pmu->pmu);
goto free;
}
out:
return rc;
free:
kfree(idxd_pmu);
idxd->idxd_pmu = NULL;
goto out;
}
void __init perfmon_init(void)
{
int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
"driver/dma/idxd/perf:online",
perf_event_cpu_online,
perf_event_cpu_offline);
if (WARN_ON(rc < 0))
return;
cpuhp_slot = rc;
cpuhp_set_up = true;
}
void __exit perfmon_exit(void)
{
if (cpuhp_set_up)
cpuhp_remove_multi_state(cpuhp_slot);
}

119
drivers/dma/idxd/perfmon.h Normal file
View File

@ -0,0 +1,119 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
#ifndef _PERFMON_H_
#define _PERFMON_H_
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/sbitmap.h>
#include <linux/dmaengine.h>
#include <linux/percpu-rwsem.h>
#include <linux/wait.h>
#include <linux/cdev.h>
#include <linux/uuid.h>
#include <linux/idxd.h>
#include <linux/perf_event.h>
#include "registers.h"
static inline struct idxd_pmu *event_to_pmu(struct perf_event *event)
{
struct idxd_pmu *idxd_pmu;
struct pmu *pmu;
pmu = event->pmu;
idxd_pmu = container_of(pmu, struct idxd_pmu, pmu);
return idxd_pmu;
}
static inline struct idxd_device *event_to_idxd(struct perf_event *event)
{
struct idxd_pmu *idxd_pmu;
struct pmu *pmu;
pmu = event->pmu;
idxd_pmu = container_of(pmu, struct idxd_pmu, pmu);
return idxd_pmu->idxd;
}
static inline struct idxd_device *pmu_to_idxd(struct pmu *pmu)
{
struct idxd_pmu *idxd_pmu;
idxd_pmu = container_of(pmu, struct idxd_pmu, pmu);
return idxd_pmu->idxd;
}
enum dsa_perf_events {
DSA_PERF_EVENT_WQ = 0,
DSA_PERF_EVENT_ENGINE,
DSA_PERF_EVENT_ADDR_TRANS,
DSA_PERF_EVENT_OP,
DSA_PERF_EVENT_COMPL,
DSA_PERF_EVENT_MAX,
};
enum filter_enc {
FLT_WQ = 0,
FLT_TC,
FLT_PG_SZ,
FLT_XFER_SZ,
FLT_ENG,
FLT_MAX,
};
#define CONFIG_RESET 0x0000000000000001
#define CNTR_RESET 0x0000000000000002
#define CNTR_ENABLE 0x0000000000000001
#define INTR_OVFL 0x0000000000000002
#define COUNTER_FREEZE 0x00000000FFFFFFFF
#define COUNTER_UNFREEZE 0x0000000000000000
#define OVERFLOW_SIZE 32
#define CNTRCFG_ENABLE BIT(0)
#define CNTRCFG_IRQ_OVERFLOW BIT(1)
#define CNTRCFG_CATEGORY_SHIFT 8
#define CNTRCFG_EVENT_SHIFT 32
#define PERFMON_TABLE_OFFSET(_idxd) \
({ \
typeof(_idxd) __idxd = (_idxd); \
((__idxd)->reg_base + (__idxd)->perfmon_offset); \
})
#define PERFMON_REG_OFFSET(idxd, offset) \
(PERFMON_TABLE_OFFSET(idxd) + (offset))
#define PERFCAP_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_PERFCAP_OFFSET))
#define PERFRST_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_PERFRST_OFFSET))
#define OVFSTATUS_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_OVFSTATUS_OFFSET))
#define PERFFRZ_REG(idxd) (PERFMON_REG_OFFSET(idxd, IDXD_PERFFRZ_OFFSET))
#define FLTCFG_REG(idxd, cntr, flt) \
(PERFMON_REG_OFFSET(idxd, IDXD_FLTCFG_OFFSET) + ((cntr) * 32) + ((flt) * 4))
#define CNTRCFG_REG(idxd, cntr) \
(PERFMON_REG_OFFSET(idxd, IDXD_CNTRCFG_OFFSET) + ((cntr) * 8))
#define CNTRDATA_REG(idxd, cntr) \
(PERFMON_REG_OFFSET(idxd, IDXD_CNTRDATA_OFFSET) + ((cntr) * 8))
#define CNTRCAP_REG(idxd, cntr) \
(PERFMON_REG_OFFSET(idxd, IDXD_CNTRCAP_OFFSET) + ((cntr) * 8))
#define EVNTCAP_REG(idxd, category) \
(PERFMON_REG_OFFSET(idxd, IDXD_EVNTCAP_OFFSET) + ((category) * 8))
#define DEFINE_PERFMON_FORMAT_ATTR(_name, _format) \
static ssize_t __perfmon_idxd_##_name##_show(struct kobject *kobj, \
struct kobj_attribute *attr, \
char *page) \
{ \
BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
return sprintf(page, _format "\n"); \
} \
static struct kobj_attribute format_attr_idxd_##_name = \
__ATTR(_name, 0444, __perfmon_idxd_##_name##_show, NULL)
#endif

120
drivers/dma/idxd/registers.h
View File

@ -24,8 +24,8 @@ union gen_cap_reg {
u64 overlap_copy:1;
u64 cache_control_mem:1;
u64 cache_control_cache:1;
u64 cmd_cap:1;
u64 rsvd:3;
u64 int_handle_req:1;
u64 dest_readback:1;
u64 drain_readback:1;
u64 rsvd2:6;
@ -120,7 +120,8 @@ union gencfg_reg {
union genctrl_reg {
struct {
u32 softerr_int_en:1;
u32 rsvd:31;
u32 halt_int_en:1;
u32 rsvd:30;
};
u32 bits;
} __packed;
@ -180,8 +181,11 @@ enum idxd_cmd {
IDXD_CMD_DRAIN_PASID,
IDXD_CMD_ABORT_PASID,
IDXD_CMD_REQUEST_INT_HANDLE,
IDXD_CMD_RELEASE_INT_HANDLE,
};
#define CMD_INT_HANDLE_IMS 0x10000
#define IDXD_CMDSTS_OFFSET 0xa8
union cmdsts_reg {
struct {
@ -193,6 +197,8 @@ union cmdsts_reg {
u32 bits;
} __packed;
#define IDXD_CMDSTS_ACTIVE 0x80000000
#define IDXD_CMDSTS_ERR_MASK 0xff
#define IDXD_CMDSTS_RES_SHIFT 8
enum idxd_cmdsts_err {
IDXD_CMDSTS_SUCCESS = 0,
@ -228,6 +234,8 @@ enum idxd_cmdsts_err {
IDXD_CMDSTS_ERR_NO_HANDLE,
};
#define IDXD_CMDCAP_OFFSET 0xb0
#define IDXD_SWERR_OFFSET 0xc0
#define IDXD_SWERR_VALID 0x00000001
#define IDXD_SWERR_OVERFLOW 0x00000002
@ -378,4 +386,112 @@ union wqcfg {
#define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32)
#define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40)
/* Following is performance monitor registers */
#define IDXD_PERFCAP_OFFSET 0x0
union idxd_perfcap {
struct {
u64 num_perf_counter:6;
u64 rsvd1:2;
u64 counter_width:8;
u64 num_event_category:4;
u64 global_event_category:16;
u64 filter:8;
u64 rsvd2:8;
u64 cap_per_counter:1;
u64 writeable_counter:1;
u64 counter_freeze:1;
u64 overflow_interrupt:1;
u64 rsvd3:8;
};
u64 bits;
} __packed;
#define IDXD_EVNTCAP_OFFSET 0x80
union idxd_evntcap {
struct {
u64 events:28;
u64 rsvd:36;
};
u64 bits;
} __packed;
struct idxd_event {
union {
struct {
u32 event_category:4;
u32 events:28;
};
u32 val;
};
} __packed;
#define IDXD_CNTRCAP_OFFSET 0x800
struct idxd_cntrcap {
union {
struct {
u32 counter_width:8;
u32 rsvd:20;
u32 num_events:4;
};
u32 val;
};
struct idxd_event events[];
} __packed;
#define IDXD_PERFRST_OFFSET 0x10
union idxd_perfrst {
struct {
u32 perfrst_config:1;
u32 perfrst_counter:1;
u32 rsvd:30;
};
u32 val;
} __packed;
#define IDXD_OVFSTATUS_OFFSET 0x30
#define IDXD_PERFFRZ_OFFSET 0x20
#define IDXD_CNTRCFG_OFFSET 0x100
union idxd_cntrcfg {
struct {
u64 enable:1;
u64 interrupt_ovf:1;
u64 global_freeze_ovf:1;
u64 rsvd1:5;
u64 event_category:4;
u64 rsvd2:20;
u64 events:28;
u64 rsvd3:4;
};
u64 val;
} __packed;
#define IDXD_FLTCFG_OFFSET 0x300
#define IDXD_CNTRDATA_OFFSET 0x200
union idxd_cntrdata {
struct {
u64 event_count_value;
};
u64 val;
} __packed;
union event_cfg {
struct {
u64 event_cat:4;
u64 event_enc:28;
};
u64 val;
} __packed;
union filter_cfg {
struct {
u64 wq:32;
u64 tc:8;
u64 pg_sz:4;
u64 xfer_sz:8;
u64 eng:8;
};
u64 val;
} __packed;
#endif

42
drivers/dma/idxd/submit.c
View File

@ -15,18 +15,30 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
desc = wq->descs[idx];
memset(desc->hw, 0, sizeof(struct dsa_hw_desc));
memset(desc->completion, 0, idxd->compl_size);
memset(desc->completion, 0, idxd->data->compl_size);
desc->cpu = cpu;
if (device_pasid_enabled(idxd))
desc->hw->pasid = idxd->pasid;
/*
* Descriptor completion vectors are 1-8 for MSIX. We will round
* robin through the 8 vectors.
* Descriptor completion vectors are 1...N for MSIX. We will round
* robin through the N vectors.
*/
wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
desc->hw->int_handle = wq->vec_ptr;
if (!idxd->int_handles) {
desc->hw->int_handle = wq->vec_ptr;
} else {
desc->vector = wq->vec_ptr;
/*
* int_handles are only for descriptor completion. However for device
* MSIX enumeration, vec 0 is used for misc interrupts. Therefore even
* though we are rotating through 1...N for descriptor interrupts, we
* need to acqurie the int_handles from 0..N-1.
*/
desc->hw->int_handle = idxd->int_handles[desc->vector - 1];
}
return desc;
}
@ -79,13 +91,15 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc)
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
{
struct idxd_device *idxd = wq->idxd;
int vec = desc->hw->int_handle;
void __iomem *portal;
int rc;
if (idxd->state != IDXD_DEV_ENABLED)
return -EIO;
if (!percpu_ref_tryget_live(&wq->wq_active))
return -ENXIO;
portal = wq->portal;
/*
@ -108,13 +122,25 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
return rc;
}
percpu_ref_put(&wq->wq_active);
/*
* Pending the descriptor to the lockless list for the irq_entry
* that we designated the descriptor to.
*/
if (desc->hw->flags & IDXD_OP_FLAG_RCI)
llist_add(&desc->llnode,
&idxd->irq_entries[vec].pending_llist);
if (desc->hw->flags & IDXD_OP_FLAG_RCI) {
int vec;
/*
* If the driver is on host kernel, it would be the value
* assigned to interrupt handle, which is index for MSIX
* vector. If it's guest then can't use the int_handle since
* that is the index to IMS for the entire device. The guest
* device local index will be used.
*/
vec = !idxd->int_handles ? desc->hw->int_handle : desc->vector;
llist_add(&desc->llnode, &idxd->irq_entries[vec].pending_llist);
}
return 0;
}

786
drivers/dma/idxd/sysfs.c
View File

File diff suppressed because it is too large Load Diff

4
drivers/dma/k3dma.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2013 - 2015 Linaro Ltd.
* Copyright (c) 2013 Hisilicon Limited.
* Copyright (c) 2013 HiSilicon Limited.
*/
#include <linux/sched.h>
#include <linux/device.h>
@ -1039,6 +1039,6 @@ static struct platform_driver k3_pdma_driver = {
module_platform_driver(k3_pdma_driver);
MODULE_DESCRIPTION("Hisilicon k3 DMA Driver");
MODULE_DESCRIPTION("HiSilicon k3 DMA Driver");
MODULE_ALIAS("platform:k3dma");
MODULE_LICENSE("GPL v2");

1
drivers/dma/qcom/gpi.c
View File

@ -2281,6 +2281,7 @@ static int gpi_probe(struct platform_device *pdev)
static const struct of_device_id gpi_of_match[] = {
{ .compatible = "qcom,sdm845-gpi-dma" },
{ .compatible = "qcom,sm8150-gpi-dma" },
{ },
};
MODULE_DEVICE_TABLE(of, gpi_of_match);

6
drivers/dma/qcom/hidma.c
View File

@ -90,12 +90,6 @@ static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach)
return container_of(dmach, struct hidma_chan, chan);
}
static inline
struct hidma_desc *to_hidma_desc(struct dma_async_tx_descriptor *t)
{
return container_of(t, struct hidma_desc, desc);
}
static void hidma_free(struct hidma_dev *dmadev)
{
INIT_LIST_HEAD(&dmadev->ddev.channels);

8
drivers/dma/xilinx/xilinx_dma.c
View File

@ -2453,6 +2453,13 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan)
return 0;
}
static void xilinx_dma_synchronize(struct dma_chan *dchan)
{
struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
tasklet_kill(&chan->tasklet);
}
/**
* xilinx_dma_channel_set_config - Configure VDMA channel
* Run-time configuration for Axi VDMA, supports:
@ -3074,6 +3081,7 @@ static int xilinx_dma_probe(struct platform_device *pdev)
xdev->common.device_free_chan_resources =
xilinx_dma_free_chan_resources;
xdev->common.device_terminate_all = xilinx_dma_terminate_all;
xdev->common.device_synchronize = xilinx_dma_synchronize;
xdev->common.device_tx_status = xilinx_dma_tx_status;
xdev->common.device_issue_pending = xilinx_dma_issue_pending;
if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {

30
drivers/pci/pci.c
View File

@ -692,6 +692,36 @@ u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
}
EXPORT_SYMBOL_GPL(pci_find_ht_capability);
/**
* pci_find_vsec_capability - Find a vendor-specific extended capability
* @dev: PCI device to query
* @vendor: Vendor ID for which capability is defined
* @cap: Vendor-specific capability ID
*
* If @dev has Vendor ID @vendor, search for a VSEC capability with
* VSEC ID @cap. If found, return the capability offset in
* config space; otherwise return 0.
*/
u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap)
{
u16 vsec = 0;
u32 header;
if (vendor != dev->vendor)
return 0;
while ((vsec = pci_find_next_ext_capability(dev, vsec,
PCI_EXT_CAP_ID_VNDR))) {
if (pci_read_config_dword(dev, vsec + PCI_VNDR_HEADER,
&header) == PCIBIOS_SUCCESSFUL &&
PCI_VNDR_HEADER_ID(header) == cap)
return vsec;
}
return 0;
}
EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
/**
* pci_find_parent_resource - return resource region of parent bus of given
* region

1
include/linux/cpuhotplug.h
View File

@ -169,6 +169,7 @@ enum cpuhp_state {
CPUHP_AP_PERF_X86_RAPL_ONLINE,
CPUHP_AP_PERF_X86_CQM_ONLINE,
CPUHP_AP_PERF_X86_CSTATE_ONLINE,
CPUHP_AP_PERF_X86_IDXD_ONLINE,
CPUHP_AP_PERF_S390_CF_ONLINE,
CPUHP_AP_PERF_S390_CFD_ONLINE,
CPUHP_AP_PERF_S390_SF_ONLINE,

1
include/linux/pci.h
View File

@ -1085,6 +1085,7 @@ u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap);
u16 pci_find_ext_capability(struct pci_dev *dev, int cap);
u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 pos, int cap);
struct pci_bus *pci_find_next_bus(const struct pci_bus *from);
u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap);
u64 pci_get_dsn(struct pci_dev *dev);