Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-24 13:11:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

virtio: features, fixes, cleanups

Browse Source 
Several new features here:
 
 - Virtio find vqs API has been reworked
   (required to fix the scalability issue we have with
    adminq, which I hope to merge later in the cycle)
 
 - vDPA driver for Marvell OCTEON
 
 - virtio fs performance improvement
 
 - mlx5 migration speedups
 
 Fixes, cleanups all over the place.
 
 Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
 -----BEGIN PGP SIGNATURE-----
 
 iQFDBAABCAAtFiEEXQn9CHHI+FuUyooNKB8NuNKNVGkFAmaXjQQPHG1zdEByZWRo
 YXQuY29tAAoJECgfDbjSjVRpnIsH/jVNqAQbe/vaBQdNMdnsA+P9A9unLbYRxYCQ
 tN73mQRIXKtnZHBRAEbMGq52HPYg8HlN2HJSgyNo6I6t8VD+PiOco7m+3GpmqEcW
 aXPOPl0BAbVoDgyutxRuuodP8Z61lBx0mG6iOxpzTXOPGlpQqtPCFHO8YnodqnPf
 tMix/5uAqgZKV2siCbw5DtzwEc0gDHU8qsD0/nyoS5nBDF9yh/ardr5P/qiyFDQH
 atCNYTOhIFU83pLAaw0fpCGbkt7gxf+5RpWVx3wkYww+/MwvYhsveRvQyaGbBz3n
 WDtET3SOtVTta98OAGIKCq/2z8f6mYXBP7vXapBgnJG3vwS/poQ=
 =LYua
 -----END PGP SIGNATURE-----

Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

Pull virtio updates from Michael Tsirkin:
 "Several new features here:

   - Virtio find vqs API has been reworked (required to fix the
     scalability issue we have with adminq, which I hope to merge later
     in the cycle)

   - vDPA driver for Marvell OCTEON

   - virtio fs performance improvement

   - mlx5 migration speedups

  Fixes, cleanups all over the place"

* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (56 commits)
  virtio: rename virtio_find_vqs_info() to virtio_find_vqs()
  virtio: remove unused virtio_find_vqs() and virtio_find_vqs_ctx() helpers
  virtio: convert the rest virtio_find_vqs() users to virtio_find_vqs_info()
  virtio_balloon: convert to use virtio_find_vqs_info()
  virtiofs: convert to use virtio_find_vqs_info()
  scsi: virtio_scsi: convert to use virtio_find_vqs_info()
  virtio_net: convert to use virtio_find_vqs_info()
  virtio_crypto: convert to use virtio_find_vqs_info()
  virtio_console: convert to use virtio_find_vqs_info()
  virtio_blk: convert to use virtio_find_vqs_info()
  virtio: rename find_vqs_info() op to find_vqs()
  virtio: remove the original find_vqs() op
  virtio: call virtio_find_vqs_info() from virtio_find_single_vq() directly
  virtio: convert find_vqs() op implementations to find_vqs_info()
  virtio_pci: convert vp_*find_vqs() ops to find_vqs_info()
  virtio: introduce virtio_queue_info struct and find_vqs_info() config op
  virtio: make virtio_find_single_vq() call virtio_find_vqs()
  virtio: make virtio_find_vqs() call virtio_find_vqs_ctx()
  caif_virtio: use virtio_find_single_vq() for single virtqueue finding
  vdpa/mlx5: Don't enable non-active VQs in .set_vq_ready()
  ...
This commit is contained in:
Linus Torvalds 2024-07-19 11:57:55 -07:00
commit f4f92db439
48 changed files with 2145 additions and 543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
MAINTAINERS
View File

@ -10786,7 +10786,7 @@ F: net/ieee802154/
F: net/mac802154/
Intel VIRTIO DATA PATH ACCELERATOR
M: Zhu Lingshan <lingshan.zhu@intel.com>
M: Zhu Lingshan <lingshan.zhu@kernel.org>
L: virtualization@lists.linux.dev
S: Supported
F: drivers/vdpa/ifcvf/
@ -13611,6 +13611,11 @@ S: Supported
F: Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.yaml
F: drivers/mmc/host/sdhci-xenon*
MARVELL OCTEON ENDPOINT VIRTIO DATA PATH ACCELERATOR
R: schalla@marvell.com
R: vattunuru@marvell.com
F: drivers/vdpa/octeon_ep/
MATROX FRAMEBUFFER DRIVER
L: linux-fbdev@vger.kernel.org
S: Orphan

8
arch/um/drivers/virt-pci.c
View File

@ -567,12 +567,14 @@ struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus)
static int um_pci_init_vqs(struct um_pci_device *dev)
{
struct virtqueue_info vqs_info[] = {
{ "cmd", um_pci_cmd_vq_cb },
{ "irq", um_pci_irq_vq_cb },
};
struct virtqueue *vqs[2];
static const char *const names[2] = { "cmd", "irq" };
vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb };
int err, i;
err = virtio_find_vqs(dev->vdev, 2, vqs, cbs, names, NULL);
err = virtio_find_vqs(dev->vdev, 2, vqs, vqs_info, NULL);
if (err)
return err;

12
arch/um/drivers/virtio_uml.c
View File

@ -1014,8 +1014,8 @@ error_kzalloc:
}
static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
struct virtqueue *vqs[],
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
@ -1031,13 +1031,15 @@ static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs,
return rc;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
vqs[i] = vu_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
ctx ? ctx[i] : false);
vqs[i] = vu_setup_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx);
if (IS_ERR(vqs[i])) {
rc = PTR_ERR(vqs[i]);
goto error_setup;

20
drivers/block/virtio_blk.c
View File

@ -964,8 +964,7 @@ static int init_vq(struct virtio_blk *vblk)
{
int err;
unsigned short i;
vq_callback_t **callbacks;
const char **names;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
unsigned short num_vqs;
unsigned short num_poll_vqs;
@ -1002,28 +1001,26 @@ static int init_vq(struct virtio_blk *vblk)
if (!vblk->vqs)
return -ENOMEM;
names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL);
callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL);
vqs_info = kcalloc(num_vqs, sizeof(*vqs_info), GFP_KERNEL);
vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL);
if (!names || !callbacks || !vqs) {
if (!vqs_info || !vqs) {
err = -ENOMEM;
goto out;
}
for (i = 0; i < num_vqs - num_poll_vqs; i++) {
callbacks[i] = virtblk_done;
vqs_info[i].callback = virtblk_done;
snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%u", i);
names[i] = vblk->vqs[i].name;
vqs_info[i].name = vblk->vqs[i].name;
}
for (; i < num_vqs; i++) {
callbacks[i] = NULL;
snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req_poll.%u", i);
names[i] = vblk->vqs[i].name;
vqs_info[i].name = vblk->vqs[i].name;
}
/* Discover virtqueues and write information to configuration. */
err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
err = virtio_find_vqs(vdev, num_vqs, vqs, vqs_info, &desc);
if (err)
goto out;
@ -1035,8 +1032,7 @@ static int init_vq(struct virtio_blk *vblk)
out:
kfree(vqs);
kfree(callbacks);
kfree(names);
kfree(vqs_info);
if (err)
kfree(vblk->vqs);
return err;

13
drivers/bluetooth/virtio_bt.c
View File

@ -254,13 +254,9 @@ static void virtbt_rx_done(struct virtqueue *vq)
static int virtbt_probe(struct virtio_device *vdev)
{
vq_callback_t *callbacks[VIRTBT_NUM_VQS] = {
[VIRTBT_VQ_TX] = virtbt_tx_done,
[VIRTBT_VQ_RX] = virtbt_rx_done,
};
const char *names[VIRTBT_NUM_VQS] = {
[VIRTBT_VQ_TX] = "tx",
[VIRTBT_VQ_RX] = "rx",
struct virtqueue_info vqs_info[VIRTBT_NUM_VQS] = {
[VIRTBT_VQ_TX] = { "tx", virtbt_tx_done },
[VIRTBT_VQ_RX] = { "rx", virtbt_rx_done },
};
struct virtio_bluetooth *vbt;
struct hci_dev *hdev;
@ -288,8 +284,7 @@ static int virtbt_probe(struct virtio_device *vdev)
INIT_WORK(&vbt->rx, virtbt_rx_work);
err = virtio_find_vqs(vdev, VIRTBT_NUM_VQS, vbt->vqs, callbacks,
names, NULL);
err = virtio_find_vqs(vdev, VIRTBT_NUM_VQS, vbt->vqs, vqs_info, NULL);
if (err)
return err;

43
drivers/char/virtio_console.c
View File

@ -1804,8 +1804,7 @@ static void config_work_handler(struct work_struct *work)
static int init_vqs(struct ports_device *portdev)
{
vq_callback_t **io_callbacks;
char **io_names;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
u32 i, j, nr_ports, nr_queues;
int err;
@ -1814,15 +1813,12 @@ static int init_vqs(struct ports_device *portdev)
nr_queues = use_multiport(portdev) ? (nr_ports + 1) * 2 : 2;
vqs = kmalloc_array(nr_queues, sizeof(struct virtqueue *), GFP_KERNEL);
io_callbacks = kmalloc_array(nr_queues, sizeof(vq_callback_t *),
GFP_KERNEL);
io_names = kmalloc_array(nr_queues, sizeof(char *), GFP_KERNEL);
vqs_info = kcalloc(nr_queues, sizeof(*vqs_info), GFP_KERNEL);
portdev->in_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *),
GFP_KERNEL);
portdev->out_vqs = kmalloc_array(nr_ports, sizeof(struct virtqueue *),
GFP_KERNEL);
if (!vqs || !io_callbacks || !io_names || !portdev->in_vqs ||
!portdev->out_vqs) {
if (!vqs || !vqs_info || !portdev->in_vqs || !portdev->out_vqs) {
err = -ENOMEM;
goto free;
}
@ -1833,30 +1829,27 @@ static int init_vqs(struct ports_device *portdev)
* 0 before others.
*/
j = 0;
io_callbacks[j] = in_intr;
io_callbacks[j + 1] = out_intr;
io_names[j] = "input";
io_names[j + 1] = "output";
vqs_info[j].callback = in_intr;
vqs_info[j + 1].callback = out_intr;
vqs_info[j].name = "input";
vqs_info[j + 1].name = "output";
j += 2;
if (use_multiport(portdev)) {
io_callbacks[j] = control_intr;
io_callbacks[j + 1] = NULL;
io_names[j] = "control-i";
io_names[j + 1] = "control-o";
vqs_info[j].callback = control_intr;
vqs_info[j].name = "control-i";
vqs_info[j + 1].name = "control-o";
for (i = 1; i < nr_ports; i++) {
j += 2;
io_callbacks[j] = in_intr;
io_callbacks[j + 1] = out_intr;
io_names[j] = "input";
io_names[j + 1] = "output";
vqs_info[j].callback = in_intr;
vqs_info[j + 1].callback = out_intr;
vqs_info[j].name = "input";
vqs_info[j + 1].name = "output";
}
}
/* Find the queues. */
err = virtio_find_vqs(portdev->vdev, nr_queues, vqs,
io_callbacks,
(const char **)io_names, NULL);
err = virtio_find_vqs(portdev->vdev, nr_queues, vqs, vqs_info, NULL);
if (err)
goto free;
@ -1874,8 +1867,7 @@ static int init_vqs(struct ports_device *portdev)
portdev->out_vqs[i] = vqs[j + 1];
}
}
kfree(io_names);
kfree(io_callbacks);
kfree(vqs_info);
kfree(vqs);
return 0;
@ -1883,8 +1875,7 @@ static int init_vqs(struct ports_device *portdev)
free:
kfree(portdev->out_vqs);
kfree(portdev->in_vqs);
kfree(io_names);
kfree(io_callbacks);
kfree(vqs_info);
kfree(vqs);
return err;

31
drivers/crypto/virtio/virtio_crypto_core.c
View File

@ -96,11 +96,10 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq)
static int virtcrypto_find_vqs(struct virtio_crypto *vi)
{
vq_callback_t **callbacks;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
int ret = -ENOMEM;
int i, total_vqs;
const char **names;
struct device *dev = &vi->vdev->dev;
/*
@ -114,26 +113,23 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
if (!vqs)
goto err_vq;
callbacks = kcalloc(total_vqs, sizeof(*callbacks), GFP_KERNEL);
if (!callbacks)
goto err_callback;
names = kcalloc(total_vqs, sizeof(*names), GFP_KERNEL);
if (!names)
goto err_names;
vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL);
if (!vqs_info)
goto err_vqs_info;
/* Parameters for control virtqueue */
callbacks[total_vqs - 1] = virtcrypto_ctrlq_callback;
names[total_vqs - 1] = "controlq";
vqs_info[total_vqs - 1].callback = virtcrypto_ctrlq_callback;
vqs_info[total_vqs - 1].name = "controlq";
/* Allocate/initialize parameters for data virtqueues */
for (i = 0; i < vi->max_data_queues; i++) {
callbacks[i] = virtcrypto_dataq_callback;
vqs_info[i].callback = virtcrypto_dataq_callback;
snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name),
"dataq.%d", i);
names[i] = vi->data_vq[i].name;
vqs_info[i].name = vi->data_vq[i].name;
}
ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, callbacks, names, NULL);
ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL);
if (ret)
goto err_find;
@ -153,18 +149,15 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi)
(unsigned long)&vi->data_vq[i]);
}
kfree(names);
kfree(callbacks);
kfree(vqs_info);
kfree(vqs);
return 0;
err_engine:
err_find:
kfree(names);
err_names:
kfree(callbacks);
err_callback:
kfree(vqs_info);
err_vqs_info:
kfree(vqs);
err_vq:
return ret;

11
drivers/firmware/arm_scmi/virtio.c
View File

@ -354,11 +354,9 @@ static void scmi_vio_deferred_tx_worker(struct work_struct *work)
scmi_vio_channel_release(vioch);
}
static const char *const scmi_vio_vqueue_names[] = { "tx", "rx" };
static vq_callback_t *scmi_vio_complete_callbacks[] = {
scmi_vio_complete_cb,
scmi_vio_complete_cb
static struct virtqueue_info scmi_vio_vqs_info[] = {
{ "tx", scmi_vio_complete_cb },
{ "rx", scmi_vio_complete_cb },
};
static unsigned int virtio_get_max_msg(struct scmi_chan_info *base_cinfo)
@ -831,8 +829,7 @@ static int scmi_vio_probe(struct virtio_device *vdev)
if (have_vq_rx)
channels[VIRTIO_SCMI_VQ_RX].is_rx = true;
ret = virtio_find_vqs(vdev, vq_cnt, vqs, scmi_vio_complete_callbacks,
scmi_vio_vqueue_names, NULL);
ret = virtio_find_vqs(vdev, vq_cnt, vqs, scmi_vio_vqs_info, NULL);
if (ret) {
dev_err(dev, "Failed to get %d virtqueue(s)\n", vq_cnt);
return ret;

10
drivers/gpio/gpio-virtio.c
View File

@ -457,15 +457,15 @@ static void virtio_gpio_free_vqs(struct virtio_device *vdev)
static int virtio_gpio_alloc_vqs(struct virtio_gpio *vgpio,
struct virtio_device *vdev)
{
const char * const names[] = { "requestq", "eventq" };
vq_callback_t *cbs[] = {
virtio_gpio_request_vq,
virtio_gpio_event_vq,
struct virtqueue_info vqs_info[] = {
{ "requestq", virtio_gpio_request_vq },
{ "eventq", virtio_gpio_event_vq },
};
struct virtqueue *vqs[2] = { NULL, NULL };
int ret;
ret = virtio_find_vqs(vdev, vgpio->irq_lines ? 2 : 1, vqs, cbs, names, NULL);
ret = virtio_find_vqs(vdev, vgpio->irq_lines ? 2 : 1, vqs,
vqs_info, NULL);
if (ret) {
dev_err(&vdev->dev, "failed to find vqs: %d\n", ret);
return ret;

9
drivers/gpu/drm/virtio/virtgpu_kms.c
View File

@ -116,11 +116,10 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev,
int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
{
static vq_callback_t *callbacks[] = {
virtio_gpu_ctrl_ack, virtio_gpu_cursor_ack
struct virtqueue_info vqs_info[] = {
{ "control", virtio_gpu_ctrl_ack },
{ "cursor", virtio_gpu_cursor_ack },
};
static const char * const names[] = { "control", "cursor" };
struct virtio_gpu_device *vgdev;
/* this will expand later */
struct virtqueue *vqs[2];
@ -207,7 +206,7 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
DRM_INFO("features: %ccontext_init\n",
vgdev->has_context_init ? '+' : '-');
ret = virtio_find_vqs(vgdev->vdev, 2, vqs, callbacks, names, NULL);
ret = virtio_find_vqs(vgdev->vdev, 2, vqs, vqs_info, NULL);
if (ret) {
DRM_ERROR("failed to find virt queues\n");
goto err_vqs;

11
drivers/iommu/virtio-iommu.c
View File

@ -1084,14 +1084,13 @@ static struct iommu_ops viommu_ops = {
static int viommu_init_vqs(struct viommu_dev *viommu)
{
struct virtio_device *vdev = dev_to_virtio(viommu->dev);
const char *names[] = { "request", "event" };
vq_callback_t *callbacks[] = {
NULL, /* No async requests */
viommu_event_handler,
struct virtqueue_info vqs_info[] = {
{ "request" },
{ "event", viommu_event_handler },
};
return virtio_find_vqs(vdev, VIOMMU_NR_VQS, viommu->vqs, callbacks,
names, NULL);
return virtio_find_vqs(vdev, VIOMMU_NR_VQS, viommu->vqs,
vqs_info, NULL);
}
static int viommu_fill_evtq(struct viommu_dev *viommu)

8
drivers/net/caif/caif_virtio.c
View File

@ -646,9 +646,7 @@ static inline void debugfs_init(struct cfv_info *cfv)
/* Setup CAIF for the a virtio device */
static int cfv_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs = cfv_release_cb;
vrh_callback_t *vrh_cbs = cfv_recv;
const char *names = "output";
const char *cfv_netdev_name = "cfvrt";
struct net_device *netdev;
struct cfv_info *cfv;
@ -675,9 +673,11 @@ static int cfv_probe(struct virtio_device *vdev)
goto err;
/* Get the TX virtio ring. This is a "guest side vring". */
err = virtio_find_vqs(vdev, 1, &cfv->vq_tx, &vq_cbs, &names, NULL);
if (err)
cfv->vq_tx = virtio_find_single_vq(vdev, cfv_release_cb, "output");
if (IS_ERR(cfv->vq_tx)) {
err = PTR_ERR(cfv->vq_tx);
goto err;
}
/* Get the CAIF configuration from virtio config space, if available */
if (vdev->config->get) {

34
drivers/net/virtio_net.c
View File

@ -5946,9 +5946,8 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu
static int virtnet_find_vqs(struct virtnet_info *vi)
{
vq_callback_t **callbacks;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
const char **names;
int ret = -ENOMEM;
int total_vqs;
bool *ctx;
@ -5965,12 +5964,9 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
if (!vqs)
goto err_vq;
callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL);
if (!callbacks)
goto err_callback;
names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL);
if (!names)
goto err_names;
vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL);
if (!vqs_info)
goto err_vqs_info;
if (!vi->big_packets || vi->mergeable_rx_bufs) {
ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL);
if (!ctx)
@ -5981,24 +5977,22 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
/* Parameters for control virtqueue, if any */
if (vi->has_cvq) {
callbacks[total_vqs - 1] = NULL;
names[total_vqs - 1] = "control";
vqs_info[total_vqs - 1].name = "control";
}
/* Allocate/initialize parameters for send/receive virtqueues */
for (i = 0; i < vi->max_queue_pairs; i++) {
callbacks[rxq2vq(i)] = skb_recv_done;
callbacks[txq2vq(i)] = skb_xmit_done;
vqs_info[rxq2vq(i)].callback = skb_recv_done;
vqs_info[txq2vq(i)].callback = skb_xmit_done;
sprintf(vi->rq[i].name, "input.%u", i);
sprintf(vi->sq[i].name, "output.%u", i);
names[rxq2vq(i)] = vi->rq[i].name;
names[txq2vq(i)] = vi->sq[i].name;
vqs_info[rxq2vq(i)].name = vi->rq[i].name;
vqs_info[txq2vq(i)].name = vi->sq[i].name;
if (ctx)
ctx[rxq2vq(i)] = true;
vqs_info[rxq2vq(i)].ctx = true;
}
ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks,
names, ctx, NULL);
ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL);
if (ret)
goto err_find;
@ -6020,10 +6014,8 @@ static int virtnet_find_vqs(struct virtnet_info *vi)
err_find:
kfree(ctx);
err_ctx:
kfree(names);
err_names:
kfree(callbacks);
err_callback:
kfree(vqs_info);
err_vqs_info:
kfree(vqs);
err_vq:
return ret;

12
drivers/net/wireless/virtual/mac80211_hwsim.c
View File

@ -6626,17 +6626,13 @@ static void hwsim_virtio_rx_done(struct virtqueue *vq)
static int init_vqs(struct virtio_device *vdev)
{
vq_callback_t *callbacks[HWSIM_NUM_VQS] = {
[HWSIM_VQ_TX] = hwsim_virtio_tx_done,
[HWSIM_VQ_RX] = hwsim_virtio_rx_done,
};
const char *names[HWSIM_NUM_VQS] = {
[HWSIM_VQ_TX] = "tx",
[HWSIM_VQ_RX] = "rx",
struct virtqueue_info vqs_info[HWSIM_NUM_VQS] = {
[HWSIM_VQ_TX] = { "tx", hwsim_virtio_tx_done },
[HWSIM_VQ_RX] = { "rx", hwsim_virtio_rx_done },
};
return virtio_find_vqs(vdev, HWSIM_NUM_VQS,
hwsim_vqs, callbacks, names, NULL);
hwsim_vqs, vqs_info, NULL);
}
static int fill_vq(struct virtqueue *vq)

10
drivers/platform/mellanox/mlxbf-tmfifo.c
View File

@ -1058,9 +1058,7 @@ static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev)
static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
unsigned int nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[],
const bool *ctx,
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev);
@ -1072,7 +1070,9 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
return -EINVAL;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
ret = -EINVAL;
goto error;
}
@ -1084,7 +1084,7 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev,
vq = vring_new_virtqueue(i, vring->num, vring->align, vdev,
false, false, vring->va,
mlxbf_tmfifo_virtio_notify,
callbacks[i], names[i]);
vqi->callback, vqi->name);
if (!vq) {
dev_err(&vdev->dev, "vring_new_virtqueue failed\n");
ret = -ENOMEM;

12
drivers/remoteproc/remoteproc_virtio.c
View File

@ -182,21 +182,21 @@ static void rproc_virtio_del_vqs(struct virtio_device *vdev)
static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[],
const bool * ctx,
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
int i, ret, queue_idx = 0;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
vqs[i] = rp_find_vq(vdev, queue_idx++, callbacks[i], names[i],
ctx ? ctx[i] : false);
vqs[i] = rp_find_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx);
if (IS_ERR(vqs[i])) {
ret = PTR_ERR(vqs[i]);
goto error;

8
drivers/rpmsg/virtio_rpmsg_bus.c
View File

@ -868,8 +868,10 @@ static void rpmsg_virtio_del_ctrl_dev(struct rpmsg_device *rpdev_ctrl)
static int rpmsg_probe(struct virtio_device *vdev)
{
vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done };
static const char * const names[] = { "input", "output" };
struct virtqueue_info vqs_info[] = {
{ "input", rpmsg_recv_done },
{ "output", rpmsg_xmit_done },
};
struct virtqueue *vqs[2];
struct virtproc_info *vrp;
struct virtio_rpmsg_channel *vch = NULL;
@ -891,7 +893,7 @@ static int rpmsg_probe(struct virtio_device *vdev)
init_waitqueue_head(&vrp->sendq);
/* We expect two virtqueues, rx and tx (and in this order) */
err = virtio_find_vqs(vdev, 2, vqs, vq_cbs, names, NULL);
err = virtio_find_vqs(vdev, 2, vqs, vqs_info, NULL);
if (err)
goto free_vrp;

13
drivers/s390/virtio/virtio_ccw.c
View File

@ -689,9 +689,7 @@ out:
static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[],
const bool *ctx,
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
struct virtio_ccw_device *vcdev = to_vc_device(vdev);
@ -705,14 +703,15 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs,
return -ENOMEM;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
vqs[i] = virtio_ccw_setup_vq(vdev, queue_idx++, callbacks[i],
names[i], ctx ? ctx[i] : false,
ccw);
vqs[i] = virtio_ccw_setup_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx, ccw);
if (IS_ERR(vqs[i])) {
ret = PTR_ERR(vqs[i]);
vqs[i] = NULL;

32
drivers/scsi/virtio_scsi.c
View File

@ -841,19 +841,16 @@ static int virtscsi_init(struct virtio_device *vdev,
int err;
u32 i;
u32 num_vqs, num_poll_vqs, num_req_vqs;
vq_callback_t **callbacks;
const char **names;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
struct irq_affinity desc = { .pre_vectors = 2 };
num_req_vqs = vscsi->num_queues;
num_vqs = num_req_vqs + VIRTIO_SCSI_VQ_BASE;
vqs = kmalloc_array(num_vqs, sizeof(struct virtqueue *), GFP_KERNEL);
callbacks = kmalloc_array(num_vqs, sizeof(vq_callback_t *),
GFP_KERNEL);
names = kmalloc_array(num_vqs, sizeof(char *), GFP_KERNEL);
vqs_info = kcalloc(num_vqs, sizeof(*vqs_info), GFP_KERNEL);
if (!callbacks || !vqs || !names) {
if (!vqs || !vqs_info) {
err = -ENOMEM;
goto out;
}
@ -869,22 +866,20 @@ static int virtscsi_init(struct virtio_device *vdev,
vscsi->io_queues[HCTX_TYPE_READ],
vscsi->io_queues[HCTX_TYPE_POLL]);
callbacks[0] = virtscsi_ctrl_done;
callbacks[1] = virtscsi_event_done;
names[0] = "control";
names[1] = "event";
vqs_info[0].callback = virtscsi_ctrl_done;
vqs_info[0].name = "control";
vqs_info[1].callback = virtscsi_event_done;
vqs_info[1].name = "event";
for (i = VIRTIO_SCSI_VQ_BASE; i < num_vqs - num_poll_vqs; i++) {
callbacks[i] = virtscsi_req_done;
names[i] = "request";
vqs_info[i].callback = virtscsi_req_done;
vqs_info[i].name = "request";
}
for (; i < num_vqs; i++) {
callbacks[i] = NULL;
names[i] = "request_poll";
}
for (; i < num_vqs; i++)
vqs_info[i].name = "request_poll";
/* Discover virtqueues and write information to configuration. */
err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
err = virtio_find_vqs(vdev, num_vqs, vqs, vqs_info, &desc);
if (err)
goto out;
@ -900,8 +895,7 @@ static int virtscsi_init(struct virtio_device *vdev,
err = 0;
out:
kfree(names);
kfree(callbacks);
kfree(vqs_info);
kfree(vqs);
if (err)
virtscsi_remove_vqs(vdev);

11
drivers/vdpa/Kconfig
View File

@ -126,4 +126,15 @@ config PDS_VDPA
With this driver, the VirtIO dataplane can be
offloaded to an AMD/Pensando DSC device.
config OCTEONEP_VDPA
tristate "vDPA driver for Octeon DPU devices"
depends on m
depends on PCI_MSI
help
This is a vDPA driver designed for Marvell's Octeon DPU devices.
This driver enables the offloading of the VirtIO dataplane to an
Octeon DPU device.
Please note that this driver must be built as a module and it
cannot be loaded until the Octeon emulation software is running.
endif # VDPA

1
drivers/vdpa/Makefile
View File

@ -8,3 +8,4 @@ obj-$(CONFIG_VP_VDPA) += virtio_pci/
obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/
obj-$(CONFIG_SNET_VDPA) += solidrun/
obj-$(CONFIG_PDS_VDPA) += pds/
obj-$(CONFIG_OCTEONEP_VDPA) += octeon_ep/

1
drivers/vdpa/ifcvf/ifcvf_main.c
View File

@ -894,4 +894,5 @@ static struct pci_driver ifcvf_driver = {
module_pci_driver(ifcvf_driver);
MODULE_DESCRIPTION("Intel IFC VF NIC driver for virtio dataplane offloading");
MODULE_LICENSE("GPL v2");

429
drivers/vdpa/mlx5/net/mlx5_vnet.c
View File

@ -48,6 +48,18 @@ MODULE_LICENSE("Dual BSD/GPL");
#define MLX5V_UNTAGGED 0x1000
/* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section
* 5.1.6.5.5 "Device operation in multiqueue mode":
*
* Multiqueue is disabled by default.
* The driver enables multiqueue by sending a command using class
* VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue
* operation, as follows: ...
*/
#define MLX5V_DEFAULT_VQ_COUNT 2
#define MLX5V_DEFAULT_VQ_SIZE 256
struct mlx5_vdpa_cq_buf {
struct mlx5_frag_buf_ctrl fbc;
struct mlx5_frag_buf frag_buf;
@ -144,10 +156,11 @@ static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx)
return idx <= mvdev->max_idx;
}
static void free_resources(struct mlx5_vdpa_net *ndev);
static void init_mvqs(struct mlx5_vdpa_net *ndev);
static int setup_driver(struct mlx5_vdpa_dev *mvdev);
static void teardown_driver(struct mlx5_vdpa_net *ndev);
static void free_fixed_resources(struct mlx5_vdpa_net *ndev);
static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev);
static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled);
static void teardown_vq_resources(struct mlx5_vdpa_net *ndev);
static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq);
static bool mlx5_vdpa_debug;
@ -862,13 +875,16 @@ static bool msix_mode_supported(struct mlx5_vdpa_dev *mvdev)
pci_msix_can_alloc_dyn(mvdev->mdev->pdev);
}
static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
static int create_virtqueue(struct mlx5_vdpa_net *ndev,
struct mlx5_vdpa_virtqueue *mvq,
bool filled)
{
int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in);
u32 out[MLX5_ST_SZ_DW(create_virtio_net_q_out)] = {};
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
struct mlx5_vdpa_mr *vq_mr;
struct mlx5_vdpa_mr *vq_desc_mr;
u64 features = filled ? mvdev->actual_features : mvdev->mlx_features;
void *obj_context;
u16 mlx_features;
void *cmd_hdr;
@ -886,7 +902,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
goto err_alloc;
}
mlx_features = get_features(ndev->mvdev.actual_features);
mlx_features = get_features(features);
cmd_hdr = MLX5_ADDR_OF(create_virtio_net_q_in, in, general_obj_in_cmd_hdr);
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
@ -894,8 +910,6 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
obj_context = MLX5_ADDR_OF(create_virtio_net_q_in, in, obj_context);
MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
mlx_features >> 3);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0,
@ -917,17 +931,36 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index);
MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent);
MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
!!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1)));
MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
if (vq_mr)
MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
!!(features & BIT_ULL(VIRTIO_F_VERSION_1)));
vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey);
if (filled) {
MLX5_SET(virtio_net_q_object, obj_context, hw_available_index, mvq->avail_idx);
MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
if (vq_mr)
MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
if (vq_desc_mr &&
MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey);
} else {
/* If there is no mr update, make sure that the existing ones are set
* modify to ready.
*/
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
if (vq_mr)
mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY;
vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
if (vq_desc_mr)
mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
}
MLX5_SET(virtio_q, vq_ctx, umem_1_id, mvq->umem1.id);
MLX5_SET(virtio_q, vq_ctx, umem_1_size, mvq->umem1.size);
@ -947,12 +980,15 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
kfree(in);
mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
mlx5_vdpa_get_mr(mvdev, vq_mr);
mvq->vq_mr = vq_mr;
if (filled) {
mlx5_vdpa_get_mr(mvdev, vq_mr);
mvq->vq_mr = vq_mr;
if (vq_desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) {
mlx5_vdpa_get_mr(mvdev, vq_desc_mr);
mvq->desc_mr = vq_desc_mr;
if (vq_desc_mr &&
MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported)) {
mlx5_vdpa_get_mr(mvdev, vq_desc_mr);
mvq->desc_mr = vq_desc_mr;
}
}
return 0;
@ -1273,6 +1309,19 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev,
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX)
MLX5_SET(virtio_net_q_object, obj_context, hw_used_index, mvq->used_idx);
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION)
MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
!!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1)));
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES) {
u16 mlx_features = get_features(ndev->mvdev.actual_features);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_12_3,
mlx_features >> 3);
MLX5_SET(virtio_net_q_object, obj_context, queue_feature_bit_mask_2_0,
mlx_features & 7);
}
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY) {
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
@ -1417,14 +1466,13 @@ static void dealloc_vector(struct mlx5_vdpa_net *ndev,
}
}
static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
static int setup_vq(struct mlx5_vdpa_net *ndev,
struct mlx5_vdpa_virtqueue *mvq,
bool filled)
{
u16 idx = mvq->index;
int err;
if (!mvq->num_ent)
return 0;
if (mvq->initialized)
return 0;
@ -1449,20 +1497,18 @@ static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
goto err_connect;
alloc_vector(ndev, mvq);
err = create_virtqueue(ndev, mvq);
err = create_virtqueue(ndev, mvq, filled);
if (err)
goto err_vq;
mvq->initialized = true;
if (mvq->ready) {
err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
if (err) {
mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n",
idx, err);
err = resume_vq(ndev, mvq);
if (err)
goto err_modify;
}
}
mvq->initialized = true;
return 0;
err_modify:
@ -1479,51 +1525,105 @@ err_fwqp:
return err;
}
static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
{
struct mlx5_virtq_attr attr;
int err;
if (!mvq->initialized)
return;
return 0;
if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)
return;
return 0;
if (modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND))
mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed\n");
if (query_virtqueue(ndev, mvq, &attr)) {
mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue\n");
return;
err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND);
if (err) {
mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed, err: %d\n", err);
return err;
}
err = query_virtqueue(ndev, mvq, &attr);
if (err) {
mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue, err: %d\n", err);
return err;
}
mvq->avail_idx = attr.available_index;
mvq->used_idx = attr.used_index;
return 0;
}
static void suspend_vqs(struct mlx5_vdpa_net *ndev)
static int suspend_vqs(struct mlx5_vdpa_net *ndev)
{
int err = 0;
int i;
for (i = 0; i < ndev->mvdev.max_vqs; i++)
suspend_vq(ndev, &ndev->vqs[i]);
for (i = 0; i < ndev->cur_num_vqs; i++) {
int local_err = suspend_vq(ndev, &ndev->vqs[i]);
err = local_err ? local_err : err;
}
return err;
}
static void resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
{
if (!mvq->initialized || !is_resumable(ndev))
return;
int err;
if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND)
return;
if (!mvq->initialized)
return 0;
if (modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY))
mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u\n", mvq->index);
if (mvq->index >= ndev->cur_num_vqs)
return 0;
switch (mvq->fw_state) {
case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT:
/* Due to a FW quirk we need to modify the VQ fields first then change state.
* This should be fixed soon. After that, a single command can be used.
*/
err = modify_virtqueue(ndev, mvq, 0);
if (err) {
mlx5_vdpa_warn(&ndev->mvdev,
"modify vq properties failed for vq %u, err: %d\n",
mvq->index, err);
return err;
}
break;
case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND:
if (!is_resumable(ndev)) {
mlx5_vdpa_warn(&ndev->mvdev, "vq %d is not resumable\n", mvq->index);
return -EINVAL;
}
break;
case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY:
return 0;
default:
mlx5_vdpa_warn(&ndev->mvdev, "resume vq %u called from bad state %d\n",
mvq->index, mvq->fw_state);
return -EINVAL;
}
err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
if (err)
mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u, err: %d\n",
mvq->index, err);
return err;
}
static void resume_vqs(struct mlx5_vdpa_net *ndev)
static int resume_vqs(struct mlx5_vdpa_net *ndev)
{
for (int i = 0; i < ndev->mvdev.max_vqs; i++)
resume_vq(ndev, &ndev->vqs[i]);
int err = 0;
for (int i = 0; i < ndev->cur_num_vqs; i++) {
int local_err = resume_vq(ndev, &ndev->vqs[i]);
err = local_err ? local_err : err;
}
return err;
}
static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
@ -2033,14 +2133,22 @@ static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps)
if (err)
return err;
for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--)
teardown_vq(ndev, &ndev->vqs[i]);
for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) {
struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
if (is_resumable(ndev))
suspend_vq(ndev, mvq);
else
teardown_vq(ndev, mvq);
}
ndev->cur_num_vqs = 2 * newqps;
} else {
ndev->cur_num_vqs = 2 * newqps;
for (i = cur_qps * 2; i < 2 * newqps; i++) {
err = setup_vq(ndev, &ndev->vqs[i]);
struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
err = mvq->initialized ? resume_vq(ndev, mvq) : setup_vq(ndev, mvq, true);
if (err)
goto clean_added;
}
@ -2285,6 +2393,7 @@ static void mlx5_vdpa_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)
}
mvq = &ndev->vqs[idx];
ndev->needs_teardown = num != mvq->num_ent;
mvq->num_ent = num;
}
@ -2324,7 +2433,6 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
struct mlx5_vdpa_virtqueue *mvq;
int err;
if (!mvdev->actual_features)
return;
@ -2340,15 +2448,11 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready
mvq = &ndev->vqs[idx];
if (!ready) {
suspend_vq(ndev, mvq);
} else {
err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
if (err) {
mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err);
} else if (mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK) {
if (resume_vq(ndev, mvq))
ready = false;
}
}
mvq->ready = ready;
}
@ -2536,14 +2640,14 @@ static int verify_driver_features(struct mlx5_vdpa_dev *mvdev, u64 features)
return 0;
}
static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev)
static int setup_virtqueues(struct mlx5_vdpa_dev *mvdev, bool filled)
{
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
int err;
int i;
for (i = 0; i < mvdev->max_vqs; i++) {
err = setup_vq(ndev, &ndev->vqs[i]);
err = setup_vq(ndev, &ndev->vqs[i], filled);
if (err)
goto err_vq;
}
@ -2559,16 +2663,10 @@ err_vq:
static void teardown_virtqueues(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_virtqueue *mvq;
int i;
for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) {
mvq = &ndev->vqs[i];
if (!mvq->initialized)
continue;
teardown_vq(ndev, mvq);
}
for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--)
teardown_vq(ndev, &ndev->vqs[i]);
}
static void update_cvq_info(struct mlx5_vdpa_dev *mvdev)
@ -2705,6 +2803,8 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
u64 old_features = mvdev->actual_features;
u64 diff_features;
int err;
print_features(mvdev, features, true);
@ -2714,20 +2814,26 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
return err;
ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
ndev->rqt_size = mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);
else
ndev->rqt_size = 1;
/* Device must start with 1 queue pair, as per VIRTIO v1.2 spec, section
* 5.1.6.5.5 "Device operation in multiqueue mode":
*
* Multiqueue is disabled by default.
* The driver enables multiqueue by sending a command using class
* VIRTIO_NET_CTRL_MQ. The command selects the mode of multiqueue
* operation, as follows: ...
*/
ndev->cur_num_vqs = 2;
/* Interested in changes of vq features only. */
if (get_features(old_features) != get_features(mvdev->actual_features)) {
for (int i = 0; i < mvdev->max_vqs; ++i) {
struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
mvq->modified_fields |= (
MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION |
MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES
);
}
}
/* When below features diverge from initial device features, VQs need a full teardown. */
#define NEEDS_TEARDOWN_MASK (BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) | \
BIT_ULL(VIRTIO_NET_F_CSUM) | \
BIT_ULL(VIRTIO_F_VERSION_1))
diff_features = mvdev->mlx_features ^ mvdev->actual_features;
ndev->needs_teardown = !!(diff_features & NEEDS_TEARDOWN_MASK);
update_cvq_info(mvdev);
return err;
@ -2816,7 +2922,7 @@ static void restore_channels_info(struct mlx5_vdpa_net *ndev)
int i;
mlx5_clear_vqs(ndev);
init_mvqs(ndev);
mvqs_set_defaults(ndev);
for (i = 0; i < ndev->mvdev.max_vqs; i++) {
mvq = &ndev->vqs[i];
ri = &mvq->ri;
@ -2848,12 +2954,12 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
if (err)
return err;
teardown_driver(ndev);
teardown_vq_resources(ndev);
}
mlx5_vdpa_update_mr(mvdev, new_mr, asid);
for (int i = 0; i < ndev->cur_num_vqs; i++)
for (int i = 0; i < mvdev->max_vqs; i++)
ndev->vqs[i].modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY |
MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
@ -2862,7 +2968,7 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
if (teardown) {
restore_channels_info(ndev);
err = setup_driver(mvdev);
err = setup_vq_resources(ndev, true);
if (err)
return err;
}
@ -2873,9 +2979,9 @@ static int mlx5_vdpa_change_map(struct mlx5_vdpa_dev *mvdev,
}
/* reslock must be held for this function */
static int setup_driver(struct mlx5_vdpa_dev *mvdev)
static int setup_vq_resources(struct mlx5_vdpa_net *ndev, bool filled)
{
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
int err;
WARN_ON(!rwsem_is_locked(&ndev->reslock));
@ -2891,7 +2997,7 @@ static int setup_driver(struct mlx5_vdpa_dev *mvdev)
if (err)
goto err_setup;
err = setup_virtqueues(mvdev);
err = setup_virtqueues(mvdev, filled);
if (err) {
mlx5_vdpa_warn(mvdev, "setup_virtqueues\n");
goto err_setup;
@ -2931,7 +3037,7 @@ out:
}
/* reslock must be held for this function */
static void teardown_driver(struct mlx5_vdpa_net *ndev)
static void teardown_vq_resources(struct mlx5_vdpa_net *ndev)
{
WARN_ON(!rwsem_is_locked(&ndev->reslock));
@ -2945,18 +3051,7 @@ static void teardown_driver(struct mlx5_vdpa_net *ndev)
destroy_rqt(ndev);
teardown_virtqueues(ndev);
ndev->setup = false;
}
static void clear_vqs_ready(struct mlx5_vdpa_net *ndev)
{
int i;
for (i = 0; i < ndev->mvdev.max_vqs; i++) {
ndev->vqs[i].ready = false;
ndev->vqs[i].modified_fields = 0;
}
ndev->mvdev.cvq.ready = false;
ndev->needs_teardown = false;
}
static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev)
@ -2997,10 +3092,22 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
goto err_setup;
}
register_link_notifier(ndev);
err = setup_driver(mvdev);
if (err) {
mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
goto err_driver;
if (ndev->needs_teardown)
teardown_vq_resources(ndev);
if (ndev->setup) {
err = resume_vqs(ndev);
if (err) {
mlx5_vdpa_warn(mvdev, "failed to resume VQs\n");
goto err_driver;
}
} else {
err = setup_vq_resources(ndev, true);
if (err) {
mlx5_vdpa_warn(mvdev, "failed to setup driver\n");
goto err_driver;
}
}
} else {
mlx5_vdpa_warn(mvdev, "did not expect DRIVER_OK to be cleared\n");
@ -3030,23 +3137,48 @@ static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev)
mvdev->group2asid[i] = 0;
}
static bool needs_vqs_reset(const struct mlx5_vdpa_dev *mvdev)
{
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[0];
if (mvdev->status & VIRTIO_CONFIG_S_DRIVER_OK)
return true;
if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT)
return true;
return mvq->modified_fields & (
MLX5_VIRTQ_MODIFY_MASK_STATE |
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS |
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_AVAIL_IDX |
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX
);
}
static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
bool vq_reset;
print_status(mvdev, 0, true);
mlx5_vdpa_info(mvdev, "performing device reset\n");
down_write(&ndev->reslock);
unregister_link_notifier(ndev);
teardown_driver(ndev);
clear_vqs_ready(ndev);
vq_reset = needs_vqs_reset(mvdev);
if (vq_reset) {
teardown_vq_resources(ndev);
mvqs_set_defaults(ndev);
}
if (flags & VDPA_RESET_F_CLEAN_MAP)
mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
ndev->mvdev.status = 0;
ndev->mvdev.suspended = false;
ndev->cur_num_vqs = 0;
ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT;
ndev->mvdev.cvq.ready = false;
ndev->mvdev.cvq.received_desc = 0;
ndev->mvdev.cvq.completed_desc = 0;
memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
@ -3059,6 +3191,8 @@ static int mlx5_vdpa_compat_reset(struct vdpa_device *vdev, u32 flags)
if (mlx5_vdpa_create_dma_mr(mvdev))
mlx5_vdpa_warn(mvdev, "create MR failed\n");
}
if (vq_reset)
setup_vq_resources(ndev, false);
up_write(&ndev->reslock);
return 0;
@ -3197,7 +3331,7 @@ static void mlx5_vdpa_free(struct vdpa_device *vdev)
ndev = to_mlx5_vdpa_ndev(mvdev);
free_resources(ndev);
free_fixed_resources(ndev);
mlx5_vdpa_destroy_mr_resources(mvdev);
if (!is_zero_ether_addr(ndev->config.mac)) {
pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
@ -3361,27 +3495,25 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
struct mlx5_vdpa_virtqueue *mvq;
int i;
int err;
mlx5_vdpa_info(mvdev, "suspending device\n");
down_write(&ndev->reslock);
unregister_link_notifier(ndev);
for (i = 0; i < ndev->cur_num_vqs; i++) {
mvq = &ndev->vqs[i];
suspend_vq(ndev, mvq);
}
err = suspend_vqs(ndev);
mlx5_vdpa_cvq_suspend(mvdev);
mvdev->suspended = true;
up_write(&ndev->reslock);
return 0;
return err;
}
static int mlx5_vdpa_resume(struct vdpa_device *vdev)
{
struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
struct mlx5_vdpa_net *ndev;
int err;
ndev = to_mlx5_vdpa_ndev(mvdev);
@ -3389,10 +3521,11 @@ static int mlx5_vdpa_resume(struct vdpa_device *vdev)
down_write(&ndev->reslock);
mvdev->suspended = false;
resume_vqs(ndev);
err = resume_vqs(ndev);
register_link_notifier(ndev);
up_write(&ndev->reslock);
return 0;
return err;
}
static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group,
@ -3467,7 +3600,7 @@ static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu)
return 0;
}
static int alloc_resources(struct mlx5_vdpa_net *ndev)
static int alloc_fixed_resources(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_net_resources *res = &ndev->res;
int err;
@ -3494,7 +3627,7 @@ err_tis:
return err;
}
static void free_resources(struct mlx5_vdpa_net *ndev)
static void free_fixed_resources(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_net_resources *res = &ndev->res;
@ -3506,7 +3639,7 @@ static void free_resources(struct mlx5_vdpa_net *ndev)
res->valid = false;
}
static void init_mvqs(struct mlx5_vdpa_net *ndev)
static void mvqs_set_defaults(struct mlx5_vdpa_net *ndev)
{
struct mlx5_vdpa_virtqueue *mvq;
int i;
@ -3518,12 +3651,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev)
mvq->ndev = ndev;
mvq->fwqp.fw = true;
mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE;
}
for (; i < ndev->mvdev.max_vqs; i++) {
mvq = &ndev->vqs[i];
memset(mvq, 0, offsetof(struct mlx5_vdpa_virtqueue, ri));
mvq->index = i;
mvq->ndev = ndev;
mvq->num_ent = MLX5V_DEFAULT_VQ_SIZE;
}
}
@ -3660,8 +3788,9 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
err = -ENOMEM;
goto err_alloc;
}
ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT;
init_mvqs(ndev);
mvqs_set_defaults(ndev);
allocate_irqs(ndev);
init_rwsem(&ndev->reslock);
config = &ndev->config;
@ -3718,8 +3847,12 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_alloc;
}
if (device_features & BIT_ULL(VIRTIO_NET_F_MQ))
if (device_features & BIT_ULL(VIRTIO_NET_F_MQ)) {
config->max_virtqueue_pairs = cpu_to_mlx5vdpa16(mvdev, max_vqs / 2);
ndev->rqt_size = max_vqs / 2;
} else {
ndev->rqt_size = 1;
}
ndev->mvdev.mlx_features = device_features;
mvdev->vdev.dma_dev = &mdev->pdev->dev;
@ -3735,7 +3868,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_res;
}
err = alloc_resources(ndev);
err = alloc_fixed_resources(ndev);
if (err)
goto err_mr;
@ -3753,12 +3886,25 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
goto err_reg;
mgtdev->ndev = ndev;
/* For virtio-vdpa, the device was set up during device register. */
if (ndev->setup)
return 0;
down_write(&ndev->reslock);
err = setup_vq_resources(ndev, false);
up_write(&ndev->reslock);
if (err)
goto err_setup_vq_res;
return 0;
err_setup_vq_res:
_vdpa_unregister_device(&mvdev->vdev);
err_reg:
destroy_workqueue(mvdev->wq);
err_res2:
free_resources(ndev);
free_fixed_resources(ndev);
err_mr:
mlx5_vdpa_destroy_mr_resources(mvdev);
err_res:
@ -3780,6 +3926,11 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *
unregister_link_notifier(ndev);
_vdpa_unregister_device(dev);
down_write(&ndev->reslock);
teardown_vq_resources(ndev);
up_write(&ndev->reslock);
wq = mvdev->wq;
mvdev->wq = NULL;
destroy_workqueue(wq);

1
drivers/vdpa/mlx5/net/mlx5_vnet.h
View File

@ -56,6 +56,7 @@ struct mlx5_vdpa_net {
struct dentry *rx_dent;
struct dentry *rx_table_dent;
bool setup;
bool needs_teardown;
u32 cur_num_vqs;
u32 rqt_size;
bool nb_registered;

4
drivers/vdpa/octeon_ep/Makefile Normal file
View File

@ -0,0 +1,4 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa.o
octep_vdpa-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa_main.o
octep_vdpa-$(CONFIG_OCTEONEP_VDPA) += octep_vdpa_hw.o

94
drivers/vdpa/octeon_ep/octep_vdpa.h Normal file
View File

@ -0,0 +1,94 @@
/* SPDX-License-Identifier: GPL-2.0-only
* Copyright (C) 2024 Marvell.
*/
#ifndef __OCTEP_VDPA_H__
#define __OCTEP_VDPA_H__
#include <linux/pci.h>
#include <linux/pci_regs.h>
#include <linux/vdpa.h>
#include <linux/virtio_pci_modern.h>
#include <uapi/linux/virtio_net.h>
#include <uapi/linux/virtio_blk.h>
#include <uapi/linux/virtio_config.h>
#include <uapi/linux/virtio_pci.h>
#include <uapi/linux/vdpa.h>
#define OCTEP_VDPA_DEVID_CN106K_PF 0xb900
#define OCTEP_VDPA_DEVID_CN106K_VF 0xb903
#define OCTEP_VDPA_DEVID_CN105K_PF 0xba00
#define OCTEP_VDPA_DEVID_CN105K_VF 0xba03
#define OCTEP_VDPA_DEVID_CN103K_PF 0xbd00
#define OCTEP_VDPA_DEVID_CN103K_VF 0xbd03
#define OCTEP_HW_MBOX_BAR 0
#define OCTEP_HW_CAPS_BAR 4
#define OCTEP_DEV_READY_SIGNATURE 0xBABABABA
#define OCTEP_EPF_RINFO(x) (0x000209f0 | ((x) << 25))
#define OCTEP_VF_MBOX_DATA(x) (0x00010210 | ((x) << 17))
#define OCTEP_PF_MBOX_DATA(x) (0x00022000 | ((x) << 4))
#define OCTEP_EPF_RINFO_RPVF(val) (((val) >> 32) & 0xF)
#define OCTEP_EPF_RINFO_NVFS(val) (((val) >> 48) & 0x7F)
#define OCTEP_FW_READY_SIGNATURE0 0xFEEDFEED
#define OCTEP_FW_READY_SIGNATURE1 0x3355ffaa
enum octep_vdpa_dev_status {
OCTEP_VDPA_DEV_STATUS_INVALID,
OCTEP_VDPA_DEV_STATUS_ALLOC,
OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT,
OCTEP_VDPA_DEV_STATUS_INIT,
OCTEP_VDPA_DEV_STATUS_READY,
OCTEP_VDPA_DEV_STATUS_UNINIT
};
struct octep_vring_info {
struct vdpa_callback cb;
void __iomem *notify_addr;
u32 __iomem *cb_notify_addr;
phys_addr_t notify_pa;
char msix_name[256];
};
struct octep_hw {
struct pci_dev *pdev;
u8 __iomem *base[PCI_STD_NUM_BARS];
struct virtio_pci_common_cfg __iomem *common_cfg;
u8 __iomem *dev_cfg;
u8 __iomem *isr;
void __iomem *notify_base;
phys_addr_t notify_base_pa;
u32 notify_off_multiplier;
u8 notify_bar;
struct octep_vring_info *vqs;
struct vdpa_callback config_cb;
u64 features;
u16 nr_vring;
u32 config_size;
int irq;
};
u8 octep_hw_get_status(struct octep_hw *oct_hw);
void octep_hw_set_status(struct octep_hw *dev, uint8_t status);
void octep_hw_reset(struct octep_hw *oct_hw);
void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id);
void octep_notify_queue(struct octep_hw *oct_hw, u16 qid);
void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length);
int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
u64 device_area);
void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num);
void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready);
bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid);
int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state);
int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state);
u16 octep_get_vq_size(struct octep_hw *oct_hw);
int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev);
u64 octep_hw_get_dev_features(struct octep_hw *oct_hw);
void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features);
u64 octep_hw_get_drv_features(struct octep_hw *oct_hw);
int octep_verify_features(u64 features);
#endif /* __OCTEP_VDPA_H__ */

517
drivers/vdpa/octeon_ep/octep_vdpa_hw.c Normal file
View File

@ -0,0 +1,517 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2024 Marvell. */
#include <linux/iopoll.h>
#include "octep_vdpa.h"
enum octep_mbox_ids {
OCTEP_MBOX_MSG_SET_VQ_STATE = 1,
OCTEP_MBOX_MSG_GET_VQ_STATE,
};
#define OCTEP_HW_TIMEOUT 10000000
#define MBOX_OFFSET 64
#define MBOX_RSP_MASK 0x00000001
#define MBOX_RC_MASK 0x0000FFFE
#define MBOX_RSP_TO_ERR(val) (-(((val) & MBOX_RC_MASK) >> 2))
#define MBOX_AVAIL(val) (((val) & MBOX_RSP_MASK))
#define MBOX_RSP(val) ((val) & (MBOX_RC_MASK | MBOX_RSP_MASK))
#define DEV_RST_ACK_BIT 7
#define FEATURE_SEL_ACK_BIT 15
#define QUEUE_SEL_ACK_BIT 15
struct octep_mbox_hdr {
u8 ver;
u8 rsvd1;
u16 id;
u16 rsvd2;
#define MBOX_REQ_SIG (0xdead)
#define MBOX_RSP_SIG (0xbeef)
u16 sig;
};
struct octep_mbox_sts {
u16 rsp:1;
u16 rc:15;
u16 rsvd;
};
struct octep_mbox {
struct octep_mbox_hdr hdr;
struct octep_mbox_sts sts;
u64 rsvd;
u32 data[];
};
static inline struct octep_mbox __iomem *octep_get_mbox(struct octep_hw *oct_hw)
{
return (struct octep_mbox __iomem *)(oct_hw->dev_cfg + MBOX_OFFSET);
}
static inline int octep_wait_for_mbox_avail(struct octep_mbox __iomem *mbox)
{
u32 val;
return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_AVAIL(val), 10,
OCTEP_HW_TIMEOUT);
}
static inline int octep_wait_for_mbox_rsp(struct octep_mbox __iomem *mbox)
{
u32 val;
return readx_poll_timeout(ioread32, &mbox->sts, val, MBOX_RSP(val), 10,
OCTEP_HW_TIMEOUT);
}
static inline void octep_write_hdr(struct octep_mbox __iomem *mbox, u16 id, u16 sig)
{
iowrite16(id, &mbox->hdr.id);
iowrite16(sig, &mbox->hdr.sig);
}
static inline u32 octep_read_sig(struct octep_mbox __iomem *mbox)
{
return ioread16(&mbox->hdr.sig);
}
static inline void octep_write_sts(struct octep_mbox __iomem *mbox, u32 sts)
{
iowrite32(sts, &mbox->sts);
}
static inline u32 octep_read_sts(struct octep_mbox __iomem *mbox)
{
return ioread32(&mbox->sts);
}
static inline u32 octep_read32_word(struct octep_mbox __iomem *mbox, u16 word_idx)
{
return ioread32(&mbox->data[word_idx]);
}
static inline void octep_write32_word(struct octep_mbox __iomem *mbox, u16 word_idx, u32 word)
{
return iowrite32(word, &mbox->data[word_idx]);
}
static int octep_process_mbox(struct octep_hw *oct_hw, u16 id, u16 qid, void *buffer,
u32 buf_size, bool write)
{
struct octep_mbox __iomem *mbox = octep_get_mbox(oct_hw);
struct pci_dev *pdev = oct_hw->pdev;
u32 *p = (u32 *)buffer;
u16 data_wds;
int ret, i;
u32 val;
if (!IS_ALIGNED(buf_size, 4))
return -EINVAL;
/* Make sure mbox space is available */
ret = octep_wait_for_mbox_avail(mbox);
if (ret) {
dev_warn(&pdev->dev, "Timeout waiting for previous mbox data to be consumed\n");
return ret;
}
data_wds = buf_size / 4;
if (write) {
for (i = 1; i <= data_wds; i++) {
octep_write32_word(mbox, i, *p);
p++;
}
}
octep_write32_word(mbox, 0, (u32)qid);
octep_write_sts(mbox, 0);
octep_write_hdr(mbox, id, MBOX_REQ_SIG);
ret = octep_wait_for_mbox_rsp(mbox);
if (ret) {
dev_warn(&pdev->dev, "Timeout waiting for mbox : %d response\n", id);
return ret;
}
val = octep_read_sig(mbox);
if ((val & 0xFFFF) != MBOX_RSP_SIG) {
dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id);
return ret;
}
val = octep_read_sts(mbox);
if (val & MBOX_RC_MASK) {
ret = MBOX_RSP_TO_ERR(val);
dev_warn(&pdev->dev, "Error while processing mbox : %d, err %d\n", id, ret);
return ret;
}
if (!write)
for (i = 1; i <= data_wds; i++)
*p++ = octep_read32_word(mbox, i);
return 0;
}
static void octep_mbox_init(struct octep_mbox __iomem *mbox)
{
iowrite32(1, &mbox->sts);
}
int octep_verify_features(u64 features)
{
/* Minimum features to expect */
if (!(features & BIT_ULL(VIRTIO_F_VERSION_1)))
return -EOPNOTSUPP;
if (!(features & BIT_ULL(VIRTIO_F_NOTIFICATION_DATA)))
return -EOPNOTSUPP;
if (!(features & BIT_ULL(VIRTIO_F_RING_PACKED)))
return -EOPNOTSUPP;
return 0;
}
u8 octep_hw_get_status(struct octep_hw *oct_hw)
{
return ioread8(&oct_hw->common_cfg->device_status);
}
void octep_hw_set_status(struct octep_hw *oct_hw, u8 status)
{
iowrite8(status, &oct_hw->common_cfg->device_status);
}
void octep_hw_reset(struct octep_hw *oct_hw)
{
u8 val;
octep_hw_set_status(oct_hw, 0 | BIT(DEV_RST_ACK_BIT));
if (readx_poll_timeout(ioread8, &oct_hw->common_cfg->device_status, val, !val, 10,
OCTEP_HW_TIMEOUT)) {
dev_warn(&oct_hw->pdev->dev, "Octeon device reset timeout\n");
return;
}
}
static int feature_sel_write_with_timeout(struct octep_hw *oct_hw, u32 select, void __iomem *addr)
{
u32 val;
iowrite32(select | BIT(FEATURE_SEL_ACK_BIT), addr);
if (readx_poll_timeout(ioread32, addr, val, val == select, 10, OCTEP_HW_TIMEOUT)) {
dev_warn(&oct_hw->pdev->dev, "Feature select%d write timeout\n", select);
return -1;
}
return 0;
}
u64 octep_hw_get_dev_features(struct octep_hw *oct_hw)
{
u32 features_lo, features_hi;
if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->device_feature_select))
return 0;
features_lo = ioread32(&oct_hw->common_cfg->device_feature);
if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->device_feature_select))
return 0;
features_hi = ioread32(&oct_hw->common_cfg->device_feature);
return ((u64)features_hi << 32) | features_lo;
}
u64 octep_hw_get_drv_features(struct octep_hw *oct_hw)
{
u32 features_lo, features_hi;
if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
return 0;
features_lo = ioread32(&oct_hw->common_cfg->guest_feature);
if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
return 0;
features_hi = ioread32(&oct_hw->common_cfg->guest_feature);
return ((u64)features_hi << 32) | features_lo;
}
void octep_hw_set_drv_features(struct octep_hw *oct_hw, u64 features)
{
if (feature_sel_write_with_timeout(oct_hw, 0, &oct_hw->common_cfg->guest_feature_select))
return;
iowrite32(features & (BIT_ULL(32) - 1), &oct_hw->common_cfg->guest_feature);
if (feature_sel_write_with_timeout(oct_hw, 1, &oct_hw->common_cfg->guest_feature_select))
return;
iowrite32(features >> 32, &oct_hw->common_cfg->guest_feature);
}
void octep_write_queue_select(struct octep_hw *oct_hw, u16 queue_id)
{
u16 val;
iowrite16(queue_id | BIT(QUEUE_SEL_ACK_BIT), &oct_hw->common_cfg->queue_select);
if (readx_poll_timeout(ioread16, &oct_hw->common_cfg->queue_select, val, val == queue_id,
10, OCTEP_HW_TIMEOUT)) {
dev_warn(&oct_hw->pdev->dev, "Queue select write timeout\n");
return;
}
}
void octep_notify_queue(struct octep_hw *oct_hw, u16 qid)
{
iowrite16(qid, oct_hw->vqs[qid].notify_addr);
}
void octep_read_dev_config(struct octep_hw *oct_hw, u64 offset, void *dst, int length)
{
u8 old_gen, new_gen, *p;
int i;
if (WARN_ON(offset + length > oct_hw->config_size))
return;
do {
old_gen = ioread8(&oct_hw->common_cfg->config_generation);
p = dst;
for (i = 0; i < length; i++)
*p++ = ioread8(oct_hw->dev_cfg + offset + i);
new_gen = ioread8(&oct_hw->common_cfg->config_generation);
} while (old_gen != new_gen);
}
int octep_set_vq_address(struct octep_hw *oct_hw, u16 qid, u64 desc_area, u64 driver_area,
u64 device_area)
{
struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
octep_write_queue_select(oct_hw, qid);
vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo,
&cfg->queue_desc_hi);
vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo,
&cfg->queue_avail_hi);
vp_iowrite64_twopart(device_area, &cfg->queue_used_lo,
&cfg->queue_used_hi);
return 0;
}
int octep_get_vq_state(struct octep_hw *oct_hw, u16 qid, struct vdpa_vq_state *state)
{
return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_GET_VQ_STATE, qid, state,
sizeof(*state), 0);
}
int octep_set_vq_state(struct octep_hw *oct_hw, u16 qid, const struct vdpa_vq_state *state)
{
struct vdpa_vq_state q_state;
memcpy(&q_state, state, sizeof(struct vdpa_vq_state));
return octep_process_mbox(oct_hw, OCTEP_MBOX_MSG_SET_VQ_STATE, qid, &q_state,
sizeof(*state), 1);
}
void octep_set_vq_num(struct octep_hw *oct_hw, u16 qid, u32 num)
{
struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
octep_write_queue_select(oct_hw, qid);
iowrite16(num, &cfg->queue_size);
}
void octep_set_vq_ready(struct octep_hw *oct_hw, u16 qid, bool ready)
{
struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
octep_write_queue_select(oct_hw, qid);
iowrite16(ready, &cfg->queue_enable);
}
bool octep_get_vq_ready(struct octep_hw *oct_hw, u16 qid)
{
struct virtio_pci_common_cfg __iomem *cfg = oct_hw->common_cfg;
octep_write_queue_select(oct_hw, qid);
return ioread16(&cfg->queue_enable);
}
u16 octep_get_vq_size(struct octep_hw *oct_hw)
{
octep_write_queue_select(oct_hw, 0);
return ioread16(&oct_hw->common_cfg->queue_size);
}
static u32 octep_get_config_size(struct octep_hw *oct_hw)
{
return sizeof(struct virtio_net_config);
}
static void __iomem *octep_get_cap_addr(struct octep_hw *oct_hw, struct virtio_pci_cap *cap)
{
struct device *dev = &oct_hw->pdev->dev;
u32 length = le32_to_cpu(cap->length);
u32 offset = le32_to_cpu(cap->offset);
u8 bar = cap->bar;
u32 len;
if (bar != OCTEP_HW_CAPS_BAR) {
dev_err(dev, "Invalid bar: %u\n", bar);
return NULL;
}
if (offset + length < offset) {
dev_err(dev, "offset(%u) + length(%u) overflows\n",
offset, length);
return NULL;
}
len = pci_resource_len(oct_hw->pdev, bar);
if (offset + length > len) {
dev_err(dev, "invalid cap: overflows bar space: %u > %u\n",
offset + length, len);
return NULL;
}
return oct_hw->base[bar] + offset;
}
/* In Octeon DPU device, the virtio config space is completely
* emulated by the device's firmware. So, the standard pci config
* read apis can't be used for reading the virtio capability.
*/
static void octep_pci_caps_read(struct octep_hw *oct_hw, void *buf, size_t len, off_t offset)
{
u8 __iomem *bar = oct_hw->base[OCTEP_HW_CAPS_BAR];
u8 *p = buf;
size_t i;
for (i = 0; i < len; i++)
*p++ = ioread8(bar + offset + i);
}
static int octep_pci_signature_verify(struct octep_hw *oct_hw)
{
u32 signature[2];
octep_pci_caps_read(oct_hw, &signature, sizeof(signature), 0);
if (signature[0] != OCTEP_FW_READY_SIGNATURE0)
return -1;
if (signature[1] != OCTEP_FW_READY_SIGNATURE1)
return -1;
return 0;
}
int octep_hw_caps_read(struct octep_hw *oct_hw, struct pci_dev *pdev)
{
struct octep_mbox __iomem *mbox;
struct device *dev = &pdev->dev;
struct virtio_pci_cap cap;
u16 notify_off;
int i, ret;
u8 pos;
oct_hw->pdev = pdev;
ret = octep_pci_signature_verify(oct_hw);
if (ret) {
dev_err(dev, "Octeon Virtio FW is not initialized\n");
return -EIO;
}
octep_pci_caps_read(oct_hw, &pos, 1, PCI_CAPABILITY_LIST);
while (pos) {
octep_pci_caps_read(oct_hw, &cap, 2, pos);
if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
dev_err(dev, "Found invalid capability vndr id: %d\n", cap.cap_vndr);
break;
}
octep_pci_caps_read(oct_hw, &cap, sizeof(cap), pos);
dev_info(dev, "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u\n",
pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
switch (cap.cfg_type) {
case VIRTIO_PCI_CAP_COMMON_CFG:
oct_hw->common_cfg = octep_get_cap_addr(oct_hw, &cap);
break;
case VIRTIO_PCI_CAP_NOTIFY_CFG:
octep_pci_caps_read(oct_hw, &oct_hw->notify_off_multiplier,
4, pos + sizeof(cap));
oct_hw->notify_base = octep_get_cap_addr(oct_hw, &cap);
oct_hw->notify_bar = cap.bar;
oct_hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
le32_to_cpu(cap.offset);
break;
case VIRTIO_PCI_CAP_DEVICE_CFG:
oct_hw->dev_cfg = octep_get_cap_addr(oct_hw, &cap);
break;
case VIRTIO_PCI_CAP_ISR_CFG:
oct_hw->isr = octep_get_cap_addr(oct_hw, &cap);
break;
}
pos = cap.cap_next;
}
if (!oct_hw->common_cfg || !oct_hw->notify_base ||
!oct_hw->dev_cfg || !oct_hw->isr) {
dev_err(dev, "Incomplete PCI capabilities");
return -EIO;
}
dev_info(dev, "common cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->common_cfg);
dev_info(dev, "device cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->dev_cfg);
dev_info(dev, "isr cfg mapped at: 0x%016llx\n", (u64)(uintptr_t)oct_hw->isr);
dev_info(dev, "notify base: 0x%016llx, notify off multiplier: %u\n",
(u64)(uintptr_t)oct_hw->notify_base, oct_hw->notify_off_multiplier);
oct_hw->config_size = octep_get_config_size(oct_hw);
oct_hw->features = octep_hw_get_dev_features(oct_hw);
ret = octep_verify_features(oct_hw->features);
if (ret) {
dev_err(&pdev->dev, "Couldn't read features from the device FW\n");
return ret;
}
oct_hw->nr_vring = vp_ioread16(&oct_hw->common_cfg->num_queues);
oct_hw->vqs = devm_kcalloc(&pdev->dev, oct_hw->nr_vring, sizeof(*oct_hw->vqs), GFP_KERNEL);
if (!oct_hw->vqs)
return -ENOMEM;
oct_hw->irq = -1;
dev_info(&pdev->dev, "Device features : %llx\n", oct_hw->features);
dev_info(&pdev->dev, "Maximum queues : %u\n", oct_hw->nr_vring);
for (i = 0; i < oct_hw->nr_vring; i++) {
octep_write_queue_select(oct_hw, i);
notify_off = vp_ioread16(&oct_hw->common_cfg->queue_notify_off);
oct_hw->vqs[i].notify_addr = oct_hw->notify_base +
notify_off * oct_hw->notify_off_multiplier;
oct_hw->vqs[i].cb_notify_addr = (u32 __iomem *)oct_hw->vqs[i].notify_addr + 1;
oct_hw->vqs[i].notify_pa = oct_hw->notify_base_pa +
notify_off * oct_hw->notify_off_multiplier;
}
mbox = octep_get_mbox(oct_hw);
octep_mbox_init(mbox);
dev_info(dev, "mbox mapped at: 0x%016llx\n", (u64)(uintptr_t)mbox);
return 0;
}

857
drivers/vdpa/octeon_ep/octep_vdpa_main.c Normal file
View File

@ -0,0 +1,857 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2024 Marvell. */
#include <linux/interrupt.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/module.h>
#include <linux/iommu.h>
#include "octep_vdpa.h"
#define OCTEP_VDPA_DRIVER_NAME "octep_vdpa"
struct octep_pf {
u8 __iomem *base[PCI_STD_NUM_BARS];
struct pci_dev *pdev;
struct resource res;
u64 vf_base;
int enabled_vfs;
u32 vf_stride;
u16 vf_devid;
};
struct octep_vdpa {
struct vdpa_device vdpa;
struct octep_hw *oct_hw;
struct pci_dev *pdev;
};
struct octep_vdpa_mgmt_dev {
struct vdpa_mgmt_dev mdev;
struct octep_hw oct_hw;
struct pci_dev *pdev;
/* Work entry to handle device setup */
struct work_struct setup_task;
/* Device status */
atomic_t status;
};
static struct octep_hw *vdpa_to_octep_hw(struct vdpa_device *vdpa_dev)
{
struct octep_vdpa *oct_vdpa;
oct_vdpa = container_of(vdpa_dev, struct octep_vdpa, vdpa);
return oct_vdpa->oct_hw;
}
static irqreturn_t octep_vdpa_intr_handler(int irq, void *data)
{
struct octep_hw *oct_hw = data;
int i;
for (i = 0; i < oct_hw->nr_vring; i++) {
if (oct_hw->vqs[i].cb.callback && ioread32(oct_hw->vqs[i].cb_notify_addr)) {
/* Acknowledge the per queue notification to the device */
iowrite32(0, oct_hw->vqs[i].cb_notify_addr);
oct_hw->vqs[i].cb.callback(oct_hw->vqs[i].cb.private);
}
}
return IRQ_HANDLED;
}
static void octep_free_irqs(struct octep_hw *oct_hw)
{
struct pci_dev *pdev = oct_hw->pdev;
if (oct_hw->irq != -1) {
devm_free_irq(&pdev->dev, oct_hw->irq, oct_hw);
oct_hw->irq = -1;
}
pci_free_irq_vectors(pdev);
}
static int octep_request_irqs(struct octep_hw *oct_hw)
{
struct pci_dev *pdev = oct_hw->pdev;
int ret, irq;
/* Currently HW device provisions one IRQ per VF, hence
* allocate one IRQ for all virtqueues call interface.
*/
ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX);
if (ret < 0) {
dev_err(&pdev->dev, "Failed to alloc msix vector");
return ret;
}
snprintf(oct_hw->vqs->msix_name, sizeof(oct_hw->vqs->msix_name),
OCTEP_VDPA_DRIVER_NAME "-vf-%d", pci_iov_vf_id(pdev));
irq = pci_irq_vector(pdev, 0);
ret = devm_request_irq(&pdev->dev, irq, octep_vdpa_intr_handler, 0,
oct_hw->vqs->msix_name, oct_hw);
if (ret) {
dev_err(&pdev->dev, "Failed to register interrupt handler\n");
goto free_irq_vec;
}
oct_hw->irq = irq;
return 0;
free_irq_vec:
pci_free_irq_vectors(pdev);
return ret;
}
static u64 octep_vdpa_get_device_features(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return oct_hw->features;
}
static int octep_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
int ret;
pr_debug("Driver Features: %llx\n", features);
ret = octep_verify_features(features);
if (ret) {
dev_warn(&oct_hw->pdev->dev,
"Must negotiate minimum features 0x%llx for this device",
BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) |
BIT_ULL(VIRTIO_F_RING_PACKED));
return ret;
}
octep_hw_set_drv_features(oct_hw, features);
return 0;
}
static u64 octep_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return octep_hw_get_drv_features(oct_hw);
}
static u8 octep_vdpa_get_status(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return octep_hw_get_status(oct_hw);
}
static void octep_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
u8 status_old;
status_old = octep_hw_get_status(oct_hw);
if (status_old == status)
return;
if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&
!(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {
if (octep_request_irqs(oct_hw))
status = status_old | VIRTIO_CONFIG_S_FAILED;
}
octep_hw_set_status(oct_hw, status);
}
static int octep_vdpa_reset(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
u8 status = octep_hw_get_status(oct_hw);
u16 qid;
if (status == 0)
return 0;
for (qid = 0; qid < oct_hw->nr_vring; qid++) {
oct_hw->vqs[qid].cb.callback = NULL;
oct_hw->vqs[qid].cb.private = NULL;
oct_hw->config_cb.callback = NULL;
oct_hw->config_cb.private = NULL;
}
octep_hw_reset(oct_hw);
if (status & VIRTIO_CONFIG_S_DRIVER_OK)
octep_free_irqs(oct_hw);
return 0;
}
static u16 octep_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return octep_get_vq_size(oct_hw);
}
static int octep_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
struct vdpa_vq_state *state)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return octep_get_vq_state(oct_hw, qid, state);
}
static int octep_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
const struct vdpa_vq_state *state)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return octep_set_vq_state(oct_hw, qid, state);
}
static void octep_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, struct vdpa_callback *cb)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
oct_hw->vqs[qid].cb = *cb;
}
static void octep_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
octep_set_vq_ready(oct_hw, qid, ready);
}
static bool octep_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return octep_get_vq_ready(oct_hw, qid);
}
static void octep_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
octep_set_vq_num(oct_hw, qid, num);
}
static int octep_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, u64 desc_area,
u64 driver_area, u64 device_area)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
pr_debug("qid[%d]: desc_area: %llx\n", qid, desc_area);
pr_debug("qid[%d]: driver_area: %llx\n", qid, driver_area);
pr_debug("qid[%d]: device_area: %llx\n\n", qid, device_area);
return octep_set_vq_address(oct_hw, qid, desc_area, driver_area, device_area);
}
static void octep_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)
{
/* Not supported */
}
static void octep_vdpa_kick_vq_with_data(struct vdpa_device *vdpa_dev, u32 data)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
u16 idx = data & 0xFFFF;
vp_iowrite32(data, oct_hw->vqs[idx].notify_addr);
}
static u32 octep_vdpa_get_generation(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return vp_ioread8(&oct_hw->common_cfg->config_generation);
}
static u32 octep_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
{
return VIRTIO_ID_NET;
}
static u32 octep_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)
{
return PCI_VENDOR_ID_CAVIUM;
}
static u32 octep_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)
{
return PAGE_SIZE;
}
static size_t octep_vdpa_get_config_size(struct vdpa_device *vdpa_dev)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
return oct_hw->config_size;
}
static void octep_vdpa_get_config(struct vdpa_device *vdpa_dev, unsigned int offset, void *buf,
unsigned int len)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
octep_read_dev_config(oct_hw, offset, buf, len);
}
static void octep_vdpa_set_config(struct vdpa_device *vdpa_dev, unsigned int offset,
const void *buf, unsigned int len)
{
/* Not supported */
}
static void octep_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, struct vdpa_callback *cb)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
oct_hw->config_cb.callback = cb->callback;
oct_hw->config_cb.private = cb->private;
}
static struct vdpa_notification_area octep_get_vq_notification(struct vdpa_device *vdpa_dev,
u16 idx)
{
struct octep_hw *oct_hw = vdpa_to_octep_hw(vdpa_dev);
struct vdpa_notification_area area;
area.addr = oct_hw->vqs[idx].notify_pa;
area.size = PAGE_SIZE;
return area;
}
static struct vdpa_config_ops octep_vdpa_ops = {
.get_device_features = octep_vdpa_get_device_features,
.set_driver_features = octep_vdpa_set_driver_features,
.get_driver_features = octep_vdpa_get_driver_features,
.get_status = octep_vdpa_get_status,
.set_status = octep_vdpa_set_status,
.reset = octep_vdpa_reset,
.get_vq_num_max = octep_vdpa_get_vq_num_max,
.get_vq_state = octep_vdpa_get_vq_state,
.set_vq_state = octep_vdpa_set_vq_state,
.set_vq_cb = octep_vdpa_set_vq_cb,
.set_vq_ready = octep_vdpa_set_vq_ready,
.get_vq_ready = octep_vdpa_get_vq_ready,
.set_vq_num = octep_vdpa_set_vq_num,
.set_vq_address = octep_vdpa_set_vq_address,
.get_vq_irq = NULL,
.kick_vq = octep_vdpa_kick_vq,
.kick_vq_with_data = octep_vdpa_kick_vq_with_data,
.get_generation = octep_vdpa_get_generation,
.get_device_id = octep_vdpa_get_device_id,
.get_vendor_id = octep_vdpa_get_vendor_id,
.get_vq_align = octep_vdpa_get_vq_align,
.get_config_size = octep_vdpa_get_config_size,
.get_config = octep_vdpa_get_config,
.set_config = octep_vdpa_set_config,
.set_config_cb = octep_vdpa_set_config_cb,
.get_vq_notification = octep_get_vq_notification,
};
static int octep_iomap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar)
{
int ret;
ret = pci_request_region(pdev, bar, OCTEP_VDPA_DRIVER_NAME);
if (ret) {
dev_err(&pdev->dev, "Failed to request BAR:%u region\n", bar);
return ret;
}
tbl[bar] = pci_iomap(pdev, bar, pci_resource_len(pdev, bar));
if (!tbl[bar]) {
dev_err(&pdev->dev, "Failed to iomap BAR:%u\n", bar);
pci_release_region(pdev, bar);
ret = -ENOMEM;
}
return ret;
}
static void octep_iounmap_region(struct pci_dev *pdev, u8 __iomem **tbl, u8 bar)
{
pci_iounmap(pdev, tbl[bar]);
pci_release_region(pdev, bar);
}
static void octep_vdpa_pf_bar_shrink(struct octep_pf *octpf)
{
struct pci_dev *pf_dev = octpf->pdev;
struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;
struct pci_bus_region bus_region;
octpf->res.start = res->start;
octpf->res.end = res->end;
octpf->vf_base = res->start;
bus_region.start = res->start;
bus_region.end = res->start - 1;
pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);
}
static void octep_vdpa_pf_bar_expand(struct octep_pf *octpf)
{
struct pci_dev *pf_dev = octpf->pdev;
struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;
struct pci_bus_region bus_region;
bus_region.start = octpf->res.start;
bus_region.end = octpf->res.end;
pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);
}
static void octep_vdpa_remove_pf(struct pci_dev *pdev)
{
struct octep_pf *octpf = pci_get_drvdata(pdev);
pci_disable_sriov(pdev);
if (octpf->base[OCTEP_HW_CAPS_BAR])
octep_iounmap_region(pdev, octpf->base, OCTEP_HW_CAPS_BAR);
if (octpf->base[OCTEP_HW_MBOX_BAR])
octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
octep_vdpa_pf_bar_expand(octpf);
}
static void octep_vdpa_vf_bar_shrink(struct pci_dev *pdev)
{
struct resource *vf_res = pdev->resource + PCI_STD_RESOURCES + 4;
memset(vf_res, 0, sizeof(*vf_res));
}
static void octep_vdpa_remove_vf(struct pci_dev *pdev)
{
struct octep_vdpa_mgmt_dev *mgmt_dev = pci_get_drvdata(pdev);
struct octep_hw *oct_hw;
int status;
oct_hw = &mgmt_dev->oct_hw;
status = atomic_read(&mgmt_dev->status);
atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT);
cancel_work_sync(&mgmt_dev->setup_task);
if (status == OCTEP_VDPA_DEV_STATUS_READY)
vdpa_mgmtdev_unregister(&mgmt_dev->mdev);
if (oct_hw->base[OCTEP_HW_CAPS_BAR])
octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
if (oct_hw->base[OCTEP_HW_MBOX_BAR])
octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR);
octep_vdpa_vf_bar_shrink(pdev);
}
static void octep_vdpa_remove(struct pci_dev *pdev)
{
if (pdev->is_virtfn)
octep_vdpa_remove_vf(pdev);
else
octep_vdpa_remove_pf(pdev);
}
static int octep_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
const struct vdpa_dev_set_config *config)
{
struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(mdev, struct octep_vdpa_mgmt_dev, mdev);
struct octep_hw *oct_hw = &mgmt_dev->oct_hw;
struct pci_dev *pdev = oct_hw->pdev;
struct vdpa_device *vdpa_dev;
struct octep_vdpa *oct_vdpa;
u64 device_features;
int ret;
oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 1, 1,
NULL, false);
if (IS_ERR(oct_vdpa)) {
dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device");
return PTR_ERR(oct_vdpa);
}
oct_vdpa->pdev = pdev;
oct_vdpa->vdpa.dma_dev = &pdev->dev;
oct_vdpa->vdpa.mdev = mdev;
oct_vdpa->oct_hw = oct_hw;
vdpa_dev = &oct_vdpa->vdpa;
device_features = oct_hw->features;
if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
if (config->device_features & ~device_features) {
dev_err(&pdev->dev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n",
config->device_features, device_features);
ret = -EINVAL;
goto vdpa_dev_put;
}
device_features &= config->device_features;
}
oct_hw->features = device_features;
dev_info(&pdev->dev, "Vdpa management device features : %llx\n", device_features);
ret = octep_verify_features(device_features);
if (ret) {
dev_warn(mdev->device,
"Must provision minimum features 0x%llx for this device",
BIT_ULL(VIRTIO_F_VERSION_1) | BIT_ULL(VIRTIO_F_ACCESS_PLATFORM) |
BIT_ULL(VIRTIO_F_NOTIFICATION_DATA) | BIT_ULL(VIRTIO_F_RING_PACKED));
goto vdpa_dev_put;
}
if (name)
ret = dev_set_name(&vdpa_dev->dev, "%s", name);
else
ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);
ret = _vdpa_register_device(&oct_vdpa->vdpa, oct_hw->nr_vring);
if (ret) {
dev_err(&pdev->dev, "Failed to register to vDPA bus");
goto vdpa_dev_put;
}
return 0;
vdpa_dev_put:
put_device(&oct_vdpa->vdpa.dev);
return ret;
}
static void octep_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *vdpa_dev)
{
_vdpa_unregister_device(vdpa_dev);
}
static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = {
.dev_add = octep_vdpa_dev_add,
.dev_del = octep_vdpa_dev_del
};
static bool get_device_ready_status(u8 __iomem *addr)
{
u64 signature = readq(addr + OCTEP_VF_MBOX_DATA(0));
if (signature == OCTEP_DEV_READY_SIGNATURE) {
writeq(0, addr + OCTEP_VF_MBOX_DATA(0));
return true;
}
return false;
}
static struct virtio_device_id id_table[] = {
{ VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
{ 0 },
};
static void octep_vdpa_setup_task(struct work_struct *work)
{
struct octep_vdpa_mgmt_dev *mgmt_dev = container_of(work, struct octep_vdpa_mgmt_dev,
setup_task);
struct pci_dev *pdev = mgmt_dev->pdev;
struct device *dev = &pdev->dev;
struct octep_hw *oct_hw;
unsigned long timeout;
int ret;
oct_hw = &mgmt_dev->oct_hw;
atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT);
/* Wait for a maximum of 5 sec */
timeout = jiffies + msecs_to_jiffies(5000);
while (!time_after(jiffies, timeout)) {
if (get_device_ready_status(oct_hw->base[OCTEP_HW_MBOX_BAR])) {
atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_INIT);
break;
}
if (atomic_read(&mgmt_dev->status) >= OCTEP_VDPA_DEV_STATUS_READY) {
dev_info(dev, "Stopping vDPA setup task.\n");
return;
}
usleep_range(1000, 1500);
}
if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_INIT) {
dev_err(dev, "BAR initialization is timed out\n");
return;
}
ret = octep_iomap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
if (ret)
return;
ret = octep_hw_caps_read(oct_hw, pdev);
if (ret < 0)
goto unmap_region;
mgmt_dev->mdev.ops = &octep_vdpa_mgmt_dev_ops;
mgmt_dev->mdev.id_table = id_table;
mgmt_dev->mdev.max_supported_vqs = oct_hw->nr_vring;
mgmt_dev->mdev.supported_features = oct_hw->features;
mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);
mgmt_dev->mdev.device = dev;
ret = vdpa_mgmtdev_register(&mgmt_dev->mdev);
if (ret) {
dev_err(dev, "Failed to register vdpa management interface\n");
goto unmap_region;
}
atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY);
return;
unmap_region:
octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);
oct_hw->base[OCTEP_HW_CAPS_BAR] = NULL;
}
static int octep_vdpa_probe_vf(struct pci_dev *pdev)
{
struct octep_vdpa_mgmt_dev *mgmt_dev;
struct device *dev = &pdev->dev;
int ret;
ret = pcim_enable_device(pdev);
if (ret) {
dev_err(dev, "Failed to enable device\n");
return ret;
}
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
if (ret) {
dev_err(dev, "No usable DMA configuration\n");
return ret;
}
pci_set_master(pdev);
mgmt_dev = devm_kzalloc(dev, sizeof(struct octep_vdpa_mgmt_dev), GFP_KERNEL);
if (!mgmt_dev)
return -ENOMEM;
ret = octep_iomap_region(pdev, mgmt_dev->oct_hw.base, OCTEP_HW_MBOX_BAR);
if (ret)
return ret;
mgmt_dev->pdev = pdev;
pci_set_drvdata(pdev, mgmt_dev);
atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ALLOC);
INIT_WORK(&mgmt_dev->setup_task, octep_vdpa_setup_task);
schedule_work(&mgmt_dev->setup_task);
dev_info(&pdev->dev, "octep vdpa mgmt device setup task is queued\n");
return 0;
}
static void octep_vdpa_assign_barspace(struct pci_dev *vf_dev, struct pci_dev *pf_dev, u8 idx)
{
struct resource *vf_res = vf_dev->resource + PCI_STD_RESOURCES + 4;
struct resource *pf_res = pf_dev->resource + PCI_STD_RESOURCES + 4;
struct octep_pf *pf = pci_get_drvdata(pf_dev);
struct pci_bus_region bus_region;
vf_res->name = pci_name(vf_dev);
vf_res->flags = pf_res->flags;
vf_res->parent = (pf_dev->resource + PCI_STD_RESOURCES)->parent;
bus_region.start = pf->vf_base + idx * pf->vf_stride;
bus_region.end = bus_region.start + pf->vf_stride - 1;
pcibios_bus_to_resource(vf_dev->bus, vf_res, &bus_region);
}
static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs)
{
struct octep_pf *pf = pci_get_drvdata(pdev);
u8 __iomem *addr = pf->base[OCTEP_HW_MBOX_BAR];
struct pci_dev *vf_pdev = NULL;
bool done = false;
int index = 0;
int ret, i;
ret = pci_enable_sriov(pdev, num_vfs);
if (ret)
return ret;
pf->enabled_vfs = num_vfs;
while ((vf_pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_ANY_ID, vf_pdev))) {
if (vf_pdev->device != pf->vf_devid)
continue;
octep_vdpa_assign_barspace(vf_pdev, pdev, index);
if (++index == num_vfs) {
done = true;
break;
}
}
if (done) {
for (i = 0; i < pf->enabled_vfs; i++)
writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i));
}
return num_vfs;
}
static int octep_sriov_disable(struct pci_dev *pdev)
{
struct octep_pf *pf = pci_get_drvdata(pdev);
if (!pci_num_vf(pdev))
return 0;
pci_disable_sriov(pdev);
pf->enabled_vfs = 0;
return 0;
}
static int octep_vdpa_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
if (num_vfs > 0)
return octep_sriov_enable(pdev, num_vfs);
else
return octep_sriov_disable(pdev);
}
static u16 octep_get_vf_devid(struct pci_dev *pdev)
{
u16 did;
switch (pdev->device) {
case OCTEP_VDPA_DEVID_CN106K_PF:
did = OCTEP_VDPA_DEVID_CN106K_VF;
break;
case OCTEP_VDPA_DEVID_CN105K_PF:
did = OCTEP_VDPA_DEVID_CN105K_VF;
break;
case OCTEP_VDPA_DEVID_CN103K_PF:
did = OCTEP_VDPA_DEVID_CN103K_VF;
break;
default:
did = 0xFFFF;
break;
}
return did;
}
static int octep_vdpa_pf_setup(struct octep_pf *octpf)
{
u8 __iomem *addr = octpf->base[OCTEP_HW_MBOX_BAR];
struct pci_dev *pdev = octpf->pdev;
int totalvfs;
size_t len;
u64 val;
totalvfs = pci_sriov_get_totalvfs(pdev);
if (unlikely(!totalvfs)) {
dev_info(&pdev->dev, "Total VFs are %d in PF sriov configuration\n", totalvfs);
return 0;
}
addr = octpf->base[OCTEP_HW_MBOX_BAR];
val = readq(addr + OCTEP_EPF_RINFO(0));
if (val == 0) {
dev_err(&pdev->dev, "Invalid device configuration\n");
return -EINVAL;
}
if (OCTEP_EPF_RINFO_RPVF(val) != BIT_ULL(0)) {
val &= ~GENMASK_ULL(35, 32);
val |= BIT_ULL(32);
writeq(val, addr + OCTEP_EPF_RINFO(0));
}
len = pci_resource_len(pdev, OCTEP_HW_CAPS_BAR);
octpf->vf_stride = len / totalvfs;
octpf->vf_devid = octep_get_vf_devid(pdev);
octep_vdpa_pf_bar_shrink(octpf);
return 0;
}
static int octep_vdpa_probe_pf(struct pci_dev *pdev)
{
struct device *dev = &pdev->dev;
struct octep_pf *octpf;
int ret;
ret = pcim_enable_device(pdev);
if (ret) {
dev_err(dev, "Failed to enable device\n");
return ret;
}
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
if (ret) {
dev_err(dev, "No usable DMA configuration\n");
return ret;
}
octpf = devm_kzalloc(dev, sizeof(*octpf), GFP_KERNEL);
if (!octpf)
return -ENOMEM;
ret = octep_iomap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
if (ret)
return ret;
pci_set_master(pdev);
pci_set_drvdata(pdev, octpf);
octpf->pdev = pdev;
ret = octep_vdpa_pf_setup(octpf);
if (ret)
goto unmap_region;
return 0;
unmap_region:
octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);
return ret;
}
static int octep_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
if (pdev->is_virtfn)
return octep_vdpa_probe_vf(pdev);
else
return octep_vdpa_probe_pf(pdev);
}
static struct pci_device_id octep_pci_vdpa_map[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_PF) },
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN106K_VF) },
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_PF) },
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN105K_VF) },
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_PF) },
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_VDPA_DEVID_CN103K_VF) },
{ 0 },
};
static struct pci_driver octep_pci_vdpa = {
.name = OCTEP_VDPA_DRIVER_NAME,
.id_table = octep_pci_vdpa_map,
.probe = octep_vdpa_probe,
.remove = octep_vdpa_remove,
.sriov_configure = octep_vdpa_sriov_configure
};
module_pci_driver(octep_pci_vdpa);
MODULE_AUTHOR("Marvell");
MODULE_DESCRIPTION("Marvell Octeon PCIe endpoint vDPA driver");
MODULE_LICENSE("GPL");

1
drivers/vdpa/vdpa.c
View File

@ -1538,4 +1538,5 @@ core_initcall(vdpa_init);
module_exit(vdpa_exit);
MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>");
MODULE_DESCRIPTION("vDPA bus");
MODULE_LICENSE("GPL v2");

105
drivers/vhost/vhost.c
View File

@ -1346,10 +1346,36 @@ static void vhost_dev_unlock_vqs(struct vhost_dev *d)
mutex_unlock(&d->vqs[i]->mutex);
}
static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq,
__virtio16 *idx)
static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq)
{
return vhost_get_avail(vq, *idx, &vq->avail->idx);
__virtio16 idx;
int r;
r = vhost_get_avail(vq, idx, &vq->avail->idx);
if (unlikely(r < 0)) {
vq_err(vq, "Failed to access available index at %p (%d)\n",
&vq->avail->idx, r);
return r;
}
/* Check it isn't doing very strange thing with available indexes */
vq->avail_idx = vhost16_to_cpu(vq, idx);
if (unlikely((u16)(vq->avail_idx - vq->last_avail_idx) > vq->num)) {
vq_err(vq, "Invalid available index change from %u to %u",
vq->last_avail_idx, vq->avail_idx);
return -EINVAL;
}
/* We're done if there is nothing new */
if (vq->avail_idx == vq->last_avail_idx)
return 0;
/*
* We updated vq->avail_idx so we need a memory barrier between
* the index read above and the caller reading avail ring entries.
*/
smp_rmb();
return 1;
}
static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,
@ -2554,38 +2580,17 @@ int vhost_get_vq_desc(struct vhost_virtqueue *vq,
{
struct vring_desc desc;
unsigned int i, head, found = 0;
u16 last_avail_idx;
__virtio16 avail_idx;
u16 last_avail_idx = vq->last_avail_idx;
__virtio16 ring_head;
int ret, access;
/* Check it isn't doing very strange things with descriptor numbers. */
last_avail_idx = vq->last_avail_idx;
if (vq->avail_idx == vq->last_avail_idx) {
if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) {
vq_err(vq, "Failed to access avail idx at %p\n",
&vq->avail->idx);
return -EFAULT;
}
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
ret = vhost_get_avail_idx(vq);
if (unlikely(ret < 0))
return ret;
if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
vq_err(vq, "Guest moved avail index from %u to %u",
last_avail_idx, vq->avail_idx);
return -EFAULT;
}
/* If there's nothing new since last we looked, return
* invalid.
*/
if (vq->avail_idx == last_avail_idx)
if (!ret)
return vq->num;
/* Only get avail ring entries after they have been
* exposed by guest.
*/
smp_rmb();
}
/* Grab the next descriptor number they're advertising, and increment
@ -2846,35 +2851,21 @@ EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
/* return true if we're sure that avaiable ring is empty */
bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
__virtio16 avail_idx;
int r;
if (vq->avail_idx != vq->last_avail_idx)
return false;
r = vhost_get_avail_idx(vq, &avail_idx);
if (unlikely(r))
return false;
r = vhost_get_avail_idx(vq);
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
if (vq->avail_idx != vq->last_avail_idx) {
/* Since we have updated avail_idx, the following
* call to vhost_get_vq_desc() will read available
* ring entries. Make sure that read happens after
* the avail_idx read.
*/
smp_rmb();
return false;
}
return true;
/* Note: we treat error as non-empty here */
return r == 0;
}
EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
/* OK, now we need to know about added descriptors. */
bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
__virtio16 avail_idx;
int r;
if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
@ -2898,25 +2889,13 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
/* They could have slipped one in as we were doing that: make
* sure it's written, then check again. */
smp_mb();
r = vhost_get_avail_idx(vq, &avail_idx);
if (r) {
vq_err(vq, "Failed to check avail idx at %p: %d\n",
&vq->avail->idx, r);
r = vhost_get_avail_idx(vq);
/* Note: we treat error as empty here */
if (unlikely(r < 0))
return false;
}
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
if (vq->avail_idx != vq->last_avail_idx) {
/* Since we have updated avail_idx, the following
* call to vhost_get_vq_desc() will read available
* ring entries. Make sure that read happens after
* the avail_idx read.
*/
smp_rmb();
return true;
}
return false;
return r;
}
EXPORT_SYMBOL_GPL(vhost_enable_notify);

1
drivers/vhost/vringh.c
View File

@ -1614,4 +1614,5 @@ EXPORT_SYMBOL(vringh_need_notify_iotlb);
#endif
MODULE_DESCRIPTION("host side of a virtio ring");
MODULE_LICENSE("GPL");

4
drivers/vhost/vsock.c
View File

@ -667,6 +667,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file)
}
vsock->guest_cid = 0; /* no CID assigned yet */
vsock->seqpacket_allow = false;
atomic_set(&vsock->queued_replies, 0);
@ -810,8 +811,7 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
goto err;
}
if (features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET))
vsock->seqpacket_allow = true;
vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {
vq = &vsock->vqs[i];

1
drivers/virtio/virtio.c
View File

@ -609,4 +609,5 @@ static void __exit virtio_exit(void)
core_initcall(virtio_init);
module_exit(virtio_exit);
MODULE_DESCRIPTION("Virtio core interface");
MODULE_LICENSE("GPL");

75
drivers/virtio/virtio_balloon.c
View File

@ -349,34 +349,49 @@ static inline void update_stat(struct virtio_balloon *vb, int idx,
#define pages_to_bytes(x) ((u64)(x) << PAGE_SHIFT)
static unsigned int update_balloon_stats(struct virtio_balloon *vb)
#ifdef CONFIG_VM_EVENT_COUNTERS
/* Return the number of entries filled by vm events */
static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb)
{
unsigned long events[NR_VM_EVENT_ITEMS];
struct sysinfo i;
unsigned int idx = 0;
long available;
unsigned long caches;
all_vm_events(events);
si_meminfo(&i);
available = si_mem_available();
caches = global_node_page_state(NR_FILE_PAGES);
#ifdef CONFIG_VM_EVENT_COUNTERS
update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
pages_to_bytes(events[PSWPIN]));
pages_to_bytes(events[PSWPIN]));
update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_OUT,
pages_to_bytes(events[PSWPOUT]));
pages_to_bytes(events[PSWPOUT]));
update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
#ifdef CONFIG_HUGETLB_PAGE
update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,
events[HTLB_BUDDY_PGALLOC]);
update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGFAIL,
events[HTLB_BUDDY_PGALLOC_FAIL]);
#endif
#endif
#endif /* CONFIG_HUGETLB_PAGE */
return idx;
}
#else /* CONFIG_VM_EVENT_COUNTERS */
static inline unsigned int update_balloon_vm_stats(struct virtio_balloon *vb)
{
return 0;
}
#endif /* CONFIG_VM_EVENT_COUNTERS */
static unsigned int update_balloon_stats(struct virtio_balloon *vb)
{
struct sysinfo i;
unsigned int idx;
long available;
unsigned long caches;
idx = update_balloon_vm_stats(vb);
si_meminfo(&i);
available = si_mem_available();
caches = global_node_page_state(NR_FILE_PAGES);
update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMFREE,
pages_to_bytes(i.freeram));
update_stat(vb, idx++, VIRTIO_BALLOON_S_MEMTOT,
@ -545,9 +560,8 @@ static void update_balloon_size_func(struct work_struct *work)
static int init_vqs(struct virtio_balloon *vb)
{
struct virtqueue_info vqs_info[VIRTIO_BALLOON_VQ_MAX] = {};
struct virtqueue *vqs[VIRTIO_BALLOON_VQ_MAX];
vq_callback_t *callbacks[VIRTIO_BALLOON_VQ_MAX];
const char *names[VIRTIO_BALLOON_VQ_MAX];
int err;
/*
@ -555,33 +569,26 @@ static int init_vqs(struct virtio_balloon *vb)
* will be NULL if the related feature is not enabled, which will
* cause no allocation for the corresponding virtqueue in find_vqs.
*/
callbacks[VIRTIO_BALLOON_VQ_INFLATE] = balloon_ack;
names[VIRTIO_BALLOON_VQ_INFLATE] = "inflate";
callbacks[VIRTIO_BALLOON_VQ_DEFLATE] = balloon_ack;
names[VIRTIO_BALLOON_VQ_DEFLATE] = "deflate";
callbacks[VIRTIO_BALLOON_VQ_STATS] = NULL;
names[VIRTIO_BALLOON_VQ_STATS] = NULL;
callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
names[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
names[VIRTIO_BALLOON_VQ_REPORTING] = NULL;
vqs_info[VIRTIO_BALLOON_VQ_INFLATE].callback = balloon_ack;
vqs_info[VIRTIO_BALLOON_VQ_INFLATE].name = "inflate";
vqs_info[VIRTIO_BALLOON_VQ_DEFLATE].callback = balloon_ack;
vqs_info[VIRTIO_BALLOON_VQ_DEFLATE].name = "deflate";
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_STATS_VQ)) {
names[VIRTIO_BALLOON_VQ_STATS] = "stats";
callbacks[VIRTIO_BALLOON_VQ_STATS] = stats_request;
vqs_info[VIRTIO_BALLOON_VQ_STATS].name = "stats";
vqs_info[VIRTIO_BALLOON_VQ_STATS].callback = stats_request;
}
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) {
names[VIRTIO_BALLOON_VQ_FREE_PAGE] = "free_page_vq";
callbacks[VIRTIO_BALLOON_VQ_FREE_PAGE] = NULL;
}
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT))
vqs_info[VIRTIO_BALLOON_VQ_FREE_PAGE].name = "free_page_vq";
if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) {
names[VIRTIO_BALLOON_VQ_REPORTING] = "reporting_vq";
callbacks[VIRTIO_BALLOON_VQ_REPORTING] = balloon_ack;
vqs_info[VIRTIO_BALLOON_VQ_REPORTING].name = "reporting_vq";
vqs_info[VIRTIO_BALLOON_VQ_REPORTING].callback = balloon_ack;
}
err = virtio_find_vqs(vb->vdev, VIRTIO_BALLOON_VQ_MAX, vqs,
callbacks, names, NULL);
vqs_info, NULL);
if (err)
return err;

9
drivers/virtio/virtio_input.c
View File

@ -185,13 +185,14 @@ static void virtinput_cfg_abs(struct virtio_input *vi, int abs)
static int virtinput_init_vqs(struct virtio_input *vi)
{
struct virtqueue_info vqs_info[] = {
{ "events", virtinput_recv_events },
{ "status", virtinput_recv_status },
};
struct virtqueue *vqs[2];
vq_callback_t *cbs[] = { virtinput_recv_events,
virtinput_recv_status };
static const char * const names[] = { "events", "status" };
int err;
err = virtio_find_vqs(vi->vdev, 2, vqs, cbs, names, NULL);
err = virtio_find_vqs(vi->vdev, 2, vqs, vqs_info, NULL);
if (err)
return err;
vi->evt = vqs[0];

12
drivers/virtio/virtio_mmio.c
View File

@ -489,9 +489,7 @@ error_available:
static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[],
const bool *ctx,
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
@ -510,13 +508,15 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
enable_irq_wake(irq);
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
ctx ? ctx[i] : false);
vqs[i] = vm_setup_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx);
if (IS_ERR(vqs[i])) {
vm_del_vqs(vdev);
return PTR_ERR(vqs[i]);

48
drivers/virtio/virtio_pci_common.c
View File

@ -285,12 +285,13 @@ void vp_del_vqs(struct virtio_device *vdev)
}
static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], bool per_vq_vectors,
const bool *ctx,
struct irq_affinity *desc)
struct virtqueue *vqs[],
struct virtqueue_info vqs_info[],
bool per_vq_vectors,
struct irq_affinity *desc)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
struct virtqueue_info *vqi;
u16 msix_vec;
int i, err, nvectors, allocated_vectors, queue_idx = 0;
@ -301,9 +302,11 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
if (per_vq_vectors) {
/* Best option: one for change interrupt, one per vq. */
nvectors = 1;
for (i = 0; i < nvqs; ++i)
if (names[i] && callbacks[i])
for (i = 0; i < nvqs; ++i) {
vqi = &vqs_info[i];
if (vqi->name && vqi->callback)
++nvectors;
}
} else {
/* Second best: one for change, shared for all vqs. */
nvectors = 2;
@ -317,20 +320,20 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
vp_dev->per_vq_vectors = per_vq_vectors;
allocated_vectors = vp_dev->msix_used_vectors;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
if (!callbacks[i])
if (!vqi->callback)
msix_vec = VIRTIO_MSI_NO_VECTOR;
else if (vp_dev->per_vq_vectors)
msix_vec = allocated_vectors++;
else
msix_vec = VP_MSIX_VQ_VECTOR;
vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
ctx ? ctx[i] : false,
msix_vec);
vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx, msix_vec);
if (IS_ERR(vqs[i])) {
err = PTR_ERR(vqs[i]);
goto error_find;
@ -343,7 +346,7 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs,
snprintf(vp_dev->msix_names[msix_vec],
sizeof *vp_dev->msix_names,
"%s-%s",
dev_name(&vp_dev->vdev.dev), names[i]);
dev_name(&vp_dev->vdev.dev), vqi->name);
err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
vring_interrupt, 0,
vp_dev->msix_names[msix_vec],
@ -361,8 +364,8 @@ error_find:
}
static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], const bool *ctx)
struct virtqueue *vqs[],
struct virtqueue_info vqs_info[])
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
int i, err, queue_idx = 0;
@ -379,12 +382,14 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs,
vp_dev->intx_enabled = 1;
vp_dev->per_vq_vectors = false;
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
ctx ? ctx[i] : false,
vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx,
VIRTIO_MSI_NO_VECTOR);
if (IS_ERR(vqs[i])) {
err = PTR_ERR(vqs[i]);
@ -400,25 +405,24 @@ out_del_vqs:
/* the config->find_vqs() implementation */
int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
struct virtqueue *vqs[], struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
int err;
/* Try MSI-X with one vector per queue. */
err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc);
err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, true, desc);
if (!err)
return 0;
/* Fallback: MSI-X with one vector for config, one shared for queues. */
err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc);
err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, false, desc);
if (!err)
return 0;
/* Is there an interrupt? If not give up. */
if (!(to_vp_device(vdev)->pci_dev->irq))
return err;
/* Finally fall back to regular interrupts. */
return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx);
return vp_find_vqs_intx(vdev, nvqs, vqs, vqs_info);
}
const char *vp_bus_name(struct virtio_device *vdev)

3
drivers/virtio/virtio_pci_common.h
View File

@ -127,8 +127,7 @@ bool vp_notify(struct virtqueue *vq);
void vp_del_vqs(struct virtio_device *vdev);
/* the config->find_vqs() implementation */
int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
struct virtqueue *vqs[], struct virtqueue_info vqs_info[],
struct irq_affinity *desc);
const char *vp_bus_name(struct virtio_device *vdev);

5
drivers/virtio/virtio_pci_modern.c
View File

@ -595,13 +595,12 @@ err:
static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
struct virtqueue *vq;
int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc);
int rc = vp_find_vqs(vdev, nvqs, vqs, vqs_info, desc);
if (rc)
return rc;

5
drivers/virtio/virtio_ring.c
View File

@ -3121,8 +3121,10 @@ dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr,
{
struct vring_virtqueue *vq = to_vvq(_vq);
if (!vq->use_dma_api)
if (!vq->use_dma_api) {
kmsan_handle_dma(virt_to_page(ptr), offset_in_page(ptr), size, dir);
return (dma_addr_t)virt_to_phys(ptr);
}
return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs);
}
@ -3244,4 +3246,5 @@ void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq,
}
EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device);
MODULE_DESCRIPTION("Virtio ring implementation");
MODULE_LICENSE("GPL");

13
drivers/virtio/virtio_vdpa.c
View File

@ -358,9 +358,7 @@ create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
vq_callback_t *callbacks[],
const char * const names[],
const bool *ctx,
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev);
@ -379,14 +377,15 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
}
for (i = 0; i < nvqs; ++i) {
if (!names[i]) {
struct virtqueue_info *vqi = &vqs_info[i];
if (!vqi->name) {
vqs[i] = NULL;
continue;
}
vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++,
callbacks[i], names[i], ctx ?
ctx[i] : false);
vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, vqi->callback,
vqi->name, vqi->ctx);
if (IS_ERR(vqs[i])) {
err = PTR_ERR(vqs[i]);
goto err_setup_vq;

62
fs/fuse/virtio_fs.c
View File

@ -51,7 +51,7 @@ struct virtio_fs_vq {
struct work_struct done_work;
struct list_head queued_reqs;
struct list_head end_reqs; /* End these requests */
struct delayed_work dispatch_work;
struct work_struct dispatch_work;
struct fuse_dev *fud;
bool connected;
long in_flight;
@ -233,7 +233,7 @@ static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
}
flush_work(&fsvq->done_work);
flush_delayed_work(&fsvq->dispatch_work);
flush_work(&fsvq->dispatch_work);
}
static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs)
@ -408,6 +408,10 @@ static void virtio_fs_hiprio_done_work(struct work_struct *work)
dec_in_flight_req(fsvq);
}
} while (!virtqueue_enable_cb(vq));
if (!list_empty(&fsvq->queued_reqs))
schedule_work(&fsvq->dispatch_work);
spin_unlock(&fsvq->lock);
}
@ -415,7 +419,7 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work)
{
struct fuse_req *req;
struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
dispatch_work.work);
dispatch_work);
int ret;
pr_debug("virtio-fs: worker %s called.\n", __func__);
@ -447,11 +451,9 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work)
ret = virtio_fs_enqueue_req(fsvq, req, true);
if (ret < 0) {
if (ret == -ENOMEM || ret == -ENOSPC) {
if (ret == -ENOSPC) {
spin_lock(&fsvq->lock);
list_add_tail(&req->list, &fsvq->queued_reqs);
schedule_delayed_work(&fsvq->dispatch_work,
msecs_to_jiffies(1));
spin_unlock(&fsvq->lock);
return;
}
@ -494,12 +496,10 @@ static int send_forget_request(struct virtio_fs_vq *fsvq,
ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC);
if (ret < 0) {
if (ret == -ENOMEM || ret == -ENOSPC) {
if (ret == -ENOSPC) {
pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
ret);
list_add_tail(&forget->list, &fsvq->queued_reqs);
schedule_delayed_work(&fsvq->dispatch_work,
msecs_to_jiffies(1));
if (!in_flight)
inc_in_flight_req(fsvq);
/* Queue is full */
@ -531,7 +531,7 @@ static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
{
struct virtio_fs_forget *forget;
struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
dispatch_work.work);
dispatch_work);
pr_debug("virtio-fs: worker %s called.\n", __func__);
while (1) {
spin_lock(&fsvq->lock);
@ -709,6 +709,12 @@ static void virtio_fs_requests_done_work(struct work_struct *work)
virtio_fs_request_complete(req, fsvq);
}
}
/* Try to push previously queued requests, as the queue might no longer be full */
spin_lock(&fsvq->lock);
if (!list_empty(&fsvq->queued_reqs))
schedule_work(&fsvq->dispatch_work);
spin_unlock(&fsvq->lock);
}
static void virtio_fs_map_queues(struct virtio_device *vdev, struct virtio_fs *fs)
@ -770,12 +776,12 @@ static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name,
if (vq_type == VQ_REQUEST) {
INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work);
INIT_DELAYED_WORK(&fsvq->dispatch_work,
virtio_fs_request_dispatch_work);
INIT_WORK(&fsvq->dispatch_work,
virtio_fs_request_dispatch_work);
} else {
INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work);
INIT_DELAYED_WORK(&fsvq->dispatch_work,
virtio_fs_hiprio_dispatch_work);
INIT_WORK(&fsvq->dispatch_work,
virtio_fs_hiprio_dispatch_work);
}
}
@ -783,14 +789,13 @@ static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name,
static int virtio_fs_setup_vqs(struct virtio_device *vdev,
struct virtio_fs *fs)
{
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
vq_callback_t **callbacks;
/* Specify pre_vectors to ensure that the queues before the
* request queues (e.g. hiprio) don't claim any of the CPUs in
* the multi-queue mapping and interrupt affinities
*/
struct irq_affinity desc = { .pre_vectors = VQ_REQUEST };
const char **names;
unsigned int i;
int ret = 0;
@ -808,20 +813,18 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
return -ENOMEM;
vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
GFP_KERNEL);
names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
fs->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*fs->mq_map), GFP_KERNEL,
dev_to_node(&vdev->dev));
if (!vqs || !callbacks || !names || !fs->mq_map) {
vqs_info = kcalloc(fs->nvqs, sizeof(*vqs_info), GFP_KERNEL);
if (!vqs || !vqs_info || !fs->mq_map) {
ret = -ENOMEM;
goto out;
}
/* Initialize the hiprio/forget request virtqueue */
callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
vqs_info[VQ_HIPRIO].callback = virtio_fs_vq_done;
virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO);
names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
vqs_info[VQ_HIPRIO].name = fs->vqs[VQ_HIPRIO].name;
/* Initialize the requests virtqueues */
for (i = VQ_REQUEST; i < fs->nvqs; i++) {
@ -829,11 +832,11 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST);
virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST);
callbacks[i] = virtio_fs_vq_done;
names[i] = fs->vqs[i].name;
vqs_info[i].callback = virtio_fs_vq_done;
vqs_info[i].name = fs->vqs[i].name;
}
ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, &desc);
ret = virtio_find_vqs(vdev, fs->nvqs, vqs, vqs_info, &desc);
if (ret < 0)
goto out;
@ -842,8 +845,7 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
virtio_fs_start_all_queues(fs);
out:
kfree(names);
kfree(callbacks);
kfree(vqs_info);
kfree(vqs);
if (ret) {
kfree(fs->vqs);
@ -1367,7 +1369,7 @@ __releases(fiq->lock)
fsvq = &fs->vqs[queue_id];
ret = virtio_fs_enqueue_req(fsvq, req, false);
if (ret < 0) {
if (ret == -ENOMEM || ret == -ENOSPC) {
if (ret == -ENOSPC) {
/*
* Virtqueue full. Retry submission from worker
* context as we might be holding fc->bg_lock.
@ -1375,8 +1377,6 @@ __releases(fiq->lock)
spin_lock(&fsvq->lock);
list_add_tail(&req->list, &fsvq->queued_reqs);
inc_in_flight_req(fsvq);
schedule_delayed_work(&fsvq->dispatch_work,
msecs_to_jiffies(1));
spin_unlock(&fsvq->lock);
return;
}
@ -1386,7 +1386,7 @@ __releases(fiq->lock)
/* Can't end request in submission context. Use a worker */
spin_lock(&fsvq->lock);
list_add_tail(&req->list, &fsvq->end_reqs);
schedule_delayed_work(&fsvq->dispatch_work, 0);
schedule_work(&fsvq->dispatch_work);
spin_unlock(&fsvq->lock);
return;
}

2
include/linux/mlx5/mlx5_ifc_vdpa.h
View File

@ -148,7 +148,9 @@ enum {
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS = (u64)1 << 6,
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_AVAIL_IDX = (u64)1 << 7,
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_USED_IDX = (u64)1 << 8,
MLX5_VIRTQ_MODIFY_MASK_QUEUE_VIRTIO_VERSION = (u64)1 << 10,
MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY = (u64)1 << 11,
MLX5_VIRTQ_MODIFY_MASK_QUEUE_FEATURES = (u64)1 << 12,
MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY = (u64)1 << 14,
};

64
include/linux/virtio_config.h
View File

@ -18,6 +18,20 @@ struct virtio_shm_region {
typedef void vq_callback_t(struct virtqueue *);
/**
* struct virtqueue_info - Info for a virtqueue passed to find_vqs().
* @name: virtqueue description. Used mainly for debugging, NULL for
* a virtqueue unused by the driver.
* @callback: A callback to invoke on a used buffer notification.
* NULL for a virtqueue that does not need a callback.
* @ctx: A flag to indicate to maintain an extra context per virtqueue.
*/
struct virtqueue_info {
const char *name;
vq_callback_t *callback;
bool ctx;
};
/**
* struct virtio_config_ops - operations for configuring a virtio device
* Note: Do not assume that a transport implements all of the operations
@ -53,10 +67,7 @@ typedef void vq_callback_t(struct virtqueue *);
* vdev: the virtio_device
* nvqs: the number of virtqueues to find
* vqs: on success, includes new virtqueues
* callbacks: array of callbacks, for each virtqueue
* include a NULL entry for vqs that do not need a callback
* names: array of virtqueue names (mainly for debugging)
* include a NULL entry for vqs unused by driver
* vqs_info: array of virtqueue info structures
* Returns 0 on success or error status
* @del_vqs: free virtqueues found by find_vqs().
* @synchronize_cbs: synchronize with the virtqueue callbacks (optional)
@ -105,9 +116,9 @@ struct virtio_config_ops {
u8 (*get_status)(struct virtio_device *vdev);
void (*set_status)(struct virtio_device *vdev, u8 status);
void (*reset)(struct virtio_device *vdev);
int (*find_vqs)(struct virtio_device *, unsigned nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
int (*find_vqs)(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
struct virtqueue_info vqs_info[],
struct irq_affinity *desc);
void (*del_vqs)(struct virtio_device *);
void (*synchronize_cbs)(struct virtio_device *);
@ -117,7 +128,7 @@ struct virtio_config_ops {
int (*set_vq_affinity)(struct virtqueue *vq,
const struct cpumask *cpu_mask);
const struct cpumask *(*get_vq_affinity)(struct virtio_device *vdev,
int index);
int index);
bool (*get_shm_region)(struct virtio_device *vdev,
struct virtio_shm_region *region, u8 id);
int (*disable_vq_and_reset)(struct virtqueue *vq);
@ -210,39 +221,30 @@ static inline bool virtio_has_dma_quirk(const struct virtio_device *vdev)
return !virtio_has_feature(vdev, VIRTIO_F_ACCESS_PLATFORM);
}
static inline
int virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs,
struct virtqueue *vqs[],
struct virtqueue_info vqs_info[],
struct irq_affinity *desc)
{
return vdev->config->find_vqs(vdev, nvqs, vqs, vqs_info, desc);
}
static inline
struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev,
vq_callback_t *c, const char *n)
{
vq_callback_t *callbacks[] = { c };
const char *names[] = { n };
struct virtqueue_info vqs_info[] = {
{ n, c },
};
struct virtqueue *vq;
int err = vdev->config->find_vqs(vdev, 1, &vq, callbacks, names, NULL,
NULL);
int err = virtio_find_vqs(vdev, 1, &vq, vqs_info, NULL);
if (err < 0)
return ERR_PTR(err);
return vq;
}
static inline
int virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[],
struct irq_affinity *desc)
{
return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, NULL, desc);
}
static inline
int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs,
struct virtqueue *vqs[], vq_callback_t *callbacks[],
const char * const names[], const bool *ctx,
struct irq_affinity *desc)
{
return vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names, ctx,
desc);
}
/**
* virtio_synchronize_cbs - synchronize with virtqueue callbacks
* @dev: the virtio device

11
include/linux/virtio_net.h
View File

@ -56,6 +56,7 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb,
unsigned int thlen = 0;
unsigned int p_off = 0;
unsigned int ip_proto;
u64 ret, remainder, gso_size;
if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
@ -98,6 +99,16 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb,
u32 off = __virtio16_to_cpu(little_endian, hdr->csum_offset);
u32 needed = start + max_t(u32, thlen, off + sizeof(__sum16));
if (hdr->gso_size) {
gso_size = __virtio16_to_cpu(little_endian, hdr->gso_size);
ret = div64_u64_rem(skb->len, gso_size, &remainder);
if (!(ret && (hdr->gso_size > needed) &&
((remainder > needed) || (remainder == 0)))) {
return -EINVAL;
}
skb_shinfo(skb)->tx_flags |= SKBFL_SHARED_FRAG;
}
if (!pskb_may_pull(skb, needed))
return -EINVAL;

16
net/vmw_vsock/virtio_transport.c
View File

@ -617,20 +617,14 @@ out:
static int virtio_vsock_vqs_init(struct virtio_vsock *vsock)
{
struct virtio_device *vdev = vsock->vdev;
static const char * const names[] = {
"rx",
"tx",
"event",
};
vq_callback_t *callbacks[] = {
virtio_vsock_rx_done,
virtio_vsock_tx_done,
virtio_vsock_event_done,
struct virtqueue_info vqs_info[] = {
{ "rx", virtio_vsock_rx_done },
{ "tx", virtio_vsock_tx_done },
{ "event", virtio_vsock_event_done },
};
int ret;
ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, callbacks, names,
NULL);
ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, vqs_info, NULL);
if (ret < 0)
return ret;

23
sound/virtio/virtio_card.c
View File

@ -110,25 +110,22 @@ static void virtsnd_event_notify_cb(struct virtqueue *vqueue)
static int virtsnd_find_vqs(struct virtio_snd *snd)
{
struct virtio_device *vdev = snd->vdev;
static vq_callback_t *callbacks[VIRTIO_SND_VQ_MAX] = {
[VIRTIO_SND_VQ_CONTROL] = virtsnd_ctl_notify_cb,
[VIRTIO_SND_VQ_EVENT] = virtsnd_event_notify_cb,
[VIRTIO_SND_VQ_TX] = virtsnd_pcm_tx_notify_cb,
[VIRTIO_SND_VQ_RX] = virtsnd_pcm_rx_notify_cb
};
static const char *names[VIRTIO_SND_VQ_MAX] = {
[VIRTIO_SND_VQ_CONTROL] = "virtsnd-ctl",
[VIRTIO_SND_VQ_EVENT] = "virtsnd-event",
[VIRTIO_SND_VQ_TX] = "virtsnd-tx",
[VIRTIO_SND_VQ_RX] = "virtsnd-rx"
struct virtqueue_info vqs_info[VIRTIO_SND_VQ_MAX] = {
[VIRTIO_SND_VQ_CONTROL] = { "virtsnd-ctl",
virtsnd_ctl_notify_cb },
[VIRTIO_SND_VQ_EVENT] = { "virtsnd-event",
virtsnd_event_notify_cb },
[VIRTIO_SND_VQ_TX] = { "virtsnd-tx",
virtsnd_pcm_tx_notify_cb },
[VIRTIO_SND_VQ_RX] = { "virtsnd-rx",
virtsnd_pcm_rx_notify_cb },
};
struct virtqueue *vqs[VIRTIO_SND_VQ_MAX] = { 0 };
unsigned int i;
unsigned int n;
int rc;
rc = virtio_find_vqs(vdev, VIRTIO_SND_VQ_MAX, vqs, callbacks, names,
NULL);
rc = virtio_find_vqs(vdev, VIRTIO_SND_VQ_MAX, vqs, vqs_info, NULL);
if (rc) {
dev_err(&vdev->dev, "failed to initialize virtqueues\n");
return rc;

9
tools/virtio/vringh_test.c
View File

@ -139,7 +139,7 @@ static int parallel_test(u64 features,
bool fast_vringh)
{
void *host_map, *guest_map;
int fd, mapsize, to_guest[2], to_host[2];
int pipe_ret, fd, mapsize, to_guest[2], to_host[2];
unsigned long xfers = 0, notifies = 0, receives = 0;
unsigned int first_cpu, last_cpu;
cpu_set_t cpu_set;
@ -161,8 +161,11 @@ static int parallel_test(u64 features,
host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
pipe(to_guest);
pipe(to_host);
pipe_ret = pipe(to_guest);
assert(!pipe_ret);
pipe_ret = pipe(to_host);
assert(!pipe_ret);
CPU_ZERO(&cpu_set);
find_cpus(&first_cpu, &last_cpu);