mirror of
https://github.com/torvalds/linux.git
synced 2024-12-05 18:41:23 +00:00
53989fad12
A test of the form:
while true; do modprobe -r cxl_pmem; modprobe cxl_pmem; done
May lead to a crash signature of the form:
BUG: unable to handle page fault for address: ffffffffc0660030
#PF: supervisor instruction fetch in kernel mode
#PF: error_code(0x0010) - not-present page
[..]
Workqueue: cxl_pmem 0xffffffffc0660030
RIP: 0010:0xffffffffc0660030
Code: Unable to access opcode bytes at RIP 0xffffffffc0660006.
[..]
Call Trace:
? process_one_work+0x4ec/0x9c0
? pwq_dec_nr_in_flight+0x100/0x100
? rwlock_bug.part.0+0x60/0x60
? worker_thread+0x2eb/0x700
In that report the 0xffffffffc0660030 address corresponds to the former
function address of cxl_nvb_update_state() from a previous load of the
module, not the current address. Fix that by arranging for ->state_work
in the 'struct cxl_nvdimm_bridge' object to be reinitialized on cxl_pmem
module reload.
Details:
Recall that CXL subsystem wants to link a CXL memory expander device to
an NVDIMM sub-hierarchy when both a persistent memory range has been
registered by the CXL platform driver (cxl_acpi) *and* when that CXL
memory expander has published persistent memory capacity (Get Partition
Info). To this end the cxl_nvdimm_bridge driver arranges to rescan the
CXL bus when either of those conditions change. The helper
bus_rescan_devices() can not be called underneath the device_lock() for
any device on that bus, so the cxl_nvdimm_bridge driver uses a workqueue
for the rescan.
Typically a driver allocates driver data to hold a 'struct work_struct'
for a driven device, but for a workqueue that may run after ->remove()
returns, driver data will have been freed. The 'struct
cxl_nvdimm_bridge' object holds the state and work_struct directly.
Unfortunately it was only arranging for that infrastructure to be
initialized once per device creation rather than the necessary once per
workqueue (cxl_pmem_wq) creation.
Introduce is_cxl_nvdimm_bridge() and cxl_nvdimm_bridge_reset() in
support of invalidating stale references to a recently destroyed
cxl_pmem_wq.
Cc: <stable@vger.kernel.org>
Fixes: 8fdcb1704f
("cxl/pmem: Add initial infrastructure for pmem support")
Reported-by: Vishal Verma <vishal.l.verma@intel.com>
Tested-by: Vishal Verma <vishal.l.verma@intel.com>
Link: https://lore.kernel.org/r/163665474585.3505991.8397182770066720755.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
385 lines
9.0 KiB
C
385 lines
9.0 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
|
|
#include <linux/libnvdimm.h>
|
|
#include <asm/unaligned.h>
|
|
#include <linux/device.h>
|
|
#include <linux/module.h>
|
|
#include <linux/ndctl.h>
|
|
#include <linux/async.h>
|
|
#include <linux/slab.h>
|
|
#include "cxlmem.h"
|
|
#include "cxl.h"
|
|
|
|
/*
|
|
* Ordered workqueue for cxl nvdimm device arrival and departure
|
|
* to coordinate bus rescans when a bridge arrives and trigger remove
|
|
* operations when the bridge is removed.
|
|
*/
|
|
static struct workqueue_struct *cxl_pmem_wq;
|
|
|
|
static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
|
|
|
|
static void clear_exclusive(void *cxlds)
|
|
{
|
|
clear_exclusive_cxl_commands(cxlds, exclusive_cmds);
|
|
}
|
|
|
|
static void unregister_nvdimm(void *nvdimm)
|
|
{
|
|
nvdimm_delete(nvdimm);
|
|
}
|
|
|
|
static int cxl_nvdimm_probe(struct device *dev)
|
|
{
|
|
struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
|
|
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
|
|
unsigned long flags = 0, cmd_mask = 0;
|
|
struct cxl_dev_state *cxlds = cxlmd->cxlds;
|
|
struct cxl_nvdimm_bridge *cxl_nvb;
|
|
struct nvdimm *nvdimm;
|
|
int rc;
|
|
|
|
cxl_nvb = cxl_find_nvdimm_bridge(cxl_nvd);
|
|
if (!cxl_nvb)
|
|
return -ENXIO;
|
|
|
|
device_lock(&cxl_nvb->dev);
|
|
if (!cxl_nvb->nvdimm_bus) {
|
|
rc = -ENXIO;
|
|
goto out;
|
|
}
|
|
|
|
set_exclusive_cxl_commands(cxlds, exclusive_cmds);
|
|
rc = devm_add_action_or_reset(dev, clear_exclusive, cxlds);
|
|
if (rc)
|
|
goto out;
|
|
|
|
set_bit(NDD_LABELING, &flags);
|
|
set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
|
|
set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
|
|
set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
|
|
nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags,
|
|
cmd_mask, 0, NULL);
|
|
if (!nvdimm) {
|
|
rc = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
dev_set_drvdata(dev, nvdimm);
|
|
rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
|
|
out:
|
|
device_unlock(&cxl_nvb->dev);
|
|
put_device(&cxl_nvb->dev);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static struct cxl_driver cxl_nvdimm_driver = {
|
|
.name = "cxl_nvdimm",
|
|
.probe = cxl_nvdimm_probe,
|
|
.id = CXL_DEVICE_NVDIMM,
|
|
};
|
|
|
|
static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds,
|
|
struct nd_cmd_get_config_size *cmd,
|
|
unsigned int buf_len)
|
|
{
|
|
if (sizeof(*cmd) > buf_len)
|
|
return -EINVAL;
|
|
|
|
*cmd = (struct nd_cmd_get_config_size) {
|
|
.config_size = cxlds->lsa_size,
|
|
.max_xfer = cxlds->payload_size,
|
|
};
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int cxl_pmem_get_config_data(struct cxl_dev_state *cxlds,
|
|
struct nd_cmd_get_config_data_hdr *cmd,
|
|
unsigned int buf_len)
|
|
{
|
|
struct cxl_mbox_get_lsa get_lsa;
|
|
int rc;
|
|
|
|
if (sizeof(*cmd) > buf_len)
|
|
return -EINVAL;
|
|
if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
|
|
return -EINVAL;
|
|
|
|
get_lsa = (struct cxl_mbox_get_lsa) {
|
|
.offset = cmd->in_offset,
|
|
.length = cmd->in_length,
|
|
};
|
|
|
|
rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_LSA, &get_lsa,
|
|
sizeof(get_lsa), cmd->out_buf, cmd->in_length);
|
|
cmd->status = 0;
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds,
|
|
struct nd_cmd_set_config_hdr *cmd,
|
|
unsigned int buf_len)
|
|
{
|
|
struct cxl_mbox_set_lsa *set_lsa;
|
|
int rc;
|
|
|
|
if (sizeof(*cmd) > buf_len)
|
|
return -EINVAL;
|
|
|
|
/* 4-byte status follows the input data in the payload */
|
|
if (struct_size(cmd, in_buf, cmd->in_length) + 4 > buf_len)
|
|
return -EINVAL;
|
|
|
|
set_lsa =
|
|
kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
|
|
if (!set_lsa)
|
|
return -ENOMEM;
|
|
|
|
*set_lsa = (struct cxl_mbox_set_lsa) {
|
|
.offset = cmd->in_offset,
|
|
};
|
|
memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
|
|
|
|
rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_SET_LSA, set_lsa,
|
|
struct_size(set_lsa, data, cmd->in_length),
|
|
NULL, 0);
|
|
|
|
/*
|
|
* Set "firmware" status (4-packed bytes at the end of the input
|
|
* payload.
|
|
*/
|
|
put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
|
|
kvfree(set_lsa);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
|
|
void *buf, unsigned int buf_len)
|
|
{
|
|
struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
|
|
unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
|
|
struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
|
|
struct cxl_dev_state *cxlds = cxlmd->cxlds;
|
|
|
|
if (!test_bit(cmd, &cmd_mask))
|
|
return -ENOTTY;
|
|
|
|
switch (cmd) {
|
|
case ND_CMD_GET_CONFIG_SIZE:
|
|
return cxl_pmem_get_config_size(cxlds, buf, buf_len);
|
|
case ND_CMD_GET_CONFIG_DATA:
|
|
return cxl_pmem_get_config_data(cxlds, buf, buf_len);
|
|
case ND_CMD_SET_CONFIG_DATA:
|
|
return cxl_pmem_set_config_data(cxlds, buf, buf_len);
|
|
default:
|
|
return -ENOTTY;
|
|
}
|
|
}
|
|
|
|
static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
|
|
struct nvdimm *nvdimm, unsigned int cmd, void *buf,
|
|
unsigned int buf_len, int *cmd_rc)
|
|
{
|
|
/*
|
|
* No firmware response to translate, let the transport error
|
|
* code take precedence.
|
|
*/
|
|
*cmd_rc = 0;
|
|
|
|
if (!nvdimm)
|
|
return -ENOTTY;
|
|
return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
|
|
}
|
|
|
|
static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb)
|
|
{
|
|
if (cxl_nvb->nvdimm_bus)
|
|
return true;
|
|
cxl_nvb->nvdimm_bus =
|
|
nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
|
|
return cxl_nvb->nvdimm_bus != NULL;
|
|
}
|
|
|
|
static int cxl_nvdimm_release_driver(struct device *dev, void *data)
|
|
{
|
|
if (!is_cxl_nvdimm(dev))
|
|
return 0;
|
|
device_release_driver(dev);
|
|
return 0;
|
|
}
|
|
|
|
static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus)
|
|
{
|
|
if (!nvdimm_bus)
|
|
return;
|
|
|
|
/*
|
|
* Set the state of cxl_nvdimm devices to unbound / idle before
|
|
* nvdimm_bus_unregister() rips the nvdimm objects out from
|
|
* underneath them.
|
|
*/
|
|
bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_release_driver);
|
|
nvdimm_bus_unregister(nvdimm_bus);
|
|
}
|
|
|
|
static void cxl_nvb_update_state(struct work_struct *work)
|
|
{
|
|
struct cxl_nvdimm_bridge *cxl_nvb =
|
|
container_of(work, typeof(*cxl_nvb), state_work);
|
|
struct nvdimm_bus *victim_bus = NULL;
|
|
bool release = false, rescan = false;
|
|
|
|
device_lock(&cxl_nvb->dev);
|
|
switch (cxl_nvb->state) {
|
|
case CXL_NVB_ONLINE:
|
|
if (!online_nvdimm_bus(cxl_nvb)) {
|
|
dev_err(&cxl_nvb->dev,
|
|
"failed to establish nvdimm bus\n");
|
|
release = true;
|
|
} else
|
|
rescan = true;
|
|
break;
|
|
case CXL_NVB_OFFLINE:
|
|
case CXL_NVB_DEAD:
|
|
victim_bus = cxl_nvb->nvdimm_bus;
|
|
cxl_nvb->nvdimm_bus = NULL;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
device_unlock(&cxl_nvb->dev);
|
|
|
|
if (release)
|
|
device_release_driver(&cxl_nvb->dev);
|
|
if (rescan) {
|
|
int rc = bus_rescan_devices(&cxl_bus_type);
|
|
|
|
dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);
|
|
}
|
|
offline_nvdimm_bus(victim_bus);
|
|
|
|
put_device(&cxl_nvb->dev);
|
|
}
|
|
|
|
static void cxl_nvdimm_bridge_state_work(struct cxl_nvdimm_bridge *cxl_nvb)
|
|
{
|
|
/*
|
|
* Take a reference that the workqueue will drop if new work
|
|
* gets queued.
|
|
*/
|
|
get_device(&cxl_nvb->dev);
|
|
if (!queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
|
|
put_device(&cxl_nvb->dev);
|
|
}
|
|
|
|
static void cxl_nvdimm_bridge_remove(struct device *dev)
|
|
{
|
|
struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
|
|
|
|
if (cxl_nvb->state == CXL_NVB_ONLINE)
|
|
cxl_nvb->state = CXL_NVB_OFFLINE;
|
|
cxl_nvdimm_bridge_state_work(cxl_nvb);
|
|
}
|
|
|
|
static int cxl_nvdimm_bridge_probe(struct device *dev)
|
|
{
|
|
struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
|
|
|
|
if (cxl_nvb->state == CXL_NVB_DEAD)
|
|
return -ENXIO;
|
|
|
|
if (cxl_nvb->state == CXL_NVB_NEW) {
|
|
cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
|
|
.provider_name = "CXL",
|
|
.module = THIS_MODULE,
|
|
.ndctl = cxl_pmem_ctl,
|
|
};
|
|
|
|
INIT_WORK(&cxl_nvb->state_work, cxl_nvb_update_state);
|
|
}
|
|
|
|
cxl_nvb->state = CXL_NVB_ONLINE;
|
|
cxl_nvdimm_bridge_state_work(cxl_nvb);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct cxl_driver cxl_nvdimm_bridge_driver = {
|
|
.name = "cxl_nvdimm_bridge",
|
|
.probe = cxl_nvdimm_bridge_probe,
|
|
.remove = cxl_nvdimm_bridge_remove,
|
|
.id = CXL_DEVICE_NVDIMM_BRIDGE,
|
|
};
|
|
|
|
/*
|
|
* Return all bridges to the CXL_NVB_NEW state to invalidate any
|
|
* ->state_work referring to the now destroyed cxl_pmem_wq.
|
|
*/
|
|
static int cxl_nvdimm_bridge_reset(struct device *dev, void *data)
|
|
{
|
|
struct cxl_nvdimm_bridge *cxl_nvb;
|
|
|
|
if (!is_cxl_nvdimm_bridge(dev))
|
|
return 0;
|
|
|
|
cxl_nvb = to_cxl_nvdimm_bridge(dev);
|
|
device_lock(dev);
|
|
cxl_nvb->state = CXL_NVB_NEW;
|
|
device_unlock(dev);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void destroy_cxl_pmem_wq(void)
|
|
{
|
|
destroy_workqueue(cxl_pmem_wq);
|
|
bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_bridge_reset);
|
|
}
|
|
|
|
static __init int cxl_pmem_init(void)
|
|
{
|
|
int rc;
|
|
|
|
set_bit(CXL_MEM_COMMAND_ID_SET_PARTITION_INFO, exclusive_cmds);
|
|
set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
|
|
set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
|
|
|
|
cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0);
|
|
if (!cxl_pmem_wq)
|
|
return -ENXIO;
|
|
|
|
rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
|
|
if (rc)
|
|
goto err_bridge;
|
|
|
|
rc = cxl_driver_register(&cxl_nvdimm_driver);
|
|
if (rc)
|
|
goto err_nvdimm;
|
|
|
|
return 0;
|
|
|
|
err_nvdimm:
|
|
cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
|
|
err_bridge:
|
|
destroy_cxl_pmem_wq();
|
|
return rc;
|
|
}
|
|
|
|
static __exit void cxl_pmem_exit(void)
|
|
{
|
|
cxl_driver_unregister(&cxl_nvdimm_driver);
|
|
cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
|
|
destroy_cxl_pmem_wq();
|
|
}
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
module_init(cxl_pmem_init);
|
|
module_exit(cxl_pmem_exit);
|
|
MODULE_IMPORT_NS(CXL);
|
|
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
|
|
MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
|