leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drm/amdkfd: CRIU checkpoint and restore events

Browse Source 
Add support to existing CRIU ioctl's to save and restore events during
criu checkpoint and restore.

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: David Yat Sin <david.yatsin@amd.com>
Signed-off-by: Rajneesh Bhardwaj <rajneesh.bhardwaj@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
This commit is contained in:
David Yat Sin 2021-03-05 16:06:41 -05:00 committed by Alex Deucher
parent 3a9822d7bd
commit 40e8a766a7
3 changed files with 280 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
View File

@ -1008,57 +1008,11 @@ static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
* through the event_page_offset field.
*/
if (args->event_page_offset) {
struct kfd_dev *kfd;
struct kfd_process_device *pdd;
void *mem, *kern_addr;
uint64_t size;
kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
if (!kfd) {
pr_err("Getting device by id failed in %s\n", __func__);
return -EINVAL;
}
mutex_lock(&p->mutex);
if (p->signal_page) {
pr_err("Event page is already set\n");
err = -EINVAL;
goto out_unlock;
}
pdd = kfd_bind_process_to_device(kfd, p);
if (IS_ERR(pdd)) {
err = PTR_ERR(pdd);
goto out_unlock;
}
mem = kfd_process_device_translate_handle(pdd,
GET_IDR_HANDLE(args->event_page_offset));
if (!mem) {
pr_err("Can't find BO, offset is 0x%llx\n",
args->event_page_offset);
err = -EINVAL;
goto out_unlock;
}
err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->adev,
mem, &kern_addr, &size);
if (err) {
pr_err("Failed to map event page to kernel\n");
goto out_unlock;
}
err = kfd_event_page_set(p, kern_addr, size);
if (err) {
pr_err("Failed to set event page\n");
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->adev, mem);
goto out_unlock;
}
p->signal_handle = args->event_page_offset;
err = kfd_kmap_event_page(p, args->event_page_offset);
mutex_unlock(&p->mutex);
if (err)
return err;
}
err = kfd_event_create(filp, p, args->event_type,
@ -1067,10 +1021,7 @@ static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
&args->event_page_offset,
&args->event_slot_index);
return err;
out_unlock:
mutex_unlock(&p->mutex);
pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
return err;
}
@ -2031,7 +1982,7 @@ static int criu_get_process_object_info(struct kfd_process *p,
if (ret)
return ret;
num_events = 0; /* TODO: Implement Events */
num_events = kfd_get_num_events(p);
num_svm_ranges = 0; /* TODO: Implement SVM-Ranges */
*num_objects = num_queues + num_events + num_svm_ranges;
@ -2040,7 +1991,7 @@ static int criu_get_process_object_info(struct kfd_process *p,
priv_size = sizeof(struct kfd_criu_process_priv_data);
priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
priv_size += queues_priv_data_size;
/* TODO: Add Events priv size */
priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
/* TODO: Add SVM ranges priv size */
*objs_priv_size = priv_size;
}
@ -2102,7 +2053,10 @@ static int criu_checkpoint(struct file *filep,
if (ret)
goto exit_unlock;
/* TODO: Dump Events */
ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data,
&priv_offset);
if (ret)
goto exit_unlock;
/* TODO: Dump SVM-Ranges */
}
@ -2410,8 +2364,8 @@ static int criu_restore_objects(struct file *filep,
goto exit;
break;
case KFD_CRIU_OBJECT_TYPE_EVENT:
/* TODO: Implement Events */
*priv_offset += sizeof(struct kfd_criu_event_priv_data);
ret = kfd_criu_restore_event(filep, p, (uint8_t __user *)args->priv_data,
priv_offset, max_priv_data_size);
if (ret)
goto exit;
break;

272
drivers/gpu/drm/amd/amdkfd/kfd_events.c
View File

@ -55,7 +55,6 @@ struct kfd_signal_page {
bool need_to_free_pages;
};
static uint64_t *page_slots(struct kfd_signal_page *page)
{
return page->kernel_address;
@ -92,7 +91,8 @@ fail_alloc_signal_store:
}
static int allocate_event_notification_slot(struct kfd_process *p,
struct kfd_event *ev)
struct kfd_event *ev,
const int *restore_id)
{
int id;
@ -104,14 +104,19 @@ static int allocate_event_notification_slot(struct kfd_process *p,
p->signal_mapped_size = 256*8;
}
/*
* Compatibility with old user mode: Only use signal slots
* user mode has mapped, may be less than
* KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
* of the event limit without breaking user mode.
*/
id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
GFP_KERNEL);
if (restore_id) {
id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
GFP_KERNEL);
} else {
/*
* Compatibility with old user mode: Only use signal slots
* user mode has mapped, may be less than
* KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
* of the event limit without breaking user mode.
*/
id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
GFP_KERNEL);
}
if (id < 0)
return id;
@ -178,9 +183,8 @@ static struct kfd_event *lookup_signaled_event_by_partial_id(
return ev;
}
static int create_signal_event(struct file *devkfd,
struct kfd_process *p,
struct kfd_event *ev)
static int create_signal_event(struct file *devkfd, struct kfd_process *p,
struct kfd_event *ev, const int *restore_id)
{
int ret;
@ -193,7 +197,7 @@ static int create_signal_event(struct file *devkfd,
return -ENOSPC;
}
ret = allocate_event_notification_slot(p, ev);
ret = allocate_event_notification_slot(p, ev, restore_id);
if (ret) {
pr_warn("Signal event wasn't created because out of kernel memory\n");
return ret;
@ -209,16 +213,22 @@ static int create_signal_event(struct file *devkfd,
return 0;
}
static int create_other_event(struct kfd_process *p, struct kfd_event *ev)
static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id)
{
/* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
* intentional integer overflow to -1 without a compiler
* warning. idr_alloc treats a negative value as "maximum
* signed integer".
*/
int id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
(uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
GFP_KERNEL);
int id;
if (restore_id)
id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
GFP_KERNEL);
else
/* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
* intentional integer overflow to -1 without a compiler
* warning. idr_alloc treats a negative value as "maximum
* signed integer".
*/
id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
(uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
GFP_KERNEL);
if (id < 0)
return id;
@ -295,8 +305,8 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
return ev->type == KFD_EVENT_TYPE_SIGNAL;
}
int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
uint64_t size)
static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
uint64_t size, uint64_t user_handle)
{
struct kfd_signal_page *page;
@ -315,10 +325,56 @@ int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
p->signal_page = page;
p->signal_mapped_size = size;
p->signal_handle = user_handle;
return 0;
}
int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
{
struct kfd_dev *kfd;
struct kfd_process_device *pdd;
void *mem, *kern_addr;
uint64_t size;
int err = 0;
if (p->signal_page) {
pr_err("Event page is already set\n");
return -EINVAL;
}
kfd = kfd_device_by_id(GET_GPU_ID(event_page_offset));
if (!kfd) {
pr_err("Getting device by id failed in %s\n", __func__);
return -EINVAL;
}
pdd = kfd_bind_process_to_device(kfd, p);
if (IS_ERR(pdd))
return PTR_ERR(pdd);
mem = kfd_process_device_translate_handle(pdd,
GET_IDR_HANDLE(event_page_offset));
if (!mem) {
pr_err("Can't find BO, offset is 0x%llx\n", event_page_offset);
return -EINVAL;
}
err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->adev,
mem, &kern_addr, &size);
if (err) {
pr_err("Failed to map event page to kernel\n");
return err;
}
err = kfd_event_page_set(p, kern_addr, size, event_page_offset);
if (err) {
pr_err("Failed to set event page\n");
amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->adev, mem);
return err;
}
return err;
}
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
uint32_t event_type, bool auto_reset, uint32_t node_id,
uint32_t *event_id, uint32_t *event_trigger_data,
@ -343,14 +399,14 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
switch (event_type) {
case KFD_EVENT_TYPE_SIGNAL:
case KFD_EVENT_TYPE_DEBUG:
ret = create_signal_event(devkfd, p, ev);
ret = create_signal_event(devkfd, p, ev, NULL);
if (!ret) {
*event_page_offset = KFD_MMAP_TYPE_EVENTS;
*event_slot_index = ev->event_id;
}
break;
default:
ret = create_other_event(p, ev);
ret = create_other_event(p, ev, NULL);
break;
}
@ -366,6 +422,166 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p,
return ret;
}
int kfd_criu_restore_event(struct file *devkfd,
struct kfd_process *p,
uint8_t __user *user_priv_ptr,
uint64_t *priv_data_offset,
uint64_t max_priv_data_size)
{
struct kfd_criu_event_priv_data *ev_priv;
struct kfd_event *ev = NULL;
int ret = 0;
ev_priv = kmalloc(sizeof(*ev_priv), GFP_KERNEL);
if (!ev_priv)
return -ENOMEM;
ev = kzalloc(sizeof(*ev), GFP_KERNEL);
if (!ev) {
ret = -ENOMEM;
goto exit;
}
if (*priv_data_offset + sizeof(*ev_priv) > max_priv_data_size) {
ret = -EINVAL;
goto exit;
}
ret = copy_from_user(ev_priv, user_priv_ptr + *priv_data_offset, sizeof(*ev_priv));
if (ret) {
ret = -EFAULT;
goto exit;
}
*priv_data_offset += sizeof(*ev_priv);
if (ev_priv->user_handle) {
ret = kfd_kmap_event_page(p, ev_priv->user_handle);
if (ret)
goto exit;
}
ev->type = ev_priv->type;
ev->auto_reset = ev_priv->auto_reset;
ev->signaled = ev_priv->signaled;
init_waitqueue_head(&ev->wq);
mutex_lock(&p->event_mutex);
switch (ev->type) {
case KFD_EVENT_TYPE_SIGNAL:
case KFD_EVENT_TYPE_DEBUG:
ret = create_signal_event(devkfd, p, ev, &ev_priv->event_id);
break;
case KFD_EVENT_TYPE_MEMORY:
memcpy(&ev->memory_exception_data,
&ev_priv->memory_exception_data,
sizeof(struct kfd_hsa_memory_exception_data));
ret = create_other_event(p, ev, &ev_priv->event_id);
break;
case KFD_EVENT_TYPE_HW_EXCEPTION:
memcpy(&ev->hw_exception_data,
&ev_priv->hw_exception_data,
sizeof(struct kfd_hsa_hw_exception_data));
ret = create_other_event(p, ev, &ev_priv->event_id);
break;
}
exit:
if (ret)
kfree(ev);
kfree(ev_priv);
mutex_unlock(&p->event_mutex);
return ret;
}
int kfd_criu_checkpoint_events(struct kfd_process *p,
uint8_t __user *user_priv_data,
uint64_t *priv_data_offset)
{
struct kfd_criu_event_priv_data *ev_privs;
int i = 0;
int ret = 0;
struct kfd_event *ev;
uint32_t ev_id;
uint32_t num_events = kfd_get_num_events(p);
if (!num_events)
return 0;
ev_privs = kvzalloc(num_events * sizeof(*ev_privs), GFP_KERNEL);
if (!ev_privs)
return -ENOMEM;
idr_for_each_entry(&p->event_idr, ev, ev_id) {
struct kfd_criu_event_priv_data *ev_priv;
/*
* Currently, all events have same size of private_data, but the current ioctl's
* and CRIU plugin supports private_data of variable sizes
*/
ev_priv = &ev_privs[i];
ev_priv->object_type = KFD_CRIU_OBJECT_TYPE_EVENT;
/* We store the user_handle with the first event */
if (i == 0 && p->signal_page)
ev_priv->user_handle = p->signal_handle;
ev_priv->event_id = ev->event_id;
ev_priv->auto_reset = ev->auto_reset;
ev_priv->type = ev->type;
ev_priv->signaled = ev->signaled;
if (ev_priv->type == KFD_EVENT_TYPE_MEMORY)
memcpy(&ev_priv->memory_exception_data,
&ev->memory_exception_data,
sizeof(struct kfd_hsa_memory_exception_data));
else if (ev_priv->type == KFD_EVENT_TYPE_HW_EXCEPTION)
memcpy(&ev_priv->hw_exception_data,
&ev->hw_exception_data,
sizeof(struct kfd_hsa_hw_exception_data));
pr_debug("Checkpointed event[%d] id = 0x%08x auto_reset = %x type = %x signaled = %x\n",
i,
ev_priv->event_id,
ev_priv->auto_reset,
ev_priv->type,
ev_priv->signaled);
i++;
}
ret = copy_to_user(user_priv_data + *priv_data_offset,
ev_privs, num_events * sizeof(*ev_privs));
if (ret) {
pr_err("Failed to copy events priv to user\n");
ret = -EFAULT;
}
*priv_data_offset += num_events * sizeof(*ev_privs);
kvfree(ev_privs);
return ret;
}
int kfd_get_num_events(struct kfd_process *p)
{
struct kfd_event *ev;
uint32_t id;
u32 num_events = 0;
idr_for_each_entry(&p->event_idr, ev, id)
num_events++;
return num_events;
}
/* Assumes that p is current. */
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)
{

27
drivers/gpu/drm/amd/amdkfd/kfd_priv.h
View File

@ -1099,7 +1099,16 @@ struct kfd_criu_queue_priv_data {
struct kfd_criu_event_priv_data {
uint32_t object_type;
uint32_t reserved;
uint64_t user_handle;
uint32_t event_id;
uint32_t auto_reset;
uint32_t type;
uint32_t signaled;
union {
struct kfd_hsa_memory_exception_data memory_exception_data;
struct kfd_hsa_hw_exception_data hw_exception_data;
};
};
int kfd_process_get_queue_info(struct kfd_process *p,
@ -1114,6 +1123,16 @@ int kfd_criu_restore_queue(struct kfd_process *p,
uint8_t __user *user_priv_data,
uint64_t *priv_data_offset,
uint64_t max_priv_data_size);
int kfd_criu_checkpoint_events(struct kfd_process *p,
uint8_t __user *user_priv_data,
uint64_t *priv_data_offset);
int kfd_criu_restore_event(struct file *devkfd,
struct kfd_process *p,
uint8_t __user *user_priv_data,
uint64_t *priv_data_offset,
uint64_t max_priv_data_size);
/* CRIU - End */
/* Queue Context Management */
@ -1277,12 +1296,14 @@ void kfd_signal_iommu_event(struct kfd_dev *dev,
void kfd_signal_hw_exception_event(u32 pasid);
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
uint64_t size);
int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset);
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
uint32_t event_type, bool auto_reset, uint32_t node_id,
uint32_t *event_id, uint32_t *event_trigger_data,
uint64_t *event_page_offset, uint32_t *event_slot_index);
int kfd_get_num_events(struct kfd_process *p);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,