leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drm/amdkfd: Reliably prevent reclaim-FS while holding DQM lock

Browse Source 
This is needed to prevent deadlocks when MMU notifiers run in
reclaim-FS context and take the DQM lock for userptr evictions.
Previously this was done by making all memory allocations under
DQM locks GFP_NOIO. This is error prone. Using
memalloc_nofs_save/restore will reliably affect all memory
allocations anywhere in the kernel while the DQM lock is held.

Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Acked-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>
This commit is contained in:
Felix Kuehling 2018-07-11 22:32:44 -04:00 committed by Oded Gabbay
parent 0337976f40
commit efeaed4d98
2 changed files with 62 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
View File

@ -240,7 +240,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
print_queue(q); print_queue(q);
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (dqm->total_queue_count >= max_num_of_queues_per_device) { if (dqm->total_queue_count >= max_num_of_queues_per_device) {
pr_warn("Can't create new usermode queue because %d queues were already created\n", pr_warn("Can't create new usermode queue because %d queues were already created\n",
@ -297,7 +297,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
dqm->total_queue_count); dqm->total_queue_count);
out_unlock: out_unlock:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -457,9 +457,9 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
{ {
int retval; int retval;
mutex_lock(&dqm->lock); dqm_lock(dqm);
retval = destroy_queue_nocpsch_locked(dqm, qpd, q); retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -471,7 +471,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
struct kfd_process_device *pdd; struct kfd_process_device *pdd;
bool prev_active = false; bool prev_active = false;
mutex_lock(&dqm->lock); dqm_lock(dqm);
pdd = kfd_get_process_device_data(q->device, q->process); pdd = kfd_get_process_device_data(q->device, q->process);
if (!pdd) { if (!pdd) {
retval = -ENODEV; retval = -ENODEV;
@ -537,7 +537,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
&q->properties, q->process->mm); &q->properties, q->process->mm);
out_unlock: out_unlock:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -570,7 +570,7 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
struct kfd_process_device *pdd; struct kfd_process_device *pdd;
int retval = 0; int retval = 0;
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (qpd->evicted++ > 0) /* already evicted, do nothing */ if (qpd->evicted++ > 0) /* already evicted, do nothing */
goto out; goto out;
@ -600,7 +600,7 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
} }
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -611,7 +611,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
struct kfd_process_device *pdd; struct kfd_process_device *pdd;
int retval = 0; int retval = 0;
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (qpd->evicted++ > 0) /* already evicted, do nothing */ if (qpd->evicted++ > 0) /* already evicted, do nothing */
goto out; goto out;
@ -633,7 +633,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0); KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -650,7 +650,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
/* Retrieve PD base */ /* Retrieve PD base */
pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm); pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
goto out; goto out;
if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
@ -695,7 +695,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
} }
qpd->evicted = 0; qpd->evicted = 0;
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -711,7 +711,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
/* Retrieve PD base */ /* Retrieve PD base */
pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm); pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */ if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
goto out; goto out;
if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */ if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
@ -739,7 +739,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
if (!retval) if (!retval)
qpd->evicted = 0; qpd->evicted = 0;
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -761,7 +761,7 @@ static int register_process(struct device_queue_manager *dqm,
/* Retrieve PD base */ /* Retrieve PD base */
pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm); pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
mutex_lock(&dqm->lock); dqm_lock(dqm);
list_add(&n->list, &dqm->queues); list_add(&n->list, &dqm->queues);
/* Update PD Base in QPD */ /* Update PD Base in QPD */
@ -771,7 +771,7 @@ static int register_process(struct device_queue_manager *dqm,
dqm->processes_count++; dqm->processes_count++;
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -786,7 +786,7 @@ static int unregister_process(struct device_queue_manager *dqm,
list_empty(&qpd->queues_list) ? "empty" : "not empty"); list_empty(&qpd->queues_list) ? "empty" : "not empty");
retval = 0; retval = 0;
mutex_lock(&dqm->lock); dqm_lock(dqm);
list_for_each_entry_safe(cur, next, &dqm->queues, list) { list_for_each_entry_safe(cur, next, &dqm->queues, list) {
if (qpd == cur->qpd) { if (qpd == cur->qpd) {
@ -799,7 +799,7 @@ static int unregister_process(struct device_queue_manager *dqm,
/* qpd not found in dqm list */ /* qpd not found in dqm list */
retval = 1; retval = 1;
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -838,7 +838,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
if (!dqm->allocated_queues) if (!dqm->allocated_queues)
return -ENOMEM; return -ENOMEM;
mutex_init(&dqm->lock); mutex_init(&dqm->lock_hidden);
INIT_LIST_HEAD(&dqm->queues); INIT_LIST_HEAD(&dqm->queues);
dqm->queue_count = dqm->next_pipe_to_allocate = 0; dqm->queue_count = dqm->next_pipe_to_allocate = 0;
dqm->sdma_queue_count = 0; dqm->sdma_queue_count = 0;
@ -867,7 +867,7 @@ static void uninitialize(struct device_queue_manager *dqm)
kfree(dqm->allocated_queues); kfree(dqm->allocated_queues);
for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++) for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
kfree(dqm->mqds[i]); kfree(dqm->mqds[i]);
mutex_destroy(&dqm->lock); mutex_destroy(&dqm->lock_hidden);
kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem); kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
} }
@ -1003,7 +1003,7 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
{ {
pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
mutex_init(&dqm->lock); mutex_init(&dqm->lock_hidden);
INIT_LIST_HEAD(&dqm->queues); INIT_LIST_HEAD(&dqm->queues);
dqm->queue_count = dqm->processes_count = 0; dqm->queue_count = dqm->processes_count = 0;
dqm->sdma_queue_count = 0; dqm->sdma_queue_count = 0;
@ -1041,9 +1041,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
init_interrupts(dqm); init_interrupts(dqm);
mutex_lock(&dqm->lock); dqm_lock(dqm);
execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0); execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return 0; return 0;
fail_allocate_vidmem: fail_allocate_vidmem:
@ -1055,9 +1055,9 @@ fail_packet_manager_init:
static int stop_cpsch(struct device_queue_manager *dqm) static int stop_cpsch(struct device_queue_manager *dqm)
{ {
mutex_lock(&dqm->lock); dqm_lock(dqm);
unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0); unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
kfd_gtt_sa_free(dqm->dev, dqm->fence_mem); kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
pm_uninit(&dqm->packets); pm_uninit(&dqm->packets);
@ -1069,11 +1069,11 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
struct kernel_queue *kq, struct kernel_queue *kq,
struct qcm_process_device *qpd) struct qcm_process_device *qpd)
{ {
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (dqm->total_queue_count >= max_num_of_queues_per_device) { if (dqm->total_queue_count >= max_num_of_queues_per_device) {
pr_warn("Can't create new kernel queue because %d queues were already created\n", pr_warn("Can't create new kernel queue because %d queues were already created\n",
dqm->total_queue_count); dqm->total_queue_count);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return -EPERM; return -EPERM;
} }
@ -1089,7 +1089,7 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
dqm->queue_count++; dqm->queue_count++;
qpd->is_debug = true; qpd->is_debug = true;
execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0); execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return 0; return 0;
} }
@ -1098,7 +1098,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
struct kernel_queue *kq, struct kernel_queue *kq,
struct qcm_process_device *qpd) struct qcm_process_device *qpd)
{ {
mutex_lock(&dqm->lock); dqm_lock(dqm);
list_del(&kq->list); list_del(&kq->list);
dqm->queue_count--; dqm->queue_count--;
qpd->is_debug = false; qpd->is_debug = false;
@ -1110,7 +1110,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
dqm->total_queue_count--; dqm->total_queue_count--;
pr_debug("Total of %d queues are accountable so far\n", pr_debug("Total of %d queues are accountable so far\n",
dqm->total_queue_count); dqm->total_queue_count);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
} }
static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
@ -1121,7 +1121,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
retval = 0; retval = 0;
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (dqm->total_queue_count >= max_num_of_queues_per_device) { if (dqm->total_queue_count >= max_num_of_queues_per_device) {
pr_warn("Can't create new usermode queue because %d queues were already created\n", pr_warn("Can't create new usermode queue because %d queues were already created\n",
@ -1188,7 +1188,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
pr_debug("Total of %d queues are accountable so far\n", pr_debug("Total of %d queues are accountable so far\n",
dqm->total_queue_count); dqm->total_queue_count);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
out_deallocate_doorbell: out_deallocate_doorbell:
@ -1197,7 +1197,8 @@ out_deallocate_sdma_queue:
if (q->properties.type == KFD_QUEUE_TYPE_SDMA) if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
deallocate_sdma_queue(dqm, q->sdma_id); deallocate_sdma_queue(dqm, q->sdma_id);
out_unlock: out_unlock:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -1314,7 +1315,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
retval = 0; retval = 0;
/* remove queue from list to prevent rescheduling after preemption */ /* remove queue from list to prevent rescheduling after preemption */
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (qpd->is_debug) { if (qpd->is_debug) {
/* /*
@ -1360,14 +1361,14 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
pr_debug("Total of %d queues are accountable so far\n", pr_debug("Total of %d queues are accountable so far\n",
dqm->total_queue_count); dqm->total_queue_count);
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
failed: failed:
failed_try_destroy_debugged_queue: failed_try_destroy_debugged_queue:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -1391,7 +1392,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
if (!dqm->asic_ops.set_cache_memory_policy) if (!dqm->asic_ops.set_cache_memory_policy)
return retval; return retval;
mutex_lock(&dqm->lock); dqm_lock(dqm);
if (alternate_aperture_size == 0) { if (alternate_aperture_size == 0) {
/* base > limit disables APE1 */ /* base > limit disables APE1 */
@ -1437,7 +1438,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
qpd->sh_mem_ape1_limit); qpd->sh_mem_ape1_limit);
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -1468,7 +1469,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
struct device_process_node *cur, *next_dpn; struct device_process_node *cur, *next_dpn;
int retval = 0; int retval = 0;
mutex_lock(&dqm->lock); dqm_lock(dqm);
/* Clear all user mode queues */ /* Clear all user mode queues */
list_for_each_entry_safe(q, next, &qpd->queues_list, list) { list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
@ -1489,7 +1490,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
} }
} }
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }
@ -1507,7 +1508,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
retval = 0; retval = 0;
mutex_lock(&dqm->lock); dqm_lock(dqm);
/* Clean all kernel queues */ /* Clean all kernel queues */
list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) { list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
@ -1562,7 +1563,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
} }
out: out:
mutex_unlock(&dqm->lock); dqm_unlock(dqm);
return retval; return retval;
} }

20
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
View File

@ -26,6 +26,8 @@
#include <linux/rwsem.h> #include <linux/rwsem.h>
#include <linux/list.h> #include <linux/list.h>
#include <linux/mutex.h>
#include <linux/sched/mm.h>
#include "kfd_priv.h" #include "kfd_priv.h"
#include "kfd_mqd_manager.h" #include "kfd_mqd_manager.h"
@ -173,8 +175,9 @@ struct device_queue_manager {
struct mqd_manager *mqds[KFD_MQD_TYPE_MAX]; struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];
struct packet_manager packets; struct packet_manager packets;
struct kfd_dev *dev; struct kfd_dev *dev;
struct mutex lock; struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
struct list_head queues; struct list_head queues;
unsigned int saved_flags;
unsigned int processes_count; unsigned int processes_count;
unsigned int queue_count; unsigned int queue_count;
unsigned int sdma_queue_count; unsigned int sdma_queue_count;
@ -219,4 +222,19 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
return (pdd->lds_base >> 60) & 0x0E; return (pdd->lds_base >> 60) & 0x0E;
} }
/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
* happens while holding this lock anywhere to prevent deadlocks when
* an MMU notifier runs in reclaim-FS context.
*/
static inline void dqm_lock(struct device_queue_manager *dqm)
{
mutex_lock(&dqm->lock_hidden);
dqm->saved_flags = memalloc_nofs_save();
}
static inline void dqm_unlock(struct device_queue_manager *dqm)
{
memalloc_nofs_restore(dqm->saved_flags);
mutex_unlock(&dqm->lock_hidden);
}
#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */