leandrof/linux
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drm/i915: Promote i915->mm.obj_lock to be irqsafe

Browse Source 
The intent is to be able to update the mm.lists from inside an irqsoff
section (e.g. from a softirq rcu workqueue), ergo we need to make the
i915->mm.obj_lock irqsafe.

v2: can_discard_pages() ensures we are shrinkable
v3: Beware shadowing of 'flags'

Fixes: 3b4fa9640c ("drm/i915: Track the purgeable objects on a separate eviction list")
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=110869
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Matthew Auld <matthew.william.auld@gmail.com>
Reviewed-by: Matthew Auld <matthew.william.auld@gmail.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190610145430.17717-1-chris@chris-wilson.co.uk
This commit is contained in:
Chris Wilson 2019-06-10 15:54:30 +01:00
parent f4d57d838c
commit a8cff4c828
8 changed files with 79 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
drivers/gpu/drm/i915/gem/i915_gem_domain.c
View File

@ -475,15 +475,20 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
}
mutex_unlock(&i915->ggtt.vm.mutex);
if (i915_gem_object_is_shrinkable(obj) &&
obj->mm.madv == I915_MADV_WILLNEED) {
struct list_head *list;
if (i915_gem_object_is_shrinkable(obj)) {
unsigned long flags;
spin_lock(&i915->mm.obj_lock);
list = obj->bind_count ?
&i915->mm.bound_list : &i915->mm.unbound_list;
list_move_tail(&obj->mm.link, list);
spin_unlock(&i915->mm.obj_lock);
spin_lock_irqsave(&i915->mm.obj_lock, flags);
if (obj->mm.madv == I915_MADV_WILLNEED) {
struct list_head *list;
list = obj->bind_count ?
&i915->mm.bound_list : &i915->mm.unbound_list;
list_move_tail(&obj->mm.link, list);
}
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
}

12
drivers/gpu/drm/i915/gem/i915_gem_object.c
View File

@ -207,9 +207,11 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
*/
if (i915_gem_object_has_pages(obj) &&
i915_gem_object_is_shrinkable(obj)) {
spin_lock(&i915->mm.obj_lock);
unsigned long flags;
spin_lock_irqsave(&i915->mm.obj_lock, flags);
list_del_init(&obj->mm.link);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
mutex_unlock(&i915->drm.struct_mutex);
@ -330,9 +332,11 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
obj->mm.madv = I915_MADV_DONTNEED;
if (i915_gem_object_has_pages(obj)) {
spin_lock(&i915->mm.obj_lock);
unsigned long flags;
spin_lock_irqsave(&i915->mm.obj_lock, flags);
list_move_tail(&obj->mm.link, &i915->mm.purge_list);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
}

16
drivers/gpu/drm/i915/gem/i915_gem_pages.c
View File

@ -57,11 +57,15 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
if (i915_gem_object_is_shrinkable(obj)) {
spin_lock(&i915->mm.obj_lock);
unsigned long flags;
spin_lock_irqsave(&i915->mm.obj_lock, flags);
i915->mm.shrink_count++;
i915->mm.shrink_memory += obj->base.size;
list_add(&obj->mm.link, &i915->mm.unbound_list);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
}
@ -151,11 +155,15 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
return pages;
if (i915_gem_object_is_shrinkable(obj)) {
spin_lock(&i915->mm.obj_lock);
unsigned long flags;
spin_lock_irqsave(&i915->mm.obj_lock, flags);
list_del(&obj->mm.link);
i915->mm.shrink_count--;
i915->mm.shrink_memory -= obj->base.size;
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
if (obj->mm.mapping) {

38
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c
View File

@ -138,7 +138,7 @@ unsigned long
i915_gem_shrink(struct drm_i915_private *i915,
unsigned long target,
unsigned long *nr_scanned,
unsigned flags)
unsigned int shrink)
{
const struct {
struct list_head *list;
@ -154,7 +154,7 @@ i915_gem_shrink(struct drm_i915_private *i915,
unsigned long scanned = 0;
bool unlock;
if (!shrinker_lock(i915, flags, &unlock))
if (!shrinker_lock(i915, shrink, &unlock))
return 0;
/*
@ -166,12 +166,12 @@ i915_gem_shrink(struct drm_i915_private *i915,
* We don't care about errors here; if we cannot wait upon the GPU,
* we will free as much as we can and hope to get a second chance.
*/
if (flags & I915_SHRINK_ACTIVE)
if (shrink & I915_SHRINK_ACTIVE)
i915_gem_wait_for_idle(i915,
I915_WAIT_LOCKED,
MAX_SCHEDULE_TIMEOUT);
trace_i915_gem_shrink(i915, target, flags);
trace_i915_gem_shrink(i915, target, shrink);
i915_retire_requests(i915);
/*
@ -179,10 +179,10 @@ i915_gem_shrink(struct drm_i915_private *i915,
* device just to recover a little memory. If absolutely necessary,
* we will force the wake during oom-notifier.
*/
if (flags & I915_SHRINK_BOUND) {
if (shrink & I915_SHRINK_BOUND) {
wakeref = intel_runtime_pm_get_if_in_use(i915);
if (!wakeref)
flags &= ~I915_SHRINK_BOUND;
shrink &= ~I915_SHRINK_BOUND;
}
/*
@ -207,8 +207,9 @@ i915_gem_shrink(struct drm_i915_private *i915,
for (phase = phases; phase->list; phase++) {
struct list_head still_in_list;
struct drm_i915_gem_object *obj;
unsigned long flags;
if ((flags & phase->bit) == 0)
if ((shrink & phase->bit) == 0)
continue;
INIT_LIST_HEAD(&still_in_list);
@ -220,50 +221,50 @@ i915_gem_shrink(struct drm_i915_private *i915,
* to be able to shrink their pages, so they remain on
* the unbound/bound list until actually freed.
*/
spin_lock(&i915->mm.obj_lock);
spin_lock_irqsave(&i915->mm.obj_lock, flags);
while (count < target &&
(obj = list_first_entry_or_null(phase->list,
typeof(*obj),
mm.link))) {
list_move_tail(&obj->mm.link, &still_in_list);
if (flags & I915_SHRINK_VMAPS &&
if (shrink & I915_SHRINK_VMAPS &&
!is_vmalloc_addr(obj->mm.mapping))
continue;
if (!(flags & I915_SHRINK_ACTIVE) &&
if (!(shrink & I915_SHRINK_ACTIVE) &&
(i915_gem_object_is_active(obj) ||
i915_gem_object_is_framebuffer(obj)))
continue;
if (!(flags & I915_SHRINK_BOUND) &&
if (!(shrink & I915_SHRINK_BOUND) &&
READ_ONCE(obj->bind_count))
continue;
if (!can_release_pages(obj))
continue;
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
if (unsafe_drop_pages(obj)) {
/* May arrive from get_pages on another bo */
mutex_lock_nested(&obj->mm.lock,
I915_MM_SHRINKER);
if (!i915_gem_object_has_pages(obj)) {
try_to_writeback(obj, flags);
try_to_writeback(obj, shrink);
count += obj->base.size >> PAGE_SHIFT;
}
mutex_unlock(&obj->mm.lock);
}
scanned += obj->base.size >> PAGE_SHIFT;
spin_lock(&i915->mm.obj_lock);
spin_lock_irqsave(&i915->mm.obj_lock, flags);
}
list_splice_tail(&still_in_list, phase->list);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
if (flags & I915_SHRINK_BOUND)
if (shrink & I915_SHRINK_BOUND)
intel_runtime_pm_put(i915, wakeref);
i915_retire_requests(i915);
@ -379,6 +380,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
struct drm_i915_gem_object *obj;
unsigned long unevictable, bound, unbound, freed_pages;
intel_wakeref_t wakeref;
unsigned long flags;
freed_pages = 0;
with_intel_runtime_pm(i915, wakeref)
@ -392,7 +394,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
* being pointed to by hardware.
*/
unbound = bound = unevictable = 0;
spin_lock(&i915->mm.obj_lock);
spin_lock_irqsave(&i915->mm.obj_lock, flags);
list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) {
if (!can_release_pages(obj))
unevictable += obj->base.size >> PAGE_SHIFT;
@ -405,7 +407,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
else
bound += obj->base.size >> PAGE_SHIFT;
}
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
if (freed_pages || unbound || bound)
pr_info("Purging GPU memory, %lu pages freed, "

5
drivers/gpu/drm/i915/gem/i915_gem_stolen.c
View File

@ -613,6 +613,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
struct drm_i915_gem_object *obj;
struct drm_mm_node *stolen;
struct i915_vma *vma;
unsigned long flags;
int ret;
if (!drm_mm_initialized(&dev_priv->mm.stolen))
@ -689,10 +690,10 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
list_move_tail(&vma->vm_link, &ggtt->vm.bound_list);
mutex_unlock(&ggtt->vm.mutex);
spin_lock(&dev_priv->mm.obj_lock);
spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
GEM_BUG_ON(i915_gem_object_is_shrinkable(obj));
obj->bind_count++;
spin_unlock(&dev_priv->mm.obj_lock);
spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
return obj;

10
drivers/gpu/drm/i915/i915_debugfs.c
View File

@ -269,6 +269,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
struct drm_i915_gem_object *obj;
u64 total_obj_size, total_gtt_size;
unsigned long total, count, n;
unsigned long flags;
int ret;
total = READ_ONCE(dev_priv->mm.shrink_count);
@ -282,7 +283,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
total_obj_size = total_gtt_size = count = 0;
spin_lock(&dev_priv->mm.obj_lock);
spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) {
if (count == total)
break;
@ -305,7 +306,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
objects[count++] = obj;
total_obj_size += obj->base.size;
}
spin_unlock(&dev_priv->mm.obj_lock);
spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
sort(objects, count, sizeof(*objects), obj_rank_by_stolen, NULL);
@ -457,6 +458,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
u64 size, mapped_size, purgeable_size, dpy_size, huge_size;
struct drm_i915_gem_object *obj;
unsigned int page_sizes = 0;
unsigned long flags;
char buf[80];
int ret;
@ -469,7 +471,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
purgeable_size = purgeable_count = 0;
huge_size = huge_count = 0;
spin_lock(&dev_priv->mm.obj_lock);
spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) {
size += obj->base.size;
++count;
@ -518,7 +520,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
page_sizes |= obj->mm.page_sizes.sg;
}
}
spin_unlock(&dev_priv->mm.obj_lock);
spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
seq_printf(m, "%u bound objects, %llu bytes\n",
count, size);

8
drivers/gpu/drm/i915/i915_gem.c
View File

@ -1138,7 +1138,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
struct list_head *list;
if (i915_gem_object_is_shrinkable(obj)) {
spin_lock(&i915->mm.obj_lock);
unsigned long flags;
spin_lock_irqsave(&i915->mm.obj_lock, flags);
if (obj->mm.madv != I915_MADV_WILLNEED)
list = &i915->mm.purge_list;
else if (obj->bind_count)
@ -1146,7 +1149,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
else
list = &i915->mm.unbound_list;
list_move_tail(&obj->mm.link, list);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
}
}

17
drivers/gpu/drm/i915/i915_vma.c
View File

@ -80,11 +80,14 @@ static void vma_print_allocator(struct i915_vma *vma, const char *reason)
static void obj_bump_mru(struct drm_i915_gem_object *obj)
{
struct drm_i915_private *i915 = to_i915(obj->base.dev);
unsigned long flags;
spin_lock_irqsave(&i915->mm.obj_lock, flags);
spin_lock(&i915->mm.obj_lock);
if (obj->bind_count)
list_move_tail(&obj->mm.link, &i915->mm.bound_list);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
obj->mm.dirty = true; /* be paranoid */
}
@ -678,8 +681,9 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
unsigned long flags;
spin_lock(&dev_priv->mm.obj_lock);
spin_lock_irqsave(&dev_priv->mm.obj_lock, flags);
if (i915_gem_object_is_shrinkable(obj))
list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
@ -687,7 +691,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
obj->bind_count++;
assert_bind_count(obj);
spin_unlock(&dev_priv->mm.obj_lock);
spin_unlock_irqrestore(&dev_priv->mm.obj_lock, flags);
}
return 0;
@ -721,8 +725,9 @@ i915_vma_remove(struct i915_vma *vma)
*/
if (vma->obj) {
struct drm_i915_gem_object *obj = vma->obj;
unsigned long flags;
spin_lock(&i915->mm.obj_lock);
spin_lock_irqsave(&i915->mm.obj_lock, flags);
GEM_BUG_ON(obj->bind_count == 0);
if (--obj->bind_count == 0 &&
@ -730,7 +735,7 @@ i915_vma_remove(struct i915_vma *vma)
obj->mm.madv == I915_MADV_WILLNEED)
list_move_tail(&obj->mm.link, &i915->mm.unbound_list);
spin_unlock(&i915->mm.obj_lock);
spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
/*
* And finally now the object is completely decoupled from this