leandrof/linux
1
0
Fork 0
forked from Minki/linux
Code Issues Pull Requests Packages Projects Releases Wiki Activity

drm/i915: Reduce duplicated forcewake logic

Browse Source 
Introduce a structure to track the individual forcewake domains and use
that to eliminate duplicate logic.

v2: - Rebase on latest dinq (Mika)
    - for_each_fw_domain macro (Mika)
    - Handle reset atomically, keeping the timer running (Mika)
    - for_each_fw_domain parameter ordering (Chris)
    - defer timer on new register access (Mika)

v3: - Fix forcewake_reset/get race by waiting pending timers

v4: - cond_resched and verbose warning on timer deletion (Chris)
    - need to run pending timers manually on reset

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> (v1)
Signed-off-by: Mika Kuoppala <mika.kuoppala@intel.com>
Acked-by: Deepak S <deepak.s@linux.intel.com> (v2)
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
This commit is contained in:
Chris Wilson 2015-01-16 11:34:37 +02:00 committed by Daniel Vetter
parent 51f6788584
commit b2cff0dbbb
3 changed files with 245 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
drivers/gpu/drm/i915/i915_debugfs.c
View File

@ -1288,14 +1288,36 @@ static int ironlake_drpc_info(struct seq_file *m)
return 0; return 0;
} }
static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
{
struct drm_info_node *node = m->private;
struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_uncore_forcewake_domain *fw_domain;
const char *domain_names[] = {
"render",
"blitter",
"media",
};
int i;
spin_lock_irq(&dev_priv->uncore.lock);
for_each_fw_domain(fw_domain, dev_priv, i) {
seq_printf(m, "%s.wake_count = %u\n",
domain_names[i],
fw_domain->wake_count);
}
spin_unlock_irq(&dev_priv->uncore.lock);
return 0;
}
static int vlv_drpc_info(struct seq_file *m) static int vlv_drpc_info(struct seq_file *m)
{ {
struct drm_info_node *node = m->private; struct drm_info_node *node = m->private;
struct drm_device *dev = node->minor->dev; struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_private *dev_priv = dev->dev_private;
u32 rpmodectl1, rcctl1, pw_status; u32 rpmodectl1, rcctl1, pw_status;
unsigned fw_rendercount = 0, fw_mediacount = 0;
intel_runtime_pm_get(dev_priv); intel_runtime_pm_get(dev_priv);
@ -1327,22 +1349,11 @@ static int vlv_drpc_info(struct seq_file *m)
seq_printf(m, "Media RC6 residency since boot: %u\n", seq_printf(m, "Media RC6 residency since boot: %u\n",
I915_READ(VLV_GT_MEDIA_RC6)); I915_READ(VLV_GT_MEDIA_RC6));
spin_lock_irq(&dev_priv->uncore.lock); return i915_gen6_forcewake_count_info(m, NULL);
fw_rendercount = dev_priv->uncore.fw_rendercount;
fw_mediacount = dev_priv->uncore.fw_mediacount;
spin_unlock_irq(&dev_priv->uncore.lock);
seq_printf(m, "Forcewake Render Count = %u\n", fw_rendercount);
seq_printf(m, "Forcewake Media Count = %u\n", fw_mediacount);
return 0;
} }
static int gen6_drpc_info(struct seq_file *m) static int gen6_drpc_info(struct seq_file *m)
{ {
struct drm_info_node *node = m->private; struct drm_info_node *node = m->private;
struct drm_device *dev = node->minor->dev; struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_private *dev_priv = dev->dev_private;
@ -1356,7 +1367,7 @@ static int gen6_drpc_info(struct seq_file *m)
intel_runtime_pm_get(dev_priv); intel_runtime_pm_get(dev_priv);
spin_lock_irq(&dev_priv->uncore.lock); spin_lock_irq(&dev_priv->uncore.lock);
forcewake_count = dev_priv->uncore.forcewake_count; forcewake_count = dev_priv->uncore.fw_domain[FW_DOMAIN_ID_RENDER].wake_count;
spin_unlock_irq(&dev_priv->uncore.lock); spin_unlock_irq(&dev_priv->uncore.lock);
if (forcewake_count) { if (forcewake_count) {
@ -1984,30 +1995,6 @@ static int i915_execlists(struct seq_file *m, void *data)
return 0; return 0;
} }
static int i915_gen6_forcewake_count_info(struct seq_file *m, void *data)
{
struct drm_info_node *node = m->private;
struct drm_device *dev = node->minor->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
unsigned forcewake_count = 0, fw_rendercount = 0, fw_mediacount = 0;
spin_lock_irq(&dev_priv->uncore.lock);
if (IS_VALLEYVIEW(dev)) {
fw_rendercount = dev_priv->uncore.fw_rendercount;
fw_mediacount = dev_priv->uncore.fw_mediacount;
} else
forcewake_count = dev_priv->uncore.forcewake_count;
spin_unlock_irq(&dev_priv->uncore.lock);
if (IS_VALLEYVIEW(dev)) {
seq_printf(m, "fw_rendercount = %u\n", fw_rendercount);
seq_printf(m, "fw_mediacount = %u\n", fw_mediacount);
} else
seq_printf(m, "forcewake count = %u\n", forcewake_count);
return 0;
}
static const char *swizzle_string(unsigned swizzle) static const char *swizzle_string(unsigned swizzle)
{ {
switch (swizzle) { switch (swizzle) {

50
drivers/gpu/drm/i915/i915_drv.h
View File

@ -596,21 +596,46 @@ struct intel_uncore_funcs {
uint64_t val, bool trace); uint64_t val, bool trace);
}; };
enum {
FW_DOMAIN_ID_RENDER = 0,
FW_DOMAIN_ID_BLITTER,
FW_DOMAIN_ID_MEDIA,
FW_DOMAIN_ID_COUNT
};
struct intel_uncore { struct intel_uncore {
spinlock_t lock; /** lock is also taken in irq contexts. */ spinlock_t lock; /** lock is also taken in irq contexts. */
struct intel_uncore_funcs funcs; struct intel_uncore_funcs funcs;
unsigned fifo_count; unsigned fifo_count;
unsigned forcewake_count; unsigned fw_domains;
unsigned fw_rendercount; struct intel_uncore_forcewake_domain {
unsigned fw_mediacount; struct drm_i915_private *i915;
unsigned fw_blittercount; int id;
unsigned wake_count;
struct timer_list force_wake_timer; struct timer_list timer;
} fw_domain[FW_DOMAIN_ID_COUNT];
#define FORCEWAKE_RENDER (1 << FW_DOMAIN_ID_RENDER)
#define FORCEWAKE_BLITTER (1 << FW_DOMAIN_ID_BLITTER)
#define FORCEWAKE_MEDIA (1 << FW_DOMAIN_ID_MEDIA)
#define FORCEWAKE_ALL (FORCEWAKE_RENDER | \
FORCEWAKE_BLITTER | \
FORCEWAKE_MEDIA)
}; };
/* Iterate over initialised fw domains */
#define for_each_fw_domain_mask(domain__, mask__, dev_priv__, i__) \
for ((i__) = 0, (domain__) = &(dev_priv__)->uncore.fw_domain[0]; \
(i__) < FW_DOMAIN_ID_COUNT; \
(i__)++, (domain__) = &(dev_priv__)->uncore.fw_domain[i__]) \
if (((mask__) & (dev_priv__)->uncore.fw_domains) & (1 << (i__)))
#define for_each_fw_domain(domain__, dev_priv__, i__) \
for_each_fw_domain_mask(domain__, FORCEWAKE_ALL, dev_priv__, i__)
#define DEV_INFO_FOR_EACH_FLAG(func, sep) \ #define DEV_INFO_FOR_EACH_FLAG(func, sep) \
func(is_mobile) sep \ func(is_mobile) sep \
func(is_i85x) sep \ func(is_i85x) sep \
@ -3167,8 +3192,10 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
* must be set to prevent GT core from power down and stale values being * must be set to prevent GT core from power down and stale values being
* returned. * returned.
*/ */
void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine); void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine); unsigned fw_domains);
void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
unsigned fw_domains);
void assert_force_wake_inactive(struct drm_i915_private *dev_priv); void assert_force_wake_inactive(struct drm_i915_private *dev_priv);
int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
@ -3200,13 +3227,6 @@ void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val);
int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val); int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val);
int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val); int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val);
#define FORCEWAKE_RENDER (1 << 0)
#define FORCEWAKE_MEDIA (1 << 1)
#define FORCEWAKE_BLITTER (1 << 2)
#define FORCEWAKE_ALL (FORCEWAKE_RENDER | FORCEWAKE_MEDIA | \
FORCEWAKE_BLITTER)
#define I915_READ8(reg) dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true) #define I915_READ8(reg) dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true)
#define I915_WRITE8(reg, val) dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true) #define I915_WRITE8(reg, val) dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true)

447
drivers/gpu/drm/i915/intel_uncore.c
View File

@ -67,7 +67,7 @@ static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
} }
static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
int fw_engine) int fw_engine)
{ {
if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0, if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0,
FORCEWAKE_ACK_TIMEOUT_MS)) FORCEWAKE_ACK_TIMEOUT_MS))
@ -93,7 +93,7 @@ static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
} }
static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
int fw_engine) int fw_engine)
{ {
u32 forcewake_ack; u32 forcewake_ack;
@ -129,7 +129,7 @@ static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
} }
static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
int fw_engine) int fw_engine)
{ {
__raw_i915_write32(dev_priv, FORCEWAKE, 0); __raw_i915_write32(dev_priv, FORCEWAKE, 0);
/* something from same cacheline, but !FORCEWAKE */ /* something from same cacheline, but !FORCEWAKE */
@ -138,7 +138,7 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
} }
static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
int fw_engine) int fw_engine)
{ {
__raw_i915_write32(dev_priv, FORCEWAKE_MT, __raw_i915_write32(dev_priv, FORCEWAKE_MT,
_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
@ -187,7 +187,7 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
} }
static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
int fw_engine) int fw_engine)
{ {
/* Check for Render Engine */ /* Check for Render Engine */
if (FORCEWAKE_RENDER & fw_engine) { if (FORCEWAKE_RENDER & fw_engine) {
@ -227,9 +227,8 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
} }
static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
int fw_engine) int fw_engine)
{ {
/* Check for Render Engine */ /* Check for Render Engine */
if (FORCEWAKE_RENDER & fw_engine) if (FORCEWAKE_RENDER & fw_engine)
__raw_i915_write32(dev_priv, FORCEWAKE_VLV, __raw_i915_write32(dev_priv, FORCEWAKE_VLV,
@ -247,37 +246,6 @@ static void __vlv_force_wake_put(struct drm_i915_private *dev_priv,
gen6_gt_check_fifodbg(dev_priv); gen6_gt_check_fifodbg(dev_priv);
} }
static void vlv_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
{
if (fw_engine & FORCEWAKE_RENDER &&
dev_priv->uncore.fw_rendercount++ != 0)
fw_engine &= ~FORCEWAKE_RENDER;
if (fw_engine & FORCEWAKE_MEDIA &&
dev_priv->uncore.fw_mediacount++ != 0)
fw_engine &= ~FORCEWAKE_MEDIA;
if (fw_engine)
dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
}
static void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
{
if (fw_engine & FORCEWAKE_RENDER) {
WARN_ON(!dev_priv->uncore.fw_rendercount);
if (--dev_priv->uncore.fw_rendercount != 0)
fw_engine &= ~FORCEWAKE_RENDER;
}
if (fw_engine & FORCEWAKE_MEDIA) {
WARN_ON(!dev_priv->uncore.fw_mediacount);
if (--dev_priv->uncore.fw_mediacount != 0)
fw_engine &= ~FORCEWAKE_MEDIA;
}
if (fw_engine)
dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
}
static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) static void __gen9_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
{ {
__raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9, __raw_i915_write32(dev_priv, FORCEWAKE_RENDER_GEN9,
@ -367,80 +335,72 @@ __gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
} }
static void
gen9_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
{
if (FORCEWAKE_RENDER & fw_engine) {
if (dev_priv->uncore.fw_rendercount++ == 0)
dev_priv->uncore.funcs.force_wake_get(dev_priv,
FORCEWAKE_RENDER);
}
if (FORCEWAKE_MEDIA & fw_engine) {
if (dev_priv->uncore.fw_mediacount++ == 0)
dev_priv->uncore.funcs.force_wake_get(dev_priv,
FORCEWAKE_MEDIA);
}
if (FORCEWAKE_BLITTER & fw_engine) {
if (dev_priv->uncore.fw_blittercount++ == 0)
dev_priv->uncore.funcs.force_wake_get(dev_priv,
FORCEWAKE_BLITTER);
}
}
static void
gen9_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
{
if (FORCEWAKE_RENDER & fw_engine) {
WARN_ON(dev_priv->uncore.fw_rendercount == 0);
if (--dev_priv->uncore.fw_rendercount == 0)
dev_priv->uncore.funcs.force_wake_put(dev_priv,
FORCEWAKE_RENDER);
}
if (FORCEWAKE_MEDIA & fw_engine) {
WARN_ON(dev_priv->uncore.fw_mediacount == 0);
if (--dev_priv->uncore.fw_mediacount == 0)
dev_priv->uncore.funcs.force_wake_put(dev_priv,
FORCEWAKE_MEDIA);
}
if (FORCEWAKE_BLITTER & fw_engine) {
WARN_ON(dev_priv->uncore.fw_blittercount == 0);
if (--dev_priv->uncore.fw_blittercount == 0)
dev_priv->uncore.funcs.force_wake_put(dev_priv,
FORCEWAKE_BLITTER);
}
}
static void gen6_force_wake_timer(unsigned long arg) static void gen6_force_wake_timer(unsigned long arg)
{ {
struct drm_i915_private *dev_priv = (void *)arg; struct intel_uncore_forcewake_domain *domain = (void *)arg;
unsigned long irqflags; unsigned long irqflags;
assert_device_not_suspended(dev_priv); assert_device_not_suspended(domain->i915);
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); spin_lock_irqsave(&domain->i915->uncore.lock, irqflags);
WARN_ON(!dev_priv->uncore.forcewake_count); if (WARN_ON(domain->wake_count == 0))
domain->wake_count++;
if (--dev_priv->uncore.forcewake_count == 0) if (--domain->wake_count == 0)
dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); domain->i915->uncore.funcs.force_wake_put(domain->i915,
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 1 << domain->id);
spin_unlock_irqrestore(&domain->i915->uncore.lock, irqflags);
} }
void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore) void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
{ {
struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_private *dev_priv = dev->dev_private;
unsigned long irqflags; unsigned long irqflags, fw = 0;
struct intel_uncore_forcewake_domain *domain;
if (del_timer_sync(&dev_priv->uncore.force_wake_timer)) int id, active_domains, retry_count = 100;
gen6_force_wake_timer((unsigned long)dev_priv);
/* Hold uncore.lock across reset to prevent any register access /* Hold uncore.lock across reset to prevent any register access
* with forcewake not set correctly * with forcewake not set correctly. Wait until all pending
* timers are run before holding.
*/ */
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); while (1) {
active_domains = 0;
for_each_fw_domain(domain, dev_priv, id) {
if (del_timer_sync(&domain->timer) == 0)
continue;
gen6_force_wake_timer((unsigned long)domain);
}
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
for_each_fw_domain(domain, dev_priv, id) {
if (timer_pending(&domain->timer))
active_domains |= (1 << id);
}
if (active_domains == 0)
break;
if (--retry_count == 0) {
DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
break;
}
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
cond_resched();
}
WARN_ON(active_domains);
for_each_fw_domain(domain, dev_priv, id)
if (domain->wake_count)
fw |= 1 << id;
if (fw)
dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
if (IS_VALLEYVIEW(dev)) if (IS_VALLEYVIEW(dev))
vlv_force_wake_reset(dev_priv); vlv_force_wake_reset(dev_priv);
@ -454,28 +414,6 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
__gen9_gt_force_wake_mt_reset(dev_priv); __gen9_gt_force_wake_mt_reset(dev_priv);
if (restore) { /* If reset with a user forcewake, try to restore */ if (restore) { /* If reset with a user forcewake, try to restore */
unsigned fw = 0;
if (IS_VALLEYVIEW(dev)) {
if (dev_priv->uncore.fw_rendercount)
fw |= FORCEWAKE_RENDER;
if (dev_priv->uncore.fw_mediacount)
fw |= FORCEWAKE_MEDIA;
} else if (IS_GEN9(dev)) {
if (dev_priv->uncore.fw_rendercount)
fw |= FORCEWAKE_RENDER;
if (dev_priv->uncore.fw_mediacount)
fw |= FORCEWAKE_MEDIA;
if (dev_priv->uncore.fw_blittercount)
fw |= FORCEWAKE_BLITTER;
} else {
if (dev_priv->uncore.forcewake_count)
fw = FORCEWAKE_ALL;
}
if (fw) if (fw)
dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
@ -485,6 +423,9 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
GT_FIFO_FREE_ENTRIES_MASK; GT_FIFO_FREE_ENTRIES_MASK;
} }
if (!restore)
assert_force_wake_inactive(dev_priv);
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
} }
@ -533,53 +474,59 @@ void intel_uncore_sanitize(struct drm_device *dev)
* be called at the beginning of the sequence followed by a call to * be called at the beginning of the sequence followed by a call to
* gen6_gt_force_wake_put() at the end of the sequence. * gen6_gt_force_wake_put() at the end of the sequence.
*/ */
void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
unsigned fw_domains)
{ {
unsigned long irqflags; unsigned long irqflags;
struct intel_uncore_forcewake_domain *domain;
int id;
if (!dev_priv->uncore.funcs.force_wake_get) if (!dev_priv->uncore.funcs.force_wake_get)
return; return;
WARN_ON(dev_priv->pm.suspended); WARN_ON(dev_priv->pm.suspended);
fw_domains &= dev_priv->uncore.fw_domains;
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
if (IS_GEN9(dev_priv->dev)) { for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {
gen9_force_wake_get(dev_priv, fw_engine); if (domain->wake_count++)
} else if (IS_VALLEYVIEW(dev_priv->dev)) { fw_domains &= ~(1 << id);
vlv_force_wake_get(dev_priv, fw_engine);
} else {
if (dev_priv->uncore.forcewake_count++ == 0)
dev_priv->uncore.funcs.force_wake_get(dev_priv,
FORCEWAKE_ALL);
} }
if (fw_domains)
dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
} }
/* /*
* see gen6_gt_force_wake_get() * see gen6_gt_force_wake_get()
*/ */
void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
unsigned fw_domains)
{ {
unsigned long irqflags; unsigned long irqflags;
struct intel_uncore_forcewake_domain *domain;
int id;
if (!dev_priv->uncore.funcs.force_wake_put) if (!dev_priv->uncore.funcs.force_wake_put)
return; return;
fw_domains &= dev_priv->uncore.fw_domains;
spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
if (IS_GEN9(dev_priv->dev)) { for_each_fw_domain_mask(domain, fw_domains, dev_priv, id) {
gen9_force_wake_put(dev_priv, fw_engine); if (WARN_ON(domain->wake_count == 0))
} else if (IS_VALLEYVIEW(dev_priv->dev)) { continue;
vlv_force_wake_put(dev_priv, fw_engine);
} else { if (--domain->wake_count)
WARN_ON(!dev_priv->uncore.forcewake_count); continue;
if (--dev_priv->uncore.forcewake_count == 0) {
dev_priv->uncore.forcewake_count++; domain->wake_count++;
mod_timer_pinned(&dev_priv->uncore.force_wake_timer, mod_timer_pinned(&domain->timer, jiffies + 1);
jiffies + 1);
}
} }
spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
@ -587,10 +534,14 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
void assert_force_wake_inactive(struct drm_i915_private *dev_priv) void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
{ {
struct intel_uncore_forcewake_domain *domain;
int i;
if (!dev_priv->uncore.funcs.force_wake_get) if (!dev_priv->uncore.funcs.force_wake_get)
return; return;
WARN_ON(dev_priv->uncore.forcewake_count > 0); for_each_fw_domain(domain, dev_priv, i)
WARN_ON(domain->wake_count);
} }
/* We give fast paths for the really cool registers */ /* We give fast paths for the really cool registers */
@ -753,19 +704,37 @@ __gen2_read(64)
trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
return val return val
static inline void __force_wake_get(struct drm_i915_private *dev_priv,
unsigned fw_domains)
{
struct intel_uncore_forcewake_domain *domain;
int i;
if (WARN_ON(!fw_domains))
return;
/* Ideally GCC would be constant-fold and eliminate this loop */
for_each_fw_domain_mask(domain, fw_domains, dev_priv, i) {
if (domain->wake_count) {
fw_domains &= ~(1 << i);
continue;
}
domain->wake_count++;
mod_timer_pinned(&domain->timer, jiffies + 1);
}
if (fw_domains)
dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
}
#define __gen6_read(x) \ #define __gen6_read(x) \
static u##x \ static u##x \
gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
GEN6_READ_HEADER(x); \ GEN6_READ_HEADER(x); \
hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \ hsw_unclaimed_reg_debug(dev_priv, reg, true, true); \
if (dev_priv->uncore.forcewake_count == 0 && \ if (NEEDS_FORCE_WAKE((dev_priv), (reg))) \
NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
dev_priv->uncore.funcs.force_wake_get(dev_priv, \
FORCEWAKE_ALL); \
dev_priv->uncore.forcewake_count++; \
mod_timer_pinned(&dev_priv->uncore.force_wake_timer, \
jiffies + 1); \
} \
val = __raw_i915_read##x(dev_priv, reg); \ val = __raw_i915_read##x(dev_priv, reg); \
hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \ hsw_unclaimed_reg_debug(dev_priv, reg, true, false); \
GEN6_READ_FOOTER; \ GEN6_READ_FOOTER; \
@ -774,45 +743,27 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
#define __vlv_read(x) \ #define __vlv_read(x) \
static u##x \ static u##x \
vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
unsigned fwengine = 0; \
GEN6_READ_HEADER(x); \ GEN6_READ_HEADER(x); \
if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) \
if (dev_priv->uncore.fw_rendercount == 0) \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
fwengine = FORCEWAKE_RENDER; \ else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) \
} else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
if (dev_priv->uncore.fw_mediacount == 0) \
fwengine = FORCEWAKE_MEDIA; \
} \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
val = __raw_i915_read##x(dev_priv, reg); \ val = __raw_i915_read##x(dev_priv, reg); \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
GEN6_READ_FOOTER; \ GEN6_READ_FOOTER; \
} }
#define __chv_read(x) \ #define __chv_read(x) \
static u##x \ static u##x \
chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
unsigned fwengine = 0; \
GEN6_READ_HEADER(x); \ GEN6_READ_HEADER(x); \
if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \ if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
if (dev_priv->uncore.fw_rendercount == 0) \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
fwengine = FORCEWAKE_RENDER; \ else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
} else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \ __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
if (dev_priv->uncore.fw_mediacount == 0) \ else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
fwengine = FORCEWAKE_MEDIA; \ __force_wake_get(dev_priv, \
} else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \ FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
if (dev_priv->uncore.fw_rendercount == 0) \
fwengine |= FORCEWAKE_RENDER; \
if (dev_priv->uncore.fw_mediacount == 0) \
fwengine |= FORCEWAKE_MEDIA; \
} \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
val = __raw_i915_read##x(dev_priv, reg); \ val = __raw_i915_read##x(dev_priv, reg); \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
GEN6_READ_FOOTER; \ GEN6_READ_FOOTER; \
} }
@ -822,32 +773,21 @@ chv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
#define __gen9_read(x) \ #define __gen9_read(x) \
static u##x \ static u##x \
gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ gen9_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
unsigned fw_engine; \
GEN6_READ_HEADER(x); \ GEN6_READ_HEADER(x); \
if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg))) \
val = __raw_i915_read##x(dev_priv, reg); \ fw_engine = 0; \
} else { \ else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
unsigned fwengine = 0; \ fw_engine = FORCEWAKE_RENDER; \
if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \ else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
if (dev_priv->uncore.fw_rendercount == 0) \ fw_engine = FORCEWAKE_MEDIA; \
fwengine = FORCEWAKE_RENDER; \ else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
} else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \ fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
if (dev_priv->uncore.fw_mediacount == 0) \ else \
fwengine = FORCEWAKE_MEDIA; \ fw_engine = FORCEWAKE_BLITTER; \
} else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \ if (fw_engine) \
if (dev_priv->uncore.fw_rendercount == 0) \ __force_wake_get(dev_priv, fw_engine); \
fwengine |= FORCEWAKE_RENDER; \ val = __raw_i915_read##x(dev_priv, reg); \
if (dev_priv->uncore.fw_mediacount == 0) \
fwengine |= FORCEWAKE_MEDIA; \
} else { \
if (dev_priv->uncore.fw_blittercount == 0) \
fwengine = FORCEWAKE_BLITTER; \
} \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
val = __raw_i915_read##x(dev_priv, reg); \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
} \
GEN6_READ_FOOTER; \ GEN6_READ_FOOTER; \
} }
@ -981,17 +921,9 @@ static void \
gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
GEN6_WRITE_HEADER; \ GEN6_WRITE_HEADER; \
hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \ hsw_unclaimed_reg_debug(dev_priv, reg, false, true); \
if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \ if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) \
if (dev_priv->uncore.forcewake_count == 0) \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
dev_priv->uncore.funcs.force_wake_get(dev_priv, \ __raw_i915_write##x(dev_priv, reg, val); \
FORCEWAKE_ALL); \
__raw_i915_write##x(dev_priv, reg, val); \
if (dev_priv->uncore.forcewake_count == 0) \
dev_priv->uncore.funcs.force_wake_put(dev_priv, \
FORCEWAKE_ALL); \
} else { \
__raw_i915_write##x(dev_priv, reg, val); \
} \
hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \ hsw_unclaimed_reg_debug(dev_priv, reg, false, false); \
hsw_unclaimed_reg_detect(dev_priv); \ hsw_unclaimed_reg_detect(dev_priv); \
GEN6_WRITE_FOOTER; \ GEN6_WRITE_FOOTER; \
@ -1000,28 +932,17 @@ gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
#define __chv_write(x) \ #define __chv_write(x) \
static void \ static void \
chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ chv_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
unsigned fwengine = 0; \
bool shadowed = is_gen8_shadowed(dev_priv, reg); \ bool shadowed = is_gen8_shadowed(dev_priv, reg); \
GEN6_WRITE_HEADER; \ GEN6_WRITE_HEADER; \
if (!shadowed) { \ if (!shadowed) { \
if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) { \ if (FORCEWAKE_CHV_RENDER_RANGE_OFFSET(reg)) \
if (dev_priv->uncore.fw_rendercount == 0) \ __force_wake_get(dev_priv, FORCEWAKE_RENDER); \
fwengine = FORCEWAKE_RENDER; \ else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) \
} else if (FORCEWAKE_CHV_MEDIA_RANGE_OFFSET(reg)) { \ __force_wake_get(dev_priv, FORCEWAKE_MEDIA); \
if (dev_priv->uncore.fw_mediacount == 0) \ else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) \
fwengine = FORCEWAKE_MEDIA; \ __force_wake_get(dev_priv, FORCEWAKE_RENDER | FORCEWAKE_MEDIA); \
} else if (FORCEWAKE_CHV_COMMON_RANGE_OFFSET(reg)) { \
if (dev_priv->uncore.fw_rendercount == 0) \
fwengine |= FORCEWAKE_RENDER; \
if (dev_priv->uncore.fw_mediacount == 0) \
fwengine |= FORCEWAKE_MEDIA; \
} \
} \ } \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
__raw_i915_write##x(dev_priv, reg, val); \ __raw_i915_write##x(dev_priv, reg, val); \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
GEN6_WRITE_FOOTER; \ GEN6_WRITE_FOOTER; \
} }
@ -1052,35 +973,22 @@ static bool is_gen9_shadowed(struct drm_i915_private *dev_priv, u32 reg)
static void \ static void \
gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \ gen9_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, \
bool trace) { \ bool trace) { \
unsigned fw_engine; \
GEN6_WRITE_HEADER; \ GEN6_WRITE_HEADER; \
if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \ if (!SKL_NEEDS_FORCE_WAKE((dev_priv), (reg)) || \
is_gen9_shadowed(dev_priv, reg)) { \ is_gen9_shadowed(dev_priv, reg)) \
__raw_i915_write##x(dev_priv, reg, val); \ fw_engine = 0; \
} else { \ else if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) \
unsigned fwengine = 0; \ fw_engine = FORCEWAKE_RENDER; \
if (FORCEWAKE_GEN9_RENDER_RANGE_OFFSET(reg)) { \ else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) \
if (dev_priv->uncore.fw_rendercount == 0) \ fw_engine = FORCEWAKE_MEDIA; \
fwengine = FORCEWAKE_RENDER; \ else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) \
} else if (FORCEWAKE_GEN9_MEDIA_RANGE_OFFSET(reg)) { \ fw_engine = FORCEWAKE_RENDER | FORCEWAKE_MEDIA; \
if (dev_priv->uncore.fw_mediacount == 0) \ else \
fwengine = FORCEWAKE_MEDIA; \ fw_engine = FORCEWAKE_BLITTER; \
} else if (FORCEWAKE_GEN9_COMMON_RANGE_OFFSET(reg)) { \ if (fw_engine) \
if (dev_priv->uncore.fw_rendercount == 0) \ __force_wake_get(dev_priv, fw_engine); \
fwengine |= FORCEWAKE_RENDER; \ __raw_i915_write##x(dev_priv, reg, val); \
if (dev_priv->uncore.fw_mediacount == 0) \
fwengine |= FORCEWAKE_MEDIA; \
} else { \
if (dev_priv->uncore.fw_blittercount == 0) \
fwengine = FORCEWAKE_BLITTER; \
} \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_get(dev_priv, \
fwengine); \
__raw_i915_write##x(dev_priv, reg, val); \
if (fwengine) \
dev_priv->uncore.funcs.force_wake_put(dev_priv, \
fwengine); \
} \
GEN6_WRITE_FOOTER; \ GEN6_WRITE_FOOTER; \
} }
@ -1132,21 +1040,24 @@ do { \
void intel_uncore_init(struct drm_device *dev) void intel_uncore_init(struct drm_device *dev)
{ {
struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_private *dev_priv = dev->dev_private;
struct intel_uncore_forcewake_domain *domain;
setup_timer(&dev_priv->uncore.force_wake_timer, int i;
gen6_force_wake_timer, (unsigned long)dev_priv);
__intel_uncore_early_sanitize(dev, false); __intel_uncore_early_sanitize(dev, false);
if (IS_GEN9(dev)) { if (IS_GEN9(dev)) {
dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get; dev_priv->uncore.funcs.force_wake_get = __gen9_force_wake_get;
dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put; dev_priv->uncore.funcs.force_wake_put = __gen9_force_wake_put;
dev_priv->uncore.fw_domains = FORCEWAKE_RENDER |
FORCEWAKE_BLITTER | FORCEWAKE_MEDIA;
} else if (IS_VALLEYVIEW(dev)) { } else if (IS_VALLEYVIEW(dev)) {
dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
dev_priv->uncore.fw_domains = FORCEWAKE_RENDER | FORCEWAKE_MEDIA;
} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) { } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get; dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put; dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
} else if (IS_IVYBRIDGE(dev)) { } else if (IS_IVYBRIDGE(dev)) {
u32 ecobus; u32 ecobus;
@ -1178,11 +1089,21 @@ void intel_uncore_init(struct drm_device *dev)
dev_priv->uncore.funcs.force_wake_put = dev_priv->uncore.funcs.force_wake_put =
__gen6_gt_force_wake_put; __gen6_gt_force_wake_put;
} }
dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
} else if (IS_GEN6(dev)) { } else if (IS_GEN6(dev)) {
dev_priv->uncore.funcs.force_wake_get = dev_priv->uncore.funcs.force_wake_get =
__gen6_gt_force_wake_get; __gen6_gt_force_wake_get;
dev_priv->uncore.funcs.force_wake_put = dev_priv->uncore.funcs.force_wake_put =
__gen6_gt_force_wake_put; __gen6_gt_force_wake_put;
dev_priv->uncore.fw_domains = FORCEWAKE_RENDER;
}
for_each_fw_domain(domain, dev_priv, i) {
domain->i915 = dev_priv;
domain->id = i;
setup_timer(&domain->timer, gen6_force_wake_timer,
(unsigned long)domain);
} }
switch (INTEL_INFO(dev)->gen) { switch (INTEL_INFO(dev)->gen) {