2016-07-13 15:03:40 +00:00
|
|
|
/*
|
|
|
|
* Copyright © 2016 Intel Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "i915_drv.h"
|
|
|
|
#include "intel_ringbuffer.h"
|
|
|
|
#include "intel_lrc.h"
|
|
|
|
|
2017-04-28 07:53:36 +00:00
|
|
|
/* Haswell does have the CXT_SIZE register however it does not appear to be
|
|
|
|
* valid. Now, docs explain in dwords what is in the context object. The full
|
|
|
|
* size is 70720 bytes, however, the power context and execlist context will
|
|
|
|
* never be saved (power context is stored elsewhere, and execlists don't work
|
|
|
|
* on HSW) - so the final size, including the extra state required for the
|
|
|
|
* Resource Streamer, is 66944 bytes, which rounds to 17 pages.
|
|
|
|
*/
|
|
|
|
#define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE)
|
|
|
|
/* Same as Haswell, but 72064 bytes now. */
|
|
|
|
#define GEN8_CXT_TOTAL_SIZE (18 * PAGE_SIZE)
|
|
|
|
|
|
|
|
#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
|
|
|
|
#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
|
|
|
|
|
|
|
|
#define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE)
|
|
|
|
|
2017-04-10 14:34:32 +00:00
|
|
|
struct engine_class_info {
|
2016-07-13 15:03:40 +00:00
|
|
|
const char *name;
|
2017-04-10 14:34:32 +00:00
|
|
|
int (*init_legacy)(struct intel_engine_cs *engine);
|
|
|
|
int (*init_execlists)(struct intel_engine_cs *engine);
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct engine_class_info intel_engine_classes[] = {
|
|
|
|
[RENDER_CLASS] = {
|
|
|
|
.name = "rcs",
|
|
|
|
.init_execlists = logical_render_ring_init,
|
|
|
|
.init_legacy = intel_init_render_ring_buffer,
|
|
|
|
},
|
|
|
|
[COPY_ENGINE_CLASS] = {
|
|
|
|
.name = "bcs",
|
|
|
|
.init_execlists = logical_xcs_ring_init,
|
|
|
|
.init_legacy = intel_init_blt_ring_buffer,
|
|
|
|
},
|
|
|
|
[VIDEO_DECODE_CLASS] = {
|
|
|
|
.name = "vcs",
|
|
|
|
.init_execlists = logical_xcs_ring_init,
|
|
|
|
.init_legacy = intel_init_bsd_ring_buffer,
|
|
|
|
},
|
|
|
|
[VIDEO_ENHANCEMENT_CLASS] = {
|
|
|
|
.name = "vecs",
|
|
|
|
.init_execlists = logical_xcs_ring_init,
|
|
|
|
.init_legacy = intel_init_vebox_ring_buffer,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
struct engine_info {
|
2017-03-01 20:26:15 +00:00
|
|
|
unsigned int hw_id;
|
2017-04-11 12:43:06 +00:00
|
|
|
unsigned int uabi_id;
|
2017-04-10 14:34:29 +00:00
|
|
|
u8 class;
|
|
|
|
u8 instance;
|
2016-07-13 15:03:40 +00:00
|
|
|
u32 mmio_base;
|
|
|
|
unsigned irq_shift;
|
2017-04-10 14:34:32 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static const struct engine_info intel_engines[] = {
|
2016-07-13 15:03:40 +00:00
|
|
|
[RCS] = {
|
2016-08-16 16:04:20 +00:00
|
|
|
.hw_id = RCS_HW,
|
2017-04-11 12:43:06 +00:00
|
|
|
.uabi_id = I915_EXEC_RENDER,
|
2017-04-10 14:34:29 +00:00
|
|
|
.class = RENDER_CLASS,
|
|
|
|
.instance = 0,
|
2016-07-13 15:03:40 +00:00
|
|
|
.mmio_base = RENDER_RING_BASE,
|
|
|
|
.irq_shift = GEN8_RCS_IRQ_SHIFT,
|
|
|
|
},
|
|
|
|
[BCS] = {
|
2016-08-16 16:04:20 +00:00
|
|
|
.hw_id = BCS_HW,
|
2017-04-11 12:43:06 +00:00
|
|
|
.uabi_id = I915_EXEC_BLT,
|
2017-04-10 14:34:29 +00:00
|
|
|
.class = COPY_ENGINE_CLASS,
|
|
|
|
.instance = 0,
|
2016-07-13 15:03:40 +00:00
|
|
|
.mmio_base = BLT_RING_BASE,
|
|
|
|
.irq_shift = GEN8_BCS_IRQ_SHIFT,
|
|
|
|
},
|
|
|
|
[VCS] = {
|
2016-08-16 16:04:20 +00:00
|
|
|
.hw_id = VCS_HW,
|
2017-04-11 12:43:06 +00:00
|
|
|
.uabi_id = I915_EXEC_BSD,
|
2017-04-10 14:34:29 +00:00
|
|
|
.class = VIDEO_DECODE_CLASS,
|
|
|
|
.instance = 0,
|
2016-07-13 15:03:40 +00:00
|
|
|
.mmio_base = GEN6_BSD_RING_BASE,
|
|
|
|
.irq_shift = GEN8_VCS1_IRQ_SHIFT,
|
|
|
|
},
|
|
|
|
[VCS2] = {
|
2016-08-16 16:04:20 +00:00
|
|
|
.hw_id = VCS2_HW,
|
2017-04-11 12:43:06 +00:00
|
|
|
.uabi_id = I915_EXEC_BSD,
|
2017-04-10 14:34:29 +00:00
|
|
|
.class = VIDEO_DECODE_CLASS,
|
|
|
|
.instance = 1,
|
2016-07-13 15:03:40 +00:00
|
|
|
.mmio_base = GEN8_BSD2_RING_BASE,
|
|
|
|
.irq_shift = GEN8_VCS2_IRQ_SHIFT,
|
|
|
|
},
|
|
|
|
[VECS] = {
|
2016-08-16 16:04:20 +00:00
|
|
|
.hw_id = VECS_HW,
|
2017-04-11 12:43:06 +00:00
|
|
|
.uabi_id = I915_EXEC_VEBOX,
|
2017-04-10 14:34:29 +00:00
|
|
|
.class = VIDEO_ENHANCEMENT_CLASS,
|
|
|
|
.instance = 0,
|
2016-07-13 15:03:40 +00:00
|
|
|
.mmio_base = VEBOX_RING_BASE,
|
|
|
|
.irq_shift = GEN8_VECS_IRQ_SHIFT,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
2017-04-28 07:53:36 +00:00
|
|
|
/**
|
|
|
|
* ___intel_engine_context_size() - return the size of the context for an engine
|
|
|
|
* @dev_priv: i915 device private
|
|
|
|
* @class: engine class
|
|
|
|
*
|
|
|
|
* Each engine class may require a different amount of space for a context
|
|
|
|
* image.
|
|
|
|
*
|
|
|
|
* Return: size (in bytes) of an engine class specific context image
|
|
|
|
*
|
|
|
|
* Note: this size includes the HWSP, which is part of the context image
|
|
|
|
* in LRC mode, but does not include the "shared data page" used with
|
|
|
|
* GuC submission. The caller should account for this if using the GuC.
|
|
|
|
*/
|
|
|
|
static u32
|
|
|
|
__intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
|
|
|
|
{
|
|
|
|
u32 cxt_size;
|
|
|
|
|
|
|
|
BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
|
|
|
|
|
|
|
|
switch (class) {
|
|
|
|
case RENDER_CLASS:
|
|
|
|
switch (INTEL_GEN(dev_priv)) {
|
|
|
|
default:
|
|
|
|
MISSING_CASE(INTEL_GEN(dev_priv));
|
2017-07-06 21:06:24 +00:00
|
|
|
case 10:
|
2017-04-28 07:53:36 +00:00
|
|
|
case 9:
|
|
|
|
return GEN9_LR_CONTEXT_RENDER_SIZE;
|
|
|
|
case 8:
|
|
|
|
return i915.enable_execlists ?
|
|
|
|
GEN8_LR_CONTEXT_RENDER_SIZE :
|
|
|
|
GEN8_CXT_TOTAL_SIZE;
|
|
|
|
case 7:
|
|
|
|
if (IS_HASWELL(dev_priv))
|
|
|
|
return HSW_CXT_TOTAL_SIZE;
|
|
|
|
|
|
|
|
cxt_size = I915_READ(GEN7_CXT_SIZE);
|
|
|
|
return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
|
|
|
|
PAGE_SIZE);
|
|
|
|
case 6:
|
|
|
|
cxt_size = I915_READ(CXT_SIZE);
|
|
|
|
return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
|
|
|
|
PAGE_SIZE);
|
|
|
|
case 5:
|
|
|
|
case 4:
|
|
|
|
case 3:
|
|
|
|
case 2:
|
|
|
|
/* For the special day when i810 gets merged. */
|
|
|
|
case 1:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
MISSING_CASE(class);
|
|
|
|
case VIDEO_DECODE_CLASS:
|
|
|
|
case VIDEO_ENHANCEMENT_CLASS:
|
|
|
|
case COPY_ENGINE_CLASS:
|
|
|
|
if (INTEL_GEN(dev_priv) < 8)
|
|
|
|
return 0;
|
|
|
|
return GEN8_LR_CONTEXT_OTHER_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-13 17:14:48 +00:00
|
|
|
static int
|
2016-07-13 15:03:40 +00:00
|
|
|
intel_engine_setup(struct drm_i915_private *dev_priv,
|
|
|
|
enum intel_engine_id id)
|
|
|
|
{
|
|
|
|
const struct engine_info *info = &intel_engines[id];
|
2017-04-10 14:34:32 +00:00
|
|
|
const struct engine_class_info *class_info;
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-13 17:14:48 +00:00
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
|
2017-04-10 14:34:32 +00:00
|
|
|
GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
|
|
|
|
class_info = &intel_engine_classes[info->class];
|
|
|
|
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-13 17:14:48 +00:00
|
|
|
GEM_BUG_ON(dev_priv->engine[id]);
|
|
|
|
engine = kzalloc(sizeof(*engine), GFP_KERNEL);
|
|
|
|
if (!engine)
|
|
|
|
return -ENOMEM;
|
2016-07-13 15:03:40 +00:00
|
|
|
|
|
|
|
engine->id = id;
|
|
|
|
engine->i915 = dev_priv;
|
2017-04-10 14:34:31 +00:00
|
|
|
WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
|
2017-04-10 14:34:32 +00:00
|
|
|
class_info->name, info->instance) >=
|
|
|
|
sizeof(engine->name));
|
2017-04-11 12:43:06 +00:00
|
|
|
engine->uabi_id = info->uabi_id;
|
2016-08-16 16:04:20 +00:00
|
|
|
engine->hw_id = engine->guc_id = info->hw_id;
|
2016-07-13 15:03:40 +00:00
|
|
|
engine->mmio_base = info->mmio_base;
|
|
|
|
engine->irq_shift = info->irq_shift;
|
2017-04-10 14:34:29 +00:00
|
|
|
engine->class = info->class;
|
|
|
|
engine->instance = info->instance;
|
2016-07-13 15:03:40 +00:00
|
|
|
|
2017-04-28 07:53:36 +00:00
|
|
|
engine->context_size = __intel_engine_context_size(dev_priv,
|
|
|
|
engine->class);
|
|
|
|
if (WARN_ON(engine->context_size > BIT(20)))
|
|
|
|
engine->context_size = 0;
|
|
|
|
|
2016-11-14 20:41:01 +00:00
|
|
|
/* Nothing to do here, execute in order of dependencies */
|
|
|
|
engine->schedule = NULL;
|
|
|
|
|
2017-03-13 02:47:11 +00:00
|
|
|
ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
|
|
|
|
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-13 17:14:48 +00:00
|
|
|
dev_priv->engine[id] = engine;
|
|
|
|
return 0;
|
2016-07-13 15:03:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2017-04-28 07:53:36 +00:00
|
|
|
* intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
|
2016-12-01 14:16:38 +00:00
|
|
|
* @dev_priv: i915 device private
|
2016-07-13 15:03:40 +00:00
|
|
|
*
|
|
|
|
* Return: non-zero if the initialization failed.
|
|
|
|
*/
|
2017-04-28 07:53:36 +00:00
|
|
|
int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
|
2016-07-13 15:03:40 +00:00
|
|
|
{
|
2016-08-10 15:22:10 +00:00
|
|
|
struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
|
2017-04-11 16:56:58 +00:00
|
|
|
const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-13 17:14:48 +00:00
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
2017-04-11 16:56:58 +00:00
|
|
|
unsigned int mask = 0;
|
2016-07-13 15:03:40 +00:00
|
|
|
unsigned int i;
|
2017-01-24 11:01:34 +00:00
|
|
|
int err;
|
2016-07-13 15:03:40 +00:00
|
|
|
|
2016-10-13 10:02:56 +00:00
|
|
|
WARN_ON(ring_mask == 0);
|
|
|
|
WARN_ON(ring_mask &
|
2016-07-13 15:03:40 +00:00
|
|
|
GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
|
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
|
|
|
|
if (!HAS_ENGINE(dev_priv, i))
|
|
|
|
continue;
|
|
|
|
|
2017-01-24 11:01:34 +00:00
|
|
|
err = intel_engine_setup(dev_priv, i);
|
|
|
|
if (err)
|
|
|
|
goto cleanup;
|
|
|
|
|
|
|
|
mask |= ENGINE_MASK(i);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Catch failures to update intel_engines table when the new engines
|
|
|
|
* are added to the driver by a warning and disabling the forgotten
|
|
|
|
* engines.
|
|
|
|
*/
|
|
|
|
if (WARN_ON(mask != ring_mask))
|
|
|
|
device_info->ring_mask = mask;
|
|
|
|
|
2017-04-11 16:56:58 +00:00
|
|
|
/* We always presume we have at least RCS available for later probing */
|
|
|
|
if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
|
|
|
|
err = -ENODEV;
|
|
|
|
goto cleanup;
|
|
|
|
}
|
|
|
|
|
2017-01-24 11:01:34 +00:00
|
|
|
device_info->num_rings = hweight32(mask);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
cleanup:
|
|
|
|
for_each_engine(engine, dev_priv, id)
|
|
|
|
kfree(engine);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2017-04-28 07:53:36 +00:00
|
|
|
* intel_engines_init() - init the Engine Command Streamers
|
2017-01-24 11:01:34 +00:00
|
|
|
* @dev_priv: i915 device private
|
|
|
|
*
|
|
|
|
* Return: non-zero if the initialization failed.
|
|
|
|
*/
|
|
|
|
int intel_engines_init(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id, err_id;
|
2017-06-16 13:03:38 +00:00
|
|
|
int err;
|
2017-01-24 11:01:34 +00:00
|
|
|
|
|
|
|
for_each_engine(engine, dev_priv, id) {
|
2017-04-10 14:34:32 +00:00
|
|
|
const struct engine_class_info *class_info =
|
|
|
|
&intel_engine_classes[engine->class];
|
2017-01-24 11:01:34 +00:00
|
|
|
int (*init)(struct intel_engine_cs *engine);
|
|
|
|
|
2016-07-13 15:03:40 +00:00
|
|
|
if (i915.enable_execlists)
|
2017-04-10 14:34:32 +00:00
|
|
|
init = class_info->init_execlists;
|
2016-07-13 15:03:40 +00:00
|
|
|
else
|
2017-04-10 14:34:32 +00:00
|
|
|
init = class_info->init_legacy;
|
2017-06-16 13:03:38 +00:00
|
|
|
|
|
|
|
err = -EINVAL;
|
|
|
|
err_id = id;
|
|
|
|
|
|
|
|
if (GEM_WARN_ON(!init))
|
|
|
|
goto cleanup;
|
2016-07-13 15:03:40 +00:00
|
|
|
|
2017-01-24 11:01:34 +00:00
|
|
|
err = init(engine);
|
2017-06-16 13:03:38 +00:00
|
|
|
if (err)
|
2016-07-13 15:03:40 +00:00
|
|
|
goto cleanup;
|
|
|
|
|
2017-03-16 17:13:03 +00:00
|
|
|
GEM_BUG_ON(!engine->submit_request);
|
2016-07-13 15:03:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
cleanup:
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-13 17:14:48 +00:00
|
|
|
for_each_engine(engine, dev_priv, id) {
|
2017-06-16 13:03:38 +00:00
|
|
|
if (id >= err_id) {
|
2017-01-24 11:01:34 +00:00
|
|
|
kfree(engine);
|
2017-06-16 13:03:38 +00:00
|
|
|
dev_priv->engine[id] = NULL;
|
|
|
|
} else {
|
2017-02-16 12:23:22 +00:00
|
|
|
dev_priv->gt.cleanup_engine(engine);
|
2017-06-16 13:03:38 +00:00
|
|
|
}
|
2016-07-13 15:03:40 +00:00
|
|
|
}
|
2017-01-24 11:01:34 +00:00
|
|
|
return err;
|
2016-07-13 15:03:40 +00:00
|
|
|
}
|
|
|
|
|
2016-10-28 12:58:46 +00:00
|
|
|
void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
|
2016-08-15 09:49:00 +00:00
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
|
|
|
|
/* Our semaphore implementation is strictly monotonic (i.e. we proceed
|
|
|
|
* so long as the semaphore value in the register/page is greater
|
|
|
|
* than the sync value), so whenever we reset the seqno,
|
|
|
|
* so long as we reset the tracking semaphore value to 0, it will
|
|
|
|
* always be before the next request's seqno. If we don't reset
|
|
|
|
* the semaphore value, then when the seqno moves backwards all
|
|
|
|
* future waits will complete instantly (causing rendering corruption).
|
|
|
|
*/
|
|
|
|
if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
|
|
|
|
I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
|
|
|
|
I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
|
|
|
|
if (HAS_VEBOX(dev_priv))
|
|
|
|
I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
|
|
|
|
}
|
2016-08-15 09:49:02 +00:00
|
|
|
if (dev_priv->semaphore) {
|
|
|
|
struct page *page = i915_vma_first_page(dev_priv->semaphore);
|
|
|
|
void *semaphores;
|
|
|
|
|
|
|
|
/* Semaphores are in noncoherent memory, flush to be safe */
|
2017-03-20 14:56:09 +00:00
|
|
|
semaphores = kmap_atomic(page);
|
2016-08-15 09:49:00 +00:00
|
|
|
memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
|
|
|
|
0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
|
2016-08-15 09:49:02 +00:00
|
|
|
drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
|
|
|
|
I915_NUM_ENGINES * gen8_semaphore_seqno_size);
|
2017-03-20 14:56:09 +00:00
|
|
|
kunmap_atomic(semaphores);
|
2016-08-15 09:49:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
|
2017-03-14 11:14:52 +00:00
|
|
|
clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
|
2016-10-28 12:58:46 +00:00
|
|
|
|
2016-08-15 09:49:00 +00:00
|
|
|
/* After manually advancing the seqno, fake the interrupt in case
|
|
|
|
* there are any waiters for that seqno.
|
|
|
|
*/
|
|
|
|
intel_engine_wakeup(engine);
|
2017-04-05 15:30:54 +00:00
|
|
|
|
|
|
|
GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
|
2016-08-15 09:49:00 +00:00
|
|
|
}
|
|
|
|
|
2016-10-28 12:58:46 +00:00
|
|
|
static void intel_engine_init_timeline(struct intel_engine_cs *engine)
|
2016-08-05 09:14:11 +00:00
|
|
|
{
|
2016-10-28 12:58:46 +00:00
|
|
|
engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
|
2016-08-05 09:14:11 +00:00
|
|
|
}
|
|
|
|
|
2016-07-13 15:03:41 +00:00
|
|
|
/**
|
|
|
|
* intel_engines_setup_common - setup engine state not requiring hw access
|
|
|
|
* @engine: Engine to setup.
|
|
|
|
*
|
|
|
|
* Initializes @engine@ structure members shared between legacy and execlists
|
|
|
|
* submission modes which do not require hardware access.
|
|
|
|
*
|
|
|
|
* Typically done early in the submission mode specific engine setup stage.
|
|
|
|
*/
|
|
|
|
void intel_engine_setup_common(struct intel_engine_cs *engine)
|
|
|
|
{
|
2016-11-14 20:41:03 +00:00
|
|
|
engine->execlist_queue = RB_ROOT;
|
|
|
|
engine->execlist_first = NULL;
|
2016-07-13 15:03:41 +00:00
|
|
|
|
2016-10-28 12:58:46 +00:00
|
|
|
intel_engine_init_timeline(engine);
|
2016-07-13 15:03:41 +00:00
|
|
|
intel_engine_init_hangcheck(engine);
|
2016-08-04 15:32:19 +00:00
|
|
|
i915_gem_batch_pool_init(engine, &engine->batch_pool);
|
2016-08-18 16:17:10 +00:00
|
|
|
|
|
|
|
intel_engine_init_cmd_parser(engine);
|
2016-07-13 15:03:41 +00:00
|
|
|
}
|
|
|
|
|
2016-08-15 09:48:59 +00:00
|
|
|
int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
|
|
|
|
{
|
|
|
|
struct drm_i915_gem_object *obj;
|
|
|
|
struct i915_vma *vma;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
WARN_ON(engine->scratch);
|
|
|
|
|
2016-12-01 14:16:36 +00:00
|
|
|
obj = i915_gem_object_create_stolen(engine->i915, size);
|
2016-08-15 09:48:59 +00:00
|
|
|
if (!obj)
|
2016-10-28 12:58:30 +00:00
|
|
|
obj = i915_gem_object_create_internal(engine->i915, size);
|
2016-08-15 09:48:59 +00:00
|
|
|
if (IS_ERR(obj)) {
|
|
|
|
DRM_ERROR("Failed to allocate scratch page\n");
|
|
|
|
return PTR_ERR(obj);
|
|
|
|
}
|
|
|
|
|
2017-01-16 15:21:30 +00:00
|
|
|
vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
|
2016-08-15 09:48:59 +00:00
|
|
|
if (IS_ERR(vma)) {
|
|
|
|
ret = PTR_ERR(vma);
|
|
|
|
goto err_unref;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
|
|
|
|
if (ret)
|
|
|
|
goto err_unref;
|
|
|
|
|
|
|
|
engine->scratch = vma;
|
2016-08-15 09:49:07 +00:00
|
|
|
DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
|
|
|
|
engine->name, i915_ggtt_offset(vma));
|
2016-08-15 09:48:59 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_unref:
|
|
|
|
i915_gem_object_put(obj);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
|
|
|
|
{
|
2016-08-15 09:49:05 +00:00
|
|
|
i915_vma_unpin_and_release(&engine->scratch);
|
2016-08-15 09:48:59 +00:00
|
|
|
}
|
|
|
|
|
2016-07-13 15:03:41 +00:00
|
|
|
/**
|
|
|
|
* intel_engines_init_common - initialize cengine state which might require hw access
|
|
|
|
* @engine: Engine to initialize.
|
|
|
|
*
|
|
|
|
* Initializes @engine@ structure members shared between legacy and execlists
|
|
|
|
* submission modes which do require hardware access.
|
|
|
|
*
|
|
|
|
* Typcally done at later stages of submission mode specific engine setup.
|
|
|
|
*
|
|
|
|
* Returns zero on success or an error code on failure.
|
|
|
|
*/
|
|
|
|
int intel_engine_init_common(struct intel_engine_cs *engine)
|
|
|
|
{
|
2017-05-04 09:33:08 +00:00
|
|
|
struct intel_ring *ring;
|
2016-07-13 15:03:41 +00:00
|
|
|
int ret;
|
|
|
|
|
2017-03-16 17:13:03 +00:00
|
|
|
engine->set_default_submission(engine);
|
|
|
|
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 15:37:20 +00:00
|
|
|
/* We may need to do things with the shrinker which
|
|
|
|
* require us to immediately switch back to the default
|
|
|
|
* context. This can cause a problem as pinning the
|
|
|
|
* default context also requires GTT space which may not
|
|
|
|
* be available. To avoid this we always pin the default
|
|
|
|
* context.
|
|
|
|
*/
|
2017-05-04 09:33:08 +00:00
|
|
|
ring = engine->context_pin(engine, engine->i915->kernel_context);
|
|
|
|
if (IS_ERR(ring))
|
|
|
|
return PTR_ERR(ring);
|
2016-07-13 15:03:41 +00:00
|
|
|
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 15:37:20 +00:00
|
|
|
ret = intel_engine_init_breadcrumbs(engine);
|
|
|
|
if (ret)
|
|
|
|
goto err_unpin;
|
|
|
|
|
2016-10-28 12:58:31 +00:00
|
|
|
ret = i915_gem_render_state_init(engine);
|
|
|
|
if (ret)
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 15:37:20 +00:00
|
|
|
goto err_unpin;
|
2016-10-28 12:58:31 +00:00
|
|
|
|
2016-08-18 16:17:10 +00:00
|
|
|
return 0;
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 15:37:20 +00:00
|
|
|
|
|
|
|
err_unpin:
|
|
|
|
engine->context_unpin(engine, engine->i915->kernel_context);
|
|
|
|
return ret;
|
2016-07-13 15:03:41 +00:00
|
|
|
}
|
2016-08-03 12:19:16 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* intel_engines_cleanup_common - cleans up the engine state created by
|
|
|
|
* the common initiailizers.
|
|
|
|
* @engine: Engine to cleanup.
|
|
|
|
*
|
|
|
|
* This cleans up everything created by the common helpers.
|
|
|
|
*/
|
|
|
|
void intel_engine_cleanup_common(struct intel_engine_cs *engine)
|
|
|
|
{
|
2016-08-15 09:48:59 +00:00
|
|
|
intel_engine_cleanup_scratch(engine);
|
|
|
|
|
2016-10-28 12:58:31 +00:00
|
|
|
i915_gem_render_state_fini(engine);
|
2016-08-03 12:19:16 +00:00
|
|
|
intel_engine_fini_breadcrumbs(engine);
|
2016-08-18 16:17:10 +00:00
|
|
|
intel_engine_cleanup_cmd_parser(engine);
|
2016-08-03 12:19:16 +00:00
|
|
|
i915_gem_batch_pool_fini(&engine->batch_pool);
|
drm/i915: Unify active context tracking between legacy/execlists/guc
The requests conversion introduced a nasty bug where we could generate a
new request in the middle of constructing a request if we needed to idle
the system in order to evict space for a context. The request to idle
would be executed (and waited upon) before the current one, creating a
minor havoc in the seqno accounting, as we will consider the current
request to already be completed (prior to deferred seqno assignment) but
ring->last_retired_head would have been updated and still could allow
us to overwrite the current request before execution.
We also employed two different mechanisms to track the active context
until it was switched out. The legacy method allowed for waiting upon an
active context (it could forcibly evict any vma, including context's),
but the execlists method took a step backwards by pinning the vma for
the entire active lifespan of the context (the only way to evict was to
idle the entire GPU, not individual contexts). However, to circumvent
the tricky issue of locking (i.e. we cannot take struct_mutex at the
time of i915_gem_request_submit(), where we would want to move the
previous context onto the active tracker and unpin it), we take the
execlists approach and keep the contexts pinned until retirement.
The benefit of the execlists approach, more important for execlists than
legacy, was the reduction in work in pinning the context for each
request - as the context was kept pinned until idle, it could short
circuit the pinning for all active contexts.
We introduce new engine vfuncs to pin and unpin the context
respectively. The context is pinned at the start of the request, and
only unpinned when the following request is retired (this ensures that
the context is idle and coherent in main memory before we unpin it). We
move the engine->last_context tracking into the retirement itself
(rather than during request submission) in order to allow the submission
to be reordered or unwound without undue difficultly.
And finally an ulterior motive for unifying context handling was to
prepare for mock requests.
v2: Rename to last_retired_context, split out legacy_context tracking
for MI_SET_CONTEXT.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161218153724.8439-3-chris@chris-wilson.co.uk
2016-12-18 15:37:20 +00:00
|
|
|
|
|
|
|
engine->context_unpin(engine, engine->i915->kernel_context);
|
2016-08-03 12:19:16 +00:00
|
|
|
}
|
2016-10-04 20:11:31 +00:00
|
|
|
|
|
|
|
u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
u64 acthd;
|
|
|
|
|
|
|
|
if (INTEL_GEN(dev_priv) >= 8)
|
|
|
|
acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
|
|
|
|
RING_ACTHD_UDW(engine->mmio_base));
|
|
|
|
else if (INTEL_GEN(dev_priv) >= 4)
|
|
|
|
acthd = I915_READ(RING_ACTHD(engine->mmio_base));
|
|
|
|
else
|
|
|
|
acthd = I915_READ(ACTHD);
|
|
|
|
|
|
|
|
return acthd;
|
|
|
|
}
|
|
|
|
|
|
|
|
u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
u64 bbaddr;
|
|
|
|
|
|
|
|
if (INTEL_GEN(dev_priv) >= 8)
|
|
|
|
bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
|
|
|
|
RING_BBADDR_UDW(engine->mmio_base));
|
|
|
|
else
|
|
|
|
bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
|
|
|
|
|
|
|
|
return bbaddr;
|
|
|
|
}
|
2016-10-12 09:05:17 +00:00
|
|
|
|
|
|
|
const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
|
|
|
|
{
|
|
|
|
switch (type) {
|
|
|
|
case I915_CACHE_NONE: return " uncached";
|
|
|
|
case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
|
|
|
|
case I915_CACHE_L3_LLC: return " L3+LLC";
|
|
|
|
case I915_CACHE_WT: return " WT";
|
|
|
|
default: return "";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline uint32_t
|
|
|
|
read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
|
|
|
|
int subslice, i915_reg_t reg)
|
|
|
|
{
|
|
|
|
uint32_t mcr;
|
|
|
|
uint32_t ret;
|
|
|
|
enum forcewake_domains fw_domains;
|
|
|
|
|
|
|
|
fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
|
|
|
|
FW_REG_READ);
|
|
|
|
fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
|
|
|
|
GEN8_MCR_SELECTOR,
|
|
|
|
FW_REG_READ | FW_REG_WRITE);
|
|
|
|
|
|
|
|
spin_lock_irq(&dev_priv->uncore.lock);
|
|
|
|
intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
|
|
|
|
|
|
|
|
mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
|
|
|
|
/*
|
|
|
|
* The HW expects the slice and sublice selectors to be reset to 0
|
|
|
|
* after reading out the registers.
|
|
|
|
*/
|
|
|
|
WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
|
|
|
|
mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
|
|
|
|
mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
|
|
|
|
I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
|
|
|
|
|
|
|
|
ret = I915_READ_FW(reg);
|
|
|
|
|
|
|
|
mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
|
|
|
|
I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
|
|
|
|
|
|
|
|
intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
|
|
|
|
spin_unlock_irq(&dev_priv->uncore.lock);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* NB: please notice the memset */
|
|
|
|
void intel_engine_get_instdone(struct intel_engine_cs *engine,
|
|
|
|
struct intel_instdone *instdone)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
u32 mmio_base = engine->mmio_base;
|
|
|
|
int slice;
|
|
|
|
int subslice;
|
|
|
|
|
|
|
|
memset(instdone, 0, sizeof(*instdone));
|
|
|
|
|
|
|
|
switch (INTEL_GEN(dev_priv)) {
|
|
|
|
default:
|
|
|
|
instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
|
|
|
|
|
|
|
|
if (engine->id != RCS)
|
|
|
|
break;
|
|
|
|
|
|
|
|
instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
|
|
|
|
for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
|
|
|
|
instdone->sampler[slice][subslice] =
|
|
|
|
read_subslice_reg(dev_priv, slice, subslice,
|
|
|
|
GEN7_SAMPLER_INSTDONE);
|
|
|
|
instdone->row[slice][subslice] =
|
|
|
|
read_subslice_reg(dev_priv, slice, subslice,
|
|
|
|
GEN7_ROW_INSTDONE);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case 7:
|
|
|
|
instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
|
|
|
|
|
|
|
|
if (engine->id != RCS)
|
|
|
|
break;
|
|
|
|
|
|
|
|
instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
|
|
|
|
instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
|
|
|
|
instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
|
|
|
|
|
|
|
|
break;
|
|
|
|
case 6:
|
|
|
|
case 5:
|
|
|
|
case 4:
|
|
|
|
instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
|
|
|
|
|
|
|
|
if (engine->id == RCS)
|
|
|
|
/* HACK: Using the wrong struct member */
|
|
|
|
instdone->slice_common = I915_READ(GEN4_INSTDONE1);
|
|
|
|
break;
|
|
|
|
case 3:
|
|
|
|
case 2:
|
|
|
|
instdone->instdone = I915_READ(GEN2_INSTDONE);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2017-02-13 17:15:14 +00:00
|
|
|
|
2017-02-16 12:23:23 +00:00
|
|
|
static int wa_add(struct drm_i915_private *dev_priv,
|
|
|
|
i915_reg_t addr,
|
|
|
|
const u32 mask, const u32 val)
|
|
|
|
{
|
|
|
|
const u32 idx = dev_priv->workarounds.count;
|
|
|
|
|
|
|
|
if (WARN_ON(idx >= I915_MAX_WA_REGS))
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
dev_priv->workarounds.reg[idx].addr = addr;
|
|
|
|
dev_priv->workarounds.reg[idx].value = val;
|
|
|
|
dev_priv->workarounds.reg[idx].mask = mask;
|
|
|
|
|
|
|
|
dev_priv->workarounds.count++;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define WA_REG(addr, mask, val) do { \
|
|
|
|
const int r = wa_add(dev_priv, (addr), (mask), (val)); \
|
|
|
|
if (r) \
|
|
|
|
return r; \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define WA_SET_BIT_MASKED(addr, mask) \
|
|
|
|
WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
|
|
|
|
|
|
|
|
#define WA_CLR_BIT_MASKED(addr, mask) \
|
|
|
|
WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
|
|
|
|
|
|
|
|
#define WA_SET_FIELD_MASKED(addr, mask, value) \
|
|
|
|
WA_REG(addr, mask, _MASKED_FIELD(mask, value))
|
|
|
|
|
|
|
|
#define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
|
|
|
|
#define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
|
|
|
|
|
|
|
|
#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
|
|
|
|
|
|
|
|
static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
|
|
|
|
i915_reg_t reg)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
struct i915_workarounds *wa = &dev_priv->workarounds;
|
|
|
|
const uint32_t index = wa->hw_whitelist_count[engine->id];
|
|
|
|
|
|
|
|
if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
WA_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
|
|
|
|
i915_mmio_reg_offset(reg));
|
|
|
|
wa->hw_whitelist_count[engine->id]++;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int gen8_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
|
|
|
|
WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
|
|
|
|
|
|
|
|
/* WaDisableAsyncFlipPerfMode:bdw,chv */
|
|
|
|
WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
|
|
|
|
|
|
|
|
/* WaDisablePartialInstShootdown:bdw,chv */
|
|
|
|
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
|
|
|
|
PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
|
|
|
|
|
|
|
|
/* Use Force Non-Coherent whenever executing a 3D context. This is a
|
|
|
|
* workaround for for a possible hang in the unlikely event a TLB
|
|
|
|
* invalidation occurs during a PSD flush.
|
|
|
|
*/
|
|
|
|
/* WaForceEnableNonCoherent:bdw,chv */
|
|
|
|
/* WaHdcDisableFetchWhenMasked:bdw,chv */
|
|
|
|
WA_SET_BIT_MASKED(HDC_CHICKEN0,
|
|
|
|
HDC_DONOT_FETCH_MEM_WHEN_MASKED |
|
|
|
|
HDC_FORCE_NON_COHERENT);
|
|
|
|
|
|
|
|
/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
|
|
|
|
* "The Hierarchical Z RAW Stall Optimization allows non-overlapping
|
|
|
|
* polygons in the same 8x4 pixel/sample area to be processed without
|
|
|
|
* stalling waiting for the earlier ones to write to Hierarchical Z
|
|
|
|
* buffer."
|
|
|
|
*
|
|
|
|
* This optimization is off by default for BDW and CHV; turn it on.
|
|
|
|
*/
|
|
|
|
WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
|
|
|
|
|
|
|
|
/* Wa4x4STCOptimizationDisable:bdw,chv */
|
|
|
|
WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* BSpec recommends 8x4 when MSAA is used,
|
|
|
|
* however in practice 16x4 seems fastest.
|
|
|
|
*
|
|
|
|
* Note that PS/WM thread counts depend on the WIZ hashing
|
|
|
|
* disable bit, which we don't touch here, but it's good
|
|
|
|
* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
|
|
|
|
*/
|
|
|
|
WA_SET_FIELD_MASKED(GEN7_GT_MODE,
|
|
|
|
GEN6_WIZ_HASHING_MASK,
|
|
|
|
GEN6_WIZ_HASHING_16x4);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int bdw_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen8_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
|
|
|
|
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
|
|
|
|
|
|
|
|
/* WaDisableDopClockGating:bdw
|
|
|
|
*
|
|
|
|
* Also see the related UCGTCL1 write in broadwell_init_clock_gating()
|
|
|
|
* to disable EUTC clock gating.
|
|
|
|
*/
|
|
|
|
WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
|
|
|
|
DOP_CLOCK_GATING_DISABLE);
|
|
|
|
|
|
|
|
WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
|
|
|
|
GEN8_SAMPLER_POWER_BYPASS_DIS);
|
|
|
|
|
|
|
|
WA_SET_BIT_MASKED(HDC_CHICKEN0,
|
|
|
|
/* WaForceContextSaveRestoreNonCoherent:bdw */
|
|
|
|
HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
|
|
|
|
/* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
|
|
|
|
(IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int chv_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen8_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* WaDisableThreadStallDopClockGating:chv */
|
|
|
|
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
|
|
|
|
|
|
|
|
/* Improve HiZ throughput on CHV. */
|
|
|
|
WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int gen9_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
|
|
|
|
GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
|
|
|
|
|
2017-06-19 21:21:47 +00:00
|
|
|
/* WaDisableKillLogic:bxt,skl,kbl */
|
|
|
|
if (!IS_COFFEELAKE(dev_priv))
|
|
|
|
I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
|
|
|
|
ECOCHK_DIS_TLB);
|
2017-02-16 12:23:23 +00:00
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
|
|
|
|
/* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
|
|
|
|
FLOW_CONTROL_ENABLE |
|
|
|
|
PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
|
|
|
|
|
|
|
|
/* Syncing dependencies between camera and graphics:skl,bxt,kbl */
|
2017-06-16 22:49:58 +00:00
|
|
|
if (!IS_COFFEELAKE(dev_priv))
|
|
|
|
WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
|
|
|
|
GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
|
2017-02-16 12:23:23 +00:00
|
|
|
|
|
|
|
/* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
|
|
|
|
WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
|
|
|
|
GEN9_DG_MIRROR_FIX_ENABLE);
|
|
|
|
|
|
|
|
/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
|
|
|
|
WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
|
|
|
|
GEN9_RHWO_OPTIMIZATION_DISABLE);
|
|
|
|
/*
|
|
|
|
* WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
|
|
|
|
* but we do that in per ctx batchbuffer as there is an issue
|
|
|
|
* with this register not getting restored on ctx restore
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
|
|
|
|
/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
|
2017-05-12 11:20:15 +00:00
|
|
|
GEN9_ENABLE_YV12_BUGFIX |
|
2017-02-16 12:23:23 +00:00
|
|
|
GEN9_ENABLE_GPGPU_PREEMPTION);
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
|
|
|
|
/* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
|
|
|
|
GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
|
|
|
|
GEN9_CCS_TLB_PREFETCH_ENABLE);
|
|
|
|
|
|
|
|
/* WaDisableMaskBasedCammingInRCC:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
|
|
|
|
WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
|
|
|
|
PIXEL_MASK_CAMMING_DISABLE);
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_SET_BIT_MASKED(HDC_CHICKEN0,
|
|
|
|
HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
|
|
|
|
HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
|
|
|
|
|
|
|
|
/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
|
|
|
|
* both tied to WaForceContextSaveRestoreNonCoherent
|
|
|
|
* in some hsds for skl. We keep the tie for all gen9. The
|
|
|
|
* documentation is a bit hazy and so we want to get common behaviour,
|
|
|
|
* even though there is no clear evidence we would need both on kbl/bxt.
|
|
|
|
* This area has been source of system hangs so we play it safe
|
|
|
|
* and mimic the skl regardless of what bspec says.
|
|
|
|
*
|
|
|
|
* Use Force Non-Coherent whenever executing a 3D context. This
|
|
|
|
* is a workaround for a possible hang in the unlikely event
|
|
|
|
* a TLB invalidation occurs during a PSD flush.
|
|
|
|
*/
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_SET_BIT_MASKED(HDC_CHICKEN0,
|
|
|
|
HDC_FORCE_NON_COHERENT);
|
|
|
|
|
2017-06-19 21:21:47 +00:00
|
|
|
/* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
|
|
|
|
I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
|
|
|
|
BDW_DISABLE_HDC_INVALIDATION);
|
2017-02-16 12:23:23 +00:00
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
if (IS_SKYLAKE(dev_priv) ||
|
|
|
|
IS_KABYLAKE(dev_priv) ||
|
2017-06-16 22:49:58 +00:00
|
|
|
IS_COFFEELAKE(dev_priv) ||
|
2017-02-16 12:23:23 +00:00
|
|
|
IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
|
|
|
|
WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
|
|
|
|
GEN8_SAMPLER_POWER_BYPASS_DIS);
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
|
|
|
|
GEN8_LQSC_FLUSH_COHERENT_LINES));
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
|
2017-02-16 12:23:23 +00:00
|
|
|
ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
u8 vals[3] = { 0, 0, 0 };
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
for (i = 0; i < 3; i++) {
|
|
|
|
u8 ss;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only consider slices where one, and only one, subslice has 7
|
|
|
|
* EUs
|
|
|
|
*/
|
|
|
|
if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* subslice_7eu[i] != 0 (because of the check above) and
|
|
|
|
* ss_max == 4 (maximum number of subslices possible per slice)
|
|
|
|
*
|
|
|
|
* -> 0 <= ss <= 3;
|
|
|
|
*/
|
|
|
|
ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
|
|
|
|
vals[i] = 3 - ss;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Tune IZ hashing. See intel_device_info_runtime_init() */
|
|
|
|
WA_SET_FIELD_MASKED(GEN7_GT_MODE,
|
|
|
|
GEN9_IZ_HASHING_MASK(2) |
|
|
|
|
GEN9_IZ_HASHING_MASK(1) |
|
|
|
|
GEN9_IZ_HASHING_MASK(0),
|
|
|
|
GEN9_IZ_HASHING(2, vals[2]) |
|
|
|
|
GEN9_IZ_HASHING(1, vals[1]) |
|
|
|
|
GEN9_IZ_HASHING(0, vals[0]));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int skl_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen9_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Actual WA is to disable percontext preemption granularity control
|
|
|
|
* until D0 which is the default case so this is equivalent to
|
|
|
|
* !WaDisablePerCtxtPreemptionGranularityControl:skl
|
|
|
|
*/
|
|
|
|
I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
|
|
|
|
_MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
|
|
|
|
|
|
|
|
/* WaEnableGapsTsvCreditFix:skl */
|
|
|
|
I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
|
|
|
|
GEN9_GAPS_TSV_CREDIT_DISABLE));
|
|
|
|
|
|
|
|
/* WaDisableGafsUnitClkGating:skl */
|
|
|
|
WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
|
|
|
|
|
|
|
|
/* WaInPlaceDecompressionHang:skl */
|
|
|
|
if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
|
|
|
|
WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
|
|
|
|
GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
|
|
|
|
|
|
|
|
/* WaDisableLSQCROPERFforOCL:skl */
|
|
|
|
ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
return skl_tune_iz_hashing(engine);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int bxt_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen9_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* WaStoreMultiplePTEenable:bxt */
|
|
|
|
/* This is a requirement according to Hardware specification */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
|
|
|
|
I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
|
|
|
|
|
|
|
|
/* WaSetClckGatingDisableMedia:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
|
|
|
|
I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
|
|
|
|
~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
|
|
|
|
}
|
|
|
|
|
|
|
|
/* WaDisableThreadStallDopClockGating:bxt */
|
|
|
|
WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
|
|
|
|
STALL_DOP_GATING_DISABLE);
|
|
|
|
|
|
|
|
/* WaDisablePooledEuLoadBalancingFix:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
|
|
|
|
WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2,
|
|
|
|
GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* WaDisableSbeCacheDispatchPortSharing:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
|
|
|
|
WA_SET_BIT_MASKED(
|
|
|
|
GEN7_HALF_SLICE_CHICKEN1,
|
|
|
|
GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
|
|
|
|
/* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
|
|
|
|
/* WaDisableObjectLevelPreemtionForInstanceId:bxt */
|
|
|
|
/* WaDisableLSQCROPERFforOCL:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
|
|
|
|
ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* WaProgramL3SqcReg1DefaultForPerf:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
|
|
|
|
I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
|
|
|
|
L3_HIGH_PRIO_CREDITS(2));
|
|
|
|
|
|
|
|
/* WaToEnableHwFixForPushConstHWBug:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
|
|
|
|
WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
|
|
|
|
GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
|
|
|
|
|
|
|
|
/* WaInPlaceDecompressionHang:bxt */
|
|
|
|
if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
|
|
|
|
WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
|
|
|
|
GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int kbl_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen9_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* WaEnableGapsTsvCreditFix:kbl */
|
|
|
|
I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
|
|
|
|
GEN9_GAPS_TSV_CREDIT_DISABLE));
|
|
|
|
|
|
|
|
/* WaDisableDynamicCreditSharing:kbl */
|
|
|
|
if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
|
|
|
|
WA_SET_BIT(GAMT_CHKN_BIT_REG,
|
|
|
|
GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
|
|
|
|
|
|
|
|
/* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
|
|
|
|
if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
|
|
|
|
WA_SET_BIT_MASKED(HDC_CHICKEN0,
|
|
|
|
HDC_FENCE_DEST_SLM_DISABLE);
|
|
|
|
|
|
|
|
/* WaToEnableHwFixForPushConstHWBug:kbl */
|
|
|
|
if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
|
|
|
|
WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
|
|
|
|
GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
|
|
|
|
|
|
|
|
/* WaDisableGafsUnitClkGating:kbl */
|
|
|
|
WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
|
|
|
|
|
|
|
|
/* WaDisableSbeCacheDispatchPortSharing:kbl */
|
|
|
|
WA_SET_BIT_MASKED(
|
|
|
|
GEN7_HALF_SLICE_CHICKEN1,
|
|
|
|
GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
|
|
|
|
|
|
|
|
/* WaInPlaceDecompressionHang:kbl */
|
|
|
|
WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
|
|
|
|
GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
|
|
|
|
|
|
|
|
/* WaDisableLSQCROPERFforOCL:kbl */
|
|
|
|
ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int glk_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen9_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* WaToEnableHwFixForPushConstHWBug:glk */
|
|
|
|
WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
|
|
|
|
GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-06-16 22:49:58 +00:00
|
|
|
static int cfl_init_workarounds(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = gen9_init_workarounds(engine);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* WaEnableGapsTsvCreditFix:cfl */
|
|
|
|
I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
|
|
|
|
GEN9_GAPS_TSV_CREDIT_DISABLE));
|
|
|
|
|
|
|
|
/* WaToEnableHwFixForPushConstHWBug:cfl */
|
|
|
|
WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
|
|
|
|
GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
|
|
|
|
|
|
|
|
/* WaDisableGafsUnitClkGating:cfl */
|
|
|
|
WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
|
|
|
|
|
|
|
|
/* WaDisableSbeCacheDispatchPortSharing:cfl */
|
|
|
|
WA_SET_BIT_MASKED(
|
|
|
|
GEN7_HALF_SLICE_CHICKEN1,
|
|
|
|
GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
|
|
|
|
|
|
|
|
/* WaInPlaceDecompressionHang:cfl */
|
|
|
|
WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
|
|
|
|
GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-02-16 12:23:23 +00:00
|
|
|
int init_workarounds_ring(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
2017-03-01 12:11:31 +00:00
|
|
|
int err;
|
2017-02-16 12:23:23 +00:00
|
|
|
|
|
|
|
WARN_ON(engine->id != RCS);
|
|
|
|
|
|
|
|
dev_priv->workarounds.count = 0;
|
2017-03-01 12:11:31 +00:00
|
|
|
dev_priv->workarounds.hw_whitelist_count[engine->id] = 0;
|
2017-02-16 12:23:23 +00:00
|
|
|
|
|
|
|
if (IS_BROADWELL(dev_priv))
|
2017-03-01 12:11:31 +00:00
|
|
|
err = bdw_init_workarounds(engine);
|
|
|
|
else if (IS_CHERRYVIEW(dev_priv))
|
|
|
|
err = chv_init_workarounds(engine);
|
|
|
|
else if (IS_SKYLAKE(dev_priv))
|
|
|
|
err = skl_init_workarounds(engine);
|
|
|
|
else if (IS_BROXTON(dev_priv))
|
|
|
|
err = bxt_init_workarounds(engine);
|
|
|
|
else if (IS_KABYLAKE(dev_priv))
|
|
|
|
err = kbl_init_workarounds(engine);
|
|
|
|
else if (IS_GEMINILAKE(dev_priv))
|
|
|
|
err = glk_init_workarounds(engine);
|
2017-06-16 22:49:58 +00:00
|
|
|
else if (IS_COFFEELAKE(dev_priv))
|
|
|
|
err = cfl_init_workarounds(engine);
|
2017-03-01 12:11:31 +00:00
|
|
|
else
|
|
|
|
err = 0;
|
|
|
|
if (err)
|
|
|
|
return err;
|
2017-02-16 12:23:23 +00:00
|
|
|
|
2017-03-01 12:11:31 +00:00
|
|
|
DRM_DEBUG_DRIVER("%s: Number of context specific w/a: %d\n",
|
|
|
|
engine->name, dev_priv->workarounds.count);
|
2017-02-16 12:23:23 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
|
|
|
|
{
|
|
|
|
struct i915_workarounds *w = &req->i915->workarounds;
|
|
|
|
u32 *cs;
|
|
|
|
int ret, i;
|
|
|
|
|
|
|
|
if (w->count == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
ret = req->engine->emit_flush(req, EMIT_BARRIER);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
cs = intel_ring_begin(req, (w->count * 2 + 2));
|
|
|
|
if (IS_ERR(cs))
|
|
|
|
return PTR_ERR(cs);
|
|
|
|
|
|
|
|
*cs++ = MI_LOAD_REGISTER_IMM(w->count);
|
|
|
|
for (i = 0; i < w->count; i++) {
|
|
|
|
*cs++ = i915_mmio_reg_offset(w->reg[i].addr);
|
|
|
|
*cs++ = w->reg[i].value;
|
|
|
|
}
|
|
|
|
*cs++ = MI_NOOP;
|
|
|
|
|
|
|
|
intel_ring_advance(req, cs);
|
|
|
|
|
|
|
|
ret = req->engine->emit_flush(req, EMIT_BARRIER);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-05-30 12:13:33 +00:00
|
|
|
static bool ring_is_idle(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
bool idle = true;
|
|
|
|
|
|
|
|
intel_runtime_pm_get(dev_priv);
|
|
|
|
|
2017-05-30 12:13:34 +00:00
|
|
|
/* First check that no commands are left in the ring */
|
|
|
|
if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
|
|
|
|
(I915_READ_TAIL(engine) & TAIL_ADDR))
|
|
|
|
idle = false;
|
|
|
|
|
2017-05-30 12:13:33 +00:00
|
|
|
/* No bit for gen2, so assume the CS parser is idle */
|
|
|
|
if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
|
|
|
|
idle = false;
|
|
|
|
|
|
|
|
intel_runtime_pm_put(dev_priv);
|
|
|
|
|
|
|
|
return idle;
|
|
|
|
}
|
|
|
|
|
2017-03-03 12:19:46 +00:00
|
|
|
/**
|
|
|
|
* intel_engine_is_idle() - Report if the engine has finished process all work
|
|
|
|
* @engine: the intel_engine_cs
|
|
|
|
*
|
|
|
|
* Return true if there are no requests pending, nothing left to be submitted
|
|
|
|
* to hardware, and that the engine is idle.
|
|
|
|
*/
|
|
|
|
bool intel_engine_is_idle(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = engine->i915;
|
|
|
|
|
2017-04-11 19:00:42 +00:00
|
|
|
/* More white lies, if wedged, hw state is inconsistent */
|
|
|
|
if (i915_terminally_wedged(&dev_priv->gpu_error))
|
|
|
|
return true;
|
|
|
|
|
2017-03-03 12:19:46 +00:00
|
|
|
/* Any inflight/incomplete requests? */
|
|
|
|
if (!i915_seqno_passed(intel_engine_get_seqno(engine),
|
|
|
|
intel_engine_last_submit(engine)))
|
|
|
|
return false;
|
|
|
|
|
2017-04-11 23:44:26 +00:00
|
|
|
if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock))
|
|
|
|
return true;
|
|
|
|
|
2017-03-03 12:19:46 +00:00
|
|
|
/* Interrupt/tasklet pending? */
|
|
|
|
if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Both ports drained, no more ELSP submission? */
|
2017-05-17 12:10:00 +00:00
|
|
|
if (port_request(&engine->execlist_port[0]))
|
2017-03-03 12:19:46 +00:00
|
|
|
return false;
|
|
|
|
|
2017-07-21 12:32:24 +00:00
|
|
|
/* ELSP is empty, but there are ready requests? */
|
|
|
|
if (READ_ONCE(engine->execlist_first))
|
|
|
|
return false;
|
|
|
|
|
2017-03-03 12:19:46 +00:00
|
|
|
/* Ring stopped? */
|
2017-05-30 12:13:33 +00:00
|
|
|
if (!ring_is_idle(engine))
|
2017-03-03 12:19:46 +00:00
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-03-03 12:19:47 +00:00
|
|
|
bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
|
2017-03-30 14:50:37 +00:00
|
|
|
if (READ_ONCE(dev_priv->gt.active_requests))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* If the driver is wedged, HW state may be very inconsistent and
|
|
|
|
* report that it is still busy, even though we have stopped using it.
|
|
|
|
*/
|
|
|
|
if (i915_terminally_wedged(&dev_priv->gpu_error))
|
|
|
|
return true;
|
|
|
|
|
2017-03-03 12:19:47 +00:00
|
|
|
for_each_engine(engine, dev_priv, id) {
|
|
|
|
if (!intel_engine_is_idle(engine))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2017-03-16 17:13:03 +00:00
|
|
|
void intel_engines_reset_default_submission(struct drm_i915_private *i915)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
|
|
|
|
for_each_engine(engine, i915, id)
|
|
|
|
engine->set_default_submission(engine);
|
|
|
|
}
|
|
|
|
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 12:10:03 +00:00
|
|
|
void intel_engines_mark_idle(struct drm_i915_private *i915)
|
|
|
|
{
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
|
|
|
|
for_each_engine(engine, i915, id) {
|
|
|
|
intel_engine_disarm_breadcrumbs(engine);
|
|
|
|
i915_gem_batch_pool_fini(&engine->batch_pool);
|
2017-06-27 15:25:10 +00:00
|
|
|
tasklet_kill(&engine->irq_tasklet);
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 12:10:03 +00:00
|
|
|
engine->no_priolist = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-13 17:15:14 +00:00
|
|
|
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
|
|
|
|
#include "selftests/mock_engine.c"
|
|
|
|
#endif
|