From 78b1a6010b46a69bcd47b723a80f92693f26d17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michel=20D=C3=A4nzer?= Date: Tue, 18 Nov 2014 18:00:08 +0900 Subject: [PATCH 01/22] drm/radeon: Use cursor_set2 hook for enabling / disabling the HW cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cursor_set2 hook provides the cursor hotspot position within the cursor image. When the hotspot position changes, we can adjust the cursor position such that the hotspot doesn't move on the screen. This prevents the cursor from appearing to intermittently jump around on the screen when the position of the hotspot within the cursor image changes. Reviewed-by: Alex Deucher Signed-off-by: Michel Dänzer Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_cursor.c | 51 +++++++++++++++++++------ drivers/gpu/drm/radeon/radeon_display.c | 2 +- drivers/gpu/drm/radeon/radeon_mode.h | 16 +++++--- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c index 9630e8d95fb4..fd4bddfd67d6 100644 --- a/drivers/gpu/drm/radeon/radeon_cursor.c +++ b/drivers/gpu/drm/radeon/radeon_cursor.c @@ -117,8 +117,10 @@ static void radeon_show_cursor(struct drm_crtc *crtc) } } +static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y); + static void radeon_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj, - uint64_t gpu_addr) + uint64_t gpu_addr, int hot_x, int hot_y) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; @@ -142,13 +144,28 @@ static void radeon_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj, /* offset is from DISP(2)_BASE_ADDRESS */ WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, radeon_crtc->legacy_cursor_offset); } + + if (hot_x != radeon_crtc->cursor_hot_x || + hot_y != radeon_crtc->cursor_hot_y) { + int x, y; + + x = radeon_crtc->cursor_x + radeon_crtc->cursor_hot_x - hot_x; + y = radeon_crtc->cursor_y + radeon_crtc->cursor_hot_y - hot_y; + + radeon_cursor_move_locked(crtc, x, y); + + radeon_crtc->cursor_hot_x = hot_x; + radeon_crtc->cursor_hot_y = hot_y; + } } -int radeon_crtc_cursor_set(struct drm_crtc *crtc, - struct drm_file *file_priv, - uint32_t handle, - uint32_t width, - uint32_t height) +int radeon_crtc_cursor_set2(struct drm_crtc *crtc, + struct drm_file *file_priv, + uint32_t handle, + uint32_t width, + uint32_t height, + int32_t hot_x, + int32_t hot_y) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; @@ -192,7 +209,7 @@ int radeon_crtc_cursor_set(struct drm_crtc *crtc, radeon_crtc->cursor_height = height; radeon_lock_cursor(crtc, true); - radeon_set_cursor(crtc, obj, gpu_addr); + radeon_set_cursor(crtc, obj, gpu_addr, hot_x, hot_y); radeon_show_cursor(crtc); radeon_lock_cursor(crtc, false); @@ -215,8 +232,7 @@ fail: return ret; } -int radeon_crtc_cursor_move(struct drm_crtc *crtc, - int x, int y) +static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) { struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); struct radeon_device *rdev = crtc->dev->dev_private; @@ -281,7 +297,6 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc, } } - radeon_lock_cursor(crtc, true); if (ASIC_IS_DCE4(rdev)) { WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); @@ -308,7 +323,21 @@ int radeon_crtc_cursor_move(struct drm_crtc *crtc, WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, (radeon_crtc->legacy_cursor_offset + (yorigin * 256))); } - radeon_lock_cursor(crtc, false); + + radeon_crtc->cursor_x = x; + radeon_crtc->cursor_y = y; return 0; } + +int radeon_crtc_cursor_move(struct drm_crtc *crtc, + int x, int y) +{ + int ret; + + radeon_lock_cursor(crtc, true); + ret = radeon_cursor_move_locked(crtc, x, y); + radeon_lock_cursor(crtc, false); + + return ret; +} diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c index f1b0fa1285bb..102116902a07 100644 --- a/drivers/gpu/drm/radeon/radeon_display.c +++ b/drivers/gpu/drm/radeon/radeon_display.c @@ -635,7 +635,7 @@ radeon_crtc_set_config(struct drm_mode_set *set) return ret; } static const struct drm_crtc_funcs radeon_crtc_funcs = { - .cursor_set = radeon_crtc_cursor_set, + .cursor_set2 = radeon_crtc_cursor_set2, .cursor_move = radeon_crtc_cursor_move, .gamma_set = radeon_crtc_gamma_set, .set_config = radeon_crtc_set_config, diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h index 64eba7ebb354..f3d87cdd5c9d 100644 --- a/drivers/gpu/drm/radeon/radeon_mode.h +++ b/drivers/gpu/drm/radeon/radeon_mode.h @@ -321,6 +321,10 @@ struct radeon_crtc { uint32_t crtc_offset; struct drm_gem_object *cursor_bo; uint64_t cursor_addr; + int cursor_x; + int cursor_y; + int cursor_hot_x; + int cursor_hot_y; int cursor_width; int cursor_height; int max_cursor_width; @@ -805,11 +809,13 @@ extern int radeon_crtc_set_base_atomic(struct drm_crtc *crtc, extern int radeon_crtc_do_set_base(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y, int atomic); -extern int radeon_crtc_cursor_set(struct drm_crtc *crtc, - struct drm_file *file_priv, - uint32_t handle, - uint32_t width, - uint32_t height); +extern int radeon_crtc_cursor_set2(struct drm_crtc *crtc, + struct drm_file *file_priv, + uint32_t handle, + uint32_t width, + uint32_t height, + int32_t hot_x, + int32_t hot_y); extern int radeon_crtc_cursor_move(struct drm_crtc *crtc, int x, int y); From 3feba08d79c32777a845c3c8a4ab93092bdf6f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michel=20D=C3=A4nzer?= Date: Tue, 18 Nov 2014 18:00:09 +0900 Subject: [PATCH 02/22] drm/radeon: Move radeon_cursor_move(_locked) to replace forward declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No functional change. Signed-off-by: Michel Dänzer Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_cursor.c | 220 ++++++++++++------------- 1 file changed, 109 insertions(+), 111 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_cursor.c b/drivers/gpu/drm/radeon/radeon_cursor.c index fd4bddfd67d6..85f38ee11888 100644 --- a/drivers/gpu/drm/radeon/radeon_cursor.c +++ b/drivers/gpu/drm/radeon/radeon_cursor.c @@ -117,7 +117,115 @@ static void radeon_show_cursor(struct drm_crtc *crtc) } } -static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y); +static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) +{ + struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); + struct radeon_device *rdev = crtc->dev->dev_private; + int xorigin = 0, yorigin = 0; + int w = radeon_crtc->cursor_width; + + if (ASIC_IS_AVIVO(rdev)) { + /* avivo cursor are offset into the total surface */ + x += crtc->x; + y += crtc->y; + } + DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y); + + if (x < 0) { + xorigin = min(-x, radeon_crtc->max_cursor_width - 1); + x = 0; + } + if (y < 0) { + yorigin = min(-y, radeon_crtc->max_cursor_height - 1); + y = 0; + } + + /* fixed on DCE6 and newer */ + if (ASIC_IS_AVIVO(rdev) && !ASIC_IS_DCE6(rdev)) { + int i = 0; + struct drm_crtc *crtc_p; + + /* + * avivo cursor image can't end on 128 pixel boundary or + * go past the end of the frame if both crtcs are enabled + * + * NOTE: It is safe to access crtc->enabled of other crtcs + * without holding either the mode_config lock or the other + * crtc's lock as long as write access to this flag _always_ + * grabs all locks. + */ + list_for_each_entry(crtc_p, &crtc->dev->mode_config.crtc_list, head) { + if (crtc_p->enabled) + i++; + } + if (i > 1) { + int cursor_end, frame_end; + + cursor_end = x - xorigin + w; + frame_end = crtc->x + crtc->mode.crtc_hdisplay; + if (cursor_end >= frame_end) { + w = w - (cursor_end - frame_end); + if (!(frame_end & 0x7f)) + w--; + } else { + if (!(cursor_end & 0x7f)) + w--; + } + if (w <= 0) { + w = 1; + cursor_end = x - xorigin + w; + if (!(cursor_end & 0x7f)) { + x--; + WARN_ON_ONCE(x < 0); + } + } + } + } + + if (ASIC_IS_DCE4(rdev)) { + WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); + WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); + WREG32(EVERGREEN_CUR_SIZE + radeon_crtc->crtc_offset, + ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); + } else if (ASIC_IS_AVIVO(rdev)) { + WREG32(AVIVO_D1CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); + WREG32(AVIVO_D1CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); + WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset, + ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); + } else { + if (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN) + y *= 2; + + WREG32(RADEON_CUR_HORZ_VERT_OFF + radeon_crtc->crtc_offset, + (RADEON_CUR_LOCK + | (xorigin << 16) + | yorigin)); + WREG32(RADEON_CUR_HORZ_VERT_POSN + radeon_crtc->crtc_offset, + (RADEON_CUR_LOCK + | (x << 16) + | y)); + /* offset is from DISP(2)_BASE_ADDRESS */ + WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, (radeon_crtc->legacy_cursor_offset + + (yorigin * 256))); + } + + radeon_crtc->cursor_x = x; + radeon_crtc->cursor_y = y; + + return 0; +} + +int radeon_crtc_cursor_move(struct drm_crtc *crtc, + int x, int y) +{ + int ret; + + radeon_lock_cursor(crtc, true); + ret = radeon_cursor_move_locked(crtc, x, y); + radeon_lock_cursor(crtc, false); + + return ret; +} static void radeon_set_cursor(struct drm_crtc *crtc, struct drm_gem_object *obj, uint64_t gpu_addr, int hot_x, int hot_y) @@ -231,113 +339,3 @@ fail: return ret; } - -static int radeon_cursor_move_locked(struct drm_crtc *crtc, int x, int y) -{ - struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc); - struct radeon_device *rdev = crtc->dev->dev_private; - int xorigin = 0, yorigin = 0; - int w = radeon_crtc->cursor_width; - - if (ASIC_IS_AVIVO(rdev)) { - /* avivo cursor are offset into the total surface */ - x += crtc->x; - y += crtc->y; - } - DRM_DEBUG("x %d y %d c->x %d c->y %d\n", x, y, crtc->x, crtc->y); - - if (x < 0) { - xorigin = min(-x, radeon_crtc->max_cursor_width - 1); - x = 0; - } - if (y < 0) { - yorigin = min(-y, radeon_crtc->max_cursor_height - 1); - y = 0; - } - - /* fixed on DCE6 and newer */ - if (ASIC_IS_AVIVO(rdev) && !ASIC_IS_DCE6(rdev)) { - int i = 0; - struct drm_crtc *crtc_p; - - /* - * avivo cursor image can't end on 128 pixel boundary or - * go past the end of the frame if both crtcs are enabled - * - * NOTE: It is safe to access crtc->enabled of other crtcs - * without holding either the mode_config lock or the other - * crtc's lock as long as write access to this flag _always_ - * grabs all locks. - */ - list_for_each_entry(crtc_p, &crtc->dev->mode_config.crtc_list, head) { - if (crtc_p->enabled) - i++; - } - if (i > 1) { - int cursor_end, frame_end; - - cursor_end = x - xorigin + w; - frame_end = crtc->x + crtc->mode.crtc_hdisplay; - if (cursor_end >= frame_end) { - w = w - (cursor_end - frame_end); - if (!(frame_end & 0x7f)) - w--; - } else { - if (!(cursor_end & 0x7f)) - w--; - } - if (w <= 0) { - w = 1; - cursor_end = x - xorigin + w; - if (!(cursor_end & 0x7f)) { - x--; - WARN_ON_ONCE(x < 0); - } - } - } - } - - if (ASIC_IS_DCE4(rdev)) { - WREG32(EVERGREEN_CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); - WREG32(EVERGREEN_CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); - WREG32(EVERGREEN_CUR_SIZE + radeon_crtc->crtc_offset, - ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); - } else if (ASIC_IS_AVIVO(rdev)) { - WREG32(AVIVO_D1CUR_POSITION + radeon_crtc->crtc_offset, (x << 16) | y); - WREG32(AVIVO_D1CUR_HOT_SPOT + radeon_crtc->crtc_offset, (xorigin << 16) | yorigin); - WREG32(AVIVO_D1CUR_SIZE + radeon_crtc->crtc_offset, - ((w - 1) << 16) | (radeon_crtc->cursor_height - 1)); - } else { - if (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN) - y *= 2; - - WREG32(RADEON_CUR_HORZ_VERT_OFF + radeon_crtc->crtc_offset, - (RADEON_CUR_LOCK - | (xorigin << 16) - | yorigin)); - WREG32(RADEON_CUR_HORZ_VERT_POSN + radeon_crtc->crtc_offset, - (RADEON_CUR_LOCK - | (x << 16) - | y)); - /* offset is from DISP(2)_BASE_ADDRESS */ - WREG32(RADEON_CUR_OFFSET + radeon_crtc->crtc_offset, (radeon_crtc->legacy_cursor_offset + - (yorigin * 256))); - } - - radeon_crtc->cursor_x = x; - radeon_crtc->cursor_y = y; - - return 0; -} - -int radeon_crtc_cursor_move(struct drm_crtc *crtc, - int x, int y) -{ - int ret; - - radeon_lock_cursor(crtc, true); - ret = radeon_cursor_move_locked(crtc, x, y); - radeon_lock_cursor(crtc, false); - - return ret; -} From 4bb62c95a7e781a238b2ab374f34b1bf91e01ddc Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 17 Nov 2014 15:08:17 -0500 Subject: [PATCH 03/22] drm/radeon: work around a hw bug in MGCG on CIK Always need to set bit 0 of RLC_CGTT_MGCG_OVERRIDE to avoid unreliable doorbell updates in some cases. Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/radeon/cik.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index bce73b6203ac..cef4cb7e5438 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -6344,6 +6344,7 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) } orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); + data |= 0x00000001; data &= 0xfffffffd; if (orig != data) WREG32(RLC_CGTT_MGCG_OVERRIDE, data); @@ -6377,7 +6378,7 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) } } else { orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); - data |= 0x00000002; + data |= 0x00000003; if (orig != data) WREG32(RLC_CGTT_MGCG_OVERRIDE, data); From 39471ad39de827657e6ab69da96496eb0943295e Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Sun, 14 Sep 2014 21:14:14 -0400 Subject: [PATCH 04/22] drm/radeon/dpm: add smc fan control for SI (v2) Enable smc fan control for SI boards. Should reduce the fan noise on systems with a higher default fan profile. v2: disable by default, add rpm controls bug: https://bugs.freedesktop.org/show_bug.cgi?id=73338 Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ppsmc.h | 5 + drivers/gpu/drm/radeon/r600_dpm.h | 3 + drivers/gpu/drm/radeon/si_dpm.c | 330 +++++++++++++++++++++++++- drivers/gpu/drm/radeon/si_dpm.h | 5 + drivers/gpu/drm/radeon/sid.h | 40 +++- drivers/gpu/drm/radeon/sislands_smc.h | 25 ++ 6 files changed, 401 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/radeon/ppsmc.h b/drivers/gpu/drm/radeon/ppsmc.h index 11c0e4d5c0bf..0c4eaa60b6ca 100644 --- a/drivers/gpu/drm/radeon/ppsmc.h +++ b/drivers/gpu/drm/radeon/ppsmc.h @@ -56,6 +56,9 @@ #define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20 #define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS 0x40 +#define FDO_MODE_HARDWARE 0 +#define FDO_MODE_PIECE_WISE_LINEAR 1 + #define PPSMC_Result_OK ((uint8_t)0x01) #define PPSMC_Result_Failed ((uint8_t)0xFF) @@ -79,6 +82,8 @@ typedef uint8_t PPSMC_Result; #define PPSMC_MSG_DisableCac ((uint8_t)0x54) #define PPSMC_TDPClampingActive ((uint8_t)0x59) #define PPSMC_TDPClampingInactive ((uint8_t)0x5A) +#define PPSMC_StartFanControl ((uint8_t)0x5B) +#define PPSMC_StopFanControl ((uint8_t)0x5C) #define PPSMC_MSG_NoDisplay ((uint8_t)0x5D) #define PPSMC_MSG_HasDisplay ((uint8_t)0x5E) #define PPSMC_MSG_UVDPowerOFF ((uint8_t)0x60) diff --git a/drivers/gpu/drm/radeon/r600_dpm.h b/drivers/gpu/drm/radeon/r600_dpm.h index 46b9d2a03018..bd499d749bc9 100644 --- a/drivers/gpu/drm/radeon/r600_dpm.h +++ b/drivers/gpu/drm/radeon/r600_dpm.h @@ -96,6 +96,9 @@ #define R600_TEMP_RANGE_MIN (90 * 1000) #define R600_TEMP_RANGE_MAX (120 * 1000) +#define FDO_PWM_MODE_STATIC 1 +#define FDO_PWM_MODE_STATIC_RPM 5 + enum r600_power_level { R600_POWER_LEVEL_LOW = 0, R600_POWER_LEVEL_MEDIUM = 1, diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index b59e1d6b27ab..cf4c420b5572 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -3396,6 +3396,15 @@ static int si_process_firmware_header(struct radeon_device *rdev) si_pi->mc_reg_table_start = tmp; + ret = si_read_smc_sram_dword(rdev, + SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + + SISLANDS_SMC_FIRMWARE_HEADER_fanTable, + &tmp, si_pi->sram_end); + if (ret) + return ret; + + si_pi->fan_table_start = tmp; + ret = si_read_smc_sram_dword(rdev, SISLANDS_SMC_FIRMWARE_HEADER_LOCATION + SISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable, @@ -5825,20 +5834,20 @@ static int si_thermal_enable_alert(struct radeon_device *rdev, if (enable) { PPSMC_Result result; - thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; - rdev->irq.dpm_thermal = true; + thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + WREG32(CG_THERMAL_INT, thermal_int); + rdev->irq.dpm_thermal = false; result = si_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt); if (result != PPSMC_Result_OK) { DRM_DEBUG_KMS("Could not enable thermal interrupts.\n"); return -EINVAL; } } else { - thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); - rdev->irq.dpm_thermal = false; + thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + WREG32(CG_THERMAL_INT, thermal_int); + rdev->irq.dpm_thermal = true; } - WREG32(CG_THERMAL_INT, thermal_int); - return 0; } @@ -5867,6 +5876,309 @@ static int si_thermal_set_temperature_range(struct radeon_device *rdev, return 0; } +static void si_fan_ctrl_set_static_mode(struct radeon_device *rdev, u32 mode) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + + if (si_pi->fan_ctrl_is_in_default_mode) { + tmp = (RREG32(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT; + si_pi->fan_ctrl_default_mode = tmp; + tmp = (RREG32(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT; + si_pi->t_min = tmp; + si_pi->fan_ctrl_is_in_default_mode = false; + } + + tmp = RREG32(CG_FDO_CTRL2) & ~TMIN_MASK; + tmp |= TMIN(0); + WREG32(CG_FDO_CTRL2, tmp); + + tmp = RREG32(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(mode); + WREG32(CG_FDO_CTRL2, tmp); +} + +static int si_thermal_setup_fan_table(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + PP_SIslands_FanTable fan_table = { FDO_MODE_HARDWARE }; + u32 duty100; + u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2; + u16 fdo_min, slope1, slope2; + u32 reference_clock, tmp; + int ret; + u64 tmp64; + + if (!si_pi->fan_table_start) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + tmp64 = (u64)rdev->pm.dpm.fan.pwm_min * duty100; + do_div(tmp64, 10000); + fdo_min = (u16)tmp64; + + t_diff1 = rdev->pm.dpm.fan.t_med - rdev->pm.dpm.fan.t_min; + t_diff2 = rdev->pm.dpm.fan.t_high - rdev->pm.dpm.fan.t_med; + + pwm_diff1 = rdev->pm.dpm.fan.pwm_med - rdev->pm.dpm.fan.pwm_min; + pwm_diff2 = rdev->pm.dpm.fan.pwm_high - rdev->pm.dpm.fan.pwm_med; + + slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100); + slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100); + + fan_table.slope1 = cpu_to_be16(slope1); + fan_table.slope2 = cpu_to_be16(slope2); + + fan_table.fdo_min = cpu_to_be16(fdo_min); + + fan_table.hys_down = cpu_to_be16(rdev->pm.dpm.fan.t_hyst); + + fan_table.hys_up = cpu_to_be16(1); + + fan_table.hys_slope = cpu_to_be16(1); + + fan_table.temp_resp_lim = cpu_to_be16(5); + + reference_clock = radeon_get_xclk(rdev); + + fan_table.refresh_period = cpu_to_be32((rdev->pm.dpm.fan.cycle_delay * + reference_clock) / 1600); + + fan_table.fdo_max = cpu_to_be16((u16)duty100); + + tmp = (RREG32(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT; + fan_table.temp_src = (uint8_t)tmp; + + ret = si_copy_bytes_to_smc(rdev, + si_pi->fan_table_start, + (u8 *)(&fan_table), + sizeof(fan_table), + si_pi->sram_end); + + if (ret) { + DRM_ERROR("Failed to load fan table to the SMC."); + rdev->pm.dpm.fan.ucode_fan_control = false; + } + + return 0; +} + +static int si_fan_ctrl_start_smc_fan_control(struct radeon_device *rdev) +{ + PPSMC_Result ret; + + ret = si_send_msg_to_smc(rdev, PPSMC_StartFanControl); + if (ret == PPSMC_Result_OK) + return 0; + else + return -EINVAL; +} + +static int si_fan_ctrl_stop_smc_fan_control(struct radeon_device *rdev) +{ + PPSMC_Result ret; + + ret = si_send_msg_to_smc(rdev, PPSMC_StopFanControl); + if (ret == PPSMC_Result_OK) + return 0; + else + return -EINVAL; +} + +#if 0 +static int si_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev, + u32 *speed) +{ + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + duty = (RREG32(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)duty * 100; + do_div(tmp64, duty100); + *speed = (u32)tmp64; + + if (*speed > 100) + *speed = 100; + + return 0; +} + +static int si_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev, + u32 speed) +{ + u32 tmp; + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + if (speed > 100) + return -EINVAL; + + if (rdev->pm.dpm.fan.ucode_fan_control) + si_fan_ctrl_stop_smc_fan_control(rdev); + + duty100 = (RREG32(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)speed * duty100; + do_div(tmp64, 100); + duty = (u32)tmp64; + + tmp = RREG32(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK; + tmp |= FDO_STATIC_DUTY(duty); + WREG32(CG_FDO_CTRL0, tmp); + + si_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + + return 0; +} + +static int si_fan_ctrl_get_fan_speed_rpm(struct radeon_device *rdev, + u32 *speed) +{ + u32 tach_period; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + tach_period = (RREG32(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT; + if (tach_period == 0) + return -ENOENT; + + *speed = 60 * xclk * 10000 / tach_period; + + return 0; +} + +static int si_fan_ctrl_set_fan_speed_rpm(struct radeon_device *rdev, + u32 speed) +{ + u32 tach_period, tmp; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + if ((speed < rdev->pm.fan_min_rpm) || + (speed > rdev->pm.fan_max_rpm)) + return -EINVAL; + + if (rdev->pm.dpm.fan.ucode_fan_control) + si_fan_ctrl_stop_smc_fan_control(rdev); + + tach_period = 60 * xclk * 10000 / (8 * speed); + tmp = RREG32(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK; + tmp |= TARGET_PERIOD(tach_period); + WREG32(CG_TACH_CTRL, tmp); + + si_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + + return 0; +} +#endif + +static void si_fan_ctrl_set_default_mode(struct radeon_device *rdev) +{ + struct si_power_info *si_pi = si_get_pi(rdev); + u32 tmp; + + if (!si_pi->fan_ctrl_is_in_default_mode) { + tmp = RREG32(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(si_pi->fan_ctrl_default_mode); + WREG32(CG_FDO_CTRL2, tmp); + + tmp = RREG32(CG_FDO_CTRL2) & TMIN_MASK; + tmp |= TMIN(si_pi->t_min); + WREG32(CG_FDO_CTRL2, tmp); + si_pi->fan_ctrl_is_in_default_mode = true; + } +} + +static void si_thermal_start_smc_fan_control(struct radeon_device *rdev) +{ + if (rdev->pm.dpm.fan.ucode_fan_control) { + si_fan_ctrl_start_smc_fan_control(rdev); + si_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + } +} + +static void si_thermal_initialize(struct radeon_device *rdev) +{ + u32 tmp; + + if (rdev->pm.fan_pulses_per_revolution) { + tmp = RREG32(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK; + tmp |= EDGE_PER_REV(rdev->pm.fan_pulses_per_revolution -1); + WREG32(CG_TACH_CTRL, tmp); + } + + tmp = RREG32(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK; + tmp |= TACH_PWM_RESP_RATE(0x28); + WREG32(CG_FDO_CTRL2, tmp); +} + +static int si_thermal_start_thermal_controller(struct radeon_device *rdev) +{ + int ret; + + si_thermal_initialize(rdev); + ret = si_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + ret = si_thermal_enable_alert(rdev, true); + if (ret) + return ret; + if (rdev->pm.dpm.fan.ucode_fan_control) { + ret = si_halt_smc(rdev); + if (ret) + return ret; + ret = si_thermal_setup_fan_table(rdev); + if (ret) + return ret; + ret = si_resume_smc(rdev); + if (ret) + return ret; + si_thermal_start_smc_fan_control(rdev); + } + + return 0; +} + +static void si_thermal_stop_thermal_controller(struct radeon_device *rdev) +{ + if (!rdev->pm.no_fan) { + si_fan_ctrl_set_default_mode(rdev); + si_fan_ctrl_stop_smc_fan_control(rdev); + } +} + int si_dpm_enable(struct radeon_device *rdev) { struct rv7xx_power_info *pi = rv770_get_pi(rdev); @@ -5979,6 +6291,8 @@ int si_dpm_enable(struct radeon_device *rdev) si_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + si_thermal_start_thermal_controller(rdev); + ni_update_current_ps(rdev, boot_ps); return 0; @@ -6019,6 +6333,7 @@ void si_dpm_disable(struct radeon_device *rdev) if (!si_is_smc_running(rdev)) return; + si_thermal_stop_thermal_controller(rdev); si_disable_ulv(rdev); si_clear_vc(rdev); if (pi->thermal_protection) @@ -6557,6 +6872,9 @@ int si_dpm_init(struct radeon_device *rdev) rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc = rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + si_pi->fan_ctrl_is_in_default_mode = true; + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; } diff --git a/drivers/gpu/drm/radeon/si_dpm.h b/drivers/gpu/drm/radeon/si_dpm.h index 8b5c06a0832d..d16bb1b5f10f 100644 --- a/drivers/gpu/drm/radeon/si_dpm.h +++ b/drivers/gpu/drm/radeon/si_dpm.h @@ -182,6 +182,7 @@ struct si_power_info { u32 dte_table_start; u32 spll_table_start; u32 papm_cfg_table_start; + u32 fan_table_start; /* CAC stuff */ const struct si_cac_config_reg *cac_weights; const struct si_cac_config_reg *lcac_config; @@ -197,6 +198,10 @@ struct si_power_info { /* SVI2 */ u8 svd_gpio_id; u8 svc_gpio_id; + /* fan control */ + bool fan_ctrl_is_in_default_mode; + u32 t_min; + u32 fan_ctrl_default_mode; }; #define SISLANDS_INITIAL_STATE_ARB_INDEX 0 diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h index 6635da9ec986..c549c16a4fe4 100644 --- a/drivers/gpu/drm/radeon/sid.h +++ b/drivers/gpu/drm/radeon/sid.h @@ -180,7 +180,10 @@ #define DIG_THERM_DPM(x) ((x) << 14) #define DIG_THERM_DPM_MASK 0x003FC000 #define DIG_THERM_DPM_SHIFT 14 - +#define CG_THERMAL_STATUS 0x704 +#define FDO_PWM_DUTY(x) ((x) << 9) +#define FDO_PWM_DUTY_MASK (0xff << 9) +#define FDO_PWM_DUTY_SHIFT 9 #define CG_THERMAL_INT 0x708 #define DIG_THERM_INTH(x) ((x) << 8) #define DIG_THERM_INTH_MASK 0x0000FF00 @@ -191,6 +194,10 @@ #define THERM_INT_MASK_HIGH (1 << 24) #define THERM_INT_MASK_LOW (1 << 25) +#define CG_MULT_THERMAL_CTRL 0x710 +#define TEMP_SEL(x) ((x) << 20) +#define TEMP_SEL_MASK (0xff << 20) +#define TEMP_SEL_SHIFT 20 #define CG_MULT_THERMAL_STATUS 0x714 #define ASIC_MAX_TEMP(x) ((x) << 0) #define ASIC_MAX_TEMP_MASK 0x000001ff @@ -199,6 +206,37 @@ #define CTF_TEMP_MASK 0x0003fe00 #define CTF_TEMP_SHIFT 9 +#define CG_FDO_CTRL0 0x754 +#define FDO_STATIC_DUTY(x) ((x) << 0) +#define FDO_STATIC_DUTY_MASK 0x0000000F +#define FDO_STATIC_DUTY_SHIFT 0 +#define CG_FDO_CTRL1 0x758 +#define FMAX_DUTY100(x) ((x) << 0) +#define FMAX_DUTY100_MASK 0x0000000F +#define FMAX_DUTY100_SHIFT 0 +#define CG_FDO_CTRL2 0x75C +#define TMIN(x) ((x) << 0) +#define TMIN_MASK 0x0000000F +#define TMIN_SHIFT 0 +#define FDO_PWM_MODE(x) ((x) << 11) +#define FDO_PWM_MODE_MASK (3 << 11) +#define FDO_PWM_MODE_SHIFT 11 +#define TACH_PWM_RESP_RATE(x) ((x) << 25) +#define TACH_PWM_RESP_RATE_MASK (0x7f << 25) +#define TACH_PWM_RESP_RATE_SHIFT 25 + +#define CG_TACH_CTRL 0x770 +# define EDGE_PER_REV(x) ((x) << 0) +# define EDGE_PER_REV_MASK (0x7 << 0) +# define EDGE_PER_REV_SHIFT 0 +# define TARGET_PERIOD(x) ((x) << 3) +# define TARGET_PERIOD_MASK 0xfffffff8 +# define TARGET_PERIOD_SHIFT 3 +#define CG_TACH_STATUS 0x774 +# define TACH_PERIOD(x) ((x) << 0) +# define TACH_PERIOD_MASK 0xffffffff +# define TACH_PERIOD_SHIFT 0 + #define GENERAL_PWRMGT 0x780 # define GLOBAL_PWRMGT_EN (1 << 0) # define STATIC_PM_EN (1 << 1) diff --git a/drivers/gpu/drm/radeon/sislands_smc.h b/drivers/gpu/drm/radeon/sislands_smc.h index 623a0b1e2d9d..3c779838d9ab 100644 --- a/drivers/gpu/drm/radeon/sislands_smc.h +++ b/drivers/gpu/drm/radeon/sislands_smc.h @@ -245,6 +245,31 @@ typedef struct SISLANDS_SMC_STATETABLE SISLANDS_SMC_STATETABLE; #define SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svd 0x11c #define SI_SMC_SOFT_REGISTER_svi_rework_gpio_id_svc 0x120 +struct PP_SIslands_FanTable +{ + uint8_t fdo_mode; + uint8_t padding; + int16_t temp_min; + int16_t temp_med; + int16_t temp_max; + int16_t slope1; + int16_t slope2; + int16_t fdo_min; + int16_t hys_up; + int16_t hys_down; + int16_t hys_slope; + int16_t temp_resp_lim; + int16_t temp_curr; + int16_t slope_curr; + int16_t pwm_curr; + uint32_t refresh_period; + int16_t fdo_max; + uint8_t temp_src; + int8_t padding2; +}; + +typedef struct PP_SIslands_FanTable PP_SIslands_FanTable; + #define SMC_SISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES 16 #define SMC_SISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES 32 From e03cea367f992e683435c41344197cd7b223b62c Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 15 Sep 2014 00:15:22 -0400 Subject: [PATCH 05/22] drm/radeon/dpm: add smc fan control for CI (v2) Enable smc fan control for CI boards. Should reduce the fan noise on systems with a higher default fan profile. v2: disable by default, add additional fan setup, rpm control bug: https://bugs.freedesktop.org/show_bug.cgi?id=73338 Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 345 ++++++++++++++++++++++++- drivers/gpu/drm/radeon/ci_dpm.h | 5 + drivers/gpu/drm/radeon/cikd.h | 40 ++- drivers/gpu/drm/radeon/ppsmc.h | 6 + drivers/gpu/drm/radeon/pptable.h | 8 + drivers/gpu/drm/radeon/r600_dpm.c | 9 + drivers/gpu/drm/radeon/radeon.h | 4 + drivers/gpu/drm/radeon/smu7_discrete.h | 30 ++- 8 files changed, 443 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index 9dbc52f3c4d1..4581d6cf90e8 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -184,6 +184,9 @@ static int ci_set_overdrive_target_tdp(struct radeon_device *rdev, u32 target_tdp); static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate); +static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev, + PPSMC_Msg msg, u32 parameter); + static struct ci_power_info *ci_get_pi(struct radeon_device *rdev) { struct ci_power_info *pi = rdev->pm.dpm.priv; @@ -355,6 +358,21 @@ static int ci_populate_dw8(struct radeon_device *rdev) return 0; } +static int ci_populate_fuzzy_fan(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + + if ((rdev->pm.dpm.fan.fan_output_sensitivity & (1 << 15)) || + (rdev->pm.dpm.fan.fan_output_sensitivity == 0)) + rdev->pm.dpm.fan.fan_output_sensitivity = + rdev->pm.dpm.fan.default_fan_output_sensitivity; + + pi->smc_powertune_table.FuzzyFan_PwmSetDelta = + cpu_to_be16(rdev->pm.dpm.fan.fan_output_sensitivity); + + return 0; +} + static int ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct radeon_device *rdev) { struct ci_power_info *pi = ci_get_pi(rdev); @@ -478,6 +496,9 @@ static int ci_populate_pm_base(struct radeon_device *rdev) if (ret) return ret; ret = ci_populate_dw8(rdev); + if (ret) + return ret; + ret = ci_populate_fuzzy_fan(rdev); if (ret) return ret; ret = ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(rdev); @@ -859,6 +880,7 @@ static int ci_thermal_enable_alert(struct radeon_device *rdev, if (enable) { thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); + WREG32_SMC(CG_THERMAL_INT, thermal_int); rdev->irq.dpm_thermal = false; result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Enable); if (result != PPSMC_Result_OK) { @@ -867,6 +889,7 @@ static int ci_thermal_enable_alert(struct radeon_device *rdev, } } else { thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; + WREG32_SMC(CG_THERMAL_INT, thermal_int); rdev->irq.dpm_thermal = true; result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Disable); if (result != PPSMC_Result_OK) { @@ -875,11 +898,324 @@ static int ci_thermal_enable_alert(struct radeon_device *rdev, } } - WREG32_SMC(CG_THERMAL_INT, thermal_int); + return 0; +} + +static void ci_fan_ctrl_set_static_mode(struct radeon_device *rdev, u32 mode) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (pi->fan_ctrl_is_in_default_mode) { + tmp = (RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT; + pi->fan_ctrl_default_mode = tmp; + tmp = (RREG32_SMC(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT; + pi->t_min = tmp; + pi->fan_ctrl_is_in_default_mode = false; + } + + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK; + tmp |= TMIN(0); + WREG32_SMC(CG_FDO_CTRL2, tmp); + + tmp = RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(mode); + WREG32_SMC(CG_FDO_CTRL2, tmp); +} + +static int ci_thermal_setup_fan_table(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + SMU7_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE }; + u32 duty100; + u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2; + u16 fdo_min, slope1, slope2; + u32 reference_clock, tmp; + int ret; + u64 tmp64; + + if (!pi->fan_table_start) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) { + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; + } + + tmp64 = (u64)rdev->pm.dpm.fan.pwm_min * duty100; + do_div(tmp64, 10000); + fdo_min = (u16)tmp64; + + t_diff1 = rdev->pm.dpm.fan.t_med - rdev->pm.dpm.fan.t_min; + t_diff2 = rdev->pm.dpm.fan.t_high - rdev->pm.dpm.fan.t_med; + + pwm_diff1 = rdev->pm.dpm.fan.pwm_med - rdev->pm.dpm.fan.pwm_min; + pwm_diff2 = rdev->pm.dpm.fan.pwm_high - rdev->pm.dpm.fan.pwm_med; + + slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100); + slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100); + + fan_table.TempMin = cpu_to_be16((50 + rdev->pm.dpm.fan.t_min) / 100); + fan_table.TempMed = cpu_to_be16((50 + rdev->pm.dpm.fan.t_med) / 100); + fan_table.TempMax = cpu_to_be16((50 + rdev->pm.dpm.fan.t_max) / 100); + + fan_table.Slope1 = cpu_to_be16(slope1); + fan_table.Slope2 = cpu_to_be16(slope2); + + fan_table.FdoMin = cpu_to_be16(fdo_min); + + fan_table.HystDown = cpu_to_be16(rdev->pm.dpm.fan.t_hyst); + + fan_table.HystUp = cpu_to_be16(1); + + fan_table.HystSlope = cpu_to_be16(1); + + fan_table.TempRespLim = cpu_to_be16(5); + + reference_clock = radeon_get_xclk(rdev); + + fan_table.RefreshPeriod = cpu_to_be32((rdev->pm.dpm.fan.cycle_delay * + reference_clock) / 1600); + + fan_table.FdoMax = cpu_to_be16((u16)duty100); + + tmp = (RREG32_SMC(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT; + fan_table.TempSrc = (uint8_t)tmp; + + ret = ci_copy_bytes_to_smc(rdev, + pi->fan_table_start, + (u8 *)(&fan_table), + sizeof(fan_table), + pi->sram_end); + + if (ret) { + DRM_ERROR("Failed to load fan table to the SMC."); + rdev->pm.dpm.fan.ucode_fan_control = false; + } return 0; } +static int ci_fan_ctrl_start_smc_fan_control(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result ret; + + if (pi->caps_od_fuzzy_fan_control_support) { + ret = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_StartFanControl, + FAN_CONTROL_FUZZY); + if (ret != PPSMC_Result_OK) + return -EINVAL; + ret = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_MSG_SetFanPwmMax, + rdev->pm.dpm.fan.default_max_fan_pwm); + if (ret != PPSMC_Result_OK) + return -EINVAL; + } else { + ret = ci_send_msg_to_smc_with_parameter(rdev, + PPSMC_StartFanControl, + FAN_CONTROL_TABLE); + if (ret != PPSMC_Result_OK) + return -EINVAL; + } + + return 0; +} + +#if 0 +static int ci_fan_ctrl_stop_smc_fan_control(struct radeon_device *rdev) +{ + PPSMC_Result ret; + + ret = ci_send_msg_to_smc(rdev, PPSMC_StopFanControl); + if (ret == PPSMC_Result_OK) + return 0; + else + return -EINVAL; +} + +static int ci_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev, + u32 *speed) +{ + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + duty = (RREG32_SMC(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)duty * 100; + do_div(tmp64, duty100); + *speed = (u32)tmp64; + + if (*speed > 100) + *speed = 100; + + return 0; +} + +static int ci_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev, + u32 speed) +{ + u32 tmp; + u32 duty, duty100; + u64 tmp64; + + if (rdev->pm.no_fan) + return -ENOENT; + + if (speed > 100) + return -EINVAL; + + if (rdev->pm.dpm.fan.ucode_fan_control) + ci_fan_ctrl_stop_smc_fan_control(rdev); + + duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT; + + if (duty100 == 0) + return -EINVAL; + + tmp64 = (u64)speed * duty100; + do_div(tmp64, 100); + duty = (u32)tmp64; + + tmp = RREG32_SMC(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK; + tmp |= FDO_STATIC_DUTY(duty); + WREG32_SMC(CG_FDO_CTRL0, tmp); + + ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + + return 0; +} + +static int ci_fan_ctrl_get_fan_speed_rpm(struct radeon_device *rdev, + u32 *speed) +{ + u32 tach_period; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + tach_period = (RREG32_SMC(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT; + if (tach_period == 0) + return -ENOENT; + + *speed = 60 * xclk * 10000 / tach_period; + + return 0; +} + +static int ci_fan_ctrl_set_fan_speed_rpm(struct radeon_device *rdev, + u32 speed) +{ + u32 tach_period, tmp; + u32 xclk = radeon_get_xclk(rdev); + + if (rdev->pm.no_fan) + return -ENOENT; + + if (rdev->pm.fan_pulses_per_revolution == 0) + return -ENOENT; + + if ((speed < rdev->pm.fan_min_rpm) || + (speed > rdev->pm.fan_max_rpm)) + return -EINVAL; + + if (rdev->pm.dpm.fan.ucode_fan_control) + ci_fan_ctrl_stop_smc_fan_control(rdev); + + tach_period = 60 * xclk * 10000 / (8 * speed); + tmp = RREG32_SMC(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK; + tmp |= TARGET_PERIOD(tach_period); + WREG32_SMC(CG_TACH_CTRL, tmp); + + ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + + return 0; +} +#endif + +static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + u32 tmp; + + if (!pi->fan_ctrl_is_in_default_mode) { + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK; + tmp |= FDO_PWM_MODE(pi->fan_ctrl_default_mode); + WREG32_SMC(CG_FDO_CTRL2, tmp); + + tmp = RREG32_SMC(CG_FDO_CTRL2) & TMIN_MASK; + tmp |= TMIN(pi->t_min); + WREG32_SMC(CG_FDO_CTRL2, tmp); + pi->fan_ctrl_is_in_default_mode = true; + } +} + +static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev) +{ + if (rdev->pm.dpm.fan.ucode_fan_control) { + ci_fan_ctrl_start_smc_fan_control(rdev); + ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC); + } +} + +static void ci_thermal_initialize(struct radeon_device *rdev) +{ + u32 tmp; + + if (rdev->pm.fan_pulses_per_revolution) { + tmp = RREG32_SMC(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK; + tmp |= EDGE_PER_REV(rdev->pm.fan_pulses_per_revolution -1); + WREG32_SMC(CG_TACH_CTRL, tmp); + } + + tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK; + tmp |= TACH_PWM_RESP_RATE(0x28); + WREG32_SMC(CG_FDO_CTRL2, tmp); +} + +static int ci_thermal_start_thermal_controller(struct radeon_device *rdev) +{ + int ret; + + ci_thermal_initialize(rdev); + ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX); + if (ret) + return ret; + ret = ci_thermal_enable_alert(rdev, true); + if (ret) + return ret; + if (rdev->pm.dpm.fan.ucode_fan_control) { + ret = ci_thermal_setup_fan_table(rdev); + if (ret) + return ret; + ci_thermal_start_smc_fan_control(rdev); + } + + return 0; +} + +static void ci_thermal_stop_thermal_controller(struct radeon_device *rdev) +{ + if (!rdev->pm.no_fan) + ci_fan_ctrl_set_default_mode(rdev); +} + #if 0 static int ci_read_smc_soft_register(struct radeon_device *rdev, u16 reg_offset, u32 *value) @@ -4841,6 +5177,8 @@ int ci_dpm_enable(struct radeon_device *rdev) ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + ci_thermal_start_thermal_controller(rdev); + ci_update_current_ps(rdev, boot_ps); return 0; @@ -4886,6 +5224,8 @@ void ci_dpm_disable(struct radeon_device *rdev) if (!ci_is_smc_running(rdev)) return; + ci_thermal_stop_thermal_controller(rdev); + if (pi->thermal_protection) ci_enable_thermal_protection(rdev, false); ci_enable_power_containment(rdev, false); @@ -5473,6 +5813,9 @@ int ci_dpm_init(struct radeon_device *rdev) rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc = rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + pi->fan_ctrl_is_in_default_mode = true; + rdev->pm.dpm.fan.ucode_fan_control = false; + return 0; } diff --git a/drivers/gpu/drm/radeon/ci_dpm.h b/drivers/gpu/drm/radeon/ci_dpm.h index 615cb2cacf2c..bb19fbf3ab8a 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.h +++ b/drivers/gpu/drm/radeon/ci_dpm.h @@ -266,6 +266,7 @@ struct ci_power_info { bool caps_automatic_dc_transition; bool caps_sclk_throttle_low_notification; bool caps_dynamic_ac_timing; + bool caps_od_fuzzy_fan_control_support; /* flags */ bool thermal_protection; bool pcie_performance_request; @@ -287,6 +288,10 @@ struct ci_power_info { struct ci_ps current_ps; struct radeon_ps requested_rps; struct ci_ps requested_ps; + /* fan control */ + bool fan_ctrl_is_in_default_mode; + u32 t_min; + u32 fan_ctrl_default_mode; }; #define CISLANDS_VOLTAGE_CONTROL_NONE 0x0 diff --git a/drivers/gpu/drm/radeon/cikd.h b/drivers/gpu/drm/radeon/cikd.h index 068cbb019326..e4e88ca8b82e 100644 --- a/drivers/gpu/drm/radeon/cikd.h +++ b/drivers/gpu/drm/radeon/cikd.h @@ -186,7 +186,10 @@ #define DIG_THERM_DPM(x) ((x) << 14) #define DIG_THERM_DPM_MASK 0x003FC000 #define DIG_THERM_DPM_SHIFT 14 - +#define CG_THERMAL_STATUS 0xC0300008 +#define FDO_PWM_DUTY(x) ((x) << 9) +#define FDO_PWM_DUTY_MASK (0xff << 9) +#define FDO_PWM_DUTY_SHIFT 9 #define CG_THERMAL_INT 0xC030000C #define CI_DIG_THERM_INTH(x) ((x) << 8) #define CI_DIG_THERM_INTH_MASK 0x0000FF00 @@ -196,7 +199,10 @@ #define CI_DIG_THERM_INTL_SHIFT 16 #define THERM_INT_MASK_HIGH (1 << 24) #define THERM_INT_MASK_LOW (1 << 25) - +#define CG_MULT_THERMAL_CTRL 0xC0300010 +#define TEMP_SEL(x) ((x) << 20) +#define TEMP_SEL_MASK (0xff << 20) +#define TEMP_SEL_SHIFT 20 #define CG_MULT_THERMAL_STATUS 0xC0300014 #define ASIC_MAX_TEMP(x) ((x) << 0) #define ASIC_MAX_TEMP_MASK 0x000001ff @@ -205,6 +211,36 @@ #define CTF_TEMP_MASK 0x0003fe00 #define CTF_TEMP_SHIFT 9 +#define CG_FDO_CTRL0 0xC0300064 +#define FDO_STATIC_DUTY(x) ((x) << 0) +#define FDO_STATIC_DUTY_MASK 0x0000000F +#define FDO_STATIC_DUTY_SHIFT 0 +#define CG_FDO_CTRL1 0xC0300068 +#define FMAX_DUTY100(x) ((x) << 0) +#define FMAX_DUTY100_MASK 0x0000000F +#define FMAX_DUTY100_SHIFT 0 +#define CG_FDO_CTRL2 0xC030006C +#define TMIN(x) ((x) << 0) +#define TMIN_MASK 0x0000000F +#define TMIN_SHIFT 0 +#define FDO_PWM_MODE(x) ((x) << 11) +#define FDO_PWM_MODE_MASK (3 << 11) +#define FDO_PWM_MODE_SHIFT 11 +#define TACH_PWM_RESP_RATE(x) ((x) << 25) +#define TACH_PWM_RESP_RATE_MASK (0x7f << 25) +#define TACH_PWM_RESP_RATE_SHIFT 25 +#define CG_TACH_CTRL 0xC0300070 +# define EDGE_PER_REV(x) ((x) << 0) +# define EDGE_PER_REV_MASK (0x7 << 0) +# define EDGE_PER_REV_SHIFT 0 +# define TARGET_PERIOD(x) ((x) << 3) +# define TARGET_PERIOD_MASK 0xfffffff8 +# define TARGET_PERIOD_SHIFT 3 +#define CG_TACH_STATUS 0xC0300074 +# define TACH_PERIOD(x) ((x) << 0) +# define TACH_PERIOD_MASK 0xffffffff +# define TACH_PERIOD_SHIFT 0 + #define CG_ECLK_CNTL 0xC05000AC # define ECLK_DIVIDER_MASK 0x7f # define ECLK_DIR_CNTL_EN (1 << 8) diff --git a/drivers/gpu/drm/radeon/ppsmc.h b/drivers/gpu/drm/radeon/ppsmc.h index 0c4eaa60b6ca..ff698b05bdf5 100644 --- a/drivers/gpu/drm/radeon/ppsmc.h +++ b/drivers/gpu/drm/radeon/ppsmc.h @@ -59,6 +59,11 @@ #define FDO_MODE_HARDWARE 0 #define FDO_MODE_PIECE_WISE_LINEAR 1 +enum FAN_CONTROL { + FAN_CONTROL_FUZZY, + FAN_CONTROL_TABLE +}; + #define PPSMC_Result_OK ((uint8_t)0x01) #define PPSMC_Result_Failed ((uint8_t)0xFF) @@ -155,6 +160,7 @@ typedef uint8_t PPSMC_Result; #define PPSMC_MSG_MASTER_DeepSleep_ON ((uint16_t) 0x18F) #define PPSMC_MSG_MASTER_DeepSleep_OFF ((uint16_t) 0x190) #define PPSMC_MSG_Remove_DC_Clamp ((uint16_t) 0x191) +#define PPSMC_MSG_SetFanPwmMax ((uint16_t) 0x19A) #define PPSMC_MSG_API_GetSclkFrequency ((uint16_t) 0x200) #define PPSMC_MSG_API_GetMclkFrequency ((uint16_t) 0x201) diff --git a/drivers/gpu/drm/radeon/pptable.h b/drivers/gpu/drm/radeon/pptable.h index 2d532996c697..4c2eec49dadc 100644 --- a/drivers/gpu/drm/radeon/pptable.h +++ b/drivers/gpu/drm/radeon/pptable.h @@ -96,6 +96,14 @@ typedef struct _ATOM_PPLIB_FANTABLE2 USHORT usTMax; // The max temperature } ATOM_PPLIB_FANTABLE2; +typedef struct _ATOM_PPLIB_FANTABLE3 +{ + ATOM_PPLIB_FANTABLE2 basicTable2; + UCHAR ucFanControlMode; + USHORT usFanPWMMax; + USHORT usFanOutputSensitivity; +} ATOM_PPLIB_FANTABLE3; + typedef struct _ATOM_PPLIB_EXTENDEDHEADER { USHORT usSize; diff --git a/drivers/gpu/drm/radeon/r600_dpm.c b/drivers/gpu/drm/radeon/r600_dpm.c index f6309bd23e01..76c6a17eeb2d 100644 --- a/drivers/gpu/drm/radeon/r600_dpm.c +++ b/drivers/gpu/drm/radeon/r600_dpm.c @@ -811,6 +811,7 @@ union power_info { union fan_info { struct _ATOM_PPLIB_FANTABLE fan; struct _ATOM_PPLIB_FANTABLE2 fan2; + struct _ATOM_PPLIB_FANTABLE3 fan3; }; static int r600_parse_clk_voltage_dep_table(struct radeon_clock_voltage_dependency_table *radeon_table, @@ -900,6 +901,14 @@ int r600_parse_extended_power_table(struct radeon_device *rdev) else rdev->pm.dpm.fan.t_max = 10900; rdev->pm.dpm.fan.cycle_delay = 100000; + if (fan_info->fan.ucFanTableFormat >= 3) { + rdev->pm.dpm.fan.control_mode = fan_info->fan3.ucFanControlMode; + rdev->pm.dpm.fan.default_max_fan_pwm = + le16_to_cpu(fan_info->fan3.usFanPWMMax); + rdev->pm.dpm.fan.default_fan_output_sensitivity = 4836; + rdev->pm.dpm.fan.fan_output_sensitivity = + le16_to_cpu(fan_info->fan3.usFanOutputSensitivity); + } rdev->pm.dpm.fan.ucode_fan_control = true; } } diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 1f61ff089c9e..5aabbe0a43f5 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -1494,6 +1494,10 @@ struct radeon_dpm_fan { u8 t_hyst; u32 cycle_delay; u16 t_max; + u8 control_mode; + u16 default_max_fan_pwm; + u16 default_fan_output_sensitivity; + u16 fan_output_sensitivity; bool ucode_fan_control; }; diff --git a/drivers/gpu/drm/radeon/smu7_discrete.h b/drivers/gpu/drm/radeon/smu7_discrete.h index 82f70c90a9ee..0b0b404ff091 100644 --- a/drivers/gpu/drm/radeon/smu7_discrete.h +++ b/drivers/gpu/drm/radeon/smu7_discrete.h @@ -431,6 +431,31 @@ struct SMU7_Discrete_MCRegisters typedef struct SMU7_Discrete_MCRegisters SMU7_Discrete_MCRegisters; +struct SMU7_Discrete_FanTable +{ + uint16_t FdoMode; + int16_t TempMin; + int16_t TempMed; + int16_t TempMax; + int16_t Slope1; + int16_t Slope2; + int16_t FdoMin; + int16_t HystUp; + int16_t HystDown; + int16_t HystSlope; + int16_t TempRespLim; + int16_t TempCurr; + int16_t SlopeCurr; + int16_t PwmCurr; + uint32_t RefreshPeriod; + int16_t FdoMax; + uint8_t TempSrc; + int8_t Padding; +}; + +typedef struct SMU7_Discrete_FanTable SMU7_Discrete_FanTable; + + struct SMU7_Discrete_PmFuses { // dw0-dw1 uint8_t BapmVddCVidHiSidd[8]; @@ -462,7 +487,10 @@ struct SMU7_Discrete_PmFuses { uint8_t BapmVddCVidHiSidd2[8]; // dw11-dw12 - uint32_t Reserved6[2]; + int16_t FuzzyFan_ErrorSetDelta; + int16_t FuzzyFan_ErrorRateSetDelta; + int16_t FuzzyFan_PwmSetDelta; + uint16_t CalcMeasPowerBlend; // dw13-dw16 uint8_t GnbLPML[16]; From 0e4ed1c1908e48ee66d9e955ce824bc1f34d103e Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 15 Sep 2014 02:15:04 -0400 Subject: [PATCH 06/22] drm/radeon/dpm: add thermal dpm support for CI Not currently used. Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 27 +++++++++++++++++++++++++++ drivers/gpu/drm/radeon/ci_dpm.h | 1 + drivers/gpu/drm/radeon/ppsmc.h | 3 +++ 3 files changed, 31 insertions(+) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index 4581d6cf90e8..83121403ce3b 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -714,6 +714,25 @@ static int ci_enable_smc_cac(struct radeon_device *rdev, bool enable) return ret; } +static int ci_enable_thermal_based_sclk_dpm(struct radeon_device *rdev, + bool enable) +{ + struct ci_power_info *pi = ci_get_pi(rdev); + PPSMC_Result smc_result = PPSMC_Result_OK; + + if (pi->thermal_sclk_dpm_enabled) { + if (enable) + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_ENABLE_THERMAL_DPM); + else + smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DISABLE_THERMAL_DPM); + } + + if (smc_result == PPSMC_Result_OK) + return 0; + else + return -EINVAL; +} + static int ci_power_control_set_level(struct radeon_device *rdev) { struct ci_power_info *pi = ci_get_pi(rdev); @@ -5177,6 +5196,12 @@ int ci_dpm_enable(struct radeon_device *rdev) ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true); + ret = ci_enable_thermal_based_sclk_dpm(rdev, true); + if (ret) { + DRM_ERROR("ci_enable_thermal_based_sclk_dpm failed\n"); + return ret; + } + ci_thermal_start_thermal_controller(rdev); ci_update_current_ps(rdev, boot_ps); @@ -5240,6 +5265,7 @@ void ci_dpm_disable(struct radeon_device *rdev) ci_reset_to_default(rdev); ci_dpm_stop_smc(rdev); ci_force_switch_to_arb_f0(rdev); + ci_enable_thermal_based_sclk_dpm(rdev, false); ci_update_current_ps(rdev, boot_ps); } @@ -5639,6 +5665,7 @@ int ci_dpm_init(struct radeon_device *rdev) pi->sclk_dpm_key_disabled = 0; pi->mclk_dpm_key_disabled = 0; pi->pcie_dpm_key_disabled = 0; + pi->thermal_sclk_dpm_enabled = 0; /* mclk dpm is unstable on some R7 260X cards with the old mc ucode */ if ((rdev->pdev->device == 0x6658) && diff --git a/drivers/gpu/drm/radeon/ci_dpm.h b/drivers/gpu/drm/radeon/ci_dpm.h index bb19fbf3ab8a..84e3d3bcf9f3 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.h +++ b/drivers/gpu/drm/radeon/ci_dpm.h @@ -239,6 +239,7 @@ struct ci_power_info { u32 sclk_dpm_key_disabled; u32 mclk_dpm_key_disabled; u32 pcie_dpm_key_disabled; + u32 thermal_sclk_dpm_enabled; struct ci_pcie_perf_range pcie_gen_performance; struct ci_pcie_perf_range pcie_lane_performance; struct ci_pcie_perf_range pcie_gen_powersaving; diff --git a/drivers/gpu/drm/radeon/ppsmc.h b/drivers/gpu/drm/radeon/ppsmc.h index ff698b05bdf5..7e5724a12f8b 100644 --- a/drivers/gpu/drm/radeon/ppsmc.h +++ b/drivers/gpu/drm/radeon/ppsmc.h @@ -162,6 +162,9 @@ typedef uint8_t PPSMC_Result; #define PPSMC_MSG_Remove_DC_Clamp ((uint16_t) 0x191) #define PPSMC_MSG_SetFanPwmMax ((uint16_t) 0x19A) +#define PPSMC_MSG_ENABLE_THERMAL_DPM ((uint16_t) 0x19C) +#define PPSMC_MSG_DISABLE_THERMAL_DPM ((uint16_t) 0x19D) + #define PPSMC_MSG_API_GetSclkFrequency ((uint16_t) 0x200) #define PPSMC_MSG_API_GetMclkFrequency ((uint16_t) 0x201) From 66648b81ce6a1ea713e299d2074d81d36cff4c94 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 17 Nov 2014 11:52:48 -0500 Subject: [PATCH 07/22] drm/radeon: fix PCC debugging message for CI DPM Add missing newline and print the bad gpio shift. Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index 83121403ce3b..c9963b589336 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -5773,7 +5773,7 @@ int ci_dpm_init(struct radeon_device *rdev) tmp |= DPM_ENABLED; break; default: - DRM_ERROR("Invalid PCC GPIO!"); + DRM_ERROR("Invalid PCC GPIO: %u!\n", gpio.shift); break; } WREG32_SMC(CNB_PWRMGT_CNTL, tmp); From f281d0a37c423cb4cc8ef91ba7136848744c955f Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Tue, 18 Nov 2014 14:32:14 -0500 Subject: [PATCH 08/22] drm/radeon/ci: apply disp voltage changes before clk changes Apply voltage changes for displays before changing clocks. Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index c9963b589336..e7816281f0d7 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -3752,6 +3752,8 @@ static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev) struct ci_power_info *pi = ci_get_pi(rdev); PPSMC_Result result; + ci_apply_disp_minimum_voltage_request(rdev); + if (!pi->sclk_dpm_key_disabled) { if (pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { result = ci_send_msg_to_smc_with_parameter(rdev, @@ -3782,8 +3784,6 @@ static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev) } } - ci_apply_disp_minimum_voltage_request(rdev); - return 0; } From e745c3c9aa5e30ae94a1be04a1d2f326fb213a56 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Tue, 18 Nov 2014 14:36:59 -0500 Subject: [PATCH 09/22] drm/radeon/ci: use different smc command for pcie dpm Use unforce levels rather than enable mask instruction. This is the preferred method. Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index e7816281f0d7..73f8c4b5bc9c 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -3773,7 +3773,7 @@ static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev) return -EINVAL; } } - +#if 0 if (!pi->pcie_dpm_key_disabled) { if (pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { result = ci_send_msg_to_smc_with_parameter(rdev, @@ -3783,7 +3783,7 @@ static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev) return -EINVAL; } } - +#endif return 0; } @@ -4247,6 +4247,14 @@ int ci_dpm_force_performance_level(struct radeon_device *rdev, } } } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) { + if (!pi->pcie_dpm_key_disabled) { + PPSMC_Result smc_result; + + smc_result = ci_send_msg_to_smc(rdev, + PPSMC_MSG_PCIeDPM_UnForceLevel); + if (smc_result != PPSMC_Result_OK) + return -EINVAL; + } ret = ci_upload_dpm_level_enable_mask(rdev); if (ret) return ret; From 7f6233ca8769a92cf4f23a0bc18bf241e9c50606 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Tue, 18 Nov 2014 14:40:26 -0500 Subject: [PATCH 10/22] drm/radeon/ci: force pcie level before sclk and mclk Preferred ordering. Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index 73f8c4b5bc9c..630434cba22d 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -4143,6 +4143,25 @@ int ci_dpm_force_performance_level(struct radeon_device *rdev, int ret; if (level == RADEON_DPM_FORCED_LEVEL_HIGH) { + if ((!pi->pcie_dpm_key_disabled) && + pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { + levels = 0; + tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask; + while (tmp >>= 1) + levels++; + if (levels) { + ret = ci_dpm_force_state_pcie(rdev, level); + if (ret) + return ret; + for (i = 0; i < rdev->usec_timeout; i++) { + tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) & + CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT; + if (tmp == levels) + break; + udelay(1); + } + } + } if ((!pi->sclk_dpm_key_disabled) && pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { levels = 0; @@ -4181,25 +4200,6 @@ int ci_dpm_force_performance_level(struct radeon_device *rdev, } } } - if ((!pi->pcie_dpm_key_disabled) && - pi->dpm_level_enable_mask.pcie_dpm_enable_mask) { - levels = 0; - tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask; - while (tmp >>= 1) - levels++; - if (levels) { - ret = ci_dpm_force_state_pcie(rdev, level); - if (ret) - return ret; - for (i = 0; i < rdev->usec_timeout; i++) { - tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) & - CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT; - if (tmp == levels) - break; - udelay(1); - } - } - } } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) { if ((!pi->sclk_dpm_key_disabled) && pi->dpm_level_enable_mask.sclk_dpm_enable_mask) { From d967be9b80a5aa3ba228a9c2d3fea91ae99e4a07 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Tue, 18 Nov 2014 15:07:33 -0500 Subject: [PATCH 11/22] drm/radeon/ci: disable needless sclk changes The current code always reprogrammed the sclk levels, but we don't currently handle disp sclk requirements so just skip it. Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/ci_dpm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index 630434cba22d..3f898d020ae6 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -3809,7 +3809,7 @@ static void ci_find_dpm_states_clocks_in_dpm_table(struct radeon_device *rdev, pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK; } else { /* XXX check display min clock requirements */ - if (0 != CISLAND_MINIMUM_ENGINE_CLOCK) + if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK) pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK; } From faffaf620f32c6e907d06570f8f75845ecb1349f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:19 +0100 Subject: [PATCH 12/22] drm/radeon: rework vm_flush parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use ring structure instead of index and provide vm_id and pd_addr separately. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/cik.c | 23 ++++++++++------------- drivers/gpu/drm/radeon/cik_sdma.c | 22 +++++++++------------- drivers/gpu/drm/radeon/ni.c | 14 +++++--------- drivers/gpu/drm/radeon/ni_dma.c | 14 +++++--------- drivers/gpu/drm/radeon/radeon.h | 5 +++-- drivers/gpu/drm/radeon/radeon_asic.h | 18 ++++++++++++------ drivers/gpu/drm/radeon/radeon_vm.c | 3 ++- drivers/gpu/drm/radeon/si.c | 18 +++++++----------- drivers/gpu/drm/radeon/si_dma.c | 19 ++++++++----------- 9 files changed, 61 insertions(+), 75 deletions(-) diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index cef4cb7e5438..3dc2be07dcde 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -5982,26 +5982,23 @@ static void cik_vm_decode_fault(struct radeon_device *rdev, * Update the page table base and flush the VM TLB * using the CP (CIK). */ -void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct radeon_ring *ring = &rdev->ring[ridx]; - int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX); - - if (vm == NULL) - return; + int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX); radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | WRITE_DATA_DST_SEL(0))); - if (vm->id < 8) { + if (vm_id < 8) { radeon_ring_write(ring, - (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); + (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); } else { radeon_ring_write(ring, - (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); + (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); } radeon_ring_write(ring, 0); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, pd_addr >> 12); /* update SH_MEM_* regs */ radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); @@ -6009,7 +6006,7 @@ void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) WRITE_DATA_DST_SEL(0))); radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); radeon_ring_write(ring, 0); - radeon_ring_write(ring, VMID(vm->id)); + radeon_ring_write(ring, VMID(vm_id)); radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6)); radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | @@ -6030,7 +6027,7 @@ void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) radeon_ring_write(ring, VMID(0)); /* HDP flush */ - cik_hdp_flush_cp_ring_emit(rdev, ridx); + cik_hdp_flush_cp_ring_emit(rdev, ring->idx); /* bits 0-15 are the VM contexts0-15 */ radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); @@ -6038,7 +6035,7 @@ void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) WRITE_DATA_DST_SEL(0))); radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); radeon_ring_write(ring, 0); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); /* compute doesn't have PFP */ if (usepfp) { diff --git a/drivers/gpu/drm/radeon/cik_sdma.c b/drivers/gpu/drm/radeon/cik_sdma.c index 4e8432d07f15..7470a2ee83bd 100644 --- a/drivers/gpu/drm/radeon/cik_sdma.c +++ b/drivers/gpu/drm/radeon/cik_sdma.c @@ -901,25 +901,21 @@ void cik_sdma_vm_pad_ib(struct radeon_ib *ib) * Update the page table base and flush the VM TLB * using sDMA (CIK). */ -void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct radeon_ring *ring = &rdev->ring[ridx]; - - if (vm == NULL) - return; - radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); - if (vm->id < 8) { - radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); + if (vm_id < 8) { + radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); } else { - radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); + radeon_ring_write(ring, (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); } - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, pd_addr >> 12); /* update SH_MEM_* regs */ radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); radeon_ring_write(ring, SRBM_GFX_CNTL >> 2); - radeon_ring_write(ring, VMID(vm->id)); + radeon_ring_write(ring, VMID(vm_id)); radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); radeon_ring_write(ring, SH_MEM_BASES >> 2); @@ -942,11 +938,11 @@ void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm radeon_ring_write(ring, VMID(0)); /* flush HDP */ - cik_sdma_hdp_flush_ring_emit(rdev, ridx); + cik_sdma_hdp_flush_ring_emit(rdev, ring->idx); /* flush TLB */ radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000)); radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); } diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index 3faee58946dd..bee432d3dd30 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c @@ -2502,15 +2502,11 @@ void cayman_vm_decode_fault(struct radeon_device *rdev, * Update the page table base and flush the VM TLB * using the CP (cayman-si). */ -void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct radeon_ring *ring = &rdev->ring[ridx]; - - if (vm == NULL) - return; - - radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0)); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0)); + radeon_ring_write(ring, pd_addr >> 12); /* flush hdp cache */ radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0)); @@ -2518,7 +2514,7 @@ void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) /* bits 0-7 are the VM contexts0-7 */ radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0)); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); /* sync PFP to ME, otherwise we might get invalid PFP reads */ radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); diff --git a/drivers/gpu/drm/radeon/ni_dma.c b/drivers/gpu/drm/radeon/ni_dma.c index f26f0a9fb522..5a72404c9d5e 100644 --- a/drivers/gpu/drm/radeon/ni_dma.c +++ b/drivers/gpu/drm/radeon/ni_dma.c @@ -446,16 +446,12 @@ void cayman_dma_vm_pad_ib(struct radeon_ib *ib) ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0); } -void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct radeon_ring *ring = &rdev->ring[ridx]; - - if (vm == NULL) - return; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); - radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2)); + radeon_ring_write(ring, pd_addr >> 12); /* flush hdp cache */ radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); @@ -465,6 +461,6 @@ void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm /* bits 0-7 are the VM contexts0-7 */ radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0)); radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); } diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 5aabbe0a43f5..39f7716343a4 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -1798,7 +1798,8 @@ struct radeon_asic_ring { void (*hdp_flush)(struct radeon_device *rdev, struct radeon_ring *ring); bool (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp, struct radeon_semaphore *semaphore, bool emit_wait); - void (*vm_flush)(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); + void (*vm_flush)(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); /* testing functions */ int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp); @@ -2850,7 +2851,7 @@ static inline void radeon_ring_write(struct radeon_ring *ring, uint32_t v) #define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_execute((rdev), (ib)) #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_parse((rdev), (ib)) #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)]->is_lockup((rdev), (cp)) -#define radeon_ring_vm_flush(rdev, r, vm) (rdev)->asic->ring[(r)]->vm_flush((rdev), (r), (vm)) +#define radeon_ring_vm_flush(rdev, r, vm_id, pd_addr) (rdev)->asic->ring[(r)->idx]->vm_flush((rdev), (r), (vm_id), (pd_addr)) #define radeon_ring_get_rptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_rptr((rdev), (r)) #define radeon_ring_get_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_wptr((rdev), (r)) #define radeon_ring_set_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->set_wptr((rdev), (r)) diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h index d8ace5b28a5b..2a45d548d5ec 100644 --- a/drivers/gpu/drm/radeon/radeon_asic.h +++ b/drivers/gpu/drm/radeon/radeon_asic.h @@ -599,7 +599,8 @@ int cayman_asic_reset(struct radeon_device *rdev); void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib); int cayman_vm_init(struct radeon_device *rdev); void cayman_vm_fini(struct radeon_device *rdev); -void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags); int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); @@ -624,7 +625,8 @@ void cayman_dma_vm_set_pages(struct radeon_device *rdev, uint32_t incr, uint32_t flags); void cayman_dma_vm_pad_ib(struct radeon_ib *ib); -void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void cayman_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); u32 cayman_gfx_get_rptr(struct radeon_device *rdev, struct radeon_ring *ring); @@ -699,7 +701,8 @@ int si_irq_set(struct radeon_device *rdev); int si_irq_process(struct radeon_device *rdev); int si_vm_init(struct radeon_device *rdev); void si_vm_fini(struct radeon_device *rdev); -void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); struct radeon_fence *si_copy_dma(struct radeon_device *rdev, uint64_t src_offset, uint64_t dst_offset, @@ -721,7 +724,8 @@ void si_dma_vm_set_pages(struct radeon_device *rdev, uint64_t addr, unsigned count, uint32_t incr, uint32_t flags); -void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); u32 si_get_xclk(struct radeon_device *rdev); uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev); int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk); @@ -793,7 +797,8 @@ int cik_irq_set(struct radeon_device *rdev); int cik_irq_process(struct radeon_device *rdev); int cik_vm_init(struct radeon_device *rdev); void cik_vm_fini(struct radeon_device *rdev); -void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); void cik_sdma_vm_copy_pages(struct radeon_device *rdev, struct radeon_ib *ib, @@ -811,7 +816,8 @@ void cik_sdma_vm_set_pages(struct radeon_device *rdev, uint32_t incr, uint32_t flags); void cik_sdma_vm_pad_ib(struct radeon_ib *ib); -void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm); +void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr); int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib); u32 cik_gfx_get_rptr(struct radeon_device *rdev, struct radeon_ring *ring); diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index dfde266529e2..9d0f87be6fa0 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -243,7 +243,8 @@ void radeon_vm_flush(struct radeon_device *rdev, if (!vm->last_flush || pd_addr != vm->pd_gpu_addr) { trace_radeon_vm_flush(pd_addr, ring, vm->id); vm->pd_gpu_addr = pd_addr; - radeon_ring_vm_flush(rdev, ring, vm); + radeon_ring_vm_flush(rdev, &rdev->ring[ring], + vm->id, vm->pd_gpu_addr); } } diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index eeea5b6a1775..e91968b04154 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c @@ -5020,27 +5020,23 @@ static void si_vm_decode_fault(struct radeon_device *rdev, block, mc_id); } -void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) { - struct radeon_ring *ring = &rdev->ring[ridx]; - - if (vm == NULL) - return; - /* write new base address */ radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | WRITE_DATA_DST_SEL(0))); - if (vm->id < 8) { + if (vm_id < 8) { radeon_ring_write(ring, - (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); + (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2); } else { radeon_ring_write(ring, - (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); + (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2); } radeon_ring_write(ring, 0); - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, pd_addr >> 12); /* flush hdp cache */ radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); @@ -5056,7 +5052,7 @@ void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) WRITE_DATA_DST_SEL(0))); radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); radeon_ring_write(ring, 0); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); /* sync PFP to ME, otherwise we might get invalid PFP reads */ radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); diff --git a/drivers/gpu/drm/radeon/si_dma.c b/drivers/gpu/drm/radeon/si_dma.c index b58f12b762d7..e8bc0a516b57 100644 --- a/drivers/gpu/drm/radeon/si_dma.c +++ b/drivers/gpu/drm/radeon/si_dma.c @@ -185,20 +185,17 @@ void si_dma_vm_set_pages(struct radeon_device *rdev, } } -void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) +void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, + unsigned vm_id, uint64_t pd_addr) + { - struct radeon_ring *ring = &rdev->ring[ridx]; - - if (vm == NULL) - return; - radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); - if (vm->id < 8) { - radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2)); + if (vm_id < 8) { + radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2)); } else { - radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2)); + radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2)); } - radeon_ring_write(ring, vm->pd_gpu_addr >> 12); + radeon_ring_write(ring, pd_addr >> 12); /* flush hdp cache */ radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); @@ -208,7 +205,7 @@ void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) /* bits 0-7 are the VM contexts0-7 */ radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0)); radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2)); - radeon_ring_write(ring, 1 << vm->id); + radeon_ring_write(ring, 1 << vm_id); } /** From 85761f605a21ef7f3feda9d272565f97ecf1aa56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:20 +0100 Subject: [PATCH 13/22] drm/radeon: stop re-reserving the BO in radeon_vm_bo_set_addr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That's useless when all callers drop the reservation immediately after calling the function. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_gem.c | 2 +- drivers/gpu/drm/radeon/radeon_kms.c | 2 -- drivers/gpu/drm/radeon/radeon_vm.c | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index c194497aa586..f752c7f56015 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -601,6 +601,7 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data, if (bo_va->it.start) { args->operation = RADEON_VA_RESULT_VA_EXIST; args->offset = bo_va->it.start * RADEON_GPU_PAGE_SIZE; + radeon_bo_unreserve(rbo); goto out; } r = radeon_vm_bo_set_addr(rdev, bo_va, args->offset, args->flags); @@ -616,7 +617,6 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data, args->operation = RADEON_VA_RESULT_ERROR; } out: - radeon_bo_unreserve(rbo); drm_gem_object_unreference_unlocked(gobj); return r; } diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c index 6eb561d33eba..f4dd26ae33e5 100644 --- a/drivers/gpu/drm/radeon/radeon_kms.c +++ b/drivers/gpu/drm/radeon/radeon_kms.c @@ -628,8 +628,6 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) RADEON_VA_IB_OFFSET, RADEON_VM_PAGE_READABLE | RADEON_VM_PAGE_SNOOPED); - - radeon_bo_unreserve(rdev->ring_tmp_bo.bo); if (r) { radeon_vm_fini(rdev, vm); kfree(fpriv); diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 9d0f87be6fa0..db0ed3af8e82 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -450,7 +450,7 @@ error: * Validate and set the offset requested within the vm address space. * Returns 0 for success, error for failure. * - * Object has to be reserved! + * Object has to be reserved and gets unreserved by this function! */ int radeon_vm_bo_set_addr(struct radeon_device *rdev, struct radeon_bo_va *bo_va, @@ -576,7 +576,7 @@ int radeon_vm_bo_set_addr(struct radeon_device *rdev, } mutex_unlock(&vm->mutex); - return radeon_bo_reserve(bo_va->bo, false); + return 0; } /** From e0602c35fe3bcbe5fe9179be78882b3101780781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:21 +0100 Subject: [PATCH 14/22] drm/radeon: remove unnecessary VM syncs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PD/PTs reservation object now contains everything needed. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_vm.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index db0ed3af8e82..d8c49ad34dee 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -701,7 +701,6 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, radeon_asic_vm_pad_ib(rdev, &ib); radeon_semaphore_sync_resv(rdev, ib.semaphore, pd->tbo.resv, false); - radeon_semaphore_sync_fence(ib.semaphore, vm->last_id_use); WARN_ON(ib.length_dw > ndw); r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) { @@ -969,7 +968,6 @@ int radeon_vm_bo_update(struct radeon_device *rdev, radeon_asic_vm_pad_ib(rdev, &ib); WARN_ON(ib.length_dw > ndw); - radeon_semaphore_sync_fence(ib.semaphore, vm->fence); r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) { radeon_ib_free(rdev, &ib); From 975700d2cc84408efa9b2360e38b1ab95368556f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:22 +0100 Subject: [PATCH 15/22] drm/radeon: split semaphore and sync object handling v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we just allocated space for four hardware semaphores in each software semaphore object. Make software semaphore objects represent only one hardware semaphore address again by splitting the sync code into it's own object. v2: fix typo in comment Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/Makefile | 3 +- drivers/gpu/drm/radeon/cik.c | 18 +- drivers/gpu/drm/radeon/cik_sdma.c | 18 +- drivers/gpu/drm/radeon/evergreen_dma.c | 18 +- drivers/gpu/drm/radeon/r600.c | 18 +- drivers/gpu/drm/radeon/r600_dma.c | 18 +- drivers/gpu/drm/radeon/radeon.h | 42 +++-- drivers/gpu/drm/radeon/radeon_cs.c | 8 +- drivers/gpu/drm/radeon/radeon_ib.c | 13 +- drivers/gpu/drm/radeon/radeon_semaphore.c | 154 +--------------- drivers/gpu/drm/radeon/radeon_sync.c | 213 ++++++++++++++++++++++ drivers/gpu/drm/radeon/radeon_vm.c | 4 +- drivers/gpu/drm/radeon/rv770_dma.c | 18 +- drivers/gpu/drm/radeon/si_dma.c | 18 +- 14 files changed, 303 insertions(+), 260 deletions(-) create mode 100644 drivers/gpu/drm/radeon/radeon_sync.c diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile index bad6caa0a727..12bc21219a0e 100644 --- a/drivers/gpu/drm/radeon/Makefile +++ b/drivers/gpu/drm/radeon/Makefile @@ -80,7 +80,8 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \ r600_dpm.o rs780_dpm.o rv6xx_dpm.o rv770_dpm.o rv730_dpm.o rv740_dpm.o \ rv770_smc.o cypress_dpm.o btc_dpm.o sumo_dpm.o sumo_smc.o trinity_dpm.o \ trinity_smc.o ni_dpm.o si_smc.o si_dpm.o kv_smc.o kv_dpm.o ci_smc.o \ - ci_dpm.o dce6_afmt.o radeon_vm.o radeon_ucode.o radeon_ib.o radeon_mn.o + ci_dpm.o dce6_afmt.o radeon_vm.o radeon_ucode.o radeon_ib.o radeon_mn.o \ + radeon_sync.o # add async DMA block radeon-y += \ diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index 3dc2be07dcde..755923bc6786 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -3994,31 +3994,27 @@ struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.blit_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_bytes, cur_size_in_bytes, control; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_bytes = size_in_bytes; @@ -4042,12 +4038,12 @@ struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } diff --git a/drivers/gpu/drm/radeon/cik_sdma.c b/drivers/gpu/drm/radeon/cik_sdma.c index 7470a2ee83bd..604e2e770951 100644 --- a/drivers/gpu/drm/radeon/cik_sdma.c +++ b/drivers/gpu/drm/radeon/cik_sdma.c @@ -541,31 +541,27 @@ struct radeon_fence *cik_copy_dma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_bytes, cur_size_in_bytes; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_bytes = size_in_bytes; @@ -586,12 +582,12 @@ struct radeon_fence *cik_copy_dma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } diff --git a/drivers/gpu/drm/radeon/evergreen_dma.c b/drivers/gpu/drm/radeon/evergreen_dma.c index 66bcfadeedd1..96535aa8659c 100644 --- a/drivers/gpu/drm/radeon/evergreen_dma.c +++ b/drivers/gpu/drm/radeon/evergreen_dma.c @@ -110,31 +110,27 @@ struct radeon_fence *evergreen_copy_dma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_dw, cur_size_in_dw; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff); r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_dw = size_in_dw; @@ -153,12 +149,12 @@ struct radeon_fence *evergreen_copy_dma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c index 56b02927cd3d..ef5d6066fa5b 100644 --- a/drivers/gpu/drm/radeon/r600.c +++ b/drivers/gpu/drm/radeon/r600.c @@ -2889,31 +2889,27 @@ struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.blit_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_bytes, cur_size_in_bytes, tmp; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff); r = radeon_ring_lock(rdev, ring, num_loops * 6 + 24); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2); @@ -2942,12 +2938,12 @@ struct radeon_fence *r600_copy_cpdma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } diff --git a/drivers/gpu/drm/radeon/r600_dma.c b/drivers/gpu/drm/radeon/r600_dma.c index aabc343b9a8f..3a58b8073f49 100644 --- a/drivers/gpu/drm/radeon/r600_dma.c +++ b/drivers/gpu/drm/radeon/r600_dma.c @@ -441,31 +441,27 @@ struct radeon_fence *r600_copy_dma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_dw, cur_size_in_dw; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE); r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_dw = size_in_dw; @@ -484,12 +480,12 @@ struct radeon_fence *r600_copy_dma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 39f7716343a4..7cda6d77aeb0 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -150,9 +150,6 @@ extern int radeon_backlight; /* number of hw syncs before falling back on blocking */ #define RADEON_NUM_SYNCS 4 -/* number of hw syncs before falling back on blocking */ -#define RADEON_NUM_SYNCS 4 - /* hardcode those limit for now */ #define RADEON_VA_IB_OFFSET (1 << 20) #define RADEON_VA_RESERVED_SIZE (8 << 20) @@ -576,10 +573,9 @@ int radeon_mode_dumb_mmap(struct drm_file *filp, * Semaphores. */ struct radeon_semaphore { - struct radeon_sa_bo *sa_bo; - signed waiters; - uint64_t gpu_addr; - struct radeon_fence *sync_to[RADEON_NUM_RINGS]; + struct radeon_sa_bo *sa_bo; + signed waiters; + uint64_t gpu_addr; }; int radeon_semaphore_create(struct radeon_device *rdev, @@ -588,19 +584,31 @@ bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring, struct radeon_semaphore *semaphore); bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring, struct radeon_semaphore *semaphore); -void radeon_semaphore_sync_fence(struct radeon_semaphore *semaphore, - struct radeon_fence *fence); -int radeon_semaphore_sync_resv(struct radeon_device *rdev, - struct radeon_semaphore *semaphore, - struct reservation_object *resv, - bool shared); -int radeon_semaphore_sync_rings(struct radeon_device *rdev, - struct radeon_semaphore *semaphore, - int waiting_ring); void radeon_semaphore_free(struct radeon_device *rdev, struct radeon_semaphore **semaphore, struct radeon_fence *fence); +/* + * Synchronization + */ +struct radeon_sync { + struct radeon_semaphore *semaphores[RADEON_NUM_SYNCS]; + struct radeon_fence *sync_to[RADEON_NUM_RINGS]; +}; + +void radeon_sync_create(struct radeon_sync *sync); +void radeon_sync_fence(struct radeon_sync *sync, + struct radeon_fence *fence); +int radeon_sync_resv(struct radeon_device *rdev, + struct radeon_sync *sync, + struct reservation_object *resv, + bool shared); +int radeon_sync_rings(struct radeon_device *rdev, + struct radeon_sync *sync, + int waiting_ring); +void radeon_sync_free(struct radeon_device *rdev, struct radeon_sync *sync, + struct radeon_fence *fence); + /* * GART structures, functions & helpers */ @@ -818,7 +826,7 @@ struct radeon_ib { struct radeon_fence *fence; struct radeon_vm *vm; bool is_const_ib; - struct radeon_semaphore *semaphore; + struct radeon_sync sync; }; struct radeon_ring { diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index a3e7aed7e680..30437aa00014 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -260,8 +260,8 @@ static int radeon_cs_sync_rings(struct radeon_cs_parser *p) continue; resv = p->relocs[i].robj->tbo.resv; - r = radeon_semaphore_sync_resv(p->rdev, p->ib.semaphore, resv, - p->relocs[i].tv.shared); + r = radeon_sync_resv(p->rdev, &p->ib.sync, resv, + p->relocs[i].tv.shared); if (r) break; @@ -285,9 +285,7 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) INIT_LIST_HEAD(&p->validated); p->idx = 0; p->ib.sa_bo = NULL; - p->ib.semaphore = NULL; p->const_ib.sa_bo = NULL; - p->const_ib.semaphore = NULL; p->chunk_ib_idx = -1; p->chunk_relocs_idx = -1; p->chunk_flags_idx = -1; @@ -582,7 +580,7 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, DRM_ERROR("Failed to sync rings: %i\n", r); goto out; } - radeon_semaphore_sync_fence(parser->ib.semaphore, vm->fence); + radeon_sync_fence(&parser->ib.sync, vm->fence); if ((rdev->family >= CHIP_TAHITI) && (parser->chunk_const_ib_idx != -1)) { diff --git a/drivers/gpu/drm/radeon/radeon_ib.c b/drivers/gpu/drm/radeon/radeon_ib.c index 3f39fcca4d07..56a17046a61f 100644 --- a/drivers/gpu/drm/radeon/radeon_ib.c +++ b/drivers/gpu/drm/radeon/radeon_ib.c @@ -64,10 +64,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring, return r; } - r = radeon_semaphore_create(rdev, &ib->semaphore); - if (r) { - return r; - } + radeon_sync_create(&ib->sync); ib->ring = ring; ib->fence = NULL; @@ -96,7 +93,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring, */ void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib) { - radeon_semaphore_free(rdev, &ib->semaphore, ib->fence); + radeon_sync_free(rdev, &ib->sync, ib->fence); radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence); radeon_fence_unref(&ib->fence); } @@ -145,11 +142,11 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, if (ib->vm) { struct radeon_fence *vm_id_fence; vm_id_fence = radeon_vm_grab_id(rdev, ib->vm, ib->ring); - radeon_semaphore_sync_fence(ib->semaphore, vm_id_fence); + radeon_sync_fence(&ib->sync, vm_id_fence); } /* sync with other rings */ - r = radeon_semaphore_sync_rings(rdev, ib->semaphore, ib->ring); + r = radeon_sync_rings(rdev, &ib->sync, ib->ring); if (r) { dev_err(rdev->dev, "failed to sync rings (%d)\n", r); radeon_ring_unlock_undo(rdev, ring); @@ -161,7 +158,7 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, if (const_ib) { radeon_ring_ib_execute(rdev, const_ib->ring, const_ib); - radeon_semaphore_free(rdev, &const_ib->semaphore, NULL); + radeon_sync_free(rdev, &const_ib->sync, NULL); } radeon_ring_ib_execute(rdev, ib->ring, ib); r = radeon_fence_emit(rdev, &ib->fence, ib->ring); diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c index 6deb08f045b7..e6ad54cdfa62 100644 --- a/drivers/gpu/drm/radeon/radeon_semaphore.c +++ b/drivers/gpu/drm/radeon/radeon_semaphore.c @@ -34,15 +34,14 @@ int radeon_semaphore_create(struct radeon_device *rdev, struct radeon_semaphore **semaphore) { - uint64_t *cpu_addr; - int i, r; + int r; *semaphore = kmalloc(sizeof(struct radeon_semaphore), GFP_KERNEL); if (*semaphore == NULL) { return -ENOMEM; } - r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &(*semaphore)->sa_bo, - 8 * RADEON_NUM_SYNCS, 8); + r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, + &(*semaphore)->sa_bo, 8, 8); if (r) { kfree(*semaphore); *semaphore = NULL; @@ -51,12 +50,7 @@ int radeon_semaphore_create(struct radeon_device *rdev, (*semaphore)->waiters = 0; (*semaphore)->gpu_addr = radeon_sa_bo_gpu_addr((*semaphore)->sa_bo); - cpu_addr = radeon_sa_bo_cpu_addr((*semaphore)->sa_bo); - for (i = 0; i < RADEON_NUM_SYNCS; ++i) - cpu_addr[i] = 0; - - for (i = 0; i < RADEON_NUM_RINGS; ++i) - (*semaphore)->sync_to[i] = NULL; + *((uint64_t *)radeon_sa_bo_cpu_addr((*semaphore)->sa_bo)) = 0; return 0; } @@ -95,146 +89,6 @@ bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ridx, return false; } -/** - * radeon_semaphore_sync_fence - use the semaphore to sync to a fence - * - * @semaphore: semaphore object to add fence to - * @fence: fence to sync to - * - * Sync to the fence using this semaphore object - */ -void radeon_semaphore_sync_fence(struct radeon_semaphore *semaphore, - struct radeon_fence *fence) -{ - struct radeon_fence *other; - - if (!fence) - return; - - other = semaphore->sync_to[fence->ring]; - semaphore->sync_to[fence->ring] = radeon_fence_later(fence, other); -} - -/** - * radeon_semaphore_sync_to - use the semaphore to sync to a reservation object - * - * @sema: semaphore object to add fence from reservation object to - * @resv: reservation object with embedded fence - * @shared: true if we should onyl sync to the exclusive fence - * - * Sync to the fence using this semaphore object - */ -int radeon_semaphore_sync_resv(struct radeon_device *rdev, - struct radeon_semaphore *sema, - struct reservation_object *resv, - bool shared) -{ - struct reservation_object_list *flist; - struct fence *f; - struct radeon_fence *fence; - unsigned i; - int r = 0; - - /* always sync to the exclusive fence */ - f = reservation_object_get_excl(resv); - fence = f ? to_radeon_fence(f) : NULL; - if (fence && fence->rdev == rdev) - radeon_semaphore_sync_fence(sema, fence); - else if (f) - r = fence_wait(f, true); - - flist = reservation_object_get_list(resv); - if (shared || !flist || r) - return r; - - for (i = 0; i < flist->shared_count; ++i) { - f = rcu_dereference_protected(flist->shared[i], - reservation_object_held(resv)); - fence = to_radeon_fence(f); - if (fence && fence->rdev == rdev) - radeon_semaphore_sync_fence(sema, fence); - else - r = fence_wait(f, true); - - if (r) - break; - } - return r; -} - -/** - * radeon_semaphore_sync_rings - sync ring to all registered fences - * - * @rdev: radeon_device pointer - * @semaphore: semaphore object to use for sync - * @ring: ring that needs sync - * - * Ensure that all registered fences are signaled before letting - * the ring continue. The caller must hold the ring lock. - */ -int radeon_semaphore_sync_rings(struct radeon_device *rdev, - struct radeon_semaphore *semaphore, - int ring) -{ - unsigned count = 0; - int i, r; - - for (i = 0; i < RADEON_NUM_RINGS; ++i) { - struct radeon_fence *fence = semaphore->sync_to[i]; - - /* check if we really need to sync */ - if (!radeon_fence_need_sync(fence, ring)) - continue; - - /* prevent GPU deadlocks */ - if (!rdev->ring[i].ready) { - dev_err(rdev->dev, "Syncing to a disabled ring!"); - return -EINVAL; - } - - if (++count > RADEON_NUM_SYNCS) { - /* not enough room, wait manually */ - r = radeon_fence_wait(fence, false); - if (r) - return r; - continue; - } - - /* allocate enough space for sync command */ - r = radeon_ring_alloc(rdev, &rdev->ring[i], 16); - if (r) { - return r; - } - - /* emit the signal semaphore */ - if (!radeon_semaphore_emit_signal(rdev, i, semaphore)) { - /* signaling wasn't successful wait manually */ - radeon_ring_undo(&rdev->ring[i]); - r = radeon_fence_wait(fence, false); - if (r) - return r; - continue; - } - - /* we assume caller has already allocated space on waiters ring */ - if (!radeon_semaphore_emit_wait(rdev, ring, semaphore)) { - /* waiting wasn't successful wait manually */ - radeon_ring_undo(&rdev->ring[i]); - r = radeon_fence_wait(fence, false); - if (r) - return r; - continue; - } - - radeon_ring_commit(rdev, &rdev->ring[i], false); - radeon_fence_note_sync(fence, ring); - - semaphore->gpu_addr += 8; - } - - return 0; -} - void radeon_semaphore_free(struct radeon_device *rdev, struct radeon_semaphore **semaphore, struct radeon_fence *fence) diff --git a/drivers/gpu/drm/radeon/radeon_sync.c b/drivers/gpu/drm/radeon/radeon_sync.c new file mode 100644 index 000000000000..ddd88fb28d99 --- /dev/null +++ b/drivers/gpu/drm/radeon/radeon_sync.c @@ -0,0 +1,213 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + */ +/* + * Authors: + * Christian König + */ + +#include +#include "radeon.h" +#include "radeon_trace.h" + +/** + * radeon_sync_create - zero init sync object + * + * @sync: sync object to initialize + * + * Just clear the sync object for now. + */ +void radeon_sync_create(struct radeon_sync *sync) +{ + unsigned i; + + for (i = 0; i < RADEON_NUM_SYNCS; ++i) + sync->semaphores[i] = NULL; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) + sync->sync_to[i] = NULL; +} + +/** + * radeon_sync_fence - use the semaphore to sync to a fence + * + * @sync: sync object to add fence to + * @fence: fence to sync to + * + * Sync to the fence using the semaphore objects + */ +void radeon_sync_fence(struct radeon_sync *sync, + struct radeon_fence *fence) +{ + struct radeon_fence *other; + + if (!fence) + return; + + other = sync->sync_to[fence->ring]; + sync->sync_to[fence->ring] = radeon_fence_later(fence, other); +} + +/** + * radeon_sync_resv - use the semaphores to sync to a reservation object + * + * @sync: sync object to add fences from reservation object to + * @resv: reservation object with embedded fence + * @shared: true if we should only sync to the exclusive fence + * + * Sync to the fence using the semaphore objects + */ +int radeon_sync_resv(struct radeon_device *rdev, + struct radeon_sync *sync, + struct reservation_object *resv, + bool shared) +{ + struct reservation_object_list *flist; + struct fence *f; + struct radeon_fence *fence; + unsigned i; + int r = 0; + + /* always sync to the exclusive fence */ + f = reservation_object_get_excl(resv); + fence = f ? to_radeon_fence(f) : NULL; + if (fence && fence->rdev == rdev) + radeon_sync_fence(sync, fence); + else if (f) + r = fence_wait(f, true); + + flist = reservation_object_get_list(resv); + if (shared || !flist || r) + return r; + + for (i = 0; i < flist->shared_count; ++i) { + f = rcu_dereference_protected(flist->shared[i], + reservation_object_held(resv)); + fence = to_radeon_fence(f); + if (fence && fence->rdev == rdev) + radeon_sync_fence(sync, fence); + else + r = fence_wait(f, true); + + if (r) + break; + } + return r; +} + +/** + * radeon_sync_rings - sync ring to all registered fences + * + * @rdev: radeon_device pointer + * @sync: sync object to use + * @ring: ring that needs sync + * + * Ensure that all registered fences are signaled before letting + * the ring continue. The caller must hold the ring lock. + */ +int radeon_sync_rings(struct radeon_device *rdev, + struct radeon_sync *sync, + int ring) +{ + unsigned count = 0; + int i, r; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + struct radeon_fence *fence = sync->sync_to[i]; + struct radeon_semaphore *semaphore; + + /* check if we really need to sync */ + if (!radeon_fence_need_sync(fence, ring)) + continue; + + /* prevent GPU deadlocks */ + if (!rdev->ring[i].ready) { + dev_err(rdev->dev, "Syncing to a disabled ring!"); + return -EINVAL; + } + + if (count >= RADEON_NUM_SYNCS) { + /* not enough room, wait manually */ + r = radeon_fence_wait(fence, false); + if (r) + return r; + continue; + } + r = radeon_semaphore_create(rdev, &semaphore); + if (r) + return r; + + sync->semaphores[count++] = semaphore; + + /* allocate enough space for sync command */ + r = radeon_ring_alloc(rdev, &rdev->ring[i], 16); + if (r) + return r; + + /* emit the signal semaphore */ + if (!radeon_semaphore_emit_signal(rdev, i, semaphore)) { + /* signaling wasn't successful wait manually */ + radeon_ring_undo(&rdev->ring[i]); + r = radeon_fence_wait(fence, false); + if (r) + return r; + continue; + } + + /* we assume caller has already allocated space on waiters ring */ + if (!radeon_semaphore_emit_wait(rdev, ring, semaphore)) { + /* waiting wasn't successful wait manually */ + radeon_ring_undo(&rdev->ring[i]); + r = radeon_fence_wait(fence, false); + if (r) + return r; + continue; + } + + radeon_ring_commit(rdev, &rdev->ring[i], false); + radeon_fence_note_sync(fence, ring); + } + + return 0; +} + +/** + * radeon_sync_free - free the sync object + * + * @rdev: radeon_device pointer + * @sync: sync object to use + * @fence: fence to use for the free + * + * Free the sync object by freeing all semaphores in it. + */ +void radeon_sync_free(struct radeon_device *rdev, + struct radeon_sync *sync, + struct radeon_fence *fence) +{ + unsigned i; + + for (i = 0; i < RADEON_NUM_SYNCS; ++i) + radeon_semaphore_free(rdev, &sync->semaphores[i], fence); +} diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index d8c49ad34dee..20ef8263d970 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -700,7 +700,7 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, if (ib.length_dw != 0) { radeon_asic_vm_pad_ib(rdev, &ib); - radeon_semaphore_sync_resv(rdev, ib.semaphore, pd->tbo.resv, false); + radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, false); WARN_ON(ib.length_dw > ndw); r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) { @@ -826,7 +826,7 @@ static void radeon_vm_update_ptes(struct radeon_device *rdev, unsigned nptes; uint64_t pte; - radeon_semaphore_sync_resv(rdev, ib->semaphore, pt->tbo.resv, false); + radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, false); if ((addr & ~mask) == (end & ~mask)) nptes = end - addr; diff --git a/drivers/gpu/drm/radeon/rv770_dma.c b/drivers/gpu/drm/radeon/rv770_dma.c index 7f34bad2e724..acff6e09cc40 100644 --- a/drivers/gpu/drm/radeon/rv770_dma.c +++ b/drivers/gpu/drm/radeon/rv770_dma.c @@ -44,31 +44,27 @@ struct radeon_fence *rv770_copy_dma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_dw, cur_size_in_dw; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4; num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFF); r = radeon_ring_lock(rdev, ring, num_loops * 5 + 8); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_dw = size_in_dw; @@ -87,12 +83,12 @@ struct radeon_fence *rv770_copy_dma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } diff --git a/drivers/gpu/drm/radeon/si_dma.c b/drivers/gpu/drm/radeon/si_dma.c index e8bc0a516b57..f5cc777e1c5f 100644 --- a/drivers/gpu/drm/radeon/si_dma.c +++ b/drivers/gpu/drm/radeon/si_dma.c @@ -226,31 +226,27 @@ struct radeon_fence *si_copy_dma(struct radeon_device *rdev, unsigned num_gpu_pages, struct reservation_object *resv) { - struct radeon_semaphore *sem = NULL; struct radeon_fence *fence; + struct radeon_sync sync; int ring_index = rdev->asic->copy.dma_ring_index; struct radeon_ring *ring = &rdev->ring[ring_index]; u32 size_in_bytes, cur_size_in_bytes; int i, num_loops; int r = 0; - r = radeon_semaphore_create(rdev, &sem); - if (r) { - DRM_ERROR("radeon: moving bo (%d).\n", r); - return ERR_PTR(r); - } + radeon_sync_create(&sync); size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT); num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff); r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11); if (r) { DRM_ERROR("radeon: moving bo (%d).\n", r); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } - radeon_semaphore_sync_resv(rdev, sem, resv, false); - radeon_semaphore_sync_rings(rdev, sem, ring->idx); + radeon_sync_resv(rdev, &sync, resv, false); + radeon_sync_rings(rdev, &sync, ring->idx); for (i = 0; i < num_loops; i++) { cur_size_in_bytes = size_in_bytes; @@ -269,12 +265,12 @@ struct radeon_fence *si_copy_dma(struct radeon_device *rdev, r = radeon_fence_emit(rdev, &fence, ring->idx); if (r) { radeon_ring_unlock_undo(rdev, ring); - radeon_semaphore_free(rdev, &sem, NULL); + radeon_sync_free(rdev, &sync, NULL); return ERR_PTR(r); } radeon_ring_unlock_commit(rdev, ring, false); - radeon_semaphore_free(rdev, &sem, fence); + radeon_sync_free(rdev, &sync, fence); return fence; } From 587cdda8f739f4c57c91d3f73a1d5b2851a86cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:23 +0100 Subject: [PATCH 16/22] drm/radeon: fence PT updates manually v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to add the real execution fence as shared. v2: fix typo Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_object.c | 19 ++++++++ drivers/gpu/drm/radeon/radeon_object.h | 2 + drivers/gpu/drm/radeon/radeon_vm.c | 65 ++++++++++++++++---------- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index 33e6c7a89c32..686e450199c5 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -818,3 +818,22 @@ int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait) ttm_bo_unreserve(&bo->tbo); return r; } + +/** + * radeon_bo_fence - add fence to buffer object + * + * @bo: buffer object in question + * @fence: fence to add + * @shared: true if fence should be added shared + * + */ +void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence, + bool shared) +{ + struct reservation_object *resv = bo->tbo.resv; + + if (shared) + reservation_object_add_shared_fence(resv, &fence->base); + else + reservation_object_add_excl_fence(resv, &fence->base); +} diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h index 1b8ec7917154..3b0b377f76cb 100644 --- a/drivers/gpu/drm/radeon/radeon_object.h +++ b/drivers/gpu/drm/radeon/radeon_object.h @@ -155,6 +155,8 @@ extern void radeon_bo_move_notify(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem); extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo); extern int radeon_bo_get_surface_reg(struct radeon_bo *bo); +extern void radeon_bo_fence(struct radeon_bo *bo, struct radeon_fence *fence, + bool shared); /* * sub allocation diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 20ef8263d970..4ca2779ed828 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -143,7 +143,7 @@ struct radeon_cs_reloc *radeon_vm_get_bos(struct radeon_device *rdev, list[0].prefered_domains = RADEON_GEM_DOMAIN_VRAM; list[0].allowed_domains = RADEON_GEM_DOMAIN_VRAM; list[0].tv.bo = &vm->page_directory->tbo; - list[0].tv.shared = false; + list[0].tv.shared = true; list[0].tiling_flags = 0; list[0].handle = 0; list_add(&list[0].tv.head, head); @@ -157,7 +157,7 @@ struct radeon_cs_reloc *radeon_vm_get_bos(struct radeon_device *rdev, list[idx].prefered_domains = RADEON_GEM_DOMAIN_VRAM; list[idx].allowed_domains = RADEON_GEM_DOMAIN_VRAM; list[idx].tv.bo = &list[idx].robj->tbo; - list[idx].tv.shared = false; + list[idx].tv.shared = true; list[idx].tiling_flags = 0; list[idx].handle = 0; list_add(&list[idx++].tv.head, head); @@ -388,35 +388,25 @@ static void radeon_vm_set_pages(struct radeon_device *rdev, static int radeon_vm_clear_bo(struct radeon_device *rdev, struct radeon_bo *bo) { - struct ttm_validate_buffer tv; - struct ww_acquire_ctx ticket; - struct list_head head; struct radeon_ib ib; unsigned entries; uint64_t addr; int r; - memset(&tv, 0, sizeof(tv)); - tv.bo = &bo->tbo; - tv.shared = false; - - INIT_LIST_HEAD(&head); - list_add(&tv.head, &head); - - r = ttm_eu_reserve_buffers(&ticket, &head, true); - if (r) + r = radeon_bo_reserve(bo, false); + if (r) return r; - r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); - if (r) - goto error; + r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); + if (r) + goto error_unreserve; addr = radeon_bo_gpu_offset(bo); entries = radeon_bo_size(bo) / 8; r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, 256); if (r) - goto error; + goto error_unreserve; ib.length_dw = 0; @@ -426,15 +416,15 @@ static int radeon_vm_clear_bo(struct radeon_device *rdev, r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) - goto error; + goto error_free; - ttm_eu_fence_buffer_objects(&ticket, &head, &ib.fence->base); + radeon_bo_fence(bo, ib.fence, false); + +error_free: radeon_ib_free(rdev, &ib); - return 0; - -error: - ttm_eu_backoff_reservation(&ticket, &head); +error_unreserve: + radeon_bo_unreserve(bo); return r; } @@ -707,6 +697,7 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, radeon_ib_free(rdev, &ib); return r; } + radeon_bo_fence(pd, ib.fence, false); radeon_fence_unref(&vm->fence); vm->fence = radeon_fence_ref(ib.fence); radeon_fence_unref(&vm->last_flush); @@ -862,6 +853,31 @@ static void radeon_vm_update_ptes(struct radeon_device *rdev, } } +/** + * radeon_vm_fence_pts - fence page tables after an update + * + * @vm: requested vm + * @start: start of GPU address range + * @end: end of GPU address range + * @fence: fence to use + * + * Fence the page tables in the range @start - @end (cayman+). + * + * Global and local mutex must be locked! + */ +static void radeon_vm_fence_pts(struct radeon_vm *vm, + uint64_t start, uint64_t end, + struct radeon_fence *fence) +{ + unsigned i; + + start >>= radeon_vm_block_size; + end >>= radeon_vm_block_size; + + for (i = start; i <= end; ++i) + radeon_bo_fence(vm->page_tables[i].bo, fence, false); +} + /** * radeon_vm_bo_update - map a bo into the vm page table * @@ -973,6 +989,7 @@ int radeon_vm_bo_update(struct radeon_device *rdev, radeon_ib_free(rdev, &ib); return r; } + radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence); radeon_fence_unref(&vm->fence); vm->fence = radeon_fence_ref(ib.fence); radeon_ib_free(rdev, &ib); From ad1a58a45ae9a35aaf68b27d21600889360680f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:24 +0100 Subject: [PATCH 17/22] drm/radeon: track VM update fences separately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note for each fence if it's a VM page table update or not. This allows us to determine the last VM update in a sync object and so to figure out if we need to flush the TLB or not. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon.h | 18 ++++++++++-------- drivers/gpu/drm/radeon/radeon_fence.c | 1 + drivers/gpu/drm/radeon/radeon_ib.c | 3 ++- drivers/gpu/drm/radeon/radeon_sync.c | 7 +++++++ drivers/gpu/drm/radeon/radeon_vm.c | 25 +++++++++++++------------ 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 7cda6d77aeb0..61b2eeabf7a4 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -360,14 +360,15 @@ struct radeon_fence_driver { }; struct radeon_fence { - struct fence base; + struct fence base; - struct radeon_device *rdev; - uint64_t seq; + struct radeon_device *rdev; + uint64_t seq; /* RB, DMA, etc. */ - unsigned ring; + unsigned ring; + bool is_vm_update; - wait_queue_t fence_wake; + wait_queue_t fence_wake; }; int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring); @@ -594,6 +595,7 @@ void radeon_semaphore_free(struct radeon_device *rdev, struct radeon_sync { struct radeon_semaphore *semaphores[RADEON_NUM_SYNCS]; struct radeon_fence *sync_to[RADEON_NUM_RINGS]; + struct radeon_fence *last_vm_update; }; void radeon_sync_create(struct radeon_sync *sync); @@ -926,8 +928,8 @@ struct radeon_vm { struct mutex mutex; /* last fence for cs using this vm */ struct radeon_fence *fence; - /* last flush or NULL if we still need to flush */ - struct radeon_fence *last_flush; + /* last flushed PD/PT update */ + struct radeon_fence *flushed_updates; /* last use of vmid */ struct radeon_fence *last_id_use; }; @@ -2975,7 +2977,7 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, struct radeon_vm *vm, int ring); void radeon_vm_flush(struct radeon_device *rdev, struct radeon_vm *vm, - int ring); + int ring, struct radeon_fence *fence); void radeon_vm_fence(struct radeon_device *rdev, struct radeon_vm *vm, struct radeon_fence *fence); diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 995167025282..d13d1b5a859f 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -140,6 +140,7 @@ int radeon_fence_emit(struct radeon_device *rdev, (*fence)->rdev = rdev; (*fence)->seq = seq; (*fence)->ring = ring; + (*fence)->is_vm_update = false; fence_init(&(*fence)->base, &radeon_fence_ops, &rdev->fence_queue.lock, rdev->fence_context + ring, seq); radeon_fence_ring_emit(rdev, ring, *fence); diff --git a/drivers/gpu/drm/radeon/radeon_ib.c b/drivers/gpu/drm/radeon/radeon_ib.c index 56a17046a61f..c39ce1f05703 100644 --- a/drivers/gpu/drm/radeon/radeon_ib.c +++ b/drivers/gpu/drm/radeon/radeon_ib.c @@ -154,7 +154,8 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib, } if (ib->vm) - radeon_vm_flush(rdev, ib->vm, ib->ring); + radeon_vm_flush(rdev, ib->vm, ib->ring, + ib->sync.last_vm_update); if (const_ib) { radeon_ring_ib_execute(rdev, const_ib->ring, const_ib); diff --git a/drivers/gpu/drm/radeon/radeon_sync.c b/drivers/gpu/drm/radeon/radeon_sync.c index ddd88fb28d99..02ac8a1de4ff 100644 --- a/drivers/gpu/drm/radeon/radeon_sync.c +++ b/drivers/gpu/drm/radeon/radeon_sync.c @@ -48,6 +48,8 @@ void radeon_sync_create(struct radeon_sync *sync) for (i = 0; i < RADEON_NUM_RINGS; ++i) sync->sync_to[i] = NULL; + + sync->last_vm_update = NULL; } /** @@ -68,6 +70,11 @@ void radeon_sync_fence(struct radeon_sync *sync, other = sync->sync_to[fence->ring]; sync->sync_to[fence->ring] = radeon_fence_later(fence, other); + + if (fence->is_vm_update) { + other = sync->last_vm_update; + sync->last_vm_update = radeon_fence_later(fence, other); + } } /** diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 4ca2779ed828..6ff5741ea403 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -190,7 +190,7 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, return NULL; /* we definately need to flush */ - radeon_fence_unref(&vm->last_flush); + vm->pd_gpu_addr = ~0ll; /* skip over VMID 0, since it is the system VM */ for (i = 1; i < rdev->vm_manager.nvm; ++i) { @@ -228,6 +228,7 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, * @rdev: radeon_device pointer * @vm: vm we want to flush * @ring: ring to use for flush + * @updates: last vm update that is waited for * * Flush the vm (cayman+). * @@ -235,13 +236,16 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, */ void radeon_vm_flush(struct radeon_device *rdev, struct radeon_vm *vm, - int ring) + int ring, struct radeon_fence *updates) { uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory); - /* if we can't remember our last VM flush then flush now! */ - if (!vm->last_flush || pd_addr != vm->pd_gpu_addr) { + if (pd_addr != vm->pd_gpu_addr || !vm->flushed_updates || + radeon_fence_is_earlier(vm->flushed_updates, updates)) { + trace_radeon_vm_flush(pd_addr, ring, vm->id); + radeon_fence_unref(&vm->flushed_updates); + vm->flushed_updates = radeon_fence_ref(updates); vm->pd_gpu_addr = pd_addr; radeon_ring_vm_flush(rdev, &rdev->ring[ring], vm->id, vm->pd_gpu_addr); @@ -272,10 +276,6 @@ void radeon_vm_fence(struct radeon_device *rdev, radeon_fence_unref(&vm->last_id_use); vm->last_id_use = radeon_fence_ref(fence); - - /* we just flushed the VM, remember that */ - if (!vm->last_flush) - vm->last_flush = radeon_fence_ref(fence); } /** @@ -418,6 +418,7 @@ static int radeon_vm_clear_bo(struct radeon_device *rdev, if (r) goto error_free; + ib.fence->is_vm_update = true; radeon_bo_fence(bo, ib.fence, false); error_free: @@ -697,10 +698,10 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, radeon_ib_free(rdev, &ib); return r; } + ib.fence->is_vm_update = true; radeon_bo_fence(pd, ib.fence, false); radeon_fence_unref(&vm->fence); vm->fence = radeon_fence_ref(ib.fence); - radeon_fence_unref(&vm->last_flush); } radeon_ib_free(rdev, &ib); @@ -989,11 +990,11 @@ int radeon_vm_bo_update(struct radeon_device *rdev, radeon_ib_free(rdev, &ib); return r; } + ib.fence->is_vm_update = true; radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence); radeon_fence_unref(&vm->fence); vm->fence = radeon_fence_ref(ib.fence); radeon_ib_free(rdev, &ib); - radeon_fence_unref(&vm->last_flush); return 0; } @@ -1124,7 +1125,7 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) vm->id = 0; vm->ib_bo_va = NULL; vm->fence = NULL; - vm->last_flush = NULL; + vm->flushed_updates = NULL; vm->last_id_use = NULL; mutex_init(&vm->mutex); vm->va = RB_ROOT; @@ -1196,7 +1197,7 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) radeon_bo_unref(&vm->page_directory); radeon_fence_unref(&vm->fence); - radeon_fence_unref(&vm->last_flush); + radeon_fence_unref(&vm->flushed_updates); radeon_fence_unref(&vm->last_id_use); mutex_destroy(&vm->mutex); From 7c42bc1aa23fc061a6ff9c2bd9208817bd54ea04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:25 +0100 Subject: [PATCH 18/22] drm/radeon: use one VMID for each ring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use multiple VMIDs for each VM, one for each ring. That allows us to execute flushes separately on each ring, still not ideal cause in a lot of cases rings can share IDs. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/cik.c | 4 +- drivers/gpu/drm/radeon/cik_sdma.c | 2 +- drivers/gpu/drm/radeon/ni.c | 6 +-- drivers/gpu/drm/radeon/ni_dma.c | 3 +- drivers/gpu/drm/radeon/radeon.h | 36 ++++++++++-------- drivers/gpu/drm/radeon/radeon_vm.c | 59 ++++++++++++++++++------------ drivers/gpu/drm/radeon/si.c | 6 +-- 7 files changed, 68 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c index 755923bc6786..3deeed33322f 100644 --- a/drivers/gpu/drm/radeon/cik.c +++ b/drivers/gpu/drm/radeon/cik.c @@ -4066,6 +4066,7 @@ struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; u32 header, control = INDIRECT_BUFFER_VALID; if (ib->is_const_ib) { @@ -4094,8 +4095,7 @@ void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); } - control |= ib->length_dw | - (ib->vm ? (ib->vm->id << 24) : 0); + control |= ib->length_dw | (vm_id << 24); radeon_ring_write(ring, header); radeon_ring_write(ring, diff --git a/drivers/gpu/drm/radeon/cik_sdma.c b/drivers/gpu/drm/radeon/cik_sdma.c index 604e2e770951..54b98379188d 100644 --- a/drivers/gpu/drm/radeon/cik_sdma.c +++ b/drivers/gpu/drm/radeon/cik_sdma.c @@ -134,7 +134,7 @@ void cik_sdma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; - u32 extra_bits = (ib->vm ? ib->vm->id : 0) & 0xf; + u32 extra_bits = (ib->vm ? ib->vm->ids[ib->ring].id : 0) & 0xf; if (rdev->wb.enabled) { u32 next_rptr = ring->wptr + 5; diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index bee432d3dd30..360de9f1f491 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c @@ -1373,6 +1373,7 @@ void cayman_fence_ring_emit(struct radeon_device *rdev, void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA | PACKET3_SH_ACTION_ENA; @@ -1395,15 +1396,14 @@ void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) #endif (ib->gpu_addr & 0xFFFFFFFC)); radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF); - radeon_ring_write(ring, ib->length_dw | - (ib->vm ? (ib->vm->id << 24) : 0)); + radeon_ring_write(ring, ib->length_dw | (vm_id << 24)); /* flush read cache over gart for this vmid */ radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl); radeon_ring_write(ring, 0xFFFFFFFF); radeon_ring_write(ring, 0); - radeon_ring_write(ring, ((ib->vm ? ib->vm->id : 0) << 24) | 10); /* poll interval */ + radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */ } static void cayman_cp_enable(struct radeon_device *rdev, bool enable) diff --git a/drivers/gpu/drm/radeon/ni_dma.c b/drivers/gpu/drm/radeon/ni_dma.c index 5a72404c9d5e..50f88611ff60 100644 --- a/drivers/gpu/drm/radeon/ni_dma.c +++ b/drivers/gpu/drm/radeon/ni_dma.c @@ -123,6 +123,7 @@ void cayman_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; if (rdev->wb.enabled) { u32 next_rptr = ring->wptr + 4; @@ -140,7 +141,7 @@ void cayman_dma_ring_ib_execute(struct radeon_device *rdev, */ while ((ring->wptr & 7) != 5) radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0)); - radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0)); + radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vm_id, 0)); radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0)); radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF)); diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 61b2eeabf7a4..79f5f5bf4c0c 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -905,33 +905,39 @@ struct radeon_vm_pt { uint64_t addr; }; +struct radeon_vm_id { + unsigned id; + uint64_t pd_gpu_addr; + /* last flushed PD/PT update */ + struct radeon_fence *flushed_updates; + /* last use of vmid */ + struct radeon_fence *last_id_use; +}; + struct radeon_vm { - struct rb_root va; - unsigned id; + struct rb_root va; /* BOs moved, but not yet updated in the PT */ - struct list_head invalidated; + struct list_head invalidated; /* BOs freed, but not yet updated in the PT */ - struct list_head freed; + struct list_head freed; /* contains the page directory */ - struct radeon_bo *page_directory; - uint64_t pd_gpu_addr; - unsigned max_pde_used; + struct radeon_bo *page_directory; + unsigned max_pde_used; /* array of page tables, one for each page directory entry */ - struct radeon_vm_pt *page_tables; + struct radeon_vm_pt *page_tables; - struct radeon_bo_va *ib_bo_va; + struct radeon_bo_va *ib_bo_va; - struct mutex mutex; + struct mutex mutex; /* last fence for cs using this vm */ - struct radeon_fence *fence; - /* last flushed PD/PT update */ - struct radeon_fence *flushed_updates; - /* last use of vmid */ - struct radeon_fence *last_id_use; + struct radeon_fence *fence; + + /* for id and flush management per ring */ + struct radeon_vm_id ids[RADEON_NUM_RINGS]; }; struct radeon_vm_manager { diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 6ff5741ea403..e38efe4962f3 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -182,15 +182,18 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, struct radeon_vm *vm, int ring) { struct radeon_fence *best[RADEON_NUM_RINGS] = {}; + struct radeon_vm_id *vm_id = &vm->ids[ring]; + unsigned choices[2] = {}; unsigned i; /* check if the id is still valid */ - if (vm->last_id_use && vm->last_id_use == rdev->vm_manager.active[vm->id]) + if (vm_id->id && vm_id->last_id_use && + vm_id->last_id_use == rdev->vm_manager.active[vm_id->id]) return NULL; /* we definately need to flush */ - vm->pd_gpu_addr = ~0ll; + vm_id->pd_gpu_addr = ~0ll; /* skip over VMID 0, since it is the system VM */ for (i = 1; i < rdev->vm_manager.nvm; ++i) { @@ -198,8 +201,8 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, if (fence == NULL) { /* found a free one */ - vm->id = i; - trace_radeon_vm_grab_id(vm->id, ring); + vm_id->id = i; + trace_radeon_vm_grab_id(i, ring); return NULL; } @@ -211,8 +214,8 @@ struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, for (i = 0; i < 2; ++i) { if (choices[i]) { - vm->id = choices[i]; - trace_radeon_vm_grab_id(vm->id, ring); + vm_id->id = choices[i]; + trace_radeon_vm_grab_id(choices[i], ring); return rdev->vm_manager.active[choices[i]]; } } @@ -239,16 +242,18 @@ void radeon_vm_flush(struct radeon_device *rdev, int ring, struct radeon_fence *updates) { uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory); + struct radeon_vm_id *vm_id = &vm->ids[ring]; - if (pd_addr != vm->pd_gpu_addr || !vm->flushed_updates || - radeon_fence_is_earlier(vm->flushed_updates, updates)) { + if (pd_addr != vm_id->pd_gpu_addr || !vm_id->flushed_updates || + radeon_fence_is_earlier(vm_id->flushed_updates, updates)) { - trace_radeon_vm_flush(pd_addr, ring, vm->id); - radeon_fence_unref(&vm->flushed_updates); - vm->flushed_updates = radeon_fence_ref(updates); - vm->pd_gpu_addr = pd_addr; + trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id); + radeon_fence_unref(&vm_id->flushed_updates); + vm_id->flushed_updates = radeon_fence_ref(updates); + vm_id->pd_gpu_addr = pd_addr; radeon_ring_vm_flush(rdev, &rdev->ring[ring], - vm->id, vm->pd_gpu_addr); + vm_id->id, vm_id->pd_gpu_addr); + } } @@ -268,14 +273,16 @@ void radeon_vm_fence(struct radeon_device *rdev, struct radeon_vm *vm, struct radeon_fence *fence) { + unsigned vm_id = vm->ids[fence->ring].id; + radeon_fence_unref(&vm->fence); vm->fence = radeon_fence_ref(fence); - radeon_fence_unref(&rdev->vm_manager.active[vm->id]); - rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); + radeon_fence_unref(&rdev->vm_manager.active[vm_id]); + rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence); - radeon_fence_unref(&vm->last_id_use); - vm->last_id_use = radeon_fence_ref(fence); + radeon_fence_unref(&vm->ids[fence->ring].last_id_use); + vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence); } /** @@ -1120,13 +1127,16 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) const unsigned align = min(RADEON_VM_PTB_ALIGN_SIZE, RADEON_VM_PTE_COUNT * 8); unsigned pd_size, pd_entries, pts_size; - int r; + int i, r; - vm->id = 0; vm->ib_bo_va = NULL; vm->fence = NULL; - vm->flushed_updates = NULL; - vm->last_id_use = NULL; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + vm->ids[i].id = 0; + vm->ids[i].flushed_updates = NULL; + vm->ids[i].last_id_use = NULL; + } mutex_init(&vm->mutex); vm->va = RB_ROOT; INIT_LIST_HEAD(&vm->invalidated); @@ -1197,8 +1207,11 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) radeon_bo_unref(&vm->page_directory); radeon_fence_unref(&vm->fence); - radeon_fence_unref(&vm->flushed_updates); - radeon_fence_unref(&vm->last_id_use); + + for (i = 0; i < RADEON_NUM_RINGS; ++i) { + radeon_fence_unref(&vm->ids[i].flushed_updates); + radeon_fence_unref(&vm->ids[i].last_id_use); + } mutex_destroy(&vm->mutex); } diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c index e91968b04154..14896ce76324 100644 --- a/drivers/gpu/drm/radeon/si.c +++ b/drivers/gpu/drm/radeon/si.c @@ -3362,6 +3362,7 @@ void si_fence_ring_emit(struct radeon_device *rdev, void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) { struct radeon_ring *ring = &rdev->ring[ib->ring]; + unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0; u32 header; if (ib->is_const_ib) { @@ -3397,14 +3398,13 @@ void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) #endif (ib->gpu_addr & 0xFFFFFFFC)); radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); - radeon_ring_write(ring, ib->length_dw | - (ib->vm ? (ib->vm->id << 24) : 0)); + radeon_ring_write(ring, ib->length_dw | (vm_id << 24)); if (!ib->is_const_ib) { /* flush read cache over gart for this vmid */ radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); - radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); + radeon_ring_write(ring, vm_id); radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | PACKET3_TC_ACTION_ENA | From 94214635f09c9211023730acdff9342fdf100aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:26 +0100 Subject: [PATCH 19/22] drm/radeon: fence BO_VAs manually MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to finally remove the VM fence and so allow concurrent use of it from different engines. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon.h | 7 +++---- drivers/gpu/drm/radeon/radeon_cs.c | 6 +++++- drivers/gpu/drm/radeon/radeon_vm.c | 17 ++++++----------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 79f5f5bf4c0c..3207bb60715e 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -456,6 +456,7 @@ struct radeon_bo_va { struct list_head bo_list; uint32_t flags; uint64_t addr; + struct radeon_fence *last_pt_update; unsigned ref_count; /* protected by vm mutex */ @@ -915,6 +916,8 @@ struct radeon_vm_id { }; struct radeon_vm { + struct mutex mutex; + struct rb_root va; /* BOs moved, but not yet updated in the PT */ @@ -932,10 +935,6 @@ struct radeon_vm { struct radeon_bo_va *ib_bo_va; - struct mutex mutex; - /* last fence for cs using this vm */ - struct radeon_fence *fence; - /* for id and flush management per ring */ struct radeon_vm_id ids[RADEON_NUM_RINGS]; }; diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c index 30437aa00014..75f22e5e999f 100644 --- a/drivers/gpu/drm/radeon/radeon_cs.c +++ b/drivers/gpu/drm/radeon/radeon_cs.c @@ -505,6 +505,9 @@ static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p, if (r) return r; + radeon_sync_resv(p->rdev, &p->ib.sync, vm->page_directory->tbo.resv, + true); + r = radeon_vm_clear_freed(rdev, vm); if (r) return r; @@ -536,6 +539,8 @@ static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p, r = radeon_vm_bo_update(rdev, bo_va, &bo->tbo.mem); if (r) return r; + + radeon_sync_fence(&p->ib.sync, bo_va->last_pt_update); } return radeon_vm_clear_invalids(rdev, vm); @@ -580,7 +585,6 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, DRM_ERROR("Failed to sync rings: %i\n", r); goto out; } - radeon_sync_fence(&parser->ib.sync, vm->fence); if ((rdev->family >= CHIP_TAHITI) && (parser->chunk_const_ib_idx != -1)) { diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index e38efe4962f3..f45761469e95 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -275,9 +275,6 @@ void radeon_vm_fence(struct radeon_device *rdev, { unsigned vm_id = vm->ids[fence->ring].id; - radeon_fence_unref(&vm->fence); - vm->fence = radeon_fence_ref(fence); - radeon_fence_unref(&rdev->vm_manager.active[vm_id]); rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence); @@ -707,8 +704,6 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, } ib.fence->is_vm_update = true; radeon_bo_fence(pd, ib.fence, false); - radeon_fence_unref(&vm->fence); - vm->fence = radeon_fence_ref(ib.fence); } radeon_ib_free(rdev, &ib); @@ -999,8 +994,8 @@ int radeon_vm_bo_update(struct radeon_device *rdev, } ib.fence->is_vm_update = true; radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence); - radeon_fence_unref(&vm->fence); - vm->fence = radeon_fence_ref(ib.fence); + radeon_fence_unref(&bo_va->last_pt_update); + bo_va->last_pt_update = radeon_fence_ref(ib.fence); radeon_ib_free(rdev, &ib); return 0; @@ -1026,6 +1021,7 @@ int radeon_vm_clear_freed(struct radeon_device *rdev, list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) { r = radeon_vm_bo_update(rdev, bo_va, NULL); radeon_bo_unref(&bo_va->bo); + radeon_fence_unref(&bo_va->last_pt_update); kfree(bo_va); if (r) return r; @@ -1084,6 +1080,7 @@ void radeon_vm_bo_rmv(struct radeon_device *rdev, bo_va->bo = radeon_bo_ref(bo_va->bo); list_add(&bo_va->vm_status, &vm->freed); } else { + radeon_fence_unref(&bo_va->last_pt_update); kfree(bo_va); } @@ -1130,8 +1127,6 @@ int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) int i, r; vm->ib_bo_va = NULL; - vm->fence = NULL; - for (i = 0; i < RADEON_NUM_RINGS; ++i) { vm->ids[i].id = 0; vm->ids[i].flushed_updates = NULL; @@ -1192,11 +1187,13 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) if (!r) { list_del_init(&bo_va->bo_list); radeon_bo_unreserve(bo_va->bo); + radeon_fence_unref(&bo_va->last_pt_update); kfree(bo_va); } } list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) { radeon_bo_unref(&bo_va->bo); + radeon_fence_unref(&bo_va->last_pt_update); kfree(bo_va); } @@ -1206,8 +1203,6 @@ void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) radeon_bo_unref(&vm->page_directory); - radeon_fence_unref(&vm->fence); - for (i = 0; i < RADEON_NUM_RINGS; ++i) { radeon_fence_unref(&vm->ids[i].flushed_updates); radeon_fence_unref(&vm->ids[i].last_id_use); From 43ac885764a589626018cc66973507213cef96ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:27 +0100 Subject: [PATCH 20/22] drm/radeon: sync PD updates as shared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We never invalidate PD entries and making them valid can run with other users in parallel. Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_vm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index f45761469e95..470451c1de62 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -695,7 +695,7 @@ int radeon_vm_update_page_directory(struct radeon_device *rdev, if (ib.length_dw != 0) { radeon_asic_vm_pad_ib(rdev, &ib); - radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, false); + radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, true); WARN_ON(ib.length_dw > ndw); r = radeon_ib_schedule(rdev, &ib, NULL, false); if (r) { From d1968e1db892e53b5783570a5738477b2a51080b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Nov 2014 14:01:28 +0100 Subject: [PATCH 21/22] drm/radeon: sync PT updates as shared v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only invalidating PTEs needs to be executed synchronized to using the PT. v2: fix sync to uses Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_vm.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c index 470451c1de62..0b10f3a03ce2 100644 --- a/drivers/gpu/drm/radeon/radeon_vm.c +++ b/drivers/gpu/drm/radeon/radeon_vm.c @@ -820,7 +820,7 @@ static void radeon_vm_update_ptes(struct radeon_device *rdev, unsigned nptes; uint64_t pte; - radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, false); + radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, true); if ((addr & ~mask) == (end & ~mask)) nptes = end - addr; @@ -980,6 +980,13 @@ int radeon_vm_bo_update(struct radeon_device *rdev, return r; ib.length_dw = 0; + if (!(bo_va->flags & RADEON_VM_PAGE_VALID)) { + unsigned i; + + for (i = 0; i < RADEON_NUM_RINGS; ++i) + radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use); + } + radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start, bo_va->it.last + 1, addr, radeon_vm_page_flags(bo_va->flags)); From 2f2624c23511b4bf0dd3d4c5ae167715513f351d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 12 Sep 2014 12:25:45 +0200 Subject: [PATCH 22/22] drm/radeon: update the VM after setting BO address v4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way the necessary VM update is kicked off immediately if all BOs involved are in GPU accessible memory. v2: fix vm lock v3: immediately update unmaps as well v4: use drm_free_large instead of kfree Tested-by: Kai Wasserbäch Signed-off-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_gem.c | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c index f752c7f56015..a748a64b38b9 100644 --- a/drivers/gpu/drm/radeon/radeon_gem.c +++ b/drivers/gpu/drm/radeon/radeon_gem.c @@ -518,6 +518,68 @@ out: return r; } +/** + * radeon_gem_va_update_vm -update the bo_va in its VM + * + * @rdev: radeon_device pointer + * @bo_va: bo_va to update + * + * Update the bo_va directly after setting it's address. Errors are not + * vital here, so they are not reported back to userspace. + */ +static void radeon_gem_va_update_vm(struct radeon_device *rdev, + struct radeon_bo_va *bo_va) +{ + struct ttm_validate_buffer tv, *entry; + struct radeon_cs_reloc *vm_bos; + struct ww_acquire_ctx ticket; + struct list_head list; + unsigned domain; + int r; + + INIT_LIST_HEAD(&list); + + tv.bo = &bo_va->bo->tbo; + tv.shared = true; + list_add(&tv.head, &list); + + vm_bos = radeon_vm_get_bos(rdev, bo_va->vm, &list); + if (!vm_bos) + return; + + r = ttm_eu_reserve_buffers(&ticket, &list, true); + if (r) + goto error_free; + + list_for_each_entry(entry, &list, head) { + domain = radeon_mem_type_to_domain(entry->bo->mem.mem_type); + /* if anything is swapped out don't swap it in here, + just abort and wait for the next CS */ + if (domain == RADEON_GEM_DOMAIN_CPU) + goto error_unreserve; + } + + mutex_lock(&bo_va->vm->mutex); + r = radeon_vm_clear_freed(rdev, bo_va->vm); + if (r) + goto error_unlock; + + if (bo_va->it.start) + r = radeon_vm_bo_update(rdev, bo_va, &bo_va->bo->tbo.mem); + +error_unlock: + mutex_unlock(&bo_va->vm->mutex); + +error_unreserve: + ttm_eu_backoff_reservation(&ticket, &list); + +error_free: + drm_free_large(vm_bos); + + if (r) + DRM_ERROR("Couldn't update BO_VA (%d)\n", r); +} + int radeon_gem_va_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { @@ -612,6 +674,8 @@ int radeon_gem_va_ioctl(struct drm_device *dev, void *data, default: break; } + if (!r) + radeon_gem_va_update_vm(rdev, bo_va); args->operation = RADEON_VA_RESULT_OK; if (r) { args->operation = RADEON_VA_RESULT_ERROR;