From 5e85eba6f50dc288c22083a7e213152bcc4b8208 Mon Sep 17 00:00:00 2001 From: Vidya Sagar Date: Tue, 13 Sep 2022 18:48:21 +0530 Subject: [PATCH 1/5] PCI/ASPM: Refactor L1 PM Substates Control Register programming Refactor the code to extract the common code to program Control Registers 1 and 2 of the L1 PM Substates capability to a new function aspm_program_l1ss() and call it for both parent and child devices. [bhelgaas: squash in update to preserve fields we're not updating from https://lore.kernel.org/r/36fa13c5-e0f8-022f-77f7-7908e4df98b8@nvidia.com] Link: https://lore.kernel.org/r/20220913131822.16557-2-vidyas@nvidia.com Signed-off-by: Vidya Sagar Signed-off-by: Bjorn Helgaas --- drivers/pci/pcie/aspm.c | 72 ++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index a8aec190986c..b4bdadc4ac35 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -455,6 +455,31 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, pci_write_config_dword(pdev, pos, val); } +static void aspm_program_l1ss(struct pci_dev *dev, u32 ctl1, u32 ctl2) +{ + u16 l1ss = dev->l1ss; + u32 l1_2_enable; + + /* + * Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be + * programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1. + */ + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL2, ctl2); + + /* + * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in + * PCI_L1SS_CTL1 must be programmed *before* setting the L1.2 + * enable bits, even though they're all in PCI_L1SS_CTL1. + */ + l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK; + ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK; + + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, ctl1); + if (l1_2_enable) + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, + ctl1 | l1_2_enable); +} + /* Calculate L1.2 PM substate timing parameters */ static void aspm_calc_l1ss_info(struct pcie_link_state *link, u32 parent_l1ss_cap, u32 child_l1ss_cap) @@ -464,7 +489,6 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, u32 t_common_mode, t_power_on, l1_2_threshold, scale, value; u32 ctl1 = 0, ctl2 = 0; u32 pctl1, pctl2, cctl1, cctl2; - u32 pl1_2_enables, cl1_2_enables; if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) return; @@ -513,39 +537,21 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, ctl2 == pctl2 && ctl2 == cctl2) return; - /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */ - pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK; - cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK; + pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE); + pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)); + aspm_program_l1ss(parent, pctl1, ctl2); - if (pl1_2_enables || cl1_2_enables) { - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_L1_2_MASK, 0); - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_L1_2_MASK, 0); - } - - /* Program T_POWER_ON times in both ports */ - pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); - pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); - - /* Program Common_Mode_Restore_Time in upstream device */ - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1); - - /* Program LTR_L1.2_THRESHOLD time in both ports */ - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_LTR_L12_TH_VALUE | - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_LTR_L12_TH_VALUE | - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); - - if (pl1_2_enables || cl1_2_enables) { - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0, - pl1_2_enables); - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0, - cl1_2_enables); - } + cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE); + cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)); + aspm_program_l1ss(child, cctl1, ctl2); } static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) From 4ff116d0d5fd8a025604b0802d93a2d5f4e465d1 Mon Sep 17 00:00:00 2001 From: Vidya Sagar Date: Tue, 13 Sep 2022 18:48:22 +0530 Subject: [PATCH 2/5] PCI/ASPM: Save L1 PM Substates Capability for suspend/resume Previously the L1 PM Substates Control Registers (CTL1 and CTL2) weren't saved and restored during suspend/resume leading to the L1 PM Substates configuration being lost post-resume. Save the L1 PM Substates Control Registers so that the configuration is retained post-resume. [bhelgaas: drop pci_is_pcie() testing; we can rely on pci_configure_ltr() having already done that] Link: https://lore.kernel.org/r/20220913131822.16557-3-vidyas@nvidia.com Signed-off-by: Vidya Sagar Signed-off-by: Bjorn Helgaas --- drivers/pci/pci.c | 7 +++++++ drivers/pci/pci.h | 4 ++++ drivers/pci/pcie/aspm.c | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 95bc329e74c0..68a49fbaabde 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1663,6 +1663,7 @@ int pci_save_state(struct pci_dev *dev) return i; pci_save_ltr_state(dev); + pci_save_aspm_l1ss_state(dev); pci_save_dpc_state(dev); pci_save_aer_state(dev); pci_save_ptm_state(dev); @@ -1769,6 +1770,7 @@ void pci_restore_state(struct pci_dev *dev) * LTR itself (in the PCIe capability). */ pci_restore_ltr_state(dev); + pci_restore_aspm_l1ss_state(dev); pci_restore_pcie_state(dev); pci_restore_pasid_state(dev); @@ -3485,6 +3487,11 @@ void pci_allocate_cap_save_buffers(struct pci_dev *dev) if (error) pci_err(dev, "unable to allocate suspend buffer for LTR\n"); + error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_L1SS, + 2 * sizeof(u32)); + if (error) + pci_err(dev, "unable to allocate suspend buffer for ASPM-L1SS\n"); + pci_allocate_vc_save_buffers(dev); } diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 785f31086313..365a844ec430 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -561,10 +561,14 @@ bool pcie_wait_for_link(struct pci_dev *pdev, bool active); void pcie_aspm_init_link_state(struct pci_dev *pdev); void pcie_aspm_exit_link_state(struct pci_dev *pdev); void pcie_aspm_powersave_config_link(struct pci_dev *pdev); +void pci_save_aspm_l1ss_state(struct pci_dev *dev); +void pci_restore_aspm_l1ss_state(struct pci_dev *dev); #else static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { } static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev) { } static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { } +static inline void pci_save_aspm_l1ss_state(struct pci_dev *dev) { } +static inline void pci_restore_aspm_l1ss_state(struct pci_dev *dev) { } #endif #ifdef CONFIG_PCIE_ECRC diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index b4bdadc4ac35..016d222b07c7 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -732,6 +732,43 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) PCI_L1SS_CTL1_L1SS_MASK, val); } +void pci_save_aspm_l1ss_state(struct pci_dev *dev) +{ + struct pci_cap_saved_state *save_state; + u16 l1ss = dev->l1ss; + u32 *cap; + + if (!l1ss) + return; + + save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS); + if (!save_state) + return; + + cap = (u32 *)&save_state->cap.data[0]; + pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL2, cap++); + pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL1, cap++); +} + +void pci_restore_aspm_l1ss_state(struct pci_dev *dev) +{ + struct pci_cap_saved_state *save_state; + u32 *cap, ctl1, ctl2; + u16 l1ss = dev->l1ss; + + if (!l1ss) + return; + + save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS); + if (!save_state) + return; + + cap = (u32 *)&save_state->cap.data[0]; + ctl2 = *cap++; + ctl1 = *cap; + aspm_program_l1ss(dev, ctl1, ctl2); +} + static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) { pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, From 9e2a03173d1b4544c1113059e61e3caa7ce5e3a4 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 4 Oct 2022 21:58:07 -0500 Subject: [PATCH 3/5] PCI/ASPM: Factor out L1 PM Substates configuration Move L1 PM Substates configuration from pcie_aspm_cap_init() to a new aspm_l1ss_init() function. No functional change intended. Link: https://lore.kernel.org/r/20221005025809.2247547-2-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Reviewed-by: Kuppuswamy Sathyanarayanan --- drivers/pci/pcie/aspm.c | 103 +++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 016d222b07c7..4535228e4a64 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -554,13 +554,65 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, aspm_program_l1ss(child, cctl1, ctl2); } +static void aspm_l1ss_init(struct pcie_link_state *link) +{ + struct pci_dev *child = link->downstream, *parent = link->pdev; + u32 parent_l1ss_cap, child_l1ss_cap; + u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; + + /* Setup L1 substate */ + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, + &parent_l1ss_cap); + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, + &child_l1ss_cap); + + if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) + parent_l1ss_cap = 0; + if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) + child_l1ss_cap = 0; + + /* + * If we don't have LTR for the entire path from the Root Complex + * to this device, we can't use ASPM L1.2 because it relies on the + * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. + */ + if (!child->ltr_path) + child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; + + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) + link->aspm_support |= ASPM_STATE_L1_1; + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) + link->aspm_support |= ASPM_STATE_L1_2; + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) + link->aspm_support |= ASPM_STATE_L1_1_PCIPM; + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) + link->aspm_support |= ASPM_STATE_L1_2_PCIPM; + + if (parent_l1ss_cap) + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + &parent_l1ss_ctl1); + if (child_l1ss_cap) + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, + &child_l1ss_ctl1); + + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) + link->aspm_enabled |= ASPM_STATE_L1_1; + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) + link->aspm_enabled |= ASPM_STATE_L1_2; + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) + link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) + link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; + + if (link->aspm_support & ASPM_STATE_L1SS) + aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap); +} + static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) { struct pci_dev *child = link->downstream, *parent = link->pdev; u32 parent_lnkcap, child_lnkcap; u16 parent_lnkctl, child_lnkctl; - u32 parent_l1ss_cap, child_l1ss_cap; - u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; struct pci_bus *linkbus = parent->subordinate; if (blacklist) { @@ -615,52 +667,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1) link->aspm_enabled |= ASPM_STATE_L1; - /* Setup L1 substate */ - pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, - &parent_l1ss_cap); - pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, - &child_l1ss_cap); - - if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) - parent_l1ss_cap = 0; - if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) - child_l1ss_cap = 0; - - /* - * If we don't have LTR for the entire path from the Root Complex - * to this device, we can't use ASPM L1.2 because it relies on the - * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. - */ - if (!child->ltr_path) - child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; - - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) - link->aspm_support |= ASPM_STATE_L1_1; - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) - link->aspm_support |= ASPM_STATE_L1_2; - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) - link->aspm_support |= ASPM_STATE_L1_1_PCIPM; - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) - link->aspm_support |= ASPM_STATE_L1_2_PCIPM; - - if (parent_l1ss_cap) - pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - &parent_l1ss_ctl1); - if (child_l1ss_cap) - pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, - &child_l1ss_ctl1); - - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) - link->aspm_enabled |= ASPM_STATE_L1_1; - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) - link->aspm_enabled |= ASPM_STATE_L1_2; - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) - link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) - link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; - - if (link->aspm_support & ASPM_STATE_L1SS) - aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap); + aspm_l1ss_init(link); /* Save default state */ link->aspm_default = link->aspm_enabled; From cfc0028627cadfa271fab0290f18731193d63d87 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 4 Oct 2022 21:58:08 -0500 Subject: [PATCH 4/5] PCI/ASPM: Ignore L1 PM Substates if device lacks capability 187f91db8237 ("PCI/ASPM: Remove struct aspm_register_info.l1ss_cap") inadvertently removed a check for existence of the L1 PM Substates (L1SS) Capability before reading it. If there is no L1SS Capability, this means we mistakenly read PCI_COMMAND and PCI_STATUS (config address 0x04) and interpret that as the PCI_L1SS_CAP register, so we may incorrectly configure L1SS. Make sure the L1SS Capability exists before trying to read it. Fixes: 187f91db8237 ("PCI/ASPM: Remove struct aspm_register_info.l1ss_cap") Link: https://lore.kernel.org/r/20221005025809.2247547-3-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Reviewed-by: Kuppuswamy Sathyanarayanan --- drivers/pci/pcie/aspm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 4535228e4a64..f12d117f44e0 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -560,6 +560,9 @@ static void aspm_l1ss_init(struct pcie_link_state *link) u32 parent_l1ss_cap, child_l1ss_cap; u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; + if (!parent->l1ss || !child->l1ss) + return; + /* Setup L1 substate */ pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, &parent_l1ss_cap); From 7afeb84d14eaaebb71f5c558ed57ca858e4304e7 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Tue, 4 Oct 2022 21:58:09 -0500 Subject: [PATCH 5/5] PCI/ASPM: Correct LTR_L1.2_THRESHOLD computation 80d7d7a904fa ("PCI/ASPM: Calculate LTR_L1.2_THRESHOLD from device characteristics") replaced a fixed value (163840ns) with one computed from T_POWER_OFF, Common_Mode_Restore_Time, etc., but it encoded the LTR_L1.2_THRESHOLD value incorrectly. This is especially a problem for small thresholds, e.g., 63ns fell into the "threshold_ns < 1024" case and was encoded as 32ns: LTR_L1.2_THRESHOLD_Scale = 1 (multiplier is 32ns) LTR_L1.2_THRESHOLD_Value = 63 >> 5 = 1 LTR_L1.2_THRESHOLD = multiplier * value = 32ns * 1 = 32ns Correct the algorithm to encode all times of 1023ns (0x3ff) or smaller exactly and larger times conservatively (the encoded threshold is never smaller than was requested). This reduces the chance of entering L1.2 when the device can't tolerate the exit latency. Fixes: 80d7d7a904fa ("PCI/ASPM: Calculate LTR_L1.2_THRESHOLD from device characteristics") Link: https://lore.kernel.org/r/20221005025809.2247547-4-helgaas@kernel.org Signed-off-by: Bjorn Helgaas Reviewed-by: Kuppuswamy Sathyanarayanan --- drivers/pci/pcie/aspm.c | 49 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index f12d117f44e0..53a1fa306e1e 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -350,29 +351,43 @@ static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) return 0; } +/* + * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1 + * register. Ports enter L1.2 when the most recent LTR value is greater + * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we + * don't enter L1.2 too aggressively. + * + * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3. + */ static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value) { - u32 threshold_ns = threshold_us * 1000; + u64 threshold_ns = (u64) threshold_us * 1000; - /* See PCIe r3.1, sec 7.33.3 and sec 6.18 */ - if (threshold_ns < 32) { - *scale = 0; + /* + * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max + * value of 0x3ff. + */ + if (threshold_ns <= 0x3ff * 1) { + *scale = 0; /* Value times 1ns */ *value = threshold_ns; - } else if (threshold_ns < 1024) { - *scale = 1; - *value = threshold_ns >> 5; - } else if (threshold_ns < 32768) { - *scale = 2; - *value = threshold_ns >> 10; - } else if (threshold_ns < 1048576) { - *scale = 3; - *value = threshold_ns >> 15; - } else if (threshold_ns < 33554432) { - *scale = 4; - *value = threshold_ns >> 20; + } else if (threshold_ns <= 0x3ff * 32) { + *scale = 1; /* Value times 32ns */ + *value = roundup(threshold_ns, 32) / 32; + } else if (threshold_ns <= 0x3ff * 1024) { + *scale = 2; /* Value times 1024ns */ + *value = roundup(threshold_ns, 1024) / 1024; + } else if (threshold_ns <= 0x3ff * 32768) { + *scale = 3; /* Value times 32768ns */ + *value = roundup(threshold_ns, 32768) / 32768; + } else if (threshold_ns <= 0x3ff * 1048576) { + *scale = 4; /* Value times 1048576ns */ + *value = roundup(threshold_ns, 1048576) / 1048576; + } else if (threshold_ns <= 0x3ff * (u64) 33554432) { + *scale = 5; /* Value times 33554432ns */ + *value = roundup(threshold_ns, 33554432) / 33554432; } else { *scale = 5; - *value = threshold_ns >> 25; + *value = 0x3ff; /* Max representable value */ } }