From c54c53d9921adef2c239cb43d5a936b63c57ebf0 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:25 -0700 Subject: [PATCH 01/24] perf/x86/intel/uncore: Add Sapphire Rapids server framework Intel Sapphire Rapids supports a discovery mechanism, that allows an uncore driver to discover the different components ("boxes") of the chip. All the generic information of the uncore boxes should be retrieved from the discovery tables. This has been enabled with the commit edae1f06c2cd ("perf/x86/intel/uncore: Parse uncore discovery tables"). Add use_discovery to indicate the case. The uncore driver doesn't need to hard code the generic information for each uncore box. But we still need to enable various functionality that cannot be directly discovered. To support these functionalities, the Sapphire Rapids server framework is introduced here. Each specific uncore unit will be added into the framework in the following patches. Add use_discovery to indicate that the discovery mechanism is required for the platform. Currently, Intel Sapphire Rapids is one of the platforms. The box ID from the discovery table is the accurate index. Use it if applicable. All the undiscovered platform-specific features will be hard code in the spr_uncores[]. Add uncore_type_customized_copy(), instead of the memcpy, to only overwrite these features. The specific uncore unit hasn't been added here. From user's perspective, there is nothing changed for now. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-2-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore.c | 26 +++++-- arch/x86/events/intel/uncore.h | 3 + arch/x86/events/intel/uncore_discovery.c | 2 +- arch/x86/events/intel/uncore_discovery.h | 3 + arch/x86/events/intel/uncore_snbep.c | 87 ++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 5 deletions(-) diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 9bf4dbbc26e2..b941ceee2289 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -865,9 +865,13 @@ static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu) sprintf(pmu->name, "uncore_%s", type->name); else sprintf(pmu->name, "uncore"); - } else - sprintf(pmu->name, "uncore_%s_%d", type->name, pmu->pmu_idx); - + } else { + /* + * Use the box ID from the discovery table if applicable. + */ + sprintf(pmu->name, "uncore_%s_%d", type->name, + type->box_ids ? type->box_ids[pmu->pmu_idx] : pmu->pmu_idx); + } } static int uncore_pmu_register(struct intel_uncore_pmu *pmu) @@ -1663,6 +1667,7 @@ struct intel_uncore_init_fun { void (*cpu_init)(void); int (*pci_init)(void); void (*mmio_init)(void); + bool use_discovery; }; static const struct intel_uncore_init_fun nhm_uncore_init __initconst = { @@ -1765,6 +1770,13 @@ static const struct intel_uncore_init_fun snr_uncore_init __initconst = { .mmio_init = snr_uncore_mmio_init, }; +static const struct intel_uncore_init_fun spr_uncore_init __initconst = { + .cpu_init = spr_uncore_cpu_init, + .pci_init = spr_uncore_pci_init, + .mmio_init = spr_uncore_mmio_init, + .use_discovery = true, +}; + static const struct intel_uncore_init_fun generic_uncore_init __initconst = { .cpu_init = intel_uncore_generic_uncore_cpu_init, .pci_init = intel_uncore_generic_uncore_pci_init, @@ -1809,6 +1821,7 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &rkl_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &adl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &spr_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init), {}, }; @@ -1832,8 +1845,13 @@ static int __init intel_uncore_init(void) uncore_init = (struct intel_uncore_init_fun *)&generic_uncore_init; else return -ENODEV; - } else + } else { uncore_init = (struct intel_uncore_init_fun *)id->driver_data; + if (uncore_no_discover && uncore_init->use_discovery) + return -ENODEV; + if (uncore_init->use_discovery && !intel_uncore_has_discovery_tables()) + return -ENODEV; + } if (uncore_init->pci_init) { pret = uncore_init->pci_init(); diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index 187d7287039c..fa0e938f0141 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -608,6 +608,9 @@ void snr_uncore_mmio_init(void); int icx_uncore_pci_init(void); void icx_uncore_cpu_init(void); void icx_uncore_mmio_init(void); +int spr_uncore_pci_init(void); +void spr_uncore_cpu_init(void); +void spr_uncore_mmio_init(void); /* uncore_nhmex.c */ void nhmex_uncore_cpu_init(void); diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index aba9bff95413..93148e215bbc 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -568,7 +568,7 @@ static bool uncore_update_uncore_type(enum uncore_access_type type_id, return true; } -static struct intel_uncore_type ** +struct intel_uncore_type ** intel_uncore_generic_init_uncores(enum uncore_access_type type_id) { struct intel_uncore_discovery_type *type; diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index 1d652939a01c..d7ccc8af6d18 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -129,3 +129,6 @@ void intel_uncore_clear_discovery_tables(void); void intel_uncore_generic_uncore_cpu_init(void); int intel_uncore_generic_uncore_pci_init(void); void intel_uncore_generic_uncore_mmio_init(void); + +struct intel_uncore_type ** +intel_uncore_generic_init_uncores(enum uncore_access_type type_id); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 76227627e3b4..c1006169580a 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* SandyBridge-EP/IvyTown uncore support */ #include "uncore.h" +#include "uncore_discovery.h" /* SNB-EP pci bus to socket mapping */ #define SNBEP_CPUNODEID 0x40 @@ -5504,3 +5505,89 @@ void icx_uncore_mmio_init(void) } /* end of ICX uncore support */ + +/* SPR uncore support */ + +#define UNCORE_SPR_NUM_UNCORE_TYPES 12 + +static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, +}; + +static void uncore_type_customized_copy(struct intel_uncore_type *to_type, + struct intel_uncore_type *from_type) +{ + if (!to_type || !from_type) + return; + + if (from_type->name) + to_type->name = from_type->name; + if (from_type->fixed_ctr_bits) + to_type->fixed_ctr_bits = from_type->fixed_ctr_bits; + if (from_type->event_mask) + to_type->event_mask = from_type->event_mask; + if (from_type->event_mask_ext) + to_type->event_mask_ext = from_type->event_mask_ext; + if (from_type->fixed_ctr) + to_type->fixed_ctr = from_type->fixed_ctr; + if (from_type->fixed_ctl) + to_type->fixed_ctl = from_type->fixed_ctl; + if (from_type->fixed_ctr_bits) + to_type->fixed_ctr_bits = from_type->fixed_ctr_bits; + if (from_type->num_shared_regs) + to_type->num_shared_regs = from_type->num_shared_regs; + if (from_type->constraints) + to_type->constraints = from_type->constraints; + if (from_type->ops) + to_type->ops = from_type->ops; + if (from_type->event_descs) + to_type->event_descs = from_type->event_descs; + if (from_type->format_group) + to_type->format_group = from_type->format_group; +} + +static struct intel_uncore_type ** +uncore_get_uncores(enum uncore_access_type type_id) +{ + struct intel_uncore_type **types, **start_types; + + start_types = types = intel_uncore_generic_init_uncores(type_id); + + /* Only copy the customized features */ + for (; *types; types++) { + if ((*types)->type_id >= UNCORE_SPR_NUM_UNCORE_TYPES) + continue; + uncore_type_customized_copy(*types, spr_uncores[(*types)->type_id]); + } + + return start_types; +} + +void spr_uncore_cpu_init(void) +{ + uncore_msr_uncores = uncore_get_uncores(UNCORE_ACCESS_MSR); +} + +int spr_uncore_pci_init(void) +{ + uncore_pci_uncores = uncore_get_uncores(UNCORE_ACCESS_PCI); + return 0; +} + +void spr_uncore_mmio_init(void) +{ + uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO); +} + +/* end of SPR uncore support */ From 949b11381f81664df3997db2ae0ec9546ab6dd85 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:26 -0700 Subject: [PATCH 02/24] perf/x86/intel/uncore: Add Sapphire Rapids server CHA support CHA merges the caching agent and Home Agent (HA) responsibilities of the chip into a single block. It's one of the Sapphire Rapids server uncore units. The layout of the control registers for a CHA uncore unit is a little bit different from the generic one. The CHA uncore unit also supports a filter register for TID. So a specific format and ops are required. Expose the common MSR ops which can be reused. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-3-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_discovery.c | 6 +- arch/x86/events/intel/uncore_discovery.h | 4 ++ arch/x86/events/intel/uncore_snbep.c | 90 +++++++++++++++++++++++- 3 files changed, 96 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 93148e215bbc..25f1c0157078 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -337,17 +337,17 @@ static const struct attribute_group generic_uncore_format_group = { .attrs = generic_uncore_formats_attr, }; -static void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) +void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); } -static void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) +void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) { wrmsrl(uncore_msr_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); } -static void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) +void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) { wrmsrl(uncore_msr_box_ctl(box), 0); } diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index d7ccc8af6d18..e836a68a8025 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -130,5 +130,9 @@ void intel_uncore_generic_uncore_cpu_init(void); int intel_uncore_generic_uncore_pci_init(void); void intel_uncore_generic_uncore_mmio_init(void); +void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); +void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box); +void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box); + struct intel_uncore_type ** intel_uncore_generic_init_uncores(enum uncore_access_type type_id); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index c1006169580a..8a470d24ae6f 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -455,6 +455,17 @@ #define ICX_NUMBER_IMC_CHN 2 #define ICX_IMC_MEM_STRIDE 0x4 +/* SPR */ +#define SPR_RAW_EVENT_MASK_EXT 0xffffff + +/* SPR CHA */ +#define SPR_CHA_PMON_CTL_TID_EN (1 << 16) +#define SPR_CHA_PMON_EVENT_MASK (SNBEP_PMON_RAW_EVENT_MASK | \ + SPR_CHA_PMON_CTL_TID_EN) +#define SPR_CHA_PMON_BOX_FILTER_TID 0x3ff + +#define SPR_C0_MSR_PMON_BOX_FILTER0 0x200e + DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); DEFINE_UNCORE_FORMAT_ATTR(event2, event, "config:0-6"); DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21"); @@ -467,6 +478,7 @@ DEFINE_UNCORE_FORMAT_ATTR(umask_ext4, umask, "config:8-15,32-55"); DEFINE_UNCORE_FORMAT_ATTR(qor, qor, "config:16"); DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19"); +DEFINE_UNCORE_FORMAT_ATTR(tid_en2, tid_en, "config:16"); DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); DEFINE_UNCORE_FORMAT_ATTR(thresh9, thresh, "config:24-35"); DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31"); @@ -5508,10 +5520,86 @@ void icx_uncore_mmio_init(void) /* SPR uncore support */ +static void spr_uncore_msr_enable_event(struct intel_uncore_box *box, + struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + struct hw_perf_event_extra *reg1 = &hwc->extra_reg; + + if (reg1->idx != EXTRA_REG_NONE) + wrmsrl(reg1->reg, reg1->config); + + wrmsrl(hwc->config_base, hwc->config); +} + +static void spr_uncore_msr_disable_event(struct intel_uncore_box *box, + struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + struct hw_perf_event_extra *reg1 = &hwc->extra_reg; + + if (reg1->idx != EXTRA_REG_NONE) + wrmsrl(reg1->reg, 0); + + wrmsrl(hwc->config_base, 0); +} + +static int spr_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event) +{ + struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; + bool tie_en = !!(event->hw.config & SPR_CHA_PMON_CTL_TID_EN); + struct intel_uncore_type *type = box->pmu->type; + + if (tie_en) { + reg1->reg = SPR_C0_MSR_PMON_BOX_FILTER0 + + HSWEP_CBO_MSR_OFFSET * type->box_ids[box->pmu->pmu_idx]; + reg1->config = event->attr.config1 & SPR_CHA_PMON_BOX_FILTER_TID; + reg1->idx = 0; + } + + return 0; +} + +static struct intel_uncore_ops spr_uncore_chabox_ops = { + .init_box = intel_generic_uncore_msr_init_box, + .disable_box = intel_generic_uncore_msr_disable_box, + .enable_box = intel_generic_uncore_msr_enable_box, + .disable_event = spr_uncore_msr_disable_event, + .enable_event = spr_uncore_msr_enable_event, + .read_counter = uncore_msr_read_counter, + .hw_config = spr_cha_hw_config, + .get_constraint = uncore_get_constraint, + .put_constraint = uncore_put_constraint, +}; + +static struct attribute *spr_uncore_cha_formats_attr[] = { + &format_attr_event.attr, + &format_attr_umask_ext4.attr, + &format_attr_tid_en2.attr, + &format_attr_edge.attr, + &format_attr_inv.attr, + &format_attr_thresh8.attr, + &format_attr_filter_tid5.attr, + NULL, +}; +static const struct attribute_group spr_uncore_chabox_format_group = { + .name = "format", + .attrs = spr_uncore_cha_formats_attr, +}; + +static struct intel_uncore_type spr_uncore_chabox = { + .name = "cha", + .event_mask = SPR_CHA_PMON_EVENT_MASK, + .event_mask_ext = SPR_RAW_EVENT_MASK_EXT, + .num_shared_regs = 1, + .ops = &spr_uncore_chabox_ops, + .format_group = &spr_uncore_chabox_format_group, +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { - NULL, + &spr_uncore_chabox, NULL, NULL, NULL, From 3ba7095beaec1bace1b5864fa92b2b7a0eaadf38 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:27 -0700 Subject: [PATCH 03/24] perf/x86/intel/uncore: Add Sapphire Rapids server IIO support The IIO stacks are responsible for managing the traffic between the PCI Express* (PCIe*) domain and the mesh domain. The IIO PMON block is situated near the IIO stacks traffic controller capturing the traffic controller as well as the PCIe* root port information. The layout of the control registers for a IIO uncore unit is a little bit different from the generic one. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-4-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 8a470d24ae6f..3b4039577f09 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5596,11 +5596,18 @@ static struct intel_uncore_type spr_uncore_chabox = { .format_group = &spr_uncore_chabox_format_group, }; +static struct intel_uncore_type spr_uncore_iio = { + .name = "iio", + .event_mask = SNBEP_PMON_RAW_EVENT_MASK, + .event_mask_ext = SNR_IIO_PMON_RAW_EVENT_MASK_EXT, + .format_group = &snr_uncore_iio_format_group, +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_chabox, - NULL, + &spr_uncore_iio, NULL, NULL, NULL, From e199eb5131591c020705deceee224b437d09ece4 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:28 -0700 Subject: [PATCH 04/24] perf/x86/intel/uncore: Add Sapphire Rapids server IRP support The IRP is responsible for maintaining coherency for the IIO traffic targeting coherent memory. The layout of the control registers for a IRP uncore unit is a little bit different from the generic one. Factor out SPR_UNCORE_COMMON_FORMAT, which can be reused by the following uncore units. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-5-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 3b4039577f09..de5a6d1f1735 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5603,12 +5603,37 @@ static struct intel_uncore_type spr_uncore_iio = { .format_group = &snr_uncore_iio_format_group, }; +static struct attribute *spr_uncore_raw_formats_attr[] = { + &format_attr_event.attr, + &format_attr_umask_ext4.attr, + &format_attr_edge.attr, + &format_attr_inv.attr, + &format_attr_thresh8.attr, + NULL, +}; + +static const struct attribute_group spr_uncore_raw_format_group = { + .name = "format", + .attrs = spr_uncore_raw_formats_attr, +}; + +#define SPR_UNCORE_COMMON_FORMAT() \ + .event_mask = SNBEP_PMON_RAW_EVENT_MASK, \ + .event_mask_ext = SPR_RAW_EVENT_MASK_EXT, \ + .format_group = &spr_uncore_raw_format_group + +static struct intel_uncore_type spr_uncore_irp = { + SPR_UNCORE_COMMON_FORMAT(), + .name = "irp", + +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_chabox, &spr_uncore_iio, - NULL, + &spr_uncore_irp, NULL, NULL, NULL, From f85ef898f8842b2a9a8f51a64eaf45ee2a8bb1f7 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:29 -0700 Subject: [PATCH 05/24] perf/x86/intel/uncore: Add Sapphire Rapids server M2PCIe support M2PCIe* blocks manage the interface between the mesh and each IIO stack. The layout of the control registers for a M2PCIe uncore unit is similar to a IRP uncore unit. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-6-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index de5a6d1f1735..890a98279fca 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5628,13 +5628,18 @@ static struct intel_uncore_type spr_uncore_irp = { }; +static struct intel_uncore_type spr_uncore_m2pcie = { + SPR_UNCORE_COMMON_FORMAT(), + .name = "m2pcie", +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_chabox, &spr_uncore_iio, &spr_uncore_irp, - NULL, + &spr_uncore_m2pcie, NULL, NULL, NULL, From 0654dfdc7e1ca30d36810ab694712da3de18440c Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:30 -0700 Subject: [PATCH 06/24] perf/x86/intel/uncore: Add Sapphire Rapids server PCU support The PCU is the primary power controller for the Sapphire Rapids. Except the name, all the information can be retrieved from the discovery tables. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-7-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 890a98279fca..913cd7aca65d 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5633,6 +5633,10 @@ static struct intel_uncore_type spr_uncore_m2pcie = { .name = "m2pcie", }; +static struct intel_uncore_type spr_uncore_pcu = { + .name = "pcu", +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { @@ -5640,7 +5644,7 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_iio, &spr_uncore_irp, &spr_uncore_m2pcie, - NULL, + &spr_uncore_pcu, NULL, NULL, NULL, From 85f2e30f987ecc73fbb5e24eda0f36ba7f337c5c Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:31 -0700 Subject: [PATCH 07/24] perf/x86/intel/uncore: Add Sapphire Rapids server IMC support The Sapphire Rapids IMC provides the interface to the DRAM and communicates to the rest of the uncore through the M2M block. The layout of the control registers for a IMC uncore unit is a little bit different from the generic one. There is a fixed counter for IMC. So a specific format and ops are required. Expose the common MMIO ops which can be reused. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-8-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_discovery.c | 10 +++---- arch/x86/events/intel/uncore_discovery.h | 6 ++++ arch/x86/events/intel/uncore_snbep.c | 35 +++++++++++++++++++++++- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 25f1c0157078..cc44311fe2d8 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -454,7 +454,7 @@ static unsigned int generic_uncore_mmio_box_ctl(struct intel_uncore_box *box) return type->box_ctls[box->dieid] + type->mmio_offsets[box->pmu->pmu_idx]; } -static void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) +void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) { unsigned int box_ctl = generic_uncore_mmio_box_ctl(box); struct intel_uncore_type *type = box->pmu->type; @@ -478,7 +478,7 @@ static void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); } -static void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) +void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) { if (!box->io_addr) return; @@ -486,7 +486,7 @@ static void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) writel(GENERIC_PMON_BOX_CTL_FRZ, box->io_addr); } -static void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box) +void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box) { if (!box->io_addr) return; @@ -505,8 +505,8 @@ static void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, writel(hwc->config, box->io_addr + hwc->config_base); } -static void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, - struct perf_event *event) +void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, + struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index e836a68a8025..97232437dfb9 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -134,5 +134,11 @@ void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box); +void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); +void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box); +void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); +void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, + struct perf_event *event); + struct intel_uncore_type ** intel_uncore_generic_init_uncores(enum uncore_access_type type_id); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 913cd7aca65d..3c9d4591b8e1 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5637,6 +5637,39 @@ static struct intel_uncore_type spr_uncore_pcu = { .name = "pcu", }; +static void spr_uncore_mmio_enable_event(struct intel_uncore_box *box, + struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!box->io_addr) + return; + + if (uncore_pmc_fixed(hwc->idx)) + writel(SNBEP_PMON_CTL_EN, box->io_addr + hwc->config_base); + else + writel(hwc->config, box->io_addr + hwc->config_base); +} + +static struct intel_uncore_ops spr_uncore_mmio_ops = { + .init_box = intel_generic_uncore_mmio_init_box, + .exit_box = uncore_mmio_exit_box, + .disable_box = intel_generic_uncore_mmio_disable_box, + .enable_box = intel_generic_uncore_mmio_enable_box, + .disable_event = intel_generic_uncore_mmio_disable_event, + .enable_event = spr_uncore_mmio_enable_event, + .read_counter = uncore_mmio_read_counter, +}; + +static struct intel_uncore_type spr_uncore_imc = { + SPR_UNCORE_COMMON_FORMAT(), + .name = "imc", + .fixed_ctr_bits = 48, + .fixed_ctr = SNR_IMC_MMIO_PMON_FIXED_CTR, + .fixed_ctl = SNR_IMC_MMIO_PMON_FIXED_CTL, + .ops = &spr_uncore_mmio_ops, +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { @@ -5646,7 +5679,7 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_m2pcie, &spr_uncore_pcu, NULL, - NULL, + &spr_uncore_imc, NULL, NULL, NULL, From f57191edaaeb01279a88ace1be5b7230bdd8c0ab Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:32 -0700 Subject: [PATCH 08/24] perf/x86/intel/uncore: Add Sapphire Rapids server M2M support The M2M blocks manage the interface between the mesh (operating on both the mesh and the SMI3 protocol) and the memory controllers. The layout of the control registers for a M2M uncore unit is a little bit different from the generic one. So a specific format and ops are required. Expose the common PCI ops which can be reused. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-9-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_discovery.c | 14 +++++------ arch/x86/events/intel/uncore_discovery.h | 8 +++++++ arch/x86/events/intel/uncore_snbep.c | 30 +++++++++++++++++++++++- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index cc44311fe2d8..6322df1dd8e0 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -377,7 +377,7 @@ static struct intel_uncore_ops generic_uncore_msr_ops = { .read_counter = uncore_msr_read_counter, }; -static void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) +void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); @@ -386,7 +386,7 @@ static void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT); } -static void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) +void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); @@ -394,7 +394,7 @@ static void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ); } -static void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box) +void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); @@ -411,8 +411,8 @@ static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box, pci_write_config_dword(pdev, hwc->config_base, hwc->config); } -static void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, - struct perf_event *event) +void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, + struct perf_event *event) { struct pci_dev *pdev = box->pci_dev; struct hw_perf_event *hwc = &event->hw; @@ -420,8 +420,8 @@ static void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, pci_write_config_dword(pdev, hwc->config_base, 0); } -static u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box, - struct perf_event *event) +u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box, + struct perf_event *event) { struct pci_dev *pdev = box->pci_dev; struct hw_perf_event *hwc = &event->hw; diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index 97232437dfb9..b85655ba835b 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -140,5 +140,13 @@ void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, struct perf_event *event); +void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); +void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box); +void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box); +void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, + struct perf_event *event); +u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box, + struct perf_event *event); + struct intel_uncore_type ** intel_uncore_generic_init_uncores(enum uncore_access_type type_id); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 3c9d4591b8e1..72ba8d44d819 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5670,6 +5670,34 @@ static struct intel_uncore_type spr_uncore_imc = { .ops = &spr_uncore_mmio_ops, }; +static void spr_uncore_pci_enable_event(struct intel_uncore_box *box, + struct perf_event *event) +{ + struct pci_dev *pdev = box->pci_dev; + struct hw_perf_event *hwc = &event->hw; + + pci_write_config_dword(pdev, hwc->config_base + 4, (u32)(hwc->config >> 32)); + pci_write_config_dword(pdev, hwc->config_base, (u32)hwc->config); +} + +static struct intel_uncore_ops spr_uncore_pci_ops = { + .init_box = intel_generic_uncore_pci_init_box, + .disable_box = intel_generic_uncore_pci_disable_box, + .enable_box = intel_generic_uncore_pci_enable_box, + .disable_event = intel_generic_uncore_pci_disable_event, + .enable_event = spr_uncore_pci_enable_event, + .read_counter = intel_generic_uncore_pci_read_counter, +}; + +#define SPR_UNCORE_PCI_COMMON_FORMAT() \ + SPR_UNCORE_COMMON_FORMAT(), \ + .ops = &spr_uncore_pci_ops + +static struct intel_uncore_type spr_uncore_m2m = { + SPR_UNCORE_PCI_COMMON_FORMAT(), + .name = "m2m", +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { @@ -5680,7 +5708,7 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_pcu, NULL, &spr_uncore_imc, - NULL, + &spr_uncore_m2m, NULL, NULL, NULL, From da5a9156cd2a3be2b00f8defb529ee3e35e5769b Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:33 -0700 Subject: [PATCH 09/24] perf/x86/intel/uncore: Add Sapphire Rapids server UPI support Sapphire Rapids uses a coherent interconnect for scaling to multiple sockets known as Intel UPI. Intel UPI technology provides a cache coherent socket to socket external communication interface between processors. The layout of the control registers for a UPI uncore unit is similar to a M2M uncore unit. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-10-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 72ba8d44d819..20045ba723b5 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5698,6 +5698,11 @@ static struct intel_uncore_type spr_uncore_m2m = { .name = "m2m", }; +static struct intel_uncore_type spr_uncore_upi = { + SPR_UNCORE_PCI_COMMON_FORMAT(), + .name = "upi", +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { @@ -5709,7 +5714,7 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { NULL, &spr_uncore_imc, &spr_uncore_m2m, - NULL, + &spr_uncore_upi, NULL, NULL, NULL, From 2a8e51eae7c83c29795622cfc794cf83436cc05d Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:34 -0700 Subject: [PATCH 10/24] perf/x86/intel/uncore: Add Sapphire Rapids server M3UPI support M3 Intel UPI is the interface between the mesh and the Intel UPI link layer. It is responsible for translating between the mesh protocol packets and the flits that are used for transmitting data across the Intel UPI interface. The layout of the control registers for a M3UPI uncore unit is similar to a UPI uncore unit. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-11-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 20045ba723b5..14b9b2367725 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5703,6 +5703,11 @@ static struct intel_uncore_type spr_uncore_upi = { .name = "upi", }; +static struct intel_uncore_type spr_uncore_m3upi = { + SPR_UNCORE_PCI_COMMON_FORMAT(), + .name = "m3upi", +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { @@ -5715,7 +5720,7 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_imc, &spr_uncore_m2m, &spr_uncore_upi, - NULL, + &spr_uncore_m3upi, NULL, NULL, }; From 0d771caf728436d9ebc2cd1d50bed71685bfe7d8 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:35 -0700 Subject: [PATCH 11/24] perf/x86/intel/uncore: Add Sapphire Rapids server MDF support The MDF subsystem is a new IP built to support the new Intel Xeon architecture that bridges multiple dies with a embedded bridge system. The layout of the control registers for a MDF uncore unit is similar to a IRP uncore unit. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-12-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 14b9b2367725..1b9ab8ed2523 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5708,6 +5708,11 @@ static struct intel_uncore_type spr_uncore_m3upi = { .name = "m3upi", }; +static struct intel_uncore_type spr_uncore_mdf = { + SPR_UNCORE_COMMON_FORMAT(), + .name = "mdf", +}; + #define UNCORE_SPR_NUM_UNCORE_TYPES 12 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { @@ -5722,7 +5727,7 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_upi, &spr_uncore_m3upi, NULL, - NULL, + &spr_uncore_mdf, }; static void uncore_type_customized_copy(struct intel_uncore_type *to_type, From 8053f2d752e2936f494ede62766a6c9e9fb674f2 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:36 -0700 Subject: [PATCH 12/24] perf/x86/intel/uncore: Add alias PMU name A perf PMU may have two PMU names. For example, Intel Sapphire Rapids server supports the discovery mechanism. Without the platform-specific support, an uncore PMU is named by a type ID plus a box ID, e.g., uncore_type_0_0, because the real name of the uncore PMU cannot be retrieved from the discovery table. With the platform-specific support later, perf has the mapping information from a type ID to a specific uncore unit. Just like the previous platforms, the uncore PMU is named by the real PMU name, e.g., uncore_cha_0. The user scripts which work well with the old numeric name may not work anymore. Add a new attribute "alias" to indicate the old numeric name. The following userspace perf tool patch will handle both names. The user scripts should work properly with the updated perf tool. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-13-git-send-email-kan.liang@linux.intel.com --- .../sysfs-bus-event_source-devices-uncore | 13 +++++++++ arch/x86/events/intel/uncore.c | 19 +++++++++---- arch/x86/events/intel/uncore.h | 1 + arch/x86/events/intel/uncore_snbep.c | 28 ++++++++++++++++++- 4 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-bus-event_source-devices-uncore diff --git a/Documentation/ABI/testing/sysfs-bus-event_source-devices-uncore b/Documentation/ABI/testing/sysfs-bus-event_source-devices-uncore new file mode 100644 index 000000000000..b56e8f019fd4 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-event_source-devices-uncore @@ -0,0 +1,13 @@ +What: /sys/bus/event_source/devices/uncore_*/alias +Date: June 2021 +KernelVersion: 5.15 +Contact: Linux kernel mailing list +Description: Read-only. An attribute to describe the alias name of + the uncore PMU if an alias exists on some platforms. + The 'perf(1)' tool should treat both names the same. + They both can be used to access the uncore PMU. + + Example: + + $ cat /sys/devices/uncore_cha_2/alias + uncore_type_0_2 diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index b941ceee2289..c72e368dd164 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -842,6 +842,18 @@ static const struct attribute_group uncore_pmu_attr_group = { .attrs = uncore_pmu_attrs, }; +void uncore_get_alias_name(char *pmu_name, struct intel_uncore_pmu *pmu) +{ + struct intel_uncore_type *type = pmu->type; + + if (type->num_boxes == 1) + sprintf(pmu_name, "uncore_type_%u", type->type_id); + else { + sprintf(pmu_name, "uncore_type_%u_%d", + type->type_id, type->box_ids[pmu->pmu_idx]); + } +} + static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu) { struct intel_uncore_type *type = pmu->type; @@ -851,12 +863,7 @@ static void uncore_get_pmu_name(struct intel_uncore_pmu *pmu) * Use uncore_type_&typeid_&boxid as name. */ if (!type->name) { - if (type->num_boxes == 1) - sprintf(pmu->name, "uncore_type_%u", type->type_id); - else { - sprintf(pmu->name, "uncore_type_%u_%d", - type->type_id, type->box_ids[pmu->pmu_idx]); - } + uncore_get_alias_name(pmu->name, pmu); return; } diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index fa0e938f0141..b9687980aab6 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -561,6 +561,7 @@ struct event_constraint * uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event); void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event); u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx); +void uncore_get_alias_name(char *pmu_name, struct intel_uncore_pmu *pmu); extern struct intel_uncore_type *empty_uncore[]; extern struct intel_uncore_type **uncore_msr_uncores; diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 1b9ab8ed2523..d0d02e048254 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5587,6 +5587,26 @@ static const struct attribute_group spr_uncore_chabox_format_group = { .attrs = spr_uncore_cha_formats_attr, }; +static ssize_t alias_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct intel_uncore_pmu *pmu = dev_to_uncore_pmu(dev); + char pmu_name[UNCORE_PMU_NAME_LEN]; + + uncore_get_alias_name(pmu_name, pmu); + return sysfs_emit(buf, "%s\n", pmu_name); +} + +static DEVICE_ATTR_RO(alias); + +static struct attribute *uncore_alias_attrs[] = { + &dev_attr_alias.attr, + NULL +}; + +ATTRIBUTE_GROUPS(uncore_alias); + static struct intel_uncore_type spr_uncore_chabox = { .name = "cha", .event_mask = SPR_CHA_PMON_EVENT_MASK, @@ -5594,6 +5614,7 @@ static struct intel_uncore_type spr_uncore_chabox = { .num_shared_regs = 1, .ops = &spr_uncore_chabox_ops, .format_group = &spr_uncore_chabox_format_group, + .attr_update = uncore_alias_groups, }; static struct intel_uncore_type spr_uncore_iio = { @@ -5601,6 +5622,7 @@ static struct intel_uncore_type spr_uncore_iio = { .event_mask = SNBEP_PMON_RAW_EVENT_MASK, .event_mask_ext = SNR_IIO_PMON_RAW_EVENT_MASK_EXT, .format_group = &snr_uncore_iio_format_group, + .attr_update = uncore_alias_groups, }; static struct attribute *spr_uncore_raw_formats_attr[] = { @@ -5620,7 +5642,8 @@ static const struct attribute_group spr_uncore_raw_format_group = { #define SPR_UNCORE_COMMON_FORMAT() \ .event_mask = SNBEP_PMON_RAW_EVENT_MASK, \ .event_mask_ext = SPR_RAW_EVENT_MASK_EXT, \ - .format_group = &spr_uncore_raw_format_group + .format_group = &spr_uncore_raw_format_group, \ + .attr_update = uncore_alias_groups static struct intel_uncore_type spr_uncore_irp = { SPR_UNCORE_COMMON_FORMAT(), @@ -5635,6 +5658,7 @@ static struct intel_uncore_type spr_uncore_m2pcie = { static struct intel_uncore_type spr_uncore_pcu = { .name = "pcu", + .attr_update = uncore_alias_groups, }; static void spr_uncore_mmio_enable_event(struct intel_uncore_box *box, @@ -5760,6 +5784,8 @@ static void uncore_type_customized_copy(struct intel_uncore_type *to_type, to_type->event_descs = from_type->event_descs; if (from_type->format_group) to_type->format_group = from_type->format_group; + if (from_type->attr_update) + to_type->attr_update = from_type->attr_update; } static struct intel_uncore_type ** From 1583971b5cb8c786df88be580cdd96a974ad591b Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:37 -0700 Subject: [PATCH 13/24] perf/x86/intel/uncore: Factor out snr_uncore_mmio_map() The IMC free-running counters on Sapphire Rapids server are also accessed by MMIO, which is similar to the previous platforms, SNR and ICX. The only difference is the device ID of the device which contains BAR address. Factor out snr_uncore_mmio_map() which ioremap the MMIO space. It can be reused in the following patch for SPR. Use the snr_uncore_mmio_map() in the icx_uncore_imc_freerunning_init_box(). There is no box_ctl for the free-running counters. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-14-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 36 +++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index d0d02e048254..9618662347e4 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -4792,13 +4792,15 @@ int snr_uncore_pci_init(void) return 0; } -static struct pci_dev *snr_uncore_get_mc_dev(int id) +#define SNR_MC_DEVICE_ID 0x3451 + +static struct pci_dev *snr_uncore_get_mc_dev(unsigned int device, int id) { struct pci_dev *mc_dev = NULL; int pkg; while (1) { - mc_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3451, mc_dev); + mc_dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, mc_dev); if (!mc_dev) break; pkg = uncore_pcibus_to_dieid(mc_dev->bus); @@ -4808,16 +4810,17 @@ static struct pci_dev *snr_uncore_get_mc_dev(int id) return mc_dev; } -static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, - unsigned int box_ctl, int mem_offset) +static int snr_uncore_mmio_map(struct intel_uncore_box *box, + unsigned int box_ctl, int mem_offset, + unsigned int device) { - struct pci_dev *pdev = snr_uncore_get_mc_dev(box->dieid); + struct pci_dev *pdev = snr_uncore_get_mc_dev(device, box->dieid); struct intel_uncore_type *type = box->pmu->type; resource_size_t addr; u32 pci_dword; if (!pdev) - return; + return -ENODEV; pci_read_config_dword(pdev, SNR_IMC_MMIO_BASE_OFFSET, &pci_dword); addr = (pci_dword & SNR_IMC_MMIO_BASE_MASK) << 23; @@ -4830,16 +4833,25 @@ static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, box->io_addr = ioremap(addr, type->mmio_map_size); if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); - return; + return -EINVAL; } - writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); + return 0; +} + +static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, + unsigned int box_ctl, int mem_offset, + unsigned int device) +{ + if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device)) + writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); } static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) { __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), - SNR_IMC_MMIO_MEM0_OFFSET); + SNR_IMC_MMIO_MEM0_OFFSET, + SNR_MC_DEVICE_ID); } static void snr_uncore_mmio_disable_box(struct intel_uncore_box *box) @@ -5413,7 +5425,8 @@ static void icx_uncore_imc_init_box(struct intel_uncore_box *box) int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - __snr_uncore_mmio_init_box(box, box_ctl, mem_offset); + __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, + SNR_MC_DEVICE_ID); } static struct intel_uncore_ops icx_uncore_mmio_ops = { @@ -5483,7 +5496,8 @@ static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), mem_offset); + snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + mem_offset, SNR_MC_DEVICE_ID); } static struct intel_uncore_ops icx_uncore_imc_freerunning_ops = { From 0378c93a92e226d99f4672e66fe4c08ee7b19e2d Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:38 -0700 Subject: [PATCH 14/24] perf/x86/intel/uncore: Support IIO free-running counters on Sapphire Rapids server Several free-running counters for IIO uncore blocks are supported on Sapphire Rapids server. They are not enumerated in the discovery tables. Extend generic_init_uncores() to support extra uncore types. The uncore types for the free-running counters is inserted right after the uncore types retrieved from the discovery table. The number of the free-running counter boxes is calculated from the max number of the corresponding standard boxes. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-15-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_discovery.c | 10 +- arch/x86/events/intel/uncore_discovery.h | 2 +- arch/x86/events/intel/uncore_snbep.c | 135 ++++++++++++++++++++++- 3 files changed, 136 insertions(+), 11 deletions(-) diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 6322df1dd8e0..3049c646fa20 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -569,7 +569,7 @@ static bool uncore_update_uncore_type(enum uncore_access_type type_id, } struct intel_uncore_type ** -intel_uncore_generic_init_uncores(enum uncore_access_type type_id) +intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra) { struct intel_uncore_discovery_type *type; struct intel_uncore_type **uncores; @@ -577,7 +577,7 @@ intel_uncore_generic_init_uncores(enum uncore_access_type type_id) struct rb_node *node; int i = 0; - uncores = kcalloc(num_discovered_types[type_id] + 1, + uncores = kcalloc(num_discovered_types[type_id] + num_extra + 1, sizeof(struct intel_uncore_type *), GFP_KERNEL); if (!uncores) return empty_uncore; @@ -606,17 +606,17 @@ intel_uncore_generic_init_uncores(enum uncore_access_type type_id) void intel_uncore_generic_uncore_cpu_init(void) { - uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR); + uncore_msr_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MSR, 0); } int intel_uncore_generic_uncore_pci_init(void) { - uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI); + uncore_pci_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_PCI, 0); return 0; } void intel_uncore_generic_uncore_mmio_init(void) { - uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO); + uncore_mmio_uncores = intel_uncore_generic_init_uncores(UNCORE_ACCESS_MMIO, 0); } diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index b85655ba835b..7280c8a3c831 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -149,4 +149,4 @@ u64 intel_generic_uncore_pci_read_counter(struct intel_uncore_box *box, struct perf_event *event); struct intel_uncore_type ** -intel_uncore_generic_init_uncores(enum uncore_access_type type_id); +intel_uncore_generic_init_uncores(enum uncore_access_type type_id, int num_extra); diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 9618662347e4..988254961fd8 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5752,6 +5752,7 @@ static struct intel_uncore_type spr_uncore_mdf = { }; #define UNCORE_SPR_NUM_UNCORE_TYPES 12 +#define UNCORE_SPR_IIO 1 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_chabox, @@ -5768,6 +5769,92 @@ static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_mdf, }; +enum perf_uncore_spr_iio_freerunning_type_id { + SPR_IIO_MSR_IOCLK, + SPR_IIO_MSR_BW_IN, + SPR_IIO_MSR_BW_OUT, + + SPR_IIO_FREERUNNING_TYPE_MAX, +}; + +static struct freerunning_counters spr_iio_freerunning[] = { + [SPR_IIO_MSR_IOCLK] = { 0x340e, 0x1, 0x10, 1, 48 }, + [SPR_IIO_MSR_BW_IN] = { 0x3800, 0x1, 0x10, 8, 48 }, + [SPR_IIO_MSR_BW_OUT] = { 0x3808, 0x1, 0x10, 8, 48 }, +}; + +static struct uncore_event_desc spr_uncore_iio_freerunning_events[] = { + /* Free-Running IIO CLOCKS Counter */ + INTEL_UNCORE_EVENT_DESC(ioclk, "event=0xff,umask=0x10"), + /* Free-Running IIO BANDWIDTH IN Counters */ + INTEL_UNCORE_EVENT_DESC(bw_in_port0, "event=0xff,umask=0x20"), + INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port1, "event=0xff,umask=0x21"), + INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port2, "event=0xff,umask=0x22"), + INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port3, "event=0xff,umask=0x23"), + INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port4, "event=0xff,umask=0x24"), + INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port5, "event=0xff,umask=0x25"), + INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port6, "event=0xff,umask=0x26"), + INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_in_port7, "event=0xff,umask=0x27"), + INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit, "MiB"), + /* Free-Running IIO BANDWIDTH OUT Counters */ + INTEL_UNCORE_EVENT_DESC(bw_out_port0, "event=0xff,umask=0x30"), + INTEL_UNCORE_EVENT_DESC(bw_out_port0.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port0.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port1, "event=0xff,umask=0x31"), + INTEL_UNCORE_EVENT_DESC(bw_out_port1.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port1.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port2, "event=0xff,umask=0x32"), + INTEL_UNCORE_EVENT_DESC(bw_out_port2.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port2.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port3, "event=0xff,umask=0x33"), + INTEL_UNCORE_EVENT_DESC(bw_out_port3.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port3.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port4, "event=0xff,umask=0x34"), + INTEL_UNCORE_EVENT_DESC(bw_out_port4.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port4.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port5, "event=0xff,umask=0x35"), + INTEL_UNCORE_EVENT_DESC(bw_out_port5.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port5.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port6, "event=0xff,umask=0x36"), + INTEL_UNCORE_EVENT_DESC(bw_out_port6.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port6.unit, "MiB"), + INTEL_UNCORE_EVENT_DESC(bw_out_port7, "event=0xff,umask=0x37"), + INTEL_UNCORE_EVENT_DESC(bw_out_port7.scale, "3.814697266e-6"), + INTEL_UNCORE_EVENT_DESC(bw_out_port7.unit, "MiB"), + { /* end: all zeroes */ }, +}; + +static struct intel_uncore_type spr_uncore_iio_free_running = { + .name = "iio_free_running", + .num_counters = 17, + .num_freerunning_types = SPR_IIO_FREERUNNING_TYPE_MAX, + .freerunning = spr_iio_freerunning, + .ops = &skx_uncore_iio_freerunning_ops, + .event_descs = spr_uncore_iio_freerunning_events, + .format_group = &skx_uncore_iio_freerunning_format_group, +}; + +#define UNCORE_SPR_MSR_EXTRA_UNCORES 1 + +static struct intel_uncore_type *spr_msr_uncores[UNCORE_SPR_MSR_EXTRA_UNCORES] = { + &spr_uncore_iio_free_running, +}; + static void uncore_type_customized_copy(struct intel_uncore_type *to_type, struct intel_uncore_type *from_type) { @@ -5803,11 +5890,13 @@ static void uncore_type_customized_copy(struct intel_uncore_type *to_type, } static struct intel_uncore_type ** -uncore_get_uncores(enum uncore_access_type type_id) +uncore_get_uncores(enum uncore_access_type type_id, int num_extra, + struct intel_uncore_type **extra) { struct intel_uncore_type **types, **start_types; + int i; - start_types = types = intel_uncore_generic_init_uncores(type_id); + start_types = types = intel_uncore_generic_init_uncores(type_id, num_extra); /* Only copy the customized features */ for (; *types; types++) { @@ -5816,23 +5905,59 @@ uncore_get_uncores(enum uncore_access_type type_id) uncore_type_customized_copy(*types, spr_uncores[(*types)->type_id]); } + for (i = 0; i < num_extra; i++, types++) + *types = extra[i]; + return start_types; } +static struct intel_uncore_type * +uncore_find_type_by_id(struct intel_uncore_type **types, int type_id) +{ + for (; *types; types++) { + if (type_id == (*types)->type_id) + return *types; + } + + return NULL; +} + +static int uncore_type_max_boxes(struct intel_uncore_type **types, + int type_id) +{ + struct intel_uncore_type *type; + int i, max = 0; + + type = uncore_find_type_by_id(types, type_id); + if (!type) + return 0; + + for (i = 0; i < type->num_boxes; i++) { + if (type->box_ids[i] > max) + max = type->box_ids[i]; + } + + return max + 1; +} + void spr_uncore_cpu_init(void) { - uncore_msr_uncores = uncore_get_uncores(UNCORE_ACCESS_MSR); + uncore_msr_uncores = uncore_get_uncores(UNCORE_ACCESS_MSR, + UNCORE_SPR_MSR_EXTRA_UNCORES, + spr_msr_uncores); + + spr_uncore_iio_free_running.num_boxes = uncore_type_max_boxes(uncore_msr_uncores, UNCORE_SPR_IIO); } int spr_uncore_pci_init(void) { - uncore_pci_uncores = uncore_get_uncores(UNCORE_ACCESS_PCI); + uncore_pci_uncores = uncore_get_uncores(UNCORE_ACCESS_PCI, 0, NULL); return 0; } void spr_uncore_mmio_init(void) { - uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO); + uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO, 0, NULL); } /* end of SPR uncore support */ From c76826a65f50038f050424365dbf3f97203f8710 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 30 Jun 2021 14:08:39 -0700 Subject: [PATCH 15/24] perf/x86/intel/uncore: Support IMC free-running counters on Sapphire Rapids server Several free-running counters for IMC uncore blocks are supported on Sapphire Rapids server. They are not enumerated in the discovery tables. The number of the free-running counter boxes is calculated from the number of corresponding standard boxes. The snbep_pci2phy_map_init() is invoked to setup the mapping from a PCI BUS to a Die ID, which is used to locate the corresponding MC device of a IMC uncore unit in the spr_uncore_imc_freerunning_init_box(). Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Andi Kleen Link: https://lore.kernel.org/r/1625087320-194204-16-git-send-email-kan.liang@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 66 +++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 988254961fd8..2558e265c087 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -5753,6 +5753,7 @@ static struct intel_uncore_type spr_uncore_mdf = { #define UNCORE_SPR_NUM_UNCORE_TYPES 12 #define UNCORE_SPR_IIO 1 +#define UNCORE_SPR_IMC 6 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = { &spr_uncore_chabox, @@ -5849,12 +5850,65 @@ static struct intel_uncore_type spr_uncore_iio_free_running = { .format_group = &skx_uncore_iio_freerunning_format_group, }; +enum perf_uncore_spr_imc_freerunning_type_id { + SPR_IMC_DCLK, + SPR_IMC_PQ_CYCLES, + + SPR_IMC_FREERUNNING_TYPE_MAX, +}; + +static struct freerunning_counters spr_imc_freerunning[] = { + [SPR_IMC_DCLK] = { 0x22b0, 0x0, 0, 1, 48 }, + [SPR_IMC_PQ_CYCLES] = { 0x2318, 0x8, 0, 2, 48 }, +}; + +static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = { + INTEL_UNCORE_EVENT_DESC(dclk, "event=0xff,umask=0x10"), + + INTEL_UNCORE_EVENT_DESC(rpq_cycles, "event=0xff,umask=0x20"), + INTEL_UNCORE_EVENT_DESC(wpq_cycles, "event=0xff,umask=0x21"), + { /* end: all zeroes */ }, +}; + +#define SPR_MC_DEVICE_ID 0x3251 + +static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +{ + int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; + + snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + mem_offset, SPR_MC_DEVICE_ID); +} + +static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = { + .init_box = spr_uncore_imc_freerunning_init_box, + .exit_box = uncore_mmio_exit_box, + .read_counter = uncore_mmio_read_counter, + .hw_config = uncore_freerunning_hw_config, +}; + +static struct intel_uncore_type spr_uncore_imc_free_running = { + .name = "imc_free_running", + .num_counters = 3, + .mmio_map_size = SNR_IMC_MMIO_SIZE, + .num_freerunning_types = SPR_IMC_FREERUNNING_TYPE_MAX, + .freerunning = spr_imc_freerunning, + .ops = &spr_uncore_imc_freerunning_ops, + .event_descs = spr_uncore_imc_freerunning_events, + .format_group = &skx_uncore_iio_freerunning_format_group, +}; + #define UNCORE_SPR_MSR_EXTRA_UNCORES 1 +#define UNCORE_SPR_MMIO_EXTRA_UNCORES 1 static struct intel_uncore_type *spr_msr_uncores[UNCORE_SPR_MSR_EXTRA_UNCORES] = { &spr_uncore_iio_free_running, }; +static struct intel_uncore_type *spr_mmio_uncores[UNCORE_SPR_MMIO_EXTRA_UNCORES] = { + &spr_uncore_imc_free_running, +}; + static void uncore_type_customized_copy(struct intel_uncore_type *to_type, struct intel_uncore_type *from_type) { @@ -5957,7 +6011,17 @@ int spr_uncore_pci_init(void) void spr_uncore_mmio_init(void) { - uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO, 0, NULL); + int ret = snbep_pci2phy_map_init(0x3250, SKX_CPUNODEID, SKX_GIDNIDMAP, true); + + if (ret) + uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO, 0, NULL); + else { + uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO, + UNCORE_SPR_MMIO_EXTRA_UNCORES, + spr_mmio_uncores); + + spr_uncore_imc_free_running.num_boxes = uncore_type_max_boxes(uncore_mmio_uncores, UNCORE_SPR_IMC) / 2; + } } /* end of SPR uncore support */ From 3f2cbe3810a60111a33f5f6267bd5a237b826fc9 Mon Sep 17 00:00:00 2001 From: Alexander Antonov Date: Tue, 6 Jul 2021 12:07:23 +0300 Subject: [PATCH 16/24] perf/x86/intel/uncore: Fix IIO cleanup mapping procedure for SNR/ICX skx_iio_cleanup_mapping() is re-used for snr and icx, but in those cases it fails to use the appropriate XXX_iio_mapping_group and as such fails to free previously allocated resources, leading to memory leaks. Fixes: 10337e95e04c ("perf/x86/intel/uncore: Enable I/O stacks to IIO PMON mapping on ICX") Signed-off-by: Alexander Antonov [peterz: Changelog] Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kan Liang Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20210706090723.41850-1-alexander.antonov@linux.intel.com --- arch/x86/events/intel/uncore_snbep.c | 42 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 2558e265c087..f665b163cb29 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -3847,6 +3847,22 @@ clear_attr_update: return ret; } +static void +pmu_iio_cleanup_mapping(struct intel_uncore_type *type, struct attribute_group *ag) +{ + struct attribute **attr = ag->attrs; + + if (!attr) + return; + + for (; *attr; attr++) + kfree((*attr)->name); + kfree(attr_to_ext_attr(*ag->attrs)); + kfree(ag->attrs); + ag->attrs = NULL; + kfree(type->topology); +} + static int skx_iio_set_mapping(struct intel_uncore_type *type) { return pmu_iio_set_mapping(type, &skx_iio_mapping_group); @@ -3854,17 +3870,7 @@ static int skx_iio_set_mapping(struct intel_uncore_type *type) static void skx_iio_cleanup_mapping(struct intel_uncore_type *type) { - struct attribute **attr = skx_iio_mapping_group.attrs; - - if (!attr) - return; - - for (; *attr; attr++) - kfree((*attr)->name); - kfree(attr_to_ext_attr(*skx_iio_mapping_group.attrs)); - kfree(skx_iio_mapping_group.attrs); - skx_iio_mapping_group.attrs = NULL; - kfree(type->topology); + pmu_iio_cleanup_mapping(type, &skx_iio_mapping_group); } static struct intel_uncore_type skx_uncore_iio = { @@ -4510,6 +4516,11 @@ static int snr_iio_set_mapping(struct intel_uncore_type *type) return pmu_iio_set_mapping(type, &snr_iio_mapping_group); } +static void snr_iio_cleanup_mapping(struct intel_uncore_type *type) +{ + pmu_iio_cleanup_mapping(type, &snr_iio_mapping_group); +} + static struct intel_uncore_type snr_uncore_iio = { .name = "iio", .num_counters = 4, @@ -4526,7 +4537,7 @@ static struct intel_uncore_type snr_uncore_iio = { .attr_update = snr_iio_attr_update, .get_topology = snr_iio_get_topology, .set_mapping = snr_iio_set_mapping, - .cleanup_mapping = skx_iio_cleanup_mapping, + .cleanup_mapping = snr_iio_cleanup_mapping, }; static struct intel_uncore_type snr_uncore_irp = { @@ -5113,6 +5124,11 @@ static int icx_iio_set_mapping(struct intel_uncore_type *type) return pmu_iio_set_mapping(type, &icx_iio_mapping_group); } +static void icx_iio_cleanup_mapping(struct intel_uncore_type *type) +{ + pmu_iio_cleanup_mapping(type, &icx_iio_mapping_group); +} + static struct intel_uncore_type icx_uncore_iio = { .name = "iio", .num_counters = 4, @@ -5130,7 +5146,7 @@ static struct intel_uncore_type icx_uncore_iio = { .attr_update = icx_iio_attr_update, .get_topology = icx_iio_get_topology, .set_mapping = icx_iio_set_mapping, - .cleanup_mapping = skx_iio_cleanup_mapping, + .cleanup_mapping = icx_iio_cleanup_mapping, }; static struct intel_uncore_type icx_uncore_irp = { From 4f32da76a1401dcd088930f0ac8658425524368b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 4 Aug 2021 12:57:10 +0100 Subject: [PATCH 17/24] perf/x86: Remove unused assignment to pointer 'e' The pointer 'e' is being assigned a value that is never read, the assignment is redundant and can be removed. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lkml.kernel.org/r/20210804115710.109608-1-colin.king@canonical.com --- arch/x86/events/core.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 3092fbf9dbe4..2a57dbed4894 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1087,10 +1087,8 @@ int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) * validate an event group (assign == NULL) */ if (!unsched && assign) { - for (i = 0; i < n; i++) { - e = cpuc->event_list[i]; + for (i = 0; i < n; i++) static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]); - } } else { for (i = n0; i < n; i++) { e = cpuc->event_list[i]; From eda8a2c599d1ff874a63de7684b430740e747dea Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 3 Aug 2021 16:15:53 +0200 Subject: [PATCH 18/24] perf/x86/intel: Replace deprecated CPU-hotplug functions The functions get_online_cpus() and put_online_cpus() have been deprecated during the CPU hotplug rework. They map directly to cpus_read_lock() and cpus_read_unlock(). Replace deprecated CPU-hotplug functions with the official version. The behavior remains unchanged. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210803141621.780504-11-bigeasy@linutronix.de --- arch/x86/events/intel/core.c | 8 ++++---- arch/x86/events/intel/pt.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index ac6fd2dabf6a..7011e87be6d0 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -5032,9 +5032,9 @@ static ssize_t freeze_on_smi_store(struct device *cdev, x86_pmu.attr_freeze_on_smi = val; - get_online_cpus(); + cpus_read_lock(); on_each_cpu(flip_smm_bit, &val, 1); - put_online_cpus(); + cpus_read_unlock(); done: mutex_unlock(&freeze_on_smi_mutex); @@ -5077,9 +5077,9 @@ static ssize_t set_sysctl_tfa(struct device *cdev, allow_tsx_force_abort = val; - get_online_cpus(); + cpus_read_lock(); on_each_cpu(update_tfa_sched, NULL, 1); - put_online_cpus(); + cpus_read_unlock(); return count; } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index b044577785bb..7f406c14715f 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -1708,7 +1708,7 @@ static __init int pt_init(void) if (!boot_cpu_has(X86_FEATURE_INTEL_PT)) return -ENODEV; - get_online_cpus(); + cpus_read_lock(); for_each_online_cpu(cpu) { u64 ctl; @@ -1716,7 +1716,7 @@ static __init int pt_init(void) if (!ret && (ctl & RTIT_CTL_TRACEEN)) prior_warn++; } - put_online_cpus(); + cpus_read_unlock(); if (prior_warn) { x86_add_exclusive(x86_lbr_exclusive_pt); From ffec09f9c7d7b21b0aff29dd5c3972f4631c0b6b Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 3 Aug 2021 16:15:54 +0200 Subject: [PATCH 19/24] perf/hw_breakpoint: Replace deprecated CPU-hotplug functions The functions get_online_cpus() and put_online_cpus() have been deprecated during the CPU hotplug rework. They map directly to cpus_read_lock() and cpus_read_unlock(). Replace deprecated CPU-hotplug functions with the official version. The behavior remains unchanged. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210803141621.780504-12-bigeasy@linutronix.de --- kernel/events/hw_breakpoint.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c index 835973444a1e..f32320ac02fd 100644 --- a/kernel/events/hw_breakpoint.c +++ b/kernel/events/hw_breakpoint.c @@ -568,7 +568,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr, if (!cpu_events) return (void __percpu __force *)ERR_PTR(-ENOMEM); - get_online_cpus(); + cpus_read_lock(); for_each_online_cpu(cpu) { bp = perf_event_create_kernel_counter(attr, cpu, NULL, triggered, context); @@ -579,7 +579,7 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr, per_cpu(*cpu_events, cpu) = bp; } - put_online_cpus(); + cpus_read_unlock(); if (likely(!err)) return cpu_events; From 6cf295b21608f9253037335f47cd0dfcce812d81 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 17 Aug 2021 17:10:44 -0500 Subject: [PATCH 20/24] perf/amd/uncore: Simplify code, use free_percpu()'s built-in check for NULL free_percpu() has its own check for NULL, no need to open-code it. Signed-off-by: Kim Phillips Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210817221048.88063-5-kim.phillips@amd.com --- arch/x86/events/amd/uncore.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 582c0ffb5e98..05bdb4cba300 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -659,11 +659,9 @@ fail_prep: fail_llc: if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) perf_pmu_unregister(&amd_nb_pmu); - if (amd_uncore_llc) - free_percpu(amd_uncore_llc); + free_percpu(amd_uncore_llc); fail_nb: - if (amd_uncore_nb) - free_percpu(amd_uncore_nb); + free_percpu(amd_uncore_nb); return ret; } From 0a0b53e0c3793c0930d258786702d48d21fc6383 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 17 Aug 2021 17:10:45 -0500 Subject: [PATCH 21/24] perf/amd/uncore: Clean up header use, use Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210817221048.88063-6-kim.phillips@amd.com --- arch/x86/events/amd/uncore.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 05bdb4cba300..7fb50ad171e9 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -12,11 +12,11 @@ #include #include #include +#include +#include -#include #include #include -#include #define NUM_COUNTERS_NB 4 #define NUM_COUNTERS_L2 4 From 9164d9493a792682143af12b182be12d7c32b195 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 17 Aug 2021 17:10:46 -0500 Subject: [PATCH 22/24] x86/cpu: Add get_llc_id() helper function Factor out a helper function rather than export cpu_llc_id, which is needed in order to be able to build the AMD uncore driver as a module. Signed-off-by: Kim Phillips Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210817221048.88063-7-kim.phillips@amd.com --- arch/x86/events/amd/uncore.c | 2 +- arch/x86/include/asm/processor.h | 2 ++ arch/x86/kernel/cpu/amd.c | 2 +- arch/x86/kernel/cpu/common.c | 6 ++++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 7fb50ad171e9..a01f9f1016d9 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -452,7 +452,7 @@ static int amd_uncore_cpu_starting(unsigned int cpu) if (amd_uncore_llc) { uncore = *per_cpu_ptr(amd_uncore_llc, cpu); - uncore->id = per_cpu(cpu_llc_id, cpu); + uncore->id = get_llc_id(cpu); uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc); *per_cpu_ptr(amd_uncore_llc, cpu) = uncore; diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index f3020c54e2cb..33dd1575104e 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -795,6 +795,8 @@ extern int set_tsc_mode(unsigned int val); DECLARE_PER_CPU(u64, msr_misc_features_shadow); +extern u16 get_llc_id(unsigned int cpu); + #ifdef CONFIG_CPU_SUP_AMD extern u32 amd_get_nodes_per_socket(void); extern u32 amd_get_highest_perf(void); diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index b7c003013d41..2131af9f2fa2 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -438,7 +438,7 @@ static void srat_detect_node(struct cpuinfo_x86 *c) node = numa_cpu_node(cpu); if (node == NUMA_NO_NODE) - node = per_cpu(cpu_llc_id, cpu); + node = get_llc_id(cpu); /* * On multi-fabric platform (e.g. Numascale NumaChip) a diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 64b805bd6a54..0f8885949e8c 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -79,6 +79,12 @@ EXPORT_SYMBOL(smp_num_siblings); /* Last level cache ID of each logical CPU */ DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID; +u16 get_llc_id(unsigned int cpu) +{ + return per_cpu(cpu_llc_id, cpu); +} +EXPORT_SYMBOL_GPL(get_llc_id); + /* correctly size the local cpu masks */ void __init setup_cpu_local_masks(void) { From 05485745ad482c1910a45f23a5c255f6a0df0f46 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 17 Aug 2021 17:10:47 -0500 Subject: [PATCH 23/24] perf/amd/uncore: Allow the driver to be built as a module Add support to build the AMD uncore driver as a module. This is in order to facilitate development without having to reboot the kernel in most cases. Signed-off-by: Kim Phillips Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210817221048.88063-8-kim.phillips@amd.com --- arch/x86/events/Kconfig | 10 ++++++++++ arch/x86/events/amd/Makefile | 5 +++-- arch/x86/events/amd/uncore.c | 28 +++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/arch/x86/events/Kconfig b/arch/x86/events/Kconfig index 39d9ded9e25a..d6cdfe631674 100644 --- a/arch/x86/events/Kconfig +++ b/arch/x86/events/Kconfig @@ -34,4 +34,14 @@ config PERF_EVENTS_AMD_POWER (CPUID Fn8000_0007_EDX[12]) interface to calculate the average power consumption on Family 15h processors. +config PERF_EVENTS_AMD_UNCORE + tristate "AMD Uncore performance events" + depends on PERF_EVENTS && CPU_SUP_AMD + default y + help + Include support for AMD uncore performance events for use with + e.g., perf stat -e amd_l3/.../,amd_df/.../. + + To compile this driver as a module, choose M here: the + module will be called 'amd-uncore'. endmenu diff --git a/arch/x86/events/amd/Makefile b/arch/x86/events/amd/Makefile index fe8795a67385..6cbe38d5fd9d 100644 --- a/arch/x86/events/amd/Makefile +++ b/arch/x86/events/amd/Makefile @@ -1,8 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_CPU_SUP_AMD) += core.o uncore.o +obj-$(CONFIG_CPU_SUP_AMD) += core.o obj-$(CONFIG_PERF_EVENTS_AMD_POWER) += power.o obj-$(CONFIG_X86_LOCAL_APIC) += ibs.o +obj-$(CONFIG_PERF_EVENTS_AMD_UNCORE) += amd-uncore.o +amd-uncore-objs := uncore.o ifdef CONFIG_AMD_IOMMU obj-$(CONFIG_CPU_SUP_AMD) += iommu.o endif - diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index a01f9f1016d9..0d04414b97d2 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -347,6 +347,7 @@ static struct pmu amd_nb_pmu = { .stop = amd_uncore_stop, .read = amd_uncore_read, .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, + .module = THIS_MODULE, }; static struct pmu amd_llc_pmu = { @@ -360,6 +361,7 @@ static struct pmu amd_llc_pmu = { .stop = amd_uncore_stop, .read = amd_uncore_read, .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, + .module = THIS_MODULE, }; static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) @@ -665,4 +667,28 @@ fail_nb: return ret; } -device_initcall(amd_uncore_init); + +static void __exit amd_uncore_exit(void) +{ + cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE); + cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); + cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); + + if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { + perf_pmu_unregister(&amd_llc_pmu); + free_percpu(amd_uncore_llc); + amd_uncore_llc = NULL; + } + + if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { + perf_pmu_unregister(&amd_nb_pmu); + free_percpu(amd_uncore_nb); + amd_uncore_nb = NULL; + } +} + +module_init(amd_uncore_init); +module_exit(amd_uncore_exit); + +MODULE_DESCRIPTION("AMD Uncore Driver"); +MODULE_LICENSE("GPL v2"); From 6a371bafe613b7746c3d3ac486bdb3035f77e029 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Tue, 17 Aug 2021 17:10:48 -0500 Subject: [PATCH 24/24] perf/x86/amd/ibs: Add bitfield definitions in new header Add with bitfield definitions for IBS MSRs, and demonstrate usage within the driver. Also move 'struct perf_ibs_data' where it can be shared with the perf tool that will soon be using it. No functional changes. Signed-off-by: Kim Phillips Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210817221048.88063-9-kim.phillips@amd.com --- arch/x86/events/amd/ibs.c | 23 +++--- arch/x86/include/asm/amd-ibs.h | 132 +++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 14 deletions(-) create mode 100644 arch/x86/include/asm/amd-ibs.h diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index ccc9ee1971e8..9739019d4b67 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -26,6 +26,7 @@ static u32 ibs_caps; #include #include +#include #define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT) #define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT @@ -100,15 +101,6 @@ struct perf_ibs { u64 (*get_count)(u64 config); }; -struct perf_ibs_data { - u32 size; - union { - u32 data[0]; /* data buffer starts here */ - u32 caps; - }; - u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; -}; - static int perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) { @@ -329,11 +321,14 @@ static int perf_ibs_set_period(struct perf_ibs *perf_ibs, static u64 get_ibs_fetch_count(u64 config) { - return (config & IBS_FETCH_CNT) >> 12; + union ibs_fetch_ctl fetch_ctl = (union ibs_fetch_ctl)config; + + return fetch_ctl.fetch_cnt << 4; } static u64 get_ibs_op_count(u64 config) { + union ibs_op_ctl op_ctl = (union ibs_op_ctl)config; u64 count = 0; /* @@ -341,12 +336,12 @@ static u64 get_ibs_op_count(u64 config) * and the lower 7 bits of CurCnt are randomized. * Otherwise CurCnt has the full 27-bit current counter value. */ - if (config & IBS_OP_VAL) { - count = (config & IBS_OP_MAX_CNT) << 4; + if (op_ctl.op_val) { + count = op_ctl.opmaxcnt << 4; if (ibs_caps & IBS_CAPS_OPCNTEXT) - count += config & IBS_OP_MAX_CNT_EXT_MASK; + count += op_ctl.opmaxcnt_ext << 20; } else if (ibs_caps & IBS_CAPS_RDWROPCNT) { - count = (config & IBS_OP_CUR_CNT) >> 32; + count = op_ctl.opcurcnt; } return count; diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd-ibs.h new file mode 100644 index 000000000000..46e1df45efc0 --- /dev/null +++ b/arch/x86/include/asm/amd-ibs.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * From PPR Vol 1 for AMD Family 19h Model 01h B1 + * 55898 Rev 0.35 - Feb 5, 2021 + */ + +#include + +/* + * IBS Hardware MSRs + */ + +/* MSR 0xc0011030: IBS Fetch Control */ +union ibs_fetch_ctl { + __u64 val; + struct { + __u64 fetch_maxcnt:16,/* 0-15: instruction fetch max. count */ + fetch_cnt:16, /* 16-31: instruction fetch count */ + fetch_lat:16, /* 32-47: instruction fetch latency */ + fetch_en:1, /* 48: instruction fetch enable */ + fetch_val:1, /* 49: instruction fetch valid */ + fetch_comp:1, /* 50: instruction fetch complete */ + ic_miss:1, /* 51: i-cache miss */ + phy_addr_valid:1,/* 52: physical address valid */ + l1tlb_pgsz:2, /* 53-54: i-cache L1TLB page size + * (needs IbsPhyAddrValid) */ + l1tlb_miss:1, /* 55: i-cache fetch missed in L1TLB */ + l2tlb_miss:1, /* 56: i-cache fetch missed in L2TLB */ + rand_en:1, /* 57: random tagging enable */ + fetch_l2_miss:1,/* 58: L2 miss for sampled fetch + * (needs IbsFetchComp) */ + reserved:5; /* 59-63: reserved */ + }; +}; + +/* MSR 0xc0011033: IBS Execution Control */ +union ibs_op_ctl { + __u64 val; + struct { + __u64 opmaxcnt:16, /* 0-15: periodic op max. count */ + reserved0:1, /* 16: reserved */ + op_en:1, /* 17: op sampling enable */ + op_val:1, /* 18: op sample valid */ + cnt_ctl:1, /* 19: periodic op counter control */ + opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */ + reserved1:5, /* 27-31: reserved */ + opcurcnt:27, /* 32-58: periodic op counter current count */ + reserved2:5; /* 59-63: reserved */ + }; +}; + +/* MSR 0xc0011035: IBS Op Data 2 */ +union ibs_op_data { + __u64 val; + struct { + __u64 comp_to_ret_ctr:16, /* 0-15: op completion to retire count */ + tag_to_ret_ctr:16, /* 15-31: op tag to retire count */ + reserved1:2, /* 32-33: reserved */ + op_return:1, /* 34: return op */ + op_brn_taken:1, /* 35: taken branch op */ + op_brn_misp:1, /* 36: mispredicted branch op */ + op_brn_ret:1, /* 37: branch op retired */ + op_rip_invalid:1, /* 38: RIP is invalid */ + op_brn_fuse:1, /* 39: fused branch op */ + op_microcode:1, /* 40: microcode op */ + reserved2:23; /* 41-63: reserved */ + }; +}; + +/* MSR 0xc0011036: IBS Op Data 2 */ +union ibs_op_data2 { + __u64 val; + struct { + __u64 data_src:3, /* 0-2: data source */ + reserved0:1, /* 3: reserved */ + rmt_node:1, /* 4: destination node */ + cache_hit_st:1, /* 5: cache hit state */ + reserved1:57; /* 5-63: reserved */ + }; +}; + +/* MSR 0xc0011037: IBS Op Data 3 */ +union ibs_op_data3 { + __u64 val; + struct { + __u64 ld_op:1, /* 0: load op */ + st_op:1, /* 1: store op */ + dc_l1tlb_miss:1, /* 2: data cache L1TLB miss */ + dc_l2tlb_miss:1, /* 3: data cache L2TLB hit in 2M page */ + dc_l1tlb_hit_2m:1, /* 4: data cache L1TLB hit in 2M page */ + dc_l1tlb_hit_1g:1, /* 5: data cache L1TLB hit in 1G page */ + dc_l2tlb_hit_2m:1, /* 6: data cache L2TLB hit in 2M page */ + dc_miss:1, /* 7: data cache miss */ + dc_mis_acc:1, /* 8: misaligned access */ + reserved:4, /* 9-12: reserved */ + dc_wc_mem_acc:1, /* 13: write combining memory access */ + dc_uc_mem_acc:1, /* 14: uncacheable memory access */ + dc_locked_op:1, /* 15: locked operation */ + dc_miss_no_mab_alloc:1, /* 16: DC miss with no MAB allocated */ + dc_lin_addr_valid:1, /* 17: data cache linear address valid */ + dc_phy_addr_valid:1, /* 18: data cache physical address valid */ + dc_l2_tlb_hit_1g:1, /* 19: data cache L2 hit in 1GB page */ + l2_miss:1, /* 20: L2 cache miss */ + sw_pf:1, /* 21: software prefetch */ + op_mem_width:4, /* 22-25: load/store size in bytes */ + op_dc_miss_open_mem_reqs:6, /* 26-31: outstanding mem reqs on DC fill */ + dc_miss_lat:16, /* 32-47: data cache miss latency */ + tlb_refill_lat:16; /* 48-63: L1 TLB refill latency */ + }; +}; + +/* MSR 0xc001103c: IBS Fetch Control Extended */ +union ic_ibs_extd_ctl { + __u64 val; + struct { + __u64 itlb_refill_lat:16, /* 0-15: ITLB Refill latency for sampled fetch */ + reserved:48; /* 16-63: reserved */ + }; +}; + +/* + * IBS driver related + */ + +struct perf_ibs_data { + u32 size; + union { + u32 data[0]; /* data buffer starts here */ + u32 caps; + }; + u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; +};