diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c index c2e5bdf3893d84aaa4e97c32e21c9f3bf6410f74..b2dd7a4eb54121098fc5834446ed9b8ba2646332 100644 --- a/arch/x86/events/zhaoxin/core.c +++ b/arch/x86/events/zhaoxin/core.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Zhaoxin PMU; + * Zhaoxin PMU; like Intel Architectural PerfMon-v2 */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -19,32 +19,29 @@ #include "../perf_event.h" /* - * Zhaoxin PerfMon, used on zxc and later. + * Zhaoxin PerfMon, used on Lujiazui and later. */ static u64 zx_pmon_event_map[PERF_COUNT_HW_MAX] __read_mostly = { - - [PERF_COUNT_HW_CPU_CYCLES] = 0x0082, - [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, - [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0515, - [PERF_COUNT_HW_CACHE_MISSES] = 0x051a, - [PERF_COUNT_HW_BUS_CYCLES] = 0x0083, + [PERF_COUNT_HW_CPU_CYCLES] = 0x0082, + [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, + [PERF_COUNT_HW_BUS_CYCLES] = 0x0083, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0028, + [PERF_COUNT_HW_BRANCH_MISSES] = 0x0029, }; static struct event_constraint zxc_event_constraints[] __read_mostly = { - FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */ EVENT_CONSTRAINT_END }; -static struct event_constraint zxd_event_constraints[] __read_mostly = { - +static struct event_constraint wudaokou_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* retired instructions */ FIXED_EVENT_CONSTRAINT(0x0082, 1), /* unhalted core clock cycles */ FIXED_EVENT_CONSTRAINT(0x0083, 2), /* unhalted bus clock cycles */ EVENT_CONSTRAINT_END }; -static __initconst const u64 zxd_hw_cache_event_ids +static __initconst const u64 wudaokou_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { @@ -148,7 +145,7 @@ static __initconst const u64 zxd_hw_cache_event_ids }, }; -static __initconst const u64 zxe_hw_cache_event_ids +static __initconst const u64 lujiazui_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { @@ -259,7 +256,10 @@ static void zhaoxin_pmu_disable_all(void) static void zhaoxin_pmu_enable_all(int added) { - wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, + x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask); } static inline u64 zhaoxin_pmu_get_status(void) @@ -286,13 +286,31 @@ static inline void zxc_pmu_ack_status(u64 ack) zhaoxin_pmu_disable_all(); } -static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc) +static inline void zhaoxin_set_masks(struct perf_event *event, int idx) { - int idx = hwc->idx - INTEL_PMC_IDX_FIXED; - u64 ctrl_val, mask; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - mask = 0xfULL << (idx * 4); + if (event->attr.exclude_host) + __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask); + if (event->attr.exclude_guest) + __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask); +} + +static inline void zhaoxin_clear_masks(struct perf_event *event, int idx) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask); + __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask); +} + +static void zhaoxin_pmu_disable_fixed(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 ctrl_val, mask; + int idx = hwc->idx; + + mask = 0xfULL << ((idx - INTEL_PMC_IDX_FIXED) * 4); rdmsrl(hwc->config_base, ctrl_val); ctrl_val &= ~mask; wrmsrl(hwc->config_base, ctrl_val); @@ -301,19 +319,23 @@ static void zhaoxin_pmu_disable_fixed(struct hw_perf_event *hwc) static void zhaoxin_pmu_disable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + zhaoxin_clear_masks(event, idx); if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { - zhaoxin_pmu_disable_fixed(hwc); + zhaoxin_pmu_disable_fixed(event); return; } x86_pmu_disable_event(event); } -static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc) +static void zhaoxin_pmu_enable_fixed(struct perf_event *event) { - int idx = hwc->idx - INTEL_PMC_IDX_FIXED; - u64 ctrl_val, bits, mask; + struct hw_perf_event *hwc = &event->hw; + u64 ctrl_val, mask, bits = 0; + int idx = hwc->idx; /* * Enable IRQ generation (0x8), @@ -326,6 +348,7 @@ static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc) if (hwc->config & ARCH_PERFMON_EVENTSEL_OS) bits |= 0x1; + idx -= INTEL_PMC_IDX_FIXED; bits <<= (idx * 4); mask = 0xfULL << (idx * 4); @@ -338,9 +361,12 @@ static void zhaoxin_pmu_enable_fixed(struct hw_perf_event *hwc) static void zhaoxin_pmu_enable_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + zhaoxin_set_masks(event, idx); if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { - zhaoxin_pmu_enable_fixed(hwc); + zhaoxin_pmu_enable_fixed(event); return; } @@ -418,9 +444,8 @@ static u64 zhaoxin_pmu_event_map(int hw_event) return zx_pmon_event_map[hw_event]; } -static struct event_constraint * -zhaoxin_get_event_constraints(struct cpu_hw_events *cpuc, int idx, - struct perf_event *event) +static struct event_constraint *zhaoxin_get_event_constraints(struct cpu_hw_events *cpuc, int idx, + struct perf_event *event) { struct event_constraint *c; @@ -456,8 +481,21 @@ static ssize_t zhaoxin_event_sysfs_show(char *page, u64 config) return x86_event_sysfs_show(page, config, event); } +static struct perf_guest_switch_msr *zhaoxin_guest_get_msrs(int *nr) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; + + arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL; + arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask; + arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask; + *nr = 1; + + return arr; +} + static const struct x86_pmu zhaoxin_pmu __initconst = { - .name = "zhaoxin_pmu", + .name = "zhaoxin", .handle_irq = zhaoxin_pmu_handle_irq, .disable_all = zhaoxin_pmu_disable_all, .enable_all = zhaoxin_pmu_enable_all, @@ -471,13 +509,15 @@ static const struct x86_pmu zhaoxin_pmu __initconst = { .max_events = ARRAY_SIZE(zx_pmon_event_map), .apic = 1, /* - * For zxd/zxe, read/write operation for PMCx MSR is 48 bits. + * For wudaokou/lujiazui, read/write operation for PMCx MSR is 48 bits. */ .max_period = (1ULL << 47) - 1, .get_event_constraints = zhaoxin_get_event_constraints, .format_attrs = zx_arch_formats_attr, .events_sysfs_show = zhaoxin_event_sysfs_show, + + .guest_get_msrs = zhaoxin_guest_get_msrs, }; static const struct { int id; char *name; } zx_arch_events_map[] __initconst = { @@ -494,11 +534,10 @@ static __init void zhaoxin_arch_events_quirk(void) { int bit; - /* disable event that reported as not presend by cpuid */ + /* disable event that reported as not present by cpuid */ for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(zx_arch_events_map)) { zx_pmon_event_map[zx_arch_events_map[bit].id] = 0; - pr_warn("CPUID marked event: \'%s\' unavailable\n", - zx_arch_events_map[bit].name); + pr_warn("CPUID marked event: \'%s\' unavailable\n", zx_arch_events_map[bit].name); } } @@ -511,7 +550,7 @@ __init int zhaoxin_pmu_init(void) unsigned int unused; int version; - pr_info("Welcome to pmu!\n"); + pr_info("Welcome to zhaoxin pmu!\n"); /* * Check whether the Architectural PerfMon supports @@ -541,7 +580,13 @@ __init int zhaoxin_pmu_init(void) switch (boot_cpu_data.x86) { case 0x06: - if (boot_cpu_data.x86_model == 0x0f || boot_cpu_data.x86_model == 0x19) { + /* + * Support Zhaoxin CPU from ZXC series, exclude Nano series through FMS. + * Nano FMS: Family=6, Model=F, Stepping=[0-A][C-D] + * ZXC FMS: Family=6, Model=F, Stepping=E-F OR Family=6, Model=0x19, Stepping=0-3 + */ + if ((boot_cpu_data.x86_model == 0x0f && boot_cpu_data.x86_stepping >= 0x0e) || + boot_cpu_data.x86_model == 0x19) { x86_pmu.max_period = x86_pmu.cntval_mask >> 1; @@ -553,9 +598,11 @@ __init int zhaoxin_pmu_init(void) zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0; zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0; zx_pmon_event_map[PERF_COUNT_HW_BUS_CYCLES] = 0; + zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0; + zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0; - pr_cont("C events, "); - break; + pr_cont("ZXC events, "); + break; } return -ENODEV; @@ -568,26 +615,50 @@ __init int zhaoxin_pmu_init(void) switch (boot_cpu_data.x86_model) { case 0x1b: - memcpy(hw_cache_event_ids, zxd_hw_cache_event_ids, + memcpy(hw_cache_event_ids, wudaokou_hw_cache_event_ids, sizeof(hw_cache_event_ids)); - x86_pmu.event_constraints = zxd_event_constraints; + x86_pmu.event_constraints = wudaokou_event_constraints; + + zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0x0515; + zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0x051a; zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0700; zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0709; - pr_cont("D events, "); + pr_cont("Wudaokou events, "); break; case 0x3b: - memcpy(hw_cache_event_ids, zxe_hw_cache_event_ids, + memcpy(hw_cache_event_ids, lujiazui_hw_cache_event_ids, sizeof(hw_cache_event_ids)); - x86_pmu.event_constraints = zxd_event_constraints; + x86_pmu.event_constraints = wudaokou_event_constraints; + + zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0x0515; + zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0x051a; + + pr_cont("Lujiazui events, "); + break; + case 0x5b: + case 0x6b: + zx_pmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = + X86_CONFIG(.event = 0x02, .umask = 0x01, .inv = 0x01, + .cmask = 0x01); + + memcpy(hw_cache_event_ids, lujiazui_hw_cache_event_ids, + sizeof(hw_cache_event_ids)); + + x86_pmu.event_constraints = wudaokou_event_constraints; + + zx_pmon_event_map[PERF_COUNT_HW_CACHE_REFERENCES] = 0x051a; + zx_pmon_event_map[PERF_COUNT_HW_CACHE_MISSES] = 0; + + if (boot_cpu_data.x86_model == 0x5b) + pr_cont("Yongfeng events, "); - zx_pmon_event_map[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0028; - zx_pmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x0029; + if (boot_cpu_data.x86_model == 0x6b) + pr_cont("Shijidadao events, "); - pr_cont("E events, "); break; default: return -ENODEV; diff --git a/arch/x86/events/zhaoxin/uncore.c b/arch/x86/events/zhaoxin/uncore.c index 4c4ea01d23c8c92c1f33c9acbdecb4c7c16c5483..5a2c0fe39df094efcb3c5fa6d491f42a641a83eb 100644 --- a/arch/x86/events/zhaoxin/uncore.c +++ b/arch/x86/events/zhaoxin/uncore.c @@ -1,196 +1,1392 @@ // SPDX-License-Identifier: GPL-2.0-only -#include - #include #include "uncore.h" static struct zhaoxin_uncore_type *empty_uncore[] = { NULL, }; static struct zhaoxin_uncore_type **uncore_msr_uncores = empty_uncore; +static struct zhaoxin_uncore_type **uncore_pci_uncores = empty_uncore; +static struct zhaoxin_uncore_type **uncore_mmio_uncores = empty_uncore; + +static bool pcidrv_registered; +static struct pci_driver *uncore_pci_driver; /* mask of cpus that collect uncore events */ static cpumask_t uncore_cpu_mask; +static cpumask_t uncore_cpu_subnode_mask; +static cpumask_t uncore_cpu_cluster_mask; /* constraint for the fixed counter */ static struct event_constraint uncore_constraint_fixed = EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL); -static int max_packages; +static int max_packages, max_subnodes, max_clusters; +static int clusters_per_subnode; +static int subnodes_per_die; +static int dies_per_socket; + +#define KH40000_MAX_SUBNODE_NUMBER 8 +static int kh40000_pcibus_limit[KH40000_MAX_SUBNODE_NUMBER]; + +/* get CPU topology register */ +#define BJ_GLOBAL_STATUS_MSR 0x1610 +#define BJ_HDW_CONFIG_MSR 0X1628 + +/* KX5000/KX6000 event control */ +#define KX5000_UNC_CTL_EV_SEL_MASK 0x000000ff +#define KX5000_UNC_CTL_UMASK_MASK 0x0000ff00 +#define KX5000_UNC_CTL_EDGE_DET (1 << 18) +#define KX5000_UNC_CTL_EN (1 << 22) +#define KX5000_UNC_CTL_INVERT (1 << 23) +#define KX5000_UNC_CTL_CMASK_MASK 0x7000000 +#define KX5000_UNC_FIXED_CTR_CTL_EN (1 << 0) + +#define KX5000_UNC_RAW_EVENT_MASK (KX5000_UNC_CTL_EV_SEL_MASK | \ + KX5000_UNC_CTL_UMASK_MASK | \ + KX5000_UNC_CTL_EDGE_DET | \ + KX5000_UNC_CTL_INVERT | \ + KX5000_UNC_CTL_CMASK_MASK) + +/* KX5000/KX6000 uncore global register */ +#define KX5000_UNC_PERF_GLOBAL_CTL 0x391 +#define KX5000_UNC_FIXED_CTR 0x394 +#define KX5000_UNC_FIXED_CTR_CTRL 0x395 + +/* KX5000/KX6000 uncore global control */ +#define KX5000_UNC_GLOBAL_CTL_EN_PC_ALL ((1ULL << 4) - 1) +#define KX5000_UNC_GLOBAL_CTL_EN_FC (1ULL << 32) + +/* KX5000/KX6000 uncore register */ +#define KX5000_UNC_PERFEVTSEL0 0x3c0 +#define KX5000_UNC_UNCORE_PMC0 0x3b0 + +/* KH40000 event control */ +#define KH40000_PMON_CTL_EV_SEL_MASK 0x000000ff +#define KH40000_PMON_CTL_UMASK_MASK 0x0000ff00 +#define KH40000_PMON_CTL_RST (1 << 17) +#define KH40000_PMON_CTL_EDGE_DET (1 << 18) +#define KH40000_PMON_CTL_EN (1 << 22) +#define KH40000_PMON_CTL_INVERT (1 << 23) +#define KH40000_PMON_CTL_THRESH_MASK 0xff000000 +#define KH40000_PMON_RAW_EVENT_MASK (KH40000_PMON_CTL_EV_SEL_MASK | \ + KH40000_PMON_CTL_UMASK_MASK | \ + KH40000_PMON_CTL_EDGE_DET | \ + KH40000_PMON_CTL_INVERT | \ + KH40000_PMON_CTL_THRESH_MASK) + +/* KH40000 LLC register*/ +#define KH40000_LLC_MSR_PMON_CTL0 0x1660 +#define KH40000_LLC_MSR_PMON_CTR0 0x165c +#define KH40000_LLC_MSR_PMON_BLK_CTL 0x1665 + +/* KH40000 HIF register*/ +#define KH40000_HIF_MSR_PMON_CTL0 0x1656 +#define KH40000_HIF_MSR_PMON_CTR0 0x1651 +#define KH40000_HIF_MSR_PMON_FIXED_CTL 0x1655 +#define KH40000_HIF_MSR_PMON_FIXED_CTR 0x1650 +#define KH40000_HIF_MSR_PMON_BLK_CTL 0x165b + +/* KH40000 ZZI(ZPI+ZOI+INI) register*/ +#define KH40000_ZZI_MSR_PMON_CTL0 0x166A +#define KH40000_ZZI_MSR_PMON_CTR0 0x1666 +#define KH40000_ZZI_MSR_PMON_BLK_CTL 0x166f + +/* KH40000 MC register*/ +#define KH40000_MC0_CHy_PMON_FIXED_CTL 0xf40 +#define KH40000_MC0_CHy_PMON_FIXED_CTR 0xf20 +#define KH40000_MC0_CHy_PMON_CTR0 0xf00 +#define KH40000_MC0_CHy_PMON_CTL0 0xf28 +#define KH40000_MC0_CHy_PMON_BLK_CTL 0xf44 + +#define KH40000_MC1_CHy_PMON_FIXED_CTL 0xf90 +#define KH40000_MC1_CHy_PMON_FIXED_CTR 0xf70 +#define KH40000_MC1_CHy_PMON_CTR0 0xf50 +#define KH40000_MC1_CHy_PMON_CTL0 0xf78 +#define KH40000_MC1_CHy_PMON_BLK_CTL 0xf94 + +/* KH40000 PCI register*/ +#define KH40000_PCI_PMON_CTR0 0xf00 +#define KH40000_PCI_PMON_CTL0 0xf28 +#define KH40000_PCI_PMON_BLK_CTL 0xf44 + +/* KH40000 ZPI_DLL register*/ +#define KH40000_ZPI_DLL_PMON_FIXED_CTL 0xf40 +#define KH40000_ZPI_DLL_PMON_FIXED_CTR 0xf20 +#define KH40000_ZPI_DLL_PMON_CTR0 0xf00 +#define KH40000_ZPI_DLL_PMON_CTL0 0xf28 +#define KH40000_ZPI_DLL_PMON_BLK_CTL 0xf44 + +/* KH40000 ZDI_DLL register*/ +#define KH40000_ZDI_DLL_PMON_FIXED_CTL 0xf40 +#define KH40000_ZDI_DLL_PMON_FIXED_CTR 0xf20 +#define KH40000_ZDI_DLL_PMON_CTR0 0xf00 +#define KH40000_ZDI_DLL_PMON_CTL0 0xf28 +#define KH40000_ZDI_DLL_PMON_BLK_CTL 0xf44 + +/* KH40000 PXPTRF register*/ +#define KH40000_PXPTRF_PMON_CTR0 0xf00 +#define KH40000_PXPTRF_PMON_CTL0 0xf28 +#define KH40000_PXPTRF_PMON_BLK_CTL 0xf44 + +/* KH40000 Box level control */ +#define KH40000_PMON_BOX_CTL_RST_CTRL (1 << 0) +#define KH40000_PMON_BOX_CTL_RST_CTRS (1 << 1) +#define KH40000_PMON_BOX_CTL_FRZ (1 << 8) +#define KH40000_PMON_PCI_BOX_PMON_EN (1 << 31) + +#define KH40000_PMON_BOX_CTL_INT (KH40000_PMON_BOX_CTL_RST_CTRL | \ + KH40000_PMON_BOX_CTL_RST_CTRS) + +#define KH40000_PMON_PCI_BOX_CTL_INT (KH40000_PMON_BOX_CTL_RST_CTRL | \ + KH40000_PMON_BOX_CTL_RST_CTRS | \ + KH40000_PMON_PCI_BOX_PMON_EN) + + +/* KX7000 event control */ +#define KX7000_PMON_CTL_EV_SEL_MASK 0x000000ff +#define KX7000_PMON_CTL_UMASK_MASK 0x0000ff00 +#define KX7000_PMON_CTL_RST (1 << 17) +#define KX7000_PMON_CTL_EDGE_DET (1 << 18) +#define KX7000_PMON_CTL_LOGIC_OP0 (1 << 19) +#define KX7000_PMON_CTL_LOGIC_OP1 (1 << 21) +#define KX7000_PMON_CTL_EN (1 << 22) +#define KX7000_PMON_CTL_INVERT (1 << 23) +#define KX7000_PMON_CTL_THRESH_MASK 0xff000000 +#define KX7000_PMON_RAW_EVENT_MASK (KX7000_PMON_CTL_EV_SEL_MASK | \ + KX7000_PMON_CTL_UMASK_MASK | \ + KX7000_PMON_CTL_EDGE_DET | \ + KX7000_PMON_CTL_LOGIC_OP0 | \ + KX7000_PMON_CTL_LOGIC_OP1 | \ + KX7000_PMON_CTL_INVERT | \ + KX7000_PMON_CTL_THRESH_MASK) + +/* KX7000 LLC register*/ +#define KX7000_LLC_MSR_PMON_CTL0 0x1979 +#define KX7000_LLC_MSR_PMON_CTR0 0x1975 +#define KX7000_LLC_MSR_PMON_BLK_CTL 0x197e + +/* KX7000 MESH register*/ +#define KX7000_MESH_MSR_PMON_CTL0 0x1983 +#define KX7000_MESH_MSR_PMON_CTR0 0x197f +#define KX7000_MESH_MSR_PMON_BLK_CTL 0x1987 + +/* KX7000 HOMESTOP register*/ +#define KX7000_HOMESTOP_MSR_PMON_CTL0 0x196a +#define KX7000_HOMESTOP_MSR_PMON_CTR0 0x1966 +#define KX7000_HOMESTOP_MSR_PMON_BLK_CTL 0x196e +#define KX7000_HOMESTOP_MSR_PMON_FIXED_CTR 0x1970 +#define KX7000_HOMESTOP_MSR_PMON_FIXED_CTL 0x1971 + +/* KX7000 CCDie ZDI_PL register*/ +#define KX7000_CCD_ZDI_PL_MSR_PMON_CTL0 0x1960 +#define KX7000_CCD_ZDI_PL_MSR_PMON_CTR0 0x195c +#define KX7000_CCD_ZDI_PL_MSR_PMON_BLK_CTL 0x1964 + +/* KX7000 cIODie ZDI_PL register*/ +#define KX7000_IOD_ZDI_PL_MSR_PMON_CTL0 0x1894 +#define KX7000_IOD_ZDI_PL_MSR_PMON_CTR0 0x1890 +#define KX7000_IOD_ZDI_PL_MSR_PMON_BLK_CTL 0x1898 +#define KX7000_IOD_ZDI_PL_MSR_PMON_FIXED_CTR 0x189A +#define KX7000_IOD_ZDI_PL_MSR_PMON_FIXED_CTL 0x189B + +/* KX7000 MC register*/ +#define KX7000_MC_A0_CHy_PMON_FIXED_CTL 0xe30 +#define KX7000_MC_A0_CHy_PMON_FIXED_CTR 0xe08 +#define KX7000_MC_A0_CHy_PMON_CTR0 0xe00 +#define KX7000_MC_A0_CHy_PMON_CTL0 0xe20 +#define KX7000_MC_A0_CHy_PMON_BLK_CTL 0xe34 + +#define KX7000_MC_A1_CHy_PMON_FIXED_CTL 0xe70 +#define KX7000_MC_A1_CHy_PMON_FIXED_CTR 0xe48 +#define KX7000_MC_A1_CHy_PMON_CTR0 0xe40 +#define KX7000_MC_A1_CHy_PMON_CTL0 0xe60 +#define KX7000_MC_A1_CHy_PMON_BLK_CTL 0xe74 + +#define KX7000_MC_B0_CHy_PMON_FIXED_CTL 0xeb0 +#define KX7000_MC_B0_CHy_PMON_FIXED_CTR 0xe88 +#define KX7000_MC_B0_CHy_PMON_CTR0 0xe80 +#define KX7000_MC_B0_CHy_PMON_CTL0 0xea0 +#define KX7000_MC_B0_CHy_PMON_BLK_CTL 0xeb4 + +#define KX7000_MC_B1_CHy_PMON_FIXED_CTL 0xef0 +#define KX7000_MC_B1_CHy_PMON_FIXED_CTR 0xec8 +#define KX7000_MC_B1_CHy_PMON_CTR0 0xec0 +#define KX7000_MC_B1_CHy_PMON_CTL0 0xee0 +#define KX7000_MC_B1_CHy_PMON_BLK_CTL 0xef4 + +#define KX7000_ZDI_DL_MMIO_PMON_CTR0 0xf00 +#define KX7000_ZDI_DL_MMIO_PMON_CTL0 0xf28 +#define KX7000_ZDI_DL_MMIO_PMON_BLK_CTL 0xf44 +#define KX7000_IOD_ZDI_DL_MMIO_BASE_OFFSET 0x168 +#define KX7000_CCD_ZDI_DL_MMIO_BASE_OFFSET 0x170 +#define KX7000_ZDI_DL_MMIO_BASE_MASK 0x3fff +#define KX7000_ZDI_DL_MMIO_BASE_MASK 0x3fff +#define KX7000_ZDI_DL_MMIO_MEM0_MASK 0xfffff000 +#define KX7000_ZDI_DL_MMIO_SIZE 0x1000 -/* CHX event control */ -#define CHX_UNC_CTL_EV_SEL_MASK 0x000000ff -#define CHX_UNC_CTL_UMASK_MASK 0x0000ff00 -#define CHX_UNC_CTL_EDGE_DET (1 << 18) -#define CHX_UNC_CTL_EN (1 << 22) -#define CHX_UNC_CTL_INVERT (1 << 23) -#define CHX_UNC_CTL_CMASK_MASK 0xff000000 -#define CHX_UNC_FIXED_CTR_CTL_EN (1 << 0) +DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); +DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); +DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); +DEFINE_UNCORE_FORMAT_ATTR(logic_op0, logic_op0, "config:19"); +DEFINE_UNCORE_FORMAT_ATTR(logic_op1, logic_op1, "config:21"); +DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); +DEFINE_UNCORE_FORMAT_ATTR(cmask3, cmask, "config:24-26"); +DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31"); -#define CHX_UNC_RAW_EVENT_MASK (CHX_UNC_CTL_EV_SEL_MASK | \ - CHX_UNC_CTL_UMASK_MASK | \ - CHX_UNC_CTL_EDGE_DET | \ - CHX_UNC_CTL_INVERT | \ - CHX_UNC_CTL_CMASK_MASK) +static void get_hdw_config_msr(void *config) +{ + u64 *data = (u64 *)config; -/* CHX global control register */ -#define CHX_UNC_PERF_GLOBAL_CTL 0x391 -#define CHX_UNC_FIXED_CTR 0x394 -#define CHX_UNC_FIXED_CTR_CTRL 0x395 + rdmsrl(BJ_HDW_CONFIG_MSR, *data); +} -/* CHX uncore global control */ -#define CHX_UNC_GLOBAL_CTL_EN_PC_ALL ((1ULL << 4) - 1) -#define CHX_UNC_GLOBAL_CTL_EN_FC (1ULL << 32) +static void get_global_status_msr(void *status) +{ + u64 *data = (u64 *)status; -/* CHX uncore register */ -#define CHX_UNC_PERFEVTSEL0 0x3c0 -#define CHX_UNC_UNCORE_PMC0 0x3b0 + rdmsrl(BJ_GLOBAL_STATUS_MSR, *data); +} -DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); -DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); -DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); -DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); -DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31"); +/*topology number : get max packages/subnode/clusters number*/ +static void get_topology_number(void) +{ + int clusters; + int subnodes; + int dies; + int packages; + u64 data; + + rdmsrl(BJ_GLOBAL_STATUS_MSR, data); + + /* check packages number */ + packages = data & 0x1; + if (packages) + max_packages = 2; + else + max_packages = 1; + + /* only Yongfeng needs die/subnode/cluster info */ + if (boot_cpu_data.x86_model != ZHAOXIN_FAM7_KH40000) + return; + + /* check dies_per_socket */ + dies = (data >> 12) & 0x1; + if (dies) + dies_per_socket = 2; + else + dies_per_socket = 1; + + /* check subnodes_per_die */ + subnodes = (data >> 32) & 0x3; + if (subnodes == 0x3) + subnodes_per_die = 2; + else + subnodes_per_die = 1; + + /* check clusters_per_subnode */ + clusters = (data >> 6) & 0x3; + if (clusters == 0x3) + clusters_per_subnode = 2; + else + clusters_per_subnode = 1; + + max_subnodes = max_packages * dies_per_socket * subnodes_per_die; + max_clusters = clusters_per_subnode * max_subnodes; +} + +static int get_pcibus_limit(void) +{ + struct pci_dev *dev; + u32 val; + int i = 0; + + dev = pci_get_device(0x1D17, 0x31B1, NULL); + if (dev == NULL) + return -ENODEV; + + pci_read_config_dword(dev, 0x94, &val); + kh40000_pcibus_limit[i++] = (val & 0x1f) << 3 | 0x7; + kh40000_pcibus_limit[i++] = (val >> 8 & 0x1f) << 3 | 0x7; + if (dies_per_socket == 2) { + kh40000_pcibus_limit[i++] = (val >> 16 & 0x1f) << 3 | 0x7; + kh40000_pcibus_limit[i++] = (val >> 24 & 0x1f) << 3 | 0x7; + } + + if (max_packages == 2) { + pci_read_config_dword(dev, 0x9c, &val); + kh40000_pcibus_limit[i++] = (val & 0x1f) << 3 | 0x7; + kh40000_pcibus_limit[i++] = (val >> 8 & 0x1f) << 3 | 0x7; + if (dies_per_socket == 2) { + kh40000_pcibus_limit[i++] = (val >> 16 & 0x1f) << 3 | 0x7; + kh40000_pcibus_limit[i++] = (val >> 24 & 0x1f) << 3 | 0x7; + } + } + + return 0; +} + +static int uncore_pcibus_to_subnodeid(struct pci_bus *bus) +{ + int i; + + for (i = 0; i < KH40000_MAX_SUBNODE_NUMBER; i++) { + if (bus->number < kh40000_pcibus_limit[i]) + break; + } + + return i; +} + +DEFINE_PER_CPU(int, zx_package_id); +DEFINE_PER_CPU(int, zx_subnode_id); +DEFINE_PER_CPU(int, zx_cluster_id); + +static void get_topology_info(void) +{ + int cpu; + int cluster_id; + int socket_id; + int die_id; + int subnode_id; + + int die_info; + int subnode_info; + int cluster_info; + + u64 config; + + for_each_present_cpu(cpu) { + smp_call_function_single(cpu, get_global_status_msr, &config, 1); + socket_id = (int)((config >> 3) & 0x1); + per_cpu(zx_package_id, cpu) = socket_id; + + /* only kh40000 needs cluster and subnode info */ + if (boot_cpu_data.x86_model != ZHAOXIN_FAM7_KH40000) + continue; + + smp_call_function_single(cpu, get_hdw_config_msr, &config, 1); + + die_info = (int)((config >> 21) & 0x3); + die_id = socket_id * dies_per_socket + die_info; + + subnode_info = (int)((config >> 20) & 0x1); + subnode_id = die_id * subnodes_per_die + subnode_info; + per_cpu(zx_subnode_id, cpu) = subnode_id; + + cluster_info = (int)((config >> 18) & 0x3); + cluster_id = subnode_id * clusters_per_subnode + cluster_info; + per_cpu(zx_cluster_id, cpu) = cluster_id; + } +} + +static int zx_topology_cluster_id(int cpu) +{ + return per_cpu(zx_cluster_id, cpu); +} + +static int zx_topology_subnode_id(int cpu) +{ + return per_cpu(zx_subnode_id, cpu); +} + +static int zx_topology_package_id(int cpu) +{ + return per_cpu(zx_package_id, cpu); +} + +DEFINE_PER_CPU(cpumask_t, zx_cluster_core_bits); +DEFINE_PER_CPU(cpumask_t, zx_subnode_core_bits); + +static void zx_gen_core_map(void) +{ + int i, nr, cpu; + int cluster_id, subnode_id; -ssize_t zx_uncore_event_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) + for_each_present_cpu(cpu) { + cluster_id = zx_topology_cluster_id(cpu); + + for (i = 0; i < 4; i++) { + nr = (cluster_id << 2) + i; + cpumask_set_cpu(nr, &per_cpu(zx_cluster_core_bits, cpu)); + } + } + + for_each_present_cpu(cpu) { + subnode_id = zx_topology_subnode_id(cpu); + + for (i = 0; i < 8; i++) { + nr = (subnode_id << 3) + i; + cpumask_set_cpu(nr, &per_cpu(zx_subnode_core_bits, cpu)); + } + } +} + +static struct cpumask *topology_cluster_core_cpumask(int cpu) +{ + return &per_cpu(zx_cluster_core_bits, cpu); +} + +static struct cpumask *topology_subnode_core_cpumask(int cpu) { - struct uncore_event_desc *event = - container_of(attr, struct uncore_event_desc, attr); + return &per_cpu(zx_subnode_core_bits, cpu); +} + +static void uncore_free_pcibus_map(void) +{ + +} + +static int kh40000_pci2node_map_init(void) +{ + return 0; +} + +ssize_t zx_uncore_event_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct uncore_event_desc *event = container_of(attr, struct uncore_event_desc, attr); + return sprintf(buf, "%s", event->config); } -/*chx uncore support */ -static void chx_uncore_msr_disable_event(struct zhaoxin_uncore_box *box, struct perf_event *event) +static struct zhaoxin_uncore_box *uncore_pmu_to_box(struct zhaoxin_uncore_pmu *pmu, int cpu) { - wrmsrl(event->hw.config_base, 0); + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + if (!strcmp(pmu->type->name, "llc")) + return pmu->boxes[zx_topology_cluster_id(cpu)]; + else + return pmu->boxes[zx_topology_subnode_id(cpu)]; + } else { + return pmu->boxes[zx_topology_package_id(cpu)]; + } } static u64 uncore_msr_read_counter(struct zhaoxin_uncore_box *box, struct perf_event *event) { u64 count; + WARN_ON_ONCE(box->cpu != smp_processor_id()); rdmsrl(event->hw.event_base, count); - return count; } -static void chx_uncore_msr_disable_box(struct zhaoxin_uncore_box *box) +static void uncore_assign_hw_event(struct zhaoxin_uncore_box *box, struct perf_event *event, + int idx) +{ + struct hw_perf_event *hwc = &event->hw; + + hwc->idx = idx; + hwc->last_tag = ++box->tags[idx]; + + if (uncore_pmc_fixed(hwc->idx)) { + hwc->event_base = uncore_fixed_ctr(box); + hwc->config_base = uncore_fixed_ctl(box); + return; + } + + hwc->config_base = uncore_event_ctl(box, hwc->idx); + hwc->event_base = uncore_perf_ctr(box, hwc->idx); +} + +void uncore_perf_event_update(struct zhaoxin_uncore_box *box, struct perf_event *event) +{ + u64 prev_count, new_count, delta; + int shift; + + if (uncore_pmc_fixed(event->hw.idx)) + shift = 64 - uncore_fixed_ctr_bits(box); + else + shift = 64 - uncore_perf_ctr_bits(box); + + /* the hrtimer might modify the previous event value */ +again: + prev_count = local64_read(&event->hw.prev_count); + new_count = uncore_read_counter(box, event); + if (local64_xchg(&event->hw.prev_count, new_count) != prev_count) + goto again; + + delta = (new_count << shift) - (prev_count << shift); + delta >>= shift; + + local64_add(delta, &event->count); +} + +/*KX5000/KX6000 uncore ops start*/ +static void kx5000_uncore_msr_disable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + wrmsrl(event->hw.config_base, 0); +} + +static void kx5000_uncore_msr_disable_box(struct zhaoxin_uncore_box *box) { - wrmsrl(CHX_UNC_PERF_GLOBAL_CTL, 0); + wrmsrl(KX5000_UNC_PERF_GLOBAL_CTL, 0); } -static void chx_uncore_msr_enable_box(struct zhaoxin_uncore_box *box) +static void kx5000_uncore_msr_enable_box(struct zhaoxin_uncore_box *box) { - wrmsrl(CHX_UNC_PERF_GLOBAL_CTL, CHX_UNC_GLOBAL_CTL_EN_PC_ALL | CHX_UNC_GLOBAL_CTL_EN_FC); + wrmsrl(KX5000_UNC_PERF_GLOBAL_CTL, + KX5000_UNC_GLOBAL_CTL_EN_PC_ALL | KX5000_UNC_GLOBAL_CTL_EN_FC); } -static void chx_uncore_msr_enable_event(struct zhaoxin_uncore_box *box, struct perf_event *event) +static void kx5000_uncore_msr_enable_event(struct zhaoxin_uncore_box *box, struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; if (hwc->idx < UNCORE_PMC_IDX_FIXED) - wrmsrl(hwc->config_base, hwc->config | CHX_UNC_CTL_EN); + wrmsrl(hwc->config_base, hwc->config | KX5000_UNC_CTL_EN); else - wrmsrl(hwc->config_base, CHX_UNC_FIXED_CTR_CTL_EN); + wrmsrl(hwc->config_base, KX5000_UNC_FIXED_CTR_CTL_EN); } -static struct attribute *chx_uncore_formats_attr[] = { +static struct attribute *kx5000_uncore_formats_attr[] = { &format_attr_event.attr, &format_attr_umask.attr, &format_attr_edge.attr, &format_attr_inv.attr, - &format_attr_cmask8.attr, + &format_attr_cmask3.attr, NULL, }; -static struct attribute_group chx_uncore_format_group = { +static struct attribute_group kx5000_uncore_format_group = { .name = "format", - .attrs = chx_uncore_formats_attr, + .attrs = kx5000_uncore_formats_attr, }; -static struct uncore_event_desc chx_uncore_events[] = { +static struct uncore_event_desc kx5000_uncore_events[] = { { /* end: all zeroes */ }, }; -static struct zhaoxin_uncore_ops chx_uncore_msr_ops = { - .disable_box = chx_uncore_msr_disable_box, - .enable_box = chx_uncore_msr_enable_box, - .disable_event = chx_uncore_msr_disable_event, - .enable_event = chx_uncore_msr_enable_event, +static struct zhaoxin_uncore_ops kx5000_uncore_msr_ops = { + .disable_box = kx5000_uncore_msr_disable_box, + .enable_box = kx5000_uncore_msr_enable_box, + .disable_event = kx5000_uncore_msr_disable_event, + .enable_event = kx5000_uncore_msr_enable_event, .read_counter = uncore_msr_read_counter, }; -static struct zhaoxin_uncore_type chx_uncore_box = { +static struct zhaoxin_uncore_type kx5000_uncore_box = { .name = "", - .num_counters = 4, + .num_counters = 4, .num_boxes = 1, .perf_ctr_bits = 48, .fixed_ctr_bits = 48, - .event_ctl = CHX_UNC_PERFEVTSEL0, - .perf_ctr = CHX_UNC_UNCORE_PMC0, - .fixed_ctr = CHX_UNC_FIXED_CTR, - .fixed_ctl = CHX_UNC_FIXED_CTR_CTRL, - .event_mask = CHX_UNC_RAW_EVENT_MASK, - .event_descs = chx_uncore_events, - .ops = &chx_uncore_msr_ops, - .format_group = &chx_uncore_format_group, -}; - -static struct zhaoxin_uncore_type *chx_msr_uncores[] = { - &chx_uncore_box, + .event_ctl = KX5000_UNC_PERFEVTSEL0, + .perf_ctr = KX5000_UNC_UNCORE_PMC0, + .fixed_ctr = KX5000_UNC_FIXED_CTR, + .fixed_ctl = KX5000_UNC_FIXED_CTR_CTRL, + .event_mask = KX5000_UNC_RAW_EVENT_MASK, + .event_descs = kx5000_uncore_events, + .ops = &kx5000_uncore_msr_ops, + .format_group = &kx5000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type *kx5000_msr_uncores[] = { + &kx5000_uncore_box, + NULL, +}; +/*KX5000/KX6000 uncore ops end*/ + +/*KH40000 msr ops start*/ +static void kh40000_uncore_msr_disable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + wrmsrl(hwc->config_base, hwc->config); +} + +static void kh40000_uncore_msr_enable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + wrmsrl(hwc->config_base, hwc->config | KH40000_PMON_CTL_EN); +} + +static void kh40000_uncore_msr_disable_box(struct zhaoxin_uncore_box *box) +{ + u64 config; + unsigned int msr; + + msr = uncore_msr_box_ctl(box); + if (msr) { + rdmsrl(msr, config); + config |= KH40000_PMON_BOX_CTL_FRZ; + wrmsrl(msr, config); + } +} + +static void kh40000_uncore_msr_enable_box(struct zhaoxin_uncore_box *box) +{ + u64 config; + unsigned int msr; + + msr = uncore_msr_box_ctl(box); + if (msr) { + rdmsrl(msr, config); + config &= ~KH40000_PMON_BOX_CTL_FRZ; + wrmsrl(msr, config); + } +} + +static void kh40000_uncore_msr_init_box(struct zhaoxin_uncore_box *box) +{ + unsigned int msr = uncore_msr_box_ctl(box); + + if (msr) { + wrmsrl(msr, KH40000_PMON_BOX_CTL_INT); + wrmsrl(msr, 0); + } +} + +static struct attribute *kh40000_uncore_formats_attr[] = { + &format_attr_event.attr, + &format_attr_umask.attr, + &format_attr_edge.attr, + &format_attr_inv.attr, + &format_attr_thresh8.attr, NULL, }; -static struct zhaoxin_uncore_box *uncore_pmu_to_box(struct zhaoxin_uncore_pmu *pmu, int cpu) +static struct attribute_group kh40000_uncore_format_group = { + .name = "format", + .attrs = kh40000_uncore_formats_attr, +}; + +static struct uncore_event_desc kh40000_uncore_llc_box_events[] = { + { /* end: all zeroes */ }, +}; + +static struct uncore_event_desc kh40000_uncore_hif_box_events[] = { + { /* end: all zeroes */ }, +}; + +static struct uncore_event_desc kh40000_uncore_zzi_box_events[] = { + { /* end: all zeroes */ }, +}; + +static struct zhaoxin_uncore_ops kh40000_uncore_msr_ops = { + .init_box = kh40000_uncore_msr_init_box, + .disable_box = kh40000_uncore_msr_disable_box, + .enable_box = kh40000_uncore_msr_enable_box, + .disable_event = kh40000_uncore_msr_disable_event, + .enable_event = kh40000_uncore_msr_enable_event, + .read_counter = uncore_msr_read_counter, +}; + +static struct zhaoxin_uncore_type kh40000_uncore_llc_box = { + .name = "llc", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .event_ctl = KH40000_LLC_MSR_PMON_CTL0, + .perf_ctr = KH40000_LLC_MSR_PMON_CTR0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_LLC_MSR_PMON_BLK_CTL, + .event_descs = kh40000_uncore_llc_box_events, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kh40000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kh40000_uncore_hif_box = { + .name = "hif", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .event_ctl = KH40000_HIF_MSR_PMON_CTL0, + .perf_ctr = KH40000_HIF_MSR_PMON_CTR0, + .fixed_ctr = KH40000_HIF_MSR_PMON_FIXED_CTR, + .fixed_ctl = KH40000_HIF_MSR_PMON_FIXED_CTL, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_HIF_MSR_PMON_BLK_CTL, + .event_descs = kh40000_uncore_hif_box_events, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kh40000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kh40000_uncore_zzi_box = { + .name = "zzi", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .event_ctl = KH40000_ZZI_MSR_PMON_CTL0, + .perf_ctr = KH40000_ZZI_MSR_PMON_CTR0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_ZZI_MSR_PMON_BLK_CTL, + .event_descs = kh40000_uncore_zzi_box_events, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kh40000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type *kh40000_msr_uncores[] = { + &kh40000_uncore_llc_box, + &kh40000_uncore_hif_box, + &kh40000_uncore_zzi_box, + NULL, +}; +/*KH40000 msr ops end*/ + +/*KH40000 pci ops start*/ +static void kh40000_uncore_pci_disable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + struct pci_dev *pdev = box->pci_dev; + struct hw_perf_event *hwc = &event->hw; + + pci_write_config_dword(pdev, hwc->config_base, hwc->config); +} + +static void kh40000_uncore_pci_enable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + struct pci_dev *pdev = box->pci_dev; + struct hw_perf_event *hwc = &event->hw; + + pci_write_config_dword(pdev, hwc->config_base, hwc->config | KH40000_PMON_CTL_EN); +} + +static void kh40000_uncore_pci_disable_box(struct zhaoxin_uncore_box *box) +{ + struct pci_dev *pdev = box->pci_dev; + int box_ctl = uncore_pci_box_ctl(box); + u32 config = 0; + + if (!pci_read_config_dword(pdev, box_ctl, &config)) { + config |= KH40000_PMON_BOX_CTL_FRZ; + pci_write_config_dword(pdev, box_ctl, config); + } +} + +static void kh40000_uncore_pci_enable_box(struct zhaoxin_uncore_box *box) +{ + struct pci_dev *pdev = box->pci_dev; + int box_ctl = uncore_pci_box_ctl(box); + u32 config = 0; + + if (!pci_read_config_dword(pdev, box_ctl, &config)) { + config &= ~KH40000_PMON_BOX_CTL_FRZ; + pci_write_config_dword(pdev, box_ctl, config); + } +} + +static u64 kh40000_uncore_pci_read_counter(struct zhaoxin_uncore_box *box, struct perf_event *event) +{ + struct pci_dev *pdev = box->pci_dev; + struct hw_perf_event *hwc = &event->hw; + u64 count = 0; + + pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count + 1); + pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count); + + return count; +} + +static void kh40000_uncore_pci_init_box(struct zhaoxin_uncore_box *box) +{ + struct pci_dev *pdev = box->pci_dev; + int box_ctl = uncore_pci_box_ctl(box); + + pci_write_config_dword(pdev, box_ctl, KH40000_PMON_PCI_BOX_CTL_INT); +} + +static struct uncore_event_desc kh40000_uncore_imc_events[] = { + { /* end: all zeroes */ }, +}; + +static struct uncore_event_desc kh40000_uncore_pci_events[] = { + { /* end: all zeroes */ }, +}; + +static struct uncore_event_desc kh40000_uncore_zpi_dll_events[] = { + { /* end: all zeroes */ }, +}; + +static struct uncore_event_desc kh40000_uncore_zdi_dll_events[] = { + { /* end: all zeroes */ }, +}; + +static struct uncore_event_desc kh40000_uncore_pxptrf_events[] = { + { /* end: all zeroes */ }, +}; + +static struct zhaoxin_uncore_ops kh40000_uncore_pci_ops = { + .init_box = kh40000_uncore_pci_init_box, + .disable_box = kh40000_uncore_pci_disable_box, + .enable_box = kh40000_uncore_pci_enable_box, + .disable_event = kh40000_uncore_pci_disable_event, + .enable_event = kh40000_uncore_pci_enable_event, + .read_counter = kh40000_uncore_pci_read_counter +}; + +static struct zhaoxin_uncore_type kh40000_uncore_mc0 = { + .name = "mc0", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .fixed_ctr = KH40000_MC0_CHy_PMON_FIXED_CTR, + .fixed_ctl = KH40000_MC0_CHy_PMON_FIXED_CTL, + .event_descs = kh40000_uncore_imc_events, + .perf_ctr = KH40000_MC0_CHy_PMON_CTR0, + .event_ctl = KH40000_MC0_CHy_PMON_CTL0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_MC0_CHy_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kh40000_uncore_format_group +}; + +static struct zhaoxin_uncore_type kh40000_uncore_mc1 = { + .name = "mc1", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .fixed_ctr = KH40000_MC1_CHy_PMON_FIXED_CTR, + .fixed_ctl = KH40000_MC1_CHy_PMON_FIXED_CTL, + .event_descs = kh40000_uncore_imc_events, + .perf_ctr = KH40000_MC1_CHy_PMON_CTR0, + .event_ctl = KH40000_MC1_CHy_PMON_CTL0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_MC1_CHy_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kh40000_uncore_format_group +}; + +static struct zhaoxin_uncore_type kh40000_uncore_pci = { + .name = "pci", + .num_counters = 4, + .num_boxes = 10, + .perf_ctr_bits = 48, + .event_descs = kh40000_uncore_pci_events, + .perf_ctr = KH40000_PCI_PMON_CTR0, + .event_ctl = KH40000_PCI_PMON_CTL0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_PCI_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kh40000_uncore_format_group +}; + +static struct zhaoxin_uncore_type kh40000_uncore_zpi_dll = { + .name = "zpi_dll", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .event_descs = kh40000_uncore_zpi_dll_events, + .perf_ctr = KH40000_ZPI_DLL_PMON_CTR0, + .event_ctl = KH40000_ZPI_DLL_PMON_CTL0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_ZPI_DLL_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kh40000_uncore_format_group +}; + +static struct zhaoxin_uncore_type kh40000_uncore_zdi_dll = { + .name = "zdi_dll", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .event_descs = kh40000_uncore_zdi_dll_events, + .perf_ctr = KH40000_ZDI_DLL_PMON_CTR0, + .event_ctl = KH40000_ZDI_DLL_PMON_CTL0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_ZDI_DLL_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kh40000_uncore_format_group +}; + +static struct zhaoxin_uncore_type kh40000_uncore_pxptrf = { + .name = "pxptrf", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .event_descs = kh40000_uncore_pxptrf_events, + .perf_ctr = KH40000_PXPTRF_PMON_CTR0, + .event_ctl = KH40000_PXPTRF_PMON_CTL0, + .event_mask = KH40000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_PXPTRF_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kh40000_uncore_format_group +}; + +enum { + KH40000_PCI_UNCORE_MC0, + KH40000_PCI_UNCORE_MC1, + KH40000_PCI_UNCORE_PCI, + KH40000_PCI_UNCORE_ZPI_DLL, + KH40000_PCI_UNCORE_ZDI_DLL, + KH40000_PCI_UNCORE_PXPTRF, +}; + +static struct zhaoxin_uncore_type *kh40000_pci_uncores[] = { + [KH40000_PCI_UNCORE_MC0] = &kh40000_uncore_mc0, + [KH40000_PCI_UNCORE_MC1] = &kh40000_uncore_mc1, + [KH40000_PCI_UNCORE_PCI] = &kh40000_uncore_pci, + [KH40000_PCI_UNCORE_ZPI_DLL] = &kh40000_uncore_zpi_dll, + [KH40000_PCI_UNCORE_ZDI_DLL] = &kh40000_uncore_zdi_dll, + [KH40000_PCI_UNCORE_PXPTRF] = &kh40000_uncore_pxptrf, + NULL, +}; + +static const struct pci_device_id kh40000_uncore_pci_ids[] = { + { /* MC Channe0/1 */ + PCI_DEVICE(0x1D17, 0x31b2), + .driver_data = UNCORE_PCI_DEV_DATA(KH40000_PCI_UNCORE_MC0, 0), + }, + + { /* ZPI_DLL */ + PCI_DEVICE(0x1D17, 0x91c1), + .driver_data = UNCORE_PCI_DEV_DATA(KH40000_PCI_UNCORE_ZPI_DLL, 0), + }, + + { /* ZDI_DLL */ + PCI_DEVICE(0x1D17, 0x3b03), + .driver_data = UNCORE_PCI_DEV_DATA(KH40000_PCI_UNCORE_ZDI_DLL, 0), + }, + + { /* PXPTRF */ + PCI_DEVICE(0x1D17, 0x31B4), + .driver_data = UNCORE_PCI_DEV_DATA(KH40000_PCI_UNCORE_PXPTRF, 0), + }, + + { /* end: all zeroes */ } +}; + +static struct pci_driver kh40000_uncore_pci_driver = { + .name = "kh40000_uncore", + .id_table = kh40000_uncore_pci_ids, +}; +/*KH40000 pci ops end*/ + + +/*KX7000 msr ops start*/ +static unsigned int kx7000_uncore_msr_offsets[] = { + 0x0, 0x13, 0x27, 0x3b, 0x4f, 0x63, 0x77, 0x8b +}; + +static struct attribute *kx7000_uncore_formats_attr[] = { + &format_attr_event.attr, + &format_attr_umask.attr, + &format_attr_edge.attr, + &format_attr_logic_op0.attr, + &format_attr_logic_op1.attr, + &format_attr_inv.attr, + &format_attr_thresh8.attr, + NULL, +}; + +static struct attribute_group kx7000_uncore_format_group = { + .name = "format", + .attrs = kx7000_uncore_formats_attr, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_mesh_box = { + .name = "mesh", + .num_counters = 4, + .num_boxes = 8, + .perf_ctr_bits = 48, + .event_ctl = KX7000_MESH_MSR_PMON_CTL0, + .perf_ctr = KX7000_MESH_MSR_PMON_CTR0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_MESH_MSR_PMON_BLK_CTL, + .msr_offsets = kx7000_uncore_msr_offsets, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_llc_box = { + .name = "llc", + .num_counters = 4, + .num_boxes = 8, + .perf_ctr_bits = 48, + .event_ctl = KX7000_LLC_MSR_PMON_CTL0, + .perf_ctr = KX7000_LLC_MSR_PMON_CTR0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_LLC_MSR_PMON_BLK_CTL, + .msr_offsets = kx7000_uncore_msr_offsets, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_hif_box = { + .name = "hif", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .event_ctl = KH40000_HIF_MSR_PMON_CTL0, + .perf_ctr = KH40000_HIF_MSR_PMON_CTR0, + .fixed_ctr = KH40000_HIF_MSR_PMON_FIXED_CTR, + .fixed_ctl = KH40000_HIF_MSR_PMON_FIXED_CTL, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_HIF_MSR_PMON_BLK_CTL, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_homestop = { + .name = "homestop", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .event_ctl = KX7000_HOMESTOP_MSR_PMON_CTL0, + .perf_ctr = KX7000_HOMESTOP_MSR_PMON_CTR0, + .fixed_ctr = KX7000_HOMESTOP_MSR_PMON_FIXED_CTR, + .fixed_ctl = KX7000_HOMESTOP_MSR_PMON_FIXED_CTL, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_HOMESTOP_MSR_PMON_BLK_CTL, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_ccd_zdi_pl = { + .name = "ccd_zdi_pl", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .event_ctl = KX7000_CCD_ZDI_PL_MSR_PMON_CTL0, + .perf_ctr = KX7000_CCD_ZDI_PL_MSR_PMON_CTR0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_CCD_ZDI_PL_MSR_PMON_BLK_CTL, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_iod_zdi_pl = { + .name = "iod_zdi_pl", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .event_ctl = KX7000_IOD_ZDI_PL_MSR_PMON_CTL0, + .perf_ctr = KX7000_IOD_ZDI_PL_MSR_PMON_CTR0, + .fixed_ctr = KX7000_IOD_ZDI_PL_MSR_PMON_FIXED_CTR, + .fixed_ctl = KX7000_IOD_ZDI_PL_MSR_PMON_FIXED_CTL, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_IOD_ZDI_PL_MSR_PMON_BLK_CTL, + .ops = &kh40000_uncore_msr_ops, + .format_group = &kx7000_uncore_format_group, +}; + + +static struct zhaoxin_uncore_type *kx7000_msr_uncores[] = { + &kx7000_uncore_llc_box, + &kx7000_uncore_mesh_box, + &kx7000_uncore_hif_box, + &kx7000_uncore_homestop, + &kx7000_uncore_ccd_zdi_pl, + &kx7000_uncore_iod_zdi_pl, + NULL, +}; +/*KX7000 msr ops end*/ + +/*KX7000 pci ops start*/ +static unsigned int kx7000_mc_ctr_lh_offsets[] = { + 0xc, 0xe, 0x10, 0x12, 0x14 +}; + +static u64 kx7000_uncore_pci_mc_read_counter(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + struct pci_dev *pdev = box->pci_dev; + struct hw_perf_event *hwc = &event->hw; + u64 count = 0; + + pci_read_config_word(pdev, hwc->event_base, (u16 *)&count + 3); + pci_read_config_dword(pdev, hwc->event_base + kx7000_mc_ctr_lh_offsets[hwc->idx], + (u32 *)&count); + + return count; +} + +static struct zhaoxin_uncore_ops kx7000_uncore_pci_mc_ops = { + .init_box = kh40000_uncore_pci_init_box, + .disable_box = kh40000_uncore_pci_disable_box, + .enable_box = kh40000_uncore_pci_enable_box, + .disable_event = kh40000_uncore_pci_disable_event, + .enable_event = kh40000_uncore_pci_enable_event, + .read_counter = kx7000_uncore_pci_mc_read_counter +}; + +static struct zhaoxin_uncore_type kx7000_uncore_mc_a0 = { + .name = "mc_a0", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .fixed_ctr = KX7000_MC_A0_CHy_PMON_FIXED_CTR, + .fixed_ctl = KX7000_MC_A0_CHy_PMON_FIXED_CTL, + .perf_ctr = KX7000_MC_A0_CHy_PMON_CTR0, + .event_ctl = KX7000_MC_A0_CHy_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_MC_A0_CHy_PMON_BLK_CTL, + .ops = &kx7000_uncore_pci_mc_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_mc_a1 = { + .name = "mc_a1", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .fixed_ctr = KX7000_MC_A1_CHy_PMON_FIXED_CTR, + .fixed_ctl = KX7000_MC_A1_CHy_PMON_FIXED_CTL, + .perf_ctr = KX7000_MC_A1_CHy_PMON_CTR0, + .event_ctl = KX7000_MC_A1_CHy_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_MC_A1_CHy_PMON_BLK_CTL, + .ops = &kx7000_uncore_pci_mc_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_mc_b0 = { + .name = "mc_b0", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .fixed_ctr = KX7000_MC_B0_CHy_PMON_FIXED_CTR, + .fixed_ctl = KX7000_MC_B0_CHy_PMON_FIXED_CTL, + .perf_ctr = KX7000_MC_B0_CHy_PMON_CTR0, + .event_ctl = KX7000_MC_B0_CHy_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_MC_B0_CHy_PMON_BLK_CTL, + .ops = &kx7000_uncore_pci_mc_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_mc_b1 = { + .name = "mc_b1", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .fixed_ctr = KX7000_MC_B1_CHy_PMON_FIXED_CTR, + .fixed_ctl = KX7000_MC_B1_CHy_PMON_FIXED_CTL, + .perf_ctr = KX7000_MC_B1_CHy_PMON_CTR0, + .event_ctl = KX7000_MC_B1_CHy_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_MC_B1_CHy_PMON_BLK_CTL, + .ops = &kx7000_uncore_pci_mc_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_pci = { + .name = "pci", + .num_counters = 4, + .num_boxes = 17, + .perf_ctr_bits = 48, + .perf_ctr = KH40000_PCI_PMON_CTR0, + .event_ctl = KH40000_PCI_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_PCI_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_pxptrf = { + .name = "pxptrf", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .event_descs = kh40000_uncore_pxptrf_events, + .perf_ctr = KH40000_PXPTRF_PMON_CTR0, + .event_ctl = KH40000_PXPTRF_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KH40000_PXPTRF_PMON_BLK_CTL, + .ops = &kh40000_uncore_pci_ops, + .format_group = &kx7000_uncore_format_group, +}; + +enum { + KX7000_PCI_UNCORE_MC_A0, + KX7000_PCI_UNCORE_MC_A1, + KX7000_PCI_UNCORE_MC_B0, + KX7000_PCI_UNCORE_MC_B1, + KX7000_PCI_UNCORE_PCI, + KX7000_PCI_UNCORE_PXPTRF, +}; + +static struct zhaoxin_uncore_type *kx7000_pci_uncores[] = { + [KX7000_PCI_UNCORE_MC_A0] = &kx7000_uncore_mc_a0, + [KX7000_PCI_UNCORE_MC_A1] = &kx7000_uncore_mc_a1, + [KX7000_PCI_UNCORE_MC_B0] = &kx7000_uncore_mc_b0, + [KX7000_PCI_UNCORE_MC_B1] = &kx7000_uncore_mc_b1, + [KX7000_PCI_UNCORE_PCI] = &kx7000_uncore_pci, + [KX7000_PCI_UNCORE_PXPTRF] = &kx7000_uncore_pxptrf, + NULL, +}; + +static const struct pci_device_id kx7000_uncore_pci_ids[] = { + { /* MC Channe A0/A1/B0/B1 */ + PCI_DEVICE(0x1D17, 0x31B2), + .driver_data = UNCORE_PCI_DEV_DATA(KX7000_PCI_UNCORE_MC_A0, 0), + }, + + { /* PXPTRF */ + PCI_DEVICE(0x1D17, 0x31B4), + .driver_data = UNCORE_PCI_DEV_DATA(KX7000_PCI_UNCORE_PXPTRF, 0), + }, + + { /* end: all zeroes */ } +}; + + +static struct pci_driver kx7000_uncore_pci_driver = { + .name = "kx7000_uncore", + .id_table = kx7000_uncore_pci_ids, +}; +/*KX7000 pci ops end*/ + +/*KX7000 mmio ops start*/ +static void kx7000_uncore_mmio_init_box(struct zhaoxin_uncore_box *box) +{ + struct pci_dev *pdev = NULL; + unsigned int box_ctl = uncore_mmio_box_ctl(box); + resource_size_t addr; + u32 pci_dword; + int mmio_base_offset; + + pdev = pci_get_device(0x1d17, 0x31b1, pdev); + if (!pdev) + return; + + if (!strcmp(box->pmu->name, "iod_zdi_dl")) + mmio_base_offset = KX7000_IOD_ZDI_DL_MMIO_BASE_OFFSET; + else + mmio_base_offset = KX7000_CCD_ZDI_DL_MMIO_BASE_OFFSET; + + pci_read_config_dword(pdev, mmio_base_offset, &pci_dword); + addr = (u64)(pci_dword & KX7000_ZDI_DL_MMIO_BASE_MASK) << 32; + + pci_read_config_dword(pdev, mmio_base_offset + 4, &pci_dword); + addr |= pci_dword & KX7000_ZDI_DL_MMIO_MEM0_MASK; + + box->io_addr = ioremap(addr, KX7000_ZDI_DL_MMIO_SIZE); + if (!box->io_addr) + return; + + writel(KH40000_PMON_PCI_BOX_CTL_INT, box->io_addr + box_ctl); +} + +static void kx7000_uncore_mmio_disable_box(struct zhaoxin_uncore_box *box) +{ + u32 config; + unsigned int box_ctl = uncore_mmio_box_ctl(box); + + if (!box->io_addr) + return; + + config = readl(box->io_addr + box_ctl); + config |= KH40000_PMON_BOX_CTL_FRZ; + writel(config, box->io_addr + box_ctl); +} + +static void kx7000_uncore_mmio_enable_box(struct zhaoxin_uncore_box *box) { - unsigned int package_id = topology_logical_package_id(cpu); + u32 config; + unsigned int box_ctl = uncore_mmio_box_ctl(box); - /* - * The unsigned check also catches the '-1' return value for non - * existent mappings in the topology map. - */ - return package_id < max_packages ? pmu->boxes[package_id] : NULL; + if (!box->io_addr) + return; + + config = readl(box->io_addr + box_ctl); + config &= ~KH40000_PMON_BOX_CTL_FRZ; + writel(config, box->io_addr + box_ctl); } -static void uncore_assign_hw_event(struct zhaoxin_uncore_box *box, - struct perf_event *event, int idx) +static void kx7000_uncore_mmio_enable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; - hwc->idx = idx; - hwc->last_tag = ++box->tags[idx]; + if (!box->io_addr) + return; - if (uncore_pmc_fixed(hwc->idx)) { - hwc->event_base = uncore_fixed_ctr(box); - hwc->config_base = uncore_fixed_ctl(box); + writel(hwc->config | KH40000_PMON_CTL_EN, box->io_addr + hwc->config_base); +} + +static void kx7000_uncore_mmio_disable_event(struct zhaoxin_uncore_box *box, + struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!box->io_addr) return; - } - hwc->config_base = uncore_event_ctl(box, hwc->idx); - hwc->event_base = uncore_perf_ctr(box, hwc->idx); + writel(hwc->config, box->io_addr + hwc->config_base); } -void uncore_perf_event_update(struct zhaoxin_uncore_box *box, struct perf_event *event) +static void uncore_mmio_exit_box(struct zhaoxin_uncore_box *box) { - u64 prev_count, new_count, delta; - int shift; + if (box->io_addr) + iounmap(box->io_addr); +} - if (uncore_pmc_fixed(event->hw.idx)) - shift = 64 - uncore_fixed_ctr_bits(box); - else - shift = 64 - uncore_perf_ctr_bits(box); +static u64 uncore_mmio_read_counter(struct zhaoxin_uncore_box *box, struct perf_event *event) +{ + u64 count = 0; + u64 count_low = 0; + u64 count_high = 0; - /* the hrtimer might modify the previous event value */ -again: - prev_count = local64_read(&event->hw.prev_count); - new_count = uncore_read_counter(box, event); - if (local64_xchg(&event->hw.prev_count, new_count) != prev_count) - goto again; + if (!box->io_addr) + return 0; - delta = (new_count << shift) - (prev_count << shift); - delta >>= shift; + count_high = readl(box->io_addr + event->hw.event_base) & 0xffff; + count_low = readl(box->io_addr + event->hw.event_base + 4); + count = (count_high << 32) + count_low; - local64_add(delta, &event->count); + return count; } +static struct zhaoxin_uncore_ops kx7000_uncore_mmio_ops = { + .init_box = kx7000_uncore_mmio_init_box, + .exit_box = uncore_mmio_exit_box, + .disable_box = kx7000_uncore_mmio_disable_box, + .enable_box = kx7000_uncore_mmio_enable_box, + .disable_event = kx7000_uncore_mmio_disable_event, + .enable_event = kx7000_uncore_mmio_enable_event, + .read_counter = uncore_mmio_read_counter, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_iod_zdi_dl = { + .name = "iod_zdi_dl", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .perf_ctr = KX7000_ZDI_DL_MMIO_PMON_CTR0, + .event_ctl = KX7000_ZDI_DL_MMIO_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_ZDI_DL_MMIO_PMON_BLK_CTL, + .ops = &kx7000_uncore_mmio_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type kx7000_uncore_ccd_zdi_dl = { + .name = "ccd_zdi_dl", + .num_counters = 4, + .num_boxes = 1, + .perf_ctr_bits = 48, + .fixed_ctr_bits = 48, + .perf_ctr = KX7000_ZDI_DL_MMIO_PMON_CTR0, + .event_ctl = KX7000_ZDI_DL_MMIO_PMON_CTL0, + .event_mask = KX7000_PMON_RAW_EVENT_MASK, + .box_ctl = KX7000_ZDI_DL_MMIO_PMON_BLK_CTL, + .ops = &kx7000_uncore_mmio_ops, + .format_group = &kx7000_uncore_format_group, +}; + +static struct zhaoxin_uncore_type *kx7000_mmio_uncores[] = { + &kx7000_uncore_iod_zdi_dl, + &kx7000_uncore_ccd_zdi_dl, + NULL, +}; + +/*KX7000 mmio ops end*/ + static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) { struct zhaoxin_uncore_box *box; @@ -226,8 +1422,7 @@ static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) static void uncore_pmu_start_hrtimer(struct zhaoxin_uncore_box *box) { - hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration), - HRTIMER_MODE_REL_PINNED); + hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration), HRTIMER_MODE_REL_PINNED); } static void uncore_pmu_cancel_hrtimer(struct zhaoxin_uncore_box *box) @@ -241,8 +1436,7 @@ static void uncore_pmu_init_hrtimer(struct zhaoxin_uncore_box *box) box->hrtimer.function = uncore_pmu_hrtimer; } -static struct zhaoxin_uncore_box *uncore_alloc_box(struct zhaoxin_uncore_type *type, - int node) +static struct zhaoxin_uncore_box *uncore_alloc_box(struct zhaoxin_uncore_type *type, int node) { int i, size, numshared = type->num_shared_regs; struct zhaoxin_uncore_box *box; @@ -259,6 +1453,8 @@ static struct zhaoxin_uncore_box *uncore_alloc_box(struct zhaoxin_uncore_type *t uncore_pmu_init_hrtimer(box); box->cpu = -1; box->package_id = -1; + box->cluster_id = -1; + box->subnode_id = -1; /* set default hrtimer timeout */ box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL; @@ -273,8 +1469,45 @@ static bool is_box_event(struct zhaoxin_uncore_box *box, struct perf_event *even return &box->pmu->pmu == event->pmu; } -static struct event_constraint * -uncore_get_event_constraint(struct zhaoxin_uncore_box *box, struct perf_event *event) +static int uncore_collect_events(struct zhaoxin_uncore_box *box, struct perf_event *leader, + bool dogrp) +{ + struct perf_event *event; + int n, max_count; + + max_count = box->pmu->type->num_counters; + if (box->pmu->type->fixed_ctl) + max_count++; + + if (box->n_events >= max_count) + return -EINVAL; + + n = box->n_events; + + if (is_box_event(box, leader)) { + box->event_list[n] = leader; + n++; + } + + if (!dogrp) + return n; + + for_each_sibling_event(event, leader) { + if (!is_box_event(box, event) || + event->state <= PERF_EVENT_STATE_OFF) + continue; + + if (n >= max_count) + return -EINVAL; + + box->event_list[n] = event; + n++; + } + return n; +} + +static struct event_constraint *uncore_get_event_constraint(struct zhaoxin_uncore_box *box, + struct perf_event *event) { struct zhaoxin_uncore_type *type = box->pmu->type; struct event_constraint *c; @@ -298,8 +1531,7 @@ uncore_get_event_constraint(struct zhaoxin_uncore_box *box, struct perf_event *e return &type->unconstrainted; } -static void uncore_put_event_constraint(struct zhaoxin_uncore_box *box, - struct perf_event *event) +static void uncore_put_event_constraint(struct zhaoxin_uncore_box *box, struct perf_event *event) { if (box->pmu->type->ops->put_constraint) box->pmu->type->ops->put_constraint(box, event); @@ -344,8 +1576,7 @@ static int uncore_assign_events(struct zhaoxin_uncore_box *box, int assign[], in } /* slow path */ if (i != n) - ret = perf_assign_events(box->event_constraint, n, - wmin, wmax, n, assign); + ret = perf_assign_events(box->event_constraint, n, wmin, wmax, n, assign); if (!assign || ret) { for (i = 0; i < n; i++) @@ -374,10 +1605,8 @@ static void uncore_pmu_event_start(struct perf_event *event, int flags) local64_set(&event->hw.prev_count, uncore_read_counter(box, event)); uncore_enable_event(box, event); - if (box->n_active == 1) { - uncore_enable_box(box); + if (box->n_active == 1) uncore_pmu_start_hrtimer(box); - } } static void uncore_pmu_event_stop(struct perf_event *event, int flags) @@ -392,10 +1621,8 @@ static void uncore_pmu_event_stop(struct perf_event *event, int flags) WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); hwc->state |= PERF_HES_STOPPED; - if (box->n_active == 0) { - uncore_disable_box(box); + if (box->n_active == 0) uncore_pmu_cancel_hrtimer(box); - } } if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { @@ -408,44 +1635,6 @@ static void uncore_pmu_event_stop(struct perf_event *event, int flags) } } -static int -uncore_collect_events(struct zhaoxin_uncore_box *box, struct perf_event *leader, - bool dogrp) -{ - struct perf_event *event; - int n, max_count; - - max_count = box->pmu->type->num_counters; - if (box->pmu->type->fixed_ctl) - max_count++; - - if (box->n_events >= max_count) - return -EINVAL; - - n = box->n_events; - - if (is_box_event(box, leader)) { - box->event_list[n] = leader; - n++; - } - - if (!dogrp) - return n; - - for_each_sibling_event(event, leader) { - if (!is_box_event(box, event) || - event->state <= PERF_EVENT_STATE_OFF) - continue; - - if (n >= max_count) - return -EINVAL; - - box->event_list[n] = event; - n++; - } - return n; -} - static int uncore_pmu_event_add(struct perf_event *event, int flags) { struct zhaoxin_uncore_box *box = uncore_event_to_box(event); @@ -461,6 +1650,7 @@ static int uncore_pmu_event_add(struct perf_event *event, int flags) return ret; hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + if (!(flags & PERF_EF_START)) hwc->state |= PERF_HES_ARCH; @@ -507,8 +1697,37 @@ static int uncore_pmu_event_add(struct perf_event *event, int flags) return 0; } -static int uncore_validate_group(struct zhaoxin_uncore_pmu *pmu, - struct perf_event *event) +static void uncore_pmu_event_del(struct perf_event *event, int flags) +{ + struct zhaoxin_uncore_box *box = uncore_event_to_box(event); + int i; + + uncore_pmu_event_stop(event, PERF_EF_UPDATE); + + for (i = 0; i < box->n_events; i++) { + if (event == box->event_list[i]) { + uncore_put_event_constraint(box, event); + + for (++i; i < box->n_events; i++) + box->event_list[i - 1] = box->event_list[i]; + + --box->n_events; + break; + } + } + + event->hw.idx = -1; + event->hw.last_tag = ~0ULL; +} + +static void uncore_pmu_event_read(struct perf_event *event) +{ + struct zhaoxin_uncore_box *box = uncore_event_to_box(event); + + uncore_perf_event_update(box, event); +} + +static int uncore_validate_group(struct zhaoxin_uncore_pmu *pmu, struct perf_event *event) { struct perf_event *leader = event->group_leader; struct zhaoxin_uncore_box *fake_box; @@ -542,36 +1761,6 @@ static int uncore_validate_group(struct zhaoxin_uncore_pmu *pmu, return ret; } -static void uncore_pmu_event_del(struct perf_event *event, int flags) -{ - struct zhaoxin_uncore_box *box = uncore_event_to_box(event); - int i; - - uncore_pmu_event_stop(event, PERF_EF_UPDATE); - - for (i = 0; i < box->n_events; i++) { - if (event == box->event_list[i]) { - uncore_put_event_constraint(box, event); - - for (++i; i < box->n_events; i++) - box->event_list[i - 1] = box->event_list[i]; - - --box->n_events; - break; - } - } - - event->hw.idx = -1; - event->hw.last_tag = ~0ULL; -} - -static void uncore_pmu_event_read(struct perf_event *event) -{ - struct zhaoxin_uncore_box *box = uncore_event_to_box(event); - - uncore_perf_event_update(box, event); -} - static int uncore_pmu_event_init(struct perf_event *event) { struct zhaoxin_uncore_pmu *pmu; @@ -603,7 +1792,7 @@ static int uncore_pmu_event_init(struct perf_event *event) event->cpu = box->cpu; event->pmu_private = box; - event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; + //event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; event->hw.idx = -1; event->hw.last_tag = ~0ULL; @@ -641,12 +1830,60 @@ static int uncore_pmu_event_init(struct perf_event *event) return ret; } -static ssize_t uncore_get_attr_cpumask(struct device *dev, struct device_attribute *attr, char *buf) +static void uncore_pmu_enable(struct pmu *pmu) +{ + struct zhaoxin_uncore_pmu *uncore_pmu; + struct zhaoxin_uncore_box *box; + + uncore_pmu = container_of(pmu, struct zhaoxin_uncore_pmu, pmu); + if (!uncore_pmu) + return; + + box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); + if (!box) + return; + + if (uncore_pmu->type->ops->enable_box) + uncore_pmu->type->ops->enable_box(box); +} + +static void uncore_pmu_disable(struct pmu *pmu) { - return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask); + struct zhaoxin_uncore_pmu *uncore_pmu; + struct zhaoxin_uncore_box *box; + + uncore_pmu = container_of(pmu, struct zhaoxin_uncore_pmu, pmu); + if (!uncore_pmu) + return; + + box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); + if (!box) + return; + + if (uncore_pmu->type->ops->disable_box) + uncore_pmu->type->ops->disable_box(box); } -static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL); +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + cpumask_t *active_mask; + struct pmu *pmu; + struct zhaoxin_uncore_pmu *uncore_pmu; + + pmu = dev_get_drvdata(dev); + uncore_pmu = container_of(pmu, struct zhaoxin_uncore_pmu, pmu); + + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + if (!strcmp(uncore_pmu->type->name, "llc")) + active_mask = &uncore_cpu_cluster_mask; + else + active_mask = &uncore_cpu_subnode_mask; + } else { + active_mask = &uncore_cpu_mask; + } + return cpumap_print_to_pagebuf(true, buf, active_mask); +} +static DEVICE_ATTR_RO(cpumask); static struct attribute *uncore_pmu_attrs[] = { &dev_attr_cpumask.attr, @@ -657,6 +1894,46 @@ static const struct attribute_group uncore_pmu_attr_group = { .attrs = uncore_pmu_attrs, }; +static int uncore_pmu_register(struct zhaoxin_uncore_pmu *pmu) +{ + int ret; + + if (!pmu->type->pmu) { + pmu->pmu = (struct pmu) { + .attr_groups = pmu->type->attr_groups, + .task_ctx_nr = perf_invalid_context, + .pmu_enable = uncore_pmu_enable, + .pmu_disable = uncore_pmu_disable, + .event_init = uncore_pmu_event_init, + .add = uncore_pmu_event_add, + .del = uncore_pmu_event_del, + .start = uncore_pmu_event_start, + .stop = uncore_pmu_event_stop, + .read = uncore_pmu_event_read, + .module = THIS_MODULE, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, + }; + } else { + pmu->pmu = *pmu->type->pmu; + pmu->pmu.attr_groups = pmu->type->attr_groups; + } + + if (pmu->type->num_boxes == 1) { + if (strlen(pmu->type->name) > 0) + sprintf(pmu->name, "uncore_%s", pmu->type->name); + else + sprintf(pmu->name, "uncore"); + } else { + sprintf(pmu->name, "uncore_%s_%d", pmu->type->name, + pmu->pmu_idx); + } + + ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); + if (!ret) + pmu->registered = true; + return ret; +} + static void uncore_pmu_unregister(struct zhaoxin_uncore_pmu *pmu) { if (!pmu->registered) @@ -667,10 +1944,19 @@ static void uncore_pmu_unregister(struct zhaoxin_uncore_pmu *pmu) static void uncore_free_boxes(struct zhaoxin_uncore_pmu *pmu) { - int package; + int i, max; + + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + if (!strcmp(pmu->type->name, "llc")) + max = max_clusters; + else + max = max_subnodes; + } else { + max = max_packages; + } - for (package = 0; package < max_packages; package++) - kfree(pmu->boxes[package]); + for (i = 0; i < max; i++) + kfree(pmu->boxes[i]); kfree(pmu->boxes); } @@ -707,7 +1993,14 @@ static int __init uncore_type_init(struct zhaoxin_uncore_type *type, bool setid) if (!pmus) return -ENOMEM; - size = max_packages*sizeof(struct zhaoxin_uncore_box *); + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + if (!strcmp(type->name, "llc")) + size = max_clusters * sizeof(struct zhaoxin_uncore_box *); + else + size = max_subnodes * sizeof(struct zhaoxin_uncore_box *); + } else { + size = max_packages * sizeof(struct zhaoxin_uncore_box *); + } for (i = 0; i < type->num_boxes; i++) { pmus[i].func_id = setid ? i : -1; @@ -756,8 +2049,7 @@ static int __init uncore_type_init(struct zhaoxin_uncore_type *type, bool setid) return -ENOMEM; } -static int __init -uncore_types_init(struct zhaoxin_uncore_type **types, bool setid) +static int __init uncore_types_init(struct zhaoxin_uncore_type **types, bool setid) { int ret; @@ -769,25 +2061,203 @@ uncore_types_init(struct zhaoxin_uncore_type **types, bool setid) return 0; } -static void uncore_change_type_ctx(struct zhaoxin_uncore_type *type, int old_cpu, - int new_cpu) +/* + * add a pci uncore device + */ +static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct zhaoxin_uncore_type *type; + struct zhaoxin_uncore_pmu *pmu; + struct zhaoxin_uncore_box *box; + struct zhaoxin_uncore_box **boxes; + char mc_dev[10]; + int loop = 1; + int i, j = 0; + int subnode_id = 0; + int ret = 0; + + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) + subnode_id = uncore_pcibus_to_subnodeid(pdev->bus); + + type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; + + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + strscpy(mc_dev, "mc0", sizeof("mc0")); + if (!strcmp(type->name, mc_dev)) + loop = 2; + } else if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KX7000) { + strscpy(mc_dev, "mc_a0", sizeof("mc_a0")); + if (!strcmp(type->name, mc_dev)) + loop = 4; + } + + boxes = kcalloc(loop, sizeof(struct zhaoxin_uncore_box *), GFP_KERNEL); + if (!boxes) + return -ENOMEM; + + for (i = 0; i < loop; i++) { + type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data) + j]; + + if (!type) + continue; + /* + * for performance monitoring unit with multiple boxes, + * each box has a different function id. + */ + pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; + + if (WARN_ON_ONCE(pmu->boxes[subnode_id] != NULL)) + return -EINVAL; + + box = uncore_alloc_box(type, NUMA_NO_NODE); + if (!box) + return -ENOMEM; + + if (pmu->func_id < 0) + pmu->func_id = pdev->devfn; + else + WARN_ON_ONCE(pmu->func_id != pdev->devfn); + + atomic_inc(&box->refcnt); + box->subnode_id = subnode_id; + box->pci_dev = pdev; + box->pmu = pmu; + uncore_box_init(box); + boxes[i] = box; + + pci_set_drvdata(pdev, boxes); + pmu->boxes[subnode_id] = box; + if (atomic_inc_return(&pmu->activeboxes) > 1) { + if (!strcmp(type->name, mc_dev)) + goto next_loop; + else + return 0; + } + /* First active box registers the pmu */ + ret = uncore_pmu_register(pmu); + if (ret) { + pci_set_drvdata(pdev, NULL); + pmu->boxes[subnode_id] = NULL; + uncore_box_exit(box); + kfree(box); + } +next_loop: + j++; + } + + return ret; +} + +static void uncore_pci_remove(struct pci_dev *pdev) +{ + struct zhaoxin_uncore_box **boxes; + struct zhaoxin_uncore_box *box; + struct zhaoxin_uncore_pmu *pmu; + int subnode_id = 0; + int i = 0; + int loop = 1; + + boxes = pci_get_drvdata(pdev); + + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + if (!strcmp(boxes[0]->pmu->type->name, "mc0")) + loop = 2; + else + loop = 1; + } else if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KX7000) { + if (!strcmp(boxes[0]->pmu->type->name, "mc_a0")) + loop = 4; + else + loop = 1; + } + + for (i = 0; i < loop; i++) { + box = boxes[i]; + pmu = box->pmu; + if (WARN_ON_ONCE(subnode_id != box->subnode_id)) + return; + + pci_set_drvdata(pdev, NULL); + pmu->boxes[subnode_id] = NULL; + if (atomic_dec_return(&pmu->activeboxes) == 0) + uncore_pmu_unregister(pmu); + + uncore_box_exit(box); + kfree(box); + } + + kfree(boxes); +} + +static int __init uncore_pci_init(void) +{ + int ret; + + ret = uncore_types_init(uncore_pci_uncores, false); + if (ret) + goto errtype; + + uncore_pci_driver->probe = uncore_pci_probe; + uncore_pci_driver->remove = uncore_pci_remove; + + ret = pci_register_driver(uncore_pci_driver); + if (ret) + goto errtype; + + pcidrv_registered = true; + return 0; + +errtype: + uncore_types_exit(uncore_pci_uncores); + uncore_free_pcibus_map(); + uncore_pci_uncores = empty_uncore; + return ret; +} + +static void uncore_pci_exit(void) +{ + if (pcidrv_registered) { + pcidrv_registered = false; + pci_unregister_driver(uncore_pci_driver); + uncore_types_exit(uncore_pci_uncores); + uncore_free_pcibus_map(); + } +} + +static void uncore_change_type_ctx(struct zhaoxin_uncore_type *type, int old_cpu, int new_cpu) { struct zhaoxin_uncore_pmu *pmu = type->pmus; struct zhaoxin_uncore_box *box; - int i, package; + int i, package_id, cluster_id = 0, subnode_id = 0; + + package_id = zx_topology_package_id(old_cpu < 0 ? new_cpu : old_cpu); + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + cluster_id = zx_topology_cluster_id(old_cpu < 0 ? new_cpu : old_cpu); + subnode_id = zx_topology_subnode_id(old_cpu < 0 ? new_cpu : old_cpu); + } - package = topology_logical_package_id(old_cpu < 0 ? new_cpu : old_cpu); for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[package]; - if (!box) - continue; + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + if (!strcmp(type->name, "llc")) { + box = pmu->boxes[cluster_id]; + if (!box) + continue; + } else { + box = pmu->boxes[subnode_id]; + if (!box) + continue; + } + } else { + box = pmu->boxes[package_id]; + if (!box) + continue; + } if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); box->cpu = new_cpu; continue; } - WARN_ON_ONCE(box->cpu != old_cpu); box->cpu = -1; if (new_cpu < 0) @@ -799,8 +2269,7 @@ static void uncore_change_type_ctx(struct zhaoxin_uncore_type *type, int old_cpu } } -static void uncore_change_context(struct zhaoxin_uncore_type **uncores, - int old_cpu, int new_cpu) +static void uncore_change_context(struct zhaoxin_uncore_type **uncores, int old_cpu, int new_cpu) { for (; *uncores; uncores++) uncore_change_type_ctx(*uncores, old_cpu, new_cpu); @@ -824,13 +2293,36 @@ static void uncore_box_unref(struct zhaoxin_uncore_type **types, int id) } } -static int uncore_event_cpu_offline(unsigned int cpu) +struct zhaoxin_uncore_type *uncore_msr_cluster_uncores[] = { + &kh40000_uncore_llc_box, + NULL, +}; + +struct zhaoxin_uncore_type *uncore_msr_subnode_uncores[] = { + &kh40000_uncore_hif_box, + &kh40000_uncore_zzi_box, + NULL, +}; + +struct zhaoxin_uncore_type *uncore_pci_subnode_uncores[] = { + &kh40000_uncore_mc0, + &kh40000_uncore_mc1, + &kh40000_uncore_pci, + &kh40000_uncore_zpi_dll, + &kh40000_uncore_zdi_dll, + &kh40000_uncore_pxptrf, + NULL, +}; + +static void kx5000_event_cpu_offline(int cpu) { int package, target; /* Check if exiting cpu is used for collecting uncore events */ + if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) - goto unref; + goto unref_cpu_mask; + /* Find a new cpu to collect uncore events */ target = cpumask_any_but(topology_core_cpumask(cpu), cpu); @@ -841,16 +2333,107 @@ static int uncore_event_cpu_offline(unsigned int cpu) target = -1; uncore_change_context(uncore_msr_uncores, cpu, target); + uncore_change_context(uncore_mmio_uncores, cpu, target); + uncore_change_context(uncore_pci_uncores, cpu, target); -unref: - /* Clear the references */ - package = topology_logical_package_id(cpu); +unref_cpu_mask: + /*clear the references*/ + package = zx_topology_package_id(cpu); uncore_box_unref(uncore_msr_uncores, package); + uncore_box_unref(uncore_mmio_uncores, package); +} + +static void kh40000_event_cpu_offline(int cpu) +{ + int cluster_target, subnode_target; + int cluster_id, subnode_id; + + cluster_id = zx_topology_cluster_id(cpu); + subnode_id = zx_topology_subnode_id(cpu); + + /* Check if exiting cpu is used for collecting uncore events */ + + if (cpumask_test_and_clear_cpu(cpu, &uncore_cpu_cluster_mask)) { + cluster_target = cpumask_any_but(topology_cluster_core_cpumask(cpu), cpu); + if (cluster_target < nr_cpu_ids) + cpumask_set_cpu(cluster_target, &uncore_cpu_cluster_mask); + else + cluster_target = -1; + uncore_change_context(uncore_msr_cluster_uncores, cpu, cluster_target); + } else { + uncore_box_unref(uncore_msr_cluster_uncores, cluster_id); + } + + if (cpumask_test_and_clear_cpu(cpu, &uncore_cpu_subnode_mask)) { + subnode_target = cpumask_any_but(topology_subnode_core_cpumask(cpu), cpu); + if (subnode_target < nr_cpu_ids) + cpumask_set_cpu(subnode_target, &uncore_cpu_subnode_mask); + else + subnode_target = -1; + uncore_change_context(uncore_msr_subnode_uncores, cpu, subnode_target); + uncore_change_context(uncore_pci_subnode_uncores, cpu, subnode_target); + } else { + uncore_box_unref(uncore_msr_subnode_uncores, subnode_id); + } +} + +static int uncore_event_cpu_offline(unsigned int cpu) +{ + unsigned int x86_model; + + x86_model = boot_cpu_data.x86_model; + + if (x86_model == ZHAOXIN_FAM7_KH40000) + kh40000_event_cpu_offline(cpu); + else + kx5000_event_cpu_offline(cpu); + + return 0; +} + +static int kx5000_allocate_boxes(struct zhaoxin_uncore_type **types, unsigned int id, + unsigned int cpu) +{ + struct zhaoxin_uncore_box *box, *tmp; + struct zhaoxin_uncore_type *type; + struct zhaoxin_uncore_pmu *pmu; + LIST_HEAD(allocated); + int i; + + /* Try to allocate all required boxes */ + for (; *types; types++) { + type = *types; + pmu = type->pmus; + + for (i = 0; i < type->num_boxes; i++, pmu++) { + if (pmu->boxes[id]) + continue; + box = uncore_alloc_box(type, cpu_to_node(cpu)); + if (!box) + goto cleanup; + box->pmu = pmu; + box->package_id = id; + list_add(&box->active_list, &allocated); + } + } + + /* Install them in the pmus */ + list_for_each_entry_safe(box, tmp, &allocated, active_list) { + list_del_init(&box->active_list); + box->pmu->boxes[id] = box; + } return 0; + +cleanup: + list_for_each_entry_safe(box, tmp, &allocated, active_list) { + list_del_init(&box->active_list); + kfree(box); + } + return -ENOMEM; } -static int allocate_boxes(struct zhaoxin_uncore_type **types, - unsigned int package, unsigned int cpu) +static int kh40000_allocate_boxes(struct zhaoxin_uncore_type **types, unsigned int id, + unsigned int cpu) { struct zhaoxin_uncore_box *box, *tmp; struct zhaoxin_uncore_type *type; @@ -862,21 +2445,25 @@ static int allocate_boxes(struct zhaoxin_uncore_type **types, for (; *types; types++) { type = *types; pmu = type->pmus; + for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[package]) + if (pmu->boxes[id]) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); if (!box) goto cleanup; box->pmu = pmu; - box->package_id = package; + if (!strcmp(type->name, "llc")) + box->cluster_id = id; + else + box->subnode_id = id; list_add(&box->active_list, &allocated); } } /* Install them in the pmus */ list_for_each_entry_safe(box, tmp, &allocated, active_list) { list_del_init(&box->active_list); - box->pmu->boxes[package] = box; + box->pmu->boxes[id] = box; } return 0; @@ -888,15 +2475,22 @@ static int allocate_boxes(struct zhaoxin_uncore_type **types, return -ENOMEM; } -static int uncore_box_ref(struct zhaoxin_uncore_type **types, - int id, unsigned int cpu) +static int uncore_box_ref(struct zhaoxin_uncore_type **types, int id, unsigned int cpu) { struct zhaoxin_uncore_type *type; struct zhaoxin_uncore_pmu *pmu; struct zhaoxin_uncore_box *box; - int i, ret; + int i, ret = 0; + + int x86_model; + + x86_model = boot_cpu_data.x86_model; + + if (x86_model == ZHAOXIN_FAM7_KH40000) + ret = kh40000_allocate_boxes(types, id, cpu); + else + ret = kx5000_allocate_boxes(types, id, cpu); - ret = allocate_boxes(types, id, cpu); if (ret) return ret; @@ -912,14 +2506,14 @@ static int uncore_box_ref(struct zhaoxin_uncore_type **types, return 0; } -static int uncore_event_cpu_online(unsigned int cpu) +static int kx5000_event_cpu_online(unsigned int cpu) { - int package, target, msr_ret; + int package, target, msr_ret, mmio_ret; - package = topology_logical_package_id(cpu); + package = zx_topology_package_id(cpu); msr_ret = uncore_box_ref(uncore_msr_uncores, package, cpu); - - if (msr_ret) + mmio_ret = uncore_box_ref(uncore_mmio_uncores, package, cpu); + if (msr_ret && mmio_ret) return -ENOMEM; /* @@ -934,45 +2528,71 @@ static int uncore_event_cpu_online(unsigned int cpu) if (!msr_ret) uncore_change_context(uncore_msr_uncores, -1, cpu); + if (!mmio_ret) + uncore_change_context(uncore_mmio_uncores, -1, cpu); + uncore_change_context(uncore_pci_uncores, -1, cpu); return 0; } -static int uncore_pmu_register(struct zhaoxin_uncore_pmu *pmu) +static int kh40000_event_cpu_online(unsigned int cpu) { - int ret; + int cluster_target, subnode_target; + int cluster_id, subnode_id; + int cluster_ret, subnode_ret; - if (!pmu->type->pmu) { - pmu->pmu = (struct pmu) { - .attr_groups = pmu->type->attr_groups, - .task_ctx_nr = perf_invalid_context, - .event_init = uncore_pmu_event_init, - .add = uncore_pmu_event_add, - .del = uncore_pmu_event_del, - .start = uncore_pmu_event_start, - .stop = uncore_pmu_event_stop, - .read = uncore_pmu_event_read, - .module = THIS_MODULE, - }; - } else { - pmu->pmu = *pmu->type->pmu; - pmu->pmu.attr_groups = pmu->type->attr_groups; + cluster_id = zx_topology_cluster_id(cpu); + subnode_id = zx_topology_subnode_id(cpu); + + cluster_ret = uncore_box_ref(uncore_msr_cluster_uncores, cluster_id, cpu); + subnode_ret = uncore_box_ref(uncore_msr_subnode_uncores, subnode_id, cpu); + + if (cluster_ret && subnode_ret) + return -ENOMEM; + + /* + * Check if there is an online cpu in the cluster or subnode + * which collects uncore events already. + */ + + cluster_target = + cpumask_any_and(&uncore_cpu_cluster_mask, topology_cluster_core_cpumask(cpu)); + subnode_target = + cpumask_any_and(&uncore_cpu_subnode_mask, topology_subnode_core_cpumask(cpu)); + + if (cluster_target < nr_cpu_ids && subnode_target < nr_cpu_ids) + return 0; + + if (!cluster_ret && cluster_target >= nr_cpu_ids) { + cpumask_set_cpu(cpu, &uncore_cpu_cluster_mask); + uncore_change_context(uncore_msr_cluster_uncores, -1, cpu); } - if (pmu->type->num_boxes == 1) { - if (strlen(pmu->type->name) > 0) - sprintf(pmu->name, "uncore_%s", pmu->type->name); - else - sprintf(pmu->name, "uncore"); - } else { - sprintf(pmu->name, "uncore_%s_%d", pmu->type->name, - pmu->pmu_idx); + if (!subnode_ret && subnode_target >= nr_cpu_ids) { + cpumask_set_cpu(cpu, &uncore_cpu_subnode_mask); + uncore_change_context(uncore_msr_subnode_uncores, -1, cpu); + uncore_change_context(uncore_pci_subnode_uncores, -1, cpu); } - ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); - if (!ret) - pmu->registered = true; - return ret; + return 0; +} + +static int uncore_event_cpu_online(unsigned int cpu) +{ + int x86_model; + int kx5000_ret = 0, kh40000_ret = 0; + + x86_model = boot_cpu_data.x86_model; + + if (x86_model == ZHAOXIN_FAM7_KH40000) + kh40000_ret = kh40000_event_cpu_online(cpu); + else + kx5000_ret = kx5000_event_cpu_online(cpu); + + if (kx5000_ret || kh40000_ret) + return -ENOMEM; + + return 0; } static int __init type_pmu_register(struct zhaoxin_uncore_type *type) @@ -1018,62 +2638,152 @@ static int __init uncore_cpu_init(void) return ret; } +static int __init uncore_mmio_init(void) +{ + struct zhaoxin_uncore_type **types = uncore_mmio_uncores; + int ret; -#define CENTAUR_UNCORE_MODEL_MATCH(model, init) \ - { X86_VENDOR_CENTAUR, 7, model, X86_FEATURE_ANY, (unsigned long)&init } + ret = uncore_types_init(types, true); + if (ret) + goto err; -#define ZHAOXIN_UNCORE_MODEL_MATCH(model, init) \ - { X86_VENDOR_ZHAOXIN, 7, model, X86_FEATURE_ANY, (unsigned long)&init } + for (; *types; types++) { + ret = type_pmu_register(*types); + if (ret) + goto err; + } + return 0; +err: + uncore_types_exit(uncore_mmio_uncores); + uncore_mmio_uncores = empty_uncore; + return ret; +} struct zhaoxin_uncore_init_fun { void (*cpu_init)(void); + int (*pci_init)(void); + void (*mmio_init)(void); +}; + +void kx5000_uncore_cpu_init(void) +{ + uncore_msr_uncores = kx5000_msr_uncores; +} + +static const struct zhaoxin_uncore_init_fun kx5000_uncore_init __initconst = { + .cpu_init = kx5000_uncore_cpu_init, +}; + +void kh40000_uncore_cpu_init(void) +{ + uncore_msr_uncores = kh40000_msr_uncores; +} + +int kh40000_uncore_pci_init(void) +{ + int ret = kh40000_pci2node_map_init();/*pci_bus to package mapping, do nothing*/ + + if (ret) + return ret; + uncore_pci_uncores = kh40000_pci_uncores; + uncore_pci_driver = &kh40000_uncore_pci_driver; + return 0; +} + +static const struct zhaoxin_uncore_init_fun kh40000_uncore_init __initconst = { + .cpu_init = kh40000_uncore_cpu_init, + .pci_init = kh40000_uncore_pci_init, }; -void chx_uncore_cpu_init(void) +void kx7000_uncore_cpu_init(void) +{ + u64 val; + int cpu; + + uncore_msr_uncores = kx7000_msr_uncores; + + /* clear bit 16 of MSR 0x1877 so that HIF can work normally */ + for_each_present_cpu(cpu) { + rdmsrl_on_cpu(cpu, 0x1877, &val); + val = val & 0xfffffffffffeffffULL; + wrmsrl_on_cpu(cpu, 0x1877, val); + } +} + +int kx7000_uncore_pci_init(void) +{ + uncore_pci_uncores = kx7000_pci_uncores; + uncore_pci_driver = &kx7000_uncore_pci_driver; + + return 0; +} + +void kx7000_uncore_mmio_init(void) { - uncore_msr_uncores = chx_msr_uncores; + uncore_mmio_uncores = kx7000_mmio_uncores; } -static const struct zhaoxin_uncore_init_fun chx_uncore_init __initconst = { - .cpu_init = chx_uncore_cpu_init, +static const struct zhaoxin_uncore_init_fun kx7000_uncore_init __initconst = { + .cpu_init = kx7000_uncore_cpu_init, + .pci_init = kx7000_uncore_pci_init, + .mmio_init = kx7000_uncore_mmio_init, }; static const struct x86_cpu_id zhaoxin_uncore_match[] __initconst = { - CENTAUR_UNCORE_MODEL_MATCH(ZHAOXIN_FAM7_CHX001, chx_uncore_init), - CENTAUR_UNCORE_MODEL_MATCH(ZHAOXIN_FAM7_CHX002, chx_uncore_init), - ZHAOXIN_UNCORE_MODEL_MATCH(ZHAOXIN_FAM7_CHX001, chx_uncore_init), - ZHAOXIN_UNCORE_MODEL_MATCH(ZHAOXIN_FAM7_CHX002, chx_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(CENTAUR, 7, ZHAOXIN_FAM7_KX5000, &kx5000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(CENTAUR, 7, ZHAOXIN_FAM7_KX6000, &kx5000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(CENTAUR, 7, ZHAOXIN_FAM7_KH40000, &kh40000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(CENTAUR, 7, ZHAOXIN_FAM7_KX7000, &kx7000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(ZHAOXIN, 7, ZHAOXIN_FAM7_KX5000, &kx5000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(ZHAOXIN, 7, ZHAOXIN_FAM7_KX6000, &kx5000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(ZHAOXIN, 7, ZHAOXIN_FAM7_KH40000, &kh40000_uncore_init), + X86_MATCH_VENDOR_FAM_MODEL(ZHAOXIN, 7, ZHAOXIN_FAM7_KX7000, &kx7000_uncore_init), {}, }; - MODULE_DEVICE_TABLE(x86cpu, zhaoxin_uncore_match); static int __init zhaoxin_uncore_init(void) { - const struct x86_cpu_id *id; + const struct x86_cpu_id *id = NULL; struct zhaoxin_uncore_init_fun *uncore_init; - int cret = 0, ret; + int pret = 0, cret = 0, mret = 0, ret; id = x86_match_cpu(zhaoxin_uncore_match); - if (!id) return -ENODEV; if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) return -ENODEV; - max_packages = topology_max_packages(); + pr_info("welcome to uncore.\n"); + + get_topology_number(); + get_topology_info(); - pr_info("welcome to uncore!\n"); + if (boot_cpu_data.x86_model == ZHAOXIN_FAM7_KH40000) { + zx_gen_core_map(); + get_pcibus_limit(); + } uncore_init = (struct zhaoxin_uncore_init_fun *)id->driver_data; + if (uncore_init->pci_init) { + pret = uncore_init->pci_init(); + if (!pret) + pret = uncore_pci_init(); + } + if (uncore_init->cpu_init) { uncore_init->cpu_init(); cret = uncore_cpu_init(); } - if (cret) + if (uncore_init->mmio_init) { + uncore_init->mmio_init(); + mret = uncore_mmio_init(); + } + + if (cret && pret && mret) return -ENODEV; ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE, @@ -1082,11 +2792,16 @@ static int __init zhaoxin_uncore_init(void) uncore_event_cpu_offline); if (ret) goto err; - pr_info("uncore init success!\n"); + return 0; + err: uncore_types_exit(uncore_msr_uncores); + uncore_types_exit(uncore_mmio_uncores); + uncore_pci_exit(); + pr_info("uncore init fail!\n"); + return ret; } module_init(zhaoxin_uncore_init); @@ -1095,7 +2810,7 @@ static void __exit zhaoxin_uncore_exit(void) { cpuhp_remove_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE); uncore_types_exit(uncore_msr_uncores); + uncore_types_exit(uncore_mmio_uncores); + uncore_pci_exit(); } module_exit(zhaoxin_uncore_exit); - -MODULE_LICENSE("GPL"); diff --git a/arch/x86/events/zhaoxin/uncore.h b/arch/x86/events/zhaoxin/uncore.h index 3521123dc95d7e64c6bcb47fe63160f977b3d956..0878b6bdd470c525894e71288773dd21f848abc3 100644 --- a/arch/x86/events/zhaoxin/uncore.h +++ b/arch/x86/events/zhaoxin/uncore.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0-only +/* SPDX-License-Identifier: GPL-2.0+ */ #include #include #include @@ -7,13 +7,13 @@ #include #include "../perf_event.h" -#define ZHAOXIN_FAM7_CHX001 0x1b -#define ZHAOXIN_FAM7_CHX002 0x3b +#define ZHAOXIN_FAM7_KX5000 0x1b +#define ZHAOXIN_FAM7_KX6000 0x3b +#define ZHAOXIN_FAM7_KH40000 0x5b +#define ZHAOXIN_FAM7_KX7000 0x6b #define UNCORE_PMU_NAME_LEN 32 #define UNCORE_PMU_HRTIMER_INTERVAL (60LL * NSEC_PER_SEC) -#define UNCORE_CHX_IMC_HRTIMER_INTERVAL (5ULL * NSEC_PER_SEC) - #define UNCORE_FIXED_EVENT 0xff #define UNCORE_PMC_IDX_MAX_GENERIC 4 @@ -22,6 +22,10 @@ #define UNCORE_PMC_IDX_MAX (UNCORE_PMC_IDX_FIXED + 1) +#define UNCORE_PCI_DEV_DATA(type, idx) ((type << 8) | idx) +#define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff) +#define UNCORE_PCI_DEV_IDX(data) (data & 0xff) + struct zhaoxin_uncore_ops; struct zhaoxin_uncore_pmu; struct zhaoxin_uncore_box; @@ -33,18 +37,21 @@ struct zhaoxin_uncore_type { int num_boxes; int perf_ctr_bits; int fixed_ctr_bits; - unsigned perf_ctr; - unsigned event_ctl; - unsigned event_mask; - unsigned event_mask_ext; - unsigned fixed_ctr; - unsigned fixed_ctl; - unsigned box_ctl; - unsigned msr_offset; - unsigned num_shared_regs:8; - unsigned single_fixed:1; - unsigned pair_ctr_ctl:1; - unsigned *msr_offsets; + unsigned int perf_ctr; + unsigned int event_ctl; + unsigned int event_mask; + unsigned int event_mask_ext; + unsigned int fixed_ctr; + unsigned int fixed_ctl; + unsigned int box_ctl; + union { + unsigned int msr_offset; + unsigned int mmio_offset; + }; + unsigned int num_shared_regs:8; + unsigned int single_fixed:1; + unsigned int pair_ctr_ctl:1; + unsigned int *msr_offsets; struct event_constraint unconstrainted; struct event_constraint *constraints; struct zhaoxin_uncore_pmu *pmus; @@ -59,28 +66,28 @@ struct zhaoxin_uncore_type { #define events_group attr_groups[2] struct zhaoxin_uncore_ops { - void (*init_box)(struct zhaoxin_uncore_box *); - void (*exit_box)(struct zhaoxin_uncore_box *); - void (*disable_box)(struct zhaoxin_uncore_box *); - void (*enable_box)(struct zhaoxin_uncore_box *); - void (*disable_event)(struct zhaoxin_uncore_box *, struct perf_event *); - void (*enable_event)(struct zhaoxin_uncore_box *, struct perf_event *); - u64 (*read_counter)(struct zhaoxin_uncore_box *, struct perf_event *); - int (*hw_config)(struct zhaoxin_uncore_box *, struct perf_event *); - struct event_constraint *(*get_constraint)(struct zhaoxin_uncore_box *, - struct perf_event *); - void (*put_constraint)(struct zhaoxin_uncore_box *, struct perf_event *); + void (*init_box)(struct zhaoxin_uncore_box *box); + void (*exit_box)(struct zhaoxin_uncore_box *box); + void (*disable_box)(struct zhaoxin_uncore_box *box); + void (*enable_box)(struct zhaoxin_uncore_box *box); + void (*disable_event)(struct zhaoxin_uncore_box *box, struct perf_event *event); + void (*enable_event)(struct zhaoxin_uncore_box *box, struct perf_event *event); + u64 (*read_counter)(struct zhaoxin_uncore_box *box, struct perf_event *event); + int (*hw_config)(struct zhaoxin_uncore_box *box, struct perf_event *event); + struct event_constraint *(*get_constraint)(struct zhaoxin_uncore_box *box, + struct perf_event *event); + void (*put_constraint)(struct zhaoxin_uncore_box *box, struct perf_event *event); }; struct zhaoxin_uncore_pmu { - struct pmu pmu; - char name[UNCORE_PMU_NAME_LEN]; - int pmu_idx; - int func_id; - bool registered; - atomic_t activeboxes; - struct zhaoxin_uncore_type *type; - struct zhaoxin_uncore_box **boxes; + struct pmu pmu; + char name[UNCORE_PMU_NAME_LEN]; + int pmu_idx; + int func_id; + bool registered; + atomic_t activeboxes; + struct zhaoxin_uncore_type *type; + struct zhaoxin_uncore_box **boxes; }; struct zhaoxin_uncore_extra_reg { @@ -92,6 +99,8 @@ struct zhaoxin_uncore_extra_reg { struct zhaoxin_uncore_box { int pci_phys_id; int package_id; /*Package ID */ + int cluster_id; + int subnode_id; int n_active; /* number of active events */ int n_events; int cpu; /* cpu to collect events */ @@ -109,34 +118,38 @@ struct zhaoxin_uncore_box { struct list_head list; struct list_head active_list; void __iomem *io_addr; - struct zhaoxin_uncore_extra_reg shared_regs[0]; + struct zhaoxin_uncore_extra_reg shared_regs[]; }; #define UNCORE_BOX_FLAG_INITIATED 0 struct uncore_event_desc { - struct kobj_attribute attr; + struct device_attribute attr; const char *config; }; -ssize_t zx_uncore_event_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf); +struct hw_info { + u64 config_info; + u64 active_state; +}; + +ssize_t zx_uncore_event_show(struct device *dev, struct device_attribute *attr, char *buf); #define ZHAOXIN_UNCORE_EVENT_DESC(_name, _config) \ -{ \ +{ \ .attr = __ATTR(_name, 0444, zx_uncore_event_show, NULL), \ - .config = _config, \ + .config = _config, \ } #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \ -static ssize_t __uncore_##_var##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, \ - char *page) \ +static ssize_t __uncore_##_var##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *page) \ { \ BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ return sprintf(page, _format "\n"); \ } \ -static struct kobj_attribute format_attr_##_var = \ +static struct device_attribute format_attr_##_var = \ __ATTR(_name, 0444, __uncore_##_var##_show, NULL) static inline bool uncore_pmc_fixed(int idx) @@ -144,72 +157,109 @@ static inline bool uncore_pmc_fixed(int idx) return idx == UNCORE_PMC_IDX_FIXED; } -static inline unsigned uncore_msr_box_offset(struct zhaoxin_uncore_box *box) +static inline unsigned int uncore_mmio_box_ctl(struct zhaoxin_uncore_box *box) +{ + return box->pmu->type->box_ctl + box->pmu->type->mmio_offset * box->pmu->pmu_idx; +} + +static inline unsigned int uncore_pci_box_ctl(struct zhaoxin_uncore_box *box) +{ + return box->pmu->type->box_ctl; +} + +static inline unsigned int uncore_pci_fixed_ctl(struct zhaoxin_uncore_box *box) +{ + return box->pmu->type->fixed_ctl; +} + +static inline unsigned int uncore_pci_fixed_ctr(struct zhaoxin_uncore_box *box) +{ + return box->pmu->type->fixed_ctr; +} + +static inline unsigned int uncore_pci_event_ctl(struct zhaoxin_uncore_box *box, int idx) +{ + return idx * 4 + box->pmu->type->event_ctl; +} + +static inline unsigned int uncore_pci_perf_ctr(struct zhaoxin_uncore_box *box, int idx) +{ + if (!strncmp(box->pmu->type->name, "mc_", 3)) + return idx * 2 + box->pmu->type->perf_ctr; + else + return idx * 8 + box->pmu->type->perf_ctr; +} + +static inline unsigned int uncore_msr_box_offset(struct zhaoxin_uncore_box *box) { struct zhaoxin_uncore_pmu *pmu = box->pmu; return pmu->type->msr_offsets ? - pmu->type->msr_offsets[pmu->pmu_idx] : - pmu->type->msr_offset * pmu->pmu_idx; + pmu->type->msr_offsets[pmu->pmu_idx] : + pmu->type->msr_offset * pmu->pmu_idx; } -static inline unsigned uncore_msr_box_ctl(struct zhaoxin_uncore_box *box) +static inline unsigned int uncore_msr_box_ctl(struct zhaoxin_uncore_box *box) { if (!box->pmu->type->box_ctl) return 0; return box->pmu->type->box_ctl + uncore_msr_box_offset(box); } -static inline unsigned uncore_msr_fixed_ctl(struct zhaoxin_uncore_box *box) +static inline unsigned int uncore_msr_fixed_ctl(struct zhaoxin_uncore_box *box) { if (!box->pmu->type->fixed_ctl) return 0; return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box); } -static inline unsigned uncore_msr_fixed_ctr(struct zhaoxin_uncore_box *box) +static inline unsigned int uncore_msr_fixed_ctr(struct zhaoxin_uncore_box *box) { return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box); } -static inline -unsigned uncore_msr_event_ctl(struct zhaoxin_uncore_box *box, int idx) +static inline unsigned int uncore_msr_event_ctl(struct zhaoxin_uncore_box *box, int idx) { return box->pmu->type->event_ctl + - (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) + - uncore_msr_box_offset(box); + (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) + uncore_msr_box_offset(box); } -static inline -unsigned uncore_msr_perf_ctr(struct zhaoxin_uncore_box *box, int idx) +static inline unsigned int uncore_msr_perf_ctr(struct zhaoxin_uncore_box *box, int idx) { return box->pmu->type->perf_ctr + - (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) + - uncore_msr_box_offset(box); + (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) + uncore_msr_box_offset(box); } -static inline -unsigned uncore_fixed_ctl(struct zhaoxin_uncore_box *box) +static inline unsigned int uncore_fixed_ctl(struct zhaoxin_uncore_box *box) { - return uncore_msr_fixed_ctl(box); + if (box->pci_dev) + return uncore_pci_fixed_ctl(box); + else + return uncore_msr_fixed_ctl(box); } -static inline -unsigned uncore_fixed_ctr(struct zhaoxin_uncore_box *box) +static inline unsigned int uncore_fixed_ctr(struct zhaoxin_uncore_box *box) { - return uncore_msr_fixed_ctr(box); + if (box->pci_dev) + return uncore_pci_fixed_ctr(box); + else + return uncore_msr_fixed_ctr(box); } -static inline -unsigned uncore_event_ctl(struct zhaoxin_uncore_box *box, int idx) +static inline unsigned int uncore_event_ctl(struct zhaoxin_uncore_box *box, int idx) { - return uncore_msr_event_ctl(box, idx); + if (box->pci_dev || box->io_addr) + return uncore_pci_event_ctl(box, idx); + else + return uncore_msr_event_ctl(box, idx); } -static inline -unsigned uncore_perf_ctr(struct zhaoxin_uncore_box *box, int idx) +static inline unsigned int uncore_perf_ctr(struct zhaoxin_uncore_box *box, int idx) { - return uncore_msr_perf_ctr(box, idx); + if (box->pci_dev || box->io_addr) + return uncore_pci_perf_ctr(box, idx); + else + return uncore_msr_perf_ctr(box, idx); } static inline int uncore_perf_ctr_bits(struct zhaoxin_uncore_box *box) @@ -239,20 +289,17 @@ static inline void uncore_enable_box(struct zhaoxin_uncore_box *box) box->pmu->type->ops->enable_box(box); } -static inline void uncore_disable_event(struct zhaoxin_uncore_box *box, - struct perf_event *event) +static inline void uncore_disable_event(struct zhaoxin_uncore_box *box, struct perf_event *event) { box->pmu->type->ops->disable_event(box, event); } -static inline void uncore_enable_event(struct zhaoxin_uncore_box *box, - struct perf_event *event) +static inline void uncore_enable_event(struct zhaoxin_uncore_box *box, struct perf_event *event) { box->pmu->type->ops->enable_event(box, event); } -static inline u64 uncore_read_counter(struct zhaoxin_uncore_box *box, - struct perf_event *event) +static inline u64 uncore_read_counter(struct zhaoxin_uncore_box *box, struct perf_event *event) { return box->pmu->type->ops->read_counter(box, event); } @@ -288,10 +335,10 @@ static inline struct zhaoxin_uncore_box *uncore_event_to_box(struct perf_event * return event->pmu_private; } - static struct zhaoxin_uncore_box *uncore_pmu_to_box(struct zhaoxin_uncore_pmu *pmu, int cpu); static u64 uncore_msr_read_counter(struct zhaoxin_uncore_box *box, struct perf_event *event); - +static void uncore_mmio_exit_box(struct zhaoxin_uncore_box *box); +static u64 uncore_mmio_read_counter(struct zhaoxin_uncore_box *box, struct perf_event *event); static void uncore_pmu_start_hrtimer(struct zhaoxin_uncore_box *box); static void uncore_pmu_cancel_hrtimer(struct zhaoxin_uncore_box *box); static void uncore_pmu_event_start(struct perf_event *event, int flags); @@ -300,9 +347,7 @@ static int uncore_pmu_event_add(struct perf_event *event, int flags); static void uncore_pmu_event_del(struct perf_event *event, int flags); static void uncore_pmu_event_read(struct perf_event *event); static void uncore_perf_event_update(struct zhaoxin_uncore_box *box, struct perf_event *event); -struct event_constraint * -uncore_get_constraint(struct zhaoxin_uncore_box *box, struct perf_event *event); +struct event_constraint *uncore_get_constraint(struct zhaoxin_uncore_box *box, + struct perf_event *event); void uncore_put_constraint(struct zhaoxin_uncore_box *box, struct perf_event *event); u64 uncore_shared_reg_config(struct zhaoxin_uncore_box *box, int idx); - -void chx_uncore_cpu_init(void);