From f5f9340b9807d44d200658ba1bbbbbb57ab72e07 Mon Sep 17 00:00:00 2001 From: Fabien Thomas Date: Wed, 28 Mar 2012 20:58:30 +0000 Subject: [PATCH] Add software PMC support. New kernel events can be added at various location for sampling or counting. This will for example allow easy system profiling whatever the processor is with known tools like pmcstat(8). Simultaneous usage of software PMC and hardware PMC is possible, for example looking at the lock acquire failure, page fault while sampling on instructions. Sponsored by: NETASQ MFC after: 1 month --- lib/libpmc/Makefile | 1 + lib/libpmc/libpmc.c | 118 ++++++-- lib/libpmc/pmc.3 | 3 + lib/libpmc/pmc.atom.3 | 1 + lib/libpmc/pmc.core.3 | 1 + lib/libpmc/pmc.core2.3 | 1 + lib/libpmc/pmc.corei7.3 | 1 + lib/libpmc/pmc.corei7uc.3 | 1 + lib/libpmc/pmc.iaf.3 | 1 + lib/libpmc/pmc.k7.3 | 1 + lib/libpmc/pmc.k8.3 | 1 + lib/libpmc/pmc.mips24k.3 | 1 + lib/libpmc/pmc.octeon.3 | 1 + lib/libpmc/pmc.p4.3 | 1 + lib/libpmc/pmc.p5.3 | 1 + lib/libpmc/pmc.p6.3 | 1 + lib/libpmc/pmc.sandybridge.3 | 1 + lib/libpmc/pmc.sandybridgeuc.3 | 1 + lib/libpmc/pmc.soft.3 | 104 +++++++ lib/libpmc/pmc.tsc.3 | 1 + lib/libpmc/pmc.ucf.3 | 1 + lib/libpmc/pmc.westmere.3 | 1 + lib/libpmc/pmc.westmereuc.3 | 1 + lib/libpmc/pmc.xscale.3 | 1 + lib/libpmc/pmclog.c | 6 + lib/libpmc/pmclog.h | 8 + sys/amd64/amd64/trap.c | 17 +- sys/amd64/include/pmc_mdep.h | 23 +- sys/arm/include/pmc_mdep.h | 2 +- sys/conf/files | 1 + sys/dev/hwpmc/hwpmc_amd.c | 10 +- sys/dev/hwpmc/hwpmc_core.c | 6 +- sys/dev/hwpmc/hwpmc_intel.c | 6 +- sys/dev/hwpmc/hwpmc_logging.c | 28 +- sys/dev/hwpmc/hwpmc_mips.c | 2 +- sys/dev/hwpmc/hwpmc_mod.c | 251 ++++++++++++++--- sys/dev/hwpmc/hwpmc_piv.c | 4 +- sys/dev/hwpmc/hwpmc_powerpc.c | 3 +- sys/dev/hwpmc/hwpmc_ppro.c | 2 +- sys/dev/hwpmc/hwpmc_soft.c | 485 +++++++++++++++++++++++++++++++++ sys/dev/hwpmc/hwpmc_soft.h | 48 ++++ sys/dev/hwpmc/hwpmc_tsc.c | 3 - sys/dev/hwpmc/hwpmc_x86.c | 7 +- sys/dev/hwpmc/hwpmc_xscale.c | 4 +- sys/dev/hwpmc/pmc_events.h | 12 +- sys/i386/i386/trap.c | 17 +- sys/i386/include/pmc_mdep.h | 29 +- sys/kern/kern_clock.c | 12 +- sys/kern/kern_lock.c | 15 + sys/kern/kern_mutex.c | 15 + sys/kern/kern_pmc.c | 161 +++++++++++ sys/kern/kern_rwlock.c | 12 + sys/kern/kern_sx.c | 12 + sys/kern/subr_trap.c | 10 + sys/mips/include/pmc_mdep.h | 2 +- sys/modules/hwpmc/Makefile | 2 +- sys/powerpc/include/pmc_mdep.h | 3 +- sys/sys/pmc.h | 66 +++-- sys/sys/pmckern.h | 97 +++++++ sys/sys/pmclog.h | 17 +- usr.sbin/pmcstat/pmcstat_log.c | 15 + 61 files changed, 1518 insertions(+), 141 deletions(-) create mode 100644 lib/libpmc/pmc.soft.3 create mode 100644 sys/dev/hwpmc/hwpmc_soft.c create mode 100644 sys/dev/hwpmc/hwpmc_soft.h diff --git a/lib/libpmc/Makefile b/lib/libpmc/Makefile index 95da88e6df1e..5edbfe6419ef 100644 --- a/lib/libpmc/Makefile +++ b/lib/libpmc/Makefile @@ -20,6 +20,7 @@ MAN+= pmc_read.3 MAN+= pmc_set.3 MAN+= pmc_start.3 MAN+= pmclog.3 +MAN+= pmc.soft.3 # PMC-dependent manual pages .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c index 4323171e7443..3b7dc376ed27 100644 --- a/lib/libpmc/libpmc.c +++ b/lib/libpmc/libpmc.c @@ -77,11 +77,12 @@ static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec, static int xscale_allocate_pmc(enum pmc_event _pe, char *_ctrspec, struct pmc_op_pmcallocate *_pmc_config); #endif - #if defined(__mips__) static int mips_allocate_pmc(enum pmc_event _pe, char* ctrspec, struct pmc_op_pmcallocate *_pmc_config); #endif /* __mips__ */ +static int soft_allocate_pmc(enum pmc_event _pe, char *_ctrspec, + struct pmc_op_pmcallocate *_pmc_config); #if defined(__powerpc__) static int ppc7450_allocate_pmc(enum pmc_event _pe, char* ctrspec, @@ -156,6 +157,8 @@ PMC_CLASSDEP_TABLE(octeon, OCTEON); PMC_CLASSDEP_TABLE(ucf, UCF); PMC_CLASSDEP_TABLE(ppc7450, PPC7450); +static struct pmc_event_descr soft_event_table[PMC_EV_DYN_COUNT]; + #undef __PMC_EV_ALIAS #define __PMC_EV_ALIAS(N,CODE) { N, PMC_EV_##CODE }, @@ -215,21 +218,22 @@ static const struct pmc_event_descr westmereuc_event_table[] = PMC_CLASS_##C, __VA_ARGS__ \ } -PMC_MDEP_TABLE(atom, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC); -PMC_MDEP_TABLE(core, IAP, PMC_CLASS_TSC); -PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC); -PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); -PMC_MDEP_TABLE(sandybridge, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); -PMC_MDEP_TABLE(westmere, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); -PMC_MDEP_TABLE(k7, K7, PMC_CLASS_TSC); -PMC_MDEP_TABLE(k8, K8, PMC_CLASS_TSC); -PMC_MDEP_TABLE(p4, P4, PMC_CLASS_TSC); -PMC_MDEP_TABLE(p5, P5, PMC_CLASS_TSC); -PMC_MDEP_TABLE(p6, P6, PMC_CLASS_TSC); -PMC_MDEP_TABLE(xscale, XSCALE, PMC_CLASS_XSCALE); -PMC_MDEP_TABLE(mips24k, MIPS24K, PMC_CLASS_MIPS24K); -PMC_MDEP_TABLE(octeon, OCTEON, PMC_CLASS_OCTEON); -PMC_MDEP_TABLE(ppc7450, PPC7450, PMC_CLASS_PPC7450); +PMC_MDEP_TABLE(atom, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC); +PMC_MDEP_TABLE(core, IAP, PMC_CLASS_SOFT, PMC_CLASS_TSC); +PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC); +PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); +PMC_MDEP_TABLE(sandybridge, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); +PMC_MDEP_TABLE(westmere, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP); +PMC_MDEP_TABLE(k7, K7, PMC_CLASS_SOFT, PMC_CLASS_TSC); +PMC_MDEP_TABLE(k8, K8, PMC_CLASS_SOFT, PMC_CLASS_TSC); +PMC_MDEP_TABLE(p4, P4, PMC_CLASS_SOFT, PMC_CLASS_TSC); +PMC_MDEP_TABLE(p5, P5, PMC_CLASS_SOFT, PMC_CLASS_TSC); +PMC_MDEP_TABLE(p6, P6, PMC_CLASS_SOFT, PMC_CLASS_TSC); +PMC_MDEP_TABLE(xscale, XSCALE, PMC_CLASS_SOFT, PMC_CLASS_XSCALE); +PMC_MDEP_TABLE(mips24k, MIPS24K, PMC_CLASS_SOFT, PMC_CLASS_MIPS24K); +PMC_MDEP_TABLE(octeon, OCTEON, PMC_CLASS_SOFT, PMC_CLASS_OCTEON); +PMC_MDEP_TABLE(ppc7450, PPC7450, PMC_CLASS_SOFT, PMC_CLASS_PPC7450); +PMC_MDEP_TABLE(generic, SOFT, PMC_CLASS_SOFT); static const struct pmc_event_descr tsc_event_table[] = { @@ -279,16 +283,24 @@ PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc); #if defined(__XSCALE__) PMC_CLASS_TABLE_DESC(xscale, XSCALE, xscale, xscale); #endif - #if defined(__mips__) PMC_CLASS_TABLE_DESC(mips24k, MIPS24K, mips24k, mips); PMC_CLASS_TABLE_DESC(octeon, OCTEON, octeon, mips); #endif /* __mips__ */ - #if defined(__powerpc__) PMC_CLASS_TABLE_DESC(ppc7450, PPC7450, ppc7450, ppc7450); #endif +static struct pmc_class_descr soft_class_table_descr = +{ + .pm_evc_name = "SOFT-", + .pm_evc_name_size = sizeof("SOFT-") - 1, + .pm_evc_class = PMC_CLASS_SOFT, + .pm_evc_event_table = NULL, + .pm_evc_event_table_size = 0, + .pm_evc_allocate_pmc = soft_allocate_pmc +}; + #undef PMC_CLASS_TABLE_DESC static const struct pmc_class_descr **pmc_class_table; @@ -343,9 +355,12 @@ static const char * pmc_state_names[] = { __PMC_STATES() }; -static int pmc_syscall = -1; /* filled in by pmc_init() */ - -static struct pmc_cpuinfo cpu_info; /* filled in by pmc_init() */ +/* + * Filled in by pmc_init(). + */ +static int pmc_syscall = -1; +static struct pmc_cpuinfo cpu_info; +static struct pmc_op_getdyneventinfo soft_event_info; /* Event masks for events */ struct pmc_masks { @@ -2179,6 +2194,25 @@ tsc_allocate_pmc(enum pmc_event pe, char *ctrspec, } #endif +static struct pmc_event_alias generic_aliases[] = { + EV_ALIAS("instructions", "SOFT-CLOCK.HARD"), + EV_ALIAS(NULL, NULL) +}; + +static int +soft_allocate_pmc(enum pmc_event pe, char *ctrspec, + struct pmc_op_pmcallocate *pmc_config) +{ + (void)ctrspec; + (void)pmc_config; + + if (pe < PMC_EV_SOFT_FIRST || pe > PMC_EV_SOFT_LAST) + return (-1); + + pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE); + return (0); +} + #if defined(__XSCALE__) static struct pmc_event_alias xscale_aliases[] = { @@ -2663,6 +2697,10 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames, ev = ppc7450_event_table; count = PMC_EVENT_TABLE_SIZE(ppc7450); break; + case PMC_CLASS_SOFT: + ev = soft_event_table; + count = soft_event_info.pm_nevent; + break; default: errno = EINVAL; return (-1); @@ -2676,6 +2714,7 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames, for (;count--; ev++, names++) *names = ev->pm_ev_name; + return (0); } @@ -2779,12 +2818,35 @@ pmc_init(void) for (n = 0; n < PMC_CLASS_TABLE_SIZE; n++) pmc_class_table[n] = NULL; + /* + * Get soft events list. + */ + soft_event_info.pm_class = PMC_CLASS_SOFT; + if (PMC_CALL(GETDYNEVENTINFO, &soft_event_info) < 0) + return (pmc_syscall = -1); + + /* Map soft events to static list. */ + for (n = 0; n < soft_event_info.pm_nevent; n++) { + soft_event_table[n].pm_ev_name = + soft_event_info.pm_events[n].pm_ev_name; + soft_event_table[n].pm_ev_code = + soft_event_info.pm_events[n].pm_ev_code; + } + soft_class_table_descr.pm_evc_event_table_size = \ + soft_event_info.pm_nevent; + soft_class_table_descr.pm_evc_event_table = \ + soft_event_table; + /* * Fill in the class table. */ n = 0; + + /* Fill soft events information. */ + pmc_class_table[n++] = &soft_class_table_descr; #if defined(__amd64__) || defined(__i386__) - pmc_class_table[n++] = &tsc_class_table_descr; + if (cpu_info.pm_cputype != PMC_CPU_GENERIC) + pmc_class_table[n++] = &tsc_class_table_descr; /* * Check if this CPU has fixed function counters. @@ -2867,6 +2929,9 @@ pmc_init(void) pmc_class_table[n] = &p4_class_table_descr; break; #endif + case PMC_CPU_GENERIC: + PMC_MDEP_INIT(generic); + break; #if defined(__XSCALE__) case PMC_CPU_INTEL_XSCALE: PMC_MDEP_INIT(xscale); @@ -3035,18 +3100,19 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu) evfence = xscale_event_table + PMC_EVENT_TABLE_SIZE(xscale); } else if (pe >= PMC_EV_MIPS24K_FIRST && pe <= PMC_EV_MIPS24K_LAST) { ev = mips24k_event_table; - evfence = mips24k_event_table + PMC_EVENT_TABLE_SIZE(mips24k -); + evfence = mips24k_event_table + PMC_EVENT_TABLE_SIZE(mips24k); } else if (pe >= PMC_EV_OCTEON_FIRST && pe <= PMC_EV_OCTEON_LAST) { ev = octeon_event_table; evfence = octeon_event_table + PMC_EVENT_TABLE_SIZE(octeon); } else if (pe >= PMC_EV_PPC7450_FIRST && pe <= PMC_EV_PPC7450_LAST) { ev = ppc7450_event_table; - evfence = ppc7450_event_table + PMC_EVENT_TABLE_SIZE(ppc7450 -); + evfence = ppc7450_event_table + PMC_EVENT_TABLE_SIZE(ppc7450); } else if (pe == PMC_EV_TSC_TSC) { ev = tsc_event_table; evfence = tsc_event_table + PMC_EVENT_TABLE_SIZE(tsc); + } else if (pe >= PMC_EV_SOFT_FIRST && pe <= PMC_EV_SOFT_LAST) { + ev = soft_event_table; + evfence = soft_event_table + soft_event_info.pm_nevent; } for (; ev != evfence; ev++) diff --git a/lib/libpmc/pmc.3 b/lib/libpmc/pmc.3 index 0a537a593314..cf74f321076d 100644 --- a/lib/libpmc/pmc.3 +++ b/lib/libpmc/pmc.3 @@ -223,6 +223,8 @@ and CPUs. .It Li PMC_CLASS_TSC The timestamp counter on i386 and amd64 architecture CPUs. +.It Li PMC_CLASS_SOFT +Software events. .El .Ss PMC Capabilities Capabilities of performance monitoring hardware are denoted using @@ -525,6 +527,7 @@ API is .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 , diff --git a/lib/libpmc/pmc.atom.3 b/lib/libpmc/pmc.atom.3 index 2ebb80483c8a..f61a1415ec43 100644 --- a/lib/libpmc/pmc.atom.3 +++ b/lib/libpmc/pmc.atom.3 @@ -1176,6 +1176,7 @@ and the underlying hardware events used on these CPUs. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.core.3 b/lib/libpmc/pmc.core.3 index 512e4921c31b..73b2cea754de 100644 --- a/lib/libpmc/pmc.core.3 +++ b/lib/libpmc/pmc.core.3 @@ -792,6 +792,7 @@ may not count some transitions. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 diff --git a/lib/libpmc/pmc.core2.3 b/lib/libpmc/pmc.core2.3 index 56c015ba9391..5c6b533164f9 100644 --- a/lib/libpmc/pmc.core2.3 +++ b/lib/libpmc/pmc.core2.3 @@ -1107,6 +1107,7 @@ and the underlying hardware events used. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.corei7.3 b/lib/libpmc/pmc.corei7.3 index 8e092500e90a..aaf1f022615f 100644 --- a/lib/libpmc/pmc.corei7.3 +++ b/lib/libpmc/pmc.corei7.3 @@ -1559,6 +1559,7 @@ Counts number of segment register loads. .Xr pmc.corei7uc 3 , .Xr pmc.westmere 3 , .Xr pmc.westmereuc 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.corei7uc.3 b/lib/libpmc/pmc.corei7uc.3 index c92698004e6b..3114c8ea9f6a 100644 --- a/lib/libpmc/pmc.corei7uc.3 +++ b/lib/libpmc/pmc.corei7uc.3 @@ -863,6 +863,7 @@ refreshed or needs to go into a power down mode. .Xr pmc.corei7 3 , .Xr pmc.westmere 3 , .Xr pmc.westmereuc 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.iaf.3 b/lib/libpmc/pmc.iaf.3 index aa39a7cc9f74..7e623d6548ac 100644 --- a/lib/libpmc/pmc.iaf.3 +++ b/lib/libpmc/pmc.iaf.3 @@ -132,6 +132,7 @@ CPU, use the event specifier .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.k7.3 b/lib/libpmc/pmc.k7.3 index 59e64896cee0..646f35275149 100644 --- a/lib/libpmc/pmc.k7.3 +++ b/lib/libpmc/pmc.k7.3 @@ -249,6 +249,7 @@ and the underlying hardware events used. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 diff --git a/lib/libpmc/pmc.k8.3 b/lib/libpmc/pmc.k8.3 index e4eb081b4d7e..628137d7ffaf 100644 --- a/lib/libpmc/pmc.k8.3 +++ b/lib/libpmc/pmc.k8.3 @@ -783,6 +783,7 @@ and the underlying hardware events used. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 diff --git a/lib/libpmc/pmc.mips24k.3 b/lib/libpmc/pmc.mips24k.3 index dc49f09e8a77..4acb0a7eab94 100644 --- a/lib/libpmc/pmc.mips24k.3 +++ b/lib/libpmc/pmc.mips24k.3 @@ -392,6 +392,7 @@ and the underlying hardware events used. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.octeon.3 b/lib/libpmc/pmc.octeon.3 index d0ef3dedabe3..be38dd8c5786 100644 --- a/lib/libpmc/pmc.octeon.3 +++ b/lib/libpmc/pmc.octeon.3 @@ -232,6 +232,7 @@ and the underlying hardware events used. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.p4.3 b/lib/libpmc/pmc.p4.3 index 3d89fc140a6c..b273edd8b1e7 100644 --- a/lib/libpmc/pmc.p4.3 +++ b/lib/libpmc/pmc.p4.3 @@ -1208,6 +1208,7 @@ and the underlying hardware events used. .Xr pmc.k8 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 diff --git a/lib/libpmc/pmc.p5.3 b/lib/libpmc/pmc.p5.3 index dae43e34894f..ebdd5054e2d5 100644 --- a/lib/libpmc/pmc.p5.3 +++ b/lib/libpmc/pmc.p5.3 @@ -444,6 +444,7 @@ and the underlying hardware events used. .Xr pmc.k8 3 , .Xr pmc.p4 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 diff --git a/lib/libpmc/pmc.p6.3 b/lib/libpmc/pmc.p6.3 index 2033cce35fc0..3fa98a75a396 100644 --- a/lib/libpmc/pmc.p6.3 +++ b/lib/libpmc/pmc.p6.3 @@ -1010,6 +1010,7 @@ and the underlying hardware events used. .Xr pmc.k8 3 , .Xr pmc.p4 3 , .Xr pmc.p5 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmclog 3 , .Xr hwpmc 4 diff --git a/lib/libpmc/pmc.sandybridge.3 b/lib/libpmc/pmc.sandybridge.3 index d530d1573dff..e0c7eb9925c8 100644 --- a/lib/libpmc/pmc.sandybridge.3 +++ b/lib/libpmc/pmc.sandybridge.3 @@ -907,6 +907,7 @@ Split locks in SQ. .Xr pmc.p5 3 , .Xr pmc.p6 3 , .Xr pmc.sandybridgeuc 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc.ucf 3 , .Xr pmc.westmere 3 , diff --git a/lib/libpmc/pmc.sandybridgeuc.3 b/lib/libpmc/pmc.sandybridgeuc.3 index 19904b37cd51..207848238732 100644 --- a/lib/libpmc/pmc.sandybridgeuc.3 +++ b/lib/libpmc/pmc.sandybridgeuc.3 @@ -208,6 +208,7 @@ Counts the number of core-outgoing entries in the coherent tracker queue. .Xr pmc.p5 3 , .Xr pmc.p6 3 , .Xr pmc.sandybridge 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc.ucf 3 , .Xr pmc.westmere 3 , diff --git a/lib/libpmc/pmc.soft.3 b/lib/libpmc/pmc.soft.3 new file mode 100644 index 000000000000..f7d0cb05ee07 --- /dev/null +++ b/lib/libpmc/pmc.soft.3 @@ -0,0 +1,104 @@ +.\" Copyright (c) 2012 Fabien Thomas. All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd March 28, 2012 +.Os +.Dt PMC.SOFT 3 +.Sh NAME +.Nm pmc.soft +.Nd measurements using software based events +.Sh LIBRARY +.Lb libpmc +.Sh SYNOPSIS +.In pmc.h +.Sh DESCRIPTION +Software events are used to collect various source of software events. +.Ss PMC Features +16 sampling counters using software events based on various sources. +These PMCs support the following capabilities: +.Bl -column "PMC_CAP_INTERRUPT" "Support" +.It Em Capability Ta Em Support +.It PMC_CAP_CASCADE Ta \&No +.It PMC_CAP_EDGE Ta \&No +.It PMC_CAP_INTERRUPT Ta Yes +.It PMC_CAP_INVERT Ta \&No +.It PMC_CAP_READ Ta Yes +.It PMC_CAP_PRECISE Ta \&No +.It PMC_CAP_SYSTEM Ta Yes +.It PMC_CAP_TAGGING Ta \&No +.It PMC_CAP_THRESHOLD Ta \&No +.It PMC_CAP_USER Ta Yes +.It PMC_CAP_WRITE Ta Yes +.El +.Ss Event Qualifiers +There is no supported event qualifier. +.Pp +The event specifiers supported by software are: +.Bl -tag -width indent +.It Li CLOCK.HARD +Hard clock ticks. +.It Li CLOCK.STAT +Stat clock ticks. +.It Li LOCK.FAILED +Lock acquisition failed. +.It Li PAGE_FAULT.ALL +All page fault type. +.It Li PAGE_FAULT.READ +Read page fault. +.It Li PAGE_FAULT.WRITE +Write page fault. +.El +.Sh SEE ALSO +.Xr pmc 3 , +.Xr pmc.atom 3 , +.Xr pmc.core 3 , +.Xr pmc.iaf 3 , +.Xr pmc.ucf 3 , +.Xr pmc.k7 3 , +.Xr pmc.k8 3 , +.Xr pmc.p4 3 , +.Xr pmc.p5 3 , +.Xr pmc.p6 3 , +.Xr pmc.corei7 3 , +.Xr pmc.corei7uc 3 , +.Xr pmc.westmereuc 3 , +.Xr pmc.tsc 3 , +.Xr pmc_cpuinfo 3 , +.Xr pmclog 3 , +.Xr hwpmc 4 +.Sh HISTORY +The +.Nm pmc +library first appeared in +.Fx 6.0 . +.Sh AUTHORS +The +.Lb libpmc +library was written by +.An "Joseph Koshy" +.Aq jkoshy@FreeBSD.org . +Software PMC was written by +.An "Fabien Thomas" +.Aq fabient@FreeBSD.org . diff --git a/lib/libpmc/pmc.tsc.3 b/lib/libpmc/pmc.tsc.3 index 1abda30f8ebe..269200549d69 100644 --- a/lib/libpmc/pmc.tsc.3 +++ b/lib/libpmc/pmc.tsc.3 @@ -68,6 +68,7 @@ maps to the TSC. .Xr pmc.p4 3 , .Xr pmc.p5 3 , .Xr pmc.p6 3 , +.Xr pmc.soft 3 , .Xr pmclog 3 , .Xr hwpmc 4 .Sh HISTORY diff --git a/lib/libpmc/pmc.ucf.3 b/lib/libpmc/pmc.ucf.3 index 5a908f52e5c0..15399e43d02e 100644 --- a/lib/libpmc/pmc.ucf.3 +++ b/lib/libpmc/pmc.ucf.3 @@ -96,6 +96,7 @@ offset C0H under device number 0 and Function 0. .Xr pmc.corei7uc 3 , .Xr pmc.westmere 3 , .Xr pmc.westmereuc 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.westmere.3 b/lib/libpmc/pmc.westmere.3 index b80aefaf3c23..547ee71eff0f 100644 --- a/lib/libpmc/pmc.westmere.3 +++ b/lib/libpmc/pmc.westmere.3 @@ -1381,6 +1381,7 @@ Counts number of SID integer 64 bit shift or move operations. .Xr pmc.corei7 3 , .Xr pmc.corei7uc 3 , .Xr pmc.westmereuc 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.westmereuc.3 b/lib/libpmc/pmc.westmereuc.3 index c322778cc18f..517c470fbe7d 100644 --- a/lib/libpmc/pmc.westmereuc.3 +++ b/lib/libpmc/pmc.westmereuc.3 @@ -1066,6 +1066,7 @@ disabled. .Xr pmc.corei7 3 , .Xr pmc.corei7uc 3 , .Xr pmc.westmere 3 , +.Xr pmc.soft 3 , .Xr pmc.tsc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , diff --git a/lib/libpmc/pmc.xscale.3 b/lib/libpmc/pmc.xscale.3 index 25e78e25a72d..521cc9c9d065 100644 --- a/lib/libpmc/pmc.xscale.3 +++ b/lib/libpmc/pmc.xscale.3 @@ -134,6 +134,7 @@ and the underlying hardware events used. .Xr pmc 3 , .Xr pmc_cpuinfo 3 , .Xr pmclog 3 , +.Xr pmc.soft 3 , .Xr hwpmc 4 .Sh HISTORY The diff --git a/lib/libpmc/pmclog.c b/lib/libpmc/pmclog.c index d9ebc674ab65..26ec9ddaab02 100644 --- a/lib/libpmc/pmclog.c +++ b/lib/libpmc/pmclog.c @@ -369,6 +369,12 @@ pmclog_get_event(void *cookie, char **data, ssize_t *len, == NULL) goto error; break; + case PMCLOG_TYPE_PMCALLOCATEDYN: + PMCLOG_READ32(le,ev->pl_u.pl_ad.pl_pmcid); + PMCLOG_READ32(le,ev->pl_u.pl_ad.pl_event); + PMCLOG_READ32(le,ev->pl_u.pl_ad.pl_flags); + PMCLOG_READSTRING(le,ev->pl_u.pl_ad.pl_evname,PMC_NAME_MAX); + break; case PMCLOG_TYPE_PMCATTACH: PMCLOG_GET_PATHLEN(pathlen,evlen,pmclog_pmcattach); PMCLOG_READ32(le,ev->pl_u.pl_t.pl_pmcid); diff --git a/lib/libpmc/pmclog.h b/lib/libpmc/pmclog.h index b7c9c847628c..2a50945e372e 100644 --- a/lib/libpmc/pmclog.h +++ b/lib/libpmc/pmclog.h @@ -88,6 +88,13 @@ struct pmclog_ev_pmcallocate { pmc_id_t pl_pmcid; }; +struct pmclog_ev_pmcallocatedyn { + uint32_t pl_event; + char pl_evname[PMC_NAME_MAX]; + uint32_t pl_flags; + pmc_id_t pl_pmcid; +}; + struct pmclog_ev_pmcattach { pmc_id_t pl_pmcid; pid_t pl_pid; @@ -146,6 +153,7 @@ struct pmclog_ev { struct pmclog_ev_map_out pl_mo; struct pmclog_ev_pcsample pl_s; struct pmclog_ev_pmcallocate pl_a; + struct pmclog_ev_pmcallocatedyn pl_ad; struct pmclog_ev_pmcattach pl_t; struct pmclog_ev_pmcdetach pl_d; struct pmclog_ev_proccsw pl_c; diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index a685f1956453..a760181168ae 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -71,6 +71,9 @@ __FBSDID("$FreeBSD$"); #include #ifdef HWPMC_HOOKS #include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); #endif #include @@ -743,8 +746,20 @@ trap_pfault(frame, usermode) */ rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL); } - if (rv == KERN_SUCCESS) + if (rv == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { + PMC_SOFT_CALL_TF( , , page_fault, all, frame); + if (ftype == VM_PROT_READ) + PMC_SOFT_CALL_TF( , , page_fault, read, + frame); + else + PMC_SOFT_CALL_TF( , , page_fault, write, + frame); + } +#endif return (0); + } nogo: if (!usermode) { if (td->td_intr_nesting_level == 0 && diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h index 4f164852d3c0..73c93feb0f2e 100644 --- a/sys/amd64/include/pmc_mdep.h +++ b/sys/amd64/include/pmc_mdep.h @@ -50,13 +50,13 @@ struct pmc_mdep; * measurement architecture have PMCs of the following classes: TSC, * IAF, IAP, UCF and UCP. */ -#define PMC_MDEP_CLASS_INDEX_TSC 0 -#define PMC_MDEP_CLASS_INDEX_K8 1 -#define PMC_MDEP_CLASS_INDEX_P4 1 -#define PMC_MDEP_CLASS_INDEX_IAP 1 -#define PMC_MDEP_CLASS_INDEX_IAF 2 -#define PMC_MDEP_CLASS_INDEX_UCP 3 -#define PMC_MDEP_CLASS_INDEX_UCF 4 +#define PMC_MDEP_CLASS_INDEX_TSC 1 +#define PMC_MDEP_CLASS_INDEX_K8 2 +#define PMC_MDEP_CLASS_INDEX_P4 2 +#define PMC_MDEP_CLASS_INDEX_IAP 2 +#define PMC_MDEP_CLASS_INDEX_IAF 3 +#define PMC_MDEP_CLASS_INDEX_UCP 4 +#define PMC_MDEP_CLASS_INDEX_UCF 5 /* * On the amd64 platform we support the following PMCs. @@ -119,6 +119,15 @@ union pmc_md_pmc { #define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +/* Build a fake kernel trapframe from current instruction pointer. */ +#define PMC_FAKE_TRAPFRAME(TF) \ + do { \ + (TF)->tf_cs = 0; (TF)->tf_rflags = 0; \ + __asm __volatile("movq %%rbp,%0" : "=r" ((TF)->tf_rbp)); \ + __asm __volatile("movq %%rsp,%0" : "=r" ((TF)->tf_rsp)); \ + __asm __volatile("call 1f \n\t1: pop %0" : "=r"((TF)->tf_rip)); \ + } while (0) + /* * Prototypes */ diff --git a/sys/arm/include/pmc_mdep.h b/sys/arm/include/pmc_mdep.h index 115d4fe14811..185fbd1744c0 100644 --- a/sys/arm/include/pmc_mdep.h +++ b/sys/arm/include/pmc_mdep.h @@ -29,7 +29,7 @@ #ifndef _MACHINE_PMC_MDEP_H_ #define _MACHINE_PMC_MDEP_H_ -#define PMC_MDEP_CLASS_INDEX_XSCALE 0 +#define PMC_MDEP_CLASS_INDEX_XSCALE 1 /* * On the ARM platform we support the following PMCs. * diff --git a/sys/conf/files b/sys/conf/files index e7d65582ea83..f913269588fd 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1260,6 +1260,7 @@ dev/hme/if_hme_sbus.c optional hme sbus dev/hptiop/hptiop.c optional hptiop scbus dev/hwpmc/hwpmc_logging.c optional hwpmc dev/hwpmc/hwpmc_mod.c optional hwpmc +dev/hwpmc/hwpmc_soft.c optional hwpmc dev/ichsmb/ichsmb.c optional ichsmb dev/ichsmb/ichsmb_pci.c optional ichsmb pci dev/ida/ida.c optional ida diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 9ffa62fbc8ac..ed49ea78a2e3 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -687,7 +687,8 @@ amd_intr(int cpu, struct trapframe *tf) wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); /* Restart the counter if logging succeeded. */ - error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf)); + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, + TRAPF_USERMODE(tf)); if (error == 0) wrmsr(evsel, config | AMD_PMC_ENABLE); } @@ -874,7 +875,7 @@ amd_pcpu_fini(struct pmc_mdep *md, int cpu) struct pmc_mdep * pmc_amd_initialize(void) { - int classindex, error, i, nclasses, ncpus; + int classindex, error, i, ncpus; struct pmc_classdep *pcd; enum pmc_cputype cputype; struct pmc_mdep *pmc_mdep; @@ -926,12 +927,9 @@ pmc_amd_initialize(void) * These processors have two classes of PMCs: the TSC and * programmable PMCs. */ - nclasses = 2; - pmc_mdep = malloc(sizeof(struct pmc_mdep) + nclasses * sizeof (struct pmc_classdep), - M_PMC, M_WAITOK|M_ZERO); + pmc_mdep = pmc_mdep_alloc(2); pmc_mdep->pmd_cputype = cputype; - pmc_mdep->pmd_nclass = nclasses; ncpus = pmc_cpu_max(); diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c index 1aac3b9eb627..6209c6f09da5 100644 --- a/sys/dev/hwpmc/hwpmc_core.c +++ b/sys/dev/hwpmc/hwpmc_core.c @@ -2239,7 +2239,7 @@ core_intr(int cpu, struct trapframe *tf) if (pm->pm_state != PMC_STATE_RUNNING) continue; - error = pmc_process_interrupt(cpu, pm, tf, + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); v = pm->pm_sc.pm_reloadcount; @@ -2326,7 +2326,7 @@ core2_intr(int cpu, struct trapframe *tf) !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; - error = pmc_process_interrupt(cpu, pm, tf, + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error) intrenable &= ~flag; @@ -2354,7 +2354,7 @@ core2_intr(int cpu, struct trapframe *tf) !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) continue; - error = pmc_process_interrupt(cpu, pm, tf, + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error) intrenable &= ~flag; diff --git a/sys/dev/hwpmc/hwpmc_intel.c b/sys/dev/hwpmc/hwpmc_intel.c index 89fd0b134e39..f7adb6c69c35 100644 --- a/sys/dev/hwpmc/hwpmc_intel.c +++ b/sys/dev/hwpmc/hwpmc_intel.c @@ -162,12 +162,10 @@ pmc_intel_initialize(void) return (NULL); } - pmc_mdep = malloc(sizeof(struct pmc_mdep) + nclasses * - sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO); + /* Allocate base class and initialize machine dependent struct */ + pmc_mdep = pmc_mdep_alloc(nclasses); pmc_mdep->pmd_cputype = cputype; - pmc_mdep->pmd_nclass = nclasses; - pmc_mdep->pmd_switch_in = intel_switch_in; pmc_mdep->pmd_switch_out = intel_switch_out; diff --git a/sys/dev/hwpmc/hwpmc_logging.c b/sys/dev/hwpmc/hwpmc_logging.c index 79170edb29a3..880bcaac0a0b 100644 --- a/sys/dev/hwpmc/hwpmc_logging.c +++ b/sys/dev/hwpmc/hwpmc_logging.c @@ -129,6 +129,7 @@ static struct mtx pmc_kthread_mtx; /* sleep lock */ /* Emit a string. Caution: does NOT update _le, so needs to be last */ #define PMCLOG_EMITSTRING(S,L) do { bcopy((S), _le, (L)); } while (0) +#define PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0) #define PMCLOG_DESPATCH(PO) \ pmclog_release((PO)); \ @@ -835,16 +836,33 @@ void pmclog_process_pmcallocate(struct pmc *pm) { struct pmc_owner *po; + struct pmc_soft *ps; po = pm->pm_owner; PMCDBG(LOG,ALL,1, "pm=%p", pm); - PMCLOG_RESERVE(po, PMCALLOCATE, sizeof(struct pmclog_pmcallocate)); - PMCLOG_EMIT32(pm->pm_id); - PMCLOG_EMIT32(pm->pm_event); - PMCLOG_EMIT32(pm->pm_flags); - PMCLOG_DESPATCH(po); + if (PMC_TO_CLASS(pm) == PMC_CLASS_SOFT) { + PMCLOG_RESERVE(po, PMCALLOCATEDYN, + sizeof(struct pmclog_pmcallocatedyn)); + PMCLOG_EMIT32(pm->pm_id); + PMCLOG_EMIT32(pm->pm_event); + PMCLOG_EMIT32(pm->pm_flags); + ps = pmc_soft_ev_acquire(pm->pm_event); + if (ps != NULL) + PMCLOG_EMITSTRING(ps->ps_ev.pm_ev_name,PMC_NAME_MAX); + else + PMCLOG_EMITNULLSTRING(PMC_NAME_MAX); + pmc_soft_ev_release(ps); + PMCLOG_DESPATCH(po); + } else { + PMCLOG_RESERVE(po, PMCALLOCATE, + sizeof(struct pmclog_pmcallocate)); + PMCLOG_EMIT32(pm->pm_id); + PMCLOG_EMIT32(pm->pm_event); + PMCLOG_EMIT32(pm->pm_flags); + PMCLOG_DESPATCH(po); + } } void diff --git a/sys/dev/hwpmc/hwpmc_mips.c b/sys/dev/hwpmc/hwpmc_mips.c index 3afa5d9b9866..8df27c908c8c 100644 --- a/sys/dev/hwpmc/hwpmc_mips.c +++ b/sys/dev/hwpmc/hwpmc_mips.c @@ -287,7 +287,7 @@ mips_pmc_intr(int cpu, struct trapframe *tf) retval = 1; if (pm->pm_state != PMC_STATE_RUNNING) continue; - error = pmc_process_interrupt(cpu, pm, tf, + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error) { /* Clear/disable the relevant counter */ diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c index 7ca7a47e12c4..7eb9f92cc513 100644 --- a/sys/dev/hwpmc/hwpmc_mod.c +++ b/sys/dev/hwpmc/hwpmc_mod.c @@ -70,6 +70,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include "hwpmc_soft.h" + /* * Types */ @@ -182,7 +184,7 @@ static int pmc_attach_one_process(struct proc *p, struct pmc *pm); static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu); static int pmc_can_attach(struct pmc *pm, struct proc *p); -static void pmc_capture_user_callchain(int cpu, struct trapframe *tf); +static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf); static void pmc_cleanup(void); static int pmc_detach_process(struct proc *p, struct pmc *pm); static int pmc_detach_one_process(struct proc *p, struct pmc *pm, @@ -206,7 +208,7 @@ static void pmc_process_csw_out(struct thread *td); static void pmc_process_exit(void *arg, struct proc *p); static void pmc_process_fork(void *arg, struct proc *p1, struct proc *p2, int n); -static void pmc_process_samples(int cpu); +static void pmc_process_samples(int cpu, int soft); static void pmc_release_pmc_descriptor(struct pmc *pmc); static void pmc_remove_owner(struct pmc_owner *po); static void pmc_remove_process_descriptor(struct pmc_process *pp); @@ -218,12 +220,16 @@ static int pmc_stop(struct pmc *pm); static int pmc_syscall_handler(struct thread *td, void *syscall_args); static void pmc_unlink_target_process(struct pmc *pmc, struct pmc_process *pp); +static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp); +static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp); +static struct pmc_mdep *pmc_generic_cpu_initialize(void); +static void pmc_generic_cpu_finalize(struct pmc_mdep *md); /* * Kernel tunables and sysctl(8) interface. */ -SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); +SYSCTL_DECL(_kern_hwpmc); static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth); @@ -1833,7 +1839,9 @@ const char *pmc_hooknames[] = { "KLDUNLOAD", "MMAP", "MUNMAP", - "CALLCHAIN" + "CALLCHAIN-NMI", + "CALLCHAIN-SOFT", + "SOFTSAMPLING" }; #endif @@ -1992,7 +2000,8 @@ pmc_hook_handler(struct thread *td, int function, void *arg) * lose the interrupt sample. */ CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask); - pmc_process_samples(PCPU_GET(cpuid)); + pmc_process_samples(PCPU_GET(cpuid), PMC_HR); + pmc_process_samples(PCPU_GET(cpuid), PMC_SR); break; @@ -2022,11 +2031,30 @@ pmc_hook_handler(struct thread *td, int function, void *arg) */ KASSERT(td == curthread, ("[pmc,%d] td != curthread", __LINE__)); - pmc_capture_user_callchain(PCPU_GET(cpuid), + + pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR, (struct trapframe *) arg); td->td_pflags &= ~TDP_CALLCHAIN; break; + case PMC_FN_USER_CALLCHAIN_SOFT: + /* + * Record a call chain. + */ + KASSERT(td == curthread, ("[pmc,%d] td != curthread", + __LINE__)); + pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR, + (struct trapframe *) arg); + td->td_pflags &= ~TDP_CALLCHAIN; + break; + + case PMC_FN_SOFT_SAMPLING: + /* + * Call soft PMC sampling intr. + */ + pmc_soft_intr((struct pmckern_soft *) arg); + break; + default: #ifdef DEBUG KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); @@ -2221,18 +2249,17 @@ pmc_destroy_pmc_descriptor(struct pmc *pm) static void pmc_wait_for_pmc_idle(struct pmc *pm) { -#ifdef DEBUG +#ifdef DEBUG volatile int maxloop; maxloop = 100 * pmc_cpu_max(); #endif - /* * Loop (with a forced context switch) till the PMC's runcount * comes down to zero. */ while (atomic_load_acq_32(&pm->pm_runcount) > 0) { -#ifdef DEBUG +#ifdef DEBUG maxloop--; KASSERT(maxloop > 0, ("[pmc,%d] (ri%d, rc%d) waiting too long for " @@ -2972,6 +2999,53 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) } break; + /* + * Retrieve soft events list. + */ + case PMC_OP_GETDYNEVENTINFO: + { + enum pmc_class cl; + enum pmc_event ev; + struct pmc_op_getdyneventinfo *gei; + struct pmc_dyn_event_descr dev; + struct pmc_soft *ps; + uint32_t nevent; + + sx_assert(&pmc_sx, SX_LOCKED); + + gei = (struct pmc_op_getdyneventinfo *) arg; + + if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0) + break; + + /* Only SOFT class is dynamic. */ + if (cl != PMC_CLASS_SOFT) { + error = EINVAL; + break; + } + + nevent = 0; + for (ev = PMC_EV_SOFT_FIRST; ev <= PMC_EV_SOFT_LAST; ev++) { + ps = pmc_soft_ev_acquire(ev); + if (ps == NULL) + continue; + bcopy(&ps->ps_ev, &dev, sizeof(dev)); + pmc_soft_ev_release(ps); + + error = copyout(&dev, + &gei->pm_events[nevent], + sizeof(struct pmc_dyn_event_descr)); + if (error != 0) + break; + nevent++; + } + if (error != 0) + break; + + error = copyout(&nevent, &gei->pm_nevent, + sizeof(nevent)); + } + break; /* * Get module statistics @@ -4022,7 +4096,7 @@ pmc_post_callchain_callback(void) */ int -pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf, +pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf, int inuserspace) { int error, callchaindepth; @@ -4035,7 +4109,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf, /* * Allocate space for a sample buffer. */ - psb = pmc_pcpu[cpu]->pc_sb; + psb = pmc_pcpu[cpu]->pc_sb[ring]; ps = psb->ps_write; if (ps->ps_nsamples) { /* in use, reader hasn't caught up */ @@ -4061,6 +4135,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf, pm->pm_runcount)); atomic_add_rel_int(&pm->pm_runcount, 1); /* hold onto PMC */ + ps->ps_pmc = pm; if ((td = curthread) && td->td_proc) ps->ps_pid = td->td_proc->p_pid; @@ -4080,11 +4155,11 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf, * Kernel stack traversals can be done immediately, * while we defer to an AST for user space traversals. */ - if (!inuserspace) + if (!inuserspace) { callchaindepth = pmc_save_kernel_callchain(ps->ps_pc, callchaindepth, tf); - else { + } else { pmc_post_callchain_callback(); callchaindepth = PMC_SAMPLE_INUSE; } @@ -4113,7 +4188,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf, */ static void -pmc_capture_user_callchain(int cpu, struct trapframe *tf) +pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf) { int i; struct pmc *pm; @@ -4124,9 +4199,7 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf) int ncallchains; #endif - sched_unpin(); /* Can migrate safely now. */ - - psb = pmc_pcpu[cpu]->pc_sb; + psb = pmc_pcpu[cpu]->pc_sb[ring]; td = curthread; KASSERT(td->td_pflags & TDP_CALLCHAIN, @@ -4172,23 +4245,25 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf) #ifdef INVARIANTS ncallchains++; #endif - } KASSERT(ncallchains > 0, ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__, cpu)); + KASSERT(td->td_pinned > 0, + ("[pmc,%d] invalid td_pinned value", __LINE__)); + sched_unpin(); /* Can migrate safely now. */ + return; } - /* * Process saved PC samples. */ static void -pmc_process_samples(int cpu) +pmc_process_samples(int cpu, int ring) { struct pmc *pm; int adjri, n; @@ -4202,18 +4277,13 @@ pmc_process_samples(int cpu) ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, PCPU_GET(cpuid), cpu)); - psb = pmc_pcpu[cpu]->pc_sb; + psb = pmc_pcpu[cpu]->pc_sb[ring]; for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */ ps = psb->ps_read; if (ps->ps_nsamples == PMC_SAMPLE_FREE) break; - if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { - /* Need a rescan at a later time. */ - CPU_SET_ATOMIC(cpu, &pmc_cpumask); - break; - } pm = ps->ps_pmc; @@ -4231,6 +4301,13 @@ pmc_process_samples(int cpu) if (pm->pm_state != PMC_STATE_RUNNING) goto entrydone; + /* If there is a pending AST wait for completion */ + if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { + /* Need a rescan at a later time. */ + CPU_SET_ATOMIC(cpu, &pmc_cpumask); + break; + } + PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu, pm, ps->ps_nsamples, ps->ps_flags, (int) (psb->ps_write - psb->ps_samples), @@ -4256,11 +4333,10 @@ pmc_process_samples(int cpu) * or a system-wide sampling PMC. Dispatch a log * entry to the PMC's owner process. */ - pmclog_process_callchain(pm, ps); entrydone: - ps->ps_nsamples = 0; /* mark entry as free */ + ps->ps_nsamples = 0; /* mark entry as free */ atomic_subtract_rel_int(&pm->pm_runcount, 1); /* increment read pointer, modulo sample size */ @@ -4584,6 +4660,76 @@ static const char *pmc_name_of_pmcclass[] = { __PMC_CLASSES() }; +/* + * Base class initializer: allocate structure and set default classes. + */ +struct pmc_mdep * +pmc_mdep_alloc(int nclasses) +{ + struct pmc_mdep *md; + int n; + + /* SOFT + md classes */ + n = 1 + nclasses; + md = malloc(sizeof(struct pmc_mdep) + n * + sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO); + if (md != NULL) { + md->pmd_nclass = n; + + /* Add base class. */ + pmc_soft_initialize(md); + } + + return md; +} + +void +pmc_mdep_free(struct pmc_mdep *md) +{ + pmc_soft_finalize(md); + free(md, M_PMC); +} + +static int +generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) +{ + (void) pc; (void) pp; + + return (0); +} + +static int +generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) +{ + (void) pc; (void) pp; + + return (0); +} + +static struct pmc_mdep * +pmc_generic_cpu_initialize(void) +{ + struct pmc_mdep *md; + + md = pmc_mdep_alloc(0); + + md->pmd_cputype = PMC_CPU_GENERIC; + + md->pmd_pcpu_init = NULL; + md->pmd_pcpu_fini = NULL; + md->pmd_switch_in = generic_switch_in; + md->pmd_switch_out = generic_switch_out; + + return (md); +} + +static void +pmc_generic_cpu_finalize(struct pmc_mdep *md) +{ + (void) md; +} + + static int pmc_initialize(void) { @@ -4643,9 +4789,12 @@ pmc_initialize(void) } md = pmc_md_initialize(); - - if (md == NULL) - return (ENOSYS); + if (md == NULL) { + /* Default to generic CPU. */ + md = pmc_generic_cpu_initialize(); + if (md == NULL) + return (ENOSYS); + } KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1, ("[pmc,%d] no classes or pmcs", __LINE__)); @@ -4717,7 +4866,25 @@ pmc_initialize(void) ps->ps_pc = sb->ps_callchains + (n * pmc_callchaindepth); - pmc_pcpu[cpu]->pc_sb = sb; + pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb; + + sb = malloc(sizeof(struct pmc_samplebuffer) + + pmc_nsamples * sizeof(struct pmc_sample), M_PMC, + M_WAITOK|M_ZERO); + sb->ps_read = sb->ps_write = sb->ps_samples; + sb->ps_fence = sb->ps_samples + pmc_nsamples; + + KASSERT(pmc_pcpu[cpu] != NULL, + ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); + + sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * + sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); + + for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) + ps->ps_pc = sb->ps_callchains + + (n * pmc_callchaindepth); + + pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb; } /* allocate space for the row disposition array */ @@ -4887,9 +5054,12 @@ pmc_cleanup(void) md->pmd_pcpu_fini(md, cpu); } - pmc_md_finalize(md); + if (md->pmd_cputype == PMC_CPU_GENERIC) + pmc_generic_cpu_finalize(md); + else + pmc_md_finalize(md); - free(md, M_PMC); + pmc_mdep_free(md); md = NULL; pmc_restore_cpu_binding(&pb); } @@ -4898,11 +5068,16 @@ pmc_cleanup(void) for (cpu = 0; cpu < maxcpu; cpu++) { if (!pmc_cpu_is_active(cpu)) continue; - KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, - ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, + KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL, + ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__, cpu)); - free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC); - free(pmc_pcpu[cpu]->pc_sb, M_PMC); + KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL, + ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__, + cpu)); + free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC); + free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC); + free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC); + free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC); free(pmc_pcpu[cpu], M_PMC); } diff --git a/sys/dev/hwpmc/hwpmc_piv.c b/sys/dev/hwpmc/hwpmc_piv.c index 8ee851828b1c..26b23a14dcd1 100644 --- a/sys/dev/hwpmc/hwpmc_piv.c +++ b/sys/dev/hwpmc/hwpmc_piv.c @@ -1463,7 +1463,7 @@ p4_intr(int cpu, struct trapframe *tf) !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; } - (void) pmc_process_interrupt(cpu, pm, tf, + (void) pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); continue; } @@ -1513,7 +1513,7 @@ p4_intr(int cpu, struct trapframe *tf) * Process the interrupt. Re-enable the PMC if * processing was successful. */ - error = pmc_process_interrupt(cpu, pm, tf, + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); /* diff --git a/sys/dev/hwpmc/hwpmc_powerpc.c b/sys/dev/hwpmc/hwpmc_powerpc.c index 8e97b97a7f4d..ccbcb2cff0ad 100644 --- a/sys/dev/hwpmc/hwpmc_powerpc.c +++ b/sys/dev/hwpmc/hwpmc_powerpc.c @@ -690,7 +690,8 @@ powerpc_intr(int cpu, struct trapframe *tf) powerpc_pmcn_write(i, v); /* Restart the counter if logging succeeded. */ - error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf)); + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, + TRAPF_USERMODE(tf)); mtspr(SPR_MMCR0, config); if (error != 0) powerpc_stop_pmc(cpu, i); diff --git a/sys/dev/hwpmc/hwpmc_ppro.c b/sys/dev/hwpmc/hwpmc_ppro.c index 8da185bf6abb..416a54037cec 100644 --- a/sys/dev/hwpmc/hwpmc_ppro.c +++ b/sys/dev/hwpmc/hwpmc_ppro.c @@ -704,7 +704,7 @@ p6_intr(int cpu, struct trapframe *tf) if (pm->pm_state != PMC_STATE_RUNNING) continue; - error = pmc_process_interrupt(cpu, pm, tf, + error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, TRAPF_USERMODE(tf)); if (error) P6_MARK_STOPPED(pc,ri); diff --git a/sys/dev/hwpmc/hwpmc_soft.c b/sys/dev/hwpmc/hwpmc_soft.c new file mode 100644 index 000000000000..c3d2dec87549 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_soft.c @@ -0,0 +1,485 @@ +/*- + * Copyright (c) 2012 Fabien Thomas + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include + +#include +#include + +#include "hwpmc_soft.h" + +/* + * Software PMC support. + */ + +#define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ + PMC_CAP_USER | PMC_CAP_SYSTEM) + +struct soft_descr { + struct pmc_descr pm_descr; /* "base class" */ +}; + +static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = +{ +#define SOFT_PMCDESCR(N) \ + { \ + .pm_descr = \ + { \ + .pd_name = #N, \ + .pd_class = PMC_CLASS_SOFT, \ + .pd_caps = SOFT_CAPS, \ + .pd_width = 64 \ + }, \ + } + + SOFT_PMCDESCR(SOFT0), + SOFT_PMCDESCR(SOFT1), + SOFT_PMCDESCR(SOFT2), + SOFT_PMCDESCR(SOFT3), + SOFT_PMCDESCR(SOFT4), + SOFT_PMCDESCR(SOFT5), + SOFT_PMCDESCR(SOFT6), + SOFT_PMCDESCR(SOFT7), + SOFT_PMCDESCR(SOFT8), + SOFT_PMCDESCR(SOFT9), + SOFT_PMCDESCR(SOFT10), + SOFT_PMCDESCR(SOFT11), + SOFT_PMCDESCR(SOFT12), + SOFT_PMCDESCR(SOFT13), + SOFT_PMCDESCR(SOFT14), + SOFT_PMCDESCR(SOFT15) +}; + +/* + * Per-CPU data structure. + */ + +struct soft_cpu { + struct pmc_hw soft_hw[SOFT_NPMCS]; + pmc_value_t soft_values[SOFT_NPMCS]; +}; + + +static struct soft_cpu **soft_pcpu; + +static int +soft_allocate_pmc(int cpu, int ri, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + enum pmc_event ev; + struct pmc_soft *ps; + + (void) cpu; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + if (a->pm_class != PMC_CLASS_SOFT) + return (EINVAL); + + if ((pm->pm_caps & SOFT_CAPS) == 0) + return (EINVAL); + + if ((pm->pm_caps & ~SOFT_CAPS) != 0) + return (EPERM); + + ev = pm->pm_event; + if (ev < PMC_EV_SOFT_FIRST || ev > PMC_EV_SOFT_LAST) + return (EINVAL); + + /* Check if event is registered. */ + ps = pmc_soft_ev_acquire(ev); + if (ps == NULL) + return (EINVAL); + pmc_soft_ev_release(ps); + + return (0); +} + +static int +soft_config_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pmc_hw *phw; + + PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + phw = &soft_pcpu[cpu]->soft_hw[ri]; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, + pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + + return (0); +} + +static int +soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) +{ + int error; + size_t copied; + const struct soft_descr *pd; + struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + phw = &soft_pcpu[cpu]->soft_hw[ri]; + pd = &soft_pmcdesc[ri]; + + if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, + PMC_NAME_MAX, &copied)) != 0) + return (error); + + pi->pm_class = pd->pm_descr.pd_class; + + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + pi->pm_enabled = TRUE; + *ppmc = phw->phw_pmc; + } else { + pi->pm_enabled = FALSE; + *ppmc = NULL; + } + + return (0); +} + +static int +soft_get_config(int cpu, int ri, struct pmc **ppm) +{ + (void) ri; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; + return (0); +} + +static int +soft_pcpu_fini(struct pmc_mdep *md, int cpu) +{ + int ri; + struct pmc_cpu *pc; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); + + free(soft_pcpu[cpu], M_PMC); + soft_pcpu[cpu] = NULL; + + ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; + + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] ri=%d", __LINE__, ri)); + + pc = pmc_pcpu[cpu]; + pc->pc_hwpmcs[ri] = NULL; + + return (0); +} + +static int +soft_pcpu_init(struct pmc_mdep *md, int cpu) +{ + int first_ri, n; + struct pmc_cpu *pc; + struct soft_cpu *soft_pc; + struct pmc_hw *phw; + + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); + KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", + __LINE__)); + + soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); + if (soft_pc == NULL) + return (ENOMEM); + + pc = pmc_pcpu[cpu]; + + KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); + + soft_pcpu[cpu] = soft_pc; + phw = soft_pc->soft_hw; + first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; + + for (n = 0; n < SOFT_NPMCS; n++, phw++) { + phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | + PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); + phw->phw_pmc = NULL; + pc->pc_hwpmcs[n + first_ri] = phw; + } + + return (0); +} + +static int +soft_read_pmc(int cpu, int ri, pmc_value_t *v) +{ + struct pmc *pm; + const struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + phw = &soft_pcpu[cpu]->soft_hw[ri]; + pm = phw->phw_pmc; + + KASSERT(pm != NULL, + ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); + + PMCDBG(MDP,REA,1,"soft-read id=%d", ri); + + *v = soft_pcpu[cpu]->soft_values[ri]; + + return (0); +} + +static int +soft_write_pmc(int cpu, int ri, pmc_value_t v) +{ + struct pmc *pm; + const struct soft_descr *pd; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; + pd = &soft_pmcdesc[ri]; + + KASSERT(pm, + ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); + + PMCDBG(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); + + soft_pcpu[cpu]->soft_values[ri] = v; + + return (0); +} + +static int +soft_release_pmc(int cpu, int ri, struct pmc *pmc) +{ + struct pmc_hw *phw; + + (void) pmc; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + phw = &soft_pcpu[cpu]->soft_hw[ri]; + + KASSERT(phw->phw_pmc == NULL, + ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + /* + * Nothing to do. + */ + return (0); +} + +static int +soft_start_pmc(int cpu, int ri) +{ + struct pmc *pm; + struct soft_cpu *pc; + struct pmc_soft *ps; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + pc = soft_pcpu[cpu]; + pm = pc->soft_hw[ri].phw_pmc; + + KASSERT(pm, + ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); + + ps = pmc_soft_ev_acquire(pm->pm_event); + if (ps == NULL) + return (EINVAL); + atomic_add_int(&ps->ps_running, 1); + pmc_soft_ev_release(ps); + + return (0); +} + +static int +soft_stop_pmc(int cpu, int ri) +{ + struct pmc *pm; + struct soft_cpu *pc; + struct pmc_soft *ps; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < SOFT_NPMCS, + ("[soft,%d] illegal row-index %d", __LINE__, ri)); + + pc = soft_pcpu[cpu]; + pm = pc->soft_hw[ri].phw_pmc; + + KASSERT(pm, + ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); + + ps = pmc_soft_ev_acquire(pm->pm_event); + /* event unregistered ? */ + if (ps != NULL) { + atomic_subtract_int(&ps->ps_running, 1); + pmc_soft_ev_release(ps); + } + + return (0); +} + +int +pmc_soft_intr(struct pmckern_soft *ks) +{ + struct pmc *pm; + struct soft_cpu *pc; + int ri, processed, error, user_mode; + + KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), + ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); + + processed = 0; + pc = soft_pcpu[ks->pm_cpu]; + + for (ri = 0; ri < SOFT_NPMCS; ri++) { + + pm = pc->soft_hw[ri].phw_pmc; + if (pm == NULL || + pm->pm_state != PMC_STATE_RUNNING || + pm->pm_event != ks->pm_ev) { + continue; + } + + processed = 1; + pc->soft_values[ri]++; + if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { + user_mode = TRAPF_USERMODE(ks->pm_tf); + error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm, + ks->pm_tf, user_mode); + if (error) { + soft_stop_pmc(ks->pm_cpu, ri); + continue; + } + + if (user_mode) { + /* If in user mode setup AST to process + * callchain out of interrupt context. + */ + curthread->td_flags |= TDF_ASTPENDING; + } + } + } + + atomic_add_int(processed ? &pmc_stats.pm_intr_processed : + &pmc_stats.pm_intr_ignored, 1); + + return (processed); +} + +void +pmc_soft_initialize(struct pmc_mdep *md) +{ + struct pmc_classdep *pcd; + + /* Add SOFT PMCs. */ + soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, + M_ZERO|M_WAITOK); + + pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; + + pcd->pcd_caps = SOFT_CAPS; + pcd->pcd_class = PMC_CLASS_SOFT; + pcd->pcd_num = SOFT_NPMCS; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 64; + + pcd->pcd_allocate_pmc = soft_allocate_pmc; + pcd->pcd_config_pmc = soft_config_pmc; + pcd->pcd_describe = soft_describe; + pcd->pcd_get_config = soft_get_config; + pcd->pcd_get_msr = NULL; + pcd->pcd_pcpu_init = soft_pcpu_init; + pcd->pcd_pcpu_fini = soft_pcpu_fini; + pcd->pcd_read_pmc = soft_read_pmc; + pcd->pcd_write_pmc = soft_write_pmc; + pcd->pcd_release_pmc = soft_release_pmc; + pcd->pcd_start_pmc = soft_start_pmc; + pcd->pcd_stop_pmc = soft_stop_pmc; + + md->pmd_npmc += SOFT_NPMCS; +} + +void +pmc_soft_finalize(struct pmc_mdep *md) +{ +#ifdef INVARIANTS + int i, ncpus; + + ncpus = pmc_cpu_max(); + for (i = 0; i < ncpus; i++) + KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", + __LINE__, i)); + + KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == + PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); +#endif + free(soft_pcpu, M_PMC); + soft_pcpu = NULL; +} diff --git a/sys/dev/hwpmc/hwpmc_soft.h b/sys/dev/hwpmc/hwpmc_soft.h new file mode 100644 index 000000000000..f82baff01667 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_soft.h @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2012 Fabien Thomas + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_HWPMC_SOFT_H_ +#define _DEV_HWPMC_SOFT_H_ 1 + +#include + +#ifdef _KERNEL + +#define PMC_CLASS_INDEX_SOFT 0 +#define SOFT_NPMCS 16 + +/* + * Prototypes. + */ + +void pmc_soft_initialize(struct pmc_mdep *md); +void pmc_soft_finalize(struct pmc_mdep *md); +int pmc_soft_intr(struct pmckern_soft *ks); + +#endif /* _KERNEL */ +#endif /* _DEV_HWPMC_SOFT_H */ diff --git a/sys/dev/hwpmc/hwpmc_tsc.c b/sys/dev/hwpmc/hwpmc_tsc.c index 0b71a5be62fe..237b7a1a9686 100644 --- a/sys/dev/hwpmc/hwpmc_tsc.c +++ b/sys/dev/hwpmc/hwpmc_tsc.c @@ -190,9 +190,6 @@ tsc_pcpu_fini(struct pmc_mdep *md, int cpu) ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; - KASSERT(ri == 0 && ri < TSC_NPMCS, ("[tsc,%d] ri=%d", __LINE__, - ri)); - pc = pmc_pcpu[cpu]; pc->pc_hwpmcs[ri] = NULL; diff --git a/sys/dev/hwpmc/hwpmc_x86.c b/sys/dev/hwpmc/hwpmc_x86.c index 72ed518f1a2d..e7485a4cc59b 100644 --- a/sys/dev/hwpmc/hwpmc_x86.c +++ b/sys/dev/hwpmc/hwpmc_x86.c @@ -48,6 +48,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include "hwpmc_soft.h" + /* * Attempt to walk a user call stack using a too-simple algorithm. * In the general case we need unwind information associated with @@ -251,8 +253,11 @@ pmc_md_initialize() /* disallow sampling if we do not have an LAPIC */ if (md != NULL && !lapic_enable_pmc()) - for (i = 1; i < md->pmd_nclass; i++) + for (i = 0; i < md->pmd_nclass; i++) { + if (i == PMC_CLASS_INDEX_SOFT) + continue; md->pmd_classdep[i].pcd_caps &= ~PMC_CAP_INTERRUPT; + } return (md); } diff --git a/sys/dev/hwpmc/hwpmc_xscale.c b/sys/dev/hwpmc/hwpmc_xscale.c index 466f3b638fa3..9b73337732eb 100644 --- a/sys/dev/hwpmc/hwpmc_xscale.c +++ b/sys/dev/hwpmc/hwpmc_xscale.c @@ -638,11 +638,9 @@ pmc_xscale_initialize() M_WAITOK|M_ZERO); /* Just one class */ - pmc_mdep = malloc(sizeof(struct pmc_mdep) + sizeof(struct pmc_classdep), - M_PMC, M_WAITOK|M_ZERO); + pmc_mdep = pmc_mdep_alloc(1); pmc_mdep->pmd_cputype = PMC_CPU_INTEL_XSCALE; - pmc_mdep->pmd_nclass = 1; pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_XSCALE]; pcd->pcd_caps = XSCALE_PMC_CAPS; diff --git a/sys/dev/hwpmc/pmc_events.h b/sys/dev/hwpmc/pmc_events.h index 2397b171aed0..98ad10f30717 100644 --- a/sys/dev/hwpmc/pmc_events.h +++ b/sys/dev/hwpmc/pmc_events.h @@ -2599,6 +2599,15 @@ __PMC_EV_ALIAS("SQ_MISC.SPLIT_LOCK", IAP_EVENT_F4H_10H) #define PMC_EV_TSC_FIRST PMC_EV_TSC_TSC #define PMC_EV_TSC_LAST PMC_EV_TSC_TSC +/* + * Software events are dynamicaly defined. + */ + +#define PMC_EV_DYN_COUNT 0x1000 + +#define PMC_EV_SOFT_FIRST 0x20000 +#define PMC_EV_SOFT_LAST (PMC_EV_SOFT_FIRST + PMC_EV_DYN_COUNT - 1) + #define __PMC_EV_UCF() \ __PMC_EV(UCF, UCLOCK) @@ -3716,6 +3725,7 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H) * 0x11100 0x0100 INTEL Pentium Pro/P-II/P-III/Pentium-M events * 0x11200 0x00FF INTEL XScale events * 0x11300 0x00FF MIPS 24K events + * 0x20000 0x1000 Software events */ #define __PMC_EVENTS() \ __PMC_EV_BLOCK(TSC, 0x01000) \ @@ -3748,6 +3758,6 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H) __PMC_EV_PPC7450() \ #define PMC_EVENT_FIRST PMC_EV_TSC_TSC -#define PMC_EVENT_LAST PMC_EV_UCP_LAST +#define PMC_EVENT_LAST PMC_EV_SOFT_LAST #endif /* _DEV_HWPMC_PMC_EVENTS_H_ */ diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 005646dc17a3..bd6019a31aaa 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -73,6 +73,9 @@ __FBSDID("$FreeBSD$"); #include #ifdef HWPMC_HOOKS #include +PMC_SOFT_DEFINE( , , page_fault, all); +PMC_SOFT_DEFINE( , , page_fault, read); +PMC_SOFT_DEFINE( , , page_fault, write); #endif #include @@ -910,8 +913,20 @@ trap_pfault(frame, usermode, eva) */ rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL); } - if (rv == KERN_SUCCESS) + if (rv == KERN_SUCCESS) { +#ifdef HWPMC_HOOKS + if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { + PMC_SOFT_CALL_TF( , , page_fault, all, frame); + if (ftype == VM_PROT_READ) + PMC_SOFT_CALL_TF( , , page_fault, read, + frame); + else + PMC_SOFT_CALL_TF( , , page_fault, write, + frame); + } +#endif return (0); + } nogo: if (!usermode) { if (td->td_intr_nesting_level == 0 && diff --git a/sys/i386/include/pmc_mdep.h b/sys/i386/include/pmc_mdep.h index 9209a2b79edf..5ce791a3b993 100644 --- a/sys/i386/include/pmc_mdep.h +++ b/sys/i386/include/pmc_mdep.h @@ -66,16 +66,16 @@ struct pmc_mdep; * measurement architecture have PMCs of the following classes: TSC, * IAF, IAP, UCF and UCP. */ -#define PMC_MDEP_CLASS_INDEX_TSC 0 -#define PMC_MDEP_CLASS_INDEX_K7 1 -#define PMC_MDEP_CLASS_INDEX_K8 1 -#define PMC_MDEP_CLASS_INDEX_P4 1 -#define PMC_MDEP_CLASS_INDEX_P5 1 -#define PMC_MDEP_CLASS_INDEX_P6 1 -#define PMC_MDEP_CLASS_INDEX_IAP 1 -#define PMC_MDEP_CLASS_INDEX_IAF 2 -#define PMC_MDEP_CLASS_INDEX_UCP 3 -#define PMC_MDEP_CLASS_INDEX_UCF 4 +#define PMC_MDEP_CLASS_INDEX_TSC 1 +#define PMC_MDEP_CLASS_INDEX_K7 2 +#define PMC_MDEP_CLASS_INDEX_K8 2 +#define PMC_MDEP_CLASS_INDEX_P4 2 +#define PMC_MDEP_CLASS_INDEX_P5 2 +#define PMC_MDEP_CLASS_INDEX_P6 2 +#define PMC_MDEP_CLASS_INDEX_IAP 2 +#define PMC_MDEP_CLASS_INDEX_IAF 3 +#define PMC_MDEP_CLASS_INDEX_UCP 4 +#define PMC_MDEP_CLASS_INDEX_UCF 5 /* * Architecture specific extensions to structures. @@ -154,6 +154,15 @@ struct pmc_mdep; #define PMC_AT_FUNCTION_EPILOGUE_RET(I) \ (((I) & 0xFF) == 0xC3) /* ret */ +/* Build a fake kernel trapframe from current instruction pointer. */ +#define PMC_FAKE_TRAPFRAME(TF) \ + do { \ + (TF)->tf_cs = 0; (TF)->tf_eflags = 0; \ + __asm __volatile("movl %%ebp,%0" : "=r" ((TF)->tf_ebp)); \ + __asm __volatile("movl %%esp,%0" : "=r" ((TF)->tf_esp)); \ + __asm __volatile("call 1f \n\t1: pop %0" : "=r"((TF)->tf_eip)); \ + } while (0) + /* * Prototypes */ diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index fdc1302429dd..57dd632fe5da 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -74,6 +74,8 @@ __FBSDID("$FreeBSD$"); #ifdef HWPMC_HOOKS #include +PMC_SOFT_DEFINE( , , clock, hard); +PMC_SOFT_DEFINE( , , clock, stat); #endif #ifdef DEVICE_POLLING @@ -446,9 +448,11 @@ hardclock_cpu(int usermode) td->td_flags |= flags; thread_unlock(td); -#ifdef HWPMC_HOOKS +#ifdef HWPMC_HOOKS if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid))) PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL); + if (td->td_intr_frame != NULL) + PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame); #endif callout_tick(); } @@ -537,6 +541,8 @@ hardclock_cnt(int cnt, int usermode) #ifdef HWPMC_HOOKS if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid))) PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL); + if (td->td_intr_frame != NULL) + PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame); #endif callout_tick(); /* We are in charge to handle this tick duty. */ @@ -758,6 +764,10 @@ statclock_cnt(int cnt, int usermode) for ( ; cnt > 0; cnt--) sched_clock(td); thread_unlock(td); +#ifdef HWPMC_HOOKS + if (td->td_intr_frame != NULL) + PMC_SOFT_CALL_TF( , , clock, stat, td->td_intr_frame); +#endif } void diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c index 00f1b11fb4f3..24526b094a6f 100644 --- a/sys/kern/kern_lock.c +++ b/sys/kern/kern_lock.c @@ -28,6 +28,7 @@ #include "opt_adaptive_lockmgrs.h" #include "opt_ddb.h" +#include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" #include @@ -53,6 +54,11 @@ __FBSDID("$FreeBSD$"); #include #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DECLARE( , , lock, failed); +#endif + CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) == (LK_ADAPTIVE | LK_NOSHARE)); CTASSERT(LK_UNLOCKED == (LK_UNLOCKED & @@ -514,6 +520,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, break; continue; } +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&lk->lock_object, &contested, &waittime); @@ -744,6 +753,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&lk->lock_object, &contested, &waittime); @@ -1056,6 +1068,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk, } while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) { +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&lk->lock_object, &contested, &waittime); diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index bfb6547e5049..f718ca099023 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include "opt_adaptive_mutexes.h" #include "opt_ddb.h" #include "opt_global.h" +#include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" #include "opt_sched.h" @@ -76,6 +77,11 @@ __FBSDID("$FreeBSD$"); #define ADAPTIVE_MUTEXES #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DEFINE( , , lock, failed); +#endif + /* * Internal utility macros. */ @@ -364,6 +370,9 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, return; } +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); if (LOCK_LOG_TEST(&m->lock_object, opts)) @@ -529,6 +538,9 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); while (!_mtx_obtain_lock(m, tid)) { @@ -600,6 +612,9 @@ thread_lock_flags_(struct thread *td, int opts, const char *file, int line) m->mtx_recurse++; break; } +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); /* Give interrupts a chance while we spin. */ diff --git a/sys/kern/kern_pmc.c b/sys/kern/kern_pmc.c index 8d9c7c04a711..2b50be0343fc 100644 --- a/sys/kern/kern_pmc.c +++ b/sys/kern/kern_pmc.c @@ -34,10 +34,17 @@ __FBSDID("$FreeBSD$"); #include "opt_hwpmc_hooks.h" #include +#include +#include +#include +#include +#include +#include #include #include #include #include +#include #ifdef HWPMC_HOOKS FEATURE(hwpmc_hooks, "Kernel support for HW PMC"); @@ -46,6 +53,9 @@ FEATURE(hwpmc_hooks, "Kernel support for HW PMC"); #define PMC_KERNEL_VERSION 0 #endif +MALLOC_DECLARE(M_PMCHOOKS); +MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks"); + const int pmc_kernel_version = PMC_KERNEL_VERSION; /* Hook variable. */ @@ -74,6 +84,28 @@ volatile int pmc_ss_count; */ struct sx pmc_sx; +/* + * PMC Soft per cpu trapframe. + */ +struct trapframe pmc_tf[MAXCPU]; + +/* + * PMC Soft use a global table to store registered events. + */ + +SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); + +static int pmc_softevents = 16; +TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "softevents", &pmc_softevents); +SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_TUN|CTLFLAG_RD, + &pmc_softevents, 0, "maximum number of soft events"); + +struct mtx pmc_softs_mtx; +int pmc_softs_count; +struct pmc_soft **pmc_softs; + +MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN); + static void pmc_init_sx(void) { @@ -182,3 +214,132 @@ pmc_cpu_max_active(void) } #endif + +/* + * Cleanup event name: + * - remove duplicate '_' + * - all uppercase + */ +static void +pmc_soft_namecleanup(char *name) +{ + char *p, *q; + + p = q = name; + + for ( ; *p == '_' ; p++) + ; + for ( ; *p ; p++) { + if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0')) + continue; + else + *q++ = toupper(*p); + } + *q = '\0'; +} + +void +pmc_soft_ev_register(struct pmc_soft *ps) +{ + static int warned = 0; + int n; + + ps->ps_running = 0; + ps->ps_ev.pm_ev_code = 0; /* invalid */ + pmc_soft_namecleanup(ps->ps_ev.pm_ev_name); + + mtx_lock_spin(&pmc_softs_mtx); + + if (pmc_softs_count >= pmc_softevents) { + /* + * XXX Reusing events can enter a race condition where + * new allocated event will be used as an old one. + */ + for (n = 0; n < pmc_softevents; n++) + if (pmc_softs[n] == NULL) + break; + if (n == pmc_softevents) { + mtx_unlock_spin(&pmc_softs_mtx); + if (!warned) { + printf("hwpmc: too many soft events, " + "increase kern.hwpmc.softevents tunable\n"); + warned = 1; + } + return; + } + + ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n; + pmc_softs[n] = ps; + } else { + ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count; + pmc_softs[pmc_softs_count++] = ps; + } + + mtx_unlock_spin(&pmc_softs_mtx); +} + +void +pmc_soft_ev_deregister(struct pmc_soft *ps) +{ + + KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL")); + + mtx_lock_spin(&pmc_softs_mtx); + + if (ps->ps_ev.pm_ev_code != 0 && + (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) { + KASSERT(ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST && + ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST, + ("pmc_soft_deregister: invalid event value")); + pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL; + } + + mtx_unlock_spin(&pmc_softs_mtx); +} + +struct pmc_soft * +pmc_soft_ev_acquire(enum pmc_event ev) +{ + struct pmc_soft *ps; + + if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents) + return NULL; + + KASSERT(ev >= PMC_EV_SOFT_FIRST && + ev <= PMC_EV_SOFT_LAST, + ("event out of range")); + + mtx_lock_spin(&pmc_softs_mtx); + + ps = pmc_softs[ev - PMC_EV_SOFT_FIRST]; + if (ps == NULL) + mtx_unlock_spin(&pmc_softs_mtx); + + return ps; +} + +void +pmc_soft_ev_release(struct pmc_soft *ps) +{ + + mtx_unlock_spin(&pmc_softs_mtx); +} + +/* + * Initialise hwpmc. + */ +static void +init_hwpmc(void *dummy __unused) +{ + if (pmc_softevents <= 0 || + pmc_softevents > PMC_EV_DYN_COUNT) { + (void) printf("hwpmc: tunable \"softevents\"=%d out of " + "range.\n", pmc_softevents); + pmc_softevents = PMC_EV_DYN_COUNT; + } + pmc_softs = malloc(pmc_softevents * sizeof(struct pmc_soft *), M_PMCHOOKS, M_NOWAIT|M_ZERO); + KASSERT(pmc_softs != NULL, ("cannot allocate soft events table")); +} + +SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL); + diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index b571532b920b..c33704181287 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$"); #include "opt_ddb.h" +#include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" #include "opt_no_adaptive_rwlocks.h" @@ -55,6 +56,11 @@ __FBSDID("$FreeBSD$"); #define ADAPTIVE_RWLOCKS #endif +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DECLARE( , , lock, failed); +#endif + #ifdef ADAPTIVE_RWLOCKS static int rowner_retries = 10; static int rowner_loops = 10000; @@ -366,6 +372,9 @@ _rw_rlock(struct rwlock *rw, const char *file, int line) } continue; } +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&rw->lock_object, &contested, &waittime); @@ -686,6 +695,9 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) while (!_rw_write_lock(rw, tid)) { #ifdef KDTRACE_HOOKS spin_cnt++; +#endif +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); #endif lock_profile_obtain_lock_failed(&rw->lock_object, &contested, &waittime); diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c index 9cb999589b1c..bcd7acd5d8d7 100644 --- a/sys/kern/kern_sx.c +++ b/sys/kern/kern_sx.c @@ -37,6 +37,7 @@ */ #include "opt_ddb.h" +#include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" #include "opt_no_adaptive_sx.h" @@ -67,6 +68,11 @@ __FBSDID("$FreeBSD$"); CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE); +#ifdef HWPMC_HOOKS +#include +PMC_SOFT_DECLARE( , , lock, failed); +#endif + /* Handy macros for sleep queues. */ #define SQ_EXCLUSIVE_QUEUE 0 #define SQ_SHARED_QUEUE 1 @@ -523,6 +529,9 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file, while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) { #ifdef KDTRACE_HOOKS spin_cnt++; +#endif +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); #endif lock_profile_obtain_lock_failed(&sx->lock_object, &contested, &waittime); @@ -811,6 +820,9 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line) } continue; } +#ifdef HWPMC_HOOKS + PMC_SOFT_CALL( , , lock, failed); +#endif lock_profile_obtain_lock_failed(&sx->lock_object, &contested, &waittime); diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index ba8b66f6589f..ab839069e67f 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include "opt_capsicum.h" +#include "opt_hwpmc_hooks.h" #include "opt_ktrace.h" #include "opt_kdtrace.h" #include "opt_sched.h" @@ -86,6 +87,10 @@ __FBSDID("$FreeBSD$"); #include #endif +#ifdef HWPMC_HOOKS +#include +#endif + #include /* @@ -192,6 +197,11 @@ ast(struct trapframe *framep) td->td_profil_ticks = 0; td->td_pflags &= ~TDP_OWEUPC; } +#ifdef HWPMC_HOOKS + /* Handle Software PMC callchain capture. */ + if (PMC_IS_PENDING_CALLCHAIN(td)) + PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, (void *) framep); +#endif if (flags & TDF_ALRMPEND) { PROC_LOCK(p); kern_psignal(p, SIGVTALRM); diff --git a/sys/mips/include/pmc_mdep.h b/sys/mips/include/pmc_mdep.h index ee7cc983c62b..1386866b30e8 100644 --- a/sys/mips/include/pmc_mdep.h +++ b/sys/mips/include/pmc_mdep.h @@ -8,7 +8,7 @@ #ifndef _MACHINE_PMC_MDEP_H_ #define _MACHINE_PMC_MDEP_H_ -#define PMC_MDEP_CLASS_INDEX_MIPS 0 +#define PMC_MDEP_CLASS_INDEX_MIPS 1 union pmc_md_op_pmcallocate { uint64_t __pad[4]; diff --git a/sys/modules/hwpmc/Makefile b/sys/modules/hwpmc/Makefile index b834f174dbff..8948805ceeca 100644 --- a/sys/modules/hwpmc/Makefile +++ b/sys/modules/hwpmc/Makefile @@ -6,7 +6,7 @@ KMOD= hwpmc -SRCS= hwpmc_mod.c hwpmc_logging.c vnode_if.h +SRCS= hwpmc_mod.c hwpmc_logging.c hwpmc_soft.c vnode_if.h .if ${MACHINE_CPUARCH} == "amd64" SRCS+= hwpmc_amd.c hwpmc_core.c hwpmc_intel.c hwpmc_piv.c hwpmc_tsc.c diff --git a/sys/powerpc/include/pmc_mdep.h b/sys/powerpc/include/pmc_mdep.h index 37531f2c0b36..345636866103 100644 --- a/sys/powerpc/include/pmc_mdep.h +++ b/sys/powerpc/include/pmc_mdep.h @@ -7,7 +7,8 @@ #ifndef _MACHINE_PMC_MDEP_H_ #define _MACHINE_PMC_MDEP_H_ -#define PMC_MDEP_CLASS_INDEX_PPC7450 0 +#define PMC_MDEP_CLASS_INDEX_PPC7450 1 + union pmc_md_op_pmcallocate { uint64_t __pad[4]; }; diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h index 49454ec89f35..7c67b9b98ade 100644 --- a/sys/sys/pmc.h +++ b/sys/sys/pmc.h @@ -39,8 +39,8 @@ #include #define PMC_MODULE_NAME "hwpmc" -#define PMC_NAME_MAX 16 /* HW counter name size */ -#define PMC_CLASS_MAX 6 /* max #classes of PMCs per-system */ +#define PMC_NAME_MAX 64 /* HW counter name size */ +#define PMC_CLASS_MAX 8 /* max #classes of PMCs per-system */ /* * Kernel<->userland API version number [MMmmpppp] @@ -83,14 +83,15 @@ __PMC_CPU(INTEL_CORE, 0x87, "Intel Core Solo/Duo") \ __PMC_CPU(INTEL_CORE2, 0x88, "Intel Core2") \ __PMC_CPU(INTEL_CORE2EXTREME, 0x89, "Intel Core2 Extreme") \ - __PMC_CPU(INTEL_ATOM, 0x8A, "Intel Atom") \ - __PMC_CPU(INTEL_COREI7, 0x8B, "Intel Core i7") \ - __PMC_CPU(INTEL_WESTMERE, 0x8C, "Intel Westmere") \ - __PMC_CPU(INTEL_SANDYBRIDGE, 0x8D, "Intel Sandy Bridge") \ - __PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \ - __PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \ - __PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \ - __PMC_CPU(PPC_7450, 0x300, "PowerPC MPC7450") + __PMC_CPU(INTEL_ATOM, 0x8A, "Intel Atom") \ + __PMC_CPU(INTEL_COREI7, 0x8B, "Intel Core i7") \ + __PMC_CPU(INTEL_WESTMERE, 0x8C, "Intel Westmere") \ + __PMC_CPU(INTEL_SANDYBRIDGE, 0x8D, "Intel Sandy Bridge") \ + __PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \ + __PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \ + __PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \ + __PMC_CPU(PPC_7450, 0x300, "PowerPC MPC7450") \ + __PMC_CPU(GENERIC, 0x400, "Generic") enum pmc_cputype { #undef __PMC_CPU @@ -99,7 +100,7 @@ enum pmc_cputype { }; #define PMC_CPU_FIRST PMC_CPU_AMD_K7 -#define PMC_CPU_LAST PMC_CPU_PPC_7450 +#define PMC_CPU_LAST PMC_CPU_GENERIC /* * Classes of PMCs @@ -119,7 +120,8 @@ enum pmc_cputype { __PMC_CLASS(XSCALE) /* Intel XScale counters */ \ __PMC_CLASS(MIPS24K) /* MIPS 24K */ \ __PMC_CLASS(OCTEON) /* Cavium Octeon */ \ - __PMC_CLASS(PPC7450) /* Motorola MPC7450 class */ + __PMC_CLASS(PPC7450) /* Motorola MPC7450 class */ \ + __PMC_CLASS(SOFT) /* Software events */ enum pmc_class { #undef __PMC_CLASS @@ -128,7 +130,7 @@ enum pmc_class { }; #define PMC_CLASS_FIRST PMC_CLASS_TSC -#define PMC_CLASS_LAST PMC_CLASS_PPC7450 +#define PMC_CLASS_LAST PMC_CLASS_SOFT /* * A PMC can be in the following states: @@ -308,7 +310,8 @@ enum pmc_event { __PMC_OP(PMCSTART, "Start a PMC") \ __PMC_OP(PMCSTOP, "Stop a PMC") \ __PMC_OP(WRITELOG, "Write a cookie to the log file") \ - __PMC_OP(CLOSELOG, "Close log file") + __PMC_OP(CLOSELOG, "Close log file") \ + __PMC_OP(GETDYNEVENTINFO, "Get dynamic events list") enum pmc_ops { @@ -338,6 +341,7 @@ enum pmc_ops { #define PMC_F_ATTACH_DONE 0x00040000 /*attached at least once */ #define PMC_CALLCHAIN_DEPTH_MAX 32 + #define PMC_CC_F_USERSPACE 0x01 /*userspace callchain*/ /* @@ -486,6 +490,7 @@ struct pmc_op_getpmcinfo { * Retrieve system CPU information. */ + struct pmc_classinfo { enum pmc_class pm_class; /* class id */ uint32_t pm_caps; /* counter capabilities */ @@ -562,6 +567,23 @@ struct pmc_op_getmsr { pmc_id_t pm_pmcid; /* allocated pmc id */ }; +/* + * OP GETDYNEVENTINFO + * + * Retrieve a PMC dynamic class events list. + */ + +struct pmc_dyn_event_descr { + char pm_ev_name[PMC_NAME_MAX]; + enum pmc_event pm_ev_code; +}; + +struct pmc_op_getdyneventinfo { + enum pmc_class pm_class; + unsigned int pm_nevent; + struct pmc_dyn_event_descr pm_events[PMC_EV_DYN_COUNT]; +}; + #ifdef _KERNEL #include @@ -572,8 +594,8 @@ struct pmc_op_getmsr { #define PMC_HASH_SIZE 16 #define PMC_MTXPOOL_SIZE 32 #define PMC_LOG_BUFFER_SIZE 4 -#define PMC_NLOGBUFFERS 16 -#define PMC_NSAMPLES 32 +#define PMC_NLOGBUFFERS 64 +#define PMC_NSAMPLES 512 #define PMC_CALLCHAIN_DEPTH 8 #define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "." @@ -829,8 +851,8 @@ struct pmc_sample { uintptr_t *ps_pc; /* (const) callchain start */ }; -#define PMC_SAMPLE_FREE ((uint16_t) 0) -#define PMC_SAMPLE_INUSE ((uint16_t) 0xFFFF) +#define PMC_SAMPLE_FREE ((uint16_t) 0) +#define PMC_SAMPLE_INUSE ((uint16_t) 0xFFFF) struct pmc_samplebuffer { struct pmc_sample * volatile ps_read; /* read pointer */ @@ -850,7 +872,7 @@ struct pmc_samplebuffer { struct pmc_cpu { uint32_t pc_state; /* physical cpu number + flags */ - struct pmc_samplebuffer *pc_sb; /* space for samples */ + struct pmc_samplebuffer *pc_sb[2]; /* space for samples */ struct pmc_hw *pc_hwpmcs[]; /* 'npmc' pointers */ }; @@ -958,7 +980,7 @@ extern struct pmc_cpu **pmc_pcpu; /* driver statistics */ extern struct pmc_op_getdriverstats pmc_stats; -#if defined(DEBUG) && DEBUG +#if defined(DEBUG) /* debug flags, major flag groups */ struct pmc_debugflags { @@ -1061,11 +1083,13 @@ MALLOC_DECLARE(M_PMC); struct pmc_mdep *pmc_md_initialize(void); /* MD init function */ void pmc_md_finalize(struct pmc_mdep *_md); /* MD fini function */ int pmc_getrowdisp(int _ri); -int pmc_process_interrupt(int _cpu, struct pmc *_pm, +int pmc_process_interrupt(int _cpu, int _soft, struct pmc *_pm, struct trapframe *_tf, int _inuserspace); int pmc_save_kernel_callchain(uintptr_t *_cc, int _maxsamples, struct trapframe *_tf); int pmc_save_user_callchain(uintptr_t *_cc, int _maxsamples, struct trapframe *_tf); +struct pmc_mdep *pmc_mdep_alloc(int nclasses); +void pmc_mdep_free(struct pmc_mdep *md); #endif /* _KERNEL */ #endif /* _SYS_PMC_H_ */ diff --git a/sys/sys/pmckern.h b/sys/sys/pmckern.h index 796c4cadab77..44f1916a9641 100644 --- a/sys/sys/pmckern.h +++ b/sys/sys/pmckern.h @@ -38,10 +38,14 @@ #define _SYS_PMCKERN_H_ #include +#include #include #include #include #include +#include + +#include #define PMC_FN_PROCESS_EXEC 1 #define PMC_FN_CSW_IN 2 @@ -52,6 +56,11 @@ #define PMC_FN_MMAP 7 #define PMC_FN_MUNMAP 8 #define PMC_FN_USER_CALLCHAIN 9 +#define PMC_FN_USER_CALLCHAIN_SOFT 10 +#define PMC_FN_SOFT_SAMPLING 11 + +#define PMC_HR 0 /* Hardware ring buffer */ +#define PMC_SR 1 /* Software ring buffer */ struct pmckern_procexec { int pm_credentialschanged; @@ -68,6 +77,79 @@ struct pmckern_map_out { size_t pm_size; /* size of unmapped region */ }; +struct pmckern_soft { + enum pmc_event pm_ev; + int pm_cpu; + struct trapframe *pm_tf; +}; + +/* + * Soft PMC. + */ + +#define PMC_SOFT_DEFINE(prov, mod, func, name) \ + struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ + { 0, { #prov "_" #mod "_" #func "." #name, 0 } }; \ + SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ + SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ + &pmc_##prov##_##mod##_##func##_##name ); \ + SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \ + SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ + &pmc_##prov##_##mod##_##func##_##name ) + +#define PMC_SOFT_DECLARE(prov, mod, func, name) \ + extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name + +/* + * PMC_SOFT_CALL can be used anywhere in the kernel. + * Require md defined PMC_FAKE_TRAPFRAME. + */ +#ifdef PMC_FAKE_TRAPFRAME +#define PMC_SOFT_CALL(pr, mo, fu, na) \ +do { \ + if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \ + struct pmckern_soft ks; \ + register_t intr; \ + intr = intr_disable(); \ + PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \ + ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ + ks.pm_cpu = PCPU_GET(cpuid); \ + ks.pm_tf = &pmc_tf[curcpu]; \ + PMC_CALL_HOOK_UNLOCKED(curthread, \ + PMC_FN_SOFT_SAMPLING, (void *) &ks); \ + intr_restore(intr); \ + } \ +} while (0) +#else +#define PMC_SOFT_CALL(pr, mo, fu, na) \ +do { \ +} while (0) +#endif + +/* + * PMC_SOFT_CALL_TF need to be used carefully. + * Userland capture will be done during AST processing. + */ +#define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \ +do { \ + if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \ + struct pmckern_soft ks; \ + register_t intr; \ + intr = intr_disable(); \ + ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ + ks.pm_cpu = PCPU_GET(cpuid); \ + ks.pm_tf = tf; \ + PMC_CALL_HOOK_UNLOCKED(curthread, \ + PMC_FN_SOFT_SAMPLING, (void *) &ks); \ + intr_restore(intr); \ + } \ +} while (0) + +struct pmc_soft { + int ps_running; + struct pmc_dyn_event_descr ps_ev; +}; + /* hook */ extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); extern int (*pmc_intr)(int _cpu, struct trapframe *_frame); @@ -84,6 +166,9 @@ extern volatile int pmc_ss_count; /* kernel version number */ extern const int pmc_kernel_version; +/* PMC soft per cpu trapframe */ +extern struct trapframe pmc_tf[MAXCPU]; + /* Hook invocation; for use within the kernel */ #define PMC_CALL_HOOK(t, cmd, arg) \ do { \ @@ -119,6 +204,10 @@ do { \ (__predict_false(atomic_load_acq_int(&(p)->p_flag) & \ P_HWPMC)) +/* Check if a thread have pending user capture. */ +#define PMC_IS_PENDING_CALLCHAIN(p) \ + (__predict_false((p)->td_pflags & TDP_CALLCHAIN)) + #define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) /* Check if a CPU has recorded samples. */ @@ -137,4 +226,12 @@ unsigned int pmc_cpu_max(void); int pmc_cpu_max_active(void); #endif +/* + * Soft events functions. + */ +void pmc_soft_ev_register(struct pmc_soft *ps); +void pmc_soft_ev_deregister(struct pmc_soft *ps); +struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); +void pmc_soft_ev_release(struct pmc_soft *ps); + #endif /* _SYS_PMCKERN_H_ */ diff --git a/sys/sys/pmclog.h b/sys/sys/pmclog.h index f2b6480a52c2..69062f7fe895 100644 --- a/sys/sys/pmclog.h +++ b/sys/sys/pmclog.h @@ -60,7 +60,13 @@ enum pmclog_type { */ PMCLOG_TYPE_MAP_IN, PMCLOG_TYPE_MAP_OUT, - PMCLOG_TYPE_CALLCHAIN + PMCLOG_TYPE_CALLCHAIN, + /* + * V3 ABI + * + * New variant of PMCLOG_TYPE_PMCALLOCATE for dynamic event. + */ + PMCLOG_TYPE_PMCALLOCATEDYN }; /* @@ -204,6 +210,14 @@ struct pmclog_userdata { uint32_t pl_userdata; } __packed; +struct pmclog_pmcallocatedyn { + PMCLOG_ENTRY_HEADER + uint32_t pl_pmcid; + uint32_t pl_event; + uint32_t pl_flags; + char pl_evname[PMC_NAME_MAX]; +} __packed; + union pmclog_entry { /* only used to size scratch areas */ struct pmclog_callchain pl_cc; struct pmclog_closelog pl_cl; @@ -213,6 +227,7 @@ union pmclog_entry { /* only used to size scratch areas */ struct pmclog_map_out pl_mo; struct pmclog_pcsample pl_s; struct pmclog_pmcallocate pl_a; + struct pmclog_pmcallocatedyn pl_ad; struct pmclog_pmcattach pl_t; struct pmclog_pmcdetach pl_d; struct pmclog_proccsw pl_c; diff --git a/usr.sbin/pmcstat/pmcstat_log.c b/usr.sbin/pmcstat/pmcstat_log.c index 1a9ddc616ec6..2ac52da58f8a 100644 --- a/usr.sbin/pmcstat/pmcstat_log.c +++ b/usr.sbin/pmcstat/pmcstat_log.c @@ -1491,6 +1491,15 @@ pmcstat_analyze_log(void) pmcstat_string_intern(ev.pl_u.pl_a.pl_evname)); break; + case PMCLOG_TYPE_PMCALLOCATEDYN: + /* + * Record the association pmc id between this + * PMC and its name. + */ + pmcstat_pmcid_add(ev.pl_u.pl_ad.pl_pmcid, + pmcstat_string_intern(ev.pl_u.pl_ad.pl_evname)); + break; + case PMCLOG_TYPE_PROCEXEC: /* @@ -1641,6 +1650,12 @@ pmcstat_print_log(void) ev.pl_u.pl_a.pl_evname, ev.pl_u.pl_a.pl_flags); break; + case PMCLOG_TYPE_PMCALLOCATEDYN: + PMCSTAT_PRINT_ENTRY("allocatedyn","0x%x \"%s\" 0x%x", + ev.pl_u.pl_ad.pl_pmcid, + ev.pl_u.pl_ad.pl_evname, + ev.pl_u.pl_ad.pl_flags); + break; case PMCLOG_TYPE_PMCATTACH: PMCSTAT_PRINT_ENTRY("attach","0x%x %d \"%s\"", ev.pl_u.pl_t.pl_pmcid,