d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add software PMC support.

Browse Source 
New kernel events can be added at various location for sampling or counting.
This will for example allow easy system profiling whatever the processor is
with known tools like pmcstat(8).

Simultaneous usage of software PMC and hardware PMC is possible, for example
looking at the lock acquire failure, page fault while sampling on
instructions.

Sponsored by: NETASQ
MFC after:	1 month
This commit is contained in:
Fabien Thomas 2012-03-28 20:58:30 +00:00
parent 1faacf5d09
commit f5f9340b98
61 changed files with 1518 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/libpmc/Makefile
View File

@ -20,6 +20,7 @@ MAN+= pmc_read.3
MAN+= pmc_set.3
MAN+= pmc_start.3
MAN+= pmclog.3
MAN+= pmc.soft.3
# PMC-dependent manual pages
.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"

118
lib/libpmc/libpmc.c
View File

@ -77,11 +77,12 @@ static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
static int xscale_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
struct pmc_op_pmcallocate *_pmc_config);
#endif
#if defined(__mips__)
static int mips_allocate_pmc(enum pmc_event _pe, char* ctrspec,
struct pmc_op_pmcallocate *_pmc_config);
#endif /* __mips__ */
static int soft_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
struct pmc_op_pmcallocate *_pmc_config);
#if defined(__powerpc__)
static int ppc7450_allocate_pmc(enum pmc_event _pe, char* ctrspec,
@ -156,6 +157,8 @@ PMC_CLASSDEP_TABLE(octeon, OCTEON);
PMC_CLASSDEP_TABLE(ucf, UCF);
PMC_CLASSDEP_TABLE(ppc7450, PPC7450);
static struct pmc_event_descr soft_event_table[PMC_EV_DYN_COUNT];
#undef __PMC_EV_ALIAS
#define __PMC_EV_ALIAS(N,CODE) { N, PMC_EV_##CODE },
@ -215,21 +218,22 @@ static const struct pmc_event_descr westmereuc_event_table[] =
PMC_CLASS_##C, __VA_ARGS__ \
}
PMC_MDEP_TABLE(atom, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
PMC_MDEP_TABLE(core, IAP, PMC_CLASS_TSC);
PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
PMC_MDEP_TABLE(sandybridge, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
PMC_MDEP_TABLE(westmere, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
PMC_MDEP_TABLE(k7, K7, PMC_CLASS_TSC);
PMC_MDEP_TABLE(k8, K8, PMC_CLASS_TSC);
PMC_MDEP_TABLE(p4, P4, PMC_CLASS_TSC);
PMC_MDEP_TABLE(p5, P5, PMC_CLASS_TSC);
PMC_MDEP_TABLE(p6, P6, PMC_CLASS_TSC);
PMC_MDEP_TABLE(xscale, XSCALE, PMC_CLASS_XSCALE);
PMC_MDEP_TABLE(mips24k, MIPS24K, PMC_CLASS_MIPS24K);
PMC_MDEP_TABLE(octeon, OCTEON, PMC_CLASS_OCTEON);
PMC_MDEP_TABLE(ppc7450, PPC7450, PMC_CLASS_PPC7450);
PMC_MDEP_TABLE(atom, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC);
PMC_MDEP_TABLE(core, IAP, PMC_CLASS_SOFT, PMC_CLASS_TSC);
PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC);
PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
PMC_MDEP_TABLE(sandybridge, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
PMC_MDEP_TABLE(westmere, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
PMC_MDEP_TABLE(k7, K7, PMC_CLASS_SOFT, PMC_CLASS_TSC);
PMC_MDEP_TABLE(k8, K8, PMC_CLASS_SOFT, PMC_CLASS_TSC);
PMC_MDEP_TABLE(p4, P4, PMC_CLASS_SOFT, PMC_CLASS_TSC);
PMC_MDEP_TABLE(p5, P5, PMC_CLASS_SOFT, PMC_CLASS_TSC);
PMC_MDEP_TABLE(p6, P6, PMC_CLASS_SOFT, PMC_CLASS_TSC);
PMC_MDEP_TABLE(xscale, XSCALE, PMC_CLASS_SOFT, PMC_CLASS_XSCALE);
PMC_MDEP_TABLE(mips24k, MIPS24K, PMC_CLASS_SOFT, PMC_CLASS_MIPS24K);
PMC_MDEP_TABLE(octeon, OCTEON, PMC_CLASS_SOFT, PMC_CLASS_OCTEON);
PMC_MDEP_TABLE(ppc7450, PPC7450, PMC_CLASS_SOFT, PMC_CLASS_PPC7450);
PMC_MDEP_TABLE(generic, SOFT, PMC_CLASS_SOFT);
static const struct pmc_event_descr tsc_event_table[] =
{
@ -279,16 +283,24 @@ PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc);
#if defined(__XSCALE__)
PMC_CLASS_TABLE_DESC(xscale, XSCALE, xscale, xscale);
#endif
#if defined(__mips__)
PMC_CLASS_TABLE_DESC(mips24k, MIPS24K, mips24k, mips);
PMC_CLASS_TABLE_DESC(octeon, OCTEON, octeon, mips);
#endif /* __mips__ */
#if defined(__powerpc__)
PMC_CLASS_TABLE_DESC(ppc7450, PPC7450, ppc7450, ppc7450);
#endif
static struct pmc_class_descr soft_class_table_descr =
{
.pm_evc_name = "SOFT-",
.pm_evc_name_size = sizeof("SOFT-") - 1,
.pm_evc_class = PMC_CLASS_SOFT,
.pm_evc_event_table = NULL,
.pm_evc_event_table_size = 0,
.pm_evc_allocate_pmc = soft_allocate_pmc
};
#undef PMC_CLASS_TABLE_DESC
static const struct pmc_class_descr **pmc_class_table;
@ -343,9 +355,12 @@ static const char * pmc_state_names[] = {
__PMC_STATES()
};
static int pmc_syscall = -1; /* filled in by pmc_init() */
static struct pmc_cpuinfo cpu_info; /* filled in by pmc_init() */
/*
* Filled in by pmc_init().
*/
static int pmc_syscall = -1;
static struct pmc_cpuinfo cpu_info;
static struct pmc_op_getdyneventinfo soft_event_info;
/* Event masks for events */
struct pmc_masks {
@ -2179,6 +2194,25 @@ tsc_allocate_pmc(enum pmc_event pe, char *ctrspec,
}
#endif
static struct pmc_event_alias generic_aliases[] = {
EV_ALIAS("instructions", "SOFT-CLOCK.HARD"),
EV_ALIAS(NULL, NULL)
};
static int
soft_allocate_pmc(enum pmc_event pe, char *ctrspec,
struct pmc_op_pmcallocate *pmc_config)
{
(void)ctrspec;
(void)pmc_config;
if (pe < PMC_EV_SOFT_FIRST || pe > PMC_EV_SOFT_LAST)
return (-1);
pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
return (0);
}
#if defined(__XSCALE__)
static struct pmc_event_alias xscale_aliases[] = {
@ -2663,6 +2697,10 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
ev = ppc7450_event_table;
count = PMC_EVENT_TABLE_SIZE(ppc7450);
break;
case PMC_CLASS_SOFT:
ev = soft_event_table;
count = soft_event_info.pm_nevent;
break;
default:
errno = EINVAL;
return (-1);
@ -2676,6 +2714,7 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
for (;count--; ev++, names++)
*names = ev->pm_ev_name;
return (0);
}
@ -2779,12 +2818,35 @@ pmc_init(void)
for (n = 0; n < PMC_CLASS_TABLE_SIZE; n++)
pmc_class_table[n] = NULL;
/*
* Get soft events list.
*/
soft_event_info.pm_class = PMC_CLASS_SOFT;
if (PMC_CALL(GETDYNEVENTINFO, &soft_event_info) < 0)
return (pmc_syscall = -1);
/* Map soft events to static list. */
for (n = 0; n < soft_event_info.pm_nevent; n++) {
soft_event_table[n].pm_ev_name =
soft_event_info.pm_events[n].pm_ev_name;
soft_event_table[n].pm_ev_code =
soft_event_info.pm_events[n].pm_ev_code;
}
soft_class_table_descr.pm_evc_event_table_size = \
soft_event_info.pm_nevent;
soft_class_table_descr.pm_evc_event_table = \
soft_event_table;
/*
* Fill in the class table.
*/
n = 0;
/* Fill soft events information. */
pmc_class_table[n++] = &soft_class_table_descr;
#if defined(__amd64__) || defined(__i386__)
pmc_class_table[n++] = &tsc_class_table_descr;
if (cpu_info.pm_cputype != PMC_CPU_GENERIC)
pmc_class_table[n++] = &tsc_class_table_descr;
/*
* Check if this CPU has fixed function counters.
@ -2867,6 +2929,9 @@ pmc_init(void)
pmc_class_table[n] = &p4_class_table_descr;
break;
#endif
case PMC_CPU_GENERIC:
PMC_MDEP_INIT(generic);
break;
#if defined(__XSCALE__)
case PMC_CPU_INTEL_XSCALE:
PMC_MDEP_INIT(xscale);
@ -3035,18 +3100,19 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
evfence = xscale_event_table + PMC_EVENT_TABLE_SIZE(xscale);
} else if (pe >= PMC_EV_MIPS24K_FIRST && pe <= PMC_EV_MIPS24K_LAST) {
ev = mips24k_event_table;
evfence = mips24k_event_table + PMC_EVENT_TABLE_SIZE(mips24k
);
evfence = mips24k_event_table + PMC_EVENT_TABLE_SIZE(mips24k);
} else if (pe >= PMC_EV_OCTEON_FIRST && pe <= PMC_EV_OCTEON_LAST) {
ev = octeon_event_table;
evfence = octeon_event_table + PMC_EVENT_TABLE_SIZE(octeon);
} else if (pe >= PMC_EV_PPC7450_FIRST && pe <= PMC_EV_PPC7450_LAST) {
ev = ppc7450_event_table;
evfence = ppc7450_event_table + PMC_EVENT_TABLE_SIZE(ppc7450
);
evfence = ppc7450_event_table + PMC_EVENT_TABLE_SIZE(ppc7450);
} else if (pe == PMC_EV_TSC_TSC) {
ev = tsc_event_table;
evfence = tsc_event_table + PMC_EVENT_TABLE_SIZE(tsc);
} else if (pe >= PMC_EV_SOFT_FIRST && pe <= PMC_EV_SOFT_LAST) {
ev = soft_event_table;
evfence = soft_event_table + soft_event_info.pm_nevent;
}
for (; ev != evfence; ev++)

3
lib/libpmc/pmc.3
View File

@ -223,6 +223,8 @@ and
CPUs.
.It Li PMC_CLASS_TSC
The timestamp counter on i386 and amd64 architecture CPUs.
.It Li PMC_CLASS_SOFT
Software events.
.El
.Ss PMC Capabilities
Capabilities of performance monitoring hardware are denoted using
@ -525,6 +527,7 @@ API is
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4 ,

1
lib/libpmc/pmc.atom.3
View File

@ -1176,6 +1176,7 @@ and the underlying hardware events used on these CPUs.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.core.3
View File

@ -792,6 +792,7 @@ may not count some transitions.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4

1
lib/libpmc/pmc.core2.3
View File

@ -1107,6 +1107,7 @@ and the underlying hardware events used.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.corei7.3
View File

@ -1559,6 +1559,7 @@ Counts number of segment register loads.
.Xr pmc.corei7uc 3 ,
.Xr pmc.westmere 3 ,
.Xr pmc.westmereuc 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.corei7uc.3
View File

@ -863,6 +863,7 @@ refreshed or needs to go into a power down mode.
.Xr pmc.corei7 3 ,
.Xr pmc.westmere 3 ,
.Xr pmc.westmereuc 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.iaf.3
View File

@ -132,6 +132,7 @@ CPU, use the event specifier
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.k7.3
View File

@ -249,6 +249,7 @@ and the underlying hardware events used.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4

1
lib/libpmc/pmc.k8.3
View File

@ -783,6 +783,7 @@ and the underlying hardware events used.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4

1
lib/libpmc/pmc.mips24k.3
View File

@ -392,6 +392,7 @@ and the underlying hardware events used.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.octeon.3
View File

@ -232,6 +232,7 @@ and the underlying hardware events used.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.p4.3
View File

@ -1208,6 +1208,7 @@ and the underlying hardware events used.
.Xr pmc.k8 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4

1
lib/libpmc/pmc.p5.3
View File

@ -444,6 +444,7 @@ and the underlying hardware events used.
.Xr pmc.k8 3 ,
.Xr pmc.p4 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4

1
lib/libpmc/pmc.p6.3
View File

@ -1010,6 +1010,7 @@ and the underlying hardware events used.
.Xr pmc.k8 3 ,
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4

1
lib/libpmc/pmc.sandybridge.3
View File

@ -907,6 +907,7 @@ Split locks in SQ.
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.sandybridgeuc 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc.ucf 3 ,
.Xr pmc.westmere 3 ,

1
lib/libpmc/pmc.sandybridgeuc.3
View File

@ -208,6 +208,7 @@ Counts the number of core-outgoing entries in the coherent tracker queue.
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.sandybridge 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc.ucf 3 ,
.Xr pmc.westmere 3 ,

104
lib/libpmc/pmc.soft.3 Normal file
View File

@ -0,0 +1,104 @@
.\" Copyright (c) 2012 Fabien Thomas. All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
.\" $FreeBSD$
.\"
.Dd March 28, 2012
.Os
.Dt PMC.SOFT 3
.Sh NAME
.Nm pmc.soft
.Nd measurements using software based events
.Sh LIBRARY
.Lb libpmc
.Sh SYNOPSIS
.In pmc.h
.Sh DESCRIPTION
Software events are used to collect various source of software events.
.Ss PMC Features
16 sampling counters using software events based on various sources.
These PMCs support the following capabilities:
.Bl -column "PMC_CAP_INTERRUPT" "Support"
.It Em Capability Ta Em Support
.It PMC_CAP_CASCADE Ta \&No
.It PMC_CAP_EDGE Ta \&No
.It PMC_CAP_INTERRUPT Ta Yes
.It PMC_CAP_INVERT Ta \&No
.It PMC_CAP_READ Ta Yes
.It PMC_CAP_PRECISE Ta \&No
.It PMC_CAP_SYSTEM Ta Yes
.It PMC_CAP_TAGGING Ta \&No
.It PMC_CAP_THRESHOLD Ta \&No
.It PMC_CAP_USER Ta Yes
.It PMC_CAP_WRITE Ta Yes
.El
.Ss Event Qualifiers
There is no supported event qualifier.
.Pp
The event specifiers supported by software are:
.Bl -tag -width indent
.It Li CLOCK.HARD
Hard clock ticks.
.It Li CLOCK.STAT
Stat clock ticks.
.It Li LOCK.FAILED
Lock acquisition failed.
.It Li PAGE_FAULT.ALL
All page fault type.
.It Li PAGE_FAULT.READ
Read page fault.
.It Li PAGE_FAULT.WRITE
Write page fault.
.El
.Sh SEE ALSO
.Xr pmc 3 ,
.Xr pmc.atom 3 ,
.Xr pmc.core 3 ,
.Xr pmc.iaf 3 ,
.Xr pmc.ucf 3 ,
.Xr pmc.k7 3 ,
.Xr pmc.k8 3 ,
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.corei7 3 ,
.Xr pmc.corei7uc 3 ,
.Xr pmc.westmereuc 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4
.Sh HISTORY
The
.Nm pmc
library first appeared in
.Fx 6.0 .
.Sh AUTHORS
The
.Lb libpmc
library was written by
.An "Joseph Koshy"
.Aq jkoshy@FreeBSD.org .
Software PMC was written by
.An "Fabien Thomas"
.Aq fabient@FreeBSD.org .

1
lib/libpmc/pmc.tsc.3
View File

@ -68,6 +68,7 @@ maps to the TSC.
.Xr pmc.p4 3 ,
.Xr pmc.p5 3 ,
.Xr pmc.p6 3 ,
.Xr pmc.soft 3 ,
.Xr pmclog 3 ,
.Xr hwpmc 4
.Sh HISTORY

1
lib/libpmc/pmc.ucf.3
View File

@ -96,6 +96,7 @@ offset C0H under device number 0 and Function 0.
.Xr pmc.corei7uc 3 ,
.Xr pmc.westmere 3 ,
.Xr pmc.westmereuc 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.westmere.3
View File

@ -1381,6 +1381,7 @@ Counts number of SID integer 64 bit shift or move operations.
.Xr pmc.corei7 3 ,
.Xr pmc.corei7uc 3 ,
.Xr pmc.westmereuc 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.westmereuc.3
View File

@ -1066,6 +1066,7 @@ disabled.
.Xr pmc.corei7 3 ,
.Xr pmc.corei7uc 3 ,
.Xr pmc.westmere 3 ,
.Xr pmc.soft 3 ,
.Xr pmc.tsc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,

1
lib/libpmc/pmc.xscale.3
View File

@ -134,6 +134,7 @@ and the underlying hardware events used.
.Xr pmc 3 ,
.Xr pmc_cpuinfo 3 ,
.Xr pmclog 3 ,
.Xr pmc.soft 3 ,
.Xr hwpmc 4
.Sh HISTORY
The

6
lib/libpmc/pmclog.c
View File

@ -369,6 +369,12 @@ pmclog_get_event(void *cookie, char **data, ssize_t *len,
== NULL)
goto error;
break;
case PMCLOG_TYPE_PMCALLOCATEDYN:
PMCLOG_READ32(le,ev->pl_u.pl_ad.pl_pmcid);
PMCLOG_READ32(le,ev->pl_u.pl_ad.pl_event);
PMCLOG_READ32(le,ev->pl_u.pl_ad.pl_flags);
PMCLOG_READSTRING(le,ev->pl_u.pl_ad.pl_evname,PMC_NAME_MAX);
break;
case PMCLOG_TYPE_PMCATTACH:
PMCLOG_GET_PATHLEN(pathlen,evlen,pmclog_pmcattach);
PMCLOG_READ32(le,ev->pl_u.pl_t.pl_pmcid);

8
lib/libpmc/pmclog.h
View File

@ -88,6 +88,13 @@ struct pmclog_ev_pmcallocate {
pmc_id_t pl_pmcid;
};
struct pmclog_ev_pmcallocatedyn {
uint32_t pl_event;
char pl_evname[PMC_NAME_MAX];
uint32_t pl_flags;
pmc_id_t pl_pmcid;
};
struct pmclog_ev_pmcattach {
pmc_id_t pl_pmcid;
pid_t pl_pid;
@ -146,6 +153,7 @@ struct pmclog_ev {
struct pmclog_ev_map_out pl_mo;
struct pmclog_ev_pcsample pl_s;
struct pmclog_ev_pmcallocate pl_a;
struct pmclog_ev_pmcallocatedyn pl_ad;
struct pmclog_ev_pmcattach pl_t;
struct pmclog_ev_pmcdetach pl_d;
struct pmclog_ev_proccsw pl_c;

17
sys/amd64/amd64/trap.c
View File

@ -71,6 +71,9 @@ __FBSDID("$FreeBSD$");
#include <sys/vmmeter.h>
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DEFINE( , , page_fault, all);
PMC_SOFT_DEFINE( , , page_fault, read);
PMC_SOFT_DEFINE( , , page_fault, write);
#endif
#include <vm/vm.h>
@ -743,8 +746,20 @@ trap_pfault(frame, usermode)
*/
rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
}
if (rv == KERN_SUCCESS)
if (rv == KERN_SUCCESS) {
#ifdef HWPMC_HOOKS
if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
PMC_SOFT_CALL_TF( , , page_fault, all, frame);
if (ftype == VM_PROT_READ)
PMC_SOFT_CALL_TF( , , page_fault, read,
frame);
else
PMC_SOFT_CALL_TF( , , page_fault, write,
frame);
}
#endif
return (0);
}
nogo:
if (!usermode) {
if (td->td_intr_nesting_level == 0 &&

23
sys/amd64/include/pmc_mdep.h
View File

@ -50,13 +50,13 @@ struct pmc_mdep;
* measurement architecture have PMCs of the following classes: TSC,
* IAF, IAP, UCF and UCP.
*/
#define PMC_MDEP_CLASS_INDEX_TSC 0
#define PMC_MDEP_CLASS_INDEX_K8 1
#define PMC_MDEP_CLASS_INDEX_P4 1
#define PMC_MDEP_CLASS_INDEX_IAP 1
#define PMC_MDEP_CLASS_INDEX_IAF 2
#define PMC_MDEP_CLASS_INDEX_UCP 3
#define PMC_MDEP_CLASS_INDEX_UCF 4
#define PMC_MDEP_CLASS_INDEX_TSC 1
#define PMC_MDEP_CLASS_INDEX_K8 2
#define PMC_MDEP_CLASS_INDEX_P4 2
#define PMC_MDEP_CLASS_INDEX_IAP 2
#define PMC_MDEP_CLASS_INDEX_IAF 3
#define PMC_MDEP_CLASS_INDEX_UCP 4
#define PMC_MDEP_CLASS_INDEX_UCF 5
/*
* On the amd64 platform we support the following PMCs.
@ -119,6 +119,15 @@ union pmc_md_pmc {
#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS)
/* Build a fake kernel trapframe from current instruction pointer. */
#define PMC_FAKE_TRAPFRAME(TF) \
do { \
(TF)->tf_cs = 0; (TF)->tf_rflags = 0; \
__asm __volatile("movq %%rbp,%0" : "=r" ((TF)->tf_rbp)); \
__asm __volatile("movq %%rsp,%0" : "=r" ((TF)->tf_rsp)); \
__asm __volatile("call 1f \n\t1: pop %0" : "=r"((TF)->tf_rip)); \
} while (0)
/*
* Prototypes
*/

2
sys/arm/include/pmc_mdep.h
View File

@ -29,7 +29,7 @@
#ifndef _MACHINE_PMC_MDEP_H_
#define _MACHINE_PMC_MDEP_H_
#define PMC_MDEP_CLASS_INDEX_XSCALE 0
#define PMC_MDEP_CLASS_INDEX_XSCALE 1
/*
* On the ARM platform we support the following PMCs.
*

1
sys/conf/files
View File

@ -1260,6 +1260,7 @@ dev/hme/if_hme_sbus.c optional hme sbus
dev/hptiop/hptiop.c optional hptiop scbus
dev/hwpmc/hwpmc_logging.c optional hwpmc
dev/hwpmc/hwpmc_mod.c optional hwpmc
dev/hwpmc/hwpmc_soft.c optional hwpmc
dev/ichsmb/ichsmb.c optional ichsmb
dev/ichsmb/ichsmb_pci.c optional ichsmb pci
dev/ida/ida.c optional ida

10
sys/dev/hwpmc/hwpmc_amd.c
View File

@ -687,7 +687,8 @@ amd_intr(int cpu, struct trapframe *tf)
wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v));
/* Restart the counter if logging succeeded. */
error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf));
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error == 0)
wrmsr(evsel, config | AMD_PMC_ENABLE);
}
@ -874,7 +875,7 @@ amd_pcpu_fini(struct pmc_mdep *md, int cpu)
struct pmc_mdep *
pmc_amd_initialize(void)
{
int classindex, error, i, nclasses, ncpus;
int classindex, error, i, ncpus;
struct pmc_classdep *pcd;
enum pmc_cputype cputype;
struct pmc_mdep *pmc_mdep;
@ -926,12 +927,9 @@ pmc_amd_initialize(void)
* These processors have two classes of PMCs: the TSC and
* programmable PMCs.
*/
nclasses = 2;
pmc_mdep = malloc(sizeof(struct pmc_mdep) + nclasses * sizeof (struct pmc_classdep),
M_PMC, M_WAITOK|M_ZERO);
pmc_mdep = pmc_mdep_alloc(2);
pmc_mdep->pmd_cputype = cputype;
pmc_mdep->pmd_nclass = nclasses;
ncpus = pmc_cpu_max();

6
sys/dev/hwpmc/hwpmc_core.c
View File

@ -2239,7 +2239,7 @@ core_intr(int cpu, struct trapframe *tf)
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, pm, tf,
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
v = pm->pm_sc.pm_reloadcount;
@ -2326,7 +2326,7 @@ core2_intr(int cpu, struct trapframe *tf)
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
error = pmc_process_interrupt(cpu, pm, tf,
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error)
intrenable &= ~flag;
@ -2354,7 +2354,7 @@ core2_intr(int cpu, struct trapframe *tf)
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
error = pmc_process_interrupt(cpu, pm, tf,
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error)
intrenable &= ~flag;

6
sys/dev/hwpmc/hwpmc_intel.c
View File

@ -162,12 +162,10 @@ pmc_intel_initialize(void)
return (NULL);
}
pmc_mdep = malloc(sizeof(struct pmc_mdep) + nclasses *
sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
/* Allocate base class and initialize machine dependent struct */
pmc_mdep = pmc_mdep_alloc(nclasses);
pmc_mdep->pmd_cputype = cputype;
pmc_mdep->pmd_nclass = nclasses;
pmc_mdep->pmd_switch_in = intel_switch_in;
pmc_mdep->pmd_switch_out = intel_switch_out;

28
sys/dev/hwpmc/hwpmc_logging.c
View File

@ -129,6 +129,7 @@ static struct mtx pmc_kthread_mtx; /* sleep lock */
/* Emit a string. Caution: does NOT update _le, so needs to be last */
#define PMCLOG_EMITSTRING(S,L) do { bcopy((S), _le, (L)); } while (0)
#define PMCLOG_EMITNULLSTRING(L) do { bzero(_le, (L)); } while (0)
#define PMCLOG_DESPATCH(PO) \
pmclog_release((PO)); \
@ -835,16 +836,33 @@ void
pmclog_process_pmcallocate(struct pmc *pm)
{
struct pmc_owner *po;
struct pmc_soft *ps;
po = pm->pm_owner;
PMCDBG(LOG,ALL,1, "pm=%p", pm);
PMCLOG_RESERVE(po, PMCALLOCATE, sizeof(struct pmclog_pmcallocate));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
PMCLOG_DESPATCH(po);
if (PMC_TO_CLASS(pm) == PMC_CLASS_SOFT) {
PMCLOG_RESERVE(po, PMCALLOCATEDYN,
sizeof(struct pmclog_pmcallocatedyn));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
ps = pmc_soft_ev_acquire(pm->pm_event);
if (ps != NULL)
PMCLOG_EMITSTRING(ps->ps_ev.pm_ev_name,PMC_NAME_MAX);
else
PMCLOG_EMITNULLSTRING(PMC_NAME_MAX);
pmc_soft_ev_release(ps);
PMCLOG_DESPATCH(po);
} else {
PMCLOG_RESERVE(po, PMCALLOCATE,
sizeof(struct pmclog_pmcallocate));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
PMCLOG_DESPATCH(po);
}
}
void

2
sys/dev/hwpmc/hwpmc_mips.c
View File

@ -287,7 +287,7 @@ mips_pmc_intr(int cpu, struct trapframe *tf)
retval = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, pm, tf,
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error) {
/* Clear/disable the relevant counter */

251
sys/dev/hwpmc/hwpmc_mod.c
View File

@ -70,6 +70,8 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include "hwpmc_soft.h"
/*
* Types
*/
@ -182,7 +184,7 @@ static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
int cpu);
static int pmc_can_attach(struct pmc *pm, struct proc *p);
static void pmc_capture_user_callchain(int cpu, struct trapframe *tf);
static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf);
static void pmc_cleanup(void);
static int pmc_detach_process(struct proc *p, struct pmc *pm);
static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
@ -206,7 +208,7 @@ static void pmc_process_csw_out(struct thread *td);
static void pmc_process_exit(void *arg, struct proc *p);
static void pmc_process_fork(void *arg, struct proc *p1,
struct proc *p2, int n);
static void pmc_process_samples(int cpu);
static void pmc_process_samples(int cpu, int soft);
static void pmc_release_pmc_descriptor(struct pmc *pmc);
static void pmc_remove_owner(struct pmc_owner *po);
static void pmc_remove_process_descriptor(struct pmc_process *pp);
@ -218,12 +220,16 @@ static int pmc_stop(struct pmc *pm);
static int pmc_syscall_handler(struct thread *td, void *syscall_args);
static void pmc_unlink_target_process(struct pmc *pmc,
struct pmc_process *pp);
static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp);
static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp);
static struct pmc_mdep *pmc_generic_cpu_initialize(void);
static void pmc_generic_cpu_finalize(struct pmc_mdep *md);
/*
* Kernel tunables and sysctl(8) interface.
*/
SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
SYSCTL_DECL(_kern_hwpmc);
static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth);
@ -1833,7 +1839,9 @@ const char *pmc_hooknames[] = {
"KLDUNLOAD",
"MMAP",
"MUNMAP",
"CALLCHAIN"
"CALLCHAIN-NMI",
"CALLCHAIN-SOFT",
"SOFTSAMPLING"
};
#endif
@ -1992,7 +2000,8 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
* lose the interrupt sample.
*/
CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask);
pmc_process_samples(PCPU_GET(cpuid));
pmc_process_samples(PCPU_GET(cpuid), PMC_HR);
pmc_process_samples(PCPU_GET(cpuid), PMC_SR);
break;
@ -2022,11 +2031,30 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
*/
KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));
pmc_capture_user_callchain(PCPU_GET(cpuid),
pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR,
(struct trapframe *) arg);
td->td_pflags &= ~TDP_CALLCHAIN;
break;
case PMC_FN_USER_CALLCHAIN_SOFT:
/*
* Record a call chain.
*/
KASSERT(td == curthread, ("[pmc,%d] td != curthread",
__LINE__));
pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR,
(struct trapframe *) arg);
td->td_pflags &= ~TDP_CALLCHAIN;
break;
case PMC_FN_SOFT_SAMPLING:
/*
* Call soft PMC sampling intr.
*/
pmc_soft_intr((struct pmckern_soft *) arg);
break;
default:
#ifdef DEBUG
KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
@ -2221,18 +2249,17 @@ pmc_destroy_pmc_descriptor(struct pmc *pm)
static void
pmc_wait_for_pmc_idle(struct pmc *pm)
{
#ifdef DEBUG
#ifdef DEBUG
volatile int maxloop;
maxloop = 100 * pmc_cpu_max();
#endif
/*
* Loop (with a forced context switch) till the PMC's runcount
* comes down to zero.
*/
while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
#ifdef DEBUG
#ifdef DEBUG
maxloop--;
KASSERT(maxloop > 0,
("[pmc,%d] (ri%d, rc%d) waiting too long for "
@ -2972,6 +2999,53 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
}
break;
/*
* Retrieve soft events list.
*/
case PMC_OP_GETDYNEVENTINFO:
{
enum pmc_class cl;
enum pmc_event ev;
struct pmc_op_getdyneventinfo *gei;
struct pmc_dyn_event_descr dev;
struct pmc_soft *ps;
uint32_t nevent;
sx_assert(&pmc_sx, SX_LOCKED);
gei = (struct pmc_op_getdyneventinfo *) arg;
if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0)
break;
/* Only SOFT class is dynamic. */
if (cl != PMC_CLASS_SOFT) {
error = EINVAL;
break;
}
nevent = 0;
for (ev = PMC_EV_SOFT_FIRST; ev <= PMC_EV_SOFT_LAST; ev++) {
ps = pmc_soft_ev_acquire(ev);
if (ps == NULL)
continue;
bcopy(&ps->ps_ev, &dev, sizeof(dev));
pmc_soft_ev_release(ps);
error = copyout(&dev,
&gei->pm_events[nevent],
sizeof(struct pmc_dyn_event_descr));
if (error != 0)
break;
nevent++;
}
if (error != 0)
break;
error = copyout(&nevent, &gei->pm_nevent,
sizeof(nevent));
}
break;
/*
* Get module statistics
@ -4022,7 +4096,7 @@ pmc_post_callchain_callback(void)
*/
int
pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf,
int inuserspace)
{
int error, callchaindepth;
@ -4035,7 +4109,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
/*
* Allocate space for a sample buffer.
*/
psb = pmc_pcpu[cpu]->pc_sb;
psb = pmc_pcpu[cpu]->pc_sb[ring];
ps = psb->ps_write;
if (ps->ps_nsamples) { /* in use, reader hasn't caught up */
@ -4061,6 +4135,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
pm->pm_runcount));
atomic_add_rel_int(&pm->pm_runcount, 1); /* hold onto PMC */
ps->ps_pmc = pm;
if ((td = curthread) && td->td_proc)
ps->ps_pid = td->td_proc->p_pid;
@ -4080,11 +4155,11 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
* Kernel stack traversals can be done immediately,
* while we defer to an AST for user space traversals.
*/
if (!inuserspace)
if (!inuserspace) {
callchaindepth =
pmc_save_kernel_callchain(ps->ps_pc,
callchaindepth, tf);
else {
} else {
pmc_post_callchain_callback();
callchaindepth = PMC_SAMPLE_INUSE;
}
@ -4113,7 +4188,7 @@ pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
*/
static void
pmc_capture_user_callchain(int cpu, struct trapframe *tf)
pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf)
{
int i;
struct pmc *pm;
@ -4124,9 +4199,7 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf)
int ncallchains;
#endif
sched_unpin(); /* Can migrate safely now. */
psb = pmc_pcpu[cpu]->pc_sb;
psb = pmc_pcpu[cpu]->pc_sb[ring];
td = curthread;
KASSERT(td->td_pflags & TDP_CALLCHAIN,
@ -4172,23 +4245,25 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf)
#ifdef INVARIANTS
ncallchains++;
#endif
}
KASSERT(ncallchains > 0,
("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
cpu));
KASSERT(td->td_pinned > 0,
("[pmc,%d] invalid td_pinned value", __LINE__));
sched_unpin(); /* Can migrate safely now. */
return;
}
/*
* Process saved PC samples.
*/
static void
pmc_process_samples(int cpu)
pmc_process_samples(int cpu, int ring)
{
struct pmc *pm;
int adjri, n;
@ -4202,18 +4277,13 @@ pmc_process_samples(int cpu)
("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
PCPU_GET(cpuid), cpu));
psb = pmc_pcpu[cpu]->pc_sb;
psb = pmc_pcpu[cpu]->pc_sb[ring];
for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
ps = psb->ps_read;
if (ps->ps_nsamples == PMC_SAMPLE_FREE)
break;
if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
/* Need a rescan at a later time. */
CPU_SET_ATOMIC(cpu, &pmc_cpumask);
break;
}
pm = ps->ps_pmc;
@ -4231,6 +4301,13 @@ pmc_process_samples(int cpu)
if (pm->pm_state != PMC_STATE_RUNNING)
goto entrydone;
/* If there is a pending AST wait for completion */
if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
/* Need a rescan at a later time. */
CPU_SET_ATOMIC(cpu, &pmc_cpumask);
break;
}
PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
pm, ps->ps_nsamples, ps->ps_flags,
(int) (psb->ps_write - psb->ps_samples),
@ -4256,11 +4333,10 @@ pmc_process_samples(int cpu)
* or a system-wide sampling PMC. Dispatch a log
* entry to the PMC's owner process.
*/
pmclog_process_callchain(pm, ps);
entrydone:
ps->ps_nsamples = 0; /* mark entry as free */
ps->ps_nsamples = 0; /* mark entry as free */
atomic_subtract_rel_int(&pm->pm_runcount, 1);
/* increment read pointer, modulo sample size */
@ -4584,6 +4660,76 @@ static const char *pmc_name_of_pmcclass[] = {
__PMC_CLASSES()
};
/*
* Base class initializer: allocate structure and set default classes.
*/
struct pmc_mdep *
pmc_mdep_alloc(int nclasses)
{
struct pmc_mdep *md;
int n;
/* SOFT + md classes */
n = 1 + nclasses;
md = malloc(sizeof(struct pmc_mdep) + n *
sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO);
if (md != NULL) {
md->pmd_nclass = n;
/* Add base class. */
pmc_soft_initialize(md);
}
return md;
}
void
pmc_mdep_free(struct pmc_mdep *md)
{
pmc_soft_finalize(md);
free(md, M_PMC);
}
static int
generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc; (void) pp;
return (0);
}
static int
generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
{
(void) pc; (void) pp;
return (0);
}
static struct pmc_mdep *
pmc_generic_cpu_initialize(void)
{
struct pmc_mdep *md;
md = pmc_mdep_alloc(0);
md->pmd_cputype = PMC_CPU_GENERIC;
md->pmd_pcpu_init = NULL;
md->pmd_pcpu_fini = NULL;
md->pmd_switch_in = generic_switch_in;
md->pmd_switch_out = generic_switch_out;
return (md);
}
static void
pmc_generic_cpu_finalize(struct pmc_mdep *md)
{
(void) md;
}
static int
pmc_initialize(void)
{
@ -4643,9 +4789,12 @@ pmc_initialize(void)
}
md = pmc_md_initialize();
if (md == NULL)
return (ENOSYS);
if (md == NULL) {
/* Default to generic CPU. */
md = pmc_generic_cpu_initialize();
if (md == NULL)
return (ENOSYS);
}
KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1,
("[pmc,%d] no classes or pmcs", __LINE__));
@ -4717,7 +4866,25 @@ pmc_initialize(void)
ps->ps_pc = sb->ps_callchains +
(n * pmc_callchaindepth);
pmc_pcpu[cpu]->pc_sb = sb;
pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb;
sb = malloc(sizeof(struct pmc_samplebuffer) +
pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
M_WAITOK|M_ZERO);
sb->ps_read = sb->ps_write = sb->ps_samples;
sb->ps_fence = sb->ps_samples + pmc_nsamples;
KASSERT(pmc_pcpu[cpu] != NULL,
("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples *
sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO);
for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
ps->ps_pc = sb->ps_callchains +
(n * pmc_callchaindepth);
pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb;
}
/* allocate space for the row disposition array */
@ -4887,9 +5054,12 @@ pmc_cleanup(void)
md->pmd_pcpu_fini(md, cpu);
}
pmc_md_finalize(md);
if (md->pmd_cputype == PMC_CPU_GENERIC)
pmc_generic_cpu_finalize(md);
else
pmc_md_finalize(md);
free(md, M_PMC);
pmc_mdep_free(md);
md = NULL;
pmc_restore_cpu_binding(&pb);
}
@ -4898,11 +5068,16 @@ pmc_cleanup(void)
for (cpu = 0; cpu < maxcpu; cpu++) {
if (!pmc_cpu_is_active(cpu))
continue;
KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL,
("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__,
cpu));
free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb, M_PMC);
KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL,
("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__,
cpu));
free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC);
free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC);
free(pmc_pcpu[cpu], M_PMC);
}

4
sys/dev/hwpmc/hwpmc_piv.c
View File

@ -1463,7 +1463,7 @@ p4_intr(int cpu, struct trapframe *tf)
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
(void) pmc_process_interrupt(cpu, pm, tf,
(void) pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
continue;
}
@ -1513,7 +1513,7 @@ p4_intr(int cpu, struct trapframe *tf)
* Process the interrupt. Re-enable the PMC if
* processing was successful.
*/
error = pmc_process_interrupt(cpu, pm, tf,
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
/*

3
sys/dev/hwpmc/hwpmc_powerpc.c
View File

@ -690,7 +690,8 @@ powerpc_intr(int cpu, struct trapframe *tf)
powerpc_pmcn_write(i, v);
/* Restart the counter if logging succeeded. */
error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf));
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
mtspr(SPR_MMCR0, config);
if (error != 0)
powerpc_stop_pmc(cpu, i);

2
sys/dev/hwpmc/hwpmc_ppro.c
View File

@ -704,7 +704,7 @@ p6_intr(int cpu, struct trapframe *tf)
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, pm, tf,
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error)
P6_MARK_STOPPED(pc,ri);

485
sys/dev/hwpmc/hwpmc_soft.c Normal file
View File

@ -0,0 +1,485 @@
/*-
* Copyright (c) 2012 Fabien Thomas
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/systm.h>
#include <sys/mutex.h>
#include <machine/cpu.h>
#include <machine/cpufunc.h>
#include "hwpmc_soft.h"
/*
* Software PMC support.
*/
#define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
PMC_CAP_USER | PMC_CAP_SYSTEM)
struct soft_descr {
struct pmc_descr pm_descr; /* "base class" */
};
static struct soft_descr soft_pmcdesc[SOFT_NPMCS] =
{
#define SOFT_PMCDESCR(N) \
{ \
.pm_descr = \
{ \
.pd_name = #N, \
.pd_class = PMC_CLASS_SOFT, \
.pd_caps = SOFT_CAPS, \
.pd_width = 64 \
}, \
}
SOFT_PMCDESCR(SOFT0),
SOFT_PMCDESCR(SOFT1),
SOFT_PMCDESCR(SOFT2),
SOFT_PMCDESCR(SOFT3),
SOFT_PMCDESCR(SOFT4),
SOFT_PMCDESCR(SOFT5),
SOFT_PMCDESCR(SOFT6),
SOFT_PMCDESCR(SOFT7),
SOFT_PMCDESCR(SOFT8),
SOFT_PMCDESCR(SOFT9),
SOFT_PMCDESCR(SOFT10),
SOFT_PMCDESCR(SOFT11),
SOFT_PMCDESCR(SOFT12),
SOFT_PMCDESCR(SOFT13),
SOFT_PMCDESCR(SOFT14),
SOFT_PMCDESCR(SOFT15)
};
/*
* Per-CPU data structure.
*/
struct soft_cpu {
struct pmc_hw soft_hw[SOFT_NPMCS];
pmc_value_t soft_values[SOFT_NPMCS];
};
static struct soft_cpu **soft_pcpu;
static int
soft_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event ev;
struct pmc_soft *ps;
(void) cpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
if (a->pm_class != PMC_CLASS_SOFT)
return (EINVAL);
if ((pm->pm_caps & SOFT_CAPS) == 0)
return (EINVAL);
if ((pm->pm_caps & ~SOFT_CAPS) != 0)
return (EPERM);
ev = pm->pm_event;
if (ev < PMC_EV_SOFT_FIRST || ev > PMC_EV_SOFT_LAST)
return (EINVAL);
/* Check if event is registered. */
ps = pmc_soft_ev_acquire(ev);
if (ps == NULL)
return (EINVAL);
pmc_soft_ev_release(ps);
return (0);
}
static int
soft_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__,
pm, phw->phw_pmc));
phw->phw_pmc = pm;
return (0);
}
static int
soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
{
int error;
size_t copied;
const struct soft_descr *pd;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
pd = &soft_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = phw->phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
soft_get_config(int cpu, int ri, struct pmc **ppm)
{
(void) ri;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
*ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
return (0);
}
static int
soft_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int ri;
struct pmc_cpu *pc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__));
free(soft_pcpu[cpu], M_PMC);
soft_pcpu[cpu] = NULL;
ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] ri=%d", __LINE__, ri));
pc = pmc_pcpu[cpu];
pc->pc_hwpmcs[ri] = NULL;
return (0);
}
static int
soft_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, n;
struct pmc_cpu *pc;
struct soft_cpu *soft_pc;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__));
KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu",
__LINE__));
soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO);
if (soft_pc == NULL)
return (ENOMEM);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu));
soft_pcpu[cpu] = soft_pc;
phw = soft_pc->soft_hw;
first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
for (n = 0; n < SOFT_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
return (0);
}
static int
soft_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
const struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
pm = phw->phw_pmc;
KASSERT(pm != NULL,
("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri));
PMCDBG(MDP,REA,1,"soft-read id=%d", ri);
*v = soft_pcpu[cpu]->soft_values[ri];
return (0);
}
static int
soft_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
const struct soft_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
pd = &soft_pmcdesc[ri];
KASSERT(pm,
("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
PMCDBG(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v);
soft_pcpu[cpu]->soft_values[ri] = v;
return (0);
}
static int
soft_release_pmc(int cpu, int ri, struct pmc *pmc)
{
struct pmc_hw *phw;
(void) pmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
phw = &soft_pcpu[cpu]->soft_hw[ri];
KASSERT(phw->phw_pmc == NULL,
("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
/*
* Nothing to do.
*/
return (0);
}
static int
soft_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct soft_cpu *pc;
struct pmc_soft *ps;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
pc = soft_pcpu[cpu];
pm = pc->soft_hw[ri].phw_pmc;
KASSERT(pm,
("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
ps = pmc_soft_ev_acquire(pm->pm_event);
if (ps == NULL)
return (EINVAL);
atomic_add_int(&ps->ps_running, 1);
pmc_soft_ev_release(ps);
return (0);
}
static int
soft_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct soft_cpu *pc;
struct pmc_soft *ps;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < SOFT_NPMCS,
("[soft,%d] illegal row-index %d", __LINE__, ri));
pc = soft_pcpu[cpu];
pm = pc->soft_hw[ri].phw_pmc;
KASSERT(pm,
("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
ps = pmc_soft_ev_acquire(pm->pm_event);
/* event unregistered ? */
if (ps != NULL) {
atomic_subtract_int(&ps->ps_running, 1);
pmc_soft_ev_release(ps);
}
return (0);
}
int
pmc_soft_intr(struct pmckern_soft *ks)
{
struct pmc *pm;
struct soft_cpu *pc;
int ri, processed, error, user_mode;
KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(),
("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu));
processed = 0;
pc = soft_pcpu[ks->pm_cpu];
for (ri = 0; ri < SOFT_NPMCS; ri++) {
pm = pc->soft_hw[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
pm->pm_event != ks->pm_ev) {
continue;
}
processed = 1;
pc->soft_values[ri]++;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
user_mode = TRAPF_USERMODE(ks->pm_tf);
error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm,
ks->pm_tf, user_mode);
if (error) {
soft_stop_pmc(ks->pm_cpu, ri);
continue;
}
if (user_mode) {
/* If in user mode setup AST to process
* callchain out of interrupt context.
*/
curthread->td_flags |= TDF_ASTPENDING;
}
}
}
atomic_add_int(processed ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (processed);
}
void
pmc_soft_initialize(struct pmc_mdep *md)
{
struct pmc_classdep *pcd;
/* Add SOFT PMCs. */
soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC,
M_ZERO|M_WAITOK);
pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT];
pcd->pcd_caps = SOFT_CAPS;
pcd->pcd_class = PMC_CLASS_SOFT;
pcd->pcd_num = SOFT_NPMCS;
pcd->pcd_ri = md->pmd_npmc;
pcd->pcd_width = 64;
pcd->pcd_allocate_pmc = soft_allocate_pmc;
pcd->pcd_config_pmc = soft_config_pmc;
pcd->pcd_describe = soft_describe;
pcd->pcd_get_config = soft_get_config;
pcd->pcd_get_msr = NULL;
pcd->pcd_pcpu_init = soft_pcpu_init;
pcd->pcd_pcpu_fini = soft_pcpu_fini;
pcd->pcd_read_pmc = soft_read_pmc;
pcd->pcd_write_pmc = soft_write_pmc;
pcd->pcd_release_pmc = soft_release_pmc;
pcd->pcd_start_pmc = soft_start_pmc;
pcd->pcd_stop_pmc = soft_stop_pmc;
md->pmd_npmc += SOFT_NPMCS;
}
void
pmc_soft_finalize(struct pmc_mdep *md)
{
#ifdef INVARIANTS
int i, ncpus;
ncpus = pmc_cpu_max();
for (i = 0; i < ncpus; i++)
KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d",
__LINE__, i));
KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class ==
PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__));
#endif
free(soft_pcpu, M_PMC);
soft_pcpu = NULL;
}

48
sys/dev/hwpmc/hwpmc_soft.h Normal file
View File

@ -0,0 +1,48 @@
/*-
* Copyright (c) 2012 Fabien Thomas
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _DEV_HWPMC_SOFT_H_
#define _DEV_HWPMC_SOFT_H_ 1
#include <sys/pmckern.h>
#ifdef _KERNEL
#define PMC_CLASS_INDEX_SOFT 0
#define SOFT_NPMCS 16
/*
* Prototypes.
*/
void pmc_soft_initialize(struct pmc_mdep *md);
void pmc_soft_finalize(struct pmc_mdep *md);
int pmc_soft_intr(struct pmckern_soft *ks);
#endif /* _KERNEL */
#endif /* _DEV_HWPMC_SOFT_H */

3
sys/dev/hwpmc/hwpmc_tsc.c
View File

@ -190,9 +190,6 @@ tsc_pcpu_fini(struct pmc_mdep *md, int cpu)
ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri;
KASSERT(ri == 0 && ri < TSC_NPMCS, ("[tsc,%d] ri=%d", __LINE__,
ri));
pc = pmc_pcpu[cpu];
pc->pc_hwpmcs[ri] = NULL;

7
sys/dev/hwpmc/hwpmc_x86.c
View File

@ -48,6 +48,8 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_param.h>
#include <vm/pmap.h>
#include "hwpmc_soft.h"
/*
* Attempt to walk a user call stack using a too-simple algorithm.
* In the general case we need unwind information associated with
@ -251,8 +253,11 @@ pmc_md_initialize()
/* disallow sampling if we do not have an LAPIC */
if (md != NULL && !lapic_enable_pmc())
for (i = 1; i < md->pmd_nclass; i++)
for (i = 0; i < md->pmd_nclass; i++) {
if (i == PMC_CLASS_INDEX_SOFT)
continue;
md->pmd_classdep[i].pcd_caps &= ~PMC_CAP_INTERRUPT;
}
return (md);
}

4
sys/dev/hwpmc/hwpmc_xscale.c
View File

@ -638,11 +638,9 @@ pmc_xscale_initialize()
M_WAITOK|M_ZERO);
/* Just one class */
pmc_mdep = malloc(sizeof(struct pmc_mdep) + sizeof(struct pmc_classdep),
M_PMC, M_WAITOK|M_ZERO);
pmc_mdep = pmc_mdep_alloc(1);
pmc_mdep->pmd_cputype = PMC_CPU_INTEL_XSCALE;
pmc_mdep->pmd_nclass = 1;
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_XSCALE];
pcd->pcd_caps = XSCALE_PMC_CAPS;

12
sys/dev/hwpmc/pmc_events.h
View File

@ -2599,6 +2599,15 @@ __PMC_EV_ALIAS("SQ_MISC.SPLIT_LOCK", IAP_EVENT_F4H_10H)
#define PMC_EV_TSC_FIRST PMC_EV_TSC_TSC
#define PMC_EV_TSC_LAST PMC_EV_TSC_TSC
/*
* Software events are dynamicaly defined.
*/
#define PMC_EV_DYN_COUNT 0x1000
#define PMC_EV_SOFT_FIRST 0x20000
#define PMC_EV_SOFT_LAST (PMC_EV_SOFT_FIRST + PMC_EV_DYN_COUNT - 1)
#define __PMC_EV_UCF() \
__PMC_EV(UCF, UCLOCK)
@ -3716,6 +3725,7 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H)
* 0x11100 0x0100 INTEL Pentium Pro/P-II/P-III/Pentium-M events
* 0x11200 0x00FF INTEL XScale events
* 0x11300 0x00FF MIPS 24K events
* 0x20000 0x1000 Software events
*/
#define __PMC_EVENTS() \
__PMC_EV_BLOCK(TSC, 0x01000) \
@ -3748,6 +3758,6 @@ __PMC_EV_ALIAS("IMPC_C0H_TRK_REQUEST.ALL", UCP_EVENT_84H_01H)
__PMC_EV_PPC7450() \
#define PMC_EVENT_FIRST PMC_EV_TSC_TSC
#define PMC_EVENT_LAST PMC_EV_UCP_LAST
#define PMC_EVENT_LAST PMC_EV_SOFT_LAST
#endif /* _DEV_HWPMC_PMC_EVENTS_H_ */

17
sys/i386/i386/trap.c
View File

@ -73,6 +73,9 @@ __FBSDID("$FreeBSD$");
#include <sys/vmmeter.h>
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DEFINE( , , page_fault, all);
PMC_SOFT_DEFINE( , , page_fault, read);
PMC_SOFT_DEFINE( , , page_fault, write);
#endif
#include <security/audit/audit.h>
@ -910,8 +913,20 @@ trap_pfault(frame, usermode, eva)
*/
rv = vm_fault(map, va, ftype, VM_FAULT_NORMAL);
}
if (rv == KERN_SUCCESS)
if (rv == KERN_SUCCESS) {
#ifdef HWPMC_HOOKS
if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
PMC_SOFT_CALL_TF( , , page_fault, all, frame);
if (ftype == VM_PROT_READ)
PMC_SOFT_CALL_TF( , , page_fault, read,
frame);
else
PMC_SOFT_CALL_TF( , , page_fault, write,
frame);
}
#endif
return (0);
}
nogo:
if (!usermode) {
if (td->td_intr_nesting_level == 0 &&

29
sys/i386/include/pmc_mdep.h
View File

@ -66,16 +66,16 @@ struct pmc_mdep;
* measurement architecture have PMCs of the following classes: TSC,
* IAF, IAP, UCF and UCP.
*/
#define PMC_MDEP_CLASS_INDEX_TSC 0
#define PMC_MDEP_CLASS_INDEX_K7 1
#define PMC_MDEP_CLASS_INDEX_K8 1
#define PMC_MDEP_CLASS_INDEX_P4 1
#define PMC_MDEP_CLASS_INDEX_P5 1
#define PMC_MDEP_CLASS_INDEX_P6 1
#define PMC_MDEP_CLASS_INDEX_IAP 1
#define PMC_MDEP_CLASS_INDEX_IAF 2
#define PMC_MDEP_CLASS_INDEX_UCP 3
#define PMC_MDEP_CLASS_INDEX_UCF 4
#define PMC_MDEP_CLASS_INDEX_TSC 1
#define PMC_MDEP_CLASS_INDEX_K7 2
#define PMC_MDEP_CLASS_INDEX_K8 2
#define PMC_MDEP_CLASS_INDEX_P4 2
#define PMC_MDEP_CLASS_INDEX_P5 2
#define PMC_MDEP_CLASS_INDEX_P6 2
#define PMC_MDEP_CLASS_INDEX_IAP 2
#define PMC_MDEP_CLASS_INDEX_IAF 3
#define PMC_MDEP_CLASS_INDEX_UCP 4
#define PMC_MDEP_CLASS_INDEX_UCF 5
/*
* Architecture specific extensions to <sys/pmc.h> structures.
@ -154,6 +154,15 @@ struct pmc_mdep;
#define PMC_AT_FUNCTION_EPILOGUE_RET(I) \
(((I) & 0xFF) == 0xC3) /* ret */
/* Build a fake kernel trapframe from current instruction pointer. */
#define PMC_FAKE_TRAPFRAME(TF) \
do { \
(TF)->tf_cs = 0; (TF)->tf_eflags = 0; \
__asm __volatile("movl %%ebp,%0" : "=r" ((TF)->tf_ebp)); \
__asm __volatile("movl %%esp,%0" : "=r" ((TF)->tf_esp)); \
__asm __volatile("call 1f \n\t1: pop %0" : "=r"((TF)->tf_eip)); \
} while (0)
/*
* Prototypes
*/

12
sys/kern/kern_clock.c
View File

@ -74,6 +74,8 @@ __FBSDID("$FreeBSD$");
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DEFINE( , , clock, hard);
PMC_SOFT_DEFINE( , , clock, stat);
#endif
#ifdef DEVICE_POLLING
@ -446,9 +448,11 @@ hardclock_cpu(int usermode)
td->td_flags |= flags;
thread_unlock(td);
#ifdef HWPMC_HOOKS
#ifdef HWPMC_HOOKS
if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
if (td->td_intr_frame != NULL)
PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame);
#endif
callout_tick();
}
@ -537,6 +541,8 @@ hardclock_cnt(int cnt, int usermode)
#ifdef HWPMC_HOOKS
if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
if (td->td_intr_frame != NULL)
PMC_SOFT_CALL_TF( , , clock, hard, td->td_intr_frame);
#endif
callout_tick();
/* We are in charge to handle this tick duty. */
@ -758,6 +764,10 @@ statclock_cnt(int cnt, int usermode)
for ( ; cnt > 0; cnt--)
sched_clock(td);
thread_unlock(td);
#ifdef HWPMC_HOOKS
if (td->td_intr_frame != NULL)
PMC_SOFT_CALL_TF( , , clock, stat, td->td_intr_frame);
#endif
}
void

15
sys/kern/kern_lock.c
View File

@ -28,6 +28,7 @@
#include "opt_adaptive_lockmgrs.h"
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_kdtrace.h"
#include <sys/cdefs.h>
@ -53,6 +54,11 @@ __FBSDID("$FreeBSD$");
#include <ddb/ddb.h>
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DECLARE( , , lock, failed);
#endif
CTASSERT(((LK_ADAPTIVE | LK_NOSHARE) & LO_CLASSFLAGS) ==
(LK_ADAPTIVE | LK_NOSHARE));
CTASSERT(LK_UNLOCKED == (LK_UNLOCKED &
@ -514,6 +520,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
break;
continue;
}
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
@ -744,6 +753,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
tid)) {
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);
@ -1056,6 +1068,9 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
}
while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&lk->lock_object,
&contested, &waittime);

15
sys/kern/kern_mutex.c
View File

@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
#include "opt_adaptive_mutexes.h"
#include "opt_ddb.h"
#include "opt_global.h"
#include "opt_hwpmc_hooks.h"
#include "opt_kdtrace.h"
#include "opt_sched.h"
@ -76,6 +77,11 @@ __FBSDID("$FreeBSD$");
#define ADAPTIVE_MUTEXES
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DEFINE( , , lock, failed);
#endif
/*
* Internal utility macros.
*/
@ -364,6 +370,9 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
return;
}
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&m->lock_object,
&contested, &waittime);
if (LOCK_LOG_TEST(&m->lock_object, opts))
@ -529,6 +538,9 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
if (LOCK_LOG_TEST(&m->lock_object, opts))
CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime);
while (!_mtx_obtain_lock(m, tid)) {
@ -600,6 +612,9 @@ thread_lock_flags_(struct thread *td, int opts, const char *file, int line)
m->mtx_recurse++;
break;
}
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&m->lock_object,
&contested, &waittime);
/* Give interrupts a chance while we spin. */

161
sys/kern/kern_pmc.c
View File

@ -34,10 +34,17 @@ __FBSDID("$FreeBSD$");
#include "opt_hwpmc_hooks.h"
#include <sys/types.h>
#include <sys/ctype.h>
#include <sys/param.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/pmc.h>
#include <sys/pmckern.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#ifdef HWPMC_HOOKS
FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
@ -46,6 +53,9 @@ FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
#define PMC_KERNEL_VERSION 0
#endif
MALLOC_DECLARE(M_PMCHOOKS);
MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks");
const int pmc_kernel_version = PMC_KERNEL_VERSION;
/* Hook variable. */
@ -74,6 +84,28 @@ volatile int pmc_ss_count;
*/
struct sx pmc_sx;
/*
* PMC Soft per cpu trapframe.
*/
struct trapframe pmc_tf[MAXCPU];
/*
* PMC Soft use a global table to store registered events.
*/
SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
static int pmc_softevents = 16;
TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "softevents", &pmc_softevents);
SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_TUN|CTLFLAG_RD,
&pmc_softevents, 0, "maximum number of soft events");
struct mtx pmc_softs_mtx;
int pmc_softs_count;
struct pmc_soft **pmc_softs;
MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN);
static void
pmc_init_sx(void)
{
@ -182,3 +214,132 @@ pmc_cpu_max_active(void)
}
#endif
/*
* Cleanup event name:
* - remove duplicate '_'
* - all uppercase
*/
static void
pmc_soft_namecleanup(char *name)
{
char *p, *q;
p = q = name;
for ( ; *p == '_' ; p++)
;
for ( ; *p ; p++) {
if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0'))
continue;
else
*q++ = toupper(*p);
}
*q = '\0';
}
void
pmc_soft_ev_register(struct pmc_soft *ps)
{
static int warned = 0;
int n;
ps->ps_running = 0;
ps->ps_ev.pm_ev_code = 0; /* invalid */
pmc_soft_namecleanup(ps->ps_ev.pm_ev_name);
mtx_lock_spin(&pmc_softs_mtx);
if (pmc_softs_count >= pmc_softevents) {
/*
* XXX Reusing events can enter a race condition where
* new allocated event will be used as an old one.
*/
for (n = 0; n < pmc_softevents; n++)
if (pmc_softs[n] == NULL)
break;
if (n == pmc_softevents) {
mtx_unlock_spin(&pmc_softs_mtx);
if (!warned) {
printf("hwpmc: too many soft events, "
"increase kern.hwpmc.softevents tunable\n");
warned = 1;
}
return;
}
ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n;
pmc_softs[n] = ps;
} else {
ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count;
pmc_softs[pmc_softs_count++] = ps;
}
mtx_unlock_spin(&pmc_softs_mtx);
}
void
pmc_soft_ev_deregister(struct pmc_soft *ps)
{
KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL"));
mtx_lock_spin(&pmc_softs_mtx);
if (ps->ps_ev.pm_ev_code != 0 &&
(ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) {
KASSERT(ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST &&
ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST,
("pmc_soft_deregister: invalid event value"));
pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL;
}
mtx_unlock_spin(&pmc_softs_mtx);
}
struct pmc_soft *
pmc_soft_ev_acquire(enum pmc_event ev)
{
struct pmc_soft *ps;
if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents)
return NULL;
KASSERT(ev >= PMC_EV_SOFT_FIRST &&
ev <= PMC_EV_SOFT_LAST,
("event out of range"));
mtx_lock_spin(&pmc_softs_mtx);
ps = pmc_softs[ev - PMC_EV_SOFT_FIRST];
if (ps == NULL)
mtx_unlock_spin(&pmc_softs_mtx);
return ps;
}
void
pmc_soft_ev_release(struct pmc_soft *ps)
{
mtx_unlock_spin(&pmc_softs_mtx);
}
/*
* Initialise hwpmc.
*/
static void
init_hwpmc(void *dummy __unused)
{
if (pmc_softevents <= 0 ||
pmc_softevents > PMC_EV_DYN_COUNT) {
(void) printf("hwpmc: tunable \"softevents\"=%d out of "
"range.\n", pmc_softevents);
pmc_softevents = PMC_EV_DYN_COUNT;
}
pmc_softs = malloc(pmc_softevents * sizeof(struct pmc_soft *), M_PMCHOOKS, M_NOWAIT|M_ZERO);
KASSERT(pmc_softs != NULL, ("cannot allocate soft events table"));
}
SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL);

12
sys/kern/kern_rwlock.c
View File

@ -35,6 +35,7 @@
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_kdtrace.h"
#include "opt_no_adaptive_rwlocks.h"
@ -55,6 +56,11 @@ __FBSDID("$FreeBSD$");
#define ADAPTIVE_RWLOCKS
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DECLARE( , , lock, failed);
#endif
#ifdef ADAPTIVE_RWLOCKS
static int rowner_retries = 10;
static int rowner_loops = 10000;
@ -366,6 +372,9 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
}
continue;
}
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&rw->lock_object,
&contested, &waittime);
@ -686,6 +695,9 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
while (!_rw_write_lock(rw, tid)) {
#ifdef KDTRACE_HOOKS
spin_cnt++;
#endif
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&rw->lock_object,
&contested, &waittime);

12
sys/kern/kern_sx.c
View File

@ -37,6 +37,7 @@
*/
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_kdtrace.h"
#include "opt_no_adaptive_sx.h"
@ -67,6 +68,11 @@ __FBSDID("$FreeBSD$");
CTASSERT((SX_NOADAPTIVE & LO_CLASSFLAGS) == SX_NOADAPTIVE);
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
PMC_SOFT_DECLARE( , , lock, failed);
#endif
/* Handy macros for sleep queues. */
#define SQ_EXCLUSIVE_QUEUE 0
#define SQ_SHARED_QUEUE 1
@ -523,6 +529,9 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
#ifdef KDTRACE_HOOKS
spin_cnt++;
#endif
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
&waittime);
@ -811,6 +820,9 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
}
continue;
}
#ifdef HWPMC_HOOKS
PMC_SOFT_CALL( , , lock, failed);
#endif
lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
&waittime);

10
sys/kern/subr_trap.c
View File

@ -45,6 +45,7 @@
__FBSDID("$FreeBSD$");
#include "opt_capsicum.h"
#include "opt_hwpmc_hooks.h"
#include "opt_ktrace.h"
#include "opt_kdtrace.h"
#include "opt_sched.h"
@ -86,6 +87,10 @@ __FBSDID("$FreeBSD$");
#include <vm/pmap.h>
#endif
#ifdef HWPMC_HOOKS
#include <sys/pmckern.h>
#endif
#include <security/mac/mac_framework.h>
/*
@ -192,6 +197,11 @@ ast(struct trapframe *framep)
td->td_profil_ticks = 0;
td->td_pflags &= ~TDP_OWEUPC;
}
#ifdef HWPMC_HOOKS
/* Handle Software PMC callchain capture. */
if (PMC_IS_PENDING_CALLCHAIN(td))
PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, (void *) framep);
#endif
if (flags & TDF_ALRMPEND) {
PROC_LOCK(p);
kern_psignal(p, SIGVTALRM);

2
sys/mips/include/pmc_mdep.h
View File

@ -8,7 +8,7 @@
#ifndef _MACHINE_PMC_MDEP_H_
#define _MACHINE_PMC_MDEP_H_
#define PMC_MDEP_CLASS_INDEX_MIPS 0
#define PMC_MDEP_CLASS_INDEX_MIPS 1
union pmc_md_op_pmcallocate {
uint64_t __pad[4];

2
sys/modules/hwpmc/Makefile
View File

@ -6,7 +6,7 @@
KMOD= hwpmc
SRCS= hwpmc_mod.c hwpmc_logging.c vnode_if.h
SRCS= hwpmc_mod.c hwpmc_logging.c hwpmc_soft.c vnode_if.h
.if ${MACHINE_CPUARCH} == "amd64"
SRCS+= hwpmc_amd.c hwpmc_core.c hwpmc_intel.c hwpmc_piv.c hwpmc_tsc.c

3
sys/powerpc/include/pmc_mdep.h
View File

@ -7,7 +7,8 @@
#ifndef _MACHINE_PMC_MDEP_H_
#define _MACHINE_PMC_MDEP_H_
#define PMC_MDEP_CLASS_INDEX_PPC7450 0
#define PMC_MDEP_CLASS_INDEX_PPC7450 1
union pmc_md_op_pmcallocate {
uint64_t __pad[4];
};

66
sys/sys/pmc.h
View File

@ -39,8 +39,8 @@
#include <machine/profile.h>
#define PMC_MODULE_NAME "hwpmc"
#define PMC_NAME_MAX 16 /* HW counter name size */
#define PMC_CLASS_MAX 6 /* max #classes of PMCs per-system */
#define PMC_NAME_MAX 64 /* HW counter name size */
#define PMC_CLASS_MAX 8 /* max #classes of PMCs per-system */
/*
* Kernel<->userland API version number [MMmmpppp]
@ -83,14 +83,15 @@
__PMC_CPU(INTEL_CORE, 0x87, "Intel Core Solo/Duo") \
__PMC_CPU(INTEL_CORE2, 0x88, "Intel Core2") \
__PMC_CPU(INTEL_CORE2EXTREME, 0x89, "Intel Core2 Extreme") \
__PMC_CPU(INTEL_ATOM, 0x8A, "Intel Atom") \
__PMC_CPU(INTEL_COREI7, 0x8B, "Intel Core i7") \
__PMC_CPU(INTEL_WESTMERE, 0x8C, "Intel Westmere") \
__PMC_CPU(INTEL_SANDYBRIDGE, 0x8D, "Intel Sandy Bridge") \
__PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \
__PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \
__PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \
__PMC_CPU(PPC_7450, 0x300, "PowerPC MPC7450")
__PMC_CPU(INTEL_ATOM, 0x8A, "Intel Atom") \
__PMC_CPU(INTEL_COREI7, 0x8B, "Intel Core i7") \
__PMC_CPU(INTEL_WESTMERE, 0x8C, "Intel Westmere") \
__PMC_CPU(INTEL_SANDYBRIDGE, 0x8D, "Intel Sandy Bridge") \
__PMC_CPU(INTEL_XSCALE, 0x100, "Intel XScale") \
__PMC_CPU(MIPS_24K, 0x200, "MIPS 24K") \
__PMC_CPU(MIPS_OCTEON, 0x201, "Cavium Octeon") \
__PMC_CPU(PPC_7450, 0x300, "PowerPC MPC7450") \
__PMC_CPU(GENERIC, 0x400, "Generic")
enum pmc_cputype {
#undef __PMC_CPU
@ -99,7 +100,7 @@ enum pmc_cputype {
};
#define PMC_CPU_FIRST PMC_CPU_AMD_K7
#define PMC_CPU_LAST PMC_CPU_PPC_7450
#define PMC_CPU_LAST PMC_CPU_GENERIC
/*
* Classes of PMCs
@ -119,7 +120,8 @@ enum pmc_cputype {
__PMC_CLASS(XSCALE) /* Intel XScale counters */ \
__PMC_CLASS(MIPS24K) /* MIPS 24K */ \
__PMC_CLASS(OCTEON) /* Cavium Octeon */ \
__PMC_CLASS(PPC7450) /* Motorola MPC7450 class */
__PMC_CLASS(PPC7450) /* Motorola MPC7450 class */ \
__PMC_CLASS(SOFT) /* Software events */
enum pmc_class {
#undef __PMC_CLASS
@ -128,7 +130,7 @@ enum pmc_class {
};
#define PMC_CLASS_FIRST PMC_CLASS_TSC
#define PMC_CLASS_LAST PMC_CLASS_PPC7450
#define PMC_CLASS_LAST PMC_CLASS_SOFT
/*
* A PMC can be in the following states:
@ -308,7 +310,8 @@ enum pmc_event {
__PMC_OP(PMCSTART, "Start a PMC") \
__PMC_OP(PMCSTOP, "Stop a PMC") \
__PMC_OP(WRITELOG, "Write a cookie to the log file") \
__PMC_OP(CLOSELOG, "Close log file")
__PMC_OP(CLOSELOG, "Close log file") \
__PMC_OP(GETDYNEVENTINFO, "Get dynamic events list")
enum pmc_ops {
@ -338,6 +341,7 @@ enum pmc_ops {
#define PMC_F_ATTACH_DONE 0x00040000 /*attached at least once */
#define PMC_CALLCHAIN_DEPTH_MAX 32
#define PMC_CC_F_USERSPACE 0x01 /*userspace callchain*/
/*
@ -486,6 +490,7 @@ struct pmc_op_getpmcinfo {
* Retrieve system CPU information.
*/
struct pmc_classinfo {
enum pmc_class pm_class; /* class id */
uint32_t pm_caps; /* counter capabilities */
@ -562,6 +567,23 @@ struct pmc_op_getmsr {
pmc_id_t pm_pmcid; /* allocated pmc id */
};
/*
* OP GETDYNEVENTINFO
*
* Retrieve a PMC dynamic class events list.
*/
struct pmc_dyn_event_descr {
char pm_ev_name[PMC_NAME_MAX];
enum pmc_event pm_ev_code;
};
struct pmc_op_getdyneventinfo {
enum pmc_class pm_class;
unsigned int pm_nevent;
struct pmc_dyn_event_descr pm_events[PMC_EV_DYN_COUNT];
};
#ifdef _KERNEL
#include <sys/malloc.h>
@ -572,8 +594,8 @@ struct pmc_op_getmsr {
#define PMC_HASH_SIZE 16
#define PMC_MTXPOOL_SIZE 32
#define PMC_LOG_BUFFER_SIZE 4
#define PMC_NLOGBUFFERS 16
#define PMC_NSAMPLES 32
#define PMC_NLOGBUFFERS 64
#define PMC_NSAMPLES 512
#define PMC_CALLCHAIN_DEPTH 8
#define PMC_SYSCTL_NAME_PREFIX "kern." PMC_MODULE_NAME "."
@ -829,8 +851,8 @@ struct pmc_sample {
uintptr_t *ps_pc; /* (const) callchain start */
};
#define PMC_SAMPLE_FREE ((uint16_t) 0)
#define PMC_SAMPLE_INUSE ((uint16_t) 0xFFFF)
#define PMC_SAMPLE_FREE ((uint16_t) 0)
#define PMC_SAMPLE_INUSE ((uint16_t) 0xFFFF)
struct pmc_samplebuffer {
struct pmc_sample * volatile ps_read; /* read pointer */
@ -850,7 +872,7 @@ struct pmc_samplebuffer {
struct pmc_cpu {
uint32_t pc_state; /* physical cpu number + flags */
struct pmc_samplebuffer *pc_sb; /* space for samples */
struct pmc_samplebuffer *pc_sb[2]; /* space for samples */
struct pmc_hw *pc_hwpmcs[]; /* 'npmc' pointers */
};
@ -958,7 +980,7 @@ extern struct pmc_cpu **pmc_pcpu;
/* driver statistics */
extern struct pmc_op_getdriverstats pmc_stats;
#if defined(DEBUG) && DEBUG
#if defined(DEBUG)
/* debug flags, major flag groups */
struct pmc_debugflags {
@ -1061,11 +1083,13 @@ MALLOC_DECLARE(M_PMC);
struct pmc_mdep *pmc_md_initialize(void); /* MD init function */
void pmc_md_finalize(struct pmc_mdep *_md); /* MD fini function */
int pmc_getrowdisp(int _ri);
int pmc_process_interrupt(int _cpu, struct pmc *_pm,
int pmc_process_interrupt(int _cpu, int _soft, struct pmc *_pm,
struct trapframe *_tf, int _inuserspace);
int pmc_save_kernel_callchain(uintptr_t *_cc, int _maxsamples,
struct trapframe *_tf);
int pmc_save_user_callchain(uintptr_t *_cc, int _maxsamples,
struct trapframe *_tf);
struct pmc_mdep *pmc_mdep_alloc(int nclasses);
void pmc_mdep_free(struct pmc_mdep *md);
#endif /* _KERNEL */
#endif /* _SYS_PMC_H_ */

97
sys/sys/pmckern.h
View File

@ -38,10 +38,14 @@
#define _SYS_PMCKERN_H_
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/proc.h>
#include <sys/sx.h>
#include <sys/pmc.h>
#include <machine/cpufunc.h>
#define PMC_FN_PROCESS_EXEC 1
#define PMC_FN_CSW_IN 2
@ -52,6 +56,11 @@
#define PMC_FN_MMAP 7
#define PMC_FN_MUNMAP 8
#define PMC_FN_USER_CALLCHAIN 9
#define PMC_FN_USER_CALLCHAIN_SOFT 10
#define PMC_FN_SOFT_SAMPLING 11
#define PMC_HR 0 /* Hardware ring buffer */
#define PMC_SR 1 /* Software ring buffer */
struct pmckern_procexec {
int pm_credentialschanged;
@ -68,6 +77,79 @@ struct pmckern_map_out {
size_t pm_size; /* size of unmapped region */
};
struct pmckern_soft {
enum pmc_event pm_ev;
int pm_cpu;
struct trapframe *pm_tf;
};
/*
* Soft PMC.
*/
#define PMC_SOFT_DEFINE(prov, mod, func, name) \
struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \
{ 0, { #prov "_" #mod "_" #func "." #name, 0 } }; \
SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \
SI_ORDER_SECOND + 1, pmc_soft_ev_register, \
&pmc_##prov##_##mod##_##func##_##name ); \
SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \
SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \
&pmc_##prov##_##mod##_##func##_##name )
#define PMC_SOFT_DECLARE(prov, mod, func, name) \
extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name
/*
* PMC_SOFT_CALL can be used anywhere in the kernel.
* Require md defined PMC_FAKE_TRAPFRAME.
*/
#ifdef PMC_FAKE_TRAPFRAME
#define PMC_SOFT_CALL(pr, mo, fu, na) \
do { \
if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \
struct pmckern_soft ks; \
register_t intr; \
intr = intr_disable(); \
PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \
ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \
ks.pm_cpu = PCPU_GET(cpuid); \
ks.pm_tf = &pmc_tf[curcpu]; \
PMC_CALL_HOOK_UNLOCKED(curthread, \
PMC_FN_SOFT_SAMPLING, (void *) &ks); \
intr_restore(intr); \
} \
} while (0)
#else
#define PMC_SOFT_CALL(pr, mo, fu, na) \
do { \
} while (0)
#endif
/*
* PMC_SOFT_CALL_TF need to be used carefully.
* Userland capture will be done during AST processing.
*/
#define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \
do { \
if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \
struct pmckern_soft ks; \
register_t intr; \
intr = intr_disable(); \
ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \
ks.pm_cpu = PCPU_GET(cpuid); \
ks.pm_tf = tf; \
PMC_CALL_HOOK_UNLOCKED(curthread, \
PMC_FN_SOFT_SAMPLING, (void *) &ks); \
intr_restore(intr); \
} \
} while (0)
struct pmc_soft {
int ps_running;
struct pmc_dyn_event_descr ps_ev;
};
/* hook */
extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg);
extern int (*pmc_intr)(int _cpu, struct trapframe *_frame);
@ -84,6 +166,9 @@ extern volatile int pmc_ss_count;
/* kernel version number */
extern const int pmc_kernel_version;
/* PMC soft per cpu trapframe */
extern struct trapframe pmc_tf[MAXCPU];
/* Hook invocation; for use within the kernel */
#define PMC_CALL_HOOK(t, cmd, arg) \
do { \
@ -119,6 +204,10 @@ do { \
(__predict_false(atomic_load_acq_int(&(p)->p_flag) & \
P_HWPMC))
/* Check if a thread have pending user capture. */
#define PMC_IS_PENDING_CALLCHAIN(p) \
(__predict_false((p)->td_pflags & TDP_CALLCHAIN))
#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0)
/* Check if a CPU has recorded samples. */
@ -137,4 +226,12 @@ unsigned int pmc_cpu_max(void);
int pmc_cpu_max_active(void);
#endif
/*
* Soft events functions.
*/
void pmc_soft_ev_register(struct pmc_soft *ps);
void pmc_soft_ev_deregister(struct pmc_soft *ps);
struct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev);
void pmc_soft_ev_release(struct pmc_soft *ps);
#endif /* _SYS_PMCKERN_H_ */

17
sys/sys/pmclog.h
View File

@ -60,7 +60,13 @@ enum pmclog_type {
*/
PMCLOG_TYPE_MAP_IN,
PMCLOG_TYPE_MAP_OUT,
PMCLOG_TYPE_CALLCHAIN
PMCLOG_TYPE_CALLCHAIN,
/*
* V3 ABI
*
* New variant of PMCLOG_TYPE_PMCALLOCATE for dynamic event.
*/
PMCLOG_TYPE_PMCALLOCATEDYN
};
/*
@ -204,6 +210,14 @@ struct pmclog_userdata {
uint32_t pl_userdata;
} __packed;
struct pmclog_pmcallocatedyn {
PMCLOG_ENTRY_HEADER
uint32_t pl_pmcid;
uint32_t pl_event;
uint32_t pl_flags;
char pl_evname[PMC_NAME_MAX];
} __packed;
union pmclog_entry { /* only used to size scratch areas */
struct pmclog_callchain pl_cc;
struct pmclog_closelog pl_cl;
@ -213,6 +227,7 @@ union pmclog_entry { /* only used to size scratch areas */
struct pmclog_map_out pl_mo;
struct pmclog_pcsample pl_s;
struct pmclog_pmcallocate pl_a;
struct pmclog_pmcallocatedyn pl_ad;
struct pmclog_pmcattach pl_t;
struct pmclog_pmcdetach pl_d;
struct pmclog_proccsw pl_c;

15
usr.sbin/pmcstat/pmcstat_log.c
View File

@ -1491,6 +1491,15 @@ pmcstat_analyze_log(void)
pmcstat_string_intern(ev.pl_u.pl_a.pl_evname));
break;
case PMCLOG_TYPE_PMCALLOCATEDYN:
/*
* Record the association pmc id between this
* PMC and its name.
*/
pmcstat_pmcid_add(ev.pl_u.pl_ad.pl_pmcid,
pmcstat_string_intern(ev.pl_u.pl_ad.pl_evname));
break;
case PMCLOG_TYPE_PROCEXEC:
/*
@ -1641,6 +1650,12 @@ pmcstat_print_log(void)
ev.pl_u.pl_a.pl_evname,
ev.pl_u.pl_a.pl_flags);
break;
case PMCLOG_TYPE_PMCALLOCATEDYN:
PMCSTAT_PRINT_ENTRY("allocatedyn","0x%x \"%s\" 0x%x",
ev.pl_u.pl_ad.pl_pmcid,
ev.pl_u.pl_ad.pl_evname,
ev.pl_u.pl_ad.pl_flags);
break;
case PMCLOG_TYPE_PMCATTACH:
PMCSTAT_PRINT_ENTRY("attach","0x%x %d \"%s\"",
ev.pl_u.pl_t.pl_pmcid,