d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add convenience APIs pmc_width() and pmc_capabilities() to -lpmc.

Browse Source 
Have pmcstat(8) and pmccontrol(8) use these APIs.

Return PMC class-related constants (PMC widths and capabilities)
with the OP GETCPUINFO call leaving OP PMCINFO to return only the
dynamic information associated with a PMC (i.e., whether enabled,
owner pid, reload count etc.).

Allow pmc_read() (i.e., OPS PMCRW) on active self-attached PMCs to
get upto-date values from hardware since we can guarantee that the
hardware is running the correct PMC at the time of the call.

Bug fixes:
 - (x86 class processors) Fix a bug that prevented an RDPMC
   instruction from being recognized as permitted till after the
   attached process had context switched out and back in again after
   a pmc_start() call.

   Tighten the rules for using RDPMC class instructions: a GETMSR
   OP is now allowed only after an OP ATTACH has been done by the
   PMC's owner to itself.  OP GETMSR is not allowed for PMCs that
   track descendants, for PMCs attached to processes other than
   their owner processes.

 - (P4/HTT processors only) Fix a bug that caused the MI and MD
   layers to get out of sync.  Add a new MD operation 'get_config()'
   as part of this fix.

 - Allow multiple system-mode PMCs at the same row-index but on
   different CPUs to be allocated.

 - Reject allocation of an administratively disabled PMC.

Misc. code cleanups and refactoring.  Improve a few comments.
This commit is contained in:
Joseph Koshy 2005-05-01 14:11:49 +00:00
parent b60d26c9b9
commit c5153e190b
14 changed files with 798 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/libpmc/Makefile
View File

@ -12,6 +12,7 @@ MAN= pmc.3
MLINKS+= \ MLINKS+= \
pmc.3 pmc_allocate.3 \ pmc.3 pmc_allocate.3 \
pmc.3 pmc_attach.3 \ pmc.3 pmc_attach.3 \
pmc.3 pmc_capabilities.3 \
pmc.3 pmc_configure_logfile.3 \ pmc.3 pmc_configure_logfile.3 \
pmc.3 pmc_cpuinfo.3 \ pmc.3 pmc_cpuinfo.3 \
pmc.3 pmc_detach.3 \ pmc.3 pmc_detach.3 \
@ -35,6 +36,7 @@ MLINKS+= \
pmc.3 pmc_set.3 \ pmc.3 pmc_set.3 \
pmc.3 pmc_start.3 \ pmc.3 pmc_start.3 \
pmc.3 pmc_stop.3 \ pmc.3 pmc_stop.3 \
pmc.3 pmc_width.3 \
pmc.3 pmc_write.3 \ pmc.3 pmc_write.3 \
pmc.3 pmc_x86_get_msr.3 pmc.3 pmc_x86_get_msr.3

30
lib/libpmc/libpmc.c
View File

@ -1981,6 +1981,36 @@ pmc_cpuinfo(const struct pmc_op_getcpuinfo **pci)
return 0; return 0;
} }
int
pmc_width(pmc_id_t pmcid, uint32_t *width)
{
unsigned int i;
enum pmc_class cl;
cl = PMC_ID_TO_CLASS(pmcid);
for (i = 0; i < cpu_info.pm_nclass; i++)
if (cpu_info.pm_classes[i].pm_class == cl) {
*width = cpu_info.pm_classes[i].pm_width;
return 0;
}
return EINVAL;
}
int
pmc_capabilities(pmc_id_t pmcid, uint32_t *caps)
{
unsigned int i;
enum pmc_class cl;
cl = PMC_ID_TO_CLASS(pmcid);
for (i = 0; i < cpu_info.pm_nclass; i++)
if (cpu_info.pm_classes[i].pm_class == cl) {
*caps = cpu_info.pm_classes[i].pm_caps;
return 0;
}
return EINVAL;
}
const char * const char *
pmc_name_of_cputype(enum pmc_cputype cp) pmc_name_of_cputype(enum pmc_cputype cp)
{ {

24
lib/libpmc/pmc.3
View File

@ -29,6 +29,7 @@
.Sh NAME .Sh NAME
.Nm pmc_allocate , .Nm pmc_allocate ,
.Nm pmc_attach , .Nm pmc_attach ,
.Nm pmc_capabilities ,
.Nm pmc_configure_logfile , .Nm pmc_configure_logfile ,
.Nm pmc_cpuinfo , .Nm pmc_cpuinfo ,
.Nm pmc_detach , .Nm pmc_detach ,
@ -53,6 +54,7 @@
.Nm pmc_start , .Nm pmc_start ,
.Nm pmc_stop , .Nm pmc_stop ,
.Nm pmc_write , .Nm pmc_write ,
.Nm pmc_width ,
.Nm pmc_x86_get_msr .Nm pmc_x86_get_msr
.Nd programming API for using hardware performance monitoring counters .Nd programming API for using hardware performance monitoring counters
.Sh LIBRARY .Sh LIBRARY
@ -73,6 +75,8 @@
.Fa "pid_t pid" .Fa "pid_t pid"
.Fc .Fc
.Ft int .Ft int
.Fn pmc_capabilities "pmc_id_t pmc" "uint32_t *caps"
.Ft int
.Fn pmc_configure_logfile "int fd" .Fn pmc_configure_logfile "int fd"
.Ft int .Ft int
.Fn pmc_cpuinfo "const struct pmc_op_getcpuinfo **cpu_info" .Fn pmc_cpuinfo "const struct pmc_op_getcpuinfo **cpu_info"
@ -130,6 +134,8 @@
.Ft int .Ft int
.Fn pmc_write "pmc_id_t pmc" "pmc_value_t value" .Fn pmc_write "pmc_id_t pmc" "pmc_value_t value"
.Ft int .Ft int
.Fn pmc_width "pmc_id_t pmc" "uint32_t *width"
.Ft int
.Fn pmc_x86_get_msr "int pmc" "uint32_t *msr" .Fn pmc_x86_get_msr "int pmc" "uint32_t *msr"
.Sh DESCRIPTION .Sh DESCRIPTION
These functions implement a high-level library for using the These functions implement a high-level library for using the
@ -252,6 +258,20 @@ the allocated PMC.
The read and write operation may be combined using The read and write operation may be combined using
.Fn pmc_rw . .Fn pmc_rw .
.Pp .Pp
The function
.Fn pmc_capabilities
sets argument
.Fa caps
to a bitmask of capabilities supported by the PMC denoted by
argument
.Fa pmc .
The function
.Fn pmc_width
sets argument
.Fa width
to the width of the PMC denoted by argument
.Fa pmc .
.Pp
The The
.Fn pmc_configure_logfile .Fn pmc_configure_logfile
function causes the function causes the
@ -3011,13 +3031,15 @@ library.
.El .El
.Pp .Pp
A call to A call to
.Fn pmc_capabilities ,
.Fn pmc_name_of_capability , .Fn pmc_name_of_capability ,
.Fn pmc_name_of_disposition , .Fn pmc_name_of_disposition ,
.Fn pmc_name_of_state , .Fn pmc_name_of_state ,
.Fn pmc_name_of_event , .Fn pmc_name_of_event ,
.Fn pmc_name_of_mode .Fn pmc_name_of_mode
and
.Fn pmc_name_of_class .Fn pmc_name_of_class
and
.Fn pmc_width
may fail with the following error: may fail with the following error:
.Bl -tag -width Er .Bl -tag -width Er
.It Bq Er EINVAL .It Bq Er EINVAL

2
lib/libpmc/pmc.h
View File

@ -38,6 +38,7 @@
int pmc_allocate(const char *_ctrspec, enum pmc_mode _mode, uint32_t _flags, int pmc_allocate(const char *_ctrspec, enum pmc_mode _mode, uint32_t _flags,
int _cpu, pmc_id_t *_pmcid); int _cpu, pmc_id_t *_pmcid);
int pmc_attach(pmc_id_t _pmcid, pid_t _pid); int pmc_attach(pmc_id_t _pmcid, pid_t _pid);
int pmc_capabilities(pmc_id_t _pmc, uint32_t *_caps);
int pmc_configure_logfile(int _fd); int pmc_configure_logfile(int _fd);
int pmc_detach(pmc_id_t _pmcid, pid_t _pid); int pmc_detach(pmc_id_t _pmcid, pid_t _pid);
int pmc_disable(int _cpu, int _pmc); int pmc_disable(int _cpu, int _pmc);
@ -50,6 +51,7 @@ int pmc_rw(pmc_id_t _pmc, pmc_value_t _newvalue, pmc_value_t *_oldvalue);
int pmc_set(pmc_id_t _pmc, pmc_value_t _value); int pmc_set(pmc_id_t _pmc, pmc_value_t _value);
int pmc_start(pmc_id_t _pmc); int pmc_start(pmc_id_t _pmc);
int pmc_stop(pmc_id_t _pmc); int pmc_stop(pmc_id_t _pmc);
int pmc_width(pmc_id_t _pmc, uint32_t *_width);
int pmc_write(pmc_id_t _pmc, pmc_value_t _value); int pmc_write(pmc_id_t _pmc, pmc_value_t _value);
int pmc_ncpu(void); int pmc_ncpu(void);

21
share/man/man4/hwpmc.4
View File

@ -287,7 +287,10 @@ system call.
Retrieve the MSR (machine specific register) number associated with Retrieve the MSR (machine specific register) number associated with
the given PMC handle. the given PMC handle.
.Pp .Pp
This operation is only valid for PMCs allocated in process-private modes. The PMC needs to be in process-private mode and allocated without the
.Va PMC_F_DESCENDANTS
modifier flag, and should be attached only to its owner process at the
time of the call.
.El .El
.Ss amd64 SPECIFIC API .Ss amd64 SPECIFIC API
AMD64 cpus support the RDPMC instruction which allows a AMD64 cpus support the RDPMC instruction which allows a
@ -303,7 +306,10 @@ system call.
Retrieve the MSR (machine specific register) number associated with Retrieve the MSR (machine specific register) number associated with
the given PMC handle. the given PMC handle.
.Pp .Pp
This operation is only valid for PMCs allocated in process-private modes. The PMC needs to be in process-private mode and allocated without the
.Va PMC_F_DESCENDANTS
modifier flag, and should be attached only to its owner process at the
time of the call.
.El .El
.Sh SYSCTL TUNABLES .Sh SYSCTL TUNABLES
The behavior of The behavior of
@ -515,7 +521,10 @@ request contained illegal flags.
.It Bq Er EINVAL .It Bq Er EINVAL
A A
.Ic PMC_OP_PMCX86GETMSR .Ic PMC_OP_PMCX86GETMSR
operation was requested for a PMC not in process-virtual mode. operation was requested for a PMC not in process-virtual mode, or
for a PMC that is not solely attached to its owner process, or for
a PMC that was allocated with flag
.Va PMC_F_DESCENDANTS .
.It Bq Er EINVAL .It Bq Er EINVAL
(On Intel Pentium 4 CPUs with HTT support) An allocation request for (On Intel Pentium 4 CPUs with HTT support) An allocation request for
a process-private PMC was issued for an event that does not support a process-private PMC was issued for an event that does not support
@ -551,6 +560,12 @@ An
.Ic PMC_OP_PMCATTACH .Ic PMC_OP_PMCATTACH
operation was issued for a target process that the current process operation was issued for a target process that the current process
does not have permission to attach to. does not have permission to attach to.
.It Bq Er EPERM
.Pq "i386 and amd64 architectures"
An
.Ic PMC_OP_PMCATTACH
operation was issued on a PMC whose MSR has been retrieved using
.Ic PMC_OP_PMCX86GETMSR .
.It Bq Er ESRCH .It Bq Er ESRCH
A process issued a PMC operation request without having allocated any PMCs. A process issued a PMC operation request without having allocated any PMCs.
.It Bq Er ESRCH .It Bq Er ESRCH

42
sys/dev/hwpmc/hwpmc_amd.c
View File

@ -360,7 +360,7 @@ amd_read_pmc(int cpu, int ri, pmc_value_t *v)
("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
cpu, ri)); cpu, ri));
mode = pm->pm_mode; mode = PMC_TO_MODE(pm);
PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class);
@ -413,7 +413,7 @@ amd_write_pmc(int cpu, int ri, pmc_value_t v)
("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__,
cpu, ri)); cpu, ri));
mode = pm->pm_mode; mode = PMC_TO_MODE(pm);
if (pd->pm_descr.pd_class == PMC_CLASS_TSC) if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
return 0; return 0;
@ -460,6 +460,18 @@ amd_config_pmc(int cpu, int ri, struct pmc *pm)
return 0; return 0;
} }
/*
* Retrieve a configured PMC pointer from hardware state.
*/
static int
amd_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc;
return 0;
}
/* /*
* Machine dependent actions taken during the context switch in of a * Machine dependent actions taken during the context switch in of a
* thread. * thread.
@ -471,10 +483,10 @@ amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
(void) pc; (void) pc;
PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
(pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0); (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
/* enable the RDPMC instruction if needed */ /* enable the RDPMC instruction if needed */
if (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
load_cr4(rcr4() | CR4_PCE); load_cr4(rcr4() | CR4_PCE);
return 0; return 0;
@ -492,7 +504,7 @@ amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
(void) pp; /* can be NULL */ (void) pp; /* can be NULL */
PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ?
(pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) == 1 : 0); (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0);
/* always turn off the RDPMC instruction */ /* always turn off the RDPMC instruction */
load_cr4(rcr4() & ~CR4_PCE); load_cr4(rcr4() & ~CR4_PCE);
@ -523,7 +535,7 @@ amd_allocate_pmc(int cpu, int ri, struct pmc *pm,
pd = &amd_pmcdesc[ri].pm_descr; pd = &amd_pmcdesc[ri].pm_descr;
/* check class match */ /* check class match */
if (pd->pd_class != pm->pm_class) if (pd->pd_class != a->pm_class)
return EINVAL; return EINVAL;
caps = pm->pm_caps; caps = pm->pm_caps;
@ -765,7 +777,7 @@ amd_intr(int cpu, uintptr_t eip)
continue; continue;
} }
mode = pm->pm_mode; mode = PMC_TO_MODE(pm);
if (PMC_IS_SAMPLING_MODE(mode) && if (PMC_IS_SAMPLING_MODE(mode) &&
AMD_PMC_HAS_OVERFLOWED(perfctr)) { AMD_PMC_HAS_OVERFLOWED(perfctr)) {
atomic_add_int(&pmc_stats.pm_intr_processed, 1); atomic_add_int(&pmc_stats.pm_intr_processed, 1);
@ -803,8 +815,6 @@ amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
return error; return error;
pi->pm_class = pd->pm_descr.pd_class; pi->pm_class = pd->pm_descr.pd_class;
pi->pm_caps = pd->pm_descr.pd_caps;
pi->pm_width = pd->pm_descr.pd_width;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE; pi->pm_enabled = TRUE;
@ -982,8 +992,17 @@ pmc_amd_initialize(void)
/* this processor has two classes of usable PMCs */ /* this processor has two classes of usable PMCs */
pmc_mdep->pmd_nclass = 2; pmc_mdep->pmd_nclass = 2;
pmc_mdep->pmd_classes[0] = PMC_CLASS_TSC;
pmc_mdep->pmd_classes[1] = AMD_PMC_CLASS; /* TSC */
pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC;
pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ;
pmc_mdep->pmd_classes[0].pm_width = 64;
/* AMD K7/K8 PMCs */
pmc_mdep->pmd_classes[1].pm_class = AMD_PMC_CLASS;
pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS;
pmc_mdep->pmd_classes[1].pm_width = 48;
pmc_mdep->pmd_nclasspmcs[0] = 1; pmc_mdep->pmd_nclasspmcs[0] = 1;
pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1);
@ -994,6 +1013,7 @@ pmc_amd_initialize(void)
pmc_mdep->pmd_read_pmc = amd_read_pmc; pmc_mdep->pmd_read_pmc = amd_read_pmc;
pmc_mdep->pmd_write_pmc = amd_write_pmc; pmc_mdep->pmd_write_pmc = amd_write_pmc;
pmc_mdep->pmd_config_pmc = amd_config_pmc; pmc_mdep->pmd_config_pmc = amd_config_pmc;
pmc_mdep->pmd_get_config = amd_get_config;
pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc;
pmc_mdep->pmd_release_pmc = amd_release_pmc; pmc_mdep->pmd_release_pmc = amd_release_pmc;
pmc_mdep->pmd_start_pmc = amd_start_pmc; pmc_mdep->pmd_start_pmc = amd_start_pmc;

4
sys/dev/hwpmc/hwpmc_intel.c
View File

@ -92,7 +92,9 @@ pmc_intel_initialize(void)
pmc_mdep->pmd_cputype = cputype; pmc_mdep->pmd_cputype = cputype;
pmc_mdep->pmd_nclass = 2; pmc_mdep->pmd_nclass = 2;
pmc_mdep->pmd_classes[0] = PMC_CLASS_TSC; pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC;
pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ;
pmc_mdep->pmd_classes[0].pm_width = 64;
pmc_mdep->pmd_nclasspmcs[0] = 1; pmc_mdep->pmd_nclasspmcs[0] = 1;
error = 0; error = 0;

394
sys/dev/hwpmc/hwpmc_mod.c
View File

@ -151,12 +151,14 @@ static struct pmc *pmc_allocate_pmc_descriptor(void);
static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
pmc_id_t pmc); pmc_id_t pmc);
static void pmc_release_pmc_descriptor(struct pmc *pmc); static void pmc_release_pmc_descriptor(struct pmc *pmc);
static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri); static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
int cpu);
static struct pmc_process *pmc_find_process_descriptor(struct proc *p, static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
uint32_t mode); uint32_t mode);
static void pmc_remove_process_descriptor(struct pmc_process *pp); static void pmc_remove_process_descriptor(struct pmc_process *pp);
static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
static void pmc_force_context_switch(void);
static void pmc_remove_owner(struct pmc_owner *po); static void pmc_remove_owner(struct pmc_owner *po);
static void pmc_maybe_remove_owner(struct pmc_owner *po); static void pmc_maybe_remove_owner(struct pmc_owner *po);
static void pmc_unlink_target_process(struct pmc *pmc, static void pmc_unlink_target_process(struct pmc *pmc,
@ -364,6 +366,7 @@ pmc_debugflags_parse(char *newstr, char *fence)
CMP_SET_FLAG_MIN("cfg", CFG); CMP_SET_FLAG_MIN("cfg", CFG);
CMP_SET_FLAG_MIN("sta", STA); CMP_SET_FLAG_MIN("sta", STA);
CMP_SET_FLAG_MIN("sto", STO); CMP_SET_FLAG_MIN("sto", STO);
CMP_SET_FLAG_MIN("int", INT);
CMP_SET_FLAG_MIN("bnd", BND); CMP_SET_FLAG_MIN("bnd", BND);
CMP_SET_FLAG_MIN("sel", SEL); CMP_SET_FLAG_MIN("sel", SEL);
else /* unrecognized keyword */ else /* unrecognized keyword */
@ -572,6 +575,27 @@ pmc_select_cpu(int cpu)
PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
} }
/*
* Force a context switch.
*
* We do this by tsleep'ing for 1 tick -- invoking mi_switch() is not
* guaranteed to force a context switch.
*/
static void
pmc_force_context_switch(void)
{
u_char curpri;
mtx_lock_spin(&sched_lock);
curpri = curthread->td_priority;
mtx_unlock_spin(&sched_lock);
(void) tsleep((void *) pmc_force_context_switch, curpri,
"pmcctx", 1);
}
/* /*
* Update the per-pmc histogram * Update the per-pmc histogram
*/ */
@ -671,7 +695,7 @@ pmc_remove_owner(struct pmc_owner *po)
* XXX rework needed. * XXX rework needed.
*/ */
if (po->po_flags & PMC_FLAG_OWNS_LOGFILE) if (po->po_flags & PMC_PO_OWNS_LOGFILE)
pmc_configure_log(po, -1); pmc_configure_log(po, -1);
} }
@ -693,7 +717,7 @@ pmc_maybe_remove_owner(struct pmc_owner *po)
*/ */
if (LIST_EMPTY(&po->po_pmcs) && if (LIST_EMPTY(&po->po_pmcs) &&
((po->po_flags & PMC_FLAG_OWNS_LOGFILE) == 0)) { ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
pmc_remove_owner(po); pmc_remove_owner(po);
FREE(po, M_PMC); FREE(po, M_PMC);
} }
@ -718,7 +742,7 @@ pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
("[pmc,%d] Illegal reference count %d for process record %p", ("[pmc,%d] Illegal reference count %d for process record %p",
__LINE__, pp->pp_refcnt, (void *) pp)); __LINE__, pp->pp_refcnt, (void *) pp));
ri = pm->pm_rowindex; ri = PMC_TO_ROWINDEX(pm);
PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
pm, ri, pp); pm, ri, pp);
@ -740,12 +764,10 @@ pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
atomic_store_rel_ptr(&pp->pp_pmcs[ri].pp_pmc, pm); atomic_store_rel_ptr(&pp->pp_pmcs[ri].pp_pmc, pm);
if (pm->pm_owner->po_owner == pp->pp_proc) if (pm->pm_owner->po_owner == pp->pp_proc)
pp->pp_flags |= PMC_FLAG_ENABLE_MSR_ACCESS; pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
pp->pp_refcnt++; pp->pp_refcnt++;
PMCDBG(PRC,TLK,2, "enable-msr %d",
(pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0);
} }
/* /*
@ -767,7 +789,7 @@ pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
("[pmc,%d] Illegal ref count %d on process record %p", ("[pmc,%d] Illegal ref count %d on process record %p",
__LINE__, pp->pp_refcnt, (void *) pp)); __LINE__, pp->pp_refcnt, (void *) pp));
ri = pm->pm_rowindex; ri = PMC_TO_ROWINDEX(pm);
PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
pm, ri, pp); pm, ri, pp);
@ -779,8 +801,11 @@ pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
pp->pp_pmcs[ri].pp_pmc = NULL; pp->pp_pmcs[ri].pp_pmc = NULL;
pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
if (pm->pm_owner->po_owner == pp->pp_proc) /* Remove owner-specific flags */
pp->pp_flags &= ~PMC_FLAG_ENABLE_MSR_ACCESS; if (pm->pm_owner->po_owner == pp->pp_proc) {
pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
}
pp->pp_refcnt--; pp->pp_refcnt--;
@ -792,9 +817,6 @@ pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
"in pmc %p", __LINE__, pp->pp_proc, pp, pm)); "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
PMCDBG(PRC,TUL,4, "unlink ptgt=%p, enable-msr=%d", ptgt,
(pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0);
LIST_REMOVE(ptgt, pt_next); LIST_REMOVE(ptgt, pt_next);
FREE(ptgt, M_PMC); FREE(ptgt, M_PMC);
} }
@ -897,7 +919,7 @@ pmc_attach_one_process(struct proc *p, struct pmc *pm)
sx_assert(&pmc_sx, SX_XLOCKED); sx_assert(&pmc_sx, SX_XLOCKED);
PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
pm->pm_rowindex, p, p->p_pid, p->p_comm); PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
/* /*
* Locate the process descriptor corresponding to process 'p', * Locate the process descriptor corresponding to process 'p',
@ -910,7 +932,7 @@ pmc_attach_one_process(struct proc *p, struct pmc *pm)
* process descriptor and PMC. * process descriptor and PMC.
*/ */
ri = pm->pm_rowindex; ri = PMC_TO_ROWINDEX(pm);
if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
return ENOMEM; return ENOMEM;
@ -944,7 +966,16 @@ pmc_attach_process(struct proc *p, struct pmc *pm)
sx_assert(&pmc_sx, SX_XLOCKED); sx_assert(&pmc_sx, SX_XLOCKED);
PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
pm->pm_rowindex, p, p->p_pid, p->p_comm); PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
/*
* If this PMC successfully allowed a GETMSR operation
* in the past, disallow further ATTACHes.
*/
if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
return EPERM;
if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
return pmc_attach_one_process(p, pm); return pmc_attach_one_process(p, pm);
@ -999,10 +1030,10 @@ pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
KASSERT(pm != NULL, KASSERT(pm != NULL,
("[pmc,%d] null pm pointer", __LINE__)); ("[pmc,%d] null pm pointer", __LINE__));
PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x", ri = PMC_TO_ROWINDEX(pm);
pm, pm->pm_rowindex, p, p->p_pid, p->p_comm, flags);
ri = pm->pm_rowindex; PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
pm, ri, p, p->p_pid, p->p_comm, flags);
if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
return ESRCH; return ESRCH;
@ -1049,7 +1080,7 @@ pmc_detach_process(struct proc *p, struct pmc *pm)
sx_assert(&pmc_sx, SX_XLOCKED); sx_assert(&pmc_sx, SX_XLOCKED);
PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
pm->pm_rowindex, p, p->p_pid, p->p_comm); PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
@ -1131,7 +1162,6 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
int cpu; int cpu;
unsigned int ri; unsigned int ri;
struct pmc *pm; struct pmc *pm;
struct pmc_hw *phw;
struct pmc_process *pp; struct pmc_process *pp;
struct pmc_owner *po; struct pmc_owner *po;
struct proc *p; struct proc *p;
@ -1183,22 +1213,22 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
* state similar to the CSW_OUT code. * state similar to the CSW_OUT code.
*/ */
phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; pm = NULL;
pm = phw->phw_pmc; (void) (*md->pmd_get_config)(cpu, ri, &pm);
PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
if (pm == NULL || if (pm == NULL ||
!PMC_IS_VIRTUAL_MODE(pm->pm_mode)) !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
continue; continue;
PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
"state=%d", ri, pp->pp_pmcs[ri].pp_pmc, "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
pm, pm->pm_state); pm, pm->pm_state);
KASSERT(pm->pm_rowindex == ri, KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] ri mismatch pmc(%d) ri(%d)", ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
__LINE__, pm->pm_rowindex, ri)); __LINE__, PMC_TO_ROWINDEX(pm), ri));
KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
("[pmc,%d] pm %p != pp_pmcs[%d] %p", ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
@ -1222,10 +1252,11 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
mtx_pool_unlock_spin(pmc_mtxpool, pm); mtx_pool_unlock_spin(pmc_mtxpool, pm);
} }
atomic_subtract_rel_32(&pm->pm_runcount,1);
KASSERT((int) pm->pm_runcount >= 0, KASSERT((int) pm->pm_runcount >= 0,
("[pmc,%d] runcount is %d", __LINE__, ri)); ("[pmc,%d] runcount is %d", __LINE__, ri));
atomic_subtract_rel_32(&pm->pm_runcount,1);
(void) md->pmd_config_pmc(cpu, ri, NULL); (void) md->pmd_config_pmc(cpu, ri, NULL);
} }
@ -1254,6 +1285,7 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
FREE(pp, M_PMC); FREE(pp, M_PMC);
} else } else
critical_exit(); /* pp == NULL */ critical_exit(); /* pp == NULL */
@ -1445,13 +1477,13 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
continue; continue;
KASSERT(PMC_IS_VIRTUAL_MODE(pm->pm_mode), KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
("[pmc,%d] Target PMC in non-virtual mode (%d)", ("[pmc,%d] Target PMC in non-virtual mode (%d)",
__LINE__, pm->pm_mode)); __LINE__, PMC_TO_MODE(pm)));
KASSERT(pm->pm_rowindex == ri, KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] Row index mismatch pmc %d != ri %d", ("[pmc,%d] Row index mismatch pmc %d != ri %d",
__LINE__, pm->pm_rowindex, ri)); __LINE__, PMC_TO_ROWINDEX(pm), ri));
/* /*
* Only PMCs that are marked as 'RUNNING' need * Only PMCs that are marked as 'RUNNING' need
@ -1510,7 +1542,6 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
struct pmc *pm; struct pmc *pm;
struct proc *p; struct proc *p;
struct pmc_cpu *pc; struct pmc_cpu *pc;
struct pmc_hw *phw;
struct pmc_process *pp; struct pmc_process *pp;
pmc_value_t newvalue, tmp; pmc_value_t newvalue, tmp;
@ -1560,18 +1591,18 @@ pmc_hook_handler(struct thread *td, int function, void *arg)
for (ri = 0; ri < md->pmd_npmc; ri++) { for (ri = 0; ri < md->pmd_npmc; ri++) {
phw = pc->pc_hwpmcs[ri]; pm = NULL;
pm = phw->phw_pmc; (void) (*md->pmd_get_config)(cpu, ri, &pm);
if (pm == NULL) /* nothing at this row index */ if (pm == NULL) /* nothing at this row index */
continue; continue;
if (!PMC_IS_VIRTUAL_MODE(pm->pm_mode)) if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
continue; /* not a process virtual PMC */ continue; /* not a process virtual PMC */
KASSERT(pm->pm_rowindex == ri, KASSERT(PMC_TO_ROWINDEX(pm) == ri,
("[pmc,%d] ri mismatch pmc(%d) ri(%d)", ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
__LINE__, pm->pm_rowindex, ri)); __LINE__, PMC_TO_ROWINDEX(pm), ri));
/* Stop hardware */ /* Stop hardware */
md->pmd_stop_pmc(cpu, ri); md->pmd_stop_pmc(cpu, ri);
@ -1838,7 +1869,7 @@ pmc_release_pmc_descriptor(struct pmc *pm)
volatile int maxloop; volatile int maxloop;
#endif #endif
u_int ri, cpu; u_int ri, cpu;
u_char curpri; enum pmc_mode mode;
struct pmc_hw *phw; struct pmc_hw *phw;
struct pmc_process *pp; struct pmc_process *pp;
struct pmc_target *ptgt, *tmp; struct pmc_target *ptgt, *tmp;
@ -1848,16 +1879,17 @@ pmc_release_pmc_descriptor(struct pmc *pm)
KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
ri = pm->pm_rowindex; ri = PMC_TO_ROWINDEX(pm);
mode = PMC_TO_MODE(pm);
PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
pm->pm_mode); mode);
/* /*
* First, we take the PMC off hardware. * First, we take the PMC off hardware.
*/ */
cpu = 0; cpu = 0;
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(mode)) {
/* /*
* A system mode PMC runs on a specific CPU. Switch * A system mode PMC runs on a specific CPU. Switch
@ -1866,7 +1898,7 @@ pmc_release_pmc_descriptor(struct pmc *pm)
pmc_save_cpu_binding(&pb); pmc_save_cpu_binding(&pb);
cpu = pm->pm_gv.pm_cpu; cpu = PMC_TO_CPU(pm);
if (pm->pm_state == PMC_STATE_RUNNING) { if (pm->pm_state == PMC_STATE_RUNNING) {
@ -1895,7 +1927,7 @@ pmc_release_pmc_descriptor(struct pmc *pm)
pmc_restore_cpu_binding(&pb); pmc_restore_cpu_binding(&pb);
} else if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { } else if (PMC_IS_VIRTUAL_MODE(mode)) {
/* /*
* A virtual PMC could be running on multiple CPUs at * A virtual PMC could be running on multiple CPUs at
@ -1924,17 +1956,11 @@ pmc_release_pmc_descriptor(struct pmc *pm)
maxloop--; maxloop--;
KASSERT(maxloop > 0, KASSERT(maxloop > 0,
("[pmc,%d] (ri%d, rc%d) waiting too long for " ("[pmc,%d] (ri%d, rc%d) waiting too long for "
"pmc to be free", __LINE__, pm->pm_rowindex, "pmc to be free", __LINE__,
pm->pm_runcount)); PMC_TO_ROWINDEX(pm), pm->pm_runcount));
#endif #endif
mtx_lock_spin(&sched_lock); pmc_force_context_switch();
curpri = curthread->td_priority;
mtx_unlock_spin(&sched_lock);
(void) tsleep((void *) pmc_release_pmc_descriptor,
curpri, "pmcrel", 1);
} }
/* /*
@ -1977,7 +2003,7 @@ pmc_release_pmc_descriptor(struct pmc *pm)
* Update row disposition * Update row disposition
*/ */
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
PMC_UNMARK_ROW_STANDALONE(ri); PMC_UNMARK_ROW_STANDALONE(ri);
else else
PMC_UNMARK_ROW_THREAD(ri); PMC_UNMARK_ROW_THREAD(ri);
@ -2007,21 +2033,20 @@ pmc_register_owner(struct proc *p, struct pmc *pmc)
if (pl == NULL) if (pl == NULL)
return ENOMEM; return ENOMEM;
if ((po = pmc_find_owner_descriptor(p)) == NULL) { if ((po = pmc_find_owner_descriptor(p)) == NULL)
if ((po = pmc_allocate_owner_descriptor(p)) == NULL) { if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
FREE(pl, M_PMC); FREE(pl, M_PMC);
return ENOMEM; return ENOMEM;
} }
po->po_flags |= PMC_FLAG_IS_OWNER; /* real owner */
}
if (pmc->pm_mode == PMC_MODE_TS) { /* XXX is this too restrictive */
if (PMC_ID_TO_MODE(pmc->pm_id) == PMC_MODE_TS) {
/* can have only one TS mode PMC per process */ /* can have only one TS mode PMC per process */
if (po->po_flags & PMC_FLAG_HAS_TS_PMC) { if (po->po_flags & PMC_PO_HAS_TS_PMC) {
FREE(pl, M_PMC); FREE(pl, M_PMC);
return EINVAL; return EINVAL;
} }
po->po_flags |= PMC_FLAG_HAS_TS_PMC; po->po_flags |= PMC_PO_HAS_TS_PMC;
} }
KASSERT(pmc->pm_owner == NULL, KASSERT(pmc->pm_owner == NULL,
@ -2067,22 +2092,41 @@ pmc_getrowdisp(int ri)
*/ */
static int static int
pmc_can_allocate_rowindex(struct proc *p, unsigned int ri) pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
{ {
enum pmc_mode mode;
struct pmc *pm;
struct pmc_list *pl; struct pmc_list *pl;
struct pmc_owner *po; struct pmc_owner *po;
struct pmc_process *pp; struct pmc_process *pp;
PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d", PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
p, p->p_pid, p->p_comm, ri); "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
/* we shouldn't have allocated a PMC at row index 'ri' */ /*
* We shouldn't have already allocated a process-mode PMC at
* row index 'ri'.
*
* We shouldn't have allocated a system-wide PMC on the same
* CPU and same RI.
*/
if ((po = pmc_find_owner_descriptor(p)) != NULL) if ((po = pmc_find_owner_descriptor(p)) != NULL)
LIST_FOREACH(pl, &po->po_pmcs, pl_next) LIST_FOREACH(pl, &po->po_pmcs, pl_next) {
if (pl->pl_pmc->pm_rowindex == ri) pm = pl->pl_pmc;
if (PMC_TO_ROWINDEX(pm) == ri) {
mode = PMC_TO_MODE(pm);
if (PMC_IS_VIRTUAL_MODE(mode))
return EEXIST; return EEXIST;
if (PMC_IS_SYSTEM_MODE(mode) &&
(int) PMC_TO_CPU(pm) == cpu)
return EEXIST;
}
}
/* we shouldn't be the target of any PMC ourselves at this index */ /*
* We also shouldn't be the target of any PMC at this index
* since otherwise a PMC_ATTACH to ourselves will fail.
*/
if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
if (pp->pp_pmcs[ri].pp_pmc) if (pp->pp_pmcs[ri].pp_pmc)
return EEXIST; return EEXIST;
@ -2139,7 +2183,7 @@ pmc_can_allocate_row(int ri, enum pmc_mode mode)
} }
/* /*
* Find a PMC descriptor with user handle 'pmc' for thread 'td'. * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
*/ */
static struct pmc * static struct pmc *
@ -2147,12 +2191,12 @@ pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
{ {
struct pmc_list *pl; struct pmc_list *pl;
KASSERT(pmcid < md->pmd_npmc, KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, pmcid, ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
md->pmd_npmc)); PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
LIST_FOREACH(pl, &po->po_pmcs, pl_next) LIST_FOREACH(pl, &po->po_pmcs, pl_next)
if (pl->pl_pmc->pm_rowindex == pmcid) if (pl->pl_pmc->pm_id == pmcid)
return pl->pl_pmc; return pl->pl_pmc;
return NULL; return NULL;
@ -2187,17 +2231,21 @@ static int
pmc_start(struct pmc *pm) pmc_start(struct pmc *pm)
{ {
int error, cpu, ri; int error, cpu, ri;
enum pmc_mode mode;
struct pmc_binding pb; struct pmc_binding pb;
KASSERT(pm != NULL, KASSERT(pm != NULL,
("[pmc,%d] null pm", __LINE__)); ("[pmc,%d] null pm", __LINE__));
PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, pm->pm_mode, mode = PMC_TO_MODE(pm);
pm->pm_rowindex); ri = PMC_TO_ROWINDEX(pm);
error = 0;
PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
pm->pm_state = PMC_STATE_RUNNING; pm->pm_state = PMC_STATE_RUNNING;
if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { if (PMC_IS_VIRTUAL_MODE(mode)) {
/* /*
* If a PMCATTACH hadn't been done on this * If a PMCATTACH hadn't been done on this
@ -2205,32 +2253,36 @@ pmc_start(struct pmc *pm)
*/ */
if (LIST_EMPTY(&pm->pm_targets)) if (LIST_EMPTY(&pm->pm_targets))
return pmc_attach_process(pm->pm_owner->po_owner, pm); error = pmc_attach_process(pm->pm_owner->po_owner, pm);
/*
* If the PMC is attached to its owner, then force a context
* switch to ensure that the MD state gets set correctly.
*/
if (error == 0 && (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER))
pmc_force_context_switch();
/* /*
* Nothing further to be done; thread context switch code * Nothing further to be done; thread context switch code
* will start/stop the PMC as appropriate. * will start/stop the hardware as appropriate.
*/ */
return 0; return error;
} }
/* /*
* A system-mode PMC. Move to the CPU associated with this * A system-wide PMC. Move to the CPU associated with this
* PMC, and start the hardware. * PMC, and start the hardware.
*/ */
pmc_save_cpu_binding(&pb); pmc_save_cpu_binding(&pb);
cpu = pm->pm_gv.pm_cpu; cpu = PMC_TO_CPU(pm);
if (pmc_cpu_is_disabled(cpu)) if (pmc_cpu_is_disabled(cpu))
return ENXIO; return ENXIO;
ri = pm->pm_rowindex;
pmc_select_cpu(cpu); pmc_select_cpu(cpu);
/* /*
@ -2238,11 +2290,13 @@ pmc_start(struct pmc *pm)
* so write out the initial value and start the PMC. * so write out the initial value and start the PMC.
*/ */
critical_enter();
if ((error = md->pmd_write_pmc(cpu, ri, if ((error = md->pmd_write_pmc(cpu, ri,
PMC_IS_SAMPLING_MODE(pm->pm_mode) ? PMC_IS_SAMPLING_MODE(mode) ?
pm->pm_sc.pm_reloadcount : pm->pm_sc.pm_reloadcount :
pm->pm_sc.pm_initial)) == 0) pm->pm_sc.pm_initial)) == 0)
error = md->pmd_start_pmc(cpu, ri); error = md->pmd_start_pmc(cpu, ri);
critical_exit();
pmc_restore_cpu_binding(&pb); pmc_restore_cpu_binding(&pb);
@ -2256,13 +2310,13 @@ pmc_start(struct pmc *pm)
static int static int
pmc_stop(struct pmc *pm) pmc_stop(struct pmc *pm)
{ {
int error, cpu; int cpu, error, ri;
struct pmc_binding pb; struct pmc_binding pb;
KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, pm->pm_mode, PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
pm->pm_rowindex); PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
pm->pm_state = PMC_STATE_STOPPED; pm->pm_state = PMC_STATE_STOPPED;
@ -2276,7 +2330,7 @@ pmc_stop(struct pmc *pm)
* switched out. * switched out.
*/ */
if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
return 0; return 0;
/* /*
@ -2288,16 +2342,22 @@ pmc_stop(struct pmc *pm)
pmc_save_cpu_binding(&pb); pmc_save_cpu_binding(&pb);
cpu = pm->pm_gv.pm_cpu; cpu = PMC_TO_CPU(pm);
KASSERT(cpu >= 0 && cpu < mp_ncpus,
("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
if (pmc_cpu_is_disabled(cpu)) if (pmc_cpu_is_disabled(cpu))
return ENXIO; return ENXIO;
pmc_select_cpu(cpu); pmc_select_cpu(cpu);
if ((error = md->pmd_stop_pmc(cpu, pm->pm_rowindex)) == 0) ri = PMC_TO_ROWINDEX(pm);
error = md->pmd_read_pmc(cpu, pm->pm_rowindex,
&pm->pm_sc.pm_initial); critical_enter();
if ((error = md->pmd_stop_pmc(cpu, ri)) == 0)
error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial);
critical_exit();
pmc_restore_cpu_binding(&pb); pmc_restore_cpu_binding(&pb);
@ -2396,11 +2456,11 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
struct pmc_op_getcpuinfo gci; struct pmc_op_getcpuinfo gci;
gci.pm_cputype = md->pmd_cputype; gci.pm_cputype = md->pmd_cputype;
gci.pm_ncpu = mp_ncpus;
gci.pm_npmc = md->pmd_npmc; gci.pm_npmc = md->pmd_npmc;
gci.pm_nclass = md->pmd_nclass; gci.pm_nclass = md->pmd_nclass;
bcopy(md->pmd_classes, &gci.pm_classes, bcopy(md->pmd_classes, &gci.pm_classes,
sizeof(gci.pm_classes)); sizeof(gci.pm_classes));
gci.pm_ncpu = mp_ncpus;
error = copyout(&gci, arg, sizeof(gci)); error = copyout(&gci, arg, sizeof(gci));
} }
break; break;
@ -2499,11 +2559,11 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
__LINE__)); __LINE__));
p->pm_ownerpid = po->po_owner->p_pid; p->pm_ownerpid = po->po_owner->p_pid;
p->pm_mode = pm->pm_mode; p->pm_mode = PMC_TO_MODE(pm);
p->pm_event = pm->pm_event; p->pm_event = pm->pm_event;
p->pm_flags = pm->pm_flags; p->pm_flags = pm->pm_flags;
if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
p->pm_reloadcount = p->pm_reloadcount =
pm->pm_sc.pm_reloadcount; pm->pm_sc.pm_reloadcount;
} }
@ -2628,6 +2688,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
int n; int n;
enum pmc_mode mode; enum pmc_mode mode;
struct pmc *pmc; struct pmc *pmc;
struct pmc_hw *phw;
struct pmc_op_pmcallocate pa; struct pmc_op_pmcallocate pa;
struct pmc_binding pb; struct pmc_binding pb;
@ -2708,10 +2769,10 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
pa.pm_ev, caps, mode, cpu); pa.pm_ev, caps, mode, cpu);
pmc = pmc_allocate_pmc_descriptor(); pmc = pmc_allocate_pmc_descriptor();
pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
PMC_ID_INVALID);
pmc->pm_event = pa.pm_ev; pmc->pm_event = pa.pm_ev;
pmc->pm_class = pa.pm_class;
pmc->pm_state = PMC_STATE_FREE; pmc->pm_state = PMC_STATE_FREE;
pmc->pm_mode = mode;
pmc->pm_caps = caps; pmc->pm_caps = caps;
pmc->pm_flags = pa.pm_flags; pmc->pm_flags = pa.pm_flags;
@ -2729,7 +2790,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
for (n = 0; n < (int) md->pmd_npmc; n++) for (n = 0; n < (int) md->pmd_npmc; n++)
if (pmc_can_allocate_row(n, mode) == 0 && if (pmc_can_allocate_row(n, mode) == 0 &&
pmc_can_allocate_rowindex( pmc_can_allocate_rowindex(
curthread->td_proc, n) == 0 && curthread->td_proc, n, cpu) == 0 &&
(PMC_IS_UNALLOCATED(cpu, n) || (PMC_IS_UNALLOCATED(cpu, n) ||
PMC_IS_SHAREABLE_PMC(cpu, n)) && PMC_IS_SHAREABLE_PMC(cpu, n)) &&
md->pmd_allocate_pmc(cpu, n, pmc, md->pmd_allocate_pmc(cpu, n, pmc,
@ -2740,7 +2801,8 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
for (n = 0; n < (int) md->pmd_npmc; n++) { for (n = 0; n < (int) md->pmd_npmc; n++) {
if (pmc_can_allocate_row(n, mode) == 0 && if (pmc_can_allocate_row(n, mode) == 0 &&
pmc_can_allocate_rowindex( pmc_can_allocate_rowindex(
curthread->td_proc, n) == 0 && curthread->td_proc, n,
PMC_CPU_ANY) == 0 &&
md->pmd_allocate_pmc(curthread->td_oncpu, md->pmd_allocate_pmc(curthread->td_oncpu,
n, pmc, &pa) == 0) n, pmc, &pa) == 0)
break; break;
@ -2760,27 +2822,37 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
break; break;
} }
PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d -> n=%d", /* Fill in the correct value in the ID field */
pmc->pm_event, pmc->pm_class, pmc->pm_mode, n); pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
/* /*
* Configure global pmc's immediately * Configure global pmc's immediately
*/ */
if (PMC_IS_SYSTEM_MODE(pmc->pm_mode)) if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
if ((error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
pmc_save_cpu_binding(&pb);
pmc_select_cpu(cpu);
phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
(error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
(void) md->pmd_release_pmc(cpu, n, pmc); (void) md->pmd_release_pmc(cpu, n, pmc);
pmc_destroy_pmc_descriptor(pmc); pmc_destroy_pmc_descriptor(pmc);
FREE(pmc, M_PMC); FREE(pmc, M_PMC);
pmc = NULL; pmc = NULL;
pmc_restore_cpu_binding(&pb);
error = EPERM;
break; break;
} }
/* pmc_restore_cpu_binding(&pb);
* Mark the row index allocated. }
*/
pmc->pm_rowindex = n;
pmc->pm_state = PMC_STATE_ALLOCATED; pmc->pm_state = PMC_STATE_ALLOCATED;
/* /*
@ -2792,14 +2864,6 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
else else
PMC_MARK_ROW_THREAD(n); PMC_MARK_ROW_THREAD(n);
/*
* If this is a system-wide CPU, mark the CPU it
* was allocated on.
*/
if (PMC_IS_SYSTEM_MODE(mode))
pmc->pm_gv.pm_cpu = cpu;
/* /*
* Register this PMC with the current thread as its owner. * Register this PMC with the current thread as its owner.
*/ */
@ -2816,7 +2880,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
* Return the allocated index. * Return the allocated index.
*/ */
pa.pm_pmcid = n; pa.pm_pmcid = pmc->pm_id;
error = copyout(&pa, arg, sizeof(pa)); error = copyout(&pa, arg, sizeof(pa));
} }
@ -2847,7 +2911,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
break; break;
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
error = EINVAL; error = EINVAL;
break; break;
} }
@ -3022,19 +3086,43 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
break; break;
} }
if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
/*
* If this PMC is attached to its owner (i.e.,
* the process requesting this operation) and
* is running, then attempt to get an
* upto-date reading from hardware for a READ.
* Writes are only allowed when the PMC is
* stopped, so only update the saved value
* field.
*
* If the PMC is not running, or is not
* attached to its owner, read/write to the
* savedvalue field.
*/
ri = PMC_TO_ROWINDEX(pm);
/* read/write the saved value in the PMC record */
mtx_pool_lock_spin(pmc_mtxpool, pm); mtx_pool_lock_spin(pmc_mtxpool, pm);
if (prw.pm_flags & PMC_F_OLDVALUE) cpu = curthread->td_oncpu;
if (prw.pm_flags & PMC_F_OLDVALUE) {
if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
(pm->pm_state == PMC_STATE_RUNNING))
error = (*md->pmd_read_pmc)(cpu, ri,
&oldvalue);
else
oldvalue = pm->pm_gv.pm_savedvalue; oldvalue = pm->pm_gv.pm_savedvalue;
}
if (prw.pm_flags & PMC_F_NEWVALUE) if (prw.pm_flags & PMC_F_NEWVALUE)
pm->pm_gv.pm_savedvalue = prw.pm_value; pm->pm_gv.pm_savedvalue = prw.pm_value;
mtx_pool_unlock_spin(pmc_mtxpool, pm); mtx_pool_unlock_spin(pmc_mtxpool, pm);
} else { /* System mode PMCs */ } else { /* System mode PMCs */
cpu = pm->pm_gv.pm_cpu; cpu = PMC_TO_CPU(pm);
ri = pm->pm_rowindex; ri = PMC_TO_ROWINDEX(pm);
if (pmc_cpu_is_disabled(cpu)) { if (pmc_cpu_is_disabled(cpu)) {
error = ENXIO; error = ENXIO;
@ -3045,6 +3133,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
pmc_save_cpu_binding(&pb); pmc_save_cpu_binding(&pb);
pmc_select_cpu(cpu); pmc_select_cpu(cpu);
critical_enter();
/* save old value */ /* save old value */
if (prw.pm_flags & PMC_F_OLDVALUE) if (prw.pm_flags & PMC_F_OLDVALUE)
if ((error = (*md->pmd_read_pmc)(cpu, ri, if ((error = (*md->pmd_read_pmc)(cpu, ri,
@ -3055,6 +3144,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
error = (*md->pmd_write_pmc)(cpu, ri, error = (*md->pmd_write_pmc)(cpu, ri,
prw.pm_value); prw.pm_value);
error: error:
critical_exit();
pmc_restore_cpu_binding(&pb); pmc_restore_cpu_binding(&pb);
if (error) if (error)
break; break;
@ -3114,7 +3204,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
break; break;
} }
if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
pm->pm_sc.pm_reloadcount = sc.pm_count; pm->pm_sc.pm_reloadcount = sc.pm_count;
else else
pm->pm_sc.pm_initial = sc.pm_count; pm->pm_sc.pm_initial = sc.pm_count;
@ -3142,9 +3232,9 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
if ((error = pmc_find_pmc(pmcid, &pm)) != 0) if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
break; break;
KASSERT(pmcid == pm->pm_rowindex, KASSERT(pmcid == pm->pm_id,
("[pmc,%d] row index %d != id %d", __LINE__, ("[pmc,%d] pmcid %x != id %x", __LINE__,
pm->pm_rowindex, pmcid)); pm->pm_id, pmcid));
if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
break; break;
@ -3184,9 +3274,9 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
if ((error = pmc_find_pmc(pmcid, &pm)) != 0) if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
break; break;
KASSERT(pmcid == pm->pm_rowindex, KASSERT(pmcid == pm->pm_id,
("[pmc,%d] row index %d != pmcid %d", __LINE__, ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
pm->pm_rowindex, pmcid)); pm->pm_id, pmcid));
if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
break; break;
@ -3234,6 +3324,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
{ {
int ri; int ri;
struct pmc *pm; struct pmc *pm;
struct pmc_target *pt;
struct pmc_op_x86_getmsr gm; struct pmc_op_x86_getmsr gm;
PMC_DOWNGRADE_SX(); PMC_DOWNGRADE_SX();
@ -3251,26 +3342,53 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
break; break;
/* /*
* The allocated PMC needs to be a process virtual PMC, * The allocated PMC has to be a process virtual PMC,
* i.e., of type T[CS]. * i.e., of type MODE_T[CS]. Global PMCs can only be
* read using the PMCREAD operation since they may be
* allocated on a different CPU than the one we could
* be running on at the time of the RDPMC instruction.
* *
* Global PMCs can only be read using the PMCREAD * The GETMSR operation is not allowed for PMCs that
* operation since they may be allocated on a * are inherited across processes.
* different CPU than the one we could be running on
* at the time of the read.
*/ */
if (!PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
(pm->pm_flags & PMC_F_DESCENDANTS)) {
error = EINVAL; error = EINVAL;
break; break;
} }
ri = pm->pm_rowindex; /*
* It only makes sense to use a RDPMC (or its
* equivalent instruction on non-x86 architectures) on
* a process that has allocated and attached a PMC to
* itself. Conversely the PMC is only allowed to have
* one process attached to it -- its owner.
*/
if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
LIST_NEXT(pt, pt_next) != NULL ||
pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
error = EINVAL;
break;
}
ri = PMC_TO_ROWINDEX(pm);
if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0) if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
break; break;
if ((error = copyout(&gm, arg, sizeof(gm))) < 0) if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
break; break;
/*
* Mark our process as using MSRs. Update machine
* state using a forced context switch.
*/
pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
pmc_force_context_switch();
} }
break; break;
#endif #endif
@ -3314,13 +3432,13 @@ pmc_configure_log(struct pmc_owner *po, int logfd)
if (po->po_logfd >= 0 && logfd < 0) { if (po->po_logfd >= 0 && logfd < 0) {
/* deconfigure log */ /* deconfigure log */
/* XXX */ /* XXX */
po->po_flags &= ~PMC_FLAG_OWNS_LOGFILE; po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
pmc_maybe_remove_owner(po); pmc_maybe_remove_owner(po);
} else if (po->po_logfd < 0 && logfd >= 0) { } else if (po->po_logfd < 0 && logfd >= 0) {
/* configure log file */ /* configure log file */
/* XXX */ /* XXX */
po->po_flags |= PMC_FLAG_OWNS_LOGFILE; po->po_flags |= PMC_PO_OWNS_LOGFILE;
/* mark process as using HWPMCs */ /* mark process as using HWPMCs */
PROC_LOCK(p); PROC_LOCK(p);
@ -3530,7 +3648,7 @@ pmc_initialize(void)
printf(PMC_MODULE_NAME ":"); printf(PMC_MODULE_NAME ":");
for (n = 0; n < (int) md->pmd_nclass; n++) for (n = 0; n < (int) md->pmd_nclass; n++)
printf(" %s(%d)", printf(" %s(%d)",
pmc_name_of_pmcclass[md->pmd_classes[n]], pmc_name_of_pmcclass[md->pmd_classes[n].pm_class],
md->pmd_nclasspmcs[n]); md->pmd_nclasspmcs[n]);
printf("\n"); printf("\n");
} }

425
sys/dev/hwpmc/hwpmc_piv.c
View File

@ -35,7 +35,7 @@ __FBSDID("$FreeBSD$");
#include <sys/smp.h> #include <sys/smp.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <machine/cputypes.h> #include <machine/apicreg.h>
#include <machine/md_var.h> #include <machine/md_var.h>
/* /*
@ -96,12 +96,52 @@ __FBSDID("$FreeBSD$");
* - Threads of multi-threaded processes that get scheduled on the same * - Threads of multi-threaded processes that get scheduled on the same
* physical CPU are handled correctly. * physical CPU are handled correctly.
* *
* HTT Detection
*
* Not all HTT capable systems will have HTT enabled since users may * Not all HTT capable systems will have HTT enabled since users may
* have turned HTT support off using the appropriate sysctls * have turned HTT support off using the appropriate sysctls
* (machdep.hlt_logical_cpus and machdep.logical_cpus_mask). We * (machdep.hlt_logical_cpus or machdep.logical_cpus_mask). We detect
* detect the presence of HTT by remembering if an initialization was * the presence of HTT by remembering if 'p4_init()' was called for a
* done for a logical CPU. * logical CPU. Note that hwpmc(4) cannot deal with a change in HTT
* status once it is loaded.
* *
* Handling HTT READ / WRITE / START / STOP
*
* PMC resources are shared across multiple logical CPUs. In each
* physical CPU's state we keep track of a 'runcount' which reflects
* the number of PMC-using processes that have been scheduled on the
* logical CPUs of this physical CPU. Process-mode PMC operations
* will actually 'start' or 'stop' hardware only if these are the
* first or last processes respectively to use the hardware. PMC
* values written by a 'write' operation are saved and are transferred
* to hardware at PMC 'start' time if the runcount is 0. If the
* runcount is greater than 0 at the time of a 'start' operation, we
* keep track of the actual hardware value at the time of the 'start'
* operation and use this to adjust the final readings at PMC 'stop'
* or 'read' time.
*
* Execution sequences:
*
* Case 1: CPUx +...- (no overlap)
* CPUy +...-
* RC 0 1 0 1 0
*
* Case 2: CPUx +........- (partial overlap)
* CPUy +........-
* RC 0 1 2 1 0
*
* Case 3: CPUx +..............- (fully overlapped)
* CPUy +.....-
* RC 0 1 2 1 0
*
* Here CPUx and CPUy are one of the two logical processors on a HTT CPU.
*
* Handling HTT CONFIG
*
* Different processes attached to the same PMC may get scheduled on
* the two logical processors in the package. We keep track of config
* and de-config operations using the CFGFLAGS fields of the per-physical
* cpu state.
*/ */
#define P4_PMCS() \ #define P4_PMCS() \
@ -386,9 +426,11 @@ static int p4_system_has_htt;
* [19 struct pmc_hw structures] * [19 struct pmc_hw structures]
* [45 ESCRs status bytes] * [45 ESCRs status bytes]
* [per-cpu spin mutex] * [per-cpu spin mutex]
* [19 flags for holding the config count and runcount] * [19 flag fields for holding config flags and a runcount]
* [19*2 saved value fields] (Thread mode PMC support) * [19*2 hw value fields] (Thread mode PMC support)
* [19*2 pmc value fields] (-do-) * or
* [19*2 EIP values] (Sampling mode PMCs)
* [19*2 pmc value fields] (Thread mode PMC support))
*/ */
struct p4_cpu { struct p4_cpu {
@ -398,12 +440,16 @@ struct p4_cpu {
char pc_escrs[P4_NESCR]; char pc_escrs[P4_NESCR];
struct mtx pc_mtx; /* spin lock */ struct mtx pc_mtx; /* spin lock */
unsigned char pc_flags[P4_NPMCS]; /* 4 bits each: {cfg,run}count */ unsigned char pc_flags[P4_NPMCS]; /* 4 bits each: {cfg,run}count */
pmc_value_t pc_saved[P4_NPMCS * P4_NHTT]; union {
pmc_value_t pc_hw[P4_NPMCS * P4_NHTT];
uintptr_t pc_ip[P4_NPMCS * P4_NHTT];
} pc_si;
pmc_value_t pc_pmc_values[P4_NPMCS * P4_NHTT]; pmc_value_t pc_pmc_values[P4_NPMCS * P4_NHTT];
}; };
#define P4_PCPU_SAVED_VALUE(PC,RI,CPU) (PC)->pc_saved[(RI)*((CPU) & 1)] #define P4_PCPU_PMC_VALUE(PC,RI,CPU) (PC)->pc_pmc_values[(RI)*((CPU) & 1)]
#define P4_PCPU_PMC_VALUE(P,R,C) (P)->pc_pmc_values[(R)*((C) & 1)] #define P4_PCPU_HW_VALUE(PC,RI,CPU) (PC)->pc_si.pc_hw[(RI)*((CPU) & 1)]
#define P4_PCPU_SAVED_IP(PC,RI,CPU) (PC)->pc_si.pc_ip[(RI)*((CPU) & 1)]
#define P4_PCPU_GET_FLAGS(PC,RI,MASK) ((PC)->pc_flags[(RI)] & (MASK)) #define P4_PCPU_GET_FLAGS(PC,RI,MASK) ((PC)->pc_flags[(RI)] & (MASK))
#define P4_PCPU_SET_FLAGS(PC,RI,MASK,VAL) do { \ #define P4_PCPU_SET_FLAGS(PC,RI,MASK,VAL) do { \
@ -417,8 +463,10 @@ struct p4_cpu {
#define P4_PCPU_GET_RUNCOUNT(PC,RI) P4_PCPU_GET_FLAGS(PC,RI,0x0F) #define P4_PCPU_GET_RUNCOUNT(PC,RI) P4_PCPU_GET_FLAGS(PC,RI,0x0F)
#define P4_PCPU_SET_RUNCOUNT(PC,RI,V) P4_PCPU_SET_FLAGS(PC,RI,0x0F,V) #define P4_PCPU_SET_RUNCOUNT(PC,RI,V) P4_PCPU_SET_FLAGS(PC,RI,0x0F,V)
#define P4_PCPU_GET_CFGCOUNT(PC,RI) (P4_PCPU_GET_FLAGS(PC,RI,0xF0) >> 4) #define P4_PCPU_GET_CFGFLAGS(PC,RI) (P4_PCPU_GET_FLAGS(PC,RI,0xF0) >> 4)
#define P4_PCPU_SET_CFGCOUNT(PC,RI,C) P4_PCPU_SET_FLAGS(PC,RI,0xF0,((C) <<4)) #define P4_PCPU_SET_CFGFLAGS(PC,RI,C) P4_PCPU_SET_FLAGS(PC,RI,0xF0,((C) <<4))
#define P4_CPU_TO_FLAG(C) (pmc_cpu_is_logical(cpu) ? 0x2 : 0x1)
/* ESCR row disposition */ /* ESCR row disposition */
static int p4_escrdisp[P4_NESCR]; static int p4_escrdisp[P4_NESCR];
@ -583,10 +631,10 @@ p4_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
(void) pc; (void) pc;
PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
(pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0); (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
/* enable the RDPMC instruction */ /* enable the RDPMC instruction */
if (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
load_cr4(rcr4() | CR4_PCE); load_cr4(rcr4() | CR4_PCE);
PMCDBG(MDP,SWI,2, "cr4=0x%x", rcr4()); PMCDBG(MDP,SWI,2, "cr4=0x%x", rcr4());
@ -642,11 +690,15 @@ p4_read_pmc(int cpu, int ri, pmc_value_t *v)
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
cpu, ri)); cpu, ri));
mode = pm->pm_mode; KASSERT(pd->pm_descr.pd_class == PMC_TO_CLASS(pm),
("[p4,%d] class mismatch pd %d != id class %d", __LINE__,
pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
mode = PMC_TO_MODE(pm);
PMCDBG(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode); PMCDBG(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { if (PMC_TO_CLASS(pm) == PMC_CLASS_TSC) {
KASSERT(PMC_IS_COUNTING_MODE(mode), KASSERT(PMC_IS_COUNTING_MODE(mode),
("[p4,%d] TSC counter in non-counting mode", __LINE__)); ("[p4,%d] TSC counter in non-counting mode", __LINE__));
*v = rdtsc(); *v = rdtsc();
@ -657,13 +709,19 @@ p4_read_pmc(int cpu, int ri, pmc_value_t *v)
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4, KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class)); ("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class));
if (PMC_IS_SYSTEM_MODE(pm->pm_mode))
tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr); tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr);
else
tmp = P4_PCPU_PMC_VALUE(pc,ri,cpu);
if (PMC_IS_SAMPLING_MODE(mode)) if (PMC_IS_VIRTUAL_MODE(mode)) {
*v = -(tmp + 1); /* undo transformation */ if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit overflow */
tmp += (P4_PERFCTR_MASK + 1) -
P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
tmp += P4_PCPU_PMC_VALUE(pc,ri,cpu);
}
if (PMC_IS_SAMPLING_MODE(mode)) /* undo transformation */
*v = P4_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else else
*v = tmp; *v = tmp;
@ -678,6 +736,7 @@ p4_read_pmc(int cpu, int ri, pmc_value_t *v)
static int static int
p4_write_pmc(int cpu, int ri, pmc_value_t v) p4_write_pmc(int cpu, int ri, pmc_value_t v)
{ {
enum pmc_mode mode;
struct pmc *pm; struct pmc *pm;
struct p4_cpu *pc; struct p4_cpu *pc;
const struct pmc_hw *phw; const struct pmc_hw *phw;
@ -697,15 +756,17 @@ p4_write_pmc(int cpu, int ri, pmc_value_t v)
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
cpu, ri)); cpu, ri));
mode = PMC_TO_MODE(pm);
PMCDBG(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri, PMCDBG(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,
pm->pm_mode, v); mode, v);
/* /*
* The P4's TSC register is writeable, but we don't allow a * The P4's TSC register is writeable, but we don't allow a
* write as changing the TSC's value could interfere with * write as changing the TSC's value could interfere with
* other parts of the system. * timekeeping and other system functions.
*/ */
if (pd->pm_descr.pd_class == PMC_CLASS_TSC) if (PMC_TO_CLASS(pm) == PMC_CLASS_TSC)
return 0; return 0;
/* /*
@ -713,10 +774,10 @@ p4_write_pmc(int cpu, int ri, pmc_value_t v)
* sampling mode PMCs, the value to be programmed into the PMC * sampling mode PMCs, the value to be programmed into the PMC
* counter is -(C+1) where 'C' is the requested sample rate. * counter is -(C+1) where 'C' is the requested sample rate.
*/ */
if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) if (PMC_IS_SAMPLING_MODE(mode))
v = -(v + 1); v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) if (PMC_IS_SYSTEM_MODE(mode))
wrmsr(pd->pm_pmc_msr, v); wrmsr(pd->pm_pmc_msr, v);
else else
P4_PCPU_PMC_VALUE(pc,ri,cpu) = v; P4_PCPU_PMC_VALUE(pc,ri,cpu) = v;
@ -730,7 +791,9 @@ p4_write_pmc(int cpu, int ri, pmc_value_t v)
* 'pm' may be NULL to indicate de-configuration. * 'pm' may be NULL to indicate de-configuration.
* *
* On HTT systems, a PMC may get configured twice, once for each * On HTT systems, a PMC may get configured twice, once for each
* "logical" CPU. * "logical" CPU. We track this using the CFGFLAGS field of the
* per-cpu state; this field is a bit mask with one bit each for
* logical CPUs 0 & 1.
*/ */
static int static int
@ -738,7 +801,7 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm)
{ {
struct pmc_hw *phw; struct pmc_hw *phw;
struct p4_cpu *pc; struct p4_cpu *pc;
int cfgcount; int cfgflags, cpuflag;
KASSERT(cpu >= 0 && cpu < mp_ncpus, KASSERT(cpu >= 0 && cpu < mp_ncpus,
("[p4,%d] illegal CPU %d", __LINE__, cpu)); ("[p4,%d] illegal CPU %d", __LINE__, cpu));
@ -753,45 +816,74 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm)
("[p4,%d] hwpmc not unconfigured before re-config", __LINE__)); ("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
mtx_lock_spin(&pc->pc_mtx); mtx_lock_spin(&pc->pc_mtx);
cfgcount = P4_PCPU_GET_CFGCOUNT(pc,ri); cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
KASSERT(cfgcount >= 0 || cfgcount <= 2, KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal cfgcount cfg=%d on cpu=%d ri=%d", __LINE__, ("[p4,%d] illegal cfgflags cfg=%d on cpu=%d ri=%d", __LINE__,
cfgcount, cpu, ri)); cfgflags, cpu, ri));
KASSERT(cfgcount == 0 || phw->phw_pmc, KASSERT(cfgflags == 0 || phw->phw_pmc,
("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count", ("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count",
__LINE__, cpu, ri)); __LINE__, cpu, ri));
PMCDBG(MDP,CFG,1, "cpu=%d ri=%d cfg=%d pm=%p", cpu, ri, cfgcount, PMCDBG(MDP,CFG,1, "cpu=%d ri=%d cfg=%d pm=%p", cpu, ri, cfgflags,
pm); pm);
cpuflag = P4_CPU_TO_FLAG(cpu);
if (pm) { /* config */ if (pm) { /* config */
if (cfgcount == 0) if (cfgflags == 0)
phw->phw_pmc = pm; phw->phw_pmc = pm;
KASSERT(phw->phw_pmc == pm, KASSERT(phw->phw_pmc == pm,
("[p4,%d] cpu=%d ri=%d config %p != hw %p", ("[p4,%d] cpu=%d ri=%d config %p != hw %p",
__LINE__, cpu, ri, pm, phw->phw_pmc)); __LINE__, cpu, ri, pm, phw->phw_pmc));
cfgcount++; cfgflags |= cpuflag;
} else { /* unconfig */ } else { /* unconfig */
--cfgcount; cfgflags &= ~cpuflag;
if (cfgcount == 0)
if (cfgflags == 0)
phw->phw_pmc = NULL; phw->phw_pmc = NULL;
} }
KASSERT(cfgcount >= 0 || cfgcount <= 2, KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__, ("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__,
cfgcount, cpu, ri)); cfgflags, cpu, ri));
P4_PCPU_SET_CFGCOUNT(pc,ri,cfgcount); P4_PCPU_SET_CFGFLAGS(pc,ri,cfgflags);
mtx_unlock_spin(&pc->pc_mtx); mtx_unlock_spin(&pc->pc_mtx);
return 0; return 0;
} }
/*
* Retrieve a configured PMC pointer from hardware state.
*/
static int
p4_get_config(int cpu, int ri, struct pmc **ppm)
{
struct p4_cpu *pc;
struct pmc_hw *phw;
int cfgflags;
pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
phw = pc->pc_hwpmcs[ri];
mtx_lock_spin(&pc->pc_mtx);
cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
mtx_unlock_spin(&pc->pc_mtx);
if (cfgflags & P4_CPU_TO_FLAG(cpu))
*ppm = phw->phw_pmc; /* PMC config'ed on this CPU */
else
*ppm = NULL;
return 0;
}
/* /*
* Allocate a PMC. * Allocate a PMC.
* *
@ -845,11 +937,11 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
pd = &p4_pmcdesc[ri]; pd = &p4_pmcdesc[ri];
PMCDBG(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x " PMCDBG(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
"reqcaps=0x%x\n", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps, "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
pm->pm_caps); pm->pm_caps);
/* check class */ /* check class */
if (pd->pm_descr.pd_class != pm->pm_class) if (pd->pm_descr.pd_class != a->pm_class)
return EINVAL; return EINVAL;
/* check requested capabilities */ /* check requested capabilities */
@ -872,7 +964,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
*/ */
if (p4_system_has_htt && if (p4_system_has_htt &&
PMC_IS_VIRTUAL_MODE(pm->pm_mode) && PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
pmc_getrowdisp(ri) != 0) pmc_getrowdisp(ri) != 0)
return EBUSY; return EBUSY;
@ -898,7 +990,8 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
*/ */
if (P4_EVENT_IS_TI(pevent) && if (P4_EVENT_IS_TI(pevent) &&
PMC_IS_VIRTUAL_MODE(pm->pm_mode) && p4_system_has_htt) PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
p4_system_has_htt)
return EINVAL; return EINVAL;
pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)]; pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
@ -917,7 +1010,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
* should also be free on the current CPU. * should also be free on the current CPU.
*/ */
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
if (P4_ESCR_ROW_DISP_IS_THREAD(escr) || if (P4_ESCR_ROW_DISP_IS_THREAD(escr) ||
pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX) pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX)
continue; continue;
@ -935,7 +1028,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
* ESCRs from rows marked as 'FREE'. * ESCRs from rows marked as 'FREE'.
*/ */
if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) { if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
if (p4_system_has_htt) { if (p4_system_has_htt) {
if (!P4_ESCR_ROW_DISP_IS_FREE(escr)) if (!P4_ESCR_ROW_DISP_IS_FREE(escr))
continue; continue;
@ -963,7 +1056,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
("[p4,%d] illegal ESCR value %d", __LINE__, escr)); ("[p4,%d] illegal ESCR value %d", __LINE__, escr));
/* mark ESCR row mode */ /* mark ESCR row mode */
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */ pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */
P4_ESCR_MARK_ROW_STANDALONE(escr); P4_ESCR_MARK_ROW_STANDALONE(escr);
} else { } else {
@ -1024,7 +1117,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue; pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue;
PMCDBG(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x " PMCDBG(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
"escr=%d escrmsr=0x%x escrval=0x%x\n", pevent->pm_cccr_select, "escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select,
cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue); cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue);
return 0; return 0;
@ -1048,7 +1141,7 @@ p4_release_pmc(int cpu, int ri, struct pmc *pm)
PMCDBG(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr); PMCDBG(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)]; pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
phw = pc->pc_hwpmcs[ri]; phw = pc->pc_hwpmcs[ri];
@ -1120,7 +1213,7 @@ p4_start_pmc(int cpu, int ri)
} }
/* start system mode PMCs directly */ /* start system mode PMCs directly */
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(escrmsr, escrvalue | escrtbits); wrmsr(escrmsr, escrvalue | escrtbits);
wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE); wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE);
return 0; return 0;
@ -1144,11 +1237,6 @@ p4_start_pmc(int cpu, int ri)
rc)); rc));
if (rc == 0) { /* 1st CPU and the non-HTT case */ if (rc == 0) { /* 1st CPU and the non-HTT case */
/*
* Enable the correct bits for this CPU.
*/
escrvalue |= escrtbits;
cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr), KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr),
("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__, ("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__,
@ -1157,36 +1245,24 @@ p4_start_pmc(int cpu, int ri)
/* write out the low 40 bits of the saved value to hardware */ /* write out the low 40 bits of the saved value to hardware */
wrmsr(pd->pm_pmc_msr, wrmsr(pd->pm_pmc_msr,
P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK); P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);
P4_PCPU_SAVED_VALUE(pc,ri,cpu) = P4_PCPU_PMC_VALUE(pc,ri,cpu) &
P4_PERFCTR_MASK;
/* Program the ESCR and CCCR and start the PMC */
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
"escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x\n", cpu, rc,
ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue,
cccrvalue);
} else if (rc == 1) { /* 2nd CPU */ } else if (rc == 1) { /* 2nd CPU */
/* /*
* Retrieve the CCCR and ESCR values from their MSRs, * Stop the PMC and retrieve the CCCR and ESCR values
* and turn on the addition T[0/1] bits for the 2nd * from their MSRs, and turn on the additional T[0/1]
* CPU. Remember the difference between the saved * bits for the 2nd CPU.
* value from the previous 'write()' operation to this
* (PMC,CPU) pair and the current PMC reading; this is
* used at PMCSTOP time to derive the correct
* increment.
*/ */
cccrvalue = rdmsr(pd->pm_cccr_msr); cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
/* check that the configuration bits read back match the PMC */
KASSERT((cccrvalue & P4_CCCR_Tx_MASK) == KASSERT((cccrvalue & P4_CCCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK), (pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK),
("[p4,%d] cpu=%d rc=%d ri=%d CCCR bits 0x%x PMC 0x%x", ("[p4,%d] Extra CCCR bits cpu=%d rc=%d ri=%d "
__LINE__, cpu, rc, ri, cccrvalue & P4_CCCR_Tx_MASK, "cccr=0x%x PMC=0x%x", __LINE__, cpu, rc, ri,
cccrvalue & P4_CCCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK)); pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK));
KASSERT(cccrvalue & P4_CCCR_ENABLE, KASSERT(cccrvalue & P4_CCCR_ENABLE,
("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running", ("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running",
@ -1196,9 +1272,6 @@ p4_start_pmc(int cpu, int ri)
"cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri, "cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,
cccrvalue, cccrtbits)); cccrvalue, cccrtbits));
/* stop PMC */
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
escrvalue = rdmsr(escrmsr); escrvalue = rdmsr(escrmsr);
KASSERT((escrvalue & P4_ESCR_Tx_MASK) == KASSERT((escrvalue & P4_ESCR_Tx_MASK) ==
@ -1207,40 +1280,33 @@ p4_start_pmc(int cpu, int ri)
"escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri, "escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,
escrvalue & P4_ESCR_Tx_MASK, escrvalue & P4_ESCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK)); pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK));
KASSERT((escrvalue & escrtbits) == 0, KASSERT((escrvalue & escrtbits) == 0,
("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d " ("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d "
"escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__, "escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__,
rc, cpu, ri, escrmsr, escrvalue, escrtbits)); rc, cpu, ri, escrmsr, escrvalue, escrtbits));
}
/* read current value and save it */ /* Enable the correct bits for this CPU. */
P4_PCPU_SAVED_VALUE(pc,ri,cpu) =
rdmsr(pd->pm_pmc_msr) & P4_PERFCTR_MASK;
/*
* program the new bits into the ESCR and CCCR,
* starting the PMC in the process.
*/
escrvalue |= escrtbits; escrvalue |= escrtbits;
cccrvalue |= cccrvalue; cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
/* Save HW value at the time of starting hardware */
P4_PCPU_HW_VALUE(pc,ri,cpu) = rdmsr(pd->pm_pmc_msr);
/* Program the ESCR and CCCR and start the PMC */
wrmsr(escrmsr, escrvalue); wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue); wrmsr(pd->pm_cccr_msr, cccrvalue);
PMCDBG(MDP,STA,2,"p4-start/2 cpu=%d rc=%d ri=%d escr=%d"
"escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x",
cpu, rc, ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr,
escrvalue, cccrvalue);
} else
panic("invalid runcount %d\n", rc);
++rc; ++rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc); P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
mtx_unlock_spin(&pc->pc_mtx); mtx_unlock_spin(&pc->pc_mtx);
PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
"escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x v=%jx", cpu, rc,
ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue,
cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu));
return 0; return 0;
} }
@ -1282,7 +1348,7 @@ p4_stop_pmc(int cpu, int ri)
PMCDBG(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri); PMCDBG(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) { if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(pd->pm_cccr_msr, wrmsr(pd->pm_cccr_msr,
pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE); pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE);
return 0; return 0;
@ -1294,12 +1360,9 @@ p4_stop_pmc(int cpu, int ri)
* On HTT machines, this PMC may be in use by two threads * On HTT machines, this PMC may be in use by two threads
* running on two logical CPUS. Thus we look at the * running on two logical CPUS. Thus we look at the
* 'pm_runcount' field and only turn off the appropriate TO/T1 * 'pm_runcount' field and only turn off the appropriate TO/T1
* bits (and keep the PMC running). * bits (and keep the PMC running) if two logical CPUs were
* using the PMC.
* *
* The 'pc_saved' field has the 'diff' between the value in
* the hardware register at PMCSTART time and the nominal
* start value for the PMC. This diff is added to the current
* PMC reading to derived the correct (absolute) return value.
*/ */
/* bits to mask */ /* bits to mask */
@ -1329,54 +1392,157 @@ p4_stop_pmc(int cpu, int ri)
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr; escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
escrvalue = rdmsr(escrmsr); escrvalue = rdmsr(escrmsr);
/* get the current PMC reading */ /* The current CPU should be running on this PMC */
tmp = rdmsr(pd->pm_pmc_msr) & P4_PERFCTR_MASK;
if (rc == 1) { /* need to keep the PMC running */
KASSERT(escrvalue & escrtbits, KASSERT(escrvalue & escrtbits,
("[p4,%d] ESCR T0/T1 mismatch cpu=%d ri=%d escrmsr=0x%x " ("[p4,%d] ESCR T0/T1 mismatch cpu=%d rc=%d ri=%d escrmsr=0x%x "
"escrvalue=0x%x tbits=0x%x", __LINE__, cpu, ri, escrmsr, "escrvalue=0x%x tbits=0x%x", __LINE__, cpu, rc, ri, escrmsr,
escrvalue, escrtbits)); escrvalue, escrtbits));
KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)) ||
KASSERT(PMC_IS_COUNTING_MODE(pm->pm_mode) ||
(cccrvalue & cccrtbits), (cccrvalue & cccrtbits),
("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x " ("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x "
"tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits)); "tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));
/* get the current hardware reading */
tmp = rdmsr(pd->pm_pmc_msr);
if (rc == 1) { /* need to keep the PMC running */
escrvalue &= ~escrtbits; escrvalue &= ~escrtbits;
cccrvalue &= ~cccrtbits; cccrvalue &= ~cccrtbits;
wrmsr(escrmsr, escrvalue); wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue); wrmsr(pd->pm_cccr_msr, cccrvalue);
} }
PMCDBG(MDP,STO,2, "p4-stop/2 cpu=%d rc=%d ri=%d escrmsr=0x%x escrval=0x%x " mtx_unlock_spin(&pc->pc_mtx);
"cccrval=0x%x", cpu, rc, ri, escrmsr, escrvalue, cccrvalue);
/* get the incremental count from this context switch */ PMCDBG(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
tmp -= P4_PCPU_SAVED_VALUE(pc,ri,cpu); "escrval=0x%x cccrval=0x%x v=%jx", cpu, rc, ri, escrmsr,
if ((int64_t) tmp < 0) /* counter wrap-around */ escrvalue, cccrvalue, tmp);
tmp = -tmp + 1;
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit counter overflow */
tmp += (P4_PERFCTR_MASK + 1) - P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp; P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;
mtx_unlock_spin(&pc->pc_mtx);
return 0; return 0;
} }
/* /*
* Handle an interrupt. * Handle an interrupt.
*
* The hardware sets the CCCR_OVF whenever a counter overflow occurs, so the handler
* examines all the 18 CCCR registers, processing the counters that have overflowed.
*
* On HTT machines, multiple logical CPUs may try to enter the NMI service
* routine at the same time.
*/ */
extern volatile lapic_t *lapic;
static void
p4_lapic_enable_pmc_interrupt(void)
{
uint32_t value;
value = lapic->lvt_pcint;
value &= ~APIC_LVT_M;
lapic->lvt_pcint = value;
}
static int static int
p4_intr(int cpu, uintptr_t eip) p4_intr(int cpu, uintptr_t eip)
{ {
(void) cpu; int i, pmc_interrupted;
(void) eip; uint32_t cccrval, pmi_ovf_mask;
struct p4_cpu *pc;
struct pmc_hw *phw;
struct pmc *pm;
pmc_value_t v;
return 0; (void) eip;
PMCDBG(MDP,INT, 1, "cpu=%d eip=%x pcint=0x%x", cpu, eip,
lapic->lvt_pcint);
pmc_interrupted = 0;
pc = (struct p4_cpu *) pmc_pcpu[cpu];
pmi_ovf_mask = pmc_cpu_is_logical(cpu) ?
P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0;
pmi_ovf_mask |= P4_CCCR_OVF;
/*
* Loop through all CCCRs, looking for ones that have the
* OVF_PMI bit set for our logical CPU.
*/
for (i = 1; i < P4_NPMCS; i++) {
cccrval = rdmsr(P4_CCCR_MSR_FIRST + i - 1);
if ((cccrval & pmi_ovf_mask) != pmi_ovf_mask)
continue;
v = rdmsr(P4_PERFCTR_MSR_FIRST + i - 1);
pmc_interrupted = 1;
PMCDBG(MDP,INT, 2, "ri=%d v=%jx", i, v);
/* Stop the counter, and turn off the overflow bit */
cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE);
wrmsr(P4_CCCR_MSR_FIRST + i - 1, cccrval);
phw = pc->pc_hwpmcs[i];
pm = phw->phw_pmc;
/*
* Ignore de-configured or stopped PMCs.
* Also ignore counting mode PMCs that may
* have overflowed their counters.
*/
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
/*
* If the previous sample hasn't been read yet, the
* sampling interrupt is coming in too fast for the
* rest of the system to cope. Do not re-enable the
* counter.
*/
if (P4_PCPU_SAVED_IP(pc,i,cpu)) {
atomic_add_int(&pmc_stats.pm_intr_ignored, 1);
continue;
}
/*
* write the the reload count and restart the
* hardware.
*/
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(
pm->pm_sc.pm_reloadcount);
wrmsr(P4_PERFCTR_MSR_FIRST + i - 1, v);
wrmsr(P4_CCCR_MSR_FIRST + i - 1,
cccrval | P4_CCCR_ENABLE);
}
if (pmc_interrupted) {
/*
* On Intel CPUs, the PMC 'pcint' entry in the LAPIC
* gets masked when a PMC interrupts the CPU. We need
* to unmask this.
*/
p4_lapic_enable_pmc_interrupt();
/* XXX: Invoke helper (non-NMI) interrupt here */
}
return pmc_interrupted;
} }
/* /*
@ -1410,8 +1576,6 @@ p4_describe(int cpu, int ri, struct pmc_info *pi,
return error; return error;
pi->pm_class = pd->pm_descr.pd_class; pi->pm_class = pd->pm_descr.pd_class;
pi->pm_caps = pd->pm_descr.pd_caps;
pi->pm_width = pd->pm_descr.pd_width;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE; pi->pm_enabled = TRUE;
@ -1456,7 +1620,9 @@ pmc_initialize_p4(struct pmc_mdep *pmc_mdep)
case PMC_CPU_INTEL_PIV: case PMC_CPU_INTEL_PIV:
pmc_mdep->pmd_npmc = P4_NPMCS; pmc_mdep->pmd_npmc = P4_NPMCS;
pmc_mdep->pmd_classes[1] = PMC_CLASS_P4; pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P4;
pmc_mdep->pmd_classes[1].pm_caps = P4_PMC_CAPS;
pmc_mdep->pmd_classes[1].pm_width = 40;
pmc_mdep->pmd_nclasspmcs[1] = 18; pmc_mdep->pmd_nclasspmcs[1] = 18;
pmc_mdep->pmd_init = p4_init; pmc_mdep->pmd_init = p4_init;
@ -1466,6 +1632,7 @@ pmc_initialize_p4(struct pmc_mdep *pmc_mdep)
pmc_mdep->pmd_read_pmc = p4_read_pmc; pmc_mdep->pmd_read_pmc = p4_read_pmc;
pmc_mdep->pmd_write_pmc = p4_write_pmc; pmc_mdep->pmd_write_pmc = p4_write_pmc;
pmc_mdep->pmd_config_pmc = p4_config_pmc; pmc_mdep->pmd_config_pmc = p4_config_pmc;
pmc_mdep->pmd_get_config = p4_get_config;
pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc; pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc;
pmc_mdep->pmd_release_pmc = p4_release_pmc; pmc_mdep->pmd_release_pmc = p4_release_pmc;
pmc_mdep->pmd_start_pmc = p4_start_pmc; pmc_mdep->pmd_start_pmc = p4_start_pmc;

38
sys/dev/hwpmc/hwpmc_ppro.c
View File

@ -336,9 +336,15 @@ p6_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
{ {
(void) pc; (void) pc;
PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS);
/* allow the RDPMC instruction if needed */ /* allow the RDPMC instruction if needed */
if (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
load_cr4(rcr4() | CR4_PCE); load_cr4(rcr4() | CR4_PCE);
PMCDBG(MDP,SWI,1, "cr4=0x%x", rcr4());
return 0; return 0;
} }
@ -348,6 +354,8 @@ p6_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
(void) pc; (void) pc;
(void) pp; /* can be NULL */ (void) pp; /* can be NULL */
PMCDBG(MDP,SWO,1, "pc=%p pp=%p cr4=0x%x", pc, pp, rcr4());
/* always turn off the RDPMC instruction */ /* always turn off the RDPMC instruction */
load_cr4(rcr4() & ~CR4_PCE); load_cr4(rcr4() & ~CR4_PCE);
@ -373,7 +381,7 @@ p6_read_pmc(int cpu, int ri, pmc_value_t *v)
return 0; return 0;
tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_MASK; tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_MASK;
if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = -tmp; *v = -tmp;
else else
*v = tmp; *v = tmp;
@ -404,7 +412,7 @@ p6_write_pmc(int cpu, int ri, pmc_value_t v)
PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
pd->pm_pmc_msr, v); pd->pm_pmc_msr, v);
if (PMC_IS_SAMPLING_MODE(pm->pm_mode)) if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = -v; v = -v;
wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_MASK); wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_MASK);
@ -425,6 +433,19 @@ p6_config_pmc(int cpu, int ri, struct pmc *pm)
return 0; return 0;
} }
/*
* Retrieve a configured PMC pointer from hardware state.
*/
static int
p6_get_config(int cpu, int ri, struct pmc **ppm)
{
*ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc;
return 0;
}
/* /*
* A pmc may be allocated to a given row index if: * A pmc may be allocated to a given row index if:
* - the event is valid for this CPU * - the event is valid for this CPU
@ -454,7 +475,7 @@ p6_allocate_pmc(int cpu, int ri, struct pmc *pm,
pm->pm_caps); pm->pm_caps);
/* check class */ /* check class */
if (pd->pm_descr.pd_class != pm->pm_class) if (pd->pm_descr.pd_class != a->pm_class)
return EINVAL; return EINVAL;
/* check requested capabilities */ /* check requested capabilities */
@ -675,8 +696,6 @@ p6_describe(int cpu, int ri, struct pmc_info *pi,
return error; return error;
pi->pm_class = pd->pm_descr.pd_class; pi->pm_class = pd->pm_descr.pd_class;
pi->pm_caps = pd->pm_descr.pd_caps;
pi->pm_width = pd->pm_descr.pd_width;
if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE; pi->pm_enabled = TRUE;
@ -695,7 +714,7 @@ p6_get_msr(int ri, uint32_t *msr)
KASSERT(ri >= 0 && ri < P6_NPMCS, KASSERT(ri >= 0 && ri < P6_NPMCS,
("[p6,%d ri %d out of range", __LINE__, ri)); ("[p6,%d ri %d out of range", __LINE__, ri));
*msr = p6_pmcdesc[ri].pm_pmc_msr; *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0;
return 0; return 0;
} }
@ -722,7 +741,9 @@ pmc_initialize_p6(struct pmc_mdep *pmc_mdep)
p6_cputype = pmc_mdep->pmd_cputype; p6_cputype = pmc_mdep->pmd_cputype;
pmc_mdep->pmd_npmc = P6_NPMCS; pmc_mdep->pmd_npmc = P6_NPMCS;
pmc_mdep->pmd_classes[1] = PMC_CLASS_P6; pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P6;
pmc_mdep->pmd_classes[1].pm_caps = P6_PMC_CAPS;
pmc_mdep->pmd_classes[1].pm_width = 40;
pmc_mdep->pmd_nclasspmcs[1] = 2; pmc_mdep->pmd_nclasspmcs[1] = 2;
pmc_mdep->pmd_init = p6_init; pmc_mdep->pmd_init = p6_init;
@ -732,6 +753,7 @@ pmc_initialize_p6(struct pmc_mdep *pmc_mdep)
pmc_mdep->pmd_read_pmc = p6_read_pmc; pmc_mdep->pmd_read_pmc = p6_read_pmc;
pmc_mdep->pmd_write_pmc = p6_write_pmc; pmc_mdep->pmd_write_pmc = p6_write_pmc;
pmc_mdep->pmd_config_pmc = p6_config_pmc; pmc_mdep->pmd_config_pmc = p6_config_pmc;
pmc_mdep->pmd_get_config = p6_get_config;
pmc_mdep->pmd_allocate_pmc = p6_allocate_pmc; pmc_mdep->pmd_allocate_pmc = p6_allocate_pmc;
pmc_mdep->pmd_release_pmc = p6_release_pmc; pmc_mdep->pmd_release_pmc = p6_release_pmc;
pmc_mdep->pmd_start_pmc = p6_start_pmc; pmc_mdep->pmd_start_pmc = p6_start_pmc;

3
sys/i386/include/pmc_mdep.h
View File

@ -126,6 +126,9 @@
#define P4_CCCR_MSR_FIRST 0x360 /* MSR_BPU_CCCR0 */ #define P4_CCCR_MSR_FIRST 0x360 /* MSR_BPU_CCCR0 */
#define P4_PERFCTR_MSR_FIRST 0x300 /* MSR_BPU_COUNTER0 */ #define P4_PERFCTR_MSR_FIRST 0x300 /* MSR_BPU_COUNTER0 */
#define P4_RELOAD_COUNT_TO_PERFCTR_VALUE(V) (1 - (V))
#define P4_PERFCTR_VALUE_TO_RELOAD_COUNT(P) (1 - (P))
/* Intel PPro, Celeron, P-II, P-III, Pentium-M PMCS */ /* Intel PPro, Celeron, P-II, P-III, Pentium-M PMCS */
#define P6_NPMCS 3 /* 1 TSC + 2 PMCs */ #define P6_NPMCS 3 /* 1 TSC + 2 PMCs */

118
sys/sys/pmc.h
View File

@ -33,13 +33,13 @@
#define PMC_MODULE_NAME "hwpmc" #define PMC_MODULE_NAME "hwpmc"
#define PMC_NAME_MAX 16 /* HW counter name size */ #define PMC_NAME_MAX 16 /* HW counter name size */
#define PMC_CLASS_MAX 4 /* #classes of PMCs in a CPU */ #define PMC_CLASS_MAX 4 /* #classes of PMCs in a system */
/* Kernel<->userland API version number [MMmmpppp] */ /* Kernel<->userland API version number [MMmmpppp] */
#define PMC_VERSION_MAJOR 0x01 #define PMC_VERSION_MAJOR 0x01
#define PMC_VERSION_MINOR 0x01 #define PMC_VERSION_MINOR 0x01
#define PMC_VERSION_PATCH 0x0001 #define PMC_VERSION_PATCH 0x0002
#define PMC_VERSION (PMC_VERSION_MAJOR << 24 | \ #define PMC_VERSION (PMC_VERSION_MAJOR << 24 | \
PMC_VERSION_MINOR << 16 | PMC_VERSION_PATCH) PMC_VERSION_MINOR << 16 | PMC_VERSION_PATCH)
@ -767,15 +767,16 @@ enum pmc_ops {
/* /*
* Flags used in operations. * Flags used in operations on PMCs.
*/ */
#define PMC_F_FORCE 0x00000001 /*OP ADMIN force operation */ #define PMC_F_FORCE 0x00000001 /*OP ADMIN force operation */
#define PMC_F_DESCENDANTS 0x00000002 /*OP ALLOCATE track descendants */ #define PMC_F_DESCENDANTS 0x00000002 /*OP ALLOCATE track descendants */
#define PMC_F_LOG_TC_CSW 0x00000004 /*OP CONFIGURELOG ctx switches */ #define PMC_F_LOG_TC_CSW 0x00000004 /*OP ALLOCATE track ctx switches */
#define PMC_F_LOG_TC_PROCEXIT 0x00000008 /*OP CONFIGURELOG log proc exits */ #define PMC_F_LOG_TC_PROCEXIT 0x00000008 /*OP ALLOCATE log proc exits */
#define PMC_F_NEWVALUE 0x00000010 /*OP RW write new value */ #define PMC_F_NEWVALUE 0x00000010 /*OP RW write new value */
#define PMC_F_OLDVALUE 0x00000020 /*OP RW get old value */ #define PMC_F_OLDVALUE 0x00000020 /*OP RW get old value */
#define PMC_F_ATTACHED_TO_OWNER 0x00010000 /*attached to owner*/
/* /*
* Cookies used to denote allocated PMCs, and the values of PMCs. * Cookies used to denote allocated PMCs, and the values of PMCs.
@ -786,6 +787,32 @@ typedef uint64_t pmc_value_t;
#define PMC_ID_INVALID (~ (pmc_id_t) 0) #define PMC_ID_INVALID (~ (pmc_id_t) 0)
/*
* PMC IDs have the following format:
*
* +--------+----------+-----------+-----------+
* | CPU | PMC MODE | PMC CLASS | ROW INDEX |
* +--------+----------+-----------+-----------+
*
* where each field is 8 bits wide. Field 'CPU' is set to the
* requested CPU for system-wide PMCs or PMC_CPU_ANY for process-mode
* PMCs. Field 'PMC MODE' is the allocated PMC mode. Field 'PMC
* CLASS' is the class of the PMC. Field 'ROW INDEX' is the row index
* for the PMC.
*
* The 'ROW INDEX' ranges over 0..NWPMCS where NHWPMCS is the total
* number of hardware PMCs on this cpu.
*/
#define PMC_ID_TO_ROWINDEX(ID) ((ID) & 0xFF)
#define PMC_ID_TO_CLASS(ID) (((ID) & 0xFF00) >> 8)
#define PMC_ID_TO_MODE(ID) (((ID) & 0xFF0000) >> 16)
#define PMC_ID_TO_CPU(ID) (((ID) & 0xFF000000) >> 24)
#define PMC_ID_MAKE_ID(CPU,MODE,CLASS,ROWINDEX) \
((((CPU) & 0xFF) << 24) | (((MODE) & 0xFF) << 16) | \
(((CLASS) & 0xFF) << 8) | ((ROWINDEX) & 0xFF))
/* /*
* Data structures for system calls supported by the pmc driver. * Data structures for system calls supported by the pmc driver.
*/ */
@ -889,17 +916,15 @@ struct pmc_op_pmcrw {
*/ */
struct pmc_info { struct pmc_info {
uint32_t pm_caps; /* counter capabilities */ char pm_name[PMC_NAME_MAX]; /* pmc name */
enum pmc_class pm_class; /* enum pmc_class */ enum pmc_class pm_class; /* enum pmc_class */
int pm_enabled; /* whether enabled */ int pm_enabled; /* whether enabled */
enum pmc_event pm_event; /* current event */
uint32_t pm_flags; /* counter flags */
enum pmc_mode pm_mode; /* current mode [enum pmc_mode] */
pid_t pm_ownerpid; /* owner, or -1 */
pmc_value_t pm_reloadcount; /* sampling counters only */
enum pmc_disp pm_rowdisp; /* FREE, THREAD or STANDLONE */ enum pmc_disp pm_rowdisp; /* FREE, THREAD or STANDLONE */
uint32_t pm_width; /* width of the PMC */ pid_t pm_ownerpid; /* owner, or -1 */
char pm_name[PMC_NAME_MAX]; /* pmc name */ enum pmc_mode pm_mode; /* current mode [enum pmc_mode] */
enum pmc_event pm_event; /* current event */
uint32_t pm_flags; /* current flags */
pmc_value_t pm_reloadcount; /* sampling counters only */
}; };
struct pmc_op_getpmcinfo { struct pmc_op_getpmcinfo {
@ -914,12 +939,18 @@ struct pmc_op_getpmcinfo {
* Retrieve system CPU information. * Retrieve system CPU information.
*/ */
struct pmc_classinfo {
enum pmc_class pm_class; /* class id */
uint32_t pm_caps; /* counter capabilities */
uint32_t pm_width; /* width of the PMC */
};
struct pmc_op_getcpuinfo { struct pmc_op_getcpuinfo {
enum pmc_cputype pm_cputype; /* what kind of CPU */ enum pmc_cputype pm_cputype; /* what kind of CPU */
uint32_t pm_nclass; /* #classes of PMCs */
uint32_t pm_ncpu; /* number of CPUs */ uint32_t pm_ncpu; /* number of CPUs */
uint32_t pm_npmc; /* #PMCs per CPU */ uint32_t pm_npmc; /* #PMCs per CPU */
enum pmc_class pm_classes[PMC_CLASS_MAX]; uint32_t pm_nclass; /* #classes of PMCs */
struct pmc_classinfo pm_classes[PMC_CLASS_MAX];
}; };
/* /*
@ -1030,7 +1061,7 @@ struct pmc_target {
* Each PMC has precisely one owner, namely the process that allocated * Each PMC has precisely one owner, namely the process that allocated
* the PMC. * the PMC.
* *
* Multiple target process may be being monitored by a PMC. The * A PMC may be attached to multiple target processes. The
* 'pm_targets' field links all the target processes being monitored * 'pm_targets' field links all the target processes being monitored
* by this PMC. * by this PMC.
* *
@ -1049,22 +1080,22 @@ struct pmc {
LIST_HEAD(,pmc_target) pm_targets; /* list of target processes */ LIST_HEAD(,pmc_target) pm_targets; /* list of target processes */
/* /*
* Global PMCs are allocated on a CPU and are not moved around. * System-wide PMCs are allocated on a CPU and are not moved
* For global PMCs we need to record the CPU the PMC was allocated * around. For system-wide PMCs we record the CPU the PMC was
* on. * allocated on in the 'CPU' field of the pmc ID.
* *
* Virtual PMCs run on whichever CPU is currently executing * Virtual PMCs run on whichever CPU is currently executing
* their owner threads. For these PMCs we need to save their * their targets' threads. For these PMCs we need to save
* current PMC counter values when they are taken off CPU. * their current PMC counter values when they are taken off
* CPU.
*/ */
union { union {
uint32_t pm_cpu; /* System-wide PMCs */
pmc_value_t pm_savedvalue; /* Virtual PMCS */ pmc_value_t pm_savedvalue; /* Virtual PMCS */
} pm_gv; } pm_gv;
/* /*
* for sampling modes, we keep track of the PMC's "reload * For sampling mode PMCs, we keep track of the PMC's "reload
* count", which is the counter value to be loaded in when * count", which is the counter value to be loaded in when
* arming the PMC for the next counting session. For counting * arming the PMC for the next counting session. For counting
* modes on PMCs that are read-only (e.g., the x86 TSC), we * modes on PMCs that are read-only (e.g., the x86 TSC), we
@ -1078,14 +1109,18 @@ struct pmc {
} pm_sc; } pm_sc;
uint32_t pm_caps; /* PMC capabilities */ uint32_t pm_caps; /* PMC capabilities */
enum pmc_class pm_class; /* class of PMC */
enum pmc_event pm_event; /* event being measured */ enum pmc_event pm_event; /* event being measured */
uint32_t pm_flags; /* additional flags PMC_F_... */ uint32_t pm_flags; /* additional flags PMC_F_... */
enum pmc_mode pm_mode; /* current mode */
struct pmc_owner *pm_owner; /* owner thread state */ struct pmc_owner *pm_owner; /* owner thread state */
uint32_t pm_rowindex; /* row index */
uint32_t pm_runcount; /* #cpus currently on */ uint32_t pm_runcount; /* #cpus currently on */
enum pmc_state pm_state; /* state (active/inactive only) */ enum pmc_state pm_state; /* current PMC state */
/*
* The PMC ID field encodes the row-index for the PMC, its
* mode, class and the CPU# associated with the PMC.
*/
pmc_id_t pm_id; /* allocated PMC id */
/* md extensions */ /* md extensions */
#if __i386__ #if __i386__
@ -1120,6 +1155,15 @@ struct pmc {
#endif #endif
}; };
/*
* Accessor macros for 'struct pmc'
*/
#define PMC_TO_MODE(P) PMC_ID_TO_MODE((P)->pm_id)
#define PMC_TO_CLASS(P) PMC_ID_TO_CLASS((P)->pm_id)
#define PMC_TO_ROWINDEX(P) PMC_ID_TO_ROWINDEX((P)->pm_id)
#define PMC_TO_CPU(P) PMC_ID_TO_CPU((P)->pm_id)
/* /*
* struct pmc_list * struct pmc_list
* *
@ -1158,11 +1202,12 @@ struct pmc_targetstate {
struct pmc_process { struct pmc_process {
LIST_ENTRY(pmc_process) pp_next; /* hash chain */ LIST_ENTRY(pmc_process) pp_next; /* hash chain */
int pp_refcnt; /* reference count */ int pp_refcnt; /* reference count */
uint32_t pp_flags; /* flags */ uint32_t pp_flags; /* flags PMC_PP_* */
struct proc *pp_proc; /* target thread */ struct proc *pp_proc; /* target thread */
struct pmc_targetstate pp_pmcs[]; /* NHWPMCs */ struct pmc_targetstate pp_pmcs[]; /* NHWPMCs */
}; };
#define PMC_PP_ENABLE_MSR_ACCESS 0x00000001
/* /*
* struct pmc_owner * struct pmc_owner
@ -1179,15 +1224,13 @@ struct pmc_process {
struct pmc_owner { struct pmc_owner {
LIST_ENTRY(pmc_owner) po_next; /* hash chain */ LIST_ENTRY(pmc_owner) po_next; /* hash chain */
LIST_HEAD(, pmc_list) po_pmcs; /* list of owned PMCs */ LIST_HEAD(, pmc_list) po_pmcs; /* list of owned PMCs */
uint32_t po_flags; /* PMC_FLAG_* */ uint32_t po_flags; /* flags PMC_PO_* */
struct proc *po_owner; /* owner proc */ struct proc *po_owner; /* owner proc */
int po_logfd; /* XXX for now */ int po_logfd; /* XXX for now */
}; };
#define PMC_FLAG_IS_OWNER 0x01 #define PMC_PO_HAS_TS_PMC 0x00000001
#define PMC_FLAG_HAS_TS_PMC 0x02 #define PMC_PO_OWNS_LOGFILE 0x00000002
#define PMC_FLAG_OWNS_LOGFILE 0x04
#define PMC_FLAG_ENABLE_MSR_ACCESS 0x08
/* /*
* struct pmc_hw -- describe the state of the PMC hardware * struct pmc_hw -- describe the state of the PMC hardware
@ -1271,12 +1314,11 @@ struct pmc_binding {
*/ */
struct pmc_mdep { struct pmc_mdep {
enum pmc_class pmd_classes[PMC_CLASS_MAX];
int pmd_nclasspmcs[PMC_CLASS_MAX];
uint32_t pmd_cputype; /* from enum pmc_cputype */ uint32_t pmd_cputype; /* from enum pmc_cputype */
uint32_t pmd_nclass; /* # PMC classes supported */
uint32_t pmd_npmc; /* max PMCs per CPU */ uint32_t pmd_npmc; /* max PMCs per CPU */
uint32_t pmd_nclass; /* # PMC classes supported */
struct pmc_classinfo pmd_classes[PMC_CLASS_MAX];
int pmd_nclasspmcs[PMC_CLASS_MAX];
/* /*
* Methods * Methods
@ -1291,6 +1333,7 @@ struct pmc_mdep {
/* configuring/reading/writing the hardware PMCs */ /* configuring/reading/writing the hardware PMCs */
int (*pmd_config_pmc)(int _cpu, int _ri, struct pmc *_pm); int (*pmd_config_pmc)(int _cpu, int _ri, struct pmc *_pm);
int (*pmd_get_config)(int _cpu, int _ri, struct pmc **_ppm);
int (*pmd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value); int (*pmd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value);
int (*pmd_write_pmc)(int _cpu, int _ri, pmc_value_t _value); int (*pmd_write_pmc)(int _cpu, int _ri, pmc_value_t _value);
@ -1392,6 +1435,7 @@ extern unsigned int pmc_debugflags; /* [Maj:12bits] [Min:16bits] [level:4] */
#define PMC_DEBUG_MIN_CFG 10 /* config */ #define PMC_DEBUG_MIN_CFG 10 /* config */
#define PMC_DEBUG_MIN_STA 11 /* start */ #define PMC_DEBUG_MIN_STA 11 /* start */
#define PMC_DEBUG_MIN_STO 12 /* stop */ #define PMC_DEBUG_MIN_STO 12 /* stop */
#define PMC_DEBUG_MIN_INT 13 /* interrupts */
/* CPU */ /* CPU */
#define PMC_DEBUG_MIN_BND 8 /* bind */ #define PMC_DEBUG_MIN_BND 8 /* bind */

2
usr.sbin/pmccontrol/pmccontrol.c
View File

@ -288,7 +288,7 @@ pmcc_do_list_events(void)
eventnamelist = NULL; eventnamelist = NULL;
for (i = 0; i < ci->pm_nclass; i++) { for (i = 0; i < ci->pm_nclass; i++) {
c = ci->pm_classes[i]; c = ci->pm_classes[i].pm_class;
printf("%s\n", pmc_name_of_class(c)); printf("%s\n", pmc_name_of_class(c));
if (pmc_event_names_of_class(c, &eventnamelist, &nevents) < 0) if (pmc_event_names_of_class(c, &eventnamelist, &nevents) < 0)

22
usr.sbin/pmcstat/pmcstat.c
View File

@ -572,21 +572,21 @@ main(int argc, char **argv)
/* compute printout widths */ /* compute printout widths */
STAILQ_FOREACH(ev, &args.pa_head, ev_next) { STAILQ_FOREACH(ev, &args.pa_head, ev_next) {
int pmc_width; int counter_width;
int pmc_display_width; int display_width;
int pmc_header_width; int header_width;
pmc_width = ppmci->pm_pmcs[ev->ev_pmcid].pm_width; (void) pmc_width(ev->ev_pmcid, &counter_width);
pmc_header_width = strlen(ev->ev_name) + 2; /* prefix '%c|' */ header_width = strlen(ev->ev_name) + 2; /* prefix '%c|' */
pmc_display_width = (int) floor(pmc_width / 3.32193) + 1; display_width = (int) floor(counter_width / 3.32193) + 1;
if (pmc_header_width > pmc_display_width) { if (header_width > display_width) {
ev->ev_fieldskip = 0; ev->ev_fieldskip = 0;
ev->ev_fieldwidth = pmc_header_width; ev->ev_fieldwidth = header_width;
} else { } else {
ev->ev_fieldskip = pmc_display_width - ev->ev_fieldskip = display_width -
pmc_header_width; header_width;
ev->ev_fieldwidth = pmc_display_width; ev->ev_fieldwidth = display_width;
} }
} }