From e829eb6d618d215b797c8e9b5f7e8c1ec4fc328d Mon Sep 17 00:00:00 2001 From: Joseph Koshy Date: Sun, 9 Nov 2008 17:37:54 +0000 Subject: [PATCH] - Separate PMC class dependent code from other kinds of machine dependencies. A 'struct pmc_classdep' structure describes operations on PMCs; 'struct pmc_mdep' contains one or more 'struct pmc_classdep' structures depending on the CPU in question. Inside PMC class dependent code, row indices are relative to the PMCs supported by the PMC class; MI code in "hwpmc_mod.c" translates global row indices before invoking class dependent operations. - Augment the OP_GETCPUINFO request with the number of PMCs present in a PMC class. - Move code common to Intel CPUs to file "hwpmc_intel.c". - Move TSC handling to file "hwpmc_tsc.c". --- sys/amd64/include/pmc_mdep.h | 35 ++- sys/conf/files.amd64 | 2 + sys/conf/files.i386 | 2 + sys/conf/files.pc98 | 2 + sys/dev/hwpmc/hwpmc_amd.c | 348 +++++++++++++----------- sys/dev/hwpmc/hwpmc_amd.h | 8 +- sys/dev/hwpmc/hwpmc_intel.c | 227 ++++++++++++++++ sys/dev/hwpmc/hwpmc_mod.c | 354 +++++++++++++++--------- sys/dev/hwpmc/hwpmc_pentium.c | 10 +- sys/dev/hwpmc/hwpmc_pentium.h | 5 +- sys/dev/hwpmc/hwpmc_piv.c | 488 ++++++++++++---------------------- sys/dev/hwpmc/hwpmc_piv.h | 7 +- sys/dev/hwpmc/hwpmc_ppro.c | 339 +++++++++++------------ sys/dev/hwpmc/hwpmc_ppro.h | 5 +- sys/dev/hwpmc/hwpmc_tsc.c | 388 +++++++++++++++++++++++++++ sys/dev/hwpmc/hwpmc_tsc.h | 43 +++ sys/dev/hwpmc/hwpmc_x86.c | 131 ++------- sys/i386/include/pmc_mdep.h | 28 ++ sys/modules/hwpmc/Makefile | 5 +- sys/sys/pmc.h | 96 ++++--- 20 files changed, 1578 insertions(+), 945 deletions(-) create mode 100644 sys/dev/hwpmc/hwpmc_intel.c create mode 100644 sys/dev/hwpmc/hwpmc_tsc.c create mode 100644 sys/dev/hwpmc/hwpmc_tsc.h diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h index a11a82ab7a1e..d4aea66f76e5 100644 --- a/sys/amd64/include/pmc_mdep.h +++ b/sys/amd64/include/pmc_mdep.h @@ -35,8 +35,34 @@ #ifndef _MACHINE_PMC_MDEP_H #define _MACHINE_PMC_MDEP_H 1 +#ifdef _KERNEL +struct pmc_mdep; +#endif + #include #include +#include + +/* + * Intel processors implementing V2 and later of the Intel performance + * measurement architecture have PMCs of the following classes: TSC, + * IAF and IAP. + */ +#define PMC_MDEP_CLASS_INDEX_TSC 0 +#define PMC_MDEP_CLASS_INDEX_K8 1 +#define PMC_MDEP_CLASS_INDEX_P4 1 +#define PMC_MDEP_CLASS_INDEX_IAF 1 +#define PMC_MDEP_CLASS_INDEX_IAP 2 + +/* + * On the amd64 platform we support the following PMCs. + * + * TSC The timestamp counter + * K8 AMD Athlon64 and Opteron PMCs in 64 bit mode. + * PIV Intel P4/HTT and P4/EMT64 + * IAP Intel Core/Core2/Atom CPUs in 64 bits mode. + * IAF Intel fixed-function PMCs in Core2 and later CPUs. + */ union pmc_md_op_pmcallocate { struct pmc_md_amd_op_pmcallocate pm_amd; @@ -55,8 +81,6 @@ union pmc_md_pmc { struct pmc_md_p4_pmc pm_p4; }; -struct pmc; - #define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_rip) #define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_rbp) #define PMC_TRAPFRAME_TO_USER_SP(TF) ((TF)->tf_rsp) @@ -88,5 +112,10 @@ struct pmc; void start_exceptions(void), end_exceptions(void); void pmc_x86_lapic_enable_pmc_interrupt(void); -#endif +struct pmc_mdep *pmc_amd_initialize(void); +void pmc_amd_finalize(struct pmc_mdep *_md); +struct pmc_mdep *pmc_intel_initialize(void); +void pmc_intel_finalize(struct pmc_mdep *_md); + +#endif /* _KERNEL */ #endif /* _MACHINE_PMC_MDEP_H */ diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index cea36950bcd5..e7a1ec74e3c2 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -188,7 +188,9 @@ dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc +dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc +dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/k8temp/k8temp.c optional k8temp dev/kbd/kbd.c optional atkbd | sc | ukbd diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index 31a7c1958b5d..33da00ef4e29 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -186,9 +186,11 @@ dev/hptrr/hptrr_os_bsd.c optional hptrr dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc +dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_pentium.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_ppro.c optional hwpmc +dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/ichwd/ichwd.c optional ichwd dev/if_ndis/if_ndis.c optional ndis diff --git a/sys/conf/files.pc98 b/sys/conf/files.pc98 index 364e8a92597d..eebc4d973272 100644 --- a/sys/conf/files.pc98 +++ b/sys/conf/files.pc98 @@ -110,9 +110,11 @@ dev/ed/if_ed_wd80x3.c optional ed isa dev/fb/fb.c optional fb | gdc dev/fe/if_fe_cbus.c optional fe isa dev/hwpmc/hwpmc_amd.c optional hwpmc +dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_pentium.c optional hwpmc dev/hwpmc/hwpmc_piv.c optional hwpmc dev/hwpmc/hwpmc_ppro.c optional hwpmc +dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/io/iodev.c optional io dev/kbd/kbd.c optional pckbd | sc | ukbd diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index fd4c5bc007d2..2ad9ac165512 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -26,7 +26,6 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. - * */ #include @@ -61,18 +60,6 @@ struct amd_descr { static struct amd_descr amd_pmcdesc[AMD_NPMCS] = { - { - .pm_descr = - { - .pd_name = "TSC", - .pd_class = PMC_CLASS_TSC, - .pd_caps = PMC_CAP_READ, - .pd_width = 64 - }, - .pm_evsel = MSR_TSC, - .pm_perfctr = 0 /* unused */ - }, - { .pm_descr = { @@ -257,6 +244,16 @@ const struct amd_event_code_map amd_event_codes[] = { const int amd_event_codes_size = sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); +/* + * Per-processor information + */ + +struct amd_cpu { + struct pmc_hw pc_amdpmcs[AMD_NPMCS]; +}; + +static struct amd_cpu **amd_pcpu; + /* * read a pmc register */ @@ -267,17 +264,17 @@ amd_read_pmc(int cpu, int ri, pmc_value_t *v) enum pmc_mode mode; const struct amd_descr *pd; struct pmc *pm; - const struct pmc_hw *phw; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); + KASSERT(amd_pcpu[cpu], + ("[amd,%d] null per-cpu, cpu %d", __LINE__, cpu)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; - pd = &amd_pmcdesc[ri]; - pm = phw->phw_pmc; + pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; + pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, @@ -287,15 +284,6 @@ amd_read_pmc(int cpu, int ri, pmc_value_t *v) PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); - /* Reading the TSC is a special case */ - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { - KASSERT(PMC_IS_COUNTING_MODE(mode), - ("[amd,%d] TSC counter in non-counting mode", __LINE__)); - *v = rdtsc(); - PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); - return 0; - } - #ifdef DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, @@ -324,18 +312,16 @@ static int amd_write_pmc(int cpu, int ri, pmc_value_t v) { const struct amd_descr *pd; - struct pmc *pm; - const struct pmc_hw *phw; enum pmc_mode mode; + struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; - pd = &amd_pmcdesc[ri]; - pm = phw->phw_pmc; + pm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; + pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, @@ -343,9 +329,6 @@ amd_write_pmc(int cpu, int ri, pmc_value_t v) mode = PMC_TO_MODE(pm); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; - #ifdef DEBUG KASSERT(pd->pm_descr.pd_class == amd_pmc_class, ("[amd,%d] unknown PMC class (%d)", __LINE__, @@ -380,7 +363,7 @@ amd_config_pmc(int cpu, int ri, struct pmc *pm) KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; + phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL, ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", @@ -397,7 +380,7 @@ amd_config_pmc(int cpu, int ri, struct pmc *pm) static int amd_get_config(int cpu, int ri, struct pmc **ppm) { - *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; + *ppm = amd_pcpu[cpu]->pc_amdpmcs[ri].phw_pmc; return 0; } @@ -474,18 +457,6 @@ amd_allocate_pmc(int cpu, int ri, struct pmc *pm, if ((pd->pd_caps & caps) != caps) return EPERM; - if (pd->pd_class == PMC_CLASS_TSC) { - /* TSC's are always allocated in system-wide counting mode */ - if (a->pm_ev != PMC_EV_TSC_TSC || - a->pm_mode != PMC_MODE_SC) - return EINVAL; - return 0; - } - -#ifdef DEBUG - KASSERT(pd->pd_class == amd_pmc_class, - ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); -#endif pe = a->pm_ev; @@ -556,7 +527,7 @@ amd_release_pmc(int cpu, int ri, struct pmc *pmc) KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; + phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; KASSERT(phw->phw_pmc == NULL, ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); @@ -588,7 +559,7 @@ amd_start_pmc(int cpu, int ri) KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; + phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; @@ -598,15 +569,6 @@ amd_start_pmc(int cpu, int ri) PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; /* TSCs are always running */ - -#ifdef DEBUG - KASSERT(pd->pm_descr.pd_class == amd_pmc_class, - ("[amd,%d] unknown PMC class (%d)", __LINE__, - pd->pm_descr.pd_class)); -#endif - KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, ri, cpu, pd->pm_descr.pd_name)); @@ -637,24 +599,13 @@ amd_stop_pmc(int cpu, int ri) KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; + phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pm = phw->phw_pmc; pd = &amd_pmcdesc[ri]; KASSERT(pm != NULL, ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, cpu, ri)); - - /* can't stop a TSC */ - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; - -#ifdef DEBUG - KASSERT(pd->pm_descr.pd_class == amd_pmc_class, - ("[amd,%d] unknown PMC class (%d)", __LINE__, - pd->pm_descr.pd_class)); -#endif - KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", __LINE__, ri, cpu, pd->pm_descr.pd_name)); @@ -677,10 +628,10 @@ amd_stop_pmc(int cpu, int ri) static int amd_intr(int cpu, struct trapframe *tf) { - int i, error, retval, ri; + int i, error, retval; uint32_t config, evsel, perfctr; struct pmc *pm; - struct pmc_cpu *pc; + struct amd_cpu *pac; struct pmc_hw *phw; pmc_value_t v; @@ -692,11 +643,10 @@ amd_intr(int cpu, struct trapframe *tf) retval = 0; - pc = pmc_pcpu[cpu]; + pac = amd_pcpu[cpu]; /* * look for all PMCs that have interrupted: - * - skip over the TSC [PMC#0] * - look for a running, sampling PMC which has overflowed * and which has a valid 'struct pmc' association * @@ -708,14 +658,12 @@ amd_intr(int cpu, struct trapframe *tf) * interrupt at a time. */ - for (i = 0; retval == 0 && i < AMD_NPMCS-1; i++) { - - ri = i + 1; /* row index; TSC is at ri == 0 */ + for (i = 0; retval == 0 && i < AMD_NPMCS; i++) { if (!AMD_PMC_HAS_OVERFLOWED(i)) continue; - phw = pc->pc_hwpmcs[ri]; + phw = &pac->pc_amdpmcs[i]; KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); @@ -769,7 +717,7 @@ amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) KASSERT(ri >= 0 && ri < AMD_NPMCS, ("[amd,%d] row-index %d out of range", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; + phw = &amd_pcpu[cpu]->pc_amdpmcs[ri]; pd = &amd_pmcdesc[ri]; if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, @@ -804,33 +752,20 @@ amd_get_msr(int ri, uint32_t *msr) ("[amd,%d] ri %d out of range", __LINE__, ri)); *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; - return 0; + + return (0); } /* * processor dependent initialization. */ -/* - * Per-processor data structure - * - * [common stuff] - * [5 struct pmc_hw pointers] - * [5 struct pmc_hw structures] - */ - -struct amd_cpu { - struct pmc_cpu pc_common; - struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; - struct pmc_hw pc_amdpmcs[AMD_NPMCS]; -}; - - static int -amd_init(int cpu) +amd_pcpu_init(struct pmc_mdep *md, int cpu) { - int n; - struct amd_cpu *pcs; + int classindex, first_ri, n; + struct pmc_cpu *pc; + struct amd_cpu *pac; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), @@ -838,29 +773,32 @@ amd_init(int cpu) PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); - pcs = malloc(sizeof(struct amd_cpu), M_PMC, + amd_pcpu[cpu] = pac = malloc(sizeof(struct amd_cpu), M_PMC, M_WAITOK|M_ZERO); - phw = &pcs->pc_amdpmcs[0]; - /* - * Initialize the per-cpu mutex and set the content of the - * hardware descriptors to a known state. + * Set the content of the hardware descriptors to a known + * state and initialize pointers in the MI per-cpu descriptor. */ + pc = pmc_pcpu[cpu]; +#if defined(__amd64__) + classindex = PMC_MDEP_CLASS_INDEX_K8; +#elif defined(__i386__) + classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? + PMC_MDEP_CLASS_INDEX_K8 : PMC_MDEP_CLASS_INDEX_K7; +#endif + first_ri = md->pmd_classdep[classindex].pcd_ri; - for (n = 0; n < AMD_NPMCS; n++, phw++) { + KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu pointer", __LINE__)); + + for (n = 0, phw = pac->pc_amdpmcs; n < AMD_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; - pcs->pc_hwpmcs[n] = phw; + pc->pc_hwpmcs[n + first_ri] = phw; } - /* Mark the TSC as shareable */ - pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; - - pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; - - return 0; + return (0); } @@ -870,11 +808,12 @@ amd_init(int cpu) */ static int -amd_cleanup(int cpu) +amd_pcpu_fini(struct pmc_mdep *md, int cpu) { - int i; + int classindex, first_ri, i; uint32_t evsel; - struct pmc_cpu *pcs; + struct pmc_cpu *pc; + struct amd_cpu *pac; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); @@ -884,7 +823,6 @@ amd_cleanup(int cpu) /* * First, turn off all PMCs on this CPU. */ - for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ evsel = rdmsr(AMD_PMC_EVSEL_0 + i); evsel &= ~AMD_PMC_ENABLE; @@ -894,25 +832,42 @@ amd_cleanup(int cpu) /* * Next, free up allocated space. */ + if ((pac = amd_pcpu[cpu]) == NULL) + return (0); - if ((pcs = pmc_pcpu[cpu]) == NULL) - return 0; + amd_pcpu[cpu] = NULL; #ifdef DEBUG - /* check the TSC */ - KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, - ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); - for (i = 1; i < AMD_NPMCS; i++) { - KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, + for (i = 0; i < AMD_NPMCS; i++) { + KASSERT(pac->pc_amdpmcs[i].phw_pmc == NULL, ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); } #endif - pmc_pcpu[cpu] = NULL; - free(pcs, M_PMC); - return 0; + pc = pmc_pcpu[cpu]; + KASSERT(pc != NULL, ("[amd,%d] NULL per-cpu state", __LINE__)); + +#if defined(__amd64__) + classindex = PMC_MDEP_CLASS_INDEX_K8; +#elif defined(__i386__) + classindex = md->pmd_cputype == PMC_CPU_AMD_K8 ? PMC_MDEP_CLASS_INDEX_K8 : + PMC_MDEP_CLASS_INDEX_K7; +#endif + first_ri = md->pmd_classdep[classindex].pcd_ri; + + /* + * Reset pointers in the MI 'per-cpu' state. + */ + for (i = 0; i < AMD_NPMCS; i++) { + pc->pc_hwpmcs[i + first_ri] = NULL; + } + + + free(pac, M_PMC); + + return (0); } /* @@ -922,11 +877,12 @@ amd_cleanup(int cpu) struct pmc_mdep * pmc_amd_initialize(void) { + int classindex, error, i, nclasses, ncpus; + struct pmc_classdep *pcd; enum pmc_cputype cputype; - enum pmc_class class; struct pmc_mdep *pmc_mdep; + enum pmc_class class; char *name; - int i; /* * The presence of hardware performance counters on the AMD @@ -939,12 +895,16 @@ pmc_amd_initialize(void) class = cputype = -1; name = NULL; switch (cpu_id & 0xF00) { +#if defined(__i386__) case 0x600: /* Athlon(tm) processor */ + classindex = PMC_MDEP_CLASS_INDEX_K7; cputype = PMC_CPU_AMD_K7; class = PMC_CLASS_K7; name = "K7"; break; +#endif case 0xF00: /* Athlon64/Opteron processor */ + classindex = PMC_MDEP_CLASS_INDEX_K8; cputype = PMC_CPU_AMD_K8; class = PMC_CLASS_K8; name = "K8"; @@ -960,53 +920,121 @@ pmc_amd_initialize(void) amd_pmc_class = class; #endif - pmc_mdep = malloc(sizeof(struct pmc_mdep), + /* + * Allocate space for pointers to PMC HW descriptors and for + * the MDEP structure used by MI code. + */ + amd_pcpu = malloc(sizeof(struct amd_cpu *) * pmc_cpu_max(), M_PMC, + M_WAITOK|M_ZERO); + + /* + * These processors have two classes of PMCs: the TSC and + * programmable PMCs. + */ + nclasses = 2; + pmc_mdep = malloc(sizeof(struct pmc_mdep) + nclasses * sizeof (struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO); - pmc_mdep->pmd_cputype = cputype; - pmc_mdep->pmd_npmc = AMD_NPMCS; + pmc_mdep->pmd_cputype = cputype; + pmc_mdep->pmd_nclass = nclasses; - /* this processor has two classes of usable PMCs */ - pmc_mdep->pmd_nclass = 2; + ncpus = pmc_cpu_max(); - /* TSC */ - pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; - pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; - pmc_mdep->pmd_classes[0].pm_width = 64; + /* Initialize the TSC. */ + error = pmc_tsc_initialize(pmc_mdep, ncpus); + if (error) + goto error; - /* AMD K7/K8 PMCs */ - pmc_mdep->pmd_classes[1].pm_class = class; - pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; - pmc_mdep->pmd_classes[1].pm_width = 48; + /* Initialize AMD K7 and K8 PMC handling. */ + pcd = &pmc_mdep->pmd_classdep[classindex]; - pmc_mdep->pmd_nclasspmcs[0] = 1; - pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); + pcd->pcd_caps = AMD_PMC_CAPS; + pcd->pcd_class = class; + pcd->pcd_num = AMD_NPMCS; + pcd->pcd_ri = pmc_mdep->pmd_npmc; + pcd->pcd_width = 48; /* fill in the correct pmc name and class */ - for (i = 1; i < AMD_NPMCS; i++) { + for (i = 0; i < AMD_NPMCS; i++) { (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", name, i-1); amd_pmcdesc[i].pm_descr.pd_class = class; } - pmc_mdep->pmd_init = amd_init; - pmc_mdep->pmd_cleanup = amd_cleanup; - pmc_mdep->pmd_switch_in = amd_switch_in; - pmc_mdep->pmd_switch_out = amd_switch_out; - pmc_mdep->pmd_read_pmc = amd_read_pmc; - pmc_mdep->pmd_write_pmc = amd_write_pmc; - pmc_mdep->pmd_config_pmc = amd_config_pmc; - pmc_mdep->pmd_get_config = amd_get_config; - pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; - pmc_mdep->pmd_release_pmc = amd_release_pmc; - pmc_mdep->pmd_start_pmc = amd_start_pmc; - pmc_mdep->pmd_stop_pmc = amd_stop_pmc; - pmc_mdep->pmd_intr = amd_intr; - pmc_mdep->pmd_describe = amd_describe; - pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ + pcd->pcd_allocate_pmc = amd_allocate_pmc; + pcd->pcd_config_pmc = amd_config_pmc; + pcd->pcd_describe = amd_describe; + pcd->pcd_get_config = amd_get_config; + pcd->pcd_get_msr = amd_get_msr; + pcd->pcd_pcpu_fini = amd_pcpu_fini; + pcd->pcd_pcpu_init = amd_pcpu_init; + pcd->pcd_read_pmc = amd_read_pmc; + pcd->pcd_release_pmc = amd_release_pmc; + pcd->pcd_start_pmc = amd_start_pmc; + pcd->pcd_stop_pmc = amd_stop_pmc; + pcd->pcd_write_pmc = amd_write_pmc; + + pmc_mdep->pmd_pcpu_init = NULL; + pmc_mdep->pmd_pcpu_fini = NULL; + pmc_mdep->pmd_intr = amd_intr; + pmc_mdep->pmd_switch_in = amd_switch_in; + pmc_mdep->pmd_switch_out = amd_switch_out; + + pmc_mdep->pmd_npmc += AMD_NPMCS; PMCDBG(MDP,INI,0,"%s","amd-initialize"); - return pmc_mdep; + return (pmc_mdep); + + error: + if (error) { + free(pmc_mdep, M_PMC); + pmc_mdep = NULL; + } + + return (NULL); +} + +/* + * Finalization code for AMD CPUs. + */ + +void +pmc_amd_finalize(struct pmc_mdep *md) +{ +#if defined(INVARIANTS) + int classindex, i, ncpus, pmcclass; +#endif + + pmc_tsc_finalize(md); + + KASSERT(amd_pcpu != NULL, ("[amd,%d] NULL per-cpu array pointer", + __LINE__)); + +#if defined(INVARIANTS) + switch (md->pmd_cputype) { +#if defined(__i386__) + case PMC_CPU_AMD_K7: + classindex = PMC_MDEP_CLASS_INDEX_K7; + pmcclass = PMC_CLASS_K7; + break; +#endif + default: + classindex = PMC_MDEP_CLASS_INDEX_K8; + pmcclass = PMC_CLASS_K8; + } + + KASSERT(md->pmd_classdep[classindex].pcd_class == pmcclass, + ("[amd,%d] pmc class mismatch", __LINE__)); + + ncpus = pmc_cpu_max(); + + for (i = 0; i < ncpus; i++) + KASSERT(amd_pcpu[i] == NULL, ("[amd,%d] non-null pcpu", + __LINE__)); +#endif + + free(amd_pcpu, M_PMC); + amd_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_amd.h b/sys/dev/hwpmc/hwpmc_amd.h index aa6417b81d7a..b995dbe08c95 100644 --- a/sys/dev/hwpmc/hwpmc_amd.h +++ b/sys/dev/hwpmc/hwpmc_amd.h @@ -44,7 +44,7 @@ #define AMD_PMC_PERFCTR_3 0xC0010007 -#define AMD_NPMCS 5 /* 1 TSC + 4 PMCs */ +#define AMD_NPMCS 4 #define AMD_PMC_COUNTERMASK 0xFF000000 #define AMD_PMC_TO_COUNTER(x) (((x) << 24) & AMD_PMC_COUNTERMASK) @@ -93,11 +93,5 @@ struct pmc_md_amd_pmc { uint32_t pm_amd_evsel; }; -/* - * Prototypes - */ - -struct pmc_mdep *pmc_amd_initialize(void); /* AMD K7/K8 PMCs */ - #endif /* _KERNEL */ #endif /* _DEV_HWPMC_AMD_H_ */ diff --git a/sys/dev/hwpmc/hwpmc_intel.c b/sys/dev/hwpmc/hwpmc_intel.c new file mode 100644 index 000000000000..3c49c57b7640 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_intel.c @@ -0,0 +1,227 @@ +/*- + * Copyright (c) 2008 Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Common code for handling Intel CPUs. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#include +#include +#include + +static int +intel_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) +{ + (void) pc; + + PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, + pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS); + + /* allow the RDPMC instruction if needed */ + if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) + load_cr4(rcr4() | CR4_PCE); + + PMCDBG(MDP,SWI,1, "cr4=0x%jx", (uintmax_t) rcr4()); + + return 0; +} + +static int +intel_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) +{ + (void) pc; + (void) pp; /* can be NULL */ + + PMCDBG(MDP,SWO,1, "pc=%p pp=%p cr4=0x%jx", pc, pp, + (uintmax_t) rcr4()); + + /* always turn off the RDPMC instruction */ + load_cr4(rcr4() & ~CR4_PCE); + + return 0; +} + +struct pmc_mdep * +pmc_intel_initialize(void) +{ + struct pmc_mdep *pmc_mdep; + enum pmc_cputype cputype; + int error, model, nclasses, ncpus; + + KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0, + ("[intel,%d] Initializing non-intel processor", __LINE__)); + + PMCDBG(MDP,INI,0, "intel-initialize cpuid=0x%x", cpu_id); + + cputype = -1; + nclasses = 2; + + switch (cpu_id & 0xF00) { +#if defined(__i386__) + case 0x500: /* Pentium family processors */ + cputype = PMC_CPU_INTEL_P5; + break; + case 0x600: /* Pentium Pro, Celeron, Pentium II & III */ + switch ((cpu_id & 0xF0) >> 4) { /* model number field */ + case 0x1: + cputype = PMC_CPU_INTEL_P6; + break; + case 0x3: case 0x5: + cputype = PMC_CPU_INTEL_PII; + break; + case 0x6: + cputype = PMC_CPU_INTEL_CL; + break; + case 0x7: case 0x8: case 0xA: case 0xB: + cputype = PMC_CPU_INTEL_PIII; + break; + case 0x9: case 0xD: + cputype = PMC_CPU_INTEL_PM; + break; + } + break; +#endif +#if defined(__i386__) || defined(__amd64__) + case 0xF00: /* P4 */ + model = ((cpu_id & 0xF0000) >> 12) | ((cpu_id & 0xF0) >> 4); + if (model >= 0 && model <= 6) /* known models */ + cputype = PMC_CPU_INTEL_PIV; + break; + } +#endif + + if ((int) cputype == -1) { + printf("pmc: Unknown Intel CPU.\n"); + return (NULL); + } + + pmc_mdep = malloc(sizeof(struct pmc_mdep) + nclasses * + sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO); + + pmc_mdep->pmd_cputype = cputype; + pmc_mdep->pmd_nclass = nclasses; + + pmc_mdep->pmd_switch_in = intel_switch_in; + pmc_mdep->pmd_switch_out = intel_switch_out; + + ncpus = pmc_cpu_max(); + + error = pmc_tsc_initialize(pmc_mdep, ncpus); + if (error) + goto error; + + switch (cputype) { +#if defined(__i386__) || defined(__amd64__) + + /* + * Intel Pentium 4 Processors, and P4/EMT64 processors. + */ + + case PMC_CPU_INTEL_PIV: + error = pmc_p4_initialize(pmc_mdep, ncpus); + + KASSERT(md->pmd_npmc == TSC_NPMCS + P4_NPMCS, ("[intel,%d] " + "incorrect npmc count %d", __LINE__, md->pmd_npmc)); + break; +#endif + +#if defined(__i386__) + /* + * P6 Family Processors + */ + + case PMC_CPU_INTEL_P6: + case PMC_CPU_INTEL_CL: + case PMC_CPU_INTEL_PII: + case PMC_CPU_INTEL_PIII: + case PMC_CPU_INTEL_PM: + error = pmc_p6_initialize(pmc_mdep, ncpus); + + KASSERT(md->pmd_npmc == TSC_NPMCS + P6_NPMCS, ("[intel,%d] " + "incorrect npmc count %d", __LINE__, md->pmd_npmc)); + break; + + /* + * Intel Pentium PMCs. + */ + + case PMC_CPU_INTEL_P5: + error = pmc_p5_initialize(pmc_mdep, ncpus); + + KASSERT(md->pmd_npmc == TSC_NPMCS + PENTIUM_NPMCS, ("[intel,%d] " + "incorrect npmc count %d", __LINE__, md->pmd_npmc)); + break; +#endif + + default: + KASSERT(0, ("[intel,%d] Unknown CPU type", __LINE__)); + } + + + error: + if (error) { + free(pmc_mdep, M_PMC); + pmc_mdep = NULL; + } + + return (pmc_mdep); +} + +void +pmc_intel_finalize(struct pmc_mdep *md) +{ + pmc_tsc_finalize(md); + + switch (md->pmd_cputype) { +#if defined(__i386__) || defined(__amd64__) + case PMC_CPU_INTEL_PIV: + pmc_p4_finalize(md); + break; +#endif +#if defined(__i386__) + case PMC_CPU_INTEL_P6: + case PMC_CPU_INTEL_CL: + case PMC_CPU_INTEL_PII: + case PMC_CPU_INTEL_PIII: + case PMC_CPU_INTEL_PM: + pmc_p6_finalize(md); + break; + case PMC_CPU_INTEL_P5: + pmc_p5_finalize(md); + break; +#endif + default: + KASSERT(0, ("[intel,%d] unknown CPU type", __LINE__)); + } +} diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c index 61c07350d4a5..32db1b872440 100644 --- a/sys/dev/hwpmc/hwpmc_mod.c +++ b/sys/dev/hwpmc/hwpmc_mod.c @@ -153,6 +153,11 @@ static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash; static LIST_HEAD(, pmc_owner) pmc_ss_owners; +/* + * A map of row indices to classdep structures. + */ +static struct pmc_classdep **pmc_rowindex_to_classdep; + /* * Prototypes */ @@ -463,7 +468,7 @@ pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS) (void) arg1; (void) arg2; /* unused parameters */ n = sizeof(pmc_debugstr); - newstr = malloc(n, M_PMC, M_ZERO|M_WAITOK); + newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO); (void) strlcpy(newstr, pmc_debugstr, n); error = sysctl_handle_string(oidp, newstr, n, req); @@ -482,6 +487,33 @@ pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS) } #endif +/* + * Map a row index to a classdep structure and return the adjusted row + * index for the PMC class index. + */ +static struct pmc_classdep * +pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri) +{ + struct pmc_classdep *pcd; + + (void) md; + + KASSERT(ri >= 0 && ri < md->pmd_npmc, + ("[pmc,%d] illegal row-index %d", __LINE__, ri)); + + pcd = pmc_rowindex_to_classdep[ri]; + + KASSERT(pcd != NULL, + ("[amd,%d] ri %d null pcd", __LINE__, ri)); + + *adjri = ri - pcd->pcd_ri; + + KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num, + ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri)); + + return (pcd); +} + /* * Concurrency Control * @@ -774,8 +806,7 @@ pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) __LINE__, pp, pm)); #endif - pt = malloc(sizeof(struct pmc_target), M_PMC, M_ZERO|M_WAITOK); - + pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO); pt->pt_process = pp; LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); @@ -1159,13 +1190,14 @@ static void pmc_process_csw_in(struct thread *td) { int cpu; - unsigned int ri; + unsigned int adjri, ri; struct pmc *pm; struct proc *p; struct pmc_cpu *pc; struct pmc_hw *phw; - struct pmc_process *pp; pmc_value_t newvalue; + struct pmc_process *pp; + struct pmc_classdep *pcd; p = td->td_proc; @@ -1212,7 +1244,8 @@ pmc_process_csw_in(struct thread *td) atomic_add_rel_32(&pm->pm_runcount, 1); /* configure the HWPMC we are going to use. */ - md->pmd_config_pmc(cpu, ri, pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + pcd->pcd_config_pmc(cpu, adjri, pm); phw = pc->pc_hwpmcs[ri]; @@ -1247,8 +1280,8 @@ pmc_process_csw_in(struct thread *td) PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); - md->pmd_write_pmc(cpu, ri, newvalue); - md->pmd_start_pmc(cpu, ri); + pcd->pcd_write_pmc(cpu, adjri, newvalue); + pcd->pcd_start_pmc(cpu, adjri); } /* @@ -1270,14 +1303,16 @@ static void pmc_process_csw_out(struct thread *td) { int cpu; - enum pmc_mode mode; - unsigned int ri; + int64_t tmp; struct pmc *pm; struct proc *p; + enum pmc_mode mode; struct pmc_cpu *pc; - struct pmc_process *pp; - int64_t tmp; pmc_value_t newvalue; + unsigned int adjri, ri; + struct pmc_process *pp; + struct pmc_classdep *pcd; + /* * Locate our process descriptor; this may be NULL if @@ -1325,8 +1360,9 @@ pmc_process_csw_out(struct thread *td) for (ri = 0; ri < md->pmd_npmc; ri++) { - pm = NULL; - (void) (*md->pmd_get_config)(cpu, ri, &pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + pm = NULL; + (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); if (pm == NULL) /* nothing at this row index */ continue; @@ -1341,7 +1377,7 @@ pmc_process_csw_out(struct thread *td) /* Stop hardware if not already stopped */ if (pm->pm_stalled == 0) - md->pmd_stop_pmc(cpu, ri); + pcd->pcd_stop_pmc(cpu, adjri); /* reduce this PMC's runcount */ atomic_subtract_rel_32(&pm->pm_runcount, 1); @@ -1361,7 +1397,7 @@ pmc_process_csw_out(struct thread *td) ("[pmc,%d] pp refcnt = %d", __LINE__, pp->pp_refcnt)); - md->pmd_read_pmc(cpu, ri, &newvalue); + pcd->pcd_read_pmc(cpu, adjri, &newvalue); tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); @@ -1412,7 +1448,7 @@ pmc_process_csw_out(struct thread *td) } /* mark hardware as free */ - md->pmd_config_pmc(cpu, ri, NULL); + pcd->pcd_config_pmc(cpu, adjri, NULL); } /* @@ -1857,8 +1893,7 @@ pmc_allocate_owner_descriptor(struct proc *p) poh = &pmc_ownerhash[hindex]; /* allocate space for N pointers and one descriptor struct */ - po = malloc(sizeof(struct pmc_owner), M_PMC, M_ZERO|M_WAITOK); - + po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO); po->po_sscount = po->po_error = po->po_flags = 0; po->po_file = NULL; po->po_owner = p; @@ -1907,12 +1942,9 @@ pmc_find_process_descriptor(struct proc *p, uint32_t mode) * Pre-allocate memory in the FIND_ALLOCATE case since we * cannot call malloc(9) once we hold a spin lock. */ - - if (mode & PMC_FLAG_ALLOCATE) { - /* allocate additional space for 'n' pmc pointers */ + if (mode & PMC_FLAG_ALLOCATE) ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc * - sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK); - } + sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO); mtx_lock_spin(&pmc_processhash_mtx); LIST_FOREACH(pp, pph, pp_next) @@ -1991,7 +2023,7 @@ pmc_allocate_pmc_descriptor(void) { struct pmc *pmc; - pmc = malloc(sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK); + pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO); if (pmc != NULL) { pmc->pm_owner = NULL; @@ -2066,19 +2098,21 @@ pmc_wait_for_pmc_idle(struct pmc *pm) static void pmc_release_pmc_descriptor(struct pmc *pm) { - u_int ri, cpu; enum pmc_mode mode; struct pmc_hw *phw; + u_int adjri, ri, cpu; struct pmc_owner *po; - struct pmc_process *pp; - struct pmc_target *ptgt, *tmp; struct pmc_binding pb; + struct pmc_process *pp; + struct pmc_classdep *pcd; + struct pmc_target *ptgt, *tmp; sx_assert(&pmc_sx, SX_XLOCKED); KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); mode = PMC_TO_MODE(pm); PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, @@ -2112,14 +2146,14 @@ pmc_release_pmc_descriptor(struct pmc *pm) PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); critical_enter(); - md->pmd_stop_pmc(cpu, ri); + pcd->pcd_stop_pmc(cpu, adjri); critical_exit(); } PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); critical_enter(); - md->pmd_config_pmc(cpu, ri, NULL); + pcd->pcd_config_pmc(cpu, adjri, NULL); critical_exit(); /* adjust the global and process count of SS mode PMCs */ @@ -2192,8 +2226,7 @@ pmc_release_pmc_descriptor(struct pmc *pm) /* * Release any MD resources */ - - (void) md->pmd_release_pmc(cpu, ri, pm); + (void) pcd->pcd_release_pmc(cpu, adjri, pm); /* * Update row disposition @@ -2408,16 +2441,19 @@ pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) static int pmc_start(struct pmc *pm) { - int error, cpu, ri; enum pmc_mode mode; struct pmc_owner *po; struct pmc_binding pb; + struct pmc_classdep *pcd; + int adjri, error, cpu, ri; KASSERT(pm != NULL, ("[pmc,%d] null pm", __LINE__)); mode = PMC_TO_MODE(pm); ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); + error = 0; PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); @@ -2430,7 +2466,7 @@ pmc_start(struct pmc *pm) */ if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) - return EDOOFUS; /* programming error */ + return (EDOOFUS); /* programming error */ /* * If this is a sampling mode PMC, log mapping information for @@ -2461,7 +2497,7 @@ pmc_start(struct pmc *pm) pmc_force_context_switch(); } - return error; + return (error); } @@ -2494,7 +2530,7 @@ pmc_start(struct pmc *pm) cpu = PMC_TO_CPU(pm); if (!pmc_cpu_is_active(cpu)) - return ENXIO; + return (ENXIO); pmc_select_cpu(cpu); @@ -2506,16 +2542,16 @@ pmc_start(struct pmc *pm) pm->pm_state = PMC_STATE_RUNNING; critical_enter(); - if ((error = md->pmd_write_pmc(cpu, ri, + if ((error = pcd->pcd_write_pmc(cpu, adjri, PMC_IS_SAMPLING_MODE(mode) ? pm->pm_sc.pm_reloadcount : pm->pm_sc.pm_initial)) == 0) - error = md->pmd_start_pmc(cpu, ri); + error = pcd->pcd_start_pmc(cpu, adjri); critical_exit(); pmc_restore_cpu_binding(&pb); - return error; + return (error); } /* @@ -2525,9 +2561,10 @@ pmc_start(struct pmc *pm) static int pmc_stop(struct pmc *pm) { - int cpu, error, ri; struct pmc_owner *po; struct pmc_binding pb; + struct pmc_classdep *pcd; + int adjri, cpu, error, ri; KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); @@ -2569,10 +2606,11 @@ pmc_stop(struct pmc *pm) pmc_select_cpu(cpu); ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); critical_enter(); - if ((error = md->pmd_stop_pmc(cpu, ri)) == 0) - error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial); + if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0) + error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial); critical_exit(); pmc_restore_cpu_binding(&pb); @@ -2589,7 +2627,7 @@ pmc_stop(struct pmc *pm) } } - return error; + return (error); } @@ -2726,13 +2764,22 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) case PMC_OP_GETCPUINFO: /* CPU information */ { struct pmc_op_getcpuinfo gci; + struct pmc_classinfo *pci; + struct pmc_classdep *pcd; + int cl; gci.pm_cputype = md->pmd_cputype; gci.pm_ncpu = pmc_cpu_max(); gci.pm_npmc = md->pmd_npmc; gci.pm_nclass = md->pmd_nclass; - bcopy(md->pmd_classes, &gci.pm_classes, - sizeof(gci.pm_classes)); + pci = gci.pm_classes; + pcd = md->pmd_classdep; + for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) { + pci->pm_caps = pcd->pcd_caps; + pci->pm_class = pcd->pcd_class; + pci->pm_width = pcd->pcd_width; + pci->pm_num = pcd->pcd_num; + } error = copyout(&gci, arg, sizeof(gci)); } break; @@ -2781,13 +2828,15 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) case PMC_OP_GETPMCINFO: { - uint32_t cpu, n, npmc; - size_t pmcinfo_size; + int ari; struct pmc *pm; - struct pmc_info *p, *pmcinfo; - struct pmc_op_getpmcinfo *gpi; + size_t pmcinfo_size; + uint32_t cpu, n, npmc; struct pmc_owner *po; struct pmc_binding pb; + struct pmc_classdep *pcd; + struct pmc_info *p, *pmcinfo; + struct pmc_op_getpmcinfo *gpi; PMC_DOWNGRADE_SX(); @@ -2819,7 +2868,12 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) for (n = 0; n < md->pmd_npmc; n++, p++) { - if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0) + pcd = pmc_ri_to_classdep(md, n, &ari); + + KASSERT(pcd != NULL, + ("[pmc,%d] null pcd ri=%d", __LINE__, n)); + + if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0) break; if (PMC_ROW_DISP_IS_STANDALONE(n)) @@ -2964,14 +3018,15 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) case PMC_OP_PMCALLOCATE: { - uint32_t caps; + int adjri, n; u_int cpu; - int n; - enum pmc_mode mode; + uint32_t caps; struct pmc *pmc; + enum pmc_mode mode; struct pmc_hw *phw; - struct pmc_op_pmcallocate pa; struct pmc_binding pb; + struct pmc_classdep *pcd; + struct pmc_op_pmcallocate pa; if ((error = copyin(arg, &pa, sizeof(pa))) != 0) break; @@ -3056,7 +3111,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) /* A valid class specifier should have been passed in. */ for (n = 0; n < md->pmd_nclass; n++) - if (md->pmd_classes[n].pm_class == pa.pm_class) + if (md->pmd_classdep[n].pcd_class == pa.pm_class) break; if (n == md->pmd_nclass) { error = EINVAL; @@ -3064,7 +3119,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) } /* The requested PMC capabilities should be feasible. */ - if ((md->pmd_classes[n].pm_caps & caps) != caps) { + if ((md->pmd_classdep[n].pcd_caps & caps) != caps) { error = EOPNOTSUPP; break; } @@ -3091,24 +3146,27 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) if (PMC_IS_SYSTEM_MODE(mode)) { pmc_select_cpu(cpu); - for (n = 0; n < (int) md->pmd_npmc; n++) + for (n = 0; n < (int) md->pmd_npmc; n++) { + pcd = pmc_ri_to_classdep(md, n, &adjri); if (pmc_can_allocate_row(n, mode) == 0 && pmc_can_allocate_rowindex( curthread->td_proc, n, cpu) == 0 && (PMC_IS_UNALLOCATED(cpu, n) || PMC_IS_SHAREABLE_PMC(cpu, n)) && - md->pmd_allocate_pmc(cpu, n, pmc, + pcd->pcd_allocate_pmc(cpu, adjri, pmc, &pa) == 0) break; + } } else { /* Process virtual mode */ for (n = 0; n < (int) md->pmd_npmc; n++) { + pcd = pmc_ri_to_classdep(md, n, &adjri); if (pmc_can_allocate_row(n, mode) == 0 && pmc_can_allocate_rowindex( curthread->td_proc, n, PMC_CPU_ANY) == 0 && - md->pmd_allocate_pmc(curthread->td_oncpu, - n, pmc, &pa) == 0) + pcd->pcd_allocate_pmc(curthread->td_oncpu, + adjri, pmc, &pa) == 0) break; } } @@ -3150,10 +3208,11 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) pmc_select_cpu(cpu); phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; + pcd = pmc_ri_to_classdep(md, n, &adjri); if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || - (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) { - (void) md->pmd_release_pmc(cpu, n, pmc); + (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) { + (void) pcd->pcd_release_pmc(cpu, adjri, pmc); pmc_destroy_pmc_descriptor(pmc); free(pmc, M_PMC); pmc = NULL; @@ -3319,19 +3378,14 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) case PMC_OP_PMCGETMSR: { - int ri; - struct pmc *pm; + int adjri, ri; + struct pmc *pm; struct pmc_target *pt; struct pmc_op_getmsr gm; + struct pmc_classdep *pcd; PMC_DOWNGRADE_SX(); - /* CPU has no 'GETMSR' support */ - if (md->pmd_get_msr == NULL) { - error = ENOSYS; - break; - } - if ((error = copyin(arg, &gm, sizeof(gm))) != 0) break; @@ -3371,8 +3425,15 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) } ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); - if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0) + /* PMC class has no 'GETMSR' support */ + if (pcd->pcd_get_msr == NULL) { + error = ENOSYS; + break; + } + + if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0) break; if ((error = copyout(&gm, arg, sizeof(gm))) < 0) @@ -3436,12 +3497,14 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) case PMC_OP_PMCRW: { - uint32_t cpu, ri; + int adjri; struct pmc *pm; - struct pmc_op_pmcrw *pprw; - struct pmc_op_pmcrw prw; - struct pmc_binding pb; + uint32_t cpu, ri; pmc_value_t oldvalue; + struct pmc_binding pb; + struct pmc_op_pmcrw prw; + struct pmc_classdep *pcd; + struct pmc_op_pmcrw *pprw; PMC_DOWNGRADE_SX(); @@ -3494,6 +3557,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) */ ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); mtx_pool_lock_spin(pmc_mtxpool, pm); cpu = curthread->td_oncpu; @@ -3501,7 +3565,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) if (prw.pm_flags & PMC_F_OLDVALUE) { if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && (pm->pm_state == PMC_STATE_RUNNING)) - error = (*md->pmd_read_pmc)(cpu, ri, + error = (*pcd->pcd_read_pmc)(cpu, adjri, &oldvalue); else oldvalue = pm->pm_gv.pm_savedvalue; @@ -3514,6 +3578,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) } else { /* System mode PMCs */ cpu = PMC_TO_CPU(pm); ri = PMC_TO_ROWINDEX(pm); + pcd = pmc_ri_to_classdep(md, ri, &adjri); if (!pmc_cpu_is_active(cpu)) { error = ENXIO; @@ -3527,12 +3592,12 @@ pmc_syscall_handler(struct thread *td, void *syscall_args) critical_enter(); /* save old value */ if (prw.pm_flags & PMC_F_OLDVALUE) - if ((error = (*md->pmd_read_pmc)(cpu, ri, + if ((error = (*pcd->pcd_read_pmc)(cpu, adjri, &oldvalue))) goto error; /* write out new value */ if (prw.pm_flags & PMC_F_NEWVALUE) - error = (*md->pmd_write_pmc)(cpu, ri, + error = (*pcd->pcd_write_pmc)(cpu, adjri, prw.pm_value); error: critical_exit(); @@ -3900,11 +3965,12 @@ pmc_capture_user_callchain(int cpu, struct trapframe *tf) static void pmc_process_samples(int cpu) { - int n, ri; struct pmc *pm; + int adjri, n, ri; struct thread *td; struct pmc_owner *po; struct pmc_sample *ps; + struct pmc_classdep *pcd; struct pmc_samplebuffer *psb; KASSERT(PCPU_GET(cpuid) == cpu, @@ -3988,7 +4054,11 @@ pmc_process_samples(int cpu) * the next hardclock tick. */ for (n = 0; n < md->pmd_npmc; n++) { - (void) (*md->pmd_get_config)(cpu,n,&pm); + pcd = pmc_ri_to_classdep(md, n, &adjri); + KASSERT(pcd != NULL, + ("[pmc,%d] null pcd ri=%d", __LINE__, n)); + (void) (*pcd->pcd_get_config)(cpu,adjri,&pm); + if (pm == NULL || /* !cfg'ed */ pm->pm_state != PMC_STATE_RUNNING || /* !active */ !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ @@ -3997,7 +4067,7 @@ pmc_process_samples(int cpu) pm->pm_stalled = 0; ri = PMC_TO_ROWINDEX(pm); - (*md->pmd_start_pmc)(cpu, ri); + (*pcd->pcd_start_pmc)(cpu, adjri); } } @@ -4025,12 +4095,13 @@ pmc_process_samples(int cpu) static void pmc_process_exit(void *arg __unused, struct proc *p) { - int is_using_hwpmcs; - int cpu; - unsigned int ri; struct pmc *pm; - struct pmc_process *pp; + int adjri, cpu; + unsigned int ri; + int is_using_hwpmcs; struct pmc_owner *po; + struct pmc_process *pp; + struct pmc_classdep *pcd; pmc_value_t newvalue, tmp; PROC_LOCK(p); @@ -4091,7 +4162,10 @@ pmc_process_exit(void *arg __unused, struct proc *p) * state similar to the CSW_OUT code. */ pm = NULL; - (void) (*md->pmd_get_config)(cpu, ri, &pm); + + pcd = pmc_ri_to_classdep(md, ri, &adjri); + + (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); @@ -4111,7 +4185,7 @@ pmc_process_exit(void *arg __unused, struct proc *p) ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); - (void) md->pmd_stop_pmc(cpu, ri); + (void) pcd->pcd_stop_pmc(cpu, adjri); KASSERT(pm->pm_runcount > 0, ("[pmc,%d] bad runcount ri %d rc %d", @@ -4120,7 +4194,7 @@ pmc_process_exit(void *arg __unused, struct proc *p) /* Stop hardware only if it is actually running */ if (pm->pm_state == PMC_STATE_RUNNING && pm->pm_stalled == 0) { - md->pmd_read_pmc(cpu, ri, &newvalue); + pcd->pcd_read_pmc(cpu, adjri, &newvalue); tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); @@ -4135,7 +4209,7 @@ pmc_process_exit(void *arg __unused, struct proc *p) KASSERT((int) pm->pm_runcount >= 0, ("[pmc,%d] runcount is %d", __LINE__, ri)); - (void) md->pmd_config_pmc(cpu, ri, NULL); + (void) pcd->pcd_config_pmc(cpu, adjri, NULL); } /* @@ -4284,10 +4358,11 @@ static const char *pmc_name_of_pmcclass[] = { static int pmc_initialize(void) { - int cpu, error, n; + int c, cpu, error, n, ri; unsigned int maxcpu; struct pmc_binding pb; struct pmc_sample *ps; + struct pmc_classdep *pcd; struct pmc_samplebuffer *sb; md = NULL; @@ -4340,14 +4415,33 @@ pmc_initialize(void) md = pmc_md_initialize(); - if (md == NULL || md->pmd_init == NULL) - return ENOSYS; + if (md == NULL) + return (ENOSYS); + + KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1, + ("[pmc,%d] no classes or pmcs", __LINE__)); + + /* Compute the map from row-indices to classdep pointers. */ + pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) * + md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); + + for (n = 0; n < md->pmd_npmc; n++) + pmc_rowindex_to_classdep[n] = NULL; + for (ri = c = 0; c < md->pmd_nclass; c++) { + pcd = &md->pmd_classdep[c]; + for (n = 0; n < pcd->pcd_num; n++, ri++) + pmc_rowindex_to_classdep[ri] = pcd; + } + + KASSERT(ri == md->pmd_npmc, + ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__, + ri, md->pmd_npmc)); maxcpu = pmc_cpu_max(); /* allocate space for the per-cpu array */ - pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), - M_PMC, M_WAITOK|M_ZERO); + pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC, + M_WAITOK|M_ZERO); /* per-cpu 'saved values' for managing process-mode PMCs */ pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc, @@ -4355,34 +4449,40 @@ pmc_initialize(void) /* Perform CPU-dependent initialization. */ pmc_save_cpu_binding(&pb); - for (cpu = 0; cpu < maxcpu; cpu++) { + error = 0; + for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) { if (!pmc_cpu_is_active(cpu)) continue; pmc_select_cpu(cpu); - if ((error = md->pmd_init(cpu)) != 0) - break; + pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) + + md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC, + M_WAITOK|M_ZERO); + if (md->pmd_pcpu_init) + error = md->pmd_pcpu_init(cpu); + for (n = 0; error == 0 && n < md->pmd_nclass; n++) + error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu); } pmc_restore_cpu_binding(&pb); - if (error != 0) - return error; + if (error) + return (error); /* allocate space for the sample array */ for (cpu = 0; cpu < maxcpu; cpu++) { if (!pmc_cpu_is_active(cpu)) continue; + sb = malloc(sizeof(struct pmc_samplebuffer) + pmc_nsamples * sizeof(struct pmc_sample), M_PMC, M_WAITOK|M_ZERO); - sb->ps_read = sb->ps_write = sb->ps_samples; sb->ps_fence = sb->ps_samples + pmc_nsamples; + KASSERT(pmc_pcpu[cpu] != NULL, ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); - sb->ps_callchains = malloc(pmc_callchaindepth * - pmc_nsamples * sizeof(uintptr_t), - M_PMC, M_WAITOK|M_ZERO); + sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * + sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) ps->ps_pc = sb->ps_callchains + @@ -4438,10 +4538,11 @@ pmc_initialize(void) if (error == 0) { printf(PMC_MODULE_NAME ":"); for (n = 0; n < (int) md->pmd_nclass; n++) { + pcd = &md->pmd_classdep[n]; printf(" %s/%d/0x%b", - pmc_name_of_pmcclass[md->pmd_classes[n].pm_class], - md->pmd_nclasspmcs[n], - md->pmd_classes[n].pm_caps, + pmc_name_of_pmcclass[pcd->pcd_class], + pcd->pcd_num, + pcd->pcd_caps, "\20" "\1INT\2USR\3SYS\4EDG\5THR" "\6REA\7WRI\10INV\11QUA\12PRC" @@ -4450,14 +4551,14 @@ pmc_initialize(void) printf("\n"); } - return error; + return (error); } /* prepare to be unloaded */ static void pmc_cleanup(void) { - int cpu; + int c, cpu; unsigned int maxcpu; struct pmc_ownerhash *ph; struct pmc_owner *po, *tmp; @@ -4538,20 +4639,9 @@ pmc_cleanup(void) KASSERT(pmc_ss_count == 0, ("[pmc,%d] Global SS count not empty", __LINE__)); - /* Free the per-cpu sample buffers. */ + /* do processor and pmc-class dependent cleanup */ maxcpu = pmc_cpu_max(); - for (cpu = 0; cpu < maxcpu; cpu++) { - if (!pmc_cpu_is_active(cpu)) - continue; - KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, - ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, - cpu)); - free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC); - free(pmc_pcpu[cpu]->pc_sb, M_PMC); - pmc_pcpu[cpu]->pc_sb = NULL; - } - /* do processor dependent cleanup */ PMCDBG(MOD,INI,3, "%s", "md cleanup"); if (md) { pmc_save_cpu_binding(&pb); @@ -4561,15 +4651,28 @@ pmc_cleanup(void) if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL) continue; pmc_select_cpu(cpu); - if (md->pmd_cleanup) - md->pmd_cleanup(cpu); + for (c = 0; c < md->pmd_nclass; c++) + md->pmd_classdep[c].pcd_pcpu_fini(md, cpu); + if (md->pmd_pcpu_fini) + md->pmd_pcpu_fini(cpu); } free(md, M_PMC); md = NULL; pmc_restore_cpu_binding(&pb); } - /* deallocate per-cpu structures */ + /* Free per-cpu descriptors. */ + for (cpu = 0; cpu < maxcpu; cpu++) { + if (!pmc_cpu_is_active(cpu)) + continue; + KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, + ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, + cpu)); + free(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC); + free(pmc_pcpu[cpu]->pc_sb, M_PMC); + free(pmc_pcpu[cpu], M_PMC); + } + free(pmc_pcpu, M_PMC); pmc_pcpu = NULL; @@ -4581,6 +4684,11 @@ pmc_cleanup(void) pmc_pmcdisp = NULL; } + if (pmc_rowindex_to_classdep) { + free(pmc_rowindex_to_classdep, M_PMC); + pmc_rowindex_to_classdep = NULL; + } + pmclog_shutdown(); sx_xunlock(&pmc_sx); /* we are done */ diff --git a/sys/dev/hwpmc/hwpmc_pentium.c b/sys/dev/hwpmc/hwpmc_pentium.c index fc26b5252649..0084fa845f64 100644 --- a/sys/dev/hwpmc/hwpmc_pentium.c +++ b/sys/dev/hwpmc/hwpmc_pentium.c @@ -44,8 +44,14 @@ __FBSDID("$FreeBSD$"); */ int -pmc_initialize_p5(struct pmc_mdep *pmc_mdep) +pmc_p5_initialize(struct pmc_mdep *pmc_mdep, int ncpus) +{ + (void) pmc_mdep; (void) ncpus; + return (ENOSYS); /* nothing here yet */ +} + +void +pmc_p5_finalize(struct pmc_mdep *pmc_mdep) { (void) pmc_mdep; - return ENOSYS; /* nothing here yet */ } diff --git a/sys/dev/hwpmc/hwpmc_pentium.h b/sys/dev/hwpmc/hwpmc_pentium.h index 36ecc64b60a6..9bb8e781fbe7 100644 --- a/sys/dev/hwpmc/hwpmc_pentium.h +++ b/sys/dev/hwpmc/hwpmc_pentium.h @@ -33,7 +33,7 @@ /* Intel Pentium PMCs */ -#define PENTIUM_NPMCS 3 /* 1 TSC + 2 PMCs */ +#define PENTIUM_NPMCS 2 #define PENTIUM_CESR_PC1 (1 << 25) #define PENTIUM_CESR_CC1_MASK 0x01C00000 #define PENTIUM_CESR_TO_CC1(C) (((C) & 0x07) << 22) @@ -66,7 +66,8 @@ struct pmc_md_pentium_pmc { * Prototypes */ -int pmc_initialize_p5(struct pmc_mdep *); /* Pentium PMCs */ +int pmc_p5_initialize(struct pmc_mdep *_md, int _ncpus); +void pmc_p5_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* _DEV_HWPMC_PENTIUM_H_ */ diff --git a/sys/dev/hwpmc/hwpmc_piv.c b/sys/dev/hwpmc/hwpmc_piv.c index b8f530ce334f..601395c135e8 100644 --- a/sys/dev/hwpmc/hwpmc_piv.c +++ b/sys/dev/hwpmc/hwpmc_piv.c @@ -153,11 +153,6 @@ __FBSDID("$FreeBSD$"); * the two logical processors in the package. We keep track of config * and de-config operations using the CFGFLAGS fields of the per-physical * cpu state. - * - * Handling TSCs - * - * TSCs are architectural state and each CPU in a HTT pair has its own - * TSC register. */ #define P4_PMCS() \ @@ -364,28 +359,6 @@ struct p4pmc_descr { }; static struct p4pmc_descr p4_pmcdesc[P4_NPMCS] = { - - /* - * TSC descriptor - */ - - { - .pm_descr = - { - .pd_name = "TSC", - .pd_class = PMC_CLASS_TSC, - .pd_caps = PMC_CAP_READ | PMC_CAP_WRITE, - .pd_width = 64 - }, - .pm_pmcnum = ~0, - .pm_cccr_msr = ~0, - .pm_pmc_msr = 0x10, - }, - - /* - * P4 PMCS - */ - #define P4_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE | \ @@ -435,8 +408,6 @@ static int p4_system_has_htt; /* * Per-CPU data structure for P4 class CPUs * - * [common stuff] - * [19 struct pmc_hw pointers] * [19 struct pmc_hw structures] * [45 ESCRs status bytes] * [per-cpu spin mutex] @@ -448,8 +419,6 @@ static int p4_system_has_htt; */ struct p4_cpu { - struct pmc_cpu pc_common; - struct pmc_hw *pc_hwpmcs[P4_NPMCS]; struct pmc_hw pc_p4pmcs[P4_NPMCS]; char pc_escrs[P4_NESCR]; struct mtx pc_mtx; /* spin lock */ @@ -463,19 +432,7 @@ struct p4_cpu { pmc_value_t pc_pmc_values[P4_NPMCS * P4_NHTT]; }; -/* - * A 'logical' CPU shares PMC resources with partner 'physical' CPU, - * except the TSC, which is architectural and hence seperate. The - * 'logical' CPU descriptor thus has pointers to the physical CPUs - * descriptor state except for the TSC (rowindex 0) which is not - * shared. - */ - -struct p4_logicalcpu { - struct pmc_cpu pc_common; - struct pmc_hw *pc_hwpmcs[P4_NPMCS]; - struct pmc_hw pc_tsc; -}; +static struct p4_cpu **p4_pcpu; #define P4_PCPU_PMC_VALUE(PC,RI,CPU) (PC)->pc_pmc_values[(RI)*((CPU) & 1)] #define P4_PCPU_HW_VALUE(PC,RI,CPU) (PC)->pc_si.pc_hw[(RI)*((CPU) & 1)] @@ -579,8 +536,8 @@ p4_find_event(enum pmc_event ev) if (p4_events[n].pm_event == ev) break; if (n == P4_NEVENTS) - return NULL; - return &p4_events[n]; + return (NULL); + return (&p4_events[n]); } /* @@ -588,13 +545,13 @@ p4_find_event(enum pmc_event ev) */ static int -p4_init(int cpu) +p4_pcpu_init(struct pmc_mdep *md, int cpu) { - int n, phycpu; char *pescr; - struct p4_cpu *pcs; - struct p4_logicalcpu *plcs; + int n, first_ri, phycpu; struct pmc_hw *phw; + struct p4_cpu *p4c; + struct pmc_cpu *pc, *plc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p4,%d] insane cpu number %d", __LINE__, cpu)); @@ -602,6 +559,8 @@ p4_init(int cpu) PMCDBG(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu, pmc_cpu_is_primary(cpu) != 0); + first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri; + /* * The two CPUs in an HT pair share their per-cpu state. * @@ -619,56 +578,50 @@ p4_init(int cpu) p4_system_has_htt = 1; phycpu = P4_TO_HTT_PRIMARY(cpu); - pcs = (struct p4_cpu *) pmc_pcpu[phycpu]; - PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pcs=%p", - cpu, phycpu, pcs); - KASSERT(pcs, - ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d", __LINE__, - cpu, phycpu)); - if (pcs == NULL) /* decline to init */ - return ENXIO; + pc = pmc_pcpu[phycpu]; + plc = pmc_pcpu[cpu]; - plcs = malloc(sizeof(struct p4_logicalcpu), - M_PMC, M_WAITOK|M_ZERO); + KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__)); - /* The TSC is architectural state and is not shared */ - plcs->pc_hwpmcs[0] = &plcs->pc_tsc; - plcs->pc_tsc.phw_state = PMC_PHW_FLAG_IS_ENABLED | - PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | - PMC_PHW_FLAG_IS_SHAREABLE; + PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu, + phycpu, pc); + KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d", + __LINE__, cpu, phycpu)); - /* Other PMCs are shared with the physical CPU */ - for (n = 1; n < P4_NPMCS; n++) - plcs->pc_hwpmcs[n] = pcs->pc_hwpmcs[n]; + /* PMCs are shared with the physical CPU. */ + for (n = 0; n < P4_NPMCS; n++) + plc->pc_hwpmcs[n + first_ri] = + pc->pc_hwpmcs[n + first_ri]; - pmc_pcpu[cpu] = (struct pmc_cpu *) plcs; - return 0; + return (0); } - pcs = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO); + p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO); - if (pcs == NULL) - return ENOMEM; - phw = pcs->pc_p4pmcs; + if (p4c == NULL) + return (ENOMEM); + + pc = pmc_pcpu[cpu]; + + KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu)); + + p4_pcpu[cpu] = p4c; + phw = p4c->pc_p4pmcs; for (n = 0; n < P4_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; - pcs->pc_hwpmcs[n] = phw; + pc->pc_hwpmcs[n + first_ri] = phw; } - /* Mark the TSC as shareable */ - pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; - - pescr = pcs->pc_escrs; + pescr = p4c->pc_escrs; for (n = 0; n < P4_NESCR; n++) *pescr++ = P4_INVALID_PMC_INDEX; - pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; - mtx_init(&pcs->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN); + mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN); - return 0; + return (0); } /* @@ -676,74 +629,39 @@ p4_init(int cpu) */ static int -p4_cleanup(int cpu) +p4_pcpu_fini(struct pmc_mdep *md, int cpu) { - int i; - struct p4_cpu *pcs; + int first_ri, i; + struct p4_cpu *p4c; + struct pmc_cpu *pc; PMCDBG(MDP,INI,0, "p4-cleanup cpu=%d", cpu); - if ((pcs = (struct p4_cpu *) pmc_pcpu[cpu]) == NULL) - return 0; + pc = pmc_pcpu[cpu]; + first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri; + + for (i = 0; i < P4_NPMCS; i++) + pc->pc_hwpmcs[i + first_ri] = NULL; + + if (!pmc_cpu_is_primary(cpu) && (cpu & 1)) + return (0); + + p4c = p4_pcpu[cpu]; + + KASSERT(p4c != NULL, ("[p4,%d] NULL pcpu", __LINE__)); /* Turn off all PMCs on this CPU */ for (i = 0; i < P4_NPMCS - 1; i++) wrmsr(P4_CCCR_MSR_FIRST + i, rdmsr(P4_CCCR_MSR_FIRST + i) & ~P4_CCCR_ENABLE); - /* - * If the CPU is physical we need to teardown the - * full MD state. - */ - if (!P4_CPU_IS_HTT_SECONDARY(cpu)) - mtx_destroy(&pcs->pc_mtx); + mtx_destroy(&p4c->pc_mtx); - free(pcs, M_PMC); + free(p4c, M_PMC); - pmc_pcpu[cpu] = NULL; + p4_pcpu[cpu] = NULL; - return 0; -} - -/* - * Context switch in. - */ - -static int -p4_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) -{ - (void) pc; - - PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, - (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); - - /* enable the RDPMC instruction */ - if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) - load_cr4(rcr4() | CR4_PCE); - - PMCDBG(MDP,SWI,2, "cr4=0x%x", (uint32_t) rcr4()); - - return 0; -} - -/* - * Context switch out. - */ - -static int -p4_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) -{ - (void) pc; - (void) pp; /* can be null */ - - PMCDBG(MDP,SWO,1, "pc=%p pp=%p", pc, pp); - - /* always disallow the RDPMC instruction */ - load_cr4(rcr4() & ~CR4_PCE); - - PMCDBG(MDP,SWO,2, "cr4=0x%x", (uint32_t) rcr4()); - - return 0; + return (0); } /* @@ -753,50 +671,27 @@ p4_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) static int p4_read_pmc(int cpu, int ri, pmc_value_t *v) { + struct pmc *pm; + pmc_value_t tmp; + struct p4_cpu *pc; enum pmc_mode mode; struct p4pmc_descr *pd; - struct pmc *pm; - struct p4_cpu *pc; - struct pmc_hw *phw; - pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p4,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < P4_NPMCS, ("[p4,%d] illegal row-index %d", __LINE__, ri)); - - if (ri == 0) { /* TSC */ -#ifdef DEBUG - pc = (struct p4_cpu *) pmc_pcpu[cpu]; - phw = pc->pc_hwpmcs[ri]; - pm = phw->phw_pmc; - - KASSERT(pm, ("[p4,%d] cpu=%d ri=%d not configured", __LINE__, - cpu, ri)); - KASSERT(PMC_TO_CLASS(pm) == PMC_CLASS_TSC, - ("[p4,%d] cpu=%d ri=%d not a TSC (%d)", __LINE__, cpu, ri, - PMC_TO_CLASS(pm))); - KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)), - ("[p4,%d] TSC counter in non-counting mode", __LINE__)); -#endif - *v = rdtsc(); - PMCDBG(MDP,REA,2, "p4-read -> %jx", *v); - return 0; - } - - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; - pd = &p4_pmcdesc[ri]; - pm = phw->phw_pmc; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + pm = pc->pc_p4pmcs[ri].phw_pmc; + pd = &p4_pmcdesc[ri]; KASSERT(pm != NULL, - ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, - cpu, ri)); + ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); KASSERT(pd->pm_descr.pd_class == PMC_TO_CLASS(pm), ("[p4,%d] class mismatch pd %d != id class %d", __LINE__, - pd->pm_descr.pd_class, PMC_TO_CLASS(pm))); + pd->pm_descr.pd_class, PMC_TO_CLASS(pm))); mode = PMC_TO_MODE(pm); @@ -822,7 +717,8 @@ p4_read_pmc(int cpu, int ri, pmc_value_t *v) *v = tmp; PMCDBG(MDP,REA,2, "p4-read -> %jx", *v); - return 0; + + return (0); } /* @@ -843,29 +739,8 @@ p4_write_pmc(int cpu, int ri, pmc_value_t v) KASSERT(ri >= 0 && ri < P4_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); - - /* - * The P4's TSC register is writeable, but we don't allow a - * write as changing the TSC's value could interfere with - * timekeeping and other system functions. - */ - if (ri == 0) { -#ifdef DEBUG - pc = (struct p4_cpu *) pmc_pcpu[cpu]; - phw = pc->pc_hwpmcs[ri]; - pm = phw->phw_pmc; - KASSERT(pm, ("[p4,%d] cpu=%d ri=%d not configured", __LINE__, - cpu, ri)); - KASSERT(PMC_TO_CLASS(pm) == PMC_CLASS_TSC, - ("[p4,%d] cpu=%d ri=%d not a TSC (%d)", __LINE__, - cpu, ri, PMC_TO_CLASS(pm))); -#endif - return 0; - } - - /* Shared PMCs */ - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + phw = &pc->pc_p4pmcs[ri]; pm = phw->phw_pmc; pd = &p4_pmcdesc[ri]; @@ -891,7 +766,7 @@ p4_write_pmc(int cpu, int ri, pmc_value_t v) else P4_PCPU_PMC_VALUE(pc,ri,cpu) = v; - return 0; + return (0); } /* @@ -914,25 +789,14 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm) KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p4,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < P4_NPMCS, ("[p4,%d] illegal row-index %d", __LINE__, ri)); PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); - if (ri == 0) { /* TSC */ - pc = (struct p4_cpu *) pmc_pcpu[cpu]; - phw = pc->pc_hwpmcs[ri]; - - KASSERT(pm == NULL || phw->phw_pmc == NULL, - ("[p4,%d] hwpmc doubly config'ed", __LINE__)); - phw->phw_pmc = pm; - return 0; - } - - /* Shared PMCs */ - - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + phw = &pc->pc_p4pmcs[ri]; KASSERT(pm == NULL || phw->phw_pmc == NULL || (p4_system_has_htt && phw->phw_pmc == pm), @@ -975,7 +839,7 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm) mtx_unlock_spin(&pc->pc_mtx); - return 0; + return (0); } /* @@ -985,19 +849,22 @@ p4_config_pmc(int cpu, int ri, struct pmc *pm) static int p4_get_config(int cpu, int ri, struct pmc **ppm) { - struct p4_cpu *pc; - struct pmc_hw *phw; int cfgflags; + struct p4_cpu *pc; - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[p4,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < P4_NPMCS, + ("[p4,%d] illegal row-index %d", __LINE__, ri)); + + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; mtx_lock_spin(&pc->pc_mtx); cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri); mtx_unlock_spin(&pc->pc_mtx); if (cfgflags & P4_CPU_TO_FLAG(cpu)) - *ppm = phw->phw_pmc; /* PMC config'ed on this CPU */ + *ppm = pc->pc_p4pmcs[ri].phw_pmc; /* PMC config'ed on this CPU */ else *ppm = NULL; @@ -1062,31 +929,23 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm, /* check class */ if (pd->pm_descr.pd_class != a->pm_class) - return EINVAL; + return (EINVAL); /* check requested capabilities */ caps = a->pm_caps; if ((pd->pm_descr.pd_caps & caps) != caps) - return EPERM; - - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { - /* TSC's are always allocated in system-wide counting mode */ - if (a->pm_ev != PMC_EV_TSC_TSC || - a->pm_mode != PMC_MODE_SC) - return EINVAL; - return 0; - } + return (EPERM); /* * If the system has HTT enabled, and the desired allocation * mode is process-private, and the PMC row disposition is not - * free (0), decline the allocation. + * FREE (0), decline the allocation. */ if (p4_system_has_htt && PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) && pmc_getrowdisp(ri) != 0) - return EBUSY; + return (EBUSY); KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4, ("[p4,%d] unknown PMC class %d", __LINE__, @@ -1094,10 +953,10 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm, if (pm->pm_event < PMC_EV_P4_FIRST || pm->pm_event > PMC_EV_P4_LAST) - return EINVAL; + return (EINVAL); if ((pevent = p4_find_event(pm->pm_event)) == NULL) - return ESRCH; + return (ESRCH); PMCDBG(MDP,ALL,2, "pevent={ev=%d,escrsel=0x%x,cccrsel=0x%x,isti=%d}", pevent->pm_event, pevent->pm_escr_eventselect, @@ -1112,9 +971,9 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm, if (P4_EVENT_IS_TI(pevent) && PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) && p4_system_has_htt) - return EINVAL; + return (EINVAL); - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; found = 0; @@ -1170,7 +1029,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm, } if (found == 0) - return ESRCH; + return (ESRCH); KASSERT((int) escr >= 0 && escr < P4_NESCR, ("[p4,%d] illegal ESCR value %d", __LINE__, escr)); @@ -1243,7 +1102,7 @@ p4_allocate_pmc(int cpu, int ri, struct pmc *pm, "escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select, cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue); - return 0; + return (0); } /* @@ -1254,21 +1113,19 @@ static int p4_release_pmc(int cpu, int ri, struct pmc *pm) { enum pmc_p4escr escr; - struct pmc_hw *phw; struct p4_cpu *pc; - if (p4_pmcdesc[ri].pm_descr.pd_class == PMC_CLASS_TSC) - return 0; + KASSERT(ri >= 0 && ri < P4_NPMCS, + ("[p4,%d] illegal row-index %d", __LINE__, ri)); escr = pm->pm_md.pm_p4.pm_p4_escr; PMCDBG(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr); if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - KASSERT(phw->phw_pmc == NULL, + KASSERT(pc->pc_p4pmcs[ri].phw_pmc == NULL, ("[p4,%d] releasing configured PMC ri=%d", __LINE__, ri)); P4_ESCR_UNMARK_ROW_STANDALONE(escr); @@ -1279,7 +1136,7 @@ p4_release_pmc(int cpu, int ri, struct pmc *pm) } else P4_ESCR_UNMARK_ROW_THREAD(escr); - return 0; + return (0); } /* @@ -1290,31 +1147,25 @@ static int p4_start_pmc(int cpu, int ri) { int rc; - uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits; struct pmc *pm; struct p4_cpu *pc; - struct pmc_hw *phw; struct p4pmc_descr *pd; + uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p4,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < P4_NPMCS, ("[p4,%d] illegal row-index %d", __LINE__, ri)); - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; - pm = phw->phw_pmc; - pd = &p4_pmcdesc[ri]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + pm = pc->pc_p4pmcs[ri].phw_pmc; + pd = &p4_pmcdesc[ri]; KASSERT(pm != NULL, - ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, - cpu, ri)); + ("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri)); PMCDBG(MDP,STA,1, "p4-start cpu=%d ri=%d", cpu, ri); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) /* TSC are always on */ - return 0; - KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4, ("[p4,%d] wrong PMC class %d", __LINE__, pd->pm_descr.pd_class)); @@ -1430,7 +1281,7 @@ p4_start_pmc(int cpu, int ri) ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue, cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu)); - return 0; + return (0); } /* @@ -1444,7 +1295,6 @@ p4_stop_pmc(int cpu, int ri) uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits; struct pmc *pm; struct p4_cpu *pc; - struct pmc_hw *phw; struct p4pmc_descr *pd; pmc_value_t tmp; @@ -1453,18 +1303,9 @@ p4_stop_pmc(int cpu, int ri) KASSERT(ri >= 0 && ri < P4_NPMCS, ("[p4,%d] illegal row index %d", __LINE__, ri)); - pd = &p4_pmcdesc[ri]; - - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; - - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; - phw = pc->pc_hwpmcs[ri]; - - KASSERT(phw != NULL, - ("[p4,%d] null phw for cpu%d, ri%d", __LINE__, cpu, ri)); - - pm = phw->phw_pmc; + pd = &p4_pmcdesc[ri]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + pm = pc->pc_p4pmcs[ri].phw_pmc; KASSERT(pm != NULL, ("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri)); @@ -1474,7 +1315,7 @@ p4_stop_pmc(int cpu, int ri) if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { wrmsr(pd->pm_cccr_msr, pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE); - return 0; + return (0); } /* @@ -1569,8 +1410,7 @@ static int p4_intr(int cpu, struct trapframe *tf) { uint32_t cccrval, ovf_mask, ovf_partner; - int i, did_interrupt, error, ri; - struct pmc_hw *phw; + int did_interrupt, error, ri; struct p4_cpu *pc; struct pmc *pm; pmc_value_t v; @@ -1578,7 +1418,7 @@ p4_intr(int cpu, struct trapframe *tf) PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf, TRAPF_USERMODE(tf)); - pc = (struct p4_cpu *) pmc_pcpu[P4_TO_HTT_PRIMARY(cpu)]; + pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; ovf_mask = P4_CPU_IS_HTT_SECONDARY(cpu) ? P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0; @@ -1597,9 +1437,7 @@ p4_intr(int cpu, struct trapframe *tf) * Loop through all CCCRs, looking for ones that have * interrupted this CPU. */ - for (i = 0; i < P4_NPMCS-1; i++) { - - ri = i + 1; /* row index */ + for (ri = 0; ri < P4_NPMCS; ri++) { /* * Check if our partner logical CPU has already marked @@ -1615,8 +1453,7 @@ p4_intr(int cpu, struct trapframe *tf) * Ignore de-configured or stopped PMCs. * Ignore PMCs not in sampling mode. */ - phw = pc->pc_hwpmcs[ri]; - pm = phw->phw_pmc; + pm = pc->pc_p4pmcs[ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { @@ -1632,7 +1469,7 @@ p4_intr(int cpu, struct trapframe *tf) * and the OVF_Tx bit for this logical * processor being set. */ - cccrval = rdmsr(P4_CCCR_MSR_FIRST + i); + cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri); if ((cccrval & ovf_mask) != ovf_mask) continue; @@ -1646,13 +1483,13 @@ p4_intr(int cpu, struct trapframe *tf) if (p4_system_has_htt && (cccrval & ovf_partner)) P4_PCPU_SET_INTRFLAG(pc, ri, 1); - v = rdmsr(P4_PERFCTR_MSR_FIRST + i); + v = rdmsr(P4_PERFCTR_MSR_FIRST + ri); PMCDBG(MDP,INT, 2, "ri=%d v=%jx", ri, v); /* Stop the counter, and reset the overflow bit */ cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE); - wrmsr(P4_CCCR_MSR_FIRST + i, cccrval); + wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval); did_interrupt = 1; @@ -1660,8 +1497,7 @@ p4_intr(int cpu, struct trapframe *tf) * Ignore de-configured or stopped PMCs. Ignore PMCs * not in sampling mode. */ - phw = pc->pc_hwpmcs[ri]; - pm = phw->phw_pmc; + pm = pc->pc_p4pmcs[ri].phw_pmc; if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING || @@ -1683,9 +1519,9 @@ p4_intr(int cpu, struct trapframe *tf) */ v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE( pm->pm_sc.pm_reloadcount); - wrmsr(P4_PERFCTR_MSR_FIRST + i, v); + wrmsr(P4_PERFCTR_MSR_FIRST + ri, v); if (error == 0) - wrmsr(P4_CCCR_MSR_FIRST + i, + wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval | P4_CCCR_ENABLE); } @@ -1718,7 +1554,6 @@ p4_describe(int cpu, int ri, struct pmc_info *pi, { int error; size_t copied; - struct pmc_hw *phw; const struct p4pmc_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), @@ -1729,26 +1564,25 @@ p4_describe(int cpu, int ri, struct pmc_info *pi, PMCDBG(MDP,OPS,1,"p4-describe cpu=%d ri=%d", cpu, ri); if (P4_CPU_IS_HTT_SECONDARY(cpu)) - return EINVAL; + return (EINVAL); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; pd = &p4_pmcdesc[ri]; if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, - PMC_NAME_MAX, &copied)) != 0) - return error; + PMC_NAME_MAX, &copied)) != 0) + return (error); pi->pm_class = pd->pm_descr.pd_class; - if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) { pi->pm_enabled = TRUE; - *ppmc = phw->phw_pmc; + *ppmc = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc; } else { pi->pm_enabled = FALSE; *ppmc = NULL; } - return 0; + return (0); } /* @@ -1770,41 +1604,52 @@ p4_get_msr(int ri, uint32_t *msr) int -pmc_initialize_p4(struct pmc_mdep *pmc_mdep) +pmc_p4_initialize(struct pmc_mdep *md, int ncpus) { + struct pmc_classdep *pcd; struct p4_event_descr *pe; + KASSERT(md != NULL, ("[p4,%d] md is NULL", __LINE__)); KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0, ("[p4,%d] Initializing non-intel processor", __LINE__)); PMCDBG(MDP,INI,1, "%s", "p4-initialize"); - switch (pmc_mdep->pmd_cputype) { + /* Allocate space for pointers to per-cpu descriptors. */ + p4_pcpu = malloc(sizeof(struct p4_cpu **) * ncpus, M_PMC, + M_ZERO|M_WAITOK); + + /* Fill in the class dependent descriptor. */ + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4]; + + switch (md->pmd_cputype) { case PMC_CPU_INTEL_PIV: - pmc_mdep->pmd_npmc = P4_NPMCS; - pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P4; - pmc_mdep->pmd_classes[1].pm_caps = P4_PMC_CAPS; - pmc_mdep->pmd_classes[1].pm_width = 40; - pmc_mdep->pmd_nclasspmcs[1] = 18; + pcd->pcd_caps = P4_PMC_CAPS; + pcd->pcd_class = PMC_CLASS_P4; + pcd->pcd_num = P4_NPMCS; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 40; - pmc_mdep->pmd_init = p4_init; - pmc_mdep->pmd_cleanup = p4_cleanup; - pmc_mdep->pmd_switch_in = p4_switch_in; - pmc_mdep->pmd_switch_out = p4_switch_out; - pmc_mdep->pmd_read_pmc = p4_read_pmc; - pmc_mdep->pmd_write_pmc = p4_write_pmc; - pmc_mdep->pmd_config_pmc = p4_config_pmc; - pmc_mdep->pmd_get_config = p4_get_config; - pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc; - pmc_mdep->pmd_release_pmc = p4_release_pmc; - pmc_mdep->pmd_start_pmc = p4_start_pmc; - pmc_mdep->pmd_stop_pmc = p4_stop_pmc; - pmc_mdep->pmd_intr = p4_intr; - pmc_mdep->pmd_describe = p4_describe; - pmc_mdep->pmd_get_msr = p4_get_msr; /* i386 */ + pcd->pcd_allocate_pmc = p4_allocate_pmc; + pcd->pcd_config_pmc = p4_config_pmc; + pcd->pcd_describe = p4_describe; + pcd->pcd_get_config = p4_get_config; + pcd->pcd_get_msr = p4_get_msr; + pcd->pcd_pcpu_fini = p4_pcpu_fini; + pcd->pcd_pcpu_init = p4_pcpu_init; + pcd->pcd_read_pmc = p4_read_pmc; + pcd->pcd_release_pmc = p4_release_pmc; + pcd->pcd_start_pmc = p4_start_pmc; + pcd->pcd_stop_pmc = p4_stop_pmc; + pcd->pcd_write_pmc = p4_write_pmc; - /* model specific munging */ + md->pmd_pcpu_fini = NULL; + md->pmd_pcpu_init = NULL; + md->pmd_intr = p4_intr; + md->pmd_npmc += P4_NPMCS; + + /* model specific configuration */ if ((cpu_id & 0xFFF) < 0xF27) { /* @@ -1824,5 +1669,26 @@ pmc_initialize_p4(struct pmc_mdep *pmc_mdep) return ENOSYS; } - return 0; + return (0); +} + +void +pmc_p4_finalize(struct pmc_mdep *md) +{ +#if defined(INVARIANTS) + int i, ncpus; +#endif + + KASSERT(p4_pcpu != NULL, + ("[p4,%d] NULL p4_pcpu", __LINE__)); + +#if defined(INVARIANTS) + ncpus = pmc_cpu_max(); + for (i = 0; i < ncpus; i++) + KASSERT(p4_pcpu[i] == NULL, ("[p4,%d] non-null pcpu %d", + __LINE__, i)); +#endif + + free(p4_pcpu, M_PMC); + p4_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_piv.h b/sys/dev/hwpmc/hwpmc_piv.h index 0837b263b639..ebde966167f5 100644 --- a/sys/dev/hwpmc/hwpmc_piv.h +++ b/sys/dev/hwpmc/hwpmc_piv.h @@ -33,7 +33,7 @@ /* Intel P4 PMCs */ -#define P4_NPMCS 19 /* 1 TSC + 18 PMCS */ +#define P4_NPMCS 18 #define P4_NESCR 45 #define P4_INVALID_PMC_INDEX -1 #define P4_MAX_ESCR_PER_EVENT 2 @@ -118,7 +118,8 @@ struct pmc_md_p4_pmc { * Prototypes */ -int pmc_initialize_p4(struct pmc_mdep *); /* Pentium IV PMCs */ +int pmc_p4_initialize(struct pmc_mdep *_md, int _ncpus); +void pmc_p4_finalize(struct pmc_mdep *md); #endif /* _KERNEL */ -#endif /* _MACHINE_PMC_MDEP_H */ +#endif /* _DEV_HWPMC_PIV_H_ */ diff --git a/sys/dev/hwpmc/hwpmc_ppro.c b/sys/dev/hwpmc/hwpmc_ppro.c index fc40e19d92a9..d97d76a07d1c 100644 --- a/sys/dev/hwpmc/hwpmc_ppro.c +++ b/sys/dev/hwpmc/hwpmc_ppro.c @@ -73,19 +73,6 @@ struct p6pmc_descr { static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = { - /* TSC */ - { - .pm_descr = - { - .pd_name = "TSC", - .pd_class = PMC_CLASS_TSC, - .pd_caps = PMC_CAP_READ, - .pd_width = 64 - }, - .pm_pmc_msr = 0x10, - .pm_evsel_msr = ~0 - }, - #define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ PMC_CAP_INVERT | PMC_CAP_QUALIFIER) @@ -306,12 +293,12 @@ p6_find_event(enum pmc_event ev) */ struct p6_cpu { - struct pmc_cpu pc_common; - struct pmc_hw *pc_hwpmcs[P6_NPMCS]; struct pmc_hw pc_p6pmcs[P6_NPMCS]; uint32_t pc_state; }; +static struct p6_cpu **p6_pcpu; + /* * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set, * with the rest of CTR0 being zero'ed out. @@ -338,10 +325,11 @@ struct p6_cpu { } while (0) static int -p6_init(int cpu) +p6_pcpu_init(struct pmc_mdep *md, int cpu) { - int n; - struct p6_cpu *pcs; + int first_ri, n; + struct p6_cpu *p6c; + struct pmc_cpu *pc; struct pmc_hw *phw; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), @@ -349,94 +337,71 @@ p6_init(int cpu) PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu); - pcs = malloc(sizeof(struct p6_cpu), M_PMC, - M_WAITOK|M_ZERO); + p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO); + pc = pmc_pcpu[cpu]; - phw = pcs->pc_p6pmcs; + KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu)); + + phw = p6c->pc_p6pmcs; + p6_pcpu[cpu] = p6c; + + first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri; for (n = 0; n < P6_NPMCS; n++, phw++) { phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); phw->phw_pmc = NULL; - pcs->pc_hwpmcs[n] = phw; + pc->pc_hwpmcs[n + first_ri] = phw; } - /* Mark the TSC as shareable */ - pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; - - pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; - - return 0; + return (0); } static int -p6_cleanup(int cpu) +p6_pcpu_fini(struct pmc_mdep *md, int cpu) { - struct pmc_cpu *pcs; + int first_ri, n; + struct p6_cpu *p6c; + struct pmc_cpu *pc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p6,%d] bad cpu %d", __LINE__, cpu)); PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu); - if ((pcs = pmc_pcpu[cpu]) != NULL) - free(pcs, M_PMC); - pmc_pcpu[cpu] = NULL; + p6c = p6_pcpu[cpu]; + p6_pcpu[cpu] = NULL; - return 0; -} + KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__)); -static int -p6_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) -{ - (void) pc; + free(p6c, M_PMC); - PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, - pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS); + first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri; + pc = pmc_pcpu[cpu]; + for (n = 0; n < P6_NPMCS; n++) + pc->pc_hwpmcs[n + first_ri] = NULL; - /* allow the RDPMC instruction if needed */ - if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) - load_cr4(rcr4() | CR4_PCE); - - PMCDBG(MDP,SWI,1, "cr4=0x%x", rcr4()); - - return 0; -} - -static int -p6_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) -{ - (void) pc; - (void) pp; /* can be NULL */ - - PMCDBG(MDP,SWO,1, "pc=%p pp=%p cr4=0x%x", pc, pp, rcr4()); - - /* always turn off the RDPMC instruction */ - load_cr4(rcr4() & ~CR4_PCE); - - return 0; + return (0); } static int p6_read_pmc(int cpu, int ri, pmc_value_t *v) { - struct pmc_hw *phw; struct pmc *pm; struct p6pmc_descr *pd; pmc_value_t tmp; - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; - pm = phw->phw_pmc; - pd = &p6_pmcdesc[ri]; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < P6_NPMCS, + ("[p6,%d] illegal row-index %d", __LINE__, ri)); + + pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; + pd = &p6_pmcdesc[ri]; KASSERT(pm, ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { - *v = rdtsc(); - return 0; - } - tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK; if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); @@ -446,26 +411,26 @@ p6_read_pmc(int cpu, int ri, pmc_value_t *v) PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, pd->pm_pmc_msr, *v); - return 0; + return (0); } static int p6_write_pmc(int cpu, int ri, pmc_value_t v) { - struct pmc_hw *phw; struct pmc *pm; struct p6pmc_descr *pd; - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; - pm = phw->phw_pmc; - pd = &p6_pmcdesc[ri]; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < P6_NPMCS, + ("[p6,%d] illegal row-index %d", __LINE__, ri)); + + pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; + pd = &p6_pmcdesc[ri]; KASSERT(pm, ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; - PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, pd->pm_pmc_msr, v); @@ -474,20 +439,26 @@ p6_write_pmc(int cpu, int ri, pmc_value_t v) wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK); - return 0; + return (0); } static int p6_config_pmc(int cpu, int ri, struct pmc *pm) { - struct pmc_hw *phw; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[p6,%d] illegal CPU %d", __LINE__, cpu)); + + KASSERT(ri >= 0 && ri < P6_NPMCS, + ("[p6,%d] illegal row-index %d", __LINE__, ri)); PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; - phw->phw_pmc = pm; + KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__, + cpu)); - return 0; + p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm; + + return (0); } /* @@ -497,9 +468,15 @@ p6_config_pmc(int cpu, int ri, struct pmc *pm) static int p6_get_config(int cpu, int ri, struct pmc **ppm) { - *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; - return 0; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[p6,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < P6_NPMCS, + ("[p6,%d] illegal row-index %d", __LINE__, ri)); + + *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; + + return (0); } @@ -521,9 +498,9 @@ p6_allocate_pmc(int cpu, int ri, struct pmc *pm, (void) cpu; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), - ("[p4,%d] illegal CPU %d", __LINE__, cpu)); + ("[p6,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < P6_NPMCS, - ("[p4,%d] illegal row-index value %d", __LINE__, ri)); + ("[p6,%d] illegal row-index value %d", __LINE__, ri)); pd = &p6_pmcdesc[ri]; @@ -533,36 +510,24 @@ p6_allocate_pmc(int cpu, int ri, struct pmc *pm, /* check class */ if (pd->pm_descr.pd_class != a->pm_class) - return EINVAL; + return (EINVAL); /* check requested capabilities */ caps = a->pm_caps; if ((pd->pm_descr.pd_caps & caps) != caps) - return EPERM; - - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { - /* TSC's are always allocated in system-wide counting mode */ - if (a->pm_ev != PMC_EV_TSC_TSC || - a->pm_mode != PMC_MODE_SC) - return EINVAL; - return 0; - } - - /* - * P6 class events - */ + return (EPERM); ev = pm->pm_event; if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST) - return EINVAL; + return (EINVAL); if ((pevent = p6_find_event(ev)) == NULL) - return ESRCH; + return (ESRCH); if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) || !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1))) - return EINVAL; + return (EINVAL); /* For certain events, Pentium M differs from the stock P6 */ allowed_unitmask = 0; @@ -577,7 +542,7 @@ p6_allocate_pmc(int cpu, int ri, struct pmc *pm, unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK; if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ - return EINVAL; + return (EINVAL); if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */ unitmask = P6_EVSEL_TO_UMASK(0x0F); @@ -612,14 +577,12 @@ p6_allocate_pmc(int cpu, int ri, struct pmc *pm, PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config); - return 0; + return (0); } static int p6_release_pmc(int cpu, int ri, struct pmc *pm) { - struct pmc_hw *phw; - (void) pm; PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm); @@ -629,12 +592,10 @@ p6_release_pmc(int cpu, int ri, struct pmc *pm) KASSERT(ri >= 0 && ri < P6_NPMCS, ("[p6,%d] illegal row-index %d", __LINE__, ri)); - phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; + KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL, + ("[p6,%d] PHW pmc non-NULL", __LINE__)); - KASSERT(phw->phw_pmc == NULL, - ("[p6,%d] PHW pmc %p != pmc %p", __LINE__, phw->phw_pmc, pm)); - - return 0; + return (0); } static int @@ -643,7 +604,6 @@ p6_start_pmc(int cpu, int ri) uint32_t config; struct pmc *pm; struct p6_cpu *pc; - struct pmc_hw *phw; const struct p6pmc_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), @@ -651,10 +611,9 @@ p6_start_pmc(int cpu, int ri) KASSERT(ri >= 0 && ri < P6_NPMCS, ("[p6,%d] illegal row-index %d", __LINE__, ri)); - pc = (struct p6_cpu *) pmc_pcpu[cpu]; - phw = pc->pc_common.pc_hwpmcs[ri]; - pm = phw->phw_pmc; - pd = &p6_pmcdesc[ri]; + pc = p6_pcpu[cpu]; + pm = pc->pc_p6pmcs[ri].phw_pmc; + pd = &p6_pmcdesc[ri]; KASSERT(pm, ("[p6,%d] starting cpu%d,ri%d with no pmc configured", @@ -662,13 +621,6 @@ p6_start_pmc(int cpu, int ri) PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; /* TSC are always running */ - - KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P6, - ("[p6,%d] unknown PMC class %d", __LINE__, - pd->pm_descr.pd_class)); - config = pm->pm_md.pm_ppro.pm_ppro_evsel; PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x", @@ -679,7 +631,7 @@ p6_start_pmc(int cpu, int ri) P6_SYNC_CTR_STATE(pc); - return 0; + return (0); } static int @@ -687,7 +639,6 @@ p6_stop_pmc(int cpu, int ri) { struct pmc *pm; struct p6_cpu *pc; - struct pmc_hw *phw; struct p6pmc_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), @@ -695,22 +646,14 @@ p6_stop_pmc(int cpu, int ri) KASSERT(ri >= 0 && ri < P6_NPMCS, ("[p6,%d] illegal row index %d", __LINE__, ri)); - pc = (struct p6_cpu *) pmc_pcpu[cpu]; - phw = pc->pc_common.pc_hwpmcs[ri]; - pm = phw->phw_pmc; - pd = &p6_pmcdesc[ri]; + pc = p6_pcpu[cpu]; + pm = pc->pc_p6pmcs[ri].phw_pmc; + pd = &p6_pmcdesc[ri]; KASSERT(pm, ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); - if (pd->pm_descr.pd_class == PMC_CLASS_TSC) - return 0; - - KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P6, - ("[p6,%d] unknown PMC class %d", __LINE__, - pd->pm_descr.pd_class)); - PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri); wrmsr(pd->pm_evsel_msr, 0); /* stop hw */ @@ -719,38 +662,35 @@ p6_stop_pmc(int cpu, int ri) P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */ PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri); - return 0; + + return (0); } static int p6_intr(int cpu, struct trapframe *tf) { - int i, error, retval, ri; + int error, retval, ri; uint32_t perf0cfg; struct pmc *pm; struct p6_cpu *pc; - struct pmc_hw *phw; pmc_value_t v; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p6,%d] CPU %d out of range", __LINE__, cpu)); retval = 0; - pc = (struct p6_cpu *) pmc_pcpu[cpu]; + pc = p6_pcpu[cpu]; /* stop both PMCs */ perf0cfg = rdmsr(P6_MSR_EVSEL0); wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN); - for (i = 0; i < P6_NPMCS-1; i++) { - ri = i + 1; + for (ri = 0; ri < P6_NPMCS; ri++) { - if (!P6_PMC_HAS_OVERFLOWED(i)) + if (!P6_PMC_HAS_OVERFLOWED(ri)) continue; - phw = pc->pc_common.pc_hwpmcs[ri]; - - if ((pm = phw->phw_pmc) == NULL || + if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL || pm->pm_state != PMC_STATE_RUNNING || !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { continue; @@ -765,7 +705,7 @@ p6_intr(int cpu, struct trapframe *tf) /* reload sampling count */ v = pm->pm_sc.pm_reloadcount; - wrmsr(P6_MSR_PERFCTR0 + i, + wrmsr(P6_MSR_PERFCTR0 + ri, P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); } @@ -783,7 +723,7 @@ p6_intr(int cpu, struct trapframe *tf) /* restart counters that can be restarted */ P6_SYNC_CTR_STATE(pc); - return retval; + return (retval); } static int @@ -795,12 +735,20 @@ p6_describe(int cpu, int ri, struct pmc_info *pi, struct pmc_hw *phw; struct p6pmc_descr *pd; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[p6,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < P6_NPMCS, + ("[p6,%d] row-index %d out of range", __LINE__, ri)); + phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; pd = &p6_pmcdesc[ri]; + KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri], + ("[p6,%d] phw mismatch", __LINE__)); + if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, PMC_NAME_MAX, &copied)) != 0) - return error; + return (error); pi->pm_class = pd->pm_descr.pd_class; @@ -812,7 +760,7 @@ p6_describe(int cpu, int ri, struct pmc_info *pi, *ppmc = NULL; } - return 0; + return (0); } static int @@ -822,58 +770,91 @@ p6_get_msr(int ri, uint32_t *msr) ("[p6,%d ri %d out of range", __LINE__, ri)); *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0; - return 0; + + return (0); } int -pmc_initialize_p6(struct pmc_mdep *pmc_mdep) +pmc_p6_initialize(struct pmc_mdep *md, int ncpus) { + struct pmc_classdep *pcd; + KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0, ("[p6,%d] Initializing non-intel processor", __LINE__)); PMCDBG(MDP,INI,1, "%s", "p6-initialize"); - switch (pmc_mdep->pmd_cputype) { + /* Allocate space for pointers to per-cpu descriptors. */ + p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC, + M_ZERO|M_WAITOK); + + /* Fill in the class dependent descriptor. */ + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6]; + + switch (md->pmd_cputype) { /* * P6 Family Processors */ - case PMC_CPU_INTEL_P6: case PMC_CPU_INTEL_CL: case PMC_CPU_INTEL_PII: case PMC_CPU_INTEL_PIII: case PMC_CPU_INTEL_PM: - p6_cputype = pmc_mdep->pmd_cputype; + p6_cputype = md->pmd_cputype; - pmc_mdep->pmd_npmc = P6_NPMCS; - pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P6; - pmc_mdep->pmd_classes[1].pm_caps = P6_PMC_CAPS; - pmc_mdep->pmd_classes[1].pm_width = 40; - pmc_mdep->pmd_nclasspmcs[1] = 2; + pcd->pcd_caps = P6_PMC_CAPS; + pcd->pcd_class = PMC_CLASS_P6; + pcd->pcd_num = P6_NPMCS; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 40; - pmc_mdep->pmd_init = p6_init; - pmc_mdep->pmd_cleanup = p6_cleanup; - pmc_mdep->pmd_switch_in = p6_switch_in; - pmc_mdep->pmd_switch_out = p6_switch_out; - pmc_mdep->pmd_read_pmc = p6_read_pmc; - pmc_mdep->pmd_write_pmc = p6_write_pmc; - pmc_mdep->pmd_config_pmc = p6_config_pmc; - pmc_mdep->pmd_get_config = p6_get_config; - pmc_mdep->pmd_allocate_pmc = p6_allocate_pmc; - pmc_mdep->pmd_release_pmc = p6_release_pmc; - pmc_mdep->pmd_start_pmc = p6_start_pmc; - pmc_mdep->pmd_stop_pmc = p6_stop_pmc; - pmc_mdep->pmd_intr = p6_intr; - pmc_mdep->pmd_describe = p6_describe; - pmc_mdep->pmd_get_msr = p6_get_msr; /* i386 */ + pcd->pcd_allocate_pmc = p6_allocate_pmc; + pcd->pcd_config_pmc = p6_config_pmc; + pcd->pcd_describe = p6_describe; + pcd->pcd_get_config = p6_get_config; + pcd->pcd_get_msr = p6_get_msr; + pcd->pcd_pcpu_fini = p6_pcpu_fini; + pcd->pcd_pcpu_init = p6_pcpu_init; + pcd->pcd_read_pmc = p6_read_pmc; + pcd->pcd_release_pmc = p6_release_pmc; + pcd->pcd_start_pmc = p6_start_pmc; + pcd->pcd_stop_pmc = p6_stop_pmc; + pcd->pcd_write_pmc = p6_write_pmc; + + md->pmd_pcpu_fini = NULL; + md->pmd_pcpu_init = NULL; + md->pmd_intr = p6_intr; + + md->pmd_npmc += P6_NPMCS; break; + default: KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__)); return ENOSYS; } - return 0; + return (0); +} + +void +pmc_p6_finalize(struct pmc_mdep *md) +{ +#if defined(INVARIANTS) + int i, ncpus; +#endif + + KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__)); + +#if defined(INVARIANTS) + ncpus = pmc_cpu_max(); + for (i = 0; i < ncpus; i++) + KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d", + __LINE__, i)); +#endif + + free(p6_pcpu, M_PMC); + p6_pcpu = NULL; } diff --git a/sys/dev/hwpmc/hwpmc_ppro.h b/sys/dev/hwpmc/hwpmc_ppro.h index f750735c85ea..c42a2b4d315d 100644 --- a/sys/dev/hwpmc/hwpmc_ppro.h +++ b/sys/dev/hwpmc/hwpmc_ppro.h @@ -33,7 +33,7 @@ /* Intel PPro, Celeron, P-II, P-III, Pentium-M PMCS */ -#define P6_NPMCS 3 /* 1 TSC + 2 PMCs */ +#define P6_NPMCS 2 /* 2 PMCs */ #define P6_EVSEL_CMASK_MASK 0xFF000000 #define P6_EVSEL_TO_CMASK(C) (((C) & 0xFF) << 24) @@ -77,7 +77,8 @@ struct pmc_md_ppro_pmc { * Prototypes */ -int pmc_initialize_p6(struct pmc_mdep *); /* Pentium Pro PMCs */ +int pmc_p6_initialize(struct pmc_mdep *_md, int _ncpus); +void pmc_p6_finalize(struct pmc_mdep *_md); #endif /* _KERNEL */ #endif /* _DEV_HWPMC_PPRO_H_ */ diff --git a/sys/dev/hwpmc/hwpmc_tsc.c b/sys/dev/hwpmc/hwpmc_tsc.c new file mode 100644 index 000000000000..0b71a5be62fe --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_tsc.c @@ -0,0 +1,388 @@ +/*- + * Copyright (c) 2008 Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include + +#include + +/* + * TSC support. + */ + +#define TSC_CAPS PMC_CAP_READ + +struct tsc_descr { + struct pmc_descr pm_descr; /* "base class" */ +}; + +static struct tsc_descr tsc_pmcdesc[TSC_NPMCS] = +{ + { + .pm_descr = + { + .pd_name = "TSC", + .pd_class = PMC_CLASS_TSC, + .pd_caps = TSC_CAPS, + .pd_width = 64 + } + } +}; + +/* + * Per-CPU data structure for TSCs. + */ + +struct tsc_cpu { + struct pmc_hw tc_hw; +}; + +static struct tsc_cpu **tsc_pcpu; + +static int +tsc_allocate_pmc(int cpu, int ri, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + (void) cpu; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < TSC_NPMCS, + ("[tsc,%d] illegal row index %d", __LINE__, ri)); + + if (a->pm_class != PMC_CLASS_TSC) + return (EINVAL); + + if ((pm->pm_caps & TSC_CAPS) == 0) + return (EINVAL); + + if ((pm->pm_caps & ~TSC_CAPS) != 0) + return (EPERM); + + if (a->pm_ev != PMC_EV_TSC_TSC || + a->pm_mode != PMC_MODE_SC) + return (EINVAL); + + return (0); +} + +static int +tsc_config_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pmc_hw *phw; + + PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + phw = &tsc_pcpu[cpu]->tc_hw; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[tsc,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, + pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + + return (0); +} + +static int +tsc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) +{ + int error; + size_t copied; + const struct tsc_descr *pd; + struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + phw = &tsc_pcpu[cpu]->tc_hw; + pd = &tsc_pmcdesc[ri]; + + if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, + PMC_NAME_MAX, &copied)) != 0) + return (error); + + pi->pm_class = pd->pm_descr.pd_class; + + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + pi->pm_enabled = TRUE; + *ppmc = phw->phw_pmc; + } else { + pi->pm_enabled = FALSE; + *ppmc = NULL; + } + + return (0); +} + +static int +tsc_get_config(int cpu, int ri, struct pmc **ppm) +{ + (void) ri; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + *ppm = tsc_pcpu[cpu]->tc_hw.phw_pmc; + + return (0); +} + +static int +tsc_get_msr(int ri, uint32_t *msr) +{ + (void) ri; + + KASSERT(ri >= 0 && ri < TSC_NPMCS, + ("[tsc,%d] ri %d out of range", __LINE__, ri)); + + *msr = MSR_TSC; + + return (0); +} + +static int +tsc_pcpu_fini(struct pmc_mdep *md, int cpu) +{ + int ri; + struct pmc_cpu *pc; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(tsc_pcpu[cpu] != NULL, ("[tsc,%d] null pcpu", __LINE__)); + + free(tsc_pcpu[cpu], M_PMC); + tsc_pcpu[cpu] = NULL; + + ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; + + KASSERT(ri == 0 && ri < TSC_NPMCS, ("[tsc,%d] ri=%d", __LINE__, + ri)); + + pc = pmc_pcpu[cpu]; + pc->pc_hwpmcs[ri] = NULL; + + return (0); +} + +static int +tsc_pcpu_init(struct pmc_mdep *md, int cpu) +{ + int ri; + struct pmc_cpu *pc; + struct tsc_cpu *tsc_pc; + + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(tsc_pcpu, ("[tsc,%d] null pcpu", __LINE__)); + KASSERT(tsc_pcpu[cpu] == NULL, ("[tsc,%d] non-null per-cpu", + __LINE__)); + + tsc_pc = malloc(sizeof(struct tsc_cpu), M_PMC, M_WAITOK|M_ZERO); + + tsc_pc->tc_hw.phw_state = PMC_PHW_FLAG_IS_ENABLED | + PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(0) | + PMC_PHW_FLAG_IS_SHAREABLE; + + tsc_pcpu[cpu] = tsc_pc; + + ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_ri; + + KASSERT(pmc_pcpu, ("[tsc,%d] null generic pcpu", __LINE__)); + + pc = pmc_pcpu[cpu]; + + KASSERT(pc, ("[tsc,%d] null generic per-cpu", __LINE__)); + + pc->pc_hwpmcs[ri] = &tsc_pc->tc_hw; + + return (0); +} + +static int +tsc_read_pmc(int cpu, int ri, pmc_value_t *v) +{ + struct pmc *pm; + enum pmc_mode mode; + const struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal ri %d", __LINE__, ri)); + + phw = &tsc_pcpu[cpu]->tc_hw; + pm = phw->phw_pmc; + + KASSERT(pm != NULL, + ("[tsc,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); + + mode = PMC_TO_MODE(pm); + + KASSERT(mode == PMC_MODE_SC, + ("[tsc,%d] illegal pmc mode %d", __LINE__, mode)); + + PMCDBG(MDP,REA,1,"tsc-read id=%d", ri); + + *v = rdtsc(); + + return (0); +} + +static int +tsc_release_pmc(int cpu, int ri, struct pmc *pmc) +{ + struct pmc_hw *phw; + + (void) pmc; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, + ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + phw = &tsc_pcpu[cpu]->tc_hw; + + KASSERT(phw->phw_pmc == NULL, + ("[tsc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + /* + * Nothing to do. + */ + return (0); +} + +static int +tsc_start_pmc(int cpu, int ri) +{ + (void) cpu; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + return (0); /* TSCs are always running. */ +} + +static int +tsc_stop_pmc(int cpu, int ri) +{ + (void) cpu; (void) ri; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + return (0); /* Cannot actually stop a TSC. */ +} + +static int +tsc_write_pmc(int cpu, int ri, pmc_value_t v) +{ + (void) cpu; (void) ri; (void) v; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[tsc,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri == 0, ("[tsc,%d] illegal row-index %d", __LINE__, ri)); + + /* + * The TSCs are used as timecounters by the kernel, so even + * though some i386 CPUs support writeable TSCs, we don't + * support writing changing TSC values through the HWPMC API. + */ + return (0); +} + +int +pmc_tsc_initialize(struct pmc_mdep *md, int maxcpu) +{ + struct pmc_classdep *pcd; + + KASSERT(md != NULL, ("[tsc,%d] md is NULL", __LINE__)); + KASSERT(md->pmd_nclass >= 1, ("[tsc,%d] dubious md->nclass %d", + __LINE__, md->pmd_nclass)); + + tsc_pcpu = malloc(sizeof(struct tsc_cpu *) * maxcpu, M_PMC, + M_ZERO|M_WAITOK); + + pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC]; + + pcd->pcd_caps = PMC_CAP_READ; + pcd->pcd_class = PMC_CLASS_TSC; + pcd->pcd_num = TSC_NPMCS; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 64; + + pcd->pcd_allocate_pmc = tsc_allocate_pmc; + pcd->pcd_config_pmc = tsc_config_pmc; + pcd->pcd_describe = tsc_describe; + pcd->pcd_get_config = tsc_get_config; + pcd->pcd_get_msr = tsc_get_msr; + pcd->pcd_pcpu_init = tsc_pcpu_init; + pcd->pcd_pcpu_fini = tsc_pcpu_fini; + pcd->pcd_read_pmc = tsc_read_pmc; + pcd->pcd_release_pmc = tsc_release_pmc; + pcd->pcd_start_pmc = tsc_start_pmc; + pcd->pcd_stop_pmc = tsc_stop_pmc; + pcd->pcd_write_pmc = tsc_write_pmc; + + md->pmd_npmc += TSC_NPMCS; + + return (0); +} + +void +pmc_tsc_finalize(struct pmc_mdep *md) +{ +#ifdef INVARIANTS + int i, ncpus; + + ncpus = pmc_cpu_max(); + for (i = 0; i < ncpus; i++) + KASSERT(tsc_pcpu[i] == NULL, ("[tsc,%d] non-null pcpu cpu %d", + __LINE__, i)); + + KASSERT(md->pmd_classdep[PMC_MDEP_CLASS_INDEX_TSC].pcd_class == + PMC_CLASS_TSC, ("[tsc,%d] class mismatch", __LINE__)); + +#else + (void) md; +#endif + + free(tsc_pcpu, M_PMC); + tsc_pcpu = NULL; +} diff --git a/sys/dev/hwpmc/hwpmc_tsc.h b/sys/dev/hwpmc/hwpmc_tsc.h new file mode 100644 index 000000000000..a8b011e6ee31 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_tsc.h @@ -0,0 +1,43 @@ +/*- + * Copyright (c) 2008 Joseph Koshy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _DEV_HWPMC_TSC_H_ +#define _DEV_HWPMC_TSC_H_ 1 + +#ifdef _KERNEL + +#define TSC_NPMCS 1 + +/* + * Prototypes. + */ + +int pmc_tsc_initialize(struct pmc_mdep *_md, int _maxcpu); +void pmc_tsc_finalize(struct pmc_mdep *_md); +#endif /* _KERNEL */ +#endif /* _DEV_HWPMC_TSC_H */ diff --git a/sys/dev/hwpmc/hwpmc_x86.c b/sys/dev/hwpmc/hwpmc_x86.c index 66bf8dd4cc65..dd9c0e9f3d60 100644 --- a/sys/dev/hwpmc/hwpmc_x86.c +++ b/sys/dev/hwpmc/hwpmc_x86.c @@ -240,120 +240,6 @@ pmc_save_kernel_callchain(uintptr_t *cc, int nframes, struct trapframe *tf) return (n); } -static struct pmc_mdep * -pmc_intel_initialize(void) -{ - struct pmc_mdep *pmc_mdep; - enum pmc_cputype cputype; - int error, model; - - KASSERT(strcmp(cpu_vendor, "GenuineIntel") == 0, - ("[intel,%d] Initializing non-intel processor", __LINE__)); - - PMCDBG(MDP,INI,0, "intel-initialize cpuid=0x%x", cpu_id); - - cputype = -1; - - switch (cpu_id & 0xF00) { -#if defined(__i386__) - case 0x500: /* Pentium family processors */ - cputype = PMC_CPU_INTEL_P5; - break; - case 0x600: /* Pentium Pro, Celeron, Pentium II & III */ - switch ((cpu_id & 0xF0) >> 4) { /* model number field */ - case 0x1: - cputype = PMC_CPU_INTEL_P6; - break; - case 0x3: case 0x5: - cputype = PMC_CPU_INTEL_PII; - break; - case 0x6: - cputype = PMC_CPU_INTEL_CL; - break; - case 0x7: case 0x8: case 0xA: case 0xB: - cputype = PMC_CPU_INTEL_PIII; - break; - case 0x9: case 0xD: - cputype = PMC_CPU_INTEL_PM; - break; - } - break; -#endif -#if defined(__i386__) || defined(__amd64__) - case 0xF00: /* P4 */ - model = ((cpu_id & 0xF0000) >> 12) | ((cpu_id & 0xF0) >> 4); - if (model >= 0 && model <= 6) /* known models */ - cputype = PMC_CPU_INTEL_PIV; - break; - } -#endif - - if ((int) cputype == -1) { - printf("pmc: Unknown Intel CPU.\n"); - return NULL; - } - - pmc_mdep = malloc(sizeof(struct pmc_mdep), - M_PMC, M_WAITOK|M_ZERO); - - pmc_mdep->pmd_cputype = cputype; - pmc_mdep->pmd_nclass = 2; - pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; - pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; - pmc_mdep->pmd_classes[0].pm_width = 64; - pmc_mdep->pmd_nclasspmcs[0] = 1; - - error = 0; - - switch (cputype) { - -#if defined(__i386__) || defined(__amd64__) - - /* - * Intel Pentium 4 Processors, and P4/EMT64 processors. - */ - - case PMC_CPU_INTEL_PIV: - error = pmc_initialize_p4(pmc_mdep); - break; -#endif - -#if defined(__i386__) - /* - * P6 Family Processors - */ - - case PMC_CPU_INTEL_P6: - case PMC_CPU_INTEL_CL: - case PMC_CPU_INTEL_PII: - case PMC_CPU_INTEL_PIII: - case PMC_CPU_INTEL_PM: - - error = pmc_initialize_p6(pmc_mdep); - break; - - /* - * Intel Pentium PMCs. - */ - - case PMC_CPU_INTEL_P5: - error = pmc_initialize_p5(pmc_mdep); - break; -#endif - - default: - KASSERT(0,("[intel,%d] Unknown CPU type", __LINE__)); - } - - if (error) { - free(pmc_mdep, M_PMC); - pmc_mdep = NULL; - } - - return pmc_mdep; -} - - /* * Machine dependent initialization for x86 class platforms. */ @@ -370,11 +256,24 @@ pmc_md_initialize() md = pmc_amd_initialize(); else if (strcmp(cpu_vendor, "GenuineIntel") == 0) md = pmc_intel_initialize(); + else + KASSERT(0, ("[x86,%d] Unknown vendor", __LINE__)); /* disallow sampling if we do not have an LAPIC */ if (md != NULL && lapic == NULL) for (i = 1; i < md->pmd_nclass; i++) - md->pmd_classes[i].pm_caps &= ~PMC_CAP_INTERRUPT; + md->pmd_classdep[i].pcd_caps &= ~PMC_CAP_INTERRUPT; - return md; + return (md); +} + +void +pmc_md_finalize(struct pmc_mdep *md) +{ + if (strcmp(cpu_vendor, "AuthenticAMD") == 0) + pmc_amd_finalize(md); + else if (strcmp(cpu_vendor, "GenuineIntel") == 0) + pmc_intel_finalize(md); + else + KASSERT(0, ("[x86,%d] Unknown vendor", __LINE__)); } diff --git a/sys/i386/include/pmc_mdep.h b/sys/i386/include/pmc_mdep.h index de0688aa8d84..720f1f3656d7 100644 --- a/sys/i386/include/pmc_mdep.h +++ b/sys/i386/include/pmc_mdep.h @@ -33,21 +33,43 @@ #ifndef _MACHINE_PMC_MDEP_H #define _MACHINE_PMC_MDEP_H 1 +#ifdef _KERNEL +struct pmc_mdep; +#endif + /* * On the i386 platform we support the following PMCs. * + * TSC The timestamp counter * K7 AMD Athlon XP/MP and other 32 bit processors. * K8 AMD Athlon64 and Opteron PMCs in 32 bit mode. * PIV Intel P4/HTT and P4/EMT64 * PPRO Intel Pentium Pro, Pentium-II, Pentium-III, Celeron and * Pentium-M processors * PENTIUM Intel Pentium MMX. + * IAP Intel Core/Core2/Atom programmable PMCs. + * IAF Intel fixed-function PMCs. */ #include /* K7 and K8 */ #include #include #include +#include + +/* + * Intel processors implementing V2 and later of the Intel performance + * measurement architecture have PMCs of the following classes: TSC, + * IAF and IAP. + */ +#define PMC_MDEP_CLASS_INDEX_TSC 0 +#define PMC_MDEP_CLASS_INDEX_K7 1 +#define PMC_MDEP_CLASS_INDEX_K8 1 +#define PMC_MDEP_CLASS_INDEX_P4 1 +#define PMC_MDEP_CLASS_INDEX_P5 1 +#define PMC_MDEP_CLASS_INDEX_P6 1 +#define PMC_MDEP_CLASS_INDEX_IAF 1 +#define PMC_MDEP_CLASS_INDEX_IAP 2 /* * Architecture specific extensions to structures. @@ -76,6 +98,7 @@ union pmc_md_pmc { }; struct pmc; +struct pmc_mdep; #define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_eip) #define PMC_TRAPFRAME_TO_FP(TF) ((TF)->tf_ebp) @@ -124,5 +147,10 @@ struct pmc; void start_exceptions(void), end_exceptions(void); void pmc_x86_lapic_enable_pmc_interrupt(void); +struct pmc_mdep *pmc_amd_initialize(void); +void pmc_amd_finalize(struct pmc_mdep *_md); +struct pmc_mdep *pmc_intel_initialize(void); +void pmc_intel_finalize(struct pmc_mdep *_md); + #endif /* _KERNEL */ #endif /* _MACHINE_PMC_MDEP_H */ diff --git a/sys/modules/hwpmc/Makefile b/sys/modules/hwpmc/Makefile index 167d7482073b..350d36152a7a 100644 --- a/sys/modules/hwpmc/Makefile +++ b/sys/modules/hwpmc/Makefile @@ -9,7 +9,7 @@ KMOD= hwpmc SRCS= hwpmc_mod.c hwpmc_logging.c vnode_if.h .if ${MACHINE_ARCH} == "amd64" -SRCS+= hwpmc_amd.c hwpmc_piv.c hwpmc_x86.c +SRCS+= hwpmc_amd.c hwpmc_intel.c hwpmc_piv.c hwpmc_tsc.c hwpmc_x86.c SRCS+= device_if.h bus_if.h .endif @@ -18,7 +18,8 @@ SRCS+= hwpmc_arm.c .endif .if ${MACHINE_ARCH} == "i386" -SRCS+= hwpmc_amd.c hwpmc_piv.c hwpmc_ppro.c hwpmc_pentium.c hwpmc_x86.c +SRCS+= hwpmc_amd.c hwpmc_intel.c hwpmc_piv.c hwpmc_ppro.c hwpmc_pentium.c +SRCS+= hwpmc_tsc.c hwpmc_x86.c SRCS+= device_if.h bus_if.h .endif diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h index d3679469f415..84c0a130e2a1 100644 --- a/sys/sys/pmc.h +++ b/sys/sys/pmc.h @@ -40,7 +40,7 @@ #define PMC_MODULE_NAME "hwpmc" #define PMC_NAME_MAX 16 /* HW counter name size */ -#define PMC_CLASS_MAX 4 /* #classes of PMCs in a system */ +#define PMC_CLASS_MAX 4 /* max #classes of PMCs per-system */ /* * Kernel<->userland API version number [MMmmpppp] @@ -472,9 +472,10 @@ struct pmc_op_getpmcinfo { */ struct pmc_classinfo { - enum pmc_class pm_class; /* class id */ + enum pmc_class pm_class; /* class id */ uint32_t pm_caps; /* counter capabilities */ uint32_t pm_width; /* width of the PMC */ + uint32_t pm_num; /* number of PMCs in class */ }; struct pmc_op_getcpuinfo { @@ -634,7 +635,7 @@ struct pmc_target { struct pmc { LIST_HEAD(,pmc_target) pm_targets; /* list of target processes */ - LIST_ENTRY(pmc) pm_next; /* owner's list */ + LIST_ENTRY(pmc) pm_next; /* owner's list */ /* * System-wide PMCs are allocated on a CPU and are not moved @@ -678,7 +679,7 @@ struct pmc { * mode, class and the CPU# associated with the PMC. */ - pmc_id_t pm_id; /* allocated PMC id */ + pmc_id_t pm_id; /* allocated PMC id */ /* md extensions */ union pmc_md_pmc pm_md; @@ -721,7 +722,7 @@ struct pmc_targetstate { struct pmc_process { LIST_ENTRY(pmc_process) pp_next; /* hash chain */ int pp_refcnt; /* reference count */ - uint32_t pp_flags; /* flags PMC_PP_* */ + uint32_t pp_flags; /* flags PMC_PP_* */ struct proc *pp_proc; /* target thread */ struct pmc_targetstate pp_pmcs[]; /* NHWPMCs */ }; @@ -839,7 +840,6 @@ struct pmc_cpu { uint32_t pc_state; /* physical cpu number + flags */ struct pmc_samplebuffer *pc_sb; /* space for samples */ struct pmc_hw *pc_hwpmcs[]; /* 'npmc' pointers */ - /* other machine dependent fields come here */ }; #define PMC_PCPU_CPU_MASK 0x000000FF @@ -862,6 +862,48 @@ struct pmc_binding { int pb_cpu; /* if so, to which CPU */ }; + +struct pmc_mdep; + +/* + * struct pmc_classdep + * + * PMC class-dependent operations. + */ +struct pmc_classdep { + uint32_t pcd_caps; /* class capabilities */ + enum pmc_class pcd_class; /* class id */ + int pcd_num; /* number of PMCs */ + int pcd_ri; /* row index of the first PMC in class */ + int pcd_width; /* width of the PMC */ + + /* configuring/reading/writing the hardware PMCs */ + int (*pcd_config_pmc)(int _cpu, int _ri, struct pmc *_pm); + int (*pcd_get_config)(int _cpu, int _ri, struct pmc **_ppm); + int (*pcd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value); + int (*pcd_write_pmc)(int _cpu, int _ri, pmc_value_t _value); + + /* pmc allocation/release */ + int (*pcd_allocate_pmc)(int _cpu, int _ri, struct pmc *_t, + const struct pmc_op_pmcallocate *_a); + int (*pcd_release_pmc)(int _cpu, int _ri, struct pmc *_pm); + + /* starting and stopping PMCs */ + int (*pcd_start_pmc)(int _cpu, int _ri); + int (*pcd_stop_pmc)(int _cpu, int _ri); + + /* description */ + int (*pcd_describe)(int _cpu, int _ri, struct pmc_info *_pi, + struct pmc **_ppmc); + + /* class-dependent initialization & finalization */ + int (*pcd_pcpu_init)(struct pmc_mdep *_md, int _cpu); + int (*pcd_pcpu_fini)(struct pmc_mdep *_md, int _cpu); + + /* machine-specific interface */ + int (*pcd_get_msr)(int _ri, uint32_t *_msr); +}; + /* * struct pmc_mdep * @@ -870,45 +912,28 @@ struct pmc_binding { struct pmc_mdep { uint32_t pmd_cputype; /* from enum pmc_cputype */ - uint32_t pmd_npmc; /* max PMCs per CPU */ - uint32_t pmd_nclass; /* # PMC classes supported */ - struct pmc_classinfo pmd_classes[PMC_CLASS_MAX]; - int pmd_nclasspmcs[PMC_CLASS_MAX]; + uint32_t pmd_npmc; /* number of PMCs per CPU */ + uint32_t pmd_nclass; /* number of PMC classes present */ /* - * Methods + * Machine dependent methods. */ - int (*pmd_init)(int _cpu); /* machine dependent initialization */ - int (*pmd_cleanup)(int _cpu); /* machine dependent cleanup */ + /* per-cpu initialization and finalization */ + int (*pmd_pcpu_init)(int _cpu); /* initialization */ + int (*pmd_pcpu_fini)(int _cpu); /* finalization */ /* thread context switch in/out */ int (*pmd_switch_in)(struct pmc_cpu *_p, struct pmc_process *_pp); int (*pmd_switch_out)(struct pmc_cpu *_p, struct pmc_process *_pp); - /* configuring/reading/writing the hardware PMCs */ - int (*pmd_config_pmc)(int _cpu, int _ri, struct pmc *_pm); - int (*pmd_get_config)(int _cpu, int _ri, struct pmc **_ppm); - int (*pmd_read_pmc)(int _cpu, int _ri, pmc_value_t *_value); - int (*pmd_write_pmc)(int _cpu, int _ri, pmc_value_t _value); - - /* pmc allocation/release */ - int (*pmd_allocate_pmc)(int _cpu, int _ri, struct pmc *_t, - const struct pmc_op_pmcallocate *_a); - int (*pmd_release_pmc)(int _cpu, int _ri, struct pmc *_pm); - - /* starting and stopping PMCs */ - int (*pmd_start_pmc)(int _cpu, int _ri); - int (*pmd_stop_pmc)(int _cpu, int _ri); - /* handle a PMC interrupt */ int (*pmd_intr)(int _cpu, struct trapframe *_tf); - int (*pmd_describe)(int _cpu, int _ri, struct pmc_info *_pi, - struct pmc **_ppmc); - - int (*pmd_get_msr)(int _ri, uint32_t *_msr); - + /* + * PMC class dependent information. + */ + struct pmc_classdep pmd_classdep[]; }; /* @@ -968,7 +993,7 @@ extern struct pmc_debugflags pmc_debugflags; #define PMC_DEBUG_MIN_FND 4 /* find */ /* MODULE */ -#define PMC_DEBUG_MIN_PMH 14 /* pmc_hook */ +#define PMC_DEBUG_MIN_PMH 14 /* pmc_hook */ #define PMC_DEBUG_MIN_PMS 15 /* pmc_syscall */ /* OWN */ @@ -1001,7 +1026,7 @@ extern struct pmc_debugflags pmc_debugflags; #define PMC_DEBUG_MIN_INT 13 /* interrupts */ /* CPU */ -#define PMC_DEBUG_MIN_BND 8 /* bind */ +#define PMC_DEBUG_MIN_BND 8 /* bind */ #define PMC_DEBUG_MIN_SEL 9 /* select */ /* LOG */ @@ -1022,6 +1047,7 @@ MALLOC_DECLARE(M_PMC); */ struct pmc_mdep *pmc_md_initialize(void); /* MD init function */ +void pmc_md_finalize(struct pmc_mdep *_md); /* MD fini function */ int pmc_getrowdisp(int _ri); int pmc_process_interrupt(int _cpu, struct pmc *_pm, struct trapframe *_tf, int _inuserspace);