Kernel hooks to support PMC sampling modes.

Reviewed by:	alc

sys/amd64/amd64/trap.c

@@ -70,6 +70,9 @@ __FBSDID("$FreeBSD$");
 #ifdef KTRACE
 #include <sys/ktrace.h>
 #endif
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
@@ -180,6 +183,19 @@ trap(frame)
 	}
 #endif
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * CPU PMCs interrupt using an NMI.  If the PMC module is
+	 * active, pass the 'rip' value to the PMC module's interrupt
+	 * handler.  A return value of '1' from the handler means that
+	 * the NMI was handled by it and we can return immediately.
+	 */
+	if (type == T_NMI && pmc_intr &&
+	    (*pmc_intr)(PCPU_GET(cpuid), (uintptr_t) frame.tf_rip,
+		TRAPF_USERMODE(&frame)))
+		goto out;
+#endif
+
 	if ((frame.tf_rflags & PSL_I) == 0) {
 		/*
 		 * Buggy application or kernel code has disabled

sys/dev/hwpmc/hwpmc_amd.c

@@ -739,7 +739,7 @@ amd_stop_pmc(int cpu, int ri)
  */
 
 static int
-amd_intr(int cpu, uintptr_t eip)
+amd_intr(int cpu, uintptr_t eip, int usermode)
 {
 	int i, retval;
 	enum pmc_mode mode;
@@ -748,6 +748,8 @@ amd_intr(int cpu, uintptr_t eip)
 	struct pmc_cpu *pc;
 	struct pmc_hw *phw;
 
+	(void) usermode;
+
 	KASSERT(cpu >= 0 && cpu < mp_ncpus,
 	    ("[amd,%d] out of range CPU %d", __LINE__, cpu));
 

sys/dev/hwpmc/hwpmc_piv.c

@@ -1452,7 +1452,7 @@ p4_lapic_enable_pmc_interrupt(void)
 
 
 static int
-p4_intr(int cpu, uintptr_t eip)
+p4_intr(int cpu, uintptr_t eip, int usermode)
 {
 	int i, pmc_interrupted;
 	uint32_t cccrval, pmi_ovf_mask;
@@ -1462,6 +1462,7 @@ p4_intr(int cpu, uintptr_t eip)
 	pmc_value_t v;
 
 	(void) eip;
+	(void) usermode;
 	PMCDBG(MDP,INT, 1, "cpu=%d eip=%x pcint=0x%x", cpu, eip,
 	    lapic->lvt_pcint);
 

sys/dev/hwpmc/hwpmc_ppro.c

@@ -672,7 +672,7 @@ p6_stop_pmc(int cpu, int ri)
 }
 
 static int
-p6_intr(int cpu, uintptr_t eip)
+p6_intr(int cpu, uintptr_t eip, int usermode)
 {
 	(void) cpu;
 	(void) eip;

sys/i386/i386/trap.c

@@ -72,6 +72,9 @@ __FBSDID("$FreeBSD$");
 #ifdef KTRACE
 #include <sys/ktrace.h>
 #endif
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
@@ -198,6 +201,20 @@ trap(frame)
 	}
 #endif
 
+#ifdef	HWPMC_HOOKS
+	/*
+	 * CPU PMCs interrupt using an NMI so we check for that first.
+	 * If the HWPMC module is active, 'pmc_hook' will point to
+	 * the function to be called.  A return value of '1' from the
+	 * hook means that the NMI was handled by it and that we can
+	 * return immediately.
+	 */
+	if (type == T_NMI && pmc_intr &&
+	    (*pmc_intr)(PCPU_GET(cpuid), (uintptr_t) frame.tf_eip,
+		TRAPF_USERMODE(&frame)))
+	    goto out;
+#endif
+
 	if ((frame.tf_eflags & PSL_I) == 0) {
 		/*
 		 * Buggy application or kernel code has disabled

sys/kern/kern_clock.c

@@ -69,6 +69,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/gmon.h>
 #endif
 
+#ifdef HWPMC_HOOKS
+#include <sys/pmckern.h>
+#endif
+
 #ifdef DEVICE_POLLING
 extern void hardclock_device_poll(void);
 #endif /* DEVICE_POLLING */
@@ -191,6 +195,11 @@ hardclock_process(frame)
 		}
 	}
 	mtx_unlock_spin_flags(&sched_lock, MTX_QUIET);
+
+#ifdef	HWPMC_HOOKS
+	if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
+		PMC_CALL_HOOK_UNLOCKED(curthread, PMC_FN_DO_SAMPLES, NULL);
+#endif
 }
 
 /*

sys/kern/kern_pmc.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2003 Joseph Koshy
+ * Copyright (c) 2003-2005, Joseph Koshy
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -35,7 +35,9 @@ struct sx pmc_sx;
 int (*pmc_hook)(struct thread *td, int function, void *arg) = NULL;
 
 /* Interrupt handler */
-int (*pmc_intr)(int cpu, uintptr_t pc) = NULL;
+int (*pmc_intr)(int cpu, uintptr_t pc, int usermode) = NULL;
+
+cpumask_t pmc_cpumask;
 
 /*
  * Since PMC(4) may not be loaded in the current kernel, the

sys/sys/pmc.h

@@ -1347,7 +1347,7 @@ struct pmc_mdep  {
 	int (*pmd_stop_pmc)(int _cpu, int _ri);
 
 	/* handle a PMC interrupt */
-	int (*pmd_intr)(int _cpu, uintptr_t _pc);
+	int (*pmd_intr)(int _cpu, uintptr_t _pc, int _usermode);
 
 	int (*pmd_describe)(int _cpu, int _ri, struct pmc_info *_pi,
 		struct pmc **_ppmc);

sys/sys/pmckern.h

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2003, Joseph Koshy
+ * Copyright (c) 2003-2005, Joseph Koshy
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,20 +39,25 @@
 #include <sys/proc.h>
 #include <sys/sx.h>
 
-#define	PMC_FN_PROCESS_EXIT		1
-#define	PMC_FN_PROCESS_EXEC		2
-#define	PMC_FN_PROCESS_FORK		3
-#define	PMC_FN_CSW_IN			4
-#define	PMC_FN_CSW_OUT			5
+#define	PMC_FN_PROCESS_EXEC		1
+#define	PMC_FN_CSW_IN			2
+#define	PMC_FN_CSW_OUT			3
+#define	PMC_FN_DO_SAMPLES		4
+
+#define	PMC_FN_PROCESS_EXIT		5	/* obsolete */
+#define	PMC_FN_PROCESS_FORK		6	/* obsolete */
 
 /* hook */
 extern int (*pmc_hook)(struct thread *_td, int _function, void *_arg);
-extern int (*pmc_intr)(int cpu, uintptr_t pc);
+extern int (*pmc_intr)(int _cpu, uintptr_t _pc, int _usermode);
 
 /* SX lock protecting the hook */
 extern struct sx pmc_sx;
 
-/* hook invocation; for use within the kernel */
+/* Per-cpu flags indicating availability of sampling data */
+extern cpumask_t pmc_cpumask;
+
+/* Hook invocation; for use within the kernel */
 #define	PMC_CALL_HOOK(t, cmd, arg)		\
 do {						\
 	sx_slock(&pmc_sx);			\
@@ -61,7 +66,7 @@ do {						\
 	sx_sunlock(&pmc_sx);			\
 } while (0)
 
-/* hook invocation that needs an exclusive lock */
+/* Hook invocation that needs an exclusive lock */
 #define	PMC_CALL_HOOK_X(t, cmd, arg)		\
 do {						\
 	sx_xlock(&pmc_sx);			\
@@ -70,22 +75,26 @@ do {						\
 	sx_xunlock(&pmc_sx);			\
 } while (0)
 
-/* context switches cannot take locks */
-#define	PMC_SWITCH_CONTEXT(t, cmd)		\
+/*
+ * Some hook invocations (e.g., from context switch and clock handling
+ * code) need to be lock-free.
+ */
+#define	PMC_CALL_HOOK_UNLOCKED(t, cmd, arg)	\
 do {						\
 	if (pmc_hook != NULL)			\
-		(pmc_hook)((t), (cmd), NULL);	\
+		(pmc_hook)((t), (cmd), (arg));	\
 } while (0)
 
+#define	PMC_SWITCH_CONTEXT(t,cmd)	PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL)
 
-/*
- * check if a process is using HWPMCs.
- */
-
+/* Check if a process is using HWPMCs.*/
 #define PMC_PROC_IS_USING_PMCS(p)				\
 	(__predict_false(atomic_load_acq_int(&(p)->p_flag) &	\
 	    P_HWPMC))
 
+/* Check if a CPU has recorded samples. */
+#define	PMC_CPU_HAS_SAMPLES(C)	(__predict_false(pmc_cpumask & (1 << (C))))
+
 /* helper functions */
 int	pmc_cpu_is_disabled(int _cpu);
 int	pmc_cpu_is_logical(int _cpu);