Add the DTrace hooks for exception handling (Function boundary trace

-fbt- provider), cyclic clock and syscalls.

sys/i386/i386/exception.s

@@ -36,6 +36,7 @@
 
 #include "opt_apic.h"
 #include "opt_hwpmc_hooks.h"
+#include "opt_kdtrace.h"
 #include "opt_npx.h"
 
 #include <machine/asmacros.h>
@@ -45,7 +46,23 @@
 #include "assym.s"
 
 #define	SEL_RPL_MASK	0x0003
+#define GSEL_KPL	0x0020	/* GSEL(GCODE_SEL, SEL_KPL) */
 
+#ifdef KDTRACE_HOOKS
+	.bss
+	.globl	dtrace_invop_jump_addr
+	.align	4
+	.type	dtrace_invop_jump_addr, @object
+        .size	dtrace_invop_jump_addr, 4
+dtrace_invop_jump_addr:
+	.zero	4
+	.globl	dtrace_invop_calltrap_addr
+	.align	4
+	.type	dtrace_invop_calltrap_addr, @object
+        .size	dtrace_invop_calltrap_addr, 4
+dtrace_invop_calltrap_addr:
+	.zero	8
+#endif
 	.text
 #ifdef HWPMC_HOOKS
 	ENTRY(start_exceptions)
@@ -95,8 +112,6 @@ IDTVEC(ofl)
 	pushl $0; TRAP(T_OFLOW)
 IDTVEC(bnd)
 	pushl $0; TRAP(T_BOUND)
-IDTVEC(ill)
-	pushl $0; TRAP(T_PRIVINFLT)
 IDTVEC(dna)
 	pushl $0; TRAP(T_DNA)
 IDTVEC(fpusegm)
@@ -152,6 +167,43 @@ calltrap:
 	MEXITCOUNT
 	jmp	doreti
 
+/*
+ * Privileged instruction fault.
+ */
+	SUPERALIGN_TEXT
+IDTVEC(ill)
+	/* Check if there is no DTrace hook registered. */
+	cmpl	$0,dtrace_invop_jump_addr
+	je	norm_ill
+
+	/* Check if this is a user fault. */
+	cmpl	$GSEL_KPL, 4(%esp)	/* Check the code segment. */
+              
+	/* If so, just handle it as a normal trap. */
+	jne	norm_ill
+              
+	/*
+	 * This is a kernel instruction fault that might have been caused
+	 * by a DTrace provider.
+	 */
+	pushal				/* Push all registers onto the stack. */
+
+	/*
+	 * Set our jump address for the jump back in the event that
+	 * the exception wasn't caused by DTrace at all.
+	 */
+	movl	$norm_ill, dtrace_invop_calltrap_addr
+
+	/* Jump to the code hooked in by DTrace. */
+	jmpl	*dtrace_invop_jump_addr
+
+	/*
+	 * Process the instruction fault in the normal way.
+	 */
+norm_ill:
+	pushl $0
+	TRAP(T_PRIVINFLT)
+
 /*
  * SYSCALL CALL GATE (old entry point for a.out binaries)
  *

sys/i386/i386/local_apic.c

@@ -35,6 +35,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_hwpmc_hooks.h"
+#include "opt_kdtrace.h"
 
 #include "opt_ddb.h"
 
@@ -65,6 +66,11 @@ __FBSDID("$FreeBSD$");
 #include <ddb/ddb.h>
 #endif
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+cyclic_clock_func_t	lapic_cyclic_clock_func[MAXCPU];
+#endif
+
 /* Sanity checks on IDT vectors. */
 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT);
 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS);
@@ -670,6 +676,17 @@ lapic_handle_timer(struct trapframe *frame)
 	(*la->la_timer_count)++;
 	critical_enter();
 
+#ifdef KDTRACE_HOOKS
+	/*
+	 * If the DTrace hooks are configured and a callback function
+	 * has been registered, then call it to process the high speed
+	 * timers.
+	 */
+	int cpu = PCPU_GET(cpuid);
+	if (lapic_cyclic_clock_func[cpu] != NULL)
+		(*lapic_cyclic_clock_func[cpu])(frame);
+#endif
+
 	/* Fire hardclock at hz. */
 	la->la_hard_ticks += hz;
 	if (la->la_hard_ticks >= lapic_timer_hz) {

sys/i386/i386/trap.c

@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_hwpmc_hooks.h"
 #include "opt_isa.h"
 #include "opt_kdb.h"
+#include "opt_kdtrace.h"
 #include "opt_ktrace.h"
 #include "opt_npx.h"
 #include "opt_trap.h"
@@ -102,6 +103,26 @@ __FBSDID("$FreeBSD$");
 #include <machine/clock.h>
 #endif
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+/*
+ * This is a hook which is initialised by the dtrace module
+ * to handle traps which might occur during DTrace probe
+ * execution.
+ */
+dtrace_trap_func_t	dtrace_trap_func;
+
+dtrace_doubletrap_func_t	dtrace_doubletrap_func;
+
+/*
+ * This is a hook which is initialised by the systrace module
+ * when it is loaded. This keeps the DTrace syscall provider
+ * implementation opaque. 
+ */
+systrace_probe_func_t	systrace_probe_func;
+#endif
+
 extern void trap(struct trapframe *frame);
 extern void syscall(struct trapframe *frame);
 
@@ -218,6 +239,25 @@ trap(struct trapframe *frame)
 	    goto out;
 #endif
 
+#ifdef KDTRACE_HOOKS
+	/*
+	 * A trap can occur while DTrace executes a probe. Before
+	 * executing the probe, DTrace blocks re-scheduling and sets
+	 * a flag in it's per-cpu flags to indicate that it doesn't
+	 * want to fault. On returning from the the probe, the no-fault
+	 * flag is cleared and finally re-scheduling is enabled.
+	 *
+	 * If the DTrace kernel module has registered a trap handler,
+	 * call it and if it returns non-zero, assume that it has
+	 * handled the trap and modified the trap frame so that this
+	 * function can return normally.
+	 */
+	if ((type == T_PROTFLT || type == T_PAGEFLT) &&
+	    dtrace_trap_func != NULL)
+		if ((*dtrace_trap_func)(frame, type))
+			goto out;
+#endif
+
 	if ((frame->tf_eflags & PSL_I) == 0) {
 		/*
 		 * Buggy application or kernel code has disabled
@@ -911,6 +951,10 @@ trap_fatal(frame, eva)
 void
 dblfault_handler()
 {
+#ifdef KDTRACE_HOOKS
+	if (dtrace_doubletrap_func != NULL)
+		(*dtrace_doubletrap_func)();
+#endif
 	printf("\nFatal double fault:\n");
 	printf("eip = 0x%x\n", PCPU_GET(common_tss.tss_eip));
 	printf("esp = 0x%x\n", PCPU_GET(common_tss.tss_esp));
@@ -1022,9 +1066,34 @@ syscall(struct trapframe *frame)
 
 		PTRACESTOP_SC(p, td, S_PT_SCE);
 
+#ifdef KDTRACE_HOOKS
+		/*
+		 * If the systrace module has registered it's probe
+		 * callback and if there is a probe active for the
+		 * syscall 'entry', process the probe.
+		 */
+		if (systrace_probe_func != NULL && callp->sy_entry != 0)
+			(*systrace_probe_func)(callp->sy_entry, code, callp,
+			    args);
+#endif
+
 		AUDIT_SYSCALL_ENTER(code, td);
 		error = (*callp->sy_call)(td, args);
 		AUDIT_SYSCALL_EXIT(error, td);
+
+		/* Save the latest error return value. */
+		td->td_errno = error;
+
+#ifdef KDTRACE_HOOKS
+		/*
+		 * If the systrace module has registered it's probe
+		 * callback and if there is a probe active for the
+		 * syscall 'return', process the probe.
+		 */
+		if (systrace_probe_func != NULL && callp->sy_return != 0)
+			(*systrace_probe_func)(callp->sy_return, code, callp,
+			    args);
+#endif
 	}
 
 	switch (error) {