diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index d9187281c75e..c5e1b4a748b4 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -393,6 +393,8 @@ statclock_process(ke, pc, user) /* * Charge the time as appropriate. */ + if (p->p_flag & P_KSES) + thread_add_ticks_intr(1, 1); ke->ke_uticks++; if (ke->ke_ksegrp->kg_nice > NZERO) cp_time[CP_NICE]++; @@ -430,6 +432,8 @@ statclock_process(ke, pc, user) ke->ke_iticks++; cp_time[CP_INTR]++; } else { + if (p->p_flag & P_KSES) + thread_add_ticks_intr(0, 1); ke->ke_sticks++; if (p != PCPU_GET(idlethread)->td_proc) cp_time[CP_SYS]++; diff --git a/sys/kern/kern_kse.c b/sys/kern/kern_kse.c index 1837e2a9d2ea..2672c4337f4e 100644 --- a/sys/kern/kern_kse.c +++ b/sys/kern/kern_kse.c @@ -374,7 +374,7 @@ kse_release(struct thread *td, struct kse_release_args *uap) msleep(p->p_sigacts, &p->p_mtx, PPAUSE|PCATCH, "pause", 0); mtx_lock_spin(&sched_lock); - td->td_flags &= ~TDF_UNBOUND; + td->td_flags |= TDF_UNBOUND; thread_schedule_upcall(td, td->td_kse); thread_exit(); } @@ -783,6 +783,7 @@ thread_export_context(struct thread *td) void *addr; int error; ucontext_t uc; + uint temp; p = td->td_proc; kg = td->td_ksegrp; @@ -820,10 +821,7 @@ thread_export_context(struct thread *td) for (;;) { mbx = (uintptr_t)kg->kg_completed; if (suword(addr, mbx)) { - PROC_LOCK(p); - psignal(p, SIGSEGV); - PROC_UNLOCK(p); - return (EFAULT); + goto bad; } PROC_LOCK(p); if (mbx == (uintptr_t)kg->kg_completed) { @@ -833,7 +831,18 @@ thread_export_context(struct thread *td) } PROC_UNLOCK(p); } + addr = (caddr_t)td->td_mailbox + + offsetof(struct kse_thr_mailbox, tm_sticks); + temp = fuword(addr) + td->td_usticks; + if (suword(addr, temp)) + goto bad; return (0); + +bad: + PROC_LOCK(p); + psignal(p, SIGSEGV); + PROC_UNLOCK(p); + return (EFAULT); } /* @@ -873,6 +882,78 @@ thread_link_mboxes(struct ksegrp *kg, struct kse *ke) return (0); } +/* + * This function should be called at statclock interrupt time + */ +int +thread_add_ticks_intr(int user, uint ticks) +{ + struct thread *td = curthread; + struct kse *ke = td->td_kse; + + if (ke->ke_mailbox == NULL) + return -1; + if (user) { + /* Current always do via ast() */ + ke->ke_flags |= KEF_ASTPENDING; + ke->ke_uuticks += ticks; + } else { + if (td->td_mailbox != NULL) + td->td_usticks += ticks; + else + ke->ke_usticks += ticks; + } + return 0; +} + +static int +thread_update_uticks(void) +{ + struct thread *td = curthread; + struct proc *p = td->td_proc; + struct kse *ke = td->td_kse; + struct kse_thr_mailbox *tmbx; + caddr_t addr; + uint uticks, sticks; + struct timespec ts; + + KASSERT(!(td->td_flags & TDF_UNBOUND), ("thread not bound.")); + + if (ke->ke_mailbox == NULL) + return 0; + + nanotime(&ts); + if (copyout(&ts, (caddr_t)&ke->ke_mailbox->km_timeofday, sizeof(ts))) + goto bad; + + uticks = ke->ke_uuticks; + ke->ke_uuticks = 0; + sticks = ke->ke_usticks; + ke->ke_usticks = 0; + tmbx = (void *)fuword((caddr_t)ke->ke_mailbox + + offsetof(struct kse_mailbox, km_curthread)); + if ((tmbx == NULL) || (tmbx == (void *)-1)) + return 0; + if (uticks) { + addr = (caddr_t)tmbx + offsetof(struct kse_thr_mailbox, tm_uticks); + uticks += fuword(addr); + if (suword(addr, uticks)) + goto bad; + } + if (sticks) { + addr = (caddr_t)tmbx + offsetof(struct kse_thr_mailbox, tm_sticks); + sticks += fuword(addr); + if (suword(addr, sticks)) + goto bad; + } + return 0; +bad: + PROC_LOCK(p); + psignal(p, SIGSEGV); + PROC_UNLOCK(p); + return -1; +} + /* * Discard the current thread and exit from its context. * @@ -1267,14 +1348,28 @@ thread_user_enter(struct proc *p, struct thread *td) (void *)fuword( (void *)&ke->ke_mailbox->km_curthread); #endif if ((td->td_mailbox == NULL) || - (td->td_mailbox == (void *)-1) || - (p->p_numthreads > max_threads_per_proc)) { + (td->td_mailbox == (void *)-1)) { td->td_mailbox = NULL; /* single thread it.. */ + mtx_lock_spin(&sched_lock); td->td_flags &= ~TDF_UNBOUND; + mtx_unlock_spin(&sched_lock); } else { - if (td->td_standin == NULL) - td->td_standin = thread_alloc(); + /* + * when thread limit reached, act like that the thread + * has already done an upcall. + */ + if (p->p_numthreads > max_threads_per_proc) { + if (td->td_standin != NULL) + thread_stash(td->td_standin); + td->td_standin = NULL; + } else { + if (td->td_standin == NULL) + td->td_standin = thread_alloc(); + } + mtx_lock_spin(&sched_lock); td->td_flags |= TDF_UNBOUND; + mtx_unlock_spin(&sched_lock); + td->td_usticks = 0; } } } @@ -1335,6 +1430,7 @@ thread_userret(struct thread *td, struct trapframe *frame) */ error = thread_export_context(td); td->td_mailbox = NULL; + td->td_usticks = 0; if (error) { /* * If we are not running on a borrowed KSE, then @@ -1352,6 +1448,7 @@ thread_userret(struct thread *td, struct trapframe *frame) td->td_flags &= ~TDF_UNBOUND; PROC_UNLOCK(td->td_proc); mtx_unlock_spin(&sched_lock); + thread_update_uticks(); return (error); /* go sync */ } thread_exit(); @@ -1424,6 +1521,7 @@ thread_userret(struct thread *td, struct trapframe *frame) } } + thread_update_uticks(); /* * To get here, we know there is no other need for our * KSE so we can proceed. If not upcalling, go back to @@ -1468,6 +1566,7 @@ thread_userret(struct thread *td, struct trapframe *frame) offsetof(struct kse_mailbox, km_curthread), 0); #else /* if user pointer arithmetic is ok in the kernel */ error = suword((caddr_t)&ke->ke_mailbox->km_curthread, 0); + ke->ke_uuticks = ke->ke_usticks = 0; #endif if (!error) return (0); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 1837e2a9d2ea..2672c4337f4e 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -374,7 +374,7 @@ kse_release(struct thread *td, struct kse_release_args *uap) msleep(p->p_sigacts, &p->p_mtx, PPAUSE|PCATCH, "pause", 0); mtx_lock_spin(&sched_lock); - td->td_flags &= ~TDF_UNBOUND; + td->td_flags |= TDF_UNBOUND; thread_schedule_upcall(td, td->td_kse); thread_exit(); } @@ -783,6 +783,7 @@ thread_export_context(struct thread *td) void *addr; int error; ucontext_t uc; + uint temp; p = td->td_proc; kg = td->td_ksegrp; @@ -820,10 +821,7 @@ thread_export_context(struct thread *td) for (;;) { mbx = (uintptr_t)kg->kg_completed; if (suword(addr, mbx)) { - PROC_LOCK(p); - psignal(p, SIGSEGV); - PROC_UNLOCK(p); - return (EFAULT); + goto bad; } PROC_LOCK(p); if (mbx == (uintptr_t)kg->kg_completed) { @@ -833,7 +831,18 @@ thread_export_context(struct thread *td) } PROC_UNLOCK(p); } + addr = (caddr_t)td->td_mailbox + + offsetof(struct kse_thr_mailbox, tm_sticks); + temp = fuword(addr) + td->td_usticks; + if (suword(addr, temp)) + goto bad; return (0); + +bad: + PROC_LOCK(p); + psignal(p, SIGSEGV); + PROC_UNLOCK(p); + return (EFAULT); } /* @@ -873,6 +882,78 @@ thread_link_mboxes(struct ksegrp *kg, struct kse *ke) return (0); } +/* + * This function should be called at statclock interrupt time + */ +int +thread_add_ticks_intr(int user, uint ticks) +{ + struct thread *td = curthread; + struct kse *ke = td->td_kse; + + if (ke->ke_mailbox == NULL) + return -1; + if (user) { + /* Current always do via ast() */ + ke->ke_flags |= KEF_ASTPENDING; + ke->ke_uuticks += ticks; + } else { + if (td->td_mailbox != NULL) + td->td_usticks += ticks; + else + ke->ke_usticks += ticks; + } + return 0; +} + +static int +thread_update_uticks(void) +{ + struct thread *td = curthread; + struct proc *p = td->td_proc; + struct kse *ke = td->td_kse; + struct kse_thr_mailbox *tmbx; + caddr_t addr; + uint uticks, sticks; + struct timespec ts; + + KASSERT(!(td->td_flags & TDF_UNBOUND), ("thread not bound.")); + + if (ke->ke_mailbox == NULL) + return 0; + + nanotime(&ts); + if (copyout(&ts, (caddr_t)&ke->ke_mailbox->km_timeofday, sizeof(ts))) + goto bad; + + uticks = ke->ke_uuticks; + ke->ke_uuticks = 0; + sticks = ke->ke_usticks; + ke->ke_usticks = 0; + tmbx = (void *)fuword((caddr_t)ke->ke_mailbox + + offsetof(struct kse_mailbox, km_curthread)); + if ((tmbx == NULL) || (tmbx == (void *)-1)) + return 0; + if (uticks) { + addr = (caddr_t)tmbx + offsetof(struct kse_thr_mailbox, tm_uticks); + uticks += fuword(addr); + if (suword(addr, uticks)) + goto bad; + } + if (sticks) { + addr = (caddr_t)tmbx + offsetof(struct kse_thr_mailbox, tm_sticks); + sticks += fuword(addr); + if (suword(addr, sticks)) + goto bad; + } + return 0; +bad: + PROC_LOCK(p); + psignal(p, SIGSEGV); + PROC_UNLOCK(p); + return -1; +} + /* * Discard the current thread and exit from its context. * @@ -1267,14 +1348,28 @@ thread_user_enter(struct proc *p, struct thread *td) (void *)fuword( (void *)&ke->ke_mailbox->km_curthread); #endif if ((td->td_mailbox == NULL) || - (td->td_mailbox == (void *)-1) || - (p->p_numthreads > max_threads_per_proc)) { + (td->td_mailbox == (void *)-1)) { td->td_mailbox = NULL; /* single thread it.. */ + mtx_lock_spin(&sched_lock); td->td_flags &= ~TDF_UNBOUND; + mtx_unlock_spin(&sched_lock); } else { - if (td->td_standin == NULL) - td->td_standin = thread_alloc(); + /* + * when thread limit reached, act like that the thread + * has already done an upcall. + */ + if (p->p_numthreads > max_threads_per_proc) { + if (td->td_standin != NULL) + thread_stash(td->td_standin); + td->td_standin = NULL; + } else { + if (td->td_standin == NULL) + td->td_standin = thread_alloc(); + } + mtx_lock_spin(&sched_lock); td->td_flags |= TDF_UNBOUND; + mtx_unlock_spin(&sched_lock); + td->td_usticks = 0; } } } @@ -1335,6 +1430,7 @@ thread_userret(struct thread *td, struct trapframe *frame) */ error = thread_export_context(td); td->td_mailbox = NULL; + td->td_usticks = 0; if (error) { /* * If we are not running on a borrowed KSE, then @@ -1352,6 +1448,7 @@ thread_userret(struct thread *td, struct trapframe *frame) td->td_flags &= ~TDF_UNBOUND; PROC_UNLOCK(td->td_proc); mtx_unlock_spin(&sched_lock); + thread_update_uticks(); return (error); /* go sync */ } thread_exit(); @@ -1424,6 +1521,7 @@ thread_userret(struct thread *td, struct trapframe *frame) } } + thread_update_uticks(); /* * To get here, we know there is no other need for our * KSE so we can proceed. If not upcalling, go back to @@ -1468,6 +1566,7 @@ thread_userret(struct thread *td, struct trapframe *frame) offsetof(struct kse_mailbox, km_curthread), 0); #else /* if user pointer arithmetic is ok in the kernel */ error = suword((caddr_t)&ke->ke_mailbox->km_curthread, 0); + ke->ke_uuticks = ke->ke_usticks = 0; #endif if (!error) return (0); diff --git a/sys/sys/kse.h b/sys/sys/kse.h index 8f5fa578b0ac..c60247a60c7e 100644 --- a/sys/sys/kse.h +++ b/sys/sys/kse.h @@ -35,6 +35,7 @@ #include #include +#include /* * This file defines the structures needed for communication between @@ -56,6 +57,8 @@ struct kse_thr_mailbox { unsigned int tm_flags; /* Thread flags */ struct kse_thr_mailbox *tm_next; /* Next thread in list */ void *tm_udata; /* For use by the UTS */ + unsigned int tm_uticks; + unsigned int tm_sticks; int tm_spare[8]; }; @@ -66,6 +69,7 @@ struct kse_thr_mailbox { * a single KSE. */ struct kse_mailbox { + int km_version; /* Mailbox version */ struct kse_thr_mailbox *km_curthread; /* Currently running thread */ struct kse_thr_mailbox *km_completed; /* Threads back from kernel */ sigset_t km_sigscaught; /* Caught signals */ @@ -73,9 +77,12 @@ struct kse_mailbox { kse_func_t *km_func; /* UTS function */ stack_t km_stack; /* UTS context */ void *km_udata; /* For use by the UTS */ + struct timespec km_timeofday; /* Time of day */ int km_spare[8]; }; +#define KSE_VER_0 0 + #ifndef _KERNEL int kse_create(struct kse_mailbox *, int); int kse_exit(void); diff --git a/sys/sys/proc.h b/sys/sys/proc.h index dd129def1247..a8f392ca0faf 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -288,6 +288,7 @@ struct thread { struct ucred *td_ucred; /* (k) Reference to credentials. */ void (*td_switchin)(void); /* (k) Switchin special func. */ struct thread *td_standin; /* (?) use this for an upcall */ + u_int td_usticks; /* Statclock hits in kernel, for UTS */ u_int td_critnest; /* (k) Critical section nest level. */ #define td_endzero td_base_pri @@ -424,6 +425,8 @@ struct kse { u_int64_t ke_uticks; /* (j) Statclock hits in user mode. */ u_int64_t ke_sticks; /* (j) Statclock hits in system mode. */ u_int64_t ke_iticks; /* (j) Statclock hits in intr. */ + u_int ke_uuticks; /* Statclock hits in user, for UTS */ + u_int ke_usticks; /* Statclock hits in kernel, for UTS */ u_char ke_oncpu; /* (j) Which cpu we are on. */ char ke_rqindex; /* (j) Run queue index. */ enum { @@ -933,6 +936,7 @@ void thread_unsuspend(struct proc *p); void thread_unsuspend_one(struct thread *td); int thread_userret(struct thread *td, struct trapframe *frame); void thread_user_enter(struct proc *p, struct thread *td); +int thread_add_ticks_intr(int user, uint ticks); void thread_sanity_check(struct thread *td, char *); #endif /* _KERNEL */