From f03b37f3e8c9e424c3a74ed50c1ed0fcf5053530 Mon Sep 17 00:00:00 2001 From: mjg Date: Mon, 12 Dec 2016 15:37:11 +0000 Subject: [PATCH] vfs: add vrefact, to be used when the vnode has to be already active This allows blind increment of relevant counters which under contention is cheaper than inc-not-zero loops at least on amd64. Use it in some of the places which are guaranteed to see already active vnodes. Reviewed by: kib (previous version) --- sys/kern/kern_descrip.c | 46 ++++++++++++++++++++--------------------- sys/kern/kern_fork.c | 2 +- sys/kern/vfs_lookup.c | 8 +++---- sys/kern/vfs_subr.c | 22 ++++++++++++++++++++ sys/sys/vnode.h | 1 + 5 files changed, 51 insertions(+), 28 deletions(-) diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c index 78cc6824ae4b..a309eda77aab 100644 --- a/sys/kern/kern_descrip.c +++ b/sys/kern/kern_descrip.c @@ -318,11 +318,11 @@ pwd_ensure_dirs(void) FILEDESC_XLOCK(fdp); if (fdp->fd_cdir == NULL) { fdp->fd_cdir = rootvnode; - VREF(rootvnode); + vrefact(rootvnode); } if (fdp->fd_rdir == NULL) { fdp->fd_rdir = rootvnode; - VREF(rootvnode); + vrefact(rootvnode); } FILEDESC_XUNLOCK(fdp); } @@ -1860,13 +1860,13 @@ fdinit(struct filedesc *fdp, bool prepfiles) FILEDESC_SLOCK(fdp); newfdp->fd_cdir = fdp->fd_cdir; if (newfdp->fd_cdir) - VREF(newfdp->fd_cdir); + vrefact(newfdp->fd_cdir); newfdp->fd_rdir = fdp->fd_rdir; if (newfdp->fd_rdir) - VREF(newfdp->fd_rdir); + vrefact(newfdp->fd_rdir); newfdp->fd_jdir = fdp->fd_jdir; if (newfdp->fd_jdir) - VREF(newfdp->fd_jdir); + vrefact(newfdp->fd_jdir); if (!prepfiles) { FILEDESC_SUNLOCK(fdp); @@ -2769,7 +2769,7 @@ _fgetvp(struct thread *td, int fd, int flags, cap_rights_t *needrightsp, error = EINVAL; } else { *vpp = fp->f_vnode; - vref(*vpp); + vrefact(*vpp); } fdrop(fp, td); @@ -2807,7 +2807,7 @@ fgetvp_rights(struct thread *td, int fd, cap_rights_t *needrightsp, *havecaps = caps; *vpp = fp->f_vnode; - vref(*vpp); + vrefact(*vpp); return (0); out: @@ -3115,10 +3115,10 @@ pwd_chroot(struct thread *td, struct vnode *vp) } } oldvp = fdp->fd_rdir; - VREF(vp); + vrefact(vp); fdp->fd_rdir = vp; if (fdp->fd_jdir == NULL) { - VREF(vp); + vrefact(vp); fdp->fd_jdir = vp; } FILEDESC_XUNLOCK(fdp); @@ -3166,17 +3166,17 @@ mountcheckdirs(struct vnode *olddp, struct vnode *newdp) continue; FILEDESC_XLOCK(fdp); if (fdp->fd_cdir == olddp) { - vref(newdp); + vrefact(newdp); fdp->fd_cdir = newdp; nrele++; } if (fdp->fd_rdir == olddp) { - vref(newdp); + vrefact(newdp); fdp->fd_rdir = newdp; nrele++; } if (fdp->fd_jdir == olddp) { - vref(newdp); + vrefact(newdp); fdp->fd_jdir = newdp; nrele++; } @@ -3185,13 +3185,13 @@ mountcheckdirs(struct vnode *olddp, struct vnode *newdp) } sx_sunlock(&allproc_lock); if (rootvnode == olddp) { - vref(newdp); + vrefact(newdp); rootvnode = newdp; nrele++; } mtx_lock(&prison0.pr_mtx); if (prison0.pr_root == olddp) { - vref(newdp); + vrefact(newdp); prison0.pr_root = newdp; nrele++; } @@ -3200,7 +3200,7 @@ mountcheckdirs(struct vnode *olddp, struct vnode *newdp) TAILQ_FOREACH(pr, &allprison, pr_list) { mtx_lock(&pr->pr_mtx); if (pr->pr_root == olddp) { - vref(newdp); + vrefact(newdp); pr->pr_root = newdp; nrele++; } @@ -3527,17 +3527,17 @@ kern_proc_filedesc_out(struct proc *p, struct sbuf *sb, ssize_t maxlen, /* ktrace vnode */ tracevp = p->p_tracevp; if (tracevp != NULL) - vref(tracevp); + vrefact(tracevp); /* text vnode */ textvp = p->p_textvp; if (textvp != NULL) - vref(textvp); + vrefact(textvp); /* Controlling tty. */ cttyvp = NULL; if (p->p_pgrp != NULL && p->p_pgrp->pg_session != NULL) { cttyvp = p->p_pgrp->pg_session->s_ttyvp; if (cttyvp != NULL) - vref(cttyvp); + vrefact(cttyvp); } fdp = fdhold(p); PROC_UNLOCK(p); @@ -3561,17 +3561,17 @@ kern_proc_filedesc_out(struct proc *p, struct sbuf *sb, ssize_t maxlen, FILEDESC_SLOCK(fdp); /* working directory */ if (fdp->fd_cdir != NULL) { - vref(fdp->fd_cdir); + vrefact(fdp->fd_cdir); export_vnode_to_sb(fdp->fd_cdir, KF_FD_TYPE_CWD, FREAD, efbuf); } /* root directory */ if (fdp->fd_rdir != NULL) { - vref(fdp->fd_rdir); + vrefact(fdp->fd_rdir); export_vnode_to_sb(fdp->fd_rdir, KF_FD_TYPE_ROOT, FREAD, efbuf); } /* jail directory */ if (fdp->fd_jdir != NULL) { - vref(fdp->fd_jdir); + vrefact(fdp->fd_jdir); export_vnode_to_sb(fdp->fd_jdir, KF_FD_TYPE_JAIL, FREAD, efbuf); } for (i = 0; fdp->fd_refcnt > 0 && i <= fdp->fd_lastfile; i++) { @@ -3661,7 +3661,7 @@ export_vnode_for_osysctl(struct vnode *vp, int type, struct kinfo_file *kif, { int error; - vref(vp); + vrefact(vp); FILEDESC_SUNLOCK(fdp); export_vnode_to_kinfo(vp, type, 0, kif, KERN_FILEDESC_PACK_KINFO); kinfo_to_okinfo(kif, okif); @@ -3788,7 +3788,7 @@ kern_proc_cwd_out(struct proc *p, struct sbuf *sb, ssize_t maxlen) if (fdp->fd_cdir == NULL) error = EINVAL; else { - vref(fdp->fd_cdir); + vrefact(fdp->fd_cdir); error = export_vnode_to_sb(fdp->fd_cdir, KF_FD_TYPE_CWD, FREAD, efbuf); } diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 4f8baf8d5a62..7e340a9cdc9b 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -547,7 +547,7 @@ do_fork(struct thread *td, struct fork_req *fr, struct proc *p2, struct thread * /* Bump references to the text vnode (for procfs). */ if (p2->p_textvp) - vref(p2->p_textvp); + vrefact(p2->p_textvp); /* * Set up linkage for kernel based threading. diff --git a/sys/kern/vfs_lookup.c b/sys/kern/vfs_lookup.c index 7885ec1e7647..73b10e15fa5e 100644 --- a/sys/kern/vfs_lookup.c +++ b/sys/kern/vfs_lookup.c @@ -255,7 +255,7 @@ namei_handle_root(struct nameidata *ndp, struct vnode **dpp) ndp->ni_pathlen--; } *dpp = ndp->ni_rootdir; - VREF(*dpp); + vrefact(*dpp); return (0); } @@ -376,7 +376,7 @@ namei(struct nameidata *ndp) */ FILEDESC_SLOCK(fdp); ndp->ni_rootdir = fdp->fd_rdir; - VREF(ndp->ni_rootdir); + vrefact(ndp->ni_rootdir); ndp->ni_topdir = fdp->fd_jdir; /* @@ -398,7 +398,7 @@ namei(struct nameidata *ndp) startdir_used = 1; } else if (ndp->ni_dirfd == AT_FDCWD) { dp = fdp->fd_cdir; - VREF(dp); + vrefact(dp); } else { rights = ndp->ni_rightsneeded; cap_rights_set(&rights, CAP_LOOKUP); @@ -956,7 +956,7 @@ lookup(struct nameidata *ndp) vput(ndp->ni_dvp); else vrele(ndp->ni_dvp); - vref(vp_crossmp); + vrefact(vp_crossmp); ndp->ni_dvp = vp_crossmp; error = VFS_ROOT(mp, compute_cn_lkflags(mp, cnp->cn_lkflags, cnp->cn_flags), &tdp); diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index b728887b4fee..22636b0ca000 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -2647,6 +2647,28 @@ vrefl(struct vnode *vp) v_incr_usecount_locked(vp); } +void +vrefact(struct vnode *vp) +{ + + CTR2(KTR_VFS, "%s: vp %p", __func__, vp); + if (__predict_false(vp->v_type == VCHR)) { + VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp, + ("%s: wrong ref counts", __func__)); + vref(vp); + return; + } +#ifdef INVARIANTS + int old = atomic_fetchadd_int(&vp->v_holdcnt, 1); + VNASSERT(old > 0, vp, ("%s: wrong hold count", __func__)); + old = atomic_fetchadd_int(&vp->v_usecount, 1); + VNASSERT(old > 0, vp, ("%s: wrong use count", __func__)); +#else + refcount_acquire(&vp->v_holdcnt); + refcount_acquire(&vp->v_usecount); +#endif +} + /* * Return reference count of a vnode. * diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h index 989af8bc13a4..7c7be966c868 100644 --- a/sys/sys/vnode.h +++ b/sys/sys/vnode.h @@ -828,6 +828,7 @@ void vput(struct vnode *vp); void vrele(struct vnode *vp); void vref(struct vnode *vp); void vrefl(struct vnode *vp); +void vrefact(struct vnode *vp); int vrefcnt(struct vnode *vp); void v_addpollinfo(struct vnode *vp);