Give struct socket structures a ref counting interface similar to

vnodes.  This will hopefully serve as a base from which we can
expand the MP code.  We currently do not attempt to obtain any
mutex or SX locks, but the door is open to add them when we nail
down exactly how that part of it is going to work.

sys/compat/svr4/svr4_stream.c

@@ -150,7 +150,6 @@ svr4_sendit(td, s, mp, flags)
 	register struct msghdr *mp;
 	int flags;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
@@ -163,8 +162,7 @@ svr4_sendit(td, s, mp, flags)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
-	if (error)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -176,16 +174,14 @@ svr4_sendit(td, s, mp, flags)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
+			error = EINVAL;
-			return (EINVAL);
+			goto done1;
 		}
 	}
 	if (mp->msg_name) {
 		error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
-		if (error) {
+		if (error)
-			fdrop(fp, td);
+			goto done1;
-			return (error);
-		}
 	} else {
 		to = 0;
 	}
@@ -211,7 +207,6 @@ svr4_sendit(td, s, mp, flags)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_sosend(so, to, &auio, 0, control,
 						     flags, td);
 	if (error) {
@@ -239,7 +234,8 @@ svr4_sendit(td, s, mp, flags)
 bad:
 	if (to)
 		FREE(to, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 	return (error);
 }
 
@@ -250,7 +246,6 @@ svr4_recvit(td, s, mp, namelenp)
 	register struct msghdr *mp;
 	caddr_t namelenp;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
@@ -264,8 +259,7 @@ svr4_recvit(td, s, mp, namelenp)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
-	if (error)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -277,8 +271,8 @@ svr4_recvit(td, s, mp, namelenp)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
+			error = EINVAL;
-			return (EINVAL);
+			goto done1;
 		}
 	}
 #ifdef KTRACE
@@ -291,7 +285,6 @@ svr4_recvit(td, s, mp, namelenp)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_soreceive(so, &fromsa, &auio,
 	    (struct mbuf **)0, mp->msg_control ? &control : (struct mbuf **)0,
 	    &mp->msg_flags);
@@ -365,7 +358,8 @@ out:
 		FREE(fromsa, M_SONAME);
 	if (control)
 		m_freem(control);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 	return (error);
 }
 

sys/kern/kern_descrip.c

@@ -60,6 +60,8 @@
 #include <sys/unistd.h>
 #include <sys/resourcevar.h>
 #include <sys/event.h>
+#include <sys/sx.h>
+#include <sys/socketvar.h>
 
 #include <machine/limits.h>
 
@@ -1425,6 +1427,51 @@ fgetvp_write(struct thread *td, int fd, struct vnode **vpp)
 	return(_fgetvp(td, fd, vpp, FWRITE));
 }
 
+/*
+ * Like fget() but loads the underlying socket, or returns an error if
+ * the descriptor does not represent a socket.
+ *
+ * We bump the ref count on the returned socket.  XXX Also obtain the SX lock in
+ * the future.
+ */
+int
+fgetsock(struct thread *td, int fd, struct socket **spp, u_int *fflagp)
+{
+	struct filedesc *fdp;
+	struct file *fp;
+	struct socket *so;
+
+	GIANT_REQUIRED;
+	fdp = td->td_proc->p_fd;
+	*spp = NULL;
+	if (fflagp)
+		*fflagp = 0;
+	if ((u_int)fd >= fdp->fd_nfiles)
+		return(EBADF);
+	if ((fp = fdp->fd_ofiles[fd]) == NULL)
+		return(EBADF);
+	if (fp->f_type != DTYPE_SOCKET)
+		return(ENOTSOCK);
+	if (fp->f_data == NULL)
+		return(EINVAL);
+	so = (struct socket *)fp->f_data;
+	if (fflagp)
+		*fflagp = fp->f_flag;
+	soref(so);
+	*spp = so;
+	return(0);
+}
+
+/*
+ * Drop the reference count on the the socket and XXX release the SX lock in
+ * the future.  The last reference closes the socket.
+ */
+void
+fputsock(struct socket *so)
+{
+	sorele(so);
+}
+
 int
 fdrop(fp, td)
 	struct file *fp;

sys/kern/sys_socket.c

@@ -182,6 +182,12 @@ soo_stat(fp, ub, td)
 	return ((*so->so_proto->pr_usrreqs->pru_sense)(so, ub));
 }
 
+/*
+ * API socket close on file pointer.  We call soclose() to close the 
+ * socket (including initiating closing protocols).  soclose() will
+ * sorele() the file reference but the actual socket will not go away
+ * until the socket's ref count hits 0.
+ */
 /* ARGSUSED */
 int
 soo_close(fp, td)
@@ -189,10 +195,12 @@ soo_close(fp, td)
 	struct thread *td;
 {
 	int error = 0;
+	struct socket *so;
 
 	fp->f_ops = &badfileops;
-	if (fp->f_data)
+	if ((so = (struct socket *)fp->f_data) != NULL) {
-		error = soclose((struct socket *)fp->f_data);
+		fp->f_data = NULL;
-	fp->f_data = 0;
+		error = soclose(so);
+	}
 	return (error);
 }

sys/kern/uipc_sockbuf.c

@@ -210,6 +210,8 @@ sodropablereq(head)
  * then we allocate a new structure, propoerly linked into the
  * data structure of the original socket, and return this.
  * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
+ *
+ * note: the ref count on the socket is 0 on return
  */
 struct socket *
 sonewconn(head, connstatus)
@@ -246,7 +248,7 @@ sonewconn3(head, connstatus, td)
 		so->so_cred = crhold(head->so_cred);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
-		sodealloc(so);
+		sotryfree(so);
 		return ((struct socket *)0);
 	}
 

sys/kern/uipc_socket.c

@@ -91,6 +91,10 @@ SYSCTL_DECL(_kern_ipc);
 static int somaxconn = SOMAXCONN;
 SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
     &somaxconn, 0, "Maximum pending socket connection queue size");
+static int numopensockets;
+SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
+    &numopensockets, 0, "Number of open sockets");
+
 
 /*
  * Socket operation routines.
@@ -106,6 +110,8 @@ SYSCTL_INT(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLFLAG_RW,
  * Note that it would probably be better to allocate socket
  * and PCB at the same time, but I'm not convinced that all
  * the protocols can be easily modified to do this.
+ *
+ * soalloc() returns a socket with a ref count of 0.
  */
 struct socket *
 soalloc(waitok)
@@ -119,11 +125,17 @@ soalloc(waitok)
 		bzero(so, sizeof *so);
 		so->so_gencnt = ++so_gencnt;
 		so->so_zone = socket_zone;
+		/* sx_init(&so->so_sxlock, "socket sxlock"); */
 		TAILQ_INIT(&so->so_aiojobq);
+		++numopensockets;
 	}
 	return so;
 }
 
+/*
+ * socreate returns a socket with a ref count of 1.  The socket should be
+ * closed with soclose().
+ */
 int
 socreate(dom, aso, type, proto, td)
 	int dom;
@@ -162,10 +174,11 @@ socreate(dom, aso, type, proto, td)
 	so->so_type = type;
 	so->so_cred = crhold(td->td_proc->p_ucred);
 	so->so_proto = prp;
+	soref(so);
 	error = (*prp->pr_usrreqs->pru_attach)(so, proto, td);
 	if (error) {
 		so->so_state |= SS_NOFDREF;
-		sofree(so);
+		sorele(so);
 		return (error);
 	}
 	*aso = so;
@@ -186,11 +199,11 @@ sobind(so, nam, td)
 	return (error);
 }
 
-void
+static void
-sodealloc(so)
+sodealloc(struct socket *so)
-	struct socket *so;
 {
 
+	KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count));
 	so->so_gencnt = ++so_gencnt;
 	if (so->so_rcv.sb_hiwat)
 		(void)chgsbsize(so->so_cred->cr_uidinfo,
@@ -210,7 +223,9 @@ sodealloc(so)
 	}
 #endif
 	crfree(so->so_cred);
+	/* sx_destroy(&so->so_sxlock); */
 	zfree(so->so_zone, so);
+	--numopensockets;
 }
 
 int
@@ -242,6 +257,8 @@ sofree(so)
 {
 	struct socket *head = so->so_head;
 
+	KASSERT(so->so_count == 0, ("socket %p so_count not 0", so));
+
 	if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0)
 		return;
 	if (head != NULL) {
@@ -272,6 +289,10 @@ sofree(so)
  * Close a socket on last file table reference removal.
  * Initiate disconnect if connected.
  * Free socket when disconnect complete.
+ *
+ * This function will sorele() the socket.  Note that soclose() may be
+ * called prior to the ref count reaching zero.  The actual socket
+ * structure will not be freed until the ref count reaches zero.
  */
 int
 soclose(so)
@@ -329,7 +350,7 @@ discard:
 	if (so->so_state & SS_NOFDREF)
 		panic("soclose: NOFDREF");
 	so->so_state |= SS_NOFDREF;
-	sofree(so);
+	sorele(so);
 	splx(s);
 	return (error);
 }
@@ -345,7 +366,7 @@ soabort(so)
 
 	error = (*so->so_proto->pr_usrreqs->pru_abort)(so);
 	if (error) {
-		sofree(so);
+		sotryfree(so);	/* note: does not decrement the ref count */
 		return error;
 	}
 	return (0);

sys/kern/uipc_socket2.c

@@ -210,6 +210,8 @@ sodropablereq(head)
  * then we allocate a new structure, propoerly linked into the
  * data structure of the original socket, and return this.
  * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED.
+ *
+ * note: the ref count on the socket is 0 on return
  */
 struct socket *
 sonewconn(head, connstatus)
@@ -246,7 +248,7 @@ sonewconn3(head, connstatus, td)
 		so->so_cred = crhold(head->so_cred);
 	if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) ||
 	    (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) {
-		sodealloc(so);
+		sotryfree(so);
 		return ((struct socket *)0);
 	}
 

sys/kern/uipc_syscalls.c

@@ -139,7 +139,7 @@ socket(td, uap)
 			fdrop(fp, td);
 		}
 	} else {
-		fp->f_data = (caddr_t)so;
+		fp->f_data = (caddr_t)so;	/* already has ref count */
 		fp->f_flag = FREAD|FWRITE;
 		fp->f_ops = &socketops;
 		fp->f_type = DTYPE_SOCKET;
@@ -164,22 +164,19 @@ bind(td, uap)
 		int	namelen;
 	} */ *uap;
 {
-	struct file *fp;
 	struct sockaddr *sa;
+	struct socket *sp;
 	int error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &sp, NULL)) != 0)
-	if (error)
 		goto done2;
-	error = getsockaddr(&sa, uap->name, uap->namelen);
+	if ((error = getsockaddr(&sa, uap->name, uap->namelen)) != 0)
-	if (error) {
+		goto done1;
-		fdrop(fp, td);
+	error = sobind(sp, sa, td);
-		goto done2;
-	}
-	error = sobind((struct socket *)fp->f_data, sa, td);
 	FREE(sa, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(sp);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -197,14 +194,13 @@ listen(td, uap)
 		int	backlog;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *sp;
 	int error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &sp, NULL)) == 0) {
-	if (error == 0) {
+		error = solisten(sp, uap->backlog, td);
-		error = solisten((struct socket *)fp->f_data, uap->backlog, td);
+		fputsock(sp);
-		fdrop(fp, td);
 	}
 	mtx_unlock(&Giant);
 	return(error);
@@ -225,13 +221,12 @@ accept1(td, uap, compat)
 	int compat;
 {
 	struct filedesc *fdp;
-	struct file *lfp = NULL;
 	struct file *nfp = NULL;
 	struct sockaddr *sa;
 	int namelen, error, s;
 	struct socket *head, *so;
 	int fd;
-	short fflag;		/* type must match fp->f_flag */
+	u_int fflag;
 
 	mtx_lock(&Giant);
 	fdp = td->td_proc->p_fd;
@@ -241,11 +236,10 @@ accept1(td, uap, compat)
 		if(error)
 			goto done2;
 	}
-	error = holdsock(fdp, uap->s, &lfp);
+	error = fgetsock(td, uap->s, &head, &fflag);
 	if (error)
 		goto done2;
 	s = splnet();
-	head = (struct socket *)lfp->f_data;
 	if ((head->so_options & SO_ACCEPTCONN) == 0) {
 		splx(s);
 		error = EINVAL;
@@ -286,7 +280,6 @@ accept1(td, uap, compat)
 	TAILQ_REMOVE(&head->so_comp, so, so_list);
 	head->so_qlen--;
 
-	fflag = lfp->f_flag;
 	error = falloc(td, &nfp, &fd);
 	if (error) {
 		/*
@@ -312,7 +305,8 @@ accept1(td, uap, compat)
 	if (head->so_sigio != NULL)
 		fsetown(fgetown(head->so_sigio), &so->so_sigio);
 
-	nfp->f_data = (caddr_t)so;
+	soref(so);			/* file descriptor reference */
+	nfp->f_data = (caddr_t)so;	/* nfp has ref count from falloc */
 	nfp->f_flag = fflag;
 	nfp->f_ops = &socketops;
 	nfp->f_type = DTYPE_SOCKET;
@@ -375,7 +369,7 @@ noconnection:
 done:
 	if (nfp != NULL)
 		fdrop(nfp, td);
-	fdrop(lfp, td);
+	fputsock(head);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -420,35 +414,31 @@ connect(td, uap)
 		int	namelen;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *so;
-	register struct socket *so;
 	struct sockaddr *sa;
 	int error, s;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
-	if (error)
 		goto done2;
-	so = (struct socket *)fp->f_data;
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
 		error = EALREADY;
-		goto done;
+		goto done1;
 	}
 	error = getsockaddr(&sa, uap->name, uap->namelen);
 	if (error)
-		goto done;
+		goto done1;
 	error = soconnect(so, sa, td);
 	if (error)
 		goto bad;
 	if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) {
 		FREE(sa, M_SONAME);
 		error = EINPROGRESS;
-		goto done;
+		goto done1;
 	}
 	s = splnet();
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-		error = tsleep((caddr_t)&so->so_timeo, PSOCK | PCATCH,
+		error = tsleep((caddr_t)&so->so_timeo, PSOCK | PCATCH, "connec", 0);
-		    "connec", 0);
 		if (error)
 			break;
 	}
@@ -462,8 +452,8 @@ bad:
 	FREE(sa, M_SONAME);
 	if (error == ERESTART)
 		error = EINTR;
-done:
+done1:
-	fdrop(fp, td);
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -499,12 +489,12 @@ socketpair(td, uap)
 		goto free2;
 	fhold(fp1);
 	sv[0] = fd;
-	fp1->f_data = (caddr_t)so1;
+	fp1->f_data = (caddr_t)so1;	/* so1 already has ref count */
 	error = falloc(td, &fp2, &fd);
 	if (error)
 		goto free3;
 	fhold(fp2);
-	fp2->f_data = (caddr_t)so2;
+	fp2->f_data = (caddr_t)so2;	/* so2 already has ref count */
 	sv[1] = fd;
 	error = soconnect2(so1, so2);
 	if (error)
@@ -552,12 +542,11 @@ sendit(td, s, mp, flags)
 	register struct msghdr *mp;
 	int flags;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
 	struct mbuf *control;
-	struct sockaddr *to;
+	struct sockaddr *to = NULL;
 	int len, error;
 	struct socket *so;
 #ifdef KTRACE
@@ -565,8 +554,7 @@ sendit(td, s, mp, flags)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
-	if (error)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -578,18 +566,14 @@ sendit(td, s, mp, flags)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
+			error = EINVAL;
-			return (EINVAL);
+			goto bad;
 		}
 	}
 	if (mp->msg_name) {
 		error = getsockaddr(&to, mp->msg_name, mp->msg_namelen);
-		if (error) {
+		if (error)
-			fdrop(fp, td);
+			goto bad;
-			return (error);
-		}
-	} else {
-		to = 0;
 	}
 	if (mp->msg_control) {
 		if (mp->msg_controllen < sizeof(struct cmsghdr)
@@ -633,7 +617,6 @@ sendit(td, s, mp, flags)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_sosend(so, to, &auio, 0, control,
 						     flags, td);
 	if (error) {
@@ -659,7 +642,7 @@ sendit(td, s, mp, flags)
 	}
 #endif
 bad:
-	fdrop(fp, td);
+	fputsock(so);
 	if (to)
 		FREE(to, M_SONAME);
 	return (error);
@@ -834,7 +817,6 @@ recvit(td, s, mp, namelenp)
 	register struct msghdr *mp;
 	caddr_t namelenp;
 {
-	struct file *fp;
 	struct uio auio;
 	register struct iovec *iov;
 	register int i;
@@ -848,8 +830,7 @@ recvit(td, s, mp, namelenp)
 	struct uio ktruio;
 #endif
 
-	error = holdsock(td->td_proc->p_fd, s, &fp);
+	if ((error = fgetsock(td, s, &so, NULL)) != 0)
-	if (error)
 		return (error);
 	auio.uio_iov = mp->msg_iov;
 	auio.uio_iovcnt = mp->msg_iovlen;
@@ -861,7 +842,7 @@ recvit(td, s, mp, namelenp)
 	iov = mp->msg_iov;
 	for (i = 0; i < mp->msg_iovlen; i++, iov++) {
 		if ((auio.uio_resid += iov->iov_len) < 0) {
-			fdrop(fp, td);
+			fputsock(so);
 			return (EINVAL);
 		}
 	}
@@ -875,7 +856,6 @@ recvit(td, s, mp, namelenp)
 	}
 #endif
 	len = auio.uio_resid;
-	so = (struct socket *)fp->f_data;
 	error = so->so_proto->pr_usrreqs->pru_soreceive(so, &fromsa, &auio,
 	    (struct mbuf **)0, mp->msg_control ? &control : (struct mbuf **)0,
 	    &mp->msg_flags);
@@ -975,7 +955,7 @@ recvit(td, s, mp, namelenp)
 		mp->msg_controllen = ctlbuf - (caddr_t)mp->msg_control;
 	}
 out:
-	fdrop(fp, td);
+	fputsock(so);
 	if (fromsa)
 		FREE(fromsa, M_SONAME);
 	if (control)
@@ -1196,14 +1176,13 @@ shutdown(td, uap)
 		int	how;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *so;
 	int error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
-	if (error == 0) {
+		error = soshutdown(so, uap->how);
-		error = soshutdown((struct socket *)fp->f_data, uap->how);
+		fputsock(so);
-		fdrop(fp, td);
 	}
 	mtx_unlock(&Giant);
 	return(error);
@@ -1224,7 +1203,7 @@ setsockopt(td, uap)
 		int	valsize;
 	} */ *uap;
 {
-	struct file *fp;
+	struct socket *so;
 	struct sockopt sopt;
 	int error;
 
@@ -1234,16 +1213,15 @@ setsockopt(td, uap)
 		return (EINVAL);
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
-	if (error == 0) {
 		sopt.sopt_dir = SOPT_SET;
 		sopt.sopt_level = uap->level;
 		sopt.sopt_name = uap->name;
 		sopt.sopt_val = uap->val;
 		sopt.sopt_valsize = uap->valsize;
 		sopt.sopt_td = td;
-		error = sosetopt((struct socket *)fp->f_data, &sopt);
+		error = sosetopt(so, &sopt);
-		fdrop(fp, td);
+		fputsock(so);
 	}
 	mtx_unlock(&Giant);
 	return(error);
@@ -1265,24 +1243,20 @@ getsockopt(td, uap)
 	} */ *uap;
 {
 	int	valsize, error;
-	struct	file *fp;
+	struct  socket *so;
 	struct	sockopt sopt;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
-	if (error)
 		goto done2;
 	if (uap->val) {
 		error = copyin((caddr_t)uap->avalsize, (caddr_t)&valsize,
 		    sizeof (valsize));
-		if (error) {
+		if (error)
-			fdrop(fp, td);
+			goto done1;
-			goto done2;
-		}
 		if (valsize < 0) {
-			fdrop(fp, td);
 			error = EINVAL;
-			goto done2;
+			goto done1;
 		}
 	} else {
 		valsize = 0;
@@ -1295,13 +1269,14 @@ getsockopt(td, uap)
 	sopt.sopt_valsize = (size_t)valsize; /* checked non-negative above */
 	sopt.sopt_td = td;
 
-	error = sogetopt((struct socket *)fp->f_data, &sopt);
+	error = sogetopt(so, &sopt);
 	if (error == 0) {
 		valsize = sopt.sopt_valsize;
 		error = copyout((caddr_t)&valsize,
 				(caddr_t)uap->avalsize, sizeof (valsize));
 	}
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -1323,21 +1298,16 @@ getsockname1(td, uap, compat)
 	} */ *uap;
 	int compat;
 {
-	struct file *fp;
+	struct socket *so;
-	register struct socket *so;
 	struct sockaddr *sa;
 	int len, error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->fdes, &fp);
+	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
-	if (error)
 		goto done2;
 	error = copyin((caddr_t)uap->alen, (caddr_t)&len, sizeof (len));
-	if (error) {
+	if (error)
-		fdrop(fp, td);
+		goto done1;
-		goto done2;
-	}
-	so = (struct socket *)fp->f_data;
 	sa = 0;
 	error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &sa);
 	if (error)
@@ -1360,7 +1330,8 @@ gotnothing:
 bad:
 	if (sa)
 		FREE(sa, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -1408,26 +1379,20 @@ getpeername1(td, uap, compat)
 	} */ *uap;
 	int compat;
 {
-	struct file *fp;
+	struct socket *so;
-	register struct socket *so;
 	struct sockaddr *sa;
 	int len, error;
 
 	mtx_lock(&Giant);
-	error = holdsock(td->td_proc->p_fd, uap->fdes, &fp);
+	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
-	if (error)
 		goto done2;
-	so = (struct socket *)fp->f_data;
 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
-		fdrop(fp, td);
 		error = ENOTCONN;
-		goto done2;
+		goto done1;
 	}
 	error = copyin((caddr_t)uap->alen, (caddr_t)&len, sizeof (len));
-	if (error) {
+	if (error)
-		fdrop(fp, td);
+		goto done1;
-		goto done2;
-	}
 	sa = 0;
 	error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, &sa);
 	if (error)
@@ -1450,7 +1415,8 @@ gotnothing:
 bad:
 	if (sa)
 		FREE(sa, M_SONAME);
-	fdrop(fp, td);
+done1:
+	fputsock(so);
 done2:
 	mtx_unlock(&Giant);
 	return (error);
@@ -1549,33 +1515,6 @@ getsockaddr(namp, uaddr, len)
 	return error;
 }
 
-/*
- * holdsock() - load the struct file pointer associated
- * with a socket into *fpp.  If an error occurs, non-zero
- * will be returned and *fpp will be set to NULL.
- */
-int
-holdsock(fdp, fdes, fpp)
-	struct filedesc *fdp;
-	int fdes;
-	struct file **fpp;
-{
-	register struct file *fp = NULL;
-	int error = 0;
-
-	if ((unsigned)fdes >= fdp->fd_nfiles ||
-	    (fp = fdp->fd_ofiles[fdes]) == NULL) {
-		error = EBADF;
-	} else if (fp->f_type != DTYPE_SOCKET) {
-		error = ENOTSOCK;
-		fp = NULL;
-	} else {
-		fhold(fp);
-	}
-	*fpp = fp;
-	return(error);
-}
-
 /*
  * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
  * XXX - The sf_buf functions are currently private to sendfile(2), so have
@@ -1678,10 +1617,9 @@ sf_buf_free(caddr_t addr, void *args)
 int
 sendfile(struct thread *td, struct sendfile_args *uap)
 {
-	struct file *fp = NULL;
 	struct vnode *vp;
 	struct vm_object *obj;
-	struct socket *so;
+	struct socket *so = NULL;
 	struct mbuf *m;
 	struct sf_buf *sf;
 	struct vm_page *pg;
@@ -1701,10 +1639,8 @@ sendfile(struct thread *td, struct sendfile_args *uap)
 		error = EINVAL;
 		goto done;
 	}
-	error = holdsock(td->td_proc->p_fd, uap->s, &fp);
+	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
-	if (error)
 		goto done;
-	so = (struct socket *)fp->f_data;
 	if (so->so_type != SOCK_STREAM) {
 		error = EINVAL;
 		goto done;
@@ -1988,8 +1924,9 @@ done:
 	}
 	if (vp)
 		vrele(vp);
-	if (fp)
+	if (so)
-		fdrop(fp, td);
+		fputsock(so);
 	mtx_unlock(&Giant);
 	return (error);
 }
+

sys/kern/uipc_usrreq.c

@@ -935,7 +935,7 @@ unp_drop(unp, errno)
 		if (unp->unp_addr)
 			FREE(unp->unp_addr, M_SONAME);
 		zfree(unp_zone, unp);
-		sofree(so);
+		sotryfree(so);
 	}
 }
 

sys/net/raw_cb.c

@@ -97,7 +97,7 @@ raw_detach(rp)
 	struct socket *so = rp->rcb_socket;
 
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	LIST_REMOVE(rp, list);
 #ifdef notdef
 	if (rp->rcb_laddr)

sys/net/raw_usrreq.c

@@ -142,8 +142,8 @@ raw_uabort(struct socket *so)
 	if (rp == 0)
 		return EINVAL;
 	raw_disconnect(rp);
-	sofree(so);
+	sotryfree(so);
-	soisdisconnected(so);
+	soisdisconnected(so);	/* XXX huh? called after the sofree()? */
 	return 0;
 }
 

sys/netatalk/ddp_pcb.c

@@ -441,7 +441,7 @@ at_pcbdetach( struct socket *so, struct ddpcb *ddp)
 {
     soisdisconnected( so );
     so->so_pcb = 0;
-    sofree( so );
+    sotryfree(so);
 
     /* remove ddp from ddp_ports list */
     if ( ddp->ddp_lsat.sat_port != ATADDR_ANYPORT &&

sys/netatalk/ddp_usrreq.c

@@ -441,7 +441,7 @@ at_pcbdetach( struct socket *so, struct ddpcb *ddp)
 {
     soisdisconnected( so );
     so->so_pcb = 0;
-    sofree( so );
+    sotryfree(so);
 
     /* remove ddp from ddp_ports list */
     if ( ddp->ddp_lsat.sat_port != ATADDR_ANYPORT &&

sys/netatm/atm_socket.c

@@ -176,7 +176,7 @@ atm_sock_detach(so)
 	 * Break links and free control blocks
 	 */
 	so->so_pcb = NULL;
-	sofree(so);
+	sotryfree(so);
 
 	atm_free((caddr_t)atp);
 

sys/netinet/in_pcb.c

@@ -563,7 +563,7 @@ in_pcbdetach(inp)
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	if (inp->inp_options)
 		(void)m_free(inp->inp_options);
 	if (rt) {

sys/netinet6/in6_pcb.c

@@ -606,7 +606,7 @@ in6_pcbdetach(inp)
 	inp->inp_gencnt = ++ipi->ipi_gencnt;
 	in_pcbremlists(inp);
 	sotoinpcb(so) = 0;
-	sofree(so);
+	sotryfree(so);
 
 	if (inp->in6p_options)
 		m_freem(inp->in6p_options);

sys/netipx/ipx_pcb.c

@@ -268,7 +268,7 @@ ipx_pcbdetach(ipxp)
 	struct socket *so = ipxp->ipxp_socket;
 
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	if (ipxp->ipxp_route.ro_rt != NULL)
 		rtfree(ipxp->ipxp_route.ro_rt);
 	remque(ipxp);

sys/netipx/ipx_usrreq.c

@@ -426,7 +426,7 @@ ipx_usr_abort(so)
 	s = splnet();
 	ipx_pcbdetach(ipxp);
 	splx(s);
-	sofree(so);
+	sotryfree(so);
 	soisdisconnected(so);
 	return (0);
 }

sys/netnatm/natm.c

@@ -133,7 +133,7 @@ natm_usr_detach(struct socket *so)
      */
     npcb_free(npcb, NPCB_DESTROY);	/* drain */
     so->so_pcb = NULL;
-    sofree(so);
+    sotryfree(so);
  out:
     splx(s);
     return (error);
@@ -481,7 +481,7 @@ struct proc *p;
 
       npcb_free(npcb, NPCB_DESTROY);	/* drain */
       so->so_pcb = NULL;
-      sofree(so);
+      sotryfree(so);
 
       break;
 

sys/netns/idp_usrreq.c

@@ -491,8 +491,8 @@ idp_usrreq(so, req, m, nam, control)
 
 	case PRU_ABORT:
 		ns_pcbdetach(nsp);
-		sofree(so);
+		sotryfree(so);
-		soisdisconnected(so);
+		soisdisconnected(so);	/* XXX huh, called after sofree()? */
 		break;
 
 	case PRU_SOCKADDR:

sys/netns/ns_pcb.c

@@ -232,7 +232,7 @@ ns_pcbdetach(nsp)
 	struct socket *so = nsp->nsp_socket;
 
 	so->so_pcb = 0;
-	sofree(so);
+	sotryfree(so);
 	if (nsp->nsp_route.ro_rt)
 		rtfree(nsp->nsp_route.ro_rt);
 	remque(nsp);

sys/nfsserver/nfs_syscalls.c

@@ -143,9 +143,12 @@ nfssvc(struct thread *td, struct nfssvc_args *uap)
 		error = copyin(uap->argp, (caddr_t)&nfsdarg, sizeof(nfsdarg));
 		if (error)
 			goto done2;
-		error = holdsock(td->td_proc->p_fd, nfsdarg.sock, &fp);
+		if ((error = fget(td, nfsdarg.sock, &fp)) != 0)
-		if (error)
 			goto done2;
+		if (fp->f_type != DTYPE_SOCKET) {
+			fdrop(fp, td);
+			goto done2;
+		}
 		/*
 		 * Get the client address for connected sockets.
 		 */

sys/sys/file.h

@@ -50,6 +50,7 @@ struct thread;
 struct uio;
 struct knote;
 struct vnode;
+struct socket;
 
 /*
  * Kernel descriptor table.
@@ -119,6 +120,9 @@ int fgetvp __P((struct thread *td, int fd, struct vnode **vpp));
 int fgetvp_read __P((struct thread *td, int fd, struct vnode **vpp));
 int fgetvp_write __P((struct thread *td, int fd, struct vnode **vpp));
 
+int fgetsock __P((struct thread *td, int fd, struct socket **spp, u_int *fflagp));
+void fputsock __P((struct socket *sp));
+
 static __inline void
 fhold(fp)
 	struct file *fp;

sys/sys/socketvar.h

@@ -38,6 +38,7 @@
 #define _SYS_SOCKETVAR_H_
 
 #include <sys/queue.h>			/* for TAILQ macros */
+#include <sys/sx.h>			/* SX locks */
 #include <sys/selinfo.h>		/* for struct selinfo */
 
 /*
@@ -52,6 +53,7 @@ struct accept_filter;
 
 struct socket {
 	struct	vm_zone *so_zone;	/* zone we were allocated from */
+	int	so_count;		/* reference count */
 	short	so_type;		/* generic type, see socket.h */
 	short	so_options;		/* from socket call, see socket.h */
 	short	so_linger;		/* time to linger while closing */
@@ -244,6 +246,28 @@ struct xsocket {
 	} \
 }
 
+/*
+ * soref()/sorele() ref-count the socket structure.  Note that you must
+ * still explicitly close the socket, but the last ref count will free
+ * the structure.
+ */
+
+#define soref(so)	do {			\
+				++so->so_count; \
+			} while (0)
+
+#define sorele(so)	do {				\
+				if (so->so_count <= 0)	\
+					panic("sorele");\
+				if (--so->so_count == 0)\
+					sofree(so);	\
+			} while (0)
+
+#define sotryfree(so)	do {				\
+				if (so->so_count == 0)	\
+					sofree(so);	\
+			} while(0)
+
 #define	sorwakeup(so)	do { \
 			  if (sb_notify(&(so)->so_rcv)) \
 			    sowakeup((so), &(so)->so_rcv); \
@@ -360,7 +384,6 @@ int	soconnect __P((struct socket *so, struct sockaddr *nam, struct thread *td));
 int	soconnect2 __P((struct socket *so1, struct socket *so2));
 int	socreate __P((int dom, struct socket **aso, int type, int proto,
 	    struct thread *td));
-void	sodealloc __P((struct socket *so));
 int	sodisconnect __P((struct socket *so));
 void	sofree __P((struct socket *so));
 int	sogetopt __P((struct socket *so, struct sockopt *sopt));