From 8d1b3828fa62900465357241711755512672361a Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Mon, 27 Mar 2000 21:38:35 +0000 Subject: [PATCH] Add a sysctl to specify the amount of UDP receive space NFS should reserve, in maximal NFS packets. Originally only 2 packets worth of space was reserved. The default is now 4, which appears to greatly improve performance for slow to mid-speed machines on gigabit networks. Add documentation and correct some prior documentation. Problem Researched by: Andrew Gallatin Approved by: jkh --- etc/defaults/rc.conf | 1 + etc/network.subr | 5 +++++ etc/rc.d/netoptions | 5 +++++ etc/rc.d/network1 | 5 +++++ etc/rc.d/network2 | 5 +++++ etc/rc.d/network3 | 5 +++++ etc/rc.d/routing | 5 +++++ etc/rc.network | 5 +++++ share/man/man5/rc.conf.5 | 9 ++++++++- sys/nfs/nfs_socket.c | 30 ++++++++++++++++++++++-------- sys/nfsclient/nfs_socket.c | 30 ++++++++++++++++++++++-------- sys/nfsserver/nfs_srvsock.c | 30 ++++++++++++++++++++++-------- 12 files changed, 110 insertions(+), 25 deletions(-) diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf index 848094640cf2..d84bd1e141bc 100644 --- a/etc/defaults/rc.conf +++ b/etc/defaults/rc.conf @@ -109,6 +109,7 @@ nfs_server_flags="-u -t -n 4" # Flags to nfsd (if enabled). single_mountd_enable="NO" # Run mountd only (or NO). mountd_flags="-r" # Flags to mountd (if NFS server enabled). nfs_reserved_port_only="NO" # Provide NFS only on secure port (or NO). +nfs_bufpackets="DEFAULT" # bufspace (in packets) for client (or DEFAULT) rpc_lockd_enable="NO" # Run NFS rpc.lockd (*broken!*) if nfs_server. rpc_statd_enable="YES" # Run NFS rpc.statd if nfs_server (or NO). portmap_enable="YES" # Run the portmapper service (or NO). diff --git a/etc/network.subr b/etc/network.subr index 0211181f0516..dd60e892eb45 100644 --- a/etc/network.subr +++ b/etc/network.subr @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/etc/rc.d/netoptions b/etc/rc.d/netoptions index 0211181f0516..dd60e892eb45 100644 --- a/etc/rc.d/netoptions +++ b/etc/rc.d/netoptions @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/etc/rc.d/network1 b/etc/rc.d/network1 index 0211181f0516..dd60e892eb45 100644 --- a/etc/rc.d/network1 +++ b/etc/rc.d/network1 @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/etc/rc.d/network2 b/etc/rc.d/network2 index 0211181f0516..dd60e892eb45 100644 --- a/etc/rc.d/network2 +++ b/etc/rc.d/network2 @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/etc/rc.d/network3 b/etc/rc.d/network3 index 0211181f0516..dd60e892eb45 100644 --- a/etc/rc.d/network3 +++ b/etc/rc.d/network3 @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/etc/rc.d/routing b/etc/rc.d/routing index 0211181f0516..dd60e892eb45 100644 --- a/etc/rc.d/routing +++ b/etc/rc.d/routing @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/etc/rc.network b/etc/rc.network index 0211181f0516..dd60e892eb45 100644 --- a/etc/rc.network +++ b/etc/rc.network @@ -493,6 +493,11 @@ network_pass3() { echo -n ' nfsd'; nfsd ${nfs_server_flags} + if [ -n "${nfs_bufpackets}" ]; then + sysctl -w vfs.nfs.bufpackets=${nfs_bufpackets} \ + > /dev/null + fi + case ${rpc_lockd_enable} in [Yy][Ee][Ss]) echo -n ' rpc.lockd'; rpc.lockd diff --git a/share/man/man5/rc.conf.5 b/share/man/man5/rc.conf.5 index 95aa2beb5922..4ef54a01e4d0 100644 --- a/share/man/man5/rc.conf.5 +++ b/share/man/man5/rc.conf.5 @@ -491,10 +491,17 @@ It is commonly needed to run CFS without real NFS used. .Ar YES , allow services like PCNFSD to make non-privileged mount requests. -.It Ar nfs_reserved_port_only +.It Ar nfs_privport (bool) If set to .Ar YES , provide NFS services only on a secure port. +.It Ar nfs_bufpackets +(integer) If set to a number, indicates the number of packets worth of +socket buffer space to reserve on an NFS client. If set to +.Ar DEFAULT , +the kernel default is used (typically 4). Using a higher number may be +useful on gigabit networks to improve performance. The minimum value is +2 and the maximum is 64. .It Ar rcp_lockd_enable (bool) If set to .Ar YES diff --git a/sys/nfs/nfs_socket.c b/sys/nfs/nfs_socket.c index b2d6bf53dbe3..7d8bec453643 100644 --- a/sys/nfs/nfs_socket.c +++ b/sys/nfs/nfs_socket.c @@ -119,11 +119,13 @@ static int proct[NFS_NPROCS] = { static int nfs_realign_test; static int nfs_realign_count; +static int nfs_bufpackets = 4; SYSCTL_DECL(_vfs_nfs); -SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RD, &nfs_realign_test, 0, ""); -SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RD, &nfs_realign_count, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RW, &nfs_realign_test, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RW, &nfs_realign_count, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, bufpackets, CTLFLAG_RW, &nfs_bufpackets, 0, ""); /* @@ -201,6 +203,7 @@ nfs_connect(nmp, rep) { register struct socket *so; int s, error, rcvreserve, sndreserve; + int pktscale; struct sockaddr *saddr; struct sockaddr_in *sin; struct proc *p = &proc0; /* only used for socreate and sobind */ @@ -301,14 +304,25 @@ nfs_connect(nmp, rep) so->so_rcv.sb_timeo = 0; so->so_snd.sb_timeo = 0; } + + /* + * Get buffer reservation size from sysctl, but impose reasonable + * limits. + */ + pktscale = nfs_bufpackets; + if (pktscale < 2) + pktscale = 2; + if (pktscale > 64) + pktscale = 64; + if (nmp->nm_sotype == SOCK_DGRAM) { - sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 2; + sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * pktscale; rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) + - NFS_MAXPKTHDR) * 2; + NFS_MAXPKTHDR) * pktscale; } else if (nmp->nm_sotype == SOCK_SEQPACKET) { - sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 2; + sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * pktscale; rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) + - NFS_MAXPKTHDR) * 2; + NFS_MAXPKTHDR) * pktscale; } else { if (nmp->nm_sotype != SOCK_STREAM) panic("nfscon sotype"); @@ -337,9 +351,9 @@ nfs_connect(nmp, rep) sosetopt(so, &sopt); } sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR + - sizeof (u_int32_t)) * 2; + sizeof (u_int32_t)) * pktscale; rcvreserve = (nmp->nm_rsize + NFS_MAXPKTHDR + - sizeof (u_int32_t)) * 2; + sizeof (u_int32_t)) * pktscale; } error = soreserve(so, sndreserve, rcvreserve); if (error) diff --git a/sys/nfsclient/nfs_socket.c b/sys/nfsclient/nfs_socket.c index b2d6bf53dbe3..7d8bec453643 100644 --- a/sys/nfsclient/nfs_socket.c +++ b/sys/nfsclient/nfs_socket.c @@ -119,11 +119,13 @@ static int proct[NFS_NPROCS] = { static int nfs_realign_test; static int nfs_realign_count; +static int nfs_bufpackets = 4; SYSCTL_DECL(_vfs_nfs); -SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RD, &nfs_realign_test, 0, ""); -SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RD, &nfs_realign_count, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RW, &nfs_realign_test, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RW, &nfs_realign_count, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, bufpackets, CTLFLAG_RW, &nfs_bufpackets, 0, ""); /* @@ -201,6 +203,7 @@ nfs_connect(nmp, rep) { register struct socket *so; int s, error, rcvreserve, sndreserve; + int pktscale; struct sockaddr *saddr; struct sockaddr_in *sin; struct proc *p = &proc0; /* only used for socreate and sobind */ @@ -301,14 +304,25 @@ nfs_connect(nmp, rep) so->so_rcv.sb_timeo = 0; so->so_snd.sb_timeo = 0; } + + /* + * Get buffer reservation size from sysctl, but impose reasonable + * limits. + */ + pktscale = nfs_bufpackets; + if (pktscale < 2) + pktscale = 2; + if (pktscale > 64) + pktscale = 64; + if (nmp->nm_sotype == SOCK_DGRAM) { - sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 2; + sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * pktscale; rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) + - NFS_MAXPKTHDR) * 2; + NFS_MAXPKTHDR) * pktscale; } else if (nmp->nm_sotype == SOCK_SEQPACKET) { - sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 2; + sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * pktscale; rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) + - NFS_MAXPKTHDR) * 2; + NFS_MAXPKTHDR) * pktscale; } else { if (nmp->nm_sotype != SOCK_STREAM) panic("nfscon sotype"); @@ -337,9 +351,9 @@ nfs_connect(nmp, rep) sosetopt(so, &sopt); } sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR + - sizeof (u_int32_t)) * 2; + sizeof (u_int32_t)) * pktscale; rcvreserve = (nmp->nm_rsize + NFS_MAXPKTHDR + - sizeof (u_int32_t)) * 2; + sizeof (u_int32_t)) * pktscale; } error = soreserve(so, sndreserve, rcvreserve); if (error) diff --git a/sys/nfsserver/nfs_srvsock.c b/sys/nfsserver/nfs_srvsock.c index b2d6bf53dbe3..7d8bec453643 100644 --- a/sys/nfsserver/nfs_srvsock.c +++ b/sys/nfsserver/nfs_srvsock.c @@ -119,11 +119,13 @@ static int proct[NFS_NPROCS] = { static int nfs_realign_test; static int nfs_realign_count; +static int nfs_bufpackets = 4; SYSCTL_DECL(_vfs_nfs); -SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RD, &nfs_realign_test, 0, ""); -SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RD, &nfs_realign_count, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RW, &nfs_realign_test, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RW, &nfs_realign_count, 0, ""); +SYSCTL_INT(_vfs_nfs, OID_AUTO, bufpackets, CTLFLAG_RW, &nfs_bufpackets, 0, ""); /* @@ -201,6 +203,7 @@ nfs_connect(nmp, rep) { register struct socket *so; int s, error, rcvreserve, sndreserve; + int pktscale; struct sockaddr *saddr; struct sockaddr_in *sin; struct proc *p = &proc0; /* only used for socreate and sobind */ @@ -301,14 +304,25 @@ nfs_connect(nmp, rep) so->so_rcv.sb_timeo = 0; so->so_snd.sb_timeo = 0; } + + /* + * Get buffer reservation size from sysctl, but impose reasonable + * limits. + */ + pktscale = nfs_bufpackets; + if (pktscale < 2) + pktscale = 2; + if (pktscale > 64) + pktscale = 64; + if (nmp->nm_sotype == SOCK_DGRAM) { - sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 2; + sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * pktscale; rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) + - NFS_MAXPKTHDR) * 2; + NFS_MAXPKTHDR) * pktscale; } else if (nmp->nm_sotype == SOCK_SEQPACKET) { - sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * 2; + sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR) * pktscale; rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) + - NFS_MAXPKTHDR) * 2; + NFS_MAXPKTHDR) * pktscale; } else { if (nmp->nm_sotype != SOCK_STREAM) panic("nfscon sotype"); @@ -337,9 +351,9 @@ nfs_connect(nmp, rep) sosetopt(so, &sopt); } sndreserve = (nmp->nm_wsize + NFS_MAXPKTHDR + - sizeof (u_int32_t)) * 2; + sizeof (u_int32_t)) * pktscale; rcvreserve = (nmp->nm_rsize + NFS_MAXPKTHDR + - sizeof (u_int32_t)) * 2; + sizeof (u_int32_t)) * pktscale; } error = soreserve(so, sndreserve, rcvreserve); if (error)