d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nfscl: Add hash lists for the NFSv4 opens

Browse Source 
A problem was reported via email, where a large (130000+) accumulation
of NFSv4 opens on an NFSv4 mount caused significant lock contention
on the mutex used to protect the client mount's open/lock state.
Although the root cause for the accumulation of opens was not
resolved, it is obvious that the NFSv4 client is not designed to
handle 100000+ opens efficiently.  When searching for an open,
usually for a match by file handle, a linear search of all opens
is done.

This patch adds a table of hash lists for the opens, hashed on
file handle.  This table will be used by future commits to
search for an open based on file handle more efficiently.

MFC after:	2 weeks
This commit is contained in:
Rick Macklem 2021-05-22 14:51:38 -07:00
parent 33c1bdfc3e
commit 3f7e14ad93
2 changed files with 21 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
sys/fs/nfs/nfsclstate.h
View File

@ -42,6 +42,7 @@ LIST_HEAD(nfsclhead, nfsclclient);
LIST_HEAD(nfsclownerhead, nfsclowner);
TAILQ_HEAD(nfscldeleghead, nfscldeleg);
LIST_HEAD(nfscldeleghash, nfscldeleg);
LIST_HEAD(nfsclopenhash, nfsclopen);
TAILQ_HEAD(nfscllayouthead, nfscllayout);
LIST_HEAD(nfscllayouthash, nfscllayout);
LIST_HEAD(nfsclflayouthead, nfsclflayout);
@ -50,6 +51,10 @@ LIST_HEAD(nfsclrecalllayouthead, nfsclrecalllayout);
#define NFSCLDELEGHASHSIZE 256
#define NFSCLDELEGHASH(c, f, l) \
(&((c)->nfsc_deleghash[ncl_hash((f), (l)) % NFSCLDELEGHASHSIZE]))
#define NFSCLOPENHASHSIZE 256
#define NFSCLOPENHASHFUNC(f, l) (ncl_hash((f), (l)) % NFSCLOPENHASHSIZE)
#define NFSCLOPENHASH(c, f, l) \
(&((c)->nfsc_openhash[NFSCLOPENHASHFUNC((f), (l))]))
#define NFSCLLAYOUTHASHSIZE 256
#define NFSCLLAYOUTHASH(c, f, l) \
(&((c)->nfsc_layouthash[ncl_hash((f), (l)) % NFSCLLAYOUTHASHSIZE]))
@ -104,6 +109,7 @@ struct nfsclclient {
struct nfsclownerhead nfsc_owner;
struct nfscldeleghead nfsc_deleg;
struct nfscldeleghash nfsc_deleghash[NFSCLDELEGHASHSIZE];
struct nfsclopenhash nfsc_openhash[NFSCLOPENHASHSIZE];
struct nfscllayouthead nfsc_layout;
struct nfscllayouthash nfsc_layouthash[NFSCLLAYOUTHASHSIZE];
struct nfscldevinfohead nfsc_devinfo;
@ -183,6 +189,7 @@ struct nfscldeleg {
*/
struct nfsclopen {
LIST_ENTRY(nfsclopen) nfso_list;
LIST_ENTRY(nfsclopen) nfso_hash;
struct nfscllockownerhead nfso_lock;
nfsv4stateid_t nfso_stateid;
struct nfsclowner *nfso_own;

15
sys/fs/nfsclient/nfs_clstate.c
View File

@ -240,9 +240,11 @@ nfscl_open(vnode_t vp, u_int8_t *nfhp, int fhlen, u_int32_t amode, int usedeleg,
*/
nowp = malloc(sizeof (struct nfsclowner),
M_NFSCLOWNER, M_WAITOK);
if (nfhp != NULL)
if (nfhp != NULL) {
nop = malloc(sizeof (struct nfsclopen) +
fhlen - 1, M_NFSCLOPEN, M_WAITOK);
nop->nfso_hash.le_prev = NULL;
}
ret = nfscl_getcl(vp->v_mount, cred, p, 1, &clp);
if (ret != 0) {
free(nowp, M_NFSCLOWNER);
@ -412,6 +414,8 @@ nfscl_newopen(struct nfsclclient *clp, struct nfscldeleg *dp,
dp->nfsdl_timestamp = NFSD_MONOSEC + 120;
nfsstatsv1.cllocalopens++;
} else {
LIST_INSERT_HEAD(NFSCLOPENHASH(clp, fhp, fhlen),
nop, nfso_hash);
nfsstatsv1.clopens++;
}
LIST_INSERT_HEAD(&owp->nfsow_open, nop, nfso_list);
@ -837,6 +841,8 @@ nfscl_getcl(struct mount *mp, struct ucred *cred, NFSPROC_T *p,
LIST_INIT(&clp->nfsc_devinfo);
for (i = 0; i < NFSCLDELEGHASHSIZE; i++)
LIST_INIT(&clp->nfsc_deleghash[i]);
for (i = 0; i < NFSCLOPENHASHSIZE; i++)
LIST_INIT(&clp->nfsc_openhash[i]);
for (i = 0; i < NFSCLLAYOUTHASHSIZE; i++)
LIST_INIT(&clp->nfsc_layouthash[i]);
clp->nfsc_flags = NFSCLFLAGS_INITED;
@ -1475,6 +1481,8 @@ nfscl_freeopen(struct nfsclopen *op, int local)
{
LIST_REMOVE(op, nfso_list);
if (op->nfso_hash.le_prev != NULL)
LIST_REMOVE(op, nfso_hash);
nfscl_freealllocks(&op->nfso_lock, local);
free(op, M_NFSCLOPEN);
if (local)
@ -1706,6 +1714,8 @@ nfscl_expireclient(struct nfsclclient *clp, struct nfsmount *nmp,
LIST_REMOVE(op, nfso_list);
op->nfso_own = towp;
LIST_INSERT_HEAD(&towp->nfsow_open, op, nfso_list);
LIST_INSERT_HEAD(NFSCLOPENHASH(clp, op->nfso_fh,
op->nfso_fhlen), op, nfso_hash);
nfsstatsv1.cllocalopens--;
nfsstatsv1.clopens++;
}
@ -1714,6 +1724,8 @@ nfscl_expireclient(struct nfsclclient *clp, struct nfsmount *nmp,
LIST_REMOVE(owp, nfsow_list);
owp->nfsow_clp = clp;
LIST_INSERT_HEAD(&clp->nfsc_owner, owp, nfsow_list);
LIST_INSERT_HEAD(NFSCLOPENHASH(clp, op->nfso_fh,
op->nfso_fhlen), op, nfso_hash);
nfsstatsv1.cllocalopenowners--;
nfsstatsv1.clopenowners++;
nfsstatsv1.cllocalopens--;
@ -4198,6 +4210,7 @@ nfscl_moveopen(vnode_t vp, struct nfsclclient *clp, struct nfsmount *nmp,
np = VTONFS(vp);
nop = malloc(sizeof (struct nfsclopen) +
lop->nfso_fhlen - 1, M_NFSCLOPEN, M_WAITOK);
nop->nfso_hash.le_prev = NULL;
newone = 0;
nfscl_newopen(clp, NULL, &owp, NULL, &op, &nop, owp->nfsow_owner,
lop->nfso_fh, lop->nfso_fhlen, cred, &newone);