d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add unlocked/SMR fast path to getblk()

Browse Source 
Convert the bufobj tries to an SMR zone/PCTRIE and add a gbincore_unlocked()
API wrapping this functionality.  Use it for a fast path in getblkx(),
falling back to locked lookup if we raced a thread changing the buf's
identity.

Reported by:	Attilio
Reviewed by:	kib, markj
Testing:	pho (in progress)
Sponsored by:	Isilon
Differential Revision:	https://reviews.freebsd.org/D25782
This commit is contained in:
Conrad Meyer 2020-07-24 17:34:04 +00:00
parent 3c30b23519
commit 68ee1dda06
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=363482
3 changed files with 58 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
sys/kern/vfs_bio.c
View File

@ -3849,7 +3849,7 @@ getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
struct buf *bp;
struct bufobj *bo;
daddr_t d_blkno;
int bsize, error, maxsize, vmio;
int bsize, error, maxsize, vmio, lockflags;
off_t offset;
CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
@ -3864,11 +3864,33 @@ getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
bo = &vp->v_bufobj;
d_blkno = dblkno;
/* Attempt lockless lookup first. */
bp = gbincore_unlocked(bo, blkno);
if (bp == NULL)
goto newbuf_unlocked;
lockflags = LK_EXCLUSIVE | LK_SLEEPFAIL |
((flags & GB_LOCK_NOWAIT) ? LK_NOWAIT : 0);
error = BUF_TIMELOCK(bp, lockflags, NULL, "getblku", slpflag,
slptimeo);
if (error == EINTR || error == ERESTART)
return (error);
else if (error != 0)
goto loop;
/* Verify buf identify has not changed since lookup. */
if (bp->b_bufobj == bo && bp->b_lblkno == blkno)
goto foundbuf_fastpath;
/* It changed, fallback to locked lookup. */
BUF_UNLOCK_RAW(bp);
loop:
BO_RLOCK(bo);
bp = gbincore(bo, blkno);
if (bp != NULL) {
int lockflags;
/*
* Buffer is in-core. If the buffer is not busy nor managed,
* it must be on a queue.
@ -3890,8 +3912,10 @@ getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
/* We timed out or were interrupted. */
else if (error != 0)
return (error);
foundbuf_fastpath:
/* If recursed, assume caller knows the rules. */
else if (BUF_LOCKRECURSED(bp))
if (BUF_LOCKRECURSED(bp))
goto end;
/*
@ -3989,6 +4013,7 @@ getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
* buffer is also considered valid (not marked B_INVAL).
*/
BO_RUNLOCK(bo);
newbuf_unlocked:
/*
* If the user does not want us to create the buffer, bail out
* here.

33
sys/kern/vfs_subr.c
View File

@ -234,6 +234,7 @@ static struct mtx __exclusive_cache_line vnode_list_mtx;
struct nfs_public nfs_pub;
static uma_zone_t buf_trie_zone;
static smr_t buf_trie_smr;
/* Zone for allocation of new vnodes - used exclusively by getnewvnode() */
static uma_zone_t vnode_zone;
@ -491,17 +492,16 @@ static int vnsz2log;
static void *
buf_trie_alloc(struct pctrie *ptree)
{
return uma_zalloc(buf_trie_zone, M_NOWAIT);
return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT));
}
static void
buf_trie_free(struct pctrie *ptree, void *node)
{
uma_zfree(buf_trie_zone, node);
uma_zfree_smr(buf_trie_zone, node);
}
PCTRIE_DEFINE(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free);
PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free,
buf_trie_smr);
/*
* Initialize the vnode management data structures.
@ -675,7 +675,8 @@ vntblinit(void *dummy __unused)
*/
buf_trie_zone = uma_zcreate("BUF TRIE", pctrie_node_size(),
NULL, NULL, pctrie_zone_init, NULL, UMA_ALIGN_PTR,
UMA_ZONE_NOFREE);
UMA_ZONE_NOFREE | UMA_ZONE_SMR);
buf_trie_smr = uma_zone_get_smr(buf_trie_zone);
uma_prealloc(buf_trie_zone, nbuf);
vnodes_created = counter_u64_alloc(M_WAITOK);
@ -2330,7 +2331,25 @@ gbincore(struct bufobj *bo, daddr_t lblkno)
bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno);
if (bp != NULL)
return (bp);
return BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno);
return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno));
}
/*
* Look up a buf using the buffer tries, without the bufobj lock. This relies
* on SMR for safe lookup, and bufs being in a no-free zone to provide type
* stability of the result. Like other lockless lookups, the found buf may
* already be invalid by the time this function returns.
*/
struct buf *
gbincore_unlocked(struct bufobj *bo, daddr_t lblkno)
{
struct buf *bp;
ASSERT_BO_UNLOCKED(bo);
bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno);
if (bp != NULL)
return (bp);
return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno));
}
/*

4
sys/sys/buf.h
View File

@ -326,6 +326,9 @@ extern const char *buf_wmesg; /* Default buffer lock message */
KASSERT(((bp)->b_flags & B_REMFREE) == 0, \
("BUF_UNLOCK %p while B_REMFREE is still set.", (bp))); \
\
BUF_UNLOCK_RAW((bp)); \
} while (0)
#define BUF_UNLOCK_RAW(bp) do { \
(void)_lockmgr_args(&(bp)->b_lock, LK_RELEASE, NULL, \
LK_WMESG_DEFAULT, LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, \
LOCK_FILE, LOCK_LINE); \
@ -547,6 +550,7 @@ void vfs_busy_pages_acquire(struct buf *bp);
void vfs_busy_pages_release(struct buf *bp);
struct buf *incore(struct bufobj *, daddr_t);
struct buf *gbincore(struct bufobj *, daddr_t);
struct buf *gbincore_unlocked(struct bufobj *, daddr_t);
struct buf *getblk(struct vnode *, daddr_t, int, int, int, int);
int getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size,
int slpflag, int slptimeo, int flags, struct buf **bpp);