zfs: add support for lockless lookup
Tested by: pho (in a patchset, previous version) Differential Revision: https://reviews.freebsd.org/D25581
This commit is contained in:
parent
172ffe702c
commit
e4cdb74faf
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=363522
@ -246,6 +246,8 @@ VTOZ(vnode_t *vp)
|
||||
#define VTOZ(VP) ((znode_t *)(VP)->v_data)
|
||||
#endif
|
||||
|
||||
#define VTOZ_SMR(VP) ((znode_t *)vn_load_v_data_smr(VP))
|
||||
|
||||
/* Called on entry to each ZFS vnode and vfs operation */
|
||||
#define ZFS_ENTER(zfsvfs) \
|
||||
{ \
|
||||
|
@ -1146,6 +1146,7 @@ zfs_acl_chown_setattr(znode_t *zp)
|
||||
|
||||
ASSERT_VOP_ELOCKED(ZTOV(zp), __func__);
|
||||
ASSERT(MUTEX_HELD(&zp->z_acl_lock));
|
||||
ASSERT_VOP_IN_SEQC(ZTOV(zp));
|
||||
|
||||
if ((error = zfs_acl_node_read(zp, &aclp, B_FALSE)) == 0)
|
||||
zp->z_mode = zfs_mode_compute(zp->z_mode, aclp,
|
||||
@ -1173,6 +1174,8 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx)
|
||||
int count = 0;
|
||||
zfs_acl_phys_t acl_phys;
|
||||
|
||||
ASSERT_VOP_IN_SEQC(ZTOV(zp));
|
||||
|
||||
mode = zp->z_mode;
|
||||
|
||||
mode = zfs_mode_compute(mode, aclp, &zp->z_pflags,
|
||||
|
@ -1391,6 +1391,9 @@ zfs_domount(vfs_t *vfsp, char *osname)
|
||||
|
||||
vfsp->vfs_data = zfsvfs;
|
||||
vfsp->mnt_flag |= MNT_LOCAL;
|
||||
#if defined(_KERNEL) && !defined(KMEM_DEBUG)
|
||||
vfsp->mnt_kern_flag |= MNTK_FPLOOKUP;
|
||||
#endif
|
||||
vfsp->mnt_kern_flag |= MNTK_LOOKUP_SHARED;
|
||||
vfsp->mnt_kern_flag |= MNTK_SHARED_WRITES;
|
||||
vfsp->mnt_kern_flag |= MNTK_EXTENDED_SHARED;
|
||||
|
@ -38,6 +38,7 @@
|
||||
#include <sys/vfs.h>
|
||||
#include <sys/vm.h>
|
||||
#include <sys/vnode.h>
|
||||
#include <sys/smr.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/kmem.h>
|
||||
@ -78,6 +79,8 @@
|
||||
#include <vm/vm_param.h>
|
||||
#include <sys/zil.h>
|
||||
|
||||
VFS_SMR_DECLARE;
|
||||
|
||||
/*
|
||||
* Programming rules.
|
||||
*
|
||||
@ -3698,6 +3701,7 @@ zfs_rename(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp,
|
||||
char *snm = scnp->cn_nameptr;
|
||||
char *tnm = tcnp->cn_nameptr;
|
||||
int error = 0;
|
||||
bool want_seqc_end = false;
|
||||
|
||||
/* Reject renames across filesystems. */
|
||||
if ((*svpp)->v_mount != tdvp->v_mount ||
|
||||
@ -3828,6 +3832,14 @@ zfs_rename(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp,
|
||||
}
|
||||
}
|
||||
|
||||
vn_seqc_write_begin(*svpp);
|
||||
vn_seqc_write_begin(sdvp);
|
||||
if (*tvpp != NULL)
|
||||
vn_seqc_write_begin(*tvpp);
|
||||
if (tdvp != *tvpp)
|
||||
vn_seqc_write_begin(tdvp);
|
||||
want_seqc_end = true;
|
||||
|
||||
vnevent_rename_src(*svpp, sdvp, scnp->cn_nameptr, ct);
|
||||
if (tzp)
|
||||
vnevent_rename_dest(*tvpp, tdvp, tnm, ct);
|
||||
@ -3914,10 +3926,20 @@ zfs_rename(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp,
|
||||
|
||||
unlockout: /* all 4 vnodes are locked, ZFS_ENTER called */
|
||||
ZFS_EXIT(zfsvfs);
|
||||
if (want_seqc_end) {
|
||||
vn_seqc_write_end(*svpp);
|
||||
vn_seqc_write_end(sdvp);
|
||||
if (*tvpp != NULL)
|
||||
vn_seqc_write_end(*tvpp);
|
||||
if (tdvp != *tvpp)
|
||||
vn_seqc_write_end(tdvp);
|
||||
want_seqc_end = false;
|
||||
}
|
||||
VOP_UNLOCK(*svpp);
|
||||
VOP_UNLOCK(sdvp);
|
||||
|
||||
out: /* original two vnodes are locked */
|
||||
MPASS(!want_seqc_end);
|
||||
if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
|
||||
zil_commit(zilog, 0);
|
||||
|
||||
@ -4861,6 +4883,31 @@ zfs_freebsd_write(ap)
|
||||
ap->a_cred, NULL));
|
||||
}
|
||||
|
||||
/*
|
||||
* VOP_FPLOOKUP_VEXEC routines are subject to special circumstances, see
|
||||
* the comment above cache_fplookup for details.
|
||||
*/
|
||||
static int
|
||||
zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v)
|
||||
{
|
||||
vnode_t *vp;
|
||||
znode_t *zp;
|
||||
uint64_t pflags;
|
||||
|
||||
vp = v->a_vp;
|
||||
zp = VTOZ_SMR(vp);
|
||||
if (__predict_false(zp == NULL))
|
||||
return (EAGAIN);
|
||||
pflags = atomic_load_64(&zp->z_pflags);
|
||||
if (pflags & ZFS_AV_QUARANTINED)
|
||||
return (EAGAIN);
|
||||
if (pflags & ZFS_XATTR)
|
||||
return (EAGAIN);
|
||||
if ((pflags & ZFS_NO_EXECS_DENIED) == 0)
|
||||
return (EAGAIN);
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
zfs_freebsd_access(ap)
|
||||
struct vop_access_args /* {
|
||||
@ -5998,6 +6045,7 @@ struct vop_vector zfs_vnodeops = {
|
||||
.vop_inactive = zfs_freebsd_inactive,
|
||||
.vop_need_inactive = zfs_freebsd_need_inactive,
|
||||
.vop_reclaim = zfs_freebsd_reclaim,
|
||||
.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
|
||||
.vop_access = zfs_freebsd_access,
|
||||
.vop_allocate = VOP_EINVAL,
|
||||
.vop_lookup = zfs_cache_lookup,
|
||||
@ -6066,6 +6114,7 @@ VFS_VOP_VECTOR_REGISTER(zfs_fifoops);
|
||||
*/
|
||||
struct vop_vector zfs_shareops = {
|
||||
.vop_default = &default_vnodeops,
|
||||
.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
|
||||
.vop_access = zfs_freebsd_access,
|
||||
.vop_inactive = zfs_freebsd_inactive,
|
||||
.vop_reclaim = zfs_freebsd_reclaim,
|
||||
|
@ -99,7 +99,12 @@ SYSCTL_INT(_debug_sizeof, OID_AUTO, znode, CTLFLAG_RD,
|
||||
*/
|
||||
krwlock_t zfsvfs_lock;
|
||||
|
||||
#if defined(_KERNEL) && !defined(KMEM_DEBUG)
|
||||
#define _ZFS_USE_SMR
|
||||
static uma_zone_t znode_uma_zone;
|
||||
#else
|
||||
static kmem_cache_t *znode_cache = NULL;
|
||||
#endif
|
||||
|
||||
/*ARGSUSED*/
|
||||
static void
|
||||
@ -366,6 +371,51 @@ zfs_znode_move(void *buf, void *newbuf, size_t size, void *arg)
|
||||
}
|
||||
#endif /* illumos */
|
||||
|
||||
#ifdef _ZFS_USE_SMR
|
||||
VFS_SMR_DECLARE;
|
||||
|
||||
static int
|
||||
zfs_znode_cache_constructor_smr(void *mem, int size __unused, void *private, int flags)
|
||||
{
|
||||
|
||||
return (zfs_znode_cache_constructor(mem, private, flags));
|
||||
}
|
||||
|
||||
static void
|
||||
zfs_znode_cache_destructor_smr(void *mem, int size __unused, void *private)
|
||||
{
|
||||
|
||||
zfs_znode_cache_destructor(mem, private);
|
||||
}
|
||||
|
||||
void
|
||||
zfs_znode_init(void)
|
||||
{
|
||||
/*
|
||||
* Initialize zcache
|
||||
*/
|
||||
rw_init(&zfsvfs_lock, NULL, RW_DEFAULT, NULL);
|
||||
ASSERT(znode_uma_zone == NULL);
|
||||
znode_uma_zone = uma_zcreate("zfs_znode_cache",
|
||||
sizeof (znode_t), zfs_znode_cache_constructor_smr,
|
||||
zfs_znode_cache_destructor_smr, NULL, NULL, 0, 0);
|
||||
VFS_SMR_ZONE_SET(znode_uma_zone);
|
||||
}
|
||||
|
||||
static znode_t *
|
||||
zfs_znode_alloc_kmem(int flags)
|
||||
{
|
||||
|
||||
return (uma_zalloc_smr(znode_uma_zone, flags));
|
||||
}
|
||||
|
||||
static void
|
||||
zfs_znode_free_kmem(znode_t *zp)
|
||||
{
|
||||
|
||||
uma_zfree_smr(znode_uma_zone, zp);
|
||||
}
|
||||
#else
|
||||
void
|
||||
zfs_znode_init(void)
|
||||
{
|
||||
@ -380,6 +430,21 @@ zfs_znode_init(void)
|
||||
kmem_cache_set_move(znode_cache, zfs_znode_move);
|
||||
}
|
||||
|
||||
static znode_t *
|
||||
zfs_znode_alloc_kmem(int flags)
|
||||
{
|
||||
|
||||
return (kmem_cache_alloc(znode_cache, flags));
|
||||
}
|
||||
|
||||
static void
|
||||
zfs_znode_free_kmem(znode_t *zp)
|
||||
{
|
||||
|
||||
kmem_cache_free(znode_cache, zp);
|
||||
}
|
||||
#endif
|
||||
|
||||
void
|
||||
zfs_znode_fini(void)
|
||||
{
|
||||
@ -393,9 +458,17 @@ zfs_znode_fini(void)
|
||||
/*
|
||||
* Cleanup zcache
|
||||
*/
|
||||
if (znode_cache)
|
||||
#ifdef _ZFS_USE_SMR
|
||||
if (znode_uma_zone) {
|
||||
uma_zdestroy(znode_uma_zone);
|
||||
znode_uma_zone = NULL;
|
||||
}
|
||||
#else
|
||||
if (znode_cache) {
|
||||
kmem_cache_destroy(znode_cache);
|
||||
znode_cache = NULL;
|
||||
znode_cache = NULL;
|
||||
}
|
||||
#endif
|
||||
rw_destroy(&zfsvfs_lock);
|
||||
}
|
||||
|
||||
@ -508,7 +581,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
|
||||
vattr.va_uid = crgetuid(kcred);
|
||||
vattr.va_gid = crgetgid(kcred);
|
||||
|
||||
sharezp = kmem_cache_alloc(znode_cache, KM_SLEEP);
|
||||
sharezp = zfs_znode_alloc_kmem(KM_SLEEP);
|
||||
ASSERT(!POINTER_IS_VALID(sharezp->z_zfsvfs));
|
||||
sharezp->z_moved = 0;
|
||||
sharezp->z_unlinked = 0;
|
||||
@ -527,7 +600,7 @@ zfs_create_share_dir(zfsvfs_t *zfsvfs, dmu_tx_t *tx)
|
||||
|
||||
zfs_acl_ids_free(&acl_ids);
|
||||
sa_handle_destroy(sharezp->z_sa_hdl);
|
||||
kmem_cache_free(znode_cache, sharezp);
|
||||
zfs_znode_free_kmem(sharezp);
|
||||
|
||||
return (error);
|
||||
}
|
||||
@ -642,13 +715,18 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
|
||||
int count = 0;
|
||||
int error;
|
||||
|
||||
zp = kmem_cache_alloc(znode_cache, KM_SLEEP);
|
||||
zp = zfs_znode_alloc_kmem(KM_SLEEP);
|
||||
|
||||
#ifndef _ZFS_USE_SMR
|
||||
KASSERT((zfsvfs->z_parent->z_vfs->mnt_kern_flag & MNTK_FPLOOKUP) == 0,
|
||||
("%s: fast path lookup enabled without smr", __func__));
|
||||
#endif
|
||||
|
||||
KASSERT(curthread->td_vp_reserved != NULL,
|
||||
("zfs_znode_alloc: getnewvnode without preallocated vnode"));
|
||||
error = getnewvnode("zfs", zfsvfs->z_parent->z_vfs, &zfs_vnodeops, &vp);
|
||||
if (error != 0) {
|
||||
kmem_cache_free(znode_cache, zp);
|
||||
zfs_znode_free_kmem(zp);
|
||||
return (NULL);
|
||||
}
|
||||
zp->z_vnode = vp;
|
||||
@ -695,7 +773,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
|
||||
sa_handle_destroy(zp->z_sa_hdl);
|
||||
zfs_vnode_forget(vp);
|
||||
zp->z_vnode = NULL;
|
||||
kmem_cache_free(znode_cache, zp);
|
||||
zfs_znode_free_kmem(zp);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
@ -1061,6 +1139,8 @@ zfs_xvattr_set(znode_t *zp, xvattr_t *xvap, dmu_tx_t *tx)
|
||||
xoap = xva_getxoptattr(xvap);
|
||||
ASSERT(xoap);
|
||||
|
||||
ASSERT_VOP_IN_SEQC(ZTOV(zp));
|
||||
|
||||
if (XVA_ISSET_REQ(xvap, XAT_CREATETIME)) {
|
||||
uint64_t times[2];
|
||||
ZFS_TIME_ENCODE(&xoap->xoa_createtime, times);
|
||||
@ -1490,7 +1570,7 @@ zfs_znode_free(znode_t *zp)
|
||||
zp->z_acl_cached = NULL;
|
||||
}
|
||||
|
||||
kmem_cache_free(znode_cache, zp);
|
||||
zfs_znode_free_kmem(zp);
|
||||
|
||||
#ifdef illumos
|
||||
VFS_RELE(zfsvfs->z_vfs);
|
||||
@ -1950,7 +2030,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
|
||||
|
||||
zfsvfs = kmem_zalloc(sizeof (zfsvfs_t), KM_SLEEP);
|
||||
|
||||
rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP);
|
||||
rootzp = zfs_znode_alloc_kmem(KM_SLEEP);
|
||||
ASSERT(!POINTER_IS_VALID(rootzp->z_zfsvfs));
|
||||
rootzp->z_moved = 0;
|
||||
rootzp->z_unlinked = 0;
|
||||
@ -1994,7 +2074,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx)
|
||||
POINTER_INVALIDATE(&rootzp->z_zfsvfs);
|
||||
|
||||
sa_handle_destroy(rootzp->z_sa_hdl);
|
||||
kmem_cache_free(znode_cache, rootzp);
|
||||
zfs_znode_free_kmem(rootzp);
|
||||
|
||||
/*
|
||||
* Create shares directory
|
||||
|
Loading…
Reference in New Issue
Block a user