zfs: add support for lockless symlink lookup

Reviewed by:	kib (previous version)
Tested by:	pho (previous version)
Differential Revision:	https://reviews.freebsd.org/D27488

sys/contrib/openzfs/include/os/freebsd/zfs/sys/zfs_znode_impl.h

@@ -53,6 +53,7 @@ extern "C" {
 #define	ZNODE_OS_FIELDS                 \
 	struct zfsvfs	*z_zfsvfs;      \
 	vnode_t		*z_vnode;       \
+	char		*z_cached_symlink;      \
 	uint64_t		z_uid;          \
 	uint64_t		z_gid;          \
 	uint64_t		z_gen;          \

sys/contrib/openzfs/module/os/freebsd/zfs/zfs_vnops_os.c

@@ -4463,6 +4463,31 @@ zfs_freebsd_fplookup_vexec(struct vop_fplookup_vexec_args *v)
 }
 #endif
 
+#if __FreeBSD_version >= 1400001
+static int
+zfs_freebsd_fplookup_symlink(struct vop_fplookup_symlink_args *v)
+{
+	vnode_t *vp;
+	znode_t *zp;
+	char *target;
+
+	vp = v->a_vp;
+	zp = VTOZ_SMR(vp);
+	if (__predict_false(zp == NULL)) {
+		return (EAGAIN);
+	}
+
+	/*
+	 * FIXME: Load consume would be sufficient but there is no primitive to do it.
+	 */
+	target = (char *)atomic_load_acq_ptr((uintptr_t *)&zp->z_cached_symlink);
+	if (target == NULL) {
+		return (EAGAIN);
+	}
+	return (cache_symlink_resolve(v->a_fpl, target, strlen(target)));
+}
+#endif
+
 #ifndef _SYS_SYSPROTO_H_
 struct vop_access_args {
 	struct vnode *a_vp;
@@ -4949,6 +4974,8 @@ zfs_freebsd_symlink(struct vop_symlink_args *ap)
 	struct componentname *cnp = ap->a_cnp;
 	vattr_t *vap = ap->a_vap;
 	znode_t *zp = NULL;
+	char *symlink;
+	size_t symlink_len;
 	int rc;
 
 	ASSERT(cnp->cn_flags & SAVENAME);
@@ -4959,8 +4986,19 @@ zfs_freebsd_symlink(struct vop_symlink_args *ap)
 
 	rc = zfs_symlink(VTOZ(ap->a_dvp), cnp->cn_nameptr, vap,
 	    ap->a_target, &zp, cnp->cn_cred, 0 /* flags */);
-	if (rc == 0)
+	if (rc == 0) {
 		*ap->a_vpp = ZTOV(zp);
+		ASSERT_VOP_ELOCKED(ZTOV(zp), __func__);
+		MPASS(zp->z_cached_symlink == NULL);
+		symlink_len = strlen(ap->a_target);
+		symlink = cache_symlink_alloc(symlink_len + 1, M_WAITOK);
+		if (symlink != NULL) {
+			memcpy(symlink, ap->a_target, symlink_len);
+			symlink[symlink_len] = '\0';
+			atomic_store_rel_ptr((uintptr_t *)&zp->z_cached_symlink,
+			    (uintptr_t)symlink);
+		}
+	}
 	return (rc);
 }
 
@@ -4975,8 +5013,36 @@ struct vop_readlink_args {
 static int
 zfs_freebsd_readlink(struct vop_readlink_args *ap)
 {
+	znode_t	*zp = VTOZ(ap->a_vp);
+	struct uio *auio;
+	char *symlink, *base;
+	size_t symlink_len;
+	int error;
+	bool trycache;
 
-	return (zfs_readlink(ap->a_vp, ap->a_uio, ap->a_cred, NULL));
+	auio = ap->a_uio;
+	trycache = false;
+	if (auio->uio_segflg == UIO_SYSSPACE && auio->uio_iovcnt == 1) {
+		base = auio->uio_iov->iov_base;
+		symlink_len = auio->uio_iov->iov_len;
+		trycache = true;
+	}
+	error = zfs_readlink(ap->a_vp, auio, ap->a_cred, NULL);
+	if (atomic_load_ptr(&zp->z_cached_symlink) != NULL ||
+	    error != 0 || !trycache) {
+		return (error);
+	}
+	symlink_len -= auio->uio_resid;
+	symlink = cache_symlink_alloc(symlink_len + 1, M_WAITOK);
+	if (symlink != NULL) {
+		memcpy(symlink, base, symlink_len);
+		symlink[symlink_len] = '\0';
+		if (!atomic_cmpset_rel_ptr((uintptr_t *)&zp->z_cached_symlink,
+		    (uintptr_t)NULL, (uintptr_t)symlink)) {
+			cache_symlink_free(symlink, symlink_len + 1);
+		}
+	}
+	return (error);
 }
 
 #ifndef _SYS_SYSPROTO_H_
@@ -5733,6 +5799,9 @@ struct vop_vector zfs_vnodeops = {
 	.vop_reclaim =		zfs_freebsd_reclaim,
 #if __FreeBSD_version >= 1300102
 	.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
+#endif
+#if __FreeBSD_version >= 1400001
+	.vop_fplookup_symlink = zfs_freebsd_fplookup_symlink,
 #endif
 	.vop_access =		zfs_freebsd_access,
 	.vop_allocate =		VOP_EINVAL,
@@ -5782,6 +5851,9 @@ struct vop_vector zfs_fifoops = {
 	.vop_fsync =		zfs_freebsd_fsync,
 #if __FreeBSD_version >= 1300102
 	.vop_fplookup_vexec = zfs_freebsd_fplookup_vexec,
+#endif
+#if __FreeBSD_version >= 1400001
+	.vop_fplookup_symlink = zfs_freebsd_fplookup_symlink,
 #endif
 	.vop_access =		zfs_freebsd_access,
 	.vop_getattr =		zfs_freebsd_getattr,
@@ -5805,6 +5877,9 @@ struct vop_vector zfs_shareops = {
 	.vop_default =		&default_vnodeops,
 #if __FreeBSD_version >= 1300121
 	.vop_fplookup_vexec =	VOP_EAGAIN,
+#endif
+#if __FreeBSD_version >= 1400001
+	.vop_fplookup_symlink =	VOP_EAGAIN,
 #endif
 	.vop_access =		zfs_freebsd_access,
 	.vop_inactive =		zfs_freebsd_inactive,

sys/contrib/openzfs/module/os/freebsd/zfs/zfs_znode.c

@@ -444,6 +444,7 @@ zfs_znode_alloc(zfsvfs_t *zfsvfs, dmu_buf_t *db, int blksz,
 	zp->z_blksz = blksz;
 	zp->z_seq = 0x7A4653;
 	zp->z_sync_cnt = 0;
+	atomic_store_ptr((uintptr_t *)&zp->z_cached_symlink, (uintptr_t)NULL);
 
 	vp = ZTOV(zp);
 
@@ -1237,6 +1238,7 @@ void
 zfs_znode_free(znode_t *zp)
 {
 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
+	char *symlink;
 
 	ASSERT(zp->z_sa_hdl == NULL);
 	zp->z_vnode = NULL;
@@ -1245,6 +1247,11 @@ zfs_znode_free(znode_t *zp)
 	list_remove(&zfsvfs->z_all_znodes, zp);
 	zfsvfs->z_nr_znodes--;
 	mutex_exit(&zfsvfs->z_znodes_lock);
+	symlink = atomic_load_ptr(&zp->z_cached_symlink);
+	if (symlink != NULL) {
+		atomic_store_rel_ptr((uintptr_t *)&zp->z_cached_symlink, (uintptr_t)NULL);
+		cache_symlink_free(symlink, strlen(symlink) + 1);
+	}
 
 	if (zp->z_acl_cached) {
 		zfs_acl_free(zp->z_acl_cached);