d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fusefs: support asynchronous cache invalidation

Browse Source 
Protocol 7.12 adds a way for the server to notify the client that it should
invalidate an inode's data cache and/or attributes.  This commit implements
that mechanism.  Unlike Linux's implementation, ours requires that the file
system also supports FUSE_EXPORT_SUPPORT (NFS-style lookups).  Otherwise the
invalidation operation will return EINVAL.

Sponsored by:	The FreeBSD Foundation
This commit is contained in:
Alan Somers 2019-06-03 17:34:01 +00:00
parent c2d70d6e6f
commit eae1ae132c
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/projects/fuse2/; revision=348560
7 changed files with 371 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sys/fs/fuse/fuse_device.c
View File

@ -515,8 +515,10 @@ fuse_device_write(struct cdev *dev, struct uio *uio, int ioflag)
case FUSE_NOTIFY_INVAL_ENTRY:
err = fuse_internal_invalidate_entry(mp, uio);
break;
case FUSE_NOTIFY_POLL:
case FUSE_NOTIFY_INVAL_INODE:
err = fuse_internal_invalidate_inode(mp, uio);
break;
case FUSE_NOTIFY_POLL:
default:
/* Not implemented */
err = ENOSYS;

90
sys/fs/fuse/fuse_internal.c
View File

@ -89,6 +89,7 @@ __FBSDID("$FreeBSD$");
#include "fuse.h"
#include "fuse_file.h"
#include "fuse_internal.h"
#include "fuse_io.h"
#include "fuse_ipc.h"
#include "fuse_node.h"
#include "fuse_file.h"
@ -106,6 +107,42 @@ static int isbzero(void *buf, size_t len);
#endif
int
fuse_internal_get_cached_vnode(struct mount* mp, ino_t ino, int flags,
struct vnode **vpp)
{
struct bintime now;
struct thread *td = curthread;
uint64_t nodeid = ino;
int error;
*vpp = NULL;
error = vfs_hash_get(mp, fuse_vnode_hash(nodeid), flags, td, vpp,
fuse_vnode_cmp, &nodeid);
if (error)
return error;
/*
* Check the entry cache timeout. We have to do this within fusefs
* instead of by using cache_enter_time/cache_lookup because those
* routines are only intended to work with pathnames, not inodes
*/
if (*vpp != NULL) {
getbinuptime(&now);
if (bintime_cmp(&(VTOFUD(*vpp)->entry_cache_timeout), &now, >)){
atomic_add_acq_long(&fuse_lookup_cache_hits, 1);
return 0;
} else {
/* Entry cache timeout */
atomic_add_acq_long(&fuse_lookup_cache_misses, 1);
cache_purge(*vpp);
vput(*vpp);
*vpp = NULL;
}
}
return 0;
}
/* Synchronously send a FUSE_ACCESS operation */
int
fuse_internal_access(struct vnode *vp,
@ -337,7 +374,6 @@ fuse_internal_invalidate_entry(struct mount *mp, struct uio *uio)
struct fuse_notify_inval_entry_out fnieo;
struct fuse_data *data = fuse_get_mpdata(mp);
struct componentname cn;
/*struct vnode *dvp;*/
struct vnode *dvp, *vp;
char name[PATH_MAX];
int err;
@ -367,8 +403,9 @@ fuse_internal_invalidate_entry(struct mount *mp, struct uio *uio)
if (fnieo.parent == FUSE_ROOT_ID)
err = VFS_ROOT(mp, LK_SHARED, &dvp);
else
err = VFS_VGET(mp, fnieo.parent, LK_SHARED, &dvp);
if (err != 0)
err = fuse_internal_get_cached_vnode( mp, fnieo.parent,
LK_SHARED, &dvp);
if (err != 0 || dvp == NULL)
return (err);
/*
* XXX we can't check dvp's generation because the FUSE invalidate
@ -391,6 +428,53 @@ fuse_internal_invalidate_entry(struct mount *mp, struct uio *uio)
return (0);
}
int
fuse_internal_invalidate_inode(struct mount *mp, struct uio *uio)
{
struct fuse_notify_inval_inode_out fniio;
struct fuse_data *data = fuse_get_mpdata(mp);
struct vnode *vp;
int err;
if (!(data->dataflags & FSESS_EXPORT_SUPPORT)) {
/*
* Linux allows file systems without export support to use
* asynchronous notification because its inode cache is indexed
* purely by the inode number. But FreeBSD's vnode is cache
* requires access to the entire vnode structure.
*/
SDT_PROBE1(fusefs, , internal, invalidate_without_export, mp);
return (EINVAL);
}
if ((err = uiomove(&fniio, sizeof(fniio), uio)) != 0)
return (err);
if (fniio.ino == FUSE_ROOT_ID)
err = VFS_ROOT(mp, LK_EXCLUSIVE, &vp);
else
err = fuse_internal_get_cached_vnode(mp, fniio.ino, LK_SHARED,
&vp);
if (err != 0 || vp == NULL)
return (err);
/*
* XXX we can't check vp's generation because the FUSE invalidate
* entry message doesn't include it. Worse case is that we invalidate
* an inode that didn't need to be invalidated.
*/
/*
* Flush and invalidate buffers if off >= 0. Technically we only need
* to flush and invalidate the range of offsets [off, off + len), but
* for simplicity's sake we do everything.
*/
if (fniio.off >= 0)
fuse_io_invalbuf(vp, curthread);
fuse_vnode_clear_attr_cache(vp);
vput(vp);
return (0);
}
/* mknod */
int
fuse_internal_mknod(struct vnode *dvp, struct vnode **vpp,

5
sys/fs/fuse/fuse_internal.h
View File

@ -193,6 +193,10 @@ fuse_validity_2_timespec(const struct fuse_entry_out *feo,
}
/* VFS ops */
int
fuse_internal_get_cached_vnode(struct mount*, ino_t, int, struct vnode**);
/* access */
static inline int
fuse_match_cred(struct ucred *basecred, struct ucred *usercred)
@ -230,6 +234,7 @@ int fuse_internal_getattr(struct vnode *vp, struct vattr *vap,
/* asynchronous invalidation */
int fuse_internal_invalidate_entry(struct mount *mp, struct uio *uio);
int fuse_internal_invalidate_inode(struct mount *mp, struct uio *uio);
/* mknod */
int fuse_internal_mknod(struct vnode *dvp, struct vnode **vpp,

24
sys/fs/fuse/fuse_vfsops.c
View File

@ -527,34 +527,14 @@ fuse_vfsop_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp)
struct fuse_dispatcher fdi;
struct fuse_entry_out *feo;
struct fuse_vnode_data *fvdat;
struct bintime now;
const char dot[] = ".";
off_t filesize;
enum vtype vtyp;
int error;
error = vfs_hash_get(mp, fuse_vnode_hash(nodeid), flags, td, vpp,
fuse_vnode_cmp, &nodeid);
if (error)
error = fuse_internal_get_cached_vnode(mp, ino, flags, vpp);
if (error || *vpp != NULL)
return error;
/*
* Check the entry cache timeout. We have to do this within fusefs
* instead of by using cache_enter_time/cache_lookup because those
* routines are only intended to work with pathnames, not inodes
*/
if (*vpp != NULL) {
getbinuptime(&now);
if (bintime_cmp(&(VTOFUD(*vpp)->entry_cache_timeout), &now, >)){
atomic_add_acq_long(&fuse_lookup_cache_hits, 1);
return 0;
} else {
/* Entry cache timeout */
atomic_add_acq_long(&fuse_lookup_cache_misses, 1);
cache_purge(*vpp);
vput(*vpp);
*vpp = NULL;
}
}
/* Do a LOOKUP, using nodeid as the parent and "." as filename */
fdisp_init(&fdi, sizeof(dot));

22
tests/sys/fs/fusefs/mockfs.cc
View File

@ -322,6 +322,13 @@ void MockFS::debug_response(const mockfs_buf_out &out) {
printf("<- INVAL_ENTRY parent=%" PRIu64 " %s\n",
out.body.inval_entry.parent, name);
break;
case FUSE_NOTIFY_INVAL_INODE:
printf("<- INVAL_INODE ino=%" PRIu64 " off=%" PRIi64
" len=%" PRIi64 "\n",
out.body.inval_inode.ino,
out.body.inval_inode.off,
out.body.inval_inode.len);
break;
default:
break;
}
@ -516,6 +523,21 @@ int MockFS::notify_inval_entry(ino_t parent, const char *name, size_t namelen)
return 0;
}
int MockFS::notify_inval_inode(ino_t ino, off_t off, ssize_t len)
{
std::unique_ptr<mockfs_buf_out> out(new mockfs_buf_out);
out->header.unique = 0; /* 0 means asynchronous notification */
out->header.error = FUSE_NOTIFY_INVAL_INODE;
out->body.inval_inode.ino = ino;
out->body.inval_inode.off = off;
out->body.inval_inode.len = len;
out->header.len = sizeof(out->header) + sizeof(out->body.inval_inode);
debug_response(*out);
write_response(*out);
return 0;
}
bool MockFS::pid_ok(pid_t pid) {
if (pid == m_pid) {
return (true);

18
tests/sys/fs/fusefs/mockfs.hh
View File

@ -178,6 +178,7 @@ union fuse_payloads_out {
fuse_init_out init;
/* The inval_entry structure should be followed by the entry's name */
fuse_notify_inval_entry_out inval_entry;
fuse_notify_inval_inode_out inval_inode;
fuse_listxattr_out listxattr;
fuse_open_out open;
fuse_statfs_out statfs;
@ -325,6 +326,23 @@ class MockFS {
*/
int notify_inval_entry(ino_t parent, const char *name, size_t namelen);
/*
* Send an asynchronous notification to invalidate an inode's cached
* data and/or attributes. Similar to libfuse's
* fuse_lowlevel_notify_inval_inode.
*
* This method will block until the client has responded, so it should
* generally be run in a separate thread from request processing.
*
* @param ino File's inode number
* @param off offset at which to begin invalidation. A
* negative offset means to invalidate attributes
* only.
* @param len Size of region of data to invalidate. 0 means
* to invalidate all cached data.
*/
int notify_inval_inode(ino_t ino, off_t off, ssize_t len);
/*
* Request handler
*

244
tests/sys/fs/fusefs/notify.cc
View File

@ -29,6 +29,10 @@
*/
extern "C" {
#include <sys/types.h>
#include <sys/sysctl.h>
#include <fcntl.h>
#include <pthread.h>
}
@ -46,14 +50,13 @@ using namespace testing;
class Notify: public FuseTest {
public:
public:
virtual void SetUp() {
m_init_flags = FUSE_EXPORT_SUPPORT;
FuseTest::SetUp();
}
void expect_lookup(uint64_t parent, const char *relpath, uint64_t ino,
Sequence &seq)
off_t size, Sequence &seq)
{
EXPECT_LOOKUP(parent, relpath)
.InSequence(seq)
@ -64,12 +67,39 @@ void expect_lookup(uint64_t parent, const char *relpath, uint64_t ino,
out.body.entry.nodeid = ino;
out.body.entry.attr.ino = ino;
out.body.entry.attr.nlink = 1;
out.body.entry.attr.size = size;
out.body.entry.attr_valid = UINT64_MAX;
out.body.entry.entry_valid = UINT64_MAX;
})));
}
};
class NotifyWriteback: public Notify {
public:
virtual void SetUp() {
const char *node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
Notify::SetUp();
if (IsSkipped())
return;
ASSERT_EQ(0, sysctlbyname(node, &val, &size, NULL, 0))
<< strerror(errno);
if (val != 2)
GTEST_SKIP() << "vfs.fusefs.data_cache_mode must be set to 2 "
"(writeback) for this test";
}
void expect_write(uint64_t ino, uint64_t offset, uint64_t size,
const void *contents)
{
FuseTest::expect_write(ino, offset, size, size, 0, 0, contents);
}
};
struct inval_entry_args {
MockFS *mock;
ino_t parent;
@ -88,6 +118,24 @@ static void* inval_entry(void* arg) {
return (void*)(intptr_t)errno;
}
struct inval_inode_args {
MockFS *mock;
ino_t ino;
off_t off;
ssize_t len;
};
static void* inval_inode(void* arg) {
const struct inval_inode_args *iia = (struct inval_inode_args*)arg;
ssize_t r;
r = iia->mock->notify_inval_inode(iia->ino, iia->off, iia->len);
if (r >= 0)
return 0;
else
return (void*)(intptr_t)errno;
}
/* Invalidate a nonexistent entry */
TEST_F(Notify, inval_entry_nonexistent)
{
@ -120,8 +168,8 @@ TEST_F(Notify, inval_entry)
Sequence seq;
pthread_t th0;
expect_lookup(FUSE_ROOT_ID, RELPATH, ino0, seq);
expect_lookup(FUSE_ROOT_ID, RELPATH, ino1, seq);
expect_lookup(FUSE_ROOT_ID, RELPATH, ino0, 0, seq);
expect_lookup(FUSE_ROOT_ID, RELPATH, ino1, 0, seq);
/* Fill the entry cache */
ASSERT_EQ(0, stat(FULLPATH, &sb)) << strerror(errno);
@ -135,7 +183,6 @@ TEST_F(Notify, inval_entry)
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_entry, &iea))
<< strerror(errno);
pthread_join(th0, &thr0_value);
/* It's not an error for an entry to not be cached */
EXPECT_EQ(0, (intptr_t)thr0_value);
/* The second lookup should return the alternate ino */
@ -171,8 +218,8 @@ TEST_F(Notify, inval_entry_below_root)
out.body.entry.attr_valid = UINT64_MAX;
out.body.entry.entry_valid = UINT64_MAX;
})));
expect_lookup(dir_ino, FNAME, ino0, seq);
expect_lookup(dir_ino, FNAME, ino1, seq);
expect_lookup(dir_ino, FNAME, ino0, 0, seq);
expect_lookup(dir_ino, FNAME, ino1, 0, seq);
/* Fill the entry cache */
ASSERT_EQ(0, stat(FULLPATH, &sb)) << strerror(errno);
@ -186,7 +233,6 @@ TEST_F(Notify, inval_entry_below_root)
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_entry, &iea))
<< strerror(errno);
pthread_join(th0, &thr0_value);
/* It's not an error for an entry to not be cached */
EXPECT_EQ(0, (intptr_t)thr0_value);
/* The second lookup should return the alternate ino */
@ -206,7 +252,7 @@ TEST_F(Notify, inval_entry_invalidates_parent_attrs)
Sequence seq;
pthread_t th0;
expect_lookup(FUSE_ROOT_ID, RELPATH, ino, seq);
expect_lookup(FUSE_ROOT_ID, RELPATH, ino, 0, seq);
EXPECT_CALL(*m_mock, process(
ResultOf([=](auto in) {
return (in.header.opcode == FUSE_GETATTR &&
@ -232,9 +278,187 @@ TEST_F(Notify, inval_entry_invalidates_parent_attrs)
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_entry, &iea))
<< strerror(errno);
pthread_join(th0, &thr0_value);
/* It's not an error for an entry to not be cached */
EXPECT_EQ(0, (intptr_t)thr0_value);
/* /'s attribute cache should be cleared */
ASSERT_EQ(0, stat("mountpoint", &sb)) << strerror(errno);
}
TEST_F(Notify, inval_inode_nonexistent)
{
struct inval_inode_args iia;
ino_t ino = 42;
void *thr0_value;
pthread_t th0;
iia.mock = m_mock;
iia.ino = ino;
iia.off = 0;
iia.len = 0;
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_inode, &iia))
<< strerror(errno);
pthread_join(th0, &thr0_value);
/* It's not an error for an inode to not be cached */
EXPECT_EQ(0, (intptr_t)thr0_value);
}
TEST_F(Notify, inval_inode_with_clean_cache)
{
const static char FULLPATH[] = "mountpoint/foo";
const static char RELPATH[] = "foo";
const char CONTENTS0[] = "abcdefgh";
const char CONTENTS1[] = "ijklmnopqrstuvwxyz";
struct inval_inode_args iia;
struct stat sb;
ino_t ino = 42;
void *thr0_value;
Sequence seq;
uid_t uid = 12345;
pthread_t th0;
ssize_t size0 = sizeof(CONTENTS0);
ssize_t size1 = sizeof(CONTENTS1);
char buf[80];
int fd;
expect_lookup(FUSE_ROOT_ID, RELPATH, ino, size0, seq);
expect_open(ino, 0, 1);
EXPECT_CALL(*m_mock, process(
ResultOf([=](auto in) {
return (in.header.opcode == FUSE_GETATTR &&
in.header.nodeid == ino);
}, Eq(true)),
_)
).WillOnce(Invoke(ReturnImmediate([=](auto i __unused, auto& out) {
SET_OUT_HEADER_LEN(out, attr);
out.body.attr.attr.mode = S_IFREG | 0644;
out.body.attr.attr_valid = UINT64_MAX;
out.body.attr.attr.size = size1;
out.body.attr.attr.uid = uid;
})));
expect_read(ino, 0, size0, size0, CONTENTS0);
expect_read(ino, 0, size1, size1, CONTENTS1);
/* Fill the data cache */
fd = open(FULLPATH, O_RDWR);
ASSERT_LE(0, fd) << strerror(errno);
ASSERT_EQ(size0, read(fd, buf, size0)) << strerror(errno);
EXPECT_EQ(0, memcmp(buf, CONTENTS0, size0));
/* Evict the data cache */
iia.mock = m_mock;
iia.ino = ino;
iia.off = 0;
iia.len = 0;
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_inode, &iia))
<< strerror(errno);
pthread_join(th0, &thr0_value);
EXPECT_EQ(0, (intptr_t)thr0_value);
/* cache attributes were been purged; this will trigger a new GETATTR */
ASSERT_EQ(0, stat(FULLPATH, &sb)) << strerror(errno);
EXPECT_EQ(uid, sb.st_uid);
EXPECT_EQ(size1, sb.st_size);
/* This read should not be serviced by cache */
ASSERT_EQ(0, lseek(fd, 0, SEEK_SET)) << strerror(errno);
ASSERT_EQ(size1, read(fd, buf, size1)) << strerror(errno);
EXPECT_EQ(0, memcmp(buf, CONTENTS1, size1));
/* Deliberately leak fd. close(2) will be tested in release.cc */
}
/* https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=238312 */
TEST_F(NotifyWriteback, DISABLED_inval_inode_with_dirty_cache)
{
const static char FULLPATH[] = "mountpoint/foo";
const static char RELPATH[] = "foo";
const char CONTENTS[] = "abcdefgh";
struct inval_inode_args iia;
ino_t ino = 42;
void *thr0_value;
Sequence seq;
pthread_t th0;
ssize_t bufsize = sizeof(CONTENTS);
int fd;
expect_lookup(FUSE_ROOT_ID, RELPATH, ino, 0, seq);
expect_open(ino, 0, 1);
/* Fill the data cache */
fd = open(FULLPATH, O_RDWR);
ASSERT_EQ(bufsize, write(fd, CONTENTS, bufsize)) << strerror(errno);
/* Evict the data cache */
expect_write(ino, 0, bufsize, CONTENTS);
iia.mock = m_mock;
iia.ino = ino;
iia.off = 0;
iia.len = 0;
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_inode, &iia))
<< strerror(errno);
pthread_join(th0, &thr0_value);
EXPECT_EQ(0, (intptr_t)thr0_value);
/* Deliberately leak fd. close(2) will be tested in release.cc */
}
/* https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=238312 */
TEST_F(NotifyWriteback, DISABLED_inval_inode_attrs_only)
{
const static char FULLPATH[] = "mountpoint/foo";
const static char RELPATH[] = "foo";
const char CONTENTS[] = "abcdefgh";
struct inval_inode_args iia;
struct stat sb;
uid_t uid = 12345;
ino_t ino = 42;
void *thr0_value;
Sequence seq;
pthread_t th0;
ssize_t bufsize = sizeof(CONTENTS);
int fd;
expect_lookup(FUSE_ROOT_ID, RELPATH, ino, 0, seq);
expect_open(ino, 0, 1);
EXPECT_CALL(*m_mock, process(
ResultOf([=](auto in) {
return (in.header.opcode == FUSE_WRITE);
}, Eq(true)),
_)
).Times(0);
EXPECT_CALL(*m_mock, process(
ResultOf([=](auto in) {
return (in.header.opcode == FUSE_GETATTR &&
in.header.nodeid == FUSE_ROOT_ID);
}, Eq(true)),
_)
).WillOnce(Invoke(ReturnImmediate([=](auto i __unused, auto& out) {
SET_OUT_HEADER_LEN(out, attr);
out.body.attr.attr.mode = S_IFREG | 0644;
out.body.attr.attr_valid = UINT64_MAX;
out.body.attr.attr.size = bufsize;
out.body.attr.attr.uid = uid;
})));
/* Fill the data cache */
fd = open(FULLPATH, O_RDWR);
ASSERT_EQ(bufsize, write(fd, CONTENTS, bufsize)) << strerror(errno);
/* Evict the attributes, but not data cache */
iia.mock = m_mock;
iia.ino = ino;
iia.off = -1;
iia.len = 0;
ASSERT_EQ(0, pthread_create(&th0, NULL, inval_inode, &iia))
<< strerror(errno);
pthread_join(th0, &thr0_value);
EXPECT_EQ(0, (intptr_t)thr0_value);
/* cache attributes were been purged; this will trigger a new GETATTR */
ASSERT_EQ(0, stat(FULLPATH, &sb)) << strerror(errno);
EXPECT_EQ(uid, sb.st_uid);
EXPECT_EQ(bufsize, sb.st_size);
/* Deliberately leak fd. close(2) will be tested in release.cc */
}