d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fusefs: implement protocol 7.23's FUSE_WRITEBACK_CACHE option

Browse Source 
As of protocol 7.23, fuse file systems can specify their cache behavior on a
per-mountpoint basis.  If they set FUSE_WRITEBACK_CACHE in
fuse_init_out.flags, then they'll get the writeback cache.  If not, then
they'll get the writethrough cache.  If they set FOPEN_DIRECT_IO in every
FUSE_OPEN response, then they'll get no cache at all.

The old vfs.fusefs.data_cache_mode sysctl is ignored for servers that use
protocol 7.23 or later.  However, it's retained for older servers,
especially for those running in jails that lack access to the new protocol.

This commit also fixes two other minor test bugs:
* WriteCluster:SetUp was using an uninitialized variable.
* Read.direct_io_pread wasn't verifying that the cache was actually
  bypassed.

Sponsored by:	The FreeBSD Foundation
This commit is contained in:
asomers 2019-06-26 17:32:31 +00:00
parent 2430baeee2
commit 014c4a07df
10 changed files with 92 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
share/man/man5/fusefs.5
View File

@ -28,7 +28,7 @@
.\" SUCH DAMAGE.
.\"
.\" $FreeBSD$
.Dd April 13, 2019
.Dd June 26, 2019
.Dt FUSEFS 5
.Os
.Sh NAME
@ -73,7 +73,7 @@ Minor version of the FUSE kernel ABI supported by this driver.
.It Va vfs.fusefs.data_cache_mode
Controls how
.Nm
will cache file data.
will cache file data for pre-7.23 file systems.
A value of 0 will disable caching entirely.
Every data access will be forwarded to the daemon.
A value of 1 will select write-through caching.
@ -84,6 +84,9 @@ Reads and writes will both be cached, and writes will occasionally be flushed
to the daemon by the page daemon.
Write-back caching is usually unsafe, especially for FUSE file systems that
require network access.
.Pp
FUSE file systems using protocol 7.23 or later specify their cache behavior
on a per-mountpoint basis, ignoring this sysctl.
.It Va vfs.fusefs.lookup_cache_enable
Controls whether
.Nm

18
sys/fs/fuse/fuse_internal.c
View File

@ -962,6 +962,13 @@ fuse_internal_init_callback(struct fuse_ticket *tick, struct uio *uio)
else
data->time_gran = 1;
if (!fuse_libabi_geq(data, 7, 23))
data->cache_mode = fuse_data_cache_mode;
else if (fiio->flags & FUSE_WRITEBACK_CACHE)
data->cache_mode = FUSE_CACHE_WB;
else
data->cache_mode = FUSE_CACHE_WT;
out:
if (err) {
fdata_set_dead(data);
@ -996,9 +1003,18 @@ fuse_internal_send_init(struct fuse_data *data, struct thread *td)
* FUSE_ATOMIC_O_TRUNC: our VFS cannot support it
* FUSE_DONT_MASK: unlike Linux, FreeBSD always applies the umask, even
* when default ACLs are in use.
* FUSE_SPLICE_WRITE, FUSE_SPLICE_MOVE, FUSE_SPLICE_READ: FreeBSD
* doesn't have splice(2).
* FUSE_FLOCK_LOCKS: not yet implemented
* FUSE_HAS_IOCTL_DIR: not yet implemented
* FUSE_AUTO_INVAL_DATA: not yet implemented
* FUSE_DO_READDIRPLUS: not yet implemented
* FUSE_READDIRPLUS_AUTO: not yet implemented
* FUSE_ASYNC_DIO: not yet implemented
* FUSE_NO_OPEN_SUPPORT: not yet implemented
*/
fiii->flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS | FUSE_EXPORT_SUPPORT
| FUSE_BIG_WRITES;
| FUSE_BIG_WRITES | FUSE_WRITEBACK_CACHE;
fuse_insert_callback(fdi.tick, fuse_internal_init_callback);
fuse_insert_message(fdi.tick, false);

4
sys/fs/fuse/fuse_io.c
View File

@ -253,7 +253,7 @@ fuse_io_dispatch(struct vnode *vp, struct uio *uio, int ioflag,
} else {
SDT_PROBE2(fusefs, , io, trace, 1,
"buffered write of vnode");
if (fuse_data_cache_mode == FUSE_CACHE_WT)
if (!fsess_opt_writeback(vnode_mount(vp)))
ioflag |= IO_SYNC;
err = fuse_write_biobackend(vp, uio, cred, fufh, ioflag,
pid);
@ -481,7 +481,7 @@ fuse_write_directbackend(struct vnode *vp, struct uio *uio,
write_flags = !pages && (
(ioflag & IO_DIRECT) ||
!fsess_opt_datacache(vnode_mount(vp)) ||
fuse_data_cache_mode != FUSE_CACHE_WB) ? 0 : FUSE_WRITE_CACHE;
!fsess_opt_writeback(vnode_mount(vp))) ? 0 : FUSE_WRITE_CACHE;
if (uio->uio_resid == 0)
return (0);

27
sys/fs/fuse/fuse_ipc.h
View File

@ -63,6 +63,12 @@
#include <sys/param.h>
#include <sys/refcount.h>
enum fuse_data_cache_mode {
FUSE_CACHE_UC,
FUSE_CACHE_WT,
FUSE_CACHE_WB,
};
struct fuse_iov {
void *base;
size_t len;
@ -209,6 +215,7 @@ struct fuse_data {
unsigned time_gran;
uint64_t notimpl;
uint64_t mnt_flag;
enum fuse_data_cache_mode cache_mode;
};
#define FSESS_DEAD 0x0001 /* session is to be closed */
@ -224,12 +231,6 @@ struct fuse_data {
FSESS_DAEMON_CAN_SPY | FSESS_PUSH_SYMLINKS_IN | \
FSESS_DEFAULT_PERMISSIONS)
enum fuse_data_cache_mode {
FUSE_CACHE_UC,
FUSE_CACHE_WT,
FUSE_CACHE_WB,
};
extern int fuse_data_cache_mode;
static inline struct fuse_data *
@ -257,13 +258,23 @@ fsess_set_notimpl(struct mount *mp, int opcode)
static inline bool
fsess_opt_datacache(struct mount *mp)
{
return (fuse_data_cache_mode != FUSE_CACHE_UC);
struct fuse_data *data = fuse_get_mpdata(mp);
return (data->cache_mode != FUSE_CACHE_UC);
}
static inline bool
fsess_opt_mmap(struct mount *mp)
{
return (fuse_data_cache_mode != FUSE_CACHE_UC);
return (fsess_opt_datacache(mp));
}
static inline bool
fsess_opt_writeback(struct mount *mp)
{
struct fuse_data *data = fuse_get_mpdata(mp);
return (data->cache_mode == FUSE_CACHE_WB);
}
/* Insert a new upgoing message */

10
sys/fs/fuse/fuse_node.c
View File

@ -108,6 +108,16 @@ SYSCTL_INT(_vfs_fusefs, OID_AUTO, node_count, CTLFLAG_RD,
int fuse_data_cache_mode = FUSE_CACHE_WT;
/*
* DEPRECATED
* This sysctl is no longer needed as of fuse protocol 7.23. Individual
* servers can select the cache behavior they need for each mountpoint:
* - writethrough: the default
* - writeback: set FUSE_WRITEBACK_CACHE in fuse_init_out.flags
* - uncached: set FOPEN_DIRECT_IO for every file
* The sysctl is retained primarily for use by jails supporting older FUSE
* protocols. It may be removed entirely once FreeBSD 11.3 and 12.0 are EOL.
*/
SYSCTL_PROC(_vfs_fusefs, OID_AUTO, data_cache_mode, CTLTYPE_INT|CTLFLAG_RW,
&fuse_data_cache_mode, 0, sysctl_fuse_cache_mode, "I",
"Zero: disable caching of FUSE file data; One: write-through caching "

24
tests/sys/fs/fusefs/io.cc
View File

@ -77,7 +77,7 @@ static void compare(const void *tbuf, const void *controlbuf, off_t baseofs,
}
class Io: public FuseTest,
public WithParamInterface<tuple<uint32_t, uint32_t, bool>> {
public WithParamInterface<tuple<bool, uint32_t, bool, bool>> {
public:
int m_backing_fd, m_control_fd, m_test_fd;
off_t m_filesize;
@ -95,9 +95,12 @@ void SetUp()
FAIL() << strerror(errno);
srandom(22'9'1982); // Seed with my birthday
m_init_flags = get<0>(GetParam());
if (get<0>(GetParam()))
m_init_flags |= FUSE_ASYNC_READ;
m_maxwrite = get<1>(GetParam());
m_async = get<2>(GetParam());
if (get<2>(GetParam()))
m_init_flags |= FUSE_WRITEBACK_CACHE;
m_async = get<3>(GetParam());
FuseTest::SetUp();
if (IsSkipped())
@ -316,15 +319,6 @@ void do_write(ssize_t size, off_t offs)
class IoCacheable: public Io {
public:
virtual void SetUp() {
const char *node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
ASSERT_EQ(0, sysctlbyname(node, &val, &size, NULL, 0))
<< strerror(errno);
if (val == 0)
GTEST_SKIP() <<
"fusefs data caching must be enabled for this test";
Io::SetUp();
}
};
@ -492,11 +486,13 @@ TEST_P(Io, resize_a_valid_buffer_while_extending)
}
INSTANTIATE_TEST_CASE_P(Io, Io,
Combine(Values(0, FUSE_ASYNC_READ), /* m_init_flags */
Combine(Bool(), /* async read */
Values(0x1000, 0x10000, 0x20000), /* m_maxwrite */
Bool(), /* writeback cache */
Bool())); /* m_async */
INSTANTIATE_TEST_CASE_P(Io, IoCacheable,
Combine(Values(0, FUSE_ASYNC_READ), /* m_init_flags */
Combine(Bool(), /* async read */
Values(0x1000, 0x10000, 0x20000), /* m_maxwrite */
Bool(), /* writeback cache */
Bool())); /* m_async */

11
tests/sys/fs/fusefs/notify.cc
View File

@ -84,20 +84,11 @@ void expect_lookup(uint64_t parent, const char *relpath, uint64_t ino,
class NotifyWriteback: public Notify {
public:
virtual void SetUp() {
const char *node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
m_init_flags |= FUSE_WRITEBACK_CACHE;
m_async = true;
Notify::SetUp();
if (IsSkipped())
return;
ASSERT_EQ(0, sysctlbyname(node, &val, &size, NULL, 0))
<< strerror(errno);
if (val != 2)
GTEST_SKIP() << "vfs.fusefs.data_cache_mode must be set to 2 "
"(writeback) for this test";
}
void expect_write(uint64_t ino, uint64_t offset, uint64_t size,

47
tests/sys/fs/fusefs/read.cc
View File

@ -92,24 +92,7 @@ class AsyncRead: public AioRead {
}
};
class ReadCacheable: public Read {
public:
virtual void SetUp() {
const char *node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
FuseTest::SetUp();
ASSERT_EQ(0, sysctlbyname(node, &val, &size, NULL, 0))
<< strerror(errno);
if (val == 0)
GTEST_SKIP() <<
"fusefs data caching must be enabled for this test";
}
};
class ReadAhead: public ReadCacheable,
class ReadAhead: public Read,
public WithParamInterface<tuple<bool, int>>
{
virtual void SetUp() {
@ -121,7 +104,7 @@ class ReadAhead: public ReadCacheable,
m_maxreadahead = val * get<1>(GetParam());
m_noclusterr = get<0>(GetParam());
ReadCacheable::SetUp();
Read::SetUp();
}
};
@ -357,6 +340,12 @@ TEST_F(Read, direct_io_pread)
fd = open(FULLPATH, O_RDONLY);
ASSERT_LE(0, fd) << strerror(errno);
ASSERT_EQ(bufsize, pread(fd, buf, bufsize, offset)) << strerror(errno);
ASSERT_EQ(0, memcmp(buf, CONTENTS, bufsize));
// With FOPEN_DIRECT_IO, the cache should be bypassed. The server will
// get a 2nd read request.
expect_read(ino, offset, bufsize, bufsize, CONTENTS);
ASSERT_EQ(bufsize, pread(fd, buf, bufsize, offset)) << strerror(errno);
ASSERT_EQ(0, memcmp(buf, CONTENTS, bufsize));
/* Deliberately leak fd. close(2) will be tested in release.cc */
@ -423,7 +412,7 @@ TEST_F(Read, eio)
* indicates EOF, because of a server-side truncation. We should invalidate
* all cached attributes. We may update the file size,
*/
TEST_F(ReadCacheable, eof)
TEST_F(Read, eof)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -453,8 +442,8 @@ TEST_F(ReadCacheable, eof)
/* Deliberately leak fd. close(2) will be tested in release.cc */
}
/* Like ReadCacheable.eof, but causes an entire buffer to be invalidated */
TEST_F(ReadCacheable, eof_of_whole_buffer)
/* Like Read.eof, but causes an entire buffer to be invalidated */
TEST_F(Read, eof_of_whole_buffer)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -490,7 +479,7 @@ TEST_F(ReadCacheable, eof_of_whole_buffer)
* With the keep_cache option, the kernel may keep its read cache across
* multiple open(2)s.
*/
TEST_F(ReadCacheable, keep_cache)
TEST_F(Read, keep_cache)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -556,7 +545,7 @@ TEST_F(Read, keep_cache_disabled)
/* Deliberately leak fd0 and fd1. */
}
TEST_F(ReadCacheable, mmap)
TEST_F(Read, mmap)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -602,7 +591,7 @@ TEST_F(ReadCacheable, mmap)
* A read via mmap comes up short, indicating that the file was truncated
* server-side.
*/
TEST_F(ReadCacheable, mmap_eof)
TEST_F(Read, mmap_eof)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -680,7 +669,7 @@ TEST_F(Read, o_direct)
ASSERT_EQ(0, lseek(fd, 0, SEEK_SET)) << strerror(errno);
ASSERT_EQ(bufsize, read(fd, buf, bufsize)) << strerror(errno);
ASSERT_EQ(0, memcmp(buf, CONTENTS, bufsize));
/* Deliberately leak fd. close(2) will be tested in release.cc */
}
@ -761,7 +750,7 @@ TEST_F(Read_7_8, read)
* If cacheing is enabled, the kernel should try to read an entire cache block
* at a time.
*/
TEST_F(ReadCacheable, cache_block)
TEST_F(Read, cache_block)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -796,7 +785,7 @@ TEST_F(ReadCacheable, cache_block)
}
/* Reading with sendfile should work (though it obviously won't be 0-copy) */
TEST_F(ReadCacheable, sendfile)
TEST_F(Read, sendfile)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -843,7 +832,7 @@ TEST_F(ReadCacheable, sendfile)
/* sendfile should fail gracefully if fuse declines the read */
/* https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=236466 */
TEST_F(ReadCacheable, DISABLED_sendfile_eio)
TEST_F(Read, DISABLED_sendfile_eio)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";

11
tests/sys/fs/fusefs/setattr.cc
View File

@ -415,8 +415,7 @@ TEST_F(Setattr, truncate) {
/*
* Truncating a file should discard cached data past the truncation point.
* This is a regression test for bug 233783. The bug only applies when
* vfs.fusefs.data_cache_mode=1 or 2, but the test should pass regardless.
* This is a regression test for bug 233783.
*
* There are two distinct failure modes. The first one is a failure to zero
* the portion of the file's final buffer past EOF. It can be reproduced by
@ -476,11 +475,6 @@ TEST_F(Setattr, truncate_discards_cached_data) {
out.body.attr.attr.mode = mode;
out.body.attr.attr.size = cur_size;
})));
/*
* The exact pattern of FUSE_WRITE operations depends on the setting of
* vfs.fusefs.data_cache_mode. But it's not important for this test.
* Just set the mocks to accept anything
*/
EXPECT_CALL(*m_mock, process(
ResultOf([=](auto in) {
return (in.header.opcode == FUSE_WRITE);
@ -510,7 +504,6 @@ TEST_F(Setattr, truncate_discards_cached_data) {
cur_size = trunc_size;
})));
/* exact pattern of FUSE_READ depends on vfs.fusefs.data_cache_mode */
EXPECT_CALL(*m_mock, process(
ResultOf([=](auto in) {
return (in.header.opcode == FUSE_READ);
@ -534,7 +527,7 @@ TEST_F(Setattr, truncate_discards_cached_data) {
ASSERT_EQ(static_cast<ssize_t>(w0_size),
pwrite(fd, w0buf, w0_size, w0_offset));
should_have_data = true;
/* Fill the cache, if data_cache_mode == 1 */
/* Fill the cache */
ASSERT_EQ(static_cast<ssize_t>(r0_size),
pread(fd, r0buf, r0_size, r0_offset));
/* 1st truncate should discard cached data */

69
tests/sys/fs/fusefs/write.cc
View File

@ -117,23 +117,6 @@ void maybe_expect_write(uint64_t ino, uint64_t offset, uint64_t size,
};
class WriteCacheable: public Write {
public:
virtual void SetUp() {
const char *node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
FuseTest::SetUp();
ASSERT_EQ(0, sysctlbyname(node, &val, &size, NULL, 0))
<< strerror(errno);
if (val == 0)
GTEST_SKIP() <<
"fusefs data caching must be enabled for this test";
}
};
sig_atomic_t Write::s_sigxfsz = 0;
class Write_7_8: public FuseTest {
@ -167,50 +150,14 @@ virtual void SetUp() {
}
};
/* Tests for the write-through cache mode */
class WriteThrough: public Write {
public:
virtual void SetUp() {
const char *cache_mode_node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
FuseTest::SetUp();
if (IsSkipped())
return;
ASSERT_EQ(0, sysctlbyname(cache_mode_node, &val, &size, NULL, 0))
<< strerror(errno);
if (val != 1)
GTEST_SKIP() << "vfs.fusefs.data_cache_mode must be set to 1 "
"(writethrough) for this test";
}
void expect_write(uint64_t ino, uint64_t offset, uint64_t isize,
uint64_t osize, const void *contents)
{
FuseTest::expect_write(ino, offset, isize, osize, 0, FUSE_WRITE_CACHE,
contents);
}
};
/* Tests for the writeback cache mode */
class WriteBack: public Write {
public:
virtual void SetUp() {
const char *node = "vfs.fusefs.data_cache_mode";
int val = 0;
size_t size = sizeof(val);
m_init_flags |= FUSE_WRITEBACK_CACHE;
FuseTest::SetUp();
if (IsSkipped())
return;
ASSERT_EQ(0, sysctlbyname(node, &val, &size, NULL, 0))
<< strerror(errno);
if (val != 2)
GTEST_SKIP() << "vfs.fusefs.data_cache_mode must be set to 2 "
"(writeback) for this test";
}
void expect_write(uint64_t ino, uint64_t offset, uint64_t isize,
@ -241,12 +188,12 @@ virtual void SetUp() {
class WriteCluster: public WriteBack {
public:
virtual void SetUp() {
if (m_maxphys < 2 * DFLTPHYS)
GTEST_SKIP() << "MAXPHYS must be at least twice DFLTPHYS"
<< " for this test";
m_async = true;
m_maxwrite = m_maxphys;
WriteBack::SetUp();
if (m_maxphys < 2 * DFLTPHYS)
GTEST_SKIP() << "MAXPHYS must be at least twice DFLTPHYS"
<< " for this test";
if (m_maxphys < 2 * m_maxbcachebuf)
GTEST_SKIP() << "MAXPHYS must be at least twice maxbcachebuf"
<< " for this test";
@ -543,7 +490,7 @@ TEST_F(Write, rlimit_fsize)
* A short read indicates EOF. Test that nothing bad happens if we get EOF
* during the R of a RMW operation.
*/
TEST_F(WriteCacheable, eof_during_rmw)
TEST_F(Write, eof_during_rmw)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -577,7 +524,7 @@ TEST_F(WriteCacheable, eof_during_rmw)
* write, then it must set the FUSE_WRITE_CACHE bit
*/
/* https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=236378 */
TEST_F(WriteCacheable, mmap)
TEST_F(Write, mmap)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -624,7 +571,7 @@ TEST_F(WriteCacheable, mmap)
free(zeros);
}
TEST_F(WriteThrough, pwrite)
TEST_F(Write, pwrite)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";
@ -1183,7 +1130,7 @@ INSTANTIATE_TEST_CASE_P(RA, TimeGran, Range(0u, 10u));
/*
* Without direct_io, writes should be committed to cache
*/
TEST_F(WriteThrough, writethrough)
TEST_F(Write, writethrough)
{
const char FULLPATH[] = "mountpoint/some_file.txt";
const char RELPATH[] = "some_file.txt";