d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

geli: use unmapped I/O

Browse Source 
Use unmapped I/O for geli. Unlike most geom providers, geli needs to
manipulate data on every read or write. Previously it would always map bios.

On my 16-core, dual socket server using geli atop md(4) devices, with 512B
sectors, this change increases geli IOPs by about 3x.

Note that geli still can't use unmapped I/O when data integrity verification
is enabled (but it could, with a little more work).  And it can't use
unmapped I/O in combination with ZFS, because ZFS uses mapped bios.

Reviewed by:	markj, kib, jhb, mjg, mat, bcr (manpages)
MFC after:	1 week
Sponsored by:	Axcient
Differential Revision:	https://reviews.freebsd.org/D25671
This commit is contained in:
Alan Somers 2020-08-26 02:44:35 +00:00
parent e6f6d0c9bc
commit 7d874f0f36
2 changed files with 68 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
sys/geom/eli/g_eli.c
View File

@ -49,6 +49,8 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <sys/vnode.h>
#include <machine/vmparam.h>
#include <vm/uma.h>
#include <geom/geom.h>
@ -972,6 +974,15 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
*/
pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
pp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
if (CRYPTO_HAS_VMPAGE) {
/*
* On DMAP architectures we can use unmapped I/O. But don't
* use it with data integrity verification. That code hasn't
* been written yet.
*/
if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0)
pp->flags |= G_PF_ACCEPT_UNMAPPED;
}
pp->mediasize = sc->sc_mediasize;
pp->sectorsize = sc->sc_sectorsize;
LIST_FOREACH(gap, &bpp->aliases, ga_next)

65
sys/geom/eli/g_eli_privacy.c
View File

@ -62,6 +62,28 @@ __FBSDID("$FreeBSD$");
MALLOC_DECLARE(M_ELI);
/*
* Copy data from a (potentially unmapped) bio to a kernelspace buffer.
*
* The buffer must have at least as much room as bp->bio_length.
*/
static void
g_eli_bio_copyin(struct bio *bp, void *kaddr)
{
struct uio uio;
struct iovec iov[1];
iov[0].iov_base = kaddr;
iov[0].iov_len = bp->bio_length;
uio.uio_iov = iov;
uio.uio_iovcnt = 1;
uio.uio_offset = 0;
uio.uio_resid = bp->bio_length;
uio.uio_segflg = UIO_SYSSPACE;
uio.uio_rw = UIO_READ;
uiomove_fromphys(bp->bio_ma, bp->bio_ma_offset, bp->bio_length, &uio);
}
/*
* The function is called after we read and decrypt data.
*
@ -98,8 +120,7 @@ g_eli_crypto_read_done(struct cryptop *crp)
*/
if (bp->bio_inbed < bp->bio_children)
return (0);
free(bp->bio_driver2, M_ELI);
bp->bio_driver2 = NULL;
if (bp->bio_error != 0) {
G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
bp->bio_error);
@ -167,6 +188,11 @@ g_eli_crypto_write_done(struct cryptop *crp)
return (0);
}
cbp->bio_data = bp->bio_driver2;
/*
* Clear BIO_UNMAPPED, which was inherited from where we cloned the bio
* in g_eli_start, because we manually set bio_data
*/
cbp->bio_flags &= ~BIO_UNMAPPED;
cbp->bio_done = g_eli_write_done;
cp = LIST_FIRST(&gp->consumer);
cbp->bio_to = cp->provider;
@ -236,10 +262,12 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
{
struct g_eli_softc *sc;
struct cryptop *crp;
vm_page_t *pages;
u_int i, nsec, secsize;
off_t dstoff;
u_char *data;
u_char *data = NULL;
int error;
int pages_offset;
G_ELI_LOGREQ(3, bp, "%s", __func__);
@ -258,16 +286,37 @@ g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
if (bp->bio_cmd == BIO_WRITE) {
data = malloc(bp->bio_length, M_ELI, M_WAITOK);
bp->bio_driver2 = data;
bcopy(bp->bio_data, data, bp->bio_length);
} else
data = bp->bio_data;
/*
* This copy could be eliminated by using crypto's output
* buffer, instead of using a single overwriting buffer.
*/
if ((bp->bio_flags & BIO_UNMAPPED) != 0)
g_eli_bio_copyin(bp, data);
else
bcopy(bp->bio_data, data, bp->bio_length);
} else {
if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
pages = bp->bio_ma;
pages_offset = bp->bio_ma_offset;
} else {
data = bp->bio_data;
}
}
for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
crp = crypto_getreq(wr->w_sid, M_WAITOK);
crypto_use_buf(crp, data, secsize);
if (data) {
crypto_use_buf(crp, data, secsize);
data += secsize;
} else {
MPASS(pages != NULL);
crypto_use_vmpage(crp, pages, secsize, pages_offset);
pages_offset += secsize;
pages += pages_offset >> PAGE_SHIFT;
pages_offset &= PAGE_MASK;
}
crp->crp_opaque = (void *)bp;
data += secsize;
if (bp->bio_cmd == BIO_WRITE) {
crp->crp_op = CRYPTO_OP_ENCRYPT;
crp->crp_callback = g_eli_crypto_write_done;