7c26a038f9
do not map the b_pages pages into buffer_map KVA. The use of the unmapped buffers eliminate the need to perform TLB shootdown for mapping on the buffer creation and reuse, greatly reducing the amount of IPIs for shootdown on big-SMP machines and eliminating up to 25-30% of the system time on i/o intensive workloads. The unmapped buffer should be explicitely requested by the GB_UNMAPPED flag by the consumer. For unmapped buffer, no KVA reservation is performed at all. The consumer might request unmapped buffer which does have a KVA reserve, to manually map it without recursing into buffer cache and blocking, with the GB_KVAALLOC flag. When the mapped buffer is requested and unmapped buffer already exists, the cache performs an upgrade, possibly reusing the KVA reservation. Unmapped buffer is translated into unmapped bio in g_vfs_strategy(). Unmapped bio carry a pointer to the vm_page_t array, offset and length instead of the data pointer. The provider which processes the bio should explicitely specify a readiness to accept unmapped bio, otherwise g_down geom thread performs the transient upgrade of the bio request by mapping the pages into the new bio_transient_map KVA submap. The bio_transient_map submap claims up to 10% of the buffer map, and the total buffer_map + bio_transient_map KVA usage stays the same. Still, it could be manually tuned by kern.bio_transient_maxcnt tunable, in the units of the transient mappings. Eventually, the bio_transient_map could be removed after all geom classes and drivers can accept unmapped i/o requests. Unmapped support can be turned off by the vfs.unmapped_buf_allowed tunable, disabling which makes the buffer (or cluster) creation requests to ignore GB_UNMAPPED and GB_KVAALLOC flags. Unmapped buffers are only enabled by default on the architectures where pmap_copy_page() was implemented and tested. In the rework, filesystem metadata is not the subject to maxbufspace limit anymore. Since the metadata buffers are always mapped, the buffers still have to fit into the buffer map, which provides a reasonable (but practically unreachable) upper bound on it. The non-metadata buffer allocations, both mapped and unmapped, is accounted against maxbufspace, as before. Effectively, this means that the maxbufspace is forced on mapped and unmapped buffers separately. The pre-patch bufspace limiting code did not worked, because buffer_map fragmentation does not allow the limit to be reached. By Jeff Roberson request, the getnewbuf() function was split into smaller single-purpose functions. Sponsored by: The FreeBSD Foundation Discussed with: jeff (previous version) Tested by: pho, scottl (previous version), jhb, bf MFC after: 2 weeks
288 lines
6.7 KiB
C
288 lines
6.7 KiB
C
/*-
|
|
* Copyright (c) 2004 Poul-Henning Kamp
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/bio.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/vnode.h>
|
|
#include <sys/mount.h>
|
|
|
|
#include <geom/geom.h>
|
|
#include <geom/geom_vfs.h>
|
|
|
|
/*
|
|
* subroutines for use by filesystems.
|
|
*
|
|
* XXX: should maybe live somewhere else ?
|
|
*/
|
|
#include <sys/buf.h>
|
|
|
|
struct g_vfs_softc {
|
|
struct mtx sc_mtx;
|
|
struct bufobj *sc_bo;
|
|
int sc_active;
|
|
int sc_orphaned;
|
|
};
|
|
|
|
static struct buf_ops __g_vfs_bufops = {
|
|
.bop_name = "GEOM_VFS",
|
|
.bop_write = bufwrite,
|
|
.bop_strategy = g_vfs_strategy,
|
|
.bop_sync = bufsync,
|
|
.bop_bdflush = bufbdflush
|
|
};
|
|
|
|
struct buf_ops *g_vfs_bufops = &__g_vfs_bufops;
|
|
|
|
static g_orphan_t g_vfs_orphan;
|
|
|
|
static struct g_class g_vfs_class = {
|
|
.name = "VFS",
|
|
.version = G_VERSION,
|
|
.orphan = g_vfs_orphan,
|
|
};
|
|
|
|
DECLARE_GEOM_CLASS(g_vfs_class, g_vfs);
|
|
|
|
static void
|
|
g_vfs_destroy(void *arg, int flags __unused)
|
|
{
|
|
struct g_consumer *cp;
|
|
|
|
g_topology_assert();
|
|
cp = arg;
|
|
if (cp->acr > 0 || cp->acw > 0 || cp->ace > 0)
|
|
g_access(cp, -cp->acr, -cp->acw, -cp->ace);
|
|
g_detach(cp);
|
|
if (cp->geom->softc == NULL)
|
|
g_wither_geom(cp->geom, ENXIO);
|
|
}
|
|
|
|
static void
|
|
g_vfs_done(struct bio *bip)
|
|
{
|
|
struct g_consumer *cp;
|
|
struct g_vfs_softc *sc;
|
|
struct buf *bp;
|
|
int destroy;
|
|
struct mount *mp;
|
|
struct vnode *vp;
|
|
struct cdev *cdevp;
|
|
|
|
/*
|
|
* Collect statistics on synchronous and asynchronous read
|
|
* and write counts for disks that have associated filesystems.
|
|
* Since this run by the g_up thread it is single threaded and
|
|
* we do not need to use atomic increments on the counters.
|
|
*/
|
|
bp = bip->bio_caller2;
|
|
vp = bp->b_vp;
|
|
if (vp == NULL) {
|
|
mp = NULL;
|
|
} else {
|
|
/*
|
|
* If not a disk vnode, use its associated mount point
|
|
* otherwise use the mountpoint associated with the disk.
|
|
*/
|
|
VI_LOCK(vp);
|
|
if (vp->v_type != VCHR ||
|
|
(cdevp = vp->v_rdev) == NULL ||
|
|
cdevp->si_devsw == NULL ||
|
|
(cdevp->si_devsw->d_flags & D_DISK) == 0)
|
|
mp = vp->v_mount;
|
|
else
|
|
mp = cdevp->si_mountpt;
|
|
VI_UNLOCK(vp);
|
|
}
|
|
if (mp != NULL) {
|
|
if (bp->b_iocmd == BIO_WRITE) {
|
|
if (LK_HOLDER(bp->b_lock.lk_lock) == LK_KERNPROC)
|
|
mp->mnt_stat.f_asyncwrites++;
|
|
else
|
|
mp->mnt_stat.f_syncwrites++;
|
|
} else {
|
|
if (LK_HOLDER(bp->b_lock.lk_lock) == LK_KERNPROC)
|
|
mp->mnt_stat.f_asyncreads++;
|
|
else
|
|
mp->mnt_stat.f_syncreads++;
|
|
}
|
|
}
|
|
|
|
cp = bip->bio_from;
|
|
sc = cp->geom->softc;
|
|
if (bip->bio_error) {
|
|
printf("g_vfs_done():");
|
|
g_print_bio(bip);
|
|
printf("error = %d\n", bip->bio_error);
|
|
}
|
|
bp->b_error = bip->bio_error;
|
|
bp->b_ioflags = bip->bio_flags;
|
|
if (bip->bio_error)
|
|
bp->b_ioflags |= BIO_ERROR;
|
|
bp->b_resid = bp->b_bcount - bip->bio_completed;
|
|
g_destroy_bio(bip);
|
|
|
|
mtx_lock(&sc->sc_mtx);
|
|
destroy = ((--sc->sc_active) == 0 && sc->sc_orphaned);
|
|
mtx_unlock(&sc->sc_mtx);
|
|
if (destroy)
|
|
g_post_event(g_vfs_destroy, cp, M_WAITOK, NULL);
|
|
|
|
bufdone(bp);
|
|
}
|
|
|
|
void
|
|
g_vfs_strategy(struct bufobj *bo, struct buf *bp)
|
|
{
|
|
struct g_vfs_softc *sc;
|
|
struct g_consumer *cp;
|
|
struct bio *bip;
|
|
|
|
cp = bo->bo_private;
|
|
sc = cp->geom->softc;
|
|
|
|
/*
|
|
* If the provider has orphaned us, just return EXIO.
|
|
*/
|
|
mtx_lock(&sc->sc_mtx);
|
|
if (sc->sc_orphaned) {
|
|
mtx_unlock(&sc->sc_mtx);
|
|
bp->b_error = ENXIO;
|
|
bp->b_ioflags |= BIO_ERROR;
|
|
bufdone(bp);
|
|
return;
|
|
}
|
|
sc->sc_active++;
|
|
mtx_unlock(&sc->sc_mtx);
|
|
|
|
bip = g_alloc_bio();
|
|
bip->bio_cmd = bp->b_iocmd;
|
|
bip->bio_offset = bp->b_iooffset;
|
|
bip->bio_length = bp->b_bcount;
|
|
bdata2bio(bp, bip);
|
|
if ((bp->b_flags & B_BARRIER) != 0) {
|
|
bip->bio_flags |= BIO_ORDERED;
|
|
bp->b_flags &= ~B_BARRIER;
|
|
}
|
|
bip->bio_done = g_vfs_done;
|
|
bip->bio_caller2 = bp;
|
|
g_io_request(bip, cp);
|
|
}
|
|
|
|
static void
|
|
g_vfs_orphan(struct g_consumer *cp)
|
|
{
|
|
struct g_geom *gp;
|
|
struct g_vfs_softc *sc;
|
|
int destroy;
|
|
|
|
g_topology_assert();
|
|
|
|
gp = cp->geom;
|
|
g_trace(G_T_TOPOLOGY, "g_vfs_orphan(%p(%s))", cp, gp->name);
|
|
sc = gp->softc;
|
|
if (sc == NULL)
|
|
return;
|
|
mtx_lock(&sc->sc_mtx);
|
|
sc->sc_orphaned = 1;
|
|
destroy = (sc->sc_active == 0);
|
|
mtx_unlock(&sc->sc_mtx);
|
|
if (destroy)
|
|
g_vfs_destroy(cp, 0);
|
|
|
|
/*
|
|
* Do not destroy the geom. Filesystem will do that during unmount.
|
|
*/
|
|
}
|
|
|
|
int
|
|
g_vfs_open(struct vnode *vp, struct g_consumer **cpp, const char *fsname, int wr)
|
|
{
|
|
struct g_geom *gp;
|
|
struct g_provider *pp;
|
|
struct g_consumer *cp;
|
|
struct g_vfs_softc *sc;
|
|
struct bufobj *bo;
|
|
int error;
|
|
|
|
g_topology_assert();
|
|
|
|
*cpp = NULL;
|
|
bo = &vp->v_bufobj;
|
|
if (bo->bo_private != vp)
|
|
return (EBUSY);
|
|
|
|
pp = g_dev_getprovider(vp->v_rdev);
|
|
if (pp == NULL)
|
|
return (ENOENT);
|
|
gp = g_new_geomf(&g_vfs_class, "%s.%s", fsname, pp->name);
|
|
sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
|
|
mtx_init(&sc->sc_mtx, "g_vfs", NULL, MTX_DEF);
|
|
sc->sc_bo = bo;
|
|
gp->softc = sc;
|
|
cp = g_new_consumer(gp);
|
|
g_attach(cp, pp);
|
|
error = g_access(cp, 1, wr, wr);
|
|
if (error) {
|
|
g_wither_geom(gp, ENXIO);
|
|
return (error);
|
|
}
|
|
vnode_create_vobject(vp, pp->mediasize, curthread);
|
|
*cpp = cp;
|
|
cp->private = vp;
|
|
bo->bo_ops = g_vfs_bufops;
|
|
bo->bo_private = cp;
|
|
bo->bo_bsize = pp->sectorsize;
|
|
|
|
return (error);
|
|
}
|
|
|
|
void
|
|
g_vfs_close(struct g_consumer *cp)
|
|
{
|
|
struct g_geom *gp;
|
|
struct g_vfs_softc *sc;
|
|
|
|
g_topology_assert();
|
|
|
|
gp = cp->geom;
|
|
sc = gp->softc;
|
|
bufobj_invalbuf(sc->sc_bo, V_SAVE, 0, 0);
|
|
sc->sc_bo->bo_private = cp->private;
|
|
gp->softc = NULL;
|
|
mtx_destroy(&sc->sc_mtx);
|
|
if (!sc->sc_orphaned || cp->provider == NULL)
|
|
g_wither_geom_close(gp, ENXIO);
|
|
g_free(sc);
|
|
}
|