d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce zfs_refcount_(add|remove)_few().

Browse Source 
There are two places where we need to add/remove several references
with semantics of zfs_refcount_(add|remove). But when debug/tracing
is disabled, it is a crime to run multiple atomic_inc() in a loop,
especially under congested pool-wide allocator lock.

Introduced new functions implement the same semantics as the loop,
but without overhead in production builds.

Reviewed-by: Rich Ercolani <rincebrain@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by:	Alexander Motin <mav@FreeBSD.org>
Sponsored by:	iXsystems, Inc.
Closes #14934
This commit is contained in:
Alexander Motin 2023-06-05 14:51:44 -04:00 committed by GitHub
parent dae3c549f5
commit 5ba4025a8d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 30 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/sys/zfs_refcount.h
View File

@ -73,13 +73,15 @@ int64_t zfs_refcount_count(zfs_refcount_t *);
int64_t zfs_refcount_add(zfs_refcount_t *, const void *);
int64_t zfs_refcount_remove(zfs_refcount_t *, const void *);
/*
* Note that (add|remove)_many add/remove one reference with "number" N,
* _not_ make N references with "number" 1, which is what vanilla
* zfs_refcount_(add|remove) would do if called N times.
* Note that (add|remove)_many adds/removes one reference with "number" N,
* _not_ N references with "number" 1, which is what (add|remove)_few does,
* or what vanilla zfs_refcount_(add|remove) called N times would do.
*
* Attempting to remove a reference with number N when none exists is a
* panic on debug kernels with reference_tracking enabled.
*/
void zfs_refcount_add_few(zfs_refcount_t *, uint64_t, const void *);
void zfs_refcount_remove_few(zfs_refcount_t *, uint64_t, const void *);
int64_t zfs_refcount_add_many(zfs_refcount_t *, uint64_t, const void *);
int64_t zfs_refcount_remove_many(zfs_refcount_t *, uint64_t, const void *);
void zfs_refcount_transfer(zfs_refcount_t *, zfs_refcount_t *);
@ -108,6 +110,10 @@ typedef struct refcount {
#define zfs_refcount_count(rc) atomic_load_64(&(rc)->rc_count)
#define zfs_refcount_add(rc, holder) atomic_inc_64_nv(&(rc)->rc_count)
#define zfs_refcount_remove(rc, holder) atomic_dec_64_nv(&(rc)->rc_count)
#define zfs_refcount_add_few(rc, number, holder) \
atomic_add_64(&(rc)->rc_count, number)
#define zfs_refcount_remove_few(rc, number, holder) \
atomic_add_64(&(rc)->rc_count, -number)
#define zfs_refcount_add_many(rc, number, holder) \
atomic_add_64_nv(&(rc)->rc_count, number)
#define zfs_refcount_remove_many(rc, number, holder) \

3
module/zfs/dmu_zfetch.c
View File

@ -520,8 +520,7 @@ dmu_zfetch_run(zstream_t *zs, boolean_t missed, boolean_t have_lock)
issued = pf_end - pf_start + ipf_end - ipf_start;
if (issued > 1) {
/* More references on top of taken in dmu_zfetch_prepare(). */
for (int i = 0; i < issued - 1; i++)
zfs_refcount_add(&zs->zs_refs, NULL);
zfs_refcount_add_few(&zs->zs_refs, issued - 1, NULL);
} else if (issued == 0) {
/* Some other thread has done our work, so drop the ref. */
if (zfs_refcount_remove(&zs->zs_refs, NULL) == 0)

6
module/zfs/metaslab.c
View File

@ -5650,8 +5650,7 @@ metaslab_class_throttle_reserve(metaslab_class_t *mc, int slots, int allocator,
* We reserve the slots individually so that we can unreserve
* them individually when an I/O completes.
*/
for (int d = 0; d < slots; d++)
zfs_refcount_add(&mca->mca_alloc_slots, zio);
zfs_refcount_add_few(&mca->mca_alloc_slots, slots, zio);
zio->io_flags |= ZIO_FLAG_IO_ALLOCATING;
return (B_TRUE);
}
@ -5665,8 +5664,7 @@ metaslab_class_throttle_unreserve(metaslab_class_t *mc, int slots,
metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
ASSERT(mc->mc_alloc_throttle_enabled);
for (int d = 0; d < slots; d++)
zfs_refcount_remove(&mca->mca_alloc_slots, zio);
zfs_refcount_remove_few(&mca->mca_alloc_slots, slots, zio);
}
static int

18
module/zfs/refcount.c
View File

@ -151,6 +151,15 @@ zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
return (zfs_refcount_add_many(rc, 1, holder));
}
void
zfs_refcount_add_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
{
if (!rc->rc_tracked)
(void) zfs_refcount_add_many(rc, number, holder);
else for (; number > 0; number--)
(void) zfs_refcount_add(rc, holder);
}
int64_t
zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
const void *holder)
@ -204,6 +213,15 @@ zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
return (zfs_refcount_remove_many(rc, 1, holder));
}
void
zfs_refcount_remove_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
{
if (!rc->rc_tracked)
(void) zfs_refcount_remove_many(rc, number, holder);
else for (; number > 0; number--)
(void) zfs_refcount_remove(rc, holder);
}
void
zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
{