freebsd-nq/module/zfs/range_tree.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright (c) 2013, 2019 by Delphix. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/spa.h>
#include <sys/dmu.h>
#include <sys/dnode.h>
#include <sys/zio.h>
#include <sys/range_tree.h>

/*
 * Range trees are tree-based data structures that can be used to
 * track free space or generally any space allocation information.
 * A range tree keeps track of individual segments and automatically
 * provides facilities such as adjacent extent merging and extent
 * splitting in response to range add/remove requests.
 *
 * A range tree starts out completely empty, with no segments in it.
 * Adding an allocation via range_tree_add to the range tree can either:
 * 1) create a new extent
 * 2) extend an adjacent extent
 * 3) merge two adjacent extents
 * Conversely, removing an allocation via range_tree_remove can:
 * 1) completely remove an extent
 * 2) shorten an extent (if the allocation was near one of its ends)
 * 3) split an extent into two extents, in effect punching a hole
 *
 * A range tree is also capable of 'bridging' gaps when adding
 * allocations. This is useful for cases when close proximity of
 * allocations is an important detail that needs to be represented
 * in the range tree. See range_tree_set_gap(). The default behavior
 * is not to bridge gaps (i.e. the maximum allowed gap size is 0).
 *
 * In order to traverse a range tree, use either the range_tree_walk()
 * or range_tree_vacate() functions.
 *
 * To obtain more accurate information on individual segment
 * operations that the range tree performs "under the hood", you can
 * specify a set of callbacks by passing a range_tree_ops_t structure
 * to the range_tree_create function. Any callbacks that are non-NULL
 * are then called at the appropriate times.
 *
 * The range tree code also supports a special variant of range trees
 * that can bridge small gaps between segments. This kind of tree is used
 * by the dsl scanning code to group I/Os into mostly sequential chunks to
 * optimize disk performance. The code here attempts to do this with as
 * little memory and computational overhead as possible. One limitation of
 * this implementation is that segments of range trees with gaps can only
 * support removing complete segments.
 */

kmem_cache_t *range_seg_cache;

/* Generic ops for managing an AVL tree alongside a range tree */
struct range_tree_ops rt_avl_ops = {
	.rtop_create = rt_avl_create,
	.rtop_destroy = rt_avl_destroy,
	.rtop_add = rt_avl_add,
	.rtop_remove = rt_avl_remove,
	.rtop_vacate = rt_avl_vacate,
};

void
range_tree_init(void)
{
	ASSERT(range_seg_cache == NULL);
	range_seg_cache = kmem_cache_create("range_seg_cache",
	    sizeof (range_seg_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
}

void
range_tree_fini(void)
{
	kmem_cache_destroy(range_seg_cache);
	range_seg_cache = NULL;
}

void
range_tree_stat_verify(range_tree_t *rt)
{
	range_seg_t *rs;
	uint64_t hist[RANGE_TREE_HISTOGRAM_SIZE] = { 0 };
	int i;

	for (rs = avl_first(&rt->rt_root); rs != NULL;
	    rs = AVL_NEXT(&rt->rt_root, rs)) {
		uint64_t size = rs->rs_end - rs->rs_start;
		int idx	= highbit64(size) - 1;

		hist[idx]++;
		ASSERT3U(hist[idx], !=, 0);
	}

	for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
		if (hist[i] != rt->rt_histogram[i]) {
			zfs_dbgmsg("i=%d, hist=%px, hist=%llu, rt_hist=%llu",
			    i, hist, hist[i], rt->rt_histogram[i]);
		}
		VERIFY3U(hist[i], ==, rt->rt_histogram[i]);
	}
}

static void
range_tree_stat_incr(range_tree_t *rt, range_seg_t *rs)
{
	uint64_t size = rs->rs_end - rs->rs_start;
	int idx = highbit64(size) - 1;

	ASSERT(size != 0);
	ASSERT3U(idx, <,
	    sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));

	rt->rt_histogram[idx]++;
	ASSERT3U(rt->rt_histogram[idx], !=, 0);
}

static void
range_tree_stat_decr(range_tree_t *rt, range_seg_t *rs)
{
	uint64_t size = rs->rs_end - rs->rs_start;
	int idx = highbit64(size) - 1;

	ASSERT(size != 0);
	ASSERT3U(idx, <,
	    sizeof (rt->rt_histogram) / sizeof (*rt->rt_histogram));

	ASSERT3U(rt->rt_histogram[idx], !=, 0);
	rt->rt_histogram[idx]--;
}

/*
 * NOTE: caller is responsible for all locking.
 */
static int
range_tree_seg_compare(const void *x1, const void *x2)
{
	const range_seg_t *r1 = (const range_seg_t *)x1;
	const range_seg_t *r2 = (const range_seg_t *)x2;

	ASSERT3U(r1->rs_start, <=, r1->rs_end);
	ASSERT3U(r2->rs_start, <=, r2->rs_end);

	return ((r1->rs_start >= r2->rs_end) - (r1->rs_end <= r2->rs_start));
}

range_tree_t *
range_tree_create_impl(range_tree_ops_t *ops, void *arg,
    int (*avl_compare) (const void *, const void *), uint64_t gap)
{
	range_tree_t *rt = kmem_zalloc(sizeof (range_tree_t), KM_SLEEP);

	avl_create(&rt->rt_root, range_tree_seg_compare,
	    sizeof (range_seg_t), offsetof(range_seg_t, rs_node));

	rt->rt_ops = ops;
	rt->rt_gap = gap;
	rt->rt_arg = arg;
	rt->rt_avl_compare = avl_compare;

	if (rt->rt_ops != NULL && rt->rt_ops->rtop_create != NULL)
		rt->rt_ops->rtop_create(rt, rt->rt_arg);

	return (rt);
}

range_tree_t *
range_tree_create(range_tree_ops_t *ops, void *arg)
{
	return (range_tree_create_impl(ops, arg, NULL, 0));
}

void
range_tree_destroy(range_tree_t *rt)
{
	VERIFY0(rt->rt_space);

	if (rt->rt_ops != NULL && rt->rt_ops->rtop_destroy != NULL)
		rt->rt_ops->rtop_destroy(rt, rt->rt_arg);

	avl_destroy(&rt->rt_root);
	kmem_free(rt, sizeof (*rt));
}

void
range_tree_adjust_fill(range_tree_t *rt, range_seg_t *rs, int64_t delta)
{
	ASSERT3U(rs->rs_fill + delta, !=, 0);
	ASSERT3U(rs->rs_fill + delta, <=, rs->rs_end - rs->rs_start);

	if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
		rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);
	rs->rs_fill += delta;
	if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
		rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
}

static void
range_tree_add_impl(void *arg, uint64_t start, uint64_t size, uint64_t fill)
{
	range_tree_t *rt = arg;
	avl_index_t where;
	range_seg_t rsearch, *rs_before, *rs_after, *rs;
	uint64_t end = start + size, gap = rt->rt_gap;
	uint64_t bridge_size = 0;
	boolean_t merge_before, merge_after;

	ASSERT3U(size, !=, 0);
	ASSERT3U(fill, <=, size);

	rsearch.rs_start = start;
	rsearch.rs_end = end;
	rs = avl_find(&rt->rt_root, &rsearch, &where);

	if (gap == 0 && rs != NULL &&
	    rs->rs_start <= start && rs->rs_end >= end) {
		zfs_panic_recover("zfs: allocating allocated segment"
		    "(offset=%llu size=%llu) of (offset=%llu size=%llu)\n",
		    (longlong_t)start, (longlong_t)size,
		    (longlong_t)rs->rs_start,
		    (longlong_t)rs->rs_end - rs->rs_start);
		return;
	}

	/*
	 * If this is a gap-supporting range tree, it is possible that we
	 * are inserting into an existing segment. In this case simply
	 * bump the fill count and call the remove / add callbacks. If the
	 * new range will extend an existing segment, we remove the
	 * existing one, apply the new extent to it and re-insert it using
	 * the normal code paths.
	 */
	if (rs != NULL) {
		ASSERT3U(gap, !=, 0);
		if (rs->rs_start <= start && rs->rs_end >= end) {
			range_tree_adjust_fill(rt, rs, fill);
			return;
		}

		avl_remove(&rt->rt_root, rs);
		if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
			rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);

		range_tree_stat_decr(rt, rs);
		rt->rt_space -= rs->rs_end - rs->rs_start;

		fill += rs->rs_fill;
		start = MIN(start, rs->rs_start);
		end = MAX(end, rs->rs_end);
		size = end - start;

		range_tree_add_impl(rt, start, size, fill);

		kmem_cache_free(range_seg_cache, rs);
		return;
	}

	ASSERT3P(rs, ==, NULL);

	/*
	 * Determine whether or not we will have to merge with our neighbors.
	 * If gap != 0, we might need to merge with our neighbors even if we
	 * aren't directly touching.
	 */
	rs_before = avl_nearest(&rt->rt_root, where, AVL_BEFORE);
	rs_after = avl_nearest(&rt->rt_root, where, AVL_AFTER);

	merge_before = (rs_before != NULL && rs_before->rs_end >= start - gap);
	merge_after = (rs_after != NULL && rs_after->rs_start <= end + gap);

	if (merge_before && gap != 0)
		bridge_size += start - rs_before->rs_end;
	if (merge_after && gap != 0)
		bridge_size += rs_after->rs_start - end;

	if (merge_before && merge_after) {
		avl_remove(&rt->rt_root, rs_before);
		if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL) {
			rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);
			rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);
		}

		range_tree_stat_decr(rt, rs_before);
		range_tree_stat_decr(rt, rs_after);

		rs_after->rs_fill += rs_before->rs_fill + fill;
		rs_after->rs_start = rs_before->rs_start;
		kmem_cache_free(range_seg_cache, rs_before);
		rs = rs_after;
	} else if (merge_before) {
		if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
			rt->rt_ops->rtop_remove(rt, rs_before, rt->rt_arg);

		range_tree_stat_decr(rt, rs_before);

		rs_before->rs_fill += fill;
		rs_before->rs_end = end;
		rs = rs_before;
	} else if (merge_after) {
		if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
			rt->rt_ops->rtop_remove(rt, rs_after, rt->rt_arg);

		range_tree_stat_decr(rt, rs_after);

		rs_after->rs_fill += fill;
		rs_after->rs_start = start;
		rs = rs_after;
	} else {
		rs = kmem_cache_alloc(range_seg_cache, KM_SLEEP);

		rs->rs_fill = fill;
		rs->rs_start = start;
		rs->rs_end = end;
		avl_insert(&rt->rt_root, rs, where);
	}

	if (gap != 0)
		ASSERT3U(rs->rs_fill, <=, rs->rs_end - rs->rs_start);
	else
		ASSERT3U(rs->rs_fill, ==, rs->rs_end - rs->rs_start);

	if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
		rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);

	range_tree_stat_incr(rt, rs);
	rt->rt_space += size + bridge_size;
}

void
range_tree_add(void *arg, uint64_t start, uint64_t size)
{
	range_tree_add_impl(arg, start, size, size);
}

static void
range_tree_remove_impl(range_tree_t *rt, uint64_t start, uint64_t size,
    boolean_t do_fill)
{
	avl_index_t where;
	range_seg_t rsearch, *rs, *newseg;
	uint64_t end = start + size;
	boolean_t left_over, right_over;

	VERIFY3U(size, !=, 0);
	VERIFY3U(size, <=, rt->rt_space);

	rsearch.rs_start = start;
	rsearch.rs_end = end;
	rs = avl_find(&rt->rt_root, &rsearch, &where);

	/* Make sure we completely overlap with someone */
	if (rs == NULL) {
		zfs_panic_recover("zfs: freeing free segment "
		    "(offset=%llu size=%llu)",
		    (longlong_t)start, (longlong_t)size);
		return;
	}

	/*
	 * Range trees with gap support must only remove complete segments
	 * from the tree. This allows us to maintain accurate fill accounting
	 * and to ensure that bridged sections are not leaked. If we need to
	 * remove less than the full segment, we can only adjust the fill count.
	 */
	if (rt->rt_gap != 0) {
		if (do_fill) {
			if (rs->rs_fill == size) {
				start = rs->rs_start;
				end = rs->rs_end;
				size = end - start;
			} else {
				range_tree_adjust_fill(rt, rs, -size);
				return;
			}
		} else if (rs->rs_start != start || rs->rs_end != end) {
			zfs_panic_recover("zfs: freeing partial segment of "
			    "gap tree (offset=%llu size=%llu) of "
			    "(offset=%llu size=%llu)",
			    (longlong_t)start, (longlong_t)size,
			    (longlong_t)rs->rs_start,
			    (longlong_t)rs->rs_end - rs->rs_start);
			return;
		}
	}

	VERIFY3U(rs->rs_start, <=, start);
	VERIFY3U(rs->rs_end, >=, end);

	left_over = (rs->rs_start != start);
	right_over = (rs->rs_end != end);

	range_tree_stat_decr(rt, rs);

	if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
		rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);

	if (left_over && right_over) {
		newseg = kmem_cache_alloc(range_seg_cache, KM_SLEEP);
		newseg->rs_start = end;
		newseg->rs_end = rs->rs_end;
		newseg->rs_fill = newseg->rs_end - newseg->rs_start;
		range_tree_stat_incr(rt, newseg);

		rs->rs_end = start;

		avl_insert_here(&rt->rt_root, newseg, rs, AVL_AFTER);
		if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
			rt->rt_ops->rtop_add(rt, newseg, rt->rt_arg);
	} else if (left_over) {
		rs->rs_end = start;
	} else if (right_over) {
		rs->rs_start = end;
	} else {
		avl_remove(&rt->rt_root, rs);
		kmem_cache_free(range_seg_cache, rs);
		rs = NULL;
	}

	if (rs != NULL) {
		/*
		 * The fill of the leftover segment will always be equal to
		 * the size, since we do not support removing partial segments
		 * of range trees with gaps.
		 */
		rs->rs_fill = rs->rs_end - rs->rs_start;
		range_tree_stat_incr(rt, rs);

		if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
			rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);
	}

	rt->rt_space -= size;
}

void
range_tree_remove(void *arg, uint64_t start, uint64_t size)
{
	range_tree_remove_impl(arg, start, size, B_FALSE);
}

void
range_tree_remove_fill(range_tree_t *rt, uint64_t start, uint64_t size)
{
	range_tree_remove_impl(rt, start, size, B_TRUE);
}

void
range_tree_resize_segment(range_tree_t *rt, range_seg_t *rs,
    uint64_t newstart, uint64_t newsize)
{
	int64_t delta = newsize - (rs->rs_end - rs->rs_start);

	range_tree_stat_decr(rt, rs);
	if (rt->rt_ops != NULL && rt->rt_ops->rtop_remove != NULL)
		rt->rt_ops->rtop_remove(rt, rs, rt->rt_arg);

	rs->rs_start = newstart;
	rs->rs_end = newstart + newsize;

	range_tree_stat_incr(rt, rs);
	if (rt->rt_ops != NULL && rt->rt_ops->rtop_add != NULL)
		rt->rt_ops->rtop_add(rt, rs, rt->rt_arg);

	rt->rt_space += delta;
}

static range_seg_t *
range_tree_find_impl(range_tree_t *rt, uint64_t start, uint64_t size)
{
	range_seg_t rsearch;
	uint64_t end = start + size;

	VERIFY(size != 0);

	rsearch.rs_start = start;
	rsearch.rs_end = end;
	return (avl_find(&rt->rt_root, &rsearch, NULL));
}

range_seg_t *
range_tree_find(range_tree_t *rt, uint64_t start, uint64_t size)
{
	range_seg_t *rs = range_tree_find_impl(rt, start, size);
	if (rs != NULL && rs->rs_start <= start && rs->rs_end >= start + size)
		return (rs);
	return (NULL);
}

void
range_tree_verify_not_present(range_tree_t *rt, uint64_t off, uint64_t size)
{
	range_seg_t *rs = range_tree_find(rt, off, size);
	if (rs != NULL)
		panic("segment already in tree; rs=%p", (void *)rs);
}

boolean_t
range_tree_contains(range_tree_t *rt, uint64_t start, uint64_t size)
{
	return (range_tree_find(rt, start, size) != NULL);
}

/*
 * Ensure that this range is not in the tree, regardless of whether
 * it is currently in the tree.
 */
void
range_tree_clear(range_tree_t *rt, uint64_t start, uint64_t size)
{
	range_seg_t *rs;

	if (size == 0)
		return;

	while ((rs = range_tree_find_impl(rt, start, size)) != NULL) {
		uint64_t free_start = MAX(rs->rs_start, start);
		uint64_t free_end = MIN(rs->rs_end, start + size);
		range_tree_remove(rt, free_start, free_end - free_start);
	}
}

void
range_tree_swap(range_tree_t **rtsrc, range_tree_t **rtdst)
{
	range_tree_t *rt;

	ASSERT0(range_tree_space(*rtdst));
	ASSERT0(avl_numnodes(&(*rtdst)->rt_root));

	rt = *rtsrc;
	*rtsrc = *rtdst;
	*rtdst = rt;
}

void
range_tree_vacate(range_tree_t *rt, range_tree_func_t *func, void *arg)
{
	range_seg_t *rs;
	void *cookie = NULL;

	if (rt->rt_ops != NULL && rt->rt_ops->rtop_vacate != NULL)
		rt->rt_ops->rtop_vacate(rt, rt->rt_arg);

	while ((rs = avl_destroy_nodes(&rt->rt_root, &cookie)) != NULL) {
		if (func != NULL)
			func(arg, rs->rs_start, rs->rs_end - rs->rs_start);
		kmem_cache_free(range_seg_cache, rs);
	}

	bzero(rt->rt_histogram, sizeof (rt->rt_histogram));
	rt->rt_space = 0;
}

void
range_tree_walk(range_tree_t *rt, range_tree_func_t *func, void *arg)
{
	for (range_seg_t *rs = avl_first(&rt->rt_root); rs;
	    rs = AVL_NEXT(&rt->rt_root, rs)) {
		func(arg, rs->rs_start, rs->rs_end - rs->rs_start);
	}
}

range_seg_t *
range_tree_first(range_tree_t *rt)
{
	return (avl_first(&rt->rt_root));
}

uint64_t
range_tree_space(range_tree_t *rt)
{
	return (rt->rt_space);
}

uint64_t
range_tree_numsegs(range_tree_t *rt)
{
	return ((rt == NULL) ? 0 : avl_numnodes(&rt->rt_root));
}

boolean_t
range_tree_is_empty(range_tree_t *rt)
{
	ASSERT(rt != NULL);
	return (range_tree_space(rt) == 0);
}

/* Generic range tree functions for maintaining segments in an AVL tree. */
void
rt_avl_create(range_tree_t *rt, void *arg)
{
	avl_tree_t *tree = arg;

	avl_create(tree, rt->rt_avl_compare, sizeof (range_seg_t),
	    offsetof(range_seg_t, rs_pp_node));
}

void
rt_avl_destroy(range_tree_t *rt, void *arg)
{
	avl_tree_t *tree = arg;

	ASSERT0(avl_numnodes(tree));
	avl_destroy(tree);
}

void
rt_avl_add(range_tree_t *rt, range_seg_t *rs, void *arg)
{
	avl_tree_t *tree = arg;
	avl_add(tree, rs);
}

void
rt_avl_remove(range_tree_t *rt, range_seg_t *rs, void *arg)
{
	avl_tree_t *tree = arg;
	avl_remove(tree, rs);
}

void
rt_avl_vacate(range_tree_t *rt, void *arg)
{
	/*
	 * Normally one would walk the tree freeing nodes along the way.
	 * Since the nodes are shared with the range trees we can avoid
	 * walking all nodes and just reinitialize the avl tree. The nodes
	 * will be freed by the range tree, so we don't want to free them here.
	 */
	rt_avl_create(rt, arg);
}

uint64_t
range_tree_min(range_tree_t *rt)
{
	range_seg_t *rs = avl_first(&rt->rt_root);
	return (rs != NULL ? rs->rs_start : 0);
}

uint64_t
range_tree_max(range_tree_t *rt)
{
	range_seg_t *rs = avl_last(&rt->rt_root);
	return (rs != NULL ? rs->rs_end : 0);
}

uint64_t
range_tree_span(range_tree_t *rt)
{
	return (range_tree_max(rt) - range_tree_min(rt));
}

/*
 * Remove any overlapping ranges between the given segment [start, end)
 * from removefrom. Add non-overlapping leftovers to addto.
 */
void
range_tree_remove_xor_add_segment(uint64_t start, uint64_t end,
    range_tree_t *removefrom, range_tree_t *addto)
{
	avl_index_t where;
	range_seg_t starting_rs = {
		.rs_start = start,
		.rs_end = start + 1
	};

	range_seg_t *curr = avl_find(&removefrom->rt_root,
	    &starting_rs, &where);

	if (curr == NULL)
		curr = avl_nearest(&removefrom->rt_root, where, AVL_AFTER);

	range_seg_t *next;
	for (; curr != NULL; curr = next) {
		next = AVL_NEXT(&removefrom->rt_root, curr);

		if (start == end)
			return;
		VERIFY3U(start, <, end);

		/* there is no overlap */
		if (end <= curr->rs_start) {
			range_tree_add(addto, start, end - start);
			return;
		}

		uint64_t overlap_start = MAX(curr->rs_start, start);
		uint64_t overlap_end = MIN(curr->rs_end, end);
		uint64_t overlap_size = overlap_end - overlap_start;
		ASSERT3S(overlap_size, >, 0);
		range_tree_remove(removefrom, overlap_start, overlap_size);

		if (start < overlap_start)
			range_tree_add(addto, start, overlap_start - start);

		start = overlap_end;
	}
	VERIFY3P(curr, ==, NULL);

	if (start != end) {
		VERIFY3U(start, <, end);
		range_tree_add(addto, start, end - start);
	} else {
		VERIFY3U(start, ==, end);
	}
}

/*
 * For each entry in rt, if it exists in removefrom, remove it
 * from removefrom. Otherwise, add it to addto.
 */
void
range_tree_remove_xor_add(range_tree_t *rt, range_tree_t *removefrom,
    range_tree_t *addto)
{
	for (range_seg_t *rs = avl_first(&rt->rt_root); rs;
	    rs = AVL_NEXT(&rt->rt_root, rs)) {
		range_tree_remove_xor_add_segment(rs->rs_start, rs->rs_end,
		    removefrom, addto);
	}
}