rb_tree: avoid extra reads in rebalancing
In RB_INSERT_COLOR and RB_REMOVE_COLOR, avoid reading a parent pointer from memory, and then reading the left-color bit from memory, and then reading the right-color bit from memory, since they're all in the same field. The compiler can't infer that only the first read is really necessary, so write the code in a way so that it doesn't have to. Drop RB_RED_LEFT and RB_RED_RIGHT macros that reach into memory to get those bits. Drop RB_COLOR, the only thing left using RB_RED_LEFT and RB_RED_RIGHT after the other changes, and go straight to DIAGNOSTIC code in subr_stats to implement RB_COLOR for its single, dubious use there. Reviewed by: alc MFC after: 3 weeks Differential Revision: https://reviews.freebsd.org/D36353
This commit is contained in:
parent
578a99c939
commit
5d91386826
@ -3406,6 +3406,13 @@ stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
|
||||
|
||||
Q_TOSTR(rbctd64->mu, -1, 10, qstr,
|
||||
sizeof(qstr));
|
||||
struct voistatdata_tdgstctd64 *parent;
|
||||
parent = RB_PARENT(rbctd64, rblnk);
|
||||
int rb_color =
|
||||
parent == NULL ? 0 :
|
||||
RB_LEFT(parent, rblnk) == rbctd64 ?
|
||||
(RB_BITS(parent, rblnk) & RB_RED_L) != 0 :
|
||||
(RB_BITS(parent, rblnk) & RB_RED_R) != 0;
|
||||
printf(" RB ctd=%3d p=%3d l=%3d r=%3d c=%2d "
|
||||
"mu=%s\n",
|
||||
(int)ARB_SELFIDX(ctd64tree, rbctd64),
|
||||
@ -3415,7 +3422,7 @@ stats_v1_vsd_tdgst_add(enum vsd_dtype vs_dtype, struct voistatdata_tdgst *tdgst,
|
||||
RB_LEFT(rbctd64, rblnk)),
|
||||
(int)ARB_SELFIDX(ctd64tree,
|
||||
RB_RIGHT(rbctd64, rblnk)),
|
||||
RB_COLOR(rbctd64, rblnk),
|
||||
rb_color,
|
||||
qstr);
|
||||
|
||||
panic("RB@%p and ARB@%p trees differ\n",
|
||||
|
@ -341,8 +341,6 @@ struct { \
|
||||
#define RB_FLIP_LEFT(elm, field) (RB_BITS(elm, field) ^= RB_RED_L)
|
||||
#define RB_FLIP_RIGHT(elm, field) (RB_BITS(elm, field) ^= RB_RED_R)
|
||||
#define RB_FLIP_ALL(elm, field) (RB_BITS(elm, field) ^= RB_RED_MASK)
|
||||
#define RB_RED_LEFT(elm, field) ((RB_BITS(elm, field) & RB_RED_L) != 0)
|
||||
#define RB_RED_RIGHT(elm, field) ((RB_BITS(elm, field) & RB_RED_R) != 0)
|
||||
#define _RB_PARENT_ONLY(elm) (__typeof(elm)) \
|
||||
((__uintptr_t)elm & ~RB_RED_MASK)
|
||||
#define RB_PARENT(elm, field) _RB_PARENT_ONLY(RB_UP(elm, field))
|
||||
@ -359,11 +357,6 @@ struct { \
|
||||
RB_LEFT(elm, field) = RB_RIGHT(elm, field) = NULL; \
|
||||
} while (/*CONSTCOND*/ 0)
|
||||
|
||||
#define RB_COLOR(elm, field) (RB_PARENT(elm, field) == NULL ? 0 : \
|
||||
RB_LEFT(RB_PARENT(elm, field), field) == elm ? \
|
||||
RB_RED_LEFT(RB_PARENT(elm, field), field) : \
|
||||
RB_RED_RIGHT(RB_PARENT(elm, field), field))
|
||||
|
||||
/*
|
||||
* Something to be invoked in a loop at the root of every modified subtree,
|
||||
* from the bottom up to the root, to update augmented node data.
|
||||
@ -489,67 +482,72 @@ name##_RB_INSERT_COLOR(struct name *head, struct type *elm) \
|
||||
* when a value has been assigned to 'child' in the previous \
|
||||
* one. \
|
||||
*/ \
|
||||
struct type *child, *gpar, *parent; \
|
||||
while ((parent = RB_PARENT(elm, field)) != NULL) { \
|
||||
gpar = RB_PARENT(parent, field); \
|
||||
struct type *child, *gpar = RB_UP(elm, field), *parent; \
|
||||
__uintptr_t red; \
|
||||
\
|
||||
while ((parent = gpar) != NULL) { \
|
||||
red = RB_BITS(parent, field); \
|
||||
gpar = RB_UP(parent, field); \
|
||||
if (RB_LEFT(parent, field) == elm) { \
|
||||
if (RB_RED_LEFT(parent, field)) { \
|
||||
if (red & RB_RED_L) { \
|
||||
RB_FLIP_LEFT(parent, field); \
|
||||
return; \
|
||||
} \
|
||||
RB_FLIP_RIGHT(parent, field); \
|
||||
if (RB_RED_RIGHT(parent, field)) { \
|
||||
if ((red & RB_RED_MASK) == 0) { \
|
||||
child = elm; \
|
||||
elm = parent; \
|
||||
continue; \
|
||||
} \
|
||||
if (RB_RED_RIGHT(elm, field)) \
|
||||
red = RB_BITS(elm, field); \
|
||||
if (red & RB_RED_R) \
|
||||
child = elm; \
|
||||
else { \
|
||||
/* coverity[uninit_use] */ \
|
||||
RB_ROTATE_LEFT(elm, child, field); \
|
||||
if (RB_RED_RIGHT(child, field)) \
|
||||
red = RB_BITS(child, field); \
|
||||
if (red & RB_RED_R) \
|
||||
RB_FLIP_LEFT(parent, field); \
|
||||
if (RB_RED_LEFT(child, field)) \
|
||||
if (red & RB_RED_L) \
|
||||
RB_FLIP_ALL(elm, field); \
|
||||
else \
|
||||
RB_FLIP_LEFT(elm, field); \
|
||||
if ((RB_BITS(child, field) & \
|
||||
RB_RED_MASK) == 0) \
|
||||
if ((red & RB_RED_MASK) == 0) \
|
||||
elm = child; \
|
||||
} \
|
||||
RB_ROTATE_RIGHT(parent, child, field); \
|
||||
} else { \
|
||||
if (RB_RED_RIGHT(parent, field)) { \
|
||||
if (red & RB_RED_R) { \
|
||||
RB_FLIP_RIGHT(parent, field); \
|
||||
return; \
|
||||
} \
|
||||
RB_FLIP_LEFT(parent, field); \
|
||||
if (RB_RED_LEFT(parent, field)) { \
|
||||
if ((red & RB_RED_MASK) == 0) { \
|
||||
child = elm; \
|
||||
elm = parent; \
|
||||
continue; \
|
||||
} \
|
||||
if (RB_RED_LEFT(elm, field)) \
|
||||
red = RB_BITS(elm, field); \
|
||||
if (red & RB_RED_L) \
|
||||
child = elm; \
|
||||
else { \
|
||||
/* coverity[uninit_use] */ \
|
||||
RB_ROTATE_RIGHT(elm, child, field); \
|
||||
if (RB_RED_LEFT(child, field)) \
|
||||
red = RB_BITS(child, field); \
|
||||
if (red & RB_RED_L) \
|
||||
RB_FLIP_RIGHT(parent, field); \
|
||||
if (RB_RED_RIGHT(child, field)) \
|
||||
if (red & RB_RED_R) \
|
||||
RB_FLIP_ALL(elm, field); \
|
||||
else \
|
||||
RB_FLIP_RIGHT(elm, field); \
|
||||
if ((RB_BITS(child, field) & \
|
||||
RB_RED_MASK) == 0) \
|
||||
if ((red & RB_RED_MASK) == 0) \
|
||||
elm = child; \
|
||||
} \
|
||||
RB_ROTATE_LEFT(parent, child, field); \
|
||||
} \
|
||||
RB_SET_PARENT(child, gpar, field); \
|
||||
gpar = _RB_PARENT_ONLY(gpar); \
|
||||
RB_UP(child, field) = gpar; \
|
||||
RB_SWAP_CHILD(head, gpar, parent, child, field); \
|
||||
RB_BITS(child, field) &= ~RB_RED_MASK; \
|
||||
if (elm != child) \
|
||||
RB_AUGMENT(elm); \
|
||||
RB_AUGMENT(parent); \
|
||||
@ -574,6 +572,8 @@ name##_RB_REMOVE_COLOR(struct name *head, \
|
||||
struct type *parent, struct type *elm) \
|
||||
{ \
|
||||
struct type *gpar, *sib; \
|
||||
__uintptr_t red; \
|
||||
\
|
||||
if (RB_LEFT(parent, field) == elm && \
|
||||
RB_RIGHT(parent, field) == elm) { \
|
||||
RB_BITS(parent, field) &= ~RB_RED_MASK; \
|
||||
@ -583,19 +583,21 @@ name##_RB_REMOVE_COLOR(struct name *head, \
|
||||
return; \
|
||||
} \
|
||||
do { \
|
||||
gpar = RB_PARENT(parent, field); \
|
||||
red = RB_BITS(parent, field); \
|
||||
gpar = RB_UP(parent, field); \
|
||||
if (RB_LEFT(parent, field) == elm) { \
|
||||
if (!RB_RED_LEFT(parent, field)) { \
|
||||
if (~red & RB_RED_L) { \
|
||||
RB_FLIP_LEFT(parent, field); \
|
||||
return; \
|
||||
} \
|
||||
if (RB_RED_RIGHT(parent, field)) { \
|
||||
if ((~red & RB_RED_MASK) == 0) { \
|
||||
RB_FLIP_RIGHT(parent, field); \
|
||||
elm = parent; \
|
||||
continue; \
|
||||
} \
|
||||
sib = RB_RIGHT(parent, field); \
|
||||
switch (RB_BITS(sib, field) & RB_RED_MASK) { \
|
||||
red = RB_BITS(sib, field); \
|
||||
switch (red & RB_RED_MASK) { \
|
||||
case RB_RED_MASK: \
|
||||
RB_FLIP_ALL(sib, field); \
|
||||
elm = parent; \
|
||||
@ -603,11 +605,12 @@ name##_RB_REMOVE_COLOR(struct name *head, \
|
||||
case RB_RED_R: \
|
||||
elm = RB_LEFT(sib, field); \
|
||||
RB_ROTATE_RIGHT(sib, elm, field); \
|
||||
if (RB_RED_LEFT(elm, field)) \
|
||||
red = RB_BITS(elm, field); \
|
||||
if (red & RB_RED_L) \
|
||||
RB_FLIP_ALL(parent, field); \
|
||||
else \
|
||||
RB_FLIP_LEFT(parent, field); \
|
||||
if (RB_RED_RIGHT(elm, field)) \
|
||||
if (red & RB_RED_R) \
|
||||
RB_FLIP_ALL(sib, field); \
|
||||
else \
|
||||
RB_FLIP_RIGHT(sib, field); \
|
||||
@ -629,17 +632,18 @@ name##_RB_REMOVE_COLOR(struct name *head, \
|
||||
} \
|
||||
RB_ROTATE_LEFT(parent, elm, field); \
|
||||
} else { \
|
||||
if (!RB_RED_RIGHT(parent, field)) { \
|
||||
if (~red & RB_RED_R) { \
|
||||
RB_FLIP_RIGHT(parent, field); \
|
||||
return; \
|
||||
} \
|
||||
if (RB_RED_LEFT(parent, field)) { \
|
||||
if ((~red & RB_RED_MASK) == 0) { \
|
||||
RB_FLIP_LEFT(parent, field); \
|
||||
elm = parent; \
|
||||
continue; \
|
||||
} \
|
||||
sib = RB_LEFT(parent, field); \
|
||||
switch (RB_BITS(sib, field) & RB_RED_MASK) { \
|
||||
red = RB_BITS(sib, field); \
|
||||
switch (red & RB_RED_MASK) { \
|
||||
case RB_RED_MASK: \
|
||||
RB_FLIP_ALL(sib, field); \
|
||||
elm = parent; \
|
||||
@ -647,11 +651,12 @@ name##_RB_REMOVE_COLOR(struct name *head, \
|
||||
case RB_RED_L: \
|
||||
elm = RB_RIGHT(sib, field); \
|
||||
RB_ROTATE_LEFT(sib, elm, field); \
|
||||
if (RB_RED_RIGHT(elm, field)) \
|
||||
red = RB_BITS(elm, field); \
|
||||
if (red & RB_RED_R) \
|
||||
RB_FLIP_ALL(parent, field); \
|
||||
else \
|
||||
RB_FLIP_RIGHT(parent, field); \
|
||||
if (RB_RED_LEFT(elm, field)) \
|
||||
if (red & RB_RED_L) \
|
||||
RB_FLIP_ALL(sib, field); \
|
||||
else \
|
||||
RB_FLIP_LEFT(sib, field); \
|
||||
@ -673,12 +678,13 @@ name##_RB_REMOVE_COLOR(struct name *head, \
|
||||
} \
|
||||
RB_ROTATE_RIGHT(parent, elm, field); \
|
||||
} \
|
||||
gpar = _RB_PARENT_ONLY(gpar); \
|
||||
RB_SET_PARENT(elm, gpar, field); \
|
||||
RB_SWAP_CHILD(head, gpar, parent, elm, field); \
|
||||
if (sib != elm) \
|
||||
RB_AUGMENT(sib); \
|
||||
break; \
|
||||
} while ((parent = RB_PARENT(elm, field)) != NULL); \
|
||||
} while ((parent = _RB_PARENT_ONLY(gpar)) != NULL); \
|
||||
}
|
||||
|
||||
#define RB_GENERATE_REMOVE(name, type, field, attr) \
|
||||
|
Loading…
Reference in New Issue
Block a user