From 204cf69651dea0189c019d97212fe735362c7e90 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Sun, 31 Mar 2013 15:21:46 +0000 Subject: [PATCH 1/4] Implement rbtree(9) specialization. New optional macros RB_TREE_DECLARE_SPECIALIZATION(static, frobnitz_tree); RB_TREE_DEFINE_SPECIALIZATION(static, frobnitz_tree, compare_frobnitz_key, compare_frobnitz_nodes); static const rb_tree_ops_t frobnitz_tree_ops = { ... RB_TREE_OPS_SPECIALIZATION(frobnitz_tree), }; can be used to open-code compare_frobnitz_key and compare_frobnitz_nodes in the tree lookup loops, to avoid the overhead of a general function pointer call. Only rb_tree_find_node is specialized for now. Later it may be fruitful to add specializations for rb_tree_find_node_{geq,leq} and rb_tree_insert_node. --- common/lib/libc/gen/rb.c | 27 +++++---------- sys/sys/rbtree.h | 88 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/common/lib/libc/gen/rb.c b/common/lib/libc/gen/rb.c index 098ab6c..36a21f3 100644 --- a/common/lib/libc/gen/rb.c +++ b/common/lib/libc/gen/rb.c @@ -79,11 +79,6 @@ static bool rb_tree_check_node(const struct rb_tree *, const struct rb_node *, #define rb_tree_check_node(a, b, c, d) true #endif -#define RB_NODETOITEM(rbto, rbn) \ - ((void *)((uintptr_t)(rbn) - (rbto)->rbto_node_offset)) -#define RB_ITEMTONODE(rbto, rbn) \ - ((rb_node_t *)((uintptr_t)(rbn) + (rbto)->rbto_node_offset)) - #define RB_SENTINEL_NODE NULL void @@ -111,20 +106,14 @@ rb_tree_init(struct rb_tree *rbt, const rb_tree_ops_t *ops) void * rb_tree_find_node(struct rb_tree *rbt, const void *key) { - const rb_tree_ops_t *rbto = rbt->rbt_ops; - rbto_compare_key_fn compare_key = rbto->rbto_compare_key; - struct rb_node *parent = rbt->rbt_root; - - while (!RB_SENTINEL_P(parent)) { - void *pobj = RB_NODETOITEM(rbto, parent); - const signed int diff = (*compare_key)(rbto->rbto_context, - pobj, key); - if (diff == 0) - return pobj; - parent = parent->rb_nodes[diff < 0]; - } - - return NULL; + const rb_tree_ops_t *const rbto = rbt->rbt_ops; + const rbto_find_node_fn find_node = rbto->rbto_find_node; + + if (__predict_true(find_node != NULL)) + return (*find_node)(rbt, key); + else + return __rb_tree_find_node_specialized(rbt, key, + rbto->rbto_compare_key); } void * diff --git a/sys/sys/rbtree.h b/sys/sys/rbtree.h index a32390b..2362afc 100644 --- a/sys/sys/rbtree.h +++ b/sys/sys/rbtree.h @@ -125,6 +125,26 @@ TAILQ_HEAD(rb_node_qh, rb_node); #define RB_TAILQ_INSERT_AFTER(a, b, c, d) do { } while (/*CONSTCOND*/0) #endif /* RBDEBUG */ +typedef struct rb_tree_ops rb_tree_ops_t; + +typedef struct rb_tree { + struct rb_node *rbt_root; + const rb_tree_ops_t *rbt_ops; + struct rb_node *rbt_minmax[2]; +#ifdef RBDEBUG + struct rb_node_qh rbt_nodes; +#endif +#ifdef RBSTATS + unsigned int rbt_count; + unsigned int rbt_insertions; + unsigned int rbt_removals; + unsigned int rbt_insertion_rebalance_calls; + unsigned int rbt_insertion_rebalance_passes; + unsigned int rbt_removal_rebalance_calls; + unsigned int rbt_removal_rebalance_passes; +#endif +} rb_tree_t; + /* * rbto_compare_nodes_fn: * return a positive value if the first node > the second node. @@ -139,31 +159,29 @@ TAILQ_HEAD(rb_node_qh, rb_node); typedef signed int (*rbto_compare_nodes_fn)(void *, const void *, const void *); typedef signed int (*rbto_compare_key_fn)(void *, const void *, const void *); +typedef void *(*rbto_find_node_fn)(struct rb_tree *, const void *); +#if 0 +typedef bool (*rbto_find_insert_fn)(struct rb_tree *, void *, + struct rb_node **, unsigned int *); +#endif -typedef struct { +struct rb_tree_ops { rbto_compare_nodes_fn rbto_compare_nodes; rbto_compare_key_fn rbto_compare_key; size_t rbto_node_offset; void *rbto_context; -} rb_tree_ops_t; - -typedef struct rb_tree { - struct rb_node *rbt_root; - const rb_tree_ops_t *rbt_ops; - struct rb_node *rbt_minmax[2]; -#ifdef RBDEBUG - struct rb_node_qh rbt_nodes; + rbto_find_node_fn rbto_find_node; +#if 0 + rbto_find_node_fn rbto_find_node_geq; + rbto_find_node_fn rbto_find_node_leq; + rbto_find_insert_fn rbto_find_insert_node; #endif -#ifdef RBSTATS - unsigned int rbt_count; - unsigned int rbt_insertions; - unsigned int rbt_removals; - unsigned int rbt_insertion_rebalance_calls; - unsigned int rbt_insertion_rebalance_passes; - unsigned int rbt_removal_rebalance_calls; - unsigned int rbt_removal_rebalance_passes; -#endif -} rb_tree_t; +}; + +#define RB_NODETOITEM(rbto, rbn) \ + ((void *)((uintptr_t)(rbn) - (rbto)->rbto_node_offset)) +#define RB_ITEMTONODE(rbto, rbn) \ + ((rb_node_t *)((uintptr_t)(rbn) + (rbto)->rbto_node_offset)) #ifdef RBSTATS #define RBSTAT_INC(v) ((void)((v)++)) @@ -173,6 +191,38 @@ typedef struct rb_tree { #define RBSTAT_DEC(v) do { } while (/*CONSTCOND*/0) #endif +#define RB_TREE_DECLARE_SPECIALIZATION(decl, prefix) \ +decl void *__rb_##prefix##_find_node(struct rb_tree *, const void *) + +#define RB_TREE_OPS_SPECIALIZATION(prefix) \ + .rbto_find_node = __rb_##prefix##_find_node + +#define RB_TREE_DEFINE_SPECIALIZATION(decl, prefix, compare_key, compare_nodes) \ +decl void * \ +__rb_##prefix##_find_node(struct rb_tree *__rbt, const void *__key) \ +{ \ + return __rb_tree_find_node_specialized(__rbt, __key, compare_key); \ +} + +static __inline void * +__rb_tree_find_node_specialized(struct rb_tree *__rbt, const void *__key, + rbto_compare_key_fn __compare_key) +{ + const rb_tree_ops_t *const __rbto = __rbt->rbt_ops; + struct rb_node *__parent = __rbt->rbt_root; + + while (!RB_SENTINEL_P(__parent)) { + void *__pobj = RB_NODETOITEM(__rbto, __parent); + const signed int __diff = (*__compare_key)(__rbto->rbto_context, + __pobj, __key); + if (__diff == 0) + return __pobj; + __parent = __parent->rb_nodes[__diff < 0]; + } + + return NULL; +} + void rb_tree_init(rb_tree_t *, const rb_tree_ops_t *); void * rb_tree_insert_node(rb_tree_t *, void *); void * rb_tree_find_node(rb_tree_t *, const void *); -- 1.8.3.1 From 22f19f9ffa6d9c2032248516f897505a5a871aed Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Sun, 31 Mar 2013 15:46:25 +0000 Subject: [PATCH 2/4] Specialize uvm map trees. --- sys/uvm/uvm_map.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sys/uvm/uvm_map.c b/sys/uvm/uvm_map.c index 4f5cd7c..de89903 100644 --- a/sys/uvm/uvm_map.c +++ b/sys/uvm/uvm_map.c @@ -298,7 +298,7 @@ static vsize_t uvm_rb_maxgap(const struct vm_map_entry *); (ROOT_ENTRY(map) == (entry) \ ? NULL : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node)) -static int +static inline int uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey) { const struct vm_map_entry *eparent = nparent; @@ -314,7 +314,7 @@ uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey) return 0; } -static int +static inline int uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey) { const struct vm_map_entry *eparent = nparent; @@ -327,11 +327,17 @@ uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey) return 0; } +RB_TREE_DECLARE_SPECIALIZATION(static, uvm_map_tree); + +RB_TREE_DEFINE_SPECIALIZATION(static, uvm_map_tree, + uvm_map_compare_key, uvm_map_compare_nodes); + static const rb_tree_ops_t uvm_map_tree_ops = { .rbto_compare_nodes = uvm_map_compare_nodes, .rbto_compare_key = uvm_map_compare_key, .rbto_node_offset = offsetof(struct vm_map_entry, rb_node), - .rbto_context = NULL + .rbto_context = NULL, + RB_TREE_OPS_SPECIALIZATION(uvm_map_tree), }; /* -- 1.8.3.1 From b88e0aaa8bda9a17f47bb3a0f48cd9d614e5683a Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Sun, 31 Mar 2013 15:46:59 +0000 Subject: [PATCH 3/4] Specialize uvm page trees. --- sys/uvm/uvm_page.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c index 623e996..4da1d9f 100644 --- a/sys/uvm/uvm_page.c +++ b/sys/uvm/uvm_page.c @@ -152,7 +152,7 @@ static void uvm_pageremove(struct uvm_object *, struct vm_page *); * per-object tree of pages */ -static signed int +static inline signed int uvm_page_compare_nodes(void *ctx, const void *n1, const void *n2) { const struct vm_page *pg1 = n1; @@ -167,7 +167,7 @@ uvm_page_compare_nodes(void *ctx, const void *n1, const void *n2) return 0; } -static signed int +static inline signed int uvm_page_compare_key(void *ctx, const void *n, const void *key) { const struct vm_page *pg = n; @@ -181,11 +181,17 @@ uvm_page_compare_key(void *ctx, const void *n, const void *key) return 0; } +RB_TREE_DECLARE_SPECIALIZATION(static, uvm_page_tree); + +RB_TREE_DEFINE_SPECIALIZATION(static, uvm_page_tree, + uvm_page_compare_key, uvm_page_compare_nodes); + const rb_tree_ops_t uvm_page_tree_ops = { .rbto_compare_nodes = uvm_page_compare_nodes, .rbto_compare_key = uvm_page_compare_key, .rbto_node_offset = offsetof(struct vm_page, rb_node), - .rbto_context = NULL + .rbto_context = NULL, + RB_TREE_OPS_SPECIALIZATION(uvm_page_tree), }; /* -- 1.8.3.1 From aed586505385f7451dc87244d7cd566a03aa3055 Mon Sep 17 00:00:00 2001 From: Taylor R Campbell Date: Tue, 4 Jun 2013 00:37:24 +0000 Subject: [PATCH 4/4] WIP: Add support for specializing rb_tree_insert_node. --- common/lib/libc/gen/rb.c | 40 ++++++++++--------------------- sys/sys/rbtree.h | 62 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 68 insertions(+), 34 deletions(-) diff --git a/common/lib/libc/gen/rb.c b/common/lib/libc/gen/rb.c index 36a21f3..cb2868c 100644 --- a/common/lib/libc/gen/rb.c +++ b/common/lib/libc/gen/rb.c @@ -162,41 +162,25 @@ void * rb_tree_insert_node(struct rb_tree *rbt, void *object) { const rb_tree_ops_t *rbto = rbt->rbt_ops; - rbto_compare_nodes_fn compare_nodes = rbto->rbto_compare_nodes; + rbto_find_insert_node_fn find_insert_node = + rbto->rbto_find_insert_node; struct rb_node *parent, *tmp, *self = RB_ITEMTONODE(rbto, object); unsigned int position; - bool rebalance; + bool rebalance, found; RBSTAT_INC(rbt->rbt_insertions); - tmp = rbt->rbt_root; - /* - * This is a hack. Because rbt->rbt_root is just a struct rb_node *, - * just like rb_node->rb_nodes[RB_DIR_LEFT], we can use this fact to - * avoid a lot of tests for root and know that even at root, - * updating RB_FATHER(rb_node)->rb_nodes[RB_POSITION(rb_node)] will - * update rbt->rbt_root. - */ - parent = (struct rb_node *)(void *)&rbt->rbt_root; - position = RB_DIR_LEFT; - /* * Find out where to place this new leaf. */ - while (!RB_SENTINEL_P(tmp)) { - void *tobj = RB_NODETOITEM(rbto, tmp); - const signed int diff = (*compare_nodes)(rbto->rbto_context, - tobj, object); - if (__predict_false(diff == 0)) { - /* - * Node already exists; return it. - */ - return tobj; - } - parent = tmp; - position = (diff < 0); - tmp = parent->rb_nodes[position]; - } + if (__predict_true(find_insert_node != NULL)) + found = (*find_insert_node)(rbt, object, &tmp, &position); + else + found = __rb_tree_find_insert_node_specialized(rbt, object, + &tmp, &position, rbto->rbto_compare_nodes); + if (__predict_false(found)) + return RB_NODETOITEM(rbto, tmp); + parent = tmp; #ifdef RBDEBUG { @@ -204,7 +188,7 @@ rb_tree_insert_node(struct rb_tree *rbt, void *object) if (position == RB_DIR_RIGHT) prev = parent; - else if (tmp != rbt->rbt_root) + else if (parent->rb_nodes[position] != rbt->rbt_root) next = parent; /* diff --git a/sys/sys/rbtree.h b/sys/sys/rbtree.h index 2362afc..c9e4a1b 100644 --- a/sys/sys/rbtree.h +++ b/sys/sys/rbtree.h @@ -160,10 +160,8 @@ typedef struct rb_tree { typedef signed int (*rbto_compare_nodes_fn)(void *, const void *, const void *); typedef signed int (*rbto_compare_key_fn)(void *, const void *, const void *); typedef void *(*rbto_find_node_fn)(struct rb_tree *, const void *); -#if 0 -typedef bool (*rbto_find_insert_fn)(struct rb_tree *, void *, +typedef int (*rbto_find_insert_node_fn)(struct rb_tree *, const void *, struct rb_node **, unsigned int *); -#endif struct rb_tree_ops { rbto_compare_nodes_fn rbto_compare_nodes; @@ -174,8 +172,8 @@ struct rb_tree_ops { #if 0 rbto_find_node_fn rbto_find_node_geq; rbto_find_node_fn rbto_find_node_leq; - rbto_find_insert_fn rbto_find_insert_node; #endif + rbto_find_insert_node_fn rbto_find_insert_node; }; #define RB_NODETOITEM(rbto, rbn) \ @@ -192,16 +190,27 @@ struct rb_tree_ops { #endif #define RB_TREE_DECLARE_SPECIALIZATION(decl, prefix) \ -decl void *__rb_##prefix##_find_node(struct rb_tree *, const void *) +decl void *__rb_##prefix##_find_node(struct rb_tree *, const void *); \ +decl int __rb_##prefix##_find_insert_node(struct rb_tree *, const void *, \ + struct rb_node **, unsigned int *) #define RB_TREE_OPS_SPECIALIZATION(prefix) \ - .rbto_find_node = __rb_##prefix##_find_node + .rbto_find_node = __rb_##prefix##_find_node, \ + .rbto_find_insert_node = __rb_##prefix##_find_insert_node #define RB_TREE_DEFINE_SPECIALIZATION(decl, prefix, compare_key, compare_nodes) \ decl void * \ __rb_##prefix##_find_node(struct rb_tree *__rbt, const void *__key) \ { \ return __rb_tree_find_node_specialized(__rbt, __key, compare_key); \ +} \ + \ +decl int \ +__rb_##prefix##_find_insert_node(struct rb_tree *__rbt, const void *__object, \ + struct rb_node **__nodep, unsigned int *__positionp) \ +{ \ + return __rb_tree_find_insert_node_specialized(__rbt, __object, \ + __nodep, __positionp, compare_nodes); \ } static __inline void * @@ -223,6 +232,47 @@ __rb_tree_find_node_specialized(struct rb_tree *__rbt, const void *__key, return NULL; } +static __inline int +__rb_tree_find_insert_node_specialized(struct rb_tree *__rbt, + const void *__object, struct rb_node **__nodep, unsigned int *__positionp, + rbto_compare_nodes_fn __compare_nodes) +{ + const rb_tree_ops_t *const __rbto = __rbt->rbt_ops; + struct rb_node *__parent, *__tmp; + unsigned int __position; + + __tmp = __rbt->rbt_root; + /* + * This is a hack. Because rbt->rbt_root is just a struct rb_node *, + * just like rb_node->rb_nodes[RB_DIR_LEFT], we can use this fact to + * avoid a lot of tests for root and know that even at root, + * updating RB_FATHER(rb_node)->rb_nodes[RB_POSITION(rb_node)] will + * update rbt->rbt_root. + */ + __parent = (struct rb_node *)(void *)&__rbt->rbt_root; + __position = RB_DIR_LEFT; + + while (!RB_SENTINEL_P(__tmp)) { + void *__tobj = RB_NODETOITEM(__rbto, __tmp); + const signed int __diff = (*__compare_nodes)(__rbto->rbto_context, + __tobj, __object); + if (__predict_false(__diff == 0)) { + /* + * Node already exists; return it. + */ + *__nodep = __tmp; + return 1; + } + __parent = __tmp; + __position = (__diff < 0); + __tmp = __parent->rb_nodes[__position]; + } + + *__nodep = __parent; + *__positionp = __position; + return 0; +} + void rb_tree_init(rb_tree_t *, const rb_tree_ops_t *); void * rb_tree_insert_node(rb_tree_t *, void *); void * rb_tree_find_node(rb_tree_t *, const void *); -- 1.8.3.1