diff --git a/alloc.c b/alloc.c index 9d5d191..a9c9489 100644 --- a/alloc.c +++ b/alloc.c @@ -8,27 +8,17 @@ #include #include "topo.h" - -static pthread_mutex_t alloc_lock; -#define NM_MAX_OBJS_PER_LVL (256) -#define MEM_OBJ_SIZE (4096) // 4k -#define MEM_OBJ_NUM (1024 * 256) // 4k * 1024 * 256 = 1GB per region -#define MEM_REGION_NUM (4) // 4 x 1GB = 4GB total -static int nm_mem_idx[NM_MAX_OBJS_PER_LVL]; -static int nm_mem_region_idx[NM_MAX_OBJS_PER_LVL]; -static void* nm_mem_regions[NM_MAX_OBJS_PER_LVL][MEM_REGION_NUM]; - -struct topo_obj; +#include "topop.h" int -topo_alloc_init(int verbose, struct topo_obj * tobj) +topo_alloc_init(struct topo_desc * desc, int verbose) { long tid; thr_self(&tid); domainset_t orig_dom; int orig_policy; - pthread_mutex_init(&alloc_lock, NULL); + pthread_mutex_init(&desc->alloc_lock, NULL); DOMAINSET_ZERO(&orig_dom); @@ -39,40 +29,37 @@ topo_alloc_init(int verbose, struct topo_obj * tobj) } domainset_t tmp_domain; - for (int i = 0; i < topo_num_numa(tobj); i++) { + for (int i = 0; i < desc->num_numa; i++) { DOMAINSET_ZERO(&tmp_domain); DOMAINSET_SET(i, &tmp_domain); ret = cpuset_setdomain(CPU_LEVEL_WHICH, CPU_WHICH_TID, tid, sizeof(tmp_domain), &tmp_domain, DOMAINSET_POLICY_PREFER); if (ret != 0) { - if (verbose) { - fprintf(stderr, "libnm: cpuset_setdomain failed with %d\n", errno); - } + fprintf(stderr, "libnm: cpuset_setdomain failed with %d\n", errno); return ret; } - for (unsigned int j = 0; j < MEM_REGION_NUM; j++) { - if ((nm_mem_regions[i][j] = mmap(NULL, MEM_OBJ_NUM * MEM_OBJ_SIZE, PROT_READ | PROT_WRITE, - MAP_ANON | MAP_ALIGNED_SUPER | MAP_NOCORE | MAP_PRIVATE | MAP_NOSYNC, - -1, 0)) == MAP_FAILED) { - if (verbose) { - fprintf(stderr, "libnm: mmap failed with %d\n", errno); - } + for (unsigned int j = 0; j < ALLOC_MEM_REGION_NUM; j++) { + if ((desc->mem_regions[i][j] = mmap(NULL, ALLOC_MEM_OBJ_NUM * ALLOC_MEM_OBJ_SIZE, PROT_READ | PROT_WRITE, + MAP_ANON | MAP_ALIGNED_SUPER | MAP_NOCORE | MAP_PRIVATE | MAP_NOSYNC, -1, 0)) == MAP_FAILED) { + fprintf(stderr, "libnm: mmap failed with %d\n", errno); return -1; } // touch the pages to prefault the pages - for (unsigned int k = 0; k < MEM_OBJ_NUM; k++) { - *(uint32_t*)((char*)nm_mem_regions[i][j] + k * MEM_OBJ_SIZE) = 0; + for (unsigned int k = 0; k < ALLOC_MEM_OBJ_NUM; k++) { + *(uint32_t*)((char*)desc->mem_regions[i][j] + k * ALLOC_MEM_OBJ_SIZE) = 0; } if (verbose) { - fprintf(stdout, "libnm: reserved %u bytes (%u MB) on node %d. vaddr: 0x%p\n", MEM_OBJ_NUM * MEM_OBJ_SIZE, MEM_OBJ_SIZE * MEM_OBJ_NUM / 1024 / 1024, i, nm_mem_regions[i][j]); + fprintf(stdout, "libnm: reserved %u bytes (%u MB) on node %d. vaddr: 0x%p\n", ALLOC_MEM_OBJ_NUM * ALLOC_MEM_OBJ_SIZE, + ALLOC_MEM_OBJ_SIZE * ALLOC_MEM_OBJ_NUM / 1024 / 1024, + i, desc->mem_regions[i][j]); } } - nm_mem_idx[i] = 0; - nm_mem_region_idx[i] = 0; + desc->mem_idx[i] = 0; + desc->mem_region_idx[i] = 0; } // restore existing thread's allocation strategy @@ -82,35 +69,35 @@ topo_alloc_init(int verbose, struct topo_obj * tobj) } void * -topo_malloc(unsigned int node, size_t size) +topo_desc_malloc(struct topo_desc * desc, unsigned int node, size_t size) { void * ret = NULL; - int num_objs = (size + MEM_OBJ_SIZE - 1) / MEM_OBJ_SIZE; + int num_objs = (size + ALLOC_MEM_OBJ_SIZE - 1) / ALLOC_MEM_OBJ_SIZE; int retry = 0; - pthread_mutex_lock(&alloc_lock); - int cur_region = nm_mem_region_idx[node]; - int cur_idx = nm_mem_idx[node]; + pthread_mutex_lock(&desc->alloc_lock); + int cur_region = desc->mem_region_idx[node]; + int cur_idx = desc->mem_idx[node]; retry: - if ((int)MEM_OBJ_NUM - cur_idx >= num_objs) { - ret = (char*)nm_mem_regions[node][cur_region] + MEM_OBJ_SIZE * cur_idx; - nm_mem_region_idx[node] = cur_region; - nm_mem_idx[node] = cur_idx + num_objs; - } else if (!retry && (cur_region < (int)MEM_REGION_NUM)) { + if ((int)ALLOC_MEM_OBJ_NUM - cur_idx >= num_objs) { + ret = (char*)desc->mem_regions[node][cur_region] + ALLOC_MEM_OBJ_SIZE * cur_idx; + desc->mem_region_idx[node] = cur_region; + desc->mem_idx[node] = cur_idx + num_objs; + } else if (!retry && (cur_region < (int)ALLOC_MEM_REGION_NUM)) { // check next region cur_region++; cur_idx = 0; retry = 1; goto retry; } - pthread_mutex_unlock(&alloc_lock); + pthread_mutex_unlock(&desc->alloc_lock); return ret; } void -topo_free(unsigned int node __attribute__((unused)), void * addr __attribute__((unused))) +topo_desc_free(struct topo_desc * desc __attribute__((unused)), unsigned int node __attribute__((unused)), void * addr __attribute__((unused))) { // dummy function } \ No newline at end of file diff --git a/inc/topo.h b/inc/topo.h index 8a30018..e2b1e7e 100644 --- a/inc/topo.h +++ b/inc/topo.h @@ -7,6 +7,8 @@ extern "C" { #endif +struct topo_desc; + int topo_num_core(); @@ -20,7 +22,7 @@ void topo_destroy(); int -topo_init(int verbose, int alloc_init); +topo_init(int enable_alloc, int verbose); void * topo_malloc(unsigned int node, size_t size); diff --git a/test/test.c b/test/test.c index e35228b..610e4f7 100644 --- a/test/test.c +++ b/test/test.c @@ -96,7 +96,7 @@ int ts_test() int main() { topo_init(1, 1); - topo_init(1, 0); + topo_init(0, 1); ts_test(); return 0; } \ No newline at end of file diff --git a/timestamp.c b/timestamp.c index e7ecdc5..1c35948 100644 --- a/timestamp.c +++ b/timestamp.c @@ -5,30 +5,30 @@ #include #include "topo.h" - -static uint64_t sysctl_tsc_freq = 0; +#include "topop.h" #define S2NS (1000000000UL) static uint64_t -tsc2ns(uint64_t tsc) +tsc2ns(uint64_t tsc, uint64_t tsc_freq) { return (uint64_t)( - (double)tsc / (double)sysctl_tsc_freq * S2NS); + (double)tsc / (double)tsc_freq * S2NS); } int -topo_ts_init(int verbose) +topo_ts_init(struct topo_desc * desc, int verbose) { int rc; - size_t sz = sizeof(sysctl_tsc_freq); + + size_t sz = sizeof(desc->tsc_freq); // init nm_tsc2ns - if ((rc = sysctlbyname( - "machdep.tsc_freq", &sysctl_tsc_freq, &sz, NULL, 0)) < 0) { + if ((rc = sysctlbyname("machdep.tsc_freq", &desc->tsc_freq, &sz, NULL, 0)) < 0) { + fprintf(stderr,"libtopo: failed to query tsc frequency via sysctl (%d)\n", errno); + } else { if (verbose) { - fprintf(stderr, - "libnm: failed to query tsc frequency via sysctl (%d)\n", errno); + fprintf(stdout,"libtopo: tsc frequency = %lu\n", desc->tsc_freq); } } @@ -36,11 +36,11 @@ topo_ts_init(int verbose) } uint64_t -topo_uptime_ns() +topo_desc_uptime_ns(struct topo_desc * desc) { unsigned int dummy; _mm_lfence(); uint64_t tsc = __rdtscp(&dummy); _mm_lfence(); - return tsc2ns(tsc); + return tsc2ns(tsc, desc->tsc_freq); } \ No newline at end of file diff --git a/topo.c b/topo.c index 6d20a46..41be77c 100644 --- a/topo.c +++ b/topo.c @@ -17,9 +17,8 @@ #include "libxml/xmlversion.h" #include "topo.h" +#include "topop.h" - -#define TOPO_MAX_CHILDREN (256) #define TOPO_INVALID (-1) #define TOPO_CACHE_L2 (2) @@ -41,9 +40,9 @@ struct topo_obj { struct topo_obj * children[TOPO_MAX_CHILDREN]; }; -extern int topo_alloc_init(int verbose, struct topo_obj * tobj); -extern int topo_ts_init(int verbose); -static struct topo_obj * g_tobj = NULL; +static struct topo_desc g_default_desc = {0}; +static _Atomic(int) initialized = 0; +static volatile int init_rc = 0; static void mask_to_cpuset(const char * mask, cpuset_t * cset) @@ -80,7 +79,7 @@ mask_to_cpuset(const char * mask, cpuset_t * cset) } static void -preorder_dump(struct topo_obj * root, int indent) { +tobj_preorder_dump(struct topo_obj * root, int indent) { cpuset_t tmpset; CPU_COPY(&root->mask, &tmpset); const char * flag_str; @@ -110,12 +109,12 @@ preorder_dump(struct topo_obj * root, int indent) { } printf("\n"); for (int i = 0; i < root->num_children; i++) { - preorder_dump(root->children[i], indent + 1); + tobj_preorder_dump(root->children[i], indent + 1); } } static xmlAttr * -find_attr(xmlNode * root, const char * attr_name) +xml_find_attr(xmlNode * root, const char * attr_name) { xmlAttr * attr = root->properties; while (attr != NULL) { @@ -129,7 +128,7 @@ find_attr(xmlNode * root, const char * attr_name) } static void -free_root(struct topo_obj * root) +tobj_free_root(struct topo_obj * root) { if (root == NULL) { return; @@ -137,7 +136,7 @@ free_root(struct topo_obj * root) if (root->num_children > 0) { for (int i = 0; i < root->num_children; i++) { - free_root(root->children[i]); + tobj_free_root(root->children[i]); } } @@ -145,21 +144,21 @@ free_root(struct topo_obj * root) } static int -populate_root(xmlNode * root, struct topo_obj * parent, struct topo_obj ** out) +tobj_populate_root(xmlNode * root, struct topo_obj * parent, struct topo_obj ** out) { int rc = 0; if (root->type == XML_ELEMENT_NODE) { if (strcmp((const char*)root->name, "group") == 0) { // this is a group, then check cache-level - xmlAttr * attr = find_attr(root, "cache-level"); + xmlAttr * attr = xml_find_attr(root, "cache-level"); if (attr == NULL) { fprintf(stderr, "libtopo: could not find attr cache-level in group.\n"); return -1; } - xmlAttr * lvl_attr = find_attr(root, "level"); + xmlAttr * lvl_attr = xml_find_attr(root, "level"); if (lvl_attr == NULL) { fprintf(stderr, "libtopo: could not find attr level in group.\n"); return -1; @@ -193,7 +192,7 @@ populate_root(xmlNode * root, struct topo_obj * parent, struct topo_obj ** out) parent = tobj; } else if (strcmp((const char*)root->name, "cpu") == 0) { - xmlAttr * attr = find_attr(root, "mask"); + xmlAttr * attr = xml_find_attr(root, "mask"); if (attr == NULL) { fprintf(stderr, "libtopo: could not find attr mask in cpu.\n"); @@ -202,7 +201,7 @@ populate_root(xmlNode * root, struct topo_obj * parent, struct topo_obj ** out) mask_to_cpuset((const char *)attr->children->content, &parent->mask); } else if (strcmp((const char *)root->name, "flag") == 0) { - xmlAttr * attr = find_attr(root, "name"); + xmlAttr * attr = xml_find_attr(root, "name"); if (attr == NULL) { fprintf(stderr, "libtopo: could not find attr name in flag.\n"); @@ -222,7 +221,7 @@ populate_root(xmlNode * root, struct topo_obj * parent, struct topo_obj ** out) xmlNode * child = root->children; while (child != NULL) { - if ((rc = populate_root(child, parent, out)) != 0) { + if ((rc = tobj_populate_root(child, parent, out)) != 0) { break; } child = child->next; @@ -232,7 +231,7 @@ populate_root(xmlNode * root, struct topo_obj * parent, struct topo_obj ** out) } static int -init(struct topo_obj **out) +tobj_populate(struct topo_obj **out) { int rc; size_t sz; @@ -262,12 +261,12 @@ init(struct topo_obj **out) errno = EINVAL; return -1; } - rc = populate_root(xmlDocGetRootElement(doc), NULL, out) != 0; + rc = tobj_populate_root(xmlDocGetRootElement(doc), NULL, out) != 0; free(buf); xmlFreeDoc(doc); if (rc != 0) { - free_root(*out); + tobj_free_root(*out); errno = EINVAL; return rc; } @@ -276,7 +275,7 @@ init(struct topo_obj **out) } static struct topo_obj * -find_node_by_mask(struct topo_obj * root, cpuset_t * set) +tobj_find_node_by_mask(struct topo_obj * root, cpuset_t * set) { if (!CPU_SUBSET(&root->mask, set)) { return NULL; @@ -284,7 +283,7 @@ find_node_by_mask(struct topo_obj * root, cpuset_t * set) struct topo_obj * best_fit; for (int i = 0; i < root->num_children; i++) { - best_fit = find_node_by_mask(root->children[i], set); + best_fit = tobj_find_node_by_mask(root->children[i], set); if (best_fit != NULL) { break; } @@ -294,7 +293,7 @@ find_node_by_mask(struct topo_obj * root, cpuset_t * set) } static struct topo_obj * -find_node_by_flag(struct topo_obj * root, int flag) +tobj_find_node_by_flag(struct topo_obj * root, int flag) { if (root->flags & flag) { return root; @@ -302,7 +301,7 @@ find_node_by_flag(struct topo_obj * root, int flag) struct topo_obj * result; for (int i = 0; i < root->num_children; i++) { - result = find_node_by_flag(root->children[i], flag); + result = tobj_find_node_by_flag(root->children[i], flag); if (result != NULL) { return result; } @@ -312,13 +311,13 @@ find_node_by_flag(struct topo_obj * root, int flag) } static int -_topo_core_to_numa(struct topo_obj * root, int coreid) +tobj_core_to_numa(struct topo_obj * root, int coreid) { cpuset_t set; CPU_ZERO(&set); CPU_SET(coreid, &set); - struct topo_obj * obj = find_node_by_mask(root, &set); + struct topo_obj * obj = tobj_find_node_by_mask(root, &set); if (obj == NULL) { return TOPO_INVALID; } @@ -341,9 +340,9 @@ _topo_core_to_numa(struct topo_obj * root, int coreid) } static int -_topo_num_numa(struct topo_obj * root) +tobj_get_num_numa(struct topo_obj * root) { - struct topo_obj * result = find_node_by_flag(root, TOPO_FLAG_NUMA); + struct topo_obj * result = tobj_find_node_by_flag(root, TOPO_FLAG_NUMA); if (result == NULL) { return 1; @@ -357,78 +356,114 @@ _topo_num_numa(struct topo_obj * root) } static int -_topo_num_core(struct topo_obj * root) +tobj_get_num_core(struct topo_obj * root) { return CPU_COUNT(&root->mask); } static void -_topo_destroy(struct topo_obj * obj) +desc_destroy(struct topo_desc * desc) { - free_root(obj); + tobj_free_root(desc->root); } static int -_topo_init(int verbose, int alloc_init, struct topo_obj ** obj) +desc_init(struct topo_desc * desc, int enable_alloc, int verbose) { - int rc = init(obj); + desc->root = NULL; + int rc = tobj_populate(&desc->root); - if (verbose) { - if (rc == 0) { - int num_cores = _topo_num_core(*obj); - fprintf(stdout, "libtopo: %d cores, %d numa domains detected.\n", num_cores, _topo_num_numa(*obj)); - for (int i = 0; i < num_cores; i++) { - fprintf(stdout, "libtopo: core #%d @ numa domain %d.\n", i, _topo_core_to_numa(*obj, i)); + if (rc == 0) { + desc->num_core = tobj_get_num_core(desc->root); + desc->num_numa = tobj_get_num_numa(desc->root); + if (desc->num_core > TOPO_MAX_CHILDREN) { + fprintf(stderr, "libtopo: too many cores - %d\n", desc->num_core); + rc = ENOMEM; + } else { + for (int i = 0; i < desc->num_core; i++) { + desc->core_to_numa_lookup[i] = tobj_core_to_numa(desc->root, i); } - fprintf(stdout, "libtopo: parsed tree:\n"); - preorder_dump(*obj, 1); - } else { - fprintf(stderr, "libtopo: failed to parse topo: %d\n", errno); + if (verbose) { + fprintf(stdout, "libtopo: %d cores, %d numa domains detected.\n", desc->num_core, desc->num_numa); + for (int i = 0; i < desc->num_core; i++) { + fprintf(stdout, "libtopo: core #%d @ numa domain %d.\n", i, desc->core_to_numa_lookup[i]); + } + fprintf(stdout, "libtopo: parsed tree:\n"); + tobj_preorder_dump(desc->root, 1); + } + } + } else { + fprintf(stderr, "libtopo: failed to parse topo: %d\n", errno); + } + + if (rc == 0) { + rc = topo_ts_init(desc, verbose); + } + + if (enable_alloc && rc == 0) { + rc = topo_alloc_init(desc, verbose); + } + + if (rc != 0) { + if (desc->root != NULL) { + tobj_free_root(desc->root); } } - rc = topo_ts_init(verbose); - - if (alloc_init && rc == 0) { - rc = topo_alloc_init(verbose, *obj); - } - return rc; } +// +// public APIs +// int topo_num_core() { - return _topo_num_core(g_tobj); + return g_default_desc.num_core; } int topo_num_numa() { - return _topo_num_numa(g_tobj); + return g_default_desc.num_numa; } int topo_core_to_numa(int coreid) { - return _topo_core_to_numa(g_tobj, coreid); + return g_default_desc.core_to_numa_lookup[coreid]; } -static _Atomic(int) initialized = 0; -static volatile int init_rc = 0; - void topo_destroy() { - _topo_destroy(g_tobj); + desc_destroy(&g_default_desc); +} + +void * +topo_malloc(unsigned int node, size_t size) +{ + return topo_desc_malloc(&g_default_desc, node, size); +} + +void +topo_free(unsigned int node, void * addr) +{ + return topo_desc_free(&g_default_desc, node, addr); +} + +uint64_t +topo_uptime_ns() +{ + return topo_desc_uptime_ns(&g_default_desc); } int -topo_init(int verbose, int alloc_init) +topo_init(int enable_alloc, int verbose) { int expected = 0; if (atomic_compare_exchange_strong(&initialized, &expected, 2)) { - init_rc = _topo_init(verbose, alloc_init, &g_tobj); + init_rc = desc_init(&g_default_desc, enable_alloc, verbose); atomic_store(&initialized, 1); } else { while(atomic_load(&initialized) != 1) {} // wait for init diff --git a/topop.h b/topop.h new file mode 100644 index 0000000..2061dbe --- /dev/null +++ b/topop.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include + +#define TOPO_MAX_CHILDREN (256) + +#define ALLOC_MAX_OBJS_PER_LVL (256) +#define ALLOC_MEM_OBJ_SIZE (4096) // 4k +#define ALLOC_MEM_OBJ_NUM (1024 * 256) // 4k * 1024 * 256 = 1GB per region +#define ALLOC_MEM_REGION_NUM (4) // 4 x 1GB = 4GB total + +struct topo_desc { + struct topo_obj * root; + int core_to_numa_lookup[TOPO_MAX_CHILDREN]; + int num_core; + int num_numa; + + // ts + uint64_t tsc_freq; + + // alloc + pthread_mutex_t alloc_lock; + int mem_idx[ALLOC_MAX_OBJS_PER_LVL]; + int mem_region_idx[ALLOC_MAX_OBJS_PER_LVL]; + void* mem_regions[ALLOC_MAX_OBJS_PER_LVL][ALLOC_MEM_REGION_NUM]; +}; + +int +topo_alloc_init(struct topo_desc * desc, int verbose); + +int +topo_ts_init(struct topo_desc *desc, int verbose); + +uint64_t +topo_desc_uptime_ns(struct topo_desc * desc); + +void * +topo_desc_malloc(struct topo_desc * desc, unsigned int node, size_t size); + +void +topo_desc_free(struct topo_desc * desc __attribute__((unused)), unsigned int node __attribute__((unused)), void * addr __attribute__((unused))); \ No newline at end of file