numam/libnm/topo.cc

#include "nm.h"
#include "nmp.h"

#include <cstdio>
#include <cstdlib>
#include <hwloc.h>

constexpr static int NM_MAX_CHILDREN = 128;

struct nm_obj {
	int level;
	int id;
	struct nm_obj *parent;
	
	int num_children;
	struct nm_obj * children[NM_MAX_CHILDREN];
};

static int size_tbl[NM_MAX_LEVEL] = { 0 };
static struct nm_obj * obj_tbl[NM_MAX_LEVEL][NM_MAX_OBJS_PER_LVL];

static hwloc_obj_t
get_parent_type(hwloc_obj_t obj, hwloc_obj_type_t type)
{
	obj = obj->parent;
	while (obj != nullptr) {
		if (obj->type == type) {
			return obj;
		}
		obj = obj->parent;
	}
	return nullptr;
}

// static int
// obj_comparator(const void * a, const void * b)
// {
// 	return ((struct nm_obj *)a)->id - ((struct nm_obj *)b)->id;
// }

static inline int 
is_level_valid(int level) 
{
	return level < (int)NM_MAX_LEVEL;
}

int
nm_obj_count(int level)
{
	if (is_level_valid(level)) {
		return size_tbl[level];
	}
	return -1;
}

struct nm_obj *
nm_obj_from_id(int level, int id)
{
	if (is_level_valid(level) && id <= size_tbl[level]) {
		return obj_tbl[level][id];
	}
	return nullptr;
}

struct nm_obj *
nm_obj_find_parent(struct nm_obj * start, int parent_level)
{
    struct nm_obj * ret = nullptr;
    while (start != nullptr) {
        if (parent_level == start->level) {
            ret = start;
            break;
        }
        start = start->parent;
    }
    return ret;
}

int
nm_obj_get_id(struct nm_obj * obj)
{
    return obj->id;
}

int
nm_obj_get_level(struct nm_obj * obj)
{
    return obj->level;
}

static int
validate_objs(int level)
{
	for (int i = 0; i < size_tbl[level]; i++) {
		struct nm_obj * each = obj_tbl[level][i];
		if (each->id != i) {
			return 0;
		}	
	}
	return 1;
}

static int
add_obj(int id, int level, struct nm_obj * parent)
{
	if (size_tbl[level] >= NM_MAX_OBJS_PER_LVL) {

		return -1;
	}

	auto each = (struct nm_obj *)malloc(sizeof(struct nm_obj));

	each->id = id;
	each->level = level;
	each->num_children = 0;
	each->parent = parent;

	if (parent != nullptr) {
		// add children
		if (parent->num_children >= NM_MAX_CHILDREN) {
			return -1;
		} else {
			parent->children[parent->num_children] = each;
			parent->num_children++;
		}
	}

	obj_tbl[level][size_tbl[level]] = each;
	size_tbl[level]++;

	return 0;
}

static int
add_level(hwloc_topology * topo, hwloc_obj_type_t hwloc_level, hwloc_obj_type_t hwloc_plevel, int level, int plevel)
{
	// populate numa nodes
	hwloc_obj_t obj = nullptr;
	hwloc_obj_t pobj = nullptr;
	struct nm_obj *parent = nullptr;
	while (true) {

		obj = hwloc_get_next_obj_by_type(topo, hwloc_level, obj);
		if (obj == nullptr) {
			break;
		}

		pobj = get_parent_type(obj, hwloc_plevel);

		if (pobj != nullptr) {
			parent = obj_tbl[plevel][pobj->logical_index];
		}

		if (add_obj(obj->logical_index, level, parent) != 0) {
			if (nm_get_verbose() > 0) {
				fprintf(stderr, "libnm: failed to add object %d.\n", obj->logical_index);
			}
			return -1;
		}

		if (nm_get_verbose() > 0) {
			fprintf(stdout, "libnm: identified id %d type %d parent %d type %d\n", obj->logical_index, level, parent == nullptr ? -1 : parent->id, plevel);
		}
	}

	// sort
	// std::qsort(obj_tbl[level], size_tbl[level], sizeof(struct nm_obj), obj_comparator);
	if (!validate_objs(level)) {
		if (nm_get_verbose() > 0) {
			fprintf(stdout, "libnm: objects are shuffled at level %d.\n", level);
		}
		return -1;
	}

	return 0;
}

int
nm_topo_init()
{
	int ret;

	// init numa stuff
	hwloc_topology *topo;
	if ((ret = hwloc_topology_init(&topo)) != 0) {
		return ret;
	}

	if ((ret = hwloc_topology_load(topo)) != 0)
		return ret;

	if ((ret = add_level(topo, HWLOC_OBJ_PACKAGE, HWLOC_OBJ_PACKAGE, NM_LEVEL_NUMA, NM_LEVEL_NUMA)) != 0) {
		return ret;
	}

	if ((ret = add_level(topo, HWLOC_OBJ_CORE, HWLOC_OBJ_PACKAGE, NM_LEVEL_CPU, NM_LEVEL_NUMA)) != 0) {
		return ret;
	}

	if ((ret = add_level(topo, HWLOC_OBJ_PU, HWLOC_OBJ_CORE, NM_LEVEL_CORE, NM_LEVEL_CPU)) != 0) {
		return ret;
	}

	return ret;
}