numam-dpdk/lib/eal/common/eal_common_lcore.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2014 Intel Corporation
 */

#include <stdlib.h>
#include <string.h>

#include <rte_common.h>
#include <rte_branch_prediction.h>
#include <rte_errno.h>
#include <rte_lcore.h>
#include <rte_log.h>

#include "eal_private.h"
#include "eal_thread.h"

unsigned int rte_get_main_lcore(void)
{
	return rte_eal_get_configuration()->main_lcore;
}

unsigned int rte_lcore_count(void)
{
	return rte_eal_get_configuration()->lcore_count;
}

int rte_lcore_index(int lcore_id)
{
	if (unlikely(lcore_id >= RTE_MAX_LCORE))
		return -1;

	if (lcore_id < 0) {
		if (rte_lcore_id() == LCORE_ID_ANY)
			return -1;

		lcore_id = (int)rte_lcore_id();
	}

	return lcore_config[lcore_id].core_index;
}

int rte_lcore_to_cpu_id(int lcore_id)
{
	if (unlikely(lcore_id >= RTE_MAX_LCORE))
		return -1;

	if (lcore_id < 0) {
		if (rte_lcore_id() == LCORE_ID_ANY)
			return -1;

		lcore_id = (int)rte_lcore_id();
	}

	return lcore_config[lcore_id].core_id;
}

rte_cpuset_t rte_lcore_cpuset(unsigned int lcore_id)
{
	return lcore_config[lcore_id].cpuset;
}

enum rte_lcore_role_t
rte_eal_lcore_role(unsigned int lcore_id)
{
	struct rte_config *cfg = rte_eal_get_configuration();

	if (lcore_id >= RTE_MAX_LCORE)
		return ROLE_OFF;
	return cfg->lcore_role[lcore_id];
}

int
rte_lcore_has_role(unsigned int lcore_id, enum rte_lcore_role_t role)
{
	struct rte_config *cfg = rte_eal_get_configuration();

	if (lcore_id >= RTE_MAX_LCORE)
		return -EINVAL;

	return cfg->lcore_role[lcore_id] == role;
}

int rte_lcore_is_enabled(unsigned int lcore_id)
{
	struct rte_config *cfg = rte_eal_get_configuration();

	if (lcore_id >= RTE_MAX_LCORE)
		return 0;
	return cfg->lcore_role[lcore_id] == ROLE_RTE;
}

unsigned int rte_get_next_lcore(unsigned int i, int skip_main, int wrap)
{
	i++;
	if (wrap)
		i %= RTE_MAX_LCORE;

	while (i < RTE_MAX_LCORE) {
		if (!rte_lcore_is_enabled(i) ||
		    (skip_main && (i == rte_get_main_lcore()))) {
			i++;
			if (wrap)
				i %= RTE_MAX_LCORE;
			continue;
		}
		break;
	}
	return i;
}

unsigned int
rte_lcore_to_socket_id(unsigned int lcore_id)
{
	return lcore_config[lcore_id].socket_id;
}

static int
socket_id_cmp(const void *a, const void *b)
{
	const int *lcore_id_a = a;
	const int *lcore_id_b = b;

	if (*lcore_id_a < *lcore_id_b)
		return -1;
	if (*lcore_id_a > *lcore_id_b)
		return 1;
	return 0;
}

/*
 * Parse /sys/devices/system/cpu to get the number of physical and logical
 * processors on the machine. The function will fill the cpu_info
 * structure.
 */
int
rte_eal_cpu_init(void)
{
	/* pointer to global configuration */
	struct rte_config *config = rte_eal_get_configuration();
	unsigned lcore_id;
	unsigned count = 0;
	unsigned int socket_id, prev_socket_id;
	int lcore_to_socket_id[RTE_MAX_LCORE];

	/*
	 * Parse the maximum set of logical cores, detect the subset of running
	 * ones and enable them by default.
	 */
	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
		lcore_config[lcore_id].core_index = count;

		/* init cpuset for per lcore config */
		CPU_ZERO(&lcore_config[lcore_id].cpuset);

		/* find socket first */
		socket_id = eal_cpu_socket_id(lcore_id);
		lcore_to_socket_id[lcore_id] = socket_id;

		if (eal_cpu_detected(lcore_id) == 0) {
			config->lcore_role[lcore_id] = ROLE_OFF;
			lcore_config[lcore_id].core_index = -1;
			continue;
		}

		/* By default, lcore 1:1 map to cpu id */
		CPU_SET(lcore_id, &lcore_config[lcore_id].cpuset);

		/* By default, each detected core is enabled */
		config->lcore_role[lcore_id] = ROLE_RTE;
		lcore_config[lcore_id].core_role = ROLE_RTE;
		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
		lcore_config[lcore_id].socket_id = socket_id;
		RTE_LOG(DEBUG, EAL, "Detected lcore %u as "
				"core %u on socket %u\n",
				lcore_id, lcore_config[lcore_id].core_id,
				lcore_config[lcore_id].socket_id);
		count++;
	}
	for (; lcore_id < CPU_SETSIZE; lcore_id++) {
		if (eal_cpu_detected(lcore_id) == 0)
			continue;
		RTE_LOG(DEBUG, EAL, "Skipped lcore %u as core %u on socket %u\n",
			lcore_id, eal_cpu_core_id(lcore_id),
			eal_cpu_socket_id(lcore_id));
	}

	/* Set the count of enabled logical cores of the EAL configuration */
	config->lcore_count = count;
	RTE_LOG(DEBUG, EAL,
			"Maximum logical cores by configuration: %u\n",
			RTE_MAX_LCORE);
	RTE_LOG(INFO, EAL, "Detected CPU lcores: %u\n", config->lcore_count);

	/* sort all socket id's in ascending order */
	qsort(lcore_to_socket_id, RTE_DIM(lcore_to_socket_id),
			sizeof(lcore_to_socket_id[0]), socket_id_cmp);

	prev_socket_id = -1;
	config->numa_node_count = 0;
	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
		socket_id = lcore_to_socket_id[lcore_id];
		if (socket_id != prev_socket_id)
			config->numa_nodes[config->numa_node_count++] =
					socket_id;
		prev_socket_id = socket_id;
	}
	RTE_LOG(INFO, EAL, "Detected NUMA nodes: %u\n", config->numa_node_count);

	return 0;
}

unsigned int
rte_socket_count(void)
{
	const struct rte_config *config = rte_eal_get_configuration();
	return config->numa_node_count;
}

int
rte_socket_id_by_idx(unsigned int idx)
{
	const struct rte_config *config = rte_eal_get_configuration();
	if (idx >= config->numa_node_count) {
		rte_errno = EINVAL;
		return -1;
	}
	return config->numa_nodes[idx];
}

static rte_rwlock_t lcore_lock = RTE_RWLOCK_INITIALIZER;
struct lcore_callback {
	TAILQ_ENTRY(lcore_callback) next;
	char *name;
	rte_lcore_init_cb init;
	rte_lcore_uninit_cb uninit;
	void *arg;
};
static TAILQ_HEAD(lcore_callbacks_head, lcore_callback) lcore_callbacks =
	TAILQ_HEAD_INITIALIZER(lcore_callbacks);

static int
callback_init(struct lcore_callback *callback, unsigned int lcore_id)
{
	if (callback->init == NULL)
		return 0;
	RTE_LOG(DEBUG, EAL, "Call init for lcore callback %s, lcore_id %u\n",
		callback->name, lcore_id);
	return callback->init(lcore_id, callback->arg);
}

static void
callback_uninit(struct lcore_callback *callback, unsigned int lcore_id)
{
	if (callback->uninit == NULL)
		return;
	RTE_LOG(DEBUG, EAL, "Call uninit for lcore callback %s, lcore_id %u\n",
		callback->name, lcore_id);
	callback->uninit(lcore_id, callback->arg);
}

static void
free_callback(struct lcore_callback *callback)
{
	free(callback->name);
	free(callback);
}

void *
rte_lcore_callback_register(const char *name, rte_lcore_init_cb init,
	rte_lcore_uninit_cb uninit, void *arg)
{
	struct rte_config *cfg = rte_eal_get_configuration();
	struct lcore_callback *callback;
	unsigned int lcore_id;

	if (name == NULL)
		return NULL;
	callback = calloc(1, sizeof(*callback));
	if (callback == NULL)
		return NULL;
	if (asprintf(&callback->name, "%s-%p", name, arg) == -1) {
		free(callback);
		return NULL;
	}
	callback->init = init;
	callback->uninit = uninit;
	callback->arg = arg;
	rte_rwlock_write_lock(&lcore_lock);
	if (callback->init == NULL)
		goto no_init;
	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
		if (cfg->lcore_role[lcore_id] == ROLE_OFF)
			continue;
		if (callback_init(callback, lcore_id) == 0)
			continue;
		/* Callback refused init for this lcore, uninitialize all
		 * previous lcore.
		 */
		while (lcore_id-- != 0) {
			if (cfg->lcore_role[lcore_id] == ROLE_OFF)
				continue;
			callback_uninit(callback, lcore_id);
		}
		free_callback(callback);
		callback = NULL;
		goto out;
	}
no_init:
	TAILQ_INSERT_TAIL(&lcore_callbacks, callback, next);
	RTE_LOG(DEBUG, EAL, "Registered new lcore callback %s (%sinit, %suninit).\n",
		callback->name, callback->init == NULL ? "NO " : "",
		callback->uninit == NULL ? "NO " : "");
out:
	rte_rwlock_write_unlock(&lcore_lock);
	return callback;
}

void
rte_lcore_callback_unregister(void *handle)
{
	struct rte_config *cfg = rte_eal_get_configuration();
	struct lcore_callback *callback = handle;
	unsigned int lcore_id;

	if (callback == NULL)
		return;
	rte_rwlock_write_lock(&lcore_lock);
	if (callback->uninit == NULL)
		goto no_uninit;
	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
		if (cfg->lcore_role[lcore_id] == ROLE_OFF)
			continue;
		callback_uninit(callback, lcore_id);
	}
no_uninit:
	TAILQ_REMOVE(&lcore_callbacks, callback, next);
	rte_rwlock_write_unlock(&lcore_lock);
	RTE_LOG(DEBUG, EAL, "Unregistered lcore callback %s-%p.\n",
		callback->name, callback->arg);
	free_callback(callback);
}

unsigned int
eal_lcore_non_eal_allocate(void)
{
	struct rte_config *cfg = rte_eal_get_configuration();
	struct lcore_callback *callback;
	struct lcore_callback *prev;
	unsigned int lcore_id;

	rte_rwlock_write_lock(&lcore_lock);
	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
		if (cfg->lcore_role[lcore_id] != ROLE_OFF)
			continue;
		cfg->lcore_role[lcore_id] = ROLE_NON_EAL;
		cfg->lcore_count++;
		break;
	}
	if (lcore_id == RTE_MAX_LCORE) {
		RTE_LOG(DEBUG, EAL, "No lcore available.\n");
		goto out;
	}
	TAILQ_FOREACH(callback, &lcore_callbacks, next) {
		if (callback_init(callback, lcore_id) == 0)
			continue;
		/* Callback refused init for this lcore, call uninit for all
		 * previous callbacks.
		 */
		prev = TAILQ_PREV(callback, lcore_callbacks_head, next);
		while (prev != NULL) {
			callback_uninit(prev, lcore_id);
			prev = TAILQ_PREV(prev, lcore_callbacks_head, next);
		}
		RTE_LOG(DEBUG, EAL, "Initialization refused for lcore %u.\n",
			lcore_id);
		cfg->lcore_role[lcore_id] = ROLE_OFF;
		cfg->lcore_count--;
		lcore_id = RTE_MAX_LCORE;
		goto out;
	}
out:
	rte_rwlock_write_unlock(&lcore_lock);
	return lcore_id;
}

void
eal_lcore_non_eal_release(unsigned int lcore_id)
{
	struct rte_config *cfg = rte_eal_get_configuration();
	struct lcore_callback *callback;

	rte_rwlock_write_lock(&lcore_lock);
	if (cfg->lcore_role[lcore_id] != ROLE_NON_EAL)
		goto out;
	TAILQ_FOREACH(callback, &lcore_callbacks, next)
		callback_uninit(callback, lcore_id);
	cfg->lcore_role[lcore_id] = ROLE_OFF;
	cfg->lcore_count--;
out:
	rte_rwlock_write_unlock(&lcore_lock);
}

int
rte_lcore_iterate(rte_lcore_iterate_cb cb, void *arg)
{
	struct rte_config *cfg = rte_eal_get_configuration();
	unsigned int lcore_id;
	int ret = 0;

	rte_rwlock_read_lock(&lcore_lock);
	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
		if (cfg->lcore_role[lcore_id] == ROLE_OFF)
			continue;
		ret = cb(lcore_id, arg);
		if (ret != 0)
			break;
	}
	rte_rwlock_read_unlock(&lcore_lock);
	return ret;
}

static int
lcore_dump_cb(unsigned int lcore_id, void *arg)
{
	struct rte_config *cfg = rte_eal_get_configuration();
	char cpuset[RTE_CPU_AFFINITY_STR_LEN];
	const char *role;
	FILE *f = arg;
	int ret;

	switch (cfg->lcore_role[lcore_id]) {
	case ROLE_RTE:
		role = "RTE";
		break;
	case ROLE_SERVICE:
		role = "SERVICE";
		break;
	case ROLE_NON_EAL:
		role = "NON_EAL";
		break;
	default:
		role = "UNKNOWN";
		break;
	}

	ret = eal_thread_dump_affinity(&lcore_config[lcore_id].cpuset, cpuset,
		sizeof(cpuset));
	fprintf(f, "lcore %u, socket %u, role %s, cpuset %s%s\n", lcore_id,
		rte_lcore_to_socket_id(lcore_id), role, cpuset,
		ret == 0 ? "" : "...");
	return 0;
}

void
rte_lcore_dump(FILE *f)
{
	rte_lcore_iterate(lcore_dump_cb, f);
}