d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

posix: Group connections of sock group on host side based on placement_id

Browse Source 
On host side the connections are created and then added to thread's
poll group. Those connections could use different NIC queues underneath.
To route all connections of poll group through single queue a unique
placement id is chosen as group_placement_id and each socket of poll
group is marked with group_placment_id using getsockopt(SO_MARK) option.

The driver could use so_mark value of skb to determine the queue to use.

Change-Id: I06bda777fe07a62133b80b2491fa7772150b3b5d
Signed-off-by: Sudheer Mogilappagari <sudheer.mogilappagari@intel.com>
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6160
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Sudheer Mogilappagari 2021-04-06 14:48:51 -07:00 committed by Tomasz Zawadzki
parent c06df11513
commit 2cbc9d4dff
4 changed files with 72 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -174,6 +174,11 @@ bool to int. We can use RPC to configure different value of `enable_placement_id
Then we can leverage SO_INCOMING_CPU to get placement_id, which aims to utilize
CPU cache locality, enabled by setting enable_placement_id=2.
A new socket placement mode called PLACEMENT_MARK has been added. Some NICs allow
sockets to be marked using the SO_MARK socket option as a hint for which hardware
queue they should be associated with. This mode leverages that by setting the same
value for all sockets within a poll group.
### thread
A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified

1
include/spdk/sock.h
View File

@ -84,6 +84,7 @@ enum spdk_placement_mode {
PLACEMENT_NONE,
PLACEMENT_NAPI,
PLACEMENT_CPU,
PLACEMENT_MARK,
};
/**

1
include/spdk_internal/sock.h
View File

@ -284,6 +284,7 @@ spdk_sock_get_placement_id(int fd, enum spdk_placement_mode mode, int *placement
switch (mode) {
case PLACEMENT_NONE:
break;
case PLACEMENT_MARK:
case PLACEMENT_NAPI: {
#if defined(SO_INCOMING_NAPI_ID)
socklen_t len = sizeof(int);

67
module/sock/posix/posix.c
View File

@ -82,6 +82,7 @@ struct spdk_posix_sock_group_impl {
struct spdk_sock_group_impl base;
int fd;
struct spdk_pending_events_list pending_events;
int placement_id;
};
static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = {
@ -368,6 +369,11 @@ posix_sock_alloc(int fd, bool enable_zero_copy)
spdk_sock_get_placement_id(sock->fd, g_spdk_posix_sock_impl_opts.enable_placement_id,
&sock->placement_id);
if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_MARK) {
/* Save placement_id */
spdk_sock_map_insert(&g_map, sock->placement_id, NULL);
}
#endif
return sock;
@ -1148,14 +1154,69 @@ posix_sock_group_impl_create(void)
group_impl->fd = fd;
TAILQ_INIT(&group_impl->pending_events);
group_impl->placement_id = -1;
if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) {
spdk_sock_map_insert(&g_map, spdk_env_get_current_core(), &group_impl->base);
group_impl->placement_id = spdk_env_get_current_core();
}
return &group_impl->base;
}
static void
posix_sock_mark(struct spdk_posix_sock_group_impl *group, struct spdk_posix_sock *sock,
int placement_id)
{
#if defined(SO_MARK)
int rc;
rc = setsockopt(sock->fd, SOL_SOCKET, SO_MARK,
&placement_id, sizeof(placement_id));
if (rc != 0) {
/* Not fatal */
SPDK_ERRLOG("Error setting SO_MARK\n");
return;
}
rc = spdk_sock_map_insert(&g_map, placement_id, &group->base);
if (rc != 0) {
/* Not fatal */
SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc);
return;
}
sock->placement_id = placement_id;
#endif
}
static void
posix_sock_update_mark(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock)
{
struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group);
if (group->placement_id == -1) {
group->placement_id = spdk_sock_map_find_free(&g_map);
/* If a free placement id is found, update existing sockets in this group */
if (group->placement_id != -1) {
struct spdk_sock *sock, *tmp;
TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) {
posix_sock_mark(group, __posix_sock(sock), group->placement_id);
}
}
}
if (group->placement_id != -1) {
/*
* group placement id is already determined for this poll group.
* Mark socket with group's placement id.
*/
posix_sock_mark(group, __posix_sock(_sock), group->placement_id);
}
}
static int
posix_sock_group_impl_add_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock)
{
@ -1193,10 +1254,12 @@ posix_sock_group_impl_add_sock(struct spdk_sock_group_impl *_group, struct spdk_
TAILQ_INSERT_TAIL(&group->pending_events, sock, link);
}
if (sock->placement_id != -1) {
if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_MARK) {
posix_sock_update_mark(_group, _sock);
} else if (sock->placement_id != -1) {
rc = spdk_sock_map_insert(&g_map, sock->placement_id, &group->base);
if (rc != 0) {
SPDK_ERRLOG("Failed to insert sock group into map: %d", rc);
SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc);
/* Do not treat this as an error. The system will continue running. */
}
}