From d9efb8f0045fd31ca63020c23f04ba205070f9d7 Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Fri, 16 Apr 2021 17:05:48 +0300 Subject: [PATCH] sock: Add new params to configure zcopy for server, client sockets When zcero copy send is enabled and used by initiator, it could significantly increase latency in some payloads. To enable more fine graing configuration of zero copy send feature, add new parameters enable_zerocopy_send_server and enable_zerocopy_send_client to spdk_sock_impl_opts to enable/disable zcopy for specific type of sockets. Exisiting enable_zerocopy_send parameter affects all types of sockets. Signed-off-by: Alexey Marchuk Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7457 (master) (cherry picked from commit 8e85b675fc9a9902316d81f8a588ba5a08c4aac4) Change-Id: I111c75608f8826980a56e210c076ab8ff16ddbdc Signed-off-by: Krzysztof Karas Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7638 Reviewed-by: Aleksey Marchuk Reviewed-by: Tomasz Zawadzki Tested-by: SPDK CI Jenkins --- CHANGELOG.md | 6 ++++++ doc/jsonrpc.md | 30 +++++++++++++++++++----------- examples/nvme/perf/perf.c | 1 + include/spdk/sock.h | 9 +++++++++ lib/sock/sock.c | 2 ++ lib/sock/sock_rpc.c | 13 +++++++++++-- module/sock/posix/posix.c | 19 +++++++++++++++---- scripts/rpc.py | 15 +++++++++++++-- scripts/rpc/sock.py | 10 +++++++++- 9 files changed, 85 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 241417bc72..f7667a84fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -179,6 +179,12 @@ sockets to be marked using the SO_MARK socket option as a hint for which hardwar queue they should be associated with. This mode leverages that by setting the same value for all sockets within a poll group. +New parameters `enable_zerocopy_send_server` and `enable_zerocopy_send_client` were added +to struct spdk_sock_impl_opts, these parameters enable or disable zero copy send for server +and client sockets which are created using `spdk_sock_listen` and `spdk_sock_listen_ext` (server); +`spdk_sock_connect` and `spdk_sock_connect_ext` (client) functions. Existing parameter +`enable_zerocopy_send` enables or disables zero copy send for both server and client sockets. + ### thread A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 876c00f4d4..2833a0125f 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -8591,7 +8591,11 @@ Example response: "recv_buf_size": 2097152, "send_buf_size": 2097152, "enable_recv_pipe": true, - "enable_zerocopy_send": true + "enable_zerocopy_send": true, + "enable_quickack": true, + "enable_placement_id": 0, + "enable_zerocopy_send_server": true, + "enable_zerocopy_send_client": false } } ~~~ @@ -8602,15 +8606,17 @@ Set parameters for the socket layer implementation. ### Parameters -Name | Optional | Type | Description ------------------------ | -------- | ----------- | ----------- -impl_name | Required | string | Name of socket implementation, e.g. posix -recv_buf_size | Optional | number | Size of socket receive buffer in bytes -send_buf_size | Optional | number | Size of socket send buffer in bytes -enable_recv_pipe | Optional | boolean | Enable or disable receive pipe -enable_zerocopy_send | Optional | boolean | Enable or disable zero copy on send -enable_quick_ack | Optional | boolean | Enable or disable quick ACK -enable_placement_id | Optional | number | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu +Name | Optional | Type | Description +--------------------------- | -------- | ----------- | ----------- +impl_name | Required | string | Name of socket implementation, e.g. posix +recv_buf_size | Optional | number | Size of socket receive buffer in bytes +send_buf_size | Optional | number | Size of socket send buffer in bytes +enable_recv_pipe | Optional | boolean | Enable or disable receive pipe +enable_zerocopy_send | Optional | boolean | Enable or disable zero copy on send for client and server sockets +enable_quick_ack | Optional | boolean | Enable or disable quick ACK +enable_placement_id | Optional | number | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu +enable_zerocopy_send_server | Optional | boolean | Enable or disable zero copy on send for server sockets +enable_zerocopy_send_client | Optional | boolean | Enable or disable zero copy on send for client sockets ### Response @@ -8632,7 +8638,9 @@ Example request: "enable_recv_pipe": false, "enable_zerocopy_send": true, "enable_quick_ack": false, - "enable_placement_id": 0 + "enable_placement_id": 0, + "enable_zerocopy_send_server": true, + "enable_zerocopy_send_client": false } } ~~~ diff --git a/examples/nvme/perf/perf.c b/examples/nvme/perf/perf.c index b345859575..844d6908fc 100644 --- a/examples/nvme/perf/perf.c +++ b/examples/nvme/perf/perf.c @@ -335,6 +335,7 @@ perf_set_sock_zcopy(const char *impl_name, bool enable) } sock_opts.enable_zerocopy_send = enable; + sock_opts.enable_zerocopy_send_client = enable; if (spdk_sock_impl_set_opts(impl_name, &sock_opts, opts_size)) { fprintf(stderr, "Failed to %s zcopy send for sock impl %s: error %d (%s)\n", diff --git a/include/spdk/sock.h b/include/spdk/sock.h index 73b6a2046c..1fcecf93ec 100644 --- a/include/spdk/sock.h +++ b/include/spdk/sock.h @@ -126,6 +126,15 @@ struct spdk_sock_impl_opts { */ uint32_t enable_placement_id; + /** + * Enable or disable use of zero copy flow on send for server sockets. Used by posix socket module. + */ + bool enable_zerocopy_send_server; + + /** + * Enable or disable use of zero copy flow on send for client sockets. Used by posix socket module. + */ + bool enable_zerocopy_send_client; }; /** diff --git a/lib/sock/sock.c b/lib/sock/sock.c index 1438f12d83..320e7474e9 100644 --- a/lib/sock/sock.c +++ b/lib/sock/sock.c @@ -791,6 +791,8 @@ spdk_sock_write_config_json(struct spdk_json_write_ctx *w) spdk_json_write_named_bool(w, "enable_zerocopy_send", opts.enable_zerocopy_send); spdk_json_write_named_bool(w, "enable_quickack", opts.enable_quickack); spdk_json_write_named_uint32(w, "enable_placement_id", opts.enable_placement_id); + spdk_json_write_named_bool(w, "enable_zerocopy_send_server", opts.enable_zerocopy_send_server); + spdk_json_write_named_bool(w, "enable_zerocopy_send_client", opts.enable_zerocopy_send_client); spdk_json_write_object_end(w); spdk_json_write_object_end(w); } else { diff --git a/lib/sock/sock_rpc.c b/lib/sock/sock_rpc.c index 602fb86991..46d725cbdb 100644 --- a/lib/sock/sock_rpc.c +++ b/lib/sock/sock_rpc.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -77,6 +77,8 @@ rpc_sock_impl_get_options(struct spdk_jsonrpc_request *request, spdk_json_write_named_bool(w, "enable_zerocopy_send", sock_opts.enable_zerocopy_send); spdk_json_write_named_bool(w, "enable_quickack", sock_opts.enable_quickack); spdk_json_write_named_uint32(w, "enable_placement_id", sock_opts.enable_placement_id); + spdk_json_write_named_bool(w, "enable_zerocopy_send_server", sock_opts.enable_zerocopy_send_server); + spdk_json_write_named_bool(w, "enable_zerocopy_send_client", sock_opts.enable_zerocopy_send_client); spdk_json_write_object_end(w); spdk_jsonrpc_end_result(request, w); free(impl_name); @@ -118,7 +120,14 @@ static const struct spdk_json_object_decoder rpc_sock_impl_set_opts_decoders[] = "enable_placement_id", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_placement_id), spdk_json_decode_uint32, true }, - + { + "enable_zerocopy_send_server", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send_server), + spdk_json_decode_bool, true + }, + { + "enable_zerocopy_send_client", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send_client), + spdk_json_decode_bool, true + } }; static void diff --git a/module/sock/posix/posix.c b/module/sock/posix/posix.c index 53dc1d4983..59125f823e 100644 --- a/module/sock/posix/posix.c +++ b/module/sock/posix/posix.c @@ -92,6 +92,8 @@ static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = { .enable_zerocopy_send = true, .enable_quickack = false, .enable_placement_id = PLACEMENT_NONE, + .enable_zerocopy_send_server = true, + .enable_zerocopy_send_client = false }; static struct spdk_sock_map g_map = { @@ -348,7 +350,7 @@ posix_sock_alloc(int fd, bool enable_zero_copy) #if defined(SPDK_ZEROCOPY) flag = 1; - if (enable_zero_copy && g_spdk_posix_sock_impl_opts.enable_zerocopy_send) { + if (enable_zero_copy) { /* Try to turn on zero copy sends */ rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag)); if (rc == 0) { @@ -441,7 +443,8 @@ posix_sock_create(const char *ip, int port, int fd, flag; int val = 1; int rc, sz; - bool enable_zero_copy = true; + bool enable_zcopy_user_opts = true; + bool enable_zcopy_impl_opts = true; assert(opts != NULL); @@ -555,6 +558,8 @@ retry: fd = -1; break; } + enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_server && + g_spdk_posix_sock_impl_opts.enable_zerocopy_send; } else if (type == SPDK_SOCK_CREATE_CONNECT) { rc = connect(fd, res->ai_addr, res->ai_addrlen); if (rc != 0) { @@ -564,6 +569,8 @@ retry: fd = -1; continue; } + enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_client && + g_spdk_posix_sock_impl_opts.enable_zerocopy_send; } flag = fcntl(fd, F_GETFL); @@ -582,9 +589,9 @@ retry: } /* Only enable zero copy for non-loopback sockets. */ - enable_zero_copy = opts->zcopy && !sock_is_loopback(fd); + enable_zcopy_user_opts = opts->zcopy && !sock_is_loopback(fd); - sock = posix_sock_alloc(fd, enable_zero_copy); + sock = posix_sock_alloc(fd, enable_zcopy_user_opts && enable_zcopy_impl_opts); if (sock == NULL) { SPDK_ERRLOG("sock allocation failed\n"); close(fd); @@ -1524,6 +1531,8 @@ posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len) GET_FIELD(enable_zerocopy_send); GET_FIELD(enable_quickack); GET_FIELD(enable_placement_id); + GET_FIELD(enable_zerocopy_send_server); + GET_FIELD(enable_zerocopy_send_client); #undef GET_FIELD #undef FIELD_OK @@ -1554,6 +1563,8 @@ posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len) SET_FIELD(enable_zerocopy_send); SET_FIELD(enable_quickack); SET_FIELD(enable_placement_id); + SET_FIELD(enable_zerocopy_send_server); + SET_FIELD(enable_zerocopy_send_client); #undef SET_FIELD #undef FIELD_OK diff --git a/scripts/rpc.py b/scripts/rpc.py index 3de543e9eb..7fecc53885 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2606,7 +2606,9 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse enable_recv_pipe=args.enable_recv_pipe, enable_zerocopy_send=args.enable_zerocopy_send, enable_quickack=args.enable_quickack, - enable_placement_id=args.enable_placement_id) + enable_placement_id=args.enable_placement_id, + enable_zerocopy_send_server=args.enable_zerocopy_send_server, + enable_zerocopy_send_client=args.enable_zerocopy_send_client) p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""") p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True) @@ -2625,8 +2627,17 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse action='store_true', dest='enable_quickack') p.add_argument('--disable-quickack', help='Disable quick ACK', action='store_false', dest='enable_quickack') + p.add_argument('--enable-zerocopy-send-server', help='Enable zerocopy on send for server sockets', + action='store_true', dest='enable_zerocopy_send_server') + p.add_argument('--disable-zerocopy-send-server', help='Disable zerocopy on send for server sockets', + action='store_false', dest='enable_zerocopy_server_client') + p.add_argument('--enable-zerocopy-send-client', help='Enable zerocopy on send for client sockets', + action='store_true', dest='enable_zerocopy_send_client') + p.add_argument('--disable-zerocopy-send-client', help='Disable zerocopy on send for client sockets', + action='store_false', dest='enable_zerocopy_send_client') p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_zerocopy_send=None, - enable_quickack=None, enable_placement_id=None) + enable_quickack=None, enable_placement_id=None, enable_zerocopy_send_server=None, + enable_zerocopy_send_client=None) def sock_set_default_impl(args): print_json(rpc.sock.sock_set_default_impl(args.client, diff --git a/scripts/rpc/sock.py b/scripts/rpc/sock.py index 081bf91b15..450822069f 100644 --- a/scripts/rpc/sock.py +++ b/scripts/rpc/sock.py @@ -18,7 +18,9 @@ def sock_impl_set_options(client, enable_recv_pipe=None, enable_zerocopy_send=None, enable_quickack=None, - enable_placement_id=None): + enable_placement_id=None, + enable_zerocopy_send_server=None, + enable_zerocopy_send_client=None): """Set parameters for the socket layer implementation. Args: @@ -29,6 +31,8 @@ def sock_impl_set_options(client, enable_zerocopy_send: enable or disable zerocopy on send (optional) enable_quickack: enable or disable quickack (optional) enable_placement_id: option for placement_id. 0:disable,1:incoming_napi,2:incoming_cpu (optional) + enable_zerocopy_send_server: enable or disable zerocopy on send for server sockets(optional) + enable_zerocopy_send_client: enable or disable zerocopy on send for client sockets(optional) """ params = {} @@ -45,6 +49,10 @@ def sock_impl_set_options(client, params['enable_quickack'] = enable_quickack if enable_placement_id is not None: params['enable_placement_id'] = enable_placement_id + if enable_zerocopy_send_server is not None: + params['enable_zerocopy_send_server'] = enable_zerocopy_send_server + if enable_zerocopy_send_client is not None: + params['enable_zerocopy_send_client'] = enable_zerocopy_send_client return client.call('sock_impl_set_options', params)