diff --git a/CHANGELOG.md b/CHANGELOG.md index 241417bc72..f7667a84fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -179,6 +179,12 @@ sockets to be marked using the SO_MARK socket option as a hint for which hardwar queue they should be associated with. This mode leverages that by setting the same value for all sockets within a poll group. +New parameters `enable_zerocopy_send_server` and `enable_zerocopy_send_client` were added +to struct spdk_sock_impl_opts, these parameters enable or disable zero copy send for server +and client sockets which are created using `spdk_sock_listen` and `spdk_sock_listen_ext` (server); +`spdk_sock_connect` and `spdk_sock_connect_ext` (client) functions. Existing parameter +`enable_zerocopy_send` enables or disables zero copy send for both server and client sockets. + ### thread A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified diff --git a/doc/jsonrpc.md b/doc/jsonrpc.md index 876c00f4d4..2833a0125f 100644 --- a/doc/jsonrpc.md +++ b/doc/jsonrpc.md @@ -8591,7 +8591,11 @@ Example response: "recv_buf_size": 2097152, "send_buf_size": 2097152, "enable_recv_pipe": true, - "enable_zerocopy_send": true + "enable_zerocopy_send": true, + "enable_quickack": true, + "enable_placement_id": 0, + "enable_zerocopy_send_server": true, + "enable_zerocopy_send_client": false } } ~~~ @@ -8602,15 +8606,17 @@ Set parameters for the socket layer implementation. ### Parameters -Name | Optional | Type | Description ------------------------ | -------- | ----------- | ----------- -impl_name | Required | string | Name of socket implementation, e.g. posix -recv_buf_size | Optional | number | Size of socket receive buffer in bytes -send_buf_size | Optional | number | Size of socket send buffer in bytes -enable_recv_pipe | Optional | boolean | Enable or disable receive pipe -enable_zerocopy_send | Optional | boolean | Enable or disable zero copy on send -enable_quick_ack | Optional | boolean | Enable or disable quick ACK -enable_placement_id | Optional | number | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu +Name | Optional | Type | Description +--------------------------- | -------- | ----------- | ----------- +impl_name | Required | string | Name of socket implementation, e.g. posix +recv_buf_size | Optional | number | Size of socket receive buffer in bytes +send_buf_size | Optional | number | Size of socket send buffer in bytes +enable_recv_pipe | Optional | boolean | Enable or disable receive pipe +enable_zerocopy_send | Optional | boolean | Enable or disable zero copy on send for client and server sockets +enable_quick_ack | Optional | boolean | Enable or disable quick ACK +enable_placement_id | Optional | number | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu +enable_zerocopy_send_server | Optional | boolean | Enable or disable zero copy on send for server sockets +enable_zerocopy_send_client | Optional | boolean | Enable or disable zero copy on send for client sockets ### Response @@ -8632,7 +8638,9 @@ Example request: "enable_recv_pipe": false, "enable_zerocopy_send": true, "enable_quick_ack": false, - "enable_placement_id": 0 + "enable_placement_id": 0, + "enable_zerocopy_send_server": true, + "enable_zerocopy_send_client": false } } ~~~ diff --git a/examples/nvme/perf/perf.c b/examples/nvme/perf/perf.c index b345859575..844d6908fc 100644 --- a/examples/nvme/perf/perf.c +++ b/examples/nvme/perf/perf.c @@ -335,6 +335,7 @@ perf_set_sock_zcopy(const char *impl_name, bool enable) } sock_opts.enable_zerocopy_send = enable; + sock_opts.enable_zerocopy_send_client = enable; if (spdk_sock_impl_set_opts(impl_name, &sock_opts, opts_size)) { fprintf(stderr, "Failed to %s zcopy send for sock impl %s: error %d (%s)\n", diff --git a/include/spdk/sock.h b/include/spdk/sock.h index 73b6a2046c..1fcecf93ec 100644 --- a/include/spdk/sock.h +++ b/include/spdk/sock.h @@ -126,6 +126,15 @@ struct spdk_sock_impl_opts { */ uint32_t enable_placement_id; + /** + * Enable or disable use of zero copy flow on send for server sockets. Used by posix socket module. + */ + bool enable_zerocopy_send_server; + + /** + * Enable or disable use of zero copy flow on send for client sockets. Used by posix socket module. + */ + bool enable_zerocopy_send_client; }; /** diff --git a/lib/sock/sock.c b/lib/sock/sock.c index 1438f12d83..320e7474e9 100644 --- a/lib/sock/sock.c +++ b/lib/sock/sock.c @@ -791,6 +791,8 @@ spdk_sock_write_config_json(struct spdk_json_write_ctx *w) spdk_json_write_named_bool(w, "enable_zerocopy_send", opts.enable_zerocopy_send); spdk_json_write_named_bool(w, "enable_quickack", opts.enable_quickack); spdk_json_write_named_uint32(w, "enable_placement_id", opts.enable_placement_id); + spdk_json_write_named_bool(w, "enable_zerocopy_send_server", opts.enable_zerocopy_send_server); + spdk_json_write_named_bool(w, "enable_zerocopy_send_client", opts.enable_zerocopy_send_client); spdk_json_write_object_end(w); spdk_json_write_object_end(w); } else { diff --git a/lib/sock/sock_rpc.c b/lib/sock/sock_rpc.c index 602fb86991..46d725cbdb 100644 --- a/lib/sock/sock_rpc.c +++ b/lib/sock/sock_rpc.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved. + * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -77,6 +77,8 @@ rpc_sock_impl_get_options(struct spdk_jsonrpc_request *request, spdk_json_write_named_bool(w, "enable_zerocopy_send", sock_opts.enable_zerocopy_send); spdk_json_write_named_bool(w, "enable_quickack", sock_opts.enable_quickack); spdk_json_write_named_uint32(w, "enable_placement_id", sock_opts.enable_placement_id); + spdk_json_write_named_bool(w, "enable_zerocopy_send_server", sock_opts.enable_zerocopy_send_server); + spdk_json_write_named_bool(w, "enable_zerocopy_send_client", sock_opts.enable_zerocopy_send_client); spdk_json_write_object_end(w); spdk_jsonrpc_end_result(request, w); free(impl_name); @@ -118,7 +120,14 @@ static const struct spdk_json_object_decoder rpc_sock_impl_set_opts_decoders[] = "enable_placement_id", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_placement_id), spdk_json_decode_uint32, true }, - + { + "enable_zerocopy_send_server", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send_server), + spdk_json_decode_bool, true + }, + { + "enable_zerocopy_send_client", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send_client), + spdk_json_decode_bool, true + } }; static void diff --git a/module/sock/posix/posix.c b/module/sock/posix/posix.c index 53dc1d4983..59125f823e 100644 --- a/module/sock/posix/posix.c +++ b/module/sock/posix/posix.c @@ -92,6 +92,8 @@ static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = { .enable_zerocopy_send = true, .enable_quickack = false, .enable_placement_id = PLACEMENT_NONE, + .enable_zerocopy_send_server = true, + .enable_zerocopy_send_client = false }; static struct spdk_sock_map g_map = { @@ -348,7 +350,7 @@ posix_sock_alloc(int fd, bool enable_zero_copy) #if defined(SPDK_ZEROCOPY) flag = 1; - if (enable_zero_copy && g_spdk_posix_sock_impl_opts.enable_zerocopy_send) { + if (enable_zero_copy) { /* Try to turn on zero copy sends */ rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag)); if (rc == 0) { @@ -441,7 +443,8 @@ posix_sock_create(const char *ip, int port, int fd, flag; int val = 1; int rc, sz; - bool enable_zero_copy = true; + bool enable_zcopy_user_opts = true; + bool enable_zcopy_impl_opts = true; assert(opts != NULL); @@ -555,6 +558,8 @@ retry: fd = -1; break; } + enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_server && + g_spdk_posix_sock_impl_opts.enable_zerocopy_send; } else if (type == SPDK_SOCK_CREATE_CONNECT) { rc = connect(fd, res->ai_addr, res->ai_addrlen); if (rc != 0) { @@ -564,6 +569,8 @@ retry: fd = -1; continue; } + enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_client && + g_spdk_posix_sock_impl_opts.enable_zerocopy_send; } flag = fcntl(fd, F_GETFL); @@ -582,9 +589,9 @@ retry: } /* Only enable zero copy for non-loopback sockets. */ - enable_zero_copy = opts->zcopy && !sock_is_loopback(fd); + enable_zcopy_user_opts = opts->zcopy && !sock_is_loopback(fd); - sock = posix_sock_alloc(fd, enable_zero_copy); + sock = posix_sock_alloc(fd, enable_zcopy_user_opts && enable_zcopy_impl_opts); if (sock == NULL) { SPDK_ERRLOG("sock allocation failed\n"); close(fd); @@ -1524,6 +1531,8 @@ posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len) GET_FIELD(enable_zerocopy_send); GET_FIELD(enable_quickack); GET_FIELD(enable_placement_id); + GET_FIELD(enable_zerocopy_send_server); + GET_FIELD(enable_zerocopy_send_client); #undef GET_FIELD #undef FIELD_OK @@ -1554,6 +1563,8 @@ posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len) SET_FIELD(enable_zerocopy_send); SET_FIELD(enable_quickack); SET_FIELD(enable_placement_id); + SET_FIELD(enable_zerocopy_send_server); + SET_FIELD(enable_zerocopy_send_client); #undef SET_FIELD #undef FIELD_OK diff --git a/scripts/rpc.py b/scripts/rpc.py index 3de543e9eb..7fecc53885 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2606,7 +2606,9 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse enable_recv_pipe=args.enable_recv_pipe, enable_zerocopy_send=args.enable_zerocopy_send, enable_quickack=args.enable_quickack, - enable_placement_id=args.enable_placement_id) + enable_placement_id=args.enable_placement_id, + enable_zerocopy_send_server=args.enable_zerocopy_send_server, + enable_zerocopy_send_client=args.enable_zerocopy_send_client) p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""") p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True) @@ -2625,8 +2627,17 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse action='store_true', dest='enable_quickack') p.add_argument('--disable-quickack', help='Disable quick ACK', action='store_false', dest='enable_quickack') + p.add_argument('--enable-zerocopy-send-server', help='Enable zerocopy on send for server sockets', + action='store_true', dest='enable_zerocopy_send_server') + p.add_argument('--disable-zerocopy-send-server', help='Disable zerocopy on send for server sockets', + action='store_false', dest='enable_zerocopy_server_client') + p.add_argument('--enable-zerocopy-send-client', help='Enable zerocopy on send for client sockets', + action='store_true', dest='enable_zerocopy_send_client') + p.add_argument('--disable-zerocopy-send-client', help='Disable zerocopy on send for client sockets', + action='store_false', dest='enable_zerocopy_send_client') p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_zerocopy_send=None, - enable_quickack=None, enable_placement_id=None) + enable_quickack=None, enable_placement_id=None, enable_zerocopy_send_server=None, + enable_zerocopy_send_client=None) def sock_set_default_impl(args): print_json(rpc.sock.sock_set_default_impl(args.client, diff --git a/scripts/rpc/sock.py b/scripts/rpc/sock.py index 081bf91b15..450822069f 100644 --- a/scripts/rpc/sock.py +++ b/scripts/rpc/sock.py @@ -18,7 +18,9 @@ def sock_impl_set_options(client, enable_recv_pipe=None, enable_zerocopy_send=None, enable_quickack=None, - enable_placement_id=None): + enable_placement_id=None, + enable_zerocopy_send_server=None, + enable_zerocopy_send_client=None): """Set parameters for the socket layer implementation. Args: @@ -29,6 +31,8 @@ def sock_impl_set_options(client, enable_zerocopy_send: enable or disable zerocopy on send (optional) enable_quickack: enable or disable quickack (optional) enable_placement_id: option for placement_id. 0:disable,1:incoming_napi,2:incoming_cpu (optional) + enable_zerocopy_send_server: enable or disable zerocopy on send for server sockets(optional) + enable_zerocopy_send_client: enable or disable zerocopy on send for client sockets(optional) """ params = {} @@ -45,6 +49,10 @@ def sock_impl_set_options(client, params['enable_quickack'] = enable_quickack if enable_placement_id is not None: params['enable_placement_id'] = enable_placement_id + if enable_zerocopy_send_server is not None: + params['enable_zerocopy_send_server'] = enable_zerocopy_send_server + if enable_zerocopy_send_client is not None: + params['enable_zerocopy_send_client'] = enable_zerocopy_send_client return client.call('sock_impl_set_options', params)