sock: Add new params to configure zcopy for server, client sockets

When zcero copy send is enabled and used by initiator,
it could significantly increase latency in some payloads.
To enable more fine graing configuration of zero copy
send feature, add new parameters enable_zerocopy_send_server
and enable_zerocopy_send_client to spdk_sock_impl_opts to
enable/disable zcopy for specific type of sockets.
Exisiting enable_zerocopy_send parameter affects all types
of sockets.

Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7457 (master)

(cherry picked from commit 8e85b675fc)
Change-Id: I111c75608f8826980a56e210c076ab8ff16ddbdc
Signed-off-by: Krzysztof Karas <krzysztof.karas@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7638
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

CHANGELOG.md

@@ -179,6 +179,12 @@ sockets to be marked using the SO_MARK socket option as a hint for which hardwar
 queue they should be associated with. This mode leverages that by setting the same
 value for all sockets within a poll group.
 
+New parameters `enable_zerocopy_send_server` and `enable_zerocopy_send_client` were added
+to struct spdk_sock_impl_opts, these parameters enable or disable zero copy send for server
+and client sockets which are created using `spdk_sock_listen` and `spdk_sock_listen_ext` (server);
+`spdk_sock_connect` and `spdk_sock_connect_ext` (client) functions. Existing parameter
+`enable_zerocopy_send`  enables or disables zero copy send for both server and client sockets.
+
 ### thread
 
 A new API `spdk_io_channel_get_io_device` was added to get the io_device for the specified

doc/jsonrpc.md

@@ -8591,7 +8591,11 @@ Example response:
     "recv_buf_size": 2097152,
     "send_buf_size": 2097152,
     "enable_recv_pipe": true,
-    "enable_zerocopy_send": true
+    "enable_zerocopy_send": true,
+    "enable_quickack": true,
+    "enable_placement_id": 0,
+    "enable_zerocopy_send_server": true,
+    "enable_zerocopy_send_client": false
   }
 }
 ~~~
@@ -8603,14 +8607,16 @@ Set parameters for the socket layer implementation.
 ### Parameters
 
 Name                        | Optional | Type        | Description
------------------------ | -------- | ----------- | -----------
+--------------------------- | -------- | ----------- | -----------
 impl_name                   | Required | string      | Name of socket implementation, e.g. posix
 recv_buf_size               | Optional | number      | Size of socket receive buffer in bytes
 send_buf_size               | Optional | number      | Size of socket send buffer in bytes
 enable_recv_pipe            | Optional | boolean     | Enable or disable receive pipe
-enable_zerocopy_send    | Optional | boolean     | Enable or disable zero copy on send
+enable_zerocopy_send        | Optional | boolean     | Enable or disable zero copy on send for client and server sockets
 enable_quick_ack            | Optional | boolean     | Enable or disable quick ACK
 enable_placement_id         | Optional | number      | Enable or disable placement_id. 0:disable,1:incoming_napi,2:incoming_cpu
+enable_zerocopy_send_server | Optional | boolean     | Enable or disable zero copy on send for server sockets
+enable_zerocopy_send_client | Optional | boolean     | Enable or disable zero copy on send for client sockets
 
 ### Response
 
@@ -8632,7 +8638,9 @@ Example request:
     "enable_recv_pipe": false,
     "enable_zerocopy_send": true,
     "enable_quick_ack": false,
-    "enable_placement_id": 0
+    "enable_placement_id": 0,
+    "enable_zerocopy_send_server": true,
+    "enable_zerocopy_send_client": false
   }
 }
 ~~~

examples/nvme/perf/perf.c

@@ -335,6 +335,7 @@ perf_set_sock_zcopy(const char *impl_name, bool enable)
 	}
 
 	sock_opts.enable_zerocopy_send = enable;
+	sock_opts.enable_zerocopy_send_client = enable;
 
 	if (spdk_sock_impl_set_opts(impl_name, &sock_opts, opts_size)) {
 		fprintf(stderr, "Failed to %s zcopy send for sock impl %s: error %d (%s)\n",

include/spdk/sock.h

@@ -126,6 +126,15 @@ struct spdk_sock_impl_opts {
 	 */
 	uint32_t enable_placement_id;
 
+	/**
+	 * Enable or disable use of zero copy flow on send for server sockets. Used by posix socket module.
+	 */
+	bool enable_zerocopy_send_server;
+
+	/**
+	 * Enable or disable use of zero copy flow on send for client sockets. Used by posix socket module.
+	 */
+	bool enable_zerocopy_send_client;
 };
 
 /**

lib/sock/sock.c

@@ -791,6 +791,8 @@ spdk_sock_write_config_json(struct spdk_json_write_ctx *w)
 			spdk_json_write_named_bool(w, "enable_zerocopy_send", opts.enable_zerocopy_send);
 			spdk_json_write_named_bool(w, "enable_quickack", opts.enable_quickack);
 			spdk_json_write_named_uint32(w, "enable_placement_id", opts.enable_placement_id);
+			spdk_json_write_named_bool(w, "enable_zerocopy_send_server", opts.enable_zerocopy_send_server);
+			spdk_json_write_named_bool(w, "enable_zerocopy_send_client", opts.enable_zerocopy_send_client);
 			spdk_json_write_object_end(w);
 			spdk_json_write_object_end(w);
 		} else {

lib/sock/sock_rpc.c

@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved.
+ *   Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -77,6 +77,8 @@ rpc_sock_impl_get_options(struct spdk_jsonrpc_request *request,
 	spdk_json_write_named_bool(w, "enable_zerocopy_send", sock_opts.enable_zerocopy_send);
 	spdk_json_write_named_bool(w, "enable_quickack", sock_opts.enable_quickack);
 	spdk_json_write_named_uint32(w, "enable_placement_id", sock_opts.enable_placement_id);
+	spdk_json_write_named_bool(w, "enable_zerocopy_send_server", sock_opts.enable_zerocopy_send_server);
+	spdk_json_write_named_bool(w, "enable_zerocopy_send_client", sock_opts.enable_zerocopy_send_client);
 	spdk_json_write_object_end(w);
 	spdk_jsonrpc_end_result(request, w);
 	free(impl_name);
@@ -118,7 +120,14 @@ static const struct spdk_json_object_decoder rpc_sock_impl_set_opts_decoders[] =
 		"enable_placement_id", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_placement_id),
 		spdk_json_decode_uint32, true
 	},
-
+	{
+		"enable_zerocopy_send_server", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send_server),
+		spdk_json_decode_bool, true
+	},
+	{
+		"enable_zerocopy_send_client", offsetof(struct spdk_rpc_sock_impl_set_opts, sock_opts.enable_zerocopy_send_client),
+		spdk_json_decode_bool, true
+	}
 };
 
 static void

module/sock/posix/posix.c

@@ -92,6 +92,8 @@ static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = {
 	.enable_zerocopy_send = true,
 	.enable_quickack = false,
 	.enable_placement_id = PLACEMENT_NONE,
+	.enable_zerocopy_send_server = true,
+	.enable_zerocopy_send_client = false
 };
 
 static struct spdk_sock_map g_map = {
@@ -348,7 +350,7 @@ posix_sock_alloc(int fd, bool enable_zero_copy)
 #if defined(SPDK_ZEROCOPY)
 	flag = 1;
 
-	if (enable_zero_copy && g_spdk_posix_sock_impl_opts.enable_zerocopy_send) {
+	if (enable_zero_copy) {
 		/* Try to turn on zero copy sends */
 		rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag));
 		if (rc == 0) {
@@ -441,7 +443,8 @@ posix_sock_create(const char *ip, int port,
 	int fd, flag;
 	int val = 1;
 	int rc, sz;
-	bool enable_zero_copy = true;
+	bool enable_zcopy_user_opts = true;
+	bool enable_zcopy_impl_opts = true;
 
 	assert(opts != NULL);
 
@@ -555,6 +558,8 @@ retry:
 				fd = -1;
 				break;
 			}
+			enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_server &&
+						 g_spdk_posix_sock_impl_opts.enable_zerocopy_send;
 		} else if (type == SPDK_SOCK_CREATE_CONNECT) {
 			rc = connect(fd, res->ai_addr, res->ai_addrlen);
 			if (rc != 0) {
@@ -564,6 +569,8 @@ retry:
 				fd = -1;
 				continue;
 			}
+			enable_zcopy_impl_opts = g_spdk_posix_sock_impl_opts.enable_zerocopy_send_client &&
+						 g_spdk_posix_sock_impl_opts.enable_zerocopy_send;
 		}
 
 		flag = fcntl(fd, F_GETFL);
@@ -582,9 +589,9 @@ retry:
 	}
 
 	/* Only enable zero copy for non-loopback sockets. */
-	enable_zero_copy = opts->zcopy && !sock_is_loopback(fd);
+	enable_zcopy_user_opts = opts->zcopy && !sock_is_loopback(fd);
 
-	sock = posix_sock_alloc(fd, enable_zero_copy);
+	sock = posix_sock_alloc(fd, enable_zcopy_user_opts && enable_zcopy_impl_opts);
 	if (sock == NULL) {
 		SPDK_ERRLOG("sock allocation failed\n");
 		close(fd);
@@ -1524,6 +1531,8 @@ posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len)
 	GET_FIELD(enable_zerocopy_send);
 	GET_FIELD(enable_quickack);
 	GET_FIELD(enable_placement_id);
+	GET_FIELD(enable_zerocopy_send_server);
+	GET_FIELD(enable_zerocopy_send_client);
 
 #undef GET_FIELD
 #undef FIELD_OK
@@ -1554,6 +1563,8 @@ posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len)
 	SET_FIELD(enable_zerocopy_send);
 	SET_FIELD(enable_quickack);
 	SET_FIELD(enable_placement_id);
+	SET_FIELD(enable_zerocopy_send_server);
+	SET_FIELD(enable_zerocopy_send_client);
 
 #undef SET_FIELD
 #undef FIELD_OK

scripts/rpc.py

@@ -2606,7 +2606,9 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
                                        enable_recv_pipe=args.enable_recv_pipe,
                                        enable_zerocopy_send=args.enable_zerocopy_send,
                                        enable_quickack=args.enable_quickack,
-                                       enable_placement_id=args.enable_placement_id)
+                                       enable_placement_id=args.enable_placement_id,
+                                       enable_zerocopy_send_server=args.enable_zerocopy_send_server,
+                                       enable_zerocopy_send_client=args.enable_zerocopy_send_client)
 
     p = subparsers.add_parser('sock_impl_set_options', help="""Set options of socket layer implementation""")
     p.add_argument('-i', '--impl', help='Socket implementation name, e.g. posix', required=True)
@@ -2625,8 +2627,17 @@ Format: 'user:u1 secret:s1 muser:mu1 msecret:ms1,user:u2 secret:s2 muser:mu2 mse
                    action='store_true', dest='enable_quickack')
     p.add_argument('--disable-quickack', help='Disable quick ACK',
                    action='store_false', dest='enable_quickack')
+    p.add_argument('--enable-zerocopy-send-server', help='Enable zerocopy on send for server sockets',
+                   action='store_true', dest='enable_zerocopy_send_server')
+    p.add_argument('--disable-zerocopy-send-server', help='Disable zerocopy on send for server sockets',
+                   action='store_false', dest='enable_zerocopy_server_client')
+    p.add_argument('--enable-zerocopy-send-client', help='Enable zerocopy on send for client sockets',
+                   action='store_true', dest='enable_zerocopy_send_client')
+    p.add_argument('--disable-zerocopy-send-client', help='Disable zerocopy on send for client sockets',
+                   action='store_false', dest='enable_zerocopy_send_client')
     p.set_defaults(func=sock_impl_set_options, enable_recv_pipe=None, enable_zerocopy_send=None,
-                   enable_quickack=None, enable_placement_id=None)
+                   enable_quickack=None, enable_placement_id=None, enable_zerocopy_send_server=None,
+                   enable_zerocopy_send_client=None)
 
     def sock_set_default_impl(args):
         print_json(rpc.sock.sock_set_default_impl(args.client,

scripts/rpc/sock.py

@@ -18,7 +18,9 @@ def sock_impl_set_options(client,
                           enable_recv_pipe=None,
                           enable_zerocopy_send=None,
                           enable_quickack=None,
-                          enable_placement_id=None):
+                          enable_placement_id=None,
+                          enable_zerocopy_send_server=None,
+                          enable_zerocopy_send_client=None):
     """Set parameters for the socket layer implementation.
 
     Args:
@@ -29,6 +31,8 @@ def sock_impl_set_options(client,
         enable_zerocopy_send: enable or disable zerocopy on send (optional)
         enable_quickack: enable or disable quickack (optional)
         enable_placement_id: option for placement_id. 0:disable,1:incoming_napi,2:incoming_cpu (optional)
+        enable_zerocopy_send_server: enable or disable zerocopy on send for server sockets(optional)
+        enable_zerocopy_send_client: enable or disable zerocopy on send for client sockets(optional)
     """
     params = {}
 
@@ -45,6 +49,10 @@ def sock_impl_set_options(client,
         params['enable_quickack'] = enable_quickack
     if enable_placement_id is not None:
         params['enable_placement_id'] = enable_placement_id
+    if enable_zerocopy_send_server is not None:
+        params['enable_zerocopy_send_server'] = enable_zerocopy_send_server
+    if enable_zerocopy_send_client is not None:
+        params['enable_zerocopy_send_client'] = enable_zerocopy_send_client
 
     return client.call('sock_impl_set_options', params)