bdev/rbd: Add key_file to the rbd_register_cluster RPC

In project practice, config_file and key_file are often used to connect
to a rados cluster, config_file includes "mon_host" and other rados
configurations like "rbd_cache", and key_file includes the secret key
and the access authority to each pool for current user. This patch adds
key_file option, user can specify config_file and key_file or only config_param
to connect rados cluster. This will make it much more flexible for users with
his/her convenience.

Signed-off-by: Tan Long <tanl12@chinatelecom.cn>
Change-Id: I6b49aad70b578bdeb3ac8ea9ca0fcbd931582025
Signed-off-by: Tan Long <tanl12@chinatelecom.cn>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10485
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: GangCao <gang.cao@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Xiaodong Liu <xiaodong.liu@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

CHANGELOG.md

@@ -28,6 +28,9 @@ removed in SPDK 22.04, and the parameter `transport_retry_count` is added and us
 
 An new parameter `bdev_retry_count` is added to the RPC `bdev_nvme_set_options`.
 
+Added 'key_file' parameter to the 'rbd_register_cluster' RPC.  It is an optional parameter to
+specify a keyring file to connect to a RADOS cluster.
+
 ### nvme
 
 New APIs, `spdk_nvme_ctrlr_disconnect`, `spdk_nvme_ctrlr_reconnect_async`, and

doc/jsonrpc.md

@@ -3606,37 +3606,42 @@ name                    | Required | string      | Registerd Rados cluster objec
 user_id                 | Optional | string      | Ceph ID (i.e. admin, not client.admin)
 config_param            | Optional | string map  | Explicit librados configuration
 config_file             | Optional | string      | File path of libraodos configuration file
+key_file                | Optional | string      | File path of libraodos key file
 
 This RPC registers a Rados Cluster object handle which is only known
-to rbd module, it uses user_id + config_param or user_id + config_file to
-identify a Rados cluster object.
-
-If no config_param is specified, Ceph configuration files must exist with
-all relevant settings for accessing the Ceph cluster. If a config map is
-passed, the configuration files are ignored and instead all key/value
-pairs are passed to rados_conf_set to configure cluster access. In
-practice, "mon_host" (= list of monitor address+port) and "key" (= the
-secret key stored in Ceph keyrings) are enough.
+to rbd module, it uses user_id + config_param or user_id + config_file +
+key_file or user_id + config_param + config_file + key_file to identify
+a Rados cluster object.
 
 When accessing the Ceph cluster as some user other than "admin" (the
 default), the "user_id" has to be set.
 
+The configuration items and secret key can be specified by setting config_param,
+config_file and key_file, all of them, or none of them. If only config_param is
+passed, all key/value pairs are passed to rados_conf_set to configure cluster access.
+In practice, "mon_host" (= list of monitor address+port) and "key" (= the secret key
+stored in Ceph keyrings) are enough. If config_file and key_file are specified, they must
+exist with all relevant settings for accessing the Ceph cluster. If config_param, config_file
+and key_file are specified, get the key/value pairs from config_file first and set to
+rados_conf_set function, then set pairs in config_param and keyring in key_file. If nothing
+is specified, it will get configuration file and key file from the default location
+/etc/ceph/ceph.conf and /etc/ceph/ceph.client.user_id.keyring.
+
 #### Result
 
 Name of newly created Rados cluster object.
 
 #### Example
 
-Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`:
+Example request:
 
 ~~
 {
   "params": {
     "name": "rbd_cluster",
-    "config_param": {
-      "mon_host": "192.168.7.1:6789,192.168.7.2:6789",
-      "key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==",
-    }
+    "user_id": cinder,
+    "config_file": "/root/ceph_conf/ceph.conf",
+    "key_file": "/root/ceph_conf/ceph.client.cinder.keyring"
   },
   "jsonrpc": "2.0",
   "method": "bdev_rbd_register_cluster",

module/bdev/rbd/bdev_rbd.c

@@ -93,6 +93,7 @@ struct bdev_rbd_cluster {
 	char *user_id;
 	char **config_param;
 	char *config_file;
+	char *key_file;
 	rados_t cluster;
 	uint32_t ref;
 	STAILQ_ENTRY(bdev_rbd_cluster) link;
@@ -109,6 +110,7 @@ bdev_rbd_cluster_free(struct bdev_rbd_cluster *entry)
 
 	bdev_rbd_free_config(entry->config_param);
 	free(entry->config_file);
+	free(entry->key_file);
 	free(entry->user_id);
 	free(entry->name);
 	free(entry);
@@ -803,6 +805,9 @@ bdev_rbd_cluster_dump_entry(const char *cluster_name, struct spdk_json_write_ctx
 		} else if (entry->config_file) {
 			spdk_json_write_named_string(w, "config_file", entry->config_file);
 		}
+		if (entry->key_file) {
+			spdk_json_write_named_string(w, "key_file", entry->key_file);
+		}
 
 		pthread_mutex_unlock(&g_map_bdev_rbd_cluster_mutex);
 		return;
@@ -910,6 +915,9 @@ dump_single_cluster_entry(struct bdev_rbd_cluster *entry, struct spdk_json_write
 	} else if (entry->config_file) {
 		spdk_json_write_named_string(w, "config_file", entry->config_file);
 	}
+	if (entry->key_file) {
+		spdk_json_write_named_string(w, "key_file", entry->key_file);
+	}
 
 	spdk_json_write_object_end(w);
 }
@@ -967,7 +975,7 @@ static const struct spdk_bdev_fn_table rbd_fn_table = {
 
 static int
 rbd_register_cluster(const char *name, const char *user_id, const char *const *config_param,
-		     const char *config_file)
+		     const char *config_file, const char *key_file)
 {
 	struct bdev_rbd_cluster *entry;
 	int rc;
@@ -1017,6 +1025,14 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c
 		}
 	}
 
+	if (key_file) {
+		entry->key_file = strdup(key_file);
+		if (entry->key_file == NULL) {
+			SPDK_ERRLOG("Failed to save the key_file=%s on entry = %p\n", key_file, entry);
+			goto err_handle;
+		}
+	}
+
 	rc = rados_create(&entry->cluster, user_id);
 	if (rc < 0) {
 		SPDK_ERRLOG("Failed to create rados_t struct\n");
@@ -1043,6 +1059,15 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c
 		}
 	}
 
+	if (key_file) {
+		rc = rados_conf_set(entry->cluster, "keyring", key_file);
+		if (rc < 0) {
+			SPDK_ERRLOG("Failed to set keyring = %s\n", key_file);
+			rados_shutdown(entry->cluster);
+			goto err_handle;
+		}
+	}
+
 	rc = rados_connect(entry->cluster);
 	if (rc < 0) {
 		SPDK_ERRLOG("Failed to connect to rbd_pool on cluster=%p\n", entry->cluster);
@@ -1104,7 +1129,8 @@ _bdev_rbd_register_cluster(void *arg)
 	int rc;
 
 	rc = rbd_register_cluster((const char *)info->name, (const char *)info->user_id,
-				  (const char *const *)info->config_param, (const char *)info->config_file);
+				  (const char *const *)info->config_param, (const char *)info->config_file,
+				  (const char *)info->key_file);
 	if (rc) {
 		ret = NULL;
 	}

module/bdev/rbd/bdev_rbd.h

@@ -44,6 +44,7 @@ struct cluster_register_info {
 	char *user_id;
 	char **config_param;
 	char *config_file;
+	char *key_file;
 };
 
 void bdev_rbd_free_config(char **config);

module/bdev/rbd/bdev_rbd_rpc.c

@@ -269,13 +269,15 @@ free_rpc_register_cluster(struct cluster_register_info *req)
 	free(req->user_id);
 	bdev_rbd_free_config(req->config_param);
 	free(req->config_file);
+	free(req->key_file);
 }
 
 static const struct spdk_json_object_decoder rpc_register_cluster_decoders[] = {
 	{"name", offsetof(struct cluster_register_info, name), spdk_json_decode_string, true},
 	{"user_id", offsetof(struct cluster_register_info, user_id), spdk_json_decode_string, true},
 	{"config_param", offsetof(struct cluster_register_info, config_param), bdev_rbd_decode_config, true},
-	{"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true}
+	{"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true},
+	{"key_file", offsetof(struct cluster_register_info, key_file), spdk_json_decode_string, true}
 };
 
 static void

scripts/rpc.py

@@ -704,7 +704,8 @@ if __name__ == "__main__":
                                                       name=args.name,
                                                       user=args.user,
                                                       config_param=config_param,
-                                                      config_file=args.config_file))
+                                                      config_file=args.config_file,
+                                                      key_file=args.key_file))
 
     p = subparsers.add_parser('bdev_rbd_register_cluster',
                               help='Add a Rados cluster with ceph rbd backend')
@@ -713,6 +714,7 @@ if __name__ == "__main__":
     p.add_argument('--config-param', action='append', metavar='key=value',
                    help="adds a key=value configuration option for rados_conf_set (default: rely on config file)")
     p.add_argument('--config-file', help="The file path of the Rados configuration file", required=False)
+    p.add_argument('--key-file', help="The file path of the Rados keyring file", required=False)
     p.set_defaults(func=bdev_rbd_register_cluster)
 
     def bdev_rbd_unregister_cluster(args):

scripts/rpc/bdev.py

@@ -747,7 +747,7 @@ def bdev_zone_block_delete(client, name):
     return client.call('bdev_zone_block_delete', params)
 
 
-def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config_file=None):
+def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config_file=None, key_file=None):
     """Create a Rados Cluster object of the Ceph RBD backend.
 
     Args:
@@ -755,6 +755,7 @@ def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config
         user: Ceph user name (optional)
         config_param: map of config keys to values (optional)
         config_file: file path of Ceph configuration file (optional)
+        key_file: file path of Ceph key file (optional)
 
     Returns:
         Name of registered Rados Cluster object.
@@ -767,6 +768,8 @@ def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config
         params['config_param'] = config_param
     if config_file is not None:
         params['config_file'] = config_file
+    if key_file is not None:
+        params['key_file'] = key_file
 
     return client.call('bdev_rbd_register_cluster', params)