vhost: fix messages results handling

Return of message handling has now changed to an enum that can
take non-negative value that is not zero in case a reply is
needed. But the code checking the variable afterwards has not
been updated, leading to success messages handling being
treated as errors.

External post and pre callbacks return type needs also to be
changed to the new enum, so that its handling is consistent.
This is done in this patch alongside with the convertion of
its only user, vhost-crypto backend.

Fixes: 0bff510b5ea6 ("vhost: unify message handling function signature")

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>

lib/librte_vhost/vhost.h

@@ -284,6 +284,16 @@ struct guest_page {
 	uint64_t size;
 };
 
+/* The possible results of a message handling function */
+enum vh_result {
+	/* Message handling failed */
+	VH_RESULT_ERR   = -1,
+	/* Message handling successful */
+	VH_RESULT_OK    =  0,
+	/* Message handling successful and reply prepared */
+	VH_RESULT_REPLY =  1,
+};
+
 /**
  * function prototype for the vhost backend to handler specific vhost user
  * messages prior to the master message handling
@@ -292,17 +302,15 @@ struct guest_page {
  *  vhost device id
  * @param msg
  *  Message pointer.
- * @param require_reply
- *  If the handler requires sending a reply, this varaible shall be written 1,
- *  otherwise 0.
  * @param skip_master
  *  If the handler requires skipping the master message handling, this variable
  *  shall be written 1, otherwise 0.
  * @return
- *  0 on success, -1 on failure
+ *  VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply,
+ *  VH_RESULT_ERR on failure
  */
-typedef int (*vhost_msg_pre_handle)(int vid, void *msg,
+typedef enum vh_result (*vhost_msg_pre_handle)(int vid, void *msg,
-		uint32_t *require_reply, uint32_t *skip_master);
+		uint32_t *skip_master);
 
 /**
  * function prototype for the vhost backend to handler specific vhost user
@@ -312,14 +320,11 @@ typedef int (*vhost_msg_pre_handle)(int vid, void *msg,
  *  vhost device id
  * @param msg
  *  Message pointer.
- * @param require_reply
- *  If the handler requires sending a reply, this varaible shall be written 1,
- *  otherwise 0.
  * @return
- *  0 on success, -1 on failure
+ *  VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply,
+ *  VH_RESULT_ERR on failure
  */
-typedef int (*vhost_msg_post_handle)(int vid, void *msg,
+typedef enum vh_result (*vhost_msg_post_handle)(int vid, void *msg);
-		uint32_t *require_reply);
 
 /**
  * pre and post vhost user message handlers

lib/librte_vhost/vhost_crypto.c

@@ -425,35 +425,33 @@ vhost_crypto_close_sess(struct vhost_crypto *vcrypto, uint64_t session_id)
 	return 0;
 }
 
-static int
+static enum vh_result
-vhost_crypto_msg_post_handler(int vid, void *msg, uint32_t *require_reply)
+vhost_crypto_msg_post_handler(int vid, void *msg)
 {
 	struct virtio_net *dev = get_device(vid);
 	struct vhost_crypto *vcrypto;
 	VhostUserMsg *vmsg = msg;
-	int ret = 0;
+	enum vh_result ret = VH_RESULT_OK;
 
-	if (dev == NULL || require_reply == NULL) {
+	if (dev == NULL) {
 		VC_LOG_ERR("Invalid vid %i", vid);
-		return -EINVAL;
+		return VH_RESULT_ERR;
 	}
 
 	vcrypto = dev->extern_data;
 	if (vcrypto == NULL) {
 		VC_LOG_ERR("Cannot find required data, is it initialized?");
-		return -ENOENT;
+		return VH_RESULT_ERR;
 	}
 
-	*require_reply = 0;
-
 	if (vmsg->request.master == VHOST_USER_CRYPTO_CREATE_SESS) {
 		vhost_crypto_create_sess(vcrypto,
 				&vmsg->payload.crypto_session);
-		*require_reply = 1;
+		ret = VH_RESULT_REPLY;
-	} else if (vmsg->request.master == VHOST_USER_CRYPTO_CLOSE_SESS)
+	} else if (vmsg->request.master == VHOST_USER_CRYPTO_CLOSE_SESS) {
-		ret = vhost_crypto_close_sess(vcrypto, vmsg->payload.u64);
+		if (vhost_crypto_close_sess(vcrypto, vmsg->payload.u64))
-	else
+			ret = VH_RESULT_ERR;
-		ret = -EINVAL;
+	}
 
 	return ret;
 }

lib/librte_vhost/vhost_user.c

@@ -71,16 +71,6 @@ static const char *vhost_message_str[VHOST_USER_MAX] = {
 	[VHOST_USER_CRYPTO_CLOSE_SESS] = "VHOST_USER_CRYPTO_CLOSE_SESS",
 };
 
-/* The possible results of a message handling function */
-enum vh_result {
-	/* Message handling failed */
-	VH_RESULT_ERR   = -1,
-	/* Message handling successful */
-	VH_RESULT_OK    =  0,
-	/* Message handling successful and reply prepared */
-	VH_RESULT_REPLY =  1,
-};
-
 static uint64_t
 get_blk_size(int fd)
 {
@@ -1738,14 +1728,11 @@ vhost_user_msg_handler(int vid, int fd)
 	}
 
 	if (dev->extern_ops.pre_msg_handle) {
-		uint32_t need_reply;
-
 		ret = (*dev->extern_ops.pre_msg_handle)(dev->vid,
-				(void *)&msg, &need_reply, &skip_master);
+				(void *)&msg, &skip_master);
-		if (ret < 0)
+		if (ret == VH_RESULT_ERR)
 			goto skip_to_reply;
-
+		else if (ret == VH_RESULT_REPLY)
-		if (need_reply)
 			send_vhost_reply(fd, &msg);
 
 		if (skip_master)
@@ -1783,15 +1770,12 @@ vhost_user_msg_handler(int vid, int fd)
 	}
 
 skip_to_post_handle:
-	if (!ret && dev->extern_ops.post_msg_handle) {
+	if (ret != VH_RESULT_ERR && dev->extern_ops.post_msg_handle) {
-		uint32_t need_reply;
-
 		ret = (*dev->extern_ops.post_msg_handle)(
-				dev->vid, (void *)&msg, &need_reply);
+				dev->vid, (void *)&msg);
-		if (ret < 0)
+		if (ret == VH_RESULT_ERR)
 			goto skip_to_reply;
-
+		else if (ret == VH_RESULT_REPLY)
-		if (need_reply)
 			send_vhost_reply(fd, &msg);
 	}
 
@@ -1800,10 +1784,10 @@ skip_to_reply:
 		vhost_user_unlock_all_queue_pairs(dev);
 
 	if (msg.flags & VHOST_USER_NEED_REPLY) {
-		msg.payload.u64 = !!ret;
+		msg.payload.u64 = ret == VH_RESULT_ERR;
 		msg.size = sizeof(msg.payload.u64);
 		send_vhost_reply(fd, &msg);
-	} else if (ret) {
+	} else if (ret == VH_RESULT_ERR) {
 		RTE_LOG(ERR, VHOST_CONFIG,
 			"vhost message handling failed.\n");
 		return -1;