diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
index 947ddb4e39..24dffdbfee 100644
--- a/lib/vhost/vhost.c
+++ b/lib/vhost/vhost.c
@@ -1564,6 +1564,10 @@ vhost_destroy_connection_cb(int vid)
 
 	if (vsession->started) {
 		rc = _stop_session(vsession);
+		if (rc != 0) {
+			pthread_mutex_unlock(&g_vhost_mutex);
+			return rc;
+		}
 	}
 
 	TAILQ_REMOVE(&vsession->vdev->vsessions, vsession, tailq);
@@ -1571,7 +1575,7 @@ vhost_destroy_connection_cb(int vid)
 	free(vsession);
 	pthread_mutex_unlock(&g_vhost_mutex);
 
-	return rc;
+	return 0;
 }
 
 void
diff --git a/lib/vhost/vhost_blk.c b/lib/vhost/vhost_blk.c
index 63a725651a..5349a2ab09 100644
--- a/lib/vhost/vhost_blk.c
+++ b/lib/vhost/vhost_blk.c
@@ -1346,11 +1346,16 @@ destroy_session_poller_cb(void *arg)
 	struct spdk_vhost_session *vsession = &bvsession->vsession;
 	int i;
 
-	if (vsession->task_cnt > 0) {
-		return SPDK_POLLER_BUSY;
-	}
+	if (vsession->task_cnt > 0 || spdk_vhost_trylock() != 0) {
+		assert(vsession->stop_retry_count > 0);
+		vsession->stop_retry_count--;
+		if (vsession->stop_retry_count == 0) {
+			SPDK_ERRLOG("%s: Timedout when destroy session (task_cnt %d)\n", vsession->name,
+				    vsession->task_cnt);
+			spdk_poller_unregister(&bvsession->stop_poller);
+			vhost_session_stop_done(vsession, -ETIMEDOUT);
+		}
 
-	if (spdk_vhost_trylock() != 0) {
 		return SPDK_POLLER_BUSY;
 	}
 
@@ -1387,6 +1392,8 @@ vhost_blk_stop_cb(struct spdk_vhost_dev *vdev,
 		vhost_blk_session_unregister_interrupts(bvsession);
 	}
 
+	/* vhost_session_send_event timeout is 3 seconds, here set retry within 4 seconds */
+	bvsession->vsession.stop_retry_count = 4000;
 	bvsession->stop_poller = SPDK_POLLER_REGISTER(destroy_session_poller_cb,
 				 bvsession, 1000);
 	return 0;
diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h
index c98b9f6177..aadf497657 100644
--- a/lib/vhost/vhost_internal.h
+++ b/lib/vhost/vhost_internal.h
@@ -166,6 +166,9 @@ struct spdk_vhost_session {
 	/* Interval used for event coalescing checking. */
 	uint64_t stats_check_interval;
 
+	/* Session's stop poller will only try limited times to destroy the session. */
+	uint32_t stop_retry_count;
+
 	struct spdk_vhost_virtqueue virtqueue[SPDK_VHOST_MAX_VQUEUES];
 
 	TAILQ_ENTRY(spdk_vhost_session) tailq;
diff --git a/lib/vhost/vhost_scsi.c b/lib/vhost/vhost_scsi.c
index 6e93457944..d7e6cff486 100644
--- a/lib/vhost/vhost_scsi.c
+++ b/lib/vhost/vhost_scsi.c
@@ -1443,11 +1443,16 @@ destroy_session_poller_cb(void *arg)
 	struct spdk_scsi_dev_session_state *state;
 	uint32_t i;
 
-	if (vsession->task_cnt > 0) {
-		return SPDK_POLLER_BUSY;
-	}
+	if (vsession->task_cnt > 0 || spdk_vhost_trylock() != 0) {
+		assert(vsession->stop_retry_count > 0);
+		vsession->stop_retry_count--;
+		if (vsession->stop_retry_count == 0) {
+			SPDK_ERRLOG("%s: Timedout when destroy session (task_cnt %d)\n", vsession->name,
+				    vsession->task_cnt);
+			spdk_poller_unregister(&svsession->stop_poller);
+			vhost_session_stop_done(vsession, -ETIMEDOUT);
+		}
 
-	if (spdk_vhost_trylock() != 0) {
 		return SPDK_POLLER_BUSY;
 	}
 
@@ -1506,6 +1511,9 @@ vhost_scsi_stop_cb(struct spdk_vhost_dev *vdev,
 	 */
 	spdk_poller_unregister(&svsession->mgmt_poller);
 
+	/* vhost_session_send_event timeout is 3 seconds, here set retry within 4 seconds */
+	svsession->vsession.stop_retry_count = 4000;
+
 	/* Wait for all pending I/Os to complete, then process all the
 	 * remaining hotremove events one last time.
 	 */