diff --git a/lib/jsonrpc/jsonrpc_internal.h b/lib/jsonrpc/jsonrpc_internal.h
index 4fc5ea8da1..2206c825f5 100644
--- a/lib/jsonrpc/jsonrpc_internal.h
+++ b/lib/jsonrpc/jsonrpc_internal.h
@@ -69,6 +69,7 @@ struct spdk_jsonrpc_server_conn {
 	uint32_t outstanding_requests;
 	struct spdk_ring *send_queue;
 	struct spdk_jsonrpc_request *send_request;
+	struct pollfd *pfd;
 };
 
 struct spdk_jsonrpc_server {
diff --git a/lib/jsonrpc/jsonrpc_server_tcp.c b/lib/jsonrpc/jsonrpc_server_tcp.c
index 8e32d5785a..018b4e0a12 100644
--- a/lib/jsonrpc/jsonrpc_server_tcp.c
+++ b/lib/jsonrpc/jsonrpc_server_tcp.c
@@ -107,13 +107,27 @@ spdk_jsonrpc_server_shutdown(struct spdk_jsonrpc_server *server)
 	free(server);
 }
 
+static void
+spdk_jsonrpc_server_conn_close(struct spdk_jsonrpc_server_conn *conn)
+{
+	conn->closed = true;
+
+	/* Set the pollfd fd to a negative value so it is ignored by poll() */
+	conn->pfd->fd = -1;
+
+	if (conn->sockfd >= 0) {
+		close(conn->sockfd);
+		conn->sockfd = -1;
+	}
+}
+
 static void
 spdk_jsonrpc_server_conn_remove(struct spdk_jsonrpc_server_conn *conn)
 {
 	struct spdk_jsonrpc_server *server = conn->server;
 	int conn_idx = conn - server->conns;
 
-	close(conn->sockfd);
+	spdk_jsonrpc_server_conn_close(conn);
 
 	spdk_ring_free(conn->send_queue);
 
@@ -159,6 +173,8 @@ spdk_jsonrpc_server_accept(struct spdk_jsonrpc_server *server)
 		pfd = &server->pollfds[conn_idx + 1];
 		pfd->fd = conn->sockfd;
 		pfd->events = POLLIN | POLLOUT;
+		pfd->revents = 0;
+		conn->pfd = pfd;
 
 		server->num_conns++;
 
@@ -389,7 +405,7 @@ spdk_jsonrpc_server_poll(struct spdk_jsonrpc_server *server)
 			rc = spdk_jsonrpc_server_conn_send(conn);
 			if (rc != 0) {
 				SPDK_TRACELOG(SPDK_TRACE_RPC, "closing conn due to send failure\n");
-				conn->closed = true;
+				spdk_jsonrpc_server_conn_close(conn);
 				continue;
 			}
 		}
@@ -398,11 +414,18 @@ spdk_jsonrpc_server_poll(struct spdk_jsonrpc_server *server)
 			rc = spdk_jsonrpc_server_conn_recv(conn);
 			if (rc != 0) {
 				SPDK_TRACELOG(SPDK_TRACE_RPC, "closing conn due to recv failure\n");
-				conn->closed = true;
+				spdk_jsonrpc_server_conn_close(conn);
 				continue;
 			}
 		}
 
+		if (pfd->revents & (POLLERR | POLLNVAL)) {
+			SPDK_TRACELOG(SPDK_TRACE_RPC, "closing conn due to poll() flag %d\n",
+				      (int)pfd->revents);
+			spdk_jsonrpc_server_conn_close(conn);
+			continue;
+		}
+
 		pfd->revents = 0;
 	}