From f7fe83f9f8a5bb345cc88fb2d230a2ceb8209584 Mon Sep 17 00:00:00 2001 From: Pawel Jakub Dawidek Date: Fri, 27 Aug 2010 14:26:37 +0000 Subject: [PATCH] Implement keepalive mechanism inside HAST protocol so we can detect secondary node failures quickly for HAST resources that are rarely modified. Remove XXX from a comment now that the guard thread never sleeps infinitely. MFC after: 2 weeks Obtained from: Wheel Systems Sp. z o.o. http://www.wheelsystems.com --- sbin/hastd/hast.h | 8 +++++- sbin/hastd/primary.c | 60 ++++++++++++++++++++++++++++++++---------- sbin/hastd/secondary.c | 14 +++++++++- 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/sbin/hastd/hast.h b/sbin/hastd/hast.h index fda62c0814f2..1f3b3860ed77 100644 --- a/sbin/hastd/hast.h +++ b/sbin/hastd/hast.h @@ -48,7 +48,12 @@ #include "proto.h" -#define HAST_PROTO_VERSION 0 +/* + * Version history: + * 0 - initial version + * 1 - HIO_KEEPALIVE added + */ +#define HAST_PROTO_VERSION 1 #define EHAST_OK 0 #define EHAST_NOENTRY 1 @@ -74,6 +79,7 @@ #define HIO_WRITE 2 #define HIO_DELETE 3 #define HIO_FLUSH 4 +#define HIO_KEEPALIVE 5 #define HAST_TIMEOUT 5 #define HAST_CONFIG "/etc/hast.conf" diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c index 52ac594ba6d5..dd26f4affb14 100644 --- a/sbin/hastd/primary.c +++ b/sbin/hastd/primary.c @@ -151,7 +151,11 @@ static pthread_mutex_t metadata_lock; */ #define HAST_NCOMPONENTS 2 /* - * Number of seconds to sleep before next reconnect try. + * Number of seconds to sleep between keepalive packets. + */ +#define KEEPALIVE_SLEEP 10 +/* + * Number of seconds to sleep between reconnect retries. */ #define RECONNECT_SLEEP 5 @@ -886,11 +890,14 @@ remote_close(struct hast_resource *res, int ncomp) sync_stop(); /* - * Wake up guard thread, so it can immediately start reconnect. + * Wake up guard thread (if we are not called from within guard thread), + * so it can immediately start reconnect. */ - mtx_lock(&hio_guard_lock); - cv_signal(&hio_guard_cond); - mtx_unlock(&hio_guard_lock); + if (!mtx_owned(&hio_guard_lock)) { + mtx_lock(&hio_guard_lock); + cv_signal(&hio_guard_cond); + mtx_unlock(&hio_guard_lock); + } } /* @@ -1734,7 +1741,7 @@ sighandler(int sig) assert(!"invalid condition"); } /* - * XXX: Racy, but if we cannot obtain hio_guard_lock here, we don't + * Racy, but if we cannot obtain hio_guard_lock here, we don't * want to risk deadlock. */ unlock = mtx_trylock(&hio_guard_lock); @@ -1851,6 +1858,32 @@ config_reload(void) pjdlog_warning("Configuration not reloaded."); } +static void +keepalive_send(struct hast_resource *res, unsigned int ncomp) +{ + struct nv *nv; + + nv = nv_alloc(); + nv_add_uint8(nv, HIO_KEEPALIVE, "cmd"); + if (nv_error(nv) != 0) { + nv_free(nv); + pjdlog_debug(1, + "keepalive_send: Unable to prepare header to send."); + return; + } + if (hast_proto_send(res, res->hr_remoteout, nv, NULL, 0) < 0) { + pjdlog_common(LOG_DEBUG, 1, errno, + "keepalive_send: Unable to send request"); + nv_free(nv); + rw_unlock(&hio_remote_lock[ncomp]); + remote_close(res, ncomp); + rw_rlock(&hio_remote_lock[ncomp]); + return; + } + nv_free(nv); + pjdlog_debug(2, "keepalive_send: Request sent."); +} + /* * Thread guards remote connections and reconnects when needed, handles * signals, etc. @@ -1874,20 +1907,19 @@ guard_thread(void *arg) sighup_received = false; config_reload(); } - /* - * If all the connection will be fine, we will sleep until - * someone wakes us up. - * If any of the connections will be broken and we won't be - * able to connect, we will sleep only for RECONNECT_SLEEP - * seconds so we can retry soon. - */ - timeout = 0; + + timeout = KEEPALIVE_SLEEP; pjdlog_debug(2, "remote_guard: Checking connections."); mtx_lock(&hio_guard_lock); for (ii = 0; ii < ncomps; ii++) { if (!ISREMOTE(ii)) continue; rw_rlock(&hio_remote_lock[ii]); + if (ISCONNECTED(res, ii)) { + assert(res->hr_remotein != NULL); + assert(res->hr_remoteout != NULL); + keepalive_send(res, ii); + } if (ISCONNECTED(res, ii)) { assert(res->hr_remotein != NULL); assert(res->hr_remoteout != NULL); diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c index c5394ff82ce0..b76f68048910 100644 --- a/sbin/hastd/secondary.c +++ b/sbin/hastd/secondary.c @@ -413,6 +413,9 @@ reqlog(int loglevel, int debuglevel, int error, struct hio *hio, const char *fmt "WRITE(%ju, %ju).", (uintmax_t)hio->hio_offset, (uintmax_t)hio->hio_length); break; + case HIO_KEEPALIVE: + (void)snprintf(msg + len, sizeof(msg) - len, "KEEPALIVE."); + break; default: (void)snprintf(msg + len, sizeof(msg) - len, "UNKNOWN(%u).", (unsigned int)hio->hio_cmd); @@ -433,6 +436,8 @@ requnpack(struct hast_resource *res, struct hio *hio) goto end; } switch (hio->hio_cmd) { + case HIO_KEEPALIVE: + break; case HIO_READ: case HIO_WRITE: case HIO_DELETE: @@ -517,7 +522,14 @@ recv_thread(void *arg) } reqlog(LOG_DEBUG, 2, -1, hio, "recv: (%p) Got request header: ", hio); - if (hio->hio_cmd == HIO_WRITE) { + if (hio->hio_cmd == HIO_KEEPALIVE) { + pjdlog_debug(2, + "recv: (%p) Moving request to the free queue.", + hio); + nv_free(hio->hio_nv); + QUEUE_INSERT(free, hio); + continue; + } else if (hio->hio_cmd == HIO_WRITE) { if (hast_proto_recv_data(res, res->hr_remotein, hio->hio_nv, hio->hio_data, MAXPHYS) < 0) { pjdlog_exit(EX_TEMPFAIL,