Add VNC Authentication support based on RFC6143 section 7.2.2.

Submitted by:	Fabian Freyer <fabian.freyer@physik.tu-berlin.de>
Reworked by:	myself
Reviewed by:	grehan, rgrimes and jilles
MFC after:	1 week.
Relnotes:	Yes.
Sponsored by:	iXsystems, Inc.
Differential Revision:	https://reviews.freebsd.org/D10818

usr.sbin/bhyve/Makefile

@@ -2,6 +2,8 @@
 # $FreeBSD$
 #
 
+.include <src.opts.mk>
+
 PROG=	bhyve
 PACKAGE=	bhyve
 
@@ -63,6 +65,12 @@ SRCS+=	vmm_instruction_emul.c
 
 LIBADD=	vmmapi md pthread z
 
+.if ${MK_OPENSSL} == "no"
+CFLAGS+=-DNO_OPENSSL
+.else
+LIBADD+=	crypto
+.endif
+
 CFLAGS+= -I${BHYVE_SYSDIR}/sys/dev/e1000
 CFLAGS+= -I${BHYVE_SYSDIR}/sys/dev/mii
 CFLAGS+= -I${BHYVE_SYSDIR}/sys/dev/usb/controller

usr.sbin/bhyve/bhyve.8

@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 3, 2017
+.Dd May 22, 2017
 .Dt BHYVE 8
 .Os
 .Sh NAME
@@ -309,7 +309,7 @@ Emergency write is advertised, but no-op at present.
 .Pp
 Framebuffer devices:
 .Bl -tag -width 10n
-.It Oo rfb= Ns Oo Ar IP: Oc Ns Ar port Oc Ns Oo ,w= Ns Ar width Oc Ns Oo ,h= Ns Ar height Oc Ns Oo ,vga= Ns Ar vgaconf Oc Ns Oo Ns ,wait Oc
+.It Oo rfb= Ns Oo Ar IP: Oc Ns Ar port Oc Ns Oo ,w= Ns Ar width Oc Ns Oo ,h= Ns Ar height Oc Ns Oo ,vga= Ns Ar vgaconf Oc Ns Oo Ns ,wait Oc Ns Oo ,password= Ns Ar password Oc
 .Bl -tag -width 8n
 .It Ar IP:port
 An
@@ -368,6 +368,11 @@ Instruct
 to only boot upon the initiation of a VNC connection, simplifying the installation
 of operating systems that require immediate keyboard input.
 This can be removed for post-installation use.
+.It password
+This type of authentication is known to be cryptographically weak and is not
+intended for use on untrusted networks.
+Many implementations will want to use stronger security, such as running
+the session over an encrypted channel provided by IPsec or SSH.
 .El
 .El
 .Pp

usr.sbin/bhyve/pci_fbuf.c

@@ -93,6 +93,7 @@ struct pci_fbuf_softc {
 
 	/* rfb server */
 	char      *rfb_host;
+	char      *rfb_password;
 	int       rfb_port;
 	int       rfb_wait;
 	int       vga_enabled;
@@ -285,7 +286,8 @@ pci_fbuf_parse_opts(struct pci_fbuf_softc *sc, char *opts)
 				goto done;
 			} else if (sc->memregs.height == 0)
 				sc->memregs.height = 1080;
-
+		} else if (!strcmp(xopts, "password")) {
+			sc->rfb_password = config;
 		} else {
 			pci_fbuf_usage(xopts);
 			ret = -1;
@@ -407,7 +409,7 @@ pci_fbuf_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 
 	memset((void *)sc->fb_base, 0, FB_SIZE);
 
-	error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait);
+	error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password);
 done:
 	if (error)
 		free(sc);

usr.sbin/bhyve/rfb.c

@@ -60,10 +60,23 @@ __FBSDID("$FreeBSD$");
 #include "rfb.h"
 #include "sockstream.h"
 
+#ifndef NO_OPENSSL
+#include <openssl/des.h>
+#endif
+
 static int rfb_debug = 0;
 #define	DPRINTF(params) if (rfb_debug) printf params
 #define	WPRINTF(params) printf params
 
+#define AUTH_LENGTH	16
+#define PASSWD_LENGTH	8
+
+#define SECURITY_TYPE_NONE 1
+#define SECURITY_TYPE_VNC_AUTH 2
+
+#define AUTH_FAILED_UNAUTH 1
+#define AUTH_FAILED_ERROR 2
+
 struct rfb_softc {
 	int		sfd;
 	pthread_t	tid;
@@ -72,9 +85,11 @@ struct rfb_softc {
 
 	int		width, height;
 
-	bool		enc_raw_ok;
-	bool		enc_zlib_ok;
-	bool		enc_resize_ok;
+	char		*password;
+
+	bool	enc_raw_ok;
+	bool	enc_zlib_ok;
+	bool	enc_resize_ok;
 
 	z_stream	zstream;
 	uint8_t		*zbuf;
@@ -208,7 +223,7 @@ static void
 rfb_send_resize_update_msg(struct rfb_softc *rc, int cfd)
 {
 	struct rfb_srvr_updt_msg supdt_msg;
-        struct rfb_srvr_rect_hdr srect_hdr;
+	struct rfb_srvr_rect_hdr srect_hdr;
 
 	/* Number of rectangles: 1 */
 	supdt_msg.type = 0;
@@ -739,8 +754,19 @@ rfb_handle(struct rfb_softc *rc, int cfd)
 {
 	const char *vbuf = "RFB 003.008\n";
 	unsigned char buf[80];
+	unsigned char *message;
+
+#ifndef NO_OPENSSL
+	unsigned char challenge[AUTH_LENGTH];
+	unsigned char keystr[PASSWD_LENGTH];
+	unsigned char crypt_expected[AUTH_LENGTH];
+
+	DES_key_schedule ks;
+	int i;
+#endif
+
 	pthread_t tid;
-        uint32_t sres;
+	uint32_t sres;
 	int len;
 
 	rc->cfd = cfd;
@@ -751,19 +777,91 @@ rfb_handle(struct rfb_softc *rc, int cfd)
 	/* 1b. Read client version */
 	len = read(cfd, buf, sizeof(buf));
 
-	/* 2a. Send security type 'none' */
+	/* 2a. Send security type */
 	buf[0] = 1;
-	buf[1] = 1; /* none */
-	stream_write(cfd, buf, 2);
+#ifndef NO_OPENSSL
+	if (rc->password) 
+		buf[1] = SECURITY_TYPE_VNC_AUTH;
+	else
+		buf[1] = SECURITY_TYPE_NONE;
+#else
+	buf[1] = SECURITY_TYPE_NONE;
+#endif
 
+	stream_write(cfd, buf, 2);
 
 	/* 2b. Read agreed security type */
 	len = stream_read(cfd, buf, 1);
 
-	/* 2c. Write back a status of 0 */
-	sres = 0;
+	/* 2c. Do VNC authentication */
+	switch (buf[0]) {
+	case SECURITY_TYPE_NONE:
+		sres = 0;
+		break;
+	case SECURITY_TYPE_VNC_AUTH:
+		/*
+		 * The client encrypts the challenge with DES, using a password
+		 * supplied by the user as the key.
+		 * To form the key, the password is truncated to
+		 * eight characters, or padded with null bytes on the right.
+		 * The client then sends the resulting 16-bytes response.
+		 */
+#ifndef NO_OPENSSL
+		strncpy(keystr, rc->password, PASSWD_LENGTH);
+
+		/* VNC clients encrypts the challenge with all the bit fields
+		 * in each byte of the password mirrored.
+		 * Here we flip each byte of the keystr.
+		 */
+		for (i = 0; i < PASSWD_LENGTH; i++) {
+			keystr[i] = (keystr[i] & 0xF0) >> 4
+				  | (keystr[i] & 0x0F) << 4;
+			keystr[i] = (keystr[i] & 0xCC) >> 2
+				  | (keystr[i] & 0x33) << 2;
+			keystr[i] = (keystr[i] & 0xAA) >> 1
+				  | (keystr[i] & 0x55) << 1;
+		}
+
+		/* Initialize a 16-byte random challenge */
+		arc4random_buf(challenge, sizeof(challenge));
+		stream_write(cfd, challenge, AUTH_LENGTH);
+
+		/* Receive the 16-byte challenge response */
+		stream_read(cfd, buf, AUTH_LENGTH);
+
+		memcpy(crypt_expected, challenge, AUTH_LENGTH);
+
+		/* Encrypt the Challenge with DES */
+		DES_set_key((C_Block *)keystr, &ks);
+		DES_ecb_encrypt((C_Block *)challenge,
+				(C_Block *)crypt_expected, &ks, DES_ENCRYPT);
+		DES_ecb_encrypt((C_Block *)(challenge + PASSWD_LENGTH),
+				(C_Block *)(crypt_expected + PASSWD_LENGTH),
+				&ks, DES_ENCRYPT);
+
+		if (memcmp(crypt_expected, buf, AUTH_LENGTH) != 0) {
+			message = "Auth Failed: Invalid Password.";
+			sres = htonl(1);
+		} else
+			sres = 0;
+#else
+		sres = 0;
+		WPRINTF(("Auth not supported, no OpenSSL in your system"));
+#endif
+
+		break;
+	}
+
+	/* 2d. Write back a status */
 	stream_write(cfd, &sres, 4);
 
+	if (sres) {
+		*((uint32_t *) buf) = htonl(strlen(message));
+		stream_write(cfd, buf, 4);
+                stream_write(cfd, message, strlen(message));
+		goto done;
+	}
+
 	/* 3a. Read client shared-flag byte */
 	len = stream_read(cfd, buf, 1);
 
@@ -869,7 +967,7 @@ sse42_supported(void)
 }
 
 int
-rfb_init(char *hostname, int port, int wait)
+rfb_init(char *hostname, int port, int wait, char *password)
 {
 	struct rfb_softc *rc;
 	struct sockaddr_in sin;
@@ -887,6 +985,8 @@ rfb_init(char *hostname, int port, int wait)
 	rc->crc_width = RFB_MAX_WIDTH;
 	rc->crc_height = RFB_MAX_HEIGHT;
 
+	rc->password = password;
+
 	rc->sfd = socket(AF_INET, SOCK_STREAM, 0);
 	if (rc->sfd < 0) {
 		perror("socket");

usr.sbin/bhyve/rfb.h

@@ -31,6 +31,6 @@
 
 #define	RFB_PORT	5900
 
-int	rfb_init(char *hostname, int port, int wait);
+int	rfb_init(char *hostname, int port, int wait, char *password);
 
 #endif /* _RFB_H_ */