From b4a4a3b82f7a0b3c684ab6470117c47b17a25981 Mon Sep 17 00:00:00 2001
From: "Alexander V. Chernikov" <melifaro@FreeBSD.org>
Date: Mon, 23 Mar 2020 12:21:32 +0000
Subject: [PATCH] Add tests verifying IPv4/IPv6 output for TCP/UDP/raw paths.

Reviewed by:	kp
MFC after:	2 weeks
Differential Revision:	https://reviews.freebsd.org/D24138
---
 tests/sys/common/Makefile        |   2 +
 tests/sys/common/net_receiver.py | 116 ++++++
 tests/sys/common/vnet.subr       |   8 +
 tests/sys/netinet/Makefile       |   2 +-
 tests/sys/netinet/output.sh      | 588 +++++++++++++++++++++++++++
 tests/sys/netinet6/Makefile      |   3 +-
 tests/sys/netinet6/output6.sh    | 658 +++++++++++++++++++++++++++++++
 7 files changed, 1375 insertions(+), 2 deletions(-)
 create mode 100755 tests/sys/common/net_receiver.py
 create mode 100755 tests/sys/netinet/output.sh
 create mode 100755 tests/sys/netinet6/output6.sh

diff --git a/tests/sys/common/Makefile b/tests/sys/common/Makefile
index 91c42d9929ca..556652cf6404 100644
--- a/tests/sys/common/Makefile
+++ b/tests/sys/common/Makefile
@@ -5,8 +5,10 @@ TESTSDIR=		${TESTSBASE}/sys/common
 ${PACKAGE}FILES+=	vnet.subr
 ${PACKAGE}FILES+=	divert.py
 ${PACKAGE}FILES+=	sender.py
+${PACKAGE}FILES+=	net_receiver.py
 
 ${PACKAGE}FILESMODE_divert.py=0555
 ${PACKAGE}FILESMODE_sender.py=0555
+${PACKAGE}FILESMODE_net_receiver.py=0555
 
 .include <bsd.test.mk>
diff --git a/tests/sys/common/net_receiver.py b/tests/sys/common/net_receiver.py
new file mode 100755
index 000000000000..cd19bf19a31d
--- /dev/null
+++ b/tests/sys/common/net_receiver.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# -
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2020 Alexander V. Chernikov
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+
+from functools import partial
+import socket
+import select
+import argparse
+import time
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='divert socket tester')
+    parser.add_argument('--sip', type=str, default='', help='IP to listen on')
+    parser.add_argument('--family', type=str, help='inet/inet6')
+    parser.add_argument('--ports', type=str, help='packet ports 1,2,3')
+    parser.add_argument('--match_str', type=str, help='match string to use')
+    parser.add_argument('--count', type=int, default=1,
+                        help='Number of messages to receive')
+    parser.add_argument('--test_name', type=str, required=True,
+                        help='test name to run')
+    return parser.parse_args()
+
+
+def test_listen_tcp(args):
+    if args.family == 'inet6':
+        fam = socket.AF_INET6
+    else:
+        fam = socket.AF_INET
+    sockets = []
+    ports = [int(port) for port in args.ports.split(',')]
+    for port in ports:
+        s = socket.socket(fam, socket.SOCK_STREAM)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        s.setblocking(0)
+        s.bind((args.sip, port))
+        print('binding on {}:{}'.format(args.sip, port))
+        s.listen(5)
+        sockets.append(s)
+    inputs = sockets
+    count = 0
+    while count < args.count:
+        readable, writable, exceptional = select.select(inputs, [], inputs)
+        for s in readable:
+            (c, address) = s.accept()
+            print('C: {}'.format(address))
+            data = c.recv(9000)
+            if args.match_str and args.match_str.encode('utf-8') != data:
+                raise Exception('Expected "{}" but got "{}"'.format(
+                    args.match_str, data.decode('utf-8')))
+            count += 1
+            c.close()
+
+
+def test_listen_udp(args):
+    if args.family == 'inet6':
+        fam = socket.AF_INET6
+    else:
+        fam = socket.AF_INET
+    sockets = []
+    ports = [int(port) for port in args.ports.split(',')]
+    for port in ports:
+        s = socket.socket(fam, socket.SOCK_DGRAM)
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        s.setblocking(0)
+        s.bind((args.sip, port))
+        print('binding on {}:{}'.format(args.sip, port))
+        sockets.append(s)
+    inputs = sockets
+    count = 0
+    while count < args.count:
+        readable, writable, exceptional = select.select(inputs, [], inputs)
+        for s in readable:
+            (data, address) = s.recvfrom(9000)
+            print('C: {}'.format(address))
+            if args.match_str and args.match_str.encode('utf-8') != data:
+                raise Exception('Expected "{}" but got "{}"'.format(
+                    args.match_str, data.decode('utf-8')))
+            count += 1
+
+
+def main():
+    args = parse_args()
+    test_ptr = globals()[args.test_name]
+    test_ptr(args)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/sys/common/vnet.subr b/tests/sys/common/vnet.subr
index d86afaf6aafa..f9a9e9c428fd 100644
--- a/tests/sys/common/vnet.subr
+++ b/tests/sys/common/vnet.subr
@@ -13,6 +13,7 @@ vnet_mkepair()
 {
 	ifname=$(ifconfig epair create)
 	echo $ifname >> created_interfaces.lst
+	echo ${ifname%a}b >> created_interfaces.lst
 	echo ${ifname%a}
 }
 
@@ -23,6 +24,13 @@ vnet_mkbridge()
 	echo ${ifname}
 }
 
+vnet_mkloopback()
+{
+	ifname=$(ifconfig lo create)
+	echo $ifname >> created_interfaces.lst
+	echo ${ifname}
+}
+
 vnet_mkjail()
 {
 	jailname=$1
diff --git a/tests/sys/netinet/Makefile b/tests/sys/netinet/Makefile
index 62ea5a3760ed..e008b6c3488b 100644
--- a/tests/sys/netinet/Makefile
+++ b/tests/sys/netinet/Makefile
@@ -9,7 +9,7 @@ ATF_TESTS_C=	ip_reass_test \
 		so_reuseport_lb_test \
 		socket_afinet
 
-ATF_TESTS_SH=	fibs_test redirect divert forward
+ATF_TESTS_SH=	fibs_test redirect divert forward output
 
 PROGS=	udp_dontroute tcp_user_cookie
 
diff --git a/tests/sys/netinet/output.sh b/tests/sys/netinet/output.sh
new file mode 100755
index 000000000000..350a7a6f0db4
--- /dev/null
+++ b/tests/sys/netinet/output.sh
@@ -0,0 +1,588 @@
+#!/usr/bin/env atf-sh
+#-
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2020 Alexander V. Chernikov
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+. $(atf_get_srcdir)/../common/vnet.subr
+
+atf_test_case "output_tcp_setup_success" "cleanup"
+output_tcp_setup_success_head()
+{
+
+	atf_set descr 'Test valid IPv4 TCP output'
+	atf_set require.user root
+}
+
+output_tcp_setup_success_body()
+{
+
+	vnet_init
+
+	net_src="192.0.2."
+	net_dst="192.0.2."
+	ip_src="${net_src}1"
+	ip_dst="${net_dst}2"
+	plen=24
+	text="testtesttst"
+	port=4242
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v4t-output_tcp_setup_success"
+
+	epair=$(vnet_mkepair)
+
+	vnet_mkjail ${jname}a ${epair}a
+	jexec ${jname}a ifconfig ${epair}a up
+	jexec ${jname}a ifconfig ${epair}a inet ${ip_src}/${plen}
+
+	vnet_mkjail ${jname}b ${epair}b
+	jexec ${jname}b ifconfig ${epair}b up
+	
+	jexec ${jname}b ifconfig ${epair}b inet ${ip_dst}/${plen}
+
+	# run listener
+	args="--family inet --ports ${port} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_tcp" ${args} &
+	cmd_pid=$!
+
+	# wait for the script init
+	counter=0
+	while [ `jexec ${jname}b sockstat -4qlp ${port} | wc -l` != "1" ]; do
+		sleep 0.01
+		counter=$((counter+1))
+		if [ ${counter} -ge 50 ]; then break; fi
+	done
+	if [ `jexec ${jname}b sockstat -4qlp ${port} | wc -l` != "1" ]; then
+		echo "App setup failed"
+		exit 1
+	fi
+
+	# run sender
+	echo -n "${text}" | jexec ${jname}a nc -N ${ip_dst} ${port}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "sender exit code $exit_code" ; fi
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+}
+
+output_tcp_setup_success_cleanup()
+{
+	vnet_cleanup
+}
+
+
+atf_test_case "output_udp_setup_success" "cleanup"
+output_udp_setup_success_head()
+{
+
+	atf_set descr 'Test valid IPv4 UDP output'
+	atf_set require.user root
+}
+
+output_udp_setup_success_body()
+{
+
+	vnet_init
+
+	net_src="192.0.2."
+	net_dst="192.0.2."
+	ip_src="${net_src}1"
+	ip_dst="${net_dst}2"
+	plen=24
+	text="testtesttst"
+	port=4242
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v4t-output_udp_setup_success"
+
+	epair=$(vnet_mkepair)
+
+	vnet_mkjail ${jname}a ${epair}a
+	jexec ${jname}a ifconfig ${epair}a up
+	jexec ${jname}a ifconfig ${epair}a inet ${ip_src}/${plen}
+
+	vnet_mkjail ${jname}b ${epair}b
+	jexec ${jname}b ifconfig ${epair}b up
+	jexec ${jname}b ifconfig ${epair}b inet ${ip_dst}/${plen}
+
+	# run listener
+	args="--family inet --ports ${port} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_udp" ${args} &
+	cmd_pid=$!
+
+	# wait for the script init
+	counter=0
+	while [ `jexec ${jname}b sockstat -4qlp ${port} | wc -l` != "1" ]; do
+		sleep 0.1
+		counterc=$((counter+1))
+		if [ ${counter} -ge 50 ]; then break; fi
+	done
+	if [ `jexec ${jname}b sockstat -4qlp ${port} | wc -l` != "1" ]; then
+		echo "App setup failed"
+		exit 1
+	fi
+
+	# run sender
+	# TODO: switch from nc to some alternative to avoid 1-second delay
+	echo -n "${text}" | jexec ${jname}a nc -uNw1 ${ip_dst} ${port}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "sender exit code $exit_code" ; fi
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+}
+
+output_udp_setup_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_test_case "output_raw_success" "cleanup"
+output_raw_success_head()
+{
+
+	atf_set descr 'Test valid IPv4 raw output'
+	atf_set require.user root
+}
+
+output_raw_success_body()
+{
+
+	vnet_init
+
+	net_src="192.0.2."
+	net_dst="192.0.2."
+	ip_src="${net_src}1"
+	ip_dst="${net_dst}2"
+	plen=24
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v4t-output_raw_success"
+
+	epair=$(vnet_mkepair)
+
+	vnet_mkjail ${jname}a ${epair}a
+	jexec ${jname}a ifconfig ${epair}a up
+	jexec ${jname}a ifconfig ${epair}a inet ${ip_src}/${plen}
+
+	vnet_mkjail ${jname}b ${epair}b
+	jexec ${jname}b ifconfig ${epair}b up
+	
+	jexec ${jname}b ifconfig ${epair}b inet ${ip_dst}/${plen}
+
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping -nc1 ${ip_dst}
+}
+
+output_raw_success_cleanup()
+{
+	vnet_cleanup
+}
+
+# Multipath tests are done the following way:
+#            epair0
+# jailA lo <        > lo jailB
+#            epair1
+# jailA has 2 routes towards /24 prefix on jailB loopback, via 2 epairs
+# jailB has 1 route towards /24 prefix on jailA loopback, via epair0
+#
+# jailA initiates connections/sends packets towards IPs on jailB loopback.
+# Script then compares amount of packets sent via epair0 and epair1
+
+mpath_check()
+{
+	if [ "`sysctl -i -n net.route.multipath`" != 1 ]; then
+		atf_skip "This test requires ROUTE_MPATH enabled"
+	fi
+}
+
+atf_test_case "output_tcp_flowid_mpath_success" "cleanup"
+output_tcp_flowid_mpath_success_head()
+{
+
+	atf_set descr 'Test valid IPv4 TCP output flowid generation'
+	atf_set require.user root
+}
+
+output_tcp_flowid_mpath_success_body()
+{
+	vnet_init
+	mpath_check
+
+	net_src="192.0.2."
+	net_dst="198.51.100."
+	ip_src="${net_src}1"
+	ip_dst="${net_dst}1"
+	plen=24
+	text="testtesttst"
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v4t-output_tcp_flowid_mpath_success"
+
+	epair0=$(vnet_mkepair)
+	epair1=$(vnet_mkepair)
+	lo_src=$(vnet_mkloopback)
+	lo_dst=$(vnet_mkloopback)
+
+	vnet_mkjail ${jname}a ${epair0}a ${epair1}a ${lo_src}
+	# Setup transit IPv4 networks
+	jexec ${jname}a ifconfig ${epair0}a up
+	jexec ${jname}a ifconfig ${epair0}a inet 203.0.113.1/30
+	jexec ${jname}a ifconfig ${epair1}a up
+	jexec ${jname}a ifconfig ${epair1}a inet 203.0.113.5/30
+	jexec ${jname}a ifconfig ${lo_src} up
+
+	vnet_mkjail ${jname}b ${epair0}b ${epair1}b ${lo_dst}
+	jexec ${jname}b ifconfig ${epair0}b up
+	jexec ${jname}b ifconfig ${epair0}b inet 203.0.113.2/30
+	jexec ${jname}b ifconfig ${epair1}b up
+	jexec ${jname}b ifconfig ${epair1}b inet 203.0.113.6/30
+	jexec ${jname}b ifconfig ${lo_dst} up
+
+	# DST ips/ports to test
+	ips="4 29 48 53 55 61 71 80 84 87 90 91 119 131 137 153 154 158 162 169 169 171 176 187 197 228 233 235 236 237 245 251"
+	ports="53540 49743 43067 9131 16734 5150 14379 40292 20634 51302 3387 24387 9282 14275 42103 26881 42461 29520 45714 11096"
+
+	jexec ${jname}a ifconfig ${lo_src} inet ${ip_src}/32
+
+	jexec ${jname}b ifconfig ${lo_dst} inet ${ip_dst}/32
+	for i in ${ips}; do
+		jexec ${jname}b ifconfig ${lo_dst} alias ${net_dst}${i}/32
+	done
+
+	# Add routes
+	# A -> towards B via epair0a 
+	jexec ${jname}a route add -4 -net ${net_dst}0/${plen} 203.0.113.2
+	# A -> towards B via epair1a
+	jexec ${jname}a route add -4 -net ${net_dst}0/${plen} 203.0.113.6
+
+	# B towards A via epair0b
+	jexec ${jname}b route add -4 -net ${net_src}0/${plen} 203.0.113.1
+	
+	# Base setup verification
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping -nc1 ${ip_dst}
+
+	# run listener
+	num_ports=`echo ${ports} | wc -w`
+	num_ips=`echo ${ips} | wc -w`
+	count_examples=$((num_ports*num_ips))
+	listener_ports=`echo ${ports} | tr ' ' '\n' | sort -n | tr '\n' ',' | sed -e 's?,$??'`
+	args="--family inet --ports ${listener_ports} --count ${count_examples} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_tcp" ${args} &
+	cmd_pid=$!
+
+	# wait for the app init
+	counter=0
+	init=0
+	while [ ${counter} -le 50 ]; do
+		_ports=`jexec ${jname}b sockstat -4ql | awk "\\\$3 == ${cmd_pid} {print \\\$6}"|awk -F: "{print \\\$2}" | sort -n | tr '\n' ','`
+		if [ "${_ports}" = "${listener_ports}," ]; then
+			init=1
+			break;
+		fi
+	done
+	if [ ${init} -eq 0 ]; then
+		jexec ${jname}b sockstat -6ql | awk "\$3 == ${cmd_pid}"
+		echo "App setup failed"
+		exit 1
+	fi
+	echo "App setup done"
+
+	# run sender
+	for _ip in ${ips}; do
+		ip="${net_dst}${_ip}"
+		for port in ${ports}; do
+			echo -n "${text}" | jexec ${jname}a nc -nN ${ip} ${port}
+			exit_code=$?
+			if [ $exit_code -ne 0 ]; then atf_fail "sender exit code $exit_code" ; fi
+		done
+	done
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+
+	pkt_0=`jexec ${jname}a netstat -Wf link -I ${epair0}a | head | awk '$1!~/^Name/{print$8}'`
+	pkt_1=`jexec ${jname}a netstat -Wf link -I ${epair1}a | head | awk '$1!~/^Name/{print$8}'`
+	if [ ${pkt_0} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	if [ ${pkt_1} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	echo "TCP Balancing: 1: ${pkt_0} 2: ${pkt_1}"
+}
+
+output_tcp_flowid_mpath_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_test_case "output_udp_flowid_mpath_success" "cleanup"
+output_udp_flowid_mpath_success_head()
+{
+
+	atf_set descr 'Test valid IPv4 UDP output flowid generation'
+	atf_set require.user root
+}
+
+output_udp_flowid_mpath_success_body()
+{
+
+	vnet_init
+	mpath_check
+
+	# Note this test will spawn around ~100 nc processes
+
+	net_src="192.0.2."
+	net_dst="198.51.100."
+	ip_src="${net_src}1"
+	ip_dst="${net_dst}1"
+	plen=24
+	text="testtesttst"
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v4t-output_tcp_flowid_mpath_success"
+
+	epair0=$(vnet_mkepair)
+	epair1=$(vnet_mkepair)
+	lo_src=$(vnet_mkloopback)
+	lo_dst=$(vnet_mkloopback)
+
+	vnet_mkjail ${jname}a ${epair0}a ${epair1}a ${lo_src}
+	# Setup transit IPv4 networks
+	jexec ${jname}a ifconfig ${epair0}a up
+	jexec ${jname}a ifconfig ${epair0}a inet 203.0.113.1/30
+	jexec ${jname}a ifconfig ${epair1}a up
+	jexec ${jname}a ifconfig ${epair1}a inet 203.0.113.5/30
+	jexec ${jname}a ifconfig ${lo_src} up
+
+	vnet_mkjail ${jname}b ${epair0}b ${epair1}b ${lo_dst}
+	jexec ${jname}b ifconfig ${epair0}b up
+	jexec ${jname}b ifconfig ${epair0}b inet 203.0.113.2/30
+	jexec ${jname}b ifconfig ${epair1}b up
+	jexec ${jname}b ifconfig ${epair1}b inet 203.0.113.6/30
+	jexec ${jname}b ifconfig ${lo_dst} up
+
+	# DST ips/ports to test
+	ips="4 29 48 53 55 61 71 80 84 87 90 91 119 131 137 153 154 158 162 169 169 171 176 187 197 228 233 235 236 237 245 251"
+	ports="53540 49743 43067 9131 16734 5150 14379 40292 20634 51302 3387 24387 9282 14275 42103 26881 42461 29520 45714 11096"
+
+	jexec ${jname}a ifconfig ${lo_src} inet ${ip_src}/32
+
+	jexec ${jname}b ifconfig ${lo_dst} inet ${ip_dst}/32
+	for i in ${ips}; do
+		jexec ${jname}b ifconfig ${lo_dst} alias ${net_dst}${i}/32
+	done
+
+	# Add routes
+	# A -> towards B via epair0a 
+	jexec ${jname}a route add -4 -net ${net_dst}0/${plen} 203.0.113.2
+	# A -> towards B via epair1a
+	jexec ${jname}a route add -4 -net ${net_dst}0/${plen} 203.0.113.6
+
+	# B towards A via epair0b
+	jexec ${jname}b route add -4 -net ${net_src}0/${plen} 203.0.113.1
+	
+	# Base setup verification
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping -nc1 ${ip_dst}
+
+	# run listener
+	num_ports=`echo ${ports} | wc -w`
+	num_ips=`echo ${ips} | wc -w`
+	count_examples=$((num_ports*num_ips))
+	listener_ports=`echo ${ports} | tr ' ' '\n' | sort -n | tr '\n' ',' | sed -e 's?,$??'`
+	args="--family inet --ports ${listener_ports} --count ${count_examples} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_udp" ${args} &
+	cmd_pid=$!
+
+	# wait for the app init
+	counter=0
+	init=0
+	while [ ${counter} -le 50 ]; do
+		_ports=`jexec ${jname}b sockstat -4ql | awk "\\\$3 == ${cmd_pid} {print \\\$6}"|awk -F: "{print \\\$2}" | sort -n | tr '\n' ','`
+		if [ "${_ports}" = "${listener_ports}," ]; then
+			init=1
+			break;
+		fi
+	done
+	if [ ${init} -eq 0 ]; then
+		jexec ${jname}b sockstat -4ql | awk "\$3 == ${cmd_pid}"
+		echo "App setup failed"
+		exit 1
+	fi
+	echo "App setup done"
+
+	# run sender
+	for _ip in ${ips}; do
+		ip="${net_dst}${_ip}"
+		for port in ${ports}; do
+			# XXX: switch to something that allows immediate exit
+			echo -n "${text}" | jexec ${jname}a nc -nuNw1 ${ip} ${port} &
+			sleep 0.01
+		done
+	done
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+
+	pkt_0=`jexec ${jname}a netstat -Wf link -I ${epair0}a | head | awk '$1!~/^Name/{print$8}'`
+	pkt_1=`jexec ${jname}a netstat -Wf link -I ${epair1}a | head | awk '$1!~/^Name/{print$8}'`
+	if [ ${pkt_0} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	if [ ${pkt_1} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	echo "UDP BALANCING: 1: ${pkt_0} 2: ${pkt_1}"
+}
+
+output_udp_flowid_mpath_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_test_case "output_raw_flowid_mpath_success" "cleanup"
+output_raw_flowid_mpath_success_head()
+{
+
+	atf_set descr 'Test valid IPv4 raw output flowid generation'
+	atf_set require.user root
+}
+
+output_raw_flowid_mpath_success_body()
+{
+
+	vnet_init
+	mpath_check
+
+	net_src="192.0.2."
+	net_dst="198.51.100."
+	ip_src="${net_src}1"
+	ip_dst="${net_dst}1"
+	plen=24
+	text="testtesttst"
+
+	jname="v4t-output_tcp_flowid_mpath_success"
+
+	epair0=$(vnet_mkepair)
+	epair1=$(vnet_mkepair)
+	lo_src=$(vnet_mkloopback)
+	lo_dst=$(vnet_mkloopback)
+
+	vnet_mkjail ${jname}a ${epair0}a ${epair1}a ${lo_src}
+	# Setup transit IPv4 networks
+	jexec ${jname}a ifconfig ${epair0}a up
+	jexec ${jname}a ifconfig ${epair0}a inet 203.0.113.1/30
+	jexec ${jname}a ifconfig ${epair1}a up
+	jexec ${jname}a ifconfig ${epair1}a inet 203.0.113.5/30
+	jexec ${jname}a ifconfig ${lo_src} up
+
+	vnet_mkjail ${jname}b ${epair0}b ${epair1}b ${lo_dst}
+	jexec ${jname}b ifconfig ${epair0}b up
+	jexec ${jname}b ifconfig ${epair0}b inet 203.0.113.2/30
+	jexec ${jname}b ifconfig ${epair1}b up
+	jexec ${jname}b ifconfig ${epair1}b inet 203.0.113.6/30
+	jexec ${jname}b ifconfig ${lo_dst} up
+
+	# DST ips/ports to test
+	ips="4 29 48 53 55 61 71 80 84 87 90 91 119 131 137 153 154 158 162 169 169 171 176 187 197 228 233 235 236 237 245 251"
+
+	jexec ${jname}a ifconfig ${lo_src} inet ${ip_src}/32
+
+	jexec ${jname}b ifconfig ${lo_dst} inet ${ip_dst}/32
+	for i in ${ips}; do
+		jexec ${jname}b ifconfig ${lo_dst} alias ${net_dst}${i}/32
+	done
+
+	# Add routes
+	# A -> towards B via epair0a 
+	jexec ${jname}a route add -4 -net ${net_dst}0/${plen} 203.0.113.2
+	# A -> towards B via epair1a
+	jexec ${jname}a route add -4 -net ${net_dst}0/${plen} 203.0.113.6
+
+	# B towards A via epair0b
+	jexec ${jname}b route add -4 -net ${net_src}0/${plen} 203.0.113.1
+	
+	# Base setup verification
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping -nc1 ${ip_dst}
+
+	# run sender
+	valid_message='1 packets transmitted, 1 packets received'
+	for _ip in ${ips}; do
+		ip="${net_dst}${_ip}"
+		atf_check -o match:"${valid_message}" jexec ${jname}a ping -nc1 ${ip}
+	done
+
+	pkt_0=`jexec ${jname}a netstat -Wf link -I ${epair0}a | head | awk '$1!~/^Name/{print$8}'`
+	pkt_1=`jexec ${jname}a netstat -Wf link -I ${epair1}a | head | awk '$1!~/^Name/{print$8}'`
+
+	jexec ${jname}a netstat -bWf link -I ${epair0}a
+	jexec ${jname}a netstat -bWf link -I ${epair1}a
+	if [ ${pkt_0} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	if [ ${pkt_1} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	echo "RAW BALANCING: 1: ${pkt_0} 2: ${pkt_1}"
+}
+
+output_raw_flowid_mpath_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case "output_tcp_setup_success"
+	atf_add_test_case "output_udp_setup_success"
+	atf_add_test_case "output_raw_success"
+	atf_add_test_case "output_tcp_flowid_mpath_success"
+	atf_add_test_case "output_udp_flowid_mpath_success"
+	atf_add_test_case "output_raw_flowid_mpath_success"
+}
+
diff --git a/tests/sys/netinet6/Makefile b/tests/sys/netinet6/Makefile
index 70d7e4a1e79a..e9f77bca1a7b 100644
--- a/tests/sys/netinet6/Makefile
+++ b/tests/sys/netinet6/Makefile
@@ -11,7 +11,8 @@ ATF_TESTS_SH=			\
 				scapyi386 \
 				redirect \
 				divert \
-				forward6
+				forward6 \
+				output6
 
 ${PACKAGE}FILES+=		exthdr.py
 ${PACKAGE}FILES+=		mld.py
diff --git a/tests/sys/netinet6/output6.sh b/tests/sys/netinet6/output6.sh
new file mode 100755
index 000000000000..fc79e598a102
--- /dev/null
+++ b/tests/sys/netinet6/output6.sh
@@ -0,0 +1,658 @@
+#!/usr/bin/env atf-sh
+#-
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Copyright (c) 2020 Alexander V. Chernikov
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+. $(atf_get_srcdir)/../common/vnet.subr
+
+atf_test_case "output6_tcp_setup_success" "cleanup"
+output6_tcp_setup_success_head()
+{
+
+	atf_set descr 'Test valid IPv6 TCP output'
+	atf_set require.user root
+}
+
+output6_tcp_setup_success_body()
+{
+
+	vnet_init
+
+	net_src="2001:db8:0:0:1::"
+	net_dst="2001:db8:0:0:1::"
+	ip_src="${net_src}1"
+	ip_dst=${net_dst}4242
+	plen=64
+	text="testtesttst"
+	port=4242
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v6t-output6_tcp_setup_success"
+
+	epair=$(vnet_mkepair)
+
+	vnet_mkjail ${jname}a ${epair}a
+	jexec ${jname}a ifconfig ${epair}a up
+	jexec ${jname}a ifconfig ${epair}a inet6 ${ip_src}/${plen}
+
+	vnet_mkjail ${jname}b ${epair}b
+	jexec ${jname}b ifconfig ${epair}b up
+	
+	jexec ${jname}b ifconfig ${epair}b inet6 ${ip_dst}/${plen}
+
+	# wait for DAD to complete
+	while [ `jexec ${jname}b ifconfig ${epair}b inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+	while [ `jexec ${jname}a ifconfig ${epair}a inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+
+	# run listener
+	args="--family inet6 --ports ${port} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_tcp" ${args} &
+	cmd_pid=$!
+
+	# wait for the script init
+	counter=0
+	while [ `jexec ${jname}b sockstat -6qlp ${port} | wc -l` != "1" ]; do
+		sleep 0.01
+		counter=$((counter+1))
+		if [ ${counter} -ge 50 ]; then break; fi
+	done
+	if [ `jexec ${jname}b sockstat -6qlp ${port} | wc -l` != "1" ]; then
+		echo "App setup failed"
+		exit 1
+	fi
+
+	# run sender
+	echo -n "${text}" | jexec ${jname}a nc -N ${ip_dst} ${port}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "sender exit code $exit_code" ; fi
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+}
+
+output6_tcp_setup_success_cleanup()
+{
+	vnet_cleanup
+}
+
+
+atf_test_case "output6_udp_setup_success" "cleanup"
+output6_udp_setup_success_head()
+{
+
+	atf_set descr 'Test valid IPv6 UDP output'
+	atf_set require.user root
+}
+
+output6_udp_setup_success_body()
+{
+
+	vnet_init
+
+	net_src="2001:db8:0:0:1::"
+	net_dst="2001:db8:0:0:1::"
+	ip_src="${net_src}1"
+	ip_dst=${net_dst}4242
+	plen=64
+	text="testtesttst"
+	port=4242
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v6t-output6_udp_setup_success"
+
+	epair=$(vnet_mkepair)
+
+	vnet_mkjail ${jname}a ${epair}a
+	jexec ${jname}a ifconfig ${epair}a up
+	jexec ${jname}a ifconfig ${epair}a inet6 ${ip_src}/${plen}
+
+	vnet_mkjail ${jname}b ${epair}b
+	jexec ${jname}b ifconfig ${epair}b up
+	jexec ${jname}b ifconfig ${epair}b inet6 ${ip_dst}/${plen}
+
+	# wait for DAD to complete
+	while [ `jexec ${jname}b ifconfig ${epair}b inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+	while [ `jexec ${jname}a ifconfig ${epair}a inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+
+	# run listener
+	args="--family inet6 --ports ${port} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_udp" ${args} &
+	cmd_pid=$!
+
+	# wait for the script init
+	counter=0
+	while [ `jexec ${jname}b sockstat -6qlp ${port} | wc -l` != "1" ]; do
+		sleep 0.1
+		counterc=$((counter+1))
+		if [ ${counter} -ge 50 ]; then break; fi
+	done
+	if [ `jexec ${jname}b sockstat -6qlp ${port} | wc -l` != "1" ]; then
+		echo "App setup failed"
+		exit 1
+	fi
+
+	# run sender
+	# TODO: switch from nc to some alternative to avoid 1-second delay
+	echo -n "${text}" | jexec ${jname}a nc -uNw1 ${ip_dst} ${port}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "sender exit code $exit_code" ; fi
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+}
+
+output6_udp_setup_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_test_case "output6_raw_success" "cleanup"
+output6_raw_success_head()
+{
+
+	atf_set descr 'Test valid IPv6 raw output'
+	atf_set require.user root
+}
+
+output6_raw_success_body()
+{
+
+	vnet_init
+
+	net_src="2001:db8:0:0:1::"
+	net_dst="2001:db8:0:0:1::"
+	ip_src="${net_src}1"
+	ip_dst=${net_dst}4242
+	plen=64
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v6t-output6_raw_success"
+
+	epair=$(vnet_mkepair)
+
+	vnet_mkjail ${jname}a ${epair}a
+	jexec ${jname}a ifconfig ${epair}a up
+	jexec ${jname}a ifconfig ${epair}a inet6 ${ip_src}/${plen}
+
+	vnet_mkjail ${jname}b ${epair}b
+	jexec ${jname}b ifconfig ${epair}b up
+	
+	jexec ${jname}b ifconfig ${epair}b inet6 ${ip_dst}/${plen}
+
+	# wait for DAD to complete
+	while [ `jexec ${jname}b ifconfig ${epair}b inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+	while [ `jexec ${jname}a ifconfig ${epair}a inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping6 -nc1 ${ip_dst}
+}
+
+output6_raw_success_cleanup()
+{
+	vnet_cleanup
+}
+
+# Multipath tests are done the following way:
+#               epair0/LL
+# jailA lo/GU <           > lo/GU jailB
+#               epair1/LL
+# jailA has 2 routes towards /64 prefix on jailB loopback, via 2 epairs
+# jailB has 1 route towards /64 prefix on jailA loopback, via epair0
+#
+# jailA initiates connections/sends packets towards IPs on jailB loopback.
+# Script then compares amount of packets sent via epair0 and epair1
+
+mpath_check()
+{
+	if [ "`sysctl -i -n net.route.multipath`" != 1 ]; then
+		atf_skip "This test requires ROUTE_MPATH enabled"
+	fi
+}
+
+atf_test_case "output6_tcp_flowid_mpath_success" "cleanup"
+output6_tcp_flowid_mpath_success_head()
+{
+
+	atf_set descr 'Test valid IPv6 TCP output flowid generation'
+	atf_set require.user root
+}
+
+output6_tcp_flowid_mpath_success_body()
+{
+	vnet_init
+	mpath_check
+
+	net_src="2001:db8:0:1"
+	net_dst="2001:db8:0:2"
+	ip_src="${net_src}::1"
+	ip_dst="${net_dst}::1"
+	plen=64
+	text="testtesttst"
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v6t-output6_tcp_flowid_mpath_success"
+
+	epair0=$(vnet_mkepair)
+	epair1=$(vnet_mkepair)
+	lo_src=$(vnet_mkloopback)
+	lo_dst=$(vnet_mkloopback)
+
+	vnet_mkjail ${jname}a ${epair0}a ${epair1}a ${lo_src}
+	jls -N
+	# enable link-local IPv6
+	jexec ${jname}a ndp -i ${epair0}a -- -disabled
+	jexec ${jname}a ifconfig ${epair0}a up
+	jexec ${jname}a ndp -i ${epair1}a -- -disabled
+	jexec ${jname}a ifconfig ${epair1}a up
+	jexec ${jname}a ifconfig ${lo_src} up
+
+	vnet_mkjail ${jname}b ${epair0}b ${epair1}b ${lo_dst}
+	jls -N
+	jexec ${jname}b ndp -i ${epair0}b -- -disabled
+	jexec ${jname}b ifconfig ${epair0}b up
+	jexec ${jname}b ndp -i ${epair1}b -- -disabled
+	jexec ${jname}b ifconfig ${epair1}b up
+	jexec ${jname}b ifconfig ${lo_dst} up
+
+	# DST ips/ports to test
+	ips="d3:c4:eb:40 2b:ff:dd:52 b1:d4:44:0e 41:2c:4d:43 66:4a:b4:be 8b:da:ac:f7 ca:d1:c4:f0 b1:31:da:d7 0c:ac:45:7a 44:9c:ce:71"
+	ports="53540 49743 43067 9131 16734 5150 14379 40292 20634 51302 3387 24387 9282 14275 42103 26881 42461 29520 45714 11096"
+
+	jexec ${jname}a ifconfig ${lo_src} inet6 ${ip_src}/128
+
+	jexec ${jname}b ifconfig ${lo_dst} inet6 ${ip_dst}/128
+	for i in ${ips}; do
+		jexec ${jname}b ifconfig ${lo_dst} inet6 ${net_dst}:${i}/128
+	done
+
+	# wait for DAD to complete
+	while [ `jexec ${jname}b ifconfig | grep inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+	while [ `jexec ${jname}a ifconfig | grep inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+
+	# Add routes
+	# A -> towards B via epair0a LL
+	ll=`jexec ${jname}b ifconfig ${epair0}b inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}a route add -6 -net ${net_dst}::/${plen} ${ll}%${epair0}a
+	# A -> towards B via epair1a LL
+	ll=`jexec ${jname}b ifconfig ${epair1}b inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}a route add -6 -net ${net_dst}::/${plen} ${ll}%${epair1}a
+
+	# B towards A via epair0b LL
+	ll=`jexec ${jname}a ifconfig ${epair1}a inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}b route add -6 -net ${net_src}::/${plen} ${ll}%${epair1}b
+	
+	# Base setup verification
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping6 -c1 ${ip_dst}
+
+	# run listener
+	num_ports=`echo ${ports} | wc -w`
+	num_ips=`echo ${ips} | wc -w`
+	count_examples=$((num_ports*num_ips))
+	listener_ports=`echo ${ports} | tr ' ' '\n' | sort -n | tr '\n' ',' | sed -e 's?,$??'`
+	args="--family inet6 --ports ${listener_ports} --count ${count_examples} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_tcp" ${args} &
+	cmd_pid=$!
+
+	# wait for the app init
+	counter=0
+	init=0
+	while [ ${counter} -le 50 ]; do
+		_ports=`jexec ${jname}b sockstat -6ql | awk "\\\$3 == ${cmd_pid} {print \\\$6}"|awk -F: "{print \\\$2}" | sort -n | tr '\n' ','`
+		if [ "${_ports}" = "${listener_ports}," ]; then
+			init=1
+			break;
+		fi
+	done
+	if [ ${init} -eq 0 ]; then
+		jexec ${jname}b sockstat -6ql | awk "\$3 == ${cmd_pid}"
+		echo "App setup failed"
+		exit 1
+	fi
+	echo "App setup done"
+
+	# run sender
+	for _ip in ${ips}; do
+		ip="${net_dst}:${_ip}"
+		for port in ${ports}; do
+			echo -n "${text}" | jexec ${jname}a nc -nN ${ip} ${port}
+			exit_code=$?
+			if [ $exit_code -ne 0 ]; then atf_fail "sender exit code $exit_code" ; fi
+		done
+	done
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+
+	pkt_0=`jexec ${jname}a netstat -Wf link -I ${epair0}a | head | awk '$1!~/^Name/{print$8}'`
+	pkt_1=`jexec ${jname}a netstat -Wf link -I ${epair1}a | head | awk '$1!~/^Name/{print$8}'`
+	if [ ${pkt_0} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	if [ ${pkt_1} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	echo "TCP Balancing: 1: ${pkt_0} 2: ${pkt_1}"
+}
+
+output6_tcp_flowid_mpath_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_test_case "output6_udp_flowid_mpath_success" "cleanup"
+output6_udp_flowid_mpath_success_head()
+{
+
+	atf_set descr 'Test valid IPv6 UDP output flowid generation'
+	atf_set require.user root
+}
+
+output6_udp_flowid_mpath_success_body()
+{
+
+	vnet_init
+	mpath_check
+
+	# Note this test will spawn around ~100 nc processes
+
+	net_src="2001:db8:0:1"
+	net_dst="2001:db8:0:2"
+	ip_src="${net_src}::1"
+	ip_dst="${net_dst}::1"
+	plen=64
+	text="testtesttst"
+
+	script_name=`dirname $0`/../common/net_receiver.py
+	script_name=`realpath ${script_name}`
+	jname="v6t-output6_udp_flowid_mpath_success"
+
+	epair0=$(vnet_mkepair)
+	epair1=$(vnet_mkepair)
+	lo_src=$(vnet_mkloopback)
+	lo_dst=$(vnet_mkloopback)
+
+	vnet_mkjail ${jname}a ${epair0}a ${epair1}a ${lo_src}
+	jls -N
+	# enable link-local IPv6
+	jexec ${jname}a ndp -i ${epair0}a -- -disabled
+	jexec ${jname}a ifconfig ${epair0}a up
+	jexec ${jname}a ndp -i ${epair1}a -- -disabled
+	jexec ${jname}a ifconfig ${epair1}a up
+	jexec ${jname}a ifconfig ${lo_src} up
+
+	vnet_mkjail ${jname}b ${epair0}b ${epair1}b ${lo_dst}
+	jls -N
+	jexec ${jname}b ndp -i ${epair0}b -- -disabled
+	jexec ${jname}b ifconfig ${epair0}b up
+	jexec ${jname}b ndp -i ${epair1}b -- -disabled
+	jexec ${jname}b ifconfig ${epair1}b up
+	jexec ${jname}b ifconfig ${lo_dst} up
+
+	# DST ips/ports to test
+	ips="d3:c4:eb:40 2b:ff:dd:52 b1:d4:44:0e 41:2c:4d:43 66:4a:b4:be 8b:da:ac:f7 ca:d1:c4:f0 b1:31:da:d7 0c:ac:45:7a 44:9c:ce:71"
+	ports="53540 49743 43067 9131 16734 5150 14379 40292 20634 51302 3387 24387 9282 14275 42103 26881 42461 29520 45714 11096"
+
+	jexec ${jname}a ifconfig ${lo_src} inet6 ${ip_src}/128
+
+	jexec ${jname}b ifconfig ${lo_dst} inet6 ${ip_dst}/128
+	for i in ${ips}; do
+		jexec ${jname}b ifconfig ${lo_dst} inet6 ${net_dst}:${i}/128
+	done
+
+	
+	# wait for DAD to complete
+	while [ `jexec ${jname}b ifconfig | grep inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+	while [ `jexec ${jname}a ifconfig | grep inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+
+	# Add routes
+	# A -> towards B via epair0a LL
+	ll=`jexec ${jname}b ifconfig ${epair0}b inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}a route add -6 -net ${net_dst}::/${plen} ${ll}%${epair0}a
+	# A -> towards B via epair1a LL
+	ll=`jexec ${jname}b ifconfig ${epair1}b inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}a route add -6 -net ${net_dst}::/${plen} ${ll}%${epair1}a
+
+	# B towards A via epair0b LL
+	ll=`jexec ${jname}a ifconfig ${epair1}a inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}b route add -6 -net ${net_src}::/${plen} ${ll}%${epair1}b
+	
+	# Base setup verification
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping6 -c1 ${ip_dst}
+
+	# run listener
+	num_ports=`echo ${ports} | wc -w`
+	num_ips=`echo ${ips} | wc -w`
+	count_examples=$((num_ports*num_ips))
+	listener_ports=`echo ${ports} | tr ' ' '\n' | sort -n | tr '\n' ',' | sed -e 's?,$??'`
+	args="--family inet6 --ports ${listener_ports} --count ${count_examples} --match_str ${text}"
+	echo jexec ${jname}b ${script_name} ${args}
+	jexec ${jname}b ${script_name} --test_name "test_listen_udp" ${args} &
+	cmd_pid=$!
+
+	# wait for the app init
+	counter=0
+	init=0
+	while [ ${counter} -le 50 ]; do
+		_ports=`jexec ${jname}b sockstat -6ql | awk "\\\$3 == ${cmd_pid} {print \\\$6}"|awk -F: "{print \\\$2}" | sort -n | tr '\n' ','`
+		if [ "${_ports}" = "${listener_ports}," ]; then
+			init=1
+			break;
+		fi
+	done
+	if [ ${init} -eq 0 ]; then
+		jexec ${jname}b sockstat -6ql | awk "\$3 == ${cmd_pid}"
+		echo "App setup failed"
+		exit 1
+	fi
+	echo "App setup done"
+
+	# run sender
+	for _ip in ${ips}; do
+		ip="${net_dst}:${_ip}"
+		for port in ${ports}; do
+			# XXX: switch to something that allows immediate exit
+			echo -n "${text}" | jexec ${jname}a nc -nuNw1 ${ip} ${port} &
+			sleep 0.01
+		done
+	done
+
+	wait ${cmd_pid}
+	exit_code=$?
+	if [ $exit_code -ne 0 ]; then atf_fail "receiver exit code $exit_code" ; fi
+
+	pkt_0=`jexec ${jname}a netstat -Wf link -I ${epair0}a | head | awk '$1!~/^Name/{print$8}'`
+	pkt_1=`jexec ${jname}a netstat -Wf link -I ${epair1}a | head | awk '$1!~/^Name/{print$8}'`
+	if [ ${pkt_0} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	if [ ${pkt_1} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	echo "UDP BALANCING: 1: ${pkt_0} 2: ${pkt_1}"
+}
+
+output6_udp_flowid_mpath_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_test_case "output6_raw_flowid_mpath_success" "cleanup"
+output6_raw_flowid_mpath_success_head()
+{
+
+	atf_set descr 'Test valid IPv6 raw output flowid generation'
+	atf_set require.user root
+}
+
+output6_raw_flowid_mpath_success_body()
+{
+
+	vnet_init
+	mpath_check
+
+	net_src="2001:db8:0:1"
+	net_dst="2001:db8:0:2"
+	ip_src="${net_src}::1"
+	ip_dst="${net_dst}::1"
+	plen=64
+	text="testtesttst"
+
+	jname="v6t-output6_raw_flowid_mpath_success"
+
+	epair0=$(vnet_mkepair)
+	epair1=$(vnet_mkepair)
+	lo_src=$(vnet_mkloopback)
+	lo_dst=$(vnet_mkloopback)
+
+	vnet_mkjail ${jname}a ${epair0}a ${epair1}a ${lo_src}
+	jls -N
+	# enable link-local IPv6
+	jexec ${jname}a ndp -i ${epair0}a -- -disabled
+	jexec ${jname}a ifconfig ${epair0}a up
+	jexec ${jname}a ndp -i ${epair1}a -- -disabled
+	jexec ${jname}a ifconfig ${epair1}a up
+	jexec ${jname}a ifconfig ${lo_src} up
+
+	vnet_mkjail ${jname}b ${epair0}b ${epair1}b ${lo_dst}
+	jls -N
+	jexec ${jname}b ndp -i ${epair0}b -- -disabled
+	jexec ${jname}b ifconfig ${epair0}b up
+	jexec ${jname}b ndp -i ${epair1}b -- -disabled
+	jexec ${jname}b ifconfig ${epair1}b up
+	jexec ${jname}b ifconfig ${lo_dst} up
+
+	# DST ips to test
+	ips="9d:33:f2:7f 48:06:9a:0b cf:96:d5:78 76:da:8e:28 d4:66:38:1e ec:43:da:7c bb:f8:93:2f d3:c4:eb:40"
+	ips="${ips} c7:31:0e:ae 8d:ed:35:2e c0:e0:22:31 82:1c:4e:28 c1:00:60:3e 6a:4f:3b:6c 8e:a7:e9:57 2b:ff:dd:52"
+	ips="${ips} 88:44:79:5d 80:62:83:11 c8:e4:17:a6 e7:2a:45:d7 5a:92:0e:04 70:fc:6a:ee ce:24:4c:68 41:2c:4d:43"
+	ips="${ips} 57:2b:5e:a7 f9:e0:69:c6 cb:b9:e6:ed 28:88:5d:fa d6:19:ac:1d dc:de:37:d8 fe:39:55:c7 b1:31:da:d7"
+
+	jexec ${jname}a ifconfig ${lo_src} inet6 ${ip_src}/128
+
+	jexec ${jname}b ifconfig ${lo_dst} inet6 ${ip_dst}/128
+	for i in ${ips}; do
+		jexec ${jname}b ifconfig ${lo_dst} inet6 ${net_dst}:${i}/128
+	done
+
+	# wait for DAD to complete
+	while [ `jexec ${jname}b ifconfig | grep inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+	while [ `jexec ${jname}a ifconfig | grep inet6 | grep -c tentative` != "0" ]; do
+		sleep 0.1
+	done
+
+	# Add routes
+	# A -> towards B via epair0a LL
+	ll=`jexec ${jname}b ifconfig ${epair0}b inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}a route add -6 -net ${net_dst}::/${plen} ${ll}%${epair0}a
+	# A -> towards B via epair1a LL
+	ll=`jexec ${jname}b ifconfig ${epair1}b inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}a route add -6 -net ${net_dst}::/${plen} ${ll}%${epair1}a
+
+	# B towards A via epair0b LL
+	ll=`jexec ${jname}a ifconfig ${epair1}a inet6 | awk '$2~/^fe80:/{print$2}' | awk -F% '{print$1}'`
+	jexec ${jname}b route add -6 -net ${net_src}::/${plen} ${ll}%${epair1}b
+	
+	# Base setup verification
+	atf_check -o match:'1 packets transmitted, 1 packets received' jexec ${jname}a ping6 -nc1 ${ip_dst}
+
+	# run sender
+	valid_message='1 packets transmitted, 1 packets received'
+	for _ip in ${ips}; do
+		ip="${net_dst}:${_ip}"
+		atf_check -o match:"${valid_message}" jexec ${jname}a ping6 -nc1 ${ip}
+	done
+
+	pkt_0=`jexec ${jname}a netstat -Wf link -I ${epair0}a | head | awk '$1!~/^Name/{print$8}'`
+	pkt_1=`jexec ${jname}a netstat -Wf link -I ${epair1}a | head | awk '$1!~/^Name/{print$8}'`
+
+	jexec ${jname}a netstat -bWf link -I ${epair0}a
+	jexec ${jname}a netstat -bWf link -I ${epair1}a
+	if [ ${pkt_0} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	if [ ${pkt_1} -le 10 ]; then
+		echo "Balancing failure: 1: ${pkt_0} 2: ${pkt_1}"
+		exit 1
+	fi
+	echo "RAW BALANCING: 1: ${pkt_0} 2: ${pkt_1}"
+}
+
+output6_raw_flowid_mpath_success_cleanup()
+{
+	vnet_cleanup
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case "output6_tcp_setup_success"
+	atf_add_test_case "output6_udp_setup_success"
+	atf_add_test_case "output6_raw_success"
+	atf_add_test_case "output6_tcp_flowid_mpath_success"
+	atf_add_test_case "output6_udp_flowid_mpath_success"
+	atf_add_test_case "output6_raw_flowid_mpath_success"
+}
+
+# end
+
+