From ddc750763746ac63cc60cc3597c7ee7be535be9b Mon Sep 17 00:00:00 2001 From: "Alexander V. Chernikov" Date: Sun, 29 Mar 2020 06:25:57 +0000 Subject: [PATCH] Switch rtsock tests to per-test jails and epair interfaces. Many rtsock tests verify the ordering of the kernel messages for the particular event. In order to avoid flaky tests due to the other tests running, switch all tests to use personal vnet-enabled jails. This removes all clashes on the IP addresses and brings back the ability to run these tests simultaneously. Reported by: olivier Reviewed by: olivier Differential Revision: https://reviews.freebsd.org/D24182 --- tests/sys/net/routing/Makefile | 3 + tests/sys/net/routing/generic_cleanup.sh | 36 ++++++++++ tests/sys/net/routing/params.h | 38 ++++++++++ tests/sys/net/routing/rtsock_common.h | 80 ++++++++++++++++++++++ tests/sys/net/routing/rtsock_config.h | 32 ++++++--- tests/sys/net/routing/test_rtsock_l3.c | 26 +++++-- tests/sys/net/routing/test_rtsock_lladdr.c | 26 +++++-- 7 files changed, 218 insertions(+), 23 deletions(-) create mode 100755 tests/sys/net/routing/generic_cleanup.sh create mode 100644 tests/sys/net/routing/params.h diff --git a/tests/sys/net/routing/Makefile b/tests/sys/net/routing/Makefile index 02e045fe4a8a..9bfd1e4ca0cd 100644 --- a/tests/sys/net/routing/Makefile +++ b/tests/sys/net/routing/Makefile @@ -7,6 +7,9 @@ TESTSDIR= ${TESTSBASE}/sys/net/routing ATF_TESTS_C += test_rtsock_l3 ATF_TESTS_C += test_rtsock_lladdr +${PACKAGE}FILES+= generic_cleanup.sh +${PACKAGE}FILESMODE_generic_cleanup.sh=0555 + # Most of the tests operates on a common IPv4/IPv6 prefix, # so running them in parallel will lead to weird results. TEST_METADATA+= is_exclusive=true diff --git a/tests/sys/net/routing/generic_cleanup.sh b/tests/sys/net/routing/generic_cleanup.sh new file mode 100755 index 000000000000..a2feb7c430f8 --- /dev/null +++ b/tests/sys/net/routing/generic_cleanup.sh @@ -0,0 +1,36 @@ +#!/bin/sh +#- +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2020 Alexander V. Chernikov +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + + +srcdir=`dirname $0` +. ${srcdir}/../../common/vnet.subr + +vnet_cleanup + diff --git a/tests/sys/net/routing/params.h b/tests/sys/net/routing/params.h new file mode 100644 index 000000000000..e17dc25cc922 --- /dev/null +++ b/tests/sys/net/routing/params.h @@ -0,0 +1,38 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 Alexander V. Chernikov + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _NET_ROUTING_PARAMS_H_ +#define _NET_ROUTING_PARAMS_H_ + +/* files to store state */ +#define JAILS_FNAME "created_jails.lst" +#define IFACES_FNAME "created_interfaces.lst" + +#endif + diff --git a/tests/sys/net/routing/rtsock_common.h b/tests/sys/net/routing/rtsock_common.h index 400fe8deae6d..23054d3e55d8 100644 --- a/tests/sys/net/routing/rtsock_common.h +++ b/tests/sys/net/routing/rtsock_common.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -64,6 +65,7 @@ #include #include "rtsock_print.h" +#include "params.h" void rtsock_update_rtm_len(struct rt_msghdr *rtm); void rtsock_validate_message(char *buffer, ssize_t len); @@ -317,6 +319,84 @@ iface_enable_ipv6(char *ifname) return (0); } +void +file_append_line(char *fname, char *text) +{ + FILE *f; + + f = fopen(fname, "a"); + fputs(text, f); + fputs("\n", f); + fclose(f); +} + +static int +vnet_wait_interface(char *vnet_name, char *ifname) +{ + char buf[512], cmd[512], *line, *token; + FILE *fp; + int i; + + snprintf(cmd, sizeof(cmd), "/usr/sbin/jexec %s /sbin/ifconfig -l", vnet_name); + for (int i = 0; i < 50; i++) { + fp = popen(cmd, "r"); + line = fgets(buf, sizeof(buf), fp); + /* cut last\n */ + if (line[0]) + line[strlen(line)-1] = '\0'; + while ((token = strsep(&line, " ")) != NULL) { + if (strcmp(token, ifname) == 0) + return (1); + } + + /* sleep 100ms */ + usleep(1000 * 100); + } + + return (0); +} + +void +vnet_switch(char *vnet_name, char *ifname) +{ + char buf[512], cmd[512], *line; + FILE *fp; + int jid, ret; + + RLOG("switching to vnet %s with interface %s", vnet_name, ifname); + snprintf(cmd, sizeof(cmd), + "/usr/sbin/jail -i -c name=%s persist vnet vnet.interface=%s", + vnet_name, ifname); + RLOG("jail cmd: \"%s\"\n", cmd); + + fp = popen(cmd, "r"); + if (fp == NULL) + atf_tc_fail("jail creation failed"); + line = fgets(buf, sizeof(buf), fp); + if (line == NULL) + atf_tc_fail("empty output from jail(8)"); + jid = strtol(line, NULL, 10); + if (jid <= 0) { + atf_tc_fail("invalid jail output: %s", line); + } + + RLOG("created jail jid=%d", jid); + file_append_line(JAILS_FNAME, vnet_name); + + /* Wait while interface appearsh inside vnet */ + if (!vnet_wait_interface(vnet_name, ifname)) { + atf_tc_fail("unable to move interface %s to jail %s", ifname, vnet_name); + } + + if (jail_attach(jid) == -1) { + RLOG_ERRNO("jail %s attach failed: ret=%d", vnet_name, errno); + atf_tc_fail("jail attach failed"); + } + + RLOG("attached to the jail"); +} + + #define SA_F_IGNORE_IFNAME 0x01 #define SA_F_IGNORE_IFTYPE 0x02 #define SA_F_IGNORE_MEMCMP 0x04 diff --git a/tests/sys/net/routing/rtsock_config.h b/tests/sys/net/routing/rtsock_config.h index e493b0b47359..9db352fa0d6f 100644 --- a/tests/sys/net/routing/rtsock_config.h +++ b/tests/sys/net/routing/rtsock_config.h @@ -30,6 +30,8 @@ #ifndef _NET_ROUTING_RTSOCK_CONFIG_H_ #define _NET_ROUTING_RTSOCK_CONFIG_H_ +#include "params.h" + struct rtsock_test_config { int ifindex; char net4_str[INET_ADDRSTRLEN]; @@ -121,8 +123,8 @@ config_setup(const atf_tc_t *tc) inet_ntop(AF_INET6, &c->net6.sin6_addr, c->net6_str, INET6_ADDRSTRLEN); inet_ntop(AF_INET6, &c->addr6.sin6_addr, c->addr6_str, INET6_ADDRSTRLEN); - c->ifname = strdup(atf_tc_get_config_var_wd(tc, "rtsock.ifname", "tap4242")); - c->autocreated_interface = atf_tc_get_config_var_as_bool_wd(tc, "rtsock.create_interface", true); + c->ifname = strdup("epair"); + c->autocreated_interface = true; if (c->autocreated_interface && (if_nametoindex(c->ifname) == 0)) { @@ -130,8 +132,15 @@ config_setup(const atf_tc_t *tc) char new_ifname[IFNAMSIZ]; strlcpy(new_ifname, c->ifname, sizeof(new_ifname)); int ret = iface_create_cloned(new_ifname); - ATF_REQUIRE_MSG(ret != 0, "tap interface creation failed: %s", strerror(errno)); + ATF_REQUIRE_MSG(ret != 0, "%s interface creation failed: %s", new_ifname, + strerror(errno)); c->ifname = strdup(new_ifname); + file_append_line(IFACES_FNAME, new_ifname); + if (strstr(new_ifname, "epair") == new_ifname) { + /* call returned epairXXXa, need to add epairXXXb */ + new_ifname[strlen(new_ifname) - 1] = 'b'; + file_append_line(IFACES_FNAME, new_ifname); + } } c->ifindex = if_nametoindex(c->ifname); ATF_REQUIRE_MSG(c->ifindex != 0, "inteface %s not found", c->ifname); @@ -143,13 +152,18 @@ config_setup(const atf_tc_t *tc) } void -config_generic_cleanup(struct rtsock_test_config *c) +config_generic_cleanup(const atf_tc_t *tc) { - if (c->ifname != NULL && c->autocreated_interface) { - iface_destroy(c->ifname); - free(c->ifname); - c->ifname = NULL; - } + const char *srcdir = atf_tc_get_config_var(tc, "srcdir"); + char cmd[512]; + int ret; + + /* XXX: sleep 100ms to avoid epair qflush panic */ + usleep(1000 * 100); + snprintf(cmd, sizeof(cmd), "%s/generic_cleanup.sh", srcdir); + ret = system(cmd); + if (ret != 0) + RLOG("'%s' failed, error %d", cmd, ret); } void diff --git a/tests/sys/net/routing/test_rtsock_l3.c b/tests/sys/net/routing/test_rtsock_l3.c index 85580b065e57..420fba345781 100644 --- a/tests/sys/net/routing/test_rtsock_l3.c +++ b/tests/sys/net/routing/test_rtsock_l3.c @@ -35,6 +35,20 @@ #include "net/bpf.h" +static void +jump_vnet(struct rtsock_test_config *c, const atf_tc_t *tc) +{ + char vnet_name[512]; + + snprintf(vnet_name, sizeof(vnet_name), "vt-%s", atf_tc_get_ident(tc)); + RLOG("jumping to %s", vnet_name); + + vnet_switch(vnet_name, c->ifname); + + /* Update ifindex cache */ + c->ifindex = if_nametoindex(c->ifname); +} + static inline struct rtsock_test_config * presetup_ipv6_iface(const atf_tc_t *tc) { @@ -43,6 +57,8 @@ presetup_ipv6_iface(const atf_tc_t *tc) c = config_setup(tc); + jump_vnet(c, tc); + ret = iface_turn_up(c->ifname); ATF_REQUIRE_MSG(ret == 0, "Unable to turn up %s", c->ifname); @@ -75,15 +91,11 @@ presetup_ipv4_iface(const atf_tc_t *tc) c = config_setup(tc); + jump_vnet(c, tc); + ret = iface_turn_up(c->ifname); ATF_REQUIRE_MSG(ret == 0, "Unable to turn up %s", c->ifname); - /* Actually open interface, so kernel writes won't fail */ - if (c->autocreated_interface) { - ret = iface_open(c->ifname); - ATF_REQUIRE_MSG(ret >= 0, "unable to open interface %s", c->ifname); - } - return (c); } @@ -235,7 +247,7 @@ verify_link_gateway(struct rt_msghdr *rtm, int ifindex) \ #define DESCRIBE_ROOT_TEST(_msg) config_describe_root_test(tc, _msg) -#define CLEANUP_AFTER_TEST config_generic_cleanup(config_setup(tc)) +#define CLEANUP_AFTER_TEST config_generic_cleanup(tc) #define RTM_DECLARE_ROOT_TEST(_name, _descr) \ ATF_TC_WITH_CLEANUP(_name); \ diff --git a/tests/sys/net/routing/test_rtsock_lladdr.c b/tests/sys/net/routing/test_rtsock_lladdr.c index 96980d4b19f7..9f535c262d07 100644 --- a/tests/sys/net/routing/test_rtsock_lladdr.c +++ b/tests/sys/net/routing/test_rtsock_lladdr.c @@ -30,6 +30,20 @@ #include "rtsock_common.h" #include "rtsock_config.h" +static void +jump_vnet(struct rtsock_test_config *c, const atf_tc_t *tc) +{ + char vnet_name[512]; + + snprintf(vnet_name, sizeof(vnet_name), "vt-%s", atf_tc_get_ident(tc)); + RLOG("jumping to %s", vnet_name); + + vnet_switch(vnet_name, c->ifname); + + /* Update ifindex cache */ + c->ifindex = if_nametoindex(c->ifname); +} + static inline struct rtsock_test_config * presetup_ipv6(const atf_tc_t *tc) { @@ -38,6 +52,8 @@ presetup_ipv6(const atf_tc_t *tc) c = config_setup(tc); + jump_vnet(c, tc); + ret = iface_turn_up(c->ifname); ATF_REQUIRE_MSG(ret == 0, "Unable to turn up %s", c->ifname); ret = iface_enable_ipv6(c->ifname); @@ -56,16 +72,12 @@ presetup_ipv4(const atf_tc_t *tc) c = config_setup(tc); + jump_vnet(c, tc); + /* assumes ifconfig doing IFF_UP */ ret = iface_setup_addr(c->ifname, c->addr4_str, c->plen4); ATF_REQUIRE_MSG(ret == 0, "ifconfig failed"); - /* Actually open interface, so kernel writes won't fail */ - if (c->autocreated_interface) { - ret = iface_open(c->ifname); - ATF_REQUIRE_MSG(ret >= 0, "unable to open interface %s", c->ifname); - } - c->rtsock_fd = rtsock_setup_socket(); return (c); @@ -96,7 +108,7 @@ prepare_route_message(struct rt_msghdr *rtm, int cmd, struct sockaddr *dst, \ #define DESCRIBE_ROOT_TEST(_msg) config_describe_root_test(tc, _msg) -#define CLEANUP_AFTER_TEST config_generic_cleanup(config_setup(tc)) +#define CLEANUP_AFTER_TEST config_generic_cleanup(tc) #define RTM_DECLARE_ROOT_TEST(_name, _descr) \ ATF_TC_WITH_CLEANUP(_name); \