From 10108cb6737ba39ad0289e84255f039a8258a6cc Mon Sep 17 00:00:00 2001 From: "Bjoern A. Zeeb" Date: Mon, 17 Feb 2020 11:08:50 +0000 Subject: [PATCH] Partially revert VNET change and expand VNET structure. Revert parts of r353274 replacing vnet_state with a shutdown flag. Not having the state flag for the current SI_SUB_* makes it harder to debug kernel or module panics related to VNET bringup or teardown. Not having the state also does not allow us to check for other dependency levels between components, e.g. for moving interfaces. Expand the VNET structure with the new boolean flag indicating that we are doing a shutdown of a given vnet and update the vnet magic cookie for the change. Update libkvm to compile with a bool in the kernel struct. Bump __FreeBSD_version for (external) module builds to more easily detect the change. Reviewed by: hselasky MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D23097 --- UPDATING | 5 +++++ lib/libkvm/kvm.c | 1 + lib/libkvm/kvm_private.c | 1 + lib/libkvm/kvm_vnet.c | 1 + sys/net/if.c | 21 +++++++++++++++++---- sys/net/vnet.c | 24 ++++++++++++++++-------- sys/net/vnet.h | 7 ++++--- sys/sys/param.h | 2 +- 8 files changed, 46 insertions(+), 16 deletions(-) diff --git a/UPDATING b/UPDATING index 87a4cdf4f730..50b44d527694 100644 --- a/UPDATING +++ b/UPDATING @@ -26,6 +26,11 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 13.x IS SLOW: disable the most expensive debugging functionality run "ln -s 'abort:false,junk:false' /etc/malloc.conf".) +20200217: + The size of struct vnet and the magic cookie have changed. + Users need to recompile libkvm and all modules using VIMAGE + together with their new kernel. + 20200212: Defining the long deprecated NO_CTF, NO_DEBUG_FILES, NO_INSTALLLIB, NO_MAN, NO_PROFILE, and NO_WARNS variables is now an error. Update diff --git a/lib/libkvm/kvm.c b/lib/libkvm/kvm.c index c22f84ed64fa..95c5a580a2dd 100644 --- a/lib/libkvm/kvm.c +++ b/lib/libkvm/kvm.c @@ -49,6 +49,7 @@ __SCCSID("@(#)kvm.c 8.2 (Berkeley) 2/13/94"); #include #include +#include #include #include diff --git a/lib/libkvm/kvm_private.c b/lib/libkvm/kvm_private.c index 2f5915c18173..ea576d8d4d2b 100644 --- a/lib/libkvm/kvm_private.c +++ b/lib/libkvm/kvm_private.c @@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include diff --git a/lib/libkvm/kvm_vnet.c b/lib/libkvm/kvm_vnet.c index eac9584cc730..79493bc5ffa8 100644 --- a/lib/libkvm/kvm_vnet.c +++ b/lib/libkvm/kvm_vnet.c @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include diff --git a/sys/net/if.c b/sys/net/if.c index 738d73616c58..42d34bb21ddb 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -322,6 +322,11 @@ SX_SYSINIT_FLAGS(ifnet_sx, &ifnet_sxlock, "ifnet_sx", SX_RECURSE); */ #define IFNET_HOLD (void *)(uintptr_t)(-1) +#ifdef VIMAGE +#define VNET_IS_SHUTTING_DOWN(_vnet) \ + ((_vnet)->vnet_shutdown && (_vnet)->vnet_state < SI_SUB_VNET_DONE) +#endif + static if_com_alloc_t *if_com_alloc[256]; static if_com_free_t *if_com_free[256]; @@ -1080,7 +1085,7 @@ if_detach_internal(struct ifnet *ifp, int vmove, struct if_clone **ifcp) #ifdef VIMAGE bool shutdown; - shutdown = ifp->if_vnet->vnet_shutdown; + shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet); #endif IFNET_WLOCK(); CK_STAILQ_FOREACH(iter, &V_ifnet, if_link) @@ -1339,6 +1344,7 @@ if_vmove_loan(struct thread *td, struct ifnet *ifp, char *ifname, int jid) struct prison *pr; struct ifnet *difp; int error; + bool shutdown; /* Try to find the prison within our visibility. */ sx_slock(&allprison_lock); @@ -1366,7 +1372,8 @@ if_vmove_loan(struct thread *td, struct ifnet *ifp, char *ifname, int jid) } /* Make sure the VNET is stable. */ - if (ifp->if_vnet->vnet_shutdown) { + shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet); + if (shutdown) { CURVNET_RESTORE(); prison_free(pr); return (EBUSY); @@ -1391,6 +1398,7 @@ if_vmove_reclaim(struct thread *td, char *ifname, int jid) struct vnet *vnet_dst; struct ifnet *ifp; int error; + bool shutdown; /* Try to find the prison within our visibility. */ sx_slock(&allprison_lock); @@ -1419,7 +1427,8 @@ if_vmove_reclaim(struct thread *td, char *ifname, int jid) } /* Make sure the VNET is stable. */ - if (ifp->if_vnet->vnet_shutdown) { + shutdown = VNET_IS_SHUTTING_DOWN(ifp->if_vnet); + if (shutdown) { CURVNET_RESTORE(); prison_free(pr); return (EBUSY); @@ -2950,11 +2959,15 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td) struct ifreq *ifr; int error; int oif_flags; +#ifdef VIMAGE + bool shutdown; +#endif CURVNET_SET(so->so_vnet); #ifdef VIMAGE /* Make sure the VNET is stable. */ - if (so->so_vnet->vnet_shutdown) { + shutdown = VNET_IS_SHUTTING_DOWN(so->so_vnet); + if (shutdown) { CURVNET_RESTORE(); return (EBUSY); } diff --git a/sys/net/vnet.c b/sys/net/vnet.c index ec1b81de6404..c5dafedbc6b2 100644 --- a/sys/net/vnet.c +++ b/sys/net/vnet.c @@ -279,6 +279,9 @@ vnet_destroy(struct vnet *vnet) LIST_REMOVE(vnet, vnet_le); VNET_LIST_WUNLOCK(); + /* Signal that VNET is being shutdown. */ + vnet->vnet_shutdown = true; + CURVNET_SET_QUIET(vnet); vnet_sysuninit(); CURVNET_RESTORE(); @@ -350,15 +353,15 @@ vnet_data_startup(void *dummy __unused) } SYSINIT(vnet_data, SI_SUB_KLD, SI_ORDER_FIRST, vnet_data_startup, NULL); +/* Dummy VNET_SYSINIT to make sure we always reach the final end state. */ static void -vnet_sysuninit_shutdown(void *unused __unused) +vnet_sysinit_done(void *unused __unused) { - /* Signal that VNET is being shutdown. */ - curvnet->vnet_shutdown = 1; + return; } -VNET_SYSUNINIT(vnet_sysuninit_shutdown, SI_SUB_VNET_DONE, SI_ORDER_FIRST, - vnet_sysuninit_shutdown, NULL); +VNET_SYSINIT(vnet_sysinit_done, SI_SUB_VNET_DONE, SI_ORDER_ANY, + vnet_sysinit_done, NULL); /* * When a module is loaded and requires storage for a virtualized global @@ -572,8 +575,10 @@ vnet_sysinit(void) struct vnet_sysinit *vs; VNET_SYSINIT_RLOCK(); - TAILQ_FOREACH(vs, &vnet_constructors, link) + TAILQ_FOREACH(vs, &vnet_constructors, link) { + curvnet->vnet_state = vs->subsystem; vs->func(vs->arg); + } VNET_SYSINIT_RUNLOCK(); } @@ -589,8 +594,10 @@ vnet_sysuninit(void) VNET_SYSINIT_RLOCK(); TAILQ_FOREACH_REVERSE(vs, &vnet_destructors, vnet_sysuninit_head, - link) + link) { + curvnet->vnet_state = vs->subsystem; vs->func(vs->arg); + } VNET_SYSINIT_RUNLOCK(); } @@ -704,7 +711,8 @@ db_vnet_print(struct vnet *vnet) db_printf(" vnet_data_mem = %p\n", vnet->vnet_data_mem); db_printf(" vnet_data_base = %#jx\n", (uintmax_t)vnet->vnet_data_base); - db_printf(" vnet_shutdown = %#08x\n", vnet->vnet_shutdown); + db_printf(" vnet_state = %#08x\n", vnet->vnet_state); + db_printf(" vnet_shutdown = %#03x\n", vnet->vnet_shutdown); db_printf("\n"); } diff --git a/sys/net/vnet.h b/sys/net/vnet.h index e02a04f97055..3886da1ccace 100644 --- a/sys/net/vnet.h +++ b/sys/net/vnet.h @@ -72,11 +72,12 @@ struct vnet { u_int vnet_magic_n; u_int vnet_ifcnt; u_int vnet_sockcnt; - u_int vnet_shutdown; /* Shutdown in progress. */ + u_int vnet_state; /* SI_SUB_* */ void *vnet_data_mem; uintptr_t vnet_data_base; -}; -#define VNET_MAGIC_N 0x3e0d8f29 + bool vnet_shutdown; /* Shutdown in progress. */ +} __aligned(CACHE_LINE_SIZE); +#define VNET_MAGIC_N 0x5e4a6f28 /* * These two virtual network stack allocator definitions are also required diff --git a/sys/sys/param.h b/sys/sys/param.h index 6cbf09c6ca8d..398038a0c31a 100644 --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -60,7 +60,7 @@ * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1300077 /* Master, propagated to newvers */ +#define __FreeBSD_version 1300078 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,