Add support for preinit, init and fini arrays. Some ABIs, in

particular on ARM, do require working init arrays.

Traditional FreeBSD crt1 calls _init and _fini of the binary, instead
of allowing runtime linker to arrange the calls.  This was probably
done to have the same crt code serve both statically and dynamically
linked binaries.  Since ABI mandates that first is called preinit
array functions, then init, and then init array functions, the init
have to be called from rtld now.

To provide binary compatibility to old FreeBSD crt1, which calls _init
itself, rtld only calls intializers and finalizers for main binary if
binary has a note indicating that new crt was used for linking.  Add
parsing of ELF notes to rtld, and cache p_osrel value since we parsed
it anyway.

The patch is inspired by init_array support for DragonflyBSD, written
by John Marino.

Reviewed by:	kan
Tested by:	andrew (arm, previous version), flo (sparc64, previous version)
MFC after:	3 weeks

libexec/rtld-elf/Makefile

@@ -10,6 +10,7 @@ SRCS=		rtld_start.S \
 MAN=		rtld.1
 CSTD?=		gnu99
 CFLAGS+=	-Wall -DFREEBSD_ELF -DIN_RTLD
+CFLAGS+=	-I${.CURDIR}/../../lib/csu/common
 .if exists(${.CURDIR}/${MACHINE_ARCH})
 RTLD_ARCH=	${MACHINE_ARCH}
 .else

libexec/rtld-elf/amd64/rtld_machdep.h

@@ -58,6 +58,9 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 #define round(size, align) \
 	(((size) + (align) - 1) & ~((align) - 1))
 #define calculate_first_tls_offset(size, align) \

libexec/rtld-elf/arm/rtld_machdep.h

@@ -48,6 +48,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 	
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 #define	TLS_TCB_SIZE	8
 typedef struct {
 	unsigned long ti_module;

libexec/rtld-elf/i386/rtld_machdep.h

@@ -58,6 +58,9 @@ reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 #define round(size, align) \
 	(((size) + (align) - 1) & ~((align) - 1))
 #define calculate_first_tls_offset(size, align) \

libexec/rtld-elf/ia64/reloc.c

@@ -586,6 +586,18 @@ call_initfini_pointer(const Obj_Entry *obj, Elf_Addr target)
 	((InitFunc) &fptr)();
 }
 
+void
+call_init_pointer(const Obj_Entry *obj, Elf_Addr target)
+{
+	struct fptr fptr;
+
+	fptr.gp = (Elf_Addr) obj->pltgot;
+	fptr.target = target;
+	dbg(" initfini: target=%p, gp=%p",
+	    (void *) fptr.target, (void *) fptr.gp);
+	((InitArrFunc) &fptr)(main_argc, main_argv, environ);
+}
+
 /* Initialize the special PLT entries. */
 void
 init_pltgot(Obj_Entry *obj)

libexec/rtld-elf/ia64/rtld_machdep.h

@@ -52,6 +52,7 @@ Elf_Addr reloc_jmpslot(Elf_Addr *, Elf_Addr, const struct Struct_Obj_Entry *,
 		       const struct Struct_Obj_Entry *, const Elf_Rel *);
 void *make_function_pointer(const Elf_Sym *, const struct Struct_Obj_Entry *);
 void call_initfini_pointer(const struct Struct_Obj_Entry *, Elf_Addr);
+void call_init_pointer(const struct Struct_Obj_Entry *, Elf_Addr);
 
 #define	TLS_TCB_SIZE	16
 

libexec/rtld-elf/map_object.c

@@ -86,6 +86,8 @@ map_object(int fd, const char *path, const struct stat *sb)
     Elf_Word stack_flags;
     Elf_Addr relro_page;
     size_t relro_size;
+    Elf_Addr note_start;
+    Elf_Addr note_end;
 
     hdr = get_elf_header(fd, path);
     if (hdr == NULL)
@@ -104,6 +106,8 @@ map_object(int fd, const char *path, const struct stat *sb)
     phdr_vaddr = 0;
     relro_page = 0;
     relro_size = 0;
+    note_start = 0;
+    note_end = 0;
     segs = alloca(sizeof(segs[0]) * hdr->e_phnum);
     stack_flags = RTLD_DEFAULT_STACK_PF_EXEC | PF_R | PF_W;
     while (phdr < phlimit) {
@@ -143,6 +147,12 @@ map_object(int fd, const char *path, const struct stat *sb)
 	    relro_page = phdr->p_vaddr;
 	    relro_size = phdr->p_memsz;
 	    break;
+
+	case PT_NOTE:
+	    note_start = (Elf_Addr)obj->relocbase + phdr->p_offset;
+	    note_end = note_start + phdr->p_filesz;
+	    digest_notes(obj, note_start, note_end);
+	    break;
 	}
 
 	++phdr;

libexec/rtld-elf/mips/rtld_machdep.h

@@ -49,6 +49,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 typedef struct {
 	unsigned long ti_module;
 	unsigned long ti_offset;

libexec/rtld-elf/powerpc/rtld_machdep.h

@@ -48,6 +48,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 /*
  * Lazy binding entry point, called via PLT.
  */

libexec/rtld-elf/powerpc64/rtld_machdep.h

@@ -48,6 +48,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *where, Elf_Addr target,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 /*
  * Lazy binding entry point, called via PLT.
  */

libexec/rtld-elf/rtld.c

@@ -61,6 +61,7 @@
 #include "libmap.h"
 #include "rtld_tls.h"
 #include "rtld_printf.h"
+#include "notes.h"
 
 #ifndef COMPAT_32BIT
 #define PATH_RTLD	"/libexec/ld-elf.so.1"
@@ -139,6 +140,7 @@ static void ref_dag(Obj_Entry *);
 static int origin_subst_one(char **, const char *, const char *,
   const char *, char *);
 static char *origin_subst(const char *, const char *);
+static void preinit_main(void);
 static int  rtld_verify_versions(const Objlist *);
 static int  rtld_verify_object_versions(Obj_Entry *);
 static void object_add_name(Obj_Entry *, const char *);
@@ -204,6 +206,12 @@ static int max_stack_flags;
 char *__progname;
 char **environ;
 
+/*
+ * Used to pass argc, argv to init functions.
+ */
+int main_argc;
+char **main_argv;
+
 /*
  * Globals to control TLS allocation.
  */
@@ -335,6 +343,8 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     __progname = obj_rtld.path;
     argv0 = argv[0] != NULL ? argv[0] : "(null)";
     environ = env;
+    main_argc = argc;
+    main_argv = argv;
 
     trust = !issetugid();
 
@@ -458,8 +468,6 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
     obj_tail = &obj_main->next;
     obj_count++;
     obj_loads++;
-    /* Make sure we don't call the main program's init and fini functions. */
-    obj_main->init = obj_main->fini = (Elf_Addr)NULL;
 
     /* Initialize a fake symbol for resolving undefined weak references. */
     sym_zero.st_info = ELF_ST_INFO(STB_GLOBAL, STT_NOTYPE);
@@ -551,7 +559,20 @@ _rtld(Elf_Addr *sp, func_ptr_type *exit_proc, Obj_Entry **objp)
       ld_bind_now != NULL && *ld_bind_now != '\0', NULL) == -1)
 	die();
 
+    if (!obj_main->crt_no_init) {
+	/*
+	 * Make sure we don't call the main program's init and fini
+	 * functions for binaries linked with old crt1 which calls
+	 * _init itself.
+	 */
+	obj_main->init = obj_main->fini = (Elf_Addr)NULL;
+	obj_main->preinit_array = obj_main->init_array =
+	    obj_main->fini_array = (Elf_Addr)NULL;
+    }
+
     wlock_acquire(rtld_bind_lock, &lockstate);
+    if (obj_main->crt_no_init)
+	preinit_main();
     objlist_call_init(&initlist, &lockstate);
     objlist_clear(&initlist);
     dbg("loading filtees");
@@ -936,10 +957,34 @@ digest_dynamic1(Obj_Entry *obj, int early, const Elf_Dyn **dyn_rpath,
 	    obj->init = (Elf_Addr) (obj->relocbase + dynp->d_un.d_ptr);
 	    break;
 
+	case DT_PREINIT_ARRAY:
+	    obj->preinit_array = (Elf_Addr)(obj->relocbase + dynp->d_un.d_ptr);
+	    break;
+
+	case DT_PREINIT_ARRAYSZ:
+	    obj->preinit_array_num = dynp->d_un.d_val / sizeof(Elf_Addr);
+	    break;
+
+	case DT_INIT_ARRAY:
+	    obj->init_array = (Elf_Addr)(obj->relocbase + dynp->d_un.d_ptr);
+	    break;
+
+	case DT_INIT_ARRAYSZ:
+	    obj->init_array_num = dynp->d_un.d_val / sizeof(Elf_Addr);
+	    break;
+
 	case DT_FINI:
 	    obj->fini = (Elf_Addr) (obj->relocbase + dynp->d_un.d_ptr);
 	    break;
 
+	case DT_FINI_ARRAY:
+	    obj->fini_array = (Elf_Addr)(obj->relocbase + dynp->d_un.d_ptr);
+	    break;
+
+	case DT_FINI_ARRAYSZ:
+	    obj->fini_array_num = dynp->d_un.d_val / sizeof(Elf_Addr);
+	    break;
+
 	/*
 	 * Don't process DT_DEBUG on MIPS as the dynamic section
 	 * is mapped read-only. DT_MIPS_RLD_MAP is used instead.
@@ -1065,6 +1110,7 @@ digest_phdr(const Elf_Phdr *phdr, int phnum, caddr_t entry, const char *path)
     Obj_Entry *obj;
     const Elf_Phdr *phlimit = phdr + phnum;
     const Elf_Phdr *ph;
+    Elf_Addr note_start, note_end;
     int nsegs = 0;
 
     obj = obj_new();
@@ -1120,6 +1166,12 @@ digest_phdr(const Elf_Phdr *phdr, int phnum, caddr_t entry, const char *path)
 	    obj->relro_page = obj->relocbase + trunc_page(ph->p_vaddr);
 	    obj->relro_size = round_page(ph->p_memsz);
 	    break;
+
+	case PT_NOTE:
+	    note_start = (Elf_Addr)obj->relocbase + ph->p_vaddr;
+	    note_end = note_start + ph->p_filesz;
+	    digest_notes(obj, note_start, note_end);
+	    break;
 	}
     }
     if (nsegs < 1) {
@@ -1131,6 +1183,44 @@ digest_phdr(const Elf_Phdr *phdr, int phnum, caddr_t entry, const char *path)
     return obj;
 }
 
+void
+digest_notes(Obj_Entry *obj, Elf_Addr note_start, Elf_Addr note_end)
+{
+	const Elf_Note *note;
+	const char *note_name;
+	uintptr_t p;
+
+	for (note = (const Elf_Note *)note_start; (Elf_Addr)note < note_end;
+	    note = (const Elf_Note *)((const char *)(note + 1) +
+	      roundup2(note->n_namesz, sizeof(Elf32_Addr)) +
+	      roundup2(note->n_descsz, sizeof(Elf32_Addr)))) {
+		if (note->n_namesz != sizeof(NOTE_FREEBSD_VENDOR) ||
+		    note->n_descsz != sizeof(int32_t))
+			continue;
+		if (note->n_type != ABI_NOTETYPE &&
+		    note->n_type != CRT_NOINIT_NOTETYPE)
+			continue;
+		note_name = (const char *)(note + 1);
+		if (strncmp(NOTE_FREEBSD_VENDOR, note_name,
+		    sizeof(NOTE_FREEBSD_VENDOR)) != 0)
+			continue;
+		switch (note->n_type) {
+		case ABI_NOTETYPE:
+			/* FreeBSD osrel note */
+			p = (uintptr_t)(note + 1);
+			p += roundup2(note->n_namesz, sizeof(Elf32_Addr));
+			obj->osrel = *(const int32_t *)(p);
+			dbg("note osrel %d", obj->osrel);
+			break;
+		case CRT_NOINIT_NOTETYPE:
+			/* FreeBSD 'crt does not call init' note */
+			obj->crt_no_init = true;
+			dbg("note crt_no_init");
+			break;
+		}
+	}
+}
+
 static Obj_Entry *
 dlcheck(void *handle)
 {
@@ -1504,11 +1594,13 @@ initlist_add_objects(Obj_Entry *obj, Obj_Entry **tail, Objlist *list)
 	initlist_add_neededs(obj->needed, list);
 
     /* Add the object to the init list. */
-    if (obj->init != (Elf_Addr)NULL)
+    if (obj->preinit_array != (Elf_Addr)NULL || obj->init != (Elf_Addr)NULL ||
+      obj->init_array != (Elf_Addr)NULL)
 	objlist_push_tail(list, obj);
 
     /* Add the object to the global fini list in the reverse order. */
-    if (obj->fini != (Elf_Addr)NULL && !obj->on_fini_list) {
+    if ((obj->fini != (Elf_Addr)NULL || obj->fini_array != (Elf_Addr)NULL)
+      && !obj->on_fini_list) {
 	objlist_push_head(&list_fini, obj);
 	obj->on_fini_list = true;
     }
@@ -1796,6 +1888,27 @@ obj_from_addr(const void *addr)
     return NULL;
 }
 
+static void
+preinit_main(void)
+{
+    Elf_Addr *preinit_addr;
+    int index;
+
+    preinit_addr = (Elf_Addr *)obj_main->preinit_array;
+    if (preinit_addr == (Elf_Addr)NULL)
+	return;
+
+    for (index = 0; index < obj_main->preinit_array_num; index++) {
+	if (preinit_addr[index] != 0 && preinit_addr[index] != 1) {
+	    dbg("calling preinit function for %s at %p", obj_main->path,
+	      (void *)preinit_addr[index]);
+	    LD_UTRACE(UTRACE_INIT_CALL, obj_main, (void *)preinit_addr[index],
+	      0, 0, obj_main->path);
+	    call_init_pointer(obj_main, preinit_addr[index]);
+	}
+    }
+}
+
 /*
  * Call the finalization functions for each of the objects in "list"
  * belonging to the DAG of "root" and referenced once. If NULL "root"
@@ -1808,6 +1921,8 @@ objlist_call_fini(Objlist *list, Obj_Entry *root, RtldLockState *lockstate)
 {
     Objlist_Entry *elm;
     char *saved_msg;
+    Elf_Addr *fini_addr;
+    int index;
 
     assert(root == NULL || root->refcount == 1);
 
@@ -1821,10 +1936,6 @@ objlist_call_fini(Objlist *list, Obj_Entry *root, RtldLockState *lockstate)
 	    if (root != NULL && (elm->obj->refcount != 1 ||
 	      objlist_find(&root->dagmembers, elm->obj) == NULL))
 		continue;
-	    dbg("calling fini function for %s at %p", elm->obj->path,
-	        (void *)elm->obj->fini);
-	    LD_UTRACE(UTRACE_FINI_CALL, elm->obj, (void *)elm->obj->fini, 0, 0,
-		elm->obj->path);
 	    /* Remove object from fini list to prevent recursive invocation. */
 	    STAILQ_REMOVE(list, elm, Struct_Objlist_Entry, link);
 	    /*
@@ -1835,7 +1946,31 @@ objlist_call_fini(Objlist *list, Obj_Entry *root, RtldLockState *lockstate)
 	     * called.
 	     */
 	    lock_release(rtld_bind_lock, lockstate);
-	    call_initfini_pointer(elm->obj, elm->obj->fini);
+
+	    /*
+	     * It is legal to have both DT_FINI and DT_FINI_ARRAY defined.
+	     * When this happens, DT_FINI_ARRAY is processed first.
+	     */
+	    fini_addr = (Elf_Addr *)elm->obj->fini_array;
+	    if (fini_addr != NULL && elm->obj->fini_array_num > 0) {
+		for (index = elm->obj->fini_array_num - 1; index >= 0;
+		  index--) {
+		    if (fini_addr[index] != 0 && fini_addr[index] != 1) {
+			dbg("calling fini function for %s at %p",
+			    elm->obj->path, (void *)fini_addr[index]);
+			LD_UTRACE(UTRACE_FINI_CALL, elm->obj,
+			    (void *)fini_addr[index], 0, 0, elm->obj->path);
+			call_initfini_pointer(elm->obj, fini_addr[index]);
+		    }
+		}
+	    }
+	    if (elm->obj->fini != (Elf_Addr)NULL) {
+		dbg("calling fini function for %s at %p", elm->obj->path,
+		    (void *)elm->obj->fini);
+		LD_UTRACE(UTRACE_FINI_CALL, elm->obj, (void *)elm->obj->fini,
+		    0, 0, elm->obj->path);
+		call_initfini_pointer(elm->obj, elm->obj->fini);
+	    }
 	    wlock_acquire(rtld_bind_lock, lockstate);
 	    /* No need to free anything if process is going down. */
 	    if (root != NULL)
@@ -1862,6 +1997,8 @@ objlist_call_init(Objlist *list, RtldLockState *lockstate)
     Objlist_Entry *elm;
     Obj_Entry *obj;
     char *saved_msg;
+    Elf_Addr *init_addr;
+    int index;
 
     /*
      * Clean init_scanned flag so that objects can be rechecked and
@@ -1879,10 +2016,6 @@ objlist_call_init(Objlist *list, RtldLockState *lockstate)
     STAILQ_FOREACH(elm, list, link) {
 	if (elm->obj->init_done) /* Initialized early. */
 	    continue;
-	dbg("calling init function for %s at %p", elm->obj->path,
-	    (void *)elm->obj->init);
-	LD_UTRACE(UTRACE_INIT_CALL, elm->obj, (void *)elm->obj->init, 0, 0,
-	    elm->obj->path);
 	/*
 	 * Race: other thread might try to use this object before current
 	 * one completes the initilization. Not much can be done here
@@ -1890,7 +2023,30 @@ objlist_call_init(Objlist *list, RtldLockState *lockstate)
 	 */
 	elm->obj->init_done = true;
 	lock_release(rtld_bind_lock, lockstate);
-	call_initfini_pointer(elm->obj, elm->obj->init);
+
+        /*
+         * It is legal to have both DT_INIT and DT_INIT_ARRAY defined.
+         * When this happens, DT_INIT is processed first.
+         */
+	if (elm->obj->init != (Elf_Addr)NULL) {
+	    dbg("calling init function for %s at %p", elm->obj->path,
+	        (void *)elm->obj->init);
+	    LD_UTRACE(UTRACE_INIT_CALL, elm->obj, (void *)elm->obj->init,
+	        0, 0, elm->obj->path);
+	    call_initfini_pointer(elm->obj, elm->obj->init);
+	}
+	init_addr = (Elf_Addr *)elm->obj->init_array;
+	if (init_addr != (Elf_Addr)NULL) {
+	    for (index = 0; index < elm->obj->init_array_num; index++) {
+		if (init_addr[index] != 0 && init_addr[index] != 1) {
+		    dbg("calling init function for %s at %p", elm->obj->path,
+			(void *)init_addr[index]);
+		    LD_UTRACE(UTRACE_INIT_CALL, elm->obj,
+			(void *)init_addr[index], 0, 0, elm->obj->path);
+		    call_init_pointer(elm->obj, init_addr[index]);
+		}
+	    }
+	}
 	wlock_acquire(rtld_bind_lock, lockstate);
     }
     errmsg_restore(saved_msg);

libexec/rtld-elf/rtld.h

@@ -71,6 +71,10 @@ extern size_t tls_static_space;
 extern int tls_dtv_generation;
 extern int tls_max_index;
 
+extern int main_argc;
+extern char **main_argv;
+extern char **environ;
+
 struct stat;
 struct Struct_Obj_Entry;
 
@@ -84,6 +88,7 @@ typedef STAILQ_HEAD(Struct_Objlist, Struct_Objlist_Entry) Objlist;
 
 /* Types of init and fini functions */
 typedef void (*InitFunc)(void);
+typedef void (*InitArrFunc)(int, char **, char **);
 
 /* Lists of shared object dependencies */
 typedef struct Struct_Needed_Entry {
@@ -213,6 +218,14 @@ typedef struct Struct_Obj_Entry {
 
     Elf_Addr init;		/* Initialization function to call */
     Elf_Addr fini;		/* Termination function to call */
+    Elf_Addr preinit_array;	/* Pre-initialization array of functions */
+    Elf_Addr init_array;	/* Initialization array of functions */
+    Elf_Addr fini_array;	/* Termination array of functions */
+    int preinit_array_num;	/* Number of entries in preinit_array */
+    int init_array_num; 	/* Number of entries in init_array */
+    int fini_array_num; 	/* Number of entries in fini_array */
+
+    int32_t osrel;		/* OSREL note value */
 
     bool mainprog : 1;		/* True if this is the main program */
     bool rtld : 1;		/* True if this is the dynamic linker */
@@ -235,6 +248,7 @@ typedef struct Struct_Obj_Entry {
     bool filtees_loaded : 1;	/* Filtees loaded */
     bool irelative : 1;		/* Object has R_MACHDEP_IRELATIVE relocs */
     bool gnu_ifunc : 1;		/* Object has references to STT_GNU_IFUNC */
+    bool crt_no_init : 1;	/* Object' crt does not call _init/_fini */
 
     struct link_map linkmap;	/* For GDB and dlinfo() */
     Objlist dldags;		/* Object belongs to these dlopened DAGs (%) */
@@ -319,6 +333,7 @@ const Elf_Sym *find_symdef(unsigned long, const Obj_Entry *,
   const Obj_Entry **, int, SymCache *, struct Struct_RtldLockState *);
 void init_pltgot(Obj_Entry *);
 void lockdflt_init(void);
+void digest_notes(Obj_Entry *, Elf_Addr, Elf_Addr);
 void obj_free(Obj_Entry *);
 Obj_Entry *obj_new(void);
 void _rtld_bind_start(void);

libexec/rtld-elf/sparc64/rtld_machdep.h

@@ -50,6 +50,9 @@ Elf_Addr reloc_jmpslot(Elf_Addr *, Elf_Addr,
 #define call_initfini_pointer(obj, target) \
 	(((InitFunc)(target))())
 
+#define call_init_pointer(obj, target) \
+	(((InitArrFunc)(target))(main_argc, main_argv, environ))
+
 #define round(size, align) \
 	(((size) + (align) - 1) & ~((align) - 1))
 #define calculate_first_tls_offset(size, align) \