1998-08-24 08:25:26 +00:00
|
|
|
/*-
|
2000-04-24 17:08:04 +00:00
|
|
|
* Copyright (c) 1998-2000 Doug Rabson
|
1998-08-24 08:25:26 +00:00
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
2003-06-11 00:56:59 +00:00
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
2008-05-23 00:49:39 +00:00
|
|
|
#include "opt_ddb.h"
|
2004-07-10 21:37:30 +00:00
|
|
|
#include "opt_gdb.h"
|
2000-04-24 17:08:04 +00:00
|
|
|
|
1998-08-24 08:25:26 +00:00
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
2004-05-19 14:36:38 +00:00
|
|
|
#ifdef GPROF
|
|
|
|
#include <sys/gmon.h>
|
|
|
|
#endif
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/kernel.h>
|
|
|
|
#include <sys/lock.h>
|
1998-08-24 08:25:26 +00:00
|
|
|
#include <sys/malloc.h>
|
2001-06-03 02:19:07 +00:00
|
|
|
#include <sys/mutex.h>
|
2006-06-20 21:31:38 +00:00
|
|
|
#include <sys/mount.h>
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
#include <sys/pcpu.h>
|
1998-08-24 08:25:26 +00:00
|
|
|
#include <sys/proc.h>
|
|
|
|
#include <sys/namei.h>
|
|
|
|
#include <sys/fcntl.h>
|
|
|
|
#include <sys/vnode.h>
|
|
|
|
#include <sys/linker.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
|
1998-08-24 08:25:26 +00:00
|
|
|
#include <machine/elf.h>
|
|
|
|
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#include <net/vnet.h>
|
|
|
|
|
2006-10-22 11:52:19 +00:00
|
|
|
#include <security/mac/mac_framework.h>
|
|
|
|
|
1998-09-11 08:46:15 +00:00
|
|
|
#include <vm/vm.h>
|
|
|
|
#include <vm/vm_param.h>
|
1999-10-11 15:19:12 +00:00
|
|
|
#ifdef SPARSE_MAPPING
|
1998-09-11 08:46:15 +00:00
|
|
|
#include <vm/vm_object.h>
|
|
|
|
#include <vm/vm_kern.h>
|
|
|
|
#include <vm/vm_extern.h>
|
1999-10-11 15:19:12 +00:00
|
|
|
#endif
|
1998-09-11 08:46:15 +00:00
|
|
|
#include <vm/pmap.h>
|
|
|
|
#include <vm/vm_map.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
|
2002-08-22 20:39:30 +00:00
|
|
|
#include <sys/link_elf.h>
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2008-05-23 00:49:39 +00:00
|
|
|
#ifdef DDB_CTF
|
|
|
|
#include <net/zlib.h>
|
|
|
|
#endif
|
|
|
|
|
2000-04-24 17:08:04 +00:00
|
|
|
#include "linker_if.h"
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2005-10-17 23:21:55 +00:00
|
|
|
#define MAXSEGS 4
|
|
|
|
|
2008-07-11 01:10:40 +00:00
|
|
|
typedef struct elf_file {
|
2010-11-08 20:57:08 +00:00
|
|
|
struct linker_file lf; /* Common fields */
|
|
|
|
int preloaded; /* Was file pre-loaded */
|
|
|
|
caddr_t address; /* Relocation address */
|
2008-07-11 01:10:40 +00:00
|
|
|
#ifdef SPARSE_MAPPING
|
2010-11-08 20:57:08 +00:00
|
|
|
vm_object_t object; /* VM object to hold file pages */
|
2008-07-11 01:10:40 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
Elf_Dyn *dynamic; /* Symbol table etc. */
|
|
|
|
Elf_Hashelt nbuckets; /* DT_HASH info */
|
|
|
|
Elf_Hashelt nchains;
|
|
|
|
const Elf_Hashelt *buckets;
|
|
|
|
const Elf_Hashelt *chains;
|
|
|
|
caddr_t hash;
|
|
|
|
caddr_t strtab; /* DT_STRTAB */
|
|
|
|
int strsz; /* DT_STRSZ */
|
|
|
|
const Elf_Sym *symtab; /* DT_SYMTAB */
|
|
|
|
Elf_Addr *got; /* DT_PLTGOT */
|
|
|
|
const Elf_Rel *pltrel; /* DT_JMPREL */
|
|
|
|
int pltrelsize; /* DT_PLTRELSZ */
|
|
|
|
const Elf_Rela *pltrela; /* DT_JMPREL */
|
|
|
|
int pltrelasize; /* DT_PLTRELSZ */
|
|
|
|
const Elf_Rel *rel; /* DT_REL */
|
|
|
|
int relsize; /* DT_RELSZ */
|
|
|
|
const Elf_Rela *rela; /* DT_RELA */
|
|
|
|
int relasize; /* DT_RELASZ */
|
|
|
|
caddr_t modptr;
|
|
|
|
const Elf_Sym *ddbsymtab; /* The symbol table we are using */
|
|
|
|
long ddbsymcnt; /* Number of symbols */
|
|
|
|
caddr_t ddbstrtab; /* String table */
|
|
|
|
long ddbstrcnt; /* number of bytes in string table */
|
|
|
|
caddr_t symbase; /* malloc'ed symbold base */
|
|
|
|
caddr_t strbase; /* malloc'ed string base */
|
|
|
|
caddr_t ctftab; /* CTF table */
|
|
|
|
long ctfcnt; /* number of bytes in CTF table */
|
|
|
|
caddr_t ctfoff; /* CTF offset table */
|
|
|
|
caddr_t typoff; /* Type offset table */
|
|
|
|
long typlen; /* Number of type entries. */
|
|
|
|
Elf_Addr pcpu_start; /* Pre-relocation pcpu set start. */
|
|
|
|
Elf_Addr pcpu_stop; /* Pre-relocation pcpu set stop. */
|
|
|
|
Elf_Addr pcpu_base; /* Relocated pcpu set address. */
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#ifdef VIMAGE
|
2010-11-08 20:57:08 +00:00
|
|
|
Elf_Addr vnet_start; /* Pre-relocation vnet set start. */
|
|
|
|
Elf_Addr vnet_stop; /* Pre-relocation vnet set stop. */
|
|
|
|
Elf_Addr vnet_base; /* Relocated vnet set address. */
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#endif
|
2008-07-11 01:10:40 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
struct link_map gdb; /* hooks for gdb */
|
2008-07-11 01:10:40 +00:00
|
|
|
#endif
|
|
|
|
} *elf_file_t;
|
|
|
|
|
2008-05-23 00:49:39 +00:00
|
|
|
#include <kern/kern_ctf.c>
|
|
|
|
|
2002-10-19 18:59:33 +00:00
|
|
|
static int link_elf_link_common_finish(linker_file_t);
|
2000-04-29 13:19:31 +00:00
|
|
|
static int link_elf_link_preload(linker_class_t cls,
|
2010-11-08 20:57:08 +00:00
|
|
|
const char *, linker_file_t *);
|
2000-04-29 13:19:31 +00:00
|
|
|
static int link_elf_link_preload_finish(linker_file_t);
|
2010-11-08 20:57:08 +00:00
|
|
|
static int link_elf_load_file(linker_class_t, const char *,
|
|
|
|
linker_file_t *);
|
|
|
|
static int link_elf_lookup_symbol(linker_file_t, const char *,
|
|
|
|
c_linker_sym_t *);
|
|
|
|
static int link_elf_symbol_values(linker_file_t, c_linker_sym_t,
|
|
|
|
linker_symval_t *);
|
|
|
|
static int link_elf_search_symbol(linker_file_t, caddr_t,
|
|
|
|
c_linker_sym_t *, long *);
|
2000-04-24 17:08:04 +00:00
|
|
|
|
|
|
|
static void link_elf_unload_file(linker_file_t);
|
2000-04-29 13:19:31 +00:00
|
|
|
static void link_elf_unload_preload(linker_file_t);
|
2001-06-13 10:58:39 +00:00
|
|
|
static int link_elf_lookup_set(linker_file_t, const char *,
|
2010-11-08 20:57:08 +00:00
|
|
|
void ***, void ***, int *);
|
2001-10-30 15:21:45 +00:00
|
|
|
static int link_elf_each_function_name(linker_file_t,
|
2010-11-08 20:57:08 +00:00
|
|
|
int (*)(const char *, void *), void *);
|
2008-05-23 00:49:39 +00:00
|
|
|
static int link_elf_each_function_nameval(linker_file_t,
|
2010-11-08 20:57:08 +00:00
|
|
|
linker_function_nameval_callback_t, void *);
|
2003-01-21 02:42:44 +00:00
|
|
|
static void link_elf_reloc_local(linker_file_t);
|
2009-05-26 21:39:09 +00:00
|
|
|
static long link_elf_symtab_get(linker_file_t, const Elf_Sym **);
|
|
|
|
static long link_elf_strtab_get(linker_file_t, caddr_t *);
|
2010-11-08 20:57:08 +00:00
|
|
|
static Elf_Addr elf_lookup(linker_file_t, Elf_Size, int);
|
2000-04-24 17:08:04 +00:00
|
|
|
|
|
|
|
static kobj_method_t link_elf_methods[] = {
|
2010-11-08 20:57:08 +00:00
|
|
|
KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol),
|
|
|
|
KOBJMETHOD(linker_symbol_values, link_elf_symbol_values),
|
|
|
|
KOBJMETHOD(linker_search_symbol, link_elf_search_symbol),
|
|
|
|
KOBJMETHOD(linker_unload, link_elf_unload_file),
|
|
|
|
KOBJMETHOD(linker_load_file, link_elf_load_file),
|
|
|
|
KOBJMETHOD(linker_link_preload, link_elf_link_preload),
|
|
|
|
KOBJMETHOD(linker_link_preload_finish, link_elf_link_preload_finish),
|
|
|
|
KOBJMETHOD(linker_lookup_set, link_elf_lookup_set),
|
|
|
|
KOBJMETHOD(linker_each_function_name, link_elf_each_function_name),
|
|
|
|
KOBJMETHOD(linker_each_function_nameval, link_elf_each_function_nameval),
|
|
|
|
KOBJMETHOD(linker_ctf_get, link_elf_ctf_get),
|
|
|
|
KOBJMETHOD(linker_symtab_get, link_elf_symtab_get),
|
|
|
|
KOBJMETHOD(linker_strtab_get, link_elf_strtab_get),
|
|
|
|
{ 0, 0 }
|
2000-04-24 17:08:04 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static struct linker_class link_elf_class = {
|
|
|
|
#if ELF_TARG_CLASS == ELFCLASS32
|
2010-11-08 20:57:08 +00:00
|
|
|
"elf32",
|
2000-04-24 17:08:04 +00:00
|
|
|
#else
|
2010-11-08 20:57:08 +00:00
|
|
|
"elf64",
|
2000-04-24 17:08:04 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
link_elf_methods, sizeof(struct elf_file)
|
2000-04-24 17:08:04 +00:00
|
|
|
};
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
static int parse_dynamic(elf_file_t);
|
|
|
|
static int relocate_file(elf_file_t);
|
|
|
|
static int link_elf_preload_parse_symbols(elf_file_t);
|
2000-04-24 17:08:04 +00:00
|
|
|
|
2004-07-10 21:37:30 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
static void r_debug_state(struct r_debug *, struct link_map *);
|
2000-04-24 17:08:04 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* A list of loaded modules for GDB to use for loading symbols.
|
|
|
|
*/
|
|
|
|
struct r_debug r_debug;
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
#define GDB_STATE(s) do { \
|
|
|
|
r_debug.r_state = s; r_debug_state(NULL, NULL); \
|
|
|
|
} while (0)
|
2000-04-24 17:08:04 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Function for the debugger to set a breakpoint on to gain control.
|
|
|
|
*/
|
2002-09-28 17:15:38 +00:00
|
|
|
static void
|
2000-10-06 05:20:02 +00:00
|
|
|
r_debug_state(struct r_debug *dummy_one __unused,
|
|
|
|
struct link_map *dummy_two __unused)
|
2000-04-24 17:08:04 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2002-10-19 18:43:37 +00:00
|
|
|
static void
|
|
|
|
link_elf_add_gdb(struct link_map *l)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
struct link_map *prev;
|
|
|
|
|
|
|
|
l->l_next = NULL;
|
|
|
|
|
|
|
|
if (r_debug.r_map == NULL) {
|
|
|
|
/* Add first. */
|
|
|
|
l->l_prev = NULL;
|
|
|
|
r_debug.r_map = l;
|
|
|
|
} else {
|
|
|
|
/* Append to list. */
|
|
|
|
for (prev = r_debug.r_map;
|
|
|
|
prev->l_next != NULL;
|
|
|
|
prev = prev->l_next)
|
|
|
|
;
|
|
|
|
l->l_prev = prev;
|
|
|
|
prev->l_next = l;
|
|
|
|
}
|
2002-10-19 18:43:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
link_elf_delete_gdb(struct link_map *l)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
if (l->l_prev == NULL) {
|
|
|
|
/* Remove first. */
|
|
|
|
if ((r_debug.r_map = l->l_next) != NULL)
|
|
|
|
l->l_next->l_prev = NULL;
|
|
|
|
} else {
|
|
|
|
/* Remove any but first. */
|
|
|
|
if ((l->l_prev->l_next = l->l_next) != NULL)
|
|
|
|
l->l_next->l_prev = l->l_prev;
|
|
|
|
}
|
2002-10-19 18:43:37 +00:00
|
|
|
}
|
2004-07-10 21:37:30 +00:00
|
|
|
#endif /* GDB */
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2002-04-21 21:08:30 +00:00
|
|
|
#ifdef __ia64__
|
|
|
|
Elf_Addr link_elf_get_gp(linker_file_t);
|
|
|
|
#endif
|
|
|
|
|
2008-07-11 01:10:40 +00:00
|
|
|
/*
|
|
|
|
* The kernel symbol table starts here.
|
|
|
|
*/
|
|
|
|
extern struct _dynamic _DYNAMIC;
|
|
|
|
|
2002-10-19 18:43:37 +00:00
|
|
|
static void
|
2008-07-08 23:51:38 +00:00
|
|
|
link_elf_error(const char *filename, const char *s)
|
2002-10-19 18:43:37 +00:00
|
|
|
{
|
2008-07-08 23:51:38 +00:00
|
|
|
if (filename == NULL)
|
|
|
|
printf("kldload: %s\n", s);
|
|
|
|
else
|
|
|
|
printf("kldload: %s: %s\n", filename, s);
|
2002-10-19 18:43:37 +00:00
|
|
|
}
|
|
|
|
|
2002-10-19 18:59:33 +00:00
|
|
|
/*
|
|
|
|
* Actions performed after linking/loading both the preloaded kernel and any
|
|
|
|
* modules; whether preloaded or dynamicly loaded.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
link_elf_link_common_finish(linker_file_t lf)
|
|
|
|
{
|
2004-07-10 21:37:30 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
|
|
|
char *newfilename;
|
2002-10-19 18:59:33 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
int error;
|
2002-10-19 19:16:03 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Notify MD code that a module is being loaded. */
|
|
|
|
error = elf_cpu_load_file(lf);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
2002-10-19 18:59:33 +00:00
|
|
|
|
2004-07-10 21:37:30 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
GDB_STATE(RT_ADD);
|
|
|
|
ef->gdb.l_addr = lf->address;
|
|
|
|
newfilename = malloc(strlen(lf->filename) + 1, M_LINKER, M_WAITOK);
|
|
|
|
strcpy(newfilename, lf->filename);
|
|
|
|
ef->gdb.l_name = newfilename;
|
|
|
|
ef->gdb.l_ld = ef->dynamic;
|
|
|
|
link_elf_add_gdb(&ef->gdb);
|
|
|
|
GDB_STATE(RT_CONSISTENT);
|
2002-10-19 18:59:33 +00:00
|
|
|
#endif
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (0);
|
2002-10-19 18:59:33 +00:00
|
|
|
}
|
|
|
|
|
1998-08-24 08:25:26 +00:00
|
|
|
static void
|
|
|
|
link_elf_init(void* arg)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
Elf_Dyn *dp;
|
|
|
|
caddr_t modptr, baseptr, sizeptr;
|
|
|
|
elf_file_t ef;
|
|
|
|
char *modname;
|
|
|
|
|
|
|
|
linker_add_class(&link_elf_class);
|
|
|
|
|
|
|
|
dp = (Elf_Dyn *)&_DYNAMIC;
|
|
|
|
modname = NULL;
|
|
|
|
modptr = preload_search_by_type("elf" __XSTRING(__ELF_WORD_SIZE) " kernel");
|
|
|
|
if (modptr == NULL)
|
|
|
|
modptr = preload_search_by_type("elf kernel");
|
|
|
|
if (modptr != NULL)
|
|
|
|
modname = (char *)preload_search_info(modptr, MODINFO_NAME);
|
|
|
|
if (modname == NULL)
|
|
|
|
modname = "kernel";
|
|
|
|
linker_kernel_file = linker_make_file(modname, &link_elf_class);
|
|
|
|
if (linker_kernel_file == NULL)
|
|
|
|
panic("%s: Can't create linker structures for kernel",
|
|
|
|
__func__);
|
|
|
|
|
|
|
|
ef = (elf_file_t) linker_kernel_file;
|
|
|
|
ef->preloaded = 1;
|
|
|
|
ef->address = 0;
|
2000-04-24 17:08:04 +00:00
|
|
|
#ifdef SPARSE_MAPPING
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->object = 0;
|
2000-04-24 17:08:04 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->dynamic = dp;
|
|
|
|
|
|
|
|
if (dp != NULL)
|
|
|
|
parse_dynamic(ef);
|
|
|
|
linker_kernel_file->address = (caddr_t) KERNBASE;
|
|
|
|
linker_kernel_file->size = -(intptr_t)linker_kernel_file->address;
|
|
|
|
|
|
|
|
if (modptr != NULL) {
|
|
|
|
ef->modptr = modptr;
|
|
|
|
baseptr = preload_search_info(modptr, MODINFO_ADDR);
|
|
|
|
if (baseptr != NULL)
|
|
|
|
linker_kernel_file->address = *(caddr_t *)baseptr;
|
|
|
|
sizeptr = preload_search_info(modptr, MODINFO_SIZE);
|
|
|
|
if (sizeptr != NULL)
|
|
|
|
linker_kernel_file->size = *(size_t *)sizeptr;
|
|
|
|
}
|
|
|
|
(void)link_elf_preload_parse_symbols(ef);
|
2000-04-24 17:08:04 +00:00
|
|
|
|
2004-07-10 21:37:30 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
r_debug.r_map = NULL;
|
|
|
|
r_debug.r_brk = r_debug_state;
|
|
|
|
r_debug.r_state = RT_CONSISTENT;
|
2000-04-24 17:08:04 +00:00
|
|
|
#endif
|
2002-10-19 18:59:33 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
(void)link_elf_link_common_finish(linker_kernel_file);
|
|
|
|
linker_kernel_file->flags |= LINKER_FILE_LINKED;
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
2006-07-14 22:39:18 +00:00
|
|
|
SYSINIT(link_elf, SI_SUB_KLD, SI_ORDER_THIRD, link_elf_init, 0);
|
1998-08-24 08:25:26 +00:00
|
|
|
|
1998-10-12 09:13:50 +00:00
|
|
|
static int
|
2000-04-29 13:19:31 +00:00
|
|
|
link_elf_preload_parse_symbols(elf_file_t ef)
|
1998-10-12 09:13:50 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
caddr_t pointer;
|
|
|
|
caddr_t ssym, esym, base;
|
|
|
|
caddr_t strtab;
|
|
|
|
int strcnt;
|
|
|
|
Elf_Sym *symtab;
|
|
|
|
int symcnt;
|
|
|
|
|
|
|
|
if (ef->modptr == NULL)
|
|
|
|
return (0);
|
|
|
|
pointer = preload_search_info(ef->modptr,
|
|
|
|
MODINFO_METADATA | MODINFOMD_SSYM);
|
|
|
|
if (pointer == NULL)
|
|
|
|
return (0);
|
|
|
|
ssym = *(caddr_t *)pointer;
|
|
|
|
pointer = preload_search_info(ef->modptr,
|
|
|
|
MODINFO_METADATA | MODINFOMD_ESYM);
|
|
|
|
if (pointer == NULL)
|
|
|
|
return (0);
|
|
|
|
esym = *(caddr_t *)pointer;
|
1998-10-12 09:13:50 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
base = ssym;
|
1998-10-12 09:13:50 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
symcnt = *(long *)base;
|
|
|
|
base += sizeof(long);
|
|
|
|
symtab = (Elf_Sym *)base;
|
|
|
|
base += roundup(symcnt, sizeof(long));
|
|
|
|
|
|
|
|
if (base > esym || base < ssym) {
|
|
|
|
printf("Symbols are corrupt!\n");
|
|
|
|
return (EINVAL);
|
|
|
|
}
|
1998-10-12 09:13:50 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
strcnt = *(long *)base;
|
|
|
|
base += sizeof(long);
|
|
|
|
strtab = base;
|
|
|
|
base += roundup(strcnt, sizeof(long));
|
1998-10-12 09:13:50 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (base > esym || base < ssym) {
|
|
|
|
printf("Symbols are corrupt!\n");
|
|
|
|
return (EINVAL);
|
|
|
|
}
|
|
|
|
|
|
|
|
ef->ddbsymtab = symtab;
|
|
|
|
ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
|
|
|
|
ef->ddbstrtab = strtab;
|
|
|
|
ef->ddbstrcnt = strcnt;
|
|
|
|
|
|
|
|
return (0);
|
1998-10-12 09:13:50 +00:00
|
|
|
}
|
|
|
|
|
1998-08-24 08:25:26 +00:00
|
|
|
static int
|
2000-04-24 17:08:04 +00:00
|
|
|
parse_dynamic(elf_file_t ef)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
Elf_Dyn *dp;
|
|
|
|
int plttype = DT_REL;
|
|
|
|
|
|
|
|
for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
|
|
|
|
switch (dp->d_tag) {
|
|
|
|
case DT_HASH:
|
|
|
|
{
|
|
|
|
/* From src/libexec/rtld-elf/rtld.c */
|
|
|
|
const Elf_Hashelt *hashtab = (const Elf_Hashelt *)
|
|
|
|
(ef->address + dp->d_un.d_ptr);
|
|
|
|
ef->nbuckets = hashtab[0];
|
|
|
|
ef->nchains = hashtab[1];
|
|
|
|
ef->buckets = hashtab + 2;
|
|
|
|
ef->chains = ef->buckets + ef->nbuckets;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case DT_STRTAB:
|
|
|
|
ef->strtab = (caddr_t) (ef->address + dp->d_un.d_ptr);
|
|
|
|
break;
|
|
|
|
case DT_STRSZ:
|
|
|
|
ef->strsz = dp->d_un.d_val;
|
|
|
|
break;
|
|
|
|
case DT_SYMTAB:
|
|
|
|
ef->symtab = (Elf_Sym*) (ef->address + dp->d_un.d_ptr);
|
|
|
|
break;
|
|
|
|
case DT_SYMENT:
|
|
|
|
if (dp->d_un.d_val != sizeof(Elf_Sym))
|
|
|
|
return (ENOEXEC);
|
|
|
|
break;
|
|
|
|
case DT_PLTGOT:
|
|
|
|
ef->got = (Elf_Addr *) (ef->address + dp->d_un.d_ptr);
|
|
|
|
break;
|
|
|
|
case DT_REL:
|
|
|
|
ef->rel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);
|
|
|
|
break;
|
|
|
|
case DT_RELSZ:
|
|
|
|
ef->relsize = dp->d_un.d_val;
|
|
|
|
break;
|
|
|
|
case DT_RELENT:
|
|
|
|
if (dp->d_un.d_val != sizeof(Elf_Rel))
|
|
|
|
return (ENOEXEC);
|
|
|
|
break;
|
|
|
|
case DT_JMPREL:
|
|
|
|
ef->pltrel = (const Elf_Rel *) (ef->address + dp->d_un.d_ptr);
|
|
|
|
break;
|
|
|
|
case DT_PLTRELSZ:
|
|
|
|
ef->pltrelsize = dp->d_un.d_val;
|
|
|
|
break;
|
|
|
|
case DT_RELA:
|
|
|
|
ef->rela = (const Elf_Rela *) (ef->address + dp->d_un.d_ptr);
|
|
|
|
break;
|
|
|
|
case DT_RELASZ:
|
|
|
|
ef->relasize = dp->d_un.d_val;
|
|
|
|
break;
|
|
|
|
case DT_RELAENT:
|
|
|
|
if (dp->d_un.d_val != sizeof(Elf_Rela))
|
|
|
|
return (ENOEXEC);
|
|
|
|
break;
|
|
|
|
case DT_PLTREL:
|
|
|
|
plttype = dp->d_un.d_val;
|
|
|
|
if (plttype != DT_REL && plttype != DT_RELA)
|
|
|
|
return (ENOEXEC);
|
|
|
|
break;
|
2004-07-10 21:37:30 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
case DT_DEBUG:
|
|
|
|
dp->d_un.d_ptr = (Elf_Addr)&r_debug;
|
|
|
|
break;
|
2000-04-24 17:08:04 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
}
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (plttype == DT_RELA) {
|
|
|
|
ef->pltrela = (const Elf_Rela *)ef->pltrel;
|
|
|
|
ef->pltrel = NULL;
|
|
|
|
ef->pltrelasize = ef->pltrelsize;
|
|
|
|
ef->pltrelsize = 0;
|
|
|
|
}
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->ddbsymtab = ef->symtab;
|
|
|
|
ef->ddbsymcnt = ef->nchains;
|
|
|
|
ef->ddbstrtab = ef->strtab;
|
|
|
|
ef->ddbstrcnt = ef->strsz;
|
1998-10-12 09:13:50 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (0);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
static int
|
|
|
|
parse_dpcpu(elf_file_t ef)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
int count;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
ef->pcpu_start = 0;
|
|
|
|
ef->pcpu_stop = 0;
|
|
|
|
error = link_elf_lookup_set(&ef->lf, "pcpu", (void ***)&ef->pcpu_start,
|
|
|
|
(void ***)&ef->pcpu_stop, &count);
|
|
|
|
/* Error just means there is no pcpu set to relocate. */
|
|
|
|
if (error != 0)
|
|
|
|
return (0);
|
|
|
|
count *= sizeof(void *);
|
|
|
|
/*
|
|
|
|
* Allocate space in the primary pcpu area. Copy in our
|
|
|
|
* initialization from the data section and then initialize
|
|
|
|
* all per-cpu storage from that.
|
|
|
|
*/
|
|
|
|
ef->pcpu_base = (Elf_Addr)(uintptr_t)dpcpu_alloc(count);
|
|
|
|
if (ef->pcpu_base == 0)
|
|
|
|
return (ENOSPC);
|
|
|
|
memcpy((void *)ef->pcpu_base, (void *)ef->pcpu_start, count);
|
|
|
|
dpcpu_copy((void *)ef->pcpu_base, count);
|
|
|
|
|
|
|
|
return (0);
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
}
|
|
|
|
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#ifdef VIMAGE
|
|
|
|
static int
|
|
|
|
parse_vnet(elf_file_t ef)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
int count;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
ef->vnet_start = 0;
|
|
|
|
ef->vnet_stop = 0;
|
|
|
|
error = link_elf_lookup_set(&ef->lf, "vnet", (void ***)&ef->vnet_start,
|
|
|
|
(void ***)&ef->vnet_stop, &count);
|
|
|
|
/* Error just means there is no vnet data set to relocate. */
|
|
|
|
if (error != 0)
|
|
|
|
return (0);
|
|
|
|
count *= sizeof(void *);
|
|
|
|
/*
|
|
|
|
* Allocate space in the primary vnet area. Copy in our
|
|
|
|
* initialization from the data section and then initialize
|
|
|
|
* all per-vnet storage from that.
|
|
|
|
*/
|
|
|
|
ef->vnet_base = (Elf_Addr)(uintptr_t)vnet_data_alloc(count);
|
|
|
|
if (ef->vnet_base == 0)
|
|
|
|
return (ENOSPC);
|
|
|
|
memcpy((void *)ef->vnet_base, (void *)ef->vnet_start, count);
|
|
|
|
vnet_data_copy((void *)ef->vnet_base, count);
|
|
|
|
|
|
|
|
return (0);
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
1998-10-09 23:55:31 +00:00
|
|
|
static int
|
2000-04-29 13:19:31 +00:00
|
|
|
link_elf_link_preload(linker_class_t cls,
|
2010-11-08 20:57:08 +00:00
|
|
|
const char* filename, linker_file_t *result)
|
1998-10-09 23:55:31 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
caddr_t modptr, baseptr, sizeptr, dynptr;
|
|
|
|
char *type;
|
|
|
|
elf_file_t ef;
|
|
|
|
linker_file_t lf;
|
|
|
|
int error;
|
|
|
|
vm_offset_t dp;
|
|
|
|
|
|
|
|
/* Look to see if we have the file preloaded */
|
|
|
|
modptr = preload_search_by_name(filename);
|
|
|
|
if (modptr == NULL)
|
|
|
|
return (ENOENT);
|
|
|
|
|
|
|
|
type = (char *)preload_search_info(modptr, MODINFO_TYPE);
|
|
|
|
baseptr = preload_search_info(modptr, MODINFO_ADDR);
|
|
|
|
sizeptr = preload_search_info(modptr, MODINFO_SIZE);
|
|
|
|
dynptr = preload_search_info(modptr,
|
|
|
|
MODINFO_METADATA | MODINFOMD_DYNAMIC);
|
|
|
|
if (type == NULL ||
|
|
|
|
(strcmp(type, "elf" __XSTRING(__ELF_WORD_SIZE) " module") != 0 &&
|
|
|
|
strcmp(type, "elf module") != 0))
|
|
|
|
return (EFTYPE);
|
|
|
|
if (baseptr == NULL || sizeptr == NULL || dynptr == NULL)
|
|
|
|
return (EINVAL);
|
|
|
|
|
|
|
|
lf = linker_make_file(filename, &link_elf_class);
|
|
|
|
if (lf == NULL)
|
|
|
|
return (ENOMEM);
|
|
|
|
|
|
|
|
ef = (elf_file_t) lf;
|
|
|
|
ef->preloaded = 1;
|
|
|
|
ef->modptr = modptr;
|
|
|
|
ef->address = *(caddr_t *)baseptr;
|
1998-10-09 23:55:31 +00:00
|
|
|
#ifdef SPARSE_MAPPING
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->object = 0;
|
1998-10-09 23:55:31 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
dp = (vm_offset_t)ef->address + *(vm_offset_t *)dynptr;
|
|
|
|
ef->dynamic = (Elf_Dyn *)dp;
|
|
|
|
lf->address = ef->address;
|
|
|
|
lf->size = *(size_t *)sizeptr;
|
|
|
|
|
|
|
|
error = parse_dynamic(ef);
|
|
|
|
if (error == 0)
|
|
|
|
error = parse_dpcpu(ef);
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#ifdef VIMAGE
|
2010-11-08 20:57:08 +00:00
|
|
|
if (error == 0)
|
|
|
|
error = parse_vnet(ef);
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
if (error != 0) {
|
|
|
|
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
link_elf_reloc_local(lf);
|
|
|
|
*result = lf;
|
|
|
|
return (0);
|
2000-04-29 13:19:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
link_elf_link_preload_finish(linker_file_t lf)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef;
|
|
|
|
int error;
|
2000-04-29 13:19:31 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
ef = (elf_file_t) lf;
|
2000-04-29 13:19:31 +00:00
|
|
|
#if 0 /* this will be more trouble than it's worth for now */
|
2010-11-08 20:57:08 +00:00
|
|
|
for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
|
|
|
|
if (dp->d_tag != DT_NEEDED)
|
|
|
|
continue;
|
|
|
|
modname = ef->strtab + dp->d_un.d_val;
|
|
|
|
error = linker_load_module(modname, lf);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
1998-10-09 23:55:31 +00:00
|
|
|
}
|
2000-04-29 13:19:31 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
error = relocate_file(ef);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
(void)link_elf_preload_parse_symbols(ef);
|
2000-04-24 17:08:04 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (link_elf_link_common_finish(lf));
|
1998-10-09 23:55:31 +00:00
|
|
|
}
|
|
|
|
|
1998-08-24 08:25:26 +00:00
|
|
|
static int
|
2002-10-19 18:59:33 +00:00
|
|
|
link_elf_load_file(linker_class_t cls, const char* filename,
|
2010-11-08 20:57:08 +00:00
|
|
|
linker_file_t* result)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
struct nameidata nd;
|
|
|
|
struct thread* td = curthread; /* XXX */
|
|
|
|
Elf_Ehdr *hdr;
|
|
|
|
caddr_t firstpage;
|
|
|
|
int nbytes, i;
|
|
|
|
Elf_Phdr *phdr;
|
|
|
|
Elf_Phdr *phlimit;
|
|
|
|
Elf_Phdr *segs[MAXSEGS];
|
|
|
|
int nsegs;
|
|
|
|
Elf_Phdr *phdyn;
|
|
|
|
Elf_Phdr *phphdr;
|
|
|
|
caddr_t mapbase;
|
|
|
|
size_t mapsize;
|
|
|
|
Elf_Off base_offset;
|
|
|
|
Elf_Addr base_vaddr;
|
|
|
|
Elf_Addr base_vlimit;
|
|
|
|
int error = 0;
|
|
|
|
int resid, flags;
|
|
|
|
elf_file_t ef;
|
|
|
|
linker_file_t lf;
|
|
|
|
Elf_Shdr *shdr;
|
|
|
|
int symtabindex;
|
|
|
|
int symstrindex;
|
|
|
|
int symcnt;
|
|
|
|
int strcnt;
|
|
|
|
int vfslocked;
|
|
|
|
|
|
|
|
shdr = NULL;
|
|
|
|
lf = NULL;
|
|
|
|
|
|
|
|
NDINIT(&nd, LOOKUP, FOLLOW | MPSAFE, UIO_SYSSPACE, filename, td);
|
|
|
|
flags = FREAD;
|
|
|
|
error = vn_open(&nd, &flags, 0, NULL);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
vfslocked = NDHASGIANT(&nd);
|
|
|
|
NDFREE(&nd, NDF_ONLY_PNBUF);
|
|
|
|
if (nd.ni_vp->v_type != VREG) {
|
|
|
|
error = ENOEXEC;
|
|
|
|
firstpage = NULL;
|
|
|
|
goto out;
|
|
|
|
}
|
2002-11-19 22:12:42 +00:00
|
|
|
#ifdef MAC
|
2010-11-08 20:57:08 +00:00
|
|
|
error = mac_kld_check_load(curthread->td_ucred, nd.ni_vp);
|
|
|
|
if (error != 0) {
|
|
|
|
firstpage = NULL;
|
|
|
|
goto out;
|
|
|
|
}
|
2002-11-19 22:12:42 +00:00
|
|
|
#endif
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/*
|
|
|
|
* Read the elf header from the file.
|
|
|
|
*/
|
|
|
|
firstpage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK);
|
|
|
|
hdr = (Elf_Ehdr *)firstpage;
|
|
|
|
error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0,
|
|
|
|
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
|
|
|
|
&resid, td);
|
|
|
|
nbytes = PAGE_SIZE - resid;
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (!IS_ELF(*hdr)) {
|
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (hdr->e_ident[EI_CLASS] != ELF_TARG_CLASS ||
|
|
|
|
hdr->e_ident[EI_DATA] != ELF_TARG_DATA) {
|
|
|
|
link_elf_error(filename, "Unsupported file layout");
|
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (hdr->e_ident[EI_VERSION] != EV_CURRENT ||
|
|
|
|
hdr->e_version != EV_CURRENT) {
|
|
|
|
link_elf_error(filename, "Unsupported file version");
|
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) {
|
|
|
|
error = ENOSYS;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (hdr->e_machine != ELF_TARG_MACH) {
|
|
|
|
link_elf_error(filename, "Unsupported machine");
|
1998-09-11 08:46:15 +00:00
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/*
|
|
|
|
* We rely on the program header being in the first page.
|
|
|
|
* This is not strictly required by the ABI specification, but
|
|
|
|
* it seems to always true in practice. And, it simplifies
|
|
|
|
* things considerably.
|
|
|
|
*/
|
|
|
|
if (!((hdr->e_phentsize == sizeof(Elf_Phdr)) &&
|
|
|
|
(hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= PAGE_SIZE) &&
|
|
|
|
(hdr->e_phoff + hdr->e_phnum*sizeof(Elf_Phdr) <= nbytes)))
|
|
|
|
link_elf_error(filename, "Unreadable program headers");
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/*
|
|
|
|
* Scan the program header entries, and save key information.
|
|
|
|
*
|
|
|
|
* We rely on there being exactly two load segments, text and data,
|
|
|
|
* in that order.
|
|
|
|
*/
|
|
|
|
phdr = (Elf_Phdr *) (firstpage + hdr->e_phoff);
|
|
|
|
phlimit = phdr + hdr->e_phnum;
|
|
|
|
nsegs = 0;
|
|
|
|
phdyn = NULL;
|
|
|
|
phphdr = NULL;
|
|
|
|
while (phdr < phlimit) {
|
|
|
|
switch (phdr->p_type) {
|
|
|
|
case PT_LOAD:
|
|
|
|
if (nsegs == MAXSEGS) {
|
|
|
|
link_elf_error(filename, "Too many sections");
|
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
/*
|
|
|
|
* XXX: We just trust they come in right order ??
|
|
|
|
*/
|
|
|
|
segs[nsegs] = phdr;
|
|
|
|
++nsegs;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PT_PHDR:
|
|
|
|
phphdr = phdr;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PT_DYNAMIC:
|
|
|
|
phdyn = phdr;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PT_INTERP:
|
|
|
|
error = ENOSYS;
|
|
|
|
goto out;
|
|
|
|
}
|
2000-04-24 17:08:04 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
++phdr;
|
|
|
|
}
|
|
|
|
if (phdyn == NULL) {
|
|
|
|
link_elf_error(filename, "Object is not dynamically-linked");
|
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
if (nsegs == 0) {
|
|
|
|
link_elf_error(filename, "No sections");
|
|
|
|
error = ENOEXEC;
|
|
|
|
goto out;
|
|
|
|
}
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/*
|
|
|
|
* Allocate the entire address space of the object, to stake
|
|
|
|
* out our contiguous region, and to establish the base
|
|
|
|
* address for relocation.
|
|
|
|
*/
|
|
|
|
base_offset = trunc_page(segs[0]->p_offset);
|
|
|
|
base_vaddr = trunc_page(segs[0]->p_vaddr);
|
|
|
|
base_vlimit = round_page(segs[nsegs - 1]->p_vaddr +
|
|
|
|
segs[nsegs - 1]->p_memsz);
|
|
|
|
mapsize = base_vlimit - base_vaddr;
|
|
|
|
|
|
|
|
lf = linker_make_file(filename, &link_elf_class);
|
|
|
|
if (lf == NULL) {
|
|
|
|
error = ENOMEM;
|
|
|
|
goto out;
|
1998-09-11 08:46:15 +00:00
|
|
|
}
|
2005-08-28 00:19:54 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
ef = (elf_file_t) lf;
|
2005-08-28 00:19:54 +00:00
|
|
|
#ifdef SPARSE_MAPPING
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->object = vm_object_allocate(OBJT_DEFAULT, mapsize >> PAGE_SHIFT);
|
|
|
|
if (ef->object == NULL) {
|
|
|
|
error = ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
ef->address = (caddr_t) vm_map_min(kernel_map);
|
|
|
|
error = vm_map_find(kernel_map, ef->object, 0,
|
|
|
|
(vm_offset_t *) &ef->address, mapsize, 1,
|
|
|
|
VM_PROT_ALL, VM_PROT_ALL, 0);
|
|
|
|
if (error != 0) {
|
|
|
|
vm_object_deallocate(ef->object);
|
|
|
|
ef->object = 0;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
ef->address = malloc(mapsize, M_LINKER, M_WAITOK);
|
|
|
|
#endif
|
|
|
|
mapbase = ef->address;
|
|
|
|
|
2005-08-28 00:19:54 +00:00
|
|
|
/*
|
2010-11-08 20:57:08 +00:00
|
|
|
* Read the text and data sections and zero the bss.
|
2005-08-28 00:19:54 +00:00
|
|
|
*/
|
2010-11-08 20:57:08 +00:00
|
|
|
for (i = 0; i < nsegs; i++) {
|
|
|
|
caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
|
|
|
|
error = vn_rdwr(UIO_READ, nd.ni_vp,
|
|
|
|
segbase, segs[i]->p_filesz, segs[i]->p_offset,
|
|
|
|
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
|
|
|
|
&resid, td);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
|
|
|
bzero(segbase + segs[i]->p_filesz,
|
|
|
|
segs[i]->p_memsz - segs[i]->p_filesz);
|
|
|
|
|
|
|
|
#ifdef SPARSE_MAPPING
|
|
|
|
/*
|
|
|
|
* Wire down the pages
|
|
|
|
*/
|
|
|
|
error = vm_map_wire(kernel_map,
|
2005-08-28 00:19:54 +00:00
|
|
|
(vm_offset_t) segbase,
|
|
|
|
(vm_offset_t) segbase + segs[i]->p_memsz,
|
|
|
|
VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES);
|
2010-11-08 20:57:08 +00:00
|
|
|
if (error != KERN_SUCCESS) {
|
|
|
|
error = ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
2005-08-28 00:19:54 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
}
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2001-10-30 15:08:51 +00:00
|
|
|
#ifdef GPROF
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Update profiling information with the new text segment. */
|
|
|
|
mtx_lock(&Giant);
|
|
|
|
kmupetext((uintfptr_t)(mapbase + segs[0]->p_vaddr - base_vaddr +
|
|
|
|
segs[0]->p_memsz));
|
|
|
|
mtx_unlock(&Giant);
|
2001-10-30 15:08:51 +00:00
|
|
|
#endif
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->dynamic = (Elf_Dyn *) (mapbase + phdyn->p_vaddr - base_vaddr);
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
lf->address = ef->address;
|
|
|
|
lf->size = mapsize;
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
error = parse_dynamic(ef);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
|
|
|
error = parse_dpcpu(ef);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#ifdef VIMAGE
|
2010-11-08 20:57:08 +00:00
|
|
|
error = parse_vnet(ef);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
link_elf_reloc_local(lf);
|
2003-01-21 02:42:44 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
VOP_UNLOCK(nd.ni_vp, 0);
|
|
|
|
error = linker_load_dependencies(lf);
|
|
|
|
vn_lock(nd.ni_vp, LK_EXCLUSIVE | LK_RETRY);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
2000-04-29 13:19:31 +00:00
|
|
|
#if 0 /* this will be more trouble than it's worth for now */
|
2010-11-08 20:57:08 +00:00
|
|
|
for (dp = ef->dynamic; dp->d_tag != DT_NULL; dp++) {
|
|
|
|
if (dp->d_tag != DT_NEEDED)
|
|
|
|
continue;
|
|
|
|
modname = ef->strtab + dp->d_un.d_val;
|
|
|
|
error = linker_load_module(modname, lf);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
2000-04-29 13:19:31 +00:00
|
|
|
}
|
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
error = relocate_file(ef);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Try and load the symbol table if it's present. (you can
|
|
|
|
* strip it!)
|
|
|
|
*/
|
|
|
|
nbytes = hdr->e_shnum * hdr->e_shentsize;
|
|
|
|
if (nbytes == 0 || hdr->e_shoff == 0)
|
|
|
|
goto nosyms;
|
|
|
|
shdr = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO);
|
|
|
|
error = vn_rdwr(UIO_READ, nd.ni_vp,
|
|
|
|
(caddr_t)shdr, nbytes, hdr->e_shoff,
|
|
|
|
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
|
|
|
|
&resid, td);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
|
|
|
symtabindex = -1;
|
|
|
|
symstrindex = -1;
|
|
|
|
for (i = 0; i < hdr->e_shnum; i++) {
|
|
|
|
if (shdr[i].sh_type == SHT_SYMTAB) {
|
|
|
|
symtabindex = i;
|
|
|
|
symstrindex = shdr[i].sh_link;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (symtabindex < 0 || symstrindex < 0)
|
|
|
|
goto nosyms;
|
1998-10-13 09:27:00 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
symcnt = shdr[symtabindex].sh_size;
|
|
|
|
ef->symbase = malloc(symcnt, M_LINKER, M_WAITOK);
|
|
|
|
strcnt = shdr[symstrindex].sh_size;
|
|
|
|
ef->strbase = malloc(strcnt, M_LINKER, M_WAITOK);
|
1998-10-13 09:27:00 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
error = vn_rdwr(UIO_READ, nd.ni_vp,
|
|
|
|
ef->symbase, symcnt, shdr[symtabindex].sh_offset,
|
|
|
|
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
|
|
|
|
&resid, td);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
|
|
|
error = vn_rdwr(UIO_READ, nd.ni_vp,
|
|
|
|
ef->strbase, strcnt, shdr[symstrindex].sh_offset,
|
|
|
|
UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
|
|
|
|
&resid, td);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
1998-10-13 09:27:00 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
ef->ddbsymcnt = symcnt / sizeof(Elf_Sym);
|
|
|
|
ef->ddbsymtab = (const Elf_Sym *)ef->symbase;
|
|
|
|
ef->ddbstrcnt = strcnt;
|
|
|
|
ef->ddbstrtab = ef->strbase;
|
1998-10-13 09:27:00 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
error = link_elf_link_common_finish(lf);
|
|
|
|
if (error != 0)
|
|
|
|
goto out;
|
2000-04-24 17:08:04 +00:00
|
|
|
|
1998-10-13 09:27:00 +00:00
|
|
|
nosyms:
|
2010-11-08 20:57:08 +00:00
|
|
|
*result = lf;
|
1998-08-24 08:25:26 +00:00
|
|
|
|
|
|
|
out:
|
2010-11-08 20:57:08 +00:00
|
|
|
VOP_UNLOCK(nd.ni_vp, 0);
|
|
|
|
vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
|
|
|
|
VFS_UNLOCK_GIANT(vfslocked);
|
|
|
|
if (error != 0 && lf != NULL)
|
|
|
|
linker_file_unload(lf, LINKER_UNLOAD_FORCE);
|
|
|
|
if (shdr != NULL)
|
|
|
|
free(shdr, M_LINKER);
|
|
|
|
if (firstpage != NULL)
|
|
|
|
free(firstpage, M_LINKER);
|
|
|
|
|
|
|
|
return (error);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
Elf_Addr
|
|
|
|
elf_relocaddr(linker_file_t lf, Elf_Addr x)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef;
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
ef = (elf_file_t)lf;
|
|
|
|
if (x >= ef->pcpu_start && x < ef->pcpu_stop)
|
|
|
|
return ((x - ef->pcpu_start) + ef->pcpu_base);
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#ifdef VIMAGE
|
2010-11-08 20:57:08 +00:00
|
|
|
if (x >= ef->vnet_start && x < ef->vnet_stop)
|
|
|
|
return ((x - ef->vnet_start) + ef->vnet_base);
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
return (x);
|
Implement a facility for dynamic per-cpu variables.
- Modules and kernel code alike may use DPCPU_DEFINE(),
DPCPU_GET(), DPCPU_SET(), etc. akin to the statically defined
PCPU_*. Requires only one extra instruction more than PCPU_* and is
virtually the same as __thread for builtin and much faster for shared
objects. DPCPU variables can be initialized when defined.
- Modules are supported by relocating the module's per-cpu linker set
over space reserved in the kernel. Modules may fail to load if there
is insufficient space available.
- Track space available for modules with a one-off extent allocator.
Free may block for memory to allocate space for an extent.
Reviewed by: jhb, rwatson, kan, sam, grehan, marius, marcel, stas
2009-06-23 22:42:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
1998-08-24 08:25:26 +00:00
|
|
|
static void
|
1998-10-09 23:55:31 +00:00
|
|
|
link_elf_unload_file(linker_file_t file)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef = (elf_file_t) file;
|
2000-04-24 17:08:04 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->pcpu_base != 0) {
|
|
|
|
dpcpu_free((void *)ef->pcpu_base,
|
|
|
|
ef->pcpu_stop - ef->pcpu_start);
|
|
|
|
}
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#ifdef VIMAGE
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->vnet_base != 0) {
|
|
|
|
vnet_data_free((void *)ef->vnet_base,
|
|
|
|
ef->vnet_stop - ef->vnet_start);
|
|
|
|
}
|
Build on Jeff Roberson's linker-set based dynamic per-CPU allocator
(DPCPU), as suggested by Peter Wemm, and implement a new per-virtual
network stack memory allocator. Modify vnet to use the allocator
instead of monolithic global container structures (vinet, ...). This
change solves many binary compatibility problems associated with
VIMAGE, and restores ELF symbols for virtualized global variables.
Each virtualized global variable exists as a "reference copy", and also
once per virtual network stack. Virtualized global variables are
tagged at compile-time, placing the in a special linker set, which is
loaded into a contiguous region of kernel memory. Virtualized global
variables in the base kernel are linked as normal, but those in modules
are copied and relocated to a reserved portion of the kernel's vnet
region with the help of a the kernel linker.
Virtualized global variables exist in per-vnet memory set up when the
network stack instance is created, and are initialized statically from
the reference copy. Run-time access occurs via an accessor macro, which
converts from the current vnet and requested symbol to a per-vnet
address. When "options VIMAGE" is not compiled into the kernel, normal
global ELF symbols will be used instead and indirection is avoided.
This change restores static initialization for network stack global
variables, restores support for non-global symbols and types, eliminates
the need for many subsystem constructors, eliminates large per-subsystem
structures that caused many binary compatibility issues both for
monitoring applications (netstat) and kernel modules, removes the
per-function INIT_VNET_*() macros throughout the stack, eliminates the
need for vnet_symmap ksym(2) munging, and eliminates duplicate
definitions of virtualized globals under VIMAGE_GLOBALS.
Bump __FreeBSD_version and update UPDATING.
Portions submitted by: bz
Reviewed by: bz, zec
Discussed with: gnn, jamie, jeff, jhb, julian, sam
Suggested by: peter
Approved by: re (kensmith)
2009-07-14 22:48:30 +00:00
|
|
|
#endif
|
2004-07-10 21:37:30 +00:00
|
|
|
#ifdef GDB
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->gdb.l_ld != NULL) {
|
|
|
|
GDB_STATE(RT_DELETE);
|
|
|
|
free((void *)(uintptr_t)ef->gdb.l_name, M_LINKER);
|
|
|
|
link_elf_delete_gdb(&ef->gdb);
|
|
|
|
GDB_STATE(RT_CONSISTENT);
|
|
|
|
}
|
2000-04-24 17:08:04 +00:00
|
|
|
#endif
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Notify MD code that a module is being unloaded. */
|
|
|
|
elf_cpu_unload_file(file);
|
2002-10-19 19:16:03 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->preloaded) {
|
|
|
|
link_elf_unload_preload(file);
|
|
|
|
return;
|
|
|
|
}
|
2002-10-19 18:59:33 +00:00
|
|
|
|
1998-09-11 08:46:15 +00:00
|
|
|
#ifdef SPARSE_MAPPING
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->object != NULL) {
|
|
|
|
vm_map_remove(kernel_map, (vm_offset_t) ef->address,
|
|
|
|
(vm_offset_t) ef->address
|
|
|
|
+ (ef->object->size << PAGE_SHIFT));
|
|
|
|
}
|
1998-09-11 08:46:15 +00:00
|
|
|
#else
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->address != NULL)
|
|
|
|
free(ef->address, M_LINKER);
|
1998-09-11 08:46:15 +00:00
|
|
|
#endif
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ef->symbase != NULL)
|
|
|
|
free(ef->symbase, M_LINKER);
|
|
|
|
if (ef->strbase != NULL)
|
|
|
|
free(ef->strbase, M_LINKER);
|
|
|
|
if (ef->ctftab != NULL)
|
|
|
|
free(ef->ctftab, M_LINKER);
|
|
|
|
if (ef->ctfoff != NULL)
|
|
|
|
free(ef->ctfoff, M_LINKER);
|
|
|
|
if (ef->typoff != NULL)
|
|
|
|
free(ef->typoff, M_LINKER);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
1998-10-09 23:55:31 +00:00
|
|
|
static void
|
2000-04-29 13:19:31 +00:00
|
|
|
link_elf_unload_preload(linker_file_t file)
|
1998-10-09 23:55:31 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
if (file->filename != NULL)
|
|
|
|
preload_delete_name(file->filename);
|
1998-10-09 23:55:31 +00:00
|
|
|
}
|
|
|
|
|
1998-09-11 08:46:15 +00:00
|
|
|
static const char *
|
2005-12-18 04:52:37 +00:00
|
|
|
symbol_name(elf_file_t ef, Elf_Size r_info)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
const Elf_Sym *ref;
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (ELF_R_SYM(r_info)) {
|
|
|
|
ref = ef->symtab + ELF_R_SYM(r_info);
|
|
|
|
return (ef->strtab + ref->st_name);
|
|
|
|
}
|
|
|
|
return (NULL);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2000-04-24 17:08:04 +00:00
|
|
|
relocate_file(elf_file_t ef)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
const Elf_Rel *rellim;
|
|
|
|
const Elf_Rel *rel;
|
|
|
|
const Elf_Rela *relalim;
|
|
|
|
const Elf_Rela *rela;
|
|
|
|
const char *symname;
|
|
|
|
|
|
|
|
/* Perform relocations without addend if there are any: */
|
|
|
|
rel = ef->rel;
|
|
|
|
if (rel != NULL) {
|
|
|
|
rellim = (const Elf_Rel *)
|
|
|
|
((const char *)ef->rel + ef->relsize);
|
|
|
|
while (rel < rellim) {
|
|
|
|
if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel,
|
|
|
|
ELF_RELOC_REL, elf_lookup)) {
|
|
|
|
symname = symbol_name(ef, rel->r_info);
|
|
|
|
printf("link_elf: symbol %s undefined\n", symname);
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
|
|
|
rel++;
|
|
|
|
}
|
1998-10-16 03:55:01 +00:00
|
|
|
}
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Perform relocations with addend if there are any: */
|
|
|
|
rela = ef->rela;
|
|
|
|
if (rela != NULL) {
|
|
|
|
relalim = (const Elf_Rela *)
|
|
|
|
((const char *)ef->rela + ef->relasize);
|
|
|
|
while (rela < relalim) {
|
|
|
|
if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela,
|
|
|
|
ELF_RELOC_RELA, elf_lookup)) {
|
|
|
|
symname = symbol_name(ef, rela->r_info);
|
|
|
|
printf("link_elf: symbol %s undefined\n",
|
|
|
|
symname);
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
|
|
|
rela++;
|
|
|
|
}
|
1998-10-16 03:55:01 +00:00
|
|
|
}
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Perform PLT relocations without addend if there are any: */
|
|
|
|
rel = ef->pltrel;
|
|
|
|
if (rel != NULL) {
|
|
|
|
rellim = (const Elf_Rel *)
|
|
|
|
((const char *)ef->pltrel + ef->pltrelsize);
|
|
|
|
while (rel < rellim) {
|
|
|
|
if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel,
|
|
|
|
ELF_RELOC_REL, elf_lookup)) {
|
|
|
|
symname = symbol_name(ef, rel->r_info);
|
|
|
|
printf("link_elf: symbol %s undefined\n",
|
|
|
|
symname);
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
|
|
|
rel++;
|
|
|
|
}
|
1998-10-16 03:55:01 +00:00
|
|
|
}
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Perform relocations with addend if there are any: */
|
|
|
|
rela = ef->pltrela;
|
|
|
|
if (rela != NULL) {
|
|
|
|
relalim = (const Elf_Rela *)
|
|
|
|
((const char *)ef->pltrela + ef->pltrelasize);
|
|
|
|
while (rela < relalim) {
|
|
|
|
if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela,
|
|
|
|
ELF_RELOC_RELA, elf_lookup)) {
|
|
|
|
symname = symbol_name(ef, rela->r_info);
|
|
|
|
printf("link_elf: symbol %s undefined\n",
|
|
|
|
symname);
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
|
|
|
rela++;
|
|
|
|
}
|
1998-10-16 03:55:01 +00:00
|
|
|
}
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (0);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
1998-09-11 08:46:15 +00:00
|
|
|
/*
|
|
|
|
* Hash function for symbol table lookup. Don't even think about changing
|
|
|
|
* this. It is specified by the System V ABI.
|
|
|
|
*/
|
|
|
|
static unsigned long
|
|
|
|
elf_hash(const char *name)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
const unsigned char *p = (const unsigned char *) name;
|
|
|
|
unsigned long h = 0;
|
|
|
|
unsigned long g;
|
|
|
|
|
|
|
|
while (*p != '\0') {
|
|
|
|
h = (h << 4) + *p++;
|
|
|
|
if ((g = h & 0xf0000000) != 0)
|
|
|
|
h ^= g >> 24;
|
|
|
|
h &= ~g;
|
|
|
|
}
|
|
|
|
return (h);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
2002-09-28 17:15:38 +00:00
|
|
|
static int
|
1999-01-27 21:50:00 +00:00
|
|
|
link_elf_lookup_symbol(linker_file_t lf, const char* name, c_linker_sym_t* sym)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef = (elf_file_t) lf;
|
|
|
|
unsigned long symnum;
|
|
|
|
const Elf_Sym* symp;
|
|
|
|
const char *strp;
|
|
|
|
unsigned long hash;
|
|
|
|
int i;
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* If we don't have a hash, bail. */
|
|
|
|
if (ef->buckets == NULL || ef->nbuckets == 0) {
|
|
|
|
printf("link_elf_lookup_symbol: missing symbol hash table\n");
|
|
|
|
return (ENOENT);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* First, search hashed global symbols */
|
|
|
|
hash = elf_hash(name);
|
|
|
|
symnum = ef->buckets[hash % ef->nbuckets];
|
|
|
|
|
|
|
|
while (symnum != STN_UNDEF) {
|
|
|
|
if (symnum >= ef->nchains) {
|
|
|
|
printf("%s: corrupt symbol table\n", __func__);
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
symp = ef->symtab + symnum;
|
|
|
|
if (symp->st_name == 0) {
|
|
|
|
printf("%s: corrupt symbol table\n", __func__);
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
|
|
|
|
|
|
|
strp = ef->strtab + symp->st_name;
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (strcmp(name, strp) == 0) {
|
|
|
|
if (symp->st_shndx != SHN_UNDEF ||
|
|
|
|
(symp->st_value != 0 &&
|
|
|
|
ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
|
|
|
|
*sym = (c_linker_sym_t) symp;
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
|
|
|
|
|
|
|
symnum = ef->chains[symnum];
|
1998-09-11 08:46:15 +00:00
|
|
|
}
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* If we have not found it, look at the full table (if loaded) */
|
|
|
|
if (ef->symtab == ef->ddbsymtab)
|
|
|
|
return (ENOENT);
|
1998-09-11 08:46:15 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Exhaustive search */
|
|
|
|
for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
|
|
|
|
strp = ef->ddbstrtab + symp->st_name;
|
|
|
|
if (strcmp(name, strp) == 0) {
|
|
|
|
if (symp->st_shndx != SHN_UNDEF ||
|
|
|
|
(symp->st_value != 0 &&
|
|
|
|
ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
|
|
|
|
*sym = (c_linker_sym_t) symp;
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
return (ENOENT);
|
|
|
|
}
|
1998-10-12 09:13:50 +00:00
|
|
|
}
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (ENOENT);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
1998-10-09 23:55:31 +00:00
|
|
|
static int
|
2010-11-08 20:57:08 +00:00
|
|
|
link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
|
|
|
|
linker_symval_t *symval)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2000-04-24 17:08:04 +00:00
|
|
|
elf_file_t ef = (elf_file_t) lf;
|
1999-01-28 00:57:57 +00:00
|
|
|
const Elf_Sym* es = (const Elf_Sym*) sym;
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2002-08-24 05:01:33 +00:00
|
|
|
if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) {
|
2010-11-08 20:57:08 +00:00
|
|
|
symval->name = ef->strtab + es->st_name;
|
|
|
|
symval->value = (caddr_t) ef->address + es->st_value;
|
|
|
|
symval->size = es->st_size;
|
|
|
|
return (0);
|
1998-10-12 09:13:50 +00:00
|
|
|
}
|
|
|
|
if (ef->symtab == ef->ddbsymtab)
|
2010-11-08 20:57:08 +00:00
|
|
|
return (ENOENT);
|
2002-08-24 05:01:33 +00:00
|
|
|
if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
|
2010-11-08 20:57:08 +00:00
|
|
|
symval->name = ef->ddbstrtab + es->st_name;
|
|
|
|
symval->value = (caddr_t) ef->address + es->st_value;
|
|
|
|
symval->size = es->st_size;
|
|
|
|
return (0);
|
1998-10-12 09:13:50 +00:00
|
|
|
}
|
2010-11-08 20:57:08 +00:00
|
|
|
return (ENOENT);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
link_elf_search_symbol(linker_file_t lf, caddr_t value,
|
2010-11-08 20:57:08 +00:00
|
|
|
c_linker_sym_t *sym, long *diffp)
|
1998-08-24 08:25:26 +00:00
|
|
|
{
|
2000-04-24 17:08:04 +00:00
|
|
|
elf_file_t ef = (elf_file_t) lf;
|
1999-12-24 15:33:36 +00:00
|
|
|
u_long off = (uintptr_t) (void *) value;
|
1998-08-24 08:25:26 +00:00
|
|
|
u_long diff = off;
|
1999-12-24 15:33:36 +00:00
|
|
|
u_long st_value;
|
1998-09-11 08:46:15 +00:00
|
|
|
const Elf_Sym* es;
|
|
|
|
const Elf_Sym* best = 0;
|
1998-08-24 08:25:26 +00:00
|
|
|
int i;
|
|
|
|
|
1998-10-12 09:13:50 +00:00
|
|
|
for (i = 0, es = ef->ddbsymtab; i < ef->ddbsymcnt; i++, es++) {
|
1998-08-24 08:25:26 +00:00
|
|
|
if (es->st_name == 0)
|
|
|
|
continue;
|
1999-12-24 15:33:36 +00:00
|
|
|
st_value = es->st_value + (uintptr_t) (void *) ef->address;
|
1999-11-28 11:59:18 +00:00
|
|
|
if (off >= st_value) {
|
|
|
|
if (off - st_value < diff) {
|
|
|
|
diff = off - st_value;
|
1998-08-24 08:25:26 +00:00
|
|
|
best = es;
|
|
|
|
if (diff == 0)
|
|
|
|
break;
|
1999-11-28 11:59:18 +00:00
|
|
|
} else if (off - st_value == diff) {
|
1998-08-24 08:25:26 +00:00
|
|
|
best = es;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (best == 0)
|
|
|
|
*diffp = off;
|
|
|
|
else
|
|
|
|
*diffp = diff;
|
1999-01-27 21:50:00 +00:00
|
|
|
*sym = (c_linker_sym_t) best;
|
1998-08-24 08:25:26 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (0);
|
1998-08-24 08:25:26 +00:00
|
|
|
}
|
2001-06-13 10:58:39 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Look up a linker set on an ELF system.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
link_elf_lookup_set(linker_file_t lf, const char *name,
|
2010-11-08 20:57:08 +00:00
|
|
|
void ***startp, void ***stopp, int *countp)
|
2001-06-13 10:58:39 +00:00
|
|
|
{
|
|
|
|
c_linker_sym_t sym;
|
|
|
|
linker_symval_t symval;
|
|
|
|
char *setsym;
|
|
|
|
void **start, **stop;
|
|
|
|
int len, error = 0, count;
|
|
|
|
|
|
|
|
len = strlen(name) + sizeof("__start_set_"); /* sizeof includes \0 */
|
2003-02-19 05:47:46 +00:00
|
|
|
setsym = malloc(len, M_LINKER, M_WAITOK);
|
2001-06-13 10:58:39 +00:00
|
|
|
|
|
|
|
/* get address of first entry */
|
|
|
|
snprintf(setsym, len, "%s%s", "__start_set_", name);
|
|
|
|
error = link_elf_lookup_symbol(lf, setsym, &sym);
|
2010-11-08 20:57:08 +00:00
|
|
|
if (error != 0)
|
2001-06-13 10:58:39 +00:00
|
|
|
goto out;
|
|
|
|
link_elf_symbol_values(lf, sym, &symval);
|
|
|
|
if (symval.value == 0) {
|
|
|
|
error = ESRCH;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
start = (void **)symval.value;
|
|
|
|
|
|
|
|
/* get address of last entry */
|
|
|
|
snprintf(setsym, len, "%s%s", "__stop_set_", name);
|
|
|
|
error = link_elf_lookup_symbol(lf, setsym, &sym);
|
2010-11-08 20:57:08 +00:00
|
|
|
if (error != 0)
|
2001-06-13 10:58:39 +00:00
|
|
|
goto out;
|
|
|
|
link_elf_symbol_values(lf, sym, &symval);
|
|
|
|
if (symval.value == 0) {
|
|
|
|
error = ESRCH;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
stop = (void **)symval.value;
|
|
|
|
|
|
|
|
/* and the number of entries */
|
|
|
|
count = stop - start;
|
|
|
|
|
|
|
|
/* and copy out */
|
2010-11-08 20:57:08 +00:00
|
|
|
if (startp != NULL)
|
2001-06-13 10:58:39 +00:00
|
|
|
*startp = start;
|
2010-11-08 20:57:08 +00:00
|
|
|
if (stopp != NULL)
|
2001-06-13 10:58:39 +00:00
|
|
|
*stopp = stop;
|
2010-11-08 20:57:08 +00:00
|
|
|
if (countp != NULL)
|
2001-06-13 10:58:39 +00:00
|
|
|
*countp = count;
|
|
|
|
|
|
|
|
out:
|
|
|
|
free(setsym, M_LINKER);
|
2010-11-08 20:57:08 +00:00
|
|
|
return (error);
|
2001-06-13 10:58:39 +00:00
|
|
|
}
|
2001-10-30 15:21:45 +00:00
|
|
|
|
|
|
|
static int
|
|
|
|
link_elf_each_function_name(linker_file_t file,
|
2010-11-08 20:57:08 +00:00
|
|
|
int (*callback)(const char *, void *), void *opaque)
|
|
|
|
{
|
|
|
|
elf_file_t ef = (elf_file_t)file;
|
|
|
|
const Elf_Sym *symp;
|
|
|
|
int i, error;
|
2001-10-30 15:21:45 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Exhaustive search */
|
|
|
|
for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
|
|
|
|
if (symp->st_value != 0 &&
|
|
|
|
ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
|
|
|
|
error = callback(ef->ddbstrtab + symp->st_name, opaque);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
}
|
2001-10-30 15:21:45 +00:00
|
|
|
}
|
2010-11-08 20:57:08 +00:00
|
|
|
return (0);
|
2001-10-30 15:21:45 +00:00
|
|
|
}
|
2002-04-21 21:08:30 +00:00
|
|
|
|
2008-05-23 00:49:39 +00:00
|
|
|
static int
|
|
|
|
link_elf_each_function_nameval(linker_file_t file,
|
|
|
|
linker_function_nameval_callback_t callback, void *opaque)
|
|
|
|
{
|
|
|
|
linker_symval_t symval;
|
|
|
|
elf_file_t ef = (elf_file_t)file;
|
|
|
|
const Elf_Sym* symp;
|
|
|
|
int i, error;
|
|
|
|
|
|
|
|
/* Exhaustive search */
|
|
|
|
for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
|
|
|
|
if (symp->st_value != 0 &&
|
|
|
|
ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
|
2010-11-08 20:57:08 +00:00
|
|
|
error = link_elf_symbol_values(file,
|
|
|
|
(c_linker_sym_t) symp, &symval);
|
|
|
|
if (error != 0)
|
2008-05-23 00:49:39 +00:00
|
|
|
return (error);
|
|
|
|
error = callback(file, i, &symval, opaque);
|
2010-11-08 20:57:08 +00:00
|
|
|
if (error != 0)
|
2008-05-23 00:49:39 +00:00
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2002-04-21 21:08:30 +00:00
|
|
|
#ifdef __ia64__
|
|
|
|
/*
|
|
|
|
* Each KLD has its own GP. The GP value for each load module is given by
|
|
|
|
* DT_PLTGOT on ia64. We need GP to construct function descriptors, but
|
|
|
|
* don't have direct access to the ELF file structure. The link_elf_get_gp()
|
|
|
|
* function returns the GP given a pointer to a generic linker file struct.
|
|
|
|
*/
|
|
|
|
Elf_Addr
|
|
|
|
link_elf_get_gp(linker_file_t lf)
|
|
|
|
{
|
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
2010-11-08 20:57:08 +00:00
|
|
|
return ((Elf_Addr)ef->got);
|
2002-04-21 21:08:30 +00:00
|
|
|
}
|
|
|
|
#endif
|
2002-04-25 01:22:16 +00:00
|
|
|
|
2002-09-27 23:12:53 +00:00
|
|
|
const Elf_Sym *
|
2005-12-18 04:52:37 +00:00
|
|
|
elf_get_sym(linker_file_t lf, Elf_Size symidx)
|
2002-09-27 23:12:53 +00:00
|
|
|
{
|
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
|
|
|
|
|
|
|
if (symidx >= ef->nchains)
|
|
|
|
return (NULL);
|
|
|
|
return (ef->symtab + symidx);
|
|
|
|
}
|
|
|
|
|
2002-10-15 05:40:07 +00:00
|
|
|
const char *
|
2005-12-18 04:52:37 +00:00
|
|
|
elf_get_symname(linker_file_t lf, Elf_Size symidx)
|
2002-10-15 05:40:07 +00:00
|
|
|
{
|
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
|
|
|
const Elf_Sym *sym;
|
|
|
|
|
|
|
|
if (symidx >= ef->nchains)
|
|
|
|
return (NULL);
|
|
|
|
sym = ef->symtab + symidx;
|
|
|
|
return (ef->strtab + sym->st_name);
|
|
|
|
}
|
|
|
|
|
2002-04-25 01:22:16 +00:00
|
|
|
/*
|
|
|
|
* Symbol lookup function that can be used when the symbol index is known (ie
|
|
|
|
* in relocations). It uses the symbol index instead of doing a fully fledged
|
|
|
|
* hash table based lookup when such is valid. For example for local symbols.
|
|
|
|
* This is not only more efficient, it's also more correct. It's not always
|
|
|
|
* the case that the symbol can be found through the hash table.
|
|
|
|
*/
|
2004-05-16 20:00:28 +00:00
|
|
|
static Elf_Addr
|
2005-12-18 04:52:37 +00:00
|
|
|
elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
|
2002-04-25 01:22:16 +00:00
|
|
|
{
|
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
|
|
|
const Elf_Sym *sym;
|
|
|
|
const char *symbol;
|
|
|
|
|
|
|
|
/* Don't even try to lookup the symbol if the index is bogus. */
|
|
|
|
if (symidx >= ef->nchains)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
sym = ef->symtab + symidx;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Don't do a full lookup when the symbol is local. It may even
|
|
|
|
* fail because it may not be found through the hash table.
|
|
|
|
*/
|
|
|
|
if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) {
|
|
|
|
/* Force lookup failure when we have an insanity. */
|
|
|
|
if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0)
|
|
|
|
return (0);
|
|
|
|
return ((Elf_Addr)ef->address + sym->st_value);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* XXX we can avoid doing a hash table based lookup for global
|
|
|
|
* symbols as well. This however is not always valid, so we'll
|
|
|
|
* just do it the hard way for now. Performance tweaks can
|
|
|
|
* always be added.
|
|
|
|
*/
|
|
|
|
|
|
|
|
symbol = ef->strtab + sym->st_name;
|
|
|
|
|
|
|
|
/* Force a lookup failure if the symbol name is bogus. */
|
|
|
|
if (*symbol == 0)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
return ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
|
|
|
|
}
|
2003-01-21 02:42:44 +00:00
|
|
|
|
|
|
|
static void
|
|
|
|
link_elf_reloc_local(linker_file_t lf)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
const Elf_Rel *rellim;
|
|
|
|
const Elf_Rel *rel;
|
|
|
|
const Elf_Rela *relalim;
|
|
|
|
const Elf_Rela *rela;
|
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
|
|
|
|
|
|
|
/* Perform relocations without addend if there are any: */
|
|
|
|
if ((rel = ef->rel) != NULL) {
|
|
|
|
rellim = (const Elf_Rel *)((const char *)ef->rel + ef->relsize);
|
|
|
|
while (rel < rellim) {
|
|
|
|
elf_reloc_local(lf, (Elf_Addr)ef->address, rel,
|
|
|
|
ELF_RELOC_REL, elf_lookup);
|
|
|
|
rel++;
|
|
|
|
}
|
2003-01-21 02:42:44 +00:00
|
|
|
}
|
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
/* Perform relocations with addend if there are any: */
|
|
|
|
if ((rela = ef->rela) != NULL) {
|
|
|
|
relalim = (const Elf_Rela *)
|
|
|
|
((const char *)ef->rela + ef->relasize);
|
|
|
|
while (rela < relalim) {
|
|
|
|
elf_reloc_local(lf, (Elf_Addr)ef->address, rela,
|
|
|
|
ELF_RELOC_RELA, elf_lookup);
|
|
|
|
rela++;
|
|
|
|
}
|
2003-01-21 02:42:44 +00:00
|
|
|
}
|
|
|
|
}
|
2009-05-26 21:39:09 +00:00
|
|
|
|
|
|
|
static long
|
|
|
|
link_elf_symtab_get(linker_file_t lf, const Elf_Sym **symtab)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
2009-05-26 21:39:09 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
*symtab = ef->ddbsymtab;
|
|
|
|
|
|
|
|
if (*symtab == NULL)
|
|
|
|
return (0);
|
|
|
|
|
|
|
|
return (ef->ddbsymcnt);
|
2009-05-26 21:39:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static long
|
|
|
|
link_elf_strtab_get(linker_file_t lf, caddr_t *strtab)
|
|
|
|
{
|
2010-11-08 20:57:08 +00:00
|
|
|
elf_file_t ef = (elf_file_t)lf;
|
2009-05-26 21:39:09 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
*strtab = ef->ddbstrtab;
|
2009-05-26 21:39:09 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
if (*strtab == NULL)
|
|
|
|
return (0);
|
2009-05-26 21:39:09 +00:00
|
|
|
|
2010-11-08 20:57:08 +00:00
|
|
|
return (ef->ddbstrcnt);
|
2009-05-26 21:39:09 +00:00
|
|
|
}
|