1998-03-07 19:24:35 +00:00
|
|
|
/*-
|
2017-11-20 19:49:47 +00:00
|
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
*
|
1998-03-07 19:24:35 +00:00
|
|
|
* Copyright (c) 1983 Regents of the University of California.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
2014-02-17 22:27:32 +00:00
|
|
|
* 3. Neither the name of the University nor the names of its contributors
|
1998-03-07 19:24:35 +00:00
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#if defined(LIBC_SCCS) && !defined(lint)
|
|
|
|
/*static char *sccsid = "from: @(#)malloc.c 5.11 (Berkeley) 2/23/91";*/
|
1999-08-28 00:22:10 +00:00
|
|
|
static char *rcsid = "$FreeBSD$";
|
1998-03-07 19:24:35 +00:00
|
|
|
#endif /* LIBC_SCCS and not lint */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* malloc.c (Caltech) 2/21/82
|
|
|
|
* Chris Kingsley, kingsley@cit-20.
|
|
|
|
*
|
|
|
|
* This is a very fast storage allocator. It allocates blocks of a small
|
|
|
|
* number of different sizes, and keeps free lists of each size. Blocks that
|
|
|
|
* don't exactly fit are passed up to the next larger size. In this
|
|
|
|
* implementation, the available sizes are 2^n-4 (or 2^n-10) bytes long.
|
|
|
|
* This is designed for use in a virtual memory environment.
|
|
|
|
*/
|
|
|
|
|
2019-05-02 15:03:16 +00:00
|
|
|
#include <sys/param.h>
|
2010-08-17 09:05:39 +00:00
|
|
|
#include <sys/sysctl.h>
|
2019-05-02 15:03:16 +00:00
|
|
|
#include <sys/mman.h>
|
2018-08-02 07:43:28 +00:00
|
|
|
#include <errno.h>
|
2003-02-13 17:35:00 +00:00
|
|
|
#include <stdarg.h>
|
2003-08-22 02:22:59 +00:00
|
|
|
#include <stddef.h>
|
1998-03-07 19:24:35 +00:00
|
|
|
#include <string.h>
|
|
|
|
#include <unistd.h>
|
2018-08-02 07:43:28 +00:00
|
|
|
#include "rtld.h"
|
2011-08-24 20:05:13 +00:00
|
|
|
#include "rtld_printf.h"
|
2018-08-02 07:43:28 +00:00
|
|
|
#include "paths.h"
|
1998-03-07 19:24:35 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Pre-allocate mmap'ed pages
|
|
|
|
*/
|
2017-11-18 13:21:22 +00:00
|
|
|
#define NPOOLPAGES (128*1024/pagesz)
|
1998-03-07 19:24:35 +00:00
|
|
|
static caddr_t pagepool_start, pagepool_end;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The overhead on a block is at least 4 bytes. When free, this space
|
|
|
|
* contains a pointer to the next free block, and the bottom two bits must
|
|
|
|
* be zero. When in use, the first byte is set to MAGIC, and the second
|
|
|
|
* byte is the size index. The remaining bytes are for alignment.
|
|
|
|
* If range checking is enabled then a second word holds the size of the
|
|
|
|
* requested block, less 1, rounded up to a multiple of sizeof(RMAGIC).
|
|
|
|
* The order of elements is critical: ov_magic must overlay the low order
|
|
|
|
* bits of ov_next, and ov_magic can not be a valid ov_next bit pattern.
|
|
|
|
*/
|
|
|
|
union overhead {
|
|
|
|
union overhead *ov_next; /* when free */
|
|
|
|
struct {
|
|
|
|
u_char ovu_magic; /* magic number */
|
|
|
|
u_char ovu_index; /* bucket # */
|
|
|
|
} ovu;
|
|
|
|
#define ov_magic ovu.ovu_magic
|
|
|
|
#define ov_index ovu.ovu_index
|
|
|
|
};
|
|
|
|
|
2018-10-29 21:08:11 +00:00
|
|
|
static void morecore(int bucket);
|
|
|
|
static int morepages(int n);
|
|
|
|
|
1998-03-07 19:24:35 +00:00
|
|
|
#define MAGIC 0xef /* magic # on accounting info */
|
|
|
|
|
|
|
|
/*
|
2019-09-02 08:03:29 +00:00
|
|
|
* nextf[i] is the pointer to the next free block of size
|
|
|
|
* (FIRST_BUCKET_SIZE << i). The overhead information precedes the data
|
|
|
|
* area returned to the user.
|
1998-03-07 19:24:35 +00:00
|
|
|
*/
|
2019-09-02 08:03:29 +00:00
|
|
|
#define FIRST_BUCKET_SIZE 8
|
1998-03-07 19:24:35 +00:00
|
|
|
#define NBUCKETS 30
|
|
|
|
static union overhead *nextf[NBUCKETS];
|
|
|
|
|
|
|
|
static int pagesz; /* page size */
|
|
|
|
|
Before calling mmap() on a shared library's text and data sections, rtld
first calls mmap() with the arguments PROT_NONE and MAP_ANON to reserve a
single, contiguous range of virtual addresses for the entire shared library.
Later, rtld calls mmap() with the the shared library's file descriptor
and the argument MAP_FIXED to place the text and data sections within the
reserved range. The rationale for mapping shared libraries in this way is
explained in the commit message for Revision 190885. However, this approach
does have an unintended, negative consequence. Since the first call to
mmap() specifies MAP_ANON and not the shared library's file descriptor, the
kernel has no idea what alignment the vm object backing the file prefers.
As a result, the reserved range's alignment is unlikely to be the same as
the vm object's, and so mapping with superpages becomes impossible. To
address this problem, this revision adds the argument MAP_ALIGNED_SUPER to
the first call to mmap() if the text section is larger than the smallest
superpage size.
To determine if the text section is larger than the smallest superpage
size, rtld must always fetch the page size information. As a result, the
private code for fetching the base page size in rtld's builtin malloc is
redundant. Eliminate it. Requested by: kib
Tested by: zbb (on arm)
Reviewed by: kib (an earlier version)
Discussed with: jhb
2014-04-11 16:55:25 +00:00
|
|
|
/*
|
|
|
|
* The array of supported page sizes is provided by the user, i.e., the
|
|
|
|
* program that calls this storage allocator. That program must initialize
|
|
|
|
* the array before making its first call to allocate storage. The array
|
|
|
|
* must contain at least one page size. The page sizes must be stored in
|
|
|
|
* increasing order.
|
|
|
|
*/
|
2010-08-17 09:05:39 +00:00
|
|
|
|
1998-03-07 19:24:35 +00:00
|
|
|
void *
|
2019-01-29 22:40:42 +00:00
|
|
|
__crt_malloc(size_t nbytes)
|
1998-03-07 19:24:35 +00:00
|
|
|
{
|
2018-10-29 21:08:11 +00:00
|
|
|
union overhead *op;
|
|
|
|
int bucket;
|
2018-10-29 21:08:19 +00:00
|
|
|
size_t amt;
|
1998-03-07 19:24:35 +00:00
|
|
|
|
|
|
|
/*
|
2019-09-02 08:03:29 +00:00
|
|
|
* First time malloc is called, setup page size.
|
1998-03-07 19:24:35 +00:00
|
|
|
*/
|
2019-09-02 08:03:29 +00:00
|
|
|
if (pagesz == 0)
|
|
|
|
pagesz = pagesizes[0];
|
1998-03-07 19:24:35 +00:00
|
|
|
/*
|
|
|
|
* Convert amount of memory requested into closest block size
|
|
|
|
* stored in hash buckets which satisfies request.
|
|
|
|
* Account for space used per block for accounting.
|
|
|
|
*/
|
2019-09-02 08:03:29 +00:00
|
|
|
amt = FIRST_BUCKET_SIZE;
|
|
|
|
bucket = 0;
|
|
|
|
while (nbytes > amt - sizeof(*op)) {
|
1998-03-07 19:24:35 +00:00
|
|
|
amt <<= 1;
|
|
|
|
bucket++;
|
2019-09-02 08:03:29 +00:00
|
|
|
if (amt == 0 || bucket >= NBUCKETS)
|
|
|
|
return (NULL);
|
1998-03-07 19:24:35 +00:00
|
|
|
}
|
|
|
|
/*
|
|
|
|
* If nothing in hash bucket right now,
|
|
|
|
* request more memory from the system.
|
|
|
|
*/
|
|
|
|
if ((op = nextf[bucket]) == NULL) {
|
|
|
|
morecore(bucket);
|
|
|
|
if ((op = nextf[bucket]) == NULL)
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
/* remove from linked list */
|
|
|
|
nextf[bucket] = op->ov_next;
|
|
|
|
op->ov_magic = MAGIC;
|
|
|
|
op->ov_index = bucket;
|
|
|
|
return ((char *)(op + 1));
|
|
|
|
}
|
|
|
|
|
2006-01-12 07:28:21 +00:00
|
|
|
void *
|
2019-01-29 22:40:42 +00:00
|
|
|
__crt_calloc(size_t num, size_t size)
|
2006-01-12 07:28:21 +00:00
|
|
|
{
|
|
|
|
void *ret;
|
|
|
|
|
|
|
|
if (size != 0 && (num * size) / size != num) {
|
|
|
|
/* size_t overflow. */
|
|
|
|
return (NULL);
|
|
|
|
}
|
|
|
|
|
2019-01-29 22:40:42 +00:00
|
|
|
if ((ret = __crt_malloc(num * size)) != NULL)
|
2006-01-12 07:28:21 +00:00
|
|
|
memset(ret, 0, num * size);
|
|
|
|
|
|
|
|
return (ret);
|
|
|
|
}
|
|
|
|
|
1998-03-07 19:24:35 +00:00
|
|
|
/*
|
|
|
|
* Allocate more memory to the indicated bucket.
|
|
|
|
*/
|
|
|
|
static void
|
2018-10-29 21:08:28 +00:00
|
|
|
morecore(int bucket)
|
1998-03-07 19:24:35 +00:00
|
|
|
{
|
2018-10-29 21:08:11 +00:00
|
|
|
union overhead *op;
|
|
|
|
int sz; /* size of desired block */
|
1998-03-07 19:24:35 +00:00
|
|
|
int amt; /* amount to allocate */
|
|
|
|
int nblks; /* how many blocks we get */
|
|
|
|
|
2019-09-02 08:03:29 +00:00
|
|
|
sz = FIRST_BUCKET_SIZE << bucket;
|
1998-03-07 19:24:35 +00:00
|
|
|
if (sz < pagesz) {
|
|
|
|
amt = pagesz;
|
|
|
|
nblks = amt / sz;
|
|
|
|
} else {
|
2019-09-02 08:03:29 +00:00
|
|
|
amt = sz;
|
1998-03-07 19:24:35 +00:00
|
|
|
nblks = 1;
|
|
|
|
}
|
|
|
|
if (amt > pagepool_end - pagepool_start)
|
2021-10-11 16:13:31 +00:00
|
|
|
if (morepages(amt / pagesz + NPOOLPAGES) == 0 &&
|
|
|
|
/* Retry with min required size */
|
|
|
|
morepages(amt / pagesz) == 0)
|
1998-03-07 19:24:35 +00:00
|
|
|
return;
|
|
|
|
op = (union overhead *)pagepool_start;
|
|
|
|
pagepool_start += amt;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Add new memory allocated to that on
|
|
|
|
* free list for this hash bucket.
|
|
|
|
*/
|
|
|
|
nextf[bucket] = op;
|
|
|
|
while (--nblks > 0) {
|
|
|
|
op->ov_next = (union overhead *)((caddr_t)op + sz);
|
|
|
|
op = (union overhead *)((caddr_t)op + sz);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2019-01-29 22:40:42 +00:00
|
|
|
__crt_free(void *cp)
|
1998-03-07 19:24:35 +00:00
|
|
|
{
|
2018-10-29 21:08:11 +00:00
|
|
|
int size;
|
|
|
|
union overhead *op;
|
1998-03-07 19:24:35 +00:00
|
|
|
|
|
|
|
if (cp == NULL)
|
|
|
|
return;
|
|
|
|
op = (union overhead *)((caddr_t)cp - sizeof (union overhead));
|
|
|
|
if (op->ov_magic != MAGIC)
|
|
|
|
return; /* sanity */
|
|
|
|
size = op->ov_index;
|
|
|
|
op->ov_next = nextf[size]; /* also clobbers ov_magic */
|
|
|
|
nextf[size] = op;
|
|
|
|
}
|
|
|
|
|
|
|
|
void *
|
2019-01-29 22:40:42 +00:00
|
|
|
__crt_realloc(void *cp, size_t nbytes)
|
1998-03-07 19:24:35 +00:00
|
|
|
{
|
2018-10-29 21:08:11 +00:00
|
|
|
u_int onb;
|
|
|
|
int i;
|
1998-03-07 19:24:35 +00:00
|
|
|
union overhead *op;
|
|
|
|
char *res;
|
|
|
|
|
|
|
|
if (cp == NULL)
|
2019-01-29 22:40:42 +00:00
|
|
|
return (__crt_malloc(nbytes));
|
1998-03-07 19:24:35 +00:00
|
|
|
op = (union overhead *)((caddr_t)cp - sizeof (union overhead));
|
2019-08-20 16:07:17 +00:00
|
|
|
if (op->ov_magic != MAGIC)
|
|
|
|
return (NULL); /* Double-free or bad argument */
|
|
|
|
i = op->ov_index;
|
1998-03-07 19:24:35 +00:00
|
|
|
onb = 1 << (i + 3);
|
2003-05-04 00:56:00 +00:00
|
|
|
if (onb < (u_int)pagesz)
|
2019-05-02 15:03:16 +00:00
|
|
|
onb -= sizeof(*op);
|
1998-03-07 19:24:35 +00:00
|
|
|
else
|
2019-05-02 15:03:16 +00:00
|
|
|
onb += pagesz - sizeof(*op);
|
1998-03-07 19:24:35 +00:00
|
|
|
/* avoid the copy if same size block */
|
2019-08-20 16:07:17 +00:00
|
|
|
if (i != 0) {
|
|
|
|
i = 1 << (i + 2);
|
|
|
|
if (i < pagesz)
|
|
|
|
i -= sizeof(*op);
|
|
|
|
else
|
|
|
|
i += pagesz - sizeof(*op);
|
1998-03-07 19:24:35 +00:00
|
|
|
}
|
2019-08-20 16:07:17 +00:00
|
|
|
if (nbytes <= onb && nbytes > (size_t)i)
|
|
|
|
return (cp);
|
2019-01-29 22:40:42 +00:00
|
|
|
if ((res = __crt_malloc(nbytes)) == NULL)
|
|
|
|
return (NULL);
|
2019-08-20 16:07:17 +00:00
|
|
|
bcopy(cp, res, (nbytes < onb) ? nbytes : onb);
|
|
|
|
__crt_free(cp);
|
1998-03-07 19:24:35 +00:00
|
|
|
return (res);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2018-10-29 21:08:28 +00:00
|
|
|
morepages(int n)
|
1998-03-07 19:24:35 +00:00
|
|
|
{
|
2019-05-16 13:13:33 +00:00
|
|
|
caddr_t addr;
|
|
|
|
int offset;
|
1998-03-07 19:24:35 +00:00
|
|
|
|
|
|
|
if (pagepool_end - pagepool_start > pagesz) {
|
2021-10-12 21:39:07 +00:00
|
|
|
addr = roundup2(pagepool_start, pagesz);
|
2018-08-02 07:43:28 +00:00
|
|
|
if (munmap(addr, pagepool_end - addr) != 0) {
|
2019-01-29 22:40:42 +00:00
|
|
|
#ifdef IN_RTLD
|
2018-08-02 07:43:28 +00:00
|
|
|
rtld_fdprintf(STDERR_FILENO, _BASENAME_RTLD ": "
|
|
|
|
"morepages: cannot munmap %p: %s\n",
|
|
|
|
addr, rtld_strerror(errno));
|
2019-01-29 22:40:42 +00:00
|
|
|
#endif
|
2018-08-02 07:43:28 +00:00
|
|
|
}
|
1998-03-07 19:24:35 +00:00
|
|
|
}
|
|
|
|
|
2021-10-11 16:13:31 +00:00
|
|
|
if (pagepool_start == MAP_FAILED)
|
|
|
|
pagepool_start = 0;
|
2021-10-12 21:39:07 +00:00
|
|
|
offset = (uintptr_t)pagepool_start - rounddown2(
|
|
|
|
(uintptr_t)pagepool_start, pagesz);
|
1998-03-07 19:24:35 +00:00
|
|
|
|
2019-05-16 13:13:33 +00:00
|
|
|
pagepool_start = mmap(0, n * pagesz, PROT_READ | PROT_WRITE,
|
|
|
|
MAP_ANON | MAP_PRIVATE, -1, 0);
|
|
|
|
if (pagepool_start == MAP_FAILED) {
|
2019-01-29 22:40:42 +00:00
|
|
|
#ifdef IN_RTLD
|
2018-08-02 07:43:28 +00:00
|
|
|
rtld_fdprintf(STDERR_FILENO, _BASENAME_RTLD ": morepages: "
|
|
|
|
"cannot mmap anonymous memory: %s\n",
|
|
|
|
rtld_strerror(errno));
|
2019-01-29 22:40:42 +00:00
|
|
|
#endif
|
2019-05-16 13:13:33 +00:00
|
|
|
return (0);
|
1998-03-07 19:24:35 +00:00
|
|
|
}
|
|
|
|
pagepool_end = pagepool_start + n * pagesz;
|
|
|
|
pagepool_start += offset;
|
|
|
|
|
2019-05-16 13:13:33 +00:00
|
|
|
return (n);
|
1998-03-07 19:24:35 +00:00
|
|
|
}
|