1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 1987, 1991, 1993
|
|
|
|
* The Regents of the University of California. All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 3. All advertising materials mentioning features or use of this software
|
|
|
|
* must display the following acknowledgement:
|
|
|
|
* This product includes software developed by the University of
|
|
|
|
* California, Berkeley and its contributors.
|
|
|
|
* 4. Neither the name of the University nor the names of its contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* @(#)kern_malloc.c 8.3 (Berkeley) 1/4/94
|
1999-08-28 01:08:13 +00:00
|
|
|
* $FreeBSD$
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
|
1998-02-23 07:41:23 +00:00
|
|
|
#include "opt_vm.h"
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/param.h>
|
1994-05-25 09:21:21 +00:00
|
|
|
#include <sys/systm.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/kernel.h>
|
2001-05-01 08:13:21 +00:00
|
|
|
#include <sys/lock.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <sys/malloc.h>
|
1996-01-29 09:58:34 +00:00
|
|
|
#include <sys/mbuf.h>
|
2000-10-20 07:29:16 +00:00
|
|
|
#include <sys/mutex.h>
|
1995-12-07 12:48:31 +00:00
|
|
|
#include <sys/vmmeter.h>
|
2001-01-21 19:25:07 +00:00
|
|
|
#include <sys/proc.h>
|
2002-04-15 04:05:53 +00:00
|
|
|
#include <sys/sysctl.h>
|
2002-11-01 18:58:12 +00:00
|
|
|
#include <sys/time.h>
|
2000-09-23 00:01:37 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <vm/vm.h>
|
2002-09-18 08:26:30 +00:00
|
|
|
#include <vm/pmap.h>
|
1995-12-07 12:48:31 +00:00
|
|
|
#include <vm/vm_param.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
#include <vm/vm_kern.h>
|
1995-12-07 12:48:31 +00:00
|
|
|
#include <vm/vm_extern.h>
|
1997-08-05 00:02:08 +00:00
|
|
|
#include <vm/vm_map.h>
|
2002-09-18 08:26:30 +00:00
|
|
|
#include <vm/vm_page.h>
|
2002-03-19 09:11:49 +00:00
|
|
|
#include <vm/uma.h>
|
|
|
|
#include <vm/uma_int.h>
|
2002-04-30 07:54:25 +00:00
|
|
|
#include <vm/uma_dbg.h>
|
1994-05-24 10:09:53 +00:00
|
|
|
|
1999-09-19 08:40:11 +00:00
|
|
|
#if defined(INVARIANTS) && defined(__i386__)
|
|
|
|
#include <machine/cpu.h>
|
|
|
|
#endif
|
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/*
|
|
|
|
* When realloc() is called, if the new size is sufficiently smaller than
|
|
|
|
* the old size, realloc() will allocate a new, smaller block to avoid
|
|
|
|
* wasting memory. 'Sufficiently smaller' is defined as: newsize <=
|
|
|
|
* oldsize / 2^n, where REALLOC_FRACTION defines the value of 'n'.
|
|
|
|
*/
|
|
|
|
#ifndef REALLOC_FRACTION
|
|
|
|
#define REALLOC_FRACTION 1 /* new block if <= half the size */
|
|
|
|
#endif
|
|
|
|
|
1999-10-03 12:18:29 +00:00
|
|
|
MALLOC_DEFINE(M_CACHE, "cache", "Various Dynamically allocated caches");
|
1999-09-11 16:41:39 +00:00
|
|
|
MALLOC_DEFINE(M_DEVBUF, "devbuf", "device driver memory");
|
|
|
|
MALLOC_DEFINE(M_TEMP, "temp", "misc temporary data buffers");
|
|
|
|
|
1999-11-22 02:45:11 +00:00
|
|
|
MALLOC_DEFINE(M_IP6OPT, "ip6opt", "IPv6 options");
|
|
|
|
MALLOC_DEFINE(M_IP6NDP, "ip6ndp", "IPv6 Neighbor Discovery");
|
|
|
|
|
2002-03-19 21:25:46 +00:00
|
|
|
static void kmeminit(void *);
|
1995-08-28 09:19:25 +00:00
|
|
|
SYSINIT(kmem, SI_SUB_KMEM, SI_ORDER_FIRST, kmeminit, NULL)
|
|
|
|
|
1997-10-12 20:26:33 +00:00
|
|
|
static MALLOC_DEFINE(M_FREE, "free", "should be on free list");
|
|
|
|
|
1998-11-10 08:46:24 +00:00
|
|
|
static struct malloc_type *kmemstatistics;
|
1997-10-10 14:06:34 +00:00
|
|
|
static char *kmembase;
|
1997-09-16 13:52:04 +00:00
|
|
|
static char *kmemlimit;
|
2000-02-16 21:11:33 +00:00
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
#define KMEM_ZSHIFT 4
|
|
|
|
#define KMEM_ZBASE 16
|
|
|
|
#define KMEM_ZMASK (KMEM_ZBASE - 1)
|
|
|
|
|
2002-04-29 17:12:02 +00:00
|
|
|
#define KMEM_ZMAX 65536
|
2002-03-19 09:11:49 +00:00
|
|
|
#define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT)
|
2002-04-15 04:05:53 +00:00
|
|
|
static u_int8_t kmemsize[KMEM_ZSIZE + 1];
|
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
/* These won't be powers of two for long */
|
|
|
|
struct {
|
2002-04-15 04:05:53 +00:00
|
|
|
int kz_size;
|
|
|
|
char *kz_name;
|
|
|
|
uma_zone_t kz_zone;
|
|
|
|
} kmemzones[] = {
|
|
|
|
{16, "16", NULL},
|
|
|
|
{32, "32", NULL},
|
|
|
|
{64, "64", NULL},
|
|
|
|
{128, "128", NULL},
|
|
|
|
{256, "256", NULL},
|
|
|
|
{512, "512", NULL},
|
|
|
|
{1024, "1024", NULL},
|
|
|
|
{2048, "2048", NULL},
|
|
|
|
{4096, "4096", NULL},
|
|
|
|
{8192, "8192", NULL},
|
2002-04-29 17:53:23 +00:00
|
|
|
{16384, "16384", NULL},
|
2002-04-29 17:12:02 +00:00
|
|
|
{32768, "32768", NULL},
|
|
|
|
{65536, "65536", NULL},
|
2002-03-19 09:11:49 +00:00
|
|
|
{0, NULL},
|
1994-05-24 10:09:53 +00:00
|
|
|
};
|
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
u_int vm_kmem_size;
|
2002-05-02 07:22:19 +00:00
|
|
|
|
|
|
|
/*
|
2002-09-18 08:26:30 +00:00
|
|
|
* The malloc_mtx protects the kmemstatistics linked list.
|
2002-05-02 07:22:19 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
struct mtx malloc_mtx;
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2002-04-15 05:24:01 +00:00
|
|
|
#ifdef MALLOC_PROFILE
|
|
|
|
uint64_t krequests[KMEM_ZSIZE + 1];
|
|
|
|
|
|
|
|
static int sysctl_kern_mprof(SYSCTL_HANDLER_ARGS);
|
|
|
|
#endif
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-04-15 05:24:01 +00:00
|
|
|
static int sysctl_kern_malloc(SYSCTL_HANDLER_ARGS);
|
1994-05-24 10:09:53 +00:00
|
|
|
|
2002-11-01 18:58:12 +00:00
|
|
|
/* time_uptime of last malloc(9) failure */
|
|
|
|
static time_t t_malloc_fail;
|
|
|
|
|
|
|
|
int
|
|
|
|
malloc_last_fail(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (time_uptime - t_malloc_fail);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
1999-01-21 08:29:12 +00:00
|
|
|
* malloc:
|
|
|
|
*
|
|
|
|
* Allocate a block of memory.
|
|
|
|
*
|
|
|
|
* If M_NOWAIT is set, this routine will not block and return NULL if
|
|
|
|
* the allocation fails.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
void *
|
|
|
|
malloc(size, type, flags)
|
|
|
|
unsigned long size;
|
1997-10-10 18:14:23 +00:00
|
|
|
struct malloc_type *type;
|
1997-10-10 14:06:34 +00:00
|
|
|
int flags;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-04-15 04:05:53 +00:00
|
|
|
int indx;
|
2002-03-19 09:11:49 +00:00
|
|
|
caddr_t va;
|
|
|
|
uma_zone_t zone;
|
2003-02-01 10:07:49 +00:00
|
|
|
#ifdef DIAGNOSTIC
|
|
|
|
unsigned long osize = size;
|
|
|
|
#endif
|
1997-10-10 18:14:23 +00:00
|
|
|
register struct malloc_type *ksp = type;
|
1997-10-10 14:06:34 +00:00
|
|
|
|
2003-03-10 20:24:54 +00:00
|
|
|
#ifdef INVARIANTS
|
2003-03-10 19:39:53 +00:00
|
|
|
/*
|
|
|
|
* To make sure that WAITOK or NOWAIT is set, but not more than
|
|
|
|
* one, and check against the API botches that are common.
|
|
|
|
*/
|
|
|
|
indx = flags & (M_WAITOK | M_NOWAIT | M_DONTWAIT | M_TRYWAIT);
|
|
|
|
if (indx != M_NOWAIT && indx != M_WAITOK) {
|
|
|
|
static struct timeval lasterr;
|
|
|
|
static int curerr, once;
|
|
|
|
if (once == 0 && ppsratecheck(&lasterr, &curerr, 1)) {
|
|
|
|
printf("Bad malloc flags: %x\n", indx);
|
|
|
|
backtrace();
|
|
|
|
flags |= M_WAITOK;
|
|
|
|
once++;
|
|
|
|
}
|
|
|
|
}
|
2003-03-10 20:24:54 +00:00
|
|
|
#endif
|
2002-04-23 18:50:25 +00:00
|
|
|
#if 0
|
|
|
|
if (size == 0)
|
|
|
|
Debugger("zero size malloc");
|
|
|
|
#endif
|
2003-03-10 19:39:53 +00:00
|
|
|
if (flags & M_WAITOK)
|
2001-09-12 08:38:13 +00:00
|
|
|
KASSERT(curthread->td_intr_nesting_level == 0,
|
2003-02-19 05:47:46 +00:00
|
|
|
("malloc(M_WAITOK) in interrupt context"));
|
2002-03-19 09:11:49 +00:00
|
|
|
if (size <= KMEM_ZMAX) {
|
2002-04-15 04:05:53 +00:00
|
|
|
if (size & KMEM_ZMASK)
|
|
|
|
size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
|
|
|
|
indx = kmemsize[size >> KMEM_ZSHIFT];
|
|
|
|
zone = kmemzones[indx].kz_zone;
|
|
|
|
#ifdef MALLOC_PROFILE
|
|
|
|
krequests[size >> KMEM_ZSHIFT]++;
|
|
|
|
#endif
|
2002-03-19 09:11:49 +00:00
|
|
|
va = uma_zalloc(zone, flags);
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&ksp->ks_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
if (va == NULL)
|
1994-05-24 10:09:53 +00:00
|
|
|
goto out;
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
ksp->ks_size |= 1 << indx;
|
|
|
|
size = zone->uz_size;
|
2002-03-19 09:11:49 +00:00
|
|
|
} else {
|
2002-04-15 04:05:53 +00:00
|
|
|
size = roundup(size, PAGE_SIZE);
|
2002-03-19 09:11:49 +00:00
|
|
|
zone = NULL;
|
|
|
|
va = uma_large_malloc(size, flags);
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&ksp->ks_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
if (va == NULL)
|
2002-03-19 09:11:49 +00:00
|
|
|
goto out;
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
2002-04-15 04:05:53 +00:00
|
|
|
ksp->ks_memuse += size;
|
1994-05-24 10:09:53 +00:00
|
|
|
ksp->ks_inuse++;
|
2002-03-19 09:11:49 +00:00
|
|
|
out:
|
1994-05-24 10:09:53 +00:00
|
|
|
ksp->ks_calls++;
|
|
|
|
if (ksp->ks_memuse > ksp->ks_maxused)
|
|
|
|
ksp->ks_maxused = ksp->ks_memuse;
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_unlock(&ksp->ks_mtx);
|
2002-11-01 18:58:12 +00:00
|
|
|
if (!(flags & M_NOWAIT))
|
2003-02-19 05:47:46 +00:00
|
|
|
KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL"));
|
2002-11-01 18:58:12 +00:00
|
|
|
if (va == NULL) {
|
|
|
|
t_malloc_fail = time_uptime;
|
|
|
|
}
|
2003-02-01 10:07:49 +00:00
|
|
|
#ifdef DIAGNOSTIC
|
|
|
|
if (!(flags & M_ZERO)) {
|
|
|
|
memset(va, 0x70, osize);
|
|
|
|
}
|
|
|
|
#endif
|
1994-05-24 10:09:53 +00:00
|
|
|
return ((void *) va);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
1999-01-21 08:29:12 +00:00
|
|
|
* free:
|
|
|
|
*
|
|
|
|
* Free a block of memory allocated by malloc.
|
|
|
|
*
|
|
|
|
* This routine may not block.
|
1994-05-24 10:09:53 +00:00
|
|
|
*/
|
|
|
|
void
|
|
|
|
free(addr, type)
|
|
|
|
void *addr;
|
1997-10-10 18:14:23 +00:00
|
|
|
struct malloc_type *type;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-09-18 08:26:30 +00:00
|
|
|
register struct malloc_type *ksp = type;
|
2002-03-19 09:11:49 +00:00
|
|
|
uma_slab_t slab;
|
|
|
|
u_long size;
|
1997-10-10 14:06:34 +00:00
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/* free(NULL, ...) does nothing */
|
|
|
|
if (addr == NULL)
|
|
|
|
return;
|
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
size = 0;
|
|
|
|
|
2002-09-18 08:26:30 +00:00
|
|
|
slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK));
|
2002-03-19 09:11:49 +00:00
|
|
|
|
|
|
|
if (slab == NULL)
|
2002-04-15 04:05:53 +00:00
|
|
|
panic("free: address %p(%p) has not been allocated.\n",
|
2002-09-18 08:26:30 +00:00
|
|
|
addr, (void *)((u_long)addr & (~UMA_SLAB_MASK)));
|
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
|
|
|
|
if (!(slab->us_flags & UMA_SLAB_MALLOC)) {
|
2002-05-02 09:07:04 +00:00
|
|
|
#ifdef INVARIANTS
|
|
|
|
struct malloc_type **mtp = addr;
|
|
|
|
#endif
|
2002-03-19 09:11:49 +00:00
|
|
|
size = slab->us_zone->uz_size;
|
2002-05-02 09:07:04 +00:00
|
|
|
#ifdef INVARIANTS
|
|
|
|
/*
|
|
|
|
* Cache a pointer to the malloc_type that most recently freed
|
|
|
|
* this memory here. This way we know who is most likely to
|
|
|
|
* have stepped on it later.
|
|
|
|
*
|
|
|
|
* This code assumes that size is a multiple of 8 bytes for
|
|
|
|
* 64 bit machines
|
|
|
|
*/
|
|
|
|
mtp = (struct malloc_type **)
|
|
|
|
((unsigned long)mtp & ~UMA_ALIGN_PTR);
|
|
|
|
mtp += (size - sizeof(struct malloc_type *)) /
|
|
|
|
sizeof(struct malloc_type *);
|
|
|
|
*mtp = type;
|
|
|
|
#endif
|
2002-03-19 09:11:49 +00:00
|
|
|
uma_zfree_arg(slab->us_zone, addr, slab);
|
|
|
|
} else {
|
|
|
|
size = slab->us_size;
|
|
|
|
uma_large_free(slab);
|
1999-05-06 18:13:11 +00:00
|
|
|
}
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&ksp->ks_mtx);
|
1994-05-24 10:09:53 +00:00
|
|
|
ksp->ks_memuse -= size;
|
|
|
|
ksp->ks_inuse--;
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_unlock(&ksp->ks_mtx);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/*
|
|
|
|
* realloc: change the size of a memory block
|
|
|
|
*/
|
|
|
|
void *
|
|
|
|
realloc(addr, size, type, flags)
|
|
|
|
void *addr;
|
|
|
|
unsigned long size;
|
|
|
|
struct malloc_type *type;
|
|
|
|
int flags;
|
|
|
|
{
|
2002-03-19 09:11:49 +00:00
|
|
|
uma_slab_t slab;
|
2002-03-13 01:42:33 +00:00
|
|
|
unsigned long alloc;
|
|
|
|
void *newaddr;
|
|
|
|
|
|
|
|
/* realloc(NULL, ...) is equivalent to malloc(...) */
|
|
|
|
if (addr == NULL)
|
|
|
|
return (malloc(size, type, flags));
|
|
|
|
|
2002-09-18 08:26:30 +00:00
|
|
|
slab = vtoslab((vm_offset_t)addr & ~(UMA_SLAB_MASK));
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-03-13 01:42:33 +00:00
|
|
|
/* Sanity check */
|
2002-03-19 09:11:49 +00:00
|
|
|
KASSERT(slab != NULL,
|
2002-03-13 01:42:33 +00:00
|
|
|
("realloc: address %p out of range", (void *)addr));
|
|
|
|
|
|
|
|
/* Get the size of the original block */
|
2002-03-19 09:11:49 +00:00
|
|
|
if (slab->us_zone)
|
|
|
|
alloc = slab->us_zone->uz_size;
|
|
|
|
else
|
|
|
|
alloc = slab->us_size;
|
2002-03-13 01:42:33 +00:00
|
|
|
|
|
|
|
/* Reuse the original block if appropriate */
|
|
|
|
if (size <= alloc
|
|
|
|
&& (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE))
|
|
|
|
return (addr);
|
|
|
|
|
|
|
|
/* Allocate a new, bigger (or smaller) block */
|
|
|
|
if ((newaddr = malloc(size, type, flags)) == NULL)
|
|
|
|
return (NULL);
|
|
|
|
|
|
|
|
/* Copy over original contents */
|
|
|
|
bcopy(addr, newaddr, min(size, alloc));
|
|
|
|
free(addr, type);
|
|
|
|
return (newaddr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* reallocf: same as realloc() but free memory on failure.
|
|
|
|
*/
|
|
|
|
void *
|
|
|
|
reallocf(addr, size, type, flags)
|
|
|
|
void *addr;
|
|
|
|
unsigned long size;
|
|
|
|
struct malloc_type *type;
|
|
|
|
int flags;
|
|
|
|
{
|
|
|
|
void *mem;
|
|
|
|
|
|
|
|
if ((mem = realloc(addr, size, type, flags)) == NULL)
|
|
|
|
free(addr, type);
|
|
|
|
return (mem);
|
|
|
|
}
|
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
/*
|
|
|
|
* Initialize the kernel memory allocator
|
|
|
|
*/
|
1995-08-28 09:19:25 +00:00
|
|
|
/* ARGSUSED*/
|
|
|
|
static void
|
1995-12-02 17:11:20 +00:00
|
|
|
kmeminit(dummy)
|
|
|
|
void *dummy;
|
1994-05-24 10:09:53 +00:00
|
|
|
{
|
2002-04-15 04:05:53 +00:00
|
|
|
u_int8_t indx;
|
2000-01-28 04:04:58 +00:00
|
|
|
u_long npg;
|
|
|
|
u_long mem_size;
|
2002-03-19 09:11:49 +00:00
|
|
|
int i;
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-04-04 21:03:38 +00:00
|
|
|
mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF);
|
2000-09-11 02:32:30 +00:00
|
|
|
|
1998-02-23 07:41:23 +00:00
|
|
|
/*
|
|
|
|
* Try to auto-tune the kernel memory size, so that it is
|
|
|
|
* more applicable for a wider range of machine sizes.
|
|
|
|
* On an X86, a VM_KMEM_SIZE_SCALE value of 4 is good, while
|
|
|
|
* a VM_KMEM_SIZE of 12MB is a fair compromise. The
|
|
|
|
* VM_KMEM_SIZE_MAX is dependent on the maximum KVA space
|
|
|
|
* available, and on an X86 with a total KVA space of 256MB,
|
|
|
|
* try to keep VM_KMEM_SIZE_MAX at 80MB or below.
|
|
|
|
*
|
|
|
|
* Note that the kmem_map is also used by the zone allocator,
|
|
|
|
* so make sure that there is enough space.
|
|
|
|
*/
|
2001-06-06 22:17:08 +00:00
|
|
|
vm_kmem_size = VM_KMEM_SIZE;
|
1998-02-23 07:41:23 +00:00
|
|
|
mem_size = cnt.v_page_count * PAGE_SIZE;
|
|
|
|
|
|
|
|
#if defined(VM_KMEM_SIZE_SCALE)
|
2001-06-06 22:17:08 +00:00
|
|
|
if ((mem_size / VM_KMEM_SIZE_SCALE) > vm_kmem_size)
|
|
|
|
vm_kmem_size = mem_size / VM_KMEM_SIZE_SCALE;
|
1998-02-23 07:41:23 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
#if defined(VM_KMEM_SIZE_MAX)
|
2001-06-06 22:17:08 +00:00
|
|
|
if (vm_kmem_size >= VM_KMEM_SIZE_MAX)
|
|
|
|
vm_kmem_size = VM_KMEM_SIZE_MAX;
|
1998-02-23 07:41:23 +00:00
|
|
|
#endif
|
|
|
|
|
1999-01-21 21:54:32 +00:00
|
|
|
/* Allow final override from the kernel environment */
|
2001-06-08 05:24:21 +00:00
|
|
|
TUNABLE_INT_FETCH("kern.vm.kmem.size", &vm_kmem_size);
|
1999-01-21 21:54:32 +00:00
|
|
|
|
2000-01-28 04:04:58 +00:00
|
|
|
/*
|
|
|
|
* Limit kmem virtual size to twice the physical memory.
|
|
|
|
* This allows for kmem map sparseness, but limits the size
|
|
|
|
* to something sane. Be careful to not overflow the 32bit
|
|
|
|
* ints while doing the check.
|
|
|
|
*/
|
|
|
|
if ((vm_kmem_size / 2) > (cnt.v_page_count * PAGE_SIZE))
|
|
|
|
vm_kmem_size = 2 * cnt.v_page_count * PAGE_SIZE;
|
1998-02-23 07:41:23 +00:00
|
|
|
|
2001-06-22 06:35:32 +00:00
|
|
|
/*
|
2001-08-03 05:05:32 +00:00
|
|
|
* In mbuf_init(), we set up submaps for mbufs and clusters, in which
|
2001-06-22 06:35:32 +00:00
|
|
|
* case we rounddown() (nmbufs * MSIZE) and (nmbclusters * MCLBYTES),
|
|
|
|
* respectively. Mathematically, this means that what we do here may
|
|
|
|
* amount to slightly more address space than we need for the submaps,
|
|
|
|
* but it never hurts to have an extra page in kmem_map.
|
|
|
|
*/
|
o Allow "buckets" in mb_alloc to be differently sized (according to
compile-time constants). That is, a "bucket" now is not necessarily
a page-worth of mbufs or clusters, but it is MBUF_BUCK_SZ, CLUS_BUCK_SZ
worth of mbufs, clusters.
o Rename {mbuf,clust}_limit to {mbuf,clust}_hiwm and introduce
{mbuf,clust}_lowm, which currently has no effect but will be used
to set the low watermarks.
o Fix netstat so that it can deal with the differently-sized buckets
and teach it about the low watermarks too.
o Make sure the per-cpu stats for an absent CPU has mb_active set to 0,
explicitly.
o Get rid of the allocate refcounts from mbuf map mess. Instead,
just malloc() the refcounts in one shot from mbuf_init()
o Clean up / update comments in subr_mbuf.c
2003-02-20 04:26:58 +00:00
|
|
|
npg = (nmbufs*MSIZE + nmbclusters*MCLBYTES + vm_kmem_size) / PAGE_SIZE;
|
1995-05-30 08:16:23 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
kmem_map = kmem_suballoc(kernel_map, (vm_offset_t *)&kmembase,
|
VM level code cleanups.
1) Start using TSM.
Struct procs continue to point to upages structure, after being freed.
Struct vmspace continues to point to pte object and kva space for kstack.
u_map is now superfluous.
2) vm_map's don't need to be reference counted. They always exist either
in the kernel or in a vmspace. The vmspaces are managed by reference
counts.
3) Remove the "wired" vm_map nonsense.
4) No need to keep a cache of kernel stack kva's.
5) Get rid of strange looking ++var, and change to var++.
6) Change more data structures to use our "zone" allocator. Added
struct proc, struct vmspace and struct vnode. This saves a significant
amount of kva space and physical memory. Additionally, this enables
TSM for the zone managed memory.
7) Keep ioopt disabled for now.
8) Remove the now bogus "single use" map concept.
9) Use generation counts or id's for data structures residing in TSM, where
it allows us to avoid unneeded restart overhead during traversals, where
blocking might occur.
10) Account better for memory deficits, so the pageout daemon will be able
to make enough memory available (experimental.)
11) Fix some vnode locking problems. (From Tor, I think.)
12) Add a check in ufs_lookup, to avoid lots of unneeded calls to bcmp.
(experimental.)
13) Significantly shrink, cleanup, and make slightly faster the vm_fault.c
code. Use generation counts, get rid of unneded collpase operations,
and clean up the cluster code.
14) Make vm_zone more suitable for TSM.
This commit is partially as a result of discussions and contributions from
other people, including DG, Tor Egge, PHK, and probably others that I
have forgotten to attribute (so let me know, if I forgot.)
This is not the infamous, final cleanup of the vnode stuff, but a necessary
step. Vnode mgmt should be correct, but things might still change, and
there is still some missing stuff (like ioopt, and physical backing of
non-merged cache files, debugging of layering concepts.)
1998-01-22 17:30:44 +00:00
|
|
|
(vm_offset_t *)&kmemlimit, (vm_size_t)(npg * PAGE_SIZE));
|
1997-08-05 00:02:08 +00:00
|
|
|
kmem_map->system_map = 1;
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-09-18 08:26:30 +00:00
|
|
|
uma_startup2();
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
|
|
|
|
int size = kmemzones[indx].kz_size;
|
|
|
|
char *name = kmemzones[indx].kz_name;
|
2002-03-19 09:11:49 +00:00
|
|
|
|
2002-04-30 07:54:25 +00:00
|
|
|
kmemzones[indx].kz_zone = uma_zcreate(name, size,
|
|
|
|
#ifdef INVARIANTS
|
2002-05-02 09:07:04 +00:00
|
|
|
mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
|
2002-04-30 07:54:25 +00:00
|
|
|
#else
|
|
|
|
NULL, NULL, NULL, NULL,
|
|
|
|
#endif
|
|
|
|
UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
|
2002-04-15 04:05:53 +00:00
|
|
|
|
2002-03-19 09:11:49 +00:00
|
|
|
for (;i <= size; i+= KMEM_ZBASE)
|
2002-04-15 04:05:53 +00:00
|
|
|
kmemsize[i >> KMEM_ZSHIFT] = indx;
|
2002-03-19 09:11:49 +00:00
|
|
|
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
1997-10-10 14:06:34 +00:00
|
|
|
}
|
|
|
|
|
1998-11-10 08:46:24 +00:00
|
|
|
void
|
|
|
|
malloc_init(data)
|
|
|
|
void *data;
|
1997-10-10 14:06:34 +00:00
|
|
|
{
|
1998-11-10 08:46:24 +00:00
|
|
|
struct malloc_type *type = (struct malloc_type *)data;
|
1997-10-10 14:06:34 +00:00
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_lock(&malloc_mtx);
|
1999-11-22 02:45:11 +00:00
|
|
|
if (type->ks_magic != M_MAGIC)
|
1997-10-28 19:01:02 +00:00
|
|
|
panic("malloc type lacks magic");
|
|
|
|
|
1997-12-05 05:36:58 +00:00
|
|
|
if (cnt.v_page_count == 0)
|
|
|
|
panic("malloc_init not allowed before vm init");
|
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
if (type->ks_next != NULL)
|
|
|
|
return;
|
|
|
|
|
1997-10-10 14:06:34 +00:00
|
|
|
type->ks_next = kmemstatistics;
|
|
|
|
kmemstatistics = type;
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_init(&type->ks_mtx, type->ks_shortdesc, "Malloc Stats", MTX_DEF);
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_unlock(&malloc_mtx);
|
1994-05-24 10:09:53 +00:00
|
|
|
}
|
1998-11-10 08:46:24 +00:00
|
|
|
|
|
|
|
void
|
|
|
|
malloc_uninit(data)
|
|
|
|
void *data;
|
|
|
|
{
|
|
|
|
struct malloc_type *type = (struct malloc_type *)data;
|
|
|
|
struct malloc_type *t;
|
|
|
|
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_lock(&malloc_mtx);
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&type->ks_mtx);
|
1999-11-22 02:45:11 +00:00
|
|
|
if (type->ks_magic != M_MAGIC)
|
1998-11-10 08:46:24 +00:00
|
|
|
panic("malloc type lacks magic");
|
|
|
|
|
|
|
|
if (cnt.v_page_count == 0)
|
|
|
|
panic("malloc_uninit not allowed before vm init");
|
|
|
|
|
|
|
|
if (type == kmemstatistics)
|
|
|
|
kmemstatistics = type->ks_next;
|
|
|
|
else {
|
|
|
|
for (t = kmemstatistics; t->ks_next != NULL; t = t->ks_next) {
|
|
|
|
if (t->ks_next == type) {
|
|
|
|
t->ks_next = type->ks_next;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
1999-05-12 11:11:27 +00:00
|
|
|
type->ks_next = NULL;
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_destroy(&type->ks_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
mtx_unlock(&malloc_mtx);
|
1998-11-10 08:46:24 +00:00
|
|
|
}
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
static int
|
|
|
|
sysctl_kern_malloc(SYSCTL_HANDLER_ARGS)
|
|
|
|
{
|
|
|
|
struct malloc_type *type;
|
|
|
|
int linesize = 128;
|
|
|
|
int curline;
|
|
|
|
int bufsize;
|
|
|
|
int first;
|
|
|
|
int error;
|
|
|
|
char *buf;
|
|
|
|
char *p;
|
|
|
|
int cnt;
|
|
|
|
int len;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
cnt = 0;
|
|
|
|
|
|
|
|
mtx_lock(&malloc_mtx);
|
|
|
|
for (type = kmemstatistics; type != NULL; type = type->ks_next)
|
|
|
|
cnt++;
|
|
|
|
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_unlock(&malloc_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
bufsize = linesize * (cnt + 1);
|
2003-02-19 05:47:46 +00:00
|
|
|
p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
|
2002-05-02 07:22:19 +00:00
|
|
|
mtx_lock(&malloc_mtx);
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
len = snprintf(p, linesize,
|
|
|
|
"\n Type InUse MemUse HighUse Requests Size(s)\n");
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
for (type = kmemstatistics; cnt != 0 && type != NULL;
|
|
|
|
type = type->ks_next, cnt--) {
|
|
|
|
if (type->ks_calls == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
curline = linesize - 2; /* Leave room for the \n */
|
2002-04-30 22:39:32 +00:00
|
|
|
len = snprintf(p, curline, "%13s%6lu%6luK%7luK%9llu",
|
2002-04-15 04:05:53 +00:00
|
|
|
type->ks_shortdesc,
|
|
|
|
type->ks_inuse,
|
|
|
|
(type->ks_memuse + 1023) / 1024,
|
|
|
|
(type->ks_maxused + 1023) / 1024,
|
|
|
|
(long long unsigned)type->ks_calls);
|
|
|
|
curline -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
first = 1;
|
2002-05-31 09:41:09 +00:00
|
|
|
for (i = 0; i < sizeof(kmemzones) / sizeof(kmemzones[0]) - 1;
|
|
|
|
i++) {
|
2002-04-15 04:05:53 +00:00
|
|
|
if (type->ks_size & (1 << i)) {
|
|
|
|
if (first)
|
|
|
|
len = snprintf(p, curline, " ");
|
|
|
|
else
|
|
|
|
len = snprintf(p, curline, ",");
|
|
|
|
curline -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
len = snprintf(p, curline,
|
|
|
|
"%s", kmemzones[i].kz_name);
|
|
|
|
curline -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
first = 0;
|
|
|
|
}
|
2002-05-31 09:41:09 +00:00
|
|
|
}
|
2002-04-15 04:05:53 +00:00
|
|
|
|
|
|
|
len = snprintf(p, 2, "\n");
|
|
|
|
p += len;
|
|
|
|
}
|
|
|
|
|
|
|
|
mtx_unlock(&malloc_mtx);
|
|
|
|
error = SYSCTL_OUT(req, buf, p - buf);
|
|
|
|
|
|
|
|
free(buf, M_TEMP);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
SYSCTL_OID(_kern, OID_AUTO, malloc, CTLTYPE_STRING|CTLFLAG_RD,
|
|
|
|
NULL, 0, sysctl_kern_malloc, "A", "Malloc Stats");
|
2002-04-15 05:24:01 +00:00
|
|
|
|
|
|
|
#ifdef MALLOC_PROFILE
|
|
|
|
|
|
|
|
static int
|
|
|
|
sysctl_kern_mprof(SYSCTL_HANDLER_ARGS)
|
|
|
|
{
|
|
|
|
int linesize = 64;
|
|
|
|
uint64_t count;
|
|
|
|
uint64_t waste;
|
|
|
|
uint64_t mem;
|
|
|
|
int bufsize;
|
|
|
|
int error;
|
|
|
|
char *buf;
|
|
|
|
int rsize;
|
|
|
|
int size;
|
|
|
|
char *p;
|
|
|
|
int len;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
bufsize = linesize * (KMEM_ZSIZE + 1);
|
|
|
|
bufsize += 128; /* For the stats line */
|
|
|
|
bufsize += 128; /* For the banner line */
|
|
|
|
waste = 0;
|
|
|
|
mem = 0;
|
|
|
|
|
2003-02-19 05:47:46 +00:00
|
|
|
p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
|
2002-04-15 05:24:01 +00:00
|
|
|
len = snprintf(p, bufsize,
|
|
|
|
"\n Size Requests Real Size\n");
|
|
|
|
bufsize -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
for (i = 0; i < KMEM_ZSIZE; i++) {
|
|
|
|
size = i << KMEM_ZSHIFT;
|
|
|
|
rsize = kmemzones[kmemsize[i]].kz_size;
|
|
|
|
count = (long long unsigned)krequests[i];
|
|
|
|
|
|
|
|
len = snprintf(p, bufsize, "%6d%28llu%11d\n",
|
|
|
|
size, (unsigned long long)count, rsize);
|
|
|
|
bufsize -= len;
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
if ((rsize * count) > (size * count))
|
|
|
|
waste += (rsize * count) - (size * count);
|
|
|
|
mem += (rsize * count);
|
|
|
|
}
|
|
|
|
|
|
|
|
len = snprintf(p, bufsize,
|
|
|
|
"\nTotal memory used:\t%30llu\nTotal Memory wasted:\t%30llu\n",
|
|
|
|
(unsigned long long)mem, (unsigned long long)waste);
|
|
|
|
p += len;
|
|
|
|
|
|
|
|
error = SYSCTL_OUT(req, buf, p - buf);
|
|
|
|
|
|
|
|
free(buf, M_TEMP);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
SYSCTL_OID(_kern, OID_AUTO, mprof, CTLTYPE_STRING|CTLFLAG_RD,
|
|
|
|
NULL, 0, sysctl_kern_mprof, "A", "Malloc Profiling");
|
|
|
|
#endif /* MALLOC_PROFILE */
|