8a171ccd92
In non regular use cases allocated memory might stay persistent in memory pool. This small patch checks every minute if there are old objects which can be released from memory pool. Right now with regular use, the pool is checked for old objects on each allocation attempt from this pool. so basically polling by its use. Now consider what happens if someone writes a lot of files and stops use of the volume or even unmounts it. So the code will no longer check if objects can be released from the pool. Already allocated objects will still stay in pool cache. this is no big issue for common use. But someone discovered this issue while doing tests. personally i know this behavior and I'm aware of it. Its no big issue. just a enhancement Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Reviewed-by: Kjeld Schouten-Lebbing <kjeld@schouten-lebbing.nl> Signed-off-by: Sebastian Gottschall <s.gottschall@dd-wrt.com> Closes #10938 Closes #10969
753 lines
20 KiB
C
753 lines
20 KiB
C
/*
|
|
* BSD 3-Clause New License (https://spdx.org/licenses/BSD-3-Clause.html)
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright notice,
|
|
* this list of conditions and the following disclaimer.
|
|
*
|
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
* this list of conditions and the following disclaimer in the documentation
|
|
* and/or other materials provided with the distribution.
|
|
*
|
|
* 3. Neither the name of the copyright holder nor the names of its
|
|
* contributors may be used to endorse or promote products derived from this
|
|
* software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* Copyright (c) 2016-2018, Klara Inc.
|
|
* Copyright (c) 2016-2018, Allan Jude
|
|
* Copyright (c) 2018-2020, Sebastian Gottschall
|
|
* Copyright (c) 2019-2020, Michael Niewöhner
|
|
* Copyright (c) 2020, The FreeBSD Foundation [1]
|
|
*
|
|
* [1] Portions of this software were developed by Allan Jude
|
|
* under sponsorship from the FreeBSD Foundation.
|
|
*/
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/sysmacros.h>
|
|
#include <sys/zfs_context.h>
|
|
#include <sys/zio_compress.h>
|
|
#include <sys/spa.h>
|
|
#include <sys/zstd/zstd.h>
|
|
|
|
#define ZSTD_STATIC_LINKING_ONLY
|
|
#include "lib/zstd.h"
|
|
#include "lib/zstd_errors.h"
|
|
|
|
kstat_t *zstd_ksp = NULL;
|
|
|
|
typedef struct zstd_stats {
|
|
kstat_named_t zstd_stat_alloc_fail;
|
|
kstat_named_t zstd_stat_alloc_fallback;
|
|
kstat_named_t zstd_stat_com_alloc_fail;
|
|
kstat_named_t zstd_stat_dec_alloc_fail;
|
|
kstat_named_t zstd_stat_com_inval;
|
|
kstat_named_t zstd_stat_dec_inval;
|
|
kstat_named_t zstd_stat_dec_header_inval;
|
|
kstat_named_t zstd_stat_com_fail;
|
|
kstat_named_t zstd_stat_dec_fail;
|
|
} zstd_stats_t;
|
|
|
|
static zstd_stats_t zstd_stats = {
|
|
{ "alloc_fail", KSTAT_DATA_UINT64 },
|
|
{ "alloc_fallback", KSTAT_DATA_UINT64 },
|
|
{ "compress_alloc_fail", KSTAT_DATA_UINT64 },
|
|
{ "decompress_alloc_fail", KSTAT_DATA_UINT64 },
|
|
{ "compress_level_invalid", KSTAT_DATA_UINT64 },
|
|
{ "decompress_level_invalid", KSTAT_DATA_UINT64 },
|
|
{ "decompress_header_invalid", KSTAT_DATA_UINT64 },
|
|
{ "compress_failed", KSTAT_DATA_UINT64 },
|
|
{ "decompress_failed", KSTAT_DATA_UINT64 },
|
|
};
|
|
|
|
/* Enums describing the allocator type specified by kmem_type in zstd_kmem */
|
|
enum zstd_kmem_type {
|
|
ZSTD_KMEM_UNKNOWN = 0,
|
|
/* Allocation type using kmem_vmalloc */
|
|
ZSTD_KMEM_DEFAULT,
|
|
/* Pool based allocation using mempool_alloc */
|
|
ZSTD_KMEM_POOL,
|
|
/* Reserved fallback memory for decompression only */
|
|
ZSTD_KMEM_DCTX,
|
|
ZSTD_KMEM_COUNT,
|
|
};
|
|
|
|
/* Structure for pooled memory objects */
|
|
struct zstd_pool {
|
|
void *mem;
|
|
size_t size;
|
|
kmutex_t barrier;
|
|
hrtime_t timeout;
|
|
};
|
|
|
|
/* Global structure for handling memory allocations */
|
|
struct zstd_kmem {
|
|
enum zstd_kmem_type kmem_type;
|
|
size_t kmem_size;
|
|
struct zstd_pool *pool;
|
|
};
|
|
|
|
/* Fallback memory structure used for decompression only if memory runs out */
|
|
struct zstd_fallback_mem {
|
|
size_t mem_size;
|
|
void *mem;
|
|
kmutex_t barrier;
|
|
};
|
|
|
|
struct zstd_levelmap {
|
|
int16_t zstd_level;
|
|
enum zio_zstd_levels level;
|
|
};
|
|
|
|
/*
|
|
* ZSTD memory handlers
|
|
*
|
|
* For decompression we use a different handler which also provides fallback
|
|
* memory allocation in case memory runs out.
|
|
*
|
|
* The ZSTD handlers were split up for the most simplified implementation.
|
|
*/
|
|
static void *zstd_alloc(void *opaque, size_t size);
|
|
static void *zstd_dctx_alloc(void *opaque, size_t size);
|
|
static void zstd_free(void *opaque, void *ptr);
|
|
|
|
/* Compression memory handler */
|
|
static const ZSTD_customMem zstd_malloc = {
|
|
zstd_alloc,
|
|
zstd_free,
|
|
NULL,
|
|
};
|
|
|
|
/* Decompression memory handler */
|
|
static const ZSTD_customMem zstd_dctx_malloc = {
|
|
zstd_dctx_alloc,
|
|
zstd_free,
|
|
NULL,
|
|
};
|
|
|
|
/* Level map for converting ZFS internal levels to ZSTD levels and vice versa */
|
|
static struct zstd_levelmap zstd_levels[] = {
|
|
{ZIO_ZSTD_LEVEL_1, ZIO_ZSTD_LEVEL_1},
|
|
{ZIO_ZSTD_LEVEL_2, ZIO_ZSTD_LEVEL_2},
|
|
{ZIO_ZSTD_LEVEL_3, ZIO_ZSTD_LEVEL_3},
|
|
{ZIO_ZSTD_LEVEL_4, ZIO_ZSTD_LEVEL_4},
|
|
{ZIO_ZSTD_LEVEL_5, ZIO_ZSTD_LEVEL_5},
|
|
{ZIO_ZSTD_LEVEL_6, ZIO_ZSTD_LEVEL_6},
|
|
{ZIO_ZSTD_LEVEL_7, ZIO_ZSTD_LEVEL_7},
|
|
{ZIO_ZSTD_LEVEL_8, ZIO_ZSTD_LEVEL_8},
|
|
{ZIO_ZSTD_LEVEL_9, ZIO_ZSTD_LEVEL_9},
|
|
{ZIO_ZSTD_LEVEL_10, ZIO_ZSTD_LEVEL_10},
|
|
{ZIO_ZSTD_LEVEL_11, ZIO_ZSTD_LEVEL_11},
|
|
{ZIO_ZSTD_LEVEL_12, ZIO_ZSTD_LEVEL_12},
|
|
{ZIO_ZSTD_LEVEL_13, ZIO_ZSTD_LEVEL_13},
|
|
{ZIO_ZSTD_LEVEL_14, ZIO_ZSTD_LEVEL_14},
|
|
{ZIO_ZSTD_LEVEL_15, ZIO_ZSTD_LEVEL_15},
|
|
{ZIO_ZSTD_LEVEL_16, ZIO_ZSTD_LEVEL_16},
|
|
{ZIO_ZSTD_LEVEL_17, ZIO_ZSTD_LEVEL_17},
|
|
{ZIO_ZSTD_LEVEL_18, ZIO_ZSTD_LEVEL_18},
|
|
{ZIO_ZSTD_LEVEL_19, ZIO_ZSTD_LEVEL_19},
|
|
{-1, ZIO_ZSTD_LEVEL_FAST_1},
|
|
{-2, ZIO_ZSTD_LEVEL_FAST_2},
|
|
{-3, ZIO_ZSTD_LEVEL_FAST_3},
|
|
{-4, ZIO_ZSTD_LEVEL_FAST_4},
|
|
{-5, ZIO_ZSTD_LEVEL_FAST_5},
|
|
{-6, ZIO_ZSTD_LEVEL_FAST_6},
|
|
{-7, ZIO_ZSTD_LEVEL_FAST_7},
|
|
{-8, ZIO_ZSTD_LEVEL_FAST_8},
|
|
{-9, ZIO_ZSTD_LEVEL_FAST_9},
|
|
{-10, ZIO_ZSTD_LEVEL_FAST_10},
|
|
{-20, ZIO_ZSTD_LEVEL_FAST_20},
|
|
{-30, ZIO_ZSTD_LEVEL_FAST_30},
|
|
{-40, ZIO_ZSTD_LEVEL_FAST_40},
|
|
{-50, ZIO_ZSTD_LEVEL_FAST_50},
|
|
{-60, ZIO_ZSTD_LEVEL_FAST_60},
|
|
{-70, ZIO_ZSTD_LEVEL_FAST_70},
|
|
{-80, ZIO_ZSTD_LEVEL_FAST_80},
|
|
{-90, ZIO_ZSTD_LEVEL_FAST_90},
|
|
{-100, ZIO_ZSTD_LEVEL_FAST_100},
|
|
{-500, ZIO_ZSTD_LEVEL_FAST_500},
|
|
{-1000, ZIO_ZSTD_LEVEL_FAST_1000},
|
|
};
|
|
|
|
/*
|
|
* This variable represents the maximum count of the pool based on the number
|
|
* of CPUs plus some buffer. We default to cpu count * 4, see init_zstd.
|
|
*/
|
|
static int pool_count = 16;
|
|
|
|
#define ZSTD_POOL_MAX pool_count
|
|
#define ZSTD_POOL_TIMEOUT 60 * 2
|
|
|
|
static struct zstd_fallback_mem zstd_dctx_fallback;
|
|
static struct zstd_pool *zstd_mempool_cctx;
|
|
static struct zstd_pool *zstd_mempool_dctx;
|
|
|
|
/*
|
|
* Try to get a cached allocated buffer from memory pool or allocate a new one
|
|
* if necessary. If a object is older than 2 minutes and does not fit the
|
|
* requested size, it will be released and a new cached entry will be allocated.
|
|
* If other pooled objects are detected without being used for 2 minutes, they
|
|
* will be released, too.
|
|
*
|
|
* The concept is that high frequency memory allocations of bigger objects are
|
|
* expensive. So if a lot of work is going on, allocations will be kept for a
|
|
* while and can be reused in that time frame.
|
|
*
|
|
* The scheduled release will be updated every time a object is reused.
|
|
*/
|
|
static void *
|
|
zstd_mempool_alloc(struct zstd_pool *zstd_mempool, size_t size)
|
|
{
|
|
struct zstd_pool *pool;
|
|
struct zstd_kmem *mem = NULL;
|
|
|
|
if (!zstd_mempool) {
|
|
return (NULL);
|
|
}
|
|
|
|
/* Seek for preallocated memory slot and free obsolete slots */
|
|
for (int i = 0; i < ZSTD_POOL_MAX; i++) {
|
|
pool = &zstd_mempool[i];
|
|
/*
|
|
* This lock is simply a marker for a pool object beeing in use.
|
|
* If it's already hold, it will be skipped.
|
|
*
|
|
* We need to create it before checking it to avoid race
|
|
* conditions caused by running in a threaded context.
|
|
*
|
|
* The lock is later released by zstd_mempool_free.
|
|
*/
|
|
if (mutex_tryenter(&pool->barrier)) {
|
|
/*
|
|
* Check if objects fits the size, if so we take it and
|
|
* update the timestamp.
|
|
*/
|
|
if (size && !mem && pool->mem && size <= pool->size) {
|
|
pool->timeout = gethrestime_sec() +
|
|
ZSTD_POOL_TIMEOUT;
|
|
mem = pool->mem;
|
|
continue;
|
|
}
|
|
|
|
/* Free memory if unused object older than 2 minutes */
|
|
if (pool->mem && gethrestime_sec() > pool->timeout) {
|
|
vmem_free(pool->mem, pool->size);
|
|
pool->mem = NULL;
|
|
pool->size = 0;
|
|
pool->timeout = 0;
|
|
}
|
|
|
|
mutex_exit(&pool->barrier);
|
|
}
|
|
}
|
|
|
|
if (!size || mem) {
|
|
return (mem);
|
|
}
|
|
|
|
/*
|
|
* If no preallocated slot was found, try to fill in a new one.
|
|
*
|
|
* We run a similar algorithm twice here to avoid pool fragmentation.
|
|
* The first one may generate holes in the list if objects get released.
|
|
* We always make sure that these holes get filled instead of adding new
|
|
* allocations constantly at the end.
|
|
*/
|
|
for (int i = 0; i < ZSTD_POOL_MAX; i++) {
|
|
pool = &zstd_mempool[i];
|
|
if (mutex_tryenter(&pool->barrier)) {
|
|
/* Object is free, try to allocate new one */
|
|
if (!pool->mem) {
|
|
mem = vmem_alloc(size, KM_SLEEP);
|
|
pool->mem = mem;
|
|
|
|
if (pool->mem) {
|
|
/* Keep track for later release */
|
|
mem->pool = pool;
|
|
pool->size = size;
|
|
mem->kmem_type = ZSTD_KMEM_POOL;
|
|
mem->kmem_size = size;
|
|
}
|
|
}
|
|
|
|
if (size <= pool->size) {
|
|
/* Update timestamp */
|
|
pool->timeout = gethrestime_sec() +
|
|
ZSTD_POOL_TIMEOUT;
|
|
|
|
return (pool->mem);
|
|
}
|
|
|
|
mutex_exit(&pool->barrier);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If the pool is full or the allocation failed, try lazy allocation
|
|
* instead.
|
|
*/
|
|
if (!mem) {
|
|
mem = vmem_alloc(size, KM_NOSLEEP);
|
|
if (mem) {
|
|
mem->pool = NULL;
|
|
mem->kmem_type = ZSTD_KMEM_DEFAULT;
|
|
mem->kmem_size = size;
|
|
}
|
|
}
|
|
|
|
return (mem);
|
|
}
|
|
|
|
/* Mark object as released by releasing the barrier mutex */
|
|
static void
|
|
zstd_mempool_free(struct zstd_kmem *z)
|
|
{
|
|
mutex_exit(&z->pool->barrier);
|
|
}
|
|
|
|
/* Convert ZFS internal enum to ZSTD level */
|
|
static int
|
|
zstd_enum_to_level(enum zio_zstd_levels level, int16_t *zstd_level)
|
|
{
|
|
if (level > 0 && level <= ZIO_ZSTD_LEVEL_19) {
|
|
*zstd_level = zstd_levels[level - 1].zstd_level;
|
|
return (0);
|
|
}
|
|
if (level >= ZIO_ZSTD_LEVEL_FAST_1 &&
|
|
level <= ZIO_ZSTD_LEVEL_FAST_1000) {
|
|
*zstd_level = zstd_levels[level - ZIO_ZSTD_LEVEL_FAST_1
|
|
+ ZIO_ZSTD_LEVEL_19].zstd_level;
|
|
return (0);
|
|
}
|
|
|
|
/* Invalid/unknown zfs compression enum - this should never happen. */
|
|
return (1);
|
|
}
|
|
|
|
/* Compress block using zstd */
|
|
size_t
|
|
zfs_zstd_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
|
|
int level)
|
|
{
|
|
size_t c_len;
|
|
int16_t zstd_level;
|
|
zfs_zstdhdr_t *hdr;
|
|
ZSTD_CCtx *cctx;
|
|
|
|
hdr = (zfs_zstdhdr_t *)d_start;
|
|
|
|
/* Skip compression if the specified level is invalid */
|
|
if (zstd_enum_to_level(level, &zstd_level)) {
|
|
ZSTDSTAT_BUMP(zstd_stat_com_inval);
|
|
return (s_len);
|
|
}
|
|
|
|
ASSERT3U(d_len, >=, sizeof (*hdr));
|
|
ASSERT3U(d_len, <=, s_len);
|
|
ASSERT3U(zstd_level, !=, 0);
|
|
|
|
cctx = ZSTD_createCCtx_advanced(zstd_malloc);
|
|
|
|
/*
|
|
* Out of kernel memory, gently fall through - this will disable
|
|
* compression in zio_compress_data
|
|
*/
|
|
if (!cctx) {
|
|
ZSTDSTAT_BUMP(zstd_stat_com_alloc_fail);
|
|
return (s_len);
|
|
}
|
|
|
|
/* Set the compression level */
|
|
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, zstd_level);
|
|
|
|
/* Use the "magicless" zstd header which saves us 4 header bytes */
|
|
ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless);
|
|
|
|
/*
|
|
* Disable redundant checksum calculation and content size storage since
|
|
* this is already done by ZFS itself.
|
|
*/
|
|
ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0);
|
|
ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0);
|
|
|
|
c_len = ZSTD_compress2(cctx,
|
|
hdr->data,
|
|
d_len - sizeof (*hdr),
|
|
s_start, s_len);
|
|
|
|
ZSTD_freeCCtx(cctx);
|
|
|
|
/* Error in the compression routine, disable compression. */
|
|
if (ZSTD_isError(c_len)) {
|
|
/*
|
|
* If we are aborting the compression because the saves are
|
|
* too small, that is not a failure. Everything else is a
|
|
* failure, so increment the compression failure counter.
|
|
*/
|
|
if (ZSTD_getErrorCode(c_len) != ZSTD_error_dstSize_tooSmall) {
|
|
ZSTDSTAT_BUMP(zstd_stat_com_fail);
|
|
}
|
|
return (s_len);
|
|
}
|
|
|
|
/*
|
|
* Encode the compressed buffer size at the start. We'll need this in
|
|
* decompression to counter the effects of padding which might be added
|
|
* to the compressed buffer and which, if unhandled, would confuse the
|
|
* hell out of our decompression function.
|
|
*/
|
|
hdr->c_len = BE_32(c_len);
|
|
|
|
/*
|
|
* Check version for overflow.
|
|
* The limit of 24 bits must not be exceeded. This allows a maximum
|
|
* version 1677.72.15 which we don't expect to be ever reached.
|
|
*/
|
|
ASSERT3U(ZSTD_VERSION_NUMBER, <=, 0xFFFFFF);
|
|
|
|
/*
|
|
* Encode the compression level as well. We may need to know the
|
|
* original compression level if compressed_arc is disabled, to match
|
|
* the compression settings to write this block to the L2ARC.
|
|
*
|
|
* Encode the actual level, so if the enum changes in the future, we
|
|
* will be compatible.
|
|
*
|
|
* The upper 24 bits store the ZSTD version to be able to provide
|
|
* future compatibility, since new versions might enhance the
|
|
* compression algorithm in a way, where the compressed data will
|
|
* change.
|
|
*
|
|
* As soon as such incompatibility occurs, handling code needs to be
|
|
* added, differentiating between the versions.
|
|
*/
|
|
hdr->version = ZSTD_VERSION_NUMBER;
|
|
hdr->level = level;
|
|
hdr->raw_version_level = BE_32(hdr->raw_version_level);
|
|
|
|
return (c_len + sizeof (*hdr));
|
|
}
|
|
|
|
/* Decompress block using zstd and return its stored level */
|
|
int
|
|
zfs_zstd_decompress_level(void *s_start, void *d_start, size_t s_len,
|
|
size_t d_len, uint8_t *level)
|
|
{
|
|
ZSTD_DCtx *dctx;
|
|
size_t result;
|
|
int16_t zstd_level;
|
|
uint32_t c_len;
|
|
const zfs_zstdhdr_t *hdr;
|
|
zfs_zstdhdr_t hdr_copy;
|
|
|
|
hdr = (const zfs_zstdhdr_t *)s_start;
|
|
c_len = BE_32(hdr->c_len);
|
|
|
|
/*
|
|
* Make a copy instead of directly converting the header, since we must
|
|
* not modify the original data that may be used again later.
|
|
*/
|
|
hdr_copy.raw_version_level = BE_32(hdr->raw_version_level);
|
|
|
|
/*
|
|
* NOTE: We ignore the ZSTD version for now. As soon as any
|
|
* incompatibility occurrs, it has to be handled accordingly.
|
|
* The version can be accessed via `hdr_copy.version`.
|
|
*/
|
|
|
|
/*
|
|
* Convert and check the level
|
|
* An invalid level is a strong indicator for data corruption! In such
|
|
* case return an error so the upper layers can try to fix it.
|
|
*/
|
|
if (zstd_enum_to_level(hdr_copy.level, &zstd_level)) {
|
|
ZSTDSTAT_BUMP(zstd_stat_dec_inval);
|
|
return (1);
|
|
}
|
|
|
|
ASSERT3U(d_len, >=, s_len);
|
|
ASSERT3U(hdr_copy.level, !=, ZIO_COMPLEVEL_INHERIT);
|
|
|
|
/* Invalid compressed buffer size encoded at start */
|
|
if (c_len + sizeof (*hdr) > s_len) {
|
|
ZSTDSTAT_BUMP(zstd_stat_dec_header_inval);
|
|
return (1);
|
|
}
|
|
|
|
dctx = ZSTD_createDCtx_advanced(zstd_dctx_malloc);
|
|
if (!dctx) {
|
|
ZSTDSTAT_BUMP(zstd_stat_dec_alloc_fail);
|
|
return (1);
|
|
}
|
|
|
|
/* Set header type to "magicless" */
|
|
ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless);
|
|
|
|
/* Decompress the data and release the context */
|
|
result = ZSTD_decompressDCtx(dctx, d_start, d_len, hdr->data, c_len);
|
|
ZSTD_freeDCtx(dctx);
|
|
|
|
/*
|
|
* Returns 0 on success (decompression function returned non-negative)
|
|
* and non-zero on failure (decompression function returned negative.
|
|
*/
|
|
if (ZSTD_isError(result)) {
|
|
ZSTDSTAT_BUMP(zstd_stat_dec_fail);
|
|
return (1);
|
|
}
|
|
|
|
if (level) {
|
|
*level = hdr_copy.level;
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
/* Decompress datablock using zstd */
|
|
int
|
|
zfs_zstd_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len,
|
|
int level __maybe_unused)
|
|
{
|
|
|
|
return (zfs_zstd_decompress_level(s_start, d_start, s_len, d_len,
|
|
NULL));
|
|
}
|
|
|
|
/* Allocator for zstd compression context using mempool_allocator */
|
|
static void *
|
|
zstd_alloc(void *opaque __maybe_unused, size_t size)
|
|
{
|
|
size_t nbytes = sizeof (struct zstd_kmem) + size;
|
|
struct zstd_kmem *z = NULL;
|
|
|
|
z = (struct zstd_kmem *)zstd_mempool_alloc(zstd_mempool_cctx, nbytes);
|
|
|
|
if (!z) {
|
|
ZSTDSTAT_BUMP(zstd_stat_alloc_fail);
|
|
return (NULL);
|
|
}
|
|
|
|
return ((void*)z + (sizeof (struct zstd_kmem)));
|
|
}
|
|
|
|
/*
|
|
* Allocator for zstd decompression context using mempool_allocator with
|
|
* fallback to reserved memory if allocation fails
|
|
*/
|
|
static void *
|
|
zstd_dctx_alloc(void *opaque __maybe_unused, size_t size)
|
|
{
|
|
size_t nbytes = sizeof (struct zstd_kmem) + size;
|
|
struct zstd_kmem *z = NULL;
|
|
enum zstd_kmem_type type = ZSTD_KMEM_DEFAULT;
|
|
|
|
z = (struct zstd_kmem *)zstd_mempool_alloc(zstd_mempool_dctx, nbytes);
|
|
if (!z) {
|
|
/* Try harder, decompression shall not fail */
|
|
z = vmem_alloc(nbytes, KM_SLEEP);
|
|
if (z) {
|
|
z->pool = NULL;
|
|
}
|
|
ZSTDSTAT_BUMP(zstd_stat_alloc_fail);
|
|
} else {
|
|
return ((void*)z + (sizeof (struct zstd_kmem)));
|
|
}
|
|
|
|
/* Fallback if everything fails */
|
|
if (!z) {
|
|
/*
|
|
* Barrier since we only can handle it in a single thread. All
|
|
* other following threads need to wait here until decompression
|
|
* is completed. zstd_free will release this barrier later.
|
|
*/
|
|
mutex_enter(&zstd_dctx_fallback.barrier);
|
|
|
|
z = zstd_dctx_fallback.mem;
|
|
type = ZSTD_KMEM_DCTX;
|
|
ZSTDSTAT_BUMP(zstd_stat_alloc_fallback);
|
|
}
|
|
|
|
/* Allocation should always be successful */
|
|
if (!z) {
|
|
return (NULL);
|
|
}
|
|
|
|
z->kmem_type = type;
|
|
z->kmem_size = nbytes;
|
|
|
|
return ((void*)z + (sizeof (struct zstd_kmem)));
|
|
}
|
|
|
|
/* Free allocated memory by its specific type */
|
|
static void
|
|
zstd_free(void *opaque __maybe_unused, void *ptr)
|
|
{
|
|
struct zstd_kmem *z = (ptr - sizeof (struct zstd_kmem));
|
|
enum zstd_kmem_type type;
|
|
|
|
ASSERT3U(z->kmem_type, <, ZSTD_KMEM_COUNT);
|
|
ASSERT3U(z->kmem_type, >, ZSTD_KMEM_UNKNOWN);
|
|
|
|
type = z->kmem_type;
|
|
switch (type) {
|
|
case ZSTD_KMEM_DEFAULT:
|
|
vmem_free(z, z->kmem_size);
|
|
break;
|
|
case ZSTD_KMEM_POOL:
|
|
zstd_mempool_free(z);
|
|
break;
|
|
case ZSTD_KMEM_DCTX:
|
|
mutex_exit(&zstd_dctx_fallback.barrier);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* Allocate fallback memory to ensure safe decompression */
|
|
static void __init
|
|
create_fallback_mem(struct zstd_fallback_mem *mem, size_t size)
|
|
{
|
|
mem->mem_size = size;
|
|
mem->mem = vmem_zalloc(mem->mem_size, KM_SLEEP);
|
|
mutex_init(&mem->barrier, NULL, MUTEX_DEFAULT, NULL);
|
|
}
|
|
|
|
/* Initialize memory pool barrier mutexes */
|
|
static void __init
|
|
zstd_mempool_init(void)
|
|
{
|
|
zstd_mempool_cctx = (struct zstd_pool *)
|
|
kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
|
|
zstd_mempool_dctx = (struct zstd_pool *)
|
|
kmem_zalloc(ZSTD_POOL_MAX * sizeof (struct zstd_pool), KM_SLEEP);
|
|
|
|
for (int i = 0; i < ZSTD_POOL_MAX; i++) {
|
|
mutex_init(&zstd_mempool_cctx[i].barrier, NULL,
|
|
MUTEX_DEFAULT, NULL);
|
|
mutex_init(&zstd_mempool_dctx[i].barrier, NULL,
|
|
MUTEX_DEFAULT, NULL);
|
|
}
|
|
}
|
|
|
|
/* Initialize zstd-related memory handling */
|
|
static int __init
|
|
zstd_meminit(void)
|
|
{
|
|
zstd_mempool_init();
|
|
|
|
/*
|
|
* Estimate the size of the fallback decompression context.
|
|
* The expected size on x64 with current ZSTD should be about 160 KB.
|
|
*/
|
|
create_fallback_mem(&zstd_dctx_fallback,
|
|
P2ROUNDUP(ZSTD_estimateDCtxSize() + sizeof (struct zstd_kmem),
|
|
PAGESIZE));
|
|
|
|
return (0);
|
|
}
|
|
|
|
/* Release object from pool and free memory */
|
|
static void __exit
|
|
release_pool(struct zstd_pool *pool)
|
|
{
|
|
mutex_destroy(&pool->barrier);
|
|
vmem_free(pool->mem, pool->size);
|
|
pool->mem = NULL;
|
|
pool->size = 0;
|
|
}
|
|
|
|
/* Release memory pool objects */
|
|
static void __exit
|
|
zstd_mempool_deinit(void)
|
|
{
|
|
for (int i = 0; i < ZSTD_POOL_MAX; i++) {
|
|
release_pool(&zstd_mempool_cctx[i]);
|
|
release_pool(&zstd_mempool_dctx[i]);
|
|
}
|
|
|
|
kmem_free(zstd_mempool_dctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
|
|
kmem_free(zstd_mempool_cctx, ZSTD_POOL_MAX * sizeof (struct zstd_pool));
|
|
zstd_mempool_dctx = NULL;
|
|
zstd_mempool_cctx = NULL;
|
|
}
|
|
|
|
/* release unused memory from pool */
|
|
|
|
void
|
|
zfs_zstd_cache_reap_now(void)
|
|
{
|
|
/*
|
|
* calling alloc with zero size seeks
|
|
* and releases old unused objects
|
|
*/
|
|
zstd_mempool_alloc(zstd_mempool_cctx, 0);
|
|
zstd_mempool_alloc(zstd_mempool_dctx, 0);
|
|
}
|
|
|
|
extern int __init
|
|
zstd_init(void)
|
|
{
|
|
/* Set pool size by using maximum sane thread count * 4 */
|
|
pool_count = (boot_ncpus * 4);
|
|
zstd_meminit();
|
|
|
|
/* Initialize kstat */
|
|
zstd_ksp = kstat_create("zfs", 0, "zstd", "misc",
|
|
KSTAT_TYPE_NAMED, sizeof (zstd_stats) / sizeof (kstat_named_t),
|
|
KSTAT_FLAG_VIRTUAL);
|
|
if (zstd_ksp != NULL) {
|
|
zstd_ksp->ks_data = &zstd_stats;
|
|
kstat_install(zstd_ksp);
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
extern void __exit
|
|
zstd_fini(void)
|
|
{
|
|
/* Deinitialize kstat */
|
|
if (zstd_ksp != NULL) {
|
|
kstat_delete(zstd_ksp);
|
|
zstd_ksp = NULL;
|
|
}
|
|
|
|
/* Release fallback memory */
|
|
vmem_free(zstd_dctx_fallback.mem, zstd_dctx_fallback.mem_size);
|
|
mutex_destroy(&zstd_dctx_fallback.barrier);
|
|
|
|
/* Deinit memory pool */
|
|
zstd_mempool_deinit();
|
|
}
|
|
|
|
#if defined(_KERNEL)
|
|
module_init(zstd_init);
|
|
module_exit(zstd_fini);
|
|
|
|
ZFS_MODULE_DESCRIPTION("ZSTD Compression for ZFS");
|
|
ZFS_MODULE_LICENSE("Dual BSD/GPL");
|
|
ZFS_MODULE_VERSION(ZSTD_VERSION_STRING);
|
|
|
|
EXPORT_SYMBOL(zfs_zstd_compress);
|
|
EXPORT_SYMBOL(zfs_zstd_decompress_level);
|
|
EXPORT_SYMBOL(zfs_zstd_decompress);
|
|
EXPORT_SYMBOL(zfs_zstd_cache_reap_now);
|
|
#endif
|