d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove duplicate lz4 implementations

Browse Source 
Port illumos change: https://www.illumos.org/issues/11667

Move lz4.c out of zfs tree to opensolaris/common/lz4, adjust it to be
usable from kernel/stand/userland builds, so we can use just one single
source. Add lz4.h to declare lz4_compress() and lz4_decompress().

MFC after:	1 month
Differential Revision:	https://reviews.freebsd.org/D22037
This commit is contained in:
Toomas Soome 2019-11-02 12:28:04 +00:00
parent 0060947db1
commit e499793e76
11 changed files with 125 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cddl/lib/libzpool/Makefile
View File

@ -8,6 +8,8 @@
.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lua
# ZFS_SHARED_SRCS
.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/zfs
# LZ4_COMMON_SRCS
.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
# KERNEL_SRCS
.PATH: ${SRCTOP}/cddl/contrib/opensolaris/lib/libzpool/common
# LIST_SRCS
@ -32,6 +34,7 @@ LIB= zpool
ZFS_COMMON_SRCS= ${ZFS_COMMON_OBJS:C/.o$/.c/} trim_map.c
ZFS_SHARED_SRCS= ${ZFS_SHARED_OBJS:C/.o$/.c/}
LZ4_COMMON_SRCS= lz4.c
LUA_SRCS= ${LUA_OBJS:C/.o$/.c/}
KERNEL_SRCS= kernel.c taskq.c util.c
LIST_SRCS= list.c
@ -39,7 +42,7 @@ UNICODE_SRCS= u8_textprep.c
LIBCMDUTILS_SRCS=nicenum.c
SRCS= ${ZFS_COMMON_SRCS} ${ZFS_SHARED_SRCS} ${LUA_SRCS} \
${KERNEL_SRCS} ${LIST_SRCS} ${ATOMIC_SRCS} \
${LZ4_COMMON_SRCS} ${KERNEL_SRCS} ${LIST_SRCS} ${ATOMIC_SRCS} \
${UNICODE_SRCS} ${LIBCMDUTILS_SRCS}
WARNS?= 0
@ -50,6 +53,7 @@ CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libzpool/common
CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs
CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lua
CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/zfs
CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
CFLAGS+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/uts/common
CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/head
CFLAGS+= -I${SRCTOP}/cddl/contrib/opensolaris/lib/libnvpair

1
stand/efi/boot1/Makefile
View File

@ -37,6 +37,7 @@ CFLAGS.zfs_module.c+= -I${ZFSSRC}
CFLAGS.zfs_module.c+= -I${SYSDIR}/cddl/boot/zfs
CFLAGS.zfs_module.c+= -I${SYSDIR}/crypto/skein
CFLAGS.zfs_module.c+= -I${SYSDIR}/cddl/contrib/opensolaris/uts/common
CFLAGS.zfs_module.c+= -I${SYSDIR}/cddl/contrib/opensolaris/common/lz4
CFLAGS+= -DEFI_ZFS_BOOT
.endif

1
stand/i386/gptzfsboot/Makefile
View File

@ -29,6 +29,7 @@ CFLAGS+=-DBOOTPROG=\"gptzfsboot\" \
-I${SYSDIR}/crypto/skein \
-I${SYSDIR}/cddl/boot/zfs \
-I${SYSDIR}/cddl/contrib/opensolaris/uts/common \
-I${SYSDIR}/cddl/contrib/opensolaris/common/lz4 \
-I${BOOTSRC}/i386/btx/lib \
-I${BOOTSRC}/i386/boot2 \
-Wall -Waggregate-return -Wbad-function-cast \

1
stand/i386/zfsboot/Makefile
View File

@ -28,6 +28,7 @@ CFLAGS+=-DBOOTPROG=\"zfsboot\" \
-I${SYSDIR}/crypto/skein \
-I${SYSDIR}/cddl/boot/zfs \
-I${SYSDIR}/cddl/contrib/opensolaris/uts/common \
-I${SYSDIR}/cddl/contrib/opensolaris/common/lz4 \
-I${BOOTSRC}/i386/boot2 \
-Wall -Waggregate-return -Wbad-function-cast -Wno-cast-align \
-Wmissing-declarations -Wmissing-prototypes -Wnested-externs \

5
stand/libsa/Makefile
View File

@ -102,6 +102,11 @@ CFLAGS+=-DHAVE_MEMCPY -I${SRCTOP}/sys/contrib/zlib
SRCS+= adler32.c crc32.c
SRCS+= infback.c inffast.c inflate.c inftrees.c zutil.c
# lz4 decompression functionality
.PATH: ${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
SRCS+= lz4.c
CFLAGS.lz4.c+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
# Create a subset of includes that are safe, as well as adjusting those that aren't
# The lists may drive people nuts, but they are explicitly opt-in
FAKE_DIRS=xlocale arpa

2
stand/libsa/zfs/Makefile.inc
View File

@ -12,4 +12,6 @@ CFLAGS+= -I${SYSDIR}/cddl/boot/zfs
CFLAGS+= -I${SYSDIR}/cddl/contrib/opensolaris/uts/common
CFLAGS+= -I${SYSDIR}/crypto/skein
CFLAGS.zfs.c+= -I${SRCTOP}/sys/cddl/contrib/opensolaris/common/lz4
CFLAGS+= -Wformat -Wall

3
sys/cddl/boot/zfs/zfssubr.c
View File

@ -26,6 +26,8 @@
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <lz4.h>
static uint64_t zfs_crc64_table[256];
#define ECKSUM 666
@ -161,7 +163,6 @@ typedef struct zio_compress_info {
#include "lzjb.c"
#include "zle.c"
#include "lz4.c"
/*
* Compression vectors.

90
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/lz4.c → sys/cddl/contrib/opensolaris/common/lz4/lz4.c
View File

@ -35,11 +35,29 @@
* Copyright (c) 2016 by Delphix. All rights reserved.
*/
#if defined(_KERNEL)
#include <sys/zfs_context.h>
#elif defined(_STANDALONE)
#include <sys/cdefs.h>
#include <stand.h>
#include <sys/types.h>
#include <sys/endian.h>
#include <assert.h>
#define ASSERT assert
#else
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <assert.h>
#define ASSERT assert
#endif
#include <lz4.h>
static int real_LZ4_compress(const char *source, char *dest, int isize,
int osize);
static int LZ4_compressBound(int isize);
static int LZ4_uncompress_unknownOutputSize(const char *source, char *dest,
int isize, int maxOutputSize);
static int LZ4_compressCtx(void *ctx, const char *source, char *dest,
@ -47,11 +65,13 @@ static int LZ4_compressCtx(void *ctx, const char *source, char *dest,
static int LZ4_compress64kCtx(void *ctx, const char *source, char *dest,
int isize, int osize);
#if defined(_KERNEL)
static kmem_cache_t *lz4_ctx_cache;
#endif
/*ARGSUSED*/
size_t
lz4_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
lz4_compress(void *s_start, void *d_start, size_t s_len, size_t d_len,
int n __unused)
{
uint32_t bufsiz;
char *dest = d_start;
@ -71,17 +91,25 @@ lz4_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
* added to the compressed buffer and which, if unhandled, would
* confuse the hell out of our decompression function.
*/
*(uint32_t *)dest = BE_32(bufsiz);
#if defined(_KERNEL)
*(uint32_t *)(void *)dest = BE_32(bufsiz);
#else
*(uint32_t *)(void *)dest = htonl(bufsiz);
#endif
return (bufsiz + sizeof (bufsiz));
}
/*ARGSUSED*/
int
lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len,
int n __unused)
{
const char *src = s_start;
uint32_t bufsiz = BE_IN32(src);
#if defined(_KERNEL)
uint32_t bufsiz = BE_IN32(s_start);
#else
uint32_t bufsiz = htonl(*(uint32_t *)s_start);
#endif
/* invalid compressed buffer size encoded at start */
if (bufsiz + sizeof (bufsiz) > s_len)
@ -105,20 +133,10 @@ lz4_decompress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n)
* or 0 if the compression fails (if LZ4_COMPRESSMIN is set).
* note : destination buffer must be already allocated.
* destination buffer must be sized to handle worst cases
* situations (input data not compressible) worst case size
* evaluation is provided by function LZ4_compressBound().
* situations (input data not compressible).
*
* Advanced Functions
*
* LZ4_compressBound() :
* Provides the maximum size that LZ4 may output in a "worst case"
* scenario (input data not compressible) primarily useful for memory
* allocation of output buffer.
*
* isize : is the input size. Max supported value is ~1.9GB
* return : maximum output size in a "worst case" scenario
* note : this function is limited by "int" range (2^31-1)
*
* LZ4_uncompress_unknownOutputSize() :
* isize : is the input size, therefore the compressed size
* maxOutputSize : is the size of the destination buffer (which must be
@ -292,9 +310,9 @@ typedef struct _U64_S {
#pragma pack()
#endif
#define A64(x) (((U64_S *)(x))->v)
#define A32(x) (((U32_S *)(x))->v)
#define A16(x) (((U16_S *)(x))->v)
#define A64(x) (((U64_S *)(__DECONST(void *, x)))->v)
#define A32(x) (((U32_S *)(__DECONST(void *, x)))->v)
#define A16(x) (((U16_S *)(__DECONST(void *, x)))->v)
/*
* Constants
@ -456,14 +474,6 @@ LZ4_NbCommonBytes(register U32 val)
#endif
/* Public functions */
static int
LZ4_compressBound(int isize)
{
return (isize + (isize / 255) + 16);
}
/* Compression functions */
/*ARGSUSED*/
@ -478,7 +488,7 @@ LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize,
HTYPE HashTable[HASHTABLESIZE] = { 0 };
#endif
const BYTE *ip = (BYTE *) source;
const BYTE *ip = (const BYTE *) source;
INITBASE(base);
const BYTE *anchor = ip;
const BYTE *const iend = ip + isize;
@ -527,7 +537,7 @@ LZ4_compressCtx(void *ctx, const char *source, char *dest, int isize,
} while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip)));
/* Catch up */
while ((ip > anchor) && (ref > (BYTE *) source) &&
while ((ip > anchor) && (ref > (const BYTE *) source) &&
unlikely(ip[-1] == ref[-1])) {
ip--;
ref--;
@ -673,7 +683,7 @@ LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize,
U16 HashTable[HASH64KTABLESIZE] = { 0 };
#endif
const BYTE *ip = (BYTE *) source;
const BYTE *ip = (const BYTE *) source;
const BYTE *anchor = ip;
const BYTE *const base = ip;
const BYTE *const iend = ip + isize;
@ -720,7 +730,7 @@ LZ4_compress64kCtx(void *ctx, const char *source, char *dest, int isize,
} while (A32(ref) != A32(ip));
/* Catch up */
while ((ip > anchor) && (ref > (BYTE *) source) &&
while ((ip > anchor) && (ref > (const BYTE *) source) &&
(ip[-1] == ref[-1])) {
ip--;
ref--;
@ -847,7 +857,11 @@ static int
real_LZ4_compress(const char *source, char *dest, int isize, int osize)
{
#if HEAPMODE
#if defined(_KERNEL)
void *ctx = kmem_cache_alloc(lz4_ctx_cache, KM_NOSLEEP);
#else
void *ctx = malloc(sizeof(struct refTables));
#endif
int result;
/*
@ -863,7 +877,11 @@ real_LZ4_compress(const char *source, char *dest, int isize, int osize)
else
result = LZ4_compressCtx(ctx, source, dest, isize, osize);
#if defined(_KERNEL)
kmem_cache_free(lz4_ctx_cache, ctx);
#else
free(ctx);
#endif
return (result);
#else
if (isize < (int)LZ4_64KLIMIT)
@ -876,7 +894,7 @@ real_LZ4_compress(const char *source, char *dest, int isize, int osize)
/*
* Note: The decoding function LZ4_uncompress_unknownOutputSize() is safe
* against "buffer overflow" attack type. They will never write nor
* against "buffer overflow" attack type. It will never write nor
* read outside of the provided output buffers.
* LZ4_uncompress_unknownOutputSize() also insures that it will never
* read outside of the input buffer. A corrupted input will produce
@ -1010,9 +1028,10 @@ LZ4_uncompress_unknownOutputSize(const char *source, char *dest, int isize,
/* write overflow error detected */
_output_error:
return (int)(-(((char *)ip) - source));
return (int)(-(((const char *)ip) - source));
}
#if defined(_KERNEL)
extern void
lz4_init(void)
{
@ -1031,3 +1050,4 @@ lz4_fini(void)
kmem_cache_destroy(lz4_ctx_cache);
#endif
}
#endif /* _KERNEL */

50
sys/cddl/contrib/opensolaris/common/lz4/lz4.h Normal file
View File

@ -0,0 +1,50 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at :
* - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
* - LZ4 source repository : http://code.google.com/p/lz4/
*/
#ifndef _LZ4_H
#define _LZ4_H
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
extern size_t lz4_compress(void *, void *, size_t, size_t, int);
extern int lz4_decompress(void *, void *, size_t, size_t, int);
#ifdef __cplusplus
}
#endif
#endif /* _LZ4_H */

1
sys/cddl/contrib/opensolaris/uts/common/Makefile.files
View File

@ -98,7 +98,6 @@ ZFS_COMMON_OBJS += \
dsl_scan.o \
zfeature.o \
gzip.o \
lz4.o \
lzjb.o \
metaslab.o \
multilist.o \

3
sys/modules/zfs/Makefile
View File

@ -18,6 +18,8 @@ SRCS+= opensolaris_nvpair_alloc_fixed.c
SRCS+= opensolaris_fnvpair.c
.PATH: ${SYSDIR}/cddl/contrib/opensolaris/common/unicode
SRCS+= u8_textprep.c
.PATH: ${SUNW}/common/lz4
SRCS+= lz4.c
.PATH: ${SYSDIR}/cddl/compat/opensolaris/kern
SRCS+= opensolaris_acl.c
@ -84,6 +86,7 @@ CFLAGS+=-I${SUNW}/uts/common/zmod
CFLAGS+=-I${SUNW}/uts/common
CFLAGS+=-I${SYSDIR}
CFLAGS+=-I${SUNW}/common/zfs
CFLAGS+=-I${SUNW}/common/lz4
CFLAGS+=-I${SUNW}/common
CFLAGS+=-DBUILDING_ZFS
CFLAGS.gcc+=-fms-extensions