Default to r2's own lz4 implementation - remove smallz4 ##util

Author: MewtR

Diff for: config-user.mk.acr

@@ -111,7 +111,6 @@ LIBZIP=@LIBZIP@
 LIBXXHASH=@LIBXXHASH@
 
 USE_SYSLZ4=@USE_SYSLZ4@
-USE_SMALLZ4=@USE_SMALLZ4@
 
 ifeq ($(USE_SYSLZ4),1)
 LZ4_LDFLAGS=@LZ4_LDFLAGS@

Diff for: configure

@@ -1,5 +1,5 @@
 #!/bin/sh
-# This script was automatically generated by ACR v2.1.4
+# This script was automatically generated by ACR v2.2.0
 # @author: pancake <nopcode.org>
 # @url:    http://www.nopcode.org
 # @repo:   git clone https://github.com/radare/acr
@@ -195,8 +195,7 @@ System types:
   --target=TARGET        configure for building compilers for TARGET [HOST]
 EOF2
 
-printf "
-Optional Features:
+printf "\nOptional Features:
   --disable-debugger     disable native debugger features
   --with-libatomic       force to use libatomic
   --with-sysmagic        force to use system's magic
@@ -232,20 +231,16 @@ Optional Features:
   --with-libversion      specify different libversion (LIBVERSION=xxx)
   --without-jemalloc     build without jemalloc
   --with-checks-level    value between 0 and 3 to enable different level of assert (see R_CHECKS_LEVEL) (R_CHECKS_LEVEL=2)
-  --disable-debug-stuff  disable the log messages and disable control-c
-"
-printf "
-Some influential environment variables:
+  --disable-debug-stuff  disable the log messages and disable control-c\n"
+printf "\nSome influential environment variables:
   CC          C compiler command
   CFLAGS      C compiler flags
   LDFLAGS     linker flags, e.g. -L<lib dir> if you have libraries in a
               nonstandard directory <lib dir>
   CPPFLAGS    C/C++ preprocessor flags, e.g. -I<include dir> if you have
               headers in a nonstandard directory <include dir>
-  CPP         C preprocessor
-"
-printf "
-Report bugs to: pancake <[email protected]>"
+  CPP         C preprocessor\n"
+printf "\nReport bugs to: pancake <[email protected]>"
 echo ""
 exit 0
 }
@@ -272,7 +267,7 @@ if [ "$QUIET" = 1 ]; then
 	echo "5.9.9"
 	exit 0
 fi
-echo "radare2-5.9.9 configuration script done with acr v2.1.4.
+echo "radare2-5.9.9 configuration script done with acr v2.2.0.
 The 'Free Software Foundation' message is only for autodetection.
 Originally written by pancake <nopcode.org>."
 exit 0
@@ -1052,7 +1047,7 @@ do_remove
 if [ "$QUIET" = 0 ]; then
 echo
 echo "Final report:"
-for A in  BUILD CC CFLAGS DEBUGGER HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO USE_NEW_ABI HAVE_EXPLICIT_MEMSET HAVE_FORK HAVE_GPERF HAVE_LIBUV HAVE_LIB_GMP WANT_DEBUGSTUFF HAVE_OPENSSL WANT_SSL_CRYPTO HAVE_PTRACE HOST LDFLAGS LIBVERSION PKGCONFIG PREFIX R_CHECKS_LEVEL TARGET USERCC USEROSTYPE USE_CAPSTONE USE_LIB_MAGIC NEW_IO_CACHE USE_SQSH USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK USE_SMALLZ4 USE_LIB_ATOMIC HAVE___ATOMIC_FETCH_ADD_8 FORCE_LIB_ATOMIC WANT_QJS ; do
+for A in  BUILD CC CFLAGS DEBUGGER HAVE_ARC4RANDOM_UNIFORM HAVE_EXPLICIT_BZERO USE_NEW_ABI HAVE_EXPLICIT_MEMSET HAVE_FORK HAVE_GPERF HAVE_LIBUV HAVE_LIB_GMP WANT_DEBUGSTUFF HAVE_OPENSSL WANT_SSL_CRYPTO HAVE_PTRACE HOST LDFLAGS LIBVERSION PKGCONFIG PREFIX R_CHECKS_LEVEL TARGET USERCC USEROSTYPE USE_CAPSTONE USE_LIB_MAGIC NEW_IO_CACHE USE_SQSH USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK USE_LIB_ATOMIC HAVE___ATOMIC_FETCH_ADD_8 FORCE_LIB_ATOMIC WANT_QJS ; do
 eval VAL="\$${A}"
 [ -z "${VAL}" ] && VAL="\"\""
 echo " - ${A} = ${VAL}"

Diff for: configure.acr

@@ -336,7 +336,7 @@ REPORT
 	HAVE_EXPLICIT_MEMSET HAVE_FORK HAVE_GPERF HAVE_LIBUV HAVE_LIB_GMP WANT_DEBUGSTUFF
 	HAVE_OPENSSL WANT_SSL_CRYPTO HAVE_PTRACE HOST LDFLAGS LIBVERSION PKGCONFIG PREFIX
 	R_CHECKS_LEVEL TARGET USERCC USEROSTYPE USE_CAPSTONE USE_LIB_MAGIC NEW_IO_CACHE USE_SQSH
-	USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK USE_SMALLZ4
+	USE_LIB_XXHASH USE_LIB_ZIP USE_PTRACE_WRAP USE_SYSLZ4 VERSION WANT_DYLINK
 	USE_LIB_ATOMIC HAVE___ATOMIC_FETCH_ADD_8 FORCE_LIB_ATOMIC WANT_QJS
 ;
 

Diff for: libr/bin/meson.build

@@ -156,8 +156,9 @@ r_bin_sources += r_bin_d_sources
 r_bin_inc = [platform_inc, include_directories('mangling', 'format')]
 
 if not get_option('use_sys_lz4')
-  library_cflags += ['-DUSE_SMALLZ4']
+  library_cflags += ['-DUSE_RLZ4']
 endif
+
 r_bin = library('r_bin', r_bin_sources,
   include_directories: r_bin_inc,
   c_args: ['-DR_API_BIN_ONLY=1'] + library_cflags,

Diff for: libr/include/r_util.h

@@ -91,6 +91,7 @@ int gettimeofday (struct timeval* p, void* tz);
 
 R_API int r_lz4_compress(ut8 *obuf, ut8 *buf, size_t buf_size, const int max_chain);
 R_API ut8 *r_lz4_decompress(const ut8* input, size_t input_size, size_t *output_size);
+R_API int r_lz4_decompress_block(ut8 *g_buf, const int comp_len, int *pp, ut8 *obuf, int osz);
 
 #ifdef __cplusplus
 extern "C" {

Diff for: libr/util/Makefile

@@ -44,8 +44,7 @@ CFLAGS+=${LZ4_CFLAGS}
 PCLIBS+=@LZ4_LDFLAGS@
 LDFLAGS+=${LZ4_LDFLAGS}
 else
-CFLAGS+=-DUSE_SMALLZ4
-OBJS+=$(SHLR)/smallz4/smallz4cat.o
+CFLAGS+=-DUSE_RLZ4
 endif
 
 OBJS+=big_gmp.o

Diff for: libr/util/meson.build

@@ -110,8 +110,9 @@ if get_option('sdb_cgen')
   r_util_sources += r_util_d_sources
 endif
 
+lz4_dep = dependency('liblz4', required: false)
 if not get_option('use_sys_lz4')
-  library_cflags += ['-DUSE_SMALLZ4']
+  library_cflags += ['-DUSE_RLZ4']
 endif
 
 r_util_deps = [ldl, mth, spp_dep, pth, utl, sdb_dep, lz4_dep, zlib_dep, platform_deps]

Diff for: libr/util/rlz4.c

@@ -4,8 +4,6 @@
 
 #include <r_util.h>
 
-// R2_600 - replace shlr/smallz4 with this code which also supports compressing
-
 #define BLOCK_SIZE (1024 * 8) /* 8K */
 #define PADDING_LITERALS 5
 #define WINDOW_BITS 10
@@ -18,7 +16,9 @@
 
 #define LOAD_16(p) *(ut16*)&g_buf[(p)]
 #define LOAD_32(p) *(ut32*)&g_buf[(p)]
+#define LOAD_32_FROM(p, x)     *(ut32 *)&x[(p)]
 #define COPY_32(d, s) *(ut32*)&g_buf[(d)] = LOAD_32((s))
+#define COPY_32_TO(d, s, x, y) *(ut32 *)&x[(d)] = LOAD_32_FROM (s, y)
 
 #define HASH_BITS 12
 #define HASH_SIZE (1 << HASH_BITS)
@@ -142,10 +142,12 @@ static int lz4_compress(ut8 *g_buf, const int uc_length, int max_chain) {
 	return op - BLOCK_SIZE;
 }
 
-static int lz4_decompress(ut8 *g_buf, const int comp_len, int *pp) {
+R_API int r_lz4_decompress_block(ut8 *g_buf, const int comp_len, int *pp, ut8 *obuf, int osz) {
 	int i, s, len, run, p = 0;
-	int ip = BLOCK_SIZE;
+	int ip = obuf? 0: BLOCK_SIZE;
+	int maxLen = obuf? osz: BLOCK_SIZE;
 	int ip_end = ip + comp_len;
+	ut8 *dst = obuf? obuf: g_buf;
 
 	for (;;) {
 		const int token = g_buf[ip++];
@@ -160,15 +162,15 @@ static int lz4_decompress(ut8 *g_buf, const int comp_len, int *pp) {
 					}
 				}
 			}
-			if ((p + run) > BLOCK_SIZE) {
+			if ((p + run) > maxLen) {
 				return -1;
 			}
 
-			COPY_32 (p, ip);
-			COPY_32 (p + 4, ip + 4);
+			COPY_32_TO (p, ip, dst, g_buf);
+			COPY_32_TO (p + 4, ip + 4, dst, g_buf);
 			for (i = 8; i < run; i += 8) {
-				COPY_32 (p + i, ip + i);
-				COPY_32 (p + 4 + i, ip + 4 + i);
+				COPY_32_TO (p + i, ip + i, dst, g_buf);
+				COPY_32_TO (p + 4 + i, ip + 4 + i, dst, g_buf);
 			}
 			p += run;
 			ip += run;
@@ -192,20 +194,20 @@ static int lz4_decompress(ut8 *g_buf, const int comp_len, int *pp) {
 				}
 			}
 		}
-		if ((p + len) > BLOCK_SIZE) {
+		if ((p + len) > maxLen) {
 			return -1;
 		}
 		if ((p - s) >= 4) {
-			COPY_32 (p, s);
-			COPY_32 (p + 4, s + 4);
+			COPY_32_TO (p, s, dst, dst);
+			COPY_32_TO (p + 4, s + 4, dst, dst);
 			for (i = 8; i < len; i += 8) {
-				COPY_32 (p + i, s + i);
-				COPY_32 (p + 4 + i, s + 4 + i);
+				COPY_32_TO (p + i, s + i, dst, dst);
+				COPY_32_TO (p + 4 + i, s + 4 + i, dst, dst);
 			}
 			p += len;
 		} else {
 			while (len-- != 0) {
-				g_buf[p++] = g_buf[s++];
+				dst[p++] = dst[s++];
 			}
 		}
 	}
@@ -217,24 +219,58 @@ R_API ut8 *r_lz4_decompress(const ut8* input, size_t input_size, size_t *output_
 	R_RETURN_VAL_IF_FAIL (input && output_size, NULL);
 	RBuffer *b = r_buf_new ();
 	ut8 g_buf[(BLOCK_SIZE + BLOCK_SIZE + EXCESS) * sizeof (ut8)];
+	bool is_legacy = true;
+	bool has_block_checksum = false, has_content_size = false, has_dictionary_id = false, has_content_checksum = false;
 	const ut8 *input_last = input + input_size;
-	while (input + 4 < input_last) {
-		if (!memcmp (input, "\x02\x21\x4c\x18", 4)) {
+
+	// Process the lz4 header
+	if (!memcmp (input, "\x02\x21\x4c\x18", 4)) {
+		input += 4;
+	} else if (!memcmp (input, "\x04\x22\x4d\x18", 4)) {
+		is_legacy = false;
+		input += 4;
+		ut8 flag = r_read_le8 (input);
+		input += 2; // skip BD byte
+		has_block_checksum = flag & 16;
+		has_content_size = flag & 8;
+		has_content_checksum = flag & 4;
+		has_dictionary_id = flag & 1;
+		if (has_content_size) {
+			input += 8;
+		}
+		if (has_dictionary_id) {
 			input += 4;
-			continue;
 		}
+		input += 1; // skip header checksum
+	}
+
+	const ut8 bytes_at_end_to_skip = has_content_checksum? 8: 4;
+	while (input + bytes_at_end_to_skip < input_last) {
 		ut32 comp_len = r_read_le32 (input);
+		bool is_compressed = is_legacy || (comp_len & 0x80000000) == 0;
+		if (!is_legacy) {
+			comp_len &= 0x7FFFFFFF;
+		}
 		input += 4;
 		int p;
 		memcpy (g_buf + BLOCK_SIZE, input, comp_len);
-		int error = lz4_decompress (g_buf, comp_len, &p);
-		if (error != 0) {
-			r_buf_free (b);
-			return NULL;
+		if (is_compressed) {
+			int error = r_lz4_decompress_block (g_buf, comp_len, &p, NULL, 0);
+			if (error != 0) {
+				r_buf_free (b);
+				return NULL;
+			}
+		} else {
+			p = comp_len;
 		}
 		r_buf_write (b, g_buf, p);
 		input += comp_len;
+
+		if (has_block_checksum) {
+			input += 4;
+		}
 	}
+
 	ut64 osz;
 	ut8 *res = r_buf_drain (b, &osz);
 	if (output_size) {

Diff for: libr/util/zip.c

@@ -6,35 +6,11 @@
 // set a maximum output buffer of 50MB
 #define MAXOUT 50000000
 
-#if USE_SMALLZ4
-#include "../../../shlr/smallz4/smallz4cat.h"
-
-struct UserPtr {
-	const ut8 * input;
-	ut64 inputPos;
-	ut8 *output;
-	ut64 outputPos;
-	ut32 *outputSize;
-	int error;
-};
-
-static void smallz4Write(const unsigned char* data, unsigned int numBytes, void *userPtr) {
-	struct UserPtr* user = (struct UserPtr*)userPtr;
-	if (data != NULL && numBytes > 0) {
-		if (*(user->outputSize) - user->outputPos < numBytes) {
-			user->error = -1;
-			return;
-		}
-		memcpy(user->output + user->outputPos, data, numBytes);
-		user->outputPos += numBytes;
-	}
-}
+#ifndef USE_RLZ4
+#define USE_RLZ4 0
+#endif
 
-static ut8 smallz4GetByte(void *userPtr) {
-	struct UserPtr* user = (struct UserPtr*)userPtr;
-	return *(user->input + (user->inputPos++));
-}
-#else
+#if !USE_RLZ4
 #include <lz4.h>
 #endif
 
@@ -116,28 +92,22 @@ static ut8 *inflatew(const ut8 *src, int srcLen, int *consumed, int *dstLen, int
 R_API ut8 *r_inflate_lz4(const ut8 *src, int srcLen, R_NULLABLE int *consumed, int *dstLen) {
 	R_RETURN_VAL_IF_FAIL (src && dstLen, NULL);
 	ut32 osz = srcLen * 5;
+	int pp = 0;
+
 	ut8 *obuf = calloc (srcLen, 5);
 	if (!obuf) {
 		return NULL;
 	}
 
-#if USE_SMALLZ4
-	struct UserPtr user = {
-		.input = src,
-		.inputPos = 0,
-		.output = obuf,
-		.outputPos = 0,
-		.outputSize = &osz,
-		.error = 0
-	};
-	int res = unlz4Block_userPtr (smallz4GetByte, smallz4Write, &user, srcLen, NULL, NULL);
-	if (res < 1 || user.error != 0)
+#if USE_RLZ4
+	int res = r_lz4_decompress_block ((ut8 *)src, srcLen, &pp, obuf, osz);
+	if (res < 0)
 #else
 	int res = LZ4_decompress_safe ((const char*)src, (char*)obuf, (uint32_t) srcLen, (uint32_t) osz);
 	if (res < 1)
 #endif
 	{
-		const int mul = srcLen / -res;
+		const int mul = USE_RLZ4? 1: srcLen / -res;
 		const int nosz = osz * (5 * (mul + 1));
 		if (nosz < osz) {
 			free (obuf);
@@ -150,24 +120,19 @@ R_API ut8 *r_inflate_lz4(const ut8 *src, int srcLen, R_NULLABLE int *consumed, i
 		}
 		obuf = nbuf;
 		osz = nosz;
-#if USE_SMALLZ4
-		user.output = obuf;
-		user.inputPos = 0;
-		user.outputPos = 0;
-		user.error = 0;
-		res = unlz4Block_userPtr (smallz4GetByte, smallz4Write, &user, srcLen, NULL, NULL);
-	}
-	user.output = NULL;
-	user.input = NULL;
+#if USE_RLZ4
+		res = r_lz4_decompress_block ((ut8 *)src, srcLen, &pp, obuf, osz);
 #else
-	}
-	res = LZ4_decompress_safe ((const char*)src, (char*)obuf, (uint32_t) srcLen, (uint32_t) osz);
+		res = LZ4_decompress_safe ((const char *)src, (char *)obuf, (uint32_t)srcLen, (uint32_t)osz);
 #endif
-	if (res > 0) {
-		*dstLen = res;
+	}
+
+	if (USE_RLZ4? res == 0: res > 0) {
+		*dstLen = USE_RLZ4? pp: res;
 		*consumed = srcLen;
 		return obuf;
 	}
+
 	*dstLen = 0;
 	*consumed = 0;
 	free (obuf);

Diff for: meson.build

@@ -586,7 +586,6 @@ if get_option('local') and get_option('default_library') == 'shared'
   rpath_lib = '$ORIGIN'
 endif
 
-subdir('shlr/smallz4')
 subdir('libr/util')
 subdir('libr/socket')
 subdir('libr/crypto')