Enable hdfs, avro and rapidJson/simdJson (#123)

* feat: enable support hdfs and avro stroage

* disable: support hdfs on macos compile option

* enable: run workflow on pull request open|reopen

* Fix minor duplicate flags

* Add rapidjson and simdjson

* Add json smoke test

---------

Co-authored-by: auxten <[email protected]>
Co-authored-by: auxten <[email protected]>

Author: nmreadelf

.github/workflows/build_wheels.yml

@@ -8,6 +8,8 @@ on:
         required: true
   release:
     types: [created]
+  pull_request:
+    types: [opened, reopened]
 
 jobs:
   build_wheels_linux:

chdb/build.sh

@@ -8,27 +8,29 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 BUILD_DIR=${PROJ_DIR}/buildlib
 
+HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1"
 # check current os type
 if [ "$(uname)" == "Darwin" ]; then
     GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0"
     UNWIND="-DUSE_UNWIND=0"
     JEMALLOC="-DENABLE_JEMALLOC=0"
     PYINIT_ENTRY="-Wl,-exported_symbol,_PyInit_${CHDB_PY_MOD}"
+    HDFS="-DENABLE_HDFS=0 -DENABLE_GSASL_LIBRARY=0 -DENABLE_KRB5=0"
     # if Darwin ARM64 (M1, M2), disable AVX
     if [ "$(uname -m)" == "arm64" ]; then
         CMAKE_TOOLCHAIN_FILE="-DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-aarch64.cmake"
-        CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+        CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DENABLE_SIMDJSON=0"
         EMBEDDED_COMPILER="-DENABLE_EMBEDDED_COMPILER=0"
         export CXX=/usr/local/opt/llvm/bin/clang++
         export CC=/usr/local/opt/llvm/bin/clang
     else
         EMBEDDED_COMPILER="-DENABLE_EMBEDDED_COMPILER=1"
         # disable AVX on Darwin for macos11
         if [ "$(sw_vers -productVersion | cut -d. -f1)" -le 11 ]; then
-            CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+            CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DENABLE_SIMDJSON=0"
         else
             # for M1, M2 using x86_64 emulation, we need to disable AVX and AVX2
-            CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+            CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DENABLE_SIMDJSON=0"
             # # If target macos version is 12, we need to test if support AVX2, 
             # # because some Mac Pro Late 2013 (MacPro6,1) support AVX but not AVX2
             # # just test it on the github action, hope you don't using Mac Pro Late 2013.
@@ -47,7 +49,7 @@ elif [ "$(uname)" == "Linux" ]; then
     PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}"
     # only x86_64, enable AVX and AVX2, enable embedded compiler
     if [ "$(uname -m)" == "x86_64" ]; then
-        CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=1"
+        CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=1 -DENABLE_SIMDJSON=1"
         EMBEDDED_COMPILER="-DENABLE_EMBEDDED_COMPILER=1"
     else
         CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=1"
@@ -66,17 +68,19 @@ cd ${BUILD_DIR}
 cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_SERVER=0 -DENABLE_CLICKHOUSE_CLIENT=0 \
     -DENABLE_CLICKHOUSE_KEEPER=0 -DENABLE_CLICKHOUSE_KEEPER_CONVERTER=0 -DENABLE_CLICKHOUSE_LOCAL=1 -DENABLE_CLICKHOUSE_SU=0 -DENABLE_CLICKHOUSE_BENCHMARK=0 \
     -DENABLE_AZURE_BLOB_STORAGE=0 -DENABLE_CLICKHOUSE_COPIER=0 -DENABLE_CLICKHOUSE_DISKS=0 -DENABLE_CLICKHOUSE_FORMAT=0 -DENABLE_CLICKHOUSE_GIT_IMPORT=0 \
-    -DENABLE_AWS_S3=1 -DENABLE_HDFS=0 -DENABLE_HIVE=0 \
+    -DENABLE_AWS_S3=1 -DENABLE_HIVE=0 -DENABLE_AVRO=1 \
     -DENABLE_CLICKHOUSE_OBFUSCATOR=0 -DENABLE_CLICKHOUSE_ODBC_BRIDGE=0 -DENABLE_ODBC=0 -DENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER=0 \
     -DENABLE_KAFKA=0 -DENABLE_MYSQL=0 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \
     -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \
-    -DENABLE_KRB5=0 -DENABLE_LDAP=0 \
+    -DENABLE_LDAP=0 \
+    ${HDFS} \
     -DENABLE_LIBRARIES=0 -DENABLE_RUST=0 \
     ${GLIBC_COMPATIBILITY} \
     -DENABLE_UTILS=0 ${EMBEDDED_COMPILER} ${UNWIND} \
     -DENABLE_ICU=0 ${JEMALLOC} \
     -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \
     -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 \
+    -DENABLE_RAPIDJSON=1 \
     -DENABLE_BROTLI=1 \
     -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \
     ${CPU_FEATURES} \

chdb/test_smoke.sh

@@ -24,5 +24,9 @@ python3 -c \
 python3 -c \
     "import chdb; res = chdb.query('select version()', 'CSV'); print(res.bytes())"
 
+# test json function
+python3 -c \
+    "import chdb; res = chdb.query('select isValidJSON(\'not a json\')', 'CSV'); print(res)"
+
 # test cli
 python3 -m chdb "select 1112222222,555" Dataframe