SuperKMeans and Full Refactor

.clang-format

@@ -0,0 +1,32 @@
+BasedOnStyle: LLVM
+
+# Indentation
+IndentWidth: 4
+TabWidth: 4
+
+# Braces
+BreakBeforeBraces: Attach
+AllowShortFunctionsOnASingleLine: InlineOnly
+
+ColumnLimit: 100
+
+# Pointer/reference alignment
+PointerAlignment: Left
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignAfterOpenBracket: BlockIndent
+AlwaysBreakTemplateDeclarations: Yes
+
+# Spaces
+SpaceBeforeParens: ControlStatements
+SpaceAfterCStyleCast: true
+SpacesInParentheses: false
+
+
+BinPackParameters: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AlwaysBreakAfterReturnType: None
+PenaltyReturnTypeOnItsOwnLine: 1024
+
+BinPackArguments: false
+AllowAllArgumentsOnNextLine: true

.clang-tidy

@@ -0,0 +1,22 @@
+Checks: >
+  -*,
+  bugprone-*,
+  clang-analyzer-*,
+  cppcoreguidelines-virtual-class-destructor,
+  modernize-pass-by-value,
+  modernize-use-emplace,
+  modernize-use-nullptr,
+  modernize-use-override,
+  modernize-use-using,
+  performance-*,
+  readability-redundant-*,
+  -bugprone-easily-swappable-parameters,
+  -performance-avoid-endl
+
+WarningsAsErrors: ''
+
+CheckOptions:
+  - key: bugprone-narrowing-conversions.WarnOnEquivalentBitWidth
+    value: false
+
+HeaderFilterRegex: 'include/superkmeans/.*'

.github/workflows/ci.yml

@@ -0,0 +1,154 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - '.gitignore'
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'LICENSE'
+      - '.gitignore'
+
+jobs:
+  format-check:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install clang-format 18.1.8
+        run: pip install clang-format==18.1.8
+
+      - name: Check C++ formatting
+        run: |
+          clang-format --version
+          ./scripts/format_check.sh
+
+  tidy-check:
+    runs-on: ubuntu-24.04
+    env:
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang-18 clang-tidy-18 libomp-18-dev libopenblas-dev cmake
+          sudo ln -sf /usr/bin/clang-tidy-18 /usr/local/bin/clang-tidy
+
+      - name: Configure
+        run: cmake -B build -DPDX_COMPILE_TESTS=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+
+      - name: Build
+        run: cmake --build build -j$(nproc)
+
+      - name: Run clang-tidy
+        run: |
+          ln -s build/compile_commands.json compile_commands.json
+          ./scripts/tidy_check.sh
+
+  cpp-build-and-test:
+    runs-on: ubuntu-24.04
+    env:
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang-18 libomp-18-dev libopenblas-dev cmake
+
+      - name: Configure
+        run: cmake -B build -DPDX_COMPILE_TESTS=ON -DCMAKE_BUILD_TYPE=Release
+
+      - name: Build tests
+        run: cmake --build build -j$(nproc) --target tests
+
+      - name: Run tests
+        run: ctest --test-dir build --output-on-failure
+
+  python:
+    runs-on: ubuntu-24.04
+    env:
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang-18 libomp-18-dev libopenblas-dev cmake
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Python bindings
+        run: pip install .
+
+      - name: Verify import
+        run: python -c "import pdxearch; print('pdxearch imported successfully')"
+
+  sanitizers-asan-ubsan:
+    runs-on: ubuntu-24.04
+    env:
+      CC: clang-18
+      CXX: clang++-18
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang-18 libomp-18-dev libopenblas-dev cmake
+
+      - name: Configure with ASan + UBSan
+        run: |
+          cmake -B build_asan -DPDX_COMPILE_TESTS=ON \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer" \
+            -DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer"
+
+      - name: Build tests
+        run: cmake --build build_asan -j$(nproc) --target tests
+
+      - name: Run tests
+        run: ctest --test-dir build_asan --output-on-failure
+
+  ci-pass:
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - format-check
+      - tidy-check
+      - cpp-build-and-test
+      - python
+      - sanitizers-asan-ubsan
+    steps:
+      - name: Check all jobs passed
+        run: |
+          if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more jobs failed or were cancelled"
+            exit 1
+          fi

.gitignore

@@ -42,6 +42,11 @@ venv
 /cmake-build-debug/
 /cmake-build-release/
 /cmake-build*/
+/build_debug/
+/build_release/
+/build_relwithdebinfo/
+/tests/cmake_test_discovery_*.json
+compile_commands.json
 /.idea/
 /dummy/
 /Testing/
@@ -62,10 +67,10 @@ pdxearch.egg-info
 /benchmarks/core_indexes/faiss_l0/*
 !/benchmarks/core_indexes/faiss_l0/*.json
 
-/benchmarks/datasets/adsampling_nary
 /benchmarks/datasets/adsampling_pdx
 /benchmarks/datasets/downloaded
-/benchmarks/datasets/nary
+/benchmarks/datasets/raw
+/benchmarks/datasets/faiss
 /benchmarks/datasets/pdx
 /benchmarks/datasets/purescan
 /benchmarks/datasets/queries
@@ -91,25 +96,12 @@ cmake_install.cmake
 /benchmarks/milvus/volumes/
 /benchmarks/python_scripts/indexes
 
-/benchmarks/BenchmarkNaryIVFADSampling
-/benchmarks/BenchmarkNaryIVFADSamplingSIMD
-/benchmarks/BenchmarkPDXADSampling
-/benchmarks/BenchmarkIVF2ADSampling
-/benchmarks/FilteredBenchmarkPDXADSampling
-/benchmarks/FilteredBenchmarkU8IVF2ADSampling
-/benchmarks/BenchmarkASYM_U8PDXADSampling
-/benchmarks/BenchmarkU8PDXADSampling
-/benchmarks/BenchmarkLEP8PDXADSampling
-/benchmarks/BenchmarkPDXIVFBOND
-/benchmarks/BenchmarkPDXBOND
-/benchmarks/BenchmarkPDXLinearScan
-/benchmarks/BenchmarkU*
-/benchmarks/G4*
-/benchmarks/BenchmarkNaryIVFLinearScan
-/benchmarks/KernelPDXL1
-/benchmarks/KernelPDXL2
-/benchmarks/KernelPDXIP
-/benchmarks/KernelNaryL1
-/benchmarks/KernelNaryL2
-/benchmarks/KernelNaryIP
-/benchmarks/BenchmarkU8*
+/benchmarks/BenchmarkEndToEnd
+/benchmarks/BenchmarkSerialization
+/benchmarks/BenchmarkPDXIVF
+/benchmarks/BenchmarkFiltered
+/benchmarks/BenchmarkSpecialFilters
+
+# Test binaries (but keep the committed test data)
+*.bin
+!tests/test_data.bin

.gitmodules

@@ -8,3 +8,6 @@
 [submodule "extern/findFFTW"]
 	path = extern/findFFTW
 	url = https://github.com/egpbos/findfftw.git
+[submodule "extern/SuperKMeans"]
+	path = extern/SuperKMeans
+	url = https://github.com/lkuffo/SuperKMeans