Update build.gradle and build.sh to separate x64 linux nmslib build with different gcc versions (#2506)

* Update build gradle to separate nmslib / faiss generation

Signed-off-by: Peter Zhu <[email protected]>

* Update scripts/build.sh to separate enable gcc switch

Signed-off-by: Peter Zhu <[email protected]>

* Remove test comments

Signed-off-by: Peter Zhu <[email protected]>

* Remove test comments

Signed-off-by: Peter Zhu <[email protected]>

* Remove test comments

Signed-off-by: Peter Zhu <[email protected]>

* Updating restart and rolling upgrade bwc test bundle.gradle

Signed-off-by: Peter Zhu <[email protected]>

* Enforce gcc10 for nmslib to compile and avx512_spr have no-op

Signed-off-by: Peter Zhu <[email protected]>

---------

Signed-off-by: Peter Zhu <[email protected]>

Author: peterzhuamazon

.github/workflows/CI.yml

@@ -40,6 +40,11 @@ jobs:
       matrix:
         java: [11, 17, 21]
 
+    env:
+       CC: gcc10-gcc
+       CXX: gcc10-g++
+       FC: gcc10-gfortran
+
     name: Build and Test k-NN Plugin on Linux
     runs-on: ubuntu-latest
     needs: Get-CI-Image-Tag

.github/workflows/test_security.yml

@@ -39,6 +39,10 @@ jobs:
     strategy:
       matrix:
         java: [21]
+    env:
+       CC: gcc10-gcc
+       CXX: gcc10-g++
+       FC: gcc10-gfortran
 
     name: Run Integration Tests on Linux
     runs-on: ubuntu-latest

build.gradle

@@ -347,13 +347,20 @@ task cmakeJniLib(type:Exec) {
     commandLine args
 }
 
+task buildNmslib(type:Exec) {
+    dependsOn cmakeJniLib
+    workingDir 'jni'
+    commandLine 'make', 'opensearchknn_nmslib', '-j', "${nproc_count}"
+}
+
 task buildJniLib(type:Exec) {
     dependsOn cmakeJniLib
     workingDir 'jni'
-    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
+    commandLine 'make', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
 }
 
 test {
+    dependsOn buildNmslib
     dependsOn buildJniLib
     systemProperty 'tests.security.manager', 'false'
     systemProperty "java.library.path", "$rootDir/jni/release"
@@ -368,6 +375,7 @@ test {
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 integTest {
     if (integTestDependOnJniLib) {
+        dependsOn buildNmslib
         dependsOn buildJniLib
     }
     systemProperty 'tests.security.manager', 'false'
@@ -468,6 +476,7 @@ task integTestRemote(type: RestIntegTestTask) {
 
 run {
     useCluster project.testClusters.integTest
+    dependsOn buildNmslib
     dependsOn buildJniLib
     doFirst {
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts

qa/restart-upgrade/build.gradle

@@ -134,6 +134,7 @@ testClusters {
 // All nodes are upgraded to latest version and run the tests
     task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
         dependsOn "testAgainstOldCluster"
+        dependsOn rootProject.tasks.buildNmslib
         dependsOn rootProject.tasks.buildJniLib
         dependsOn rootProject.tasks.assemble
         useCluster testClusters."${baseName}"

qa/rolling-upgrade/build.gradle

@@ -52,6 +52,7 @@ task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 task testAgainstOneThirdUpgradedCluster(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}"
+    dependsOn rootProject.tasks.buildNmslib
     dependsOn rootProject.tasks.buildJniLib
     dependsOn rootProject.tasks.assemble
     dependsOn "testAgainstOldCluster"

scripts/build.sh

@@ -131,16 +131,24 @@ cd $work_dir
 ./gradlew :buildJniLib -Davx512.enabled=false -Davx512_spr.enabled=false -Davx2.enabled=false -Dbuild.lib.commit_patches=false -Dnproc.count=${NPROC_COUNT:-1}
 
 if [ "$PLATFORM" != "windows" ] && [ "$ARCHITECTURE" = "x64" ]; then
+  echo "Building k-NN library nmslib with gcc 10 on non-windows x64"
+  rm -rf jni/CMakeCache.txt jni/CMakeFiles
+  env CC=gcc10-gcc CXX=gcc10-g++ FC=gcc10-gfortran ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
+
   echo "Building k-NN library after enabling AVX2"
   # Skip applying patches as patches were applied already from previous :buildJniLib task
   # If we apply patches again, it fails with conflict
+  rm -rf jni/CMakeCache.txt jni/CMakeFiles
   ./gradlew :buildJniLib -Davx2.enabled=true -Davx512.enabled=false -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 
   echo "Building k-NN library after enabling AVX512"
   ./gradlew :buildJniLib -Davx512.enabled=true -Davx512_spr.enabled=false -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 
   echo "Building k-NN library after enabling AVX512_SPR"
   ./gradlew :buildJniLib -Davx512_spr.enabled=true -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
+
+else
+  ./gradlew :buildNmslib -Dbuild.lib.commit_patches=false -Dbuild.lib.apply_patches=false
 fi
 
 ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
@@ -149,7 +157,7 @@ fi
 # Add lib to zip
 zipPath=$(find "$(pwd)/build/distributions" -path \*.zip)
 distributions="$(dirname "${zipPath}")"
-mkdir $distributions/lib
+mkdir -p $distributions/lib
 libPrefix="libopensearchknn"
 if [ "$PLATFORM" = "windows" ]; then
     libPrefix="opensearchknn"