fix(forecast): auto-expand replicas for default results index on 3AZ domains (#1615)

When running forecasting on a 3AZ domain with awareness attributes=3, the
forecaster failed to initialize with:

  IllegalArgumentException: Validation Failed: 1: expected total copies
  needs to be a multiple of total awareness attributes [3];

The default forecast results index was created with:
  - number_of_shards: 1
  - number_of_replicas: 1

This gives 2 total copies, which violates the 3AZ awareness requirement and
causes index creation to fail, so all forecasting results errored on 3AZ
domains.

This change updates IndexManagement.java to:
  - set auto_expand_replicas to "0-2" for the default forecast results index

Single-AZ domains remain replica-free, while 3AZ domains automatically
allocate 2 replicas (3 total copies), satisfying the awareness constraint and
allowing forecasters to initialize successfully.

Test:
1. manual test

Signed-off-by: kaituo <[email protected]>

Author: kaituo

.github/workflows/test_bwc.yml

@@ -22,16 +22,15 @@ jobs:
 
     name: Test Anomaly detection BWC
     runs-on: ubuntu-latest
-    container:
-      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
-      # this image tag is subject to change as more dependencies and updates will arrive over time
-      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-      - name: Run start commands
-        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+      # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+      - name: Remove unnecessary files Linux
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
@@ -45,18 +44,16 @@ jobs:
       - name: Assemble anomaly-detection
         run: |
           plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
-          chown -R 1000:1000 `pwd`
           echo plugin_version $plugin_version
-          su `id -un 1000` -c "./gradlew assemble"
+          ./gradlew assemble
           echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
-          su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"
+          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
           echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
           ls ./build/distributions/
-          su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version"
+          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
           echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version ..."
           ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/$plugin_version
       - name: Run AD Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests ..."
-          chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false"
+          ./gradlew bwcTestSuite -Dtests.security.manager=false

CHANGELOG.md

@@ -11,7 +11,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 - Adding auto create as an optional field on detectors ([#1602](https://github.com/opensearch-project/anomaly-detection/pull/1602))
 
 ### Bug Fixes
-
+- fix(forecast): auto-expand replicas for default results index on 3AZ domains ([#1615](https://github.com/opensearch-project/anomaly-detection/pull/1615))
 
 ### Infrastructure
 - Test: Prevent oversized bulk requests in synthetic data test ([#1603](https://github.com/opensearch-project/anomaly-detection/pull/1603))

build.gradle

@@ -54,7 +54,7 @@ buildscript {
         js_resource_folder = "src/test/resources/job-scheduler"
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
         job_scheduler_version = System.getProperty("job_scheduler.version", opensearch_build)
-        bwcVersionShort = "2.20.0"
+        bwcVersionShort = "2.19.1"
         bwcVersion = bwcVersionShort + ".0"
         bwcOpenSearchADDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' +
                 'opensearch/plugins/opensearch-anomaly-detection-' + bwcVersion + '.zip'

src/main/java/org/opensearch/timeseries/indices/IndexManagement.java

@@ -290,8 +290,8 @@ protected void choosePrimaryShards(CreateIndexRequest request, boolean hiddenInd
                     .builder()
                     // put 1 primary shards per hot node if possible
                     .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, getNumberOfPrimaryShards())
-                    // 1 replica for better search performance and fail-over
-                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                    // Support up to 2 replicas at least
+                    .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, customResultIndexAutoExpandReplica)
                     .put(IndexMetadata.SETTING_INDEX_HIDDEN, hiddenIndex)
             );
     }
@@ -1403,13 +1403,6 @@ protected void initResultIndexDirectly(
         if (defaultResultIndex) {
             adminClient.indices().create(request, markMappingUpToDate(resultIndex, actionListener));
         } else {
-            request
-                .settings(
-                    Settings
-                        .builder()
-                        // Support up to 2 replicas at least
-                        .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, customResultIndexAutoExpandReplica)
-                );
             adminClient.indices().create(request, actionListener);
         }
     }

src/test/java/org/opensearch/timeseries/ODFERestTestCase.java

@@ -56,6 +56,7 @@
 import org.apache.hc.core5.http.HttpEntity;
 import org.apache.hc.core5.http.HttpHeaders;
 import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.apache.hc.core5.http.message.BasicHeader;
 import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
 import org.apache.hc.core5.reactor.ssl.TlsDetails;
@@ -438,7 +439,7 @@ public Response createRoleMapping(String role, ArrayList<String> users) throws I
         for (int i = 0; i < users.size(); i++) {
             usersString.add(users.get(i));
         }
-        return TestHelpers
+        Response response = TestHelpers
             .makeRequest(
                 client(),
                 "PUT",
@@ -450,10 +451,14 @@ public Response createRoleMapping(String role, ArrayList<String> users) throws I
                     ),
                 ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana"))
             );
+        // Consume the body so the underlying HTTP connection can be released back to the pool; otherwise the dispatcher
+        // thread stays tied to the response and leaks when the test shuts down.
+        EntityUtils.consume(response.getEntity());
+        return response;
     }
 
     public Response deleteUser(String user) throws IOException {
-        return TestHelpers
+        Response response = TestHelpers
             .makeRequest(
                 client(),
                 "DELETE",
@@ -462,6 +467,10 @@ public Response deleteUser(String user) throws IOException {
                 "",
                 ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana"))
             );
+        // Consume the body so the underlying HTTP connection can be released back to the pool; otherwise the dispatcher
+        // thread stays tied to the response and leaks when the test shuts down.
+        EntityUtils.consume(response.getEntity());
+        return response;
     }
 
     public Response deleteAllResourceSharingRecords() throws IOException {
@@ -473,7 +482,7 @@ public Response deleteAllResourceSharingRecords() throws IOException {
         params.put("conflicts", "proceed");
         params.put("refresh", "true");
 
-        return TestHelpers
+        Response response = TestHelpers
             .makeRequest(
                 adminClient(),
                 "POST",
@@ -483,6 +492,10 @@ public Response deleteAllResourceSharingRecords() throws IOException {
                 ImmutableList
                     .of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana"), new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"))
             );
+        // Consume the body so the underlying HTTP connection can be released back to the pool; otherwise the dispatcher
+        // thread stays tied to the response and leaks when the test shuts down.
+        EntityUtils.consume(response.getEntity());
+        return response;
     }
 
 }