Merge branch 'apache:master' into bugfix/2963-fix-npe-while-rebalancing-task

Author: jacob-netguardians

bump-snapshot-mac.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+echo There are $# arguments to $0: $*
+version=`grep -A 1 "<artifactId>helix</artifactId>" pom.xml |tail -1 | awk 'BEGIN {FS="[<,>]"};{print $3}'`
+if [ "$#" -eq 1 ]; then
+  new_version=$1
+elif [ "$#" -eq 2 ]; then
+  version=$1
+  new_version=$2
+else
+  echo "ERROR: Need oldVersion and newVersion two arguments"
+  exit 0
+fi
+
+
+# Move Ivy
+mv helix-admin-webapp/helix-admin-webapp-$version-SNAPSHOT.ivy helix-admin-webapp/helix-admin-webapp-$new_version-SNAPSHOT.ivy
+mv helix-agent/helix-agent-$version-SNAPSHOT.ivy helix-agent/helix-agent-$new_version-SNAPSHOT.ivy
+mv helix-common/helix-common-$version-SNAPSHOT.ivy helix-common/helix-common-$new_version-SNAPSHOT.ivy
+mv helix-core/helix-core-$version-SNAPSHOT.ivy helix-core/helix-core-$new_version-SNAPSHOT.ivy
+mv helix-lock/helix-lock-$version-SNAPSHOT.ivy helix-lock/helix-lock-$new_version-SNAPSHOT.ivy
+mv helix-rest/helix-rest-$version-SNAPSHOT.ivy helix-rest/helix-rest-$new_version-SNAPSHOT.ivy
+mv metadata-store-directory-common/metadata-store-directory-common-$version-SNAPSHOT.ivy metadata-store-directory-common/metadata-store-directory-common-$new_version-SNAPSHOT.ivy
+mv metrics-common/metrics-common-$version-SNAPSHOT.ivy metrics-common/metrics-common-$new_version-SNAPSHOT.ivy
+mv zookeeper-api/zookeeper-api-$version-SNAPSHOT.ivy zookeeper-api/zookeeper-api-$new_version-SNAPSHOT.ivy
+mv helix-view-aggregator/helix-view-aggregator-$version-SNAPSHOT.ivy helix-view-aggregator/helix-view-aggregator-$new_version-SNAPSHOT.ivy
+mv meta-client/meta-client-$version-SNAPSHOT.ivy meta-client/meta-client-$new_version-SNAPSHOT.ivy
+
+find . -type f -name '*.ivy' -exec sed -i '' s/$version-SNAPSHOT/$new_version-SNAPSHOT/g {} +
+find . -type f -name 'pom.xml' -exec sed -i '' s/$version-SNAPSHOT/$new_version-SNAPSHOT/g {} +
+
+
+echo Bump up is DONE! Please double check and commit!
+#END

helix-admin-webapp/helix-admin-webapp-1.4.2-SNAPSHOT.ivy renamed to helix-admin-webapp/helix-admin-webapp-1.4.3-SNAPSHOT.ivy

@@ -21,7 +21,7 @@ under the License.
 <ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
 	<info organisation="org.apache.helix"
 		module="helix-agent"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="integration"
 		publication="20141020152553"
 	>
@@ -31,7 +31,7 @@ under the License.
 		<m:properties__osgi.export>org.apache.helix.agent*;version=&quot;${project.version};-noimport:=true</m:properties__osgi.export>
 		<m:properties__svnImpl>svn</m:properties__svnImpl>
 		<m:properties__scmSkipDeletedFiles>false</m:properties__scmSkipDeletedFiles>
-		<m:properties__currentRelease>1.4.2-SNAPSHOT</m:properties__currentRelease>
+		<m:properties__currentRelease>1.4.3-SNAPSHOT</m:properties__currentRelease>
 		<m:properties__osgi.import>
       org.apache.helix*,
       org.apache.commons.cli;version=&quot;[1.2,2)&quot;,
@@ -46,7 +46,7 @@ under the License.
 		<m:properties__distMgmtSnapshotsUrl>https://repository.apache.org/content/repositories/snapshots</m:properties__distMgmtSnapshotsUrl>
 		<m:properties__SKIP_INTEGRATION_TESTS>true</m:properties__SKIP_INTEGRATION_TESTS>
 		<m:properties__gpg.useagent>true</m:properties__gpg.useagent>
-		<m:dependency.management__org.apache.helix__helix-core__version>1.4.2-SNAPSHOT</m:dependency.management__org.apache.helix__helix-core__version>
+		<m:dependency.management__org.apache.helix__helix-core__version>1.4.3-SNAPSHOT</m:dependency.management__org.apache.helix__helix-core__version>
 		<m:properties__project.build.sourceEncoding>UTF-8</m:properties__project.build.sourceEncoding>
 		<m:dependency.management__org.testng__testng__version>6.0.1</m:dependency.management__org.testng__testng__version>
 		<m:properties__helix.release.preparationGoals>clean install</m:properties__helix.release.preparationGoals>
@@ -77,12 +77,12 @@ under the License.
 		<artifact name="helix-agent" type="source" ext="jar" conf="sources" m:classifier="sources"/>
 	</publications>
 	<dependencies>
-		<dependency org="org.apache.helix" name="helix-core" rev="1.4.2-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
+		<dependency org="org.apache.helix" name="helix-core" rev="1.4.3-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="org.restlet.jse" name="org.restlet" rev="2.2.1" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="org.testng" name="testng" rev="6.0.1" force="true" conf="test->runtime(*),master(*)">
 			<exclude org="junit" module="junit" name="*" type="*" ext="*" conf="" matcher="exact"/>
 		</dependency>
-		<dependency org="org.apache.helix" name="helix-core" rev="1.4.2-SNAPSHOT" force="true" conf="test->runtime(*),master(*)">
+		<dependency org="org.apache.helix" name="helix-core" rev="1.4.3-SNAPSHOT" force="true" conf="test->runtime(*),master(*)">
 			<artifact name="helix-core" type="test-jar" ext="jar" conf="" m:classifier="tests"/>
 		</dependency>
     <dependency org="org.slf4j" name="slf4j-api" rev="1.7.32" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)">
@@ -92,7 +92,7 @@ under the License.
         <artifact name="log4j-slf4j-impl" ext="jar"/>
     </dependency>
 		<override org="org.mockito" module="mockito-all" matcher="exact" rev="1.9.5"/>
-		<override org="org.apache.helix" module="helix-core" matcher="exact" rev="1.4.2-SNAPSHOT"/>
+		<override org="org.apache.helix" module="helix-core" matcher="exact" rev="1.4.3-SNAPSHOT"/>
 		<override org="org.restlet.jse" module="org.restlet" matcher="exact" rev="2.2.1"/>
 		<override org="junit" module="junit" matcher="exact" rev="4.11"/>
 		<override org="org.testng" module="testng" matcher="exact" rev="6.0.1"/>

helix-agent/helix-agent-1.4.2-SNAPSHOT.ivy renamed to helix-agent/helix-agent-1.4.3-SNAPSHOT.ivy

@@ -20,7 +20,7 @@ under the License.
 <ivy-module version="1.0">
 	<info organisation="org.apache.helix"
 		module="helix-common"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="integration"
 		publication="20170128141623"
 	/>

helix-common/helix-common-1.4.2-SNAPSHOT.ivy renamed to helix-common/helix-common-1.4.3-SNAPSHOT.ivy

@@ -20,7 +20,7 @@ under the License.
 <ivy-module version="1.0" xmlns:m="http://ant.apache.org/ivy/maven">
 	<info organisation="org.apache.helix"
 		module="helix-core"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="release"
 		publication="2011111505113547"
 	>

helix-core/helix-core-1.4.2-SNAPSHOT.ivy renamed to helix-core/helix-core-1.4.3-SNAPSHOT.ivy

@@ -642,8 +642,8 @@ public InstanceOperation getInstanceOperation() {
               .build();
     }
 
-    // Always respect the HELIX_ENABLED being set to false when instance operation is unset
-    // for backwards compatibility.
+    // if instance operation is not DISABLED, we always respect the HELIX_ENABLED being set to false
+    // when instance operation is unset for backwards compatibility.
     if (!_record.getBooleanField(InstanceConfigProperty.HELIX_ENABLED.name(),
         HELIX_ENABLED_DEFAULT_VALUE)
         && (InstanceConstants.INSTANCE_DISABLED_OVERRIDABLE_OPERATIONS.contains(
@@ -656,6 +656,17 @@ public InstanceOperation getInstanceOperation() {
           .build();
     }
 
+    // if instance operation is DISABLE, we override it to ENABLE if HELIX_ENABLED set to true
+    if (activeInstanceOperation.getOperation() == InstanceConstants.InstanceOperation.DISABLE) {
+      // it is not likely that HELIX_ENABLED is unset, because when we set operation to disable,
+      // we always set HELIX_ENABLED to false
+      // If instance is enabled by old version helix (not having instance operation), the instance config
+      // will have HELIX_ENABLED set to true. In this case, we should override the instance operation to ENABLE
+      if ("true".equals(_record.getSimpleField(InstanceConfigProperty.HELIX_ENABLED.name()))) {
+        return new InstanceOperation.Builder().setOperation(InstanceConstants.InstanceOperation.ENABLE).build();
+      }
+    }
+
     return activeInstanceOperation;
   }
 

helix-core/src/main/java/org/apache/helix/model/InstanceConfig.java

@@ -27,7 +27,6 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -330,6 +329,11 @@ protected synchronized boolean verifyState() {
         }
       }
 
+      // Process resources to retain only those requested
+      if (_resources != null && !_resources.isEmpty()) {
+        idealStates.keySet().retainAll(_resources);
+        extViews.keySet().retainAll(_resources);
+      }
       for (String resourceName : idealStates.keySet()) {
         IdealState is = idealStates.get(resourceName);
         ExternalView extView = extViews.get(resourceName);

helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java

@@ -74,6 +74,22 @@ public void testSetInstanceEnableWithReason() {
         InstanceConstants.InstanceDisabledType.USER_OPERATION.toString());
   }
 
+  @Test
+  public void testEnabledInstanceBackwardCompatibility() {
+    // if instance is disabled by instanceOperation, and enabled by HELIX_ENABLED, the instance should be enabled
+    InstanceConfig instanceConfig =
+        new InstanceConfig.Builder().setInstanceOperation(InstanceConstants.InstanceOperation.DISABLE).build("id");
+    Assert.assertFalse(instanceConfig.getInstanceEnabled());
+    // assume an old version client enabled the instance by setting HELIX_ENABLED to true
+    instanceConfig.getRecord().setSimpleField(InstanceConfig.InstanceConfigProperty.HELIX_ENABLED.name(), "true");
+    Assert.assertTrue(instanceConfig.getInstanceEnabled());
+    instanceConfig.setInstanceOperation(InstanceConstants.InstanceOperation.ENABLE);
+
+    // disable the instance by setting HELIX_ENABLED to false
+    instanceConfig.getRecord().setSimpleField(InstanceConfig.InstanceConfigProperty.HELIX_ENABLED.name(), "false");
+    Assert.assertFalse(instanceConfig.getInstanceEnabled());
+  }
+
   @Test
   public void testGetParsedDomainEmptyDomain() {
     InstanceConfig instanceConfig = new InstanceConfig(new ZNRecord("id"));

helix-core/src/test/java/org/apache/helix/model/TestInstanceConfig.java

@@ -20,6 +20,7 @@
  */
 
 import java.util.Arrays;
+import java.util.Collections;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
@@ -263,4 +264,60 @@ public void testSleepTransition() throws InterruptedException {
     // The new participant causes rebalance, but the state transitions are all stuck
     Assert.assertFalse(strictMatchVerifier.verify(3000));
   }
+
+  @Test
+  public void testExternalViewBestPossibleVerifierWorkWithWaged() throws InterruptedException {
+    // Create a WAGED resource with 2 replicas and verify that it is balanced
+    String testDB1 = "resource-wagedDB1";
+    _setupTool.addResourceToCluster(_clusterName, testDB1, 1,
+        BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.toString());
+    _admin.enableWagedRebalance(_clusterName, Collections.singletonList(testDB1));
+    IdealState db1IdealState = _admin.getResourceIdealState(_clusterName, testDB1);
+    db1IdealState.setReplicas(Integer.toString(2));
+    db1IdealState.getRecord().setListField(testDB1 + "_0",
+        Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()));
+    _admin.setResourceIdealState(_clusterName, testDB1, db1IdealState);
+
+    // Verify that the resource is balanced
+    BestPossibleExternalViewVerifier bestPossibleVerifier =
+        new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+            .setResources(Sets.newHashSet(testDB1))
+            .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+            .build();
+    Assert.assertTrue(bestPossibleVerifier.verify(10000));
+
+    // Re-start a new participant with sleeping transition(all state model transition cannot finish)
+    // This should fail the verification of the second resource
+    _participants[2].syncStop();
+    Thread.sleep(1000);
+
+    _participants[2] = new MockParticipantManager(ZK_ADDR, _clusterName, _participants[2].getInstanceName());
+    _participants[2].setTransition(new SleepTransition(99999999));
+    _participants[2].syncStart();
+
+    // Create the second WAGED resource with 2 replicas and verify that it is NOT balanced
+    String testDB2 = "resource-wagedDB2";
+    _setupTool.addResourceToCluster(_clusterName, testDB2, 1,
+        BuiltInStateModelDefinitions.MasterSlave.name(), RebalanceMode.FULL_AUTO.toString());
+    _admin.enableWagedRebalance(_clusterName, Collections.singletonList(testDB2));
+    IdealState db2IdealState = _admin.getResourceIdealState(_clusterName, testDB2);
+    db2IdealState.setReplicas(Integer.toString(2));
+    db2IdealState.getRecord().setListField(testDB2 + "_0",
+        Arrays.asList(_participants[1].getInstanceName(), _participants[2].getInstanceName()));
+    _admin.setResourceIdealState(_clusterName, testDB2, db2IdealState);
+
+    // Verify that the resource is NOT balanced
+    bestPossibleVerifier = new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+        .setResources(Sets.newHashSet(testDB2))
+        .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+        .build();
+    Assert.assertFalse(bestPossibleVerifier.verify(3000));
+
+    // The verification of the first resource should still pass
+    bestPossibleVerifier = new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+        .setResources(Sets.newHashSet(testDB1))
+        .setWaitTillVerify(TestHelper.DEFAULT_REBALANCE_PROCESSING_WAIT_TIME)
+        .build();
+    Assert.assertTrue(bestPossibleVerifier.verify(10000));
+  }
 }

helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java

@@ -6071,9 +6071,9 @@ [email protected]:
     pretty-bytes "^5.3.0"
 
 cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
-  integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
+  version "7.0.6"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
+  integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
   dependencies:
     path-key "^3.1.0"
     shebang-command "^2.0.0"

helix-front/yarn.lock

@@ -20,9 +20,9 @@ under the License.
 <ivy-module version="1.0">
 	<info organisation="org.apache.helix"
 		module="helix-lock"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="integration"
-		publication="202001.4.25923"
+		publication="202001.4.35923"
 	/>
 	<configurations>
 		<conf name="default" visibility="public" description="runtime dependencies and master artifact can be used with this conf" extends="runtime,master"/>
@@ -43,6 +43,6 @@ under the License.
     <dependency org="org.apache.logging.log4j" name="log4j-slf4j-impl" rev="2.17.1" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)">
         <artifact name="log4j-slf4j-impl" ext="jar"/>
     </dependency>
-		<dependency org="org.apache.helix" name="helix-core" rev="1.4.2-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
+		<dependency org="org.apache.helix" name="helix-core" rev="1.4.3-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 	</dependencies>
 </ivy-module>

helix-lock/helix-lock-1.4.2-SNAPSHOT.ivy renamed to helix-lock/helix-lock-1.4.3-SNAPSHOT.ivy

@@ -20,7 +20,7 @@ under the License.
 <ivy-module version="1.0">
 	<info organisation="org.apache.helix"
 		module="helix-rest"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="integration"
 		publication="20170128141623"
 	/>
@@ -46,7 +46,7 @@ under the License.
     <dependency org="org.yaml" name="snakeyaml" rev="1.30">
         <artifact name="snakeyaml" m:classifier="sources" ext="jar"/>
     </dependency>
-		<dependency org="org.apache.helix" name="helix-core" rev="1.4.2-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
+		<dependency org="org.apache.helix" name="helix-core" rev="1.4.3-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.12.6.1" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="com.fasterxml.jackson.core" name="jackson-core" rev="2.12.6" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="commons-cli" name="commons-cli" rev="1.2" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>

helix-rest/helix-rest-1.4.2-SNAPSHOT.ivy renamed to helix-rest/helix-rest-1.4.3-SNAPSHOT.ivy

@@ -18,7 +18,7 @@ under the License.
 <ivy-module version="1.0">
 	<info organisation="org.apache.helix"
 		module="helix-view-aggregator"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="integration"
 		publication="20170128141623"
 	/>
@@ -41,7 +41,7 @@ under the License.
     <dependency org="org.apache.logging.log4j" name="log4j-slf4j-impl" rev="2.17.1" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)">
         <artifact name="log4j-slf4j-impl" ext="jar"/>
     </dependency>
-		<dependency org="org.apache.helix" name="helix-core" rev="1.4.2-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
+		<dependency org="org.apache.helix" name="helix-core" rev="1.4.3-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="1.8.5" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="1.8.5" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 		<dependency org="commons-cli" name="commons-cli" rev="1.2" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>

helix-view-aggregator/helix-view-aggregator-1.4.2-SNAPSHOT.ivy renamed to helix-view-aggregator/helix-view-aggregator-1.4.3-SNAPSHOT.ivy

@@ -20,9 +20,9 @@ under the License.
 <ivy-module version="1.0">
 	<info organisation="org.apache.helix"
 		module="meta-client"
-		revision="1.4.2-SNAPSHOT"
+		revision="1.4.3-SNAPSHOT"
 		status="integration"
-		publication="202001.4.25923"
+		publication="202001.4.35923"
 	/>
 	<configurations>
 		<conf name="default" visibility="public" description="runtime dependencies and master artifact can be used with this conf" extends="runtime,master"/>
@@ -43,6 +43,6 @@ under the License.
     <dependency org="org.apache.logging.log4j" name="log4j-slf4j-impl" rev="2.17.1" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)">
         <artifact name="log4j-slf4j-impl" ext="jar"/>
     </dependency>
-		<dependency org="org.apache.helix" name="zookeeper-api" rev="1.4.2-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
+		<dependency org="org.apache.helix" name="zookeeper-api" rev="1.4.3-SNAPSHOT" force="true" conf="compile->compile(*),master(*);runtime->runtime(*)"/>
 	</dependencies>
 </ivy-module>