5688 paralise st

docs/development/system-tests.md

@@ -72,14 +72,21 @@ of that class, and look at the tests in that module for examples.
 
 This test module contains several JUnit test suites:
 - QuickSystemTestSuite (the default)
-- NightlyFunctionalSystemTestSuite
-- NightlyPerformanceSystemTestSuite
+- SlowSuite1-3
+- ExpensiveSuite1-3
 
-Tests that are tagged as Slow or Expensive will not be included in the quick suite. The quick suite is intended to run
+Tests that are tagged as Slow(1-3) or Expensive(1-3) will not be included in the quick suite. The quick suite is intended to run
 in around 40 minutes. The nightly functional suite includes tests tagged as Slow, and will take a bit longer. The
 nightly performance suite includes all tests, including ones tagged as Expensive. The performance tests work with a
 larger bulk of data. They take time to run and can be costly to run frequently.
 
+Due to the time taken to run all the system tests for either a functional or performance run, multiple test suites
+now run in parallel. This is achievied by copying the Java, Python, Scripts and CodeStyle folders of Sleeper into a
+temporary folder per batch (quick, slow1-3, expensive1-3). The nightly run script then runs each of these in parallel
+with a short 60 seconds delay between each one starting up. Once all have completed it then uploads the results to S3.
+When adding a new Slow or Expensive System test add either the Slow1-3 or Expensive 1-3 tag.
+
+
 ### Running tests
 
 This command will build the system and run the default, quick system test suite:

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/AutoDeleteS3ObjectsST.java

@@ -25,7 +25,7 @@
 import sleeper.query.core.output.ResultsOutput;
 import sleeper.query.runner.output.S3ResultsOutput;
 import sleeper.systemtest.dsl.SleeperSystemTest;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow3;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.nio.file.Path;
@@ -38,7 +38,7 @@
 
 @SystemTest
 // Slow because it needs to do multiple CDK deployments
-@Slow
+@Slow3
 public class AutoDeleteS3ObjectsST {
 
     @TempDir

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/AutoStopEcsTaskST.java

@@ -23,7 +23,7 @@
 import sleeper.core.statestore.FileReference;
 import sleeper.core.util.PollWithRetries;
 import sleeper.systemtest.dsl.SleeperSystemTest;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow1;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -36,7 +36,7 @@
 
 @SystemTest
 // Slow because it needs to do multiple CDK deployments
-@Slow
+@Slow1
 public class AutoStopEcsTaskST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionCreationST.java

@@ -25,7 +25,7 @@
 import sleeper.core.util.PollWithRetries;
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.compaction.FoundCompactionJobs;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow1;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -45,7 +45,7 @@
 import static sleeper.systemtest.suite.testutil.FileReferenceSystemTestHelper.fileFactory;
 
 @SystemTest
-@Slow // Slow because it deploys its own instance so it can drain the whole compaction jobs queue
+@Slow1 // Slow because it deploys its own instance so it can drain the whole compaction jobs queue
 @Execution(SAME_THREAD)
 public class CompactionCreationST {
 

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionDataFusionPerformanceST.java

@@ -26,7 +26,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
+import sleeper.systemtest.suite.testutil.Expensive1;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -41,7 +41,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMPACTION_PERFORMANCE_DATAFUSION;
 
 @SystemTest
-@Expensive // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+@Expensive1 // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
 public class CompactionDataFusionPerformanceST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionOnEC2ST.java

@@ -25,7 +25,7 @@
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
 import sleeper.systemtest.dsl.sourcedata.RowNumbers;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow1;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.nio.file.Path;
@@ -41,7 +41,7 @@
 import static sleeper.systemtest.suite.testutil.TestResources.exampleString;
 
 @SystemTest
-@Slow
+@Slow1
 public class CompactionOnEC2ST {
     @TempDir
     private Path tempDir;

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionPerformanceST.java

@@ -26,7 +26,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
+import sleeper.systemtest.suite.testutil.Expensive1;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -40,7 +40,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMPACTION_PERFORMANCE;
 
 @SystemTest
-@Expensive // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+@Expensive1 // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
 public class CompactionPerformanceST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/CompactionVeryLargeST.java

@@ -28,7 +28,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
+import sleeper.systemtest.suite.testutil.Expensive2;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -43,7 +43,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMPACTION_PERFORMANCE_DATAFUSION;
 
 @SystemTest
-@Expensive // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
+@Expensive2 // Expensive because it takes a long time to compact this many rows on fairly large ECS instances.
 public class CompactionVeryLargeST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/EksBulkImportST.java

@@ -26,7 +26,7 @@
 import sleeper.systemtest.dsl.instance.SystemTestParameters;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
 import sleeper.systemtest.dsl.util.SystemTestSchema;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow1;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.util.Map;
@@ -46,7 +46,7 @@
 // Slow because it needs to do two CDK deployments, one to add the EKS cluster and one to remove it.
 // Each CDK deployment takes around 20 minutes.
 // If we left the EKS cluster around, there would be extra costs as the control pane is persistent.
-@Slow
+@Slow1
 public class EksBulkImportST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/EmrBulkImportPerformanceST.java

@@ -23,7 +23,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
+import sleeper.systemtest.suite.testutil.Expensive2;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -36,7 +36,7 @@
 import static sleeper.systemtest.suite.testutil.FileReferenceSystemTestHelper.numberOfRowsIn;
 
 @SystemTest
-@Expensive // Expensive because it takes a lot of very costly EMR instances to import this many rows.
+@Expensive2 // Expensive because it takes a lot of very costly EMR instances to import this many rows.
 public class EmrBulkImportPerformanceST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/EmrPersistentBulkImportST.java

@@ -25,7 +25,7 @@
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
 import sleeper.systemtest.dsl.util.SystemTestSchema;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow2;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.util.Map;
@@ -45,7 +45,7 @@
 // Each CDK deployment takes around 20 minutes.
 // If we leave the EMR cluster deployed, the costs for the EMR instances add up to hundreds of pounds quite quickly.
 // With the CDK deployments, the cluster doesn't stay around for very long as it only imports 100 rows.
-@Slow
+@Slow2
 public class EmrPersistentBulkImportST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/GarbageCollectionST.java

@@ -24,6 +24,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.sourcedata.RowNumbers;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.Test1;
 
 import java.nio.file.Path;
 import java.time.Duration;
@@ -41,6 +42,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.MAIN;
 
 @SystemTest
+@Test1
 public class GarbageCollectionST {
     @TempDir
     private Path tempDir;

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/IngestPerformanceST.java

@@ -23,7 +23,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
+import sleeper.systemtest.suite.testutil.Expensive3;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -36,7 +36,7 @@
 import static sleeper.systemtest.suite.testutil.FileReferenceSystemTestHelper.numberOfRowsIn;
 
 @SystemTest
-@Expensive // Expensive because it takes a long time to ingest this many rows on fairly large ECS instances.
+@Expensive3 // Expensive because it takes a long time to ingest this many rows on fairly large ECS instances.
 public class IngestPerformanceST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/MultipleTablesST.java

@@ -24,7 +24,7 @@
 import sleeper.core.schema.Schema;
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.util.SystemTestSchema;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow2;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.util.Map;
@@ -45,7 +45,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.MAIN;
 
 @SystemTest
-@Slow // Slow because compactions run for 200 tables in one task
+@Slow2 // Slow because compactions run for 200 tables in one task
 public class MultipleTablesST {
     private final Schema schema = SystemTestSchema.DEFAULT_SCHEMA;
     private static final int NUMBER_OF_TABLES = 200;

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/OptionalFeaturesDisabledST.java

@@ -24,7 +24,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.sourcedata.RowNumbers;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow2;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.nio.file.Path;
@@ -38,7 +38,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.OPTIONAL_FEATURES_DISABLED;
 
 @SystemTest
-@Slow // Slow because it deploys a separate instance just for this test, and the CDK is slow
+@Slow2 // Slow because it deploys a separate instance just for this test, and the CDK is slow
 public class OptionalFeaturesDisabledST {
 
     @TempDir

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/ParallelCompactionsST.java

@@ -25,7 +25,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.extension.AfterTestReports;
 import sleeper.systemtest.dsl.reporting.SystemTestReports;
-import sleeper.systemtest.suite.testutil.Expensive;
+import sleeper.systemtest.suite.testutil.Expensive3;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -42,7 +42,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.PARALLEL_COMPACTIONS;
 
 @SystemTest
-@Expensive
+@Expensive3
 public class ParallelCompactionsST {
 
     @BeforeEach

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/QueryST.java

@@ -26,6 +26,7 @@
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.util.SystemTestSchema;
 import sleeper.systemtest.suite.testutil.SystemTest;
+import sleeper.systemtest.suite.testutil.Test2;
 
 import java.nio.file.Path;
 import java.util.stream.LongStream;
@@ -38,6 +39,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.MAIN;
 
 @SystemTest
+@Test2
 public class QueryST {
     @TempDir
     private Path tempDir;

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/RedeployOptionalStacksST.java

@@ -21,7 +21,7 @@
 
 import sleeper.core.properties.model.OptionalStack;
 import sleeper.systemtest.dsl.SleeperSystemTest;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow3;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.util.LinkedHashSet;
@@ -31,7 +31,7 @@
 
 @SystemTest
 // Slow because it needs to do many CDK deployments
-@Slow
+@Slow3
 public class RedeployOptionalStacksST {
 
     private static final Set<OptionalStack> REDEPLOYABLE_STACKS = new LinkedHashSet<>(OptionalStack.all());

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/StateStoreCommitterThroughputST.java

@@ -23,7 +23,7 @@
 import sleeper.core.statestore.FileReferenceFactory;
 import sleeper.systemtest.dsl.SleeperSystemTest;
 import sleeper.systemtest.dsl.statestore.StateStoreCommitMessage;
-import sleeper.systemtest.suite.testutil.Slow;
+import sleeper.systemtest.suite.testutil.Slow3;
 import sleeper.systemtest.suite.testutil.SystemTest;
 
 import java.time.Duration;
@@ -49,7 +49,7 @@
 import static sleeper.systemtest.suite.fixtures.SystemTestInstance.COMMITTER_THROUGHPUT;
 
 @SystemTest
-@Slow
+@Slow3
 @Execution(SAME_THREAD)
 public class StateStoreCommitterThroughputST {
 

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/NightlyFunctionalSystemTestSuite.java

@@ -21,6 +21,6 @@
 import sleeper.systemtest.suite.testutil.SystemTestSuite;
 
 @SystemTestSuite
-@ExcludeTags("expensive")
+@ExcludeTags({"expensive1", "expensive2", "expensive3"})
 public class NightlyFunctionalSystemTestSuite {
 }

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/QuickSystemTestSuite.java

@@ -21,6 +21,6 @@
 import sleeper.systemtest.suite.testutil.SystemTestSuite;
 
 @SystemTestSuite
-@ExcludeTags({"slow", "expensive"})
+@ExcludeTags({"slow1", "slow2", "slow3", "expensive1", "expensive2", "expensive3"})
 public class QuickSystemTestSuite {
 }

java/system-test/system-test-suite/src/test/java/sleeper/systemtest/suite/suites/parallel/ExpensiveSuite1.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2022-2025 Crown Copyright
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sleeper.systemtest.suite.suites.parallel;
+
+import org.junit.platform.suite.api.IncludeTags;
+
+import sleeper.systemtest.suite.testutil.SystemTestSuite;
+
+@SystemTestSuite
+@IncludeTags("expensive1")
+public class ExpensiveSuite1 {
+}