Add End-to-End tests to CI job (#1043)

* Added end-to-end tests to CI

Signed-off-by: Norman Jordan <[email protected]>

* Correcting some expected results

Signed-off-by: Norman Jordan <[email protected]>

---------

Signed-off-by: Norman Jordan <[email protected]>

Author: normanj-bitquill

.github/workflows/test-and-build-workflow.yml

@@ -33,7 +33,7 @@ jobs:
         run: echo "SBT_OPTS=-Xmx2G" >> $GITHUB_ENV
 
       - name: Integ Test
-        run: sbt integtest/integration
+        run: sbt integtest/integration e2etest/test
 
       - name: Upload test report
         if: always() # Ensures the artifact is saved even if tests fail

e2e-test/src/test/resources/opensearch/queries/ppl/query_121.results

@@ -8,7 +8,7 @@
   ],
   "datarows": [
     [
-      5
+      3
     ]
   ],
   "total": 1,

e2e-test/src/test/resources/opensearch/queries/ppl/query_125.results

@@ -8,7 +8,7 @@
   ],
   "datarows": [
     [
-      4
+      3
     ]
   ],
   "total": 1,

e2e-test/src/test/resources/spark/queries/ppl/query_121.results

@@ -1,2 +1,2 @@
 distinct_count(country)
-5
+3

e2e-test/src/test/resources/spark/queries/ppl/query_125.results

@@ -1,2 +1,2 @@
 distinct_count(major)
-4
+3

e2e-test/src/test/scala/org/opensearch/spark/e2e/EndToEndITSuite.scala

@@ -75,7 +75,22 @@ class EndToEndITSuite extends AnyFlatSpec with TableDrivenPropertyChecks with Be
     val dockerProcess = new ProcessBuilder("docker", "compose", "up", "-d")
       .directory(new File(DOCKER_INTEG_DIR))
       .start()
-    dockerProcess.waitFor(5, TimeUnit.MINUTES)
+    var stopReading = false
+    new Thread() {
+      override def run(): Unit = {
+        val reader = new BufferedReader(new InputStreamReader(dockerProcess.getInputStream))
+        var line = reader.readLine()
+        while (!stopReading && line != null) {
+          logInfo("*** " + line)
+          line = reader.readLine()
+        }
+      }
+    }.start()
+    val completed = dockerProcess.waitFor(30, TimeUnit.MINUTES)
+    stopReading = true
+    if (!completed) {
+      throw new IllegalStateException("Unable to start docker cluster")
+    }
 
     if (dockerProcess.exitValue() != 0) {
       logError("Unable to start docker cluster")
@@ -97,7 +112,7 @@ class EndToEndITSuite extends AnyFlatSpec with TableDrivenPropertyChecks with Be
     val dockerProcess = new ProcessBuilder("docker", "compose", "down")
       .directory(new File(DOCKER_INTEG_DIR))
       .start()
-    dockerProcess.waitFor(2, TimeUnit.MINUTES)
+    dockerProcess.waitFor(10, TimeUnit.MINUTES)
 
     if (dockerProcess.exitValue() != 0) {
       logError("Unable to stop docker cluster")