Merge branch 'main' into ppl-projection-command

Author: YANG-DB

.github/workflows/stalled.yml

@@ -0,0 +1,28 @@
+name: Label Stalled PRs
+on:
+  schedule:
+    - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT
+permissions:
+  pull-requests: write
+jobs:
+  stale:
+    if: github.repository == 'opensearch-project/opensearch-spark'
+    runs-on: ubuntu-latest
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Stale PRs
+        uses: actions/stale@v9
+        with:
+          repo-token: ${{ steps.github_app_token.outputs.token }}
+          stale-pr-label: 'stalled'
+          stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.'
+          days-before-pr-stale: 30
+          days-before-issue-stale: -1
+          days-before-pr-close: -1
+          days-before-issue-close: -1

build.sbt

@@ -55,9 +55,6 @@ lazy val testScalastyle = taskKey[Unit]("testScalastyle")
 // - .inAll applies the rule to all dependencies, not just direct dependencies
 val packagesToShade = Seq(
   "com.amazonaws.cloudwatch.**",
-  "com.fasterxml.jackson.core.**",
-  "com.fasterxml.jackson.dataformat.**",
-  "com.fasterxml.jackson.databind.**",
   "com.google.**",
   "com.sun.jna.**",
   "com.thoughtworks.paranamer.**",
@@ -325,6 +322,28 @@ lazy val integtest = (project in file("integ-test"))
 lazy val integration = taskKey[Unit]("Run integration tests")
 lazy val awsIntegration = taskKey[Unit]("Run AWS integration tests")
 
+lazy val e2etest = (project in file("e2e-test"))
+  .dependsOn(flintCommons % "test->package", flintSparkIntegration % "test->package", pplSparkIntegration % "test->package", sparkSqlApplication % "test->package")
+  .settings(
+    commonSettings,
+    name := "e2e-test",
+    scalaVersion := scala212,
+    libraryDependencies ++= Seq(
+      "org.scalatest" %% "scalatest" % "3.2.15" % "test",
+      "org.apache.spark" %% "spark-connect-client-jvm" % "3.5.3" % "test",
+      "com.amazonaws" % "aws-java-sdk-s3" % "1.12.568" % "test",
+      "com.softwaremill.sttp.client3" %% "core" % "3.10.2" % "test",
+      "com.softwaremill.sttp.client3" %% "play2-json" % "3.10.2",
+      "com.typesafe.play" %% "play-json" % "2.9.2" % "test",
+    ),
+    libraryDependencies ++= deps(sparkVersion),
+    javaOptions ++= Seq(
+      s"-DappJar=${(sparkSqlApplication / assembly).value.getAbsolutePath}",
+      s"-DextensionJar=${(flintSparkIntegration / assembly).value.getAbsolutePath}",
+      s"-DpplJar=${(pplSparkIntegration / assembly).value.getAbsolutePath}",
+    )
+  )
+
 lazy val standaloneCosmetic = project
   .settings(
     name := "opensearch-spark-standalone",

docker/integ-test/.env

@@ -5,9 +5,13 @@ MASTER_UI_PORT=8080
 MASTER_PORT=7077
 UI_PORT=4040
 SPARK_CONNECT_PORT=15002
-PPL_JAR=../../ppl-spark-integration/target/scala-2.12/ppl-spark-integration-assembly-0.7.0-SNAPSHOT.jar
-FLINT_JAR=../../flint-spark-integration/target/scala-2.12/flint-spark-integration-assembly-0.7.0-SNAPSHOT.jar
+PPL_JAR=./ppl-spark-integration/target/scala-2.12/ppl-spark-integration-assembly-0.7.0-SNAPSHOT.jar
+FLINT_JAR=./flint-spark-integration/target/scala-2.12/flint-spark-integration-assembly-0.7.0-SNAPSHOT.jar
+SQL_APP_JAR=./spark-sql-application/target/scala-2.12/sql-job-assembly-0.7.0-SNAPSHOT.jar
 OPENSEARCH_NODE_MEMORY=512m
 OPENSEARCH_ADMIN_PASSWORD=C0rrecthorsebatterystaple.
 OPENSEARCH_PORT=9200
+OPENSEARCH_PA_PORT=9600
 OPENSEARCH_DASHBOARDS_PORT=5601
+S3_ACCESS_KEY=Vt7jnvi5BICr1rkfsheT
+S3_SECRET_KEY=5NK3StGvoGCLUWvbaGN0LBUf9N6sjE94PEzLdqwO

docker/integ-test/configuration-updater/apply-configuration.sh

@@ -0,0 +1,93 @@
+/bin/sh
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+# Login to Minio
+curl -q \
+     -c /tmp/minio-cookies.txt \
+     -H 'Content-Type: application/json' \
+     -d '{"accessKey": "minioadmin", "secretKey": "minioadmin"}' \
+     http://minio-S3:9001/api/v1/login
+# Delete the test bucket
+curl -b /tmp/minio-cookies.txt \
+     -X DELETE \
+     http://minio-S3:9001/api/v1/buckets/test
+# Create the integ-test bucket
+curl -q \
+     -b /tmp/minio-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -d '{"name": "integ-test", "versioning": {"enabled": true, "excludePrefixes": [], "excludeFolders": false}, "locking": true}' \
+     http://minio-S3:9001/api/v1/buckets
+# Create the test-resources bucket
+curl -q \
+     -b /tmp/minio-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -d '{"name": "test-resources", "versioning": {"enabled": false, "excludePrefixes": [], "excludeFolders": false}, "locking": true}' \
+     http://minio-S3:9001/api/v1/buckets
+# Create the access key
+curl -q \
+     -b /tmp/minio-cookies.txt \
+     -X GET
+     "http://minio-S3:9001/api/v1/service-accounts/${S3_ACCESS_KEY}"
+if [ "$?" -ne "0" ]; then
+  curl -q \
+       -b /tmp/minio-cookies.txt \
+       -X POST \
+       -H 'Content-Type: application/json' \
+       -d "{\"policy\": \"\", \"accessKey\": \"${S3_ACCESS_KEY}\", \"secretKey\": \"${S3_SECRET_KEY}\", \"description\": \"\", \"comment\": \"\", \"name\": \"\", \"expiry\": null}" \
+       http://minio-S3:9001/api/v1/service-account-credentials
+fi
+
+# Login to OpenSearch Dashboards
+echo ">>> Login to OpenSearch dashboards"
+curl -q \
+     -c /tmp/opensearch-cookies.txt \
+     -X POST \
+     -H 'Content-Type: application/json' \
+     -H 'Osd-Version: 2.18.0' \
+     -H 'Osd-Xsrf: fetch' \
+     -d "{\"username\": \"admin\", \"password\": \"${OPENSEARCH_ADMIN_PASSWORD}\"}" \
+     'http://opensearch-dashboards:5601/auth/login?dataSourceId='
+if [ "$?" -eq "0" ]; then
+  echo "    >>> Login successful"
+else
+  echo "    >>> Login failed"
+fi
+
+# Create the S3/Glue datasource
+curl -q \
+     -b /tmp/opensearch-cookies.txt \
+     -X GET \
+     http://localhost:5601/api/directquery/dataconnections/mys3
+if [ "$?" -ne "0" ]; then
+  echo ">>> Creating datasource"
+  curl -q \
+       -b /tmp/opensearch-cookies.txt \
+       -X POST \
+       -H 'Content-Type: application/json' \
+       -H 'Osd-Version: 2.18.0' \
+       -H 'Osd-Xsrf: fetch' \
+       -d "{\"name\": \"mys3\", \"allowedRoles\": [], \"connector\": \"s3glue\", \"properties\": {\"glue.auth.type\": \"iam_role\", \"glue.auth.role_arn\": \"arn:aws:iam::123456789012:role/S3Access\", \"glue.indexstore.opensearch.uri\": \"http://opensearch:9200\", \"glue.indexstore.opensearch.auth\": \"basicauth\", \"glue.indexstore.opensearch.auth.username\": \"admin\", \"glue.indexstore.opensearch.auth.password\": \"${OPENSEARCH_ADMIN_PASSWORD}\"}}" \
+       http://opensearch-dashboards:5601/api/directquery/dataconnections
+  if [ "$?" -eq "0" ]; then
+    echo "    >>> S3 datasource created"
+  else
+    echo "    >>> Failed to create S3 datasource"
+  fi
+
+  echo ">>> Setting cluster settings"
+  curl -v \
+       -u "admin:${OPENSEARCH_ADMIN_PASSWORD}" \
+       -X PUT \
+       -H 'Content-Type: application/json' \
+       -d '{"persistent": {"plugins.query.executionengine.spark.config": "{\"applicationId\":\"integ-test\",\"executionRoleARN\":\"arn:aws:iam::xxxxx:role/emr-job-execution-role\",\"region\":\"us-west-2\", \"sparkSubmitParameters\": \"--conf spark.dynamicAllocation.enabled=false\"}"}}' \
+       http://opensearch:9200/_cluster/settings
+  if [ "$?" -eq "0" ]; then
+    echo "    >>> Successfully set cluster settings"
+  else
+    echo "    >>> Failed to set cluster settings"
+  fi
+fi