Merge branch 'apache:master' into master

Author: laxmanchekka

.asf.yaml

@@ -31,12 +31,12 @@ github:
     rebase: false
   collaborators:
     - sullis
-    - shenyu0127
-    - tibrewalpratik17
-    - abhioncbr
-    - zhtaoxiang
+    - yashmayya
     - shounakmk219
     - itschrispeck
+    - zhtaoxiang
     - soumitra-st
-    - swaminathanmanish
-    - yashmayya
+    - suddendust
+    - vrajat
+    - deepthi912
+    - timveil

.github/dependabot.yml

@@ -25,6 +25,19 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 20
+    ignore:
+    - dependency-name: "org.apache.pinot:pinot-dropwizard"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["0.10.0"]
+    - dependency-name: "org.apache.pinot:pinot-yammer"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["0.10.0"]
+    - dependency-name: "commons-io:commons-io"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["2.11.0"]
+    - dependency-name: "com.yammer.metrics:metrics-core"
+      # Locked test resource for pinot-spi/src/test/java/org/apache/pinot/spi/plugin/ClassLoaderTest.java
+      versions: ["2.1.5"]
 
   - package-ecosystem: "npm"
     directory: "/pinot-controller/src/main/resources"

.github/workflows/pinot_tests.yml

@@ -123,6 +123,11 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+            --add-opens=java.base/java.nio=ALL-UNNAMED
+            --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+            --add-opens=java.base/java.lang=ALL-UNNAMED
+            --add-opens=java.base/java.util=ALL-UNNAMED
+            --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
       - name: Unit Test
         env:
@@ -140,6 +145,11 @@ jobs:
             --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
             --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+            --add-opens=java.base/java.nio=ALL-UNNAMED
+            --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+            --add-opens=java.base/java.lang=ALL-UNNAMED
+            --add-opens=java.base/java.util=ALL-UNNAMED
+            --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
         run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh

@@ -29,5 +29,5 @@ netstat -i
 cd pinot-integration-tests || exit 1
 if [ "$RUN_TEST_SET" == "1" ]; then
   mvn test \
-      -P github-actions,custom-cluster-integration-test-suite || exit 1
+      -P github-actions,codecoverage,custom-cluster-integration-test-suite || exit 1
 fi

.github/workflows/scripts/pr-tests/.pinot_tests_integration.sh

@@ -29,9 +29,9 @@ netstat -i
 cd pinot-integration-tests || exit 1
 if [ "$RUN_TEST_SET" == "1" ]; then
   mvn test \
-      -P github-actions,integration-tests-set-1 && exit 0 || exit 1
+      -P github-actions,codecoverage,integration-tests-set-1 && exit 0 || exit 1
 fi
 if [ "$RUN_TEST_SET" == "2" ]; then
   mvn test \
-      -P github-actions,integration-tests-set-2 && exit 0 || exit 1
+      -P github-actions,codecoverage,integration-tests-set-2 && exit 0 || exit 1
 fi

.github/workflows/scripts/pr-tests/.pinot_tests_unit.sh

@@ -37,7 +37,7 @@ if [ "$RUN_TEST_SET" == "1" ]; then
       -pl 'pinot-core' \
       -pl 'pinot-query-planner' \
       -pl 'pinot-query-runtime' \
-      -P github-actions,no-integration-tests || exit 1
+      -P github-actions,codecoverage,no-integration-tests || exit 1
 fi
 if [ "$RUN_TEST_SET" == "2" ]; then
   mvn test -T 16 \
@@ -48,5 +48,7 @@ if [ "$RUN_TEST_SET" == "2" ]; then
     -pl '!pinot-query-planner' \
     -pl '!pinot-query-runtime' \
     -pl '!:pinot-yammer' \
-    -P github-actions,no-integration-tests || exit 1
+    -P github-actions,codecoverage,no-integration-tests || exit 1
 fi
+
+mvn jacoco:report-aggregate@report -P codecoverage || exit 1

.gitignore

@@ -6,7 +6,8 @@ cscope.*
 .externalToolBuilders/
 maven-eclipse.xml
 target/
-examples/
+/examples/
+/logs/
 bin/
 */bin/
 .idea
@@ -45,7 +46,8 @@ yarn-error.log*
 quickstart*
 
 #build symlink directory
-build*
+build
+build/*
 
 #helm related files
 kubernetes/helm/**/charts/

.mvn/extensions.xml

@@ -28,6 +28,6 @@
     <extension>
         <groupId>com.gradle</groupId>
         <artifactId>common-custom-user-data-maven-extension</artifactId>
-        <version>2</version>
+        <version>2.0.1</version>
     </extension>
 </extensions>

README.md

@@ -87,6 +87,15 @@ SELECT sum(clicks), sum(impressions) FROM AdAnalyticsTable
        daysSinceEpoch TOP 100
 ```
 
+## Contributing to Pinot
+Please refer to the [Contribution Guide](https://docs.pinot.apache.org/developers/developers-and-contributors/contribution-guidelines) for 
+more information on how to contribute to Apache Pinot.
+
+Normal Pinot builds are done using the `mvn clean install` command. 
+However this command can take a long time to run.
+For faster builds it is recommended to use `mvn verify -Ppinot-fastdev`, which disables some plugins that are not 
+actually needed for development. 
+
 ## Building Pinot
 More detailed instructions can be found at [Quick Demo](https://docs.pinot.apache.org/basics/getting-started/quick-start) section in the documentation.
 ```
@@ -95,7 +104,9 @@ $ git clone https://github.com/apache/pinot.git
 $ cd pinot
 
 # Build Pinot
-$ mvn clean install -DskipTests -Pbin-dist
+# -Pbin-dist is required to build the binary distribution
+# -Pbuild-shaded-jar is required to build the shaded jar, which is necessary for some features like spark connectors
+$ mvn clean install -DskipTests -Pbin-dist -Pbuild-shaded-jar
 
 # Run the Quick Demo
 $ cd build/

docker/images/pinot-base/pinot-base-runtime/ms-openjdk.dockerfile

@@ -20,12 +20,19 @@
 ARG JAVA_VERSION=11
 ARG JDK_IMAGE=mcr.microsoft.com/openjdk/jdk
 
-FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu
+FROM ${JDK_IMAGE}:${JAVA_VERSION}-ubuntu AS builder
+
+FROM ubuntu:24.10
+ARG JAVA_VERSION
+ENV LANG=en_US.UTF-8
+ENV JAVA_HOME="/usr/lib/jvm/msopenjdk-${JAVA_VERSION}"
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+COPY --from=builder $JAVA_HOME $JAVA_HOME
 
 LABEL [email protected]
 
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends vim less wget curl git python sysstat procps linux-tools-generic libtasn1-6 zstd && \
+  apt-get install -y --no-install-recommends vim less wget curl git python-is-python3 sysstat procps linux-tools-generic libtasn1-6 zstd ca-certificates && \
   rm -rf /var/lib/apt/lists/*
 
 RUN case `uname -m` in \