Arrow Flight Server bootstrap logic

CHANGELOG-3.0.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added pull-based Ingestion (APIs, for ingestion source, a Kafka plugin, and IngestionEngine that pulls data from the ingestion source) ([#16958](https://github.com/opensearch-project/OpenSearch/pull/16958))
 - Added ConfigurationUtils to core for the ease of configuration parsing [#17223](https://github.com/opensearch-project/OpenSearch/pull/17223)
 - Add execution_hint to cardinality aggregator request (#[17312](https://github.com/opensearch-project/OpenSearch/pull/17312))
+- Arrow Flight RPC plugin with Flight server bootstrap logic and client for internode communication ([#16962](https://github.com/opensearch-project/OpenSearch/pull/16962))
 
 ### Dependencies
 - Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))

codecov.yml

@@ -4,6 +4,7 @@ codecov:
 ignore:
   - "test"
   - "benchmarks"
+  - "plugins/arrow-flight-rpc/**/org/apache/arrow/flight/**"
 
 coverage:
   precision: 2

gradle/libs.versions.toml

@@ -82,7 +82,7 @@ opentelemetry         = "1.46.0"
 opentelemetrysemconv  = "1.29.0-alpha"
 
 # arrow dependencies
-arrow                 = "17.0.0"
+arrow                 = "18.1.0"
 flatbuffers           = "2.0.0"
 
 [libraries]

libs/arrow-spi/build.gradle

@@ -10,79 +10,11 @@
  */
 
 testingConventions.enabled = false
+
 dependencies {
   api project(':libs:opensearch-core')
-  api "org.apache.arrow:arrow-vector:${versions.arrow}"
-  api "org.apache.arrow:arrow-format:${versions.arrow}"
-  api "org.apache.arrow:arrow-memory-core:${versions.arrow}"
-  runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}"
-  runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}"
-  runtimeOnly "io.netty:netty-buffer:${versions.netty}"
-  runtimeOnly "io.netty:netty-common:${versions.netty}"
-
-  runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
-  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
-  runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-
-  implementation "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
 tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
-
-tasks.named('thirdPartyAudit').configure {
-  ignoreMissingClasses(
-    // Logging frameworks
-    'org.apache.commons.logging.Log',
-    'org.apache.commons.logging.LogFactory',
-    'org.apache.log4j.Level',
-    'org.apache.log4j.Logger',
-    'org.slf4j.impl.StaticLoggerBinder',
-    'org.slf4j.impl.StaticMDCBinder',
-    'org.slf4j.impl.StaticMarkerBinder',
-
-    // Reactor BlockHound
-    'reactor.blockhound.BlockHound$Builder',
-    'reactor.blockhound.integration.BlockHoundIntegration'
-  )
-
-  ignoreViolations(
-    "io.netty.util.internal.PlatformDependent0",
-    "io.netty.util.internal.PlatformDependent0\$1",
-    "io.netty.util.internal.PlatformDependent0\$2",
-    "io.netty.util.internal.PlatformDependent0\$3",
-    "io.netty.util.internal.PlatformDependent0\$4",
-    "io.netty.util.internal.PlatformDependent0\$6",
-    "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef",
-    "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef",
-    "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields",
-    "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields",
-    "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields",
-    "io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode",
-    "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField",
-    "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField",
-    "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField",
-    "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField",
-    "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField",
-    "io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess",
-    "io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess",
-    "io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess",
-    "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField",
-    "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField",
-    "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField",
-    "org.apache.arrow.memory.ArrowBuf",
-    "org.apache.arrow.memory.util.ByteFunctionHelpers",
-    "org.apache.arrow.memory.util.MemoryUtil",
-    "org.apache.arrow.memory.util.MemoryUtil\$1",
-    "org.apache.arrow.memory.util.hash.MurmurHasher",
-    "org.apache.arrow.memory.util.hash.SimpleHasher",
-    "org.apache.arrow.vector.BaseFixedWidthVector",
-    "org.apache.arrow.vector.BitVectorHelper",
-    "org.apache.arrow.vector.Decimal256Vector",
-    "org.apache.arrow.vector.DecimalVector",
-    "org.apache.arrow.vector.util.DecimalUtility",
-    "org.apache.arrow.vector.util.VectorAppender"
-  )
-}