Skip to content

Commit 6d3fd37

Browse files
rishabhmauryarishabhmaurya
authored
Library changes for Apache Arrow integration (#16691)
* Library changes for arrow integration Signed-off-by: Rishabh Maurya <[email protected]> * Bump guava 32->33 Signed-off-by: Rishabh Maurya <[email protected]> * add support for onCancel and Cancellable for BatchedJob in lib:arrow module Signed-off-by: Rishabh Maurya <[email protected]> * address PR comments Signed-off-by: Rishabh Maurya <[email protected]> * Move StreamTicket to an interface Signed-off-by: Rishabh Maurya <[email protected]> * remove jackson dependencies Signed-off-by: Rishabh Maurya <[email protected]> * make sl4j runtime only Signed-off-by: Rishabh Maurya <[email protected]> * introduce factory for stream ticket Signed-off-by: Rishabh Maurya <[email protected]> * Address PR comments Signed-off-by: Rishabh Maurya <[email protected]> --------- Signed-off-by: Rishabh Maurya <[email protected]>
1 parent ad982c2 commit 6d3fd37

File tree

54 files changed

+13613
-6
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+13613
-6
lines changed

‎gradle/libs.versions.toml

+5-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jettison = "1.5.4"
2020
woodstox = "6.4.0"
2121
kotlin = "1.7.10"
2222
antlr4 = "4.13.1"
23-
guava = "32.1.1-jre"
23+
guava = "33.2.1-jre"
2424
protobuf = "3.25.5"
2525
jakarta_annotation = "1.3.5"
2626
google_http_client = "1.44.1"
@@ -79,3 +79,7 @@ resteasy = "6.2.4.Final"
7979
# opentelemetry dependencies
8080
opentelemetry = "1.41.0"
8181
opentelemetrysemconv = "1.27.0-alpha"
82+
83+
# arrow dependencies
84+
arrow = "17.0.0"
85+
flatbuffers = "2.0.0"

‎libs/arrow-spi/build.gradle

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*
8+
* Modifications Copyright OpenSearch Contributors. See
9+
* GitHub history for details.
10+
*/
11+
12+
testingConventions.enabled = false
13+
dependencies {
14+
api project(':libs:opensearch-core')
15+
api "org.apache.arrow:arrow-vector:${versions.arrow}"
16+
api "org.apache.arrow:arrow-format:${versions.arrow}"
17+
api "org.apache.arrow:arrow-memory-core:${versions.arrow}"
18+
runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}"
19+
runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}"
20+
runtimeOnly "io.netty:netty-buffer:${versions.netty}"
21+
runtimeOnly "io.netty:netty-common:${versions.netty}"
22+
23+
runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
24+
runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
25+
runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
26+
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
27+
28+
implementation "commons-codec:commons-codec:${versions.commonscodec}"
29+
}
30+
31+
tasks.named('forbiddenApisMain').configure {
32+
replaceSignatureFiles 'jdk-signatures'
33+
}
34+
35+
tasks.named('thirdPartyAudit').configure {
36+
ignoreMissingClasses(
37+
// Logging frameworks
38+
'org.apache.commons.logging.Log',
39+
'org.apache.commons.logging.LogFactory',
40+
'org.apache.log4j.Level',
41+
'org.apache.log4j.Logger',
42+
'org.slf4j.impl.StaticLoggerBinder',
43+
'org.slf4j.impl.StaticMDCBinder',
44+
'org.slf4j.impl.StaticMarkerBinder',
45+
46+
// Reactor BlockHound
47+
'reactor.blockhound.BlockHound$Builder',
48+
'reactor.blockhound.integration.BlockHoundIntegration'
49+
)
50+
51+
ignoreViolations(
52+
"io.netty.util.internal.PlatformDependent0",
53+
"io.netty.util.internal.PlatformDependent0\$1",
54+
"io.netty.util.internal.PlatformDependent0\$2",
55+
"io.netty.util.internal.PlatformDependent0\$3",
56+
"io.netty.util.internal.PlatformDependent0\$4",
57+
"io.netty.util.internal.PlatformDependent0\$6",
58+
"io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef",
59+
"io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef",
60+
"io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields",
61+
"io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields",
62+
"io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields",
63+
"io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode",
64+
"io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField",
65+
"io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField",
66+
"io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField",
67+
"io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField",
68+
"io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField",
69+
"io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess",
70+
"io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess",
71+
"io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess",
72+
"io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField",
73+
"io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField",
74+
"io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField",
75+
"org.apache.arrow.memory.ArrowBuf",
76+
"org.apache.arrow.memory.util.ByteFunctionHelpers",
77+
"org.apache.arrow.memory.util.MemoryUtil",
78+
"org.apache.arrow.memory.util.MemoryUtil\$1",
79+
"org.apache.arrow.memory.util.hash.MurmurHasher",
80+
"org.apache.arrow.memory.util.hash.SimpleHasher",
81+
"org.apache.arrow.vector.BaseFixedWidthVector",
82+
"org.apache.arrow.vector.BitVectorHelper",
83+
"org.apache.arrow.vector.Decimal256Vector",
84+
"org.apache.arrow.vector.DecimalVector",
85+
"org.apache.arrow.vector.util.DecimalUtility",
86+
"org.apache.arrow.vector.util.VectorAppender"
87+
)
88+
}

‎libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
5d052f20fd1193840eb59818515e710156c364b2

0 commit comments

Comments
 (0)