Skip to content

Commit 164a875

Browse files
luohaoluohao
committed
Add Lance connector
1 parent 8073611 commit 164a875

File tree

88 files changed

+9245
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

88 files changed

+9245
-0
lines changed

‎lib/trino-lance-file/pom.xml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<parent>
6+
<groupId>io.trino</groupId>
7+
<artifactId>trino-root</artifactId>
8+
<version>479-SNAPSHOT</version>
9+
<relativePath>../../pom.xml</relativePath>
10+
</parent>
11+
12+
<artifactId>trino-lance-file</artifactId>
13+
<description>Trino - Lance file format</description>
14+
15+
<properties>
16+
<air.test.jvm.additional-arguments>${air.test.jvm.additional-arguments.default}
17+
--add-opens=java.base/java.nio=ALL-UNNAMED
18+
--sun-misc-unsafe-memory-access=allow</air.test.jvm.additional-arguments>
19+
</properties>
20+
21+
<dependencies>
22+
<dependency>
23+
<groupId>build.buf.gen</groupId>
24+
<artifactId>lancedb_lance_protocolbuffers_java</artifactId>
25+
</dependency>
26+
27+
<dependency>
28+
<groupId>com.fasterxml.jackson.core</groupId>
29+
<artifactId>jackson-annotations</artifactId>
30+
</dependency>
31+
32+
<dependency>
33+
<groupId>com.google.errorprone</groupId>
34+
<artifactId>error_prone_annotations</artifactId>
35+
</dependency>
36+
37+
<dependency>
38+
<groupId>com.google.guava</groupId>
39+
<artifactId>guava</artifactId>
40+
</dependency>
41+
42+
<dependency>
43+
<groupId>com.google.protobuf</groupId>
44+
<artifactId>protobuf-java</artifactId>
45+
</dependency>
46+
47+
<dependency>
48+
<groupId>io.github.luohao</groupId>
49+
<artifactId>fastlanes-java</artifactId>
50+
</dependency>
51+
52+
<dependency>
53+
<groupId>io.trino</groupId>
54+
<artifactId>trino-filesystem</artifactId>
55+
</dependency>
56+
57+
<dependency>
58+
<groupId>io.trino</groupId>
59+
<artifactId>trino-memory-context</artifactId>
60+
</dependency>
61+
62+
<dependency>
63+
<groupId>io.trino</groupId>
64+
<artifactId>trino-plugin-toolkit</artifactId>
65+
</dependency>
66+
67+
<dependency>
68+
<groupId>it.unimi.dsi</groupId>
69+
<artifactId>fastutil</artifactId>
70+
</dependency>
71+
72+
<dependency>
73+
<groupId>jakarta.annotation</groupId>
74+
<artifactId>jakarta.annotation-api</artifactId>
75+
</dependency>
76+
77+
<dependency>
78+
<groupId>io.airlift</groupId>
79+
<artifactId>slice</artifactId>
80+
<scope>provided</scope>
81+
</dependency>
82+
83+
<dependency>
84+
<groupId>io.trino</groupId>
85+
<artifactId>trino-spi</artifactId>
86+
<scope>provided</scope>
87+
</dependency>
88+
89+
<dependency>
90+
<groupId>org.openjdk.jol</groupId>
91+
<artifactId>jol-core</artifactId>
92+
<scope>provided</scope>
93+
</dependency>
94+
95+
<dependency>
96+
<groupId>com.lancedb</groupId>
97+
<artifactId>lance-core</artifactId>
98+
<scope>test</scope>
99+
</dependency>
100+
101+
<dependency>
102+
<groupId>io.airlift</groupId>
103+
<artifactId>junit-extensions</artifactId>
104+
<scope>test</scope>
105+
</dependency>
106+
107+
<dependency>
108+
<groupId>io.trino</groupId>
109+
<artifactId>trino-main</artifactId>
110+
<scope>test</scope>
111+
</dependency>
112+
113+
<dependency>
114+
<groupId>io.trino</groupId>
115+
<artifactId>trino-testing-services</artifactId>
116+
<scope>test</scope>
117+
</dependency>
118+
119+
<dependency>
120+
<groupId>org.apache.arrow</groupId>
121+
<artifactId>arrow-memory-core</artifactId>
122+
<scope>test</scope>
123+
</dependency>
124+
125+
<dependency>
126+
<groupId>org.apache.arrow</groupId>
127+
<artifactId>arrow-vector</artifactId>
128+
<scope>test</scope>
129+
</dependency>
130+
131+
<dependency>
132+
<groupId>org.apache.calcite.avatica</groupId>
133+
<artifactId>avatica-core</artifactId>
134+
<version>1.25.0</version>
135+
<scope>test</scope>
136+
</dependency>
137+
138+
<dependency>
139+
<groupId>org.assertj</groupId>
140+
<artifactId>assertj-core</artifactId>
141+
<scope>test</scope>
142+
</dependency>
143+
144+
<dependency>
145+
<groupId>org.junit.jupiter</groupId>
146+
<artifactId>junit-jupiter-api</artifactId>
147+
<scope>test</scope>
148+
</dependency>
149+
150+
<dependency>
151+
<groupId>org.junit.jupiter</groupId>
152+
<artifactId>junit-jupiter-engine</artifactId>
153+
<scope>test</scope>
154+
</dependency>
155+
156+
<dependency>
157+
<groupId>org.openjdk.jmh</groupId>
158+
<artifactId>jmh-core</artifactId>
159+
<scope>test</scope>
160+
</dependency>
161+
162+
<dependency>
163+
<groupId>org.openjdk.jmh</groupId>
164+
<artifactId>jmh-generator-annprocess</artifactId>
165+
<scope>test</scope>
166+
</dependency>
167+
</dependencies>
168+
169+
<repositories>
170+
<repository>
171+
<releases>
172+
<enabled>true</enabled>
173+
</releases>
174+
<snapshots>
175+
<enabled>false</enabled>
176+
</snapshots>
177+
<id>buf</id>
178+
<name>Buf Maven Repository</name>
179+
<url>https://buf.build/gen/maven</url>
180+
</repository>
181+
</repositories>
182+
</project>

‎lib/trino-lance-file/src/main/java/io/trino/lance/file/AbstractLanceDataSource.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.lance.file;
15+
16+
import io.airlift.slice.Slice;
17+
import io.airlift.slice.Slices;
18+
19+
import java.io.IOException;
20+
21+
import static java.lang.Math.min;
22+
import static java.lang.Math.toIntExact;
23+
import static java.util.Objects.requireNonNull;
24+
25+
public abstract class AbstractLanceDataSource
26+
implements LanceDataSource
27+
{
28+
private final LanceDataSourceId id;
29+
private final long estimatedSize;
30+
private long readTimeNanos;
31+
private long readBytes;
32+
33+
public AbstractLanceDataSource(LanceDataSourceId id, long estimatedSize)
34+
{
35+
this.id = requireNonNull(id, "id is null");
36+
this.estimatedSize = estimatedSize;
37+
}
38+
39+
@Override
40+
public LanceDataSourceId getId()
41+
{
42+
return id;
43+
}
44+
45+
@Override
46+
public long getReadBytes()
47+
{
48+
return readBytes;
49+
}
50+
51+
@Override
52+
public long getReadTimeNanos()
53+
{
54+
return readTimeNanos;
55+
}
56+
57+
@Override
58+
public long getEstimatedSize()
59+
{
60+
return estimatedSize;
61+
}
62+
63+
@Override
64+
public Slice readTail(int length)
65+
throws IOException
66+
{
67+
long start = System.nanoTime();
68+
69+
Slice tailSlice = readTailInternal(length);
70+
71+
readTimeNanos += System.nanoTime() - start;
72+
readBytes += tailSlice.length();
73+
74+
return tailSlice;
75+
}
76+
77+
protected Slice readTailInternal(int length)
78+
throws IOException
79+
{
80+
int readSize = toIntExact(min(estimatedSize, length));
81+
return readFully(estimatedSize - readSize, readSize);
82+
}
83+
84+
@Override
85+
public long getRetainedSize()
86+
{
87+
return 0;
88+
}
89+
90+
@Override
91+
public final Slice readFully(long position, int length)
92+
throws IOException
93+
{
94+
byte[] buffer = new byte[length];
95+
readFully(position, buffer, 0, length);
96+
return Slices.wrappedBuffer(buffer);
97+
}
98+
99+
protected abstract void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength)
100+
throws IOException;
101+
102+
private void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength)
103+
throws IOException
104+
{
105+
long start = System.nanoTime();
106+
107+
readInternal(position, buffer, bufferOffset, bufferLength);
108+
109+
readTimeNanos += System.nanoTime() - start;
110+
readBytes += bufferLength;
111+
}
112+
113+
@Override
114+
public final String toString()
115+
{
116+
return id.toString();
117+
}
118+
}

‎lib/trino-lance-file/src/main/java/io/trino/lance/file/FileLanceDataSource.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.lance.file;
15+
16+
import java.io.File;
17+
import java.io.FileNotFoundException;
18+
import java.io.IOException;
19+
import java.io.RandomAccessFile;
20+
21+
public class FileLanceDataSource
22+
extends AbstractLanceDataSource
23+
{
24+
private final RandomAccessFile input;
25+
26+
public FileLanceDataSource(File path)
27+
throws FileNotFoundException
28+
{
29+
super(new LanceDataSourceId(path.getPath()), path.length());
30+
this.input = new RandomAccessFile(path, "r");
31+
}
32+
33+
@Override
34+
protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength)
35+
throws IOException
36+
{
37+
input.seek(position);
38+
input.readFully(buffer, bufferOffset, bufferLength);
39+
}
40+
41+
@Override
42+
public void close()
43+
throws IOException
44+
{
45+
input.close();
46+
}
47+
}

‎lib/trino-lance-file/src/main/java/io/trino/lance/file/LanceDataSource.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.lance.file;
15+
16+
import io.airlift.slice.Slice;
17+
18+
import java.io.Closeable;
19+
import java.io.IOException;
20+
21+
public interface LanceDataSource
22+
extends Closeable
23+
{
24+
LanceDataSourceId getId();
25+
26+
long getReadBytes();
27+
28+
long getReadTimeNanos();
29+
30+
long getEstimatedSize();
31+
32+
long getRetainedSize();
33+
34+
Slice readTail(int length)
35+
throws IOException;
36+
37+
Slice readFully(long position, int length)
38+
throws IOException;
39+
40+
@Override
41+
default void close()
42+
throws IOException
43+
{
44+
}
45+
}

0 commit comments

Comments
 (0)