Skip to content

Commit afe74cc

Browse files
luohaoluohao
committed
fixup! Add docs and update product-tests
1 parent 29a0e87 commit afe74cc

File tree

11 files changed

+221
-81
lines changed

11 files changed

+221
-81
lines changed

‎docs/src/main/sphinx/connector.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ Ignite <connector/ignite>
2626
JMX <connector/jmx>
2727
Kafka <connector/kafka>
2828
Lakehouse <connector/lakehouse>
29+
Lance <connector/lance>
2930
Loki <connector/loki>
3031
MariaDB <connector/mariadb>
3132
Memory <connector/memory>

‎docs/src/main/sphinx/connector/lance.md‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Lance connector
2+
3+
## General configuration
4+
5+
To configure the Lance connector, create a catalog properties file `etc/catalog/example.properties` with the following content
6+
7+
```text
8+
connector.name=lance
9+
```
10+
11+
You must configure a [namespace](https://lancedb.github.io/lance/format/namespace/) type.
12+
Currently only [directory namespace](https://lancedb.github.io/lance/format/namespace/impls/dir/) is supported.
13+
14+
```text
15+
lance.namespace.type=directory
16+
```
17+
18+
## Lance Namespace configuration
19+
### Directory namespace
20+
Lance directory namespace is a lightweight and simple single-level Lance namespace that contains only a list of tables. All tables reside in the default namespace.
21+
22+
The following configuration properties are available:
23+
24+
:::{list-table}
25+
:widths: 30, 10, 60
26+
:header-rows: 1
27+
28+
* - Property name
29+
- Required
30+
- Description
31+
* - `lance.namespace.directory.warehouse.location`
32+
- Yes
33+
- The root directory URI of the namespace where tables are stored.
34+
:::
35+
36+
37+
## File system access configuration
38+
39+
The connector supports accessing the following file systems:
40+
41+
* [](/object-storage/file-system-azure)
42+
* [](/object-storage/file-system-gcs)
43+
* [](/object-storage/file-system-s3)
44+
* [](/object-storage/file-system-hdfs)
45+
46+
You must enable and configure the specific file system access.

‎plugin/trino-lance/src/main/java/io/trino/plugin/lance/LanceMetadata.java‎

Lines changed: 4 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -56,22 +56,6 @@ public LanceMetadata(TrinoCatalog catalog)
5656
this.catalog = requireNonNull(catalog, "catalog is null");
5757
}
5858

59-
private static long getSnapshotIdFromVersion(ConnectorTableVersion version)
60-
{
61-
io.trino.spi.type.Type versionType = version.getVersionType();
62-
return switch (version.getPointerType()) {
63-
// TODO: list and search versions to do temporal time travel
64-
case TEMPORAL -> throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables with temporal version");
65-
case TARGET_ID -> {
66-
if (versionType != BIGINT) {
67-
throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables: unsupported type for table version " + versionType.getDisplayName());
68-
}
69-
// TODO: support String type target id
70-
yield (long) version.getVersion();
71-
}
72-
};
73-
}
74-
7559
@Override
7660
public List<String> listSchemaNames(ConnectorSession session)
7761
{
@@ -118,22 +102,16 @@ public Iterator<RelationColumnsMetadata> streamRelationColumns(ConnectorSession
118102
@Override
119103
public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName, Optional<ConnectorTableVersion> startVersion, Optional<ConnectorTableVersion> endVersion)
120104
{
121-
if (startVersion.isPresent()) {
122-
throw new TrinoException(NOT_SUPPORTED, "Read table with startRowPosition version is not supported");
105+
if (startVersion.isPresent() || endVersion.isPresent()) {
106+
throw new TrinoException(NOT_SUPPORTED, "This connector does not support versioned tables");
123107
}
108+
124109
Optional<BaseTable> loadedTable = catalog.loadTable(session, tableName);
125110
if (loadedTable.isEmpty()) {
126111
return null;
127112
}
128113
BaseTable baseTable = loadedTable.get();
129-
Optional<Long> version;
130-
if (endVersion.isPresent()) {
131-
version = Optional.of(getSnapshotIdFromVersion(endVersion.get()));
132-
}
133-
else {
134-
version = Optional.empty();
135-
}
136-
return new LanceTableHandle(tableName, baseTable.loadManifest(version), baseTable.getTableLocation().toString());
114+
return new LanceTableHandle(tableName, baseTable.loadManifest(Optional.empty()), baseTable.getTableLocation().toString());
137115
}
138116

139117
@Override

‎testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeLance.java‎

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeLanceMinio.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.tests.product.launcher.env.environment;
15+
16+
import com.google.common.collect.ImmutableMap;
17+
import com.google.inject.Inject;
18+
import io.trino.tests.product.launcher.docker.DockerFiles;
19+
import io.trino.tests.product.launcher.env.DockerContainer;
20+
import io.trino.tests.product.launcher.env.Environment;
21+
import io.trino.tests.product.launcher.env.EnvironmentConfig;
22+
import io.trino.tests.product.launcher.env.EnvironmentProvider;
23+
import io.trino.tests.product.launcher.env.common.Minio;
24+
import io.trino.tests.product.launcher.env.common.StandardMultinode;
25+
import io.trino.tests.product.launcher.env.common.TestsEnvironment;
26+
import io.trino.tests.product.launcher.testcontainers.PortBinder;
27+
import org.testcontainers.containers.BindMode;
28+
import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy;
29+
30+
import java.io.File;
31+
import java.io.IOException;
32+
import java.io.UncheckedIOException;
33+
import java.nio.file.Files;
34+
import java.nio.file.Path;
35+
import java.nio.file.attribute.FileAttribute;
36+
import java.nio.file.attribute.PosixFilePermission;
37+
import java.nio.file.attribute.PosixFilePermissions;
38+
import java.util.Set;
39+
40+
import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts;
41+
import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS;
42+
import static io.trino.tests.product.launcher.env.common.Minio.MINIO_CONTAINER_NAME;
43+
import static java.util.Objects.requireNonNull;
44+
import static org.testcontainers.utility.MountableFile.forHostPath;
45+
46+
@TestsEnvironment
47+
public class EnvMultinodeLanceMinio
48+
extends EnvironmentProvider
49+
{
50+
private static final File HIVE_JDBC_PROVIDER = new File("testing/trino-product-tests-launcher/target/hive-jdbc.jar");
51+
private static final String S3_BUCKET_NAME = "test-bucket";
52+
// TODO: use official image from Lance once https://github.com/lancedb/lance-spark/issues/102 is fixed
53+
private static final String SPARK_IMAGE = "docker.io/hluoulh/lance-spark:3.5.7-15";
54+
private static final String SPARK_CONTAINER_NAME = "spark";
55+
private static final int SPARK_THRIFT_PORT = 10213;
56+
private static final String SPARK = "spark";
57+
private static final String MINIO = "minio";
58+
59+
private final DockerFiles.ResourceProvider configDir;
60+
private final PortBinder portBinder;
61+
private final String hadoopImagesVersion;
62+
63+
@Inject
64+
public EnvMultinodeLanceMinio(StandardMultinode standardMultinode, Minio minio, DockerFiles dockerFiles, EnvironmentConfig config, PortBinder portBinder)
65+
{
66+
super(standardMultinode, minio);
67+
this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/multinode-lance-minio");
68+
this.portBinder = requireNonNull(portBinder, "portBinder is null");
69+
this.hadoopImagesVersion = requireNonNull(config, "config is null").getHadoopImagesVersion();
70+
}
71+
72+
@Override
73+
public void extendEnvironment(Environment.Builder builder)
74+
{
75+
builder.configureContainer(TESTS, dockerContainer -> dockerContainer.withEnv("S3_BUCKET", S3_BUCKET_NAME));
76+
77+
builder.addContainer(createSparkContainer()).containerDependsOn(SPARK, MINIO);
78+
// initialize buckets in minio
79+
FileAttribute<Set<PosixFilePermission>> posixFilePermissions = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--"));
80+
Path minioBucketDirectory;
81+
try {
82+
minioBucketDirectory = Files.createTempDirectory("test-bucket-contents", posixFilePermissions);
83+
minioBucketDirectory.toFile().deleteOnExit();
84+
}
85+
catch (IOException e) {
86+
throw new UncheckedIOException(e);
87+
}
88+
builder.configureContainer(MINIO_CONTAINER_NAME, container ->
89+
{
90+
container.withCopyFileToContainer(forHostPath(minioBucketDirectory), "/data/" + S3_BUCKET_NAME);
91+
});
92+
93+
builder.addConnector("lance", forHostPath(configDir.getPath("lance.properties")));
94+
95+
builder.configureContainer(TESTS, dockerContainer -> dockerContainer
96+
.withFileSystemBind(HIVE_JDBC_PROVIDER.getParent(), "/docker/jdbc", BindMode.READ_ONLY));
97+
}
98+
99+
@SuppressWarnings("resource")
100+
private DockerContainer createSparkContainer()
101+
{
102+
DockerContainer container = new DockerContainer(SPARK_IMAGE, SPARK_CONTAINER_NAME)
103+
.withCopyFileToContainer(
104+
forHostPath(configDir.getPath("spark-defaults.conf")),
105+
"/opt/spark/conf/spark-defaults.conf")
106+
.withEnv(ImmutableMap.of("HIVE_SERVER2_THRIFT_PORT", String.valueOf(SPARK_THRIFT_PORT)))
107+
.withStartupCheckStrategy(new IsRunningStartupCheckStrategy())
108+
.waitingFor(forSelectedPorts(SPARK_THRIFT_PORT));
109+
portBinder.exposePort(container, SPARK_THRIFT_PORT);
110+
return container;
111+
}
112+
}

‎testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteLance.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
import com.google.common.collect.ImmutableList;
1717
import io.trino.tests.product.launcher.env.EnvironmentConfig;
18-
import io.trino.tests.product.launcher.env.environment.EnvMultinodeLance;
18+
import io.trino.tests.product.launcher.env.environment.EnvMultinodeLanceMinio;
1919
import io.trino.tests.product.launcher.suite.Suite;
2020
import io.trino.tests.product.launcher.suite.SuiteTestRun;
2121

@@ -31,7 +31,7 @@ public class SuiteLance
3131
public List<SuiteTestRun> getTestRuns(EnvironmentConfig config)
3232
{
3333
return ImmutableList.of(
34-
SuiteTestRun.testOnEnvironment(EnvMultinodeLance.class)
34+
SuiteTestRun.testOnEnvironment(EnvMultinodeLanceMinio.class)
3535
.withGroups(CONFIGURED_FEATURES, LANCE)
3636
.build());
3737
}

‎testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-lance-minio/jvm.config‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
-server
2+
--add-opens=java.base/java.nio=ALL-UNNAMED
3+
-Xmx2G
4+
-XX:G1HeapRegionSize=32M
5+
-XX:+ExplicitGCInvokesConcurrent
6+
-XX:+ExitOnOutOfMemoryError
7+
-XX:+HeapDumpOnOutOfMemoryError
8+
-XX:-OmitStackTraceInFastThrow
9+
-XX:ReservedCodeCacheSize=150M
10+
-XX:PerMethodRecompilationCutoff=10000
11+
-XX:PerBytecodeRecompilationCutoff=10000
12+
-Djdk.attach.allowAttachSelf=true
13+
-Duser.timezone=Asia/Kathmandu
14+
-XX:ErrorFile=/docker/logs/product-tests-presto-jvm-error-file.log
15+
# Allow loading dynamic agent used by JOL
16+
-XX:+EnableDynamicAgentLoading
17+
-XX:+ExitOnOutOfMemoryError

‎testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-lance-minio/lance.properties‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
connector.name=lance
2+
lance.namespace.type=directory
3+
lance.namespace.directory.warehouse.location=s3://test-bucket/lance
4+
fs.native-s3.enabled=true
5+
fs.hadoop.enabled=false
6+
s3.region=us-east-1
7+
s3.aws-access-key=minio-access-key
8+
s3.aws-secret-key=minio-secret-key
9+
s3.endpoint=http://minio:9080/
10+
s3.path-style-access=true

‎testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-lance-minio/spark-defaults.conf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
spark.sql.catalog.lance=com.lancedb.lance.spark.LanceNamespaceSparkCatalog
2+
spark.sql.catalog.lance.impl=dir
3+
spark.sql.catalog.lance.root=s3://test-bucket/lance
4+
spark.sql.catalog.lance.storage.endpoint=http://minio:9080
5+
spark.sql.catalog.lance.storage.aws_allow_http=true
6+
spark.sql.catalog.lance.storage.access_key_id=minio-access-key
7+
spark.sql.catalog.lance.storage.secret_access_key=minio-secret-key
8+
spark.sql.catalog.lance.storage.region=us-east-1
9+
spark.sql.defaultCatalog=lance

‎testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-lance/lance.properties‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

0 commit comments

Comments
 (0)