Skip to content

Commit 164ad92

Browse files
ejeffrliejeffrli
authored
Implement EnvironmentProperties for remaining non-jdbc connectors (awslabs#12)
* Implement EnvironmentProperties for remaining non-jdbc connectors
1 parent bb0053f commit 164ad92

File tree

57 files changed

+496
-93
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+496
-93
lines changed

‎athena-aws-cmdb/src/main/java/com/amazonaws/athena/connectors/aws/cmdb/AwsCmdbCompositeHandler.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
package com.amazonaws.athena.connectors.aws.cmdb;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
2323
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2424

2525
/**
@@ -31,6 +31,6 @@ public class AwsCmdbCompositeHandler
3131
{
3232
public AwsCmdbCompositeHandler()
3333
{
34-
super(new AwsCmdbMetadataHandler(GlueConnectionUtils.getGlueConnection()), new AwsCmdbRecordHandler(GlueConnectionUtils.getGlueConnection()));
34+
super(new AwsCmdbMetadataHandler(new EnvironmentProperties().createEnvironment()), new AwsCmdbRecordHandler(new EnvironmentProperties().createEnvironment()));
3535
}
3636
}

‎athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/connection/ClouderaHiveEnvironmentProperties.java ‎athena-cloudera-hive/src/main/java/com/amazonaws/athena/connectors/cloudera/ClouderaHiveEnvironmentProperties.java

+8-5
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,19 @@
1717
* limitations under the License.
1818
* #L%
1919
*/
20-
package com.amazonaws.athena.connector.lambda.connection;
20+
package com.amazonaws.athena.connectors.cloudera;
21+
22+
import com.amazonaws.athena.connectors.jdbc.JdbcEnvironmentProperties;
2123

2224
import java.util.Map;
2325

26+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.HIVE_CONFS;
27+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.HIVE_VARS;
28+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.SECRET_NAME;
29+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.SESSION_CONFS;
30+
2431
public class ClouderaHiveEnvironmentProperties extends JdbcEnvironmentProperties
2532
{
26-
private static final String SESSION_CONFS = "SESSION_CONFS";
27-
private static final String HIVE_CONFS = "HIVE_CONFS";
28-
private static final String HIVE_VARS = "HIVE_VARS";
29-
3033
@Override
3134
protected String getConnectionStringPrefix(Map<String, String> connectionProperties)
3235
{

‎athena-cloudera-hive/src/main/java/com/amazonaws/athena/connectors/cloudera/HiveCompositeHandler.java

-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
*/
2020
package com.amazonaws.athena.connectors.cloudera;
2121

22-
import com.amazonaws.athena.connector.lambda.connection.ClouderaHiveEnvironmentProperties;
2322
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2423

2524
/**

‎athena-cloudera-impala/src/main/java/com/amazonaws/athena/connectors/cloudera/ImpalaCompositeHandler.java

-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
*/
2121
package com.amazonaws.athena.connectors.cloudera;
2222

23-
import com.amazonaws.athena.connector.lambda.connection.ImpalaEnvironmentProperties;
2423
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2524

2625
/**

‎athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/connection/ImpalaEnvironmentProperties.java ‎athena-cloudera-impala/src/main/java/com/amazonaws/athena/connectors/cloudera/ImpalaEnvironmentProperties.java

+10-2
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,23 @@
1717
* limitations under the License.
1818
* #L%
1919
*/
20-
package com.amazonaws.athena.connector.lambda.connection;
20+
package com.amazonaws.athena.connectors.cloudera;
21+
22+
import com.amazonaws.athena.connectors.jdbc.JdbcEnvironmentProperties;
2123

2224
import java.util.Map;
2325

24-
public class ImpalaEnvironmentProperties extends SaphanaEnvironmentProperties
26+
public class ImpalaEnvironmentProperties extends JdbcEnvironmentProperties
2527
{
2628
@Override
2729
protected String getConnectionStringPrefix(Map<String, String> connectionProperties)
2830
{
2931
return "impala://jdbc:impala://";
3032
}
33+
34+
@Override
35+
protected String getDatabase(Map<String, String> connectionProperties)
36+
{
37+
return "/";
38+
}
3139
}

‎athena-cloudwatch-metrics/src/main/java/com/amazonaws/athena/connectors/cloudwatch/metrics/MetricsCompositeHandler.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
package com.amazonaws.athena.connectors.cloudwatch.metrics;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
2323
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2424

2525
/**
@@ -31,6 +31,6 @@ public class MetricsCompositeHandler
3131
{
3232
public MetricsCompositeHandler()
3333
{
34-
super(new MetricsMetadataHandler(GlueConnectionUtils.getGlueConnection()), new MetricsRecordHandler(GlueConnectionUtils.getGlueConnection()));
34+
super(new MetricsMetadataHandler(new EnvironmentProperties().createEnvironment()), new MetricsRecordHandler(new EnvironmentProperties().createEnvironment()));
3535
}
3636
}

‎athena-cloudwatch/src/main/java/com/amazonaws/athena/connectors/cloudwatch/CloudwatchCompositeHandler.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
package com.amazonaws.athena.connectors.cloudwatch;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
2323
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2424

2525
/**
@@ -31,6 +31,6 @@ public class CloudwatchCompositeHandler
3131
{
3232
public CloudwatchCompositeHandler()
3333
{
34-
super(new CloudwatchMetadataHandler(GlueConnectionUtils.getGlueConnection()), new CloudwatchRecordHandler(GlueConnectionUtils.getGlueConnection()));
34+
super(new CloudwatchMetadataHandler(new EnvironmentProperties().createEnvironment()), new CloudwatchRecordHandler(new EnvironmentProperties().createEnvironment()));
3535
}
3636
}

‎athena-datalakegen2/src/main/java/com/amazonaws/athena/connectors/datalakegen2/DataLakeGen2CompositeHandler.java

-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
*/
2020
package com.amazonaws.athena.connectors.datalakegen2;
2121

22-
import com.amazonaws.athena.connector.lambda.connection.DataLakeGen2EnvironmentProperties;
2322
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2423

2524
/**

‎athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/connection/DataLakeGen2EnvironmentProperties.java ‎athena-datalakegen2/src/main/java/com/amazonaws/athena/connectors/datalakegen2/DataLakeGen2EnvironmentProperties.java

+24-2
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,37 @@
1717
* limitations under the License.
1818
* #L%
1919
*/
20-
package com.amazonaws.athena.connector.lambda.connection;
20+
package com.amazonaws.athena.connectors.datalakegen2;
21+
22+
import com.amazonaws.athena.connectors.jdbc.JdbcEnvironmentProperties;
2123

2224
import java.util.Map;
2325

24-
public class DataLakeGen2EnvironmentProperties extends SqlServerEnvironmentProperties
26+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DATABASE;
27+
28+
public class DataLakeGen2EnvironmentProperties extends JdbcEnvironmentProperties
2529
{
2630
@Override
2731
protected String getConnectionStringPrefix(Map<String, String> connectionProperties)
2832
{
2933
return "datalakegentwo://jdbc:sqlserver://";
3034
}
35+
36+
@Override
37+
protected String getDatabase(Map<String, String> connectionProperties)
38+
{
39+
return ";databaseName=" + connectionProperties.get(DATABASE);
40+
}
41+
42+
@Override
43+
protected String getJdbcParametersSeparator()
44+
{
45+
return ";";
46+
}
47+
48+
@Override
49+
protected String getDelimiter()
50+
{
51+
return ";";
52+
}
3153
}

‎athena-db2-as400/src/main/java/com/amazonaws/athena/connectors/db2as400/Db2As400CompositeHandler.java

-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
*/
2020
package com.amazonaws.athena.connectors.db2as400;
2121

22-
import com.amazonaws.athena.connector.lambda.connection.Db2As400EnvironmentProperties;
2322
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2423

2524
/**

‎athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/connection/Db2As400EnvironmentProperties.java ‎athena-db2-as400/src/main/java/com/amazonaws/athena/connectors/db2as400/Db2As400EnvironmentProperties.java

+9-4
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,27 @@
1717
* limitations under the License.
1818
* #L%
1919
*/
20-
package com.amazonaws.athena.connector.lambda.connection;
20+
package com.amazonaws.athena.connectors.db2as400;
21+
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
2123

2224
import java.util.HashMap;
2325
import java.util.Map;
2426

27+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DEFAULT;
28+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.HOST;
29+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.JDBC_PARAMS;
30+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.SECRET_NAME;
31+
2532
public class Db2As400EnvironmentProperties extends EnvironmentProperties
2633
{
27-
private static final String JDBC_PARAMS = "JDBC_PARAMS";
28-
private static final String DEFAULT = "default";
2934
@Override
3035
public Map<String, String> connectionPropertiesToEnvironment(Map<String, String> connectionProperties)
3136
{
3237
HashMap<String, String> environment = new HashMap<>();
3338

3439
// now construct jdbc string
35-
String connectionString = "db2as400://jdbc:as400://" + connectionProperties.get("HOST")
40+
String connectionString = "db2as400://jdbc:as400://" + connectionProperties.get(HOST)
3641
+ ";" + connectionProperties.getOrDefault(JDBC_PARAMS, "");
3742

3843
if (connectionProperties.containsKey(SECRET_NAME)) {

‎athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2CompositeHandler.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
*/
2020
package com.amazonaws.athena.connectors.db2;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
2322
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2423

2524
/**
@@ -32,6 +31,6 @@ public class Db2CompositeHandler extends CompositeHandler
3231
{
3332
public Db2CompositeHandler()
3433
{
35-
super(new Db2MetadataHandler(GlueConnectionUtils.getGlueConnection()), new Db2RecordHandler(GlueConnectionUtils.getGlueConnection()));
34+
super(new Db2MetadataHandler(new Db2EnvironmentProperties().createEnvironment()), new Db2RecordHandler(new Db2EnvironmentProperties().createEnvironment()));
3635
}
3736
}

‎athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/connection/Db2EnvironmentProperties.java ‎athena-db2/src/main/java/com/amazonaws/athena/connectors/db2/Db2EnvironmentProperties.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@
1717
* limitations under the License.
1818
* #L%
1919
*/
20-
package com.amazonaws.athena.connector.lambda.connection;
20+
package com.amazonaws.athena.connectors.db2;
21+
22+
import com.amazonaws.athena.connectors.jdbc.JdbcEnvironmentProperties;
2123

2224
import java.util.Map;
2325

26+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DATABASE;
27+
2428
public class Db2EnvironmentProperties extends JdbcEnvironmentProperties
2529
{
2630
@Override

‎athena-docdb/src/main/java/com/amazonaws/athena/connectors/docdb/DocDBCompositeHandler.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
*/
2020
package com.amazonaws.athena.connectors.docdb;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
2322
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2423

2524
/**
@@ -31,6 +30,6 @@ public class DocDBCompositeHandler
3130
{
3231
public DocDBCompositeHandler()
3332
{
34-
super(new DocDBMetadataHandler(GlueConnectionUtils.getGlueConnection()), new DocDBRecordHandler(GlueConnectionUtils.getGlueConnection()));
33+
super(new DocDBMetadataHandler(new DocDBEnvironmentProperties().createEnvironment()), new DocDBRecordHandler(new DocDBEnvironmentProperties().createEnvironment()));
3534
}
3635
}

‎athena-docdb/src/main/java/com/amazonaws/athena/connectors/docdb/DocDBEnvironmentProperties.java

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*-
2+
* #%L
3+
* Amazon Athena Query Federation SDK
4+
* %%
5+
* Copyright (C) 2019 - 2024 Amazon Web Services
6+
* %%
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
* #L%
19+
*/
20+
package com.amazonaws.athena.connectors.docdb;
21+
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
23+
24+
import java.util.HashMap;
25+
import java.util.Map;
26+
27+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.DEFAULT_DOCDB;
28+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.HOST;
29+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.JDBC_PARAMS;
30+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.PORT;
31+
import static com.amazonaws.athena.connector.lambda.connection.EnvironmentConstants.SECRET_NAME;
32+
33+
public class DocDBEnvironmentProperties extends EnvironmentProperties
34+
{
35+
@Override
36+
public Map<String, String> connectionPropertiesToEnvironment(Map<String, String> connectionProperties)
37+
{
38+
Map<String, String> environment = new HashMap<>();
39+
40+
String connectionString = "mongodb://${" + connectionProperties.get(SECRET_NAME) + "}@"
41+
+ connectionProperties.get(HOST) + connectionProperties.get(PORT) + "/?"
42+
+ connectionProperties.getOrDefault(JDBC_PARAMS, "");
43+
environment.put(DEFAULT_DOCDB, connectionString);
44+
return environment;
45+
}
46+
}

‎athena-dynamodb/src/main/java/com/amazonaws/athena/connectors/dynamodb/DynamoDBCompositeHandler.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
package com.amazonaws.athena.connectors.dynamodb;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
2323
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2424

2525
/**
@@ -31,6 +31,6 @@ public class DynamoDBCompositeHandler
3131
{
3232
public DynamoDBCompositeHandler()
3333
{
34-
super(new DynamoDBMetadataHandler(GlueConnectionUtils.getGlueConnection()), new DynamoDBRecordHandler(GlueConnectionUtils.getGlueConnection()));
34+
super(new DynamoDBMetadataHandler(new EnvironmentProperties().createEnvironment()), new DynamoDBRecordHandler(new EnvironmentProperties().createEnvironment()));
3535
}
3636
}

‎athena-elasticsearch/src/main/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchCompositeHandler.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
*/
2020
package com.amazonaws.athena.connectors.elasticsearch;
2121

22-
import com.amazonaws.athena.connector.lambda.GlueConnectionUtils;
22+
import com.amazonaws.athena.connector.lambda.connection.EnvironmentProperties;
2323
import com.amazonaws.athena.connector.lambda.handlers.CompositeHandler;
2424

2525
/**
@@ -31,6 +31,6 @@ public class ElasticsearchCompositeHandler
3131
{
3232
public ElasticsearchCompositeHandler()
3333
{
34-
super(new ElasticsearchMetadataHandler(GlueConnectionUtils.getGlueConnection()), new ElasticsearchRecordHandler(GlueConnectionUtils.getGlueConnection()));
34+
super(new ElasticsearchMetadataHandler(new EnvironmentProperties().createEnvironment()), new ElasticsearchRecordHandler(new EnvironmentProperties().createEnvironment()));
3535
}
3636
}

‎athena-federation-sdk/src/main/java/com/amazonaws/athena/connector/lambda/connection/EnvironmentConstants.java

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*-
2+
* #%L
3+
* Amazon Athena Query Federation SDK
4+
* %%
5+
* Copyright (C) 2019 - 2024 Amazon Web Services
6+
* %%
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
* #L%
19+
*/
20+
package com.amazonaws.athena.connector.lambda.connection;
21+
22+
public final class EnvironmentConstants
23+
{
24+
private EnvironmentConstants() {}
25+
26+
public static final int CONNECT_TIMEOUT = 250;
27+
28+
// Lambda environment variable keys
29+
public static final String DEFAULT_GLUE_CONNECTION = "glue_connection";
30+
public static final String SECRET_NAME = "secret_name";
31+
public static final String SPILL_KMS_KEY_ID = "spill_kms_key_id";
32+
public static final String KMS_KEY_ID = "kms_key_id";
33+
public static final String DEFAULT = "default";
34+
public static final String DEFAULT_DOCDB = "default_docdb";
35+
public static final String DEFAULT_HBASE = "default_hbase";
36+
37+
// glue connection property names
38+
public static final String HOST = "HOST";
39+
public static final String PORT = "PORT";
40+
public static final String JDBC_PARAMS = "JDBC_PARAMS";
41+
public static final String DATABASE = "DATABASE";
42+
public static final String SESSION_CONFS = "SESSION_CONFS";
43+
public static final String HIVE_CONFS = "HIVE_CONFS";
44+
public static final String HIVE_VARS = "HIVE_VARS";
45+
public static final String WAREHOUSE = "WAREHOUSE";
46+
public static final String SCHEMA = "SCHEMA";
47+
public static final String PROJECT_ID = "PROJECT_ID";
48+
public static final String CLUSTER_RES_ID = "CLUSTER_RESOURCE_ID";
49+
public static final String GRAPH_TYPE = "GRAPH_TYPE";
50+
public static final String HBASE_PORT = "HBASE_PORT";
51+
public static final String ZOOKEEPER_PORT = "ZOOKEEPER_PORT";
52+
public static final String CUSTOM_AUTH_TYPE = "CUSTOM_AUTH_TYPE";
53+
public static final String GLUE_CERTIFICATES_S3_REFERENCE = "CERTIFICATE_S3_REFERENCE";
54+
}

0 commit comments

Comments
 (0)