Skip to content

Commit 95b2c1b

Browse files
Michael Hacketthackett123
Michael Hackett
authored and
hackett123
committed
Integ Test Config File Fixes
- Modify all json config files to fix malformed json, and add required field for query execution - Consume new config variable when starting query execution to avoid failures for missing output location
1 parent 96f8c19 commit 95b2c1b

File tree

22 files changed

+61
-24
lines changed

22 files changed

+61
-24
lines changed

‎athena-cloudera-hive/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"connector_name" : "hive", /* connector name */
33
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
4+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
45
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
56
"region": "<region>", /* aws region name */
67
"hive_db_name": "<db name>", /* hive db name */
@@ -26,7 +27,7 @@
2627
},
2728
"user_settings" : { /* User customizable settings */
2829
"hive_db_name": "<Enter Value>", /* Name of the Db used by the Hive integration tests */
29-
"hive_db_port": <Enter Value>, /* Port number (int) associated with the hive */
30+
"hive_db_port": "<Enter Value>", /* Port number (int) associated with the hive */
3031
"hive_table_test": "<Enter Value>" /* Hive table name */
3132
}
3233
}

‎athena-cloudera-impala/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"connector_name" : "impala", /* connector name */
33
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
4+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
45
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
56
"region": "<region>", /* aws region name */
67
"impala_db_name": "<db name>", /* impala db name */
@@ -26,7 +27,7 @@
2627
},
2728
"user_settings" : { /* User customizable settings */
2829
"impala_db_name": "<Enter Value>", /* Name of the Db used by the impala integration tests */
29-
"impala_db_port": <Enter, Value>, /* Port number (int) associated with the impala */
30+
"impala_db_port": "<Enter, Value>", /* Port number (int) associated with the impala */
3031
"impala_table_test": "<Enter Value>" /* impala table name */
3132
}
3233
}

‎athena-cloudwatch/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<spill bucket>", /* The S3 bucket used for spilling excess data */
67
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
78
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
8-
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
9+
"spill_put_request_headers": "" /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
910
},
1011
"vpc_configuration" : { /* This connector does not use a VPC configuration */
1112
"vpc_id": "", /* Leave empty */

‎athena-datalakegen2/etc/test-config.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "",/* Secret name used to retrieve user credentials from SecretsManager. */
45
"region": "us-east-1",/* aws region name */
56
"schema_name": "<schema name>", /* datalakegen2 schema name */

‎athena-docdb/etc/test-config.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<spill bucket>", /* The S3 bucket used for spilling excess data */
67
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
78
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
8-
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
9+
"spill_put_request_headers": "" /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
910
},
1011
"vpc_configuration" : { /* VPC configuration for DB instances within a VPC */
1112
"vpc_id": "<Enter Value>", /* The VPC Id (e.g. vpc-xxx) */
@@ -15,7 +16,7 @@
1516
},
1617
"user_settings" : { /* User customizable settings */
1718
"docdb_db_name": "<Enter Value>", /* Name of the Db used by the DocumentDB integration tests */
18-
"docdb_db_port": <Enter Value>, /* Port number (int) associated with the Redshift cluster endpoint (usually 27017) */
19+
"docdb_db_port": "<Enter Value>", /* Port number (int) associated with the Redshift cluster endpoint (usually 27017) */
1920
"docdb_table_movies": "<Enter Value>" /* MongoDB table name */
2021
}
2122
}

‎athena-dynamodb/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<spill bucket>", /* The S3 bucket used for spilling excess data */
67
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
78
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
8-
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
9+
"spill_put_request_headers": "" /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
910
},
1011
"vpc_configuration" : { /* This connector does not use a VPC configuration */
1112
"vpc_id": "", /* Leave empty */

‎athena-elasticsearch/etc/test-config.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "", /* Not Used */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<Enter Value>", /* The S3 bucket used for spilling excess data */

‎athena-federation-integ-test/src/main/java/com/amazonaws/athena/connector/integ/IntegrationTestBase.java

+17-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.amazonaws.services.athena.model.GetQueryResultsResult;
3535
import com.amazonaws.services.athena.model.ListDatabasesRequest;
3636
import com.amazonaws.services.athena.model.ListDatabasesResult;
37+
import com.amazonaws.services.athena.model.ResultConfiguration;
3738
import com.amazonaws.services.athena.model.Row;
3839
import com.amazonaws.services.athena.model.StartQueryExecutionRequest;
3940
import org.slf4j.Logger;
@@ -65,6 +66,7 @@ public abstract class IntegrationTestBase
6566
private static final Logger logger = LoggerFactory.getLogger(IntegrationTestBase.class);
6667

6768
private static final String TEST_CONFIG_WORK_GROUP = "athena_work_group";
69+
private static final String TEST_CONFIG_RESULT_LOCATION = "athena_result_location";
6870
private static final String TEST_CONFIG_USER_SETTINGS = "user_settings";
6971
private static final String ATHENA_QUERY_QUEUED_STATE = "QUEUED";
7072
private static final String ATHENA_QUERY_RUNNING_STATE = "RUNNING";
@@ -97,6 +99,7 @@ public abstract class IntegrationTestBase
9799
private final Optional<ConnectorVpcAttributes> vpcAttributes;
98100
private final Optional<SecretsManagerCredentials> secretCredentials;
99101
private final String athenaWorkgroup;
102+
private final String athenaResultLocation;
100103
private CloudFormationClient cloudFormationClient;
101104

102105
public IntegrationTestBase()
@@ -127,6 +130,7 @@ protected void setSpecificResource(final Stack stack)
127130
lambdaFunctionName = connectorStackProvider.getLambdaFunctionName();
128131
athenaClient = AmazonAthenaClientBuilder.defaultClient();
129132
athenaWorkgroup = getAthenaWorkgroup();
133+
athenaResultLocation = getAthenaResultLocation();
130134
}
131135

132136
/**
@@ -145,6 +149,17 @@ private String getAthenaWorkgroup()
145149
return athenaWorkgroup;
146150
}
147151

152+
private String getAthenaResultLocation()
153+
throws RuntimeException
154+
{
155+
String athenaResultLocation = "s3://" + testConfig.getStringItem(TEST_CONFIG_RESULT_LOCATION).orElseThrow(() ->
156+
new RuntimeException(TEST_CONFIG_RESULT_LOCATION + " must be specified in test-config.json."));
157+
158+
logger.info("Athena Result Location: {}", athenaResultLocation);
159+
160+
return athenaResultLocation;
161+
}
162+
148163
/**
149164
* Public accessor for the framework generate lambda function name used in generating the lambda function.
150165
* @return The name of the lambda function.
@@ -311,7 +326,8 @@ public GetQueryResultsResult startQueryExecution(String query)
311326
{
312327
StartQueryExecutionRequest startQueryExecutionRequest = new StartQueryExecutionRequest()
313328
.withWorkGroup(athenaWorkgroup)
314-
.withQueryString(query);
329+
.withQueryString(query)
330+
.withResultConfiguration(new ResultConfiguration().withOutputLocation(athenaResultLocation));
315331

316332
String queryExecutionId = sendAthenaQuery(startQueryExecutionRequest);
317333
logger.info("Query: [{}], Query Id: [{}]", query, queryExecutionId);

‎athena-google-bigquery/etc/test-config.json

+10-9
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,25 @@
11
{
2-
"connector_name" : "bigquery", /* connector name */
3-
"athena_work_group" : "AmazonAthenaPreviewFunctionality", "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
4-
"secrets_manager_secret" : "", /* Secret name used to retrieve user credentials from SecretsManager. */
2+
"connector_name": "bigquery", /* connector name */
3+
"athena_work_group": "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
4+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
5+
"secrets_manager_secret": "", /* Secret name used to retrieve user credentials from SecretsManager. */
56
"schema_name": "<bigquery dataset name>", /* bigquery dataset name */
67
"table_name": "<table name>", /* bigquery table name */
78
"table_name2": "<table name>", /* bigquery table name */
89
"table_name3": "<table name>", /* bigquery table name */
910
"table_name4": "<table name>", /* bigquery table name */
1011
"table_name5": "<table name>", /* bigquery table name */
11-
"environment_vars" : {
12-
"spill_bucket" : "<spill bucket>", /* The S3 bucket used for spilling excess data */
13-
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
14-
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
12+
"environment_vars": {
13+
"spill_bucket": "<spill bucket>", /* The S3 bucket used for spilling excess data */
14+
"spill_prefix": "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
15+
"disable_spill_encryption": "false", /* If set to true encryption for spilled data is disabled (default: false) */
1516
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
16-
"region" : "us-east-1", /* region name */
17+
"region": "us-east-1", /* region name */
1718
"secret_manager_gcp_creds_name": "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
1819
"gcp_project_id": "<google bigquery project ID>", /* google bigquery project ID*/
1920
"concurrencyLimit": "10" /* concurrencyLimit will be used for splits*/
2021
},
21-
"vpc_configuration" : {
22+
"vpc_configuration": {
2223
"region": "<region>", /* aws region name */
2324
"vpc_id": "<Enter Value>", /* The VPC Id (e.g. vpc-xxx) */
2425
"security_group_id": "<Enter Value>", /* The Security Group Id (e.g. sg-xxx) */

‎athena-hbase/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<Enter Value>", /* The S3 bucket used for spilling excess data */
67
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
78
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
8-
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
9+
"spill_put_request_headers": "" /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
910
},
1011
"vpc_configuration" : { /* VPC configuration for DB instances within a VPC */
1112
"vpc_id": "<Enter Value>", /* The VPC Id (e.g. vpc-xxx) */

‎athena-hortonworks-hive/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"connector_name" : "hive", /* connector name */
33
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
4+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
45
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
56
"region": "<region>", /* aws region name */
67
"hive_db_name": "<db name>", /* hive db name */
@@ -26,7 +27,7 @@
2627
},
2728
"user_settings" : { /* User customizable settings */
2829
"hive_db_name": "<Enter Value>", /* Name of the Db used by the Hive integration tests */
29-
"hive_db_port": <Enter Value>, /* Port number (int) associated with the hive */
30+
"hive_db_port": "<Enter Value>", /* Port number (int) associated with the hive */
3031
"hive_table_test": "<Enter Value>" /* Hive table name */
3132
}
3233
}

‎athena-mysql/etc/test-config.json

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<spill bucket>", /* The S3 bucket used for spilling excess data */
67
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
78
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
8-
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
9+
"spill_put_request_headers": "" /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
910
},
1011
"vpc_configuration" : { /* VPC configuration for DB instances within a VPC */
1112
"vpc_id": "<VPC Id>", /* The VPC Id (e.g. vpc-xxx) */
@@ -15,7 +16,7 @@
1516
},
1617
"user_settings" : { /* User customizable settings */
1718
"mysql_db_name": "<DB Name>", /* Name of the Db used by the MySql integration tests */
18-
"mysql_db_port": <Port (Integer)>, /* Port number associated with the MySql instance endpoint */
19+
"mysql_db_port": "<Port (Integer)>", /* Port number associated with the MySql instance endpoint */
1920
"mysql_table_movies": "<Table Name>", /* MySql table name */
2021
"mysql_table_bday": "<Table Name>" /* MySql table name */
2122
}

‎athena-oracle/etc/test-config.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"connector_name" : "oracle", /* connector name */
33
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
4+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
45
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. */
56
"region": "<region>", /* aws region name */
67
"oracle_db_name": "<oracle db name>", /* oracle db name */

‎athena-postgresql/etc/test-config.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"athena_work_group": "FederationIntegrationTests",
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret": "federation/integ/postgresql",
45
"environment_vars":
56
{

‎athena-redis/etc/test-config.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"athena_work_group" : "FederationIntegrationTests", /* The Athena Workgroup used for running integration tests (default: FederationIntegrationTests) */
3+
"athena_result_location": "<bucket_path (without s3://)>", /* The s3 bucket path used to store results of Athena queries */
34
"secrets_manager_secret" : "<secret name>", /* Secret name used to retrieve user credentials from SecretsManager. Make sure to enter a dummy value for the username key */
45
"environment_vars" : { /* Parameters used by the connector's internal logic */
56
"spill_bucket" : "<spill bucket>", /* The S3 bucket used for spilling excess data */
67
"spill_prefix" : "athena-spill", /* The prefix within the S3 spill bucket (default: athena-spill) */
78
"disable_spill_encryption" : "false", /* If set to true encryption for spilled data is disabled (default: false) */
8-
"spill_put_request_headers": "", /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
9+
"spill_put_request_headers": "" /* JSON encoded map of request headers and values for the s3 putObject request used for spilling. This is a String not an object. Optional.*/
910
},
1011
"vpc_configuration" : { /* VPC configuration for Redis instances within a VPC */
1112
"vpc_id": "<Enter Value>", /* The VPC Id (e.g. vpc-xxx) */

0 commit comments

Comments
 (0)