Skip to content

Commit 0d2a1c7

Browse files
YANG-DBYANG-DB
authored
Fix flint skipping index syntax issues (#1846)
 * update flint related issues for - vpc flow - cloud trail - multiple records protocol support Signed-off-by: YANGDB <[email protected]> * update flint vega ip sankey visualization query Signed-off-by: YANGDB <[email protected]> * update flint vega ip sankey visualization query Signed-off-by: YANGDB <[email protected]> --------- Signed-off-by: YANGDB <[email protected]>
1 parent 59f0d57 commit 0d2a1c7

15 files changed

+335
-146
lines changed

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/aws_vpc_flow_flint-live-1.0.0.ndjson

+17-17
Large diffs are not rendered by default.

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/aws_vpc_flow_flint-pre_agg-1.0.0.ndjson

+18-18
Large diffs are not rendered by default.

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_mv_vpc-1.0.0.sql

Whitespace-only changes.

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_skipping_index-1.0.0.sql

+5-6
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,11 @@ CREATE SKIPPING INDEX ON {table_name} (
22
accountid BLOOM_FILTER,
33
region VALUE_SET,
44
severity_id VALUE_SET,
5-
src_endpoint.ip BLOOM_FILTER,
6-
dst_endpoint.ip BLOOM_FILTER,
7-
src_endpoint.svc_name VALUE_SET,
8-
dst_endpoint.svc_name VALUE_SET,
9-
request_processing_time MIN_MAX,
10-
traffic.bytes MIN_MAX
5+
`src_endpoint.ip` BLOOM_FILTER,
6+
`dst_endpoint.ip` BLOOM_FILTER,
7+
`src_endpoint.svc_name` VALUE_SET,
8+
`dst_endpoint.svc_name` VALUE_SET,
9+
`traffic.bytes` MIN_MAX
1110
) WITH (
1211
auto_refresh = true,
1312
refresh_interval = '15 Minutes',

‎server/adaptors/integrations/__data__/repository/amazon_vpc_flow/assets/create_table_parquet_vpc-1.0.0.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -58,5 +58,5 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
5858
accountid STRING,
5959
eventday STRING
6060
)
61-
USING json
61+
USING parquet
6262
LOCATION '{s3_bucket_location}'

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/aws_cloudtrail-flint-1.0.0.ndjson

+17-17
Large diffs are not rendered by default.

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_mv_cloud-trail-1.0.0.sql

+41-42
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,50 @@
11
CREATE MATERIALIZED VIEW {table_name}__mview AS
22
SELECT
3-
rec.userIdentity.type AS `aws.cloudtrail.userIdentity.type`,
4-
rec.userIdentity.principalId AS `aws.cloudtrail.userIdentity.principalId`,
5-
rec.userIdentity.arn AS `aws.cloudtrail.userIdentity.arn`,
6-
rec.userIdentity.accountId AS `aws.cloudtrail.userIdentity.accountId`,
7-
rec.userIdentity.invokedBy AS `aws.cloudtrail.userIdentity.invokedBy`,
8-
rec.userIdentity.accessKeyId AS `aws.cloudtrail.userIdentity.accessKeyId`,
9-
rec.userIdentity.userName AS `aws.cloudtrail.userIdentity.userName`,
10-
rec.userIdentity.sessionContext.attributes.mfaAuthenticated AS `aws.cloudtrail.userIdentity.sessionContext.attributes.mfaAuthenticated`,
11-
CAST(rec.userIdentity.sessionContext.attributes.creationDate AS TIMESTAMP) AS `aws.cloudtrail.userIdentity.sessionContext.attributes.creationDate`,
12-
rec.userIdentity.sessionContext.sessionIssuer.type AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type`,
13-
rec.userIdentity.sessionContext.sessionIssuer.principalId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.principalId`,
14-
rec.userIdentity.sessionContext.sessionIssuer.arn AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.arn`,
15-
rec.userIdentity.sessionContext.sessionIssuer.accountId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.accountId`,
16-
rec.userIdentity.sessionContext.sessionIssuer.userName AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.userName`,
17-
rec.userIdentity.sessionContext.ec2RoleDelivery AS `aws.cloudtrail.userIdentity.sessionContext.ec2RoleDelivery`,
3+
userIdentity.type AS `aws.cloudtrail.userIdentity.type`,
4+
userIdentity.principalId AS `aws.cloudtrail.userIdentity.principalId`,
5+
userIdentity.arn AS `aws.cloudtrail.userIdentity.arn`,
6+
userIdentity.accountId AS `aws.cloudtrail.userIdentity.accountId`,
7+
userIdentity.invokedBy AS `aws.cloudtrail.userIdentity.invokedBy`,
8+
userIdentity.accessKeyId AS `aws.cloudtrail.userIdentity.accessKeyId`,
9+
userIdentity.userName AS `aws.cloudtrail.userIdentity.userName`,
10+
userIdentity.sessionContext.attributes.mfaAuthenticated AS `aws.cloudtrail.userIdentity.sessionContext.attributes.mfaAuthenticated`,
11+
CAST( userIdentity.sessionContext.attributes.creationDate AS TIMESTAMP) AS `aws.cloudtrail.userIdentity.sessionContext.attributes.creationDate`,
12+
userIdentity.sessionContext.sessionIssuer.type AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type`,
13+
userIdentity.sessionContext.sessionIssuer.principalId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.principalId`,
14+
userIdentity.sessionContext.sessionIssuer.arn AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.arn`,
15+
userIdentity.sessionContext.sessionIssuer.accountId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.accountId`,
16+
userIdentity.sessionContext.sessionIssuer.userName AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.userName`,
17+
userIdentity.sessionContext.ec2RoleDelivery AS `aws.cloudtrail.userIdentity.sessionContext.ec2RoleDelivery`,
1818

19-
rec.eventVersion AS `aws.cloudtrail.eventVersion`,
20-
CAST(rec.eventTime AS TIMESTAMP) AS `@timestamp`,
21-
rec.eventSource AS `aws.cloudtrail.eventSource`,
22-
rec.eventName AS `aws.cloudtrail.eventName`,
23-
rec.eventCategory AS `aws.cloudtrail.eventCategory`,
24-
rec.eventType AS `aws.cloudtrail.eventType`,
25-
rec.eventId AS `aws.cloudtrail.eventId`,
19+
eventVersion AS `aws.cloudtrail.eventVersion`,
20+
CAST( eventTime AS TIMESTAMP) AS `@timestamp`,
21+
eventSource AS `aws.cloudtrail.eventSource`,
22+
eventName AS `aws.cloudtrail.eventName`,
23+
eventCategory AS `aws.cloudtrail.eventCategory`,
24+
eventType AS `aws.cloudtrail.eventType`,
25+
eventId AS `aws.cloudtrail.eventId`,
2626

27-
rec.awsRegion AS `aws.cloudtrail.awsRegion`,
28-
rec.sourceIPAddress AS `aws.cloudtrail.sourceIPAddress`,
29-
rec.userAgent AS `aws.cloudtrail.userAgent`,
30-
rec.errorCode AS `errorCode`,
31-
rec.errorMessage AS `errorMessage`,
32-
rec.requestParameters AS `aws.cloudtrail.requestParameter`,
33-
rec.responseElements AS `aws.cloudtrail.responseElements`,
34-
rec.additionalEventData AS `aws.cloudtrail.additionalEventData`,
35-
rec.requestId AS `aws.cloudtrail.requestId`,
36-
rec.resources AS `aws.cloudtrail.resources`,
37-
rec.apiVersion AS `aws.cloudtrail.apiVersion`,
38-
rec.readOnly AS `aws.cloudtrail.readOnly`,
39-
rec.recipientAccountId AS `aws.cloudtrail.recipientAccountId`,
40-
rec.serviceEventDetails AS `aws.cloudtrail.serviceEventDetails`,
41-
rec.sharedEventId AS `aws.cloudtrail.sharedEventId`,
42-
rec.vpcEndpointId AS `aws.cloudtrail.vpcEndpointId`,
43-
rec.tlsDetails.tlsVersion AS `aws.cloudtrail.tlsDetails.tls_version`,
44-
rec.tlsDetails.cipherSuite AS `aws.cloudtrail.tlsDetailscipher_suite`,
45-
rec.tlsDetails.clientProvidedHostHeader AS `aws.cloudtrail.tlsDetailsclient_provided_host_header`
27+
awsRegion AS `aws.cloudtrail.awsRegion`,
28+
sourceIPAddress AS `aws.cloudtrail.sourceIPAddress`,
29+
userAgent AS `aws.cloudtrail.userAgent`,
30+
errorCode AS `errorCode`,
31+
errorMessage AS `errorMessage`,
32+
requestParameters AS `aws.cloudtrail.requestParameter`,
33+
responseElements AS `aws.cloudtrail.responseElements`,
34+
additionalEventData AS `aws.cloudtrail.additionalEventData`,
35+
requestId AS `aws.cloudtrail.requestId`,
36+
resources AS `aws.cloudtrail.resources`,
37+
apiVersion AS `aws.cloudtrail.apiVersion`,
38+
readOnly AS `aws.cloudtrail.readOnly`,
39+
recipientAccountId AS `aws.cloudtrail.recipientAccountId`,
40+
serviceEventDetails AS `aws.cloudtrail.serviceEventDetails`,
41+
sharedEventId AS `aws.cloudtrail.sharedEventId`,
42+
vpcEndpointId AS `aws.cloudtrail.vpcEndpointId`,
43+
tlsDetails.tlsVersion AS `aws.cloudtrail.tlsDetails.tls_version`,
44+
tlsDetails.cipherSuite AS `aws.cloudtrail.tlsDetailscipher_suite`,
45+
tlsDetails.clientProvidedHostHeader AS `aws.cloudtrail.tlsDetailsclient_provided_host_header`
4646
FROM
4747
{table_name}
48-
LATERAL VIEW explode(Records) myTable AS rec
4948
WITH (
5049
auto_refresh = true,
5150
refresh_interval = '15 Minute',

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_mv_cloud-trail-records-1.0.0.sql

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
CREATE MATERIALIZED VIEW {table_name}__mview AS
2+
SELECT
3+
rec.userIdentity.type AS `aws.cloudtrail.userIdentity.type`,
4+
rec.userIdentity.principalId AS `aws.cloudtrail.userIdentity.principalId`,
5+
rec.userIdentity.arn AS `aws.cloudtrail.userIdentity.arn`,
6+
rec.userIdentity.accountId AS `aws.cloudtrail.userIdentity.accountId`,
7+
rec.userIdentity.invokedBy AS `aws.cloudtrail.userIdentity.invokedBy`,
8+
rec.userIdentity.accessKeyId AS `aws.cloudtrail.userIdentity.accessKeyId`,
9+
rec.userIdentity.userName AS `aws.cloudtrail.userIdentity.userName`,
10+
rec.userIdentity.sessionContext.attributes.mfaAuthenticated AS `aws.cloudtrail.userIdentity.sessionContext.attributes.mfaAuthenticated`,
11+
CAST(rec.userIdentity.sessionContext.attributes.creationDate AS TIMESTAMP) AS `aws.cloudtrail.userIdentity.sessionContext.attributes.creationDate`,
12+
rec.userIdentity.sessionContext.sessionIssuer.type AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type`,
13+
rec.userIdentity.sessionContext.sessionIssuer.principalId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.principalId`,
14+
rec.userIdentity.sessionContext.sessionIssuer.arn AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.arn`,
15+
rec.userIdentity.sessionContext.sessionIssuer.accountId AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.accountId`,
16+
rec.userIdentity.sessionContext.sessionIssuer.userName AS `aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.userName`,
17+
rec.userIdentity.sessionContext.ec2RoleDelivery AS `aws.cloudtrail.userIdentity.sessionContext.ec2RoleDelivery`,
18+
19+
rec.eventVersion AS `aws.cloudtrail.eventVersion`,
20+
CAST(rec.eventTime AS TIMESTAMP) AS `@timestamp`,
21+
rec.eventSource AS `aws.cloudtrail.eventSource`,
22+
rec.eventName AS `aws.cloudtrail.eventName`,
23+
rec.eventCategory AS `aws.cloudtrail.eventCategory`,
24+
rec.eventType AS `aws.cloudtrail.eventType`,
25+
rec.eventId AS `aws.cloudtrail.eventId`,
26+
27+
rec.awsRegion AS `aws.cloudtrail.awsRegion`,
28+
rec.sourceIPAddress AS `aws.cloudtrail.sourceIPAddress`,
29+
rec.userAgent AS `aws.cloudtrail.userAgent`,
30+
rec.errorCode AS `errorCode`,
31+
rec.errorMessage AS `errorMessage`,
32+
rec.requestParameters AS `aws.cloudtrail.requestParameter`,
33+
rec.responseElements AS `aws.cloudtrail.responseElements`,
34+
rec.additionalEventData AS `aws.cloudtrail.additionalEventData`,
35+
rec.requestId AS `aws.cloudtrail.requestId`,
36+
rec.resources AS `aws.cloudtrail.resources`,
37+
rec.apiVersion AS `aws.cloudtrail.apiVersion`,
38+
rec.readOnly AS `aws.cloudtrail.readOnly`,
39+
rec.recipientAccountId AS `aws.cloudtrail.recipientAccountId`,
40+
rec.serviceEventDetails AS `aws.cloudtrail.serviceEventDetails`,
41+
rec.sharedEventId AS `aws.cloudtrail.sharedEventId`,
42+
rec.vpcEndpointId AS `aws.cloudtrail.vpcEndpointId`,
43+
rec.tlsDetails.tlsVersion AS `aws.cloudtrail.tlsDetails.tls_version`,
44+
rec.tlsDetails.cipherSuite AS `aws.cloudtrail.tlsDetailscipher_suite`,
45+
rec.tlsDetails.clientProvidedHostHeader AS `aws.cloudtrail.tlsDetailsclient_provided_host_header`
46+
FROM
47+
{table_name}
48+
LATERAL VIEW explode(Records) explodedCloudTrailsTable AS rec
49+
WITH (
50+
auto_refresh = true,
51+
refresh_interval = '15 Minute',
52+
checkpoint_location = '{s3_checkpoint_location}',
53+
watermark_delay = '1 Minute',
54+
extra_options = '{ "{table_name}": { "maxFilesPerTrigger": "10" }}'
55+
)

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_skipping_index-1.0.0.sql

+9-9
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
CREATE SKIPPING INDEX ON {table_name} (
2-
rec.userIdentity.principalId BLOOM_FILTER,
3-
rec.userIdentity.accountId BLOOM_FILTER,
4-
rec.userIdentity.userName BLOOM_FILTER,
5-
rec.sourceIPAddress BLOOM_FILTER,
6-
rec.eventId BLOOM_FILTER,
7-
rec.userIdentity.type VALUE_SET,
8-
rec.eventName VALUE_SET,
9-
rec.eventType VALUE_SET,
10-
rec.awsRegion VALUE_SET
2+
`userIdentity.principalId` BLOOM_FILTER,
3+
`userIdentity.accountId` BLOOM_FILTER,
4+
`userIdentity.userName` BLOOM_FILTER,
5+
`sourceIPAddress` BLOOM_FILTER,
6+
`eventId` BLOOM_FILTER,
7+
`userIdentity.type` VALUE_SET,
8+
`eventName` VALUE_SET,
9+
`eventType` VALUE_SET,
10+
`awsRegion` VALUE_SET
1111
) WITH (
1212
auto_refresh = true,
1313
refresh_interval = '15 Minutes',

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-1.0.0.sql

+6-7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
2-
Records ARRAY<STRUCT<
1+
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
32
eventVersion STRING,
43
userIdentity STRUCT<
54
type:STRING,
@@ -56,10 +55,10 @@ CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
5655
cipherSuite:STRING,
5756
clientProvidedHostHeader:STRING
5857
>
59-
>>
60-
) USING json
61-
LOCATION '{s3_bucket_location}'
58+
)
59+
USING json
6260
OPTIONS (
63-
compression='gzip',
64-
recursivefilelookup='true'
61+
PATH '{s3_bucket_location}',
62+
recursivefilelookup='true',
63+
multiline 'true'
6564
)

‎server/adaptors/integrations/__data__/repository/aws_cloudtrail/assets/create_table_cloud-trail-records-1.0.0.sql

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
CREATE EXTERNAL TABLE IF NOT EXISTS {table_name} (
2+
Records ARRAY<STRUCT<
3+
eventVersion STRING,
4+
userIdentity STRUCT<
5+
type:STRING,
6+
principalId:STRING,
7+
arn:STRING,
8+
accountId:STRING,
9+
invokedBy:STRING,
10+
accessKeyId:STRING,
11+
userName:STRING,
12+
sessionContext:STRUCT<
13+
attributes:STRUCT<
14+
mfaAuthenticated:STRING,
15+
creationDate:STRING
16+
>,
17+
sessionIssuer:STRUCT<
18+
type:STRING,
19+
principalId:STRING,
20+
arn:STRING,
21+
accountId:STRING,
22+
userName:STRING
23+
>,
24+
ec2RoleDelivery:STRING,
25+
webIdFederationData:MAP<STRING,STRING>
26+
>
27+
>,
28+
eventTime STRING,
29+
eventSource STRING,
30+
eventName STRING,
31+
awsRegion STRING,
32+
sourceIPAddress STRING,
33+
userAgent STRING,
34+
errorCode STRING,
35+
errorMessage STRING,
36+
requestParameters STRING,
37+
responseElements STRING,
38+
additionalEventData STRING,
39+
requestId STRING,
40+
eventId STRING,
41+
resources ARRAY<STRUCT<
42+
arn:STRING,
43+
accountId:STRING,
44+
type:STRING
45+
>>,
46+
eventType STRING,
47+
apiVersion STRING,
48+
readOnly STRING,
49+
recipientAccountId STRING,
50+
serviceEventDetails STRING,
51+
sharedEventId STRING,
52+
vpcEndpointId STRING,
53+
eventCategory STRING,
54+
tlsDetails STRUCT<
55+
tlsVersion:STRING,
56+
cipherSuite:STRING,
57+
clientProvidedHostHeader:STRING
58+
>
59+
>>
60+
)
61+
USING json
62+
LOCATION '{s3_bucket_location}'
63+
OPTIONS (
64+
compression='gzip',
65+
recursivefilelookup='true',
66+
multiline 'true'
67+
)

0 commit comments

Comments
 (0)