Skip to content

Commit be1df0f

Browse files
dai-chendai-chen
authored
Introduce basic sanity test for MV used by Observability Integrations (#995)
Signed-off-by: Chen Dai <[email protected]>
1 parent 6b69197 commit be1df0f

File tree

5 files changed

+635
-0
lines changed

5 files changed

+635
-0
lines changed

‎integ-test/src/integration/resources/aws-logs/cloud_trail.sql

+181
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
CREATE TABLE {table_name} (
2+
eventVersion STRING,
3+
userIdentity STRUCT<
4+
type:STRING,
5+
principalId:STRING,
6+
arn:STRING,
7+
accountId:STRING,
8+
invokedBy:STRING,
9+
accessKeyId:STRING,
10+
userName:STRING,
11+
sessionContext:STRUCT<
12+
attributes:STRUCT<
13+
mfaAuthenticated:STRING,
14+
creationDate:STRING
15+
>,
16+
sessionIssuer:STRUCT<
17+
type:STRING,
18+
principalId:STRING,
19+
arn:STRING,
20+
accountId:STRING,
21+
userName:STRING
22+
>,
23+
ec2RoleDelivery:STRING,
24+
webIdFederationData:MAP<STRING,STRING>
25+
>
26+
>,
27+
eventTime STRING,
28+
eventSource STRING,
29+
eventName STRING,
30+
awsRegion STRING,
31+
sourceIPAddress STRING,
32+
userAgent STRING,
33+
errorCode STRING,
34+
errorMessage STRING,
35+
requestParameters STRING,
36+
responseElements STRING,
37+
additionalEventData STRING,
38+
requestId STRING,
39+
eventId STRING,
40+
resources ARRAY<STRUCT<
41+
arn:STRING,
42+
accountId:STRING,
43+
type:STRING
44+
>>,
45+
eventType STRING,
46+
apiVersion STRING,
47+
readOnly STRING,
48+
recipientAccountId STRING,
49+
serviceEventDetails STRING,
50+
sharedEventId STRING,
51+
vpcEndpointId STRING,
52+
eventCategory STRING,
53+
tlsDetails STRUCT<
54+
tlsVersion:STRING,
55+
cipherSuite:STRING,
56+
clientProvidedHostHeader:STRING
57+
>
58+
)
59+
USING json
60+
OPTIONS (
61+
recursivefilelookup='true',
62+
multiline 'true'
63+
);
64+
65+
INSERT INTO {table_name} VALUES
66+
(
67+
'1.08',
68+
NAMED_STRUCT(
69+
'type', 'IAMUser',
70+
'principalId', 'AWS123456789012',
71+
'arn', 'arn:aws:iam::123456789012:user/ExampleUser',
72+
'accountId', '123456789012',
73+
'invokedBy', null,
74+
'accessKeyId', 'AKIA1234567890',
75+
'userName', 'ExampleUser',
76+
'sessionContext', NAMED_STRUCT(
77+
'attributes', NAMED_STRUCT(
78+
'mfaAuthenticated', 'true',
79+
'creationDate', '2023-11-01T05:00:00Z'
80+
),
81+
'sessionIssuer', NAMED_STRUCT(
82+
'type', 'Role',
83+
'principalId', 'ARO123456789012',
84+
'arn', 'arn:aws:iam::123456789012:role/MyRole',
85+
'accountId', '123456789012',
86+
'userName', 'MyRole'
87+
),
88+
'ec2RoleDelivery', 'true',
89+
'webIdFederationData', MAP()
90+
)
91+
),
92+
'2023-11-01T05:00:00Z',
93+
'sts.amazonaws.com',
94+
'AssumeRole',
95+
'us-east-1',
96+
'198.51.100.45',
97+
'AWS CLI',
98+
null,
99+
null,
100+
null,
101+
null,
102+
null,
103+
'request-id-1',
104+
'event-id-1',
105+
ARRAY(NAMED_STRUCT(
106+
'arn', 'arn:aws:iam::123456789012:role/MyRole',
107+
'accountId', '123456789012',
108+
'type', 'AWS::IAM::Role'
109+
)),
110+
'AwsApiCall',
111+
'2015-03-31',
112+
'true',
113+
'123456789012',
114+
null,
115+
null,
116+
null,
117+
'Management',
118+
NAMED_STRUCT(
119+
'tlsVersion', 'TLSv1.2',
120+
'cipherSuite', 'ECDHE-RSA-AES128-GCM-SHA256',
121+
'clientProvidedHostHeader', null
122+
)
123+
),
124+
(
125+
'1.08',
126+
NAMED_STRUCT(
127+
'type', 'IAMUser',
128+
'principalId', 'AWS123456789012',
129+
'arn', 'arn:aws:iam::123456789012:user/ExampleUser',
130+
'accountId', '123456789012',
131+
'invokedBy', null,
132+
'accessKeyId', 'AKIA1234567890',
133+
'userName', 'ExampleUser',
134+
'sessionContext', NAMED_STRUCT(
135+
'attributes', NAMED_STRUCT(
136+
'mfaAuthenticated', 'true',
137+
'creationDate', '2023-11-01T05:06:00Z'
138+
),
139+
'sessionIssuer', NAMED_STRUCT(
140+
'type', 'Role',
141+
'principalId', 'ARO123456789012',
142+
'arn', 'arn:aws:iam::123456789012:role/MyRole',
143+
'accountId', '123456789012',
144+
'userName', 'MyRole'
145+
),
146+
'ec2RoleDelivery', 'true',
147+
'webIdFederationData', MAP()
148+
)
149+
),
150+
'2023-11-01T05:06:00Z',
151+
'sts.amazonaws.com',
152+
'AssumeRole',
153+
'us-east-1',
154+
'198.51.100.45',
155+
'AWS CLI',
156+
null,
157+
null,
158+
null,
159+
null,
160+
null,
161+
'request-id-2',
162+
'event-id-2',
163+
ARRAY(NAMED_STRUCT(
164+
'arn', 'arn:aws:iam::123456789012:role/MyRole',
165+
'accountId', '123456789012',
166+
'type', 'AWS::IAM::Role'
167+
)),
168+
'AwsApiCall',
169+
'2015-03-31',
170+
'true',
171+
'123456789012',
172+
null,
173+
null,
174+
null,
175+
'Management',
176+
NAMED_STRUCT(
177+
'tlsVersion', 'TLSv1.2',
178+
'cipherSuite', 'ECDHE-RSA-AES128-GCM-SHA256',
179+
'clientProvidedHostHeader', null
180+
)
181+
);

‎integ-test/src/integration/resources/aws-logs/vpc_flow.sql

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
CREATE TABLE {table_name} (
2+
version INT,
3+
accountId STRING,
4+
interfaceId STRING,
5+
srcAddr STRING,
6+
dstAddr STRING,
7+
srcPort INT,
8+
dstPort INT,
9+
protocol BIGINT,
10+
packets BIGINT,
11+
bytes BIGINT,
12+
start BIGINT,
13+
`end` BIGINT,
14+
action STRING,
15+
logStatus STRING
16+
)
17+
USING csv
18+
OPTIONS (
19+
sep = ' ',
20+
recursiveFileLookup = 'true'
21+
);
22+
23+
INSERT INTO {table_name} VALUES
24+
(1, '123456789012', 'eni-abc123', '10.0.0.1', '10.0.0.2', 12345, 80, 6, 10, 200, 1698814800, 1698814860, 'ACCEPT', 'OK'), -- 05:00:00 to 05:01:00 UTC
25+
(2, '123456789012', 'eni-def456', '10.0.0.1', '10.0.0.2', 12346, 443, 6, 5, 150, 1698814900, 1698814960, 'ACCEPT', 'OK'), -- 05:01:40 to 05:02:40 UTC
26+
(3, '123456789013', 'eni-ghi789', '10.0.0.3', '10.0.0.4', 12347, 22, 6, 15, 300, 1698815400, 1698815460, 'ACCEPT', 'OK'), -- 05:10:00 to 05:11:00 UTC
27+
(4, '123456789013', 'eni-jkl012', '10.0.0.5', '10.0.0.6', 12348, 21, 6, 20, 400, 1698815500, 1698815560, 'REJECT', 'OK'), -- 05:11:40 to 05:12:40 UTC
28+
(5, '123456789014', 'eni-mno345', '10.0.0.7', '10.0.0.8', 12349, 25, 6, 25, 500, 1698816000, 1698816060, 'ACCEPT', 'OK') -- 05:20:00 to 05:21:00 UTC
29+
;

‎integ-test/src/integration/resources/aws-logs/waf.sql

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
CREATE TABLE {table_name} (
2+
timestamp STRING,
3+
webaclId STRING,
4+
action STRING,
5+
formatVersion INT,
6+
httpRequest STRUCT<
7+
clientIp: STRING,
8+
country: STRING,
9+
headers: ARRAY<STRUCT<
10+
name: STRING,
11+
value: STRING
12+
>>,
13+
uri: STRING,
14+
args: STRING,
15+
httpVersion: STRING,
16+
httpMethod: STRING,
17+
requestId: STRING
18+
>,
19+
httpSourceId STRING,
20+
httpSourceName STRING,
21+
requestBodySize INT,
22+
requestBodySizeInspectedByWAF INT,
23+
terminatingRuleId STRING,
24+
terminatingRuleType STRING,
25+
ruleGroupList ARRAY<STRUCT<
26+
ruleId: STRING,
27+
ruleAction: STRING
28+
>>,
29+
rateBasedRuleList ARRAY<STRUCT<
30+
ruleId: STRING
31+
>>,
32+
nonTerminatingMatchingRules ARRAY<STRUCT<
33+
ruleId: STRING
34+
>>
35+
)
36+
USING json
37+
OPTIONS (
38+
recursivefilelookup = 'true'
39+
);
40+
41+
INSERT INTO {table_name} VALUES
42+
(
43+
1698814800000, -- 2023-11-01T05:00:00Z
44+
'webacl-12345',
45+
'ALLOW',
46+
1,
47+
NAMED_STRUCT(
48+
'clientIp', '192.0.2.1',
49+
'country', 'US',
50+
'headers', ARRAY(NAMED_STRUCT('name', 'User-Agent', 'value', 'Mozilla/5.0')),
51+
'uri', '/index.html',
52+
'args', 'query=example',
53+
'httpVersion', 'HTTP/1.1',
54+
'httpMethod', 'GET',
55+
'requestId', 'req-1'
56+
),
57+
'source-1',
58+
'http-source',
59+
500,
60+
450,
61+
'rule-1',
62+
'REGULAR',
63+
ARRAY(NAMED_STRUCT('ruleId', 'group-rule-1', 'ruleAction', 'ALLOW')),
64+
ARRAY(),
65+
ARRAY()
66+
),
67+
(
68+
1698815400000, -- 2023-11-01T05:10:00Z
69+
'webacl-67890',
70+
'BLOCK',
71+
1,
72+
NAMED_STRUCT(
73+
'clientIp', '192.0.2.2',
74+
'country', 'CA',
75+
'headers', ARRAY(NAMED_STRUCT('name', 'Referer', 'value', 'example.com')),
76+
'uri', '/login.html',
77+
'args', '',
78+
'httpVersion', 'HTTP/2',
79+
'httpMethod', 'POST',
80+
'requestId', 'req-2'
81+
),
82+
'source-2',
83+
'http-source',
84+
750,
85+
600,
86+
'rule-2',
87+
'RATE_BASED',
88+
ARRAY(NAMED_STRUCT('ruleId', 'group-rule-2', 'ruleAction', 'BLOCK')),
89+
ARRAY(),
90+
ARRAY()
91+
);

0 commit comments

Comments
 (0)