Skip to content

Commit d786715

Browse files
maxdaymaxday
authored
Add metrics and logs integration testing (#2)
integration-tests
1 parent 96417c2 commit d786715

File tree

21 files changed

+1508
-1
lines changed

21 files changed

+1508
-1
lines changed

‎.github/workflows/integration_tests.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: build
2+
3+
on:
4+
schedule:
5+
- cron: "0 12 * * *" #every day at midday
6+
7+
jobs:
8+
9+
integration-test:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout
13+
uses: actions/checkout@v2
14+
15+
- name: Set up Node 14
16+
uses: actions/setup-node@v1
17+
with:
18+
node-version: 14
19+
20+
- name: Cache Node modules
21+
id: cache-node-modules
22+
uses: actions/cache@v2
23+
with:
24+
path: "**/node_modules"
25+
key: ${{ runner.os }}-modules-${{ hashFiles('**/yarn.lock') }}
26+
27+
- name: Install Serverless Framework
28+
run: sudo yarn global add serverless --prefix /usr/local
29+
30+
- name: Install dependencies
31+
if: steps.cache-node-modules.outputs.cache-hit != 'true'
32+
working-directory: integration_tests
33+
run: yarn install
34+
35+
- name: Install dependencies
36+
if: steps.cache-node-modules.outputs.cache-hit != 'true'
37+
working-directory: integration_tests/recorder-extension/src
38+
run: yarn install
39+
40+
- name: Create folder
41+
run: pwd && mkdir -p ~/dd
42+
43+
- name: Checkout the datadog-agent
44+
uses: actions/checkout@v2
45+
with:
46+
repository: DataDog/datadog-agent
47+
#ref: refs/heads/release/lambda-extension-v9
48+
ref: refs/heads/maxday/integration-test # todo : change after merge of current PRs
49+
path: "datadog-agent"
50+
51+
- name: Build the layer
52+
run: ./scripts/build_binary_and_layer.sh
53+
54+
55+
- name: Run tests
56+
env:
57+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
58+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
59+
# Todo : retrieve this layer version automatically from AWS
60+
NODE_LAYER_VERSION: 55
61+
run: cp -R ../.layers . && ./integration_tests/run.sh

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
integration_tests/.serverless
2+
integration_tests/bin
3+
integration_tests/node_modules
4+
integration_tests/recorder-extension/node_modules/
5+
integration_tests/recorder-extension/ext.zip
6+
integration_tests/.DS_Store
17
.DS_Store
28
.layers
39
extensions

‎integration_tests/recorder-extension/a_recorder‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
exec "/opt/extensions/src/index.js"

‎integration_tests/recorder-extension/package.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"name": "recorder-extension",
3+
"version": "1.0.0",
4+
"main": "index.js",
5+
"dependencies": {
6+
"express": "^4.17.1",
7+
"node-fetch": "^2.6.1",
8+
"protobufjs": "^6.11.2"
9+
}
10+
}

‎integration_tests/recorder-extension/src/agent_payload.proto‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
syntax = "proto3";
2+
3+
// file partially copied from https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto
4+
5+
package datadog.agentpayload;
6+
7+
message CommonMetadata {
8+
string agent_version = 1;
9+
string timezone = 2;
10+
double current_epoch = 3;
11+
string internal_ip = 4;
12+
string public_ip = 5;
13+
string api_key = 6;
14+
}
15+
16+
message SketchPayload {
17+
message Sketch {
18+
message Distribution {
19+
int64 ts = 1;
20+
int64 cnt = 2;
21+
double min = 3;
22+
double max = 4;
23+
double avg = 5;
24+
double sum = 6;
25+
repeated double v = 7;
26+
repeated uint32 g = 8;
27+
repeated uint32 delta = 9;
28+
repeated double buf = 10;
29+
}
30+
message Dogsketch {
31+
int64 ts = 1;
32+
int64 cnt = 2;
33+
double min = 3;
34+
double max = 4;
35+
double avg = 5;
36+
double sum = 6;
37+
repeated sint32 k = 7;
38+
repeated uint32 n = 8;
39+
}
40+
string metric = 1;
41+
string host = 2;
42+
repeated Distribution distributions = 3 [(gogoproto.nullable) = false];
43+
repeated string tags = 4;
44+
reserved 5, 6;
45+
reserved "distributionsK", "distributionsC";
46+
repeated Dogsketch dogsketches = 7 [(gogoproto.nullable) = false];
47+
}
48+
repeated Sketch sketches = 1 [(gogoproto.nullable) = false];
49+
CommonMetadata metadata = 2 [(gogoproto.nullable) = false];
50+
}

‎integration_tests/recorder-extension/src/index.js‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env node
2+
3+
const express = require('express');
4+
const fetch = require('node-fetch');
5+
const bodyParser = require('body-parser');
6+
const protobuf = require('protobufjs');
7+
8+
const BASE_URL = `http://${process.env.AWS_LAMBDA_RUNTIME_API}/2020-01-01/extension`;
9+
const SHUTDOWN_EVENT = 'SHUTDOWN';
10+
11+
const handleShutdown = async () => {
12+
console.log('SHUTDOWN received in recorder');
13+
//making sure that the server won't be closed when the extension will send data
14+
await new Promise(r => setTimeout(r, 1000));
15+
console.log('SHUTDOWN end');
16+
}
17+
18+
async function register() {
19+
const res = await fetch(`${BASE_URL}/register`, {
20+
method: 'post',
21+
body: JSON.stringify({
22+
'events': [
23+
SHUTDOWN_EVENT
24+
],
25+
}),
26+
headers: {
27+
'Content-Type': 'application/json',
28+
'Lambda-Extension-Name': 'a_recorder',
29+
}
30+
});
31+
32+
if (!res.ok) {
33+
console.error('register failed', await res.text());
34+
}
35+
return res.headers.get('lambda-extension-identifier');
36+
}
37+
38+
async function next(extensionId) {
39+
const res = await fetch(`${BASE_URL}/event/next`, {
40+
method: 'get',
41+
headers: {
42+
'Content-Type': 'application/json',
43+
'Lambda-Extension-Identifier': extensionId,
44+
}
45+
});
46+
47+
if (!res.ok) {
48+
console.error('next failed', await res.text());
49+
return null;
50+
}
51+
52+
return await res.json();
53+
}
54+
55+
(async function main() {
56+
57+
const app = express();
58+
const options = {
59+
inflate: true,
60+
limit: '300kb',
61+
type: 'application/x-protobuf'
62+
};
63+
64+
app.use(bodyParser.raw(options));
65+
app.use(bodyParser.json());
66+
67+
const extensionId = await register();
68+
69+
const port = 3333;
70+
71+
app.get('/*', (req, res) => {
72+
console.log("GET", req.url);
73+
res.sendStatus(200);
74+
});
75+
76+
app.post('/api/beta/sketches*', async (req, res) => {
77+
const root = await protobuf.load('/opt/extensions/src/agent_payload.proto');
78+
const SketchPayload = root.lookupType('datadog.agentpayload.SketchPayload');
79+
const obj = SketchPayload.decode(req.body);
80+
for(let i = 0; i < obj.sketches.length; ++i) {
81+
console.log("[sketch]", JSON.stringify(obj.sketches[i]));
82+
}
83+
res.sendStatus(200);
84+
});
85+
86+
87+
app.post('/v1/input', async (req, res) => {
88+
if(JSON.stringify(req.body) !== '{}') { // to avoid printing empty logs due to the connectivity test
89+
for(let i = 0; i < req.body.length; ++i) {
90+
const logString = JSON.stringify(req.body[i]);
91+
if(logString.indexOf("[sketch]") === -1 && logString.indexOf("[log]") === -1) { // avoid inception
92+
console.log("[log]", logString);
93+
}
94+
}
95+
}
96+
res.sendStatus(200);
97+
});
98+
99+
app.post('/*', (req, res) => {
100+
console.log("POST", req.url);
101+
res.sendStatus(200);
102+
});
103+
104+
app.listen(port);
105+
106+
process.on('SIGINT', async () => await handleShutdown());
107+
process.on('SIGTERM', async () => await handleShutdown());
108+
109+
while (true) {
110+
const event = await next(extensionId);
111+
if(event.eventType === SHUTDOWN_EVENT) {
112+
await handleShutdown();
113+
break;
114+
} else {
115+
await handleShutdown();
116+
throw new Error('Unexpected event');
117+
}
118+
}
119+
})();

‎integration_tests/recorder-extension/src/package.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"name": "recorder-extension",
3+
"version": "1.0.0",
4+
"main": "index.js",
5+
"dependencies": {
6+
"express": "^4.17.1",
7+
"node-fetch": "^2.6.1",
8+
"protobufjs": "^6.11.2"
9+
}
10+
}

0 commit comments

Comments
 (0)