Skip to content

Commit 37a394e

Browse files
CookiezaursCookiezaurs
committed
Script to compare aggregated events data with granural data
1 parent 419a8f7 commit 37a394e

File tree

1 file changed

+272
-0
lines changed

1 file changed

+272
-0
lines changed

‎bin/scripts/data-reports/compare_drill_aggregated.js

+272
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
/*
2+
* Script compares aggregated data from events collection with drill data for given period.
3+
* Place script ins
4+
* {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
5+
*
6+
* Usage:
7+
* node compare_drill_aggregated.js
8+
*/
9+
var period = "7days"; //Chose any of formats: "Xdays" ("7days","100days") or ["1-1-2024", "1-10-2024"],
10+
var app_list = []; //List with apps
11+
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12+
var eventMap = {}; //If left empty will run for all alls/events.
13+
14+
var use_union_with = true; //
15+
16+
const Promise = require("bluebird");
17+
const crypto = require("crypto");
18+
var pluginManager = require("../../../plugins/pluginManager");
19+
var fetch = require("../../../api/parts/data/fetch.js");
20+
var countlyCommon = require("../../../api/lib/countly.common.js");
21+
var common = require("../../../api/utils/common.js");
22+
var moment = require("moment");
23+
24+
var endReport = {};
25+
26+
console.log("Script compares numbers in aggregated data with drill data for given period.");
27+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
28+
common.db = countlyDb;
29+
console.log("Connected to databases...");
30+
getAppList({db: countlyDb}, function(err, apps) {
31+
if (!apps || !apps.length) {
32+
console.log("No apps found");
33+
return close();
34+
}
35+
else {
36+
console.log("Apps found:", apps.length);
37+
Promise.each(apps, function(app) {
38+
return new Promise(function(resolve, reject) {
39+
console.log("Processing app: ", app.name);
40+
countlyDb.collection("events").findOne({_id: app._id}, {'list': 1}, function(err, events) {
41+
if (err) {
42+
console.log("Error getting events: ", err);
43+
reject();
44+
}
45+
else {
46+
events = events || {};
47+
events.list = events.list || [];
48+
events.list = events.list.filter(function(ee) {
49+
if (ee.indexOf("[CLY]_") === 0) {
50+
return false;
51+
}
52+
else {
53+
return true;
54+
}
55+
});
56+
57+
if (eventMap && eventMap[app._id + ""]) {
58+
var listBF = events.list.length;
59+
events.list = events.list.filter(function(ee) {
60+
if (eventMap && eventMap[app._id + ""]) {
61+
return eventMap[app._id + ""].indexOf(ee) > -1;
62+
}
63+
else {
64+
return false;
65+
}
66+
});
67+
if (events.list.length != listBF) {
68+
console.log(" Filtered events based on eventMap: ", events.list.length, " from ", listBF);
69+
}
70+
71+
}
72+
if (events && events.list && events.list.length) {
73+
endReport[app._id] = {"name": app.name, "total": events.list.length, "bad": 0};
74+
Promise.each(events.list, function(event) {
75+
return new Promise(function(resolve2, reject2) {
76+
console.log(" Processing event: ", event);
77+
var params = {
78+
app_id: app._id + "",
79+
appTimezone: app.timezone,
80+
qstring: { period: period},
81+
time: common.initTimeObj(app.timezone, Date.now().valueOf())
82+
};
83+
84+
//fetch drill data
85+
var periodObject = countlyCommon.getPeriodObj({"appTimezone": app.timezone, "qstring": {"period": period}});
86+
getDataFromDrill({event: event, app_id: app._id + "", timezone: app.timezone, drillDb: drillDb, periodObj: periodObject}, function(err, drillData) {
87+
if (err) {
88+
console.log(" Error getting drill data: ", err);
89+
reject2();
90+
}
91+
else {
92+
//console.log(" Drill data loaded");
93+
//console.log(JSON.stringify(drillData));
94+
//fetch aggregated data
95+
96+
var collectionName = "events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
97+
98+
fetch.getTimeObjForEvents(collectionName, params, null, function(data) {
99+
var mergedData = {};
100+
var totals = {"c": 0, "s": 0, "dur": 0};
101+
for (var z = 0; z < periodObject.currentPeriodArr.length; z++) {
102+
var date = periodObject.currentPeriodArr[z].split(".");
103+
if (data && data[date[0]] && data[date[0]][date[1]] && data[date[0]][date[1]][date[2]]) {
104+
mergedData[periodObject.currentPeriodArr[z]] = {};
105+
mergedData[periodObject.currentPeriodArr[z]].c = data[date[0]][date[1]][date[2]].c || 0;
106+
mergedData[periodObject.currentPeriodArr[z]].s = data[date[0]][date[1]][date[2]].s || 0;
107+
mergedData[periodObject.currentPeriodArr[z]].dur = data[date[0]][date[1]][date[2]].dur || 0;
108+
totals.c += data[date[0]][date[1]][date[2]].c || 0;
109+
totals.s += data[date[0]][date[1]][date[2]].s || 0;
110+
totals.dur += data[date[0]][date[1]][date[2]].dur || 0;
111+
}
112+
}
113+
//console.log(" Aggregated data loaded");
114+
//console.log(JSON.stringify(mergedData));
115+
var report = {"totals": {}, "data": {}};
116+
var haveAnything = false;
117+
for (var key in totals) {
118+
if (totals[key] != drillData.totals[key]) {
119+
report.totals[key] = (totals[key] || 0) - (drillData.totals[key] || 0);
120+
haveAnything = true;
121+
}
122+
}
123+
for (var z = 0; z < periodObject.currentPeriodArr.length; z++) {
124+
if (drillData.data[periodObject.currentPeriodArr[z]]) {
125+
if (mergedData[periodObject.currentPeriodArr[z]]) {
126+
var diff = {};
127+
for (var key in mergedData[periodObject.currentPeriodArr[z]]) {
128+
diff[key] = (mergedData[periodObject.currentPeriodArr[z]][key] || 0) - (drillData.data[periodObject.currentPeriodArr[z]][key] || 0);
129+
}
130+
if (diff.c || diff.s || diff.dur) {
131+
report.data[periodObject.currentPeriodArr[z]] = diff;
132+
haveAnything = true;
133+
}
134+
}
135+
else {
136+
report.data[periodObject.currentPeriodArr[z]] = {};
137+
report.data[periodObject.currentPeriodArr[z]].c = -1 * drillData.data[periodObject.currentPeriodArr[z]].c;
138+
report.data[periodObject.currentPeriodArr[z]].s = -1 * drillData.data[periodObject.currentPeriodArr[z]].s;
139+
report.data[periodObject.currentPeriodArr[z]].dur = -1 * drillData.data[periodObject.currentPeriodArr[z]].dur;
140+
haveAnything;
141+
}
142+
}
143+
else {
144+
if (mergedData[periodObject.currentPeriodArr[z]]) {
145+
report.data[periodObject.currentPeriodArr[z]] = mergedData[periodObject.currentPeriodArr[z]];
146+
haveAnything = true;
147+
}
148+
}
149+
}
150+
if (haveAnything) {
151+
console.log(" " + JSON.stringify(report));
152+
endReport[app._id]["bad"]++;
153+
endReport[app._id]["events"] = endReport[app._id]["events"] || {};
154+
endReport[app._id]["events"][event] = {"e": event, report: report};
155+
}
156+
resolve2();
157+
});
158+
}
159+
});
160+
});
161+
}).then(function() {
162+
console.log("Finished processing app: ", app.name);
163+
resolve();
164+
}).catch(function(eee) {
165+
console.log("Error processing app: ", app.name);
166+
console.log(eee);
167+
reject();
168+
});
169+
}
170+
else {
171+
resolve();
172+
}
173+
}
174+
175+
});
176+
});
177+
178+
}).then(function() {
179+
console.log("Finished");
180+
console.log(JSON.stringify(endReport));
181+
close();
182+
}).catch(function(eee) {
183+
console.log("Error while fetching data");
184+
console.log(eee);
185+
console.log("EXITING...");
186+
console.log(JSON.stringify(endReport));
187+
close();
188+
});
189+
}
190+
});
191+
192+
function getDataFromDrill(options, callback) {
193+
var startDate = options.periodObj.start;
194+
var endDate = options.periodObj.end;
195+
196+
var tmpArr = options.periodObj.currentPeriodArr[0].split(".");
197+
startDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
198+
if (options.timezone) {
199+
startDate.tz(options.timezone);
200+
}
201+
startDate = startDate.valueOf() - startDate.utcOffset() * 60000;
202+
203+
tmpArr = options.periodObj.currentPeriodArr[options.periodObj.currentPeriodArr.length - 1].split(".");
204+
endDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
205+
if (options.timezone) {
206+
endDate.tz(options.timezone);
207+
}
208+
endDate = endDate.valueOf() - endDate.utcOffset() * 60000;
209+
210+
let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
211+
212+
var query = {"ts": {"$gte": startDate, "$lt": endDate}};
213+
var query2 = {"ts": {"$gte": startDate, "$lt": endDate}};
214+
/*query.e = options.event;
215+
query.a = options.app_id;*/
216+
217+
var pipeline = [
218+
{"$match": query},
219+
];
220+
/*if (use_union_with) {
221+
let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
222+
pipeline.push({"$unionWith": { "coll": collection, "pipeline": [{"$match": query2}]}});
223+
}*/
224+
225+
pipeline.push({"$group": {"_id": "$d", "c": {"$sum": "$c"}, "s": {"$sum": "$s"}, "dur": {"$sum": "$dur"}}});
226+
//console.log(JSON.stringify(pipeline));
227+
options.drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
228+
if (err) {
229+
console.log(err);
230+
}
231+
var result = {"data": {}, "totals": {"c": 0, "s": 0, "dur": 0}};
232+
if (data && data.length > 0) {
233+
for (var z = 0; z < data.length; z++) {
234+
var iid = data[z]._id.split(":").join(".");
235+
if (options.periodObj.currentPeriodArr.indexOf(iid) !== -1) {
236+
result.data[iid] = data[z];
237+
result.totals.c += data[z].c || 0;
238+
result.totals.s += data[z].s || 0;
239+
result.totals.dur += data[z].dur || 0;
240+
}
241+
}
242+
}
243+
callback(err, result);
244+
});
245+
246+
}
247+
248+
function close() {
249+
countlyDb.close();
250+
drillDb.close();
251+
console.log("Done.");
252+
}
253+
254+
function getAppList(options, calllback) {
255+
var query = {};
256+
if (app_list && app_list.length > 0) {
257+
var listed = [];
258+
for (var z = 0; z < app_list.length; z++) {
259+
listed.push(options.db.ObjectID(app_list[z]));
260+
}
261+
query = {_id: {$in: listed}};
262+
}
263+
options.db.collection("apps").find(query, {"name": 1, "timezone": 1}).toArray(function(err, apps) {
264+
if (err) {
265+
console.log("Error getting apps: ", err);
266+
}
267+
calllback(err, apps);
268+
});
269+
270+
271+
}
272+
});

0 commit comments

Comments
 (0)