Skip to content

Commit e3f532d

Browse files
CookiezaursCookiezaurs
authored
Merge pull request #5625 from Countly/anna/master
Script to compare aggregated events data with granural data
2 parents eb3c24e + 5b8f76a commit e3f532d

File tree

1 file changed

+262
-0
lines changed

1 file changed

+262
-0
lines changed

‎bin/scripts/data-reports/compare_drill_aggregated.js

+262
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
/*
2+
* Script compares aggregated data from events collection with drill data for given period.
3+
* Place script ins
4+
* {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
5+
*
6+
* Usage:
7+
* node compare_drill_aggregated.js
8+
*/
9+
var period = "7days"; //Chose any of formats: "Xdays" ("7days","100days") or ["1-1-2024", "1-10-2024"],
10+
var app_list = []; //List with apps
11+
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12+
var eventMap = {}; //If left empty will run for all alls/events.
13+
var verbose = false; //true to show more output
14+
15+
16+
const Promise = require("bluebird");
17+
const crypto = require("crypto");
18+
var pluginManager = require("../../../plugins/pluginManager");
19+
var fetch = require("../../../api/parts/data/fetch.js");
20+
var countlyCommon = require("../../../api/lib/countly.common.js");
21+
var common = require("../../../api/utils/common.js");
22+
var moment = require("moment");
23+
24+
var endReport = {};
25+
26+
console.log("Script compares numbers in aggregated data with drill data for given period.");
27+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
28+
common.db = countlyDb;
29+
console.log("Connected to databases...");
30+
getAppList({db: countlyDb}, function(err, apps) {
31+
if (!apps || !apps.length) {
32+
console.log("No apps found");
33+
return close();
34+
}
35+
else {
36+
console.log("Apps found:", apps.length);
37+
Promise.each(apps, function(app) {
38+
return new Promise(function(resolve, reject) {
39+
console.log("Processing app: ", app.name);
40+
countlyDb.collection("events").findOne({_id: app._id}, {'list': 1}, function(err, events) {
41+
if (err) {
42+
console.log("Error getting events: ", err);
43+
reject();
44+
}
45+
else {
46+
events = events || {};
47+
events.list = events.list || [];
48+
events.list = events.list.filter(function(ee) {
49+
if (ee.indexOf("[CLY]_") === 0) {
50+
return false;
51+
}
52+
else {
53+
return true;
54+
}
55+
});
56+
57+
if (eventMap && eventMap[app._id + ""]) {
58+
var listBF = events.list.length;
59+
events.list = events.list.filter(function(ee) {
60+
if (eventMap && eventMap[app._id + ""]) {
61+
return eventMap[app._id + ""].indexOf(ee) > -1;
62+
}
63+
else {
64+
return false;
65+
}
66+
});
67+
if (events.list.length != listBF) {
68+
console.log(" Filtered events based on eventMap: ", events.list.length, " from ", listBF);
69+
}
70+
71+
}
72+
if (events && events.list && events.list.length) {
73+
endReport[app._id] = {"name": app.name, "total": events.list.length, "bad": 0};
74+
Promise.each(events.list, function(event) {
75+
return new Promise(function(resolve2, reject2) {
76+
console.log(" Processing event: ", event);
77+
var params = {
78+
app_id: app._id + "",
79+
appTimezone: app.timezone,
80+
qstring: { period: period},
81+
time: common.initTimeObj(app.timezone, Date.now().valueOf())
82+
};
83+
84+
//fetch drill data
85+
var periodObject = countlyCommon.getPeriodObj({"appTimezone": app.timezone, "qstring": {"period": period}});
86+
getDataFromDrill({event: event, app_id: app._id + "", timezone: app.timezone, drillDb: drillDb, periodObj: periodObject}, function(err, drillData) {
87+
if (err) {
88+
console.log(" Error getting drill data: ", err);
89+
reject2();
90+
}
91+
else {
92+
if (verbose) {
93+
console.log(" Drill data loaded");
94+
console.log(JSON.stringify(drillData));
95+
}
96+
//fetch aggregated data
97+
var collectionName = "events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
98+
99+
fetch.getTimeObjForEvents(collectionName, params, null, function(data) {
100+
var mergedData = {};
101+
var totals = {"c": 0, "s": 0, "dur": 0};
102+
for (var z0 = 0; z0 < periodObject.currentPeriodArr.length; z0++) {
103+
var date = periodObject.currentPeriodArr[z0].split(".");
104+
if (data && data[date[0]] && data[date[0]][date[1]] && data[date[0]][date[1]][date[2]]) {
105+
mergedData[periodObject.currentPeriodArr[z0]] = {};
106+
mergedData[periodObject.currentPeriodArr[z0]].c = data[date[0]][date[1]][date[2]].c || 0;
107+
mergedData[periodObject.currentPeriodArr[z0]].s = data[date[0]][date[1]][date[2]].s || 0;
108+
mergedData[periodObject.currentPeriodArr[z0]].dur = data[date[0]][date[1]][date[2]].dur || 0;
109+
totals.c += data[date[0]][date[1]][date[2]].c || 0;
110+
totals.s += data[date[0]][date[1]][date[2]].s || 0;
111+
totals.dur += data[date[0]][date[1]][date[2]].dur || 0;
112+
}
113+
}
114+
if (verbose) {
115+
console.log(" Aggregated data loaded");
116+
console.log(JSON.stringify(mergedData));
117+
}
118+
var report = {"totals": {}, "data": {}};
119+
var haveAnything = false;
120+
for (var key in totals) {
121+
if (totals[key] != drillData.totals[key]) {
122+
report.totals[key] = (totals[key] || 0) - (drillData.totals[key] || 0);
123+
haveAnything = true;
124+
}
125+
}
126+
for (var z = 0; z < periodObject.currentPeriodArr.length; z++) {
127+
if (drillData.data[periodObject.currentPeriodArr[z]]) {
128+
if (mergedData[periodObject.currentPeriodArr[z]]) {
129+
var diff = {};
130+
for (var key0 in mergedData[periodObject.currentPeriodArr[z]]) {
131+
diff[key0] = (mergedData[periodObject.currentPeriodArr[z]][key0] || 0) - (drillData.data[periodObject.currentPeriodArr[z]][key0] || 0);
132+
}
133+
if (diff.c || diff.s || diff.dur) {
134+
report.data[periodObject.currentPeriodArr[z]] = diff;
135+
haveAnything = true;
136+
}
137+
}
138+
else {
139+
report.data[periodObject.currentPeriodArr[z]] = {};
140+
report.data[periodObject.currentPeriodArr[z]].c = -1 * drillData.data[periodObject.currentPeriodArr[z]].c;
141+
report.data[periodObject.currentPeriodArr[z]].s = -1 * drillData.data[periodObject.currentPeriodArr[z]].s;
142+
report.data[periodObject.currentPeriodArr[z]].dur = -1 * drillData.data[periodObject.currentPeriodArr[z]].dur;
143+
haveAnything;
144+
}
145+
}
146+
else {
147+
if (mergedData[periodObject.currentPeriodArr[z]]) {
148+
report.data[periodObject.currentPeriodArr[z]] = mergedData[periodObject.currentPeriodArr[z]];
149+
haveAnything = true;
150+
}
151+
}
152+
}
153+
if (haveAnything) {
154+
console.log(" " + JSON.stringify(report));
155+
endReport[app._id]["bad"]++;
156+
endReport[app._id]["events"] = endReport[app._id]["events"] || {};
157+
endReport[app._id]["events"][event] = {"e": event, report: report};
158+
}
159+
resolve2();
160+
});
161+
}
162+
});
163+
});
164+
}).then(function() {
165+
console.log("Finished processing app: ", app.name);
166+
resolve();
167+
}).catch(function(eee) {
168+
console.log("Error processing app: ", app.name);
169+
console.log(eee);
170+
reject();
171+
});
172+
}
173+
else {
174+
resolve();
175+
}
176+
}
177+
178+
});
179+
});
180+
181+
}).then(function() {
182+
console.log("Finished");
183+
console.log(JSON.stringify(endReport));
184+
close();
185+
}).catch(function(eee) {
186+
console.log("Error while fetching data");
187+
console.log(eee);
188+
console.log("EXITING...");
189+
console.log(JSON.stringify(endReport));
190+
close();
191+
});
192+
}
193+
});
194+
195+
function getDataFromDrill(options, callback) {
196+
var tmpArr = options.periodObj.currentPeriodArr[0].split(".");
197+
var startDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
198+
if (options.timezone) {
199+
startDate.tz(options.timezone);
200+
}
201+
startDate = startDate.valueOf() - startDate.utcOffset() * 60000;
202+
203+
tmpArr = options.periodObj.currentPeriodArr[options.periodObj.currentPeriodArr.length - 1].split(".");
204+
var endDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
205+
if (options.timezone) {
206+
endDate.tz(options.timezone);
207+
}
208+
endDate = endDate.valueOf() - endDate.utcOffset() * 60000;
209+
210+
let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
211+
var query = {"ts": {"$gte": startDate, "$lt": endDate}};
212+
var pipeline = [
213+
{"$match": query},
214+
];
215+
216+
pipeline.push({"$group": {"_id": "$d", "c": {"$sum": "$c"}, "s": {"$sum": "$s"}, "dur": {"$sum": "$dur"}}});
217+
options.drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
218+
if (err) {
219+
console.log(err);
220+
}
221+
var result = {"data": {}, "totals": {"c": 0, "s": 0, "dur": 0}};
222+
if (data && data.length > 0) {
223+
for (var z = 0; z < data.length; z++) {
224+
var iid = data[z]._id.split(":").join(".");
225+
if (options.periodObj.currentPeriodArr.indexOf(iid) !== -1) {
226+
result.data[iid] = data[z];
227+
result.totals.c += data[z].c || 0;
228+
result.totals.s += data[z].s || 0;
229+
result.totals.dur += data[z].dur || 0;
230+
}
231+
}
232+
}
233+
callback(err, result);
234+
});
235+
236+
}
237+
238+
function close() {
239+
countlyDb.close();
240+
drillDb.close();
241+
console.log("Done.");
242+
}
243+
244+
function getAppList(options, calllback) {
245+
var query = {};
246+
if (app_list && app_list.length > 0) {
247+
var listed = [];
248+
for (var z = 0; z < app_list.length; z++) {
249+
listed.push(options.db.ObjectID(app_list[z]));
250+
}
251+
query = {_id: {$in: listed}};
252+
}
253+
options.db.collection("apps").find(query, {"name": 1, "timezone": 1}).toArray(function(err, apps) {
254+
if (err) {
255+
console.log("Error getting apps: ", err);
256+
}
257+
calllback(err, apps);
258+
});
259+
260+
261+
}
262+
});

0 commit comments

Comments
 (0)