Skip to content

Commit bfc8a8e

Browse files
ArtursKadikisArtursKadikis
committed
Merge branch 'master' into release.24.10
# Conflicts: # CHANGELOG.md
2 parents 0378d39 + a30812c commit bfc8a8e

File tree

1 file changed

+182
-90
lines changed

1 file changed

+182
-90
lines changed

‎bin/scripts/data-reports/compare_drill_aggregated.js

+182-90
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
/*
2-
* Script compares aggregated data from events collection with drill data for given period.
3-
* Place script ins
4-
* {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
5-
*
6-
* Usage:
7-
* node compare_drill_aggregated.js
8-
*/
9-
var period = "7days"; //Chose any of formats: "Xdays" ("7days","100days") or ["1-1-2024", "1-10-2024"],
10-
var app_list = []; //List with apps
11-
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12-
var eventMap = {}; //If left empty will run for all alls/events.
13-
var verbose = false; //true to show more output
14-
2+
* Script compares aggregated data from events collection with drill data for a given period.
3+
* Place the script in {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
4+
*
5+
* Usage:
6+
* node compare_drill_aggregated.js
7+
*
8+
*/
9+
var period = "7days"; // Choose any of formats: "Xdays" ("7days", "100days") or ["1-1-2024", "1-10-2024"],
10+
var app_list = []; // List with apps
11+
// Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12+
var eventMap = {}; // If left empty will run for all apps/events.
13+
const path = './summary_report.csv'; // Specify a valid path: 'summary_report.csv'
14+
var union_with_old_collection = true; // False if all sessions are stored in drill_events collection
15+
var verbose = false; // true to show more output
1516

1617
const Promise = require("bluebird");
1718
const crypto = require("crypto");
19+
const fs = require('fs');
1820
var pluginManager = require("../../../plugins/pluginManager");
1921
var fetch = require("../../../api/parts/data/fetch.js");
2022
var countlyCommon = require("../../../api/lib/countly.common.js");
@@ -23,7 +25,7 @@ var moment = require("moment");
2325

2426
var endReport = {};
2527

26-
console.log("Script compares numbers in aggregated data with drill data for given period.");
28+
console.log("Script compares numbers in aggregated data with drill data for the given period.");
2729
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
2830
common.db = countlyDb;
2931
console.log("Connected to databases...");
@@ -67,8 +69,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
6769
if (events.list.length != listBF) {
6870
console.log(" Filtered events based on eventMap: ", events.list.length, " from ", listBF);
6971
}
70-
7172
}
73+
7274
if (events && events.list && events.list.length) {
7375
endReport[app._id] = {"name": app.name, "total": events.list.length, "bad": 0};
7476
Promise.each(events.list, function(event) {
@@ -81,7 +83,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
8183
time: common.initTimeObj(app.timezone, Date.now().valueOf())
8284
};
8385

84-
//fetch drill data
86+
// Fetch drill data
8587
var periodObject = countlyCommon.getPeriodObj({"appTimezone": app.timezone, "qstring": {"period": period}});
8688
getDataFromDrill({event: event, app_id: app._id + "", timezone: app.timezone, drillDb: drillDb, periodObj: periodObject}, function(err, drillData) {
8789
if (err) {
@@ -93,7 +95,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
9395
console.log(" Drill data loaded");
9496
console.log(JSON.stringify(drillData));
9597
}
96-
//fetch aggregated data
98+
99+
// Fetch aggregated data
97100
var collectionName = "events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
98101

99102
fetch.getTimeObjForEvents(collectionName, params, null, function(data) {
@@ -111,10 +114,12 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
111114
totals.dur += data[date[0]][date[1]][date[2]].dur || 0;
112115
}
113116
}
117+
114118
if (verbose) {
115119
console.log(" Aggregated data loaded");
116120
console.log(JSON.stringify(mergedData));
117121
}
122+
118123
var report = {"totals": {}, "data": {}};
119124
var haveAnything = false;
120125
for (var key in totals) {
@@ -140,7 +145,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
140145
report.data[periodObject.currentPeriodArr[z]].c = -1 * drillData.data[periodObject.currentPeriodArr[z]].c;
141146
report.data[periodObject.currentPeriodArr[z]].s = -1 * drillData.data[periodObject.currentPeriodArr[z]].s;
142147
report.data[periodObject.currentPeriodArr[z]].dur = -1 * drillData.data[periodObject.currentPeriodArr[z]].dur;
143-
haveAnything;
148+
haveAnything = true;
144149
}
145150
}
146151
else {
@@ -150,113 +155,200 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
150155
}
151156
}
152157
}
158+
159+
let aggCount = totals.c || 0;
160+
let drillCount = drillData.totals.c || 0;
161+
let percentageDiff = 0;
162+
if (drillCount !== 0) {
163+
percentageDiff = ((drillCount - aggCount) / drillCount) * 100;
164+
}
165+
else {
166+
if (aggCount !== 0) {
167+
// If drillCount is 0, and aggCount is not 0, show a large difference
168+
percentageDiff = (aggCount > 0 ? 100 : -100); // 100% or -100% depending on the sign of aggCount
169+
}
170+
else {
171+
percentageDiff = 0; // Both counts are 0, no difference
172+
}
173+
}
174+
175+
console.log("----------------------------------------------");
176+
console.log("- Application name:", app.name);
177+
console.log("- Event name:", event);
178+
console.log("- Counts in Aggregated data:", aggCount);
179+
console.log("- Counts in Drill data:", drillCount);
180+
console.log("- Percentage difference between Drill data and Aggregated data:", percentageDiff.toFixed(2) + "%");
181+
console.log("-----------------------------------------------");
182+
183+
// Store detailed report in endReport
184+
endReport[app._id]["events"] = endReport[app._id]["events"] || {};
185+
endReport[app._id]["events"][event] = {
186+
"e": event,
187+
"aggregated_count": aggCount,
188+
"drill_count": drillCount,
189+
"percentage_difference": percentageDiff.toFixed(2)
190+
};
191+
153192
if (haveAnything) {
154-
console.log(" " + JSON.stringify(report));
193+
// Increment "bad" count if haveAnything is true
155194
endReport[app._id]["bad"]++;
156-
endReport[app._id]["events"] = endReport[app._id]["events"] || {};
157-
endReport[app._id]["events"][event] = {"e": event, report: report};
158195
}
196+
159197
resolve2();
160198
});
161199
}
162200
});
163201
});
164202
}).then(function() {
165203
console.log("Finished processing app: ", app.name);
204+
console.log("---------------------------------");
166205
resolve();
167206
}).catch(function(eee) {
168207
console.log("Error processing app: ", app.name);
208+
console.log("---------------------------------");
169209
console.log(eee);
170210
reject();
171211
});
172212
}
173213
else {
214+
console.log("No events in the App");
215+
console.log("---------------------------------");
174216
resolve();
175217
}
176218
}
177-
178219
});
179220
});
180-
181221
}).then(function() {
182222
console.log("Finished");
183-
console.log(JSON.stringify(endReport));
184-
close();
223+
try {
224+
// Complete CSV after processing the apps
225+
console.log("\nSummary Report (CSV-like):");
226+
console.log("App,Event,Aggregated,Drill,% Difference");
227+
228+
let csvRows = ["App,Event,Aggregated,Drill,PercentageDifference"];
229+
for (let appId in endReport) {
230+
let appReport = endReport[appId];
231+
for (let event in appReport.events) {
232+
let eventReport = appReport.events[event];
233+
csvRows.push(`${appReport.name},${eventReport.e},${eventReport.aggregated_count},${eventReport.drill_count},${eventReport.percentage_difference}`);
234+
}
235+
}
236+
fs.writeFileSync(path, csvRows.join("\n"));
237+
console.log("Summary report saved to '" + path + "'.");
238+
239+
console.log(JSON.stringify(endReport));
240+
close();
241+
}
242+
catch (err) {
243+
console.error("Failed to save partial report:", err);
244+
}
185245
}).catch(function(eee) {
186246
console.log("Error while fetching data");
187247
console.log(eee);
188-
console.log("EXITING...");
189-
console.log(JSON.stringify(endReport));
190-
close();
191-
});
192-
}
193-
});
194-
195-
function getDataFromDrill(options, callback) {
196-
var tmpArr = options.periodObj.currentPeriodArr[0].split(".");
197-
var startDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
198-
if (options.timezone) {
199-
startDate.tz(options.timezone);
200-
}
201-
startDate = startDate.valueOf() - startDate.utcOffset() * 60000;
202-
203-
tmpArr = options.periodObj.currentPeriodArr[options.periodObj.currentPeriodArr.length - 1].split(".");
204-
var endDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
205-
if (options.timezone) {
206-
endDate.tz(options.timezone);
207-
}
208-
endDate = endDate.valueOf() - endDate.utcOffset() * 60000;
209-
210-
let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
211-
var query = {"ts": {"$gte": startDate, "$lt": endDate}};
212-
var pipeline = [
213-
{"$match": query},
214-
];
248+
try {
249+
// Complete CSV after processing the apps
250+
console.log("\nSummary Report (CSV-like):");
251+
console.log("App,Event,Aggregated,Drill,% Difference");
215252

216-
pipeline.push({"$group": {"_id": "$d", "c": {"$sum": "$c"}, "s": {"$sum": "$s"}, "dur": {"$sum": "$dur"}}});
217-
options.drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
218-
if (err) {
219-
console.log(err);
220-
}
221-
var result = {"data": {}, "totals": {"c": 0, "s": 0, "dur": 0}};
222-
if (data && data.length > 0) {
223-
for (var z = 0; z < data.length; z++) {
224-
var iid = data[z]._id.split(":").join(".");
225-
if (options.periodObj.currentPeriodArr.indexOf(iid) !== -1) {
226-
result.data[iid] = data[z];
227-
result.totals.c += data[z].c || 0;
228-
result.totals.s += data[z].s || 0;
229-
result.totals.dur += data[z].dur || 0;
253+
let csvRows = ["App,Event,Aggregated,Drill,PercentageDifference"];
254+
for (let appId in endReport) {
255+
let appReport = endReport[appId];
256+
for (let event in appReport.events) {
257+
let eventReport = appReport.events[event];
258+
csvRows.push(`${appReport.name},${eventReport.e},${eventReport.aggregated_count},${eventReport.drill_count},${eventReport.percentage_difference}`);
259+
}
230260
}
261+
fs.writeFileSync(path, csvRows.join("\n"));
262+
console.log("Partial summary report saved to '" + path + "'.");
263+
}
264+
catch (err) {
265+
console.error("Failed to save partial report:", err);
266+
}
267+
return close();
268+
});
269+
}
270+
function close() {
271+
try {
272+
if (countlyDb?.close) {
273+
countlyDb.close();
274+
}
275+
if (drillDb?.close) {
276+
drillDb.close();
231277
}
232278
}
233-
callback(err, result);
234-
});
235-
279+
catch (err) {
280+
console.error("Error occurred while closing database connections:", err);
281+
}
282+
finally {
283+
console.log("Done.");
284+
console.log("EXITING...");
285+
}
286+
}
287+
});
288+
}).catch(function(eee) {
289+
console.log("Error while fetching data");
290+
console.log(eee);
291+
});
292+
function getDataFromDrill(options, callback) {
293+
var tmpArr = options.periodObj.currentPeriodArr[0].split(".");
294+
var startDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
295+
if (options.timezone) {
296+
startDate.tz(options.timezone);
236297
}
237-
238-
function close() {
239-
countlyDb.close();
240-
drillDb.close();
241-
console.log("Done.");
298+
startDate = startDate.valueOf() - startDate.utcOffset() * 60000;
299+
tmpArr = options.periodObj.currentPeriodArr[options.periodObj.currentPeriodArr.length - 1].split(".");
300+
var endDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
301+
if (options.timezone) {
302+
endDate.tz(options.timezone);
303+
}
304+
endDate = endDate.valueOf() - endDate.utcOffset() * 60000;
305+
var query = { "ts": { "$gte": startDate, "$lt": endDate }, "a": options.app_id, "e": options.event };
306+
var pipeline = [];
307+
pipeline.push({ "$match": query });
308+
if (union_with_old_collection) {
309+
let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
310+
var query2 = { "ts": { "$gte": startDate, "$lt": endDate } };
311+
pipeline.push({ "$unionWith": { "coll": collection, "pipeline": [{ "$match": query2 }] } });
242312
}
243313

244-
function getAppList(options, calllback) {
245-
var query = {};
246-
if (app_list && app_list.length > 0) {
247-
var listed = [];
248-
for (var z = 0; z < app_list.length; z++) {
249-
listed.push(options.db.ObjectID(app_list[z]));
250-
}
251-
query = {_id: {$in: listed}};
314+
pipeline.push({ "$group": { "_id": "$d", "c": { "$sum": "$c" }, "s": { "$sum": "$s" }, "dur": { "$sum": "$dur" } } });
315+
options.drillDb.collection("drill_events").aggregate(pipeline, { "allowDiskUse": true }).toArray(function(err, data) {
316+
if (err) {
317+
console.log(err);
318+
callback(err, null);
319+
return;
252320
}
253-
options.db.collection("apps").find(query, {"name": 1, "timezone": 1}).toArray(function(err, apps) {
254-
if (err) {
255-
console.log("Error getting apps: ", err);
321+
var result = { "data": {}, "totals": { "c": 0, "s": 0, "dur": 0 } };
322+
if (data && data.length > 0) {
323+
for (var z = 0; z < data.length; z++) {
324+
var iid = data[z]._id.split(":").join(".");
325+
if (options.periodObj.currentPeriodArr.indexOf(iid) !== -1) {
326+
result.data[iid] = data[z];
327+
result.totals.c += data[z].c || 0;
328+
result.totals.s += data[z].s || 0;
329+
result.totals.dur += data[z].dur || 0;
330+
}
256331
}
257-
calllback(err, apps);
258-
});
259-
260-
332+
}
333+
else {
334+
console.log("No drill data found for event:", options.event);
335+
}
336+
callback(err, result);
337+
});
338+
}
339+
function getAppList(options, callback) {
340+
var query = {};
341+
if (app_list && app_list.length > 0) {
342+
var listed = [];
343+
for (var z = 0; z < app_list.length; z++) {
344+
listed.push(options.db.ObjectID(app_list[z]));
345+
}
346+
query = { _id: { $in: listed } };
261347
}
262-
});
348+
options.db.collection("apps").find(query, { "name": 1, "timezone": 1 }).toArray(function(err, apps) {
349+
if (err) {
350+
console.log("Error getting apps: ", err);
351+
}
352+
callback(err, apps);
353+
});
354+
}

0 commit comments

Comments
 (0)