1
1
/*
2
- * Script compares aggregated data from events collection with drill data for given period.
3
- * Place script ins
4
- * {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
5
- *
6
- * Usage:
7
- * node compare_drill_aggregated.js
8
- */
9
- var period = "7days" ; //Chose any of formats: "Xdays" ("7days","100days") or ["1-1-2024", "1-10-2024"],
10
- var app_list = [ ] ; //List with apps
11
- //Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12
- var eventMap = { } ; //If left empty will run for all alls/events.
13
- var verbose = false ; //true to show more output
14
-
2
+ * Script compares aggregated data from events collection with drill data for a given period.
3
+ * Place the script in {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
4
+ *
5
+ * Usage:
6
+ * node compare_drill_aggregated.js
7
+ *
8
+ */
9
+ var period = "7days" ; // Choose any of formats: "Xdays" ("7days", "100days") or ["1-1-2024", "1-10-2024"],
10
+ var app_list = [ ] ; // List with apps
11
+ // Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12
+ var eventMap = { } ; // If left empty will run for all apps/events.
13
+ const path = './summary_report.csv' ; // Specify a valid path: 'summary_report.csv'
14
+ var union_with_old_collection = true ; // False if all sessions are stored in drill_events collection
15
+ var verbose = false ; // true to show more output
15
16
16
17
const Promise = require ( "bluebird" ) ;
17
18
const crypto = require ( "crypto" ) ;
19
+ const fs = require ( 'fs' ) ;
18
20
var pluginManager = require ( "../../../plugins/pluginManager" ) ;
19
21
var fetch = require ( "../../../api/parts/data/fetch.js" ) ;
20
22
var countlyCommon = require ( "../../../api/lib/countly.common.js" ) ;
@@ -23,7 +25,7 @@ var moment = require("moment");
23
25
24
26
var endReport = { } ;
25
27
26
- console . log ( "Script compares numbers in aggregated data with drill data for given period." ) ;
28
+ console . log ( "Script compares numbers in aggregated data with drill data for the given period." ) ;
27
29
Promise . all ( [ pluginManager . dbConnection ( "countly" ) , pluginManager . dbConnection ( "countly_drill" ) ] ) . then ( async function ( [ countlyDb , drillDb ] ) {
28
30
common . db = countlyDb ;
29
31
console . log ( "Connected to databases..." ) ;
@@ -67,8 +69,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
67
69
if ( events . list . length != listBF ) {
68
70
console . log ( " Filtered events based on eventMap: " , events . list . length , " from " , listBF ) ;
69
71
}
70
-
71
72
}
73
+
72
74
if ( events && events . list && events . list . length ) {
73
75
endReport [ app . _id ] = { "name" : app . name , "total" : events . list . length , "bad" : 0 } ;
74
76
Promise . each ( events . list , function ( event ) {
@@ -81,7 +83,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
81
83
time : common . initTimeObj ( app . timezone , Date . now ( ) . valueOf ( ) )
82
84
} ;
83
85
84
- //fetch drill data
86
+ // Fetch drill data
85
87
var periodObject = countlyCommon . getPeriodObj ( { "appTimezone" : app . timezone , "qstring" : { "period" : period } } ) ;
86
88
getDataFromDrill ( { event : event , app_id : app . _id + "" , timezone : app . timezone , drillDb : drillDb , periodObj : periodObject } , function ( err , drillData ) {
87
89
if ( err ) {
@@ -93,7 +95,8 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
93
95
console . log ( " Drill data loaded" ) ;
94
96
console . log ( JSON . stringify ( drillData ) ) ;
95
97
}
96
- //fetch aggregated data
98
+
99
+ // Fetch aggregated data
97
100
var collectionName = "events" + crypto . createHash ( 'sha1' ) . update ( event + app . _id ) . digest ( 'hex' ) ;
98
101
99
102
fetch . getTimeObjForEvents ( collectionName , params , null , function ( data ) {
@@ -111,10 +114,12 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
111
114
totals . dur += data [ date [ 0 ] ] [ date [ 1 ] ] [ date [ 2 ] ] . dur || 0 ;
112
115
}
113
116
}
117
+
114
118
if ( verbose ) {
115
119
console . log ( " Aggregated data loaded" ) ;
116
120
console . log ( JSON . stringify ( mergedData ) ) ;
117
121
}
122
+
118
123
var report = { "totals" : { } , "data" : { } } ;
119
124
var haveAnything = false ;
120
125
for ( var key in totals ) {
@@ -140,7 +145,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
140
145
report . data [ periodObject . currentPeriodArr [ z ] ] . c = - 1 * drillData . data [ periodObject . currentPeriodArr [ z ] ] . c ;
141
146
report . data [ periodObject . currentPeriodArr [ z ] ] . s = - 1 * drillData . data [ periodObject . currentPeriodArr [ z ] ] . s ;
142
147
report . data [ periodObject . currentPeriodArr [ z ] ] . dur = - 1 * drillData . data [ periodObject . currentPeriodArr [ z ] ] . dur ;
143
- haveAnything ;
148
+ haveAnything = true ;
144
149
}
145
150
}
146
151
else {
@@ -150,113 +155,200 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
150
155
}
151
156
}
152
157
}
158
+
159
+ let aggCount = totals . c || 0 ;
160
+ let drillCount = drillData . totals . c || 0 ;
161
+ let percentageDiff = 0 ;
162
+ if ( drillCount !== 0 ) {
163
+ percentageDiff = ( ( drillCount - aggCount ) / drillCount ) * 100 ;
164
+ }
165
+ else {
166
+ if ( aggCount !== 0 ) {
167
+ // If drillCount is 0, and aggCount is not 0, show a large difference
168
+ percentageDiff = ( aggCount > 0 ? 100 : - 100 ) ; // 100% or -100% depending on the sign of aggCount
169
+ }
170
+ else {
171
+ percentageDiff = 0 ; // Both counts are 0, no difference
172
+ }
173
+ }
174
+
175
+ console . log ( "----------------------------------------------" ) ;
176
+ console . log ( "- Application name:" , app . name ) ;
177
+ console . log ( "- Event name:" , event ) ;
178
+ console . log ( "- Counts in Aggregated data:" , aggCount ) ;
179
+ console . log ( "- Counts in Drill data:" , drillCount ) ;
180
+ console . log ( "- Percentage difference between Drill data and Aggregated data:" , percentageDiff . toFixed ( 2 ) + "%" ) ;
181
+ console . log ( "-----------------------------------------------" ) ;
182
+
183
+ // Store detailed report in endReport
184
+ endReport [ app . _id ] [ "events" ] = endReport [ app . _id ] [ "events" ] || { } ;
185
+ endReport [ app . _id ] [ "events" ] [ event ] = {
186
+ "e" : event ,
187
+ "aggregated_count" : aggCount ,
188
+ "drill_count" : drillCount ,
189
+ "percentage_difference" : percentageDiff . toFixed ( 2 )
190
+ } ;
191
+
153
192
if ( haveAnything ) {
154
- console . log ( " " + JSON . stringify ( report ) ) ;
193
+ // Increment "bad" count if haveAnything is true
155
194
endReport [ app . _id ] [ "bad" ] ++ ;
156
- endReport [ app . _id ] [ "events" ] = endReport [ app . _id ] [ "events" ] || { } ;
157
- endReport [ app . _id ] [ "events" ] [ event ] = { "e" : event , report : report } ;
158
195
}
196
+
159
197
resolve2 ( ) ;
160
198
} ) ;
161
199
}
162
200
} ) ;
163
201
} ) ;
164
202
} ) . then ( function ( ) {
165
203
console . log ( "Finished processing app: " , app . name ) ;
204
+ console . log ( "---------------------------------" ) ;
166
205
resolve ( ) ;
167
206
} ) . catch ( function ( eee ) {
168
207
console . log ( "Error processing app: " , app . name ) ;
208
+ console . log ( "---------------------------------" ) ;
169
209
console . log ( eee ) ;
170
210
reject ( ) ;
171
211
} ) ;
172
212
}
173
213
else {
214
+ console . log ( "No events in the App" ) ;
215
+ console . log ( "---------------------------------" ) ;
174
216
resolve ( ) ;
175
217
}
176
218
}
177
-
178
219
} ) ;
179
220
} ) ;
180
-
181
221
} ) . then ( function ( ) {
182
222
console . log ( "Finished" ) ;
183
- console . log ( JSON . stringify ( endReport ) ) ;
184
- close ( ) ;
223
+ try {
224
+ // Complete CSV after processing the apps
225
+ console . log ( "\nSummary Report (CSV-like):" ) ;
226
+ console . log ( "App,Event,Aggregated,Drill,% Difference" ) ;
227
+
228
+ let csvRows = [ "App,Event,Aggregated,Drill,PercentageDifference" ] ;
229
+ for ( let appId in endReport ) {
230
+ let appReport = endReport [ appId ] ;
231
+ for ( let event in appReport . events ) {
232
+ let eventReport = appReport . events [ event ] ;
233
+ csvRows . push ( `${ appReport . name } ,${ eventReport . e } ,${ eventReport . aggregated_count } ,${ eventReport . drill_count } ,${ eventReport . percentage_difference } ` ) ;
234
+ }
235
+ }
236
+ fs . writeFileSync ( path , csvRows . join ( "\n" ) ) ;
237
+ console . log ( "Summary report saved to '" + path + "'." ) ;
238
+
239
+ console . log ( JSON . stringify ( endReport ) ) ;
240
+ close ( ) ;
241
+ }
242
+ catch ( err ) {
243
+ console . error ( "Failed to save partial report:" , err ) ;
244
+ }
185
245
} ) . catch ( function ( eee ) {
186
246
console . log ( "Error while fetching data" ) ;
187
247
console . log ( eee ) ;
188
- console . log ( "EXITING..." ) ;
189
- console . log ( JSON . stringify ( endReport ) ) ;
190
- close ( ) ;
191
- } ) ;
192
- }
193
- } ) ;
194
-
195
- function getDataFromDrill ( options , callback ) {
196
- var tmpArr = options . periodObj . currentPeriodArr [ 0 ] . split ( "." ) ;
197
- var startDate = moment ( new Date ( Date . UTC ( parseInt ( tmpArr [ 0 ] ) , parseInt ( tmpArr [ 1 ] ) - 1 , parseInt ( tmpArr [ 2 ] ) ) ) ) ;
198
- if ( options . timezone ) {
199
- startDate . tz ( options . timezone ) ;
200
- }
201
- startDate = startDate . valueOf ( ) - startDate . utcOffset ( ) * 60000 ;
202
-
203
- tmpArr = options . periodObj . currentPeriodArr [ options . periodObj . currentPeriodArr . length - 1 ] . split ( "." ) ;
204
- var endDate = moment ( new Date ( Date . UTC ( parseInt ( tmpArr [ 0 ] ) , parseInt ( tmpArr [ 1 ] ) - 1 , parseInt ( tmpArr [ 2 ] ) ) ) ) . add ( 1 , 'days' ) ;
205
- if ( options . timezone ) {
206
- endDate . tz ( options . timezone ) ;
207
- }
208
- endDate = endDate . valueOf ( ) - endDate . utcOffset ( ) * 60000 ;
209
-
210
- let collection = "drill_events" + crypto . createHash ( 'sha1' ) . update ( options . event + options . app_id ) . digest ( 'hex' ) ;
211
- var query = { "ts" : { "$gte" : startDate , "$lt" : endDate } } ;
212
- var pipeline = [
213
- { "$match" : query } ,
214
- ] ;
248
+ try {
249
+ // Complete CSV after processing the apps
250
+ console . log ( "\nSummary Report (CSV-like):" ) ;
251
+ console . log ( "App,Event,Aggregated,Drill,% Difference" ) ;
215
252
216
- pipeline . push ( { "$group" : { "_id" : "$d" , "c" : { "$sum" : "$c" } , "s" : { "$sum" : "$s" } , "dur" : { "$sum" : "$dur" } } } ) ;
217
- options . drillDb . collection ( collection ) . aggregate ( pipeline , { "allowDiskUse" : true } ) . toArray ( function ( err , data ) {
218
- if ( err ) {
219
- console . log ( err ) ;
220
- }
221
- var result = { "data" : { } , "totals" : { "c" : 0 , "s" : 0 , "dur" : 0 } } ;
222
- if ( data && data . length > 0 ) {
223
- for ( var z = 0 ; z < data . length ; z ++ ) {
224
- var iid = data [ z ] . _id . split ( ":" ) . join ( "." ) ;
225
- if ( options . periodObj . currentPeriodArr . indexOf ( iid ) !== - 1 ) {
226
- result . data [ iid ] = data [ z ] ;
227
- result . totals . c += data [ z ] . c || 0 ;
228
- result . totals . s += data [ z ] . s || 0 ;
229
- result . totals . dur += data [ z ] . dur || 0 ;
253
+ let csvRows = [ "App,Event,Aggregated,Drill,PercentageDifference" ] ;
254
+ for ( let appId in endReport ) {
255
+ let appReport = endReport [ appId ] ;
256
+ for ( let event in appReport . events ) {
257
+ let eventReport = appReport . events [ event ] ;
258
+ csvRows . push ( `${ appReport . name } ,${ eventReport . e } ,${ eventReport . aggregated_count } ,${ eventReport . drill_count } ,${ eventReport . percentage_difference } ` ) ;
259
+ }
230
260
}
261
+ fs . writeFileSync ( path , csvRows . join ( "\n" ) ) ;
262
+ console . log ( "Partial summary report saved to '" + path + "'." ) ;
263
+ }
264
+ catch ( err ) {
265
+ console . error ( "Failed to save partial report:" , err ) ;
266
+ }
267
+ return close ( ) ;
268
+ } ) ;
269
+ }
270
+ function close ( ) {
271
+ try {
272
+ if ( countlyDb ?. close ) {
273
+ countlyDb . close ( ) ;
274
+ }
275
+ if ( drillDb ?. close ) {
276
+ drillDb . close ( ) ;
231
277
}
232
278
}
233
- callback ( err , result ) ;
234
- } ) ;
235
-
279
+ catch ( err ) {
280
+ console . error ( "Error occurred while closing database connections:" , err ) ;
281
+ }
282
+ finally {
283
+ console . log ( "Done." ) ;
284
+ console . log ( "EXITING..." ) ;
285
+ }
286
+ }
287
+ } ) ;
288
+ } ) . catch ( function ( eee ) {
289
+ console . log ( "Error while fetching data" ) ;
290
+ console . log ( eee ) ;
291
+ } ) ;
292
+ function getDataFromDrill ( options , callback ) {
293
+ var tmpArr = options . periodObj . currentPeriodArr [ 0 ] . split ( "." ) ;
294
+ var startDate = moment ( new Date ( Date . UTC ( parseInt ( tmpArr [ 0 ] ) , parseInt ( tmpArr [ 1 ] ) - 1 , parseInt ( tmpArr [ 2 ] ) ) ) ) ;
295
+ if ( options . timezone ) {
296
+ startDate . tz ( options . timezone ) ;
236
297
}
237
-
238
- function close ( ) {
239
- countlyDb . close ( ) ;
240
- drillDb . close ( ) ;
241
- console . log ( "Done." ) ;
298
+ startDate = startDate . valueOf ( ) - startDate . utcOffset ( ) * 60000 ;
299
+ tmpArr = options . periodObj . currentPeriodArr [ options . periodObj . currentPeriodArr . length - 1 ] . split ( "." ) ;
300
+ var endDate = moment ( new Date ( Date . UTC ( parseInt ( tmpArr [ 0 ] ) , parseInt ( tmpArr [ 1 ] ) - 1 , parseInt ( tmpArr [ 2 ] ) ) ) ) . add ( 1 , 'days' ) ;
301
+ if ( options . timezone ) {
302
+ endDate . tz ( options . timezone ) ;
303
+ }
304
+ endDate = endDate . valueOf ( ) - endDate . utcOffset ( ) * 60000 ;
305
+ var query = { "ts" : { "$gte" : startDate , "$lt" : endDate } , "a" : options . app_id , "e" : options . event } ;
306
+ var pipeline = [ ] ;
307
+ pipeline . push ( { "$match" : query } ) ;
308
+ if ( union_with_old_collection ) {
309
+ let collection = "drill_events" + crypto . createHash ( 'sha1' ) . update ( options . event + options . app_id ) . digest ( 'hex' ) ;
310
+ var query2 = { "ts" : { "$gte" : startDate , "$lt" : endDate } } ;
311
+ pipeline . push ( { "$unionWith" : { "coll" : collection , "pipeline" : [ { "$match" : query2 } ] } } ) ;
242
312
}
243
313
244
- function getAppList ( options , calllback ) {
245
- var query = { } ;
246
- if ( app_list && app_list . length > 0 ) {
247
- var listed = [ ] ;
248
- for ( var z = 0 ; z < app_list . length ; z ++ ) {
249
- listed . push ( options . db . ObjectID ( app_list [ z ] ) ) ;
250
- }
251
- query = { _id : { $in : listed } } ;
314
+ pipeline . push ( { "$group" : { "_id" : "$d" , "c" : { "$sum" : "$c" } , "s" : { "$sum" : "$s" } , "dur" : { "$sum" : "$dur" } } } ) ;
315
+ options . drillDb . collection ( "drill_events" ) . aggregate ( pipeline , { "allowDiskUse" : true } ) . toArray ( function ( err , data ) {
316
+ if ( err ) {
317
+ console . log ( err ) ;
318
+ callback ( err , null ) ;
319
+ return ;
252
320
}
253
- options . db . collection ( "apps" ) . find ( query , { "name" : 1 , "timezone" : 1 } ) . toArray ( function ( err , apps ) {
254
- if ( err ) {
255
- console . log ( "Error getting apps: " , err ) ;
321
+ var result = { "data" : { } , "totals" : { "c" : 0 , "s" : 0 , "dur" : 0 } } ;
322
+ if ( data && data . length > 0 ) {
323
+ for ( var z = 0 ; z < data . length ; z ++ ) {
324
+ var iid = data [ z ] . _id . split ( ":" ) . join ( "." ) ;
325
+ if ( options . periodObj . currentPeriodArr . indexOf ( iid ) !== - 1 ) {
326
+ result . data [ iid ] = data [ z ] ;
327
+ result . totals . c += data [ z ] . c || 0 ;
328
+ result . totals . s += data [ z ] . s || 0 ;
329
+ result . totals . dur += data [ z ] . dur || 0 ;
330
+ }
256
331
}
257
- calllback ( err , apps ) ;
258
- } ) ;
259
-
260
-
332
+ }
333
+ else {
334
+ console . log ( "No drill data found for event:" , options . event ) ;
335
+ }
336
+ callback ( err , result ) ;
337
+ } ) ;
338
+ }
339
+ function getAppList ( options , callback ) {
340
+ var query = { } ;
341
+ if ( app_list && app_list . length > 0 ) {
342
+ var listed = [ ] ;
343
+ for ( var z = 0 ; z < app_list . length ; z ++ ) {
344
+ listed . push ( options . db . ObjectID ( app_list [ z ] ) ) ;
345
+ }
346
+ query = { _id : { $in : listed } } ;
261
347
}
262
- } ) ;
348
+ options . db . collection ( "apps" ) . find ( query , { "name" : 1 , "timezone" : 1 } ) . toArray ( function ( err , apps ) {
349
+ if ( err ) {
350
+ console . log ( "Error getting apps: " , err ) ;
351
+ }
352
+ callback ( err , apps ) ;
353
+ } ) ;
354
+ }
0 commit comments