Skip to content

Commit 1a9f67b

Browse files
ArtursKadikisArtursKadikis
authored
Merge branch 'next' into changelog_24.10.0
2 parents dc4d4b2 + 73c1257 commit 1a9f67b

File tree

16 files changed

+3128
-190
lines changed

16 files changed

+3128
-190
lines changed

‎CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ Enterprise Features:
3232
- [users] UI improvements
3333
- [views] Added a quick transition to drill
3434

35+
## Version 24.05.15
36+
Enterprise fixes:
37+
- [nps] Fixed UI issues in the widget editor related to the "user consent" section
38+
- [ratings] Fixed rendering issue for escaped values
39+
3540
## Version 24.05.14
3641
Fixes:
3742
- [code] Added better handling for countly servers while deployed using subdirectory

‎bin/scripts/data-reports/compare_drill_aggregated.js

+262
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
/*
2+
* Script compares aggregated data from events collection with drill data for given period.
3+
* Place script ins
4+
* {countly}/bin/scripts/data-reports/compare_drill_aggregated.js
5+
*
6+
* Usage:
7+
* node compare_drill_aggregated.js
8+
*/
9+
var period = "7days"; //Chose any of formats: "Xdays" ("7days","100days") or ["1-1-2024", "1-10-2024"],
10+
var app_list = []; //List with apps
11+
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
12+
var eventMap = {}; //If left empty will run for all alls/events.
13+
var verbose = false; //true to show more output
14+
15+
16+
const Promise = require("bluebird");
17+
const crypto = require("crypto");
18+
var pluginManager = require("../../../plugins/pluginManager");
19+
var fetch = require("../../../api/parts/data/fetch.js");
20+
var countlyCommon = require("../../../api/lib/countly.common.js");
21+
var common = require("../../../api/utils/common.js");
22+
var moment = require("moment");
23+
24+
var endReport = {};
25+
26+
console.log("Script compares numbers in aggregated data with drill data for given period.");
27+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
28+
common.db = countlyDb;
29+
console.log("Connected to databases...");
30+
getAppList({db: countlyDb}, function(err, apps) {
31+
if (!apps || !apps.length) {
32+
console.log("No apps found");
33+
return close();
34+
}
35+
else {
36+
console.log("Apps found:", apps.length);
37+
Promise.each(apps, function(app) {
38+
return new Promise(function(resolve, reject) {
39+
console.log("Processing app: ", app.name);
40+
countlyDb.collection("events").findOne({_id: app._id}, {'list': 1}, function(err, events) {
41+
if (err) {
42+
console.log("Error getting events: ", err);
43+
reject();
44+
}
45+
else {
46+
events = events || {};
47+
events.list = events.list || [];
48+
events.list = events.list.filter(function(ee) {
49+
if (ee.indexOf("[CLY]_") === 0) {
50+
return false;
51+
}
52+
else {
53+
return true;
54+
}
55+
});
56+
57+
if (eventMap && eventMap[app._id + ""]) {
58+
var listBF = events.list.length;
59+
events.list = events.list.filter(function(ee) {
60+
if (eventMap && eventMap[app._id + ""]) {
61+
return eventMap[app._id + ""].indexOf(ee) > -1;
62+
}
63+
else {
64+
return false;
65+
}
66+
});
67+
if (events.list.length != listBF) {
68+
console.log(" Filtered events based on eventMap: ", events.list.length, " from ", listBF);
69+
}
70+
71+
}
72+
if (events && events.list && events.list.length) {
73+
endReport[app._id] = {"name": app.name, "total": events.list.length, "bad": 0};
74+
Promise.each(events.list, function(event) {
75+
return new Promise(function(resolve2, reject2) {
76+
console.log(" Processing event: ", event);
77+
var params = {
78+
app_id: app._id + "",
79+
appTimezone: app.timezone,
80+
qstring: { period: period},
81+
time: common.initTimeObj(app.timezone, Date.now().valueOf())
82+
};
83+
84+
//fetch drill data
85+
var periodObject = countlyCommon.getPeriodObj({"appTimezone": app.timezone, "qstring": {"period": period}});
86+
getDataFromDrill({event: event, app_id: app._id + "", timezone: app.timezone, drillDb: drillDb, periodObj: periodObject}, function(err, drillData) {
87+
if (err) {
88+
console.log(" Error getting drill data: ", err);
89+
reject2();
90+
}
91+
else {
92+
if (verbose) {
93+
console.log(" Drill data loaded");
94+
console.log(JSON.stringify(drillData));
95+
}
96+
//fetch aggregated data
97+
var collectionName = "events" + crypto.createHash('sha1').update(event + app._id).digest('hex');
98+
99+
fetch.getTimeObjForEvents(collectionName, params, null, function(data) {
100+
var mergedData = {};
101+
var totals = {"c": 0, "s": 0, "dur": 0};
102+
for (var z0 = 0; z0 < periodObject.currentPeriodArr.length; z0++) {
103+
var date = periodObject.currentPeriodArr[z0].split(".");
104+
if (data && data[date[0]] && data[date[0]][date[1]] && data[date[0]][date[1]][date[2]]) {
105+
mergedData[periodObject.currentPeriodArr[z0]] = {};
106+
mergedData[periodObject.currentPeriodArr[z0]].c = data[date[0]][date[1]][date[2]].c || 0;
107+
mergedData[periodObject.currentPeriodArr[z0]].s = data[date[0]][date[1]][date[2]].s || 0;
108+
mergedData[periodObject.currentPeriodArr[z0]].dur = data[date[0]][date[1]][date[2]].dur || 0;
109+
totals.c += data[date[0]][date[1]][date[2]].c || 0;
110+
totals.s += data[date[0]][date[1]][date[2]].s || 0;
111+
totals.dur += data[date[0]][date[1]][date[2]].dur || 0;
112+
}
113+
}
114+
if (verbose) {
115+
console.log(" Aggregated data loaded");
116+
console.log(JSON.stringify(mergedData));
117+
}
118+
var report = {"totals": {}, "data": {}};
119+
var haveAnything = false;
120+
for (var key in totals) {
121+
if (totals[key] != drillData.totals[key]) {
122+
report.totals[key] = (totals[key] || 0) - (drillData.totals[key] || 0);
123+
haveAnything = true;
124+
}
125+
}
126+
for (var z = 0; z < periodObject.currentPeriodArr.length; z++) {
127+
if (drillData.data[periodObject.currentPeriodArr[z]]) {
128+
if (mergedData[periodObject.currentPeriodArr[z]]) {
129+
var diff = {};
130+
for (var key0 in mergedData[periodObject.currentPeriodArr[z]]) {
131+
diff[key0] = (mergedData[periodObject.currentPeriodArr[z]][key0] || 0) - (drillData.data[periodObject.currentPeriodArr[z]][key0] || 0);
132+
}
133+
if (diff.c || diff.s || diff.dur) {
134+
report.data[periodObject.currentPeriodArr[z]] = diff;
135+
haveAnything = true;
136+
}
137+
}
138+
else {
139+
report.data[periodObject.currentPeriodArr[z]] = {};
140+
report.data[periodObject.currentPeriodArr[z]].c = -1 * drillData.data[periodObject.currentPeriodArr[z]].c;
141+
report.data[periodObject.currentPeriodArr[z]].s = -1 * drillData.data[periodObject.currentPeriodArr[z]].s;
142+
report.data[periodObject.currentPeriodArr[z]].dur = -1 * drillData.data[periodObject.currentPeriodArr[z]].dur;
143+
haveAnything;
144+
}
145+
}
146+
else {
147+
if (mergedData[periodObject.currentPeriodArr[z]]) {
148+
report.data[periodObject.currentPeriodArr[z]] = mergedData[periodObject.currentPeriodArr[z]];
149+
haveAnything = true;
150+
}
151+
}
152+
}
153+
if (haveAnything) {
154+
console.log(" " + JSON.stringify(report));
155+
endReport[app._id]["bad"]++;
156+
endReport[app._id]["events"] = endReport[app._id]["events"] || {};
157+
endReport[app._id]["events"][event] = {"e": event, report: report};
158+
}
159+
resolve2();
160+
});
161+
}
162+
});
163+
});
164+
}).then(function() {
165+
console.log("Finished processing app: ", app.name);
166+
resolve();
167+
}).catch(function(eee) {
168+
console.log("Error processing app: ", app.name);
169+
console.log(eee);
170+
reject();
171+
});
172+
}
173+
else {
174+
resolve();
175+
}
176+
}
177+
178+
});
179+
});
180+
181+
}).then(function() {
182+
console.log("Finished");
183+
console.log(JSON.stringify(endReport));
184+
close();
185+
}).catch(function(eee) {
186+
console.log("Error while fetching data");
187+
console.log(eee);
188+
console.log("EXITING...");
189+
console.log(JSON.stringify(endReport));
190+
close();
191+
});
192+
}
193+
});
194+
195+
function getDataFromDrill(options, callback) {
196+
var tmpArr = options.periodObj.currentPeriodArr[0].split(".");
197+
var startDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2]))));
198+
if (options.timezone) {
199+
startDate.tz(options.timezone);
200+
}
201+
startDate = startDate.valueOf() - startDate.utcOffset() * 60000;
202+
203+
tmpArr = options.periodObj.currentPeriodArr[options.periodObj.currentPeriodArr.length - 1].split(".");
204+
var endDate = moment(new Date(Date.UTC(parseInt(tmpArr[0]), parseInt(tmpArr[1]) - 1, parseInt(tmpArr[2])))).add(1, 'days');
205+
if (options.timezone) {
206+
endDate.tz(options.timezone);
207+
}
208+
endDate = endDate.valueOf() - endDate.utcOffset() * 60000;
209+
210+
let collection = "drill_events" + crypto.createHash('sha1').update(options.event + options.app_id).digest('hex');
211+
var query = {"ts": {"$gte": startDate, "$lt": endDate}};
212+
var pipeline = [
213+
{"$match": query},
214+
];
215+
216+
pipeline.push({"$group": {"_id": "$d", "c": {"$sum": "$c"}, "s": {"$sum": "$s"}, "dur": {"$sum": "$dur"}}});
217+
options.drillDb.collection(collection).aggregate(pipeline, {"allowDiskUse": true}).toArray(function(err, data) {
218+
if (err) {
219+
console.log(err);
220+
}
221+
var result = {"data": {}, "totals": {"c": 0, "s": 0, "dur": 0}};
222+
if (data && data.length > 0) {
223+
for (var z = 0; z < data.length; z++) {
224+
var iid = data[z]._id.split(":").join(".");
225+
if (options.periodObj.currentPeriodArr.indexOf(iid) !== -1) {
226+
result.data[iid] = data[z];
227+
result.totals.c += data[z].c || 0;
228+
result.totals.s += data[z].s || 0;
229+
result.totals.dur += data[z].dur || 0;
230+
}
231+
}
232+
}
233+
callback(err, result);
234+
});
235+
236+
}
237+
238+
function close() {
239+
countlyDb.close();
240+
drillDb.close();
241+
console.log("Done.");
242+
}
243+
244+
function getAppList(options, calllback) {
245+
var query = {};
246+
if (app_list && app_list.length > 0) {
247+
var listed = [];
248+
for (var z = 0; z < app_list.length; z++) {
249+
listed.push(options.db.ObjectID(app_list[z]));
250+
}
251+
query = {_id: {$in: listed}};
252+
}
253+
options.db.collection("apps").find(query, {"name": 1, "timezone": 1}).toArray(function(err, apps) {
254+
if (err) {
255+
console.log("Error getting apps: ", err);
256+
}
257+
calllback(err, apps);
258+
});
259+
260+
261+
}
262+
});

‎bin/scripts/device_list/amazon.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"KFSNWI":"Fire Max 11 (2023, 13th Gen)",
44
"KFRAWI (2GB)":"Fire HD 8 (2022, 2GB)",
55
"KFRAPWI (3GB)":"Fire HD 8 (2022, 3GB)",
6+
"KFRAPWI (4GB)":"Fire HD 8 (2024, 4GB)",
67
"KFQUWI": "Fire 7 (2022)",
78
"KFTRWI (3GB)":"Fire HD 10 (2021, 3GB)",
89
"KFTRPWI (4GB)":"Fire HD 10 (2021, 4GB)",
@@ -52,8 +53,9 @@
5253
"AFTTIFF55":"Onida HD/FHD - Fire TV (2020)",
5354
"AFTSHN02":"TCL 32 FHD, 40 FHD Fire TV (2023)",
5455
"AFTMD002":"TCL Class S3 1080p LED Smart TV with Fire TV (2023)",
55-
"AFTMD001":"Fire TV - TCL Q6 4K QLED HDR (2023)",
56+
"AFTMA475B1":"TCL 4K Smart Fire TV (2024)",
5657
"AFTMD001":"Fire TV - TCL S4 Series 4K UHD HDR LED (2023)",
58+
"AFTDEC012E":"Fire TV - TCL S4/S5/Q5/Q6 Series 4K UHD HDR LED (2024)",
5759
"AFTKA002":"Fire TV 2-Series (2023)",
5860
"AFTKAUK002":"Fire TV 2-Series (2023)",
5961
"AFTHA004":"Toshiba 4K UHD - Fire TV (2022)",

‎bin/scripts/device_list/apple.json

+4
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@
5656
"iPhone15,5":"iPhone 15 Plus",
5757
"iPhone16,1":"iPhone 15 Pro",
5858
"iPhone16,2":"iPhone 15 Pro Max",
59+
"iPhone17,1":"iPhone 16 Pro",
60+
"iPhone17,2":"iPhone 16 Pro Max",
61+
"iPhone17,3":"iPhone 16",
62+
"iPhone17,4":"iPhone 16 Plus",
5963
"iPod1,1":"iPod touch 1G",
6064
"iPod2,1":"iPod touch 2G",
6165
"iPod3,1":"iPod touch 3G",

‎bin/scripts/device_list/generate.js

+16-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// run as
22
// node generate.js > countly.device.list.js
3+
// resulting file should be copied to "/frontend/express/public/javascripts/countly"
34

45
//https://www.theiphonewiki.com/wiki/Models
56
//https://gist.github.com/adamawolf/3048717
@@ -15,11 +16,17 @@ var amazon = require("./amazon.json");
1516
for (var i in amazon) {
1617
devices[i] = amazon[i];
1718
}
19+
20+
// Informative messages are writer to stderr so they don't interfere with the stdout piping to a file
21+
// When downloading the CSV file it will be UTF-16 LE. It needs to be transformed to UTF-8 (non BOM version)
22+
// converting with notepad++ or vscode might not work on a windows device. Try on a mac device
23+
process.stderr.write("Starting CSV parsing\n");
1824
var csv = require('csvtojson');
1925
csv()
2026
//from https://support.google.com/googleplay/answer/1727131?hl=en-GB
2127
.fromFile("./supported_devices.csv")
2228
.on('json', (jsonObj)=>{
29+
//process.stderr.write("Parsed data/json line: " + jsonObj);
2330
var d = jsonObj["Marketing Name"] + "";
2431
var i = jsonObj["Model"] + "";
2532
if (i != d && d.trim().length) {
@@ -33,6 +40,14 @@ csv()
3340
}
3441
}
3542
})
43+
// .on('data', (data)=>{
44+
// //process.stderr.write("Parsed data line: " + data);
45+
// })
46+
// .on('error', (err)=>{
47+
// process.stderr.write("Error while parsing: " + err);
48+
// })
3649
.on('done', ()=>{
50+
process.stderr.write("CSV parsing 'done' trigger\n");
3751
process.stdout.write("/**\n * Object with device models as keys and pretty/marketing device names as values\n * @name countlyDeviceList\n * @global\n * @namespace countlyDeviceList\n */\nvar countlyDeviceList = " + JSON.stringify(devices, null, 4) + ";\n/*global module*/\nif (typeof module !== 'undefined' && module.exports) {\n module.exports = countlyDeviceList;\n}");
38-
});
52+
});
53+
process.stderr.write("Ending CSV parsing\n");

0 commit comments

Comments
 (0)