Skip to content

Commit 38e3f98

Browse files
ar2rsawseenar2rsawseen
committed
Merge branch 'master' into flex
2 parents d3e8983 + d9a3210 commit 38e3f98

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+3326
-3061
lines changed

‎CHANGELOG.md

+23
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,26 @@
1+
## Version 24.05.13
2+
Fixes:
3+
- [alerts] Fixed bugs related to NPS alerts
4+
- [crash] Reworked symbol files upload to support larger symbol files
5+
- [push] Fixed bug that would surface when sending Array or Object related payload
6+
7+
Enterprise fixes:
8+
- [ab-testing] Slight improvements to the UI and UX
9+
- [remote-config] Slight improvements to the UI and UX
10+
11+
Enterprise Features:
12+
- [ab-testing] Improved UI for selecting AB test expiration
13+
14+
## Version 24.05.12
15+
Fixes:
16+
- [dashboards] Fixes for dashboards grid
17+
- [dasboards] UI fix for dashboard widget action menu
18+
- [push] Refactored fcm API related code
19+
- [reports] Use config for encryption key in reports
20+
21+
Enterprise fixes:
22+
- [retention] Fixes for byval retention query calculation
23+
124
## Version 24.05.11
225
Fixes:
326
- [cache] Use a cursor without timeout

‎api/api.js

+13-5
Original file line numberDiff line numberDiff line change
@@ -370,10 +370,18 @@ plugins.connectToAllDatabases().then(function() {
370370
}
371371

372372
const form = new formidable.IncomingForm(formidableOptions);
373-
req.body = '';
374-
req.on('data', (data) => {
375-
req.body += data;
376-
});
373+
if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) {
374+
req.body = [];
375+
req.on('data', (data) => {
376+
req.body.push(data);
377+
});
378+
}
379+
else {
380+
req.body = '';
381+
req.on('data', (data) => {
382+
req.body += data;
383+
});
384+
}
377385

378386
let multiFormData = false;
379387
// Check if we have 'multipart/form-data'
@@ -432,4 +440,4 @@ plugins.connectToAllDatabases().then(function() {
432440

433441
plugins.loadConfigs(common.db);
434442
}
435-
});
443+
});

‎api/config.sample.js

+1
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ var countlyConfig = {
9999
* @property {string} algorithm - name of the algorithm to use for encryption. The algorithm is dependent on OpenSSL, examples are 'aes192', etc. On recent OpenSSL releases, openssl list-cipher-algorithms will display the available cipher algorithms. Default value is aes-256-cbc
100100
* @property {string} input_encoding - how encryption input is encoded. Used as output for decrypting. Default utf-8.
101101
* @property {string} output_encoding - how encryption output is encoded. Used as input for decrypting. Default hex.
102+
* @property {string} reports_key - key used for encryption of reports links
102103
*/
103104
encryption: {},
104105

‎api/parts/data/cache.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -971,7 +971,7 @@ class StreamedCollection {
971971
let [col, last] = await createCollection(this.db, this.name, 1e7);
972972

973973
this.col = col;
974-
this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, noCursorTimeout: true, numberOfRetries: -1}).stream();
974+
this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, numberOfRetries: -1}).stream();
975975

976976
this.stream.on('data', doc => {
977977
if (this.inserts.indexOf(doc._id.toString()) !== -1) {

‎api/parts/data/fetch.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1589,7 +1589,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
15891589
options = {};
15901590
}
15911591

1592-
if (typeof options === "undefined") {
1592+
if (!options) {
15931593
options = {};
15941594
}
15951595

@@ -1679,7 +1679,7 @@ function fetchTimeObj(collection, params, isCustomEvent, options, callback) {
16791679

16801680
var zeroDocs = [zeroIdToFetch];
16811681
var monthDocs = [monthIdToFetch];
1682-
if (!(options && options.dontBreak)) {
1682+
if (!options.dontBreak) {
16831683
for (let i = 0; i < common.base64.length; i++) {
16841684
zeroDocs.push(zeroIdToFetch + "_" + common.base64[i]);
16851685
monthDocs.push(monthIdToFetch + "_" + common.base64[i]);

‎api/parts/mgmt/cms.js

+12-1
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,21 @@ function syncCMSDataToDB(params) {
157157
}
158158

159159
cmsApi.saveEntries = function(params) {
160+
161+
var entries = [];
162+
try {
163+
entries = JSON.parse(params.qstring.entries);
164+
}
165+
catch (ex) {
166+
log.e(params.qstring.entries);
167+
common.returnMessage(params, 400, 'Invalid entries parameter');
168+
return;
169+
}
170+
160171
transformAndStoreData(
161172
Object.assign({dataTransformed: true}, params),
162173
null,
163-
JSON.parse(params.qstring.entries),
174+
entries,
164175
function(err1) {
165176
if (err1) {
166177
log.e('An error occured while storing entries in DB: ' + err1);

‎api/utils/requestProcessor.js

+9
Original file line numberDiff line numberDiff line change
@@ -1824,6 +1824,9 @@ const processRequest = (params) => {
18241824
switch (paths[3]) {
18251825
case 'all':
18261826
validateRead(params, 'core', () => {
1827+
if (!params.qstring.query) {
1828+
params.qstring.query = {};
1829+
}
18271830
if (typeof params.qstring.query === "string") {
18281831
try {
18291832
params.qstring.query = JSON.parse(params.qstring.query);
@@ -1864,6 +1867,9 @@ const processRequest = (params) => {
18641867
break;
18651868
case 'count':
18661869
validateRead(params, 'core', () => {
1870+
if (!params.qstring.query) {
1871+
params.qstring.query = {};
1872+
}
18671873
if (typeof params.qstring.query === "string") {
18681874
try {
18691875
params.qstring.query = JSON.parse(params.qstring.query);
@@ -1896,6 +1902,9 @@ const processRequest = (params) => {
18961902
break;
18971903
case 'list':
18981904
validateRead(params, 'core', () => {
1905+
if (!params.qstring.query) {
1906+
params.qstring.query = {};
1907+
}
18991908
if (typeof params.qstring.query === "string") {
19001909
try {
19011910
params.qstring.query = JSON.parse(params.qstring.query);

‎bin/scripts/expire-data/countly_single_app_expireDataBatches.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ function processDrillCollection(collection, seconds, callback) {
209209

210210
function generateIterationList(z) {
211211
z = (start === 0 && z) ? z : start;
212-
if (timeSpan === 0 && start === 0) {
212+
if (timeSpan === 0 && z === 0) {
213213
listed.push({"collection": collection.collection, "db": collection.db, "start": 0, "end": end, "query": {"ts": {"$lt": end}}});
214214
}
215215
else if (timeSpan === 0) {

‎bin/scripts/fix-data/regenerate_aggregated_data.js

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
/*
2+
* Scrript triggers data regeneration for events and sessions.
3+
* It stores progress in county.data_regeneration_progress collection.
4+
* Collection can be dropped once regeneration is complete. Data in there ensures that if script dies unexpectedly, then upon staring again it will not regenerate collections again.
5+
* mongosh countly
6+
* db.data_regeneration_progress.drop();
7+
*
8+
* script path: {countly}/bin/scripts/fix-data/
9+
*
10+
* To run script:
11+
* node regenerate_aggregated_data.js
12+
*/
13+
//Adjust settings below before running script
14+
var regenerate_events = true; //if true will regenerate all custom events aggregated data
15+
var regenerate_sessions = false;
16+
var period = "732days"; //any valid period
17+
18+
//Each app IDis listed as string, for example var appList = ["6075f94b7e5e0d392902520c",6075f94b7e5e0d392902520d]
19+
var appList = [];//If left empty, will run for all apps.
20+
//For each app defined there only listed events will be regenerated. If left empty, all events will be regenerated.
21+
//Example var eventMap = {"6075f94b7e5e0d392902520c":["Logout","Login"],"6075f94b7e5e0d392902520d":["Logout","Login","Buy"]};
22+
var eventMap = {}; //If left empty will run for all alls/events.
23+
24+
25+
//End of adjustable settings
26+
27+
const common = require("../../../api/utils/common.js");
28+
const pluginManager = require('../../../plugins/pluginManager.js');
29+
const asyncjs = require('async');
30+
const drill = require('../../../plugins/drill/api/parts/data/drill.js');
31+
32+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
33+
console.log("Connected to databases...");
34+
common.db = countlyDb;
35+
common.drillDb = drillDb;
36+
//get all apps
37+
try {
38+
var query = {};
39+
if (appList && appList.length) {
40+
query._id = {$in: appList.map(app_id=>common.db.ObjectID(app_id))};
41+
}
42+
const apps = await countlyDb.collection("apps").find(query, {_id: 1, name: 1, timezone: 1}).toArray();
43+
if (!apps || !apps.length) {
44+
return close();
45+
}
46+
try {
47+
//for each app serially process users
48+
asyncjs.eachSeries(apps, async function(app) {
49+
console.log("Processing app: ", app.name);
50+
//get all drill collections for this app
51+
var skipped_ec = 0;
52+
if (regenerate_events) {
53+
var events = await countlyDb.collection("events").findOne({_id: app._id}, {'list': 1});
54+
if (events && events.list && events.list.length) {
55+
events.list = events.list.filter(function(ee) {
56+
if (ee.indexOf("[CLY]_") === 0) {
57+
return false;
58+
}
59+
else if (eventMap && eventMap[app._id + ""]) {
60+
return eventMap[app._id + ""].indexOf(ee) > -1;
61+
}
62+
else {
63+
return true;
64+
}
65+
});
66+
for (var z = 0; z < events.list.length; z++) {
67+
var qq = {_id: app._id + ""};
68+
qq[events.list[z]] = {$exists: true};
69+
var doc = await countlyDb.collection("data_regeneration_progress").findOne(qq);
70+
if (!doc) {
71+
var event = events.list[z];
72+
console.log(" Processing event: ", event);
73+
var params = {
74+
appTimezone: app.timezone,
75+
time: common.initTimeObj(app.timezone),
76+
qstring: {
77+
app_id: app._id + "",
78+
event: event,
79+
period: period
80+
}
81+
};
82+
try {
83+
await new Promise((resolve)=>{
84+
drill.calculateEvents(params, function() {
85+
resolve();
86+
});
87+
});
88+
var update = {};
89+
update[event] = Date.now();
90+
await countlyDb.collection("data_regeneration_progress").updateOne({_id: app._id + ""}, {"$set": update}, {"upsert": true});
91+
92+
}
93+
catch (err) {
94+
console.log(err);
95+
}
96+
}
97+
else {
98+
skipped_ec++;
99+
}
100+
}
101+
}
102+
else {
103+
console.log(" No events found for app: ", app.name);
104+
}
105+
if (skipped_ec) {
106+
console.log(" Skipped ", skipped_ec, " events as they are marked as recalculated");
107+
}
108+
}
109+
if (regenerate_sessions) {
110+
var doc2 = await countlyDb.collection("data_regeneration_progress").findOne({_id: app._id + "", "[CLY]_session": {$exists: true}});
111+
if (!doc2) {
112+
console.log(" Processing sessions");
113+
var params2 = {
114+
appTimezone: app.timezone,
115+
time: common.initTimeObj(app.timezone),
116+
qstring: {
117+
app_id: app._id + "",
118+
period: period
119+
}
120+
};
121+
try {
122+
await new Promise((resolve)=>{
123+
drill.calculateSessions(params2, function() {
124+
resolve();
125+
});
126+
});
127+
await countlyDb.collection("data_regeneration_progress").updateOne({_id: app._id + ""}, {"$set": { "[CLY]_session": Date.now()}}, {"upsert": true});
128+
}
129+
catch (err) {
130+
console.log(err);
131+
}
132+
}
133+
else {
134+
console.log(" Sessions already processed for app: ", app.name);
135+
}
136+
}
137+
138+
}, function(err) {
139+
return close(err);
140+
});
141+
}
142+
catch (err) {
143+
return close(err);
144+
}
145+
}
146+
catch (err) {
147+
return close(err);
148+
}
149+
150+
function close(err) {
151+
if (err) {
152+
console.log(err);
153+
console.log('EXITED WITH ERROR');
154+
}
155+
console.log("Closing connections...");
156+
countlyDb.close();
157+
drillDb.close();
158+
console.log("DONE");
159+
}
160+
161+
});

0 commit comments

Comments
 (0)