Merge branch 'next' into SER-1991-ss-taskmanager-uses-comment-field-for-identifying-operation

cihadtekin · web-flow · commit b9d6a673eb06 · 2024-11-22T10:58:48.000+03:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -6,7 +6,7 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   pull_request:
-    branches: [ master, next ]
+    branches: [ master, next, release.24.10 ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,41 @@
+## Version 24.10.2
+Fixes:
+- [core] Correct aggregated collection cleanup on event omitting
+- [core] Fixed bug where changing passwords results in the loss of the "Global Admin" role
+- [core] Fixed bug where exporting incoming data logs could result in "Incorrect parameter \"data\" error
+- [core] Removed use of commands which needs admin rights from report manager.
+- [crash] Fixed bug in crash ingestion for scenarios where the "app version" is not a string.
+- [script] Fixing bug with "delete_old_members" script that led to malformed requests
+
+Enterprise fixes:
+- [nps] Fixed bug that showed the wrong nps preview title
+
 ## Version 24.10.1
-Integrating 24.05.16 fixes
+Fixes:
+- [core] Replaced "Users" with "Sessions" label on technology home widgets
+- [push] Improved ability to observe push related errors
+- [push] Replaced push plugin with an earlier version of the plugin
+
+Enterprise fixes:
+- [cohorts] Fixed issues with nightly cleanup
+- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
+- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
+- [flows] Fixed bug in case of null data in schema
+- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
+- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [nps] Fixed bug where it was possible to submit empty nps surveys
+- [ratings] Fixed bug with user consent
+- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
+
+Security:
+- Bumped cookie-parser from 1.4.6 to 1.4.7
+- Bumped express-rate-limit from 7.4.0 to 7.4.1
+- Bumped moment-timezone from 0.5.45 to 0.5.46
+- Bumped sass from 1.79.3 to 1.79.4
+- Fixing minor vulnerability that would allow for unauthorized file upload
+
+Enterprise Features:
+- [block] Added a way to filter crashes by their error (stacktrace)
 
 ## Version 24.10
 Fixes:
@@ -35,39 +71,9 @@ Enterprise Features:
 - [users] UI improvements
 - [views] Added a quick transition to drill
 
-## Version 24.05.17
-Fixes:
-- [push] Improved ability to observe push related errors
-
-Enterprise fixes:
-- [cohorts] Fixed issues with nightly cleanup
-- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
-- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
-- [flows] Fixed bug in case of null data in schema
-- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
-- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
-
-Security:
-- Fixing minor vulnerability that would allow for unauthorized file upload
-
-## Version 24.05.16
-Fixes:
-- [core] Replaced "Users" with "Sessions" label on technology home widgets
-- [push] Replaced push plugin with an earlier version of the plugin
-
-Enterprise fixes:
-- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
-- [nps] Fixed bug where it was possible to submit empty nps surveys
-- [ratings] Fixed bug with user consent
-
-Security:
-- Bumped cookie-parser from 1.4.6 to 1.4.7
-- Bumped express-rate-limit from 7.4.0 to 7.4.1
-- Bumped moment-timezone from 0.5.45 to 0.5.46
-- Bumped sass from 1.79.3 to 1.79.4
-
 ## Version 24.05.15
 Enterprise fixes:
+- [ab-testing] Fixed JSON.parse issue preventing creation of AB tests
 - [nps] Fixed UI issues in the widget editor related to the "user consent" section
 - [ratings] Fixed rendering issue for escaped values
 
diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js
@@ -1172,7 +1172,7 @@ const processRequest = (params) => {
                                         return new Promise(function(resolve) {
                                             var collectionNameWoPrefix = common.crypto.createHash('sha1').update(obj.key + params.qstring.app_id).digest('hex');
                                             //removes all document for current segment
-                                            common.db.collection("events" + collectionNameWoPrefix).remove({"s": {$in: obj.list}}, {multi: true}, function(err3) {
+                                            common.db.collection("events_data").remove({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: obj.list}}, {multi: true}, function(err3) {
                                                 if (err3) {
                                                     console.log(err3);
                                                 }
@@ -1187,7 +1187,7 @@ const processRequest = (params) => {
                                                         unsetUs["meta_v2." + obj.list[p]] = "";
                                                     }
                                                     //clears out meta data for segments
-                                                    common.db.collection("events" + collectionNameWoPrefix).update({$or: my_query}, {$unset: unsetUs}, {multi: true}, function(err4) {
+                                                    common.db.collection("events_data").update({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, $or: my_query}, {$unset: unsetUs}, {multi: true}, function(err4) {
                                                         if (err4) {
                                                             console.log(err4);
                                                         }
@@ -1231,7 +1231,6 @@ const processRequest = (params) => {
                                                         else {
                                                             resolve();
                                                         }
-
                                                     });
                                                 }
                                                 else {
diff --git a/api/utils/taskmanager.js b/api/utils/taskmanager.js
@@ -1080,4 +1080,4 @@ function getResult(callback, options) {
         }
     };
 }
-module.exports = taskmanager;
+module.exports = taskmanager;
diff --git a/bin/scripts/member-managament/delete_old_members.js b/bin/scripts/member-managament/delete_old_members.js
@@ -44,7 +44,7 @@ Promise.all([pluginManager.dbConnection("countly")]).spread(function(countlyDb)
                         Url: SERVER_URL + "/i/users/delete",
                         body: {
                             api_key: API_KEY,
-                            args: JSON.stringify({user_ids: [(data._id + "")]})
+                            args: {user_ids: [data._id + ""]}
                         }
                     }, function(data) {
                         if (data.err) {
@@ -99,8 +99,7 @@ function sendRequest(params, callback) {
         const options = {
             uri: url.href,
             method: params.requestType,
-            json: true,
-            body: body,
+            json: body,
             strictSSL: false
         };
 
diff --git a/frontend/express/public/core/user-management/javascripts/countly.views.js b/frontend/express/public/core/user-management/javascripts/countly.views.js
@@ -949,8 +949,20 @@
         watch: {
             'groups': function() {
                 if (this.groups.length > 0) {
-                    // Remove global admin role if user is assigned to any group
-                    this.$refs.userDrawer.editedObject.global_admin = false;
+                    // Remove global admin role if the assigned groups does not have global admin access
+                    var groupHasGlobalAdmin = false;
+
+                    this.groups.forEach(function(grpId) {
+                        var group = groupsModel.data().find(function(grp) {
+                            return grpId === grp._id;
+                        });
+
+                        if (group && group.global_admin === true) {
+                            groupHasGlobalAdmin = true;
+                        }
+                    });
+
+                    this.$refs.userDrawer.editedObject.global_admin = groupHasGlobalAdmin;
                 }
 
                 if (this.groups.length === 0) {
@@ -1152,4 +1164,4 @@
     countlyVue.container.registerData("user-management/edit-user-drawer", {
         component: Drawer
     });
-})();
+})();
diff --git a/plugins/crashes/api/api.js b/plugins/crashes/api/api.js
@@ -464,7 +464,12 @@ plugins.setConfigs("crashes", {
                         }
                         updateUser.hadAnyNonfatalCrash = report.ts;
                     }
-                    let updateData = { $inc: {} };
+
+                    if ('app_version' in report && typeof report.app_version !== 'string') {
+                        report.app_version += '';
+                    }
+                    let updateData = {$inc: {}};
+
                     updateData.$inc["data.crashes"] = 1;
                     if (Object.keys(updateUser).length) {
                         updateData.$set = updateUser;
diff --git a/plugins/crashes/tests.js b/plugins/crashes/tests.js
@@ -3144,6 +3144,35 @@ describe('Testing Crashes', function() {
         });
     });
 
+    describe('Crash app version', async() => {
+        it('should process crash app version as string', async() => {
+            const crashData = {
+                "_error": "error",
+                "_app_version": 123, // app version is number
+                "_os": "android",
+            };
+
+            await request.get('/i')
+                .query({ app_key: APP_KEY, device_id: DEVICE_ID, crash: JSON.stringify(crashData) })
+                .expect(200);
+
+            const crashGroupQuery = JSON.stringify({
+                latest_version: { $in: [`${crashData._app_version}`] },
+            });
+            let crashGroupResponse = await request
+                .get('/o')
+                .query({ method: 'crashes', api_key: API_KEY_ADMIN, app_id: APP_ID, query: crashGroupQuery });
+            const crashGroup = crashGroupResponse.body.aaData[0];
+            crashGroupResponse = await request
+                .get(`/o?`)
+                .query({ method: 'crashes', api_key: API_KEY_ADMIN, app_id: APP_ID, group: crashGroup._id });
+
+            const crash = crashGroupResponse.body.data[0];
+
+            crash.app_version.should.equal(`${crashData._app_version}`);
+        });
+    });
+
     describe('Reset app', function() {
         it('should reset data', function(done) {
             var params = {app_id: APP_ID, period: "reset"};
diff --git a/plugins/logger/frontend/public/javascripts/countly.views.js b/plugins/logger/frontend/public/javascripts/countly.views.js
@@ -136,6 +136,44 @@
             formatExportFunction: function() {
                 var tableData = this.logsData;
                 var table = [];
+                var sanitizeQueryData = function(data) {
+                    try {
+                        // If data is already a string, parse it first
+                        let queryObject = typeof data === 'string' ? JSON.parse(data) : data;
+
+                        // Handle nested JSON strings within the object
+                        Object.keys(queryObject).forEach(key => {
+                            if (typeof queryObject[key] === 'string') {
+                                // Try to parse if it looks like JSON
+                                if (queryObject[key].startsWith('{') || queryObject[key].startsWith('[')) {
+                                    try {
+                                        queryObject[key] = JSON.parse(queryObject[key]);
+                                        if (typeof queryObject[key] === 'object' && queryObject[key] !== null) {
+                                            queryObject[key] = sanitizeQueryData(queryObject[key]);
+                                        }
+                                    }
+                                    catch (e) {
+                                        // If parsing fails, keep decoded string
+                                    }
+                                }
+                                queryObject[key] = countlyCommon.unescapeHtml(queryObject[key]);
+                            }
+                            else if (typeof queryObject[key] === 'object' && queryObject[key] !== null) {
+                                // Recursively handle nested objects
+                                Object.keys(queryObject[key]).forEach(nestedKey => {
+                                    if (typeof queryObject[key][nestedKey] === 'string') {
+                                        queryObject[key][nestedKey] = countlyCommon.unescapeHtml(queryObject[key][nestedKey]);
+                                    }
+                                });
+                            }
+                        });
+                        return JSON.stringify(queryObject);
+                    }
+                    catch (err) {
+                        return data; // Return original data if processing fails
+                    }
+                };
+
                 for (var i = 0; i < tableData.length; i++) {
                     var item = {};
                     item[CV.i18n('logger.requests').toUpperCase()] = countlyCommon.formatTimeAgoText(tableData[i].reqts).text;
@@ -158,16 +196,15 @@
                     }
                     if (tableData[i].q) {
                         try {
-                            item[CV.i18n('logger.request-query').toUpperCase()] = JSON.stringify(tableData[i].q);
+                            item[CV.i18n('logger.request-query').toUpperCase()] = sanitizeQueryData(tableData[i].q);
                         }
                         catch (err) {
                             item[CV.i18n('logger.request-header').toUpperCase()] = "-";
                         }
                     }
                     if (tableData[i].h) {
                         try {
-                            var stringifiedHeader = JSON.stringify(tableData[i].h);
-                            item["REQUEST HEADER"] = stringifiedHeader.replace(/&quot;/g, '"');
+                            item["REQUEST HEADER"] = sanitizeQueryData(tableData[i].h);
                         }
                         catch (err) {
                             item["REQUEST HEADER"] = "-";
diff --git a/plugins/views/scripts/omitViewSegments.js b/plugins/views/scripts/omitViewSegments.js
@@ -110,7 +110,7 @@ function getAppList(options, callback) {
     if (app_list && app_list.length > 0) {
         var listed = [];
         for (var z = 0; z < app_list.length; z++) {
-            listed.push(options.db.ObjectId(app_list[z]));
+            listed.push(options.db.ObjectID(app_list[z]));
         }
         query = {_id: {$in: listed}};
     }
diff --git a/test/2.api/15.event.managment.js b/test/2.api/15.event.managment.js
@@ -402,7 +402,10 @@ describe('Testing event settings', function() {
 
         it('checking for segmentation in  collections(test3)', function(done) {
             var collectionNameWoPrefix = crypto.createHash('sha1').update("test3" + APP_ID).digest('hex');
-            testUtils.db.collection("events" + collectionNameWoPrefix).find({"s": {$in: ["my_segment"]}}).toArray(function(err, res) {
+            testUtils.db.collection("events_data").find({"_id": {"$regex": ("^" + APP_ID + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: ["my_segment"]}}).toArray(function(err, res) {
+                if (err) {
+                    console.log(err);
+                }
                 if (res.length == 0) {
                     done();
                 }
@@ -465,7 +468,7 @@ describe('Testing event settings', function() {
 
         it('checking for segmentation in  collections(test3)', function(done) {
             var collectionNameWoPrefix = crypto.createHash('sha1').update("test3" + APP_ID).digest('hex');
-            testUtils.db.collection("events" + collectionNameWoPrefix).find({"s": {$in: ["my_segment"]}}).toArray(function(err, res) {
+            testUtils.db.collection("events_data").find({"_id": {"$regex": ("^" + APP_ID + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: ["my_segment"]}}).toArray(function(err, res) {
                 if (res.length == 0) {
                     done();
                 }
@@ -516,7 +519,7 @@ describe('Testing event settings', function() {
 
         it('checking for segmentation in  collections(t1)', function(done) {
             var collectionNameWoPrefix = crypto.createHash('sha1').update("t1" + APP_ID).digest('hex');
-            testUtils.db.collection("events" + collectionNameWoPrefix).find({"s": {$in: ["s"]}}).toArray(function(err, res) {
+            testUtils.db.collection("events_data").find({"_id": {"$regex": ("^" + APP_ID + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: ["s"]}}).toArray(function(err, res) {
                 if (res.length == 0) {
                     done();
                 }
@@ -676,7 +679,7 @@ describe('Testing event settings', function() {
 
         it('checking for segmentation in  collections(t5)', function(done) {
             var collectionNameWoPrefix = crypto.createHash('sha1').update("t5" + APP_ID).digest('hex');
-            testUtils.db.collection("events" + collectionNameWoPrefix).find({"s": {$in: ["bad_segment"]}}).toArray(function(err, res) {
+            testUtils.db.collection("events_data").find({"_id": {"$regex": ("^" + APP_ID + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: ["bad_segment"]}}).toArray(function(err, res) {
                 if (res.length == 0) {
                     done();
                 }

Original file line number	Diff line number	Diff line change
`@@ -1172,7 +1172,7 @@ const processRequest = (params) => {`
`1172`	`1172`	`return new Promise(function(resolve) {`
`1173`	`1173`	`var collectionNameWoPrefix = common.crypto.createHash('sha1').update(obj.key + params.qstring.app_id).digest('hex');`
`1174`	`1174`	`//removes all document for current segment`
`1175`		`- common.db.collection("events" + collectionNameWoPrefix).remove({"s": {$in: obj.list}}, {multi: true}, function(err3) {`
	`1175`	`+ common.db.collection("events_data").remove({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, "s": {$in: obj.list}}, {multi: true}, function(err3) {`
`1176`	`1176`	`if (err3) {`
`1177`	`1177`	`console.log(err3);`
`1178`	`1178`	`}`
`@@ -1187,7 +1187,7 @@ const processRequest = (params) => {`
`1187`	`1187`	`unsetUs["meta_v2." + obj.list[p]] = "";`
`1188`	`1188`	`}`
`1189`	`1189`	`//clears out meta data for segments`
`1190`		`- common.db.collection("events" + collectionNameWoPrefix).update({$or: my_query}, {$unset: unsetUs}, {multi: true}, function(err4) {`
	`1190`	`+ common.db.collection("events_data").update({"_id": {"$regex": ("^" + params.qstring.app_id + "_" + collectionNameWoPrefix + "_.*")}, $or: my_query}, {$unset: unsetUs}, {multi: true}, function(err4) {`
`1191`	`1191`	`if (err4) {`
`1192`	`1192`	`console.log(err4);`
`1193`	`1193`	`}`
`@@ -1231,7 +1231,6 @@ const processRequest = (params) => {`
`1231`	`1231`	`else {`
`1232`	`1232`	`resolve();`
`1233`	`1233`	`}`
`1234`		`-`
`1235`	`1234`	`});`
`1236`	`1235`	`}`
`1237`	`1236`	`else {`
Original file line number	Diff line number	Diff line change
`@@ -1080,4 +1080,4 @@ function getResult(callback, options) {`
`1080`	`1080`	`}`
`1081`	`1081`	`};`
`1082`	`1082`	`}`
`1083`		`-module.exports = taskmanager;`
	`1083`	`+module.exports = taskmanager;`
Original file line number	Diff line number	Diff line change
`@@ -464,7 +464,12 @@ plugins.setConfigs("crashes", {`
`464`	`464`	`}`
`465`	`465`	`updateUser.hadAnyNonfatalCrash = report.ts;`
`466`	`466`	`}`
`467`		`- let updateData = { $inc: {} };`
	`467`	`+`
	`468`	`+ if ('app_version' in report && typeof report.app_version !== 'string') {`
	`469`	`+ report.app_version += '';`
	`470`	`+ }`
	`471`	`+ let updateData = {$inc: {}};`
	`472`	`+`
`468`	`473`	`updateData.$inc["data.crashes"] = 1;`
`469`	`474`	`if (Object.keys(updateUser).length) {`
`470`	`475`	`updateData.$set = updateUser;`
Original file line number	Diff line number	Diff line change
`@@ -110,7 +110,7 @@ function getAppList(options, callback) {`
`110`	`110`	`if (app_list && app_list.length > 0) {`
`111`	`111`	`var listed = [];`
`112`	`112`	`for (var z = 0; z < app_list.length; z++) {`
`113`		`- listed.push(options.db.ObjectId(app_list[z]));`
	`113`	`+ listed.push(options.db.ObjectID(app_list[z]));`
`114`	`114`	`}`
`115`	`115`	`query = {_id: {$in: listed}};`
`116`	`116`	`}`