Merge branch 'master' into SER-1984-ss-reworking-cache-js-not-to-use-tailable-cursor-and-capped-collection

Author: cihadtekin

.github/workflows/main.yml

@@ -10,6 +10,11 @@ on:
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
+    inputs:
+      custom_tag:
+        description: 'Custom Docker tag (optional)'
+        required: false
+        default: ''
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -74,7 +79,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name  }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -131,7 +136,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -178,7 +183,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -231,7 +236,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -281,7 +286,8 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          npm run cy:run:dashboard
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
+          npm run cy:run:dashboard --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}
@@ -308,7 +314,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -354,7 +360,8 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          npm run cy:run:onboarding
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
+          npm run cy:run:onboarding --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}

CHANGELOG.md

@@ -1,3 +1,41 @@
+## Version 24.05.XX
+Fixes:
+- [script] Fixing bug with "delete_old_members" script that led to malformed requests
+
+## Version 24.05.17
+Fixes:
+- [push] Improved ability to observe push related errors
+
+Enterprise fixes:
+- [cohorts] Fixed issues with nightly cleanup
+- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
+- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
+- [flows] Fixed bug in case of null data in schema
+- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
+
+Security:
+- Fixing minor vulnerability that would allow for unauthorized file upload
+
+Enterprise Features:
+- [block] Added a way to filter crashes by their error (stacktrace)
+
+## Version 24.05.16
+Fixes:
+- [core] Replaced "Users" with "Sessions" label on technology home widgets
+- [push] Replaced push plugin with an earlier version of the plugin
+
+Enterprise fixes:
+- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
+- [nps] Fixed bug where it was possible to submit empty nps surveys
+- [ratings] Fixed bug with user consent
+
+Security:
+- Bumped cookie-parser from 1.4.6 to 1.4.7
+- Bumped express-rate-limit from 7.4.0 to 7.4.1
+- Bumped moment-timezone from 0.5.45 to 0.5.46
+- Bumped sass from 1.79.3 to 1.79.4
+
 ## Version 24.05.15
 Enterprise fixes:
 - [nps] Fixed UI issues in the widget editor related to the "user consent" section

bin/scripts/expire-data/delete_custom_events_regex.js

@@ -6,7 +6,6 @@
  */
 
 
-const { ObjectId } = require('mongodb');
 const pluginManager = require('../../../plugins/pluginManager.js');
 const common = require('../../../api/utils/common.js');
 const drillCommon = require('../../../plugins/drill/api/common.js');
@@ -25,7 +24,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
 
     //GET APP
     try {
-        const app = await countlyDb.collection("apps").findOne({_id: ObjectId(APP_ID)}, {_id: 1, name: 1});
+        const app = await countlyDb.collection("apps").findOne({_id: countlyDb.ObjectID(APP_ID)}, {_id: 1, name: 1});
         console.log("App:", app.name);
         //GET EVENTS
         var events = [];
@@ -51,6 +50,27 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
                     }
                 ]).toArray();
                 events = events.length ? events[0].list : [];
+                const metaEvents = await drillDb.collection("drill_meta").aggregate([
+                    {
+                        $match: {
+                            'app_id': app._id + "",
+                            "type": "e",
+                            "e": { $regex: regex, $options: CASE_INSENSITIVE ? "i" : "", $nin: events }
+                        }
+                    },
+                    {
+                        $group: {
+                            _id: "$e"
+                        }
+                    },
+                    {
+                        $project: {
+                            _id: 0,
+                            e: "$_id"
+                        }
+                    }
+                ]).toArray();
+                events = events.concat(metaEvents.map(e => e.e));
             }
             catch (err) {
                 close("Invalid regex");
@@ -86,6 +106,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
         close(err);
     }
 
+
     async function deleteDrillEvents(appId, events) {
         for (let i = 0; i < events.length; i++) {
             var collectionName = drillCommon.getCollectionName(events[i], appId);

bin/scripts/member-managament/delete_old_members.js

@@ -44,7 +44,7 @@ Promise.all([pluginManager.dbConnection("countly")]).spread(function(countlyDb)
                         Url: SERVER_URL + "/i/users/delete",
                         body: {
                             api_key: API_KEY,
-                            args: JSON.stringify({user_ids: [(data._id + "")]})
+                            args: {user_ids: [data._id + ""]}
                         }
                     }, function(data) {
                         if (data.err) {
@@ -99,8 +99,7 @@ function sendRequest(params, callback) {
         const options = {
             uri: url.href,
             method: params.requestType,
-            json: true,
-            body: body,
+            json: body,
             strictSSL: false
         };
 

frontend/express/app.js

@@ -603,6 +603,10 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_
     app.use(function(req, res, next) {
         var contentType = req.headers['content-type'];
         if (req.method.toLowerCase() === 'post' && contentType && contentType.indexOf('multipart/form-data') >= 0) {
+            if (!req.session?.uid || Date.now() > req.session?.expires) {
+                res.status(401).send('Unauthorized');
+                return;
+            }
             var form = new formidable.IncomingForm();
             form.uploadDir = __dirname + '/uploads';
             form.parse(req, function(err, fields, files) {

frontend/express/public/core/device-and-type/templates/technologyHomeWidget.html

@@ -12,7 +12,7 @@
 								<div class="version-graph-block bu-p-3" :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}`">
 									<div class="bu-columns version-graph-title">
 										<div class="bu-column" :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-name`">{{item2.name}}</div>
-										<div class="bu-column" :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-value`">{{formatNumber(item2.value)}} {{item2.value > 1 ? 'Users' : 'User'}}<span class="divider" :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-divider`"> | </span><span :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-percent`">{{item2.percent}}% </span></div>
+										<div class="bu-column" :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-value`">{{formatNumber(item2.value)}} {{item2.value > 1 ? i18n('common.sessions') : i18n('common.session')}}<span class="divider" :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-divider`"> | </span><span :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-percent`">{{item2.percent}}% </span></div>
 									</div>
 									<cly-progress-bar :data-test-id="`cly-section-${item.title.toLowerCase().replaceAll(/\s/g, '-')}-item-${idx2}-progress-bar`" :entities="item2.bar" :height=8></cly-progress-bar>
 								</div>

frontend/express/public/localization/dashboard/dashboard.properties

@@ -255,6 +255,8 @@ common.selected-with-count ={0} Selected
 common.selected = Selected
 common.select-all-with-count = Select all {0}
 common.deselect = Deselect
+common.session = Session
+common.sessions = Sessions
 
 #vue
 common.undo = Undo

package-lock.json

@@ -50,7 +50,7 @@
     "ejs": "3.1.10",
     "errorhandler": "1.5.1",
     "express": "4.21.0",
-    "express-rate-limit": "7.4.0",
+    "express-rate-limit": "7.4.1",
     "express-session": "1.18.0",
     "form-data": "^4.0.0",
     "formidable": "2.1.1",

package.json

@@ -1131,11 +1131,12 @@
                 }
                 if (doc.actionType === 'EVENT_MERGE' && doc.isRegexMerge === true) {
                     doc.actionType = 'merge-regex';
+                    doc.eventTransformTargetRegex = doc.transformTarget[0];
                 }
                 else {
                     doc.actionType = doc.actionType.split('_')[1].toLowerCase();
                 }
-                doc.isExistingEvent = 'true';
+                doc.isExistingEvent = doc.isExistingEvent ? 'true' : 'false';
                 // doc.tab;
                 // delete doc.transformType;
                 doc.name = countlyCommon.unescapeHtml(doc.name);

plugins/data-manager/frontend/public/javascripts/countly.views.js

@@ -0,0 +1,12 @@
+# Countly Push Plugin
+
+These URLs need to be accessible by the Countly instance:
+
+- for Android:
+	- fcm.googleapis.com
+	- accounts.google.com
+- for IOS:
+	- api.development.push.apple.com
+	- api.push.apple.com
+- for Huawei
+	- push-api.cloud.huawei.com