Merge branch 'release.24.10' into flex

Author: ar2rsawseen

.eslintrc.json

@@ -237,9 +237,6 @@
                 "for-direction": [
                     "off"
                 ],
-                "no-console": [
-                    "off"
-                ],
                 "no-control-regex": [
                     "off"
                 ],
@@ -343,4 +340,4 @@
             }
         }
     ]
-}
+}

.github/workflows/main.yml

@@ -10,6 +10,11 @@ on:
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
+    inputs:
+      custom_tag:
+        description: 'Custom Docker tag (optional)'
+        required: false
+        default: ''
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -64,7 +69,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:6.0
+        image: mongo:8.0
         options: >-
           --health-cmd mongosh
           --health-interval 10s
@@ -74,7 +79,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name  }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -121,7 +126,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:6.0
+        image: mongo:8.0
         options: >-
           --health-cmd mongosh
           --health-interval 10s
@@ -131,7 +136,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -155,6 +160,11 @@ jobs:
         shell: bash
         run: "sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Run tests
         shell: bash
         working-directory: /opt/countly
@@ -168,7 +178,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:6.0
+        image: mongo:8.0
         options: >-
           --health-cmd mongosh
           --health-interval 10s
@@ -178,7 +188,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -202,6 +212,11 @@ jobs:
         shell: bash
         run: "sed -i 's/port: 3001,/port: 3001, workers: 1,/' /opt/countly/api/config.js"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Install plugins
         shell: bash
         run: |
@@ -221,7 +236,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:6.0
+        image: mongo:8.0
         options: >-
           --health-cmd mongosh
           --health-interval 10s
@@ -231,7 +246,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -255,6 +270,11 @@ jobs:
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Prepare environment
         shell: bash
         working-directory: /opt/countly
@@ -281,7 +301,8 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          npm run cy:run:dashboard
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
+          npm run cy:run:dashboard --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}
@@ -298,7 +319,7 @@ jobs:
 
     services:
       mongodb:
-        image: mongo:6.0
+        image: mongo:8.0
         options: >-
           --health-cmd mongosh
           --health-interval 10s
@@ -308,7 +329,7 @@ jobs:
           - 27017:27017
 
     container:
-      image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
+      image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
       env:
         COUNTLY_CONFIG__MONGODB_HOST: mongodb
         COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -332,6 +353,11 @@ jobs:
         shell: bash
         run: "sed -i 's/mongosh --quiet/mongosh --host mongodb --quiet/' /opt/countly/bin/backup/import_events.sh && sed -i 's/mongoimport --db/mongoimport --host mongodb --db/' /opt/countly/bin/backup/import_events.sh"
 
+      - name: NPM install
+        shell: bash
+        working-directory: /opt/countly
+        run: npm install
+
       - name: Prepare environment
         shell: bash
         working-directory: /opt/countly
@@ -354,7 +380,8 @@ jobs:
           /sbin/my_init &
           cd ui-tests
           npm install
-          npm run cy:run:onboarding
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
+          npm run cy:run:onboarding --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
 
       - name: Upload UI tests artifacts
         if: ${{ failure() }}

CHANGELOG.md

@@ -1,3 +1,57 @@
+## Version 24.10
+Fixes:
+- [core] Interpreting carrier value of "--" as an unknown value
+- [crash] Fixed issues with visualisation of Flutter stack traces
+- [dashoards] If a user is given access to a dashboard, they will now be able to see all widgets even if they don't have access to the underlying data
+- [density] UI fixed
+- [events] Fixed issue where slower loading data would replace newest event selection
+
+Enterprise fixes:
+- [crashes-jira] Preventing crashing when jira authentification fails
+- [formulas] Removing HTML from tooltip localization
+
+Features:
+- Making user guide icons stand out more before first use
+- [alerts] UI improvements
+- [alerts] small improvements in email notifications
+- [code] Unifying bottom menu for bulk operations
+- [core] Aggregated events data collections merged together
+- [nps] UI improvements
+- [surveys] UI improvements
+- upgraded mongoDB to version 7
+- upgraded nodejs to version 20
+
+Enterprise Features:
+- [crash_symbolication] Added ability to set custom domain override for the symbolication server
+- [drill] Drill "by query" vizualization improved
+- [drill] Drill data collections merged together
+- [events] Added a quick transition to drill
+- [hooks] Added audit log entries for hook "create", "edit", "delete", "update" actions
+- [users] "Last Session" column in User Profiles is now sortable
+- [users] Crashes in the crashes tab can now be sorted
+- [users] UI improvements
+- [views] Added a quick transition to drill
+
+## Version 24.05.15
+Enterprise fixes:
+- [nps] Fixed UI issues in the widget editor related to the "user consent" section
+- [ratings] Fixed rendering issue for escaped values
+
+## Version 24.05.14
+Fixes:
+- [code] Added better handling for countly servers while deployed using subdirectory
+- [data-manager] Added missing "Create" button for event groups
+- [data-manager] Fixes for auto toggling data masking setting on redaction change
+- [populator] Fixed issues in case of wrongly structured data
+- [push] token shouldn't be revoked if it's already in db
+- [server-stats] Fixed a data point calculation bug
+
+Enterprise fixes:
+- [push_approver] Fixed issue with the plugin not handling being "soft disabled"
+- [surveys] fixed a bug with the Survey widget where the "submit" button could become non interactable
+- [surveys] fixed issues with dashboard editor in relation to the "terms and conditions"
+- [surveys] fixed issues with previewing, creating and editing nps widgets
+
 ## Version 24.05.13
 Fixes:
 - [alerts] Fixed bugs related to NPS alerts
@@ -14,7 +68,7 @@ Enterprise Features:
 ## Version 24.05.12
 Fixes:
 - [dashboards] Fixes for dashboards grid
-- [dasboards] UI fix for dashboard widget action menu
+- [dashboards] UI fix for dashboard widget action menu
 - [push] Refactored fcm API related code
 - [reports] Use config for encryption key in reports
 
@@ -32,7 +86,7 @@ Fixes:
 - [logs]  Show collected problems on logger
 
 Enterprise fixes:
-- [data-manager] Fixes n UI to allow events starting with "/"
+- [data-manager] Fixes in UI to allow events starting with "/"
 - [flows] Fixes for flows step generation
 - [surveys] Other is not allowed as a valid answer for required questions
 

Gruntfile.js

@@ -241,7 +241,7 @@ module.exports = function(grunt) {
                         'frontend/express/public/javascripts/dom/drop/drop-theme-countly.min.css',
                         'frontend/express/public/javascripts/utils/tooltipster/tooltipster.bundle.min.css',
                         'frontend/express/public/stylesheets/bulma/bulma-custom.css',
-                        'frontend/express/public/stylesheets/styles/manifest.css',
+                        'frontend/express/public/stylesheets/styles/manifest2.css',
                         'frontend/express/public/stylesheets/vue/element-tiptap.css',
                     ]
                 }
@@ -324,7 +324,25 @@ module.exports = function(grunt) {
 
     grunt.registerTask('default', ['mochaTest']);
 
-    grunt.registerTask('dist', ['sass', 'concat', 'uglify', 'cssmin']);
+    grunt.registerTask('replace-paths', 'Replace image paths in prod CSS files', function() {
+        var cssFiles = [
+            {
+                filepath: 'frontend/express/public/stylesheets/styles/manifest.css',
+                lookup: '../../images',
+                replacement: '../images',
+                newPath: 'frontend/express/public/stylesheets/styles/manifest2.css'
+            }
+        ];
+
+        cssFiles.forEach(function(file) {
+            var content = grunt.file.read(file.filepath);
+            var newContent = content.replace(/\.\.\/\.\.\/images/g, file.replacement);
+            grunt.file.write(file.newPath, newContent);
+            grunt.log.writeln('Processed file: ' + file.filepath);
+        });
+    });
+
+    grunt.registerTask('dist', ['sass', 'concat', 'uglify', 'replace-paths', 'cssmin']);
 
     grunt.registerTask('plugins', 'Minify plugin JS / CSS files and copy images', function() {
         var js = [], css = [], img = [], fs = require('fs'), path = require('path');

api/api.js

@@ -156,9 +156,9 @@ plugins.connectToAllDatabases().then(function() {
             console.log("Successfully stored batch state");
         }
         catch (ex) {
-            console.log("Could not store batch state");
+            console.log("Could not store batch state", ex);
         }
-        process.exit(code);
+        process.exit(typeof code === "number" ? code : 1);
     }
 
     /**

api/config.sample.js

@@ -107,7 +107,7 @@ var countlyConfig = {
     * Specifies where to store files. Value "fs" means file system or basically storing files on hard drive. Another currently supported option is "gridfs" storing files in MongoDB database using GridFS. By default fallback to "fs";
     * @type {string} [default=fs]
     */
-    fileStorage: "fs",
+    fileStorage: "gridfs",
     /**
     * Specifies after how long time configurations are reloded from data base. Default value is 10000 (10 seconds)
     * @type {integer} [default=10000]
@@ -121,7 +121,7 @@ var countlyConfig = {
     /** 
      * Share same database connection pool between databases
      */
-    shared_connection: false,
+    shared_connection: true,
     /**
      * Simple SMTP mail sender configuration.
      * Can only be used when you don't have custom mailer extend ({@code countly/extend/mail.js}).

api/lib/countly.common.js

@@ -1533,9 +1533,6 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
         periodObj = countlyCommon.periodObj;
     }
 
-    if (!rangeArray) {
-        return {"chartData": tableData};
-    }
     var periodMin = 0,
         periodMax = 0,
         dataObj = {},
@@ -1544,6 +1541,10 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
         propertyFunctions = underscore.pluck(dataProperties, "func"),
         propertyValue = 0;
 
+    if (!rangeArray) {
+        return {"chartData": tableData};
+    }
+
     if (!periodObj.isSpecialPeriod) {
         periodMin = periodObj.periodMin;
         periodMax = (periodObj.periodMax + 1);

api/parts/data/batcher.js

@@ -232,7 +232,7 @@ class WriteBatcher {
                         updateOne: {
                             filter: {_id: this.data[db][collection][key].id},
                             update: this.data[db][collection][key].value,
-                            upsert: true
+                            upsert: this.data[db][collection][key].upsert
                         }
                     });
                 }
@@ -340,18 +340,25 @@ class WriteBatcher {
      *  @param {string} id - id of the document
      *  @param {object} operation - operation
      *  @param {string} db - name of the database for which to write data
+     *  @param {object=} options - options for operation ((upsert: false) - if you don't want to upsert document)
      */
-    add(collection, id, operation, db = "countly") {
+    add(collection, id, operation, db = "countly", options) {
         if (!this.shared || cluster.isMaster) {
             if (!this.data[db][collection]) {
                 this.data[db][collection] = {};
             }
             if (!this.data[db][collection][id]) {
-                this.data[db][collection][id] = {id: id, value: operation};
+                this.data[db][collection][id] = {id: id, value: operation, upsert: true};
+                if (options && options.upsert === false) {
+                    this.data[db][collection][id].upsert = false;
+                }
                 batcherStats.update_queued++;
             }
             else {
                 this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
+                if (options && options.upsert === false) {
+                    this.data[db][collection][id].upsert = this.data[db][collection][id].upsert || false;
+                }
             }
             if (!this.process) {
                 this.flush(db, collection);