Merge branch 'master' into fix-install-scripts

Author: ar2rsawseen

CHANGELOG.md

@@ -1,3 +1,62 @@
+## Version 22.09.16
+Fixes:
+- [dashboard] fixed incorrectly changing widgets with number visualisation
+- [core] Fix decoding of special characters in ui
+- [core] Fix for vulnerable password generation
+- [core] Sanitize file names for localisation and themes
+- [hooks] Fix calling of localhost
+- [data-manager] fixed bug in category change for events
+- [install] run wget without sudo during installation
+- [populator] Fix for empty users created for ab-testing
+- [settings] Fix for API settings missing from app level configuration
+
+Enterprise fixes:
+- [data-manager] Fix bug in changing visibility for event
+- [cohorts] Fixed element sizes of cohort steps, inside cohort creation form
+- [ab-testing] Change python3 to python3.8 for CentOS 8
+- [ab-testing] Set default timezone for models installation scripts
+- [drill] Send segmentation request as POST
+
+## Version 22.09.15
+Fixes:
+- [compliance-hub] use 'change' instead of 'after' for filter
+- [core] app user export to database (not using filesystem anymore) !!!changes export format!!!
+- [core] do not fetch masking config if masking is not enabled
+- [core] fixed parsing of special characters in event keys
+- [core] only use custom period when set explicitly in model file
+- [core] set activePeriod as current day in periodObject if single day selected
+- [dashboards] fixed bug with not fully loaded graphs for events and crashes for some periods
+- [data-manager] fixed localization for data masking toggle
+- [dbviewer] correct read access check fixed
+- [dbviewer] fixed server error on invalid queries
+- [events] fixed display bug in the all events view for events with ampersand in its name enterprise
+- [install] do not overwrite supervisord.conf in upgrades
+- [install] online and offline setups for CentOS/RHEL 7
+- [networking] support for ipv6
+- [period] end date was set as 00:00Am in custom period selections
+- [populator] added UI check for maximum time input that prevents non-number inputs
+- [populator] populating with template create SDK requests with template document properties
+- [push] fixed wrong error deserialization
+- [security] deepExtend manual object copy replaced with lodash merge
+- [security] jquery validation xss vulnerability fix
+- [UI] graph notes back link is fixed
+
+Enterprise fixes:
+- [ab-testing] Fixes for setup.
+- [active-directory] Remove tlsKey for active directory  client
+- [cohorts] Fixes for displaying special characters
+- [data-manager] Ability to mask device id
+- [data-manager] [users]  Fixes for & in events name
+- [drill] Added index on eventTimeline collection for field app to have faster deletion on app delete/clear.
+- [drill] Fixed bug in timeline on single event deletion.
+- [drill] Make sure only preset values are used in meta regeneration and no new values are added.
+- [drill] Meta cleanup endpoint and function in drill. Clears out wrongly saved infromation in meta about user properties.
+- [retention] Fixes for showing cohort names in retention view.
+- [retention] Retention label set according to selected result type.
+- [revenue] Null check for revenue widgets
+- [users]  Fixes for displaying special characters
+- [users] sidebar properties value change after page has loaded
+
 ## Version 22.09.14
 Fixes:
 - [core] Always use random initialization vector if not provided for encryption

Dockerfile-core

@@ -1,4 +1,4 @@
-FROM phusion/baseimage:bionic-1.0.0
+FROM phusion/baseimage:focal-1.2.0
 
 ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,enterpriseinfo,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,assistant,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards
 # Enterprise Edition:

README.md

@@ -3,7 +3,9 @@
 
 <p align="right">
 
-[![Build Status](https://api.travis-ci.org/Countly/countly-server.png?branch=master)](https://travis-ci.org/Countly/countly-server) [![Install Countly on DigitalOcean](https://count.ly/github/install-on-digital-ocean.svg)](http://do.count.ly)
+![CI](https://github.com/countly/countly-server/actions/workflows/main.yml/badge.svg)
+![CodeQL Analysis](https://github.com/countly/countly-server/actions/workflows/codeql-analysis.yml/badge.svg)
+
 
 </p>
 

api/lib/countly.common.js

@@ -699,6 +699,7 @@ countlyCommon.extractChartData = function(db, clearFunction, chartData, dataProp
 * @param {object} data - countly metric model data
 * @param {object} props - object where key is output property name and value could be string as key from data object or function to create new value based on existing ones
 * @param {function} clearObject - function to prefill all expected properties as u, t, n, etc with 0, so you would not have null in the result which won't work when drawing graphs
+* @param {object} periodObject - period object override
 * @returns {object} object with sparkleline data for each property
 * @example
 * var sparkLines = countlyCommon.getSparklineData(countlySession.getDb(), {
@@ -723,8 +724,8 @@ countlyCommon.extractChartData = function(db, clearFunction, chartData, dataProp
 *   "avg-events":"1.6222222222222222,1.5555555555555556,1.6,1.6363636363636365,1.6486486486486487,1,1,1,1,1,1.8333333333333333,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1.4137931034482758,1,1,1,1"
 * }
 */
-countlyCommon.getSparklineData = function(data, props, clearObject) {
-    var _periodObj = countlyCommon.periodObj;
+countlyCommon.getSparklineData = function(data, props, clearObject, periodObject) {
+    var _periodObj = periodObject || countlyCommon.periodObj;
     var sparkLines = {};
     for (let p in props) {
         sparkLines[p] = [];
@@ -1530,6 +1531,7 @@ countlyCommon.timeString = function(timespent) {
 * @param {array} unique - array of all properties that are unique from properties array. We need to apply estimation to them
 * @param {object} totalUserOverrideObj - using unique property as key and total_users estimation property as value for all unique metrics that we want to have total user estimation overridden
 * @param {object} prevTotalUserOverrideObj - using unique property as key and total_users estimation property as value for all unique metrics that we want to have total user estimation overridden for previous period
+* @param {object} periodObject period object override for calculation
 * @returns {object} dashboard data object
 * @example
 * countlyCommon.getDashboardData(countlySession.getDb(), ["t", "n", "u", "d", "e", "p", "m"], ["u", "p", "m"], {u:"users"});
@@ -1544,7 +1546,7 @@ countlyCommon.timeString = function(timespent) {
 *      "m":{"total":86,"prev-total":0,"change":"NA","trend":"u","isEstimate":true}
 * }
 */
-countlyCommon.getDashboardData = function(data, properties, unique, totalUserOverrideObj, prevTotalUserOverrideObj) {
+countlyCommon.getDashboardData = function(data, properties, unique, totalUserOverrideObj, prevTotalUserOverrideObj, periodObject) {
     /**
     * Clear object, bu nulling out predefined properties, that does not exist
     * @param {object} obj - object to clear
@@ -1568,7 +1570,7 @@ countlyCommon.getDashboardData = function(data, properties, unique, totalUserOve
         return obj;
     }
 
-    var _periodObj = countlyCommon.periodObj,
+    var _periodObj = periodObject || countlyCommon.periodObj,
         dataArr = {},
         tmp_x,
         tmp_y,

api/lib/countly.model.js

@@ -71,7 +71,7 @@ countlyModel.create = function(fetchValue) {
     };
     //Private Properties
     var _Db = {},
-        _period = "30days",
+        _period = null,
         _metas = {},
         _uniques = ["u"],
         _metrics = ["t", "u", "n"],
@@ -507,6 +507,10 @@ countlyModel.create = function(fetchValue) {
     * @returns {array} object to use when displaying number {value: 123, change: 12, sparkline: [1,2,3,4,5,6,7]}
     */
     countlyMetric.getNumber = function(metric, isSparklineNotRequired) {
+        var periodObject = null;
+        if (this.getPeriod()) { // only set custom period if it was explicitly set on the model object
+            periodObject = countlyCommon.getPeriodObj({qstring: {}}, this.getPeriod());
+        }
         metric = metric || _metrics[0];
         var metrics = [metric];
         //include other default metrics for data correction
@@ -517,7 +521,7 @@ countlyModel.create = function(fetchValue) {
         if (metric === "n") {
             metrics.push("u");
         }
-        var data = countlyCommon.getDashboardData(this.getDb(), metrics, _uniques, { u: this.getTotalUsersObj().users }, { u: this.getTotalUsersObj(true).users });
+        var data = countlyCommon.getDashboardData(this.getDb(), metrics, _uniques, { u: this.getTotalUsersObj().users }, { u: this.getTotalUsersObj(true).users }, periodObject);
         if (isSparklineNotRequired) {
             return data[metric];
         }
@@ -535,7 +539,7 @@ countlyModel.create = function(fetchValue) {
             }
 
             return obj;
-        });
+        }, periodObject);
         for (let i in data) {
             if (sparkLines[i]) {
                 data[i].sparkline = sparkLines[i].split(",").map(function(item) {
@@ -553,10 +557,13 @@ countlyModel.create = function(fetchValue) {
     */
     countlyMetric.getTimelineData = function() {
         var dataProps = [];
+        var periodObject = null;
         for (let i = 0; i < _metrics.length; i++) {
             dataProps.push({ name: _metrics[i] });
         }
-        var periodObject = countlyCommon.getPeriodObj({qstring: {}}, this.getPeriod());
+        if (this.getPeriod()) { // only set custom period if it was explicitly set on the model object
+            periodObject = countlyCommon.getPeriodObj({qstring: {}}, this.getPeriod());
+        }
         var data = countlyCommon.extractData(this.getDb(), this.clearObject, dataProps, periodObject);
         var ret = {};
         for (let i = 0; i < data.length; i++) {

api/parts/data/usage.js

@@ -392,11 +392,11 @@ usage.returnAllProcessedMetrics = function(params) {
             }
 
             // We check if country data logging is on and user's country is the configured country of the app
-            if (tmpMetric.name === "country" && (plugins.getConfig("api").country_data === false || params.app_cc !== params.user.country)) {
+            if (tmpMetric.name === "country" && (plugins.getConfig("api", params.app && params.app.plugins, true).country_data === false || params.app_cc !== params.user.country)) {
                 continue;
             }
             // We check if city data logging is on and user's country is the configured country of the app
-            if (tmpMetric.name === "city" && (plugins.getConfig("api").city_data === false || params.app_cc !== params.user.country)) {
+            if (tmpMetric.name === "city" && (plugins.getConfig("api", params.app && params.app.plugins, true).city_data === false || params.app_cc !== params.user.country)) {
                 continue;
             }
 

api/utils/common.js

@@ -1322,6 +1322,32 @@ common.validateArgs = function(args, argProperties, returnErrors) {
                 }
             }
 
+            if (argProperties[arg].regex) {
+                try {
+                    var re = new RegExp(argProperties[arg].regex);
+                    if (!re.test(args[arg])) {
+                        if (returnErrors) {
+                            returnObj.errors.push(arg + " is not correct format");
+                            returnObj.result = false;
+                            argState = false;
+                        }
+                        else {
+                            return false;
+                        }
+                    }
+                }
+                catch (ex) {
+                    if (returnErrors) {
+                        returnObj.errors.push('Incorrect regex: ' + args[arg]);
+                        returnObj.result = false;
+                        argState = false;
+                    }
+                    else {
+                        return false;
+                    }
+                }
+            }
+
             if (argState && returnErrors && !argProperties[arg]['exclude-from-ret-obj']) {
                 returnObj.obj[arg] = parsed === undefined ? args[arg] : parsed;
             }
@@ -2584,6 +2610,44 @@ common.reviver = (key, value) => {
     }
 };
 
+/**
+ * Shuffle string using crypto.getRandomValues
+ * @param {string} text - text to be shuffled
+ * @returns {string} shuffled password
+ */
+common.shuffleString = function(text) {
+    var j, x, i;
+    for (i = text.length; i; i--) {
+        j = Math.floor(Math.random() * i);
+        x = text[i - 1];
+        text[i - 1] = text[j];
+        text[j] = x;
+    }
+
+    return text.join("");
+};
+
+/**
+ * Gets a random string from given character set string with given length
+ * @param {string} charSet - charSet string
+ * @param {number} length - length of the random string. default 1 
+ * @returns {string} random string from charset
+ */
+common.getRandomValue = function(charSet, length = 1) {
+    const randomValues = crypto.getRandomValues(new Uint8Array(charSet.length));
+    let randomValue = "";
+
+    if (length > charSet.length) {
+        length = charSet.length;
+    }
+
+    for (let i = 0; i < length; i++) {
+        randomValue += charSet[randomValues[i] % charSet.length];
+    }
+
+    return randomValue;
+};
+
 /**
     * Generate random password
     * @param {number} length - length of the password
@@ -2605,29 +2669,20 @@ common.generatePassword = function(length, no_special) {
     }
 
     //1 char
-    text.push(upchars.charAt(Math.floor(Math.random() * upchars.length)));
+    text.push(this.getRandomValue(upchars));
     //1 number
-    text.push(numbers.charAt(Math.floor(Math.random() * numbers.length)));
+    text.push(this.getRandomValue(numbers));
     //1 special char
     if (!no_special) {
-        text.push(specials.charAt(Math.floor(Math.random() * specials.length)));
+        text.push(this.getRandomValue(specials));
         length--;
     }
 
-    var j, x, i;
     //5 any chars
-    for (i = 0; i < Math.max(length - 2, 5); i++) {
-        text.push(all.charAt(Math.floor(Math.random() * all.length)));
-    }
+    text.push(this.getRandomValue(all, Math.max(length - 2, 5)));
 
     //randomize order
-    for (i = text.length; i; i--) {
-        j = Math.floor(Math.random() * i);
-        x = text[i - 1];
-        text[i - 1] = text[j];
-        text[j] = x;
-    }
-    return text.join("");
+    return this.shuffleString(text);
 };
 
 /**

api/utils/localization.js

@@ -8,7 +8,8 @@ var locale = {},
     fs = require('fs'),
     path = require('path'),
     parser = require('properties-parser'),
-    log = require('./log')('api:localization');
+    log = require('./log')('api:localization'),
+    common = require('./common');
 
 var dir = path.resolve(__dirname, '../../frontend/express/public/localization/min');
 var file = "locale";
@@ -72,7 +73,7 @@ locale.getProperty = function(lang, name, callback) {
     }
     else if (!localized[lang]) {
         localized[lang] = JSON.parse(JSON.stringify(orig));
-        fs.readFile(dir + '/' + file + '_' + lang + '.properties', 'utf8', function(err, local_properties) {
+        fs.readFile(dir + '/' + common.sanitizeFilename(file) + '_' + common.sanitizeFilename(lang) + '.properties', 'utf8', function(err, local_properties) {
             if (!err && local_properties) {
                 local_properties = parser.parse(local_properties);
                 for (var i in local_properties) {
@@ -103,7 +104,7 @@ locale.getProperties = function(lang, callback) {
     }
     else if (!localized[lang]) {
         localized[lang] = JSON.parse(JSON.stringify(orig));
-        fs.readFile(dir + '/' + file + '_' + lang + '.properties', 'utf8', function(err, local_properties) {
+        fs.readFile(dir + '/' + common.sanitizeFilename(file) + '_' + common.sanitizeFilename(lang) + '.properties', 'utf8', function(err, local_properties) {
             if (!err && local_properties) {
                 local_properties = parser.parse(local_properties);
                 for (var i in local_properties) {

bin/countly.install.sh

@@ -13,6 +13,7 @@ fi
 
 sudo bash "$DIR/scripts/init_countly_user.sh"
 cd "$DIR/../"
+
 sudo su countly -c "/bin/bash $DIR/scripts/check_countly_user_permissions.sh > /dev/null 2>&1"
 
 if [ ! -f ./permission_test_file.txt ]; then
@@ -23,7 +24,6 @@ else
     sudo rm -f ./permission_test_file.txt
 fi
 
-
 if [ "$totalm" -lt "1800" ]; then
     echo "Countly requires at least 2Gb of RAM"
     if [ "$COUNTLY_OVERWRITE_MEM_REQUIREMENT" != "1" ]; then

bin/docker/modify.sh

@@ -31,7 +31,7 @@ if [ "${COUNTLY_CONTAINER}" != "frontend" ]; then
 	# install monngodb tools to have mongoexport
 	if [ "${ID}" == "debian" ] || [ "${ID}" == "ubuntu" ]; then
 	    wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian10-x86_64-100.5.4.deb
-            apt install ./mongodb-database-tools-*-100.5.4.deb
+            apt install -y --allow-downgrades ./mongodb-database-tools-*-100.5.4.deb
 	    rm ./mongodb-database-tools-*-100.5.4.deb
 	else
             wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-rhel70-x86_64-100.5.4.rpm

frontend/express/app.js

@@ -354,7 +354,7 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_
         if (!loadedThemes[theme]) {
             var tempThemeFiles = {css: [], js: []};
             if (theme && theme.length) {
-                var themeDir = path.resolve(__dirname, "public/themes/" + theme + "/");
+                var themeDir = path.resolve(__dirname, "public/themes/" + common.sanitizeFilename(theme) + "/");
                 fs.readdir(themeDir, function(err, list) {
                     if (err) {
                         if (callback) {
@@ -978,14 +978,14 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_
             }
             plgns.forEach(plugin => {
                 try {
-                    let contents = fs.readdirSync(__dirname + `/../../plugins/${plugin}/frontend/public/javascripts`) || [];
+                    let contents = fs.readdirSync(__dirname + `/../../plugins/${common.sanitizeFilename(plugin)}/frontend/public/javascripts`) || [];
                     toDashboard.javascripts.push.apply(toDashboard.javascripts, contents.filter(n => typeof n === 'string' && n.includes('.js') && n.length > 3 && n.indexOf('.js') === n.length - 3).map(n => `${plugin}/javascripts/${n}`));
                 }
                 catch (e) {
                     console.log('Error while reading folder of plugin %s: %j', plugin, e.stack);
                 }
                 try {
-                    let contents = fs.readdirSync(__dirname + `/../../plugins/${plugin}/frontend/public/stylesheets`) || [];
+                    let contents = fs.readdirSync(__dirname + `/../../plugins/${common.sanitizeFilename(plugin)}/frontend/public/stylesheets`) || [];
                     toDashboard.stylesheets.push.apply(toDashboard.stylesheets, contents.filter(n => typeof n === 'string' && n.includes('.css') && n.length > 4 && n.indexOf('.css') === n.length - 4).map(n => `${plugin}/stylesheets/${n}`));
                 }
                 catch (e) {