Merge branch 'master' of https://github.com/Countly/countly-server into master-bugfix

Author: Cookiezaurs

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
-      
+
       - name: Github Actions Azure connection fix
         run: |
           # Workaround for https://github.com/actions/runner-images/issues/675#issuecomment-1381389712
@@ -31,27 +31,27 @@ jobs:
       - name: Installing Countly
         shell: bash
         run: sudo bash ./bin/countly.install.sh
-        
+
       - name: NodeJS version
         shell: bash
         run: node --version
-        
+
       - name: NPM version
         shell: bash
         run: npm --version
-        
+
       - name: Mongo version
         shell: bash
         run: mongo --version
-        
+
       - name: ShellCheck
         shell: bash
         run: countly shellcheck
-        
+
       - name: ESLint
         shell: bash
         run: npx eslint .
-        
+
       - name: DistFiles
         shell: bash
         run: sudo countly task dist-all
@@ -63,19 +63,19 @@ jobs:
       - name: Output API Logs
         if: ${{ always() }}
         run: cat log/countly-api.log
-        
+
       - name: Output Dashboard Logs
         if: ${{ always() }}
         run: cat log/countly-dashboard.log
-        
+
       - name: Output MongoDB Logs
         if: ${{ always() }}
         run: sudo cat /var/log/mongodb/mongod.log
 
       - name: Output Nginx Logs
         if: ${{ always() }}
         run: sudo cat /var/log/nginx/error.log
-        
+
       - name: Send slack notification
         id: slack
         if: always()

.shellcheckrc

@@ -1,3 +1,62 @@
+## Version 22.09.16
+Fixes:
+- [dashboard] fixed incorrectly changing widgets with number visualisation
+- [core] Fix decoding of special characters in ui
+- [core] Fix for vulnerable password generation
+- [core] Sanitize file names for localisation and themes
+- [hooks] Fix calling of localhost
+- [data-manager] fixed bug in category change for events
+- [install] run wget without sudo during installation
+- [populator] Fix for empty users created for ab-testing
+- [settings] Fix for API settings missing from app level configuration
+
+Enterprise fixes:
+- [data-manager] Fix bug in changing visibility for event
+- [cohorts] Fixed element sizes of cohort steps, inside cohort creation form
+- [ab-testing] Change python3 to python3.8 for CentOS 8
+- [ab-testing] Set default timezone for models installation scripts
+- [drill] Send segmentation request as POST
+
+## Version 22.09.15
+Fixes:
+- [compliance-hub] use 'change' instead of 'after' for filter
+- [core] app user export to database (not using filesystem anymore) !!!changes export format!!!
+- [core] do not fetch masking config if masking is not enabled
+- [core] fixed parsing of special characters in event keys
+- [core] only use custom period when set explicitly in model file
+- [core] set activePeriod as current day in periodObject if single day selected
+- [dashboards] fixed bug with not fully loaded graphs for events and crashes for some periods
+- [data-manager] fixed localization for data masking toggle
+- [dbviewer] correct read access check fixed
+- [dbviewer] fixed server error on invalid queries
+- [events] fixed display bug in the all events view for events with ampersand in its name enterprise
+- [install] do not overwrite supervisord.conf in upgrades
+- [install] online and offline setups for CentOS/RHEL 7
+- [networking] support for ipv6
+- [period] end date was set as 00:00Am in custom period selections
+- [populator] added UI check for maximum time input that prevents non-number inputs
+- [populator] populating with template create SDK requests with template document properties
+- [push] fixed wrong error deserialization
+- [security] deepExtend manual object copy replaced with lodash merge
+- [security] jquery validation xss vulnerability fix
+- [UI] graph notes back link is fixed
+
+Enterprise fixes:
+- [ab-testing] Fixes for setup.
+- [active-directory] Remove tlsKey for active directory  client
+- [cohorts] Fixes for displaying special characters
+- [data-manager] Ability to mask device id
+- [data-manager] [users]  Fixes for & in events name
+- [drill] Added index on eventTimeline collection for field app to have faster deletion on app delete/clear.
+- [drill] Fixed bug in timeline on single event deletion.
+- [drill] Make sure only preset values are used in meta regeneration and no new values are added.
+- [drill] Meta cleanup endpoint and function in drill. Clears out wrongly saved infromation in meta about user properties.
+- [retention] Fixes for showing cohort names in retention view.
+- [retention] Retention label set according to selected result type.
+- [revenue] Null check for revenue widgets
+- [users]  Fixes for displaying special characters
+- [users] sidebar properties value change after page has loaded
+
 ## Version 22.09.14
 Fixes:
 - [core] Always use random initialization vector if not provided for encryption

CHANGELOG.md

@@ -1,4 +1,4 @@
-FROM phusion/baseimage:bionic-1.0.0
+FROM phusion/baseimage:focal-1.2.0
 
 ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,enterpriseinfo,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,assistant,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards
 # Enterprise Edition:

Dockerfile-core

@@ -3,7 +3,9 @@
 
 <p align="right">
 
-[![Build Status](https://api.travis-ci.org/Countly/countly-server.png?branch=master)](https://travis-ci.org/Countly/countly-server) [![Install Countly on DigitalOcean](https://count.ly/github/install-on-digital-ocean.svg)](http://do.count.ly)
+![CI](https://github.com/countly/countly-server/actions/workflows/main.yml/badge.svg)
+![CodeQL Analysis](https://github.com/countly/countly-server/actions/workflows/codeql-analysis.yml/badge.svg)
+
 
 </p>
 

README.md

@@ -699,6 +699,7 @@ countlyCommon.extractChartData = function(db, clearFunction, chartData, dataProp
 * @param {object} data - countly metric model data
 * @param {object} props - object where key is output property name and value could be string as key from data object or function to create new value based on existing ones
 * @param {function} clearObject - function to prefill all expected properties as u, t, n, etc with 0, so you would not have null in the result which won't work when drawing graphs
+* @param {object} periodObject - period object override
 * @returns {object} object with sparkleline data for each property
 * @example
 * var sparkLines = countlyCommon.getSparklineData(countlySession.getDb(), {
@@ -723,8 +724,8 @@ countlyCommon.extractChartData = function(db, clearFunction, chartData, dataProp
 *   "avg-events":"1.6222222222222222,1.5555555555555556,1.6,1.6363636363636365,1.6486486486486487,1,1,1,1,1,1.8333333333333333,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1.4137931034482758,1,1,1,1"
 * }
 */
-countlyCommon.getSparklineData = function(data, props, clearObject) {
-    var _periodObj = countlyCommon.periodObj;
+countlyCommon.getSparklineData = function(data, props, clearObject, periodObject) {
+    var _periodObj = periodObject || countlyCommon.periodObj;
     var sparkLines = {};
     for (let p in props) {
         sparkLines[p] = [];
@@ -1530,6 +1531,7 @@ countlyCommon.timeString = function(timespent) {
 * @param {array} unique - array of all properties that are unique from properties array. We need to apply estimation to them
 * @param {object} totalUserOverrideObj - using unique property as key and total_users estimation property as value for all unique metrics that we want to have total user estimation overridden
 * @param {object} prevTotalUserOverrideObj - using unique property as key and total_users estimation property as value for all unique metrics that we want to have total user estimation overridden for previous period
+* @param {object} periodObject period object override for calculation
 * @returns {object} dashboard data object
 * @example
 * countlyCommon.getDashboardData(countlySession.getDb(), ["t", "n", "u", "d", "e", "p", "m"], ["u", "p", "m"], {u:"users"});
@@ -1544,7 +1546,7 @@ countlyCommon.timeString = function(timespent) {
 *      "m":{"total":86,"prev-total":0,"change":"NA","trend":"u","isEstimate":true}
 * }
 */
-countlyCommon.getDashboardData = function(data, properties, unique, totalUserOverrideObj, prevTotalUserOverrideObj) {
+countlyCommon.getDashboardData = function(data, properties, unique, totalUserOverrideObj, prevTotalUserOverrideObj, periodObject) {
     /**
     * Clear object, bu nulling out predefined properties, that does not exist
     * @param {object} obj - object to clear
@@ -1568,7 +1570,7 @@ countlyCommon.getDashboardData = function(data, properties, unique, totalUserOve
         return obj;
     }
 
-    var _periodObj = countlyCommon.periodObj,
+    var _periodObj = periodObject || countlyCommon.periodObj,
         dataArr = {},
         tmp_x,
         tmp_y,

api/lib/countly.common.js

@@ -71,7 +71,7 @@ countlyModel.create = function(fetchValue) {
     };
     //Private Properties
     var _Db = {},
-        _period = "30days",
+        _period = null,
         _metas = {},
         _uniques = ["u"],
         _metrics = ["t", "u", "n"],
@@ -507,6 +507,10 @@ countlyModel.create = function(fetchValue) {
     * @returns {array} object to use when displaying number {value: 123, change: 12, sparkline: [1,2,3,4,5,6,7]}
     */
     countlyMetric.getNumber = function(metric, isSparklineNotRequired) {
+        var periodObject = null;
+        if (this.getPeriod()) { // only set custom period if it was explicitly set on the model object
+            periodObject = countlyCommon.getPeriodObj({qstring: {}}, this.getPeriod());
+        }
         metric = metric || _metrics[0];
         var metrics = [metric];
         //include other default metrics for data correction
@@ -517,7 +521,7 @@ countlyModel.create = function(fetchValue) {
         if (metric === "n") {
             metrics.push("u");
         }
-        var data = countlyCommon.getDashboardData(this.getDb(), metrics, _uniques, { u: this.getTotalUsersObj().users }, { u: this.getTotalUsersObj(true).users });
+        var data = countlyCommon.getDashboardData(this.getDb(), metrics, _uniques, { u: this.getTotalUsersObj().users }, { u: this.getTotalUsersObj(true).users }, periodObject);
         if (isSparklineNotRequired) {
             return data[metric];
         }
@@ -535,7 +539,7 @@ countlyModel.create = function(fetchValue) {
             }
 
             return obj;
-        });
+        }, periodObject);
         for (let i in data) {
             if (sparkLines[i]) {
                 data[i].sparkline = sparkLines[i].split(",").map(function(item) {
@@ -553,10 +557,13 @@ countlyModel.create = function(fetchValue) {
     */
     countlyMetric.getTimelineData = function() {
         var dataProps = [];
+        var periodObject = null;
         for (let i = 0; i < _metrics.length; i++) {
             dataProps.push({ name: _metrics[i] });
         }
-        var periodObject = countlyCommon.getPeriodObj({qstring: {}}, this.getPeriod());
+        if (this.getPeriod()) { // only set custom period if it was explicitly set on the model object
+            periodObject = countlyCommon.getPeriodObj({qstring: {}}, this.getPeriod());
+        }
         var data = countlyCommon.extractData(this.getDb(), this.clearObject, dataProps, periodObject);
         var ret = {};
         for (let i = 0; i < data.length; i++) {

api/lib/countly.model.js

@@ -392,11 +392,11 @@ usage.returnAllProcessedMetrics = function(params) {
             }
 
             // We check if country data logging is on and user's country is the configured country of the app
-            if (tmpMetric.name === "country" && (plugins.getConfig("api").country_data === false || params.app_cc !== params.user.country)) {
+            if (tmpMetric.name === "country" && (plugins.getConfig("api", params.app && params.app.plugins, true).country_data === false || params.app_cc !== params.user.country)) {
                 continue;
             }
             // We check if city data logging is on and user's country is the configured country of the app
-            if (tmpMetric.name === "city" && (plugins.getConfig("api").city_data === false || params.app_cc !== params.user.country)) {
+            if (tmpMetric.name === "city" && (plugins.getConfig("api", params.app && params.app.plugins, true).city_data === false || params.app_cc !== params.user.country)) {
                 continue;
             }
 

api/parts/data/usage.js

@@ -9,7 +9,7 @@ var usersApi = {},
     mail = require('./mail.js'),
     countlyConfig = require('./../../../frontend/express/config.js'),
     plugins = require('../../../plugins/pluginManager.js'),
-    { hasAdminAccess, getUserApps, getAdminApps } = require('./../../utils/rights.js');
+    { hasAdminAccess, getUserApps, getAdminApps, hasReadRight } = require('./../../utils/rights.js');
 
 const countlyCommon = require('../../lib/countly.common.js');
 const log = require('../../utils/log.js')('core:mgmt.users');
@@ -925,7 +925,7 @@ usersApi.fetchNotes = async function(params) {
             appIds = await usersApi.fetchUserAppIds(params);
         }
         filteredAppIds = appIds.filter((appId) => {
-            if (hasAdminAccess(params.member, appId)) {
+            if (hasAdminAccess(params.member, appId) || hasReadRight('core', appId, params.member)) {
                 return true;
             }
             return false;