Merge pull request #15 from Countly/next

Next

Author: frknbasaran

CHANGELOG.md

@@ -1,3 +1,58 @@
+## Version 19.08.1
+
+**Fixes**
+* [EChartMap] refactor library loading
+* [EchartMap] fixed date change refresh bug
+* [api] fixed i/tasks/name - calling correct function
+* [api] fixed total user correction for server side models
+* [api] increase user count in aggregated data for country when country changes
+* [crashes] fixed crash menu arrow
+* [crashes] update minidump
+* [events] fixed getting undefined _activeEvent in some situations
+* [events] fixed issue with hiding/showing event whan there is '.' in event name
+* [events] fixed issue with matching events in frontend if their keys have special symbols like "&" in them
+* [frontend] changed duration display format
+* [frontend] fixed check for admin apps (some sections that should be shown to admin, where not visible)
+* [frontend] fixed chrome autofill prevention
+* [frontend] fixed countries screen color problem
+* [frontend] fixed keeping filtered events after changing segments
+* [frontend] fixed localization for visits in top
+* [frontend] fixed total user correction in 0 cases
+* [frontend] fixed unknown country check
+* [frontend] fixed uploading app icon on first app screen
+* [frontend] fixes for 0 values having colors on the map
+* [frontend] improved Internet Explorer 11 support
+* [logger] let large texts scroll in table cell
+* [prelogin] fixed issue with showing messages in forgot page
+* [push] GCM Deprecation
+* [push] included in data migration
+* [push] show only for mobile type
+* [scripts] fixed db upgrade script running as separate script
+* [sdk] updated bundled Web SDK
+* [views] view deletion added in systemlogs
+
+**Enterprise fixes**
+* [attribution] prevent user from creating campaign ID with " or '
+* [cohorts] cohort drawer ui bugs fixed
+* [concurrent_users] fixed alerts table
+* [crash_symbols] order the symbols when fetching to always use last symbol if multiple same symbols provided
+* [crashes-jira] fixed check for correct crashes view
+* [dashboards] fixed user estimation correction in custom dashboards
+* [drill] fixed country map bugs
+* [drill] fixed punchcard value formatting
+* [funnels] fixed delete multiple rows bug
+* [funnels] fixed dragging steps only by drag handler
+* [restrict] css fix for hiding menus
+* [users] also validate funnel step segmentation
+* [users] fix custom column selection bugs
+
+**Enterprise Improvements**
+* [cohorts] added configuration to control minimal cohort regeneration time
+* [concurrent_users] added legacy live plugin endpoint support
+* [drill] return undefined values too in BY queries
+* [users] added cursor pointer on view message button
+* [users] updated to use long name and value transformations for custom selected fields
+
 ## Version 19.08
 
 **Fixes**

Gruntfile.js

@@ -33,6 +33,7 @@ module.exports = function(grunt) {
             },
             utils: {
                 src: [
+                    'frontend/express/public/javascripts/utils/polyfills.js',
                     'frontend/express/public/javascripts/utils/underscore-min.js',
                     'frontend/express/public/javascripts/utils/prefixfree.min.js',
                     'frontend/express/public/javascripts/utils/moment/moment-with-locales.min.js',

README.md

@@ -98,7 +98,7 @@ Countly is a privacy-focused and 360-degree analytics platform with several, uni
 * Configurable and extensible via open source [plugins](https://count.ly/plugins).
 * Modern and easy to use web based dashboard with a focus on user experience, which makes getting complex insights a breeze.
 * Tracking more than 2500 web sites and 16000 mobile applications.
-* Collecting more than 60 billion datapoints worlwide.
+* Collecting more than 60 billion datapoints worldwide.
 
 ## Differences between Community Edition & Enterprise Edition
 

api/jobs/topEvents.js

@@ -4,11 +4,12 @@ const crypto = require("crypto");
 const countlyApi = {
     data: {
         fetch: require("../parts/data/fetch.js"),
-        lib: require("../lib/countly.model.js")
+        lib: require("../lib/countly.model.js"),
+        common: require("../lib/countly.common.js")
     }
 };
 const countlyEvents = countlyApi.data.lib.load("event");
-
+const countlyCommon = countlyApi.data.common;
 /** Class for job of top events widget **/
 class TopEventsJob extends job.Job {
     /**
@@ -62,9 +63,14 @@ class TopEventsJob extends job.Job {
                                         doneEventMap();
                                     });
                                 }, () => {
+                                    const _data = {};
+                                    Object.keys(data).forEach(key => {
+                                        const encodeKey = countlyCommon.encode(key);
+                                        _data[encodeKey] = data[key];
+                                    });
                                     const ts = Math.round(new Date().getTime() / 1000);
                                     const period = givenPeriod === "hour" ? "today" : givenPeriod;
-                                    db.collection(collectionName).insert({app_id: _id, ts, period, data}, (error, records) => {
+                                    db.collection(collectionName).insert({app_id: _id, ts, period, data: _data}, (error, records) => {
                                         if (!error && records) {
                                             eachCallback();
                                         }

api/lib/countly.common.js

@@ -98,6 +98,17 @@ function getPeriodObject() {
     };
 
     endTimestamp = _currMoment.clone().utc().endOf("day");
+
+    if (_period && _period.indexOf(",") !== -1) {
+        try {
+            _period = JSON.parse(_period);
+        }
+        catch (SyntaxError) {
+            console.log("period JSON parse failed");
+            _period = "month";
+        }
+    }
+
     if (Array.isArray(_period)) {
         var fromDate, toDate;
 
@@ -192,6 +203,17 @@ function getPeriodObject() {
             isSpecialPeriod: true
         });
     }
+    //incorrect period, defaulting to 30 days
+    else {
+        let nDays = 30;
+
+        startTimestamp = _currMoment.clone().utc().startOf("day").subtract(nDays - 1, "days");
+        cycleDuration = moment.duration(nDays, "days");
+        Object.assign(periodObject, {
+            dateString: "D MMM",
+            isSpecialPeriod: true
+        });
+    }
 
     Object.assign(periodObject, {
         start: startTimestamp.valueOf(),
@@ -831,7 +853,7 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
             }
 
             if (propertyNames.indexOf("u") !== -1 && Object.keys(tmpPropertyObj).length) {
-                if (totalUserOverrideObj && totalUserOverrideObj[rangeArray[j]]) {
+                if (totalUserOverrideObj && typeof totalUserOverrideObj[rangeArray[j]] !== "undefined") {
 
                     tmpPropertyObj.u = totalUserOverrideObj[rangeArray[j]];
 
@@ -953,7 +975,14 @@ countlyCommon.extractBarData = function(db, rangeArray, clearFunction, fetchFunc
         dataProps.push({name: "n"});
         dataProps.push({name: "t"});
     }
+    if (metric === "n") {
+        dataProps.push({name: "u"});
+    }
     var rangeData = countlyCommon.extractTwoLevelData(db, rangeArray, clearFunction, dataProps, totalUserOverrideObj);
+    rangeData.chartData = countlyCommon.mergeMetricsByName(rangeData.chartData, "range");
+    rangeData.chartData = underscore.sortBy(rangeData.chartData, function(obj) {
+        return -obj[metric];
+    });
     var rangeNames = underscore.pluck(rangeData.chartData, 'range'),
         rangeTotal = underscore.pluck(rangeData.chartData, metric),
         barData = [],
@@ -1330,7 +1359,7 @@ countlyCommon.extractMetric = function(db, rangeArray, clearFunction, dataProper
             }
 
             if (propertyNames.indexOf("u") !== -1 && Object.keys(tmpPropertyObj).length) {
-                if (totalUserOverrideObj && totalUserOverrideObj[rangeArray[j]]) {
+                if (totalUserOverrideObj && typeof totalUserOverrideObj[rangeArray[j]] !== "undefined") {
 
                     tmpPropertyObj.u = totalUserOverrideObj[rangeArray[j]];
 
@@ -1607,7 +1636,7 @@ countlyCommon.getDashboardData = function(data, properties, unique, totalUserOve
 
     // Total users can't be less than new users
     if (typeof current.u !== "undefined" && typeof current.n !== "undefined" && current.u < current.n) {
-        if (totalUserOverrideObj && typeof totalUserOverrideObj.u !== "undefined" && totalUserOverrideObj.u) {
+        if (totalUserOverrideObj && typeof totalUserOverrideObj.u !== "undefined") {
             current.n = current.u;
         }
         else {
@@ -1636,7 +1665,7 @@ countlyCommon.getDashboardData = function(data, properties, unique, totalUserOve
     //check if we can correct data using total users correction
     if (totalUserOverrideObj) {
         for (let i = 0; i < unique.length; i++) {
-            if (dataArr[unique[i]] && typeof totalUserOverrideObj[unique[i]] !== "undefined" && totalUserOverrideObj[unique[i]]) {
+            if (dataArr[unique[i]] && typeof totalUserOverrideObj[unique[i]] !== "undefined") {
                 dataArr[unique[i]].is_estimate = false;
             }
         }

api/lib/countly.model.js

@@ -488,7 +488,16 @@ countlyModel.create = function(fetchValue) {
     */
     countlyMetric.getNumber = function(metric) {
         metric = metric || _metrics[0];
-        var data = countlyCommon.getDashboardData(this.getDb(), [metric], _uniques, {u: this.getTotalUsersObj().users}, {u: this.getTotalUsersObj(true).users});
+        var metrics = [metric];
+        //include other default metrics for data correction
+        if (metric === "u") {
+            metrics.push("n");
+            metrics.push("t");
+        }
+        if (metric === "n") {
+            metrics.push("u");
+        }
+        var data = countlyCommon.getDashboardData(this.getDb(), metrics, _uniques, {u: this.getTotalUsersObj().users}, {u: this.getTotalUsersObj(true).users});
         var ob = {};
         ob[metric] = metric;
         var sparkLines = countlyCommon.getSparklineData(this.getDb(), ob, function(obj) {
@@ -505,9 +514,11 @@ countlyModel.create = function(fetchValue) {
             return obj;
         });
         for (let i in data) {
-            data[i].sparkline = sparkLines[i].split(",").map(function(item) {
-                return parseInt(item);
-            });
+            if (sparkLines[i]) {
+                data[i].sparkline = sparkLines[i].split(",").map(function(item) {
+                    return parseInt(item);
+                });
+            }
         }
         return data[metric];
     };

api/parts/data/fetch.js

@@ -1108,8 +1108,9 @@ fetch.fetchDataTopEvents = function(params) {
             // eslint-disable-next-line no-shadow
             const { app_id, data, _id, ts, period } = result;
             let _data = Object.keys(data).map(function(key) {
+                const decodeKey = countlyCommon.decode(key);
                 const { sparkline, total, change } = data[key].data.count;
-                return { name: key, data: sparkline, count: total, trend: change };
+                return { name: decodeKey, data: sparkline, count: total, trend: change };
             });
             const sortByCount = _data.sort((a, b) => b.count - a.count).slice(0, limit);
             common.returnOutput(params, { _id, app_id, ts, period, data: sortByCount });
@@ -1277,7 +1278,9 @@ fetch.getTotalUsersObjWithOptions = function(metric, params, options, callback)
     var shortcodesForMetrics = {
         "devices": "d",
         "app_versions": "av",
+        "os": "p",
         "platforms": "p",
+        "os_versions": "pv",
         "platform_versions": "pv",
         "resolutions": "r",
         "countries": "cc",

api/parts/data/usage.js

@@ -789,6 +789,11 @@ function processUserSession(dbAppUser, params, done) {
             return true;
         }
 
+        if (dbAppUser.cc !== params.user.country) {
+            monthObjUpdate.push(params.user.country + '.' + common.dbMap.unique);
+            zeroObjUpdate.push(params.user.country + '.' + common.dbMap.unique);
+        }
+
         // Calculate the frequency range of the user
 
         if ((params.time.timestamp - userLastSeenTimestamp) >= (sessionFrequencyMax * 60 * 60)) {
@@ -842,17 +847,23 @@ function processUserSession(dbAppUser, params, done) {
         for (let k = 0; k < uniqueLevelsZero.length; k++) {
             if (uniqueLevelsZero[k] === "Y") {
                 updateUsersZero['d.' + common.dbMap.unique] = 1;
-                updateUsersZero['d.' + params.user.country + '.' + common.dbMap.unique] = 1;
+                if (dbAppUser.cc === params.user.country) {
+                    updateUsersZero['d.' + params.user.country + '.' + common.dbMap.unique] = 1;
+                }
             }
             else {
                 updateUsersZero['d.' + uniqueLevelsZero[k] + '.' + common.dbMap.unique] = 1;
-                updateUsersZero['d.' + uniqueLevelsZero[k] + '.' + params.user.country + '.' + common.dbMap.unique] = 1;
+                if (dbAppUser.cc === params.user.country) {
+                    updateUsersZero['d.' + uniqueLevelsZero[k] + '.' + params.user.country + '.' + common.dbMap.unique] = 1;
+                }
             }
         }
 
         for (let l = 0; l < uniqueLevelsMonth.length; l++) {
             updateUsersMonth['d.' + uniqueLevelsMonth[l] + '.' + common.dbMap.unique] = 1;
-            updateUsersMonth['d.' + uniqueLevelsMonth[l] + '.' + params.user.country + '.' + common.dbMap.unique] = 1;
+            if (dbAppUser.cc === params.user.country) {
+                updateUsersMonth['d.' + uniqueLevelsMonth[l] + '.' + params.user.country + '.' + common.dbMap.unique] = 1;
+            }
         }
 
         plugins.dispatch("/session/begin", {

api/parts/mgmt/users.js

@@ -918,4 +918,16 @@ usersApi.fetchNotes = async function(params) {
     return true;
 };
 
+usersApi.ackNotification = function(params) {
+    common.db.collection('members').updateOne({"_id": common.db.ObjectID(params.member._id)}, {$set: {['notes.' + params.qstring.path]: true}}, err => {
+        if (err) {
+            log.e('Error while acking member notification', err);
+            return common.returnMessage(params, 500, 'Unknown error');
+        }
+        else {
+            common.returnOutput(params, {['notes.' + params.qstring.path]: true});
+        }
+    });
+};
+
 module.exports = usersApi;

api/utils/requestProcessor.js

@@ -236,6 +236,9 @@ const processRequest = (params) => {
                 case 'deleteOwnAccount':
                     validateUserForWriteAPI(countlyApi.mgmt.users.deleteOwnAccount, params);
                     break;
+                case 'ack':
+                    validateUserForWriteAPI(countlyApi.mgmt.users.ackNotification, params);
+                    break;
                 default:
                     if (!plugins.dispatch(apiPath, {
                         params: params,
@@ -594,7 +597,7 @@ const processRequest = (params) => {
                     break;
                 case 'name':
                     validateUserForWrite(params, () => {
-                        taskmanager.deleteResult({
+                        taskmanager.nameResult({
                             db: common.db,
                             id: params.qstring.task_id,
                             name: params.qstring.name
@@ -925,12 +928,14 @@ const processRequest = (params) => {
                         for (let i = 0; i < idss.length; i++) {
 
                             if (idss[i].indexOf('.') !== -1) {
-                                updateThese.$unset["map." + idss[i].replace(/\./g, ':')] = 1;
-                                updateThese.$unset["segments." + idss[i].replace(/\./g, ':')] = 1;
+                                updateThese.$unset["map." + idss[i].replace(/\./g, '\\u002e')] = 1;
+                                updateThese.$unset["segments." + idss[i].replace(/\./g, '\\u002e')] = 1;
+                                updateThese.$unset["omitted_segments." + idss[i].replace(/\./g, '\\u002e')] = 1;
                             }
                             else {
                                 updateThese.$unset["map." + idss[i]] = 1;
                                 updateThese.$unset["segments." + idss[i]] = 1;
+                                updateThese.$unset["omitted_segments." + idss[i]] = 1;
                             }
                         }
 
@@ -1053,6 +1058,7 @@ const processRequest = (params) => {
 
                             for (let i = 0; i < idss.length; i++) {
 
+                                var baseID = idss[i].replace(/\\u002e/g, ".");
                                 if (!update_array.map[idss[i]]) {
                                     update_array.map[idss[i]] = {};
                                 }
@@ -1074,7 +1080,7 @@ const processRequest = (params) => {
 
                                 if (params.qstring.set_visibility === 'hide' && event && event.overview && Array.isArray(event.overview)) {
                                     for (let j = 0; j < event.overview.length; j++) {
-                                        if (event.overview[j].eventKey === idss[i]) {
+                                        if (event.overview[j].eventKey === baseID) {
                                             event.overview.splice(j, 1);
                                             j = j - 1;
                                         }

bin/scripts/checking_versions.js

@@ -75,7 +75,24 @@ function compareVersions(a, b) {
         }
     }
 
-    return Math.sign(a.length - b.length);
+    if (aParts.length === bParts.length) {
+        return 0;
+    }
+
+    let longestArray = aParts;
+    if (bParts.length > longestArray.length) {
+        longestArray = bParts;
+    }
+
+    const continueIndex = Math.min(aParts.length, bParts.length);
+
+    for (let i = continueIndex; i < longestArray.length; i += 1) {
+        if (parseInt(longestArray[i], 10) > 0) {
+            return longestArray === bParts ? -1 : +1;
+        }
+    }
+
+    return 0;
 }
 
 versions = versions.sort(compareVersions);