Merge pull request #14 from Countly/master

master sync.

Author: frknbasaran

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = false
+
+[*.{js,html}]
+indent_style = space
+indent_size = 4

.eslintrc.json

@@ -1,6 +1,7 @@
 {
     "extends": "eslint:recommended",
     "rules": {
+        "require-atomic-updates": "off",
         "block-spacing": [
             "error",
             "always"
@@ -28,10 +29,7 @@
             "error",
             "all"
         ],
-        "eol-last": [
-            "error",
-            "never"
-        ],
+        "eol-last": "off",
         "func-call-spacing": [
             "error",
             "never"
@@ -126,8 +124,7 @@
         {
             "files": [
                 "frontend/express/public/javascripts/countly/*.js",
-                "plugins/*/frontend/public/javascripts/countly.models.js",
-                "plugins/*/frontend/public/javascripts/countly.views.js" 
+                "plugins/*/frontend/public/javascripts/countly.*.js"
             ],
             "env": {
                 "browser": true
@@ -204,7 +201,7 @@
                 "node": true
             },
             "parserOptions": {
-                "ecmaVersion": 2017
+                "ecmaVersion": 9
             },
             "rules": {
                 "no-console": "off",
@@ -281,7 +278,7 @@
                 "node": true
             },
             "parserOptions": {
-                "ecmaVersion": 2017
+                "ecmaVersion": 9
             },
             "rules": {
                 "no-console": "off"
@@ -299,7 +296,7 @@
                 "mocha": true
             },
             "parserOptions": {
-                "ecmaVersion": 2017
+                "ecmaVersion": 9
             },
             "rules": {
                 "no-console": "off",

.travis.yml

@@ -1,4 +1,5 @@
 language: nodejs
+dist: trusty
 node_js: "8"
 env:
   global:

CHANGELOG.md

@@ -1,3 +1,167 @@
+## Version 19.08
+
+**Fixes**
+* [api] fixed error on deleting user with exported data
+* [api] improved tops speed with aggregation pipeline
+* [app_versions] fixing displaying empty state
+* [assistant] handling case, when document could not be read
+* [browser] correctly convert metric to collection for correct data
+* [configs] display correct values in app configuration after save
+* [events] fixed event overview incorrectly formats duration
+* [frontend] added configuration element for google maps api key
+* [frontend] correct event total calculation for segmented view of aggregated data
+* [frontend] fixed Drop throws error when datatable is empty
+* [frontend] fixed japanese locale dates
+* [frontend] fixed refreshing total user correction for today
+* [frontend] reduced session extend calls.
+* [loyalty] fix table ordering
+* [push] Correct system log on push credentials update error
+* [push] Fixing populator creating messages for wrong app
+* [slipping-away] fix fetch user list bug
+* [source] fixed localization key in configs
+* [star-rating] lots of bug and ui fixes
+* [ui] Fixing expand row icon switches on refresh
+* [ui] changed email reports % colors.
+* [ui] disabling annoying chrome autocomplete
+* [ui] fix bug with Ubuntu font on Windows
+* [ui] prevent double scrolling
+* [ui] set app title attribute when switching apps
+* [ui] users and new users country sorting problem resolved
+* [views] fixes with tokens expiring when viewing heatmaps
+
+**Enterprise fixes**
+* [attribution] encode link if it is not a custom scheme
+* [attribution] fixed using last campaign urls when creating new campaign
+* [block] fixed creating empty users for new blocked users
+* [cohorts] fixed editing cohorts with big lists
+* [cohorts] handle empy user sets correctly and more efficiently
+* [dashboards] block all popups for dashboard image rendering in emails
+* [dashboards] fixed sending email reports if creator is deleted
+* [dashboards] improve screenshot rendering speed with cache for emails
+* [drill] actions button avaialble without query
+* [drill] better bucket and value selection for report manager data
+* [drill] big list values are not selected when replaying query bugfix
+* [drill] cohorts querying fixes
+* [drill] correct user count for multi BY queries
+* [drill] fixed generating month ticks for more than 1 year
+* [drill] query performance improved with new indexes
+* [flows] fix flows logic bug about app data clearing
+* [flows] fix views event logic bug
+* [funnels] allow to create funnels without events but with views
+* [funnels] fixed editing funnels with grouping elements
+* [funnels] top percentage update bug fixed
+* [groups] add uppercase to allowed characters for name and group id
+* [revenue] fix updating overview widget on period changes
+* [whitelabeling] replacement of word Countly with company name in all localized strings
+
+**New Features**
+* [api] allow updating multiple app users
+* [applications] application lock mechanism
+* [authorization] tokens now support url parameters as limits
+* [crashes] multi thread error support
+* [crashes] new crash stack processing for android and javascript
+* [crashes] new dropdown menu UI in crash groups
+* [dbviewer] added custom field support to dbviewer sort
+* [events] display top events for event overview
+* [frontend] option to delete your own account
+* [plugins] asynchronously check plugin enabling/disabling procedure
+* [push] Adding push events to user’s timeline
+* [push] allow building target audience just before scheduled date
+* [push] option to view recipients of a push
+* [push] proxy authentication support
+* [push] real-time event triggered push notifications
+* [report-manager] smarter more often regenerating automatic reports
+* [security] added password secret salt support in configuration file
+* [slipping-away-users] add flexible periods setting support
+* [two-factor-auth] enable two factor authentication through Microsoft or Google authenticator apps
+* [ui] added native tab behaivor to cly-select element
+* [ui] allow uploading dashboard profile pictures
+* [ui] customize graph colors through config file
+* [ui] moved some management tools to top menu
+* [ui] new graph note system
+* [ui] new menu design
+* [ui] new prelogin page design
+* [views] select columns to display functionality
+
+**New Enterprise Features**
+* [ab-testing] new plugin to perform AB testing
+* [attribution] allow providing custom domain for campaign links
+* [attribution] allow reattribution
+* [concurrent_users] new and more performant and detailed version of live plugin
+* [crash_symbolication] support for native crashes and symbolication
+* [crashes-jira] plugin to tie caught crashes with JIRA issues
+* [drill] make Drill configuration on app level
+* [formulas] new plugin to perform arithmetical computations on selected metrics
+* [funnels] allow sorting steps by drag and drop when editing funnels
+* [geo] Allowing float as geo radius
+* [users] added region support
+* [users] select columns to display functionality
+
+**Development related**
+* [api] moved account deletion to backend api.
+* [api] provide cancel request on app not exists
+* [api] refactored getPeriodObj
+* [cmd] added new upgrade subcommands for automatic multiple version upgrades
+* [cmd] script to upgrade countly to ee
+* [cmd] use password prompt to mask sensitive data
+* [common] add email for validation
+* [configs] improved config parsing and usage
+* [configs] more session and cookie settings in config file
+* [core] Increasing default heap size to 2Gb for API
+* [crashes] make crash identification model independent
+* [db] log incorrect Object ID to info level
+* [example] AWS SES - Simple Email Service example
+* [example] nginx example config of blocking access outside intranet
+* [frontend] redirect rather than render on POST processing
+* [frontend] replaced old google library loader with new version
+* [frontend] set cookie only when theme is customizable on user level
+* [frontend] trust estimation correction more than new users data
+* [members] moved all member functions to single separate members utility
+* [scripts] allow countly user to enable plugins
+* [scripts] correctly backup nginx config on ubuntu
+* [scripts] update_translation.js perfs and reliability boost
+* [security] HTML escaping in all localized strings
+* [security] remove flash cross domain policy
+* [taskmanager] no need to decode html (breaks json parsing)
+* [tests] additional way to parse CSRF from body in case of minified html
+* [ui] new menu management system
+* [ui] updated font awesome lib
+
+## Version 19.02.1
+
+**Fixes**
+* [alerts] fixed compare value bug
+* [assistant] fixed callback in case of failure to fetch rss feed
+* [crashes] additional checks for database failures
+* [data] fixed none tracking mode
+* [frontend] allow to use device_list on server side too
+* [frontend] fixed missing texts for formatSecond
+* [frontend] handle logout GET with redirect just in case
+* [push] fixed credentials setting validation
+* [star-rating] created separate filter popup for comments and ratings tab
+* [star-rating] trigger_button_text field problem solved
+
+**Enterprise fixes**
+* [attribution] do the regex check on click matching instead of direct match
+* [attribution] fixed encoding redirect url
+* [cohorts] fixed period check
+* [crash_symbolication] fixed api checks and texts
+* [drill] fixed BY query for array properties
+* [drill] fixed jumping order of BY properties
+* [remote-config] correctly check for parameter length
+* [remote-config] fixed for boolean values
+* [users] correctly check for number type when displaying user properties
+
+**New Features**
+* [frontend] add css class to #content based on route name
+* [frontend] add css class to body based on selected language
+* [populator] mark users generated with populator by custom property
+
+**Development related**
+* [ide] added a project editorconfig file
+* [log] do not log failed CSRF checks
+* [sdk] updated web and nodejs sdks
+
 ## Version 19.02
 
 **Fixes**

Dockerfile-core

@@ -1,5 +1,8 @@
 FROM phusion/baseimage:0.9.16
 
+ARG COUNTLY_CONFIG_API_MONGODB_HOST=localhost
+ARG COUNTLY_CONFIG_FRONTEND_MONGODB_HOST=localhost
+
 CMD ["/sbin/my_init"]
 
 ## Setup Countly

Gruntfile.js

@@ -33,6 +33,7 @@ module.exports = function(grunt) {
             },
             utils: {
                 src: [
+                    'frontend/express/public/javascripts/utils/polyfills.js',
                     'frontend/express/public/javascripts/utils/underscore-min.js',
                     'frontend/express/public/javascripts/utils/prefixfree.min.js',
                     'frontend/express/public/javascripts/utils/moment/moment-with-locales.min.js',
@@ -54,8 +55,16 @@ module.exports = function(grunt) {
                     'frontend/express/public/javascripts/utils/selectize.min.js',
                     'frontend/express/public/javascripts/utils/vue.min.js',
                     'frontend/express/public/javascripts/utils/jquery.xss.js',
-                    'frontend/express/public/javascripts/countly/countly.common.js'
-
+                    'frontend/express/public/javascripts/countly/countly.common.js',
+                    'frontend/express/public/javascripts/utils/simpleUpload.min.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/codemirror.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/javascript.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/json2.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/jsonlint.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/minify.json.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/jsoneditor.js',
+                    'frontend/express/public/javascripts/utils/Sortable.min.js',
+                    'frontend/express/public/javascripts/utils/vuedraggable.umd.min.js'
                 ],
                 dest: 'frontend/express/public/javascripts/min/countly.utils.concat.js'
             },
@@ -127,6 +136,7 @@ module.exports = function(grunt) {
                         'frontend/express/public/stylesheets/main.css',
                         'frontend/express/public/stylesheets/amaranjs/amaran.min.css',
                         'frontend/express/public/stylesheets/selectize/selectize.css',
+                        'frontend/express/public/stylesheets/jsoneditor/codemirror.css',
                         'frontend/express/public/javascripts/dom/tipsy/tipsy.css',
                         'frontend/express/public/javascripts/visualization/rickshaw/rickshaw.min.css',
                         'frontend/express/public/javascripts/dom/pace/pace-theme-flash.css',

LICENSE

@@ -1,7 +1,7 @@
 Countly Product Analytics - Community Edition License
 --------------------------------------------------
 
-Copyright (C) 2011-2018 Countly, https://count.ly
+© Countly, https://count.ly
 
 Countly is provided under AGPL v3 with modified Section 7. In accordance 
 with Section 7 of the AGPL, the Works included in this package or repository 

README.md

@@ -3,7 +3,7 @@
 
 <p align="right">
 
-[![Build Status](https://api.travis-ci.org/Countly/countly-server.png?branch=master)](https://travis-ci.org/Countly/countly-server) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/98c2726f2d734697a5f1ac0d453f0a06)](https://app.codacy.com/app/ar2rsawseen/countly-server?utm_source=github.com&utm_medium=referral&utm_content=Countly/countly-server&utm_campaign=Badge_Grade_Dashboard) [![Install Countly on DigitalOcean](https://do.count.ly/button.svg?v2)](http://do.count.ly)
+[![Build Status](https://api.travis-ci.org/Countly/countly-server.png?branch=master)](https://travis-ci.org/Countly/countly-server) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/98c2726f2d734697a5f1ac0d453f0a06)](https://app.codacy.com/app/ar2rsawseen/countly-server?utm_source=github.com&utm_medium=referral&utm_content=Countly/countly-server&utm_campaign=Badge_Grade_Dashboard) [![Install Countly on DigitalOcean](https://count.ly/github/install-on-digital-ocean.svg)](http://do.count.ly)
 
 </p>
 
@@ -118,7 +118,7 @@ There are several ways to install Countly:
 
 1. The following command will download and install Countly on your **Ubuntu** or **CentOS** server. 
 
-    `wget -qO- http://c.ly/install | bash`
+    `wget -qO- https://c.ly/install | bash`
 
 2. If you have a Digital Ocean account, [click here and install on Digital Ocean](http://do.count.ly) with a single click.
 

api/api.js

@@ -10,10 +10,17 @@ const log = require('./utils/log.js')('core:api');
 const common = require('./utils/common.js');
 const {processRequest} = require('./utils/requestProcessor');
 const versionInfo = require('../frontend/express/version.info');
+const frontendConfig = require('../frontend/express/config.js');
+const {CacheMaster, CacheWorker} = require('./parts/data/cache.js');
 
 var t = ["countly:", "api"];
 
 if (cluster.isMaster) {
+    console.log("Starting master");
+    if (!common.checkDatabaseConfigMatch(countlyConfig.mongodb, frontendConfig.mongodb)) {
+        log.w('API AND FRONTEND DATABASE CONFIGS ARE DIFFERENT');
+    }
+    common.db = plugins.dbConnection();
     t.push("master");
     t.push("node");
     t.push(process.argv[1]);
@@ -201,8 +208,11 @@ const passToMaster = (worker) => {
 };
 
 if (cluster.isMaster) {
-    console.log("Starting master");
-    common.db = plugins.dbConnection();
+    common.cache = new CacheMaster(common.db);
+    common.cache.start().then(plugins.dispatch.bind(plugins, '/cache/init', {}), e => {
+        console.log(e);
+        process.exit(1);
+    });
 
     const workerCount = (countlyConfig.api.workers)
         ? countlyConfig.api.workers
@@ -228,17 +238,22 @@ if (cluster.isMaster) {
 
     // Allow configs to load & scanner to find all jobs classes
     setTimeout(() => {
+        jobs.job('api:topEvents').replace().schedule('every 1 day');
         jobs.job('api:ping').replace().schedule('every 1 day');
         jobs.job('api:clear').replace().schedule('every 1 day');
         jobs.job('api:clearTokens').replace().schedule('every 1 day');
-        jobs.job('api:task').replace().schedule('every 59 mins starting on the 59 min');
+        jobs.job('api:task').replace().schedule('every 1 hour on the first min');
         jobs.job('api:userMerge').replace().schedule('every 1 hour on the 10th min');
     }, 10000);
 }
 else {
     console.log("Starting worker", process.pid, "parent:", process.ppid);
     const taskManager = require('./utils/taskmanager.js');
     common.db = plugins.dbConnection(countlyConfig);
+
+    common.cache = new CacheWorker(common.db);
+    common.cache.start();
+
     //since process restarted mark running tasks as errored
     taskManager.errorResults({db: common.db});
 
@@ -288,7 +303,7 @@ else {
             const headers = {};
             headers["Access-Control-Allow-Origin"] = "*";
             headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS";
-            headers["Access-Control-Allow-Headers"] = "countly-token";
+            headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type";
             res.writeHead(200, headers);
             res.end();
         }

api/jobs/ping.js

@@ -59,6 +59,7 @@ class PingJob extends job.Job {
                                 timestamp: Math.floor(date.getTime() / 1000),
                                 hour: date.getHours(),
                                 dow: date.getDay(),
+                                no_meta: true,
                                 events: JSON.stringify([
                                     {
                                         key: "PING",