Merge branch 'next' into bugs/arturs-ui-fixes

# Conflicts:
#	plugins/plugins/frontend/public/javascripts/countly.views.js

Author: ar2rsawseen

.github/workflows/deploy.yml

@@ -0,0 +1,42 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Deploy
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master, next ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      
+      - name: Enable command line
+        shell: bash
+        run: sudo bash ./bin/scripts/detect.init.sh
+        
+      - name: Installing dependencies
+        shell: bash
+        run: npm install
+     
+      - name: Deploy server
+        shell: bash
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+        run: bash ./bin/scripts/deploy.sh
+
+      - name: Deploy documentation
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: bash ./bin/scripts/generateDocs.sh

.github/workflows/docker-image.yml

@@ -99,3 +99,34 @@ jobs:
           push: true
           file: ./Dockerfile-frontend
           tags: countly/frontend:latest,countly/frontend:${{ steps.vars.outputs.tag }}
+          
+  push_all:
+    name: Push Countly Single-All Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+        
+      - name: Set output
+        id: vars
+        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+      
+      - name: Check output
+        env:
+          RELEASE_VERSION: ${{ steps.vars.outputs.tag }}
+        run: |
+          echo $RELEASE_VERSION
+          echo ${{ steps.vars.outputs.tag }}
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          push: true
+          file: ./Dockerfile-core
+          tags: countly/countly-core:latest,countly/countly-core:${{ steps.vars.outputs.tag }}

CHANGELOG.md

@@ -1,3 +1,142 @@
+## Version 22.03.10
+
+Fixes:
+- [dashboards] The graph tooltip should overflow from the widget's bounding box
+- [push] Expired credentials case
+- [ui] Pressing enter refreshes the page
+
+Enterprise fixes:
+- [auth_plugins] Remove old UI artifacts
+- [drill] View regeneration from drill data
+- [surveys] Fixed tooltip localization
+
+## Version 22.03.9
+
+Fixes:
+- [batcher] more no fallback errors
+- [compliance-hub] Table column design fix
+- [push] Arrays for sent messages
+- [push] Method to get notifications sent to a particular user
+- [push] Missing audit logs calls
+- [push] Missing capping / sleeping support
+- [push] Parsing only string args for legacy api
+- [push] Set test pushes as prod in upgrade script
+- [ui] Chart label scaling issue fixed
+- [versions] Showing mongodb version in versions view
+- [views] Using isoWeek for unique views values
+
+Enterprise Fixes:
+- [ab-testing] fix of undefined array
+- [attribution] Added null check for unpopulated data case
+- [attribution] Fixed the issue where metadata was not displayed in segments
+- [users] Always translate surveys
+- [users] Fix for sorting session table
+- [users] Handled formating case for numbers
+- [users] Missing tooltip added
+- [users] Optimize calculating event count for session for single user in aggregation pipeline
+- [white-labeling] Default button color code changed
+
+## Version 22.03.8
+
+Fixes:
+- [dependency] update dependencies
+- [ui] fixed scaling issues and x-axis labels on charts
+- [ui] tab title should update when switching app types
+
+Enterprise fixes:
+- [crashes-jira] new UI integration
+- [funnels] save button's label changed for edit case.
+- [retention_segments] fixed retention filter query
+- [users] fixed export query
+- [users] fixed query bookmark
+
+## Version 22.03.7
+
+Fixes:
+- [compare] Adding tests
+- [members] Secure password reset token generation
+- [push] Allowing no platform results for migrated messages
+- [push] Migrating messages in batches
+- [remote-config] Fix for percentage bars
+- [upgrade] Remove even more files
+- [views] Fix view regeneration
+
+Enterprise fixes:
+- [flows] ui fix
+
+## Version 22.03.6
+
+Fixes:
+- [compare] loading state fix
+- [ui] X-axis overflow handling changes
+- [upgrade] add drill indexes
+- [upgrade] do not upgrade new users that have permission object
+
+Fixes Enterprise:
+- [attribution] added information toast when clicked show/hide button
+- [surveys] handling appearance null cases for sdk api
+- [users] add engagement score
+
+## Version 22.03.5
+
+Fixes:
+
+- [core] Fixing logging issues
+- [crashes] Fixed app_version filter
+- [dashboards] Add dashboard name to audit logs when widget deleted
+- [dashboards] Show that app is deleted in widgets if app was deleted
+- [docker] Fixed dependency for HTML PDF on centos images
+- [grunt] Fixed production mode
+- [push] Destroy connection on credentials validation failure
+- [push] Fixing change streams in sender until driver update
+- [push] Longer muticasts
+- [push] Self signed proxy certificate support
+- [star-rating] Added custom rating symbol support
+
+## Version 22.03.4
+
+Fixes:
+- [push] added missing files
+
+## Version 22.03.3
+
+Fixes:
+- [crashes] defaults for optional fields
+- [dashboards] custom period for events widget
+- [dbviewer] fixed exporting of other databases
+- [frontend] fixed sorting of analytics tables
+- [populator] add push events
+- [push] credential validation fixes
+- [push] logger fixes
+- [push] remove send job
+- [report-manager] fixed localization strings
+
+Enterprise Fixes:
+- [attribution] new ui transition
+- [drill] allow storing objects/arrays in event segments
+- [flows] correct view name mapping if start and end point selected for flow
+- [revenue] average paying customer data fixes and css improvement
+- [surveys] fix to show logo preview
+- [users] fixed decoding properties in user profiles
+
+## Version 22.03.2
+
+Fixes:
+- [crashes] fixed chart color
+- [crashes] fixed crashes stats
+- [populator] added control when input has a comma
+- [push] Revert "delete unsued preview push images" 
+- [push] fixes in upgrade script
+- [push] making sure audience pusher works if there's no token
+- [push] proxy support, better configs & lots of fixes
+
+Enterprise fixes:
+- [ab-testing] fixes for detail page to show variants
+- [revenue] session data included in the countly revenue
+- [surveys] changes to show device id if userName not present
+- [surveys] changes to show logo on edit
+- [surveys] fixed logo deletion
+
 ## Version 22.03.1
 
 Enterprise fixes
@@ -29,6 +168,11 @@ Fixes:
 - [scripts] fixed Compiler Version for 22.03 upgrade and installation
 - [upgrade] remove old files
 
+## Version 21.11.4
+
+Fixes:
+- [members] Secure password reset token generation
+
 ## Version 21.11.3
 
 Eneterprise Fixes:

Dockerfile-centos-api

@@ -29,7 +29,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
 	ln -s /usr/bin/node /usr/bin/nodejs && \
 	\
 	yum -y install centos-release-scl && \
-	yum -y install openssl-devel devtoolset-7-gcc-c++ make git wget unzip make binutils autoconf automake makedepend libtool pkgconfig zlib-devel libxml2-devel python-setuptools which && \
+	yum -y install openssl-devel devtoolset-7-gcc-c++ make git wget unzip bzip2 make binutils autoconf automake makedepend libtool pkgconfig zlib-devel libxml2-devel python-setuptools which && \
 	source /opt/rh/devtoolset-7/enable && \
     \
 	# modify standard distribution

Dockerfile-centos-frontend

@@ -28,7 +28,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
 	ln -s /usr/bin/node /usr/bin/nodejs && \
 	\
 	yum -y install centos-release-scl && \
-	yum -y install openssl-devel devtoolset-7-gcc-c++ make git wget unzip make binutils autoconf automake makedepend libtool pkgconfig zlib-devel libxml2-devel python-setuptools && \
+	yum -y install openssl-devel devtoolset-7-gcc-c++ make git wget unzip bzip2 make binutils autoconf automake makedepend libtool pkgconfig zlib-devel libxml2-devel python-setuptools && \
 	source /opt/rh/devtoolset-7/enable && \
 	\
 	# modify standard distribution

Dockerfile-core

@@ -72,7 +72,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
     chown -R countly:countly /opt/countly && \
     # cleanup
     npm remove -y --no-save mocha nyc should supertest && \
-    apt-get -y remove build-essential libkrb5-dev git sqlite3 wget \
+    apt-get -y remove build-essential libkrb5-dev sqlite3 wget \
         gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \
     apt-get -y install gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
     apt-get clean && \

Gruntfile.js

@@ -155,6 +155,8 @@ module.exports = function(grunt) {
                     'frontend/express/public/javascripts/countly/countly.version.history.js',
                     'frontend/express/public/javascripts/countly/countly.analytics.js',
                     'frontend/express/public/javascripts/countly/countly.views.js',
+                    'frontend/express/public/core/device-and-type/javascripts/countly.models.js',
+                    'frontend/express/public/core/device-and-type/javascripts/countly.views.js',
                     'frontend/express/public/core/user-activity/javascripts/countly.models.js',
                     'frontend/express/public/core/user-activity/javascripts/countly.views.js',
                     'frontend/express/public/core/session-overview/javascripts/countly.models.js',
@@ -168,21 +170,18 @@ module.exports = function(grunt) {
                     'frontend/express/public/core/report-manager/javascripts/countly.views.js',
                     'frontend/express/public/core/app-resolution/javascripts/countly.views.js',
                     'frontend/express/public/core/platform/javascripts/countly.views.js',
-                    'frontend/express/public/core/devices-and-types/javascripts/countly.models.js',
-                    'frontend/express/public/core/devices-and-types/javascripts/countly.views.js',
                     'frontend/express/public/core/carrier/javascripts/countly.models.js',
                     'frontend/express/public/core/carrier/javascripts/countly.views.js',
-                    'frontend/express/public/core/events/javascripts/countly.overview.models.js',
-                    'frontend/express/public/core/events/javascripts/countly.overview.views.js',
                     'frontend/express/public/core/events/javascripts/countly.details.models.js',
                     'frontend/express/public/core/events/javascripts/countly.details.views.js',
+                    'frontend/express/public/core/events/javascripts/countly.overview.models.js',
+                    'frontend/express/public/core/events/javascripts/countly.overview.views.js',
                     'frontend/express/public/core/user-management/javascripts/countly.models.js',
                     'frontend/express/public/core/user-management/javascripts/countly.views.js',
                     'frontend/express/public/core/app-management/javascripts/countly.models.js',
                     'frontend/express/public/core/app-management/javascripts/countly.views.js',
                     'frontend/express/public/core/logs/javascripts/countly.views.js',
                     'frontend/express/public/core/token-manager/javascripts/countly.views.js',
-                    'frontend/express/public/core/report-manager/javascripts/countly.views.js',
                     'frontend/express/public/core/geo-countries/javascripts/countly.cities.models.js',
                     'frontend/express/public/core/geo-countries/javascripts/countly.models.js',
                     'frontend/express/public/core/user-analytics-overview/javascripts/countly.views.js',
@@ -224,6 +223,7 @@ module.exports = function(grunt) {
                         'frontend/express/public/stylesheets/countly-checkbox/countly.checkbox.css',
                         'frontend/express/public/javascripts/dom/tipsy/tipsy.css',
                         'frontend/express/public/javascripts/dom/gridstack/gridstack.css',
+                        'frontend/express/public/javascripts/dom/gridstack/gridstack-extra.css',
                         'frontend/express/public/javascripts/visualization/rickshaw/rickshaw.min.css',
                         'frontend/express/public/javascripts/dom/pace/pace-theme-flash.css',
                         'frontend/express/public/javascripts/dom/drop/drop-theme-countly.min.css',

api/parts/data/batcher.js

@@ -51,7 +51,7 @@ class InsertBatcher {
      *  @param {string} collection - name of the collection for which to write data
      */
     async flush(db, collection) {
-        var no_fallback_errors = [10334, 17419];
+        var no_fallback_errors = [10334, 17419, 14, 56];
         if (this.data[db][collection].length) {
             var docs = this.data[db][collection];
             this.data[db][collection] = [];
@@ -188,7 +188,7 @@ class WriteBatcher {
      *  @param {string} collection - name of the collection for which to write data
      */
     async flush(db, collection) {
-        var no_fallback_errors = [10334, 17419];
+        var no_fallback_errors = [10334, 17419, 14, 56];
         if (Object.keys(this.data[db][collection]).length) {
             var queries = [];
             for (let key in this.data[db][collection]) {

api/parts/data/cache.js

@@ -54,7 +54,7 @@ class DataStore {
     constructor(size, age, dispose, Cls) {
         this.size = size;
         this.age = age;
-        this.lru = new LRU({max: size || Number.MAX_SAFE_INTEGER, maxAge: age || Number.MAX_SAFE_INTEGER, dispose: dispose, noDisposeOnSet: true, updateAgeOnGet: true});
+        this.lru = new LRU({max: size || 10000, ttl: age || Number.MAX_SAFE_INTEGER, dispose: dispose, noDisposeOnSet: true, updateAgeOnGet: true});
         if (Cls) {
             this.Cls = Cls;
             this.Clas = require('../../../' + Cls[0])[Cls[1]];
@@ -588,7 +588,7 @@ class CacheMaster {
         this.operators[group] = {init, Cls, read, write, update, remove};
 
         if (!size && size !== 0) {
-            size = config.api && config.api.cache && config.api.cache[group] && config.api.cache[group].size !== undefined ? config.api.cache[group].size : Number.MAX_SAFE_INTEGER;
+            size = config.api && config.api.cache && config.api.cache[group] && config.api.cache[group].size !== undefined ? config.api.cache[group].size : 10000;
         }
 
         if (!age && age !== 0) {

api/parts/data/fetch.js

@@ -1444,7 +1444,7 @@ fetch.getTotalUsersObjWithOptions = function(metric, params, options, callback)
             }
 
             if (groupBy === "users") {
-                options.db.collection("app_users" + params.app_id).find(match).count(function(error, appUsersDbResult) {
+                options.db.collection("app_users" + params.app_id).count(match, function(error, appUsersDbResult) {
                     if (!error && appUsersDbResult) {
                         callback([{ "_id": "users", "u": appUsersDbResult }]);
                     }
@@ -2045,8 +2045,8 @@ fetch.alljobs = async function(metric, params) {
 fetch.jobDetails = async function(metric, params) {
     const columns = ["schedule", "next", "finished", "status", "data", "duration"];
     let sort = {};
+    const total = await common.db.collection('jobs').count({ name: params.qstring.name });
     const cursor = common.db.collection('jobs').find({ name: params.qstring.name });
-    const total = await cursor.count();
     sort[columns[params.qstring.iSortCol_0 || 0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1;
     cursor.sort(sort);
     cursor.skip(Number(params.qstring.iDisplayStart || 0));

api/parts/mgmt/app_users.js

@@ -330,7 +330,7 @@ usersApi.count = function(app_id, query, callback) {
         query: query
     });
 
-    common.db.collection('app_users' + app_id).find(query).count(callback);
+    common.db.collection('app_users' + app_id).count(query, callback);
 };
 
 /**

api/parts/mgmt/users.js

@@ -666,7 +666,7 @@ usersApi.deleteOwnAccount = function(params) {
         verifyMemberArgon2Hash(params.member.email, params.qstring.password, (err, member) => {
             if (member) {
                 if (member.global_admin) {
-                    common.db.collection('members').find({'global_admin': true}).count(function(err2, count) {
+                    common.db.collection('members').count({'global_admin': true}, function(err2, count) {
                         if (err2) {
                             console.log(err2);
                             common.returnMessage(params, 400, 'Mongo error');
@@ -972,7 +972,7 @@ usersApi.fetchNotes = async function(params) {
         log.e(' got error while paring query notes request', e);
     }
     let count = 0;
-    common.db.collection('notes').find(query).count(function(error, noteCount) {
+    common.db.collection('notes').count(query, function(error, noteCount) {
         if (!error && noteCount) {
             count = noteCount;
             common.db.collection('notes').find(query)

api/utils/common.js

@@ -637,6 +637,7 @@ common.initTimeObj = function(appTimezone, reqTimestamp) {
         daily: tmpMoment.format("YYYY.M.D"),
         hourly: tmpMoment.format("YYYY.M.D.H"),
         weekly: Math.ceil(tmpMoment.format("DDD") / 7),
+        weeklyISO: tmpMoment.isoWeek(),
         month: tmpMoment.format("M"),
         day: tmpMoment.format("D"),
         hour: tmpMoment.format("H")