Merge branch 'next' into ar2rsawseen/feature/ingestion

# Conflicts:
#	api/utils/requestProcessor.js

Author: ar2rsawseen

.eslintrc.json

@@ -251,6 +251,28 @@
                 ]
             }
         },
+        {
+            "files": [ "plugins/content/api/positioning/**/*.js" ],
+            "parserOptions": {
+                "ecmaVersion": 2023,
+                "sourceType": "module"
+            },
+            "env": {
+                "node": true,
+                "es2023": true
+            }
+        },
+        {
+            "files": [ "plugins/content/api/positioning/**/*.cjs" ],
+            "parserOptions": {
+                "ecmaVersion": 2023,
+                "sourceType": "commonjs"
+            },
+            "env": {
+                "node": true,
+                "es2023": true
+            }
+        },
         {
             "files": [
                 "api/**/*.js",

.github/workflows/deploy.yml

@@ -6,7 +6,7 @@ name: Deploy
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   push:
-    branches: [ master, next ]
+    branches: [ master, next, release.24.10, release.24.12 ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

.github/workflows/main.yml

@@ -6,7 +6,7 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   pull_request:
-    branches: [ master, next, release.24.10 ]
+    branches: [ master, next, release.*, flex ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

.github/workflows/stable-je-deploy.yml

@@ -0,0 +1,28 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Deploy Journey Engine
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ next ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+      
+      - name: Deploy server
+        shell: bash
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.STABLE_JE_SSH_PRIVATE_KEY }}
+        run: bash ./bin/scripts/deploy-je.sh

CHANGELOG.md

@@ -1,38 +1,147 @@
+## Version 25.x.x
+Features:
+- [alerts] alerts table default order should be by creation time newest at the top
+- [core] allow tracking Countly dashboard usage with Countly
+- [sdk] Improved and added new Server Config options
+
+Enterprise Features:
+- [journey_engine] Editing/Deleting/Duplication of blocks and version management
+
 ## Version 24.12
 Features:
 - [audit-logs] Exported audit logs from UI now would have "BEFORE" and "AFTER" fields
 - [core] Ability to mark reports as 'dirty' to make sure they are regenerated in full
 - [core] Adding a cancel button to "create new app" form
 - [core] Adding a nightly job to delete old data
+- [core] Fixed a bug causing events to not being loaded when there's an escaped character in the event name
 - [core] Redirecting user to a newly created app
 - [core] Removing HTML from localization files
 - [core] Showing a flex banner on sidebar if the version is Countly Lite
 - [crashes] Adding confirmation for deleting crash groups
 - [dashoards] Fixed the "Add/manage notes" button that did not work for the technology widget
 - [dbviewer] Preventing aggregation of using any stages which might open user to harmful actions (like $merge, $out, $lookup, $uninonWith) for all users except global admin
+- [gridfs] fixes for moving to Promises
 - [nps] Fixing issues with default logo selection
 - [populator] Adding ability to select features to populate and other small improvements
+- [push] Fixed bug where IOS credentials get mixed up while sending messages from different apps at the same time
+- [push] Fixed bug where it crashes in connection pool growth because of a type mismatch in an if condition
+- [reports] Fixes report generation failure due to SSL error
 - [star-rating] Removed unnecessary limitation with using cohorts for targeting
-- [surveys] Removed unnecessary limitation with using cohorts for targeting
+- [system-utility] Fixed: Mongo error (code: 26) in some Countly instances when the profiler gets run for the first time
+- [user-management] Global admins can now disable 2FA for individual users
 
 Enterprise Features:
 - [cohorts] Adding ability to edit cohorts. This deletes historical calculations
 - [content] Adding "Content Builder" feature
 - [core] Adding support For SingleStore Kai
 - [flows] Adding UX improvements to the editor
 - [journey_engine] Adding "Journey Engine" feature
+- [ldap] Fixed issues that would lead to configuration options not being picked up
 - [remote-config] Moving enable/disable functionality to the dropdown
+- [surveys] "Select one" text in the widget can be edited now
+- [surveys] Removed unnecessary limitation with using cohorts for targeting
 
 Dependencies:
-- Bump countly-sdk-nodejs from 22.6.0 to 24.10.0
-- Bump countly-sdk-web from 24.4.1 to 24.11.0
+- Bump countly-sdk-nodejs from 24.10.0 to 24.10.1
+- Bump countly-sdk-web from 24.11.2 to 24.11.3
+- Bump express from 4.21.1 to 4.21.2
+- Bump express-rate-limit from 7.4.1 to 7.5.0
 - Bump form-data from 4.0.0 to 4.0.1
 - Bump jimp from 0.22.12 to 1.6.0
 - Bump jsdoc from 4.0.3 to 4.0.4
+- Bump mocha from 10.2.0 to 10.8.2
 - Bump nodemailer from 6.9.15 to 6.9.16
-- Bump puppeteer from 23.8.0 to 23.9.0
+- Bump puppeteer from 23.10.4 to 23.11.1
+- Bump sass from 1.81.0 to 1.83.3
 - Bump tslib from 2.7.0 to 2.8.1
 
+## Version 24.10.7
+Fixes:
+- [data-manager] Modifying existing values when segment values want to be updated in the Data Manager
+- [drill] Fix for UI error when push plugin is not enabled
+
+Enterprise fixes:
+- [drill] Fixed empty events list in drill section
+
+Features:
+- [core] Add self tracking capability
+- [hooks] Added remote config changes to internal actions
+- [system-utility] New endpoint: /take-heap-snapshot.
+- [system-utility] Using nodejs fs to write profiler files instead of gridfs.
+
+## Version 24.10.6
+Fixes:
+- [push] Using apns-id header as message result in debug mode
+- [server-stats] Fix data point calculation in job
+- [TopEventsJob] preserver previous state if overwriting fails
+- [ui] scroll top on step changes in drawers
+
+Enterprise fixes:
+- [drill] Encoding url component before changing history state
+- [drill] Fixed drill meta regeneration
+- [drill] [license] Update license loader to enable supplying db client
+- [users] Format data points displayed in user sidebar
+- [cohorts] Unescape drill texts in cohort component
+   
+Dependencies:
+- Bump fs-extra from 11.2.0 to 11.3.0
+- Bump nodemailer from 6.9.16 to 6.10.0
+
+Enterprise Dependencies:
+- Bump nanoid in /plugins/cognito from 2.1.11 to 3.3.8
+- Bump shortid in /plugins/cognito from 2.2.16 to 2.2.17
+
+## Version 24.10.3
+Fixes:
+- [dashboards] Fixing issue where dashboard widgets go into single column
+
+Security:
+- Bump puppeteer from 17.1.3 to 23.8.0
+- Bump express from 4.21.0 to 4.21.1 
+- Bump sass from 1.79.4 to 1.81.0
+- Bump express-session from 1.18.0 to 1.18.1
+- Bump cross-spawn from 7.0.3 to 7.0.6 in /ui-tests
+- Bump cross-spawn from 7.0.3 to 7.0.6 in /plugins/hooks
+
+## Version 24.10.2
+Fixes:
+- [core] Correct aggregated collection cleanup on event omitting
+- [core] Fixed bug where changing passwords results in the loss of the "Global Admin" role
+- [core] Fixed bug where exporting incoming data logs could result in "Incorrect parameter \"data\" error
+- [core] Removed use of commands which needs admin rights from report manager.
+- [crash] Fixed bug in crash ingestion for scenarios where the "app version" is not a string.
+- [script] Fixing bug with "delete_old_members" script that led to malformed requests
+
+Enterprise fixes:
+- [nps] Fixed bug that showed the wrong nps preview title
+
+## Version 24.10.1
+Fixes:
+- [core] Replaced "Users" with "Sessions" label on technology home widgets
+- [push] Improved ability to observe push related errors
+- [push] Replaced push plugin with an earlier version of the plugin
+
+Enterprise fixes:
+- [cohorts] Fixed issues with nightly cleanup
+- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
+- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
+- [flows] Fixed bug in case of null data in schema
+- [license] Fixed bug with MAU type of licenses that would prevent the server from starting
+- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [nps] Fixed bug where it was possible to submit empty nps surveys
+- [ratings] Fixed bug with user consent
+- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
+
+Security:
+- Bumped cookie-parser from 1.4.6 to 1.4.7
+- Bumped express-rate-limit from 7.4.0 to 7.4.1
+- Bumped moment-timezone from 0.5.45 to 0.5.46
+- Bumped sass from 1.79.3 to 1.79.4
+- Fixing minor vulnerability that would allow for unauthorized file upload
+
+Enterprise Features:
+- [block] Added a way to filter crashes by their error (stacktrace)
+
 ## Version 24.10
 Fixes:
 - [core] Correct aggregated collection cleanup on event omitting
@@ -69,6 +178,42 @@ Enterprise Features:
 - [users] UI improvements
 - [views] Added a quick transition to drill
 
+## Version 24.05.21
+Fixes:
+- [core] Fixed a bug causing events to not being loaded when there's an escaped character in the event name
+- [gridfs] fixes for moving to Promises
+- [reports] Fixes report generation failure due to SSL error
+- [surveys] "Select one" text in the widget can be edited now
+- [system-utility] Fixed: Mongo error (code: 26) in some Countly instances when the profiler gets run for the first time
+
+Dependencies:
+- Bump countly-sdk-nodejs from 24.10.0 to 24.10.1
+- Bump countly-sdk-web from 24.11.2 to 24.11.4
+- Bump express-rate-limit from 7.4.1 to 7.5.0
+- Bump puppeteer from 23.10.4 to 23.11.1
+- Bump sass from 1.81.0 to 1.83.4
+
+## Version 24.05.20
+Fixes:
+- [push] Fixed bug where IOS credentials get mixed up while sending messages from different apps at the same time
+- [push] Fixed bug where it crashes in connection pool growth because of a type mismatch in an if condition
+
+Security:
+- [cohorts] Prevent query injection on cohort creation
+
+Dependencies:
+- Bump countly-sdk-nodejs from 22.6.0 to 24.10.0
+- Bump countly-sdk-web from 24.4.1 to 24.11.0
+- Bump express from 4.21.1 to 4.21.2
+- Bump form-data from 4.0.0 to 4.0.1
+- Bump jimp from 0.22.12 to 1.6.0
+- Bump jsdoc from 4.0.3 to 4.0.4
+- Bump mocha from 10.2.0 to 10.8.2
+- Bump mongodb from 4.9.1 to 4.17.2
+- Bump nodemailer from 6.9.15 to 6.9.16
+- Bump puppeteer from 23.8.0 to 23.9.0
+- Bump tslib from 2.7.0 to 2.8.1
+
 ## Version 24.05.19
 Fixes:
 - [dashboards] Fixing issue where dashboard widgets go into single column
@@ -4460,4 +4605,3 @@ This version provides several features and bugfixes to both server and SDKs. The
     A user of an application can only view analytics for that application 
     and cannot edit its settings.
   * Added csfr protection to all methods provided through app.js.
-

Dockerfile-api

@@ -1,4 +1,4 @@
-FROM node:hydrogen-bullseye-slim
+FROM node:iron-bookworm-slim
 
 ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager,guides
 # Countly Enterprise:
@@ -25,15 +25,14 @@ COPY . .
 
 # install required dependencies which slim image doesn't have
 RUN apt-get update && \
-	apt-get install -y iputils-ping procps net-tools telnet apt-transport-https curl wget git python2 make gcc g++ unzip && \
-	ln -s /usr/bin/python2.7 /usr/bin/python
+	apt-get install -y iputils-ping procps net-tools telnet apt-transport-https curl wget git make gcc g++ unzip xz-utils
 
 RUN apt-get update && \
 	apt-get upgrade -y && \
 	cd /usr/src && \
 	wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \
 	tar -xf Python-3.8.12.tar.xz && \
-	apt-get install -y build-essential sudo zlib1g-dev libssl1.1 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \
+	apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \
 	cd Python-3.8.12 && \
 	./configure --enable-optimizations --enable-shared && \
 	make && \
@@ -51,14 +50,15 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo
 	# preinstall
 	cp -n ./api/config.sample.js ./api/config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
+	HOME=/tmp npm install -g npm@latest && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \
 	bash /opt/countly/bin/scripts/detect.init.sh && \
 	\
 	# cleanup & chown
 	npm remove -y --no-save mocha nyc should supertest && \
-	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 && \
+	apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \
 	apt-get install -y libgbm-dev libgbm1 gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
 	apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
 	rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \

Dockerfile-centos-api

@@ -34,7 +34,7 @@ RUN yum update -y
 RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.rpm" && \
 	rpm -i /tmp/tini.rpm && \
 	\
-	curl -sL https://rpm.nodesource.com/setup_18.x | bash - && \
+	curl -sL https://rpm.nodesource.com/setup_20.x | bash - && \
 	yum install -y nodejs python3.8 python2 python38-libs python38-devel python38-pip nss libdrm libgbm cyrus-sasl* && \
 	ln -s /usr/bin/node /usr/bin/nodejs && \
 	unlink /usr/bin/python3 && \
@@ -45,14 +45,15 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
 	yum group install -y "Development Tools" && \
 	yum install -y epel-release && \
 	yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \
-	yum install -y https://pkgs.dyn.su/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
+	yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
 	yum install -y wget openssl-devel make git libsqlite* sqlite unzip bzip2 && \
 	# modify standard distribution
 	./bin/docker/modify.sh && \
 	\
 	# preinstall
 	cp -n ./api/config.sample.js ./api/config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
+	HOME=/tmp npm install -g npm@latest && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \
@@ -73,4 +74,4 @@ USER 1001:0
 
 ENTRYPOINT ["/usr/bin/tini", "-v", "--"]
 
-CMD ["/opt/countly/bin/docker/cmd.sh"]
+CMD ["/opt/countly/bin/docker/cmd.sh"]

Dockerfile-centos-frontend

@@ -32,7 +32,7 @@ RUN yum update -y
 RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.rpm" && \
         rpm -i /tmp/tini.rpm && \
         \
-        curl -sL https://rpm.nodesource.com/setup_18.x | bash - && \
+        curl -sL https://rpm.nodesource.com/setup_20.x | bash - && \
         yum install -y nodejs python3.8 python2 python38-libs python38-devel python38-pip nss libdrm libgbm cyrus-sasl* && \
         ln -s /usr/bin/node /usr/bin/nodejs && \
         unlink /usr/bin/python3 && \
@@ -43,7 +43,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
         yum group install -y "Development Tools" && \
         yum install -y epel-release && \
         yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \
-	      yum install -y https://pkgs.dyn.su/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
+	      yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \
         yum install -y wget openssl-devel make git sqlite libsqlite* unzip bzip2 && \
         # modify standard distribution
         ./bin/docker/modify.sh && \
@@ -52,6 +52,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo
 	cp -n ./frontend/express/public/javascripts/countly/countly.config.sample.js ./frontend/express/public/javascripts/countly/countly.config.js && \
 	cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \
 	cp -n ./api/config.sample.js ./api/config.js && \
+	HOME=/tmp npm install -g npm@latest && \
 	HOME=/tmp npm install --unsafe-perm=true --allow-root && \
 	HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \
 	./bin/docker/preinstall.sh && \

Dockerfile-core

@@ -40,7 +40,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \
         gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev \
         libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \
     # node
-    wget -qO- https://deb.nodesource.com/setup_18.x | bash - && \
+    wget -qO- https://deb.nodesource.com/setup_20.x | bash - && \
     # data_migration (mongo clients)
     wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - && \
     echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list && \