diff --git a/.eslintrc.json b/.eslintrc.json index ce496cb3e45..8948a31958e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -251,6 +251,28 @@ ] } }, + { + "files": [ "plugins/content/api/positioning/**/*.js" ], + "parserOptions": { + "ecmaVersion": 2023, + "sourceType": "module" + }, + "env": { + "node": true, + "es2023": true + } + }, + { + "files": [ "plugins/content/api/positioning/**/*.cjs" ], + "parserOptions": { + "ecmaVersion": 2023, + "sourceType": "commonjs" + }, + "env": { + "node": true, + "es2023": true + } + }, { "files": [ "api/**/*.js", diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b8ccc05252a..73a66a78ecc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,7 +6,7 @@ name: Deploy on: # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ master, next ] + branches: [ master, next, release.24.10, release.24.12 ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/stable-je-deploy.yml b/.github/workflows/stable-je-deploy.yml new file mode 100644 index 00000000000..6393bd70357 --- /dev/null +++ b/.github/workflows/stable-je-deploy.yml @@ -0,0 +1,28 @@ +# This is a basic workflow to help you get started with Actions + +name: Deploy Journey Engine + +# Controls when the workflow will run +on: + # Triggers the workflow on push or pull request events but only for the master branch + push: + branches: [ next ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + deploy: + runs-on: ubuntu-latest + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + + - name: Deploy server + shell: bash + env: + SSH_PRIVATE_KEY: ${{ secrets.STABLE_JE_SSH_PRIVATE_KEY }} + run: bash ./bin/scripts/deploy-je.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index dac4e84a6cf..7a4991cf7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,34 @@ +## Version 25.x.x +Features: +- [alerts] alerts table default order should be by creation time newest at the top +- [core] allow tracking Countly dashboard usage with Countly +- [sdk] Improved and added new Server Config options + +Enterprise Features: +- [journey_engine] Editing/Deleting/Duplication of blocks and version management + ## Version 24.12 Features: - [audit-logs] Exported audit logs from UI now would have "BEFORE" and "AFTER" fields - [core] Ability to mark reports as 'dirty' to make sure they are regenerated in full - [core] Adding a cancel button to "create new app" form - [core] Adding a nightly job to delete old data +- [core] Fixed a bug causing events to not being loaded when there's an escaped character in the event name - [core] Redirecting user to a newly created app - [core] Removing HTML from localization files - [core] Showing a flex banner on sidebar if the version is Countly Lite - [crashes] Adding confirmation for deleting crash groups - [dashoards] Fixed the "Add/manage notes" button that did not work for the technology widget - [dbviewer] Preventing aggregation of using any stages which might open user to harmful actions (like $merge, $out, $lookup, $uninonWith) for all users except global admin +- [gridfs] fixes for moving to Promises - [nps] Fixing issues with default logo selection - [populator] Adding ability to select features to populate and other small improvements +- [push] Fixed bug where IOS credentials get mixed up while sending messages from different apps at the same time +- [push] Fixed bug where it crashes in connection pool growth because of a type mismatch in an if condition +- [reports] Fixes report generation failure due to SSL error - [star-rating] Removed unnecessary limitation with using cohorts for targeting -- [surveys] Removed unnecessary limitation with using cohorts for targeting +- [system-utility] Fixed: Mongo error (code: 26) in some Countly instances when the profiler gets run for the first time +- [user-management] Global admins can now disable 2FA for individual users Enterprise Features: - [cohorts] Adding ability to edit cohorts. This deletes historical calculations @@ -21,18 +36,112 @@ Enterprise Features: - [core] Adding support For SingleStore Kai - [flows] Adding UX improvements to the editor - [journey_engine] Adding "Journey Engine" feature +- [ldap] Fixed issues that would lead to configuration options not being picked up - [remote-config] Moving enable/disable functionality to the dropdown +- [surveys] "Select one" text in the widget can be edited now +- [surveys] Removed unnecessary limitation with using cohorts for targeting Dependencies: -- Bump countly-sdk-nodejs from 22.6.0 to 24.10.0 -- Bump countly-sdk-web from 24.4.1 to 24.11.0 +- Bump countly-sdk-nodejs from 24.10.0 to 24.10.1 +- Bump countly-sdk-web from 24.11.2 to 24.11.3 +- Bump express from 4.21.1 to 4.21.2 +- Bump express-rate-limit from 7.4.1 to 7.5.0 - Bump form-data from 4.0.0 to 4.0.1 - Bump jimp from 0.22.12 to 1.6.0 - Bump jsdoc from 4.0.3 to 4.0.4 +- Bump mocha from 10.2.0 to 10.8.2 - Bump nodemailer from 6.9.15 to 6.9.16 -- Bump puppeteer from 23.8.0 to 23.9.0 +- Bump puppeteer from 23.10.4 to 23.11.1 +- Bump sass from 1.81.0 to 1.83.3 - Bump tslib from 2.7.0 to 2.8.1 +## Version 24.10.7 +Fixes: +- [data-manager] Modifying existing values when segment values want to be updated in the Data Manager +- [drill] Fix for UI error when push plugin is not enabled + +Enterprise fixes: +- [drill] Fixed empty events list in drill section + +Features: +- [core] Add self tracking capability +- [hooks] Added remote config changes to internal actions +- [system-utility] New endpoint: /take-heap-snapshot. +- [system-utility] Using nodejs fs to write profiler files instead of gridfs. + +## Version 24.10.6 +Fixes: +- [push] Using apns-id header as message result in debug mode +- [server-stats] Fix data point calculation in job +- [TopEventsJob] preserver previous state if overwriting fails +- [ui] scroll top on step changes in drawers + +Enterprise fixes: +- [drill] Encoding url component before changing history state +- [drill] Fixed drill meta regeneration +- [drill] [license] Update license loader to enable supplying db client +- [users] Format data points displayed in user sidebar +- [cohorts] Unescape drill texts in cohort component + +Dependencies: +- Bump fs-extra from 11.2.0 to 11.3.0 +- Bump nodemailer from 6.9.16 to 6.10.0 + +Enterprise Dependencies: +- Bump nanoid in /plugins/cognito from 2.1.11 to 3.3.8 +- Bump shortid in /plugins/cognito from 2.2.16 to 2.2.17 + +## Version 24.10.3 +Fixes: +- [dashboards] Fixing issue where dashboard widgets go into single column + +Security: +- Bump puppeteer from 17.1.3 to 23.8.0 +- Bump express from 4.21.0 to 4.21.1 +- Bump sass from 1.79.4 to 1.81.0 +- Bump express-session from 1.18.0 to 1.18.1 +- Bump cross-spawn from 7.0.3 to 7.0.6 in /ui-tests +- Bump cross-spawn from 7.0.3 to 7.0.6 in /plugins/hooks + +## Version 24.10.2 +Fixes: +- [core] Correct aggregated collection cleanup on event omitting +- [core] Fixed bug where changing passwords results in the loss of the "Global Admin" role +- [core] Fixed bug where exporting incoming data logs could result in "Incorrect parameter \"data\" error +- [core] Removed use of commands which needs admin rights from report manager. +- [crash] Fixed bug in crash ingestion for scenarios where the "app version" is not a string. +- [script] Fixing bug with "delete_old_members" script that led to malformed requests + +Enterprise fixes: +- [nps] Fixed bug that showed the wrong nps preview title + +## Version 24.10.1 +Fixes: +- [core] Replaced "Users" with "Sessions" label on technology home widgets +- [push] Improved ability to observe push related errors +- [push] Replaced push plugin with an earlier version of the plugin + +Enterprise fixes: +- [cohorts] Fixed issues with nightly cleanup +- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations +- [drill] Fixed wrong pie chart label tooltip in dashboard widget +- [flows] Fixed bug in case of null data in schema +- [license] Fixed bug with MAU type of licenses that would prevent the server from starting +- [nps] Fixed bug in the editor where the "internal name" field was not mandatory +- [nps] Fixed bug where it was possible to submit empty nps surveys +- [ratings] Fixed bug with user consent +- [ratings] Fixed UI bug where "Internal name" was not a mandatory field + +Security: +- Bumped cookie-parser from 1.4.6 to 1.4.7 +- Bumped express-rate-limit from 7.4.0 to 7.4.1 +- Bumped moment-timezone from 0.5.45 to 0.5.46 +- Bumped sass from 1.79.3 to 1.79.4 +- Fixing minor vulnerability that would allow for unauthorized file upload + +Enterprise Features: +- [block] Added a way to filter crashes by their error (stacktrace) + ## Version 24.10 Fixes: - [core] Correct aggregated collection cleanup on event omitting @@ -69,6 +178,42 @@ Enterprise Features: - [users] UI improvements - [views] Added a quick transition to drill +## Version 24.05.21 +Fixes: +- [core] Fixed a bug causing events to not being loaded when there's an escaped character in the event name +- [gridfs] fixes for moving to Promises +- [reports] Fixes report generation failure due to SSL error +- [surveys] "Select one" text in the widget can be edited now +- [system-utility] Fixed: Mongo error (code: 26) in some Countly instances when the profiler gets run for the first time + +Dependencies: +- Bump countly-sdk-nodejs from 24.10.0 to 24.10.1 +- Bump countly-sdk-web from 24.11.2 to 24.11.4 +- Bump express-rate-limit from 7.4.1 to 7.5.0 +- Bump puppeteer from 23.10.4 to 23.11.1 +- Bump sass from 1.81.0 to 1.83.4 + +## Version 24.05.20 +Fixes: +- [push] Fixed bug where IOS credentials get mixed up while sending messages from different apps at the same time +- [push] Fixed bug where it crashes in connection pool growth because of a type mismatch in an if condition + +Security: +- [cohorts] Prevent query injection on cohort creation + +Dependencies: +- Bump countly-sdk-nodejs from 22.6.0 to 24.10.0 +- Bump countly-sdk-web from 24.4.1 to 24.11.0 +- Bump express from 4.21.1 to 4.21.2 +- Bump form-data from 4.0.0 to 4.0.1 +- Bump jimp from 0.22.12 to 1.6.0 +- Bump jsdoc from 4.0.3 to 4.0.4 +- Bump mocha from 10.2.0 to 10.8.2 +- Bump mongodb from 4.9.1 to 4.17.2 +- Bump nodemailer from 6.9.15 to 6.9.16 +- Bump puppeteer from 23.8.0 to 23.9.0 +- Bump tslib from 2.7.0 to 2.8.1 + ## Version 24.05.19 Fixes: - [dashboards] Fixing issue where dashboard widgets go into single column @@ -4460,4 +4605,3 @@ This version provides several features and bugfixes to both server and SDKs. The A user of an application can only view analytics for that application and cannot edit its settings. * Added csfr protection to all methods provided through app.js. - diff --git a/Dockerfile-api b/Dockerfile-api index 41e8e6fcd4c..567f0801b6d 100644 --- a/Dockerfile-api +++ b/Dockerfile-api @@ -1,4 +1,4 @@ -FROM node:hydrogen-bullseye-slim +FROM node:iron-bookworm-slim ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager,guides # Countly Enterprise: @@ -25,15 +25,14 @@ COPY . . # install required dependencies which slim image doesn't have RUN apt-get update && \ - apt-get install -y iputils-ping procps net-tools telnet apt-transport-https curl wget git python2 make gcc g++ unzip && \ - ln -s /usr/bin/python2.7 /usr/bin/python + apt-get install -y iputils-ping procps net-tools telnet apt-transport-https curl wget git make gcc g++ unzip xz-utils RUN apt-get update && \ apt-get upgrade -y && \ cd /usr/src && \ wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \ tar -xf Python-3.8.12.tar.xz && \ - apt-get install -y build-essential sudo zlib1g-dev libssl1.1 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \ + apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \ cd Python-3.8.12 && \ ./configure --enable-optimizations --enable-shared && \ make && \ @@ -51,6 +50,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo # preinstall cp -n ./api/config.sample.js ./api/config.js && \ cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \ + HOME=/tmp npm install -g npm@latest && \ HOME=/tmp npm install --unsafe-perm=true --allow-root && \ HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \ ./bin/docker/preinstall.sh && \ @@ -58,7 +58,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo \ # cleanup & chown npm remove -y --no-save mocha nyc should supertest && \ - apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 && \ + apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \ apt-get install -y libgbm-dev libgbm1 gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \ diff --git a/Dockerfile-centos-api b/Dockerfile-centos-api index 7a56ce89416..37c5d20f10b 100644 --- a/Dockerfile-centos-api +++ b/Dockerfile-centos-api @@ -34,7 +34,7 @@ RUN yum update -y RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.rpm" && \ rpm -i /tmp/tini.rpm && \ \ - curl -sL https://rpm.nodesource.com/setup_18.x | bash - && \ + curl -sL https://rpm.nodesource.com/setup_20.x | bash - && \ yum install -y nodejs python3.8 python2 python38-libs python38-devel python38-pip nss libdrm libgbm cyrus-sasl* && \ ln -s /usr/bin/node /usr/bin/nodejs && \ unlink /usr/bin/python3 && \ @@ -45,7 +45,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo yum group install -y "Development Tools" && \ yum install -y epel-release && \ yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \ - yum install -y https://pkgs.dyn.su/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \ + yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \ yum install -y wget openssl-devel make git libsqlite* sqlite unzip bzip2 && \ # modify standard distribution ./bin/docker/modify.sh && \ @@ -53,6 +53,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo # preinstall cp -n ./api/config.sample.js ./api/config.js && \ cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \ + HOME=/tmp npm install -g npm@latest && \ HOME=/tmp npm install --unsafe-perm=true --allow-root && \ HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \ ./bin/docker/preinstall.sh && \ @@ -73,4 +74,4 @@ USER 1001:0 ENTRYPOINT ["/usr/bin/tini", "-v", "--"] -CMD ["/opt/countly/bin/docker/cmd.sh"] \ No newline at end of file +CMD ["/opt/countly/bin/docker/cmd.sh"] diff --git a/Dockerfile-centos-frontend b/Dockerfile-centos-frontend index 5817654f5c9..d49991e232d 100644 --- a/Dockerfile-centos-frontend +++ b/Dockerfile-centos-frontend @@ -32,7 +32,7 @@ RUN yum update -y RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.rpm" && \ rpm -i /tmp/tini.rpm && \ \ - curl -sL https://rpm.nodesource.com/setup_18.x | bash - && \ + curl -sL https://rpm.nodesource.com/setup_20.x | bash - && \ yum install -y nodejs python3.8 python2 python38-libs python38-devel python38-pip nss libdrm libgbm cyrus-sasl* && \ ln -s /usr/bin/node /usr/bin/nodejs && \ unlink /usr/bin/python3 && \ @@ -43,7 +43,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo yum group install -y "Development Tools" && \ yum install -y epel-release && \ yum install -y pango.x86_64 libXcomposite.x86_64 libXcursor.x86_64 libXdamage.x86_64 libXext.x86_64 libXi.x86_64 libXtst.x86_64 cups-libs.x86_64 libXScrnSaver.x86_64 libXrandr.x86_64 GConf2.x86_64 alsa-lib.x86_64 atk.x86_64 gtk3.x86_64 xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-utils xorg-x11-fonts-cyrillic xorg-x11-fonts-Type1 xorg-x11-fonts-misc && \ - yum install -y https://pkgs.dyn.su/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \ + yum install -y https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-2.el8.noarch.rpm && \ yum install -y wget openssl-devel make git sqlite libsqlite* unzip bzip2 && \ # modify standard distribution ./bin/docker/modify.sh && \ @@ -52,6 +52,7 @@ RUN curl -s -L -o /tmp/tini.rpm "https://github.com/krallin/tini/releases/downlo cp -n ./frontend/express/public/javascripts/countly/countly.config.sample.js ./frontend/express/public/javascripts/countly/countly.config.js && \ cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \ cp -n ./api/config.sample.js ./api/config.js && \ + HOME=/tmp npm install -g npm@latest && \ HOME=/tmp npm install --unsafe-perm=true --allow-root && \ HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \ ./bin/docker/preinstall.sh && \ diff --git a/Dockerfile-core b/Dockerfile-core index 8bbf7ae4fec..f525a43cc5f 100644 --- a/Dockerfile-core +++ b/Dockerfile-core @@ -40,7 +40,7 @@ RUN useradd -r -M -U -d /opt/countly -s /bin/false countly && \ gcc g++ make binutils autoconf automake autotools-dev libtool pkg-config zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev \ libevent-dev libjansson-dev libjemalloc-dev cython python3-dev python-setuptools && \ # node - wget -qO- https://deb.nodesource.com/setup_18.x | bash - && \ + wget -qO- https://deb.nodesource.com/setup_20.x | bash - && \ # data_migration (mongo clients) wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | sudo apt-key add - && \ echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list && \ diff --git a/Dockerfile-frontend b/Dockerfile-frontend index 66cc47a3b1b..4db8d33266f 100644 --- a/Dockerfile-frontend +++ b/Dockerfile-frontend @@ -1,4 +1,4 @@ -FROM node:hydrogen-bullseye-slim +FROM node:iron-bookworm-slim ARG COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager,guides # Countly Enterprise: @@ -21,15 +21,14 @@ WORKDIR /opt/countly COPY . . # install required dependencies which slim image doesn't have RUN apt-get update && \ - apt-get install -y iputils-ping net-tools telnet apt-transport-https procps curl wget git python2 make gcc g++ unzip && \ - ln -s /usr/bin/python2.7 /usr/bin/python + apt-get install -y iputils-ping net-tools telnet apt-transport-https procps curl wget git make gcc g++ unzip xz-utils RUN apt-get update && \ apt-get upgrade -y && \ cd /usr/src && \ wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz && \ tar -xf Python-3.8.12.tar.xz && \ - apt-get install -y build-essential sudo zlib1g-dev libssl1.1 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \ + apt-get install -y build-essential sudo zlib1g-dev libssl3 libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libsqlite3-dev libreadline-dev libffi-dev curl libbz2-dev && \ cd Python-3.8.12 && \ ./configure --enable-optimizations --enable-shared && \ make && \ @@ -48,6 +47,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo cp -n ./api/config.sample.js ./api/config.js && \ cp -n ./frontend/express/config.sample.js ./frontend/express/config.js && \ cp -n ./frontend/express/public/javascripts/countly/countly.config.sample.js ./frontend/express/public/javascripts/countly/countly.config.js && \ + HOME=/tmp npm install -g npm@latest && \ HOME=/tmp npm install --unsafe-perm=true --allow-root && \ HOME=/tmp npm install argon2 --build-from-source --unsafe-perm=true --allow-root && \ ./bin/docker/preinstall.sh && \ @@ -56,7 +56,7 @@ RUN curl -s -L -o /tmp/tini.deb "https://github.com/krallin/tini/releases/downlo \ # cleanup & chown npm remove -y --no-save mocha nyc should supertest puppeteer && \ - apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 && \ + apt-get remove -y git gcc g++ make automake autoconf libtool pkg-config unzip sqlite3 wget && \ apt-get autoremove -y && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ rm -rf test /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /var/log/* /root/.npm && \ diff --git a/README.md b/README.md index 04f9491a218..3f4ad6978b4 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ ## 🔗 Quick links * [Countly Website](https://countly.com) -* [Countly Server installation guide](https://support.count.ly/hc/en-us/articles/360036862332-Installing-the-Countly-Server) -* [Countly SDKs, download and documentation links](https://support.count.ly/hc/en-us/articles/360037236571-Downloading-and-Installing-SDKs) +* [Countly Server installation guide](https://support.countly.com/hc/en-us/articles/360036862332-Installing-the-Countly-Server) +* [Countly SDKs, download and documentation links](https://support.countly.com/hc/en-us/articles/360037236571-Downloading-and-Installing-SDKs) * [Countly Community on Discord](https://discord.gg/countly) -* [User Guides for Countly features](https://support.count.ly/hc/en-us/sections/7039354168729-User-Guides-Countly-22-x) +* [User Guides for Countly features](https://support.countly.com/hc/en-us/sections/360007405211-User-Guides) ## 🌟 What is Countly? @@ -53,8 +53,8 @@ This repository includes server-side part of Countly, with the following feature Countly can collect and visualize data from mobile, web and desktop applications. Using the write-API you can send data into Countly from any source. For more information please check the below resources: -* [List of Countly SDKs, documentation and download information](https://support.count.ly/hc/en-us/articles/360037236571-Downloading-and-Installing-SDKs) -* [SDK development guide to build your own SDK](https://support.count.ly/hc/en-us/articles/360037753291-SDK-development-guide) +* [List of Countly SDKs, documentation and download information](https://support.countly.com/hc/en-us/articles/360037236571-Downloading-and-Installing-SDKs) +* [SDK development guide to build your own SDK](https://support.countly.com/hc/en-us/articles/360037753291-SDK-development-guide) * [Countly Server Write API to send data into Countly from any source](https://api.count.ly/reference/i) ## 🛠️ Installing and upgrading Countly server @@ -69,15 +69,15 @@ There are several ways to install Countly: 2. For bash lovers, we provide a beautiful installation script (`bin/countly.install.sh`) in countly-server package which installs everything required to run Countly Server. For this, you need a stable release of this repository [available here](https://github.com/Countly/countly-server/releases). -3. Countly Lite also has Docker support - [see our official Docker repository](https://registry.hub.docker.com/r/countly/countly-server/) and [installation instructions for Docker](https://support.count.ly/hc/en-us/articles/360036862332-Installing-the-Countly-Server). +3. Countly Lite also has Docker support - [see our official Docker repository](https://registry.hub.docker.com/r/countly/countly-server/) and [installation instructions for Docker](https://support.countly.com/hc/en-us/articles/360036862332-Installing-the-Countly-Server). -If you want to upgrade Countly from a previous version, please take a look at [upgrading documentation](https://support.count.ly/hc/en-us/articles/360037443652-Upgrading-the-Countly-Server). +If you want to upgrade Countly from a previous version, please take a look at [upgrading documentation](https://support.countly.com/hc/en-us/articles/360037443652-Upgrading-the-Countly-Server). ## 🧩 API, extensibility and plugins Countly has a [well-defined API](https://api.count.ly), that reads and writes data from/to the Countly backend. Countly dashboard is built using the read API, so it's possible to fetch any information you see on the dashboard using the API. -Countly is extensible using the plugin architecture. If you would like to modify any exiting feature by extending it or changing it, or if you would like to add completely new capabilities to Countly you can modify existing plugins or create new ones. We suggest [you read this document](https://support.count.ly/hc/en-us/articles/360036862392-Introduction) if you would like to start with plugin development. +Countly is extensible using the plugin architecture. If you would like to modify any exiting feature by extending it or changing it, or if you would like to add completely new capabilities to Countly you can modify existing plugins or create new ones. We suggest [you read this document](https://support.countly.com/hc/en-us/articles/360036862392-Introduction) if you would like to start with plugin development. ## 💚 Community @@ -93,7 +93,7 @@ Security is very important to us. If you discover any issue regarding security, * **NodeJS** — An open-source, cross-platform JavaScript runtime environment * **Linux** — What we all love using ;-) -Plus lots of [open source libraries](https://support.count.ly/hc/en-us/articles/360037092232-Open-source-components)! +Plus lots of [open source libraries](https://support.countly.com/hc/en-us/articles/360037092232-Open-source-components)! ## 🤝 How can I help you with your efforts? diff --git a/api/api.js b/api/api.js index 2745ffef8ff..667d6870b5c 100644 --- a/api/api.js +++ b/api/api.js @@ -113,8 +113,8 @@ plugins.connectToAllDatabases().then(function() { password_rotation: 3, password_autocomplete: true, robotstxt: "User-agent: *\nDisallow: /", - dashboard_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000 ; includeSubDomains\nX-Content-Type-Options: nosniff", - api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nAccess-Control-Allow-Origin:*", + dashboard_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000; includeSubDomains; preload\nX-Content-Type-Options: nosniff", + api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000; includeSubDomains; preload\nAccess-Control-Allow-Origin:*", dashboard_rate_limit_window: 60, dashboard_rate_limit_requests: 500, proxy_hostname: "", diff --git a/api/jobs/topEvents.js b/api/jobs/topEvents.js index 44913509632..20d3af998ac 100644 --- a/api/jobs/topEvents.js +++ b/api/jobs/topEvents.js @@ -19,8 +19,8 @@ class TopEventsJob extends job.Job { /** * TopEvents initialize function */ - init() { - this.getAllApps(); + async init() { + return this.getAllApps(); } /** @@ -144,6 +144,7 @@ class TopEventsJob extends job.Job { } catch (error) { log.e("TopEvents Job has a error: ", error); + throw error; } } @@ -157,7 +158,18 @@ class TopEventsJob extends job.Job { const encodedData = this.encodeEvents(data); const timeSecond = this.timeSecond(); const currentPeriood = this.mutatePeriod(period); - await new Promise((res, rej) => common.db.collection(TopEventsJob.COLLECTION_NAME).insert({ app_id: _id, ts: timeSecond, period: currentPeriood, data: encodedData, totalCount: totalCount, prevTotalCount: prevTotalCount, totalSum: totalSum, prevTotalSum: prevTotalSum, totalDuration: totalDuration, prevTotalDuration: prevTotalDuration, prevSessionCount: sessionData.prevSessionCount, totalSessionCount: sessionData.totalSessionCount, prevUsersCount: usersData.prevUsersCount, totalUsersCount: usersData.totalUsersCount }, (error, records) => !error && records ? res(records) : rej(error))); + await new Promise((res, rej) => common.db.collection(TopEventsJob.COLLECTION_NAME).findOneAndReplace( + { + app_id: _id, period: currentPeriood, + }, + { + app_id: _id, ts: timeSecond, period: currentPeriood, data: encodedData, totalCount: totalCount, prevTotalCount: prevTotalCount, totalSum: totalSum, prevTotalSum: prevTotalSum, totalDuration: totalDuration, prevTotalDuration: prevTotalDuration, prevSessionCount: sessionData.prevSessionCount, totalSessionCount: sessionData.totalSessionCount, prevUsersCount: usersData.prevUsersCount, totalUsersCount: usersData.totalUsersCount + }, + { + upsert: true + }, + (error, records) => !error && records ? res(records) : rej(error)) + ); } /** @@ -169,7 +181,6 @@ class TopEventsJob extends job.Job { const getEvents = await new Promise((res, rej) => common.db.collection("events").findOne({ _id: app._id }, (errorEvents, result) => errorEvents ? rej(errorEvents) : res(result))); if (getEvents && 'list' in getEvents) { const eventMap = this.eventsFilter(getEvents.list); - await new Promise((res, rej) => common.db.collection(TopEventsJob.COLLECTION_NAME).remove({ app_id: app._id }, (error, result) => error ? rej(error) : res(result))); if (eventMap && eventMap instanceof Array) { for (const period of TopEventsJob.PERIODS) { const data = {}; @@ -211,9 +222,14 @@ class TopEventsJob extends job.Job { * @param {Db} db connection * @param {done} done callback */ - run(db, done) { - this.init(); - done(); + async run(db, done) { + try { + await this.init(); + done(); + } + catch (error) { + done(error); + } } } diff --git a/api/parts/mgmt/app_users.js b/api/parts/mgmt/app_users.js index 539149f9c13..e438db69ccb 100644 --- a/api/parts/mgmt/app_users.js +++ b/api/parts/mgmt/app_users.js @@ -509,6 +509,7 @@ usersApi.mergeOtherPlugins = function(options, callback) { if (result && result.length) { for (let index = 0; index < result.length; index++) { if (result[index].status === "rejected") { + log.e(result[index]); retry = true; break; } diff --git a/api/utils/common.js b/api/utils/common.js index af59a3ea0c8..16b00eba4c4 100644 --- a/api/utils/common.js +++ b/api/utils/common.js @@ -693,7 +693,13 @@ common.getDate = function(timestamp, timezone) { * @returns {number} current day of the year */ common.getDOY = function(timestamp, timezone) { - var endDate = (timestamp) ? moment.unix(timestamp * 1000) : moment(); + var endDate; + if (timestamp && timestamp.toString().length === 13) { + endDate = (timestamp) ? moment.unix(timestamp / 1000) : moment(); + } + else { + endDate = (timestamp) ? moment.unix(timestamp) : moment(); + } if (timezone) { endDate.tz(timezone); diff --git a/api/utils/countly-request/package-lock.json b/api/utils/countly-request/package-lock.json index 55eef687cec..57af8f88f04 100644 --- a/api/utils/countly-request/package-lock.json +++ b/api/utils/countly-request/package-lock.json @@ -13,7 +13,7 @@ "hpagent": "^1.2.0" }, "devDependencies": { - "mocha": "^10.2.0", + "mocha": "^10.8.2", "should": "^13.2.3" } }, @@ -77,10 +77,11 @@ } }, "node_modules/ansi-colors": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", - "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", + "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -132,7 +133,8 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/binary-extensions": { "version": "2.2.0", @@ -144,13 +146,13 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, + "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "node_modules/braces": { @@ -303,19 +305,14 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true - }, "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", "dev": true, + "license": "MIT", "dependencies": { - "ms": "2.1.2" + "ms": "^2.1.3" }, "engines": { "node": ">=6.0" @@ -326,12 +323,6 @@ } } }, - "node_modules/debug/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/decamelize": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-4.0.0.tgz", @@ -378,10 +369,11 @@ } }, "node_modules/diff": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", - "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", + "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -462,7 +454,8 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/fsevents": { "version": "2.3.2", @@ -502,20 +495,21 @@ } }, "node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", "dev": true, + "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "minimatch": "^5.0.1", + "once": "^1.3.0" }, "engines": { - "node": "*" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -533,18 +527,6 @@ "node": ">= 6" } }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/got": { "version": "11.8.5", "resolved": "https://registry.npmjs.org/got/-/got-11.8.5.tgz", @@ -616,7 +598,9 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", "dev": true, + "license": "ISC", "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -626,7 +610,8 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -773,10 +758,11 @@ } }, "node_modules/minimatch": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", - "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", "dev": true, + "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" }, @@ -784,42 +770,33 @@ "node": ">=10" } }, - "node_modules/minimatch/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, "node_modules/mocha": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", - "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", + "version": "10.8.2", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.8.2.tgz", + "integrity": "sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg==", "dev": true, + "license": "MIT", "dependencies": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" + "ansi-colors": "^4.1.3", + "browser-stdout": "^1.3.1", + "chokidar": "^3.5.3", + "debug": "^4.3.5", + "diff": "^5.2.0", + "escape-string-regexp": "^4.0.0", + "find-up": "^5.0.0", + "glob": "^8.1.0", + "he": "^1.2.0", + "js-yaml": "^4.1.0", + "log-symbols": "^4.1.0", + "minimatch": "^5.1.6", + "ms": "^2.1.3", + "serialize-javascript": "^6.0.2", + "strip-json-comments": "^3.1.1", + "supports-color": "^8.1.1", + "workerpool": "^6.5.1", + "yargs": "^16.2.0", + "yargs-parser": "^20.2.9", + "yargs-unparser": "^2.0.0" }, "bin": { "_mocha": "bin/_mocha", @@ -827,10 +804,6 @@ }, "engines": { "node": ">= 14.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mochajs" } }, "node_modules/ms": { @@ -839,18 +812,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, - "node_modules/nanoid": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", - "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", - "dev": true, - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -926,15 +887,6 @@ "node": ">=8" } }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -972,6 +924,7 @@ "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", "dev": true, + "license": "MIT", "dependencies": { "safe-buffer": "^5.1.0" } @@ -1031,13 +984,15 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", "dev": true, + "license": "BSD-3-Clause", "dependencies": { "randombytes": "^2.1.0" } @@ -1162,10 +1117,11 @@ } }, "node_modules/workerpool": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", - "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", - "dev": true + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", + "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", + "dev": true, + "license": "Apache-2.0" }, "node_modules/wrap-ansi": { "version": "7.0.0", @@ -1217,10 +1173,11 @@ } }, "node_modules/yargs-parser": { - "version": "20.2.4", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.4.tgz", - "integrity": "sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA==", + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, + "license": "ISC", "engines": { "node": ">=10" } diff --git a/api/utils/countly-request/package.json b/api/utils/countly-request/package.json old mode 100755 new mode 100644 index 2ef44d17835..3c5080bdb45 --- a/api/utils/countly-request/package.json +++ b/api/utils/countly-request/package.json @@ -18,7 +18,7 @@ "hpagent": "^1.2.0" }, "devDependencies": { - "mocha": "^10.2.0", + "mocha": "^10.8.2", "should": "^13.2.3" } } diff --git a/api/utils/countlyFs.js b/api/utils/countlyFs.js index fc8ed4fb705..5568367d9c8 100644 --- a/api/utils/countlyFs.js +++ b/api/utils/countlyFs.js @@ -68,7 +68,7 @@ countlyFs.gridfs = {}; **/ function beforeSave(category, filename, options, callback, done) { log.d("checking file", filename); - ob.getId(category, filename, function(err, res) { + ob.getId(category, filename, async function(err, res) { log.d("file state", filename, err, res); if (options.forceClean) { ob.clearFile(category, filename, done); @@ -80,15 +80,20 @@ countlyFs.gridfs = {}; else if (options.writeMode === "overwrite") { var bucket = new GridFSBucket(db, { bucketName: category }); log.d("deleting file", filename); - bucket.delete(res, function(error) { - log.d("deleted", filename, error); - if (!error) { - setTimeout(done, 1); - } - else if (callback) { - callback(error); - } - }); + let errHandle = null; + try { + await bucket.delete(res); + } + catch (error) { + errHandle = error; + } + log.d("deleted", filename, errHandle); + if (!errHandle) { + setTimeout(done, 1); + } + else if (callback) { + callback(errHandle); + } } else { if (callback) { @@ -116,6 +121,7 @@ countlyFs.gridfs = {}; * }); */ ob.getId = function(category, filename, callback) { + log.d("getId", category, filename); db.collection(category + ".files").findOne({ filename: filename }, {_id: 1}, function(err, res) { if (callback) { callback(err, (res && res._id) ? res._id : false); @@ -144,6 +150,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } + log.d("exists", category, dest, options); var query = {}; if (options.id) { query._id = options.id; @@ -184,7 +191,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("saveFile", category, dest, source, options); var filename = dest.split(path.sep).pop(); beforeSave(category, filename, options, callback, function() { save(category, filename, fs.createReadStream(source), options, callback); @@ -218,6 +225,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } + log.d("saveData", category, dest, typeof data, options); beforeSave(category, filename, options, callback, function() { var readStream = new Readable; readStream.push(data); @@ -253,6 +261,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } + log.d("saveStream", category, dest, typeof readStream, options); beforeSave(category, filename, options, callback, function() { save(category, filename, readStream, options, callback); }); @@ -271,7 +280,7 @@ countlyFs.gridfs = {}; * console.log("Finished", err); * }); */ - ob.rename = function(category, dest, source, options, callback) { + ob.rename = async function(category, dest, source, options, callback) { var newname = dest.split(path.sep).pop(); var oldname = source.split(path.sep).pop(); if (typeof options === "function") { @@ -281,25 +290,35 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("rename", category, dest, source, options); if (options.id) { let bucket = new GridFSBucket(db, { bucketName: category }); - bucket.rename(options.id, newname, function(error) { - if (callback) { - callback(error); - } - }); + let errHandle = null; + try { + await bucket.rename(options.id, newname); + } + catch (error) { + errHandle = error; + } + if (callback) { + callback(errHandle); + } } else { - db.collection(category + ".files").findOne({ filename: oldname }, {_id: 1}, function(err, res) { + db.collection(category + ".files").findOne({ filename: oldname }, {_id: 1}, async function(err, res) { if (!err) { if (res && res._id) { let bucket = new GridFSBucket(db, { bucketName: category }); - bucket.rename(res._id, newname, function(error) { - if (callback) { - callback(error); - } - }); + let errHandle = null; + try { + await bucket.rename(res._id, newname); + } + catch (error) { + errHandle = error; + } + if (callback) { + callback(errHandle); + } } else { if (callback) { @@ -391,7 +410,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("deleteFile", category, dest, options); if (options.id) { ob.deleteFileById(category, options.id, callback); } @@ -426,13 +445,19 @@ countlyFs.gridfs = {}; * console.log("Finished", err); * }); */ - ob.deleteAll = function(category, dest, callback) { + ob.deleteAll = async function(category, dest, callback) { + log.d("deleteAll", category, dest); var bucket = new GridFSBucket(db, { bucketName: category }); - bucket.drop(function(error) { - if (callback) { - callback(error); - } - }); + let errHandle = null; + try { + await bucket.drop(); + } + catch (error) { + errHandle = error; + } + if (callback) { + callback(errHandle); + } }; /** @@ -457,7 +482,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("getStream", category, dest, options); if (callback) { if (options.id) { ob.getStreamById(category, options.id, callback); @@ -490,7 +515,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("getData", category, dest, options); if (options.id) { ob.getDataById(category, options.id, callback); } @@ -536,7 +561,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("getSize", category, dest, options); var query = {}; if (options.id) { query._id = options.id; @@ -571,7 +596,7 @@ countlyFs.gridfs = {}; if (!options) { options = {}; } - + log.d("getStats", category, dest, options); var query = {}; if (options.id) { query._id = options.id; @@ -608,6 +633,7 @@ countlyFs.gridfs = {}; * }); */ ob.getDataById = function(category, id, callback) { + log.d("getDataById", category, id); var bucket = new GridFSBucket(db, { bucketName: category }); var downloadStream = bucket.openDownloadStream(id); downloadStream.on('error', function(error) { @@ -639,6 +665,7 @@ countlyFs.gridfs = {}; * }); */ ob.getStreamById = function(category, id, callback) { + log.d("getStreamById", category, id); if (callback) { var bucket = new GridFSBucket(db, { bucketName: category }); callback(null, bucket.openDownloadStream(id)); @@ -656,17 +683,17 @@ countlyFs.gridfs = {}; * }); */ ob.deleteFileById = async function(category, id, callback) { + log.d("deleteFileById", category, id); var bucket = new GridFSBucket(db, { bucketName: category }); + let errHandle = null; try { await bucket.delete(id); - if (callback) { - callback(null); - } } - catch (ee) { - if (callback) { - callback(ee); - } + catch (error) { + errHandle = error; + } + if (callback) { + callback(errHandle); } }; @@ -681,6 +708,7 @@ countlyFs.gridfs = {}; * }); */ ob.clearFile = function(category, filename, callback) { + log.d("clearFile", category, filename); db.collection(category + ".files").deleteMany({ filename: filename }, function(err1, res1) { log.d("deleting files", category, { filename: filename }, err1, res1 && res1.result); db.collection(category + ".chunks").deleteMany({ files_id: filename }, function(err2, res2) { @@ -697,6 +725,7 @@ countlyFs.gridfs = {}; * @param {function} callback - function called when files found or query errored, providing error object as first param and a list of filename, creation date and size as secondas second */ ob.listFiles = function(category, callback) { + log.d("listFiles", category); const bucket = new GridFSBucket(db, { bucketName: category }); bucket.find().toArray() .then((records) => callback( diff --git a/api/utils/render.js b/api/utils/render.js index ba65f9d11cc..ee01d946878 100644 --- a/api/utils/render.js +++ b/api/utils/render.js @@ -67,7 +67,7 @@ exports.renderView = function(options, cb) { XDG_CONFIG_HOME: pathModule.resolve(__dirname, "../../.cache/chrome/tmp/.chromium"), XDG_CACHE_HOME: pathModule.resolve(__dirname, "../../.cache/chrome/tmp/.chromium") }, - args: ['--no-sandbox', '--disable-setuid-sandbox'], + args: ['--no-sandbox', '--disable-setuid-sandbox', '--ignore-certificate-errors'], ignoreHTTPSErrors: true, userDataDir: pathModule.resolve(__dirname, "../../dump/chrome") }; diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js old mode 100755 new mode 100644 index a314ef9facb..cee4ca4c9fd --- a/api/utils/requestProcessor.js +++ b/api/utils/requestProcessor.js @@ -3265,6 +3265,80 @@ const processFetchRequest = (params, app, done) => { }); }; +/** + * Process Bulk Request + * @param {number} i - request number in bulk + * @param {array} requests - array of requests to process + * @param {params} params - params object + * @returns {void} void + */ +const processBulkRequest = (i, requests, params) => { + const appKey = params.qstring.app_key; + if (i === requests.length) { + common.unblockResponses(params); + if ((params.qstring.safe_api_response || plugins.getConfig("api", params.app && params.app.plugins, true).safe) && !params.res.finished) { + common.returnMessage(params, 200, 'Success'); + } + return; + } + + if (!requests[i] || (!requests[i].app_key && !appKey)) { + return processBulkRequest(i + 1, requests, params); + } + if (params.qstring.safe_api_response) { + requests[i].safe_api_response = true; + } + params.req.body = JSON.stringify(requests[i]); + const tmpParams = { + 'app_id': '', + 'app_cc': '', + 'ip_address': requests[i].ip_address || common.getIpAddress(params.req), + 'user': { + 'country': requests[i].country_code || 'Unknown', + 'city': requests[i].city || 'Unknown' + }, + 'qstring': requests[i], + 'href': "/i", + 'res': params.res, + 'req': params.req, + 'promises': [], + 'bulk': true, + 'populator': params.qstring.populator, + 'blockResponses': true + }; + + tmpParams.qstring.app_key = (requests[i].app_key || appKey) + ""; + + if (!tmpParams.qstring.device_id) { + return processBulkRequest(i + 1, requests, params); + } + else { + //make sure device_id is string + tmpParams.qstring.device_id += ""; + tmpParams.app_user_id = common.crypto.createHash('sha1') + .update(tmpParams.qstring.app_key + tmpParams.qstring.device_id + "") + .digest('hex'); + } + + return validateAppForWriteAPI(tmpParams, () => { + /** + * Dispatches /sdk/end event upon finishing processing request + **/ + function resolver() { + plugins.dispatch("/sdk/end", {params: tmpParams}, () => { + processBulkRequest(i + 1, requests, params); + }); + } + + Promise.all(tmpParams.promises) + .then(resolver) + .catch((error) => { + console.log(error); + resolver(); + }); + }); +}; + /** * @param {object} params - params object * @param {String} type - source type diff --git a/bin/commands/scripts/apidocs.sh b/bin/commands/scripts/apidocs.sh index 60c9fda72b6..21b079a780f 100644 --- a/bin/commands/scripts/apidocs.sh +++ b/bin/commands/scripts/apidocs.sh @@ -12,5 +12,7 @@ then elif [ "$1" = "generate" ]; then echo 'yes' echo "$DIR/../../../../plugins/" + npm install apidoc; + npm install apidoc-template; "$DIR/../../../node_modules/.bin/apidoc" -c "$DIR/../../../apidoc.json" -f "api/.*\\.js$" -i "$DIR/../../../plugins/" -o "$DIR/../../../frontend/express/public/apidoc/" -t "$DIR/../../../node_modules/apidoc-template/template/"; fi diff --git a/bin/commands/scripts/docs.sh b/bin/commands/scripts/docs.sh index 9db21208285..a2fb5f6c1de 100644 --- a/bin/commands/scripts/docs.sh +++ b/bin/commands/scripts/docs.sh @@ -20,7 +20,7 @@ elif [ "$1" = "generate" ]; then npx jsdoc "$DIR/../../../frontend/express/app.js" "$DIR/../../../frontend/express/config.sample.js" "$DIR/../../../frontend/express/version.info.js" "$DIR/../../../frontend/express/locale.conf.js" "$DIR/../../../frontend/express/libs/" -R "$DIR/../../../README.md" -c "$DIR/../../../jsdoc_conf.json" -d "$DIR/../../../frontend/express/public/docs/app" ; #apidoc - npx apidoc -i "$DIR/../../../" -o "$DIR/../../../frontend/express/public/docs/apidoc" -f ".*\\.js$" -e "node_modules" ; + npm install apidoc; npm install apidoc-template; npx apidoc -i "$DIR/../../../" -o "$DIR/../../../frontend/express/public/docs/apidoc" -f ".*\\.js$" -e "node_modules" ; #add redirect for main folder echo "
" > "$DIR/../../../frontend/express/public/docs/index.html" diff --git a/bin/countly.install_rhel.sh b/bin/countly.install_rhel.sh index ba4577a1b47..0b135def439 100644 --- a/bin/countly.install_rhel.sh +++ b/bin/countly.install_rhel.sh @@ -48,9 +48,9 @@ cp "$DIR/config/supervisord.example.conf" "$DIR/config/supervisord.conf" #Install raven-release for ipa-gothic-fonts required by puppeteer if [[ "$CENTOS_MAJOR" = "9" ]]; then - sudo rpm -ivh https://pkgs.dyn.su/el8/base/x86_64/ipa-gothic-fonts-003.03-15.el8.noarch.rpm + sudo rpm -ivh https://pkgs.sysadmins.ws/el8/base/x86_64/ipa-gothic-fonts-003.03-15.el8.noarch.rpm else - sudo yum install -y https://pkgs.dyn.su/el8/base/x86_64/raven-release-1.0-3.el8.noarch.rpm + sudo yum install https://pkgs.sysadmins.ws/el8/base/x86_64/raven-release-1.0-3.el8.noarch.rpm sudo yum install -y ipa-gothic-fonts fi diff --git a/bin/scripts/deploy-je.sh b/bin/scripts/deploy-je.sh new file mode 100644 index 00000000000..267308def03 --- /dev/null +++ b/bin/scripts/deploy-je.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Generate a timestamp for the log file +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") +LOG_FILE="deploy_$TIMESTAMP.log" + +# Check if the event is a push to the "next" branch in the "Countly/countly-server" repository +if [ -z "$GITHUB_HEAD_REF" ] && [ "$GITHUB_REPOSITORY" == "Countly/countly-server" ]; then + GITHUB_BRANCH=${GITHUB_REF#refs/heads/} + echo "$GITHUB_BRANCH" + if [ "$GITHUB_BRANCH" == "next" ]; then + echo "$SSH_PRIVATE_KEY" > je-deploy-key + mkdir -p ~/.ssh + mv je-deploy-key ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Log the deployment command with a timestamped log file + ssh -oStrictHostKeyChecking=no "stable-je@stable-je-cb.count.ly" \ + "sudo su -c 'bash /opt/deploy.sh > /var/log/github-action/$LOG_FILE 2>&1 &'" + fi +fi + diff --git a/bin/scripts/member-managament/delete_old_members.js b/bin/scripts/member-managament/delete_old_members.js index a52639e79f9..13f88681426 100644 --- a/bin/scripts/member-managament/delete_old_members.js +++ b/bin/scripts/member-managament/delete_old_members.js @@ -17,7 +17,8 @@ var days = 30; //query states not logged in in last N days , but logged in at least once var ts = Math.round(Date.now() / 1000) - days * 24 * 60 * 60; -var query = {"$and": [{"last_login": {"$lt": ts}}, {"last_login": {"$exists": true}}]}; +var query = {"$and": [{"last_login": {"$lt": ts}}, {"last_login": {"$exists": true}}, {"global_admin": {"$ne": true}}]}; +// {"global_admin": {"$ne": true}} ensures that global admins are excluded from deletion. //although mogodb does not return null on $lt, keep like above for safety @@ -120,4 +121,4 @@ function sendRequest(params, callback) { console.log(e); callback({"err": 'Failed to send'}); } -} \ No newline at end of file +} diff --git a/bin/scripts/member-managament/disable_2fa.js b/bin/scripts/member-managament/disable_2fa.js new file mode 100644 index 00000000000..61eb794816b --- /dev/null +++ b/bin/scripts/member-managament/disable_2fa.js @@ -0,0 +1,54 @@ +/* +Script should be placed in ./bin/scripts/member-managament/disable_2fa.js + +Script is used to disable user(s) 2FA by email. +*/ + +var pluginManager = require('../../../plugins/pluginManager.js'); + +const dry_run = false; //if set true, there will be only information outputted about users in the email list, but 2FA disable operation will not be triggered. +//const EMAILS = ["test@mail.com", "test2@mail.com"]; +const EMAILS = ['']; + +if (dry_run) { + console.log("This is a dry run"); + console.log("Members will only be listed, 2FA will not be disabled"); +} + +pluginManager.dbConnection().then(async(countlyDb) => { + try { + // Find the users by email + let users = []; + users = await getUsers(countlyDb, EMAILS); + + console.log(`The following ${users.length} user(s) 2FA will be disabled: `); + console.log(JSON.stringify(users)); + if (!dry_run) { + await countlyDb.collection('members').updateMany({_id: {$in: users.map(user=>user._id)}}, + { + $set: {"two_factor_auth.enabled": false}, + $unset: {"two_factor_auth.secret_token": ""} + }); + console.log("All done"); + } + } + catch (error) { + console.log("ERROR: "); + console.log(error); + } + finally { + countlyDb.close(); + } +}); + +function getUsers(db, emails) { + const query = {}; + if (emails?.length) { + query.email = { + $in: emails + }; + } + return db.collection('members').find(query, { + projection: { _id: 1, email: 1 } + }).toArray(); +} \ No newline at end of file diff --git a/bin/upgrade/24.12/upgrade.mongo.80.sh b/bin/upgrade/24.12/upgrade.mongo.80.sh index 46fea7095d3..193f67dc9ca 100644 --- a/bin/upgrade/24.12/upgrade.mongo.80.sh +++ b/bin/upgrade/24.12/upgrade.mongo.80.sh @@ -126,6 +126,7 @@ mongosh --nodb --eval 'var conn; print("Waiting for MongoDB connection on port 2 if [ "$isAuth" -eq "1" ]; then echo "run this command with authentication to upgrade to 8.0" + # shellcheck disable=SC2028 echo "mongosh admin --eval \"db.adminCommand( { setFeatureCompatibilityVersion: \\\8.0\\\", confirm: true } )\"" elif ! mongosh admin --eval "printjson(db.adminCommand( { getParameter: 1, featureCompatibilityVersion: 1 } ))" ; then echo "Could not connect to MongodB, run this command when Mongo is up and running" diff --git a/bin/upgrade/DEV/add_creation_date_for_existing_alerts.js b/bin/upgrade/DEV/add_creation_date_for_existing_alerts.js new file mode 100644 index 00000000000..7ad693b0331 --- /dev/null +++ b/bin/upgrade/DEV/add_creation_date_for_existing_alerts.js @@ -0,0 +1,25 @@ +// Script that adds creation date for existing alerts. + +const pluginManager = require('../../../plugins/pluginManager.js'); + +pluginManager.dbConnection().then(async(countlyDb) => { + try { + await countlyDb.collection('alerts').updateMany( + { createdAt: { $exists: false } }, + [ + { + $set: { + createdAt: { $toDouble: { $toDate: "$_id" } } + } + } + ] + ); + console.log("Finished adding creation date for existing alerts."); + } + catch (error) { + console.log(`Error adding creation date for existing alerts: ${error}`); + } + finally { + countlyDb.close(); + } +}); \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 40523cb6c1a..62e26ea1a0c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,10 +16,10 @@ services: countly-api: image: 'countly/api:latest' - # Countly Enterprise: image: 'gcr.io/countly-01/api:20.11.2' + # Countly Enterprise: image: 'gcr.io/countly-01/api:24.12' environment: - COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager - # Countly Enterprise: - COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,license,drill,funnels,retention_segments,flows,cohorts,surveys,remote-config,ab-testing,formulas,activity-map,concurrent_users,revenue,logger,systemlogs,populator,reports,crashes,push,geo,block,restrict,users,star-rating,slipping-away-users,compare,server-stats,assistant,dbviewer,crash_symbolication,crashes-jira,groups,white-labeling,alerts,times-of-day,compliance-hub,onboarding,active_users,performance-monitoring,config-transfer,consolidate,data-manager,hooks,dashboards,sdk + # Countly Enterprise: - COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,license,drill,funnels,retention_segments,flows,cohorts,surveys,remote-config,ab-testing,formulas,activity-map,concurrent_users,revenue,logger,systemlogs,populator,reports,crashes,push,geo,block,users,star-rating,slipping-away-users,compare,server-stats,dbviewer,crash_symbolication,crashes-jira,groups,white-labeling,alerts,times-of-day,compliance-hub,onboarding,active_users,performance-monitoring,config-transfer,consolidate,data-manager,hooks,dashboards,sdk - COUNTLY_CONFIG__MONGODB_HOST=mongodb - COUNTLY_CONFIG_API_API_WORKERS=4 # CPU core count - COUNTLY_CONFIG__FILESTORAGE="gridfs" @@ -37,10 +37,10 @@ services: countly-frontend: image: 'countly/frontend:latest' - # Countly Enterprise: image: 'gcr.io/countly-01/frontend:20.11.2' + # Countly Enterprise: image: 'gcr.io/countly-01/frontend:24.12' environment: - COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,logger,systemlogs,populator,reports,crashes,push,star-rating,slipping-away-users,compare,server-stats,dbviewer,times-of-day,compliance-hub,alerts,onboarding,consolidate,remote-config,hooks,dashboards,sdk,data-manager - # Countly Enterprise: - COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,license,drill,funnels,retention_segments,flows,cohorts,surveys,remote-config,ab-testing,formulas,activity-map,concurrent_users,revenue,logger,systemlogs,populator,reports,crashes,push,geo,block,restrict,users,star-rating,slipping-away-users,compare,server-stats,assistant,dbviewer,crash_symbolication,crashes-jira,groups,white-labeling,alerts,times-of-day,compliance-hub,onboarding,active_users,performance-monitoring,config-transfer,consolidate,data-manager,hooks,dashboards,sdk + # Countly Enterprise: - COUNTLY_PLUGINS=mobile,web,desktop,plugins,density,locale,browser,sources,views,license,drill,funnels,retention_segments,flows,cohorts,surveys,remote-config,ab-testing,formulas,activity-map,concurrent_users,revenue,logger,systemlogs,populator,reports,crashes,push,geo,block,users,star-rating,slipping-away-users,compare,server-stats,dbviewer,crash_symbolication,crashes-jira,groups,white-labeling,alerts,times-of-day,compliance-hub,onboarding,active_users,performance-monitoring,config-transfer,consolidate,data-manager,hooks,dashboards,sdk - COUNTLY_CONFIG__MONGODB_HOST=mongodb - NODE_OPTIONS="--max-old-space-size=2048" networks: diff --git a/frontend/express/app.js b/frontend/express/app.js index 7c8e8341988..77eaa3d0fed 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -6,6 +6,22 @@ // Set process name process.title = "countly: dashboard node " + process.argv[1]; +var fs = require('fs'); +var path = require('path'); +var IS_FLEX = false; + +if (fs.existsSync(path.resolve('/opt/deployment_env.json'))) { + var deploymentConf = fs.readFileSync('/opt/deployment_env.json', 'utf8'); + try { + if (JSON.parse(deploymentConf).DEPLOYMENT_ID) { + IS_FLEX = true; + } + } + catch (e) { + IS_FLEX = false; + } +} + var versionInfo = require('./version.info'), pack = require('../../package.json'), COUNTLY_VERSION = versionInfo.version, @@ -27,8 +43,6 @@ var versionInfo = require('./version.info'), } }), crypto = require('crypto'), - fs = require('fs'), - path = require('path'), jimp = require('jimp'), flash = require('connect-flash'), cookieParser = require('cookie-parser'), @@ -66,7 +80,13 @@ var COUNTLY_NAMED_TYPE = "Countly Lite v" + COUNTLY_VERSION; var COUNTLY_TYPE_CE = true; var COUNTLY_TRIAL = (versionInfo.trial) ? true : false; var COUNTLY_TRACK_TYPE = "OSS"; -if (versionInfo.footer) { + +if (IS_FLEX) { + COUNTLY_NAMED_TYPE = "Countly v" + COUNTLY_VERSION; + COUNTLY_TYPE_CE = false; + COUNTLY_TRACK_TYPE = "Flex"; +} +else if (versionInfo.footer) { COUNTLY_NAMED_TYPE = versionInfo.footer; COUNTLY_TYPE_CE = false; if (COUNTLY_NAMED_TYPE === "Countly Cloud") { @@ -116,8 +136,8 @@ plugins.setConfigs("frontend", { session_timeout: 30, use_google: true, code: true, - google_maps_api_key: "", offline_mode: false, + self_tracking: "", }); if (!plugins.isPluginEnabled('tracker')) { @@ -137,7 +157,6 @@ plugins.setUserConfigs("frontend", { session_timeout: false, use_google: false, code: false, - google_maps_api_key: "" }); plugins.setConfigs("security", { @@ -151,8 +170,8 @@ plugins.setConfigs("security", { password_rotation: 3, password_autocomplete: true, robotstxt: "User-agent: *\nDisallow: /", - dashboard_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000 ; includeSubDomains\nX-Content-Type-Options: nosniff", - api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nAccess-Control-Allow-Origin:*", + dashboard_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000; includeSubDomains; preload\nX-Content-Type-Options: nosniff", + api_additional_headers: "X-Frame-Options:deny\nX-XSS-Protection:1; mode=block\nStrict-Transport-Security:max-age=31536000; includeSubDomains; preload\nAccess-Control-Allow-Origin:*", dashboard_rate_limit_window: 60, dashboard_rate_limit_requests: 500 }); @@ -907,8 +926,9 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ countly_tracking = plugins.isPluginEnabled('tracker') ? true : plugins.getConfig('frontend').countly_tracking, countly_domain = plugins.getConfig('api').domain, licenseNotification, licenseError; + var isLocked = false; configs.export_limit = plugins.getConfig("api").export_limit; - app.loadThemeFiles(configs.theme, function(theme) { + app.loadThemeFiles(configs.theme, async function(theme) { if (configs._user.theme) { res.cookie("theme", configs.theme); } @@ -922,6 +942,13 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ if (member.upgrade) { countlyDb.collection('members').update({"_id": member._id}, {$unset: {upgrade: ""}}, function() {}); } + if (IS_FLEX) { + let locked = await countlyDb.collection('mycountly').findOne({_id: 'lockServer'}); + if (locked?.isLocked === true) { + isLocked = true; + } + + } if (req.session.licenseError) { licenseError = req.session.licenseError; @@ -989,6 +1016,8 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ helpCenterLink: COUNTLY_HELPCENTER_LINK, featureRequestLink: COUNTLY_FEATUREREQUEST_LINK, }, + mycountly: IS_FLEX, + isLocked: isLocked, }; diff --git a/frontend/express/public/.well-known/security.txt b/frontend/express/public/.well-known/security.txt new file mode 100644 index 00000000000..1db00744410 --- /dev/null +++ b/frontend/express/public/.well-known/security.txt @@ -0,0 +1,7 @@ +# If you would like to report a security issue with Countly Server, Countly SDKs +# please get in touch via the below method +Contact: mailto:security@count.ly +Expires: 2025-03-14T00:00:00.000Z +Preferred-Languages: en +Canonical: https://securitytxt.org/.well-known/security.txt +Policy: https://countly.com/legal/privacy-policy \ No newline at end of file diff --git a/frontend/express/public/core/app-management/stylesheets/_main.scss b/frontend/express/public/core/app-management/stylesheets/_main.scss index b3261f3bd02..21c4ae006b5 100644 --- a/frontend/express/public/core/app-management/stylesheets/_main.scss +++ b/frontend/express/public/core/app-management/stylesheets/_main.scss @@ -85,7 +85,7 @@ } } &__colorpicker{ - .picker-body {overflow: auto;} + .cly-vue-color-picker__picker {overflow: auto;} } &__button{ &:active, &:focus, &:hover {background-color: unset !important;border-color: unset !important;} @@ -193,4 +193,4 @@ width: 177%; } } -} \ No newline at end of file +} diff --git a/frontend/express/public/core/date-presets/javascripts/countly.views.js b/frontend/express/public/core/date-presets/javascripts/countly.views.js old mode 100755 new mode 100644 index a0830b9e534..03478c00e13 --- a/frontend/express/public/core/date-presets/javascripts/countly.views.js +++ b/frontend/express/public/core/date-presets/javascripts/countly.views.js @@ -1,4 +1,4 @@ -/*global Vue app, countlyVue, CV, countlyGlobal, groupsModel, _, CountlyHelpers, countlyPresets*/ +/*global Vue app, countlyVue, CV, countlyGlobal, groupsModel, _, CountlyHelpers, countlyPresets, countlyAuth*/ (function() { @@ -459,11 +459,13 @@ }); }; - app.route("/manage/date-presets", "date-presets", function() { - const PresetManagementView = getManagementView(); - this.renderWhenReady(PresetManagementView); - }); + if (countlyAuth.validateCreate('core')) { + app.route("/manage/date-presets", "date-presets", function() { + const PresetManagementView = getManagementView(); + this.renderWhenReady(PresetManagementView); + }); - app.addMenu("management", {code: "presets", permission: "core", url: "#/manage/date-presets", text: "sidebar.management.presets", priority: 30}); + app.addMenu("management", {code: "presets", permission: "core", url: "#/manage/date-presets", text: "sidebar.management.presets", priority: 30}); + } })(); \ No newline at end of file diff --git a/frontend/express/public/core/events/javascripts/countly.details.models.js b/frontend/express/public/core/events/javascripts/countly.details.models.js index 913f7f34c62..04a5fcbcf8b 100644 --- a/frontend/express/public/core/events/javascripts/countly.details.models.js +++ b/frontend/express/public/core/events/javascripts/countly.details.models.js @@ -895,6 +895,9 @@ return countlyAllEvents.service.fetchAllEventsData(context, period) .then(function(res) { if (res) { + if (Array.isArray(res.list)) { + res.list = res.list.map(eventName => countlyCommon.unescapeHtml(eventName)); + } context.commit("setAllEventsData", res); if ((!context.state.selectedEventName) || (res.map && res.map[context.state.selectedEventName] && !res.map[context.state.selectedEventName].is_visible) || (res.list && res.list.indexOf(context.state.selectedEventName) === -1)) { var appId = countlyCommon.ACTIVE_APP_ID; diff --git a/frontend/express/public/core/onboarding/javascripts/countly.views.js b/frontend/express/public/core/onboarding/javascripts/countly.views.js index 222b069a9d8..e8a147cd5b7 100644 --- a/frontend/express/public/core/onboarding/javascripts/countly.views.js +++ b/frontend/express/public/core/onboarding/javascripts/countly.views.js @@ -37,7 +37,7 @@ types: Object.keys(app.appTypes), appTemplates: appTemplates, populatorProgress: 0, - populatorMaxTime: 60, + populatorMaxTime: 30, isPopulatorFinished: false, isCountlyEE: countlyGlobal.plugins.includes('drill'), selectedAppTemplate: null, diff --git a/frontend/express/public/core/user-management/javascripts/countly.models.js b/frontend/express/public/core/user-management/javascripts/countly.models.js index 147224b3f7b..a87d85aff40 100644 --- a/frontend/express/public/core/user-management/javascripts/countly.models.js +++ b/frontend/express/public/core/user-management/javascripts/countly.models.js @@ -215,5 +215,21 @@ callback(err.responseJSON.result); }); }; + countlyUserManagement.disableTwoFactorAuth = function(id, callback) { + return $.ajax({ + type: "GET", + url: countlyGlobal.path + "/i/two-factor-auth", + data: { + method: "admin_disable", + uid: id + }, + success: function() { + callback(); + }, + error: function(err) { + callback(err.responseJSON.result); + } + }); + }; })((window.countlyUserManagement = window.countlyUserManagement || {})); diff --git a/frontend/express/public/core/user-management/javascripts/countly.views.js b/frontend/express/public/core/user-management/javascripts/countly.views.js index 011c875c7d2..6cb5584eb56 100644 --- a/frontend/express/public/core/user-management/javascripts/countly.views.js +++ b/frontend/express/public/core/user-management/javascripts/countly.views.js @@ -65,6 +65,7 @@ }, roleMap: roleMap, showLogs: countlyGlobal.plugins.indexOf('systemlogs') > -1, + twoFactorAuth: countlyGlobal.plugins.indexOf('two-factor-auth') > -1, tableDynamicCols: tableDynamicCols, userManagementPersistKey: 'userManagement_table_' + countlyCommon.ACTIVE_APP_ID, isGroupPluginEnabled: isGroupPluginEnabled @@ -114,6 +115,9 @@ } }, methods: { + is2faEnabled: function(row) { + return countlyGlobal.member.global_admin && this.twoFactorAuth && row.two_factor_auth && row.two_factor_auth.enabled; + }, handleCommand: function(command, index) { switch (command) { case "delete-user": @@ -157,6 +161,21 @@ }); }); break; + case 'disable-2fa': + countlyUserManagement.disableTwoFactorAuth(index, function(err) { + if (err) { + CountlyHelpers.notify({ + message: CV.i18n('two-factor-auth.faildisable_title'), + type: 'error' + }); + return; + } + CountlyHelpers.notify({ + message: CV.i18n('two-factor-auth.disable_title'), + type: 'success' + }); + }); + break; } }, handleSubmitFilter: function(newFilter) { diff --git a/frontend/express/public/core/user-management/templates/data-table.html b/frontend/express/public/core/user-management/templates/data-table.html index 571376f6630..bf8558ea974 100644 --- a/frontend/express/public/core/user-management/templates/data-table.html +++ b/frontend/express/public/core/user-management/templates/data-table.html @@ -75,6 +75,7 @@Loading...