diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f54991fa72..e431f69100a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - + - name: Github Actions Azure connection fix run: | # Workaround for https://github.com/actions/runner-images/issues/675#issuecomment-1381389712 @@ -31,27 +31,27 @@ jobs: - name: Installing Countly shell: bash run: sudo bash ./bin/countly.install.sh - + - name: NodeJS version shell: bash run: node --version - + - name: NPM version shell: bash run: npm --version - + - name: Mongo version shell: bash run: mongo --version - + - name: ShellCheck shell: bash run: countly shellcheck - + - name: ESLint shell: bash run: npx eslint . - + - name: DistFiles shell: bash run: sudo countly task dist-all @@ -63,11 +63,11 @@ jobs: - name: Output API Logs if: ${{ always() }} run: cat log/countly-api.log - + - name: Output Dashboard Logs if: ${{ always() }} run: cat log/countly-dashboard.log - + - name: Output MongoDB Logs if: ${{ always() }} run: sudo cat /var/log/mongodb/mongod.log @@ -75,7 +75,7 @@ jobs: - name: Output Nginx Logs if: ${{ always() }} run: sudo cat /var/log/nginx/error.log - + - name: Send slack notification id: slack if: always() diff --git a/bin/config/supervisord.example.conf b/bin/config/supervisord.example.conf index 28dd4571feb..a0b80b382fd 100644 --- a/bin/config/supervisord.example.conf +++ b/bin/config/supervisord.example.conf @@ -2,16 +2,16 @@ file=/tmp/supervisor.sock [supervisord] -logfile=/var/log/supervisord.log +logfile=%(here)s/../../log/supervisord/supervisord.log logfile_maxbytes=50MB logfile_backups=10 loglevel=warn -pidfile=/var/log/supervisord.pid +pidfile=%(here)s/../../log/supervisord/supervisord.pid nodaemon=false minfds=1024 minprocs=200 -user=root -childlogdir=/var/log/ +user=countly +childlogdir=%(here)s/../../log/supervisord/ [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface diff --git a/bin/config/supervisord.ldaps.conf b/bin/config/supervisord.ldaps.conf index d99c1a427cc..f58973e5df1 100644 --- a/bin/config/supervisord.ldaps.conf +++ b/bin/config/supervisord.ldaps.conf @@ -2,16 +2,16 @@ file=/tmp/supervisor.sock [supervisord] -logfile=/var/log/supervisord.log +logfile=%(here)s/../../log/supervisord/supervisord.log logfile_maxbytes=50MB logfile_backups=10 loglevel=warn -pidfile=/var/log/supervisord.pid +pidfile=%(here)s/../../log/supervisord/supervisord.pid nodaemon=false minfds=1024 minprocs=200 -user=root -childlogdir=/var/log/ +user=countly +childlogdir=%(here)s/../../log/supervisord/ [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface diff --git a/bin/countly.install.sh b/bin/countly.install.sh index c8c55c99a14..9be90fee282 100755 --- a/bin/countly.install.sh +++ b/bin/countly.install.sh @@ -14,6 +14,16 @@ fi sudo bash "$DIR/scripts/init_countly_user.sh" cd "$DIR/../" +sudo su countly -c "/bin/bash $DIR/scripts/check_countly_user_permissions.sh > /dev/null 2>&1" + +if [ ! -f ./permission_test_file.txt ]; then + PARENT_DIR=$(cd ./../ && pwd) + echo "Permission error, you cannot install Countly under ${PARENT_DIR}." + exit 1 +else + sudo rm -f ./permission_test_file.txt +fi + if [ "$totalm" -lt "1800" ]; then echo "Countly requires at least 2Gb of RAM" if [ "$COUNTLY_OVERWRITE_MEM_REQUIREMENT" != "1" ]; then diff --git a/bin/countly.install_rhel.sh b/bin/countly.install_rhel.sh index 351aae1dae8..923517254db 100755 --- a/bin/countly.install_rhel.sh +++ b/bin/countly.install_rhel.sh @@ -24,10 +24,10 @@ if grep -q -i "release 8" /etc/redhat-release ; then fi sudo yum -y install epel-release + # see https://github.com/koalaman/shellcheck/issues/1871 wget https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz sudo tar -C /usr/local/bin/ -xf shellcheck-v0.7.1.linux.x86_64.tar.xz --no-anchored 'shellcheck' --strip=1 - sudo ln -sf /usr/local/bin/shellcheck /usr/bin/shellcheck if [ ! -x "$(command -v python)" ]; then @@ -52,11 +52,11 @@ enabled=1" | sudo tee /etc/yum.repos.d/nginx.repo >/dev/null sudo yum install devtoolset-8 -y sudo yum install devtoolset-8-gcc* -y + #shellcheck source=/dev/null source /opt/rh/devtoolset-8/enable && echo -e "\nsource /opt/rh/devtoolset-8/enable" | sudo tee -a /etc/profile sudo yum install -y epel-release sudo yum install -y ShellCheck - else echo "Unsupported OS version, only support RHEL/Centos 8 and 7" exit 1 @@ -111,6 +111,7 @@ sudo yum install numactl -y #install mongodb sudo bash "$DIR/scripts/mongodb.install.sh" + if [ "$INSIDE_DOCKER" == "1" ]; then sudo sed -i 's/ fork/# fork/g' /etc/mongod.conf sudo mongod -f /etc/mongod.conf & diff --git a/bin/countly.install_ubuntu.sh b/bin/countly.install_ubuntu.sh index d766a5b284c..044713a9c44 100644 --- a/bin/countly.install_ubuntu.sh +++ b/bin/countly.install_ubuntu.sh @@ -2,6 +2,8 @@ set -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + UBUNTU_YEAR="$(lsb_release -sr | cut -d '.' -f 1)"; if [[ "$UBUNTU_YEAR" != "18" && "$UBUNTU_YEAR" != "20" && "$UBUNTU_YEAR" != "22" ]]; then @@ -9,17 +11,16 @@ if [[ "$UBUNTU_YEAR" != "18" && "$UBUNTU_YEAR" != "20" && "$UBUNTU_YEAR" != "22" exit 1 fi -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - bash "$DIR/scripts/logo.sh"; #update package index sudo apt-get update -sudo apt-get -y install wget build-essential libkrb5-dev git sqlite3 unzip bzip2 shellcheck +sudo apt-get -y install wget build-essential libkrb5-dev git sqlite3 unzip bzip2 shellcheck curl gnupg2 ca-certificates lsb-release if [[ "$UBUNTU_YEAR" = "22" ]]; then sudo apt-get -y install python2 python2-dev + if [[ ! -h /usr/bin/python ]]; then sudo ln -s /usr/bin/python2.7 /usr/bin/python sudo ln -s /usr/bin/python2-config /usr/bin/python-config @@ -30,11 +31,10 @@ fi #Install GCC && G++> 7 version sudo apt-get -y install software-properties-common -sudo apt-get -y install build-essential -sudo apt-get install gcc g++ make +sudo apt-get -y install gcc g++ make #Install dependancies required by the puppeteer -sudo apt-get -y install libgbm-dev libgbm1 gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget +sudo apt-get -y install libgbm-dev libgbm1 gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils if sudo apt-cache pkgnames | grep -q python-software-properties; then sudo apt-get -y install python-software-properties @@ -46,15 +46,7 @@ if ! (command -v apt-add-repository >/dev/null) then sudo apt-get -y install software-properties-common fi -#add node.js repo -#echo | apt-add-repository ppa:chris-lea/node.js -wget -qO- https://deb.nodesource.com/setup_14.x | sudo -E bash - - -#update once more after adding new repos -sudo apt-get update - #install nginx -sudo apt-get -y install curl gnupg2 ca-certificates lsb-release echo "deb http://nginx.org/packages/ubuntu $(lsb_release -cs) nginx" \ | sudo tee /etc/apt/sources.list.d/nginx.list curl -fsSL https://nginx.org/keys/nginx_signing.key | sudo apt-key add - @@ -62,13 +54,17 @@ sudo apt-get update sudo apt-get -y install nginx #install node.js +#add node.js repo +#echo | apt-add-repository ppa:chris-lea/node.js +wget -qO- https://deb.nodesource.com/setup_14.x | sudo -E bash - +sudo apt-get update sudo apt-get -y install nodejs || (echo "Failed to install nodejs." ; exit) set +e NODE_JS_CMD=$(which nodejs) set -e if [[ -z "$NODE_JS_CMD" ]]; then - sudo ln -s "$(which node)" /usr/bin/nodejs + sudo ln -s "$(which node)" /usr/bin/nodejs fi #if npm is not installed, install it too @@ -77,8 +73,7 @@ if ! (command -v npm >/dev/null) then fi #install supervisor -if [ "$INSIDE_DOCKER" != "1" ] -then +if [ "$INSIDE_DOCKER" != "1" ]; then sudo apt-get -y install supervisor || (echo "Failed to install supervisor." ; exit) cp "$DIR/config/supervisord.example.conf" "$DIR/config/supervisord.conf" fi @@ -96,15 +91,15 @@ npm config set prefix "$DIR/../.local/" #install mongodb sudo bash "$DIR/scripts/mongodb.install.sh" -if [ "$INSIDE_DOCKER" == "1" ] -then - bash "$DIR/commands/docker/mongodb.sh" & - until mongo --eval "db.stats()" | grep "collections" - do +if [ "$INSIDE_DOCKER" == "1" ]; then + bash "$DIR/commands/docker/mongodb.sh" & + + until mongo --eval "db.stats()" | grep "collections"; do echo echo "waiting for MongoDB to allocate files..." sleep 1 done + sleep 3 fi @@ -116,8 +111,7 @@ countly save /etc/nginx/nginx.conf "$DIR/config/nginx" sudo cp "$DIR/config/nginx.server.conf" /etc/nginx/conf.d/default.conf sudo cp "$DIR/config/nginx.conf" /etc/nginx/nginx.conf -if [ "$INSIDE_DOCKER" != "1" ] -then +if [ "$INSIDE_DOCKER" != "1" ]; then sudo /etc/init.d/nginx restart fi @@ -130,7 +124,7 @@ cp -n "$DIR/../api/config.sample.js" "$DIR/../api/config.js" cp -n "$DIR/../frontend/express/config.sample.js" "$DIR/../frontend/express/config.js" if [ ! -f "$DIR/../plugins/plugins.json" ]; then - cp "$DIR/../plugins/plugins.default.json" "$DIR/../plugins/plugins.json" + cp "$DIR/../plugins/plugins.default.json" "$DIR/../plugins/plugins.json" fi if [ ! -f "/etc/timezone" ]; then @@ -163,14 +157,12 @@ sudo countly task dist-all sudo countly check after install #finally start countly api and dashboard -if [ "$INSIDE_DOCKER" != "1" ] -then +if [ "$INSIDE_DOCKER" != "1" ]; then sudo countly start fi -bash "$DIR/scripts/done.sh"; - -if [ "$INSIDE_DOCKER" == "1" ] -then +if [ "$INSIDE_DOCKER" == "1" ]; then kill -2 "$(pgrep mongo)" -fi \ No newline at end of file +fi + +bash "$DIR/scripts/done.sh"; \ No newline at end of file diff --git a/bin/installer.sh b/bin/installer.sh index 83805fbfc2a..a3f96deb5c5 100755 --- a/bin/installer.sh +++ b/bin/installer.sh @@ -46,12 +46,4 @@ else fi fi -DATE=$(date +%Y-%m-%d:%H:%M:%S) -if [[ ! -z "$APT_GET_CMD" ]]; then - bash countly/bin/countly.install.sh 2>&1 | tee "countly/log/countly-install-$DATE.log" -elif [[ ! -z "$YUM_CMD" ]]; then - bash countly/bin/countly.install_rhel.sh 2>&1 | tee "countly/log/countly-install-$DATE.log" -else - echo "error can't install Countly" - exit 1; -fi \ No newline at end of file +bash countly/bin/countly.install.sh \ No newline at end of file diff --git a/bin/scripts/check_countly_user_permissions.sh b/bin/scripts/check_countly_user_permissions.sh new file mode 100644 index 00000000000..1e2ce3bb4b3 --- /dev/null +++ b/bin/scripts/check_countly_user_permissions.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +touch permission_test_file.txt \ No newline at end of file diff --git a/bin/scripts/init_countly_user.sh b/bin/scripts/init_countly_user.sh index 82d701ea387..799923693dd 100644 --- a/bin/scripts/init_countly_user.sh +++ b/bin/scripts/init_countly_user.sh @@ -3,6 +3,7 @@ #get current directory DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" COUNTLY_DIR="$( cd "$DIR"/../../ && pwd )" + #check if user not created yet if [ "$(getent passwd countly)x" == 'x' ]; then #create countly user diff --git a/bin/scripts/mongodb.install.sh b/bin/scripts/mongodb.install.sh index a81e7c57bfe..fc23bb67106 100644 --- a/bin/scripts/mongodb.install.sh +++ b/bin/scripts/mongodb.install.sh @@ -358,76 +358,57 @@ function mongodb_check() { } if [ $# -eq 0 ]; then + #install latest mongodb if [ -f /etc/redhat-release ]; then - #install latest mongodb + CENTOS_RELEASE="$(rpm --eval '%{centos_ver}')" - #select source based on release - if grep -q -i "release 6" /etc/redhat-release ; then - echo "[mongodb-org-4.4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/6/mongodb-org/4.4/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc" > /etc/yum.repos.d/mongodb-org-4.4.repo - elif grep -q -i "release 7" /etc/redhat-release ; then - echo "[mongodb-org-4.4] -name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/4.4/x86_64/ -gpgcheck=1 -enabled=1 -gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc" > /etc/yum.repos.d/mongodb-org-4.4.repo - elif grep -q -i "release 8" /etc/redhat-release ; then - echo "[mongodb-org-4.4] + if [[ "$CENTOS_RELEASE" != "7" && "$CENTOS_RELEASE" != "8" ]]; then + echo "Unsupported OS version, only support CentOS/RHEL 8 and 7" + exit 1 + fi + + echo "[mongodb-org-4.4] name=MongoDB Repository -baseurl=https://repo.mongodb.org/yum/redhat/8/mongodb-org/4.4/x86_64/ +baseurl=https://repo.mongodb.org/yum/redhat/${CENTOS_RELEASE}/mongodb-org/4.4/x86_64/ gpgcheck=1 enabled=1 gpgkey=https://www.mongodb.org/static/pgp/server-4.4.asc" > /etc/yum.repos.d/mongodb-org-4.4.repo - fi + yum install -y mongodb-org fi if [ -f /etc/lsb-release ]; then - #install latest mongodb - wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add - UBUNTU_YEAR="$(lsb_release -sr | cut -d '.' -f 1)"; - if [ "$UBUNTU_YEAR" == "16" ]; then - echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list ; - elif [ "$UBUNTU_YEAR" == "18" ]; then - echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list ; - elif [ "$UBUNTU_YEAR" == "22" ]; then + if [[ "$UBUNTU_YEAR" != "18" && "$UBUNTU_YEAR" != "20" && "$UBUNTU_YEAR" != "22" ]]; then + echo "Unsupported OS version, only support Ubuntu 22, 20 and 18" + exit 1 + fi + + #mongodb 4.4 is not supported officially on Ubuntu 22 + if [[ "$UBUNTU_YEAR" == "22" ]]; then + #we can install binaries of Ubuntu 20 if we pre-install required libssl wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb ; dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb ; rm -rf libssl1.1_1.1.1f-1ubuntu2_amd64.deb - echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list ; + + UBUNTU_RELEASE="focal" else - echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list ; + UBUNTU_RELEASE="$(lsb_release -cs)" fi + + wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add - + echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu ${UBUNTU_RELEASE}/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.4.list ; + apt-get update - #install mongodb DEBIAN_FRONTEND="noninteractive" apt-get -y install mongodb-org || (echo "Failed to install mongodb." ; exit) fi #backup config and remove configuration to prevent duplicates mongodb_configure - if [ -f /etc/redhat-release ]; then - #mongodb might need to be started - if grep -q -i "release 6" /etc/redhat-release ; then - service mongod restart > /dev/null || echo "mongodb service does not exist" - else - systemctl restart mongod > /dev/null || echo "mongodb systemctl job does not exist" - fi - fi - - if [ -f /etc/lsb-release ]; then - if [[ "$(/sbin/init --version)" =~ upstart ]]; then - restart mongod > /dev/null || echo "mongodb upstart job does not exist" - else - systemctl restart mongod || echo "mongodb systemctl job does not exist" - fi 2> /dev/null - fi + #mongodb might need to be restarted + systemctl restart mongod > /dev/null || echo "mongodb systemctl job does not exist" mongodb_check elif [ "$1" == "check" ]; then diff --git a/log/supervisord/supervisord.log b/log/supervisord/supervisord.log new file mode 100644 index 00000000000..e69de29bb2d