Skip to content

Commit 5c68c2c

Browse files
CookiezaursCookiezaurs
authored
Merge branch 'next' into configurations-issues
2 parents acb5111 + b859b6a commit 5c68c2c

File tree

24 files changed

+788
-595
lines changed

24 files changed

+788
-595
lines changed

‎.github/workflows/main.yml

+19-12
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ on:
1010

1111
# Allows you to run this workflow manually from the Actions tab
1212
workflow_dispatch:
13+
inputs:
14+
custom_tag:
15+
description: 'Custom Docker tag (optional)'
16+
required: false
17+
default: ''
1318

1419
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
1520
jobs:
@@ -64,7 +69,7 @@ jobs:
6469

6570
services:
6671
mongodb:
67-
image: mongo:7.0
72+
image: mongo:8.0
6873
options: >-
6974
--health-cmd mongosh
7075
--health-interval 10s
@@ -74,7 +79,7 @@ jobs:
7479
- 27017:27017
7580

7681
container:
77-
image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
82+
image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
7883
env:
7984
COUNTLY_CONFIG__MONGODB_HOST: mongodb
8085
COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -121,7 +126,7 @@ jobs:
121126

122127
services:
123128
mongodb:
124-
image: mongo:7.0
129+
image: mongo:8.0
125130
options: >-
126131
--health-cmd mongosh
127132
--health-interval 10s
@@ -131,7 +136,7 @@ jobs:
131136
- 27017:27017
132137

133138
container:
134-
image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
139+
image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
135140
env:
136141
COUNTLY_CONFIG__MONGODB_HOST: mongodb
137142
COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -173,7 +178,7 @@ jobs:
173178

174179
services:
175180
mongodb:
176-
image: mongo:7.0
181+
image: mongo:8.0
177182
options: >-
178183
--health-cmd mongosh
179184
--health-interval 10s
@@ -183,7 +188,7 @@ jobs:
183188
- 27017:27017
184189

185190
container:
186-
image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
191+
image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
187192
env:
188193
COUNTLY_CONFIG__MONGODB_HOST: mongodb
189194
COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -231,7 +236,7 @@ jobs:
231236

232237
services:
233238
mongodb:
234-
image: mongo:7.0
239+
image: mongo:8.0
235240
options: >-
236241
--health-cmd mongosh
237242
--health-interval 10s
@@ -241,7 +246,7 @@ jobs:
241246
- 27017:27017
242247

243248
container:
244-
image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
249+
image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
245250
env:
246251
COUNTLY_CONFIG__MONGODB_HOST: mongodb
247252
COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -296,7 +301,8 @@ jobs:
296301
/sbin/my_init &
297302
cd ui-tests
298303
npm install
299-
npm run cy:run:dashboard
304+
xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
305+
npm run cy:run:dashboard --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
300306
301307
- name: Upload UI tests artifacts
302308
if: ${{ failure() }}
@@ -313,7 +319,7 @@ jobs:
313319

314320
services:
315321
mongodb:
316-
image: mongo:6.0
322+
image: mongo:8.0
317323
options: >-
318324
--health-cmd mongosh
319325
--health-interval 10s
@@ -323,7 +329,7 @@ jobs:
323329
- 27017:27017
324330

325331
container:
326-
image: countly/countly-core:pipelines-${{ github.base_ref || github.ref_name }}
332+
image: countly/countly-core:pipelines-${{ inputs.custom_tag || github.base_ref || github.ref_name }}
327333
env:
328334
COUNTLY_CONFIG__MONGODB_HOST: mongodb
329335
COUNTLY_CONFIG_API_PREVENT_JOBS: true
@@ -374,7 +380,8 @@ jobs:
374380
/sbin/my_init &
375381
cd ui-tests
376382
npm install
377-
npm run cy:run:onboarding
383+
xvfb-run --auto-servernum --server-args="-screen 0 1280x1024x24" \
384+
npm run cy:run:onboarding --headless --no-sandbox --disable-gpu --disable-dev-shm-usage
378385
379386
- name: Upload UI tests artifacts
380387
if: ${{ failure() }}

‎api/parts/mgmt/app_users.js

+62-46
Original file line numberDiff line numberDiff line change
@@ -764,6 +764,7 @@ usersApi.merge = function(app_id, newAppUser, new_id, old_id, new_device_id, old
764764
var deleteMyExport = function(exportID) { //tries to delete packed file, exported folder and saved export in gridfs
765765
//normally there should be only export in gridfs. Cleaning all to be sure.
766766
//rejects only if there stays any saved data for export
767+
log.d("deleteMyExport:" + exportID);
767768
return new Promise(function(resolve, reject) {
768769
//remove archive
769770
var errors = [];
@@ -775,7 +776,6 @@ var deleteMyExport = function(exportID) { //tries to delete packed file, exporte
775776
errors.push(err);
776777
}
777778
}
778-
779779
countlyFs.gridfs.deleteFile("appUsers", path.resolve(__dirname, './../../../export/AppUser/' + exportID + '.tar.gz'), {id: exportID + '.tar.gz'}, function(error) {
780780
if (error && error.message && error.message.substring(0, 12) !== "FileNotFound" && error.message.substring(0, 14) !== 'File not found') {
781781
log.e(error.message.substring(0, 14));
@@ -806,6 +806,18 @@ var deleteMyExport = function(exportID) { //tries to delete packed file, exporte
806806
}
807807
});
808808

809+
810+
}).then(function() {
811+
return new Promise(function(resolve, reject) {
812+
common.db.collection("exports").remove({"_eid": exportID}, function(err0) {
813+
if (err0) {
814+
reject(err0);
815+
}
816+
else {
817+
resolve();
818+
}
819+
});
820+
});
809821
});
810822
};
811823

@@ -822,46 +834,42 @@ usersApi.deleteExport = function(filename, params, callback) {
822834
//remove archive
823835
deleteMyExport(base_name[0]).then(
824836
function() {
825-
common.db.collection("exports").remove({"_eid": base_name[0]}, function(err0) {
826-
if (err0) {
827-
log.e(err0);
828-
}
829-
if (name_parts.length === 3 && name_parts[2] !== 'HASH') {
830-
//update user info
831-
common.db.collection('app_users' + name_parts[1]).update({"uid": name_parts[2]}, {$unset: {"appUserExport": ""}}, {upsert: false, multi: true}, function(err) {
832-
if (err) {
833-
callback(err, "");
834-
}
835-
else {
836-
plugins.dispatch("/systemlogs", {
837-
params: params,
838-
action: "export_app_user_deleted",
839-
data: {
840-
result: "ok",
841-
uids: name_parts[2],
842-
id: base_name[0],
843-
app_id: name_parts[1],
844-
info: "Exported data deleted"
845-
}
846-
});
847-
callback(null, "Export deleted");
848-
}
849-
});
850-
}
851-
else {
852-
plugins.dispatch("/systemlogs", {
853-
params: params,
854-
action: "export_app_user_deleted",
855-
data: {
856-
result: "ok",
857-
id: base_name[0],
858-
app_id: name_parts[1],
859-
info: "Exported data deleted"
860-
}
861-
});
862-
callback(null, "Export deleted");
863-
}
864-
});
837+
if (name_parts.length === 3 && name_parts[2] !== 'HASH') {
838+
//update user info
839+
common.db.collection('app_users' + name_parts[1]).update({"uid": name_parts[2]}, {$unset: {"appUserExport": ""}}, {upsert: false, multi: true}, function(err) {
840+
if (err) {
841+
callback(err, "");
842+
}
843+
else {
844+
plugins.dispatch("/systemlogs", {
845+
params: params,
846+
action: "export_app_user_deleted",
847+
data: {
848+
result: "ok",
849+
uids: name_parts[2],
850+
id: base_name[0],
851+
app_id: name_parts[1],
852+
info: "Exported data deleted"
853+
}
854+
});
855+
callback(null, "Export deleted");
856+
}
857+
});
858+
}
859+
else {
860+
plugins.dispatch("/systemlogs", {
861+
params: params,
862+
action: "export_app_user_deleted",
863+
data: {
864+
result: "ok",
865+
id: base_name[0],
866+
app_id: name_parts[1],
867+
info: "Exported data deleted"
868+
}
869+
});
870+
callback(null, "Export deleted");
871+
}
872+
865873
},
866874
function(err) {
867875
console.log(err);
@@ -1073,12 +1081,20 @@ usersApi.export = function(app_id, query, params, callback) {
10731081
// else {
10741082
// resolve();
10751083
// }
1076-
new Promise(function(resolve) {
1077-
log.d("collection marked");
1078-
//export data from metric_changes
10791084

1080-
export_safely({projection: {"appUserExport": 0}, export_id: export_id, app_id: app_id, args: [...dbargs, "--collection", "metric_changes" + app_id, "-q", '{"uid":{"$in": ["' + res[0].uid.join('","') + '"]}}', "--out", export_folder + "/metric_changes" + app_id + ".json"]}).finally(function() {
1081-
resolve();
1085+
//try deleting old export
1086+
deleteMyExport(export_id).then(function(err) {
1087+
if (err) {
1088+
log.e(err);
1089+
}
1090+
log.d("old export deleted");
1091+
return new Promise(function(resolve) {
1092+
log.d("collection marked");
1093+
//export data from metric_changes
1094+
1095+
export_safely({projection: {"appUserExport": 0}, export_id: export_id, app_id: app_id, args: [...dbargs, "--collection", "metric_changes" + app_id, "-q", '{"uid":{"$in": ["' + res[0].uid.join('","') + '"]}}', "--out", export_folder + "/metric_changes" + app_id + ".json"]}).finally(function() {
1096+
resolve();
1097+
});
10821098
});
10831099
}).then(function() {
10841100
log.d("metric_changes exported");

‎api/utils/countlyFs.js

+10-4
Original file line numberDiff line numberDiff line change
@@ -601,13 +601,19 @@ countlyFs.gridfs = {};
601601
* console.log("Finished", err);
602602
* });
603603
*/
604-
ob.deleteFileById = function(category, id, callback) {
604+
ob.deleteFileById = async function(category, id, callback) {
605605
var bucket = new GridFSBucket(db, { bucketName: category });
606-
bucket.delete(id, function(error) {
606+
try {
607+
await bucket.delete(id);
607608
if (callback) {
608-
callback(error);
609+
callback(null);
609610
}
610-
});
611+
}
612+
catch (ee) {
613+
if (callback) {
614+
callback(ee);
615+
}
616+
}
611617
};
612618

613619
/**

‎bin/scripts/data-reports/check_comparison_report.js

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
//Script to look for meaningful differences in drill vs aggregated data.
2+
var data = {"601ba83d99111111d9": {"name": "My Good test APP", "total": 25, "bad": 0}, "6026972d182f7e014cd82114": {"name": "My BAD APP", "total": 100, "bad": 2, "events": {"BAD EVENT": {"e": "BAD EVENT", "report": {"totals": {"c": 1000}, "data": {"2024.10.2": {"c": 1000, "s": 0, "dur": -9.094947017729282e-13}}}}}}}; //output of compare_drill_aggregated.js script
3+
4+
//Should be adjusted based on data amount.
5+
var total_treshold = 100; //If modulus daily value is bigger than this, it will be autputted
6+
var daily_treshold = 50; //If for any day any value is biiger that this - it will be outputted.
7+
8+
for (var appid in data) {
9+
if (data[appid].bad > 0) {
10+
for (var event in data[appid].events) {
11+
var is_any_total_bad = false;
12+
for (var m in data[appid].events[event].report.totals) {
13+
if (data[appid].events[event].report.totals[m] > total_treshold || data[appid].events[event].report.totals[m] < -1 * total_treshold) {
14+
is_any_total_bad = true;
15+
}
16+
}
17+
if (is_any_total_bad) {
18+
console.log("total difference in APP:" + appid + " for event " + event + " " + JSON.stringify(data[appid].events[event].report.totals));
19+
20+
for (var date in data[appid].events[event].report.data) {
21+
var is_any_bad = false;
22+
for (var m2 in data[appid].events[event].report.data[date]) {
23+
if (data[appid].events[event].report.data[date][m2] > daily_treshold || data[appid].events[event].report.data[date][m2] < daily_treshold * -1) {
24+
is_any_bad = true;
25+
}
26+
}
27+
if (is_any_bad) {
28+
console.log(" " + date + " : " + JSON.stringify(data[appid].events[event].report.data[date]));
29+
}
30+
}
31+
32+
}
33+
}
34+
}
35+
}

‎bin/scripts/mongodb.install.sh

+6-6
Original file line numberDiff line numberDiff line change
@@ -341,12 +341,12 @@ if [ $# -eq 0 ]; then
341341
exit 1
342342
fi
343343

344-
echo "[mongodb-org-7.0]
344+
echo "[mongodb-org-8.0]
345345
name=MongoDB Repository
346-
baseurl=https://repo.mongodb.org/yum/redhat/${CENTOS_MAJOR}/mongodb-org/7.0/x86_64/
346+
baseurl=https://repo.mongodb.org/yum/redhat/${CENTOS_MAJOR}/mongodb-org/8.0/x86_64/
347347
gpgcheck=1
348348
enabled=1
349-
gpgkey=https://www.mongodb.org/static/pgp/server-7.0.asc" > /etc/yum.repos.d/mongodb-org-7.0.repo
349+
gpgkey=https://pgp.mongodb.com/server-8.0.asc" > /etc/yum.repos.d/mongodb-org-8.0.repo
350350

351351
yum install -y mongodb-org
352352
elif [ -f /etc/lsb-release ]; then
@@ -359,12 +359,12 @@ gpgkey=https://www.mongodb.org/static/pgp/server-7.0.asc" > /etc/yum.repos.d/mon
359359
exit 1
360360
fi
361361

362-
wget -qO - https://www.mongodb.org/static/pgp/server-7.0.asc | sudo apt-key add -
363-
echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu ${UBUNTU_RELEASE}/mongodb-org/7.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-7.0.list ;
362+
curl -fsSL https://www.mongodb.org/static/pgp/server-8.0.asc | sudo gpg -o /usr/share/keyrings/mongodb-server-8.0.gpg --dearmor
363+
echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] https://repo.mongodb.org/apt/ubuntu ${UBUNTU_RELEASE}/mongodb-org/8.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-8.0.list
364364
apt-get update
365365
DEBIAN_FRONTEND="noninteractive" apt-get install -y mongodb-org || (echo "Failed to install mongodb." ; exit)
366366
else
367-
echo "Unsupported OS or version, only CentOS/RHEL 8 or 9 and Ubuntu 20 or 22."
367+
echo "Unsupported OS or version, only CentOS/RHEL 8 or 9 and Ubuntu 20 or 22 or 24."
368368
exit 1
369369
fi
370370

0 commit comments

Comments
 (0)