Skip to content

Commit db39334

Browse files
committed
Merge branch 'master' into next
# Conflicts: # package-lock.json # package.json # plugins/star-rating/frontend/public/templates/comments-table.html
2 parents da7f9b7 + 9d1fe3c commit db39334

File tree

35 files changed

+593
-125
lines changed

35 files changed

+593
-125
lines changed

CHANGELOG.md

+15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,18 @@
1+
## Version 23.06.15
2+
Fixes:
3+
- [crashes] Add config for activating custom field cleanup job
4+
- [data-migration] Fix for upload request
5+
- [core] Remove sensitive fields from API responses
6+
- [dashboards] Validating links for note widgets
7+
- [star-rating] Css changes for ratings comments table
8+
- [star-rating] Fix for targeting reset on toggle
9+
- [members] Fix full Name updates in db
10+
11+
Enterprise fixes:
12+
- [crash_symbolication] Fix for symbol file upload
13+
- [data-manager] Fix for disabled input in view transformations
14+
- [data-manager] Fix for duplicate events being created in event transformation
15+
116
## Version 23.06.14
217
Fixes:
318
- [cache] Fixing initialization race conditions (group store is not initialized)

Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,4 @@ RUN apt-get update && apt-get install -y sudo && \
5353
apt-get install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils && \
5454
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
5555
rm -rf /tmp/* /tmp/.??* /var/tmp/* /var/tmp/.??* /root/.npm && \
56-
mkdir /opt/countly/.npm && chown -R 1001:0 /opt/countly/.npm
56+
mkdir -p /opt/countly/.npm && chown -R 1001:0 /opt/countly/.npm

api/parts/data/cache.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -971,7 +971,7 @@ class StreamedCollection {
971971
let [col, last] = await createCollection(this.db, this.name, 1e7);
972972

973973
this.col = col;
974-
this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, noCursorTimeout: true, numberOfRetries: -1}).stream();
974+
this.stream = col.find({_id: {$gt: last}}, {tailable: true, awaitData: true, numberOfRetries: -1}).stream();
975975

976976
this.stream.on('data', doc => {
977977
if (this.inserts.indexOf(doc._id.toString()) !== -1) {

api/parts/data/usage.js

+14-4
Original file line numberDiff line numberDiff line change
@@ -195,11 +195,15 @@ usage.processSessionDuration = function(params, callback) {
195195
var dbDateIds = common.getDateIds(params);
196196

197197
common.writeBatcher.add("users", params.app_id + "_" + dbDateIds.month + "_" + postfix, {'$inc': updateUsers});
198+
params.qstring.session_duration = session_duration;
199+
200+
if (!params.qstring.begin_session) {
201+
plugins.dispatch("/session/duration", {
202+
params: params,
203+
session_duration: session_duration
204+
});
205+
}
198206

199-
plugins.dispatch("/session/duration", {
200-
params: params,
201-
session_duration: session_duration
202-
});
203207
if (callback) {
204208
callback();
205209
}
@@ -1141,6 +1145,12 @@ plugins.register("/sdk/user_properties", async function(ob) {
11411145
dbAppUser: params.app_user,
11421146
updates: ob.updates
11431147
});
1148+
if (params.qstring.session_duration) {
1149+
plugins.dispatch("/session/duration", {
1150+
params: params,
1151+
session_duration: params.qstring.session_duration
1152+
});
1153+
}
11441154
}
11451155
else {
11461156
userProps.lsid = params.request_id;

api/parts/mgmt/users.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ usersApi.getUserById = function(params) {
4242
}
4343
common.db.collection('members').findOne({ _id: common.db.ObjectID(params.qstring.id) }, {
4444
password: 0,
45-
appSortList: 0
45+
appSortList: 0,
46+
api_key: 0
4647
}, function(err, member) {
4748

4849
if (!member || err) {
@@ -72,7 +73,8 @@ usersApi.getUserById = function(params) {
7273
usersApi.getAllUsers = function(params) {
7374
common.db.collection('members').find({}, {
7475
password: 0,
75-
appSortList: 0
76+
appSortList: 0,
77+
api_key: 0,
7678
}).toArray(function(err, members) {
7779

7880
if (!members || err) {

api/utils/common.js

-1
Original file line numberDiff line numberDiff line change
@@ -3174,7 +3174,6 @@ common.sanitizeHTML = (html) => {
31743174
let attributeName = matches[1];
31753175
let attributeValue = matches[2];
31763176
if (allowedAttributes.indexOf(attributeName) > -1) {
3177-
31783177
var attributeValueStart = fullAttribute.indexOf(attributeValue);
31793178
if (attributeValueStart >= 1) {
31803179
var attributeWithQuote = fullAttribute.substring(attributeValueStart - 1);

api/utils/requestProcessor.js

+8-6
Original file line numberDiff line numberDiff line change
@@ -1946,20 +1946,22 @@ const processRequest = (params) => {
19461946
common.returnMessage(params, 400, 'Missing parameter "collection"');
19471947
return false;
19481948
}
1949-
if (typeof params.qstring.query === "string") {
1949+
if (typeof params.qstring.filter === "string") {
19501950
try {
1951-
params.qstring.query = JSON.parse(params.qstring.query, common.reviver);
1951+
params.qstring.query = JSON.parse(params.qstring.filter, common.reviver);
19521952
}
19531953
catch (ex) {
1954-
params.qstring.query = null;
1954+
common.returnMessage(params, 400, "Failed to parse query. " + ex.message);
1955+
return false;
19551956
}
19561957
}
1957-
if (typeof params.qstring.filter === "string") {
1958+
else if (typeof params.qstring.query === "string") {
19581959
try {
1959-
params.qstring.query = JSON.parse(params.qstring.filter, common.reviver);
1960+
params.qstring.query = JSON.parse(params.qstring.query, common.reviver);
19601961
}
19611962
catch (ex) {
1962-
params.qstring.query = null;
1963+
common.returnMessage(params, 400, "Failed to parse query. " + ex.message);
1964+
return false;
19631965
}
19641966
}
19651967
if (typeof params.qstring.projection === "string") {

api/utils/taskmanager.js

+73-1
Original file line numberDiff line numberDiff line change
@@ -741,8 +741,80 @@ taskmanager.getTableQueryResult = async function(options, callback) {
741741
}
742742
const count = await options.db.collection("long_tasks").count(options.query);
743743
return options.db.collection("long_tasks").find(options.query, options.projection).sort(sortBy).skip(skip).limit(limit).toArray((err, list) => {
744+
//check if there are any reports connected to widgets and change view links to correct dashboards if there are any.
744745
if (!err) {
745-
callback(null, {list, count});
746+
var ids_drill = [];
747+
var ids_formulas = [];
748+
for (var z = 0; z < list.length; z++) {
749+
if (list[z].linked_to) {
750+
list[z].dashboard_report = true;
751+
if (list[z].linked_to._issuer === "wqm:drill") {
752+
ids_drill.push(list[z]._id);
753+
}
754+
755+
if (list[z].linked_to._issuer === "wqm:formulas") {
756+
ids_formulas.push(list[z]._id);
757+
}
758+
}
759+
}
760+
if (ids_drill.length > 0 || ids_formulas.length > 0) {
761+
var query = {};
762+
if (ids_formulas.length > 0 && ids_drill.length > 0) {
763+
query.$or = [{"cmetrics": {$in: ids_formulas}}, {"drill_report": {$in: ids_drill}}];
764+
}
765+
else if (ids_formulas.length > 0) {
766+
query.cmetrics = {$in: ids_formulas};
767+
}
768+
else if (ids_drill.length > 0) {
769+
query.drill_report = {$in: ids_drill};
770+
}
771+
common.db.collection("widgets").aggregate([
772+
{"$match": query},
773+
{"$project": {"wid": {"$toString": "$_id"}, "drill_report": "$drill_report", "cmetrics": "$cmetrics"}},
774+
{"$unionWith": {"coll": "dashboards", "pipeline": [{"$project": {"did": "$_id", "wid": "$widgets"}}, {"$unwind": "$wid"}, {"$project": {"did": "$did", "wid": {"$toString": "$wid"}}}]}},
775+
{"$group": {"_id": "$wid", "did": {"$push": "$did"}, "drill_report": {"$push": "$drill_report"}, "cmetrics": {"$push": "$cmetrics"}}}
776+
], function(err1, res) {
777+
if (err1) {
778+
log.e(err1);
779+
}
780+
if (res) {
781+
var map_report = {};
782+
for (var k = 0; k < res.length; k++) {
783+
if (Array.isArray(res[k].did)) {
784+
res[k].did = res[k].did[0];
785+
}
786+
787+
if (Array.isArray(res[k].drill_report)) {
788+
res[k].drill_report = res[k].drill_report[0];
789+
}
790+
791+
if (Array.isArray(res[k].cmetrics)) {
792+
res[k].cmetrics = res[k].cmetrics[0];
793+
}
794+
if (res[k].drill_report && res[k].drill_report.length > 0) {
795+
for (var p = 0; p < res[k].drill_report.length; p++) {
796+
map_report[res[k].drill_report[p]] = res[k].did;
797+
}
798+
}
799+
if (res[k].cmetrics && res[k].cmetrics.length > 0) {
800+
for (var p2 = 0; p2 < res[k].cmetrics.length; p2++) {
801+
map_report[res[k].cmetrics[p2]] = res[k].did;
802+
}
803+
}
804+
}
805+
for (var kk = 0; kk < list.length; kk++) {
806+
if (map_report[list[kk]._id]) {
807+
list[kk].view = "#/custom/" + map_report[list[kk]._id];
808+
list[kk].have_dashboard_widget = true;
809+
}
810+
}
811+
}
812+
callback(null, {list, count});
813+
});
814+
}
815+
else {
816+
callback(null, {list, count});
817+
}
746818
}
747819
else {
748820
callback(err, {});

bin/scripts/data-reports/datapoints_based_on_drill.js

+12-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,18 @@ var startDate = new Date("2021-03-01T00:00:00");
1111
var endDate = new Date("2021-04-01T00:00:00");
1212

1313
var plugins = require("../../../plugins/pluginManager");
14-
var map = require("./db_mapping.json");
14+
var maps = require("./db_mapping.json");
15+
var map;
16+
for (let i = 0; i < maps.length; i++) {
17+
if (maps[i].name === "countly_drill") {
18+
map = maps[i];
19+
break;
20+
}
21+
}
22+
if (!map) {
23+
console.log("No mapping found for countly_drill");
24+
process.exit();
25+
}
1526
var asyncjs = require("async");
1627
var remap = {};
1728
for (var key in map.collections) {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/**
2+
* Description: Deletes custom events from countly and drill databases
3+
* Server: countly
4+
* Path: $(countly dir)/bin/scripts/expire-data
5+
* Command: node delete_custom_events.js
6+
*/
7+
8+
9+
const { ObjectId } = require('mongodb');
10+
const pluginManager = require('../../../plugins/pluginManager.js');
11+
const common = require('../../../api/utils/common.js');
12+
const drillCommon = require('../../../plugins/drill/api/common.js');
13+
14+
const APP_ID = "";
15+
const EVENTS = []; //leave empty to delete all custom events
16+
17+
Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) {
18+
console.log("Connected to databases...");
19+
20+
//SET COMMON DB AND DRILL DB
21+
common.db = countlyDb;
22+
common.drillDb = drillDb;
23+
24+
//GET APP
25+
try {
26+
const app = await countlyDb.collection("apps").findOne({_id: ObjectId(APP_ID)}, {_id: 1, name: 1});
27+
console.log("App:", app.name);
28+
//GET EVENTS
29+
var events = EVENTS;
30+
if (!events.length) {
31+
events = await countlyDb.collection("events").findOne({_id: app._id}, {_id: 0, list: 1});
32+
events = (events && events.list) || [];
33+
}
34+
if (!events.length) {
35+
close("No events found");
36+
}
37+
else {
38+
//DELETE EVENTS
39+
try {
40+
console.log(1 + ") Deleting drill events:");
41+
await deleteDrillEvents(app._id, events);
42+
console.log(2 + ") Deleting countly events:");
43+
await deleteCountlyEvents(app._id, events);
44+
console.log(3 + ") Deleting event times:");
45+
await deleteEventTimes(app._id, events);
46+
console.log(4 + ") Deleting event groups:");
47+
await deleteEventGroups(app._id, events);
48+
console.log(5 + ") Deleting event keys:");
49+
await deleteEventKeys(app._id, events);
50+
close();
51+
}
52+
catch (err) {
53+
close(err);
54+
}
55+
}
56+
}
57+
catch (err) {
58+
console.log("App not found");
59+
close(err);
60+
}
61+
62+
async function deleteDrillEvents(appId, events) {
63+
for (let i = 0; i < events.length; i++) {
64+
var collectionName = drillCommon.getCollectionName(events[i], appId);
65+
await drillDb.collection(collectionName).drop();
66+
console.log("Dropped collection:", collectionName);
67+
}
68+
await drillDb.collection('drill_bookmarks').remove({'app_id': appId, 'event_key': {$in: events}});
69+
console.log("Cleared drill_bookmarks");
70+
await drillDb.collection("drill_meta").remove({'app_id': (appId + ""), "type": "e", "e": {$in: events}});
71+
console.log("Cleared drill_meta");
72+
}
73+
74+
async function deleteCountlyEvents(appId, events) {
75+
for (let i = 0; i < events.length; i++) {
76+
var collectionName = 'events' + drillCommon.getEventHash(events[i], appId);
77+
await countlyDb.collection(collectionName).drop();
78+
console.log("Dropped collection:", collectionName);
79+
}
80+
}
81+
82+
async function deleteEventTimes(appId, events) {
83+
await countlyDb.collection("eventTimes" + appId).update({}, {"$pull": {"e": {"e": {$in: events}}}}, {"multi": true});
84+
console.log("Cleared eventTimes");
85+
await countlyDb.collection("timelineStatus").remove({'app_id': (appId + ""), "event": {$in: events}});
86+
console.log("Cleared timelineStatus");
87+
}
88+
89+
async function deleteEventKeys(appId, events) {
90+
const unsetQuery = {};
91+
for (let i = 0; i < events.length; i++) {
92+
unsetQuery[`segments.${events[i]}`] = 1;
93+
unsetQuery[`map.${events[i]}`] = 1;
94+
unsetQuery[`omitted_segments.${events[i]}`] = 1;
95+
}
96+
await countlyDb.collection("events").updateOne({_id: appId}, {$pull: {list: {$in: events}}, $unset: unsetQuery}, {$pull: {"overview": {eventKey: {$in: events}}}});
97+
console.log("Cleared events:", events);
98+
}
99+
100+
async function deleteEventGroups(appId, events) {
101+
await countlyDb.collection("event_groups").update({'app_id': (appId + "")}, {"$pull": {"source_events": {$in: events}}}, {"multi": true});
102+
console.log("Cleared event_groups");
103+
}
104+
105+
function close(err) {
106+
if (err) {
107+
console.log("Finished with errors:", err);
108+
}
109+
else {
110+
console.log("Finished successfully.");
111+
}
112+
countlyDb.close();
113+
drillDb.close();
114+
}
115+
});

0 commit comments

Comments
 (0)