Skip to content

Commit 7b409a0

Browse files
pnrgencpnrgenc
committed
[SER-690] Delete dashboard widgets if a related Drill query is deleted
This script detects also deleted funnels & formulas data to remove from widgets [SER-688] [SER-689]
1 parent 1695414 commit 7b409a0

File tree

3 files changed

+277
-0
lines changed

3 files changed

+277
-0
lines changed

‎bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js

+178
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
var plugins = require('../../../../plugins/pluginManager');
2+
const fetch = require('node-fetch');
3+
const https = require('https');
4+
5+
// PLEASE ENTER BELOW INFORMATIONS BEFORE RUNNING THIS SCRIPT
6+
const API_KEY = "YOUR_API_KEY";
7+
const SERVER_URL = "http://localhost";
8+
9+
10+
async function sendRequest(params) {
11+
try {
12+
const url = new URL(params.Url || SERVER_URL);
13+
const options = {
14+
method: params.requestType,
15+
headers: {
16+
'Content-Type': 'application/json',
17+
},
18+
agent: new https.Agent({
19+
rejectUnauthorized: false,
20+
}),
21+
};
22+
23+
const response = await fetch(url.href, options);
24+
if (response.status === 200 || response.status === 201) {
25+
return true;
26+
}
27+
else {
28+
return { err: 'There was an error while sending a request.', code: response.status + " " + response.statusText };
29+
}
30+
}
31+
catch (error) {
32+
return { err: 'There was an error while sending a request.', code: error.code || 'Unknown' };
33+
}
34+
}
35+
36+
async function recheckFunnelWidgets(countlyDb) {
37+
console.log("Detecting deleted data for funnels...");
38+
39+
const widgets = await countlyDb.collection('widgets').find({ widget_type: 'funnels', funnel_type: { $exists: true, $ne: [] } }, { funnel_type: 1 }).toArray();
40+
if (!widgets || !widgets.length) {
41+
console.log("No widgets found.");
42+
return;
43+
}
44+
const funnelIdsInWidgets = widgets.map(widget => widget.funnel_type[0].split('***')[1]);
45+
const existingFunnels = await countlyDb.collection('funnels').find({ _id: { $in: funnelIdsInWidgets } }, { _id: 1, app_id: 1 }).toArray();
46+
if (!existingFunnels || !existingFunnels.length) {
47+
console.log("No funnels found.");
48+
return;
49+
}
50+
const formattedExistingFunnels = existingFunnels.map(funnel => funnel.app_id + "***" + funnel._id.toString());
51+
const missingFunnelIds = widgets.filter(function(result) {
52+
return !formattedExistingFunnels.includes(result.funnel_type[0]);
53+
}).map(function(result) {
54+
return result.funnel_type[0];
55+
});
56+
57+
if (missingFunnelIds.length) {
58+
const matchOperator = {
59+
widget_type: "funnels",
60+
"funnel_type": {
61+
$in: missingFunnelIds
62+
}
63+
};
64+
65+
try {
66+
var response = await sendRequest({
67+
requestType: 'GET',
68+
Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator),
69+
});
70+
if (!response.err) {
71+
console.log("Deleted funnels removed from widgets successfully.");
72+
}
73+
}
74+
catch (error) {
75+
console.log('Error while sending a request: ', error);
76+
}
77+
}
78+
else {
79+
console.log("No deleted funnels found in widgets.");
80+
}
81+
}
82+
83+
async function recheckFormulasWidgets(countlyDb) {
84+
console.log("Detecting deleted data for formulas...");
85+
86+
const widgets = await countlyDb.collection('widgets').find({ widget_type: 'formulas', cmetric_refs: { $exists: true, $ne: [] } }, { "cmetric_refs._id": 1 }).toArray();
87+
if (!widgets || !widgets.length) {
88+
console.log("No widgets found.");
89+
return;
90+
}
91+
const ids = widgets.map(item => countlyDb.ObjectID(item.cmetric_refs[0]._id));
92+
const existingFormulas = await countlyDb.collection('calculated_metrics').find({ _id: { $in: ids } }, { _id: 1 }).toArray();
93+
if (!existingFormulas || !existingFormulas.length) {
94+
console.log("No formulas found.");
95+
return;
96+
}
97+
const missingFormulasIds = widgets.filter(widget => {
98+
return !existingFormulas.some(formula => String(formula._id) === widget.cmetric_refs[0]._id);
99+
}).map(function(result) {
100+
return result.cmetric_refs[0]._id;
101+
});
102+
103+
if (missingFormulasIds.length) {
104+
const matchOperator = {
105+
widget_type: "formulas",
106+
"cmetric_refs": {
107+
$elemMatch: {
108+
_id: {
109+
$in: missingFormulasIds
110+
}
111+
}
112+
}
113+
};
114+
115+
try {
116+
var response = await sendRequest({
117+
requestType: 'GET',
118+
Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator),
119+
});
120+
if (!response.err) {
121+
console.log("Deleted formulas removed from widgets successfully.");
122+
}
123+
}
124+
catch (error) {
125+
console.log('Error while sending a request: ', error);
126+
}
127+
}
128+
else {
129+
console.log("No deleted formulas found in widgets.");
130+
}
131+
}
132+
133+
async function recheckDrillWidgets() {
134+
console.log("Detecting deleted data for drill...");
135+
const matchOperator = {
136+
"widget_type": "drill",
137+
"drill_query": { $size: 0 }
138+
};
139+
140+
try {
141+
var response = await sendRequest({
142+
requestType: 'GET',
143+
Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator),
144+
});
145+
if (!response.err) {
146+
console.log("Deleted drills removed from widgets successfully.");
147+
}
148+
}
149+
catch (error) {
150+
console.log('Error while sending a request: ', error);
151+
}
152+
}
153+
154+
155+
plugins.dbConnection().then(async(countlyDb) => {
156+
try {
157+
await recheckFunnelWidgets(countlyDb);
158+
}
159+
catch (error) {
160+
console.log('Error in recheckFunnelWidgets:', error);
161+
}
162+
163+
try {
164+
await recheckFormulasWidgets(countlyDb);
165+
}
166+
catch (error) {
167+
console.log('Error in recheckFormulasWidgets:', error);
168+
}
169+
170+
try {
171+
await recheckDrillWidgets();
172+
}
173+
catch (error) {
174+
console.log('Error in recheckDrillWidgets:', error);
175+
}
176+
countlyDb.close();
177+
});
178+

‎plugins/dashboards/api/api.js

+9
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,15 @@ plugins.setConfigs("dashboards", {
546546
return true;
547547
});
548548

549+
plugins.register("/o/dashboards/recheck_widgets", function(ob) {
550+
var params = ob.params;
551+
var apiKey = params.qstring.apiKey;
552+
var matchOperator = params.qstring.matchOperator;
553+
var result = customDashboards.callWidgetRecheck(apiKey, matchOperator);
554+
return common.returnOutput(params, result);
555+
});
556+
557+
549558
/**
550559
* @api {get} /i/dashboards/create Create a dashboard
551560
* @apiName CreateDashboard

‎plugins/dashboards/api/parts/dashboards.js

+90
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ var countlyModel = require("../../../../api/lib/countly.model.js"),
99
countlyCommon = require('../../../../api/lib/countly.common'),
1010
fetch = require("../../../../api/parts/data/fetch.js"),
1111
log = common.log('dashboards:api'),
12+
requestProcessor = require('../../../../api/utils/requestProcessor.js'),
1213
plugins = require("../../../pluginManager.js");
1314

1415
/** @lends module:api/parts/data/dashboard */
@@ -563,6 +564,95 @@ dashboard.fetchNoteData = async function(params, apps, widget) {
563564
};
564565

565566

567+
/**
568+
* Remove deleted records from widgets
569+
*
570+
* @param {string} apiKey user's api key
571+
* @param {object} matchOperator match operator for aggregation
572+
* @returns {boolean} true if success
573+
*/
574+
dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator) {
575+
try {
576+
if (!apiKey || !matchOperator) {
577+
log.e('missing parameters for removeDeletedRecordsFromWidgets', apiKey, matchOperator);
578+
return false;
579+
}
580+
581+
if (typeof matchOperator === 'string') {
582+
matchOperator = JSON.parse(matchOperator);
583+
}
584+
585+
var pipeline = [
586+
{
587+
$match: matchOperator
588+
},
589+
{
590+
$lookup: {
591+
from: "dashboards",
592+
localField: "_id",
593+
foreignField: "widgets",
594+
as: "dashboard"
595+
}
596+
},
597+
{
598+
$unwind: "$dashboard"
599+
},
600+
{
601+
$project: {
602+
_id: 0,
603+
widget_id: "$_id",
604+
dashboard_id: "$dashboard._id",
605+
}
606+
}
607+
];
608+
var widgets = await common.db.collection('widgets').aggregate(pipeline, {allowDiskUse: true}).toArray();
609+
610+
for (const widget of widgets) {
611+
var dashboardId = widget.dashboard_id;
612+
var widgetId = widget.widget_id;
613+
if (!dashboardId || !widgetId) {
614+
log.e('dashbordId or widgetId could not found in remove widget');
615+
continue;
616+
}
617+
var params = {
618+
'req': {
619+
url: "/i/dashboards/remove-widget?dashboard_id=" + dashboardId + "&widget_id=" + widgetId + "&api_key=" + apiKey
620+
},
621+
//adding custom processing for API responses
622+
'APICallback': function(err, responseData, headers, returnCode) {
623+
if (err) {
624+
log.e('Error while removing widget from dashboard', err, responseData, headers, returnCode);
625+
return false;
626+
}
627+
else {
628+
return true;
629+
}
630+
}
631+
};
632+
requestProcessor.processRequest(params);
633+
}
634+
return true;
635+
}
636+
catch (error) {
637+
log.e('Invalid request for remove widget', error);
638+
return false;
639+
}
640+
};
641+
642+
643+
dashboard.callWidgetRecheck = function(apiKey, matchOperator) {
644+
return plugins.dispatch("/dashboard/clean-deleted-widgets", {
645+
api_key: apiKey,
646+
match: matchOperator
647+
});
648+
};
649+
650+
plugins.register("/dashboard/clean-deleted-widgets", async function(ob) {
651+
var response = await dashboard.removeDeletedRecordsFromWidgets(ob.api_key, ob.match);
652+
return response;
653+
}, true);
654+
655+
566656
/**
567657
* Function to fetch technology analytics data for app
568658
* @param {Object} params - params object

0 commit comments

Comments
 (0)