From 7b409a0777958d7ca142f97aea8753be4211636f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C4=B1nar=20Gen=C3=A7?= Date: Thu, 15 Jun 2023 19:53:55 +0300 Subject: [PATCH 1/3] [SER-690] Delete dashboard widgets if a related Drill query is deleted This script detects also deleted funnels & formulas data to remove from widgets [SER-688] [SER-689] --- .../remove_deleted_recors_from_widgets.js | 178 ++++++++++++++++++ plugins/dashboards/api/api.js | 9 + plugins/dashboards/api/parts/dashboards.js | 90 +++++++++ 3 files changed, 277 insertions(+) create mode 100644 bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js diff --git a/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js b/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js new file mode 100644 index 00000000000..8b0312d1269 --- /dev/null +++ b/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js @@ -0,0 +1,178 @@ +var plugins = require('../../../../plugins/pluginManager'); +const fetch = require('node-fetch'); +const https = require('https'); + +// PLEASE ENTER BELOW INFORMATIONS BEFORE RUNNING THIS SCRIPT +const API_KEY = "YOUR_API_KEY"; +const SERVER_URL = "http://localhost"; + + +async function sendRequest(params) { + try { + const url = new URL(params.Url || SERVER_URL); + const options = { + method: params.requestType, + headers: { + 'Content-Type': 'application/json', + }, + agent: new https.Agent({ + rejectUnauthorized: false, + }), + }; + + const response = await fetch(url.href, options); + if (response.status === 200 || response.status === 201) { + return true; + } + else { + return { err: 'There was an error while sending a request.', code: response.status + " " + response.statusText }; + } + } + catch (error) { + return { err: 'There was an error while sending a request.', code: error.code || 'Unknown' }; + } +} + +async function recheckFunnelWidgets(countlyDb) { + console.log("Detecting deleted data for funnels..."); + + const widgets = await countlyDb.collection('widgets').find({ widget_type: 'funnels', funnel_type: { $exists: true, $ne: [] } }, { funnel_type: 1 }).toArray(); + if (!widgets || !widgets.length) { + console.log("No widgets found."); + return; + } + const funnelIdsInWidgets = widgets.map(widget => widget.funnel_type[0].split('***')[1]); + const existingFunnels = await countlyDb.collection('funnels').find({ _id: { $in: funnelIdsInWidgets } }, { _id: 1, app_id: 1 }).toArray(); + if (!existingFunnels || !existingFunnels.length) { + console.log("No funnels found."); + return; + } + const formattedExistingFunnels = existingFunnels.map(funnel => funnel.app_id + "***" + funnel._id.toString()); + const missingFunnelIds = widgets.filter(function(result) { + return !formattedExistingFunnels.includes(result.funnel_type[0]); + }).map(function(result) { + return result.funnel_type[0]; + }); + + if (missingFunnelIds.length) { + const matchOperator = { + widget_type: "funnels", + "funnel_type": { + $in: missingFunnelIds + } + }; + + try { + var response = await sendRequest({ + requestType: 'GET', + Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator), + }); + if (!response.err) { + console.log("Deleted funnels removed from widgets successfully."); + } + } + catch (error) { + console.log('Error while sending a request: ', error); + } + } + else { + console.log("No deleted funnels found in widgets."); + } +} + +async function recheckFormulasWidgets(countlyDb) { + console.log("Detecting deleted data for formulas..."); + + const widgets = await countlyDb.collection('widgets').find({ widget_type: 'formulas', cmetric_refs: { $exists: true, $ne: [] } }, { "cmetric_refs._id": 1 }).toArray(); + if (!widgets || !widgets.length) { + console.log("No widgets found."); + return; + } + const ids = widgets.map(item => countlyDb.ObjectID(item.cmetric_refs[0]._id)); + const existingFormulas = await countlyDb.collection('calculated_metrics').find({ _id: { $in: ids } }, { _id: 1 }).toArray(); + if (!existingFormulas || !existingFormulas.length) { + console.log("No formulas found."); + return; + } + const missingFormulasIds = widgets.filter(widget => { + return !existingFormulas.some(formula => String(formula._id) === widget.cmetric_refs[0]._id); + }).map(function(result) { + return result.cmetric_refs[0]._id; + }); + + if (missingFormulasIds.length) { + const matchOperator = { + widget_type: "formulas", + "cmetric_refs": { + $elemMatch: { + _id: { + $in: missingFormulasIds + } + } + } + }; + + try { + var response = await sendRequest({ + requestType: 'GET', + Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator), + }); + if (!response.err) { + console.log("Deleted formulas removed from widgets successfully."); + } + } + catch (error) { + console.log('Error while sending a request: ', error); + } + } + else { + console.log("No deleted formulas found in widgets."); + } +} + +async function recheckDrillWidgets() { + console.log("Detecting deleted data for drill..."); + const matchOperator = { + "widget_type": "drill", + "drill_query": { $size: 0 } + }; + + try { + var response = await sendRequest({ + requestType: 'GET', + Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator), + }); + if (!response.err) { + console.log("Deleted drills removed from widgets successfully."); + } + } + catch (error) { + console.log('Error while sending a request: ', error); + } +} + + +plugins.dbConnection().then(async(countlyDb) => { + try { + await recheckFunnelWidgets(countlyDb); + } + catch (error) { + console.log('Error in recheckFunnelWidgets:', error); + } + + try { + await recheckFormulasWidgets(countlyDb); + } + catch (error) { + console.log('Error in recheckFormulasWidgets:', error); + } + + try { + await recheckDrillWidgets(); + } + catch (error) { + console.log('Error in recheckDrillWidgets:', error); + } + countlyDb.close(); +}); + diff --git a/plugins/dashboards/api/api.js b/plugins/dashboards/api/api.js index 1fe872b6c49..418de22bd8e 100644 --- a/plugins/dashboards/api/api.js +++ b/plugins/dashboards/api/api.js @@ -546,6 +546,15 @@ plugins.setConfigs("dashboards", { return true; }); + plugins.register("/o/dashboards/recheck_widgets", function(ob) { + var params = ob.params; + var apiKey = params.qstring.apiKey; + var matchOperator = params.qstring.matchOperator; + var result = customDashboards.callWidgetRecheck(apiKey, matchOperator); + return common.returnOutput(params, result); + }); + + /** * @api {get} /i/dashboards/create Create a dashboard * @apiName CreateDashboard diff --git a/plugins/dashboards/api/parts/dashboards.js b/plugins/dashboards/api/parts/dashboards.js index 2a6159a5bd2..73cf230c6bb 100644 --- a/plugins/dashboards/api/parts/dashboards.js +++ b/plugins/dashboards/api/parts/dashboards.js @@ -9,6 +9,7 @@ var countlyModel = require("../../../../api/lib/countly.model.js"), countlyCommon = require('../../../../api/lib/countly.common'), fetch = require("../../../../api/parts/data/fetch.js"), log = common.log('dashboards:api'), + requestProcessor = require('../../../../api/utils/requestProcessor.js'), plugins = require("../../../pluginManager.js"); /** @lends module:api/parts/data/dashboard */ @@ -563,6 +564,95 @@ dashboard.fetchNoteData = async function(params, apps, widget) { }; +/** + * Remove deleted records from widgets + * + * @param {string} apiKey user's api key + * @param {object} matchOperator match operator for aggregation + * @returns {boolean} true if success + */ +dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator) { + try { + if (!apiKey || !matchOperator) { + log.e('missing parameters for removeDeletedRecordsFromWidgets', apiKey, matchOperator); + return false; + } + + if (typeof matchOperator === 'string') { + matchOperator = JSON.parse(matchOperator); + } + + var pipeline = [ + { + $match: matchOperator + }, + { + $lookup: { + from: "dashboards", + localField: "_id", + foreignField: "widgets", + as: "dashboard" + } + }, + { + $unwind: "$dashboard" + }, + { + $project: { + _id: 0, + widget_id: "$_id", + dashboard_id: "$dashboard._id", + } + } + ]; + var widgets = await common.db.collection('widgets').aggregate(pipeline, {allowDiskUse: true}).toArray(); + + for (const widget of widgets) { + var dashboardId = widget.dashboard_id; + var widgetId = widget.widget_id; + if (!dashboardId || !widgetId) { + log.e('dashbordId or widgetId could not found in remove widget'); + continue; + } + var params = { + 'req': { + url: "/i/dashboards/remove-widget?dashboard_id=" + dashboardId + "&widget_id=" + widgetId + "&api_key=" + apiKey + }, + //adding custom processing for API responses + 'APICallback': function(err, responseData, headers, returnCode) { + if (err) { + log.e('Error while removing widget from dashboard', err, responseData, headers, returnCode); + return false; + } + else { + return true; + } + } + }; + requestProcessor.processRequest(params); + } + return true; + } + catch (error) { + log.e('Invalid request for remove widget', error); + return false; + } +}; + + +dashboard.callWidgetRecheck = function(apiKey, matchOperator) { + return plugins.dispatch("/dashboard/clean-deleted-widgets", { + api_key: apiKey, + match: matchOperator + }); +}; + +plugins.register("/dashboard/clean-deleted-widgets", async function(ob) { + var response = await dashboard.removeDeletedRecordsFromWidgets(ob.api_key, ob.match); + return response; +}, true); + + /** * Function to fetch technology analytics data for app * @param {Object} params - params object From 525c641c068fd85900b5fcd2a562322eb94a0389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C4=B1nar=20Gen=C3=A7?= Date: Mon, 26 Jun 2023 18:12:05 +0300 Subject: [PATCH 2/3] [SER-690] Removed requestProcessor when removing widgets --- .../remove_deleted_recors_from_widgets.js | 68 +++---------------- plugins/dashboards/api/parts/dashboards.js | 65 +++++++++++------- 2 files changed, 49 insertions(+), 84 deletions(-) diff --git a/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js b/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js index 8b0312d1269..4ee3d7af78a 100644 --- a/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js +++ b/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js @@ -1,37 +1,5 @@ var plugins = require('../../../../plugins/pluginManager'); -const fetch = require('node-fetch'); -const https = require('https'); - -// PLEASE ENTER BELOW INFORMATIONS BEFORE RUNNING THIS SCRIPT -const API_KEY = "YOUR_API_KEY"; -const SERVER_URL = "http://localhost"; - - -async function sendRequest(params) { - try { - const url = new URL(params.Url || SERVER_URL); - const options = { - method: params.requestType, - headers: { - 'Content-Type': 'application/json', - }, - agent: new https.Agent({ - rejectUnauthorized: false, - }), - }; - - const response = await fetch(url.href, options); - if (response.status === 200 || response.status === 201) { - return true; - } - else { - return { err: 'There was an error while sending a request.', code: response.status + " " + response.statusText }; - } - } - catch (error) { - return { err: 'There was an error while sending a request.', code: error.code || 'Unknown' }; - } -} +const dashboard = require('../../../../plugins/dashboards/api/parts/dashboards.js'); async function recheckFunnelWidgets(countlyDb) { console.log("Detecting deleted data for funnels..."); @@ -63,13 +31,7 @@ async function recheckFunnelWidgets(countlyDb) { }; try { - var response = await sendRequest({ - requestType: 'GET', - Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator), - }); - if (!response.err) { - console.log("Deleted funnels removed from widgets successfully."); - } + return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb); } catch (error) { console.log('Error while sending a request: ', error); @@ -113,13 +75,7 @@ async function recheckFormulasWidgets(countlyDb) { }; try { - var response = await sendRequest({ - requestType: 'GET', - Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator), - }); - if (!response.err) { - console.log("Deleted formulas removed from widgets successfully."); - } + return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb); } catch (error) { console.log('Error while sending a request: ', error); @@ -130,7 +86,7 @@ async function recheckFormulasWidgets(countlyDb) { } } -async function recheckDrillWidgets() { +async function recheckDrillWidgets(countlyDb) { console.log("Detecting deleted data for drill..."); const matchOperator = { "widget_type": "drill", @@ -138,20 +94,13 @@ async function recheckDrillWidgets() { }; try { - var response = await sendRequest({ - requestType: 'GET', - Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator), - }); - if (!response.err) { - console.log("Deleted drills removed from widgets successfully."); - } + return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb); } catch (error) { console.log('Error while sending a request: ', error); } } - plugins.dbConnection().then(async(countlyDb) => { try { await recheckFunnelWidgets(countlyDb); @@ -168,11 +117,12 @@ plugins.dbConnection().then(async(countlyDb) => { } try { - await recheckDrillWidgets(); + await recheckDrillWidgets(countlyDb); } catch (error) { console.log('Error in recheckDrillWidgets:', error); } - countlyDb.close(); + finally { + countlyDb.close(); + } }); - diff --git a/plugins/dashboards/api/parts/dashboards.js b/plugins/dashboards/api/parts/dashboards.js index 73cf230c6bb..bdd32bd9916 100644 --- a/plugins/dashboards/api/parts/dashboards.js +++ b/plugins/dashboards/api/parts/dashboards.js @@ -9,7 +9,6 @@ var countlyModel = require("../../../../api/lib/countly.model.js"), countlyCommon = require('../../../../api/lib/countly.common'), fetch = require("../../../../api/parts/data/fetch.js"), log = common.log('dashboards:api'), - requestProcessor = require('../../../../api/utils/requestProcessor.js'), plugins = require("../../../pluginManager.js"); /** @lends module:api/parts/data/dashboard */ @@ -563,18 +562,17 @@ dashboard.fetchNoteData = async function(params, apps, widget) { return widget; }; - /** * Remove deleted records from widgets - * - * @param {string} apiKey user's api key + * @param {object} params params object * @param {object} matchOperator match operator for aggregation + * @param {object} db database object - if coming from script * @returns {boolean} true if success */ -dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator) { +dashboard.removeDeletedRecordsFromWidgets = async function(params, matchOperator, db) { try { - if (!apiKey || !matchOperator) { - log.e('missing parameters for removeDeletedRecordsFromWidgets', apiKey, matchOperator); + if (!params || !matchOperator) { + log.e('missing parameters for removeDeletedRecordsFromWidgets', params, matchOperator); return false; } @@ -582,6 +580,10 @@ dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator matchOperator = JSON.parse(matchOperator); } + if (typeof db !== 'undefined') { + common.db = db; + } + var pipeline = [ { $match: matchOperator @@ -614,22 +616,17 @@ dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator log.e('dashbordId or widgetId could not found in remove widget'); continue; } - var params = { - 'req': { - url: "/i/dashboards/remove-widget?dashboard_id=" + dashboardId + "&widget_id=" + widgetId + "&api_key=" + apiKey - }, - //adding custom processing for API responses - 'APICallback': function(err, responseData, headers, returnCode) { - if (err) { - log.e('Error while removing widget from dashboard', err, responseData, headers, returnCode); - return false; + + await new Promise((resolve, reject) => { + dashboard.deleteWidget(params, dashboardId, widgetId, function(success) { + if (success) { + resolve(); } else { - return true; + reject(); } - } - }; - requestProcessor.processRequest(params); + }); + }); } return true; } @@ -639,16 +636,34 @@ dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator } }; +dashboard.deleteWidget = function(params, dashboardId, widgetId, callback) { + common.db.collection("dashboards").update({_id: common.db.ObjectID(dashboardId)}, { $pull: {widgets: common.db.ObjectID(widgetId)}}, function(dashboardErr) { + if (!dashboardErr) { + common.db.collection("widgets").findAndModify({_id: common.db.ObjectID(widgetId)}, {}, {}, {remove: true}, function(widgetErr, widgetResult) { + if (widgetErr || !widgetResult || !widgetResult.value) { + common.returnMessage(params, 500, "Failed to remove widget"); + callback(false); + } + else { + var logData = widgetResult.value; + logData.dashboard = dashboard.name; -dashboard.callWidgetRecheck = function(apiKey, matchOperator) { - return plugins.dispatch("/dashboard/clean-deleted-widgets", { - api_key: apiKey, - match: matchOperator + plugins.dispatch("/systemlogs", {params: params, action: "widget_deleted", data: logData}); + plugins.dispatch("/dashboard/widget/deleted", {params: params, widget: widgetResult.value}); + common.returnMessage(params, 200, 'Success'); + callback(true); + } + }); + } + else { + common.returnMessage(params, 500, "Failed to remove widget"); + callback(false); + } }); }; plugins.register("/dashboard/clean-deleted-widgets", async function(ob) { - var response = await dashboard.removeDeletedRecordsFromWidgets(ob.api_key, ob.match); + var response = await dashboard.removeDeletedRecordsFromWidgets(ob.params, ob.match); return response; }, true); From de647495aab215d7d860c2eb60d336b2f9c9b0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C4=B1nar=20Gen=C3=A7?= Date: Tue, 11 Jul 2023 13:17:39 +0300 Subject: [PATCH 3/3] [SER-690] Removed unused request from dashboard --- plugins/dashboards/api/api.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/plugins/dashboards/api/api.js b/plugins/dashboards/api/api.js index 418de22bd8e..1fe872b6c49 100644 --- a/plugins/dashboards/api/api.js +++ b/plugins/dashboards/api/api.js @@ -546,15 +546,6 @@ plugins.setConfigs("dashboards", { return true; }); - plugins.register("/o/dashboards/recheck_widgets", function(ob) { - var params = ob.params; - var apiKey = params.qstring.apiKey; - var matchOperator = params.qstring.matchOperator; - var result = customDashboards.callWidgetRecheck(apiKey, matchOperator); - return common.returnOutput(params, result); - }); - - /** * @api {get} /i/dashboards/create Create a dashboard * @apiName CreateDashboard