[SER-690] Delete dashboard widgets if a related Drill query is deleted

bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js

@@ -0,0 +1,178 @@
+var plugins = require('../../../../plugins/pluginManager');
+const fetch = require('node-fetch');
+const https = require('https');
+
+// PLEASE ENTER BELOW INFORMATIONS BEFORE RUNNING THIS SCRIPT
+const API_KEY = "YOUR_API_KEY";
+const SERVER_URL = "http://localhost";
+
+
+async function sendRequest(params) {
+    try {
+        const url = new URL(params.Url || SERVER_URL);
+        const options = {
+            method: params.requestType,
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            agent: new https.Agent({
+                rejectUnauthorized: false,
+            }),
+        };
+
+        const response = await fetch(url.href, options);
+        if (response.status === 200 || response.status === 201) {
+            return true;
+        }
+        else {
+            return { err: 'There was an error while sending a request.', code: response.status + " " + response.statusText };
+        }
+    }
+    catch (error) {
+        return { err: 'There was an error while sending a request.', code: error.code || 'Unknown' };
+    }
+}
+
+async function recheckFunnelWidgets(countlyDb) {
+    console.log("Detecting deleted data for funnels...");
+
+    const widgets = await countlyDb.collection('widgets').find({ widget_type: 'funnels', funnel_type: { $exists: true, $ne: [] } }, { funnel_type: 1 }).toArray();
+    if (!widgets || !widgets.length) {
+        console.log("No widgets found.");
+        return;
+    }
+    const funnelIdsInWidgets = widgets.map(widget => widget.funnel_type[0].split('***')[1]);
+    const existingFunnels = await countlyDb.collection('funnels').find({ _id: { $in: funnelIdsInWidgets } }, { _id: 1, app_id: 1 }).toArray();
+    if (!existingFunnels || !existingFunnels.length) {
+        console.log("No funnels found.");
+        return;
+    }
+    const formattedExistingFunnels = existingFunnels.map(funnel => funnel.app_id + "***" + funnel._id.toString());
+    const missingFunnelIds = widgets.filter(function(result) {
+        return !formattedExistingFunnels.includes(result.funnel_type[0]);
+    }).map(function(result) {
+        return result.funnel_type[0];
+    });
+
+    if (missingFunnelIds.length) {
+        const matchOperator = {
+            widget_type: "funnels",
+            "funnel_type": {
+                $in: missingFunnelIds
+            }
+        };
+
+        try {
+            var response = await sendRequest({
+                requestType: 'GET',
+                Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator),
+            });
+            if (!response.err) {
+                console.log("Deleted funnels removed from widgets successfully.");
+            }
+        }
+        catch (error) {
+            console.log('Error while sending a request: ', error);
+        }
+    }
+    else {
+        console.log("No deleted funnels found in widgets.");
+    }
+}
+
+async function recheckFormulasWidgets(countlyDb) {
+    console.log("Detecting deleted data for formulas...");
+
+    const widgets = await countlyDb.collection('widgets').find({ widget_type: 'formulas', cmetric_refs: { $exists: true, $ne: [] } }, { "cmetric_refs._id": 1 }).toArray();
+    if (!widgets || !widgets.length) {
+        console.log("No widgets found.");
+        return;
+    }
+    const ids = widgets.map(item => countlyDb.ObjectID(item.cmetric_refs[0]._id));
+    const existingFormulas = await countlyDb.collection('calculated_metrics').find({ _id: { $in: ids } }, { _id: 1 }).toArray();
+    if (!existingFormulas || !existingFormulas.length) {
+        console.log("No formulas found.");
+        return;
+    }
+    const missingFormulasIds = widgets.filter(widget => {
+        return !existingFormulas.some(formula => String(formula._id) === widget.cmetric_refs[0]._id);
+    }).map(function(result) {
+        return result.cmetric_refs[0]._id;
+    });
+
+    if (missingFormulasIds.length) {
+        const matchOperator = {
+            widget_type: "formulas",
+            "cmetric_refs": {
+                $elemMatch: {
+                    _id: {
+                        $in: missingFormulasIds
+                    }
+                }
+            }
+        };
+
+        try {
+            var response = await sendRequest({
+                requestType: 'GET',
+                Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator),
+            });
+            if (!response.err) {
+                console.log("Deleted formulas removed from widgets successfully.");
+            }
+        }
+        catch (error) {
+            console.log('Error while sending a request: ', error);
+        }
+    }
+    else {
+        console.log("No deleted formulas found in widgets.");
+    }
+}
+
+async function recheckDrillWidgets() {
+    console.log("Detecting deleted data for drill...");
+    const matchOperator = {
+        "widget_type": "drill",
+        "drill_query": { $size: 0 }
+    };
+
+    try {
+        var response = await sendRequest({
+            requestType: 'GET',
+            Url: SERVER_URL + "/o/dashboards/recheck_widgets?apiKey=" + API_KEY + "&matchOperator=" + JSON.stringify(matchOperator),
+        });
+        if (!response.err) {
+            console.log("Deleted drills removed from widgets successfully.");
+        }
+    }
+    catch (error) {
+        console.log('Error while sending a request: ', error);
+    }
+}
+
+
+plugins.dbConnection().then(async(countlyDb) => {
+    try {
+        await recheckFunnelWidgets(countlyDb);
+    }
+    catch (error) {
+        console.log('Error in recheckFunnelWidgets:', error);
+    }
+
+    try {
+        await recheckFormulasWidgets(countlyDb);
+    }
+    catch (error) {
+        console.log('Error in recheckFormulasWidgets:', error);
+    }
+
+    try {
+        await recheckDrillWidgets();
+    }
+    catch (error) {
+        console.log('Error in recheckDrillWidgets:', error);
+    }
+    countlyDb.close();
+});
+

plugins/dashboards/api/api.js

@@ -546,6 +546,15 @@ plugins.setConfigs("dashboards", {
         return true;
     });
 
+    plugins.register("/o/dashboards/recheck_widgets", function(ob) {
+        var params = ob.params;
+        var apiKey = params.qstring.apiKey;
+        var matchOperator = params.qstring.matchOperator;
+        var result = customDashboards.callWidgetRecheck(apiKey, matchOperator);
+        return common.returnOutput(params, result);
+    });
+
+
     /**
      * @api {get} /i/dashboards/create Create a dashboard
      * @apiName CreateDashboard

plugins/dashboards/api/parts/dashboards.js

@@ -9,6 +9,7 @@ var countlyModel = require("../../../../api/lib/countly.model.js"),
     countlyCommon = require('../../../../api/lib/countly.common'),
     fetch = require("../../../../api/parts/data/fetch.js"),
     log = common.log('dashboards:api'),
+    requestProcessor = require('../../../../api/utils/requestProcessor.js'),
     plugins = require("../../../pluginManager.js");
 
 /** @lends module:api/parts/data/dashboard */
@@ -563,6 +564,95 @@ dashboard.fetchNoteData = async function(params, apps, widget) {
 };
 
 
+/**
+ * Remove deleted records from widgets
+ * 
+ * @param {string} apiKey user's api key
+ * @param {object} matchOperator match operator for aggregation
+ * @returns {boolean} true if success
+ */
+dashboard.removeDeletedRecordsFromWidgets = async function(apiKey, matchOperator) {
+    try {
+        if (!apiKey || !matchOperator) {
+            log.e('missing parameters for removeDeletedRecordsFromWidgets', apiKey, matchOperator);
+            return false;
+        }
+
+        if (typeof matchOperator === 'string') {
+            matchOperator = JSON.parse(matchOperator);
+        }
+
+        var pipeline = [
+            {
+                $match: matchOperator
+            },
+            {
+                $lookup: {
+                    from: "dashboards",
+                    localField: "_id",
+                    foreignField: "widgets",
+                    as: "dashboard"
+                }
+            },
+            {
+                $unwind: "$dashboard"
+            },
+            {
+                $project: {
+                    _id: 0,
+                    widget_id: "$_id",
+                    dashboard_id: "$dashboard._id",
+                }
+            }
+        ];
+        var widgets = await common.db.collection('widgets').aggregate(pipeline, {allowDiskUse: true}).toArray();
+
+        for (const widget of widgets) {
+            var dashboardId = widget.dashboard_id;
+            var widgetId = widget.widget_id;
+            if (!dashboardId || !widgetId) {
+                log.e('dashbordId or widgetId could not found in remove widget');
+                continue;
+            }
+            var params = {
+                'req': {
+                    url: "/i/dashboards/remove-widget?dashboard_id=" + dashboardId + "&widget_id=" + widgetId + "&api_key=" + apiKey
+                },
+                //adding custom processing for API responses
+                'APICallback': function(err, responseData, headers, returnCode) {
+                    if (err) {
+                        log.e('Error while removing widget from dashboard', err, responseData, headers, returnCode);
+                        return false;
+                    }
+                    else {
+                        return true;
+                    }
+                }
+            };
+            requestProcessor.processRequest(params);
+        }
+        return true;
+    }
+    catch (error) {
+        log.e('Invalid request for remove widget', error);
+        return false;
+    }
+};
+
+
+dashboard.callWidgetRecheck = function(apiKey, matchOperator) {
+    return plugins.dispatch("/dashboard/clean-deleted-widgets", {
+        api_key: apiKey,
+        match: matchOperator
+    });
+};
+
+plugins.register("/dashboard/clean-deleted-widgets", async function(ob) {
+    var response = await dashboard.removeDeletedRecordsFromWidgets(ob.api_key, ob.match);
+    return response;
+}, true);
+
+
 /**
  * Function to fetch technology analytics data for app
  * @param  {Object} params - params object