Merge pull request #4311 from Countly/SER-690

[SER-690] Delete dashboard widgets if a related Drill query is deleted

Author: Cookiezaurs

bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js

@@ -0,0 +1,128 @@
+var plugins = require('../../../../plugins/pluginManager');
+const dashboard = require('../../../../plugins/dashboards/api/parts/dashboards.js');
+
+async function recheckFunnelWidgets(countlyDb) {
+    console.log("Detecting deleted data for funnels...");
+
+    const widgets = await countlyDb.collection('widgets').find({ widget_type: 'funnels', funnel_type: { $exists: true, $ne: [] } }, { funnel_type: 1 }).toArray();
+    if (!widgets || !widgets.length) {
+        console.log("No widgets found.");
+        return;
+    }
+    const funnelIdsInWidgets = widgets.map(widget => widget.funnel_type[0].split('***')[1]);
+    const existingFunnels = await countlyDb.collection('funnels').find({ _id: { $in: funnelIdsInWidgets } }, { _id: 1, app_id: 1 }).toArray();
+    if (!existingFunnels || !existingFunnels.length) {
+        console.log("No funnels found.");
+        return;
+    }
+    const formattedExistingFunnels = existingFunnels.map(funnel => funnel.app_id + "***" + funnel._id.toString());
+    const missingFunnelIds = widgets.filter(function(result) {
+        return !formattedExistingFunnels.includes(result.funnel_type[0]);
+    }).map(function(result) {
+        return result.funnel_type[0];
+    });
+
+    if (missingFunnelIds.length) {
+        const matchOperator = {
+            widget_type: "funnels",
+            "funnel_type": {
+                $in: missingFunnelIds
+            }
+        };
+
+        try {
+           return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb);
+        }
+        catch (error) {
+            console.log('Error while sending a request: ', error);
+        }
+    }
+    else {
+        console.log("No deleted funnels found in widgets.");
+    }
+}
+
+async function recheckFormulasWidgets(countlyDb) {
+    console.log("Detecting deleted data for formulas...");
+
+    const widgets = await countlyDb.collection('widgets').find({ widget_type: 'formulas', cmetric_refs: { $exists: true, $ne: [] } }, { "cmetric_refs._id": 1 }).toArray();
+    if (!widgets || !widgets.length) {
+        console.log("No widgets found.");
+        return;
+    }
+    const ids = widgets.map(item => countlyDb.ObjectID(item.cmetric_refs[0]._id));
+    const existingFormulas = await countlyDb.collection('calculated_metrics').find({ _id: { $in: ids } }, { _id: 1 }).toArray();
+    if (!existingFormulas || !existingFormulas.length) {
+        console.log("No formulas found.");
+        return;
+    }
+    const missingFormulasIds = widgets.filter(widget => {
+        return !existingFormulas.some(formula => String(formula._id) === widget.cmetric_refs[0]._id);
+    }).map(function(result) {
+        return result.cmetric_refs[0]._id;
+    });
+
+    if (missingFormulasIds.length) {
+        const matchOperator = {
+            widget_type: "formulas",
+            "cmetric_refs": {
+                $elemMatch: {
+                    _id: {
+                        $in: missingFormulasIds
+                    }
+                }
+            }
+        };
+
+        try {
+            return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb);
+        }
+        catch (error) {
+            console.log('Error while sending a request: ', error);
+        }
+    }
+    else {
+        console.log("No deleted formulas found in widgets.");
+    }
+}
+
+async function recheckDrillWidgets(countlyDb) {
+    console.log("Detecting deleted data for drill...");
+    const matchOperator = {
+        "widget_type": "drill",
+        "drill_query": { $size: 0 }
+    };
+
+    try {
+        return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb);
+    }
+    catch (error) {
+        console.log('Error while sending a request: ', error);
+    }
+}
+
+plugins.dbConnection().then(async(countlyDb) => {
+    try {
+        await recheckFunnelWidgets(countlyDb);
+    }
+    catch (error) {
+        console.log('Error in recheckFunnelWidgets:', error);
+    }
+
+    try {
+        await recheckFormulasWidgets(countlyDb);
+    }
+    catch (error) {
+        console.log('Error in recheckFormulasWidgets:', error);
+    }
+
+    try {
+        await recheckDrillWidgets(countlyDb);
+    }
+    catch (error) {
+        console.log('Error in recheckDrillWidgets:', error);
+    }
+    finally {
+        countlyDb.close();
+    }
+});

plugins/dashboards/api/parts/dashboards.js

@@ -562,6 +562,111 @@ dashboard.fetchNoteData = async function(params, apps, widget) {
     return widget;
 };
 
+/**
+ * Remove deleted records from widgets
+ * @param {object} params params object
+ * @param {object} matchOperator match operator for aggregation
+ * @param {object} db database object - if coming from script
+ * @returns {boolean} true if success
+ */
+dashboard.removeDeletedRecordsFromWidgets = async function(params, matchOperator, db) {
+    try {
+        if (!params || !matchOperator) {
+            log.e('missing parameters for removeDeletedRecordsFromWidgets', params, matchOperator);
+            return false;
+        }
+
+        if (typeof matchOperator === 'string') {
+            matchOperator = JSON.parse(matchOperator);
+        }
+
+        if (typeof db !== 'undefined') {
+            common.db = db;
+        }
+
+        var pipeline = [
+            {
+                $match: matchOperator
+            },
+            {
+                $lookup: {
+                    from: "dashboards",
+                    localField: "_id",
+                    foreignField: "widgets",
+                    as: "dashboard"
+                }
+            },
+            {
+                $unwind: "$dashboard"
+            },
+            {
+                $project: {
+                    _id: 0,
+                    widget_id: "$_id",
+                    dashboard_id: "$dashboard._id",
+                }
+            }
+        ];
+        var widgets = await common.db.collection('widgets').aggregate(pipeline, {allowDiskUse: true}).toArray();
+
+        for (const widget of widgets) {
+            var dashboardId = widget.dashboard_id;
+            var widgetId = widget.widget_id;
+            if (!dashboardId || !widgetId) {
+                log.e('dashbordId or widgetId could not found in remove widget');
+                continue;
+            }
+
+            await new Promise((resolve, reject) => {
+                dashboard.deleteWidget(params, dashboardId, widgetId, function(success) {
+                    if (success) {
+                        resolve();
+                    }
+                    else {
+                        reject();
+                    }
+                });
+            });
+        }
+        return true;
+    }
+    catch (error) {
+        log.e('Invalid request for remove widget', error);
+        return false;
+    }
+};
+
+dashboard.deleteWidget = function(params, dashboardId, widgetId, callback) {
+    common.db.collection("dashboards").update({_id: common.db.ObjectID(dashboardId)}, { $pull: {widgets: common.db.ObjectID(widgetId)}}, function(dashboardErr) {
+        if (!dashboardErr) {
+            common.db.collection("widgets").findAndModify({_id: common.db.ObjectID(widgetId)}, {}, {}, {remove: true}, function(widgetErr, widgetResult) {
+                if (widgetErr || !widgetResult || !widgetResult.value) {
+                    common.returnMessage(params, 500, "Failed to remove widget");
+                    callback(false);
+                }
+                else {
+                    var logData = widgetResult.value;
+                    logData.dashboard = dashboard.name;
+
+                    plugins.dispatch("/systemlogs", {params: params, action: "widget_deleted", data: logData});
+                    plugins.dispatch("/dashboard/widget/deleted", {params: params, widget: widgetResult.value});
+                    common.returnMessage(params, 200, 'Success');
+                    callback(true);
+                }
+            });
+        }
+        else {
+            common.returnMessage(params, 500, "Failed to remove widget");
+            callback(false);
+        }
+    });
+};
+
+plugins.register("/dashboard/clean-deleted-widgets", async function(ob) {
+    var response = await dashboard.removeDeletedRecordsFromWidgets(ob.params, ob.match);
+    return response;
+}, true);
+
 
 /**
  * Function to fetch technology analytics data for app