diff --git a/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js b/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js new file mode 100644 index 00000000000..4ee3d7af78a --- /dev/null +++ b/bin/upgrade/DEV/scripts/remove_deleted_recors_from_widgets.js @@ -0,0 +1,128 @@ +var plugins = require('../../../../plugins/pluginManager'); +const dashboard = require('../../../../plugins/dashboards/api/parts/dashboards.js'); + +async function recheckFunnelWidgets(countlyDb) { + console.log("Detecting deleted data for funnels..."); + + const widgets = await countlyDb.collection('widgets').find({ widget_type: 'funnels', funnel_type: { $exists: true, $ne: [] } }, { funnel_type: 1 }).toArray(); + if (!widgets || !widgets.length) { + console.log("No widgets found."); + return; + } + const funnelIdsInWidgets = widgets.map(widget => widget.funnel_type[0].split('***')[1]); + const existingFunnels = await countlyDb.collection('funnels').find({ _id: { $in: funnelIdsInWidgets } }, { _id: 1, app_id: 1 }).toArray(); + if (!existingFunnels || !existingFunnels.length) { + console.log("No funnels found."); + return; + } + const formattedExistingFunnels = existingFunnels.map(funnel => funnel.app_id + "***" + funnel._id.toString()); + const missingFunnelIds = widgets.filter(function(result) { + return !formattedExistingFunnels.includes(result.funnel_type[0]); + }).map(function(result) { + return result.funnel_type[0]; + }); + + if (missingFunnelIds.length) { + const matchOperator = { + widget_type: "funnels", + "funnel_type": { + $in: missingFunnelIds + } + }; + + try { + return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb); + } + catch (error) { + console.log('Error while sending a request: ', error); + } + } + else { + console.log("No deleted funnels found in widgets."); + } +} + +async function recheckFormulasWidgets(countlyDb) { + console.log("Detecting deleted data for formulas..."); + + const widgets = await countlyDb.collection('widgets').find({ widget_type: 'formulas', cmetric_refs: { $exists: true, $ne: [] } }, { "cmetric_refs._id": 1 }).toArray(); + if (!widgets || !widgets.length) { + console.log("No widgets found."); + return; + } + const ids = widgets.map(item => countlyDb.ObjectID(item.cmetric_refs[0]._id)); + const existingFormulas = await countlyDb.collection('calculated_metrics').find({ _id: { $in: ids } }, { _id: 1 }).toArray(); + if (!existingFormulas || !existingFormulas.length) { + console.log("No formulas found."); + return; + } + const missingFormulasIds = widgets.filter(widget => { + return !existingFormulas.some(formula => String(formula._id) === widget.cmetric_refs[0]._id); + }).map(function(result) { + return result.cmetric_refs[0]._id; + }); + + if (missingFormulasIds.length) { + const matchOperator = { + widget_type: "formulas", + "cmetric_refs": { + $elemMatch: { + _id: { + $in: missingFormulasIds + } + } + } + }; + + try { + return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb); + } + catch (error) { + console.log('Error while sending a request: ', error); + } + } + else { + console.log("No deleted formulas found in widgets."); + } +} + +async function recheckDrillWidgets(countlyDb) { + console.log("Detecting deleted data for drill..."); + const matchOperator = { + "widget_type": "drill", + "drill_query": { $size: 0 } + }; + + try { + return await dashboard.removeDeletedRecordsFromWidgets({member: {username: 'unknown'}}, matchOperator, countlyDb); + } + catch (error) { + console.log('Error while sending a request: ', error); + } +} + +plugins.dbConnection().then(async(countlyDb) => { + try { + await recheckFunnelWidgets(countlyDb); + } + catch (error) { + console.log('Error in recheckFunnelWidgets:', error); + } + + try { + await recheckFormulasWidgets(countlyDb); + } + catch (error) { + console.log('Error in recheckFormulasWidgets:', error); + } + + try { + await recheckDrillWidgets(countlyDb); + } + catch (error) { + console.log('Error in recheckDrillWidgets:', error); + } + finally { + countlyDb.close(); + } +}); diff --git a/plugins/dashboards/api/parts/dashboards.js b/plugins/dashboards/api/parts/dashboards.js index 2a6159a5bd2..bdd32bd9916 100644 --- a/plugins/dashboards/api/parts/dashboards.js +++ b/plugins/dashboards/api/parts/dashboards.js @@ -562,6 +562,111 @@ dashboard.fetchNoteData = async function(params, apps, widget) { return widget; }; +/** + * Remove deleted records from widgets + * @param {object} params params object + * @param {object} matchOperator match operator for aggregation + * @param {object} db database object - if coming from script + * @returns {boolean} true if success + */ +dashboard.removeDeletedRecordsFromWidgets = async function(params, matchOperator, db) { + try { + if (!params || !matchOperator) { + log.e('missing parameters for removeDeletedRecordsFromWidgets', params, matchOperator); + return false; + } + + if (typeof matchOperator === 'string') { + matchOperator = JSON.parse(matchOperator); + } + + if (typeof db !== 'undefined') { + common.db = db; + } + + var pipeline = [ + { + $match: matchOperator + }, + { + $lookup: { + from: "dashboards", + localField: "_id", + foreignField: "widgets", + as: "dashboard" + } + }, + { + $unwind: "$dashboard" + }, + { + $project: { + _id: 0, + widget_id: "$_id", + dashboard_id: "$dashboard._id", + } + } + ]; + var widgets = await common.db.collection('widgets').aggregate(pipeline, {allowDiskUse: true}).toArray(); + + for (const widget of widgets) { + var dashboardId = widget.dashboard_id; + var widgetId = widget.widget_id; + if (!dashboardId || !widgetId) { + log.e('dashbordId or widgetId could not found in remove widget'); + continue; + } + + await new Promise((resolve, reject) => { + dashboard.deleteWidget(params, dashboardId, widgetId, function(success) { + if (success) { + resolve(); + } + else { + reject(); + } + }); + }); + } + return true; + } + catch (error) { + log.e('Invalid request for remove widget', error); + return false; + } +}; + +dashboard.deleteWidget = function(params, dashboardId, widgetId, callback) { + common.db.collection("dashboards").update({_id: common.db.ObjectID(dashboardId)}, { $pull: {widgets: common.db.ObjectID(widgetId)}}, function(dashboardErr) { + if (!dashboardErr) { + common.db.collection("widgets").findAndModify({_id: common.db.ObjectID(widgetId)}, {}, {}, {remove: true}, function(widgetErr, widgetResult) { + if (widgetErr || !widgetResult || !widgetResult.value) { + common.returnMessage(params, 500, "Failed to remove widget"); + callback(false); + } + else { + var logData = widgetResult.value; + logData.dashboard = dashboard.name; + + plugins.dispatch("/systemlogs", {params: params, action: "widget_deleted", data: logData}); + plugins.dispatch("/dashboard/widget/deleted", {params: params, widget: widgetResult.value}); + common.returnMessage(params, 200, 'Success'); + callback(true); + } + }); + } + else { + common.returnMessage(params, 500, "Failed to remove widget"); + callback(false); + } + }); +}; + +plugins.register("/dashboard/clean-deleted-widgets", async function(ob) { + var response = await dashboard.removeDeletedRecordsFromWidgets(ob.params, ob.match); + return response; +}, true); + /** * Function to fetch technology analytics data for app