Merge pull request #5986 from Countly/ar2rsawseen/24.10

DBViewever improvements

Author: ar2rsawseen

api/utils/requestProcessor.js

@@ -7,7 +7,7 @@ const Promise = require('bluebird');
 const url = require('url');
 const common = require('./common.js');
 const countlyCommon = require('../lib/countly.common.js');
-const { validateAppAdmin, validateUser, validateRead, validateUserForRead, validateUserForWrite, validateGlobalAdmin, dbUserHasAccessToCollection, validateUpdate, validateDelete, validateCreate } = require('./rights.js');
+const { validateAppAdmin, validateUser, validateRead, validateUserForRead, validateUserForWrite, validateGlobalAdmin, dbUserHasAccessToCollection, validateUpdate, validateDelete, validateCreate, getBaseAppFilter } = require('./rights.js');
 const authorize = require('./authorizer.js');
 const taskmanager = require('./taskmanager.js');
 const plugins = require('../../plugins/pluginManager.js');
@@ -2128,7 +2128,7 @@ const processRequest = (params) => {
                         }
 
                         dbUserHasAccessToCollection(params, params.qstring.collection, (hasAccess) => {
-                            if (hasAccess) {
+                            if (hasAccess || (params.qstring.db === "countly_drill" && params.qstring.collection === "drill_events") || (params.qstring.db === "countly" && params.qstring.collection === "events_data")) {
                                 var dbs = { countly: common.db, countly_drill: common.drillDb, countly_out: common.outDb, countly_fs: countlyFs.gridfs.getHandler() };
                                 var db = "";
                                 if (params.qstring.db && dbs[params.qstring.db]) {
@@ -2137,6 +2137,23 @@ const processRequest = (params) => {
                                 else {
                                     db = common.db;
                                 }
+                                if (!params.member.global_admin && params.qstring.collection === "drill_events" || params.qstring.collection === "events_data") {
+                                    var base_filter = getBaseAppFilter(params.member, params.qstring.db, params.qstring.collection);
+                                    if (base_filter && Object.keys(base_filter).length > 0) {
+                                        params.qstring.query = params.qstring.query || {};
+                                        for (var key in base_filter) {
+                                            if (params.qstring.query[key]) {
+                                                params.qstring.query.$and = params.qstring.query.$and || [];
+                                                params.qstring.query.$and.push({[key]: base_filter[key]});
+                                                params.qstring.query.$and.push({[key]: params.qstring.query[key]});
+                                                delete params.qstring.query[key];
+                                            }
+                                            else {
+                                                params.qstring.query[key] = base_filter[key];
+                                            }
+                                        }
+                                    }
+                                }
                                 countlyApi.data.exports.fromDatabase({
                                     db: db,
                                     params: params,

api/utils/rights.js

@@ -1083,7 +1083,32 @@ function validateWrite(params, feature, accessType, callback, callbackParam) {
         });
     });
 }
-
+/**
+ * Creates filter object  to filter by member allowed collections
+ * @param {object} member - members object from params
+ * @param {string} dbName  - database name as string
+ * @param {string} collectionName  - collection Name
+ * @returns {object} filter object
+ */
+exports.getBaseAppFilter = function(member, dbName, collectionName) {
+    var base_filter = {};
+    var apps = exports.getUserApps(member);
+    if (dbName === "countly_drill" && collectionName === "drill_events") {
+        if (Array.isArray(apps) && apps.length > 0) {
+            base_filter.a = {"$in": apps};
+        }
+    }
+    else if (dbName === "countly" && collectionName === "events_data") {
+        var in_array = [];
+        if (Array.isArray(apps) && apps.length > 0) {
+            for (var i = 0; i < apps.length; i++) {
+                in_array.push(new RegExp("^" + apps[i] + "_.*"));
+            }
+            base_filter = {"_id": {"$in": in_array}};
+        }
+    }
+    return base_filter;
+};
 /**
 * Validate user for create access by api_key for provided app_id (both required parameters for the request).
 * @param {params} params - {@link params} object

plugins/dbviewer/api/api.js

@@ -5,19 +5,88 @@ var common = require('../../../api/utils/common.js'),
     countlyFs = require('../../../api/utils/countlyFs.js'),
     _ = require('underscore'),
     taskManager = require('../../../api/utils/taskmanager.js'),
-    { getCollectionName, dbUserHasAccessToCollection, dbLoadEventsData, validateUser, getUserApps, validateGlobalAdmin, hasReadRight } = require('../../../api/utils/rights.js'),
+    { getCollectionName, dbUserHasAccessToCollection, dbLoadEventsData, validateUser, getUserApps, validateGlobalAdmin, hasReadRight, getBaseAppFilter } = require('../../../api/utils/rights.js'),
     exported = {};
 const { MongoInvalidArgumentError } = require('mongodb');
 
 const { EJSON } = require('bson');
 
 const FEATURE_NAME = 'dbviewer';
+const whiteListedAggregationStages = {
+    "$addFields": true,
+    "$bucket": true,
+    "$bucketAuto": true,
+    //"$changeStream": false,
+    //"$changeStreamSplitLargeEvents": false,
+    //"$collStats": false,
+    "$count": true,
+    //"$currentOp": false,
+    "$densify": true,
+    //"$documents": false
+    "$facet": true,
+    "$fill": true,
+    "$geoNear": true,
+    "$graphLookup": true,
+    "$group": true,
+    //"$indexStats": false,
+    "$limit": true,
+    //"$listLocalSessions": false
+    //"$listSampledQueries": false
+    //"$listSearchIndexes": false
+    //"$listSessions": false
+    //"$lookup": false
+    "$match": true,
+    //"$merge": false
+    //"$mergeCursors": false
+    //"$out": false
+    //"$planCacheStats": false,
+    "$project": true,
+    "$querySettings": true,
+    "$redact": true,
+    "$replaceRoot": true,
+    "$replaceWith": true,
+    "$sample": true,
+    "$search": true,
+    "$searchMeta": true,
+    "$set": true,
+    "$setWindowFields": true,
+    //"$sharedDataDistribution": false,
+    "$skip": true,
+    "$sort": true,
+    "$sortByCount": true,
+    //"$unionWith": false,
+    "$unset": true,
+    "$unwind": true,
+    "$vectorSearch": true //atlas specific
+};
 var spawn = require('child_process').spawn,
     child;
+
 (function() {
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
     });
+    /**
+     * Function removes not allowed aggregation stages from the pipeline
+     * @param {array} aggregation  - current aggregation pipeline
+     * @returns {object} changes - object with information which operations were removed
+     */
+    function escapeNotAllowedAggregationStages(aggregation) {
+        var changes = {};
+        for (var z = 0; z < aggregation.length; z++) {
+            for (var key in aggregation[z]) {
+                if (!whiteListedAggregationStages[key]) {
+                    changes[key] = true;
+                    delete aggregation[z][key];
+                }
+            }
+            if (Object.keys(aggregation[z]).length === 0) {
+                aggregation.splice(z, 1);
+                z--;
+            }
+        }
+        return changes;
+    }
 
     /**
      * @api {get} /o/db Access database
@@ -179,6 +248,25 @@ var spawn = require('child_process').spawn,
                 filter = {};
             }
 
+            var base_filter = {};
+            if (!params.member.global_admin) {
+                base_filter = getBaseAppFilter(params.member, dbNameOnParam, params.qstring.collection);
+            }
+
+            if (base_filter && Object.keys(base_filter).length > 0) {
+                for (var key in base_filter) {
+                    if (filter[key]) {
+                        filter.$and = filter.$and || [];
+                        filter.$and.push({[key]: base_filter[key]});
+                        filter.$and.push({[key]: filter[key]});
+                        delete filter[key];
+                    }
+                    else {
+                        filter[key] = base_filter[key];
+                    }
+                }
+            }
+
             if (dbs[dbNameOnParam]) {
                 try {
                     var cursor = dbs[dbNameOnParam].collection(params.qstring.collection).find(filter, { projection });
@@ -191,6 +279,7 @@ var spawn = require('child_process').spawn,
                         common.returnMessage(params, 400, "Invalid collection name: Collection names can not contain '$' or other invalid characters");
                     }
                     else {
+                        log.e(error);
                         common.returnMessage(params, 500, "An unexpected error occurred.");
                     }
                     return false;
@@ -291,7 +380,7 @@ var spawn = require('child_process').spawn,
                             async.each(results, function(col, done) {
                                 if (col.collectionName.indexOf("system.indexes") === -1 && col.collectionName.indexOf("sessions_") === -1) {
                                     userHasAccess(params, col.collectionName, params.qstring.app_id, function(hasAccess) {
-                                        if (hasAccess) {
+                                        if (hasAccess || col.collectionName === "events_data" || col.collectionName === "drill_events") {
                                             ob = parseCollectionName(col.collectionName, lookup);
                                             db.collections[ob.pretty] = ob.name;
                                         }
@@ -318,8 +407,9 @@ var spawn = require('child_process').spawn,
         * Get aggregated result by the parameter on the url
         * @param {string} collection - collection will be applied related query
         * @param {object} aggregation - aggregation object
+        * @param {object} changes - object referencing removed stages from pipeline
         * */
-        function aggregate(collection, aggregation) {
+        function aggregate(collection, aggregation, changes) {
             if (params.qstring.iDisplayLength) {
                 aggregation.push({ "$limit": parseInt(params.qstring.iDisplayLength) });
             }
@@ -339,6 +429,10 @@ var spawn = require('child_process').spawn,
                 else if (collection === 'auth_tokens') {
                     aggregation.splice(addProjectionAt, 0, {"$addFields": {"_id": "***redacted***"}});
                 }
+                else if ((collection === "events_data" || collection === "drill_events") && !params.member.global_admin) {
+                    var base_filter = getBaseAppFilter(params.member, dbNameOnParam, params.qstring.collection);
+                    aggregation.splice(0, 0, {"$match": base_filter});
+                }
                 // check task is already running?
                 taskManager.checkIfRunning({
                     db: dbs[dbNameOnParam],
@@ -375,7 +469,7 @@ var spawn = require('child_process').spawn,
                             },
                             outputData: function(aggregationErr, result) {
                                 if (!aggregationErr) {
-                                    common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: 0, iTotalDisplayRecords: 0, "aaData": result });
+                                    common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: 0, iTotalDisplayRecords: 0, "aaData": result, "removed": (changes || {}) });
                                 }
                                 else {
                                     common.returnMessage(params, 500, aggregationErr);
@@ -409,7 +503,12 @@ var spawn = require('child_process').spawn,
 
             if (appId) {
                 if (hasReadRight(FEATURE_NAME, appId, parameters.member)) {
-                    return dbUserHasAccessToCollection(parameters, collection, appId, callback);
+                    if (collection === "events_data" || collection === "drill_events") {
+                        return callback(true);
+                    }
+                    else {
+                        return dbUserHasAccessToCollection(parameters, collection, appId, callback);
+                    }
                 }
             }
             else {
@@ -485,10 +584,14 @@ var spawn = require('child_process').spawn,
                 }
                 else {
                     userHasAccess(params, params.qstring.collection, function(hasAccess) {
-                        if (hasAccess) {
+                        if (hasAccess || params.qstring.collection === "events_data" || params.qstring.collection === "drill_events") {
                             try {
                                 let aggregation = EJSON.parse(params.qstring.aggregation);
-                                aggregate(params.qstring.collection, aggregation);
+                                var changes = escapeNotAllowedAggregationStages(aggregation);
+                                if (changes && Object.keys(changes).length > 0) {
+                                    log.d("Removed stages from pipeline: ", JSON.stringify(changes));
+                                }
+                                aggregate(params.qstring.collection, aggregation, changes);
                             }
                             catch (e) {
                                 common.returnMessage(params, 500, 'Aggregation object is not valid.');
@@ -508,7 +611,7 @@ var spawn = require('child_process').spawn,
                 }
                 else {
                     userHasAccess(params, params.qstring.collection, function(hasAccess) {
-                        if (hasAccess) {
+                        if (hasAccess || params.qstring.collection === "events_data" || params.qstring.collection === "drill_events") {
                             dbGetCollection();
                         }
                         else {

plugins/dbviewer/frontend/public/javascripts/countly.views.js

@@ -534,6 +534,13 @@
                                 if (res.aaData.length) {
                                     self.fields = Object.keys(map);
                                 }
+                                if (res.removed && typeof res.removed === 'object' && Object.keys(res.removed).length > 0) {
+                                    self.removed = CV.i18n('dbviewer.removed-warning') + Object.keys(res.removed).join(", ");
+
+                                }
+                                else {
+                                    self.removed = "";
+                                }
                             }
                             if (err) {
                                 var message = CV.i18n('dbviewer.server-error');
@@ -559,7 +566,7 @@
                     }
                 },
                 updatePath: function(query) {
-                    window.location.hash = "#/manage/db/aggregate/" + this.db + "/" + this.collection + "/" + query;
+                    app.navigate("#/manage/db/aggregate/" + this.db + "/" + this.collection + "/" + query);
                 },
                 getCollectionName: function() {
                     var self = this;

plugins/dbviewer/frontend/public/localization/dbviewer.properties

@@ -36,6 +36,7 @@ dbviewer.generate-aggregate-report= Generate aggregate report
 dbviewer.back-to-dbviewer = Back to DB Viewer
 dbviewer.invalid-pipeline = Invalid pipeline object, please check pipeline input.
 dbviewer.server-error = There was a server error. There might be more information in logs.
+dbviewer.removed-warning = Some stages are removed from aggregation pipleine. Following stages are allowed only with global admin rights:
 dbviewer.not-found-data = Couldn't find any results
 dbviewer.execute-aggregation = Execute Aggregation on {0}
 dbviewer.prepare-new-aggregation = Prepare New Aggregation

plugins/dbviewer/frontend/public/templates/aggregate.html

@@ -25,6 +25,7 @@
                 </el-button>
             </div>
         </cly-section>
+            <cly-notification v-if="removed" class="bu-mb-5 cly-vue-events-all__alerts" :text="removed" ></cly-notification>
         <cly-section>
             <cly-datatable-n :force-loading="queryLoading" :rows="aggregationResult" :prevent-default-sort="true">
                 <template v-slot="scope">