Merge pull request #104 from Countly/CustomStorageOption

Custom Storage Option

Author: turtledreams

CHANGELOG.md

@@ -1,9 +1,10 @@
-## xx.xx.xx
+## 24.10.0
 - Default max segmentation value count changed from 30 to 100
-- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file.
-- Added a new init time config option (conf.storage_type) which can make user set among these storage options:
+- Mitigated an issue where SDK could create an unintended dump file
+- Added a new init time config option (conf.storage_type) which can make user set the SDK storage option:
   - File Storage
   - Memory Only Storage
+- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods
 
 ## 22.06.0
 - Fixed a bug where remote config requests were rejected

lib/countly-bulk.js

@@ -25,7 +25,7 @@ var cc = require("./countly-common");
 var BulkUser = require("./countly-bulk-user");
 var CountlyStorage = require("./countly-storage");
 
-const StorageTypes = cc.storageTypeEnums;
+CountlyBulk.StorageTypes = cc.storageTypeEnums;
 
 /**
  * @lends module:lib/countly-bulk
@@ -39,7 +39,6 @@ const StorageTypes = cc.storageTypeEnums;
  * @param {number} [conf.fail_timeout=60] - set time in seconds to wait after failed connection to server in seconds
  * @param {number} [conf.session_update=60] - how often in seconds should session be extended
  * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch
- * @param {boolean} [conf.persist_queue=false] - persistently store queue until processed, default is false if you want to keep queue in memory and process all in one process run
  * @param {boolean} [conf.force_post=false] - force using post method for all requests
  * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc
  * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request
@@ -50,6 +49,8 @@ const StorageTypes = cc.storageTypeEnums;
  * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
  * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
  * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
+ * @param {Object} conf.custom_storage_method - user given storage methods
+ * @param {boolean} [conf.persist_queue=false] - *DEPRECATED* persistent mode instead of using in-memory queue. Use storage_type and storage_path instead
  * @example
  * var server = new CountlyBulk({
  *   app_key: "{YOUR-API-KEY}",
@@ -75,8 +76,6 @@ function CountlyBulk(conf) {
     var maxBreadcrumbCount = 100;
     var maxStackTraceLinesPerThread = 30;
     var maxStackTraceLineLength = 200;
-    var storageType = StorageTypes.FILE;
-
     cc.debugBulk = conf.debug || false;
     if (!conf.app_key) {
         cc.log(cc.logLevelEnums.ERROR, "CountlyBulk, 'app_key' is missing.");
@@ -105,9 +104,13 @@ function CountlyBulk(conf) {
     conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
     conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
     conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
-    conf.storage_type = conf.storage_type || storageType;
 
-    CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue);
+    // bulk mode is memory only by default
+    if (typeof conf.storage_type === "undefined" && conf.persist_queue === false) {
+        conf.storage_type = CountlyBulk.StorageTypes.MEMORY;
+    }
+
+    CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.custom_storage_method);
 
     this.conf = conf;
     /**
@@ -599,6 +602,27 @@ function CountlyBulk(conf) {
     var requestQueue = CountlyStorage.storeGet("cly_req_queue", []);
     var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {});
     var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []);
+    /**
+     * getBulkEventQueue is a testing purposed method which returns the event queue object
+     * @returns {Object} eventQueue
+     */
+    this._getBulkEventQueue = function() {
+        return eventQueue;
+    };
+    /**
+     * getBulkRequestQueue is a testing purposed method which returns the request queue object
+     * @returns {Object} requestQueue
+     */
+    this._getBulkRequestQueue = function() {
+        return requestQueue;
+    };
+    /**
+     * getBulkQueue is a testing purposed method which returns the bulk queue object
+     * @returns {Object} bulkQueue
+     */
+    this._getBulkQueue = function() {
+        return bulkQueue;
+    };
 }
 
 module.exports = CountlyBulk;

lib/countly-common.js

@@ -54,7 +54,6 @@ var cc = {
     storageTypeEnums: {
         FILE: "file",
         MEMORY: "memory",
-        CUSTOM: "custom",
     },
 
     /**

lib/countly-storage.js

@@ -2,14 +2,50 @@ const fs = require('fs');
 const path = require('path');
 var cc = require("./countly-common");
 
+// Constants
+const defaultPath = "../data/"; // Default storage path
+const defaultBulkPath = "../bulk_data/"; // Default bulk storage path
+const StorageTypes = cc.storageTypeEnums;
+const defaultStorageType = StorageTypes.FILE;
+const customTypeName = "custom";
+
 var storagePath;
-var __data = {};
-var defaultPath = "../data/"; // Default path
-var defaultBulkPath = "../bulk_data/"; // Default path
+let storageMethod = {};
+var __cache = {};
 var asyncWriteLock = false;
 var asyncWriteQueue = [];
-let storageMethod = {};
-const StorageTypes = cc.storageTypeEnums;
+
+/**
+ * Sets the storage method, by default sets file storage and storage path.
+ * @param {String} userPath - User provided storage path
+ * @param {StorageTypes} storageType - Whether to use memory only storage or not
+ * @param {Boolean} isBulk - Whether the storage is for bulk data
+ * @param {varies} customStorageMethod - Storage methods provided by the user
+ */
+var initStorage = function(userPath, storageType, isBulk = false, customStorageMethod = null) {
+    cc.log(cc.logLevelEnums.INFO, `Initializing storage with userPath: [${userPath}], storageType: [${storageType}], isBulk: [${isBulk}], customStorageMethod type: [${typeof customStorageMethod}].`);
+
+    // set storage type
+    storageType = storageType || defaultStorageType;
+    storageMethod = fileStorage; // file storage is default
+
+    if (storageType === StorageTypes.MEMORY) {
+        storageMethod = memoryStorage;
+        cc.log(cc.logLevelEnums.DEBUG, `Using memory storage!`);
+    }
+
+    // at this point we either use memory or file storage. If custom storage is provided, check if it is valid and use it instead
+    if (isCustomStorageValid(customStorageMethod)) {
+        storageMethod = customStorageMethod;
+        storageType = customTypeName;
+        cc.log(cc.logLevelEnums.DEBUG, `Using custom storage!`);
+    }
+
+    // set storage path if not memory storage
+    if (storageType !== StorageTypes.MEMORY) {
+        setStoragePath(userPath, isBulk);
+    }
+};
 
 // Memory-only storage methods
 const memoryStorage = {
@@ -22,7 +58,7 @@ const memoryStorage = {
     storeSet: function(key, value, callback) {
         if (key) {
             cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`);
-            __data[key] = value;
+            __cache[key] = value;
             if (typeof callback === "function") {
                 callback(null);
             }
@@ -39,14 +75,14 @@ const memoryStorage = {
      */
     storeGet: function(key, def) {
         cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`);
-        return typeof __data[key] !== "undefined" ? __data[key] : def;
+        return typeof __cache[key] !== "undefined" ? __cache[key] : def;
     },
     /**
      *  Remove value from memory
      *  @param {String} key - key of value to remove
      */
     storeRemove: function(key) {
-        delete __data[key];
+        delete __cache[key];
     },
 };
 
@@ -59,7 +95,7 @@ const fileStorage = {
      *  @param {Function} callback - callback to call when done storing
      */
     storeSet: function(key, value, callback) {
-        __data[key] = value;
+        __cache[key] = value;
         if (!asyncWriteLock) {
             asyncWriteLock = true;
             writeFile(key, value, callback);
@@ -76,7 +112,7 @@ const fileStorage = {
      */
     storeGet: function(key, def) {
         cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
-        if (typeof __data[key] === "undefined") {
+        if (typeof __cache[key] === "undefined") {
             var ob = readFile(key);
             var obLen;
             // check if the 'read object' is empty or not
@@ -90,17 +126,17 @@ const fileStorage = {
 
             // if empty or falsy set default value
             if (!ob || obLen === 0) {
-                __data[key] = def;
+                __cache[key] = def;
             }
             // else set the value read file has
             else {
-                __data[key] = ob[key];
+                __cache[key] = ob[key];
             }
         }
-        return __data[key];
+        return __cache[key];
     },
     storeRemove: function(key) {
-        delete __data[key];
+        delete __cache[key];
         var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
         fs.access(filePath, fs.constants.F_OK, (accessErr) => {
             if (accessErr) {
@@ -119,35 +155,31 @@ const fileStorage = {
     },
 };
 
-/**
- * Sets the storage method, by default sets file storage and storage path.
- * @param {String} userPath - User provided storage path
- * @param {StorageTypes} storageType - Whether to use memory only storage or not
- * @param {Boolean} isBulk - Whether the storage is for bulk data
- * @param {Boolean} persistQueue - Whether to persist the queue until processed
- */
-var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) {
-    if (storageType === StorageTypes.MEMORY) {
-        storageMethod = memoryStorage;
+var isCustomStorageValid = function(storage) {
+    if (!storage) {
+        return false;
+    }
+    if (typeof storage.storeSet !== 'function') {
+        return false;
+    }
+    if (typeof storage.storeGet !== 'function') {
+        return false;
     }
-    else {
-        storageMethod = fileStorage;
-        setStoragePath(userPath, isBulk, persistQueue);
+    if (typeof storage.storeRemove !== 'function') {
+        return false;
     }
+    return true;
 };
 
 /**
  * Sets the storage path, defaulting to a specified path if none is provided.
  * @param {String} userPath - User provided storage path
  * @param {Boolean} isBulk - Whether the storage is for bulk data
- * @param {Boolean} persistQueue - Whether to persist the queue until processed
  */
-var setStoragePath = function(userPath, isBulk = false, persistQueue = false) {
+var setStoragePath = function(userPath, isBulk = false) {
     storagePath = userPath || (isBulk ? defaultBulkPath : defaultPath);
 
-    if (!isBulk || persistQueue) {
-        createDirectory(path.resolve(__dirname, storagePath));
-    }
+    createDirectory(path.resolve(__dirname, storagePath));
 };
 
 /**
@@ -178,9 +210,10 @@ var createDirectory = function(dir) {
  */
 var resetStorage = function() {
     storagePath = undefined;
-    __data = {};
+    __cache = {};
     asyncWriteLock = false;
     asyncWriteQueue = [];
+    storageMethod = {};
 };
 
 /**
@@ -226,10 +259,10 @@ var readFile = function(key) {
  *  Force store data synchronously on unrecoverable errors to preserve it for next launch
  */
 var forceStore = function() {
-    for (var i in __data) {
+    for (var i in __cache) {
         var dir = path.resolve(__dirname, `${getStoragePath()}__${i}.json`);
         var ob = {};
-        ob[i] = __data[i];
+        ob[i] = __cache[i];
         try {
             fs.writeFileSync(dir, JSON.stringify(ob));
         }
@@ -292,7 +325,12 @@ var getStorageType = function() {
     if (storageMethod === memoryStorage) {
         return StorageTypes.MEMORY;
     }
-    return StorageTypes.FILE;
+
+    if (storageMethod === fileStorage) {
+        return StorageTypes.FILE;
+    }
+
+    return null;
 };
 
 module.exports = {

lib/countly.js

@@ -29,8 +29,7 @@ var Bulk = require("./countly-bulk");
 var CountlyStorage = require("./countly-storage");
 
 var Countly = {};
-const StorageTypes = cc.storageTypeEnums;
-
+Countly.StorageTypes = cc.storageTypeEnums;
 Countly.Bulk = Bulk;
 (function() {
     var SDK_VERSION = "24.10.0";
@@ -72,7 +71,7 @@ Countly.Bulk = Bulk;
     var maxStackTraceLinesPerThread = 30;
     var maxStackTraceLineLength = 200;
     var deviceIdType = null;
-    var storageType = StorageTypes.FILE;
+    var heartBeatTimer = null;
     /**
     * Array with list of available features that you can require consent for
     */
@@ -124,6 +123,7 @@ Countly.Bulk = Bulk;
  * @param {string} conf.metrics._locale - locale or language of the device in ISO format
  * @param {string} conf.metrics._store - source from where the user/device/installation came from
  * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
+ * @param {Object} conf.custom_storage_method - user given storage methods
  * @example
  * Countly.init({
  *   app_key: "{YOUR-APP-KEY}",
@@ -168,11 +168,12 @@ Countly.Bulk = Bulk;
             Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount;
             Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
             Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength;
-            conf.storage_type = conf.storage_type || storageType;
+            conf.storage_path = conf.storage_path || Countly.storage_path;
+            conf.storage_type = conf.storage_type || Countly.storage_type;
             // Common module debug value is set to init time debug value
             cc.debug = conf.debug;
 
-            CountlyStorage.initStorage(conf.storage_path, conf.storage_type);
+            CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, conf.custom_storage_method);
 
             // clear stored device ID if flag is set
             if (conf.clear_stored_device_id) {
@@ -320,6 +321,10 @@ Countly.Bulk = Bulk;
         maxStackTraceLinesPerThread = 30;
         maxStackTraceLineLength = 200;
         deviceIdType = null;
+        if (heartBeatTimer) {
+            clearInterval(heartBeatTimer);
+            heartBeatTimer = null;
+        }
 
         // cc DEBUG
         cc.debug = false;
@@ -334,6 +339,7 @@ Countly.Bulk = Bulk;
 
         // device_id
         Countly.device_id = undefined;
+        Countly.device_id_type = undefined;
         Countly.remote_config = undefined;
         Countly.require_consent = false;
         Countly.debug = undefined;
@@ -1478,7 +1484,7 @@ Countly.Bulk = Bulk;
             }, "heartBeat", false);
         }
 
-        setTimeout(heartBeat, beatInterval);
+        heartBeatTimer = setTimeout(heartBeat, beatInterval);
     }
 
     /**