Skip to content

Commit 0d33cb3

Browse files
ar2rsawseenar2rsawseen
authored
Merge branch 'next' into initial-screen
2 parents 7dbc209 + e24394b commit 0d33cb3

File tree

45 files changed

+1262
-344
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1262
-344
lines changed

‎api/lib/countly.common.js

+15-6
Original file line numberDiff line numberDiff line change
@@ -615,7 +615,7 @@ countlyCommon.extractTwoLevelData = function(db, rangeArray, clearFunction, data
615615
}
616616

617617
if (propertyNames.indexOf("u") !== -1 && Object.keys(tmpPropertyObj).length) {
618-
if (countlyCommon.periodObj.periodContainsToday && totalUserOverrideObj && totalUserOverrideObj[rangeArray[j]]) {
618+
if (totalUserOverrideObj && totalUserOverrideObj[rangeArray[j]]) {
619619

620620
tmpPropertyObj.u = totalUserOverrideObj[rangeArray[j]];
621621

@@ -1114,7 +1114,7 @@ countlyCommon.extractMetric = function(db, rangeArray, clearFunction, dataProper
11141114
}
11151115

11161116
if (propertyNames.indexOf("u") !== -1 && Object.keys(tmpPropertyObj).length) {
1117-
if (countlyCommon.periodObj.periodContainsToday && totalUserOverrideObj && totalUserOverrideObj[rangeArray[j]]) {
1117+
if (totalUserOverrideObj && totalUserOverrideObj[rangeArray[j]]) {
11181118

11191119
tmpPropertyObj.u = totalUserOverrideObj[rangeArray[j]];
11201120

@@ -1233,6 +1233,7 @@ countlyCommon.timeString = function(timespent) {
12331233
* @param {array} properties - array of all properties to extract
12341234
* @param {array} unique - array of all properties that are unique from properties array. We need to apply estimation to them
12351235
* @param {object} totalUserOverrideObj - using unique property as key and total_users estimation property as value for all unique metrics that we want to have total user estimation overridden
1236+
* @param {object} prevTotalUserOverrideObj - using unique property as key and total_users estimation property as value for all unique metrics that we want to have total user estimation overridden for previous period
12361237
* @returns {object} dashboard data object
12371238
* @example
12381239
* countlyCommon.getDashboardData(countlySession.getDb(), ["t", "n", "u", "d", "e", "p", "m"], ["u", "p", "m"], {u:"users"});
@@ -1247,7 +1248,7 @@ countlyCommon.timeString = function(timespent) {
12471248
* "m":{"total":86,"prev-total":0,"change":"NA","trend":"u","isEstimate":true}
12481249
* }
12491250
*/
1250-
countlyCommon.getDashboardData = function(data, properties, unique, totalUserOverrideObj) {
1251+
countlyCommon.getDashboardData = function(data, properties, unique, totalUserOverrideObj, prevTotalUserOverrideObj) {
12511252
/**
12521253
* Clear object, bu nulling out predefined properties, that does not exist
12531254
* @param {object} obj - object to clear
@@ -1372,17 +1373,25 @@ countlyCommon.getDashboardData = function(data, properties, unique, totalUserOve
13721373
}
13731374

13741375
//check if we can correct data using total users correction
1375-
if (_periodObj.periodContainsToday && totalUserOverrideObj) {
1376+
if (totalUserOverrideObj) {
13761377
for (let i = 0; i < unique.length; i++) {
13771378
if (current[unique[i]] && typeof totalUserOverrideObj[unique[i]] !== "undefined" && totalUserOverrideObj[unique[i]]) {
13781379
current[unique[i]] = totalUserOverrideObj[unique[i]];
13791380
}
13801381
}
13811382
}
13821383

1384+
if (prevTotalUserOverrideObj) {
1385+
for (let i = 0; i < unique.length; i++) {
1386+
if (previous[unique[i]] && typeof prevTotalUserOverrideObj[unique[i]] !== "undefined" && prevTotalUserOverrideObj[unique[i]]) {
1387+
previous[unique[i]] = prevTotalUserOverrideObj[unique[i]];
1388+
}
1389+
}
1390+
}
1391+
13831392
// Total users can't be less than new users
13841393
if (typeof current.u !== "undefined" && typeof current.n !== "undefined" && current.u < current.n) {
1385-
if (_periodObj.periodContainsToday && totalUserOverrideObj && typeof totalUserOverrideObj.u !== "undefined" && totalUserOverrideObj.u) {
1394+
if (totalUserOverrideObj && typeof totalUserOverrideObj.u !== "undefined" && totalUserOverrideObj.u) {
13861395
current.n = current.u;
13871396
}
13881397
else {
@@ -1409,7 +1418,7 @@ countlyCommon.getDashboardData = function(data, properties, unique, totalUserOve
14091418
}
14101419

14111420
//check if we can correct data using total users correction
1412-
if (_periodObj.periodContainsToday && totalUserOverrideObj) {
1421+
if (totalUserOverrideObj) {
14131422
for (let i = 0; i < unique.length; i++) {
14141423
if (dataArr[unique[i]] && typeof totalUserOverrideObj[unique[i]] !== "undefined" && totalUserOverrideObj[unique[i]]) {
14151424
dataArr[unique[i]].is_estimate = false;

‎api/lib/countly.model.js

+11-4
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,8 @@ countlyModel.create = function(fetchValue) {
7474
_metas = {},
7575
_uniques = ["u"],
7676
_metrics = ["t", "u", "n"],
77-
_totalUsersObj = {};
77+
_totalUsersObj = {},
78+
_prevTotalUsersObj = {};
7879

7980
/**
8081
* Reset/delete all retrieved metric data, like when changing app or selected time period
@@ -118,17 +119,23 @@ countlyModel.create = function(fetchValue) {
118119
* Set total user object for this metric to use for unique user correction
119120
* @memberof module:api/lib/countly.model~countlyMetric
120121
* @param {object} totalUsersObj - object with total user data from {@link module:api/parts/data/fetch.getTotalUsersObj}
122+
* @param {object} prevTotalUserObj - object with total user data from {@link module:api/parts/data/fetch.getTotalUsersObj} for previous period
121123
*/
122-
countlyMetric.setTotalUsersObj = function(totalUsersObj) {
124+
countlyMetric.setTotalUsersObj = function(totalUsersObj, prevTotalUserObj) {
123125
_totalUsersObj = totalUsersObj;
126+
_prevTotalUsersObj = prevTotalUserObj;
124127
};
125128

126129
/**
127130
* Get total user object for this metric to use for unique user correction
128131
* @memberof module:api/lib/countly.model~countlyMetric
132+
* @param {boolean} prev - get correction data for previous period
129133
* @returns {object} object with total user data from {@link module:api/parts/data/fetch.getTotalUsersObj}
130134
*/
131-
countlyMetric.getTotalUsersObj = function() {
135+
countlyMetric.getTotalUsersObj = function(prev) {
136+
if (prev) {
137+
return _prevTotalUsersObj;
138+
}
132139
return _totalUsersObj;
133140
};
134141

@@ -481,7 +488,7 @@ countlyModel.create = function(fetchValue) {
481488
*/
482489
countlyMetric.getNumber = function(metric) {
483490
metric = metric || _metrics[0];
484-
var data = countlyCommon.getDashboardData(this.getDb(), [metric], _uniques, {u: this.getTotalUsersObj().users}, this.clearObject);
491+
var data = countlyCommon.getDashboardData(this.getDb(), [metric], _uniques, {u: this.getTotalUsersObj().users}, {u: this.getTotalUsersObj(true).users});
485492
var ob = {};
486493
ob[metric] = metric;
487494
var sparkLines = countlyCommon.getSparklineData(this.getDb(), ob, function(obj) {

‎api/lib/countly.users.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ function create() {
3030
e: "events"
3131
};
3232
var ret = {};
33-
var data = countlyCommon.getDashboardData(countlySession.getDb(), ["t", "n", "u", "d", "e"], ["u"], {u: countlySession.getTotalUsersObj().users}, countlySession.clearObject);
33+
var data = countlyCommon.getDashboardData(countlySession.getDb(), ["t", "n", "u", "d", "e"], ["u"], {u: countlySession.getTotalUsersObj().users}, {u: countlySession.getTotalUsersObj(true).users});
3434
for (let i in data) {
3535
ret[map[i]] = data[i];
3636
}

‎api/parts/data/fetch.js

+109-94
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ fetch.fetchDashboard = function(params) {
324324
fetch.getTotalUsersObj("users", params, function(dbTotalUsersObj) {
325325
countlyCommon.setPeriod(period.period);
326326

327-
countlySession.setTotalUsersObj(fetch.formatTotalUsersObj(dbTotalUsersObj));
327+
countlySession.setTotalUsersObj(fetch.formatTotalUsersObj(dbTotalUsersObj), fetch.formatTotalUsersObj(dbTotalUsersObj, true));
328328

329329
var data = {
330330
out: period.out,
@@ -440,7 +440,7 @@ fetch.fetchAllApps = function(params) {
440440

441441
fetch.getTotalUsersObj("users", params, function(dbTotalUsersObj) {
442442
countlySession.setDb(usersDoc || {});
443-
countlySession.setTotalUsersObj(fetch.formatTotalUsersObj(dbTotalUsersObj));
443+
countlySession.setTotalUsersObj(fetch.formatTotalUsersObj(dbTotalUsersObj), fetch.formatTotalUsersObj(dbTotalUsersObj, true));
444444

445445
var sessionData = countlySession.getSessionData();
446446
var charts = {
@@ -553,7 +553,7 @@ fetch.fetchCountries = function(params) {
553553
fetch.getTotalUsersObj("countries", params, function(dbTotalUsersObj) {
554554
countlyCommon.setPeriod(period.period);
555555

556-
countlyLocation.setTotalUsersObj(fetch.formatTotalUsersObj(dbTotalUsersObj));
556+
countlyLocation.setTotalUsersObj(fetch.formatTotalUsersObj(dbTotalUsersObj), fetch.formatTotalUsersObj(dbTotalUsersObj, true));
557557

558558
var data = {
559559
out: period.out,
@@ -974,7 +974,7 @@ fetch.getTotalUsersObj = function(metric, params, callback) {
974974
};
975975

976976
/**
977-
* Get data for estimating total users count if period contains today with options
977+
* Get data for estimating total users count allowing plugins to add their own data
978978
* @param {string} metric - name of the collection where to get data from
979979
* @param {params} params - params object with app_id and date
980980
* @param {object=} options - additional optional settings
@@ -998,7 +998,7 @@ fetch.getTotalUsersObjWithOptions = function(metric, params, options, callback)
998998

999999
/*
10001000
List of shortcodes in app_users document for different metrics
1001-
*/
1001+
*/
10021002
var shortcodesForMetrics = {
10031003
"devices": "d",
10041004
"app_versions": "av",
@@ -1010,116 +1010,126 @@ fetch.getTotalUsersObjWithOptions = function(metric, params, options, callback)
10101010
"carriers": "c"
10111011
};
10121012

1013+
if (!params.time) {
1014+
params.time = common.initTimeObj(params.appTimezone, params.qstring.timestamp);
1015+
}
1016+
10131017
/*
1014-
This API endpoint /o?method=total_users should only be used if
1015-
selected period contains today
1016-
*/
1017-
if (periodObj.periodContainsToday) {
1018-
/*
1019-
Aggregation query uses this variable for $match operation
1020-
We skip uid-sequence document and filter results by last session timestamp
1021-
*/
1022-
var match = {ls: countlyCommon.getTimestampRangeQuery(params, true)};
1018+
Aggregation query uses this variable for $match operation
1019+
We skip uid-sequence document and filter results by last session timestamp
1020+
*/
1021+
var match = {ls: countlyCommon.getTimestampRangeQuery(params, true)};
10231022

1024-
/*
1025-
Let plugins register their short codes and match queries
1026-
*/
1027-
plugins.dispatch("/o/method/total_users", {
1028-
shortcodesForMetrics: shortcodesForMetrics,
1029-
match: match
1030-
});
1023+
/*
1024+
Let plugins register their short codes and match queries
1025+
*/
1026+
plugins.dispatch("/o/method/total_users", {
1027+
shortcodesForMetrics: shortcodesForMetrics,
1028+
match: match
1029+
});
10311030

1032-
/*
1033-
Aggregation query uses this variable for $group operation
1034-
If there is no corresponding shortcode default is to count all
1035-
users in this period
1036-
*/
1037-
var groupBy = (shortcodesForMetrics[metric]) ? "$" + shortcodesForMetrics[metric] : "users";
1031+
var ob = { params: params, period: periodObj, metric: metric, options: options, result: [], shortcodesForMetrics: shortcodesForMetrics, match: match};
10381032

1033+
plugins.dispatch("/estimation/correction", ob, function() {
10391034
/*
1040-
In app users we store city information even if user is not from
1041-
the selected timezone country of the app. We $match to get city
1042-
information only for users in app's configured country
1043-
*/
1044-
if (metric === "cities") {
1045-
match.cc = params.app_cc;
1046-
}
1047-
1048-
options.db.collection("app_users" + params.app_id).aggregate([
1049-
{$match: match},
1050-
{
1051-
$group: {
1052-
_id: groupBy,
1053-
u: { $sum: 1 }
1054-
}
1035+
If no plugin has returned any estimation corrections then
1036+
this API endpoint /o?method=total_users should only be used if
1037+
selected period contains today
1038+
*/
1039+
if (ob.result.length === 0 && periodObj.periodContainsToday) {
1040+
1041+
/*
1042+
Aggregation query uses this variable for $group operation
1043+
If there is no corresponding shortcode default is to count all
1044+
users in this period
1045+
*/
1046+
var groupBy = (shortcodesForMetrics[metric]) ? "$" + shortcodesForMetrics[metric] : "users";
1047+
1048+
/*
1049+
In app users we store city information even if user is not from
1050+
the selected timezone country of the app. We $match to get city
1051+
information only for users in app's configured country
1052+
*/
1053+
if (metric === "cities") {
1054+
match.cc = params.app_cc;
10551055
}
1056-
], { allowDiskUse: true }, function(error, appUsersDbResult) {
1057-
1058-
if (plugins.getConfig("api", params.app && params.app.plugins, true).metric_changes && shortcodesForMetrics[metric]) {
10591056

1060-
var metricChangesMatch = {ts: countlyCommon.getTimestampRangeQuery(params, true)};
1061-
1062-
metricChangesMatch[shortcodesForMetrics[metric] + ".o"] = { "$exists": true };
1063-
1064-
/*
1065-
We track changes to metrics such as app version in metric_changesAPPID collection;
1066-
{ "uid" : "2", "ts" : 1462028715, "av" : { "o" : "1:0:1", "n" : "1:1" } }
1067-
1068-
While returning a total user result for any metric, we check metric_changes to see
1069-
if any metric change happened in the selected period and include this in the result
1070-
*/
1071-
options.db.collection("metric_changes" + params.app_id).aggregate([
1072-
{$match: metricChangesMatch},
1073-
{
1074-
$group: {
1075-
_id: '$' + shortcodesForMetrics[metric] + ".o",
1076-
uniqDeviceIds: { $addToSet: '$uid'}
1077-
}
1078-
},
1079-
{$unwind: "$uniqDeviceIds"},
1080-
{
1081-
$group: {
1082-
_id: "$_id",
1083-
u: { $sum: 1 }
1084-
}
1057+
options.db.collection("app_users" + params.app_id).aggregate([
1058+
{$match: match},
1059+
{
1060+
$group: {
1061+
_id: groupBy,
1062+
u: { $sum: 1 }
10851063
}
1086-
], { allowDiskUse: true }, function(err, metricChangesDbResult) {
1064+
}
1065+
], { allowDiskUse: true }, function(error, appUsersDbResult) {
1066+
1067+
if (plugins.getConfig("api", params.app && params.app.plugins, true).metric_changes && shortcodesForMetrics[metric]) {
1068+
1069+
var metricChangesMatch = {ts: countlyCommon.getTimestampRangeQuery(params, true)};
1070+
1071+
metricChangesMatch[shortcodesForMetrics[metric] + ".o"] = { "$exists": true };
1072+
1073+
/*
1074+
We track changes to metrics such as app version in metric_changesAPPID collection;
1075+
{ "uid" : "2", "ts" : 1462028715, "av" : { "o" : "1:0:1", "n" : "1:1" } }
1076+
1077+
While returning a total user result for any metric, we check metric_changes to see
1078+
if any metric change happened in the selected period and include this in the result
1079+
*/
1080+
options.db.collection("metric_changes" + params.app_id).aggregate([
1081+
{$match: metricChangesMatch},
1082+
{
1083+
$group: {
1084+
_id: '$' + shortcodesForMetrics[metric] + ".o",
1085+
uniqDeviceIds: { $addToSet: '$uid'}
1086+
}
1087+
},
1088+
{$unwind: "$uniqDeviceIds"},
1089+
{
1090+
$group: {
1091+
_id: "$_id",
1092+
u: { $sum: 1 }
1093+
}
1094+
}
1095+
], { allowDiskUse: true }, function(err, metricChangesDbResult) {
10871096

1088-
if (metricChangesDbResult) {
1089-
var appUsersDbResultIndex = _.pluck(appUsersDbResult, '_id');
1097+
if (metricChangesDbResult) {
1098+
var appUsersDbResultIndex = _.pluck(appUsersDbResult, '_id');
10901099

1091-
for (let i = 0; i < metricChangesDbResult.length; i++) {
1092-
var itemIndex = appUsersDbResultIndex.indexOf(metricChangesDbResult[i]._id);
1100+
for (let i = 0; i < metricChangesDbResult.length; i++) {
1101+
var itemIndex = appUsersDbResultIndex.indexOf(metricChangesDbResult[i]._id);
10931102

1094-
if (itemIndex === -1) {
1095-
appUsersDbResult.push(metricChangesDbResult[i]);
1096-
}
1097-
else {
1098-
appUsersDbResult[itemIndex].u += metricChangesDbResult[i].u;
1103+
if (itemIndex === -1) {
1104+
appUsersDbResult.push(metricChangesDbResult[i]);
1105+
}
1106+
else {
1107+
appUsersDbResult[itemIndex].u += metricChangesDbResult[i].u;
1108+
}
10991109
}
11001110
}
1101-
}
1102-
1111+
callback(appUsersDbResult);
1112+
});
1113+
}
1114+
else {
11031115
callback(appUsersDbResult);
1104-
});
1105-
}
1106-
else {
1107-
callback(appUsersDbResult);
1108-
}
1109-
});
1110-
}
1111-
else {
1112-
callback([]);
1113-
}
1116+
}
1117+
});
1118+
}
1119+
else {
1120+
callback(ob.result);
1121+
}
1122+
});
11141123
};
11151124

11161125
/**
11171126
* Format total users object based on propeties it has (converting short metric values to long proper ones, etc)
11181127
* @param {object} obj - total users object
11191128
* @param {string} forMetric - for which metric to format result
1129+
* @param {boolean} prev - get data for previous period, if available
11201130
* @returns {object} total users object with formated values
11211131
**/
1122-
fetch.formatTotalUsersObj = function(obj, forMetric) {
1132+
fetch.formatTotalUsersObj = function(obj, forMetric, prev) {
11231133
var tmpObj = {},
11241134
processingFunction;
11251135

@@ -1133,7 +1143,12 @@ fetch.formatTotalUsersObj = function(obj, forMetric) {
11331143
for (let i = 0; i < obj.length; i++) {
11341144
var tmpKey = (processingFunction) ? processingFunction(obj[i]._id) : obj[i]._id;
11351145

1136-
tmpObj[tmpKey] = obj[i].u;
1146+
if (prev) {
1147+
tmpObj[tmpKey] = obj[i].pu || 0;
1148+
}
1149+
else {
1150+
tmpObj[tmpKey] = obj[i].u;
1151+
}
11371152
}
11381153
}
11391154

0 commit comments

Comments
 (0)