Skip to content

Commit 0e8d0ee

Browse files
ArtursKadikisArtursKadikis
authored
Merge branch 'master' into SER-2110
2 parents 7fd42a0 + f9fa851 commit 0e8d0ee

File tree

13 files changed

+199
-46
lines changed

13 files changed

+199
-46
lines changed

‎CHANGELOG.md

+4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## Version 24.05.XX
2+
Enterprise fixes:
3+
- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
4+
15
## Version 24.05.16
26
Fixes:
37
- [core] Replaced "Users" with "Sessions" label on technology home widgets

‎bin/scripts/expire-data/delete_custom_events_regex.js

+23-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
*/
77

88

9-
const { ObjectId } = require('mongodb');
109
const pluginManager = require('../../../plugins/pluginManager.js');
1110
const common = require('../../../api/utils/common.js');
1211
const drillCommon = require('../../../plugins/drill/api/common.js');
@@ -25,7 +24,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
2524

2625
//GET APP
2726
try {
28-
const app = await countlyDb.collection("apps").findOne({_id: ObjectId(APP_ID)}, {_id: 1, name: 1});
27+
const app = await countlyDb.collection("apps").findOne({_id: countlyDb.ObjectID(APP_ID)}, {_id: 1, name: 1});
2928
console.log("App:", app.name);
3029
//GET EVENTS
3130
var events = [];
@@ -51,6 +50,27 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
5150
}
5251
]).toArray();
5352
events = events.length ? events[0].list : [];
53+
const metaEvents = await drillDb.collection("drill_meta").aggregate([
54+
{
55+
$match: {
56+
'app_id': app._id + "",
57+
"type": "e",
58+
"e": { $regex: regex, $options: CASE_INSENSITIVE ? "i" : "", $nin: events }
59+
}
60+
},
61+
{
62+
$group: {
63+
_id: "$e"
64+
}
65+
},
66+
{
67+
$project: {
68+
_id: 0,
69+
e: "$_id"
70+
}
71+
}
72+
]).toArray();
73+
events = events.concat(metaEvents.map(e => e.e));
5474
}
5575
catch (err) {
5676
close("Invalid regex");
@@ -86,6 +106,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
86106
close(err);
87107
}
88108

109+
89110
async function deleteDrillEvents(appId, events) {
90111
for (let i = 0; i < events.length; i++) {
91112
var collectionName = drillCommon.getCollectionName(events[i], appId);

‎plugins/push/README.md

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# Countly Push Plugin
2+
3+
These URLs need to be accessible by the Countly instance:
4+
5+
- for Android:
6+
- fcm.googleapis.com
7+
- accounts.google.com
8+
- for IOS:
9+
- api.development.push.apple.com
10+
- api.push.apple.com
11+
- for Huawei
12+
- push-api.cloud.huawei.com

‎plugins/push/api/api.js

+6
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,10 @@ plugins.register('/master', function() {
8383
common.dbUniqueMap.users.push(common.dbMap['messaging-enabled'] = DBMAP.MESSAGING_ENABLED);
8484
fields(platforms, true).forEach(f => common.dbUserMap[f] = f);
8585
PUSH.cache = common.cache.cls(PUSH_CACHE_GROUP);
86+
setTimeout(() => {
87+
const jobManager = require('../../../api/parts/jobs');
88+
jobManager.job("push:clear-stats").replace().schedule("at 3:00 am every 7 days");
89+
}, 10000);
8690
});
8791

8892
plugins.register('/master/runners', runners => {
@@ -361,6 +365,7 @@ plugins.register('/i/app_users/export', ({app_id, uids, export_commands, dbargs,
361365
* @apiDefine PushMessageBody
362366
*
363367
* @apiBody {ObjectID} app Application ID
368+
* @apiBody {Boolean} saveStats Store each individual push records into push_stats for debugging
364369
* @apiBody {String[]} platforms Array of platforms to send to
365370
* @apiBody {String="draft"} [status] Message status, only set to draft when creating or editing a draft message, don't set otherwise
366371
* @apiBody {Object} filter={} User profile filter to limit recipients of this message
@@ -410,6 +415,7 @@ plugins.register('/i/app_users/export', ({app_id, uids, export_commands, dbargs,
410415
*
411416
* @apiSuccess {ObjectID} _id Message ID
412417
* @apiSuccess {ObjectID} app Application ID
418+
* @apiSuccess {Boolean} saveStats Store each individual push records into push_stats for debugging
413419
* @apiSuccess {String[]} platforms Array of platforms to send to
414420
* @apiSuccess {Number} state Message state, for internal use
415421
* @apiSuccess {String="created", "inactive", "draft", "scheduled", "sending", "sent", "stopped", "failed"} [status] Message status: "created" is for messages yet to be scheduled (put into queue), "inactive" - cannot be scheduled (approval required for push approver plugin), "draft", "scheduled", "sending", "sent", "stopped" - automated message has been stopped, "failed" - failed to send all notifications

‎plugins/push/api/jobs/clear-stats.js

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
'use strict';
2+
/**
3+
* @typedef {import("mongodb").Db} MongoDb
4+
*/
5+
6+
const { Job } = require('../../../../api/parts/jobs/job.js');
7+
const log = require('../../../../api/utils/log.js')('job:push:clear-stats');
8+
9+
const EXPIRY = 30 * 24 * 60 * 60 * 1000; // 30 days
10+
11+
/**
12+
* Clears push_stats collection
13+
*/
14+
class ClearStatsJob extends Job {
15+
/**
16+
* Clears push_stats based on EXPIRY and MAX_RECORDS
17+
* @param {MongoDb} db - db connection
18+
*/
19+
async run(db) {
20+
log.d('Clearing push_stats');
21+
await db.collection("push_stats").deleteMany({
22+
d: { $lte: new Date(Date.now() - EXPIRY) }
23+
});
24+
}
25+
}
26+
27+
module.exports = ClearStatsJob;

‎plugins/push/api/jobs/util/resultor.js

+69-29
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,23 @@
11
const { FRAME, FRAME_NAME } = require('../../send/proto'),
22
{ DoFinish } = require('./do_finish'),
33
{ ERROR, TriggerKind, State, Status, PushError, Result } = require('../../send/data');
4+
/**
5+
* @typedef {import("mongodb").ObjectId} ObjectId
6+
*/
7+
8+
/**
9+
* PushStat object (collection: push_stats)
10+
* @typedef {Object} PushStat
11+
* @property {ObjectId} a - application id
12+
* @property {ObjectId} m - message id from "messages" collection
13+
* @property {string} u - uid from app_users{appId}
14+
* @property {string} t - token from "push_{appId}" collection
15+
* @property {string=} r - id returned from provider
16+
* @property {Date} d - date this message sent to this user
17+
* @property {string=} e - error message
18+
* @property {string} p - platform: "a" for android, "i" for ios and "h" for huawei
19+
* @property {string} f - token type: "p" for production
20+
*/
421

522
/**
623
* Stream responsible for handling sending results:
@@ -34,7 +51,10 @@ class Resultor extends DoFinish {
3451
this.fatalErrors = {}; // {mid: []}
3552
this.toDelete = []; // [push id, push id, ...]
3653
this.count = 0; // number of results cached
37-
this.last = null; // time of last data from
54+
this.last = null; // time of last data from
55+
56+
/** @type {PushStat[]} */
57+
this.pushStats = [];
3858

3959
this.data.on('app', app => {
4060
this.changed[app._id] = {};
@@ -118,11 +138,17 @@ class Resultor extends DoFinish {
118138
if (id < 0) {
119139
return;
120140
}
121-
let {p, m, pr} = this.data.pushes[id],
141+
const p = this.data.pushes[id];
142+
let {p: platform, m, pr} = p,
122143
msg = this.data.message(m),
123144
result,
124145
rp, rl;
125146

147+
// additional fields to keep this in push_stats
148+
if (msg && msg.saveStats) {
149+
this.pushStats.push({ a: p.a, m: p.m, p: p.p, f: p.f, u: p.u, t: p.t, d: new Date, r: null, e: results.toString() });
150+
}
151+
126152
if (msg) {
127153
result = msg.result;
128154
result.lastRun.processed++;
@@ -131,7 +157,7 @@ class Resultor extends DoFinish {
131157
else {
132158
result = this.noMessage[m] || (this.noMessage[m] = new Result());
133159
}
134-
rp = result.sub(p, undefined, PLATFORM[p].parent);
160+
rp = result.sub(platform, undefined, PLATFORM[platform].parent);
135161
rl = rp.sub(pr.la || 'default');
136162

137163
result.processed++;
@@ -141,8 +167,8 @@ class Resultor extends DoFinish {
141167
rl.recordError(results.message, 1);
142168
rl.processed++;
143169

144-
if (PLATFORM[p].parent) {
145-
rp = result.sub(PLATFORM[p].parent),
170+
if (PLATFORM[platform].parent) {
171+
rp = result.sub(PLATFORM[platform].parent),
146172
rl = rp.sub(pr.la || 'default');
147173
rp.recordError(results.message, 1);
148174
rp.processed++;
@@ -159,29 +185,39 @@ class Resultor extends DoFinish {
159185
}
160186
else {
161187
results.forEach(res => {
162-
let id, token;
163-
if (typeof res === 'string') {
164-
this.log.d('Ok for %s', id);
165-
id = res;
166-
}
167-
else {
188+
let id, resultId, token;
189+
190+
if (Array.isArray(res)) {
168191
this.log.d('New token for %s', id);
169192
id = res[0];
170193
token = res[1];
171194
}
195+
else {
196+
id = res;
197+
}
198+
199+
if (typeof id !== "string") {
200+
resultId = id.r;
201+
id = id.p;
202+
}
172203

173204
let p = this.data.pushes[id];
174205
if (!p) { // 2 or more resultors on one pool
175206
return;
176207
}
177208

178-
this.data.decSending(p.m);
179-
180-
let m = this.data.message(p.m),
209+
let msg = this.data.message(p.m),
181210
result, rp, rl;
182211

183-
if (m) {
184-
result = m.result;
212+
// additional fields to keep this in push_stats
213+
if (msg && msg.saveStats) {
214+
this.pushStats.push({ a: p.a, m: p.m, p: p.p, f: p.f, u: p.u, t: p.t, d: new Date, r: resultId, e: null });
215+
}
216+
217+
this.data.decSending(p.m);
218+
219+
if (msg) {
220+
result = msg.result;
185221
result.lastRun.processed++;
186222
}
187223
else {
@@ -220,14 +256,6 @@ class Resultor extends DoFinish {
220256
});
221257
this.log.d('Added %d results', results.length);
222258
}
223-
224-
// // in case no more data is expected, we can safely close the stream
225-
// if (this.check()) {
226-
// for (let _ in this.state.pushes) {
227-
// return;
228-
// }
229-
// this.do_flush(() => this.end());
230-
// }
231259
}
232260
else if (frame & FRAME.ERROR) {
233261
let error = results.messageError(),
@@ -241,28 +269,35 @@ class Resultor extends DoFinish {
241269
return;
242270
}
243271
this.log.d('Error %d %s for %s', results.type, results.name, id);
244-
let {m, p, pr} = this.data.pushes[id],
272+
const p = this.data.pushes[id];
273+
let {m, p: platform, pr} = p,
245274
result, rp, rl;
275+
let msg = this.data.message(m);
276+
277+
// additional fields to keep this in push_stats
278+
if (msg && msg.saveStats) {
279+
this.pushStats.push({ a: p.a, m: p.m, p: p.p, f: p.f, u: p.u, t: p.t, d: new Date, r: null, e: results.toString() });
280+
}
281+
246282
mids[m] = (mids[m] || 0) + 1;
247283
delete this.data.pushes[id];
248284
this.toDelete.push(id);
249285

250-
let msg = this.data.message(m);
251286
if (msg) {
252287
result = msg.result;
253288
}
254289
else {
255290
result = this.noMessage[m] || (this.noMessage[m] = new Result());
256291
}
257292

258-
rp = result.sub(p, undefined, PLATFORM[p].parent);
293+
rp = result.sub(platform, undefined, PLATFORM[platform].parent);
259294
rl = rp.sub(pr.la || 'default');
260295

261296
rp.processed++;
262297
rl.processed++;
263298

264-
if (PLATFORM[p].parent) {
265-
rp = result.sub(PLATFORM[p].parent),
299+
if (PLATFORM[platform].parent) {
300+
rp = result.sub(PLATFORM[platform].parent),
266301
rl = rp.sub(pr.la || 'default');
267302
rp.processed++;
268303
rl.processed++;
@@ -514,6 +549,11 @@ class Resultor extends DoFinish {
514549
}
515550
}
516551

552+
if (this.pushStats.length) {
553+
promises.push(this.db.collection("push_stats").insertMany(this.pushStats));
554+
this.pushStats = [];
555+
}
556+
517557
Promise.all(promises).then(() => {
518558
this.log.d('do_flush done');
519559
callback();

‎plugins/push/api/send/data/message.js

+27-3
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,14 @@ class Message extends Mongoable {
3333

3434
/**
3535
* Validation scheme of this class
36-
*
36+
*
3737
* @returns {object} validateArgs scheme
3838
*/
3939
static get scheme() {
4040
return {
4141
_id: { required: false, type: 'ObjectID' },
4242
app: { required: true, type: 'ObjectID' },
43+
saveStats: { required: false, type: 'Boolean' },
4344
platforms: { required: true, type: 'String[]', in: () => require('../platforms').platforms },
4445
state: { type: 'Number' },
4546
status: { type: 'String', in: Object.values(Status) },
@@ -148,9 +149,32 @@ class Message extends Mongoable {
148149
}
149150
}
150151

152+
/**
153+
* Getter for message.saveStats
154+
*
155+
* @returns {boolean} saveStats
156+
*/
157+
get saveStats() {
158+
return this._data.saveStats;
159+
}
160+
161+
/**
162+
* Setter for message.saveStats
163+
*
164+
* @param {boolean} value value to set
165+
*/
166+
set saveStats(value) {
167+
if (typeof value !== "boolean") {
168+
this._data.saveStats = false;
169+
}
170+
else {
171+
this._data.saveStats = value;
172+
}
173+
}
174+
151175
/**
152176
* Getter for platforms
153-
*
177+
*
154178
* @returns {string[]|undefined} platforms array
155179
*/
156180
get platforms() {
@@ -159,7 +183,7 @@ class Message extends Mongoable {
159183

160184
/**
161185
* Setter for platforms
162-
*
186+
*
163187
* @param {string[]|undefined} arr platforms array
164188
*/
165189
set platforms(arr) {

0 commit comments

Comments
 (0)