Skip to content

Commit e96415b

Browse files
ar2rsawseenar2rsawseen
authored
Merge pull request #470 from Cookiezaurs/GDPR
[core]app_users saving exports in gridfs
2 parents 36c7d3c + ff89d35 commit e96415b

File tree

3 files changed

+107
-55
lines changed

3 files changed

+107
-55
lines changed

‎api/parts/mgmt/app_users.js

+89-43
Original file line numberDiff line numberDiff line change
@@ -144,23 +144,13 @@ var crypto = require('crypto');
144144
{
145145
for(var i=0;i<res[0].exported.length; i++)//delete exports if exist
146146
{
147-
if(res[0].exported[i].substr(res[0].exported[i].length-7)==".tar.gz")
148-
{
149-
if (fs.existsSync(res[0].exported[i])) {
150-
try {fs.unlinkSync(res[0].exported[i]);}catch(err){ console.log(err);}
151-
}
152-
}
153-
else
154-
{
155-
if(fs.existsSync(res[0].exported[i]))
156-
{
157-
fse.remove(res[0].exported[i],
158-
err => {
159-
if(err){console.log(err);}
160-
}
161-
);
162-
}
163-
}
147+
var id = res[0].exported[i].split("/");
148+
id = id.substr(id.length-7);
149+
150+
deleteMyExport(id).then(
151+
function(res){},
152+
function(err){ console.log(err);}
153+
);
164154
}
165155
}
166156
//deleting userimages(if they exist);
@@ -414,6 +404,47 @@ var crypto = require('crypto');
414404
};
415405

416406

407+
var deleteMyExport = function(exportID)
408+
{//tries to delete packed file, exported folder and saved export in gridfs
409+
//normally there should be only export in gridfs. Cleaning all to be sure.
410+
//rejects only if there stays any saved data for export
411+
return new Promise(function (resolve, reject) {
412+
//remove archive
413+
errors = [];
414+
if (fs.existsSync(path.resolve(__dirname,'./../../../export/AppUser/'+exportID+'.tar.gz'))) {
415+
try {fs.unlinkSync(path.resolve(__dirname,'./../../../export/AppUser/'+exportID+'.tar.gz'));}
416+
catch(err){ errors.push(err);}
417+
}
418+
419+
countlyFs.gridfs.deleteFile("appUsers", path.resolve(__dirname,'./../../../export/AppUser/'+exportID+'.tar.gz'), {id:exportID+'.tar.gz'}, function(error)
420+
{
421+
if(error && error.message && error.message.substring(0,12)!="FileNotFound")
422+
errors.push(error.message);
423+
if(fs.existsSync(path.resolve(__dirname,'./../../../export/AppUser/'+exportID)))
424+
{
425+
fse.remove(path.resolve(__dirname,'./../../../export/AppUser/'+exportID),
426+
err => {
427+
if(err)
428+
errors.push(error);
429+
if(errors.length==0)
430+
resolve();
431+
else
432+
reject(errors);
433+
}
434+
);
435+
}
436+
else
437+
{
438+
if(errors.length==0)
439+
resolve();
440+
else
441+
reject(errors);
442+
}
443+
});
444+
445+
});
446+
}
447+
417448
usersApi.deleteExport=function(filename,params,callback)
418449
{
419450
if(filename && filename!='')
@@ -429,25 +460,7 @@ var crypto = require('crypto');
429460
else
430461
{
431462
//remove archive
432-
if (fs.existsSync(path.resolve(__dirname,'./../../../export/AppUser/'+base_name[0]+'.tar.gz'))) {
433-
try {fs.unlinkSync(path.resolve(__dirname,'./../../../export/AppUser/'+base_name[0]+'.tar.gz'));}catch(err){ callback(err,"");}
434-
}
435-
436-
new Promise(function (resolve, reject) {
437-
if(fs.existsSync(path.resolve(__dirname,'./../../../export/AppUser/'+base_name[0])))
438-
{
439-
fse.remove(path.resolve(__dirname,'./../../../export/AppUser/'+base_name[0]),
440-
err => {
441-
if(err)
442-
reject(err);
443-
else
444-
resolve();
445-
}
446-
);
447-
}
448-
else
449-
resolve();
450-
}).then(
463+
deleteMyExport(base_name[0]).then(
451464
function(res)
452465
{
453466
if(name_parts.length==3 && name_parts[2]!='HASH')
@@ -471,9 +484,10 @@ var crypto = require('crypto');
471484
},
472485
function(err)
473486
{
474-
callback(err,"");
487+
console.log(err);
488+
callback("There was some errors during deleting export. Please look in log for more information","");
475489
}
476-
);
490+
);
477491
}
478492
}
479493
else
@@ -615,8 +629,27 @@ var crypto = require('crypto');
615629
Promise.all(commands).then(
616630
function(result) {
617631
//pack export
618-
clear_out_empty_files(path.resolve(__dirname,'./../../../export/AppUser/'+export_filename))
619-
.then(function(){ return run_command("tar -cvf "+export_filename+".tar.gz"+" "+export_filename)})
632+
clear_out_empty_files(path.resolve(__dirname,'./../../../export/AppUser/'+export_filename))//remove empty files
633+
.then(function(){ return run_command("tar -cvf "+export_filename+".tar.gz"+" "+export_filename)}) //create archive
634+
.then(function(){
635+
return new Promise(function(resolve, reject){/*save export in gridFS*/
636+
var my_filename = path.resolve(__dirname,'./../../../export/AppUser/'+export_filename+'.tar.gz')
637+
countlyFs.gridfs.saveFile("appUsers", my_filename, my_filename, {id:export_filename+".tar.gz",writeMode:"overwrite"}, function(err){
638+
if(err)
639+
{
640+
console.log(err);
641+
reject("unable to store exported file. There is more information in log.");
642+
}
643+
else
644+
{
645+
//remove archive, because it is saved in db.
646+
try {fs.unlinkSync(my_filename);}
647+
catch(err){ console.log(err);}
648+
resolve();//resolve anyway because export is still OK
649+
}
650+
});
651+
});
652+
})
620653
.then(
621654
function(result) {
622655
fse.remove(export_folder, err => {
@@ -643,10 +676,23 @@ var crypto = require('crypto');
643676
});
644677
},
645678
function(error) {
646-
plugins.dispatch("/systemlogs", {params:params, action:"export_app_user", data:{result:"error",uids:res[0].uid.join(", "),app_id:app_id,info:"Error during packing exported files",export_folder:export_folder}});
647-
callback({message:"Export successful. Unable to pack exported files.",filename:export_filename},"");
679+
plugins.dispatch("/systemlogs", {params:params, action:"export_app_user", data:{result:"error",uids:res[0].uid.join(", "),app_id:app_id,info:err,export_folder:export_folder}});
680+
usersApi.deleteExport(export_filename,params,function(err,msg){
681+
if(err)
682+
callback({mesage:"Storing exported files failed. Unable to clean up file system.",filename:'appUser_'+app_id+'_'+eid},"");
683+
else
684+
callback({mesage:"Storing exported files failed. Partially exported data deleted.",filename:'appUser_'+app_id+'_'+eid},"");
685+
});
648686
}
649-
);
687+
).catch(err => {
688+
plugins.dispatch("/systemlogs", {params:params, action:"export_app_user", data:{result:"error",info:err}});
689+
usersApi.deleteExport(export_filename,params,function(err,msg){
690+
if(err)
691+
callback({mesage:"Storing exported files failed. Unable to clean up file system.",filename:'appUser_'+app_id+'_'+eid},"");
692+
else
693+
callback({mesage:"Storing exported files failed. Partially exported data deleted.",filename:'appUser_'+app_id+'_'+eid},"");
694+
});
695+
});
650696
},
651697
function(error) {
652698
console.log(error);

‎api/utils/requestProcessor.js

+18-11
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ const plugins = require('../../plugins/pluginManager.js');
1414
const versionInfo = require('../../frontend/express/version.info');
1515
const log = require('./log.js')('core:api');
1616
const fs = require('fs');
17+
var countlyFs = require('./countlyFs.js');
1718
var path = require('path');
1819
const validateUserForWriteAPI = validateUser;
1920
const validateUserForDataReadAPI = validateUserForRead;
@@ -1012,20 +1013,26 @@ const processRequest = (params) => {
10121013
validateUserForRead(params, function(){
10131014
var filename = paths[4].split('.');
10141015
var myfile = '../../export/AppUser/'+filename[0]+'.tar.gz';
1015-
fs.stat(myfile,function(err,stat)
1016-
{
1017-
if(err)
1018-
{
1016+
1017+
countlyFs.gridfs.getSize("appUsers", myfile, {id:filename[0]+'.tar.gz'}, function(error,size){
1018+
if(error)
1019+
common.returnMessage(params, 400, error);
1020+
else if(size==0)
10191021
common.returnMessage(params, 400, "Export doesn't exist");
1020-
}
10211022
else
10221023
{
1023-
var readStream = fs.createReadStream(myfile);
1024-
params.res.writeHead(200, {
1025-
'Content-Type': 'application/x-gzip',
1026-
'Content-Length': stat.size
1027-
});
1028-
readStream.pipe(params.res);
1024+
countlyFs.gridfs.getStream("appUsers", myfile, {id:filename[0]+'.tar.gz'}, function(error,stream){
1025+
if(error)
1026+
common.returnMessage(params, 400, "Export doesn't exist");
1027+
else
1028+
{
1029+
params.res.writeHead(200, {
1030+
'Content-Type': 'application/x-gzip',
1031+
'Content-Length': size
1032+
});
1033+
stream.pipe(params.res);
1034+
}
1035+
});
10291036
}
10301037
});
10311038
});

‎plugins/data_migration/api/data_migration_helper.js

-1
Original file line numberDiff line numberDiff line change
@@ -1064,7 +1064,6 @@ module.exports = function(my_db){
10641064
log_me(my_logpath,"Files packed",false);
10651065
if(params.qstring.only_export && params.qstring.only_export==true)
10661066
{
1067-
update_progress(exportid,"packing","progress",100,"",true,{},params);
10681067
log_me(my_logpath,"Starting clean up",false);
10691068
self.clean_up_data('export',exportid,false).then(
10701069
function(result){

0 commit comments

Comments
 (0)