Skip to content

Commit 59ecd7e

Browse files
aborigeneaborigene
committed
Fix to log rotation manipulation
1 parent 82a71af commit 59ecd7e

File tree

2 files changed

+138
-28
lines changed

2 files changed

+138
-28
lines changed

‎src/main/java/com/appdynamics/extension/webdispatcher/webdispatcherExtension/ExtensionController.java

+15-10
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ class CoreActivities extends Thread implements Runnable
2525
Connection h2connection;
2626
String test_value;
2727
Logger LOGGER;
28-
public CoreActivities(balancingData bd, Connection h2con) {
28+
public CoreActivities(balancingData bd, Connection h2con, Logger LocalLogger) {
2929
CoreActivities_bd = bd;
3030
h2connection = h2con;
31+
LOGGER = LocalLogger;
3132
}
3233

33-
public CoreActivities(balancingData bd) {
34+
public CoreActivities(balancingData bd, Logger LocalLogger) {
3435
CoreActivities_bd = bd;
36+
LOGGER = LocalLogger;
3537
}
3638

3739
public void run() {
@@ -62,15 +64,17 @@ class DistributionCoeficient extends Thread
6264
Dictionary<String, String> URL_total_reqs;
6365
Dictionary<String, String> URL_host_total_reqs;
6466

65-
public DistributionCoeficient(balancingData bd, Connection h2con) {
67+
public DistributionCoeficient(balancingData bd, Connection h2con, Logger LocalLogger) {
6668
DistributionCoeficient_bd = bd;
6769
h2connection = h2con;
68-
LOGGER = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
70+
//LOGGER = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
71+
LOGGER = LocalLogger;
6972
}
7073

71-
public DistributionCoeficient(balancingData bd) {
74+
public DistributionCoeficient(balancingData bd, Logger LocalLogger) {
7275
DistributionCoeficient_bd = bd;
73-
LOGGER = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
76+
//LOGGER = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
77+
LOGGER = LocalLogger;
7478
}
7579

7680
public void run()
@@ -198,6 +202,7 @@ private static void setUpLogger(){
198202
try{
199203
FileHandler fileTxt = new FileHandler("webDispatcher.log", 10000000, 5, true);
200204
fileTxt.setFormatter(new SimpleFormatter());
205+
//fileTxt.ge
201206
LOGGER.addHandler(fileTxt);
202207
LOGGER.setLevel(Level.WARNING);
203208
LOGGER.setUseParentHandlers(false);//removeHandler(ConsoleHandler.class);
@@ -238,10 +243,10 @@ public static void main( String[] args )
238243
Statement stm = con.createStatement();
239244
stm.execute("create table distribution (method varchar(10) not null, url varchar(255) not null, target_host varchar(255) not null, request_number int not null)");
240245

241-
Runnable CoreActivitiesRunnable = new CoreActivities(balancingDataInstance, con);
246+
Runnable CoreActivitiesRunnable = new CoreActivities(balancingDataInstance, con, LOGGER);
242247
new Thread(CoreActivitiesRunnable).start();
243248

244-
Runnable DistributionCoeficient = new DistributionCoeficient(balancingDataInstance, con);
249+
Runnable DistributionCoeficient = new DistributionCoeficient(balancingDataInstance, con, LOGGER);
245250
new Thread(DistributionCoeficient).start();
246251

247252
} catch (SQLException ex) {
@@ -250,10 +255,10 @@ public static void main( String[] args )
250255
//lgr.log(Level.SEVERE, ex.getMessage(), ex);
251256
LOGGER.log(Level.WARNING, "H2 Database could not be initialized, balance distribution will not be possible: ", ex.getMessage());
252257

253-
Runnable CoreActivitiesRunnable = new CoreActivities(balancingDataInstance);
258+
Runnable CoreActivitiesRunnable = new CoreActivities(balancingDataInstance, LOGGER);
254259
new Thread(CoreActivitiesRunnable).start();
255260

256-
Runnable DistributionCoeficient = new DistributionCoeficient(balancingDataInstance);
261+
Runnable DistributionCoeficient = new DistributionCoeficient(balancingDataInstance, LOGGER);
257262
new Thread(DistributionCoeficient).start();
258263
}
259264
}

‎src/main/java/com/appdynamics/extension/webdispatcher/webdispatcherExtension/WebDispatcherLogRead.java

+123-18
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,36 @@
2626
import java.sql.ResultSet;
2727
import java.sql.SQLException;
2828
import java.sql.Statement;
29+
import java.util.Iterator;
2930
import java.util.NoSuchElementException;
31+
import java.util.logging.Handler;
3032
import java.util.logging.Level;
3133
import java.util.logging.Logger;
3234

3335
public class WebDispatcherLogRead{
3436

35-
Logger LOGGER;
37+
final private static Logger LOGGER = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
3638
String MetricRootProperty;
3739

38-
public WebDispatcherLogRead(String MetricRoot){
39-
LOGGER = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
40+
public WebDispatcherLogRead(String MetricRoot){
4041
MetricRootProperty = MetricRoot;
42+
setUpLogger();
4143
}
4244

45+
private static void setUpLogger(){
46+
try{
47+
FileHandler fileTxt = new FileHandler("webDispatcherLogReader.log", 10000000, 5, true);
48+
fileTxt.setFormatter(new SimpleFormatter());
49+
//fileTxt.ge
50+
LOGGER.addHandler(fileTxt);
51+
LOGGER.setLevel(Level.INFO);
52+
LOGGER.setUseParentHandlers(false);//removeHandler(ConsoleHandler.class);
53+
}
54+
catch (IOException e){
55+
LOGGER.log(Level.WARNING, "Unable to write log: {0}", e.getMessage());
56+
}
57+
}
58+
4359
private boolean clearReadProperties(ConfigReader configuration){
4460
try{
4561
Properties writeProperties = new Properties();
@@ -366,7 +382,6 @@ public void processFile(Path path, Long initialPosition, ConfigReader configurat
366382
}
367383
catch (IOException e){
368384
LOGGER.log(Level.INFO, "Line does no exist on this file, this file will not be processed.");
369-
370385
return;
371386
}
372387
}
@@ -539,7 +554,7 @@ public int compare(String f1, String f2){
539554
LOGGER.log(Level.INFO, "Complete path to rotated file to process: {0}", i);
540555
try {
541556
long time_difference = Files.getLastModifiedTime(Paths.get(i)).toMillis()-System.currentTimeMillis();
542-
if (time_difference<=1500){//file is within time range, lets read an see we find the line stored
557+
if (time_difference<=150000){//file is within time range, lets read an see we find the line stored
543558
//System.out.println("We found a file: "+i+"FileTime: "+ Files.getLastModifiedTime(Paths.get(i)).toString());
544559
LOGGER.log(Level.INFO, "Found a file to process {0}: ", i);
545560
//result.
@@ -575,7 +590,7 @@ public int compare(String f1, String f2){
575590
}
576591
}
577592

578-
public void ReadFile(int readIntervalSeconds, String log_file_location, String log_file_name, balancingData bd, Connection h2con) {
593+
public void ReadFile(int readIntervalSeconds, String log_file_location, String configured_log_file_name, balancingData bd, Connection h2con) {
579594
ConfigReader configuration = new ConfigReader();
580595
//setUpLogger();
581596
if (clearReadProperties(configuration)) LOGGER.log(Level.WARNING, "Brand new execution, clearing read.properties");
@@ -589,6 +604,7 @@ public void ReadFile(int readIntervalSeconds, String log_file_location, String l
589604
Long finalPosition, initialPosition;
590605
initialPosition = 0L;
591606
finalPosition = 0L;
607+
String log_file_name = configured_log_file_name;
592608
String most_recent_log_file_name=log_file_name;
593609

594610
//getting most recent file name based on the file name patter
@@ -627,16 +643,17 @@ public void ReadFile(int readIntervalSeconds, String log_file_location, String l
627643
initialPosition = Long.parseLong(position);
628644
Path path = Paths.get(log_file_name);
629645
Long fileSize = Files.size(path);
630-
631-
if (fileSize<previousFileSize) {//file has rotated, we need to read the previous file before reading the new one.
646+
LOGGER.log(Level.INFO, "Current file size: {0} ", fileSize);
647+
LOGGER.log(Level.INFO, "Previous file size: {0} ", previousFileSize);
648+
if ((fileSize<previousFileSize) && (fileSize!=0)){//file has rotated, we need to read the previous file before reading the new one.
632649
final Long currentPosition = initialPosition;
633-
//System.out.println("Todo###############################################################...");
634-
//System.out.println(log_file.getParent());
635650
LOGGER.log(Level.INFO, "Parent from log file: {0}", log_file.getParent());
636651
try (Stream<Path> walk = Files.walk(Paths.get(log_file.getParent()))) {
637652
List<String> result;
638653
result = walk.filter(Files::isRegularFile)
639-
.map(x -> x.toString()).collect(Collectors.toList());
654+
.map(x -> x.toString())
655+
.filter(f -> f.endsWith(".log"))
656+
.filter(f -> f.contains(configured_log_file_name.replace(".log", ""))).collect(Collectors.toList());
640657
result.sort(new Comparator<String>(){//sorting the stream by file modified time from newest to oldest
641658
@Override
642659
public int compare(String f1, String f2){
@@ -651,16 +668,22 @@ public int compare(String f1, String f2){
651668
}
652669
});
653670

671+
result.forEach(i->{LOGGER.log(Level.INFO, "Listing files: {0}", i);});
654672
result.forEach(i->{
655673
int index = i.lastIndexOf(File.separator);
656674
String relative_file_name = i.substring(index+1);
657-
if ((relative_file_name.startsWith(log_file.getName().replace(".log", "")))&&(relative_file_name.contains(".log"))){
675+
//if ((relative_file_name.startsWith(configured_log_file_name))&&(relative_file_name.contains(".log"))){
658676
LOGGER.log(Level.INFO, "Rotated file to process: {0}", relative_file_name);
659677
LOGGER.log(Level.INFO, "Complete path to rotated file to process: {0}", i);
660678
try {
661-
long time_difference = Files.getLastModifiedTime(Paths.get(i)).toMillis()-System.currentTimeMillis();
662-
if (time_difference<=1500){//file is within time range, lets read an see we find the line stored
679+
long time_difference = System.currentTimeMillis()-Files.getLastModifiedTime(Paths.get(i)).toMillis();
680+
LOGGER.log(Level.INFO, "Candidate to process {0}: ", i);
681+
LOGGER.log(Level.INFO, "Time difference {0}: ", time_difference);
682+
LOGGER.log(Level.INFO, "Current time {0}: ", System.currentTimeMillis());
683+
LOGGER.log(Level.INFO, "File time time {0}: ", Files.getLastModifiedTime(Paths.get(i)).toMillis());
684+
if (time_difference<=360000){//file is within 6 minute time range, lets read an see we find the line stored, ideally this would be 1.5 minute adding more time to acommodate to some fluctuations
663685
LOGGER.log(Level.INFO, "Found a file to process {0}: ", i);
686+
LOGGER.log(Level.INFO, "Current position {0}: ", currentPosition);
664687
synchronized(h2con){
665688
processFile(Paths.get(i), currentPosition, configuration, true, bd, h2con);
666689
h2con.notify();
@@ -671,7 +694,7 @@ public int compare(String f1, String f2){
671694
LOGGER.log(Level.WARNING, "Error reading rotated files: ", e.getMessage());
672695
}
673696

674-
}
697+
//}
675698

676699
});
677700
} catch (IOException e) {
@@ -680,10 +703,23 @@ public int compare(String f1, String f2){
680703
}
681704
}
682705
else {
683-
synchronized(h2con){
684-
processFile(path, initialPosition, configuration, false, bd, h2con);
685-
h2con.notify();
706+
if (fileSize>previousFileSize){//current read file has grown, we need to continue processing it
707+
synchronized(h2con){
708+
processFile(path, initialPosition, configuration, false, bd, h2con);
709+
h2con.notify();
710+
}
711+
}
712+
else{//file has not changed or current file is zero length, we need to check exactly what happened
713+
long currentFileLastChangeTime = Files.getLastModifiedTime(Paths.get(log_file_name)).toMillis();
714+
String new_log_filename = ProcessNewFiles(log_file_location, configured_log_file_name, bd, h2con, currentFileLastChangeTime, configuration);
715+
if (!new_log_filename.equals("")){//no new files have been found will keep watching for new files
716+
LOGGER.log(Level.WARNING, "All new files have been processed, updating file handler loop and continuing. New file to read is {0}.", new_log_filename);
717+
log_file_name = new_log_filename;
718+
}
719+
else{
720+
LOGGER.log(Level.WARNING, "There were no knew files to process, will keep watching for new files or updates to the current one. Current file is {0}.", log_file_name);
686721
}
722+
}
687723
}
688724
readWriteFileAccess.close();
689725
Thread.sleep(readIntervalSeconds*1000);
@@ -696,4 +732,73 @@ public int compare(String f1, String f2){
696732
e.printStackTrace();
697733
}
698734
}
735+
736+
private String ProcessNewFiles(String log_file_location, String configured_log_file_name, balancingData bd, Connection h2con, long currentFileLastChangedTime, ConfigReader configuration){
737+
String new_file_name = "";
738+
LOGGER.log(Level.INFO, "Starting to process newFiles at {0}...", log_file_location);
739+
LOGGER.log(Level.INFO, "Configured file name {0}...", configured_log_file_name);
740+
//configured_log_file_name=;
741+
try (Stream<Path> walk = Files.walk(Paths.get(log_file_location))) {
742+
List<String> result = walk.filter(Files::isRegularFile)
743+
.map(x -> x.toString())
744+
.filter(f -> f.endsWith(".log"))
745+
.filter(f -> f.contains(configured_log_file_name.replace(".log", ""))).collect(Collectors.toList());
746+
//.filter(pathItem -> pathItem.contains(".log")).collect(Collectors.toList());
747+
result.sort(new Comparator<String>(){//sorting the stream by file modified time from oldest to newest
748+
@Override
749+
public int compare(String f1, String f2){
750+
try {
751+
return Files.getLastModifiedTime(Paths.get(f1)).compareTo(Files.getLastModifiedTime(Paths.get(f2)));//from oldest to newest
752+
}
753+
catch (IOException e){
754+
LOGGER.log(Level.WARNING, "There was an error while sorting files, file list will not be sorted: {0}", e.getMessage());
755+
return 0;
756+
}
757+
}
758+
});
759+
760+
result.forEach(item->{
761+
LOGGER.log(Level.INFO, "Found an item {0}: ", item);
762+
763+
LOGGER.log(Level.INFO, "Contains {0} ", item.contains(configured_log_file_name));
764+
765+
});
766+
for(Iterator<String> i = result.iterator(); i.hasNext();){
767+
String item = i.next();
768+
LOGGER.log(Level.INFO, "Found an item {0}: ", item);
769+
int index = item.lastIndexOf(File.separator);
770+
String relative_file_name = item.substring(index+1);
771+
if ((relative_file_name.startsWith(configured_log_file_name))&&(relative_file_name.contains(".log"))){
772+
try {
773+
long newFileChangeTime = Files.getLastModifiedTime(Paths.get(item)).toMillis();
774+
if (newFileChangeTime>=currentFileLastChangedTime){//file is within time range, lets read an see we find the line stored
775+
LOGGER.log(Level.INFO, "Found a file to process {0}: ", item);
776+
synchronized(h2con){
777+
processFile(Paths.get(item), 0L, configuration, true, bd, h2con);
778+
h2con.notify();
779+
}
780+
}
781+
}
782+
catch (IOException e){
783+
LOGGER.log(Level.WARNING, "Error reading new files after rotation: ", e.getMessage());
784+
}
785+
}
786+
if (!i.hasNext()) new_file_name = item;
787+
}
788+
789+
790+
}
791+
catch (IOException e) {
792+
LOGGER.log(Level.WARNING, "Error while walking through files: ", e.getMessage());
793+
e.printStackTrace();
794+
}
795+
return new_file_name;
796+
797+
}
798+
799+
/*private void processNewFilesResult(String item, int result_size, String configured_log_file_name, long currentFileLastChangedTime, Connection h2con, ConfigReader configuration, int iterations){
800+
801+
802+
}
803+
}*/
699804
}

0 commit comments

Comments
 (0)