Logfetch improvements and bug fixes

scripts/logfetch/logfetch_base.py

@@ -26,6 +26,9 @@ def tasks_for_requests(args):
             tasks = [task["taskId"]["id"] for task in all_tasks_for_request(args, request) if log_matches(task["taskId"]["deployId"], args.deployId)]
         else:
             tasks = [task["taskId"]["id"] for task in all_tasks_for_request(args, request)]
+            log('Found {0} matching tasks\n'.format(len(tasks)), args, False)
+            if hasattr(args, 'task_count') and len(tasks) > args.task_count:
+                log(colored('Max task count set to {0}, excluding {1} tasks. Run again with --task-count={2} to search all matching tasks\n'.format(args.task_count, len(tasks) - args.task_count, len(tasks)), 'yellow'), args, False)
             tasks = tasks[0:args.task_count] if hasattr(args, 'task_count') else tasks
         all_tasks = all_tasks + tasks
     if not all_tasks:
@@ -50,7 +53,7 @@ def all_tasks_for_request(args, request):
         elif len(active_tasks) == 0:
             return historical_tasks
         else:
-            return active_tasks + [h for h in historical_tasks if date_range_overlaps(args, int(str(h['updatedAt'])[0:-3]), int(str(h['taskId']['startedAt'])[0:-3]))]
+            return active_tasks + [h for h in historical_tasks if date_range_overlaps(args, int(str(h['taskId']['startedAt'])[0:-3]), int(str(h['updatedAt'])[0:-3]))]
     else:
         return active_tasks
 
@@ -80,7 +83,9 @@ def date_range_overlaps(args, start, end):
             return True
         elif end_datetime > args.start and end_datetime < args.end:
             return True
-        elif end_datetime > args.end and start_datetime > args.start:
+        elif end_datetime > args.end and start_datetime < args.end:
+            return True
+        elif end_datetime > args.start and start_datetime < args.start:
             return True
         else:
             return False

scripts/logfetch/s3_logs.py

@@ -79,16 +79,20 @@ def logs_for_all_requests(args):
         tasks_progress = 0
         tasks_goal = len(tasks)
         for task in tasks:
-            s3_logs = logfetch_base.get_json_response(s3_task_logs_uri(args, task), args, s3_params)
+            s3_logs = logfetch_base.get_json_response(s3_task_logs_uri(args, task), args)
             logs = logs + s3_logs if s3_logs else logs
             tasks_progress += 1
             logfetch_base.update_progress_bar(tasks_progress, tasks_goal, 'S3 Log Finder', args.silent or args.verbose)
+        found_logs = []
+        keys = []
+        for log in logs:
+            if not log['key'] in keys:
+                found_logs.append(log)
+                keys.append(log['key'])
         logfetch_base.log(colored('\nAlso searching s3 history...\n', 'cyan'), args, False)
         for request in logfetch_base.all_requests(args):
             s3_logs = logfetch_base.get_json_response(s3_request_logs_uri(args, request), args, s3_params)
             logs = logs + s3_logs if s3_logs else logs
-        found_logs = []
-        keys = []
         for log in logs:
             if not log['key'] in keys:
                 found_logs.append(log)

scripts/setup.py

@@ -11,7 +11,7 @@
 
 setup(
     name='singularity-logfetch',
-    version='0.29.2',
+    version='0.29.3',
     description='Singularity log fetching and searching',
     author="HubSpot",
     author_email='[email protected]',