feat: optimizing refresh routine

Author: matheuslab

.gitignore

@@ -4,4 +4,5 @@ __pycache__
 build/
 dist/
 output/
-*.txt
+*.txt
+*.csv

src/costs.py

@@ -1,18 +1,21 @@
 import requests
 import pandas as pd
 from initial import timestamps
+import os
 
 
-def getCostsDataFrame(apiKey=None, fleetId='', timestamps=timestamps):
+def getCostsDataFrame(apiKey=None, fleetName='', timestamps=timestamps):
     frames = []
+    if os.path.exists(os.path.expanduser('~/cobliBI/costs.csv')): 
+        frames.append(pd.read_csv(os.path.expanduser('~/cobliBI/costs.csv')))
     for month in timestamps:
         start_timestamp, end_timestamp = timestamps[month]
         costsURL = f"https://api.cobli.co/herbie-1.1/costs/report?begin={start_timestamp}&end={end_timestamp}&tz=America%2FFortaleza"
         costsResponse = requests.get(costsURL, headers={'cobli-api-key': apiKey})
         frames.append(pd.read_excel(costsResponse.content, 0))
     dataframe = pd.concat(frames)
-    dataframe.assign(Frota = [fleetId] * len(dataframe.index))
-    dataframe.to_csv('costs.csv')
+    dataframe['Frota'] = [fleetName] * len(dataframe.index)
+    dataframe.to_csv(os.path.expanduser('~/cobliBI/costs.csv'), index = False)
     return dataframe
 
 costs_function_name = 'getCostsDataFrame'

src/generate_bi_script.py

@@ -31,19 +31,19 @@ def check_api_keys(apiKeyList: dict):
 
 
 def generate_script(apiKeyList: dict):
-    with open('./main_file.txt', 'w') as main_file:
-        main_file.write(initialSource + '\n\n')
-        main_file.write(costsScript + '\n')
-        main_file.write(incidentsScript + '\n')
-        main_file.write(productivityScript + '\n')
-        for key in apiKeyList:
-            main_file.write(
-                f"{costs_function_name}('{apiKeyList[key]}', '{key}')\n\n")
-            main_file.write(
-                f"{incidents_function_name}('{apiKeyList[key]}', '{key}')\n\n")
-            main_file.write(
-                f"{productivity_function_name}('{apiKeyList[key]}', '{key}')\n\n")
-    with open('./data_file.txt', 'w') as data_file:
+    print(initialSource + '\n\n')
+    print(costsScript + '\n')
+    print(incidentsScript + '\n')
+    print(productivityScript + '\n')
+    for key in apiKeyList:
+        print(
+            f"costs = {costs_function_name}('{apiKeyList[key]}', '{key}')\n\n")
+        print(
+            f"incidents = {incidents_function_name}('{apiKeyList[key]}', '{key}')\n\n")
+        print(
+            f"productivity = {productivity_function_name}('{apiKeyList[key]}', '{key}')\n\n")
+    os.makedirs(os.path.expanduser('~/cobliBI'), exist_ok=True)
+    with open(os.path.expanduser('~/cobliBI/data_file.txt'), 'w') as data_file:
         data_file.write(initial.now)
 
 gui(check_api_keys)

src/incidents.py

@@ -1,20 +1,22 @@
 import requests
 import pandas as pd
 from initial import timestamps
+import os
 
 
-def getIncidentsDataFrame(apiKey=None, fleetId='', timestamps=timestamps):
+def getIncidentsDataFrame(apiKey=None, fleetName='', timestamps=timestamps):
     frames = []
+    if os.path.exists(os.path.expanduser('~/cobliBI/incidents.csv')): 
+        frames.append(pd.read_csv(os.path.expanduser('~/cobliBI/incidents.csv')))
     for month in timestamps:
         start_timestamp, end_timestamp = timestamps[month]
         incidentsURL = f"https://api.cobli.co/herbie-1.1/stats/incidents/report?begin={start_timestamp}&end={end_timestamp}&tz=America%2FFortaleza"
         incidentsResponse = requests.get(
             incidentsURL, headers={'cobli-api-key': apiKey})
         frames.append(pd.read_excel(incidentsResponse.content, 0))
     dataframe = pd.concat(frames)
-    
-    dataframe.assign(Frota = [fleetId] * len(dataframe.index))
-    dataframe.to_csv('incidents.csv')
+    dataframe['Frota'] = [fleetName] * len(dataframe.index)
+    dataframe.to_csv(os.path.expanduser('~/cobliBI/incidents.csv'), index = False)
     return dataframe
 
 incidents_function_name = 'getIncidentsDataFrame'

src/initial.py

@@ -9,23 +9,26 @@
 format_string = "%d/%m/%Y %H:%M:%S.%f"
 
 now = datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S.%f")
+formattedCurrentDate = datetime.datetime.strptime(now, format_string)
 
 timestamps = dict()
 
+def utc_to_epoch(date):
+    return int(time.mktime(datetime.datetime.strptime(date, format_string).timetuple()))*1000
+
 for monthsAgo in range(6):
-    end_date_str = (datetime.datetime.strptime(now, format_string) - pd.DateOffset(months=monthsAgo)).strftime(format_string)
-    end_timestamp_value = int(time.mktime(
-    datetime.datetime.strptime(end_date_str, format_string).timetuple()))*1000
+    end_date_str = (formattedCurrentDate - pd.DateOffset(months=monthsAgo)).strftime(format_string)
+    end_timestamp_value = utc_to_epoch(end_date_str)
 
-    start_date_str = (datetime.datetime.strptime(now, format_string) - pd.DateOffset(months=monthsAgo+1)).strftime(format_string)
-    start_timestamp_value = int(time.mktime(
-    datetime.datetime.strptime(start_date_str, format_string).timetuple()))*1000
+    start_date_str = (formattedCurrentDate - pd.DateOffset(months=monthsAgo+1)).strftime(format_string)
+    start_timestamp_value = utc_to_epoch(start_date_str)
 
     timestamps[monthsAgo] = (start_timestamp_value, end_timestamp_value)
 
-if os.path.exists('./data_file.txt'): 
-    with open('./data_file.txt', 'r') as data_file:
+if os.path.exists(os.path.expanduser('~/cobliBI/data_file.txt')): 
+    with open(os.path.expanduser('~/cobliBI/data_file.txt'), 'r') as data_file:
         last_refresh = data_file.read()
 
     if last_refresh:
-        timestamps = [last_refresh, now]
+        shortTimestamp = dict()
+        shortTimestamp[0] = [utc_to_epoch(last_refresh), utc_to_epoch(now)]

src/productivity.py

@@ -1,19 +1,22 @@
 import requests
 import pandas as pd
 from initial import timestamps
+import os
 
 
-def getProductivityDataFrame(apiKey=None, fleetId='', timestamps=timestamps):
+def getProductivityDataFrame(apiKey=None, fleetName='', timestamps=timestamps):
     frames = []
+    if os.path.exists(os.path.expanduser('~/cobliBI/productivity.csv')): 
+        frames.append(pd.read_csv(os.path.expanduser('~/cobliBI/productivity.csv')))
     for month in timestamps:
         start_timestamp, end_timestamp = timestamps[month]
         productivityURL = f"https://api.cobli.co/herbie-1.1/stats/performance/vehicle/report?begin={start_timestamp}&end={end_timestamp}&tz=America%2FFortaleza"
         productivityResponse = requests.get(
             productivityURL, headers={'cobli-api-key': apiKey})
         frames.append(pd.read_excel(productivityResponse.content, 2))
     dataframe = pd.concat(frames)
-    dataframe.assign(Frota = [fleetId] * len(dataframe.index))
-    dataframe.to_csv('productivity.csv')
+    dataframe['Frota'] = [fleetName] * len(dataframe.index)
+    dataframe.to_csv(os.path.expanduser('~/cobliBI/productivity.csv'), index = False)
     return dataframe
 
 productivity_function_name = 'getProductivityDataFrame'