Initial data to data-table display client

datastore/requirements.txt

@@ -1,3 +1,4 @@
 flask==2.0.2
 gunicorn==20.1.0
-requests==2.27.0
+requests==2.27.0
+pandas==1.3.3

datastore/src/app.py

@@ -1,14 +1,64 @@
 from flask import Flask, jsonify
 from os import environ
+import os
+import pandas as pd
 
+from data_loading import *
+
+## Get Parameters
 SECRET_KEY = environ.get("SECRET_KEY")
 print("SECRET KEY", SECRET_KEY)
+
+## Load data
+mcc_list=[1,2]
+
+# Directions for locating file at TACC
+file_url_root ='https://api.a2cps.org/files/v2/download/public/system/a2cps.storage.community/reports'
+
+current_folder = os.path.dirname(__file__)
+data_folder  ='data'
+
+# URLS
+subjects_raw_url = load_report_data(file_url_root, 'subjects', mcc_list)
+
+# Local files
+subjects_raw = load_raw_subjects_files(current_folder, data_folder, mcc_list)
+
+subjects = {}
+weekly_data = pd.DataFrame()
+
+for mcc in subjects_raw.keys():
+    mcc_data = pd.DataFrame.from_dict(subjects_raw[mcc], orient = 'index')
+    mcc_data['mcc'] = mcc
+    if weekly_data.empty:
+        weekly_data = mcc_data
+    else:
+        weekly_data = pd.concat([weekly_data, mcc_data])
+
+weekly_data.reset_index(inplace=True)
+if 'index' in weekly_data.columns:
+    weekly_data.rename(columns={"index": "record_id"}, inplace=True)
+
+subjects['weekly'] = weekly_data.to_dict('index')
+
+# local_data_store = load_data(file_url_root, reports, mcc_list)
+
+
 app = Flask(__name__)
 
 @app.route("/api")
 def api():
-    d = { "key": "value" }
-    return jsonify(d)
+    datafeeds = {
+        "weekly": {"date" : "today", "data" : 'tbd'},
+        "consort": {"date" : "today", "data" : 'tbd'},
+        "blood": {"date" : "today", "data" : 'tbd'},
+    }
+    return jsonify(datafeeds)
+
+@app.route("/subjects")
+def api_weekly():
+
+    return jsonify(subjects)
 
 if __name__ == "__main__":
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0')

datastore/src/data/consort.csv

@@ -0,0 +1,37 @@
+source,target,value,timestamp,mcc
+Screened Patients,Declined - Not interested in research,116,2022-01-12T16:00:59,1
+Screened Patients,Declined - COVID-related,3,2022-01-12T16:00:59,1
+Screened Patients,Declined - Compensation insufficient,2,2022-01-12T16:00:59,1
+Screened Patients,Declined - Specific study procedure,124,2022-01-12T16:00:59,1
+Screened Patients,Declined - Time related,388,2022-01-12T16:00:59,1
+Screened Patients,Declined - No reason provided,345,2022-01-12T16:00:59,1
+Screened Patients,Consented Patients,271,2022-01-12T16:00:59,1
+Consented Patients,Withdrawl Prior to Surgery - Subject chose to discontinue the study,10,2022-01-12T16:00:59,1
+Consented Patients,Withdrawl Prior to Surgery - Site PI chose to discontinue subject participation,4,2022-01-12T16:00:59,1
+Consented Patients,"Withdrawl Prior to Surgery - Subject is lost to follow-up, unable to locate",2,2022-01-12T16:00:59,1
+Consented Patients,Patients Reaching Baseline,228,2022-01-12T16:00:59,1
+Patients Reaching Baseline,Patients With Surgery,217,2022-01-12T16:00:59,1
+Patients With Surgery,Early Terminations - Subject chose to discontinue the study,2,2022-01-12T16:00:59,1
+Patients With Surgery,"Early Terminations - Subject is lost to follow-up, unable to locate",1,2022-01-12T16:00:59,1
+Patients With Surgery,Patients Reaching Week 6,191,2022-01-12T16:00:59,1
+Patients Reaching Week 6,Early Terminations - Subject chose to discontinue the study,1,2022-01-12T16:00:59,1
+Patients Reaching Week 6,Patients Reaching Month 3,124,2022-01-12T16:00:59,1
+Patients Reaching Month 3,Patients Reaching Month 6,46,2022-01-12T16:00:59,1
+Screened Patients,Declined - Not interested in research,116,2022-01-12T16:01:36,2
+Screened Patients,Declined - COVID-related,3,2022-01-12T16:01:36,2
+Screened Patients,Declined - Compensation insufficient,2,2022-01-12T16:01:36,2
+Screened Patients,Declined - Specific study procedure,124,2022-01-12T16:01:36,2
+Screened Patients,Declined - Time related,388,2022-01-12T16:01:36,2
+Screened Patients,Declined - No reason provided,345,2022-01-12T16:01:36,2
+Screened Patients,Consented Patients,271,2022-01-12T16:01:36,2
+Consented Patients,Withdrawl Prior to Surgery - Subject chose to discontinue the study,10,2022-01-12T16:01:36,2
+Consented Patients,Withdrawl Prior to Surgery - Site PI chose to discontinue subject participation,4,2022-01-12T16:01:36,2
+Consented Patients,"Withdrawl Prior to Surgery - Subject is lost to follow-up, unable to locate",2,2022-01-12T16:01:36,2
+Consented Patients,Patients Reaching Baseline,228,2022-01-12T16:01:36,2
+Patients Reaching Baseline,Patients With Surgery,217,2022-01-12T16:01:36,2
+Patients With Surgery,Early Terminations - Subject chose to discontinue the study,2,2022-01-12T16:01:36,2
+Patients With Surgery,"Early Terminations - Subject is lost to follow-up, unable to locate",1,2022-01-12T16:01:36,2
+Patients With Surgery,Patients Reaching Week 6,191,2022-01-12T16:01:36,2
+Patients Reaching Week 6,Early Terminations - Subject chose to discontinue the study,1,2022-01-12T16:01:36,2
+Patients Reaching Week 6,Patients Reaching Month 3,124,2022-01-12T16:01:36,2
+Patients Reaching Month 3,Patients Reaching Month 6,46,2022-01-12T16:01:36,2

datastore/src/data_loading.py

@@ -0,0 +1,40 @@
+import requests
+import json
+import os
+import pandas as pd
+import sqlite3
+import datetime
+from datetime import datetime, timedelta
+
+def load_report_data(file_url_root, report, mcc_list):
+    report_data = {}
+    for mcc in mcc_list:
+        mcc=str(mcc)
+        report_suffix = report + '-' + str(mcc) + '-latest.json'
+        file_url = '/'.join([file_url_root, report, report_suffix])
+        report_data[mcc] = {}
+        r = requests.get(file_url)
+        report_data[mcc]['r_status'] = r.status_code
+        if r.status_code == 200:
+            mcc_json = r.json()
+        else:
+            mcc_json = None
+        report_data[mcc]['raw_data'] = mcc_json
+    return report_data
+
+def load_raw_subjects_files(current_folder, data_folder, mcc_list):
+    file_report = {}
+    filename = 'subjects-[mcc]-latest.json'
+    for mcc in mcc_list:
+        filename_mcc = filename.replace('[mcc]',str(mcc))
+        filepath = os.path.join(current_folder, data_folder, filename_mcc)
+        with open(filepath , 'r') as json_file:
+            file_report[str(mcc)] = json.loads(json_file.read())
+    return file_report
+
+
+def load_data(file_url_root, reports, mcc_list):
+    local_data_store = {}
+    for report in reports:
+        local_data_store[report] = load_report_data(file_url_root, report, mcc_list)
+    return local_data_store

datastore_client/requirements.txt

@@ -1,4 +1,5 @@
 flask==2.0.2
 gunicorn==20.1.0
 requests==2.27.0
-dash==2.0.0
+pandas==1.3.3
+dash==2.0.0

datastore_client/src/app.py

@@ -1,24 +1,60 @@
 import dash
-from dash import html
+from dash import html, dash_table as dt
 import requests
 import flask
 import traceback
 
+import requests
+import json
+import pandas as pd
+
 server = flask.Flask('app')
 
+# ---------------------------------
+#   Get Data From datastore
+# ---------------------------------
 try:
-    response = requests.get("http://datastore:8050/api")
+    response = requests.get("http://datastore:8050/subjects")
+    subjects = response.json()
+    print(subjects.keys())
 except Exception as e:
+    subjects = {'status': 'no response'}
     traceback.print_exc()
 
-print("data from datastore", response.json())
+weekly = pd.DataFrame.from_dict(subjects['weekly'], orient='index').reset_index()
+#
+# print("data from datastore:", datafeed)
+
+# ---------------------------------
+#   Page components
+# ---------------------------------
+
+df = pd.read_csv('https://git.io/Juf1t')
+datafeeds_div = html.Div([
+    html.Div([
+        json.dumps(weekly.to_dict('records'))
+        ]),
+    dt.DataTable(
+        id='tbl_weekly', data=weekly.to_dict('records'),
+        columns=[{"name": i, "id": i} for i in weekly.columns],
+    ),
+    dt.DataTable(
+        id='tbl', data=df.to_dict('records'),
+        columns=[{"name": i, "id": i} for i in df.columns],
+    )
+])
+
+# ---------------------------------
+#   build app
+# ---------------------------------
 
 app = dash.Dash('app', server=server)
 
 
 app.layout = html.Div([
-    html.H1('Hello World'),
+    html.H1('Monday. Blurgh'),
+    datafeeds_div
 ])
 
 if __name__ == '__main__':
-    app.run_server()
+    app.run_server()

docker-compose.dev.yml

@@ -18,7 +18,7 @@ services:
     env_file:
       - ./datastore/.env
     networks:
-      - datastore_net 
+      - datastore_net
   datastore_client:
     build: ./datastore_client
     depends_on:
@@ -38,7 +38,7 @@ services:
     env_file:
       - ./datastore_client/.env
     networks:
-      - datastore_net 
+      - datastore_net
 
 networks:
   datastore_net:
@@ -47,4 +47,3 @@ networks:
       driver: default
       config:
           - subnet: 172.16.238.0/24
-