github reporting updates

Author: jeffreyaven

docker-compose.yml

@@ -14,6 +14,8 @@ services:
       STACKQL_GITHUB_USERNAME: ${STACKQL_GITHUB_USERNAME}
       STACKQL_GITHUB_PASSWORD: ${STACKQL_GITHUB_PASSWORD}
       GOOGLE_CREDENTIALS: ${GOOGLE_CREDENTIALS}
+    dns: 
+      - 8.8.8.8
     command:
       - /bin/sh
       - -c

notebooks/github/sprint_reporting/includes/includes.ipynb

@@ -10,7 +10,8 @@
     "from IPython.display import display, Markdown, HTML, clear_output\n",
     "import plotly.express as px\n",
     "import pandas as pd\n",
-    "import sys, datetime, base64, time, psycopg2\n",
+    "from datetime import datetime, timedelta, timezone\n",
+    "import sys, base64, time, psycopg2\n",
     "sys.path.append('/jupyter/ext')\n",
     "import stackql\n",
     "from psycopg2.extras import RealDictCursor\n",

notebooks/github/sprint_reporting/sprint_reporting.ipynb

@@ -20,10 +20,18 @@
     "# show header\n",
     "owner = owner.value\n",
     "repo = repo.value\n",
+    "sprint_number = sprint_number.value\n",
     "start_date = start_date.value.strftime('%Y-%m-%d')\n",
     "end_date = end_date.value.strftime('%Y-%m-%d')\n",
-    "header_text = f\"# Sprint {sprint_number.value}\\n\"\n",
-    "header_text += f\"#### {start_date} to {end_date}\\n\"\n",
+    "aest = timezone(timedelta(hours=10))\n",
+    "start_date_aest = datetime.fromisoformat(f\"{start_date}T00:00:00\").replace(tzinfo=aest)\n",
+    "end_date_aest = datetime.fromisoformat(f\"{end_date}T00:00:00\").replace(tzinfo=aest)\n",
+    "start_date_utc = start_date_aest.astimezone(timezone.utc)\n",
+    "end_date_utc = end_date_aest.astimezone(timezone.utc)\n",
+    "start_date_iso_utc = start_date_utc.isoformat()\n",
+    "end_date_iso_utc = end_date_utc.isoformat()\n",
+    "header_text = f\"# Sprint {sprint_number}\\n\"\n",
+    "header_text += f\"#### {start_date} to {end_date} (AEST)\\n\"\n",
     "display(Markdown(header_text))"
    ]
   },
@@ -50,8 +58,10 @@
     "    JSON_EXTRACT(commit, '$$.author.name') as author,\n",
     "    DATE(DATETIME(JSON_EXTRACT(commit, '$$.author.date'), '+10 hours')) as date\n",
     "    FROM github.repos.commits\n",
-    "    WHERE owner = '$owner' AND repo = '$repo' AND\n",
-    "    date > '$start_date' AND date <= '$end_date') c\n",
+    "    WHERE until = '$end_date_iso_utc' \n",
+    "    AND since = '$start_date_iso_utc'\n",
+    "    AND owner = '$owner' \n",
+    "    AND repo = '$repo') c\n",
     "GROUP BY c.author"
    ]
   },
@@ -87,8 +97,10 @@
     "DATE(DATETIME(JSON_EXTRACT(commit, '$$.author.date'), '+10 hours')) as date,\n",
     "TIME(DATETIME(JSON_EXTRACT(commit, '$$.author.date'), '+10 hours')) as time\n",
     "FROM github.repos.commits\n",
-    "WHERE owner = '$owner' AND repo = '$repo' AND\n",
-    "date > '$start_date' AND date <= '$end_date'"
+    "WHERE until = '$end_date_iso_utc' \n",
+    "AND since = '$start_date_iso_utc'\n",
+    "AND owner = '$owner' \n",
+    "AND repo = '$repo'"
    ]
   },
   {
@@ -102,7 +114,7 @@
     "commits_df = stackql_df\n",
     "commit_shas = commits_df['sha'].tolist()\n",
     "display(commits_df)\n",
-    "display(create_download_link(commits_df))"
+    "display(create_download_link(commits_df, filename=f\"{sprint_number}_commits.csv\"))"
    ]
   },
   {
@@ -128,9 +140,12 @@
     "FROM\n",
     "github.repos.activities\n",
     "WHERE \n",
-    "    owner = '$owner' AND \n",
-    "    repo = '$repo'  AND\n",
-    "    date > '$start_date' AND date <= '$end_date'"
+    "owner = '$owner' \n",
+    "AND repo = '$repo'\n",
+    "AND date > '$start_date' \n",
+    "AND date <= '$end_date'\n",
+    "AND direction = 'desc'\n",
+    "AND time_period = 'month'"
    ]
   },
   {
@@ -142,7 +157,7 @@
     "\n",
     "# display activity and download link\n",
     "display(stackql_df)\n",
-    "display(create_download_link(stackql_df))"
+    "display(create_download_link(stackql_df, filename=f\"{sprint_number}_activity.csv\"))"
    ]
   },
   {
@@ -200,6 +215,7 @@
     "WHERE \n",
     "    owner = '$owner' AND \n",
     "    repo = '$repo' AND\n",
+    "    since = '$start_date_iso_utc' AND\n",
     "    updated_at > '$start_date' AND updated_at <= '$end_date'"
    ]
   },
@@ -302,7 +318,7 @@
     "\n",
     "# display files with download link\n",
     "display(files_df)\n",
-    "display(create_download_link(files_df))"
+    "display(create_download_link(files_df, filename=f\"{sprint_number}_files.csv\"))"
    ]
   }
  ],

scripts/entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 /srv/stackql/stackql --version
 echo "starting stackql server..."
-nohup /srv/stackql/stackql --http.response.pageLimit=1 --pgsrv.port=5444 srv &
+nohup /srv/stackql/stackql --http.response.pageLimit=-1 --pgsrv.port=5444 srv &
 echo "stackql server started"
 start-notebook.sh --NotebookApp.token=''