From 15e7bfae61a4f05d2513970c04016f2dd1de4aa4 Mon Sep 17 00:00:00 2001
From: TimDiam0nd <54599998+TimDiam0nd@users.noreply.github.com>
Date: Thu, 29 Aug 2024 08:28:46 +0100
Subject: [PATCH 1/3] Remove json whitespace in ingest()

Update ingest function so that JSON objects contain no whitespace between colons and commas, to reduce body size and also make raw log searching json easier
---
 common/ingest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/ingest.py b/common/ingest.py
index ba239ff..03996fd 100644
--- a/common/ingest.py
+++ b/common/ingest.py
@@ -94,7 +94,7 @@ def ingest(data: list[Any], log_type: str):
   # [{"logText": str(log1)}, {"logText": str(log2)}, ...]
   parsed_data = list(
       map(
-          lambda i: {"logText": str(json.dumps(i).encode("utf-8"), "utf-8")},
+          lambda i: {"logText": str(json.dumps(i, separators=(',', ':')).encode("utf-8"), "utf-8")},
           data,
       )
   )

From c6716fe2506db7613b25526388a79abaecc68e32 Mon Sep 17 00:00:00 2001
From: TimDiam0nd <54599998+TimDiam0nd@users.noreply.github.com>
Date: Thu, 29 Aug 2024 09:03:50 +0100
Subject: [PATCH 2/3] Update utils.py

---
 common/utils.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/common/utils.py b/common/utils.py
index f02225f..b23becc 100644
--- a/common/utils.py
+++ b/common/utils.py
@@ -138,3 +138,20 @@ def cloud_logging(message: str, severity: str = "INFO") -> None:
     severity (str): severity of the message. Defaults to "INFO".
   """
   print(json.dumps({"severity": severity, "message": message}))
+
+
+def cleanup(object: dict | list) -> dict | list:
+    """Remove empty lists and dicts from an object
+    
+    Args:
+        object (dict | list): The object to cleanup
+
+    Returns:
+        dict | list: The cleaned object
+    """
+    if isinstance(object, dict):
+        return {k: cleanup(v) for k, v in object.items() if v and cleanup(v)}
+    elif isinstance(object, list):
+        return [cleanup(v) for v in object if v and cleanup(v)]
+    else:
+        return object

From 5cafa666db7f7258b7fca5a1b76d954e08d1d4d3 Mon Sep 17 00:00:00 2001
From: TimDiam0nd <54599998+TimDiam0nd@users.noreply.github.com>
Date: Thu, 29 Aug 2024 09:04:30 +0100
Subject: [PATCH 3/3] Update ingest.py

---
 common/ingest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/ingest.py b/common/ingest.py
index 03996fd..af80367 100644
--- a/common/ingest.py
+++ b/common/ingest.py
@@ -94,7 +94,7 @@ def ingest(data: list[Any], log_type: str):
   # [{"logText": str(log1)}, {"logText": str(log2)}, ...]
   parsed_data = list(
       map(
-          lambda i: {"logText": str(json.dumps(i, separators=(',', ':')).encode("utf-8"), "utf-8")},
+          lambda i: {"logText": str(json.dumps(utils.cleanup(i), separators=(',', ':')).encode("utf-8"), "utf-8")},
           data,
       )
   )