diff --git a/common/ingest.py b/common/ingest.py index ba239ff..af80367 100644 --- a/common/ingest.py +++ b/common/ingest.py @@ -94,7 +94,7 @@ def ingest(data: list[Any], log_type: str): # [{"logText": str(log1)}, {"logText": str(log2)}, ...] parsed_data = list( map( - lambda i: {"logText": str(json.dumps(i).encode("utf-8"), "utf-8")}, + lambda i: {"logText": str(json.dumps(utils.cleanup(i), separators=(',', ':')).encode("utf-8"), "utf-8")}, data, ) ) diff --git a/common/utils.py b/common/utils.py index f02225f..b23becc 100644 --- a/common/utils.py +++ b/common/utils.py @@ -138,3 +138,20 @@ def cloud_logging(message: str, severity: str = "INFO") -> None: severity (str): severity of the message. Defaults to "INFO". """ print(json.dumps({"severity": severity, "message": message})) + + +def cleanup(object: dict | list) -> dict | list: + """Remove empty lists and dicts from an object + + Args: + object (dict | list): The object to cleanup + + Returns: + dict | list: The cleaned object + """ + if isinstance(object, dict): + return {k: cleanup(v) for k, v in object.items() if v and cleanup(v)} + elif isinstance(object, list): + return [cleanup(v) for v in object if v and cleanup(v)] + else: + return object