post-indicator generation and output fetching workflow

Author: ciaransweet

.gitignore

@@ -10,3 +10,6 @@ wheels/
 .venv
 
 .idea
+generated-indicators*
+output*
+inputs*

Dockerfile

@@ -0,0 +1,16 @@
+FROM amazonlinux:latest
+
+# Install dependencies
+RUN yum install -y awscli
+
+# Set working directory
+WORKDIR /app
+
+# Copy the Bash script into the container
+COPY prepare-output.sh /prepare-output.sh
+
+# Ensure script is executable
+RUN chmod +x /prepare-output.sh
+
+# Set entrypoint to use the script
+ENTRYPOINT ["bash", "/prepare-output.sh"]

fetch_hazard_indicator.cwl

@@ -0,0 +1,83 @@
+cwlVersion: v1.2
+$graph:
+  - class: Workflow
+    id: fetch-hazard-indicator
+    label: Fetch a hazard indicator from S3
+    doc: Fetch a hazard indicator produced with os_climate_hazard
+    requirements:
+      ResourceRequirement:
+        coresMax: 2
+        ramMax: 4096
+
+    inputs:
+      hazard_indicator_key:
+        type: string
+        default: ""
+      aws_access_key_id:
+        type: string
+        default: ""
+      aws_secret_access_key:
+        type: string
+        default: ""
+      aws_session_token:
+        type: string
+        default: ""
+
+    outputs:
+      - id: fetched-indicator
+        type: Directory
+        outputSource:
+          - fetch-indicator-step/indicator-results
+
+    steps:
+      fetch-indicator-step:
+        run: "#fetch-indicator-command"
+        in:
+          hazard_indicator_key: hazard_indicator_key
+          aws_access_key_id: aws_access_key_id
+          aws_secret_access_key: aws_secret_access_key
+          aws_session_token: aws_session_token
+        out:
+          - indicator-results
+
+
+  - class: CommandLineTool
+    id: fetch-indicator-command
+
+    hints:
+      DockerRequirement:
+        dockerPull: indicator-fetching:latest
+
+    requirements:
+      ResourceRequirement:
+        coresMax: 2
+        ramMax: 4096
+      NetworkAccess:
+        networkAccess: true
+      EnvVarRequirement:
+          envDef:
+            AWS_ACCESS_KEY_ID: $(inputs.aws_access_key_id)
+            AWS_SECRET_ACCESS_KEY: $(inputs.aws_secret_access_key)
+            AWS_SESSION_TOKEN: $(inputs.aws_session_token)
+            AWS_DEFAULT_REGION: "eu-west-2"
+
+    inputs:
+      hazard_indicator_key:
+        type: string
+      aws_access_key_id:
+        type: string
+      aws_secret_access_key:
+        type: string
+      aws_session_token:
+        type: string
+
+    outputs:
+      indicator-results:
+        type: Directory
+        outputBinding:
+          glob: output
+
+    baseCommand: ["prepare-output.sh"]
+
+    arguments:
+      - valueFrom: $(inputs.hazard_indicator_key)

json-to-command.py

@@ -0,0 +1,10 @@
+import json
+import sys
+
+def main(input_path):
+    with open(input_path, "r") as f:
+        jase = json.load(f)
+    print('"' + json.dumps(jase).replace('"', '\\"') + '"')
+
+if __name__ == '__main__':
+    main(sys.argv[1])

prepare-output.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+VALUE="$2"
+SRC_BUCKET="ukri-eodh-hazard-tiles-test"
+SRC_PATH="indicator-generation/$VALUE"
+DEST_DIR="output"
+
+# Create output directory if it doesn't exist
+mkdir -p "$DEST_DIR"
+
+# Sync data from S3
+aws s3 cp "s3://$SRC_BUCKET/$SRC_PATH/" "$DEST_DIR/" --recursive --exclude "*" \
+    --include "cubified/*" \
+    --include "days_tas_above_*.json" \
+    --include "degree_days_*.json" \
+    --include "collection.json"
+
+echo "Files have been organized into '$DEST_DIR'."

pyproject.toml

@@ -9,7 +9,8 @@ dependencies = [
     "stactools-osc-hazard",
     "typer>=0.15.2",
     "cf-xarray==0.9.4",
-    "xarray==2023.1.0"
+    "xarray==2023.1.0",
+    "cwltool>=3.1.20250110105449",
 ]
 
 [tool.uv.sources]