Initial commit

docker-local-pkg/.gitignore → .gitignore

@@ -1,5 +1,5 @@
 .venv/*
-.zen/*
+*/.zen/*
 uv.lock
 __pycache__/*
 *.pyc

trigger-multiple-templates/main.py

@@ -0,0 +1,82 @@
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from zenml import pipeline, step
+from zenml.client import Client
+from zenml.exceptions import EntityExistsError
+
+STACK_NAME = "axj-gcp-stack"
+PROJECT_NAME = "default"
+BASE_TEMPLATE_NAME = "simple_ml_pipeline_template"
+MAIN_TRIGGER_TEMPLATE_NAME = "main_trigger_pipeline"
+
+
+@step(enable_cache=False)
+def trigger_pipeline(template_name: str, configs: List[Optional[Dict[str, Any]]]):
+
+    client = Client()
+    template = client.get_run_template(template_name)
+
+    for config in configs:
+        time.sleep(1)
+
+        # Trigger the pipeline with the updated configuration
+        run = Client().trigger_pipeline(
+            template_id=template.id,
+            run_configuration=config,
+        )
+
+        print(f"Run of pipeline triggered with {config['steps']['train_model']['parameters']['n_estimators']} estimators.")
+
+
+@pipeline
+def main_trigger_pipeline():
+    trigger_pipeline(template_name=BASE_TEMPLATE_NAME)
+
+
+if __name__ == "__main__":
+    client = Client()
+
+    client.set_active_project("default")
+
+    remote_stack = client.get_stack("axj-gcp-stack")
+    os.environ["ZENML_ACTIVE_STACK_ID"] = str(remote_stack.id)
+
+    try:
+        from pipeline import simple_ml_pipeline
+
+        template = simple_ml_pipeline.create_run_template(name=BASE_TEMPLATE_NAME)
+    except EntityExistsError:
+        template = client.get_run_template(BASE_TEMPLATE_NAME)
+
+    configs: List[Optional[Dict[str, Any]]] = []
+    for i in range(10):
+        config: Optional[Dict[str, Any]] = template.config_template
+
+        config["steps"]["train_model"]["parameters"]["n_estimators"] = i*10
+
+        configs.append(config)
+
+    params = {
+            "trigger_pipeline": {
+                "parameters": {
+                    "configs": configs,
+                    "template_name": BASE_TEMPLATE_NAME
+                }
+            }
+        }
+
+    collective_template = main_trigger_pipeline.with_options(
+        step_configurations=params
+    )
+
+    collective_template.create_run_template(name=MAIN_TRIGGER_TEMPLATE_NAME)
+
+
+
+
+
+
+
+

trigger-multiple-templates/pipeline.py

@@ -0,0 +1,47 @@
+from typing import Tuple
+
+import numpy as np
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import RandomForestClassifier
+
+from zenml import pipeline, step
+from zenml.config import DockerSettings
+
+
+@step
+def load_data() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """Load the iris dataset and split it into train and test sets."""
+    iris = load_iris()
+    X_train, X_test, y_train, y_test = train_test_split(
+        iris.data, iris.target, test_size=0.2, random_state=42
+    )
+    return X_train, X_test, y_train, y_test
+
+@step(enable_cache=False)
+def train_model(
+    X_train: np.ndarray,
+    y_train: np.ndarray,
+    n_estimators: int = 100,
+) -> RandomForestClassifier:
+    """Train a random forest classifier."""
+    model = RandomForestClassifier(n_estimators=n_estimators, random_state=42)
+    model.fit(X_train, y_train)
+    return model
+
+@pipeline(
+    settings = {"docker": 
+        DockerSettings(
+            requirements="requirements.txt",
+            python_package_installer="uv",
+            prevent_build_reuse=True
+        )
+    }
+)
+def simple_ml_pipeline():
+    """A simple ML pipeline that loads data, trains a model, and evaluates it."""
+    # Load and split the data
+    X_train, X_test, y_train, y_test = load_data()
+
+    # Train the model
+    model = train_model(X_train, y_train)

trigger-multiple-templates/requirements.txt

@@ -0,0 +1,3 @@
+zenml>=0.81.0
+scikit-learn>=1.3.0
+numpy>=1.24.0