Introduce multi level templating

docker-local-pkg/.gitignore → .gitignore

@@ -1,5 +1,5 @@
 .venv/*
-.zen/*
+*/.zen/*
 uv.lock
 __pycache__/*
 *.pyc

docker-local-pkg/.dockerignore

@@ -4,5 +4,4 @@
 .gitignore
 .DS_Store
 .idea
-.pytest_cache
-src/demo
+.pytest_cache

docker-local-pkg/README.md

@@ -26,25 +26,29 @@ The solution works by:
 
 ## Setup
 ```sh
-uv sync --dev # needed once
-uv run zenml init # needed once
-uv run zenml login # needed once
-uv run zenml project set default # needed once
+uv sync --dev
+uv run zenml init
+uv run zenml login
+uv run zenml project set default
 ```
 
 ## Running Pipelines
 
 ### Local Execution
 ```sh
-uv run zenml stack set default # run on default local stack
+uv run zenml stack set default
 uv run training-pipeline
 ```
 
 This will run the pipeline defined in `src/demo/pipelines/training_pipeline.py` using your local environment with the demo package installed in editable mode.
 
 ### Remote Execution
+
+In this case we will run on GCP but this will also work for other remote orchestrators.
+
 ```sh
-uv run zenml stack set <...> # Set some stack with a docker based orchestrator
+uv run zenml stack set <...> 
+uv run zenml integration install gcp
 uv run training-pipeline
 ```
 

docker-local-pkg/pyproject.toml

@@ -5,8 +5,7 @@ description = "Demo"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "pandas",
-    "zenml[server]==0.81.0"
+    "pandas",  "zenml"
 ]
 
 [build-system]

docker-local-pkg/src/demo/pipelines/training_pipeline.py

@@ -2,19 +2,18 @@
 
 from zenml import pipeline
 from zenml.config import DockerSettings
-from demo.steps.training_step import train
+from demo.steps.training_step import train  # in order for this import to work you need the DockerSettings defined below
+
 
 @pipeline(
     enable_cache=False,
     settings = {"docker": 
-        DockerSettings(
-            dockerfile="Dockerfile", 
-            build_context_root=".",
-            # build_config={"build_options": {"platform": "linux/amd64"}}, # If you are 
-            python_package_installer="uv",
-            prevent_build_reuse=True
-        )
-    }
+            DockerSettings(
+                python_package_installer="uv",
+                local_project_install_command="uv pip install -e . --no-deps",
+                allow_including_files_in_images=True,
+            )
+        }
 )
 def training_pipeline() -> None:
     train() 

trigger-multiple-templates/README.md

@@ -0,0 +1,85 @@
+# Trigger Multiple ML Pipelines
+
+This project demonstrates how to use ZenML to create a single run template to use as a singular endpoint to trigger many instances of another run template.
+This can be useful when you want to create many slightly different version of a model based on the same code but with different parameters or data. 
+
+
+## Setup
+
+1. Create and activate a virtual environment:
+
+```bash
+# Create a new virtual environment
+pyenv virtualenv trigger-multiple-templates
+
+# Activate the virtual environment
+pyenv activate trigger-multiple-templates
+```
+
+2. Install dependencies using uv:
+
+```bash
+# Install requirements
+uv pip install -r requirements.txt
+```
+
+## Project Structure
+
+- `main.py`: Contains the main trigger pipeline that orchestrates multiple runs
+- `pipeline.py`: Defines the base ML pipeline for training models
+- `requirements.txt`: Lists the project dependencies
+- `.python-version`: Specifies the Python version for pyenv
+
+## How It Works
+
+The project consists of two main components:
+
+1. **Base ML Pipeline** (`pipeline.py`):
+   - Loads the Iris dataset
+   - Splits the data into training and test sets
+   - Trains a Random Forest classifier with configurable number of estimators
+   - Uses Docker settings with uv as the package installer
+
+2. **Trigger Pipeline** (`main.py`):
+   - Creates a template from the base ML pipeline
+   - Generates multiple configurations with different numbers of estimators (0 to 90, in steps of 10)
+   - Triggers multiple pipeline runs with these configurations
+   - Uses a GCP stack for execution
+
+## Creating the Run Templates
+
+To create the run template:
+
+```bash
+python main.py
+```
+
+This will:
+1. Set up the base ML pipeline template
+2. Create multiple configurations with different numbers of estimators to be used as the default config
+3. Set up the main_trigger_pipeline which serves as the controller
+
+## Trigger runs when ready
+
+Once the cloud builder has created all the run templates, here is how you can now trigger the runs
+
+![Created Templats](../.assets/CreatedTemplates.png)
+
+*Figure 1: Here is what the ready run templates will look like in your Dashboard*
+
+![Run Configuration](../.assets/RunMainPipeline.png)
+
+*Figure 2: Once you clicked into the run template for the main pipeline, you can now decide to run it*
+
+![Multiple Runs](../.assets/ChangeConfig.png)
+
+*Figure 3: Heere you now have the option to change the configuration that is passed onto the actual training pipelines*
+
+![Run Details](../.assets/10runs.png)
+
+*Figure 4: Here are 10 instances of the training pipelines running in parallel, triggered by our main pipeline*
+
+
+## Note
+
+The pipeline is configured to use a GCP stack named "axj-gcp-stack". Make sure you go ahead and change stack, project and other names before you run this. 

trigger-multiple-templates/main.py

@@ -0,0 +1,92 @@
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from zenml import pipeline, step
+from zenml.client import Client
+from zenml.exceptions import EntityExistsError
+from zenml.config import DockerSettings
+
+STACK_NAME = "axj-gcp-stack"
+PROJECT_NAME = "default"
+BASE_TEMPLATE_NAME = "simple_ml_pipeline_template"
+MAIN_TRIGGER_TEMPLATE_NAME = "main_trigger_pipeline"
+
+
+@step(enable_cache=False)
+def trigger_run_templates(template_name: str, configs: List[Optional[Dict[str, Any]]]):
+
+    client = Client()
+    template = client.get_run_template(template_name)
+
+    for config in configs:
+        time.sleep(1)
+
+        # Trigger the pipeline with the updated configuration
+        run = Client().trigger_pipeline(
+            template_id=template.id,
+            run_configuration=config,
+        )
+
+        print(f"Run of pipeline triggered with {config['steps']['train_model']['parameters']['n_estimators']} estimators.")
+
+
+@pipeline(
+    settings = {"docker": 
+        DockerSettings(
+            python_package_installer="uv"
+        )
+    }
+)
+def main_trigger_pipeline():
+
+    trigger_run_templates(template_name=BASE_TEMPLATE_NAME)
+
+
+if __name__ == "__main__":
+
+    # Set the active project and stack
+    client = Client()
+    client.set_active_project(PROJECT_NAME)
+    remote_stack = client.get_stack(STACK_NAME)
+    os.environ["ZENML_ACTIVE_STACK_ID"] = str(remote_stack.id)
+
+    # Create the run template for the base pipeline
+    try:
+        from pipeline import simple_ml_pipeline
+
+        template = simple_ml_pipeline.create_run_template(name=BASE_TEMPLATE_NAME)
+    except EntityExistsError:
+        template = client.get_run_template(BASE_TEMPLATE_NAME)
+
+    # Create a batch of configs to use for the base pipeline
+    configs: List[Optional[Dict[str, Any]]] = []
+    for i in range(10):
+        config: Optional[Dict[str, Any]] = template.config_template
+
+        config["steps"]["train_model"]["parameters"]["n_estimators"] = i*10
+
+        configs.append(config)
+
+    # Create the run template for the main trigger pipeline
+    params = {
+            "trigger_run_templates": {
+                "parameters": {
+                    "configs": configs,
+                    "template_name": BASE_TEMPLATE_NAME
+                }
+            }
+        }
+    collective_template = main_trigger_pipeline.with_options(
+        step_configurations=params
+    )
+
+    collective_template.create_run_template(name=MAIN_TRIGGER_TEMPLATE_NAME)
+
+
+
+
+
+
+
+

trigger-multiple-templates/pipeline.py

@@ -0,0 +1,47 @@
+from typing import Tuple
+
+import numpy as np
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import RandomForestClassifier
+
+from zenml import pipeline, step
+from zenml.config import DockerSettings
+
+
+@step
+def load_data() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    """Load the iris dataset and split it into train and test sets."""
+    iris = load_iris()
+    X_train, X_test, y_train, y_test = train_test_split(
+        iris.data, iris.target, test_size=0.2, random_state=42
+    )
+    return X_train, X_test, y_train, y_test
+
+@step(enable_cache=False)
+def train_model(
+    X_train: np.ndarray,
+    y_train: np.ndarray,
+    n_estimators: int = 100,
+) -> RandomForestClassifier:
+    """Train a random forest classifier."""
+    model = RandomForestClassifier(n_estimators=n_estimators, random_state=42)
+    model.fit(X_train, y_train)
+    return model
+
+@pipeline(
+    settings = {"docker": 
+        DockerSettings(
+            requirements="requirements.txt",
+            python_package_installer="uv",
+            prevent_build_reuse=True
+        )
+    }
+)
+def simple_ml_pipeline():
+    """A simple ML pipeline that loads data, trains a model, and evaluates it."""
+    # Load and split the data
+    X_train, X_test, y_train, y_test = load_data()
+
+    # Train the model
+    model = train_model(X_train, y_train)

trigger-multiple-templates/requirements.txt

@@ -0,0 +1,3 @@
+zenml>=0.83.0
+scikit-learn>=1.3.0
+numpy>=1.24.0