Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .github/scripts/test-deployment-ansible.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export OS_AUTH_TYPE=v3applicationcredential
export OS_INTERFACE=public
export OS_IDENTITY_API_VERSION=3

# --- Step 1 ---
# --- Step 1 ---
echo "Clone catalog repository (ref: '${CATALOG_REF}')"
PATH_TO_CATALOG_REPO="${GITHUB_WORKSPACE}/ewc-community-hub"
git clone \
Expand All @@ -20,7 +20,7 @@ git clone \
"https://github.com/ewcloud/ewc-community-hub.git" \
"${PATH_TO_CATALOG_REPO}"

# --- Step 2 ---
# --- Step 2 ---
echo "Extract metadata of '${ITEM_NAME}' from catalog"
PATH_TO_CATALOG="${PATH_TO_CATALOG_REPO}/items.yaml"
item_metadata=$(yq '.spec.items."'"$ITEM_NAME"'" | del(.description)' $PATH_TO_CATALOG) # Drop the description attribute(for succinct step summary)
Expand Down Expand Up @@ -51,7 +51,7 @@ if [[ "${INPUT_SPEC_JSON}" != "{}" ]]; then
to_entries[]
| "--item-inputs"
, (.key + "=" +
if .value == null then
if .value == null then
(null | tojson)
elif (.value | type) == "array" or (.value | type) == "object" then
(.value | tojson)
Expand Down Expand Up @@ -87,7 +87,7 @@ chmod 600 "$ANSIBLE_SSH_PUBLIC_KEY_FILE"
export EWC_CLI_SSH_PUBLIC_KEY_PATH=$ANSIBLE_SSH_PUBLIC_KEY_FILE
export EWC_CLI_SSH_PRIVATE_KEY_PATH=$ANSIBLE_SSH_PRIVATE_KEY_FILE

# --- Step 7 ---
# --- Step 7 ---
echo "Login"

EWCCLI_LOGIN_EXIT_CODE=0
Expand All @@ -97,10 +97,10 @@ set +e
EWCCLI_LOGIN_EXIT_CODE=$?
set -e

# --- Step 8 ---
# --- Step 8 ---
echo "Deploy (including VM provisioning)"

if [ -z "${EXTRA_VARS}" ]; then
if [ -z "${EXTRA_VARS}" ]; then
EWCCLI_DEPLOY_CMD=(ewc hub --path-to-catalog "${PATH_TO_CATALOG}" deploy "${ITEM_NAME}" --server-name "github-vm-${GITHUB_RUN_ID}" --external-ip)
else
EWCCLI_DEPLOY_CMD=(ewc hub --path-to-catalog "${PATH_TO_CATALOG}" deploy "${ITEM_NAME}" --server-name "github-vm-${GITHUB_RUN_ID}" --external-ip "${EXTRA_VARS[@]}")
Expand All @@ -112,7 +112,7 @@ set +e
EWCCLI_DEPLOY_EXIT_CODE=$?
set -e

# --- Step 9 ---
# --- Step 9 ---
echo "Cleanup"

EWCCLI_CLEANUP_EXIT_CODE=0
Expand Down Expand Up @@ -197,9 +197,9 @@ cp $GITHUB_STEP_SUMMARY "$ARTIFACTS_DIR/summary.md"
# NOTE: As of 18.02.2026, this is needed for cleaning up any created Floating IPs, since the EWCCLI does not remove them and is unclear if it reuses existing consistently
echo "Unlocking the Floating IP (best-effort approach)"
floating_ip=$(openstack server show "github-vm-${GITHUB_RUN_ID}" -f json | jq '.addresses[].[] | select( (startswith("192.") or startswith("10.")) | not )' | tr -d '"' ) || true
openstack floating ip delete $floating_ip || true
openstack floating ip delete $floating_ip || true

# --- Step 13 ---
# --- Step 13 ---
echo "Re-rasing test errors (if any)"
if [ "$EWCCLI_STATUS" = "failing" ]; then
echo "::error::One or more failures caught during testing. See the summary or logs for details"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-deployment-ansible-ecmwf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ env:
ANSIBLE_SSH_PUBLIC_KEY: '${{ secrets.ECMWF_ANSIBLE_SSH_PUBLIC_KEY }}'
ITEM_NAME: ${{ inputs.itemName }}
CATALOG_REF: ${{ inputs.catalogRef }}
INPUT_SPEC_JSON: '${{ inputs.inputSpecJson }}'
INPUT_SPEC_JSON: '${{ inputs.inputSpecJson }}'

jobs:
deploy-test:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-deployment-ansible-eumetsat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ env:
ANSIBLE_SSH_PUBLIC_KEY: '${{ secrets.EUMETSAT_ANSIBLE_SSH_PUBLIC_KEY }}'
ITEM_NAME: ${{ inputs.itemName }}
CATALOG_REF: ${{ inputs.catalogRef }}
INPUT_SPEC_JSON: '${{ inputs.inputSpecJson }}'
INPUT_SPEC_JSON: '${{ inputs.inputSpecJson }}'

jobs:
deploy-test:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/unittest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:

- name: Install Python dependencies
run: pip install -e .[test]

- name: Run pre-commit
run: pre-commit run --all-files

Expand All @@ -43,6 +43,6 @@ jobs:

- name: Install Python dependencies
run: pip install -e .[test]

- name: Run pytest
run: pytest
2 changes: 2 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
# See the LICENSE file for more details

---
exclude: ^ewccli/tests/

repos:
- repo: https://github.com/Lucas-C/pre-commit-hooks
rev: v1.5.5
Expand Down
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,8 @@ Info required for a profile:
[my-profile]
federee = EUMETSAT or ECMWF
tenant_name = eumetsat-ewc-communityhub
application_credential_id =
application_credential_secret =
application_credential_id =
application_credential_secret =
ssh_public_key_path =
ssh_private_key_path =
```
Expand Down Expand Up @@ -225,7 +225,7 @@ where ITEM is taken from `ewc hub list` command under Item column.

If you would like to test the deployment of:
* **an Item with private source code (local or remote)**

OR
* **a new Item, not yet published in the EWC Community Hub**

Expand Down Expand Up @@ -283,14 +283,14 @@ spec:
published: true
```

where
where

- `sources` can be (only the first element in the list is considered):
- Public repo URL (e.g. https://github.com/your-repo.git) if your repository is public already
- Absolute path to a directory with the Item (e.g. `/home/murdaca/custom-items/new-item`). The path needs to point to a directory that needs to exists an not be empty. (WARNING: No local path are accepted!)
- `pathToMainFile` is the relattive path to your directory or repository
- `pathToRequirementsFile` is the relattive path to your directory or repository
- `publicIP` is a flag used to enable deployment of
- `publicIP` is a flag used to enable deployment of
- `ewccli.inputs` is the list of inputs you want the user to be able to provide, they can be mandatory or optional, respecively with default key not set or set.

### Running a test
Expand Down
6 changes: 4 additions & 2 deletions ewccli/backends/kubernetes/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@

"""Helpers methods for Kubernetes backend."""

from typing import Any, List, Dict

def get_status_from_conditions(conditions: list) -> str:

def get_status_from_conditions(conditions: List[Dict[str, Any]]) -> Any:
"""
Extract a human-readable status from conditions list.

Expand All @@ -27,7 +29,7 @@ def get_status_from_conditions(conditions: list) -> str:
return f"{first.get('type')}: {first.get('status')}"


def get_reason_from_conditions(conditions: list) -> str:
def get_reason_from_conditions(conditions: List[Dict[str, Any]]) -> Any:
"""
Extract the reason from the 'Ready' condition if available,
otherwise fallback to the first condition's reason.
Expand Down
58 changes: 24 additions & 34 deletions ewccli/backends/openstack/backend_ostack.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import time
import sys
import os
from typing import Tuple, Optional, Any
from typing import Tuple, Optional, Any, Dict
from collections import namedtuple
from pathlib import Path

Expand Down Expand Up @@ -65,9 +65,7 @@ def __init__(
)
self.auth_url = auth_url
else:
config = (
OpenStackConfig()
) # ~/.config/openstack/clouds.yaml, to change use OS_CLIENT_CONFIG_FILE
config = OpenStackConfig() # ~/.config/openstack/clouds.yaml, to change use OS_CLIENT_CONFIG_FILE
# Get the default cloud if no name is specified
cloud = (
config.get_one()
Expand Down Expand Up @@ -133,14 +131,14 @@ def create_server(
server_name: str,
image_name: str,
flavour_name: str,
networks: tuple,
networks: Optional[tuple],
keypair_name: str,
sec_groups: tuple,
sec_groups: Optional[tuple],
attempts: int = 1,
retry_delay_s: int = 30,
wait_time_s: int = 600,
dry_run: bool = False,
) -> Tuple[ServerResult, Optional[str], dict[Any, Any]]:
) -> Tuple[ServerResult, Optional[str], Dict[str, Any]]:
"""Create an OpenStack server.

Automatically deletes and retries the creation process until a server is available.
Expand Down Expand Up @@ -388,16 +386,12 @@ def create_server(
new_server,
)


def find_latest_image(
self,
conn: openstack.connection.Connection,
prefix: str
):
def find_latest_image(self, conn: openstack.connection.Connection, prefix: str):
"""
Select the latest image for CPU or GPU families with special rules.
"""
import re

TIMESTAMP_RE = r"\d{14}"

def is_cpu_image(prefix: str, name: str):
Expand All @@ -420,11 +414,13 @@ def is_cpu_image(prefix: str, name: str):
def is_gpu_rocky(name: str):
# Prefix: Rocky-9-GPU
# Match: Rocky-9.<minor>-GPU-<timestamp>
return bool(re.match(
rf"^Rocky-9\.\d+-GPU-{TIMESTAMP_RE}$",
name,
re.IGNORECASE,
))
return bool(
re.match(
rf"^Rocky-9\.\d+-GPU-{TIMESTAMP_RE}$",
name,
re.IGNORECASE,
)
)

def is_gpu_ubuntu(name: str):
# Prefix: Ubuntu 22.04 NVIDIA_AI
Expand All @@ -451,7 +447,11 @@ def image_matches(name: str, prefix: str):
return False

# Filter matching images
matches = [img for img in conn.compute.images() if image_matches(name=img.name, prefix=prefix)]
matches = [
img
for img in conn.compute.images()
if image_matches(name=img.name, prefix=prefix)
]

if not matches:
return None
Expand All @@ -460,7 +460,6 @@ def image_matches(name: str, prefix: str):
matches.sort(key=lambda img: img.created_at, reverse=True)
return matches[0]


def check_server_inputs(
self,
conn: openstack.connection.Connection,
Expand All @@ -474,7 +473,9 @@ def check_server_inputs(
image = conn.compute.find_image(image_name)

if not image:
total_images = ewc_hub_config.EWC_CLI_CPU_IMAGES + [ewc_hub_config.EWC_CLI_GPU_IMAGES_SITE_MAP[federee]]
total_images = ewc_hub_config.EWC_CLI_CPU_IMAGES + [
ewc_hub_config.EWC_CLI_GPU_IMAGES_SITE_MAP[federee]
]
error_message = (
f"❌ Unsupported OS image for the EWC CLI: {image_name}\n\n"
f"🖥️ EWC Supported images (short names): [bold green]{', '.join(total_images)}[/bold green]\n"
Expand Down Expand Up @@ -528,7 +529,6 @@ def check_server_inputs(

return True, ""


def list_servers(
self,
conn: openstack.connection.Connection,
Expand All @@ -550,7 +550,6 @@ def list_servers(
image_map = {image.id: image.name for image in conn.compute.images()}

for server in conn.compute.servers():

if (
not (
server.metadata.get("deployed")
Expand Down Expand Up @@ -822,12 +821,7 @@ def remove_network(
_LOGGER.warning(f"{network_name} not found for server {server.name}")
return NetworkResult(True, False)


def ssh_key_matches_openstack(
self,
public_key_path: str,
keypair: dict
) -> bool:
def ssh_key_matches_openstack(self, public_key_path: str, keypair: dict) -> bool:
"""
Check whether the local SSH public key matches the OpenStack keypair.

Expand All @@ -852,7 +846,6 @@ def ssh_key_matches_openstack(

return local_key == openstack_key


def create_keypair(
self,
conn: openstack.connection.Connection,
Expand All @@ -877,11 +870,8 @@ def create_keypair(
existing_key = conn.compute.find_keypair(keypair_name)

if existing_key:

match = OpenstackBackend.ssh_key_matches_openstack(
conn,
keypair=existing_key,
public_key_path=public_key_path
conn, keypair=existing_key, public_key_path=public_key_path
)

if not match:
Expand Down
Loading
Loading