Merge branch 'main' into fix/importTasksWithPasswords-CMEM-5932

Author: seebi

.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: v7.0.0
+_commit: v7.1.0
 _src_path: gh:eccenca/cmem-plugin-template
 author_mail: [email protected]
 author_name: eccenca GmbH

.gitlab-ci.yml

@@ -1,6 +1,6 @@
 ---
 default:
-  image: docker-registry.eccenca.com/eccenca-python:v3.11.4
+  image: docker-registry.eccenca.com/eccenca-python:v3.11.9-2
   # all jobs can be interrupted in case a new commit is pushed
   interruptible: true
   before_script:

.idea/cmem-plugin-base.iml

@@ -6,11 +6,30 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 
 ## [Unreleased]
 
+### Added
+
+- Custom actions for Workflow plugins (CMEM-5576)
+- Added explicit_schema parameter to FlexibleOutputSchema (CMEM-6444)
+
 ### Fixed
 
-- Check if passwords can be decrypted, i.e., if the key is valid (CMEM-5932).
+- Check if passwords can be decrypted, i.e., if the key is valid (CMEM-5932)
+
+## [4.9.0] 2025-02-20
+
+### Added
+
+- TypedEntitySchema - A custom entity schema that holds entities of a specific type (e.g. files)
+- FileEntitySchema - Entity schema that holds a collection of files
+- testing module with different context classes for tests:
+  - TestUserContext
+  - TestPluginContext
+  - TestTaskContext
+  - TestExecutionContext
+  - TestSystemContext
+
 
-## [4.8.0] 2024-09-12
+## [4.8.0] 2024-09-12 - shipped with DI v24.3.0
 
 ### Added
 

CHANGELOG.md

@@ -157,9 +157,7 @@ tasks:
     <<: *preparation
     cmds:
       # ignore 51358 safety - dev dependency only
-      # ignore 74735 jinja2 - dev dependency only
-      # ignore 75180 pip - dev dependency only
-      - poetry run safety check -i 51358 -i 74735 -i 75180
+      - poetry run safety check -i 51358
 
   check:ruff:
     desc: Complain about everything else

Taskfile.yaml

@@ -9,7 +9,8 @@
 from pkgutil import get_data
 from typing import Any, ClassVar
 
-from cmem_plugin_base.dataintegration.plugins import TransformPlugin, WorkflowPlugin
+from cmem_plugin_base.dataintegration.context import PluginContext
+from cmem_plugin_base.dataintegration.plugins import PluginBase, TransformPlugin, WorkflowPlugin
 from cmem_plugin_base.dataintegration.types import (
     ParameterType,
     ParameterTypes,
@@ -96,6 +97,84 @@ def __init__(  # noqa: PLR0913
         self.visible = visible
 
 
+class PluginAction:
+    """Custom plugin action.
+
+    Plugin actions provide additional functionality besides the default execution.
+    They can be triggered from the plugin UI.
+    Each action is based on a method on the plugin class. Besides the self parameter,
+    the method can have one additional parameter of type PluginContext.
+    The return value of the method will be converted to a string and displayed in the UI.
+    The string may use Markdown formatting.
+    The method may return None, in which case no output will be displayed.
+    It may raise an exception to signal an error to the user.
+
+    :param name: The name of the method.
+    :param label: A human-readable label of the action
+    :param description: A human-readable description of the action
+    :param icon: An optional custom icon.
+    """
+
+    def __init__(self, name: str, label: str, description: str, icon: Icon | None = None):
+        self.name = name
+        self.label = label
+        self.description = description
+        self.icon = icon
+        self.validated = False
+        self.provide_plugin_context = False  # Will be set by validate()
+
+    def validate(self, plugin_class: type) -> None:
+        """Validate the action and set the `provide_plugin_context` boolean.
+
+        :param plugin_class: The plugin class
+        """
+        # Get the method from the class.
+        try:
+            method = getattr(plugin_class, self.name)
+        except AttributeError:
+            raise TypeError(
+                f"Plugin class '{plugin_class.__name__}' does not have a method named '{self.name}'"
+            ) from None
+        if not callable(method):
+            raise TypeError(f"'{self.name}' in class '{plugin_class.__name__}' is not a function.")
+
+        # Check parameters
+        parameters = list(inspect.signature(method).parameters.values())
+        if len(parameters) == 1:
+            self.provide_plugin_context = False
+        elif len(parameters) - 1 == 1:
+            if parameters[1].annotation is PluginContext:
+                self.provide_plugin_context = True
+            else:
+                raise TypeError(
+                    f"Argument of method '{self.name}' in {plugin_class.__name__} must "
+                    f"be typed PluginContext (it's {parameters[1].annotation})."
+                )
+        else:
+            raise TypeError(
+                f"Method '{self.name}' in {plugin_class.__name__} has more than one"
+                f" argument (besides 'self')."
+            )
+        self.validated = True
+
+    def execute(self, plugin: PluginBase, context: PluginContext) -> str | None:
+        """Call the action.
+
+        :param plugin: The plugin instance on which the action is called.
+        :param context: The plugin context
+        :return: The result of the action as string
+        """
+        if not self.validated:
+            raise ValueError("Action must be validated before it can be executed.")
+        if self.provide_plugin_context:
+            result = getattr(plugin, self.name)(context)
+        else:
+            result = getattr(plugin, self.name)()
+        if result is None:
+            return None
+        return str(result)
+
+
 class PluginDescription:
     """A plugin description.
 
@@ -106,6 +185,7 @@ class PluginDescription:
     :param categories: The categories to which this plugin belongs to.
     :param parameters: Available plugin parameters
     :param icon: An optional custom plugin icon.
+    :param actions: Custom plugin actions.
     """
 
     def __init__(  # noqa: PLR0913
@@ -118,6 +198,7 @@ def __init__(  # noqa: PLR0913
         categories: list[str] | None = None,
         parameters: list[PluginParameter] | None = None,
         icon: Icon | None = None,
+        actions: list[PluginAction] | None = None,
     ) -> None:
         #  Set the type of the plugin. Same as the class name of the plugin
         #  base class, e.g., 'WorkflowPlugin'.
@@ -153,6 +234,12 @@ def __init__(  # noqa: PLR0913
         else:
             self.parameters = parameters
         self.icon = icon
+        if actions is None:
+            self.actions = []
+        else:
+            self.actions = actions
+        for action in self.actions:
+            action.validate(plugin_class)
 
 
 @dataclass
@@ -233,6 +320,7 @@ class Plugin:
     :param categories: The categories to which this plugin belongs to.
     :param parameters: Available plugin parameters.
     :param icon: Optional custom plugin icon.
+    :param actions: Custom plugin actions
     """
 
     plugins: ClassVar[list[PluginDescription]] = []
@@ -246,12 +334,14 @@ def __init__(  # noqa: PLR0913
         categories: list[str] | None = None,
         parameters: list[PluginParameter] | None = None,
         icon: Icon | None = None,
+        actions: list[PluginAction] | None = None,
     ):
         self.label = label
         self.description = description
         self.documentation = documentation
         self.plugin_id = plugin_id
         self.icon = icon
+        self.actions = actions
         if categories is None:
             self.categories = []
         else:
@@ -272,6 +362,7 @@ def __call__(self, func: type):
             categories=self.categories,
             parameters=self.retrieve_parameters(func),
             icon=self.icon,
+            actions=self.actions,
         )
         Plugin.plugins.append(plugin_desc)
         return func
@@ -304,7 +395,7 @@ def retrieve_parameters(self, plugin_class: type) -> list[PluginParameter]:
                 # Special handling of PluginContext parameter
                 if isinstance(param.param_type, PluginContextParameterType):
                     param.visible = False  # Should never be visible in the UI
-                    param.default_value = ""  # dummy value
+                    param.default_value = ""  # default value
 
                 if param.default_value is None and sig_param.default != _empty:
                     param.default_value = sig_param.default

cmem_plugin_base/dataintegration/description.py

@@ -1,4 +1,5 @@
 """DI Resource Parameter Type."""
+# ruff: noqa: A005
 
 from typing import Any
 

cmem_plugin_base/dataintegration/parameter/resource.py

@@ -22,8 +22,15 @@ class FlexibleSchemaPort(Port):
     Flexible input ports will adapt the schema to the connected output.
     Flexible output ports will adapt the schema to the connected input.
     It is not allowed to connect two flexible ports.
+
+    :param explicit_schema: Indicates whether an output port has an explicitly defined and quickly
+                            retrievable schema (like CSV). This allows for connecting it directly to
+                            flexible input ports in which case the explict schema will be used.
     """
 
+    def __init__(self, explicit_schema: bool = False):
+        self.explicit_schema = explicit_schema
+
 
 class UnknownSchemaPort(Port):
     """Port for which the schema is not known in advance.

cmem_plugin_base/dataintegration/ports.py

@@ -0,0 +1,16 @@
+"""Custom entity schema that holds entities of a specific type (e.g. files)"""
+
+
+def type_uri(suffix: str) -> str:
+    """Create a new entity schema type URI."""
+    return "https://vocab.eccenca.com/di/entity/" + suffix
+
+
+def path_uri(suffix: str) -> str:
+    """Create a new entity schema path."""
+    return "https://vocab.eccenca.com/di/entity/" + suffix
+
+
+def instance_uri(suffix: str) -> str:
+    """Create a new typed entity instance URI"""
+    return "https://eccenca.com/di/entity/" + suffix

cmem_plugin_base/dataintegration/typed_entities/__init__.py

@@ -0,0 +1,64 @@
+"""File entities"""
+
+from cmem_plugin_base.dataintegration.entity import Entity, EntityPath
+from cmem_plugin_base.dataintegration.typed_entities import instance_uri, path_uri, type_uri
+from cmem_plugin_base.dataintegration.typed_entities.typed_entities import (
+    TypedEntitySchema,
+)
+
+
+class File:
+    """A file entity that can be held in a FileEntitySchema."""
+
+    def __init__(self, path: str, file_type: str, mime: str | None) -> None:
+        self.path = path
+        self.file_type = file_type
+        self.mime = mime
+
+
+class LocalFile(File):
+    """A file that's located on the local file system."""
+
+    def __init__(self, path: str, mime: str | None = None) -> None:
+        super().__init__(path, "Local", mime)
+
+
+class ProjectFile(File):
+    """A project file"""
+
+    def __init__(self, path: str, mime: str | None = None) -> None:
+        super().__init__(path, "Project", mime)
+
+
+class FileEntitySchema(TypedEntitySchema[File]):
+    """Entity schema that holds a collection of files."""
+
+    def __init__(self):
+        super().__init__(
+            type_uri=type_uri("File"),
+            paths=[
+                EntityPath(path_uri("filePath")),
+                EntityPath(path_uri("fileType")),
+                EntityPath(path_uri("mimeType")),
+            ],
+        )
+
+    def to_entity(self, value: File) -> Entity:
+        """Create a generic entity from a file"""
+        return Entity(
+            uri=instance_uri(value.path),
+            values=[[value.path], [value.file_type], [value.mime or ""]],
+        )
+
+    def from_entity(self, entity: Entity) -> File:
+        """Create a file entity from a generic entity."""
+        path = entity.values[0][0]
+        file_type = entity.values[1][0]
+        mime = entity.values[2][0] if entity.values[2][0] else None
+        match file_type:
+            case "Local":
+                return LocalFile(path, mime)
+            case "Project":
+                return ProjectFile(path, mime)
+            case _:
+                raise ValueError(f"File '{path}' has unexpected type '{file_type}'.")

cmem_plugin_base/dataintegration/typed_entities/file.py

@@ -0,0 +1,56 @@
+"""Custom entity schema that holds entities of a specific type (e.g. files)"""
+
+from abc import abstractmethod
+from collections.abc import Iterator, Sequence
+from typing import Generic, TypeVar
+
+from cmem_plugin_base.dataintegration.entity import Entities, Entity, EntityPath, EntitySchema
+
+T = TypeVar("T")
+
+
+class TypedEntitySchema(EntitySchema, Generic[T]):
+    """A custom entity schema that holds entities of a specific type (e.g. files)."""
+
+    def __init__(self, type_uri: str, paths: Sequence[EntityPath]):
+        super().__init__(type_uri, paths)
+
+    @abstractmethod
+    def to_entity(self, value: T) -> Entity:
+        """Create a generic entity from a typed entity."""
+
+    @abstractmethod
+    def from_entity(self, entity: Entity) -> T:
+        """Create a typed entity from a generic entity.
+
+        Implementations may assume that the incoming schema matches the schema expected by
+        this typed schema, i.e., schema validation is not required.
+        """
+
+    def to_entities(self, values: Iterator[T]) -> "TypedEntities[T]":
+        """Given a collection of values, create a new typed entities instance."""
+        return TypedEntities(values, self)
+
+    def from_entities(self, entities: Entities) -> "TypedEntities[T]":
+        """Create typed entities from generic entities.
+
+        Returns None if the entities do not match the target type.
+        """
+        # TODO(robert): add validation
+        # CMEM-6095
+        if entities.schema.type_uri == self.type_uri:
+            if isinstance(entities, TypedEntities):
+                return entities
+            return TypedEntities(map(self.from_entity, entities.entities), self)
+        raise ValueError(
+            f"Expected entities of type '{self.type_uri}' but got '{entities.schema.type_uri}'."
+        )
+
+
+class TypedEntities(Entities, Generic[T]):
+    """Collection of entities of a particular type."""
+
+    def __init__(self, values: Iterator[T], schema: TypedEntitySchema[T]):
+        super().__init__(map(schema.to_entity, values), schema)
+        self.values = values
+        self.schema = schema

cmem_plugin_base/dataintegration/typed_entities/typed_entities.py

@@ -1,4 +1,5 @@
 """Parameter types."""
+# ruff: noqa: A005
 
 from collections.abc import Iterable
 from dataclasses import dataclass