From eb7f3eec31191ebb97f8d6847b070ebcf62a05a4 Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 9 Oct 2025 16:20:14 +0100 Subject: [PATCH 01/36] feat: add OPEN_CATEGORICAL_VARIABLE_TYPE --- orchestrator/schema/domain.py | 43 ++++++++++---- tests/schema/test_domain.py | 105 ++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 10 deletions(-) diff --git a/orchestrator/schema/domain.py b/orchestrator/schema/domain.py index 53f46444..7a1574cd 100644 --- a/orchestrator/schema/domain.py +++ b/orchestrator/schema/domain.py @@ -21,6 +21,7 @@ class VariableTypeEnum(str, enum.Enum): CATEGORICAL_VARIABLE_TYPE = ( "CATEGORICAL_VARIABLE_TYPE" # the value of the variable is a category label ) + OPEN_CATEGORICAL_VARIABLE_TYPE = "OPEN_CATEGORICAL_VARIABLE_TYPE" # the value of the variable is a category label but all categories are not known in advance BINARY_VARIABLE_TYPE = "BINARY_VARIABLE_TYPE" # the value of the variable is binary UNKNOWN_VARIABLE_TYPE = "UNKNOWN_VARIABLE_TYPE" # the type of value of the variable is unknown/unspecified IDENTIFIER_VARIABLE_TYPE = "IDENTIFIER_VARIABLE_TYPE" # the value is some type of, possible unique, identifier @@ -233,6 +234,9 @@ def variableType_matches_values(cls, value, values: "pydantic.FieldValidationInf elif value == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: assert values.data.get("values") is None assert values.data.get("interval") is None + elif value == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE: + assert values.data.get("interval") is None + assert values.data.get("domainRange") is None return value @@ -324,9 +328,10 @@ def domain_values(self) -> list: if self.variableType in { VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, VariableTypeEnum.UNKNOWN_VARIABLE_TYPE, + VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, }: raise ValueError( - "Cannot generate domain values for continuous or unknown variables" + "Cannot generate domain values for continuous, unknown or open categorical variables" ) if self.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE: return [False, True] @@ -363,9 +368,12 @@ def valueInDomain(self, value): retval = True elif self.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE: retval = value in self.values - elif self.variableType == VariableTypeEnum.UNKNOWN_VARIABLE_TYPE: - # If the domain is unknown we just return True - # This is required if the value is from a PropertyType with this domain for self-consistency + elif self.variableType in [ + VariableTypeEnum.UNKNOWN_VARIABLE_TYPE, + VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + ]: + # If the domain is unknown or open categorical we just return True + # This is required if the value is from a PropertyType with these domains for self-consistency # e.g. If we have a ConstitutiveProperty(identifier="smiles", PropertyDomain(type=UNKNOWN_VARIABLE_TYPE) # And then if we ask is smiles = (CO2) in the domain it should return True. retval = True @@ -391,6 +399,14 @@ def isSubDomain(self, otherDomain: "PropertyDomain") -> bool: if otherDomain.variableType == VariableTypeEnum.UNKNOWN_VARIABLE_TYPE: # We can return immediately as there is nothing else we can do with UNKNOWN return True + if ( + otherDomain.variableType + == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + ): + # Infinite size domain cannot be subdomain of open_categorical domain + import math + + return self.size == math.inf # Variables not of same type cannot be subdomains in the following situations # A_ this domain is unknown and the other is not-unknown @@ -438,7 +454,13 @@ def isSubDomain(self, otherDomain: "PropertyDomain") -> bool: # (Note the case where otherDomain is discrete/binary is excluded above) # i.e. we can check this by computing the set difference retval = len(s.difference(o)) == 0 - + elif self.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE: + # The case where other is Unknown or open categorical are handled above + # for all other cases we are not a subdomain + return otherDomain.variableType in [ + VariableTypeEnum.UNKNOWN_VARIABLE_TYPE, + VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + ] elif self.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: # If the other domain has no range then the receiver is a subdomain if not otherDomain.domainRange: @@ -550,8 +572,8 @@ def isSubDomain(self, otherDomain: "PropertyDomain") -> bool: def size(self) -> float | int: """Returns the size (number of elements) in the domain if this is countable. - Returns math.inf if the size is not countable. - This includes any domain with CONTINUOUS_VARIABLE_TYPE, UNKNOWN_VARIABLE_TYPE ir IDENTIFIER_VARIABLE_TYPE. + Returns math.inf if the size is not countable or is unknown/open categorical. + This includes any domain with CONTINUOUS_VARIABLE_TYPE, UNKNOWN_VARIABLE_TYPE or OPEN_CATEGORICAL_VARIABLE_TYPE. It also includes any unbounded domain with DISCRETE_VARIABLE_TYPE. """ @@ -561,9 +583,10 @@ def size(self) -> float | int: self.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE ): # noqa: SIM114 size = math.inf - elif self.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE and ( - self.domainRange is None and self.values is None - ): + elif ( + self.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE + and (self.domainRange is None and self.values is None) + ) or self.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE: size = math.inf else: if self.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE: diff --git a/tests/schema/test_domain.py b/tests/schema/test_domain.py index 89c9e4c4..bdd8248b 100644 --- a/tests/schema/test_domain.py +++ b/tests/schema/test_domain.py @@ -826,3 +826,108 @@ def test_domain_values(): # Test 0 is in domain values of discrete var of range [0,1] d = PropertyDomain(domainRange=[0, 1], interval=1) assert d.domain_values == [0] + + +def test_open_categorical_variable_type_property_domain(): + import math + + import pydantic + + from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum + + d = PropertyDomain(variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE) + # Type is set + assert d.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + # valueInDomain always True + assert d.valueInDomain("anything") is True + assert d.valueInDomain(1234) is True + assert d.valueInDomain([1, 2, 3]) is True + # size is inf + assert d.size == math.inf + # domain_values raises ValueError + with pytest.raises( + ValueError, + match="Cannot generate domain values for continuous, unknown or open categorical variables", + ): + _ = d.domain_values + # isSubDomain only for open categorical + d2 = PropertyDomain(variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE) + assert d.isSubDomain(d2) + # Not a subdomain of categorical + d3 = PropertyDomain(values=["a", "b"]) + assert not d.isSubDomain(d3) + # But categorical can be subdomain of open categorical + assert d3.isSubDomain(d) + + # Check d.isSubDomain(OTHER) behaviour + + # Open categorical is not a subdomain of any other domain except UNKNOWN_VARIABLE_TYPE + assert d.isSubDomain( + PropertyDomain(variableType=VariableTypeEnum.UNKNOWN_VARIABLE_TYPE) + ) + # Open categorical is not a subdomain of continuous + assert not d.isSubDomain( + PropertyDomain(variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE) + ) + # Open categorical is not a subdomain of discrete + assert not d.isSubDomain( + PropertyDomain(variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, interval=1) + ) + + # Check OTHER.isSubDomain(d) behaviour + + # Continuous is not a subdomain of open categorical + assert not PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE + ).isSubDomain(d) + # Discrete with no bounds is not a subdomain of open categorical + assert not PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, interval=1 + ).isSubDomain(d) + # Discrete with bounds is a subdomain of open categorical + assert PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, + domainRange=[0, 2], + interval=1, + ).isSubDomain(d) + # Unknown is not a subdomain of open categorical + assert not PropertyDomain( + variableType=VariableTypeEnum.UNKNOWN_VARIABLE_TYPE + ).isSubDomain(d) + + # Test serialization retains variable type + dump = d.model_dump() + assert dump["variableType"] == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + dser = PropertyDomain.model_validate(dump) + assert dser.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + + # Test values can be passed for open categorical + d = PropertyDomain( + variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + values=["a", "b", "c"], + ) + assert d.values == ["a", "b", "c"] + assert d.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + # Test domain range cannot be passed for open categorical + with pytest.raises( + pydantic.ValidationError, match="1 validation error for PropertyDomain" + ): + d = PropertyDomain( + variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + domainRange=[0, 1], + ) + # Test interval cannot be passed for open categorical + with pytest.raises( + pydantic.ValidationError, match="1 validation error for PropertyDomain" + ): + d = PropertyDomain( + variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, interval=1 + ) + + # Test serialization retains variable type and values + dump = d.model_dump() + assert dump["variableType"] == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + assert dump["values"] == ["a", "b", "c"] + dser = PropertyDomain.model_validate(dump) + assert dser.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + assert dser.values == ["a", "b", "c"] From a1c99424e29ca58e45ad4025360a47ee909e8991 Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 9 Oct 2025 20:54:22 +0100 Subject: [PATCH 02/36] test: update to new exception msg --- tests/schema/test_domain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/schema/test_domain.py b/tests/schema/test_domain.py index bdd8248b..1bd0baee 100644 --- a/tests/schema/test_domain.py +++ b/tests/schema/test_domain.py @@ -808,7 +808,7 @@ def test_domain_values(): # Test continuous variables raise ValueError when domain_values is called with pytest.raises( ValueError, - match="Cannot generate domain values for continuous or unknown variables", + match="Cannot generate domain values for continuous, unknown or open categorical variables", ): PropertyDomain( variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE @@ -817,7 +817,7 @@ def test_domain_values(): # Test unknown variables raise ValueError when domain_values is called with pytest.raises( ValueError, - match="Cannot generate domain values for continuous or unknown variables", + match="Cannot generate domain values for continuous, unknown or open categorical variables", ): PropertyDomain( variableType=VariableTypeEnum.UNKNOWN_VARIABLE_TYPE From f9745156784acfb2ba7eb60f51a783d4af042a1a Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 9 Oct 2025 20:56:54 +0100 Subject: [PATCH 03/36] refactor: isSubDomain method It was getting long and complex. Create one function for each domain type that checks if another domain is a sub-domain of a domain of that type. --- orchestrator/schema/domain.py | 338 ++++++++++++++++------------------ 1 file changed, 163 insertions(+), 175 deletions(-) diff --git a/orchestrator/schema/domain.py b/orchestrator/schema/domain.py index 7a1574cd..e761c2d6 100644 --- a/orchestrator/schema/domain.py +++ b/orchestrator/schema/domain.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MIT import enum +import math import typing import numpy as np @@ -66,6 +67,138 @@ def _internal_range_values(lower, upper, interval) -> list: return list(np.round(values, 10)) +def is_subdomain_of_unknown_domain(unknownDomain, testDomain): + return True + + +def is_subdomain_of_continuous_domain(continuousDomain, testDomain): + + if testDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: + if continuousDomain.domainRange is None: + return True + return min(continuousDomain.domainRange) <= min(testDomain.domainRange) and max( + continuousDomain.domainRange + ) >= max(testDomain.domainRange) + if testDomain.variableType in [ + VariableTypeEnum.UNKNOWN_VARIABLE_TYPE, + VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, + ]: + return False + if testDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE: + if continuousDomain.domainRange is None: + return True + + if testDomain.size == math.inf: + return False + + return min(continuousDomain.domainRange) <= min( + testDomain.domain_values + ) and max(continuousDomain.domainRange) > max(testDomain.domain_values) + + # The only variable type left is BINARY + # Check 0,1 is within our domainRange + return continuousDomain.domainRange is None or ( + min(continuousDomain.domainRange) <= 0 and max(continuousDomain.domainRange) > 1 + ) + + +def is_subdomain_of_discrete_domain(discreteDomain, testDomain): + if testDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE: + if discreteDomain.interval: + if testDomain.interval: + # We both have an interval + # Our interval must be divisible by domain interval + if testDomain.interval % discreteDomain.interval == 0: + # Now we have to check the ranges + if testDomain.domainRange and discreteDomain.domainRange: + # Both have ranges - values must be subsets of each other + retval = set(testDomain.domain_values).issubset( + discreteDomain.domain_values + ) + elif ( + testDomain.domainRange and not discreteDomain.domainRange + ) or (not testDomain.domainRange and discreteDomain.domainRange): + # If we have a range and the other doesn't, it's a subdomain; if we don't have a range and the other does, it's not a subdomain + retval = ( + testDomain.domainRange and not discreteDomain.domainRange + ) + else: + # Neither have ranges + retval = True + else: + retval = False + else: + # they have a domain range and interval we have values + # convert their domain range to values + retval = set(testDomain.domain_values).issubset( + discreteDomain.domain_values + ) + else: + retval = set(testDomain.domain_values).issubset( + discreteDomain.domain_values + ) + return retval + if testDomain.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE: + return all(x in discreteDomain.domain_values for x in [0, 1]) + + # All other domains are false (CONTINUOUS, OPEN_CATEGORICAL, UNKNOWN and CATEGORICAL + return False + + +def is_subdomain_of_categorical_domain(categoricalDomain, testDomain): + """Checks if the other domain is a subdomain of the categorical domain""" + # Check against all members of VariableTypeEnum + + if testDomain.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE: + return all(x in categoricalDomain.values for x in testDomain.values) + if testDomain.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE: + return all(x in categoricalDomain.domain_values for x in [0, 1]) + if testDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE: + return testDomain.size != math.inf and all( + x in categoricalDomain.domain_values for x in testDomain.domain_values + ) + if testDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: + return False + + # All other domains are false: OPEN_CATEGORICAL, UNKNOWN, CONTINUOUS + return False + + +def is_subdomain_of_binary_domain(binaryDomain, testDomain): + if testDomain.variableType in [ + VariableTypeEnum.DISCRETE_VARIABLE_TYPE, + VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, + VariableTypeEnum.BINARY_VARIABLE_TYPE, + ]: + if testDomain.size <= 2: + return all( + x in binaryDomain.domain_values for x in testDomain.domain_values + ) + return False + + # All other domains are false: OPEN_CATEGORICAL, UNKNOWN, CONTINUOUS except BINARY + return testDomain.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE + + +def is_subdomain_of_open_categorical_domain(openCategoricalDomain, testDomain): + + if testDomain.variableType in [ + VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + *VariableTypeEnum.BINARY_VARIABLE_TYPE, + VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, + ]: + return True + if testDomain.variableType in [ + VariableTypeEnum.UNKNOWN_VARIABLE_TYPE, + VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, + ]: + return False + + # Only domain left is DISCRETE + return testDomain.size != math.inf + + class ProbabilityFunction(pydantic.BaseModel): identifier: ProbabilityFunctionsEnum = pydantic.Field( default=ProbabilityFunctionsEnum.UNIFORM @@ -387,186 +520,43 @@ def valueInDomain(self, value): return retval def isSubDomain(self, otherDomain: "PropertyDomain") -> bool: - """Checks if the receiver is a subdomain of otherDomain. + """Checks if self is a subdomain of otherDomain. If the two domains are identical this method returns True""" if self is otherDomain: return True - if self.variableType != otherDomain.variableType: - - if otherDomain.variableType == VariableTypeEnum.UNKNOWN_VARIABLE_TYPE: - # We can return immediately as there is nothing else we can do with UNKNOWN - return True - if ( - otherDomain.variableType - == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE - ): - # Infinite size domain cannot be subdomain of open_categorical domain - import math - - return self.size == math.inf - - # Variables not of same type cannot be subdomains in the following situations - # A_ this domain is unknown and the other is not-unknown - # B_ this domain is categorical and the other domain is discrete or binary (you can't create a categorical domain that could be a discrete domain or binary domain) - # C_ this domain is continuous and the other is discrete/categorical/binary - - # Note: binary/discrete could be subdomains of categorical if categorical mixes numbers and strings - # e.g. binary is a subdomain of [True, False, "George"] - - # A - if ( - self.variableType == VariableTypeEnum.UNKNOWN_VARIABLE_TYPE - and otherDomain.variableType != VariableTypeEnum.UNKNOWN_VARIABLE_TYPE - ): - return False - - # B - if ( - otherDomain.variableType - in [ - VariableTypeEnum.DISCRETE_VARIABLE_TYPE, - VariableTypeEnum.BINARY_VARIABLE_TYPE, - ] - ) and self.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE: - return False - - # C - if ( - self.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE - and otherDomain.variableType - in [ - VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, - VariableTypeEnum.BINARY_VARIABLE_TYPE, - VariableTypeEnum.DISCRETE_VARIABLE_TYPE, - ] - ): - return False - - retval = True - if self.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE: - s = set(self.values) - o = set(otherDomain.values) - - # The receiver is a subdomain if all its values are in otherDomain values - # (Note the case where otherDomain is discrete/binary is excluded above) - # i.e. we can check this by computing the set difference - retval = len(s.difference(o)) == 0 - elif self.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE: - # The case where other is Unknown or open categorical are handled above - # for all other cases we are not a subdomain - return otherDomain.variableType in [ - VariableTypeEnum.UNKNOWN_VARIABLE_TYPE, - VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, - ] - elif self.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: - # If the other domain has no range then the receiver is a subdomain - if not otherDomain.domainRange: - retval = True - elif otherDomain.values: # If the other domain has values - use those - retval = all( - min(self.domainRange) <= x < max(self.domainRange) - for x in otherDomain.values - ) - else: - retval = bool( - min(self.domainRange) >= min(otherDomain.domainRange) - and max(self.domainRange) <= max(otherDomain.domainRange) - ) - elif ( - self.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE - and otherDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE - ): - # There are four situations - # 1. Us: DomainRange Other: DomainRange - # 2. Us: DomainRange Other: Values - # 3. Us: Values Other: Values - # 4. Us: Values Other: DomainRange - if otherDomain.interval: - if self.interval: - # We both have an interval - # Our interval must be divisible by domain interval - if self.interval % otherDomain.interval == 0: - # No we have to check the ranges - if self.domainRange and otherDomain.domainRange: - # Both have ranges - values must be subsets of each other - s = set(self.domain_values) - o = set(otherDomain.domain_values) - retval = len(s.difference(o)) == 0 - elif self.domainRange and not otherDomain.domainRange: - # We have a range and the other doesn't - retval = True - elif not self.domainRange and otherDomain.domainRange: - # We don't have a range and the other does - can't be subdomain - retval = False - else: - # Neither have ranges - retval = True - else: - retval = False - else: - # they have a domain range and interval we have values - # convert their domain range to values - s = set(self.values) - o = set(otherDomain.domain_values) - retval = len(s.difference(o)) == 0 - else: - if self.values: - # we both have values - s = set(self.values) - o = set(otherDomain.values) - retval = len(s.difference(o)) == 0 - else: - # we have a domain range and interval, and they have values - # convert the domain range to values - s = set(self.domain_values) - o = set(otherDomain.values) - retval = len(s.difference(o)) == 0 - elif ( - self.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE - and otherDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE - ): - if not otherDomain.domainRange: - # If there is no domain on the continuous variable we are subdomain - retval = True - elif self.values: # If we have values - use those - retval = all( - min(otherDomain.domainRange) <= x < max(otherDomain.domainRange) - for x in self.values - ) - else: - retval = bool( - min(self.domainRange) >= min(otherDomain.domainRange) - and max(self.domainRange) <= max(otherDomain.domainRange) - ) - elif ( - self.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE - and otherDomain.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE - ): - # We are a subdomain of binary domain if our values are [0,1], [1], or [0] - # AND we have less than 3 values - retval = all(x in [0, 1] for x in self.domain_values) and self.size <= 2 - elif self.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE: - if otherDomain.variableType in [ - VariableTypeEnum.DISCRETE_VARIABLE_TYPE, - VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, - ]: - retval = all( - x in otherDomain.domain_values for x in [0, 1, True, False] - ) - elif otherDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: - retval = ( - ( - max(otherDomain.domainRange) > 1 - and min(otherDomain.domainRange) <= 0 - ) - if otherDomain.domainRange - else True - ) + if self == otherDomain: + return True - return retval + # If variable types are the same, handle in each function + if otherDomain.variableType == VariableTypeEnum.UNKNOWN_VARIABLE_TYPE: + return is_subdomain_of_unknown_domain( + unknownDomain=otherDomain, testDomain=self + ) + if otherDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: + return is_subdomain_of_continuous_domain( + continuousDomain=otherDomain, testDomain=self + ) + if otherDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE: + return is_subdomain_of_discrete_domain( + discreteDomain=otherDomain, testDomain=self + ) + if otherDomain.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE: + return is_subdomain_of_categorical_domain( + categoricalDomain=otherDomain, testDomain=self + ) + if otherDomain.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE: + return is_subdomain_of_binary_domain( + binaryDomain=otherDomain, testDomain=self + ) + if otherDomain.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE: + return is_subdomain_of_open_categorical_domain( + openCategoricalDomain=otherDomain, testDomain=self + ) + # fallback to previous logic if unknown type + raise ValueError(f"Internal error: Unknown variable type {self.variableType}") @property def size(self) -> float | int: @@ -577,8 +567,6 @@ def size(self) -> float | int: It also includes any unbounded domain with DISCRETE_VARIABLE_TYPE. """ - import math - if ( self.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE ): # noqa: SIM114 From e7a40a5e339c2a36af5593a9dc6b449d2ba19286 Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 9 Oct 2025 20:58:18 +0100 Subject: [PATCH 04/36] chore: simplification --- orchestrator/schema/domain.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orchestrator/schema/domain.py b/orchestrator/schema/domain.py index e761c2d6..32d49fae 100644 --- a/orchestrator/schema/domain.py +++ b/orchestrator/schema/domain.py @@ -169,7 +169,6 @@ def is_subdomain_of_binary_domain(binaryDomain, testDomain): if testDomain.variableType in [ VariableTypeEnum.DISCRETE_VARIABLE_TYPE, VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, - VariableTypeEnum.BINARY_VARIABLE_TYPE, ]: if testDomain.size <= 2: return all( From 1232d7df9c6263cd1dcc527be3808757381eaeb9 Mon Sep 17 00:00:00 2001 From: michaelj Date: Mon, 13 Oct 2025 10:00:07 +0100 Subject: [PATCH 05/36] feat: update open categorical types in entityspace Since the entityspace is fixed on creation strictly it can't have an open categorical dimension. We convert it to categorical with the values passed which will always be a sub-domain of the experiments open-categorical domain. --- orchestrator/schema/entityspace.py | 24 +++++++++++++++++++++--- tests/schema/test_entityspace.py | 23 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/orchestrator/schema/entityspace.py b/orchestrator/schema/entityspace.py index 9dabb1d4..ddef433c 100644 --- a/orchestrator/schema/entityspace.py +++ b/orchestrator/schema/entityspace.py @@ -3,7 +3,7 @@ import typing -from orchestrator.schema.domain import VariableTypeEnum +from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum from orchestrator.schema.entity import Entity from orchestrator.schema.property import ConstitutiveProperty from orchestrator.schema.property_value import ( @@ -26,8 +26,26 @@ def __init__( constitutiveProperties: list[ConstitutiveProperty], ): - self._constitutiveProperties = constitutiveProperties - self._propertyLookup = {c.identifier: c for c in self._constitutiveProperties} + self._propertyLookup = {c.identifier: c for c in constitutiveProperties} + # Update open-categorical type to categorical -> once in an entityspace the category can't be open anymore + for c in constitutiveProperties: + if ( + c.propertyDomain.variableType + == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + ): + # ConstitutiveProperty is immutable so we need to create a new one + propertyDomain = PropertyDomain( + variableType=VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, + values=c.propertyDomain.values, + probabilityFunction=c.propertyDomain.probabilityFunction, + ) + c = ConstitutiveProperty( + identifier=c.identifier, + propertyDomain=propertyDomain, + ) + self._propertyLookup[c.identifier] = c + + self._constitutiveProperties = list(self._propertyLookup.values()) @property def config(self) -> list[ConstitutiveProperty]: diff --git a/tests/schema/test_entityspace.py b/tests/schema/test_entityspace.py index d738f861..3e476ac6 100644 --- a/tests/schema/test_entityspace.py +++ b/tests/schema/test_entityspace.py @@ -374,3 +374,26 @@ def test_entity_space_iterators(measurement_space_from_single_parameterized_expe ), "Expected the first five random points to not be identical to first five sequential points" assert len(set(random)) == len(random), "Expected no points to be duplicated" + + +def test_entity_space_updates_open_categorical_property(): + es = EntitySpaceRepresentation( + constitutiveProperties=[ + ConstitutiveProperty( + identifier="open_categorical_prop", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + values=["a", "b", "c"], + ), + ) + ] + ) + assert ( + es.constitutiveProperties[0].propertyDomain.variableType + == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE + ), "Expected open categorical property to be updated to categorical" + assert es.constitutiveProperties[0].propertyDomain.values == [ + "a", + "b", + "c", + ], "Expected values to be retained" From 5535780d6f9a44ad4a6a79bcb56dea7d64d7ceac Mon Sep 17 00:00:00 2001 From: michaelj Date: Mon, 13 Oct 2025 18:09:00 +0100 Subject: [PATCH 06/36] chore: doc strings --- orchestrator/schema/domain.py | 56 ++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/orchestrator/schema/domain.py b/orchestrator/schema/domain.py index 32d49fae..17f0a2c5 100644 --- a/orchestrator/schema/domain.py +++ b/orchestrator/schema/domain.py @@ -68,11 +68,24 @@ def _internal_range_values(lower, upper, interval) -> list: def is_subdomain_of_unknown_domain(unknownDomain, testDomain): + """Returns True if the testDomain is a subdomain of the unknownDomain + Parameters: + unknownDomain: A PropertyDomain with variableType UNKNOWN_VARIABLE_TYPE + testDomain: A PropertyDomain with any variableType + Returns: + True if the testDomain is a subdomain of the unknownDomain + """ return True def is_subdomain_of_continuous_domain(continuousDomain, testDomain): - + """Returns True if the testDomain is a subdomain of the continuousDomain + Parameters: + continuousDomain: A PropertyDomain with variableType CONTINUOUS_VARIABLE_TYPE + testDomain: A PropertyDomain with any variableType + Returns: + True if the testDomain is a subdomain of the continuousDomain + """ if testDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: if continuousDomain.domainRange is None: return True @@ -104,6 +117,13 @@ def is_subdomain_of_continuous_domain(continuousDomain, testDomain): def is_subdomain_of_discrete_domain(discreteDomain, testDomain): + """Returns True if the testDomain is a subdomain of the discreteDomain + Parameters: + discreteDomain: A PropertyDomain with variableType DISCRETE_VARIABLE_TYPE + testDomain: A PropertyDomain with any variableType + Returns: + True if the testDomain is a subdomain of the discreteDomain + """ if testDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE: if discreteDomain.interval: if testDomain.interval: @@ -147,7 +167,13 @@ def is_subdomain_of_discrete_domain(discreteDomain, testDomain): def is_subdomain_of_categorical_domain(categoricalDomain, testDomain): - """Checks if the other domain is a subdomain of the categorical domain""" + """Returns True if the testDomain is a subdomain of the categoricalDomain + Parameters: + categoricalDomain: A PropertyDomain with variableType CATEGORICAL_VARIABLE_TYPE + testDomain: A PropertyDomain with any variableType + Returns: + True if the testDomain is a subdomain of the categoricalDomain + """ # Check against all members of VariableTypeEnum if testDomain.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE: @@ -158,14 +184,25 @@ def is_subdomain_of_categorical_domain(categoricalDomain, testDomain): return testDomain.size != math.inf and all( x in categoricalDomain.domain_values for x in testDomain.domain_values ) - if testDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE: - return False # All other domains are false: OPEN_CATEGORICAL, UNKNOWN, CONTINUOUS return False def is_subdomain_of_binary_domain(binaryDomain, testDomain): + """Returns True if the testDomain is a subdomain of the binaryDomain + + The cases where this returns True are + - testDomain is a BINARY_VARIABLE_TYPE + - testDomain is a DISCRETE_VARIABLE_TYPE or CATEGORICAL_VARIABLE_TYPE with size <= 2 + and all values in testDomain are in binaryDomain.domain_values + + Parameters: + binaryDomain: A PropertyDomain with variableType BINARY_VARIABLE_TYPE + testDomain: A PropertyDomain with any variableType + Returns: + True if the testDomain is a subdomain of the binaryDomain + """ if testDomain.variableType in [ VariableTypeEnum.DISCRETE_VARIABLE_TYPE, VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, @@ -181,7 +218,18 @@ def is_subdomain_of_binary_domain(binaryDomain, testDomain): def is_subdomain_of_open_categorical_domain(openCategoricalDomain, testDomain): + """Returns True if the testDomain is a subdomain of the openCategoricalDomain + The cases where this returns True are: + - testDomain is an OPEN_CATEGORICAL_VARIABLE_TYPE, BINARY_VARIABLE_TYPE or CATEGORICAL_VARIABLE_TYPE + - testDomain is a Discrete_VARIABLE_TYPE with size != math.inf + + Parameters: + openCategoricalDomain: A PropertyDomain with variableType OPEN_CATEGORICAL_VARIABLE_TYPE + testDomain: A PropertyDomain with any variableType + Returns: + True if the testDomain is a subdomain of the openCategoricalDomain + """ if testDomain.variableType in [ VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, *VariableTypeEnum.BINARY_VARIABLE_TYPE, From 1dcfd141debe74239acb1ae15c4b54d84b430167 Mon Sep 17 00:00:00 2001 From: michaelj Date: Mon, 13 Oct 2025 19:56:06 +0100 Subject: [PATCH 07/36] feat: custom_experiments as decorated functions as plugins Just decorate a python function with @custom_experiments to add it - No need for YAML - No need for ado_actuators namespace --- .codespellrc | 2 +- .../optimization_test_functions.py | 61 ++-- .../custom_experiments/pyproject.toml | 5 + .../modules/actuators/custom_experiments.py | 289 ++++++++++++++++-- orchestrator/modules/actuators/registry.py | 20 -- 5 files changed, 300 insertions(+), 77 deletions(-) diff --git a/.codespellrc b/.codespellrc index 53b6021c..1c237ac9 100644 --- a/.codespellrc +++ b/.codespellrc @@ -2,4 +2,4 @@ skip = toxenv,./examples/pfas-generative-models/data,./website/theme,adorchestrator.egg-info,*.svg,examples/pfas-generative-models/data/GM_Comparison/Transfromer/Sample_0/test_generations.csv count = quiet-level = 3 -ignore-words-list = ser,Transfromer \ No newline at end of file +ignore-words-list = ser,Transfromer,discus diff --git a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py b/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py index 2d911b5c..35f95359 100644 --- a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py +++ b/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py @@ -5,41 +5,54 @@ from nevergrad.functions import ArtificialFunction -from orchestrator.schema.entity import Entity -from orchestrator.schema.experiment import Experiment -from orchestrator.schema.observed_property import ObservedPropertyValue -from orchestrator.schema.property_value import ValueTypeEnum +from orchestrator.modules.actuators.custom_experiments import custom_experiment +from orchestrator.schema.domain import PropertyDomain +from orchestrator.schema.property import ConstitutiveProperty moduleLog = logging.getLogger() -def artificial_function( - entity: Entity, experiment: Experiment, parameters: dict | None -): +@custom_experiment( + [ + ConstitutiveProperty( + identifier="x0", + propertyDomain=PropertyDomain(variableType="CONTINUOUS_VARIABLE_TYPE"), + ), + ConstitutiveProperty( + identifier="x1", + propertyDomain=PropertyDomain(variableType="CONTINUOUS_VARIABLE_TYPE"), + ), + ConstitutiveProperty( + identifier="x2", + propertyDomain=PropertyDomain(variableType="CONTINUOUS_VARIABLE_TYPE"), + ), + ConstitutiveProperty( + identifier="name", + propertyDomain=PropertyDomain( + values=["discus", "sphere", "cigar", "griewank", "rosenbrock", "st1"] + ), + ), + ConstitutiveProperty( + identifier="num_blocks", + propertyDomain=PropertyDomain( + domainRange=[1, 10], variableType="DISCRETE_VARIABLE_TYPE", interval=1 + ), + ), + ] +) +def artificial_function(x0: float, x1: float, x2: float, name: str, num_blocks: int): import numpy as np - # parameters is a dictionary of key:value pairs of the experiment required/optional inputs - # defined in custom_experiments.yaml - parameters = experiment.propertyValuesFromEntity(entity) - # Get the function from nevergrad.functions.ArtificialFunction func = ArtificialFunction( - name=parameters["name"], - num_blocks=parameters["num_blocks"], - block_dimension=int( - len(entity.constitutive_property_values) / parameters["num_blocks"] - ), + name=name, + num_blocks=num_blocks, + block_dimension=int(3 / num_blocks), translation_factor=0.0, ) # Call the nevergrad function - value = func(np.asarray([v.value for v in entity.constitutive_property_values])) + value = func(np.asarray([x0, x1, x2])) - # Return the function value to ado - pv = ObservedPropertyValue( - value=value, - property=experiment.observedPropertyForTargetIdentifier("function_value"), - valueType=ValueTypeEnum.NUMERIC_VALUE_TYPE, - ) - return [pv] + return {"function_value": value} diff --git a/examples/optimization_test_functions/custom_experiments/pyproject.toml b/examples/optimization_test_functions/custom_experiments/pyproject.toml index caf5ce9e..7072055a 100644 --- a/examples/optimization_test_functions/custom_experiments/pyproject.toml +++ b/examples/optimization_test_functions/custom_experiments/pyproject.toml @@ -32,3 +32,8 @@ fallback-version = "0.0.0" tagged-metadata = true dirty = true bump = true + +#Add entry point for custom experiments +[project.entry-points."ado.custom_experiments"] +optimization_test_functions = "optimization_test_functions.optimization_test_functions" + diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 95dc5238..33486c98 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -1,8 +1,10 @@ # Copyright (c) IBM Corporation # SPDX-License-Identifier: MIT +import inspect import typing import uuid +from functools import wraps import pydantic import ray @@ -11,16 +13,19 @@ from orchestrator.core.actuatorconfiguration.config import GenericActuatorParameters from orchestrator.modules.actuators.base import ( ActuatorBase, - ActuatorModuleConf, DeprecatedExperimentError, ) from orchestrator.modules.actuators.measurement_queue import MeasurementQueue -from orchestrator.modules.module import load_module_class_or_function from orchestrator.schema.entity import ( CheckRequiredObservedPropertyValuesPresent, Entity, ) from orchestrator.schema.experiment import Experiment +from orchestrator.schema.observed_property import ( + ObservedProperty, + ObservedPropertyValue, +) +from orchestrator.schema.property import ConstitutiveProperty from orchestrator.schema.reference import ExperimentReference from orchestrator.schema.request import MeasurementRequest, MeasurementRequestStateEnum from orchestrator.schema.result import ValidMeasurementResult @@ -29,6 +34,165 @@ configure_logging() +# Module-level catalog for custom experiments +_custom_experiments_catalog = orchestrator.modules.actuators.catalog.ExperimentCatalog( + catalogIdentifier="CustomExperiments" +) + + +def custom_experiment(properties: list[ConstitutiveProperty]): + """ + Decorator for custom experiment functions. + + Args: + properties: List of ConstitutiveProperty instances defining the input values the Entity must have + + Returns: + A decorator that wraps a function to work with ADO's custom experiment system + + Example: + from orchestrator.schema.property import ConstitutiveProperty + + @custom_experiment([ + ConstitutiveProperty(identifier="temperature"), + ConstitutiveProperty(identifier="pressure") + ]) + def my_experiment(temperature: float, pressure: float) -> dict[str, float]: + # Function takes temperature and pressure as parameters + # Returns a dict of observed property values + return { + "density": temperature * pressure * 0.001, + "viscosity": temperature / pressure * 0.1 + } + """ + + def decorator(func): + @wraps(func) + def wrapper( + entity: Entity, experiment: Experiment + ) -> list[ObservedPropertyValue]: + """ + Wrapper function that converts Entity+Experiment to dict and calls the wrapped function. + + Args: + entity: The entity to measure + experiment: The experiment configuration + + Returns: + List of ObservedPropertyValue instances + """ + # Convert Entity+Experiment to dict using propertyValuesFromEntity + input_values = experiment.propertyValuesFromEntity(entity) + + # Call the wrapped function with the input values + result_dict = func(**input_values) + + # Convert the result dict to ObservedPropertyValue list + observed_property_values = [] + for property_identifier, value in result_dict.items(): + # Create ObservedProperty for this result + observed_property = ObservedProperty(identifier=property_identifier) + + # Create ObservedPropertyValue + observed_property_value = ObservedPropertyValue( + property=observed_property, value=value + ) + observed_property_values.append(observed_property_value) + + return observed_property_values + + # Validate that the properties match the function parameters + func_signature = inspect.signature(func) + func_param_names = set(func_signature.parameters.keys()) + property_identifiers = {prop.identifier for prop in properties} + + if func_param_names != property_identifiers: + raise ValueError( + f"Function parameter names {func_param_names} do not match " + f"property identifiers {property_identifiers}" + ) + + # Store decorator arguments as function attributes + wrapper._decorator_properties = properties + wrapper._decorator_func = func + wrapper._is_custom_experiment = True + + # Create and store the Experiment instance + experiment = Experiment( + actuatorIdentifier="custom_experiments", + identifier=func.__name__, + requiredProperties=tuple(properties), + optionalProperties=(), + defaultParameterization=(), + deprecated=False, + ) + wrapper._experiment = experiment + + # Add the experiment to the module-level catalog + _custom_experiments_catalog.addExperiment(experiment) + + return wrapper + + return decorator + + +def get_custom_experiment_info(func) -> dict: + """ + Helper function to access decorator arguments and experiment from a decorated function. + + Args: + func: A function decorated with @custom_experiment + + Returns: + Dict containing decorator properties and experiment instance + + Raises: + ValueError: If the function is not decorated with @custom_experiment + """ + if not hasattr(func, "_is_custom_experiment") or not func._is_custom_experiment: + raise ValueError("Function is not decorated with @custom_experiment") + + return { + "properties": func._decorator_properties, + "experiment": func._experiment, + "original_func": func._decorator_func, + } + + +def load_custom_experiments_from_entry_points(): + """ + Load custom experiments from entry points. + + This function searches for entry points under 'ado.custom_experiments' and loads + any decorated functions from those modules. + """ + try: + import importlib + import importlib.metadata + + # Get all entry points for ado.custom_experiments + entry_points = importlib.metadata.entry_points() + custom_experiment_groups = entry_points.get("ado.custom_experiments", []) + + for entry_point in custom_experiment_groups: + entry_point.load() + + except ImportError: + # importlib.metadata not available (Python < 3.8) + pass + + +def get_custom_experiments_catalog() -> ( + orchestrator.modules.actuators.catalog.ExperimentCatalog +): + """ + Get the module-level catalog of custom experiments. + + Returns: + The ExperimentCatalog containing all registered custom experiments + """ + return _custom_experiments_catalog + async def custom_experiment_wrapper( function: typing.Callable, @@ -89,42 +253,54 @@ def __init__(self, queue, params: dict | None = None): self.log.debug(f"Queue is {self._stateUpdateQueue}") self.log.debug(f"Params are {params}") - import orchestrator.modules.actuators.registry - - # Load custom_experiments catalog from registry - registry = ( - orchestrator.modules.actuators.registry.ActuatorRegistry.globalRegistry() - ) - self._catalog = registry.catalogForActuatorIdentifier(self.__class__.identifier) - + # Use the module-level catalog by calling the class method + self._catalog = type(self).catalog() self.log.debug(f"Catalog is {self._catalog}") self._functionImplementations = {} for experiment in self._catalog.experiments: - # We cannot do a check of the dependencies here because - # (a) Catalog addition to registry is non-deterministic so any dependent experiment - # may not be in the registry yet - - # Store function name in the experiments metadata - try: - experiment_module_conf = ActuatorModuleConf.model_validate( - experiment.metadata.get("module", None) - ) - except pydantic.ValidationError: - self.log.exception( - f"Experiment {experiment} did not provide a valid module configuration - skipping" - ) - else: - self._functionImplementations[experiment.identifier] = ( - load_module_class_or_function(experiment_module_conf) - ) - + # For custom experiments, we need to find the decorated function + # The experiment identifier should match the function name + function_name = experiment.identifier + + # Search for the function in the current module and loaded modules + found_function = None + + # First, try to find it in the current module + import sys + + current_module = sys.modules[__name__] + if hasattr(current_module, function_name): + potential_function = getattr(current_module, function_name) + if ( + hasattr(potential_function, "_is_custom_experiment") + and potential_function._is_custom_experiment + ): + found_function = potential_function + + # If not found, search in other loaded modules + if not found_function: + for module_name, module in sys.modules.items(): + if module and hasattr(module, function_name): + potential_function = getattr(module, function_name) + if ( + hasattr(potential_function, "_is_custom_experiment") + and potential_function._is_custom_experiment + ): + found_function = potential_function + break + + if found_function: + self._functionImplementations[experiment.identifier] = found_function self.log.info( f"Experiment name: {experiment.identifier}. " - f"Function Name: {experiment_module_conf.moduleFunction}. " f"Function Implementation: {self._functionImplementations[experiment.identifier]}. " f"Experiment: {experiment}" ) + else: + self.log.warning( + f"Could not find function implementation for experiment {experiment.identifier}" + ) self.log.debug("Completed init") @@ -207,13 +383,62 @@ async def submit( # We only send one request return [requestid] + +def load_custom_experiments_legacy(identifier): + import importlib.resources + import logging + import pkgutil + from pathlib import Path + + import ado_actuators as plugins + import yaml + + from orchestrator.modules.actuators.catalog import ActuatorCatalogExtension + from orchestrator.modules.actuators.registry import ( + CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME, + ) + + logger = logging.getLogger("custom_experiments") + + for module in pkgutil.iter_modules(plugins.__path__, f"{plugins.__name__}."): + module_contents = { + entry.name for entry in importlib.resources.files(module.name).iterdir() + } + + if CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME in module_contents: + logger.debug(f"Found {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME}") + + experiments_configuration_file = Path( + str(importlib.resources.files(module.name)) + ) / Path(CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME) + + try: + catalog_extension = ActuatorCatalogExtension.model_validate( + yaml.safe_load(experiments_configuration_file.read_text()) + ) + except pydantic.ValidationError: + logger.exception( + f"{module.name}'s {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME} raised a validation error" + ) + raise + + logger.debug(f"Adding catalog extension {catalog_extension!s}") + # Check if catalog extension is for this actuator + if catalog_extension.actuatorIdentifier == identifier: + logger.debug( + f"Adding catalog extension {catalog_extension!s} for actuator {identifier}" + ) + _custom_experiments_catalog.update(catalog_extension) + @classmethod def catalog( cls, actuator_configuration: GenericActuatorParameters | None = None ) -> orchestrator.modules.actuators.catalog.ExperimentCatalog: - return orchestrator.modules.actuators.catalog.ExperimentCatalog( - catalogIdentifier="CustomExperiments" - ) + + load_custom_experiments_legacy(cls.identifier) + # Load custom experiments from entry points before returning catalog + load_custom_experiments_from_entry_points() + return get_custom_experiments_catalog() def current_catalog( self, diff --git a/orchestrator/modules/actuators/registry.py b/orchestrator/modules/actuators/registry.py index 590bbef6..183546bb 100644 --- a/orchestrator/modules/actuators/registry.py +++ b/orchestrator/modules/actuators/registry.py @@ -13,7 +13,6 @@ GenericActuatorParameters, ) from orchestrator.modules.actuators.catalog import ( - ActuatorCatalogExtension, ExperimentCatalog, ) from orchestrator.schema.measurementspace import MeasurementSpace @@ -198,25 +197,6 @@ def __init__( actuatorid=actuator_class.identifier, actuatorClass=actuator_class, ) - elif CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME in module_contents: - self.log.debug(f"Found {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME}") - - experiments_configuration_file = Path( - str(importlib.resources.files(module.name)) - ) / Path(CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME) - - try: - catalog_extension = ActuatorCatalogExtension.model_validate( - yaml.safe_load(experiments_configuration_file.read_text()) - ) - except pydantic.ValidationError: - self.log.exception( - f"{module.name}'s {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME} raised a validation error" - ) - raise - - self.log.debug(f"Adding catalog extension {catalog_extension!s}") - self.updateCatalogs(catalogExtension=catalog_extension) def __str__(self): From 789fbd36d76b7e75e0aa407c8a7ba1dbfe633490 Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 14 Oct 2025 19:37:50 +0100 Subject: [PATCH 08/36] feat: agent interface --- orchestrator/cli/core/cli.py | 3 +++ .../vllm_performance_test/execute_benchmark.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/orchestrator/cli/core/cli.py b/orchestrator/cli/core/cli.py index 34c45203..b183ac74 100644 --- a/orchestrator/cli/core/cli.py +++ b/orchestrator/cli/core/cli.py @@ -16,6 +16,7 @@ from orchestrator.cli.commands.describe import register_describe_command from orchestrator.cli.commands.edit import register_edit_command from orchestrator.cli.commands.get import register_get_command +from orchestrator.cli.commands.hamilton import register_hamilton_command from orchestrator.cli.commands.show import register_show_command from orchestrator.cli.commands.template import register_template_command from orchestrator.cli.commands.upgrade import register_upgrade_command @@ -59,6 +60,8 @@ register_template_command(app) register_upgrade_command(app) register_version_command(app) +print("registering hamilton command") +register_hamilton_command(app) @app.callback() diff --git a/plugins/actuators/vllm_performance/ado_actuators/vllm_performance/vllm_performance_test/execute_benchmark.py b/plugins/actuators/vllm_performance/ado_actuators/vllm_performance/vllm_performance_test/execute_benchmark.py index e7a39770..e55a99a5 100644 --- a/plugins/actuators/vllm_performance/ado_actuators/vllm_performance/vllm_performance_test/execute_benchmark.py +++ b/plugins/actuators/vllm_performance/ado_actuators/vllm_performance/vllm_performance_test/execute_benchmark.py @@ -85,9 +85,9 @@ def execute_benchmark( if __name__ == "__main__": results = execute_benchmark( interpreter="python3.10", - base_url="https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/", + base_url="https://test-oss-discovery-dev.apps.morrigan.accelerated-discovery.res.ibm.com/", data_set="random", - model="deepseek-ai/DeepSeek-V2.5", + model="openai/gpt-oss-20b", request_rate=None, max_concurrency=None, hf_token=os.getenv("HF_TOKEN"), From f705b2977d1561a1ca44e82d108933ed15e104f1 Mon Sep 17 00:00:00 2001 From: michaelj Date: Mon, 20 Oct 2025 19:19:24 +0100 Subject: [PATCH 09/36] refactor: update optimization_test_functions To use new decorator --- .../custom_experiments.yaml | 38 ------------------- .../optimization_test_functions/__init__.py | 0 .../custom_experiments/pyproject.toml | 4 +- 3 files changed, 2 insertions(+), 40 deletions(-) delete mode 100644 examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/custom_experiments.yaml rename examples/optimization_test_functions/custom_experiments/{ado_actuators => }/optimization_test_functions/__init__.py (100%) diff --git a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/custom_experiments.yaml b/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/custom_experiments.yaml deleted file mode 100644 index e5db85dd..00000000 --- a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/custom_experiments.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) IBM Corporation -# SPDX-License-Identifier: MIT - -experiments: - - identifier: 'nevergrad_opt_3d_test_func' - actuatorIdentifier: "custom_experiments" - metadata: - module: - moduleName: ado_actuators.optimization_test_functions.optimization_test_functions - moduleFunction: artificial_function - targetProperties: - - identifier: "function_value" - requiredProperties: - - identifier: "x0" - propertyDomain: - variableType: "CONTINUOUS_VARIABLE_TYPE" - - identifier: "x1" - propertyDomain: - variableType: "CONTINUOUS_VARIABLE_TYPE" - - identifier: "x2" - propertyDomain: - variableType: "CONTINUOUS_VARIABLE_TYPE" - optionalProperties: - - identifier: "name" - propertyDomain: - values: ['discus', 'sphere', 'cigar', 'griewank', 'rosenbrock', 'st1'] - - identifier: "num_blocks" - propertyDomain: - domainRange: [1,10] - variableType: "DISCRETE_VARIABLE_TYPE" - interval: 1 - defaultParameterization: - - value: "rosenbrock" - property: - identifier: "name" - - value: 1 - property: - identifier: "num_blocks" diff --git a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/__init__.py b/examples/optimization_test_functions/custom_experiments/optimization_test_functions/__init__.py similarity index 100% rename from examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/__init__.py rename to examples/optimization_test_functions/custom_experiments/optimization_test_functions/__init__.py diff --git a/examples/optimization_test_functions/custom_experiments/pyproject.toml b/examples/optimization_test_functions/custom_experiments/pyproject.toml index 7072055a..837b9cc7 100644 --- a/examples/optimization_test_functions/custom_experiments/pyproject.toml +++ b/examples/optimization_test_functions/custom_experiments/pyproject.toml @@ -13,12 +13,12 @@ dependencies = [ [tool.hatch.build] # Include the full namespace path include = [ - "ado_actuators/optimization_test_functions/**", + "optimization_test_functions/**", ] [tool.hatch.build.targets.wheel] only-include = [ - "ado_actuators/optimization_test_functions" + "optimization_test_functions" ] [tool.hatch.version] From 7ee5d85fcb9ee5517daa2231f3ffa68320f859c4 Mon Sep 17 00:00:00 2001 From: michaelj Date: Mon, 20 Oct 2025 19:20:11 +0100 Subject: [PATCH 10/36] refactor: update optimization_test_functions To use new decorator --- .../optimization_test_functions.py | 58 ------------- .../optimization_test_functions.py | 82 +++++++++++++++++++ 2 files changed, 82 insertions(+), 58 deletions(-) delete mode 100644 examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py create mode 100644 examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py diff --git a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py b/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py deleted file mode 100644 index 35f95359..00000000 --- a/examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/optimization_test_functions.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) IBM Corporation -# SPDX-License-Identifier: MIT - -import logging - -from nevergrad.functions import ArtificialFunction - -from orchestrator.modules.actuators.custom_experiments import custom_experiment -from orchestrator.schema.domain import PropertyDomain -from orchestrator.schema.property import ConstitutiveProperty - -moduleLog = logging.getLogger() - - -@custom_experiment( - [ - ConstitutiveProperty( - identifier="x0", - propertyDomain=PropertyDomain(variableType="CONTINUOUS_VARIABLE_TYPE"), - ), - ConstitutiveProperty( - identifier="x1", - propertyDomain=PropertyDomain(variableType="CONTINUOUS_VARIABLE_TYPE"), - ), - ConstitutiveProperty( - identifier="x2", - propertyDomain=PropertyDomain(variableType="CONTINUOUS_VARIABLE_TYPE"), - ), - ConstitutiveProperty( - identifier="name", - propertyDomain=PropertyDomain( - values=["discus", "sphere", "cigar", "griewank", "rosenbrock", "st1"] - ), - ), - ConstitutiveProperty( - identifier="num_blocks", - propertyDomain=PropertyDomain( - domainRange=[1, 10], variableType="DISCRETE_VARIABLE_TYPE", interval=1 - ), - ), - ] -) -def artificial_function(x0: float, x1: float, x2: float, name: str, num_blocks: int): - - import numpy as np - - # Get the function from nevergrad.functions.ArtificialFunction - func = ArtificialFunction( - name=name, - num_blocks=num_blocks, - block_dimension=int(3 / num_blocks), - translation_factor=0.0, - ) - - # Call the nevergrad function - value = func(np.asarray([x0, x1, x2])) - - return {"function_value": value} diff --git a/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py b/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py new file mode 100644 index 00000000..a863dfcc --- /dev/null +++ b/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py @@ -0,0 +1,82 @@ +# Copyright (c) IBM Corporation +# SPDX-License-Identifier: MIT + +import logging +import typing + +from nevergrad.functions import ArtificialFunction + +from orchestrator.modules.actuators.custom_experiments import custom_experiment +from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum +from orchestrator.schema.property import ( + ConstitutiveProperty, +) + +moduleLog = logging.getLogger() + + +# Decorate the python function with @custom_experiment +# This tells ado +# - The domains of all your parameters i.e. what are valid values they can take +# - The name of the output variable +# - If you return a named tuple -> No point really as need to declare a NamedTuple class or TypedDict which is longer than below +# - If you use keyword args -> Can extract optional parameters and parameterization +@custom_experiment( + required_properties=[ + ConstitutiveProperty( + identifier="x0", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE + ), + ), + ConstitutiveProperty( + identifier="x1", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE + ), + ), + ConstitutiveProperty( + identifier="x2", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE + ), + ), + ], + optional_properties=[ + ConstitutiveProperty( + identifier="num_blocks", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, + domainRange=[1, 10], + interval=1, + ), + ), + ConstitutiveProperty( + identifier="name", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, + values=["discus", "sphere", "cigar", "griewank", "rosenbrock", "st1"], + ), + ), + ], + parameterization={"num_blocks": 1, "name": "rosenbrock"}, + output_properties=["function_value"], +) +def nevergrad_opt_3d_test_func( + x0: float, x1: float, x2: float, name: str, num_blocks: int +) -> dict[str, typing.Any]: + + import numpy as np + + # Get the function from nevergrad.functions.ArtificialFunction + func = ArtificialFunction( + name=name, + num_blocks=num_blocks, + block_dimension=int(3 / num_blocks), + translation_factor=0.0, + ) + + # Call the nevergrad function + value = func(np.asarray([x0, x1, x2])) + + return {"function_value": value} From 2bcf49e60a3bf7d73cbd740afd4ba368628892c8 Mon Sep 17 00:00:00 2001 From: michaelj Date: Mon, 20 Oct 2025 19:23:03 +0100 Subject: [PATCH 11/36] refactor: further updates for using decorator --- .../modules/actuators/custom_experiments.py | 335 +++++++++--------- 1 file changed, 176 insertions(+), 159 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 33486c98..88c94a8d 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MIT import inspect +import logging import typing import uuid from functools import wraps @@ -13,9 +14,11 @@ from orchestrator.core.actuatorconfiguration.config import GenericActuatorParameters from orchestrator.modules.actuators.base import ( ActuatorBase, + ActuatorModuleConf, DeprecatedExperimentError, ) from orchestrator.modules.actuators.measurement_queue import MeasurementQueue +from orchestrator.modules.module import load_module_class_or_function from orchestrator.schema.entity import ( CheckRequiredObservedPropertyValuesPresent, Entity, @@ -25,7 +28,12 @@ ObservedProperty, ObservedPropertyValue, ) -from orchestrator.schema.property import ConstitutiveProperty +from orchestrator.schema.property import ( + AbstractPropertyDescriptor, + ConstitutiveProperty, + ConstitutivePropertyDescriptor, +) +from orchestrator.schema.property_value import ConstitutivePropertyValue from orchestrator.schema.reference import ExperimentReference from orchestrator.schema.request import MeasurementRequest, MeasurementRequestStateEnum from orchestrator.schema.result import ValidMeasurementResult @@ -40,60 +48,95 @@ ) -def custom_experiment(properties: list[ConstitutiveProperty]): +def derive_optional_properties_and_parameterization(func, required_properties): + func_signature = inspect.signature(func) + req_property_identifiers = {prop.identifier for prop in required_properties} + derived_optional_properties = [] + derived_parameterization = {} + for param in func_signature.parameters.values(): + if param.name in req_property_identifiers: + continue + if ( + param.kind + in (inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + and param.default is not inspect.Parameter.empty + ): + derived_optional_properties.append( + ConstitutiveProperty(identifier=param.name) + ) + derived_parameterization[param.name] = param.default + return derived_optional_properties, derived_parameterization + + +def custom_experiment( + required_properties: list[ConstitutiveProperty | ObservedProperty], + output_properties: list[str], + optional_properties: list[ConstitutiveProperty] | None = None, + parameterization: dict[str, typing.Any] | None = None, + metadata: dict[str, typing.Any] | None = None, +): """ Decorator for custom experiment functions. Args: - properties: List of ConstitutiveProperty instances defining the input values the Entity must have + required_properties: List of ConstitutiveProperty instances that are required input values. + output_properties: List of strings identifying the output property names. + optional_properties: List of ConstitutiveProperty instances that are optional input values. + parameterization: Tuple of parameters for default parameterization. + metadata: Metadata for the experiment Returns: A decorator that wraps a function to work with ADO's custom experiment system Example: - from orchestrator.schema.property import ConstitutiveProperty - - @custom_experiment([ - ConstitutiveProperty(identifier="temperature"), - ConstitutiveProperty(identifier="pressure") - ]) - def my_experiment(temperature: float, pressure: float) -> dict[str, float]: - # Function takes temperature and pressure as parameters - # Returns a dict of observed property values - return { - "density": temperature * pressure * 0.001, - "viscosity": temperature / pressure * 0.1 - } + + mass = ConstitutiveProperty(identifier="mass", propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, + domainRange=[0, 100], + )) + volume = ConstitutiveProperty(identifier="volume", propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, + domainRange=[0, 100], + )) + + @custom_experiment( + required_properties=[mass, volume], + output_properties=["density"] + ) + def calculate_density(mass, volume): + density_value = mass / volume if volume else None + return { + "density": density_value + } """ + metadata = metadata if metadata else {} + def decorator(func): + # Set up dynamic optional_properties and parameterization if none were provided + _optional_properties = optional_properties + _parameterization = parameterization + if _optional_properties is None: + _optional_properties, _parameterization = ( + derive_optional_properties_and_parameterization( + func, required_properties + ) + ) + @wraps(func) def wrapper( entity: Entity, experiment: Experiment ) -> list[ObservedPropertyValue]: """ Wrapper function that converts Entity+Experiment to dict and calls the wrapped function. - - Args: - entity: The entity to measure - experiment: The experiment configuration - - Returns: - List of ObservedPropertyValue instances """ - # Convert Entity+Experiment to dict using propertyValuesFromEntity input_values = experiment.propertyValuesFromEntity(entity) - - # Call the wrapped function with the input values result_dict = func(**input_values) - - # Convert the result dict to ObservedPropertyValue list observed_property_values = [] for property_identifier, value in result_dict.items(): - # Create ObservedProperty for this result - observed_property = ObservedProperty(identifier=property_identifier) - - # Create ObservedPropertyValue + observed_property = experiment.observedPropertyForTargetIdentifier( + property_identifier + ) observed_property_value = ObservedPropertyValue( property=observed_property, value=value ) @@ -101,30 +144,47 @@ def wrapper( return observed_property_values - # Validate that the properties match the function parameters + # Validate that the required property identifiers match the function parameters func_signature = inspect.signature(func) func_param_names = set(func_signature.parameters.keys()) - property_identifiers = {prop.identifier for prop in properties} - - if func_param_names != property_identifiers: + req_property_identifiers = {prop.identifier for prop in required_properties} + opt_property_identifiers = {prop.identifier for prop in optional_properties} + experiment_prop_identifiers = ( + req_property_identifiers | opt_property_identifiers + ) + if not experiment_prop_identifiers.issubset(func_param_names): raise ValueError( - f"Function parameter names {func_param_names} do not match " - f"property identifiers {property_identifiers}" + f"Function parameter names {func_param_names} must include all required property identifiers {req_property_identifiers}" ) # Store decorator arguments as function attributes - wrapper._decorator_properties = properties - wrapper._decorator_func = func + wrapper._decorator_required_properties = required_properties + wrapper._decorator_optional_properties = _optional_properties + wrapper._decorator_parameterization = _parameterization + wrapper._original_func = func wrapper._is_custom_experiment = True + metadata["experiment_function"] = wrapper + # Create and store the Experiment instance experiment = Experiment( actuatorIdentifier="custom_experiments", identifier=func.__name__, - requiredProperties=tuple(properties), - optionalProperties=(), - defaultParameterization=(), + requiredProperties=tuple(required_properties), + optionalProperties=tuple(_optional_properties), + targetProperties=[ + AbstractPropertyDescriptor(identifier=p) for p in output_properties + ], + defaultParameterization=tuple( + [ + ConstitutivePropertyValue( + property=ConstitutivePropertyDescriptor(identifier=k), value=v + ) + for k, v in _parameterization.items() + ] + ), deprecated=False, + metadata=metadata, ) wrapper._experiment = experiment @@ -136,27 +196,52 @@ def wrapper( return decorator -def get_custom_experiment_info(func) -> dict: - """ - Helper function to access decorator arguments and experiment from a decorated function. +def load_custom_experiments_from_catalog_extensions(identifier): + import importlib.resources + import logging + import pkgutil + from pathlib import Path - Args: - func: A function decorated with @custom_experiment + import ado_actuators as plugins + import yaml - Returns: - Dict containing decorator properties and experiment instance + from orchestrator.modules.actuators.catalog import ActuatorCatalogExtension + from orchestrator.modules.actuators.registry import ( + CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME, + ) - Raises: - ValueError: If the function is not decorated with @custom_experiment - """ - if not hasattr(func, "_is_custom_experiment") or not func._is_custom_experiment: - raise ValueError("Function is not decorated with @custom_experiment") + logger = logging.getLogger("custom_experiments") + + for module in pkgutil.iter_modules(plugins.__path__, f"{plugins.__name__}."): + module_contents = { + entry.name for entry in importlib.resources.files(module.name).iterdir() + } - return { - "properties": func._decorator_properties, - "experiment": func._experiment, - "original_func": func._decorator_func, - } + if CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME in module_contents: + logger.debug(f"Found {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME}") + + experiments_configuration_file = Path( + str(importlib.resources.files(module.name)) + ) / Path(CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME) + + try: + catalog_extension = ActuatorCatalogExtension.model_validate( + yaml.safe_load(experiments_configuration_file.read_text()) + ) + except pydantic.ValidationError: + logger.exception( + f"{module.name}'s {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME} raised a validation error" + ) + raise + + logger.debug(f"Adding catalog extension {catalog_extension!s}") + for experiment in catalog_extension.experiments: + if experiment.actuatorIdentifier == "custom_experiments": + _custom_experiments_catalog.addExperiment(experiment) + else: + logger.warning( + f"Cannot add catalog extension for {experiment.actuatorIdentifier} - only custom_experiments supports catalog extensions" + ) def load_custom_experiments_from_entry_points(): @@ -172,14 +257,14 @@ def load_custom_experiments_from_entry_points(): # Get all entry points for ado.custom_experiments entry_points = importlib.metadata.entry_points() - custom_experiment_groups = entry_points.get("ado.custom_experiments", []) - + custom_experiment_groups = entry_points.select(group="ado.custom_experiments") for entry_point in custom_experiment_groups: + print(entry_point) entry_point.load() - except ImportError: + except ImportError as error: + logging.getLogger("load_custom_experiments").warning(error) # importlib.metadata not available (Python < 3.8) - pass def get_custom_experiments_catalog() -> ( @@ -197,23 +282,26 @@ def get_custom_experiments_catalog() -> ( async def custom_experiment_wrapper( function: typing.Callable, parameters: dict, - measurementRequest: MeasurementRequest, - targetExperiment: Experiment, + measurement_request: MeasurementRequest, + target_experiment: Experiment, queue: MeasurementQueue, ): """ :param function: The function to call :param parameters: The custom parameters to the function - :param measurementRequest: The entity and custom experiment to be measured - :param targetExperiment: The experiment to execute. + :param measurement_request: The entity and custom experiment to be measured + :param target_experiment: The experiment to execute. Required as the measurementRequest only includes an ExperimentReference :param queue: The queue to put the result on :return: """ measurement_results = [] - for entity in measurementRequest.entities: - values = function(entity, targetExperiment, parameters=parameters) + for entity in measurement_request.entities: + if target_experiment.metadata.get("experiment_function"): + values = function(entity, target_experiment) + else: + values = function(entity, target_experiment, parameters=parameters) # Record the results in the entity if len(values) > 0: @@ -223,12 +311,12 @@ async def custom_experiment_wrapper( measurement_results.append(measurement_result) if len(measurement_results) > 0: - measurementRequest.measurements = measurement_results - measurementRequest.status = MeasurementRequestStateEnum.SUCCESS + measurement_request.measurements = measurement_results + measurement_request.status = MeasurementRequestStateEnum.SUCCESS else: - measurementRequest.status = MeasurementRequestStateEnum.FAILED + measurement_request.status = MeasurementRequestStateEnum.FAILED - await queue.put_async(measurementRequest, block=False) + await queue.put_async(measurement_request, block=False) @ray.remote @@ -259,39 +347,18 @@ def __init__(self, queue, params: dict | None = None): self._functionImplementations = {} for experiment in self._catalog.experiments: - # For custom experiments, we need to find the decorated function - # The experiment identifier should match the function name - function_name = experiment.identifier - - # Search for the function in the current module and loaded modules - found_function = None - - # First, try to find it in the current module - import sys - - current_module = sys.modules[__name__] - if hasattr(current_module, function_name): - potential_function = getattr(current_module, function_name) - if ( - hasattr(potential_function, "_is_custom_experiment") - and potential_function._is_custom_experiment - ): - found_function = potential_function - - # If not found, search in other loaded modules - if not found_function: - for module_name, module in sys.modules.items(): - if module and hasattr(module, function_name): - potential_function = getattr(module, function_name) - if ( - hasattr(potential_function, "_is_custom_experiment") - and potential_function._is_custom_experiment - ): - found_function = potential_function - break - - if found_function: - self._functionImplementations[experiment.identifier] = found_function + if module := experiment.metadata.get("module"): + experiment_module_conf = ActuatorModuleConf.model_validate(module) + function = ( + load_module_class_or_function(experiment_module_conf) + if experiment_module_conf + else None + ) + else: + function = experiment.metadata.get("experiment_function") + + if function: + self._functionImplementations[experiment.identifier] = function self.log.info( f"Experiment name: {experiment.identifier}. " f"Function Implementation: {self._functionImplementations[experiment.identifier]}. " @@ -299,7 +366,7 @@ def __init__(self, queue, params: dict | None = None): ) else: self.log.warning( - f"Could not find function implementation for experiment {experiment.identifier}" + f"Experiment in custom_experiment catalog is missing required metadata (either experiment_function or module): {experiment}" ) self.log.debug("Completed init") @@ -364,9 +431,6 @@ async def submit( self.log.debug(f"Create measurement request {request}") # TODO: Allow functions to specify if they should be remote - experiment = self._catalog.experimentForReference(request.experimentReference) - function = experiment.metadata.get("function", experiment.identifier) - self.log.debug(f"Calling custom experiment {function}") await custom_experiment_wrapper( self._functionImplementations[ @@ -383,59 +447,12 @@ async def submit( # We only send one request return [requestid] - -def load_custom_experiments_legacy(identifier): - import importlib.resources - import logging - import pkgutil - from pathlib import Path - - import ado_actuators as plugins - import yaml - - from orchestrator.modules.actuators.catalog import ActuatorCatalogExtension - from orchestrator.modules.actuators.registry import ( - CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME, - ) - - logger = logging.getLogger("custom_experiments") - - for module in pkgutil.iter_modules(plugins.__path__, f"{plugins.__name__}."): - module_contents = { - entry.name for entry in importlib.resources.files(module.name).iterdir() - } - - if CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME in module_contents: - logger.debug(f"Found {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME}") - - experiments_configuration_file = Path( - str(importlib.resources.files(module.name)) - ) / Path(CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME) - - try: - catalog_extension = ActuatorCatalogExtension.model_validate( - yaml.safe_load(experiments_configuration_file.read_text()) - ) - except pydantic.ValidationError: - logger.exception( - f"{module.name}'s {CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME} raised a validation error" - ) - raise - - logger.debug(f"Adding catalog extension {catalog_extension!s}") - # Check if catalog extension is for this actuator - if catalog_extension.actuatorIdentifier == identifier: - logger.debug( - f"Adding catalog extension {catalog_extension!s} for actuator {identifier}" - ) - _custom_experiments_catalog.update(catalog_extension) - @classmethod def catalog( cls, actuator_configuration: GenericActuatorParameters | None = None ) -> orchestrator.modules.actuators.catalog.ExperimentCatalog: - load_custom_experiments_legacy(cls.identifier) + load_custom_experiments_from_catalog_extensions(cls.identifier) # Load custom experiments from entry points before returning catalog load_custom_experiments_from_entry_points() return get_custom_experiments_catalog() From ed1f5a838a113427f93ebe7b1ae6157733a543b3 Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 21 Oct 2025 09:27:43 +0100 Subject: [PATCH 12/36] docs: temporarily change included file --- website/docs/actuators/creating-custom-experiments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/actuators/creating-custom-experiments.md b/website/docs/actuators/creating-custom-experiments.md index 6d943e6f..9503b78d 100644 --- a/website/docs/actuators/creating-custom-experiments.md +++ b/website/docs/actuators/creating-custom-experiments.md @@ -54,7 +54,7 @@ An example experiment description file is: ```yaml {% - include "../../../examples/optimization_test_functions/custom_experiments/ado_actuators/optimization_test_functions/custom_experiments.yaml" + include "../../../examples/ml-multi-cloud/custom_experiment/ado_actuators/ml_multi_cloud_custom_experiments/custom_experiments.yaml" %} ``` From e5229e10c668b459f535fc6eac98dadc656efbc7 Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 22 Oct 2025 10:37:51 +0100 Subject: [PATCH 13/36] fix: minor --- orchestrator/modules/actuators/custom_experiments.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 88c94a8d..98934609 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -154,7 +154,7 @@ def wrapper( ) if not experiment_prop_identifiers.issubset(func_param_names): raise ValueError( - f"Function parameter names {func_param_names} must include all required property identifiers {req_property_identifiers}" + f"Function parameter names {func_param_names} must include all property identifiers {experiment_prop_identifiers}. Missing identifiers: {experiment_prop_identifiers - func_param_names}" ) # Store decorator arguments as function attributes @@ -259,7 +259,6 @@ def load_custom_experiments_from_entry_points(): entry_points = importlib.metadata.entry_points() custom_experiment_groups = entry_points.select(group="ado.custom_experiments") for entry_point in custom_experiment_groups: - print(entry_point) entry_point.load() except ImportError as error: From 2b667f81d69402190fa814154ad455cfed7aea73 Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 22 Oct 2025 12:30:36 +0100 Subject: [PATCH 14/36] fix: can't store function As it can't be serialized --- .../modules/actuators/custom_experiments.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 98934609..0dff699c 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -18,7 +18,7 @@ DeprecatedExperimentError, ) from orchestrator.modules.actuators.measurement_queue import MeasurementQueue -from orchestrator.modules.module import load_module_class_or_function +from orchestrator.modules.module import ModuleTypeEnum, load_module_class_or_function from orchestrator.schema.entity import ( CheckRequiredObservedPropertyValuesPresent, Entity, @@ -148,7 +148,11 @@ def wrapper( func_signature = inspect.signature(func) func_param_names = set(func_signature.parameters.keys()) req_property_identifiers = {prop.identifier for prop in required_properties} - opt_property_identifiers = {prop.identifier for prop in optional_properties} + opt_property_identifiers = ( + {prop.identifier for prop in optional_properties} + if optional_properties + else set() + ) experiment_prop_identifiers = ( req_property_identifiers | opt_property_identifiers ) @@ -164,7 +168,12 @@ def wrapper( wrapper._original_func = func wrapper._is_custom_experiment = True - metadata["experiment_function"] = wrapper + # Create an ActuatorModuleConf instance describing where the function is + metadata["module"] = ActuatorModuleConf( + moduleType=ModuleTypeEnum.ACTUATOR, + moduleName=func.__module__, + moduleFunction=func.__name__, + ) # Create and store the Experiment instance experiment = Experiment( @@ -297,10 +306,13 @@ async def custom_experiment_wrapper( measurement_results = [] for entity in measurement_request.entities: - if target_experiment.metadata.get("experiment_function"): - values = function(entity, target_experiment) - else: + # Inspect function to see if it has a keyword parameter "parameters" + func_signature = inspect.signature(function) + func_param_names = set(func_signature.parameters.keys()) + if "parameters" in func_param_names: values = function(entity, target_experiment, parameters=parameters) + else: + values = function(entity, target_experiment) # Record the results in the entity if len(values) > 0: @@ -346,6 +358,8 @@ def __init__(self, queue, params: dict | None = None): self._functionImplementations = {} for experiment in self._catalog.experiments: + + function = None if module := experiment.metadata.get("module"): experiment_module_conf = ActuatorModuleConf.model_validate(module) function = ( @@ -353,8 +367,6 @@ def __init__(self, queue, params: dict | None = None): if experiment_module_conf else None ) - else: - function = experiment.metadata.get("experiment_function") if function: self._functionImplementations[experiment.identifier] = function From 78cdf18ceb8578b8b12008da95f143f52288b38b Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 23 Oct 2025 14:29:21 +0100 Subject: [PATCH 15/36] feat: enhanced inference of domain also refactor code into more primitive functions --- .../modules/actuators/custom_experiments.py | 250 +++++++++++++++--- 1 file changed, 216 insertions(+), 34 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 0dff699c..977b981f 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -48,24 +48,212 @@ ) -def derive_optional_properties_and_parameterization(func, required_properties): +def _infer_domain_and_property(identifier, annotation, default): + """This function infers the domain of a parameter from its type and default value. + Parameters: + - identifier: The name of the parameter + - annotation: The type of the parameter. Must be a valid python type + - default: The default value of the parameter + Returns: + - A ConstitutiveProperty instance with the inferred domain + Exceptions: + - ValueError: If the parameter is not supported i.e. the domain cannot be inferred + """ + import logging + + logger = logging.getLogger("custom_experiments") + from typing import get_args, get_origin + + from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum + + if annotation is int: + domain = PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, interval=1 + ) + elif annotation is float: + domain = PropertyDomain(variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE) + elif annotation is bool: + domain = PropertyDomain(variableType=VariableTypeEnum.BINARY_VARIABLE_TYPE) + elif annotation is str: + domain = PropertyDomain( + variableType=VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE, + values=[default], + ) + elif get_origin(annotation) in [ + getattr(typing, "Literal", None), + getattr(__import__("typing_extensions"), "Literal", None), + ] or str(annotation).startswith("typing.Literal"): + vals = list(get_args(annotation)) + domain = PropertyDomain( + variableType=VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE, values=vals + ) + else: + logger.warning( + f"Error parameter '{identifier}' - unsupported annotation: {annotation}" + ) + raise ValueError(f"Unsupported annotation: {annotation}") + + return ConstitutiveProperty(identifier=identifier, propertyDomain=domain) + + +def derive_required_properties_from_signature(func, optional_idents): + """This function derives the required properties from the function signature. + + The required properties are the positional parameters of the function that are not in optional_idents. + + Parameters: + - func: The function to derive the required properties from + - optional_idents: The identifiers of the optional properties + Returns: + - A list of ConstitutiveProperty instances + """ + func_signature = inspect.signature(func) - req_property_identifiers = {prop.identifier for prop in required_properties} - derived_optional_properties = [] - derived_parameterization = {} + required_props = [] for param in func_signature.parameters.values(): - if param.name in req_property_identifiers: + if param.name in optional_idents: + continue + if ( + param.kind + in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + and param.default is inspect.Parameter.empty + ): + print(param) + inferred_prop = _infer_domain_and_property( + param.name, param.annotation, None + ) + required_props.append(inferred_prop) + return required_props + + +def get_parameterization( + properties: list[ConstitutiveProperty], func_signature: inspect.Signature +) -> dict[str, typing.Any]: + """This function derives the parameterization of properties and function signature. + + The parameterization of a property is the default value of the corresponding parameter in func_signature + + Parameters: + - properties: The properties to derive the parameterization for + - func_signature: The function signature to derive the parameterization from + Returns: + - A dictionary of property identifiers and their default values + Exceptions: + - ValueError: If a parameterization cannot be""" + param_map = {p.name: p for p in func_signature.parameters.values()} + results = {} + missing = [] + for prop in properties: + param = param_map.get(prop.identifier, None) + if param and param.default is not inspect.Parameter.empty: + results[prop.identifier] = param.default + else: + missing.append(prop.identifier) + if missing: + raise ValueError(f"Parameterization missing for: {missing}") + return results + + +def derive_optional_properties_and_parameterization( + func: typing.Callable, required_properties: list[ConstitutiveProperty] +) -> tuple[list[ConstitutiveProperty], dict[str, typing.Any]]: + """This function derives the optional properties and their parameterization from the function signature. + + The optional properties are the keyword parameters of the function that are not in required_properties. + The parameterization of an optional property is the default value of the corresponding parameter in func_signature. + + Parameters: + - func: The function to derive the optional properties and parameterization from + - required_properties: The properties that are required input values. + Returns: + - A tuple. The first element is a list of optional properties, the second element is a dictionary of property identifiers and their default values + Exceptions: + - ValueError: If a parameterization cannot be derived for any optional property (unexpected) + - ValueError: If a domain cannot be inferred for any optional property""" + optional_properties = [] + for param in inspect.signature(func).parameters.values(): + if param.name in {prop.identifier for prop in required_properties}: continue if ( param.kind in (inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) and param.default is not inspect.Parameter.empty ): - derived_optional_properties.append( - ConstitutiveProperty(identifier=param.name) + inferred_prop = _infer_domain_and_property( + param.name, param.annotation, param.default + ) + optional_properties.append(inferred_prop) + + return optional_properties, get_parameterization( + optional_properties, inspect.signature(func) + ) + + +def check_parameters_and_infer( + _optional_properties: list[ConstitutiveProperty] | None, + _parameterization: dict | None, + _required_properties: list[ConstitutiveProperty] | None, + func, +): + logger = logging.getLogger("custom_experiment_decorator") + + # Set up dynamic optional_properties and parameterization if none were provided + _optional_properties = _optional_properties if _optional_properties else [] + + if not _required_properties: + try: + _required_properties = derive_required_properties_from_signature( + func, {prop.identifier for prop in _optional_properties} + ) + except ValueError as error: + logger.critical( + f"No required properties provided and they could not be derived from signature: {error}" + ) + raise error + if _optional_properties and not _parameterization: + try: + _parameterization = get_parameterization( + _optional_properties, inspect.signature(func) + ) + except ValueError as error: + logger.critical( + f"Optional properties provided but parameterization could not be derived from signature: {error}" + ) + raise error + if not _optional_properties and not _parameterization: + try: + _optional_properties, _parameterization = ( + derive_optional_properties_and_parameterization( + func, _required_properties + ) + ) + except ValueError as error: + logger.critical( + f"No optional properties provided and theycould not be derived from signature: {error}" ) - derived_parameterization[param.name] = param.default - return derived_optional_properties, derived_parameterization + raise error + + return _optional_properties, _parameterization, _required_properties + + +def check_parameters_valid(_optional_properties, _required_properties, func): + # Validate that the property identifiers match the function parameters + func_signature = inspect.signature(func) + func_param_names = set(func_signature.parameters.keys()) + req_property_identifiers = {prop.identifier for prop in _required_properties} + opt_property_identifiers = ( + {prop.identifier for prop in _optional_properties} + if _optional_properties + else set() + ) + experiment_prop_identifiers = req_property_identifiers | opt_property_identifiers + if not experiment_prop_identifiers.issubset(func_param_names): + raise ValueError( + f"Function parameter names {func_param_names} must include all property identifiers {experiment_prop_identifiers}. Missing identifiers: {experiment_prop_identifiers - func_param_names}" + ) def custom_experiment( @@ -111,17 +299,9 @@ def calculate_density(mass, volume): """ metadata = metadata if metadata else {} + logger = logging.getLogger("custom_experiment_decorator") def decorator(func): - # Set up dynamic optional_properties and parameterization if none were provided - _optional_properties = optional_properties - _parameterization = parameterization - if _optional_properties is None: - _optional_properties, _parameterization = ( - derive_optional_properties_and_parameterization( - func, required_properties - ) - ) @wraps(func) def wrapper( @@ -144,25 +324,27 @@ def wrapper( return observed_property_values - # Validate that the required property identifiers match the function parameters - func_signature = inspect.signature(func) - func_param_names = set(func_signature.parameters.keys()) - req_property_identifiers = {prop.identifier for prop in required_properties} - opt_property_identifiers = ( - {prop.identifier for prop in optional_properties} - if optional_properties - else set() - ) - experiment_prop_identifiers = ( - req_property_identifiers | opt_property_identifiers + # If we were not given information on required/optional properties + # or parameterization try to infer it + # This function will log a critical error message and raise exception + # if inference is required (because user did not provide explicit information) + # but it could not be done (missing annotation, invalid annotation etc.) + _optional_properties, _parameterization, _required_properties = ( + check_parameters_and_infer( + optional_properties, parameterization, required_properties, func + ) ) - if not experiment_prop_identifiers.issubset(func_param_names): - raise ValueError( - f"Function parameter names {func_param_names} must include all property identifiers {experiment_prop_identifiers}. Missing identifiers: {experiment_prop_identifiers - func_param_names}" + + try: + check_parameters_valid(_optional_properties, _required_properties, func) + except ValueError as error: + logger.critical( + f"Unable to generate custom function via decorator: {error}" ) + raise # Store decorator arguments as function attributes - wrapper._decorator_required_properties = required_properties + wrapper._decorator_required_properties = _required_properties wrapper._decorator_optional_properties = _optional_properties wrapper._decorator_parameterization = _parameterization wrapper._original_func = func @@ -179,7 +361,7 @@ def wrapper( experiment = Experiment( actuatorIdentifier="custom_experiments", identifier=func.__name__, - requiredProperties=tuple(required_properties), + requiredProperties=tuple(_required_properties), optionalProperties=tuple(_optional_properties), targetProperties=[ AbstractPropertyDescriptor(identifier=p) for p in output_properties From e179c7154fa147508019a0ed807efd4a54718a56 Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 23 Oct 2025 14:29:54 +0100 Subject: [PATCH 16/36] test: decoration functions --- tests/actuators/test_custom_experiments.py | 149 +++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 tests/actuators/test_custom_experiments.py diff --git a/tests/actuators/test_custom_experiments.py b/tests/actuators/test_custom_experiments.py new file mode 100644 index 00000000..59933bb5 --- /dev/null +++ b/tests/actuators/test_custom_experiments.py @@ -0,0 +1,149 @@ +# Copyright (c) IBM Corporation +# SPDX-License-Identifier: MIT + +import re + +import pytest + +from orchestrator.modules.actuators import custom_experiments +from orchestrator.schema.domain import VariableTypeEnum + +# Used in test_literal_domain +try: + from typing import Literal +except ImportError: + from typing import Literal + + +def test_infer_domain_and_property_type(): + """Tests that given a parameter name, its type and a default the correct behaviour is observed""" + + fn = custom_experiments._infer_domain_and_property + # int + p = fn("a", int, 42) + assert p.propertyDomain.variableType == VariableTypeEnum.DISCRETE_VARIABLE_TYPE + # float + p = fn("b", float, 3.1) + assert p.propertyDomain.variableType == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE + # bool + p = fn("c", bool, True) + assert p.propertyDomain.variableType == VariableTypeEnum.BINARY_VARIABLE_TYPE + # str + p = fn("d", str, "hello") + assert ( + p.propertyDomain.variableType == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + ) + assert p.propertyDomain.values == ["hello"] + # Literal + p = fn("e", Literal["X", "Y"], "X") + assert p.propertyDomain.variableType == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE + assert set(p.propertyDomain.values) == {"X", "Y"} + # bytes is not supported + with pytest.raises(ValueError, match=r"Unsupported annotation: "): + _ = fn("f", bytes, b"err") + + +def test_derive_required_properties_from_signature_basic(): + def f(a: int, b: float, c: int = 1): + pass + + result = custom_experiments.derive_required_properties_from_signature( + f, optional_idents={} + ) + # a, b expected (no-domain), c skipped as optional + ids = {r.identifier for r in result} + assert ids == {"a", "b"} + + # Check that missing annotation raises error + def f(a, b: float, c: int = 1): + pass + + with pytest.raises( + ValueError, match=r"Unsupported annotation: " + ): + custom_experiments.derive_required_properties_from_signature( + f, optional_idents={} + ) + + +def test_get_parameterization_success_and_failure(): + import inspect + + from orchestrator.schema.property import ConstitutiveProperty + + def g(x=7, y=9): + pass + + sig = inspect.signature(g) + ps = [ConstitutiveProperty(identifier="x"), ConstitutiveProperty(identifier="y")] + paramz = custom_experiments.get_parameterization(ps, sig) + assert paramz["x"] == 7 + assert paramz["y"] == 9 + + def g2(x): + pass + + sig2 = inspect.signature(g2) + with pytest.raises( + ValueError, match=re.escape("Parameterization missing for: ['x']") + ): + custom_experiments.get_parameterization( + [ConstitutiveProperty(identifier="x")], sig2 + ) + + +def test_derive_optional_properties_and_parameterization_basic_types_and_unsupported(): + # covers int, float, bool, str, literal, and unsupported + + def fn( + i: int = 1, + f: float = 2.0, + b: bool = False, + s: str = "abc", + lit: Literal["A", "B"] = "A", + ): + pass + + optionals, _ = custom_experiments.derive_optional_properties_and_parameterization( + fn, [] + ) + types = {p.identifier: p.propertyDomain.variableType for p in optionals} + print(types) + assert types["i"] == VariableTypeEnum.DISCRETE_VARIABLE_TYPE + assert types["f"] == VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE + assert types["b"] == VariableTypeEnum.BINARY_VARIABLE_TYPE + assert types["s"] == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE + assert types["lit"] == VariableTypeEnum.CATEGORICAL_VARIABLE_TYPE + + # bytes is not supported for inference - check fails + def fn( + i: int = 1, + f: float = 2.0, + b: bool = False, + s: str = "abc", + lit: Literal["A", "B"] = "A", + n: bytes = b"foo", + ): + pass + + with pytest.raises(ValueError, match="Unsupported annotation: "): + optionals, _ = ( + custom_experiments.derive_optional_properties_and_parameterization(fn, []) + ) + + # i has no annotation - check fails + def fn( + i=1, + f: float = 2.0, + b: bool = False, + s: str = "abc", + lit: Literal["A", "B"] = "A", + ): + pass + + with pytest.raises( + ValueError, match=r"Unsupported annotation: " + ): + optionals, _ = ( + custom_experiments.derive_optional_properties_and_parameterization(fn, []) + ) From 2c22adbeb32f5359b01989dcc04b5786c923d284 Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 23 Oct 2025 15:03:23 +0100 Subject: [PATCH 17/36] docs(website): Update custom experiment docs --- .../actuators/creating-custom-experiments.md | 390 ++++++++++++------ 1 file changed, 253 insertions(+), 137 deletions(-) diff --git a/website/docs/actuators/creating-custom-experiments.md b/website/docs/actuators/creating-custom-experiments.md index 9503b78d..ba6aa2ea 100644 --- a/website/docs/actuators/creating-custom-experiments.md +++ b/website/docs/actuators/creating-custom-experiments.md @@ -1,185 +1,301 @@ > [!NOTE] > -> The +> For a full worked example, see > [search a space with an optimizer](../examples/best-configuration-search.md) -> example contains the code described here -Often you might want to use an experiment that is a simple python function. A -typical example is a cost-function to be used with an optimization. `ado` -provides a way to add such functions as experiments without having to create an -Actuator class. +`ado` enables you to use Python functions as experiments by registering + them as custom experiments using a decorator. -The process involves creating a python package with two files +## The structure of a custom experiment package -- A python module with your functions -- A yaml file describing the experiments they provide +Your custom experiment should be in a standard Python package e.g. -And then installing this package. - -## Custom experiment package structure +```text +$YOUR_REPO_NAME/ + pyproject.toml + my_custom_experiment/ # Change to whatever name you like + __init__.py + # Python file with your decorated function(s) - can have any name + experiments.py +``` -To create a package with one or more custom experiments you **must** use the -following package structure +In addition, you must register an entry-point to the group `ado.custom_experiments` +in your `pyproject.toml` so `ado` can find your custom_experiment automatically: - -```text -$YOUR_REPO_NAME - - setup.py - - ado_actuators/ # This is `ado`'s namespaced package for actuator plugins and custom experiments - - $YOUR_CUSTOM_EXPERIMENT_PLUGIN_PACKAGE/ # Your package with custom experiments - - __init__.py - - $EXPERIMENTS.py # Python file with your function in it - - custom_experiments.yaml # A yaml file describing the custom experiments your package provides +```toml +[project.entry-points."ado.custom_experiments"] +#This should be python file with your decorated function(s). +my_experiment = "my_custom_package.experiments" ``` - -## Writing the experiment catalog +>[!NOTE] +> +> Note: You can have more than one decorated function in a module, +> and register more than one entry-point i.e. have functions +> in different modules. -The experiment catalog contains the following critical pieces of information: +## Decorating your custom experiment function -1. What your experiment is called -2. Who is going to execute your experiment - for custom python functions this - will **always** be the special actuator "custom_experiments" -3. What the python function that executes your experiment is called -4. What properties your experiment measures -5. The properties from other experiments this experiment requires as input - if - your function does not require properties from another experiment you don't - need this field +**To define a custom experiment, decorate your function with `@custom_experiment`.** -A catalog can define multiple experiments. Each one is a new element in the -top-level list. +In the simplest case: -An example experiment description file is: +- type the parameters (using python `typing`) +- return the output in a dictionary of key value pairs +- define the keys of these dictionary in the `output_properties` +parameter of the decorator -```yaml -{% - include "../../../examples/ml-multi-cloud/custom_experiment/ado_actuators/ml_multi_cloud_custom_experiments/custom_experiments.yaml" -%} +```python +from typing import Dict, Any +from orchestrator.modules.actuators.custom_experiments import custom_experiment + +@custom_experiment( + output_properties=["density"] +) +def calculate_density(mass: float, volume: float) -> Dict[str, Any]: + density_value = mass / volume if volume else None + return {"density": density_value} ``` -This YAML describes: +**Experiment Naming:** -- a single experiment called `nevergrad_opt_3d_test_func` -- the measurement will be executed using a python function called - `artificial_function` - - the function is in the module - `ado_actuators.optimization_test_functions.optimization_test_functions` i.e. - `ado_actuators.$YOUR_CUSTOM_EXPERIMENT_PLUGIN_PACKAGE.$EXPERIMENTS` - following above package layout) - - this name will always start with `ado_actuators` -- The experiment has a set of required input properties - `x0`, `x1` and `x2` - - and set of optional input properties - `name` and `num_blocks` - - these input properties are what the python function is expected to use -- The experiment measures a single target property `function_value` - - so applying this experiment will return a value of an observed property - called `nevergrad_opt_3d_test_func.total_cost` +The experiment will be registered with the name of the decorated +Python function (e.g., `calculate_density`). -## Writing your custom experiment functions +**Required Properties:** -The python function that implements the experiment described in the catalog must +Each positional parameter in the signature will become a +required property. -1. Be called the name you gave in the catalog (`metadata.function` field) -2. Have a specific signature and return value +**Return Value:** -A snippet of the above function, `artifical_function`, showing the signature and -return value is: +The function must return a dictionary whose keys are output names +e.g. "density" above, and the value is the measured value. - -```python -import typing -from orchestrator.schema.experiment import Experiment -from orchestrator.schema.entity import Entity -from orchestrator.schema.property_value import PropertyValue - - -def artificial_function( - entity: Entity, - experiment: Experiment, - parameters=None, # deprecated field -) -> typing.List[PropertyValue]: - """ - - :param entity: The entity to be measured - :param experiment: The Experiment object representing the exact Experiment to perform - Required as multiple experiments can measure this property - :param parameters: A dictionary. - :return: A list of PropertyValue objects - """ - # parameters is a dictionary of key:value pairs of the experiment required/optional inputs - # defined in custom_experiments.yaml - parameters = experiment.propertyValuesFromEntity(entity) - - #Experiment logic elided - ... - - # At end return the results - pv = PropertyValue( - value=value, - property=experiment.observedPropertyForTargetIdentifier("function_value"), - valueType=ValueTypeEnum.NUMERIC_VALUE_TYPE, - ) - return [pv] -``` - +**Property Domains:** + +`ado` will infer the domains of your positional (non-keyword) inputs as follows: + +- floats -> continuous domain over the real numbers +- ints -> discrete domain over the integers +- literal -> categorical domain whose values are the literal values + +> [!IMPORTANT] +> +> If a positional parameter has a different type to above e.g.string +> `ado` cannot automatically determine a domain and you will get an +> exception on trying to use the function. +> In this case see +> [define the domain of input parameters](#defining-the-domains-of-required-properties) -In the above function `entity` and `experiment` are `ado` objects describing -what to measure and what to measure it with. Since the custom experiment package -only defined one experiment (see -[Writing the experiment catalog](#writing-the-experiment-catalog)) the -`experiment` object will represent the `nevergrad_opt_3d_test_func` experiment. -The `entity` and `experiment` objects can be converted into a dictionary of -required and optional input properties names and values using -`experiment.propertyValuesFromEntity` +### Keyword parameters and optional properties -Once the values to return have been calculated the function has to create -`PropertyValue` objects as shown above. +Keyword parameters in your function signature will be converted to +optional properties of the custom experiment. +The `parameterization` for the optional properties is the value of +keyword in the signature. -To find out more about the class instances passed to this function check the -`ado` source code. +The domain inference rules are the same as given above with one addition, +types other than float,int and literal, are assigned an open categorical domain +with a single "known" value, the keyword parameters default. -!!! warning end - - If your function returns properties with different names than those - you specified in the catalog for the experiment entry, they will be ignored. +--- -## Using your custom experiments: the custom_experiments actuator +## Using your custom experiment -All your custom experiments in `ado` are accessed via a special actuator called -_custom_experiments_. +### Adding your custom experiment to `ado` -### Add your experiments to `ado` +To add your experiments to `ado`: -First to add your experiments to `ado` run `pip install` in the same directory -as your -[custom experiment packages `setup.py`](#custom-experiment-package-structure) +1. Install your package (e.g. `pip install -e .` in your package’s root). +2. Run: -Confirm the experiment has been added: + ```shell + ado describe actuators --details + ``` + +All custom experiments are made available in `ado` through +the special actuator called `custom_experiments`. +Your experiment will be listed under the `custom_experiments` actuator +using the function's name. + +### Testing your custom experiment + +You can test your custom experiment +using the [`run_experiment`](run_experiment.md) command line tool. +Save the following YAML to a file `point.yaml` + +```yaml +entity: + mass:8 + volume:4 +experiments: +- actuatorIdentifier: custom_experiments + experimentIdentifier: calculate_density +``` + +then execute: ```commandline -ado describe actuators --details +run_experiment point.yaml ``` -If the custom experiment was the one defined in -[above](#writing-the-experiment-catalog) you would see a new experiment entry -for the _objective_functions_ actuator called `ml-multicloud-cost-v1.0`. +### Using your custom experiment in a `discoveryspace` -### Add a custom experiment to a `discoveryspace` +To use a custom experiment in `discoveryspace` +you specify it in its `measurementspace` - exactly like other experiments. -To use a custom experiment you declare it the `measurementspace` of a -`discoveryspaces` - exactly like other experiments. The only difference is you -used the `custom_experiments` actuator. +Here is a toy example using the `calculate_density` custom experiment +defined above: ```yaml -{% include "../../../examples/optimization_test_functions/space.yaml" %} +sampleStoreIdentifier: dfe035 +entitySpace: +- identifier: mass + propertyDomain: + domainRange: [1,10] + interval: 1 +- identifier: volume + propertyDomain: + domainRange: [1,10] +experiments: +- actuatorIdentifier: custom_experiments + experimentIdentifier: calculate_density +``` + +--- + +## Advanced configuration of custom experiments + +The simplest case described in +[decorating you custom experiment function](#decorating-your-custom-experiment-function) +is enough to get started with a custom experiment. +However, if your function has particular types or if you want to refine +domain information you need to access more advanced features of the decorator. + +### Defining the domains of required properties + +Python functions don't carry any domain information so in many cases +the domain inferred from the type will be too broad. +In this case you can define the domains explicitly in the decorator. + +> [!IMPORTANT] +> +> Once you define one required property explicitly you must define them all explicitly. + +Defining the domain explicitly enables: + +- Better input validation when creating `spaces` +- Automated construction of relevant discovery spaces (via `ado template`) +- Control of what are considered required and optional properties +- Finer grained control of the domain e.g. you can have a float +parameter but make the domain discrete + +In the following example, we explicitly indicate that the mass and volume parameters +of our `calculate_density` function are positive numbers. + +```python +from typing import Dict, Any +from orchestrator.modules.actuators.custom_experiments import custom_experiment +from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum +from orchestrator.schema.property import ConstitutiveProperty + +mass = ConstitutiveProperty( + identifier="mass", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, + domainRange=[1, 100] + ) +) +volume = ConstitutiveProperty( + identifier="volume", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, + domainRange=[1, 100] + ) +) + +@custom_experiment( + required_properties=[mass, volume], + output_properties=["density"] +) +def calculate_density(mass, volume) -> Dict[str, Any]: + density_value = mass / volume if volume else None + return {"density": density_value} +``` + +> [!NOTE] +> +> Every non-keyword parameter in your python function is **required**. +> However you can make any keyword parameter also required + +### Defining the domains of optional properties + +Similarly to required properties you can define domains for +the **optional properties** via the `optional_properties` parameter to the decorator. +This is also a list of `ConstitutiveProperty` instances +which define the parameters domains. +Default values for the optional properties must be given either in the function signature +i.e. as keyword args, or via the `parameterization` parameter to the decorator. + +> [!IMPORTANT] +> +> Once you define one optional property explicitly you must define them +> all explicitly. +> Similarly once you define the parameterization of an optional +> property explicitly you must define the all explicitly. + +```python + +round_result = ConstitutiveProperty( + identifier="round_result", + propertyDomain=PropertyDomain( + variableType=VariableTypeEnum.BINARY_VARIABLE_TYPE, + ) +) + +@custom_experiment( + required_properties=[mass, volume], + #round_result will get its default value from the keyword arg + optional_properties=[round_result], + output_properties=["density"], + metadata={"description": "Calculates density from mass and volume"} +) +def calculate_density(mass, volume, round_result: bool = False): + density_value = mass / volume if volume else None + if round_result and density_value is not None: + density_value = round(density_value, 2) + return {"density": density_value} ``` -Note `ado` will validate the measurement space as normal. So in this case if the -custom experiment `benchmark_performance` from the `replay` actuator is not -included the space creation will fail. +The above registers `round_result` as an optional properties +of the experiment, with its value in the function signature as the default parameterization. + +### Supplying metadata + +You can also supply a `metadata` dictionary to the "metadata" parameter of +the decorator. +Use this to record experiment-level documentation, categories, etc. +This is illustrated in the above example. + +--- + +## Using your decorated function in code + +The decorated function is wrapped to take `ado` internal +data structures, and you would not typically need to +call it directly. However, the decorated experiment function is +still regular Python and can be called: + +```python +# Access the original function (undecorated) +original = calculate_density._decorator_func +print(original(8, 4)) # {'density': 2} +``` ## Next Steps -Follow the +See [search a space with an optimizer](../examples/best-configuration-search.md) -example to see how the custom experiment described here works in practice. +for a complete practical workflow using custom experiments. From f6bbca67507b7b79358d59150abe34fc043c919c Mon Sep 17 00:00:00 2001 From: michaelj Date: Thu, 23 Oct 2025 15:05:10 +0100 Subject: [PATCH 18/36] fix: comment for future --- orchestrator/cli/core/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/orchestrator/cli/core/cli.py b/orchestrator/cli/core/cli.py index b183ac74..672f6b4d 100644 --- a/orchestrator/cli/core/cli.py +++ b/orchestrator/cli/core/cli.py @@ -16,7 +16,6 @@ from orchestrator.cli.commands.describe import register_describe_command from orchestrator.cli.commands.edit import register_edit_command from orchestrator.cli.commands.get import register_get_command -from orchestrator.cli.commands.hamilton import register_hamilton_command from orchestrator.cli.commands.show import register_show_command from orchestrator.cli.commands.template import register_template_command from orchestrator.cli.commands.upgrade import register_upgrade_command @@ -60,8 +59,7 @@ register_template_command(app) register_upgrade_command(app) register_version_command(app) -print("registering hamilton command") -register_hamilton_command(app) +# register_hamilton_command(app) @app.callback() From 74ee4bf0103e860e55ee2f918f0e0218623299d9 Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 08:45:53 +0000 Subject: [PATCH 19/36] feat: Add ExperimentModuleConf for clarity --- orchestrator/modules/actuators/custom_experiments.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 977b981f..1c62dea4 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -18,7 +18,11 @@ DeprecatedExperimentError, ) from orchestrator.modules.actuators.measurement_queue import MeasurementQueue -from orchestrator.modules.module import ModuleTypeEnum, load_module_class_or_function +from orchestrator.modules.module import ( + ModuleConf, + ModuleTypeEnum, + load_module_class_or_function, +) from orchestrator.schema.entity import ( CheckRequiredObservedPropertyValuesPresent, Entity, @@ -48,6 +52,10 @@ ) +class ExperimentModuleConf(ModuleConf): + moduleType: ModuleTypeEnum = pydantic.Field(default=ModuleTypeEnum.EXPERIMENT) + + def _infer_domain_and_property(identifier, annotation, default): """This function infers the domain of a parameter from its type and default value. Parameters: @@ -543,7 +551,7 @@ def __init__(self, queue, params: dict | None = None): function = None if module := experiment.metadata.get("module"): - experiment_module_conf = ActuatorModuleConf.model_validate(module) + experiment_module_conf = ExperimentModuleConf.model_validate(module) function = ( load_module_class_or_function(experiment_module_conf) if experiment_module_conf From 0173311b446f425258ac41366fdce7daf630572c Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 08:46:31 +0000 Subject: [PATCH 20/36] refactor: change parameter name to output_property_identifiers --- orchestrator/modules/actuators/custom_experiments.py | 10 +++++----- website/docs/actuators/creating-custom-experiments.md | 8 +++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 1c62dea4..2c2b3c79 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -266,7 +266,7 @@ def check_parameters_valid(_optional_properties, _required_properties, func): def custom_experiment( required_properties: list[ConstitutiveProperty | ObservedProperty], - output_properties: list[str], + output_property_identifiers: list[str], optional_properties: list[ConstitutiveProperty] | None = None, parameterization: dict[str, typing.Any] | None = None, metadata: dict[str, typing.Any] | None = None, @@ -276,7 +276,7 @@ def custom_experiment( Args: required_properties: List of ConstitutiveProperty instances that are required input values. - output_properties: List of strings identifying the output property names. + output_property_identifiers: List of strings identifying the output property names. optional_properties: List of ConstitutiveProperty instances that are optional input values. parameterization: Tuple of parameters for default parameterization. metadata: Metadata for the experiment @@ -372,7 +372,8 @@ def wrapper( requiredProperties=tuple(_required_properties), optionalProperties=tuple(_optional_properties), targetProperties=[ - AbstractPropertyDescriptor(identifier=p) for p in output_properties + AbstractPropertyDescriptor(identifier=p) + for p in output_property_identifiers ], defaultParameterization=tuple( [ @@ -462,7 +463,6 @@ def load_custom_experiments_from_entry_points(): except ImportError as error: logging.getLogger("load_custom_experiments").warning(error) - # importlib.metadata not available (Python < 3.8) def get_custom_experiments_catalog() -> ( @@ -566,7 +566,7 @@ def __init__(self, queue, params: dict | None = None): f"Experiment: {experiment}" ) else: - self.log.warning( + self.log.error( f"Experiment in custom_experiment catalog is missing required metadata (either experiment_function or module): {experiment}" ) diff --git a/website/docs/actuators/creating-custom-experiments.md b/website/docs/actuators/creating-custom-experiments.md index ba6aa2ea..c7e4f1f9 100644 --- a/website/docs/actuators/creating-custom-experiments.md +++ b/website/docs/actuators/creating-custom-experiments.md @@ -50,8 +50,9 @@ parameter of the decorator from typing import Dict, Any from orchestrator.modules.actuators.custom_experiments import custom_experiment + @custom_experiment( - output_properties=["density"] + output_property_identifiers=["density"] ) def calculate_density(mass: float, volume: float) -> Dict[str, Any]: density_value = mass / volume if volume else None @@ -204,7 +205,7 @@ from orchestrator.schema.property import ConstitutiveProperty mass = ConstitutiveProperty( identifier="mass", propertyDomain=PropertyDomain( - variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, domainRange=[1, 100] ) ) @@ -216,9 +217,10 @@ volume = ConstitutiveProperty( ) ) + @custom_experiment( required_properties=[mass, volume], - output_properties=["density"] + output_property_identifiers=["density"] ) def calculate_density(mass, volume) -> Dict[str, Any]: density_value = mass / volume if volume else None From c63a789a2869894b216b726f7974042d75da7f39 Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 08:55:05 +0000 Subject: [PATCH 21/36] docs(website): improvements --- .../actuators/creating-custom-experiments.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/website/docs/actuators/creating-custom-experiments.md b/website/docs/actuators/creating-custom-experiments.md index c7e4f1f9..c4547c69 100644 --- a/website/docs/actuators/creating-custom-experiments.md +++ b/website/docs/actuators/creating-custom-experiments.md @@ -1,7 +1,7 @@ > [!NOTE] > -> For a full worked example, see +> For a fully working example, see > [search a space with an optimizer](../examples/best-configuration-search.md) `ado` enables you to use Python functions as experiments by registering @@ -31,9 +31,9 @@ my_experiment = "my_custom_package.experiments" >[!NOTE] > -> Note: You can have more than one decorated function in a module, -> and register more than one entry-point i.e. have functions -> in different modules. +> 1. You can have more than one decorated function in a module. +> 2. If you want to have functions in different modules you +> need to register each module as an entrypoint. ## Decorating your custom experiment function @@ -43,7 +43,7 @@ In the simplest case: - type the parameters (using python `typing`) - return the output in a dictionary of key value pairs -- define the keys of these dictionary in the `output_properties` +- define the keys of this dictionary in the `output_property_identifiers` parameter of the decorator ```python @@ -190,8 +190,8 @@ Defining the domain explicitly enables: - Better input validation when creating `spaces` - Automated construction of relevant discovery spaces (via `ado template`) - Control of what are considered required and optional properties -- Finer grained control of the domain e.g. you can have a float -parameter but make the domain discrete +- Finer grained control of the domain (e.g. you can have a float +parameter but make the domain discrete) In the following example, we explicitly indicate that the mass and volume parameters of our `calculate_density` function are positive numbers. @@ -230,7 +230,8 @@ def calculate_density(mass, volume) -> Dict[str, Any]: > [!NOTE] > > Every non-keyword parameter in your python function is **required**. -> However you can make any keyword parameter also required +> However, you can make any keyword parameter required by adding it to +> required_properties parameter of the decorator ### Defining the domains of optional properties @@ -261,7 +262,7 @@ round_result = ConstitutiveProperty( required_properties=[mass, volume], #round_result will get its default value from the keyword arg optional_properties=[round_result], - output_properties=["density"], + output_property_identifiers=["density"], metadata={"description": "Calculates density from mass and volume"} ) def calculate_density(mass, volume, round_result: bool = False): From 1b4fd3ecf135d95cb97711d307bf09e8abc0220d Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 08:55:29 +0000 Subject: [PATCH 22/36] feat: new module type --- orchestrator/modules/module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/orchestrator/modules/module.py b/orchestrator/modules/module.py index 9611bc4a..43238872 100644 --- a/orchestrator/modules/module.py +++ b/orchestrator/modules/module.py @@ -16,6 +16,7 @@ class ModuleTypeEnum(enum.Enum): SAMPLE_STORE = "sample_store" GENERIC = "generic" SAMPLER = "sampler" + EXPERIMENT = "experiment" class ModuleConf(pydantic.BaseModel): From 5866b25caf2527603ca3cedabb3d290fae60a58a Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 08:55:54 +0000 Subject: [PATCH 23/36] refactor: to new parameter name --- .../optimization_test_functions/optimization_test_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py b/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py index a863dfcc..b81231df 100644 --- a/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py +++ b/examples/optimization_test_functions/custom_experiments/optimization_test_functions/optimization_test_functions.py @@ -60,7 +60,7 @@ ), ], parameterization={"num_blocks": 1, "name": "rosenbrock"}, - output_properties=["function_value"], + output_property_identifiers=["function_value"], ) def nevergrad_opt_3d_test_func( x0: float, x1: float, x2: float, name: str, num_blocks: int From 128c82674e8edb1c072a78e4f81e25ddded06c49 Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 08:56:36 +0000 Subject: [PATCH 24/36] chore: fix import --- tests/actuators/test_custom_experiments.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/actuators/test_custom_experiments.py b/tests/actuators/test_custom_experiments.py index 59933bb5..bb0f6882 100644 --- a/tests/actuators/test_custom_experiments.py +++ b/tests/actuators/test_custom_experiments.py @@ -2,18 +2,13 @@ # SPDX-License-Identifier: MIT import re +from typing import Literal import pytest from orchestrator.modules.actuators import custom_experiments from orchestrator.schema.domain import VariableTypeEnum -# Used in test_literal_domain -try: - from typing import Literal -except ImportError: - from typing import Literal - def test_infer_domain_and_property_type(): """Tests that given a parameter name, its type and a default the correct behaviour is observed""" From 0f07ff86eaa97d6009778aba39dc7d47f5c2c59d Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 09:03:14 +0000 Subject: [PATCH 25/36] chore: typing --- orchestrator/modules/actuators/custom_experiments.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index 2c2b3c79..f13f1c2c 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -56,7 +56,9 @@ class ExperimentModuleConf(ModuleConf): moduleType: ModuleTypeEnum = pydantic.Field(default=ModuleTypeEnum.EXPERIMENT) -def _infer_domain_and_property(identifier, annotation, default): +def _infer_domain_and_property( + identifier: str, annotation: type, default: typing.Any +) -> ConstitutiveProperty: """This function infers the domain of a parameter from its type and default value. Parameters: - identifier: The name of the parameter @@ -104,7 +106,9 @@ def _infer_domain_and_property(identifier, annotation, default): return ConstitutiveProperty(identifier=identifier, propertyDomain=domain) -def derive_required_properties_from_signature(func, optional_idents): +def derive_required_properties_from_signature( + func: typing.Callable, optional_idents: list[str] +) -> list[ConstitutiveProperty]: """This function derives the required properties from the function signature. The required properties are the positional parameters of the function that are not in optional_idents. From 273997de866d66b7b53715210774fd9557c9568f Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 12:13:07 +0000 Subject: [PATCH 26/36] fix: not creating correct model --- orchestrator/modules/actuators/custom_experiments.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index f13f1c2c..ec979a69 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -14,7 +14,6 @@ from orchestrator.core.actuatorconfiguration.config import GenericActuatorParameters from orchestrator.modules.actuators.base import ( ActuatorBase, - ActuatorModuleConf, DeprecatedExperimentError, ) from orchestrator.modules.actuators.measurement_queue import MeasurementQueue @@ -362,9 +361,9 @@ def wrapper( wrapper._original_func = func wrapper._is_custom_experiment = True - # Create an ActuatorModuleConf instance describing where the function is - metadata["module"] = ActuatorModuleConf( - moduleType=ModuleTypeEnum.ACTUATOR, + # Create an ExperimentModuleConf instance describing where the function is + metadata["module"] = ExperimentModuleConf( + moduleType=ModuleTypeEnum.EXPERIMENT, moduleName=func.__module__, moduleFunction=func.__name__, ) From 4a9aa572849df35771bdb9482362eae92292562d Mon Sep 17 00:00:00 2001 From: michaelj Date: Tue, 28 Oct 2025 19:53:02 +0000 Subject: [PATCH 27/36] fix: incorrect anchor --- website/docs/actuators/working-with-actuators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/actuators/working-with-actuators.md b/website/docs/actuators/working-with-actuators.md index d425d48b..566c987e 100644 --- a/website/docs/actuators/working-with-actuators.md +++ b/website/docs/actuators/working-with-actuators.md @@ -11,7 +11,7 @@ specific documentation for various actuators available. You can also add [your own custom experiments](creating-custom-experiments.md) using the special actuator -[_custom_experiments_](creating-custom-experiments.md#using-your-custom-experiments-the-custom_experiments-actuator) +[_custom_experiments_](creating-custom-experiments.md#using-your-custom-experiment). !!! info end From e2822f3a2daf75c36651e45106eaaa5ae2753c5f Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 29 Oct 2025 14:08:57 +0000 Subject: [PATCH 28/36] fix: Handle exceptions from custom functions --- .../modules/actuators/custom_experiments.py | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/orchestrator/modules/actuators/custom_experiments.py b/orchestrator/modules/actuators/custom_experiments.py index ec979a69..de98d6d6 100644 --- a/orchestrator/modules/actuators/custom_experiments.py +++ b/orchestrator/modules/actuators/custom_experiments.py @@ -39,7 +39,7 @@ from orchestrator.schema.property_value import ConstitutivePropertyValue from orchestrator.schema.reference import ExperimentReference from orchestrator.schema.request import MeasurementRequest, MeasurementRequestStateEnum -from orchestrator.schema.result import ValidMeasurementResult +from orchestrator.schema.result import InvalidMeasurementResult, ValidMeasurementResult from orchestrator.utilities.environment import enable_ray_actor_coverage from orchestrator.utilities.logging import configure_logging @@ -405,7 +405,6 @@ def load_custom_experiments_from_catalog_extensions(identifier): import pkgutil from pathlib import Path - import ado_actuators as plugins import yaml from orchestrator.modules.actuators.catalog import ActuatorCatalogExtension @@ -413,6 +412,14 @@ def load_custom_experiments_from_catalog_extensions(identifier): CATALOG_EXTENSIONS_CONFIGURATION_FILE_NAME, ) + try: + import ado_actuators as plugins + except ImportError: + logging.getLogger("custom_experiments").info( + "ado_actuators namespace package has not been created yet" + ) + return + logger = logging.getLogger("custom_experiments") for module in pkgutil.iter_modules(plugins.__path__, f"{plugins.__name__}."): @@ -502,15 +509,23 @@ async def custom_experiment_wrapper( # Inspect function to see if it has a keyword parameter "parameters" func_signature = inspect.signature(function) func_param_names = set(func_signature.parameters.keys()) - if "parameters" in func_param_names: - values = function(entity, target_experiment, parameters=parameters) - else: - values = function(entity, target_experiment) + try: + if "parameters" in func_param_names: + values = function(entity, target_experiment, parameters=parameters) + else: + values = function(entity, target_experiment) - # Record the results in the entity - if len(values) > 0: - measurement_result = ValidMeasurementResult( - entityIdentifier=entity.identifier, measurements=values + # Record the results in the entity + if len(values) > 0: + measurement_result = ValidMeasurementResult( + entityIdentifier=entity.identifier, measurements=values + ) + measurement_results.append(measurement_result) + except Exception as error: + measurement_result = InvalidMeasurementResult( + entityIdentifier=entity.identifier, + experimentReference=target_experiment.reference, + reason=f"Unexpected exception: {error}", ) measurement_results.append(measurement_result) From 02e618eec9bb87e184e0a93a6de63b76f48e9dee Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 29 Oct 2025 14:09:44 +0000 Subject: [PATCH 29/36] fix: Handle exceptions from submit() --- orchestrator/utilities/run_experiment.py | 33 +++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/orchestrator/utilities/run_experiment.py b/orchestrator/utilities/run_experiment.py index 0ed6959d..da2f80ef 100644 --- a/orchestrator/utilities/run_experiment.py +++ b/orchestrator/utilities/run_experiment.py @@ -84,13 +84,23 @@ def execute_local( queue=queue, params=config ) actuator = actuators[experiment.actuatorIdentifier] - # Submit the measurement request asynchronously. - actuator.submit.remote( - entities=[entity], - experimentReference=experiment.reference, - requesterid="run_experiment", - requestIndex=0, - ) + # Submit the measurement request asynchronously, handle errors gracefully. + try: + actuator.submit.remote( + entities=[entity], + experimentReference=experiment.reference, + requesterid="run_experiment", + requestIndex=0, + ) + except Exception as e: + print( + f"[ERROR] Failed to submit measurement request to actuator '{experiment.actuatorIdentifier}': {e}" + ) + import traceback + + traceback.print_exc() + # Either skip, or return None, or propagate. Let's return None. + return None return queue.get() return execute_local @@ -241,8 +251,13 @@ def run( if valid: print(f"Executing: {reference.experimentIdentifier}") request = execute(reference, entity) - print("Result:") - print(f"{request.series_representation(output_format='target')}\n") + if request is None: + print( + "Measurement request failed unexpectedly. Skipping this experiment." + ) + else: + print("Result:") + print(f"{request.series_representation(output_format='target')}\n") else: print("Entity is not valid") From e5c6f0e25877acebc5ae518cd111cc983d20f741 Mon Sep 17 00:00:00 2001 From: michaelj Date: Sun, 2 Nov 2025 15:23:39 +0000 Subject: [PATCH 30/36] feat: vector property domain A domain for a property whose value is a vector. --- orchestrator/schema/domain.py | 5 +++++ orchestrator/schema/property.py | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/orchestrator/schema/domain.py b/orchestrator/schema/domain.py index 6ea713d7..9e26e45e 100644 --- a/orchestrator/schema/domain.py +++ b/orchestrator/schema/domain.py @@ -26,6 +26,7 @@ class VariableTypeEnum(str, enum.Enum): BINARY_VARIABLE_TYPE = "BINARY_VARIABLE_TYPE" # the value of the variable is binary UNKNOWN_VARIABLE_TYPE = "UNKNOWN_VARIABLE_TYPE" # the type of value of the variable is unknown/unspecified IDENTIFIER_VARIABLE_TYPE = "IDENTIFIER_VARIABLE_TYPE" # the value is some type of, possible unique, identifier + VECTOR_VARIABLE_TYPE = "VECTOR_VARIABLE_TYPE" # the value is a vector class ProbabilityFunctionsEnum(str, enum.Enum): @@ -416,6 +417,10 @@ def variableType_matches_values(cls, value, values: "pydantic.FieldValidationInf elif value == VariableTypeEnum.OPEN_CATEGORICAL_VARIABLE_TYPE: assert values.data.get("interval") is None assert values.data.get("domainRange") is None + elif value == VariableTypeEnum.VECTOR_VARIABLE_TYPE: + raise ValueError( + "Vector variables are not supported by PropertyDomain - use VectorPropertyDomain instead" + ) return value diff --git a/orchestrator/schema/property.py b/orchestrator/schema/property.py index 11cb7f1b..88b385a3 100644 --- a/orchestrator/schema/property.py +++ b/orchestrator/schema/property.py @@ -2,11 +2,32 @@ # SPDX-License-Identifier: MIT import enum +from typing import Annotated import pydantic from pydantic import ConfigDict from orchestrator.schema.domain import PropertyDomain +from orchestrator.schema.vector_domain import VectorPropertyDomain + + +def domain_type_discriminator(domain): + + if isinstance(domain, PropertyDomain): + return "scalar" + if isinstance(domain, VectorPropertyDomain): + return "vector" + if isinstance(domain, dict): + return "vector" if domain.get("element_domain") else "scalar" + + raise ValueError(f"Unable to determine domain type for domain: {domain}") + + +Domain = Annotated[ + Annotated[PropertyDomain, pydantic.Tag("scalar")] + | Annotated[VectorPropertyDomain, pydantic.Tag("vector")], + pydantic.Discriminator(domain_type_discriminator), +] class MeasuredPropertyTypeEnum(str, enum.Enum): @@ -117,7 +138,7 @@ class Property(pydantic.BaseModel): metadata: dict | None = pydantic.Field( default=None, description="Metadata on the property" ) - propertyDomain: PropertyDomain = pydantic.Field( + propertyDomain: Domain = pydantic.Field( default=PropertyDomain(), description="Provides information on the variable type and the valid values it can take", ) From 999ec5a3c1cc648dbcd05c9b45a04749fd5d04b6 Mon Sep 17 00:00:00 2001 From: michaelj Date: Sun, 2 Nov 2025 15:31:21 +0000 Subject: [PATCH 31/36] test: vector domain --- tests/schema/test_vector_domain.py | 92 ++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/schema/test_vector_domain.py diff --git a/tests/schema/test_vector_domain.py b/tests/schema/test_vector_domain.py new file mode 100644 index 00000000..c3e263d1 --- /dev/null +++ b/tests/schema/test_vector_domain.py @@ -0,0 +1,92 @@ +# Copyright (c) IBM Corporation +# SPDX-License-Identifier: MIT + +import math + +import pydantic +import pytest + +from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum +from orchestrator.schema.vector_domain import VectorPropertyDomain + + +@pytest.fixture +def simple_element_domain(): + # Discrete domain: {1, 2, 3} + return PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, + values=[1, 2, 3], + ) + + +def test_vector_property_domain_valid_vector(simple_element_domain): + vpd = VectorPropertyDomain(element_domain=simple_element_domain, number_elements=2) + assert vpd.valueInDomain([1, 2]) + assert vpd.valueInDomain([2, 3]) + assert not vpd.valueInDomain([1, 999]) # 999 not in element domain + assert not vpd.valueInDomain([1]) # Too short + assert not vpd.valueInDomain([1, 2, 3]) # Too long + + +def test_vector_property_domain_domain_values(simple_element_domain): + vpd = VectorPropertyDomain(element_domain=simple_element_domain, number_elements=2) + values = vpd.domain_values + # Should be the cartesian product + expected = [(a, b) for a in [1, 2, 3] for b in [1, 2, 3]] + assert set(values) == set(expected) + assert len(values) == 9 # 3^2 + + +def test_vector_property_domain_size(simple_element_domain): + vpd = VectorPropertyDomain(element_domain=simple_element_domain, number_elements=3) + assert vpd.size == 27 + # Inf if element_domain not countable + # Make continuous domain (should not allow domain_values) + from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum + + cd = PropertyDomain( + variableType=VariableTypeEnum.CONTINUOUS_VARIABLE_TYPE, domainRange=[0, 1] + ) + vpd_cont = VectorPropertyDomain(element_domain=cd, number_elements=2) + assert math.isinf(vpd_cont.size) + with pytest.raises(Exception, match="element_domain must be discrete"): + _ = vpd_cont.domain_values + + +def test_vector_property_domain_isSubDomain(simple_element_domain): + eldom_small = PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, values=[1] + ) + eldom_big = PropertyDomain( + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, values=[1, 2, 3] + ) + vpd_small = VectorPropertyDomain(element_domain=eldom_small, number_elements=2) + vpd_big = VectorPropertyDomain(element_domain=eldom_big, number_elements=3) + # Check: fewer dims + assert vpd_small.isSubDomain(vpd_big) + # Reverse: should fail (more dims) + assert not vpd_big.isSubDomain(vpd_small) + # Same number dims but element subdomain wrong + vpd2 = VectorPropertyDomain(element_domain=eldom_big, number_elements=2) + assert not vpd2.isSubDomain(vpd_small) + + +def test_vector_property_domain_eq(simple_element_domain): + vpd1 = VectorPropertyDomain(element_domain=simple_element_domain, number_elements=2) + vpd2 = VectorPropertyDomain(element_domain=simple_element_domain, number_elements=2) + assert vpd1 == vpd2 + vpd3 = VectorPropertyDomain(element_domain=simple_element_domain, number_elements=3) + assert vpd1 != vpd3 + + +def test_vector_property_domain_variableType_guard(simple_element_domain): + # If someone tries to construct it with wrong variableType, should raise error + + from orchestrator.schema.domain import VariableTypeEnum + + with pytest.raises(pydantic.ValidationError, match="VariableType must be VECTOR"): + VectorPropertyDomain( + element_domain=simple_element_domain, + number_elements=2, + variableType=VariableTypeEnum.DISCRETE_VARIABLE_TYPE, + ) From aa6183d30237ff06f8ea8a266511bab1de6aa983 Mon Sep 17 00:00:00 2001 From: michaelj Date: Fri, 21 Nov 2025 10:17:29 +0000 Subject: [PATCH 32/36] feat(schema): add vector domain module --- orchestrator/schema/vector_domain.py | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 orchestrator/schema/vector_domain.py diff --git a/orchestrator/schema/vector_domain.py b/orchestrator/schema/vector_domain.py new file mode 100644 index 00000000..8f649f67 --- /dev/null +++ b/orchestrator/schema/vector_domain.py @@ -0,0 +1,88 @@ +# Copyright (c) IBM Corporation +# SPDX-License-Identifier: MIT + +import itertools + +import pydantic +from pydantic import BaseModel, ConfigDict, Field + +from orchestrator.schema.domain import PropertyDomain, VariableTypeEnum + + +class VectorPropertyDomain(BaseModel): + element_domain: PropertyDomain = Field(..., description="Domain of elements") + number_elements: int = Field(..., description="Length/dimension of the vector") + variableType: VariableTypeEnum = Field( + default=VariableTypeEnum.VECTOR_VARIABLE_TYPE + ) + + model_config = ConfigDict(frozen=True, extra="forbid") + + @pydantic.field_validator("variableType") + def variableType_matches_values(cls, value, values: "pydantic.FieldValidationInfo"): + if value != VariableTypeEnum.VECTOR_VARIABLE_TYPE: + raise ValueError("VariableType must be VECTOR_VARIABLE_TYPE") + return value + + def valueInDomain(self, value: list) -> bool: + """Check that all elements in the vector are in the element_domain.""" + if not isinstance( + value, (list, tuple) + ): # or len(value) != self.number_elements: + return False + + return all(self.element_domain.valueInDomain(v) for v in value) + + def isSubDomain(self, otherDomain: "VectorPropertyDomain") -> bool: + """Must be a subdomain only to another VectorPropertyDomain.""" + # Must be a VectorPropertyDomain and have the proper variableType (robustness) + if not hasattr(otherDomain, "variableType") or ( + otherDomain.variableType != self.variableType + ): + return False + # Must have equal or fewer dimensions + if self.number_elements > otherDomain.number_elements: + return False + # Each element subdomain + return self.element_domain.isSubDomain(otherDomain.element_domain) + + @property + def domain_values(self) -> list: + # The cartesian product of the element domain values, number_elements times + # Returns a list of vectors + try: + elem_values = self.element_domain.domain_values + except Exception as e: + raise ValueError( + f"element_domain must be discrete and have domain_values: {e!s}" + ) + # Cartesian product + return list(itertools.product(elem_values, repeat=self.number_elements)) + + @property + def size(self) -> int: + """Returns the size (number of possible vectors) if countable.""" + + n_elem_values = len(self.element_domain.domain_values) + return n_elem_values**self.number_elements + + def __eq__(self, other): + if not isinstance(other, VectorPropertyDomain): + return False + return ( + self.number_elements == other.number_elements + and self.element_domain == other.element_domain + and self.variableType == other.variableType + ) + + def _repr_pretty_(self, p, cycle=False): + if cycle: + p.text("Cycle detected") + else: + p.text(f"Type: {self.variableType}") + p.breakable() + p.text(f"Number of elements: {self.number_elements}") + p.breakable() + with p.group(2, "Element Domain:"): + p.break_() + p.pretty(self.element_domain) From e49d399af9630e72211e6495d171e8ff36941406 Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 26 Nov 2025 09:25:50 +0000 Subject: [PATCH 33/36] chore(core): update str rep --- orchestrator/schema/entityspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestrator/schema/entityspace.py b/orchestrator/schema/entityspace.py index ddef433c..40f99798 100644 --- a/orchestrator/schema/entityspace.py +++ b/orchestrator/schema/entityspace.py @@ -97,7 +97,7 @@ def size(self) -> int: def __str__(self): return ( - f"Explicit entity-space defined by {len(self._constitutiveProperties)}" + f"entityspace defined by {len(self._constitutiveProperties)}" f" constitutive properties: {[cp.identifier for cp in self._constitutiveProperties]}" ) From 918431f3d1753f4ff5fa69a3ce96f6ac8d570232 Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 26 Nov 2025 09:30:14 +0000 Subject: [PATCH 34/36] fix(core): domain value types domain value types would be numpy types. PropertyValues created using this values would be cast to int/float and serialized to json. However with vectors the value is a list and the list contents are not changed. This meant vector values would fail to be serialized to json. This is the simplest change to fix this. --- orchestrator/schema/domain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/orchestrator/schema/domain.py b/orchestrator/schema/domain.py index 9e26e45e..c9a82e13 100644 --- a/orchestrator/schema/domain.py +++ b/orchestrator/schema/domain.py @@ -59,13 +59,13 @@ def _internal_range_values(lower, upper, interval) -> list: """ if not is_float_range(interval=interval, domain_range=[lower, upper]): - return list(np.arange(lower, upper, interval)) + return [int(el) for el in np.arange(lower, upper, interval)] num = int(np.floor((upper - lower) / interval)) + 1 values = [lower + i * interval for i in range(num)] if values[-1] == upper: values = values[:-1] # values = np.linspace(lower, upper, num)[:-1] - return list(np.round(values, 10)) + return [float(el) for el in np.round(values, 10)] def is_subdomain_of_unknown_domain(unknownDomain, testDomain): From 1efba4a40ea143801df526b76514bad81ea597f5 Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 26 Nov 2025 09:31:15 +0000 Subject: [PATCH 35/36] chore(logs): improve error message --- orchestrator/core/discoveryspace/samplers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/orchestrator/core/discoveryspace/samplers.py b/orchestrator/core/discoveryspace/samplers.py index 60f2feff..c13d54cf 100644 --- a/orchestrator/core/discoveryspace/samplers.py +++ b/orchestrator/core/discoveryspace/samplers.py @@ -351,7 +351,7 @@ def iterator_closure( entitySpace=entitySpace ): raise ValueError( - f"Cannot use ExplicitEntitySpaceGridSampleGenerator with {entitySpace}" + f"ExplicitEntitySpaceGridSampleGenerator is not compatible with {entitySpace}" ) def sequential_iterator() -> typing.Generator[list[Entity], None, None]: @@ -404,7 +404,7 @@ def entitySpaceIterator( entitySpace=entitySpace ): raise ValueError( - f"Cannot use ExplicitEntitySpaceGridSampleGenerator with {entitySpace}" + f"ExplicitEntitySpaceGridSampleGenerator is not compatible with {entitySpace}" ) def iterator_closure( @@ -465,7 +465,7 @@ async def iterator_closure( entitySpace=entitySpace ): raise ValueError( - f"Cannot use ExplicitEntitySpaceGridSampleGenerator with {entitySpace}" + f"ExplicitEntitySpaceGridSampleGenerator is not compatible with {entitySpace}" ) async def sequential_iterator() -> ( From ce103d78045c3ad6fa4e104e5cfa47086d93f66f Mon Sep 17 00:00:00 2001 From: michaelj Date: Wed, 26 Nov 2025 09:32:26 +0000 Subject: [PATCH 36/36] fix(core): valueInDomain and size for vector domains --- orchestrator/schema/vector_domain.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/orchestrator/schema/vector_domain.py b/orchestrator/schema/vector_domain.py index 8f649f67..2157f44d 100644 --- a/orchestrator/schema/vector_domain.py +++ b/orchestrator/schema/vector_domain.py @@ -26,9 +26,7 @@ def variableType_matches_values(cls, value, values: "pydantic.FieldValidationInf def valueInDomain(self, value: list) -> bool: """Check that all elements in the vector are in the element_domain.""" - if not isinstance( - value, (list, tuple) - ): # or len(value) != self.number_elements: + if not isinstance(value, (list, tuple)) or len(value) != self.number_elements: return False return all(self.element_domain.valueInDomain(v) for v in value) @@ -63,7 +61,7 @@ def domain_values(self) -> list: def size(self) -> int: """Returns the size (number of possible vectors) if countable.""" - n_elem_values = len(self.element_domain.domain_values) + n_elem_values = self.element_domain.size return n_elem_values**self.number_elements def __eq__(self, other):