feat: add entitymatching and delete tags

implement ENG-4427

Author: cowan-macady

.coveragerc

@@ -12,8 +12,9 @@ omit =
   indykite_sdk/identity/helper.py
   indykite_sdk/model/unique_name_identifier.py
   indykite_sdk/model/data_access.py
-  indykite_sdk/model/email_attachment.py
-  indykite_sdk/identity/consent.py
+  indykite_sdk/model/property.py
+  indykite_sdk/model/scope_item.py
+  indykite_sdk/model/audience_item.py
   indykite_sdk/utils/*
   tests/*
   examples/*

Pipfile

@@ -22,6 +22,7 @@ pytest-cov = "4.1.0"
 authlib = "*"
 grpcio-tools = "*"
 urllib3 = "*"
+numpy = "*"
 
 [requires]
 python_version = "3.11"

Pipfile.lock

@@ -22,9 +22,10 @@ ignore:
     "indykite_sdk/gnostic"
   - "indykite_sdk/identity/helper.py"
   - "indykite_sdk/model/unique_name_identifier.py"
-  - "indykite_sdk/model/email_attachment.py"
   - "indykite_sdk/model/data_access.py"
-  - "indykite_sdk/identity/consent.py"
+  - "indykite_sdk/model/property.py"
+  - "indykite_sdk/model/scope_item.py"
+  - "indykite_sdk/model/audience_item.py"
   - "indykite_sdk/utils"
   - "tests"
   - "examples"

codecov.yaml

@@ -43,6 +43,10 @@ def __init__(self, token_source=None):
                               validate_external_data_resolver_content_type,
                               create_external_data_resolver_config_node,
                               update_external_data_resolver_config_node,
-                              external_data_resolver_config
+                              external_data_resolver_config,
+                              create_entity_matching_pipeline_config_node,
+                              update_entity_matching_pipeline_config_node,
+                              entity_matching_pipeline_config,
+                              validate_entity_matching_status
                               )
     from .create_application_with_agent_credentials import create_application_with_agent_credentials

indykite_sdk/buf/validate/validate_pb2.py

@@ -8,6 +8,7 @@
 from indykite_sdk.model.authorization_policy_config_status import Status
 from indykite_sdk.indykite.config.v1beta1 import model_pb2
 from indykite_sdk.model.token_status import ExternalTokenStatus
+from indykite_sdk.model.entity_matching_status import Status as EntityMatchingStatus
 from indykite_sdk.model.external_data_resolver_config_content_type import ContentType
 import indykite_sdk.utils.logger as logger
 
@@ -564,6 +565,141 @@ def external_data_resolver_config(self,
         return logger.logger_error(exception)
 
 
+def create_entity_matching_pipeline_config_node(self,
+                                              location,
+                                              name,
+                                              display_name,
+                                              description,
+                                              entity_matching_pipeline_config,
+                                              bookmarks=[]):
+    """
+    create entity matching pipeline config node
+    :param self:
+    :param location: string gid id
+    :param name: string pattern: ^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])$
+    :param display_name: string
+    :param description: string
+    :param entity_matching_pipeline_config: EntityMatchingPipelineConfig object
+    :param bookmarks: list of strings with pattern: ^[a-zA-Z0-9_-]{40,}$
+    :return: deserialized CreateConfigNode instance
+    """
+    sys.excepthook = logger.handle_excepthook
+    try:
+        if entity_matching_pipeline_config and not isinstance(
+            entity_matching_pipeline_config,
+            model_pb2.EntityMatchingPipelineConfig):
+            raise TypeError("EntityMatchingPipelineConfig must be an object")
+
+        response = self.stub.CreateConfigNode(
+            pb2.CreateConfigNodeRequest(
+                location=location,
+                name=name,
+                display_name=wrappers.StringValue(value=display_name),
+                description=wrappers.StringValue(value=description),
+                entity_matching_pipeline_config=entity_matching_pipeline_config,
+                bookmarks=bookmarks
+            )
+        )
+    except Exception as exception:
+        return logger.logger_error(exception)
+
+    if not response:
+        return None
+    return CreateConfigNode.deserialize(response)
+
+
+def update_entity_matching_pipeline_config_node(self,
+                                              config_node_id,
+                                              etag,
+                                              display_name,
+                                              description,
+                                              entity_matching_pipeline_config,
+                                              bookmarks=[]):
+    """
+    update entity matching pipeline
+    :param self:
+    :param config_node_id: string gid id
+    :param display_name: string
+    :param description: string
+    :param entity_matching_pipeline_config: EntityMatchingPipelineConfig object
+    :param bookmarks: list of strings with pattern: ^[a-zA-Z0-9_-]{40,}$
+    :return: deserialized UpdateConfigNode instance
+    """
+    sys.excepthook = logger.handle_excepthook
+    try:
+        if entity_matching_pipeline_config and not isinstance(
+            entity_matching_pipeline_config,
+            model_pb2.EntityMatchingPipelineConfig):
+            raise TypeError("EntityMatchingPipelineConfig must be an object")
+        response = self.stub.UpdateConfigNode(
+            pb2.UpdateConfigNodeRequest(
+                id=config_node_id,
+                etag=wrappers.StringValue(value=etag),
+                display_name=wrappers.StringValue(value=display_name),
+                description=wrappers.StringValue(value=description),
+                entity_matching_pipeline_config=entity_matching_pipeline_config,
+                bookmarks=bookmarks
+            )
+        )
+    except Exception as exception:
+        return logger.logger_error(exception)
+
+    if not response:
+        return None
+    return UpdateConfigNode.deserialize(response)
+
+
+def entity_matching_pipeline_config(self,
+                                  node_filter,
+                                  similarity_score_cutoff=None,
+                                  property_mapping_status=None,
+                                  property_mapping_message=None,
+                                  entity_matching_status=None,
+                                  entity_matching_message=None,
+                                  property_mappings=None,
+                                  rerun_interval=None,
+                                  last_run_time=None,
+                                  report_url=None,
+                                  report_type=None):
+
+    """
+    create EntityMatchingPipelineConfig
+    :param self:
+    :param node_filter: EntityMatchingPipelineConfig.NodeFilter object
+    :param similarity_score_cutoff: float
+    :param property_mapping_status: EntityMatchingPipelineConfig.Status object
+    :param property_mapping_message: google.protobuf.StringValue
+    :param entity_matching_status: EntityMatchingPipelineConfig.Status
+    :param entity_matching_message: google.protobuf.StringValue
+    :param property_mappings: array of EntityMatchingPipelineConfig.PropertyMapping
+    :param rerun_interval: string
+    :param last_run_time: google.protobuf.Timestamp
+    :param report_url: google.protobuf.StringValue
+    :param report_type: google.protobuf.StringValue
+    :return: EntityMatchingPipelineConfig object
+    """
+    sys.excepthook = logger.handle_excepthook
+    try:
+        property_mapping_status = self.validate_entity_matching_status(property_mapping_status)
+        entity_matching_status = self.validate_entity_matching_status(entity_matching_status)
+        external_config = model_pb2.EntityMatchingPipelineConfig(
+            node_filter=node_filter,
+            similarity_score_cutoff=similarity_score_cutoff,
+            property_mapping_status=property_mapping_status,
+            property_mapping_message=wrappers.StringValue(value=property_mapping_message),
+            entity_matching_status=entity_matching_status,
+            entity_matching_message=wrappers.StringValue(value=entity_matching_message),
+            property_mappings=property_mappings,
+            rerun_interval=rerun_interval,
+            last_run_time=last_run_time,
+            report_url=wrappers.StringValue(value=report_url),
+            report_type=wrappers.StringValue(value=report_type)
+            )
+        return external_config
+    except Exception as exception:
+        return logger.logger_error(exception)
+
+
 def validate_data_points(self, data_points):
     """
     validate data_points requirement
@@ -641,6 +777,24 @@ def validate_external_data_resolver_content_type(self, content_type):
         return logger.logger_error(exception)
 
 
+def validate_entity_matching_status(self, status):
+    """
+    validate entity matching status requirement
+    :param self:
+    :param status: number
+    :return: status, none  or error
+    """
+    try:
+        if not status:
+            return None
+        statuses = [s.value for s in EntityMatchingStatus]
+        if status and status not in statuses:
+            raise TypeError("status must be a member of EntityMatchingPipelineConfig.Status")
+        return status
+    except Exception as exception:
+        return logger.logger_error(exception)
+
+
 def list_config_node_versions(self, id_config_node):
     """
     list config nodes versions of the specified config node

indykite_sdk/config/__init__.py

@@ -0,0 +1,20 @@
+import sys
+from indykite_sdk.utils import jwt_credentials
+from indykite_sdk.utils.logger import handle_excepthook, logger_error
+
+
+class EntityMatchingClient(object):
+
+    def __init__(self, token_source=None):
+        sys.excepthook = handle_excepthook
+        try:
+            self.channel, self.stub, self.credentials, self.token_source = jwt_credentials.get_credentials(
+                client="entitymatching",
+                token_source=token_source
+            )
+        except Exception as exception:
+            return logger_error(exception)
+
+    # Imported methods
+    from .read_suggested_property_mapping import read_suggested_property_mapping
+    from .run_entity_matching_pipeline import run_entity_matching_pipeline

indykite_sdk/config/config_node.py

@@ -0,0 +1,26 @@
+import sys
+
+from indykite_sdk.indykite.entitymatching.v1beta1 import entity_matching_api_pb2 as pb2
+from indykite_sdk.model.entity_matching import ReadSuggestedPropertyMappingResponse
+import indykite_sdk.utils.logger as logger
+
+
+def read_suggested_property_mapping(self, id):
+    """
+    read suggested property mapping
+    :param self:
+    :param id: confg node GID id
+    :return: deserialized ReadSuggestedPropertyMappingResponse
+    """
+    sys.excepthook = logger.handle_excepthook
+    try:
+        response = self.stub.ReadSuggestedPropertyMapping(
+            pb2.ReadSuggestedPropertyMappingRequest(
+                id=id
+            )
+        )
+        if not response:
+            return None
+        return ReadSuggestedPropertyMappingResponse.deserialize(response)
+    except Exception as exception:
+        return logger.logger_error(exception)

indykite_sdk/entitymatching/__init__.py

@@ -0,0 +1,30 @@
+import sys
+
+from indykite_sdk.indykite.entitymatching.v1beta1 import entity_matching_api_pb2 as pb2
+from indykite_sdk.model.entity_matching import RunEntityMatchingPipelineResponse
+import indykite_sdk.utils.logger as logger
+
+
+def run_entity_matching_pipeline(self, id, similarity_score_cutoff, custom_property_mappings=None):
+    """
+    run entitymatching pipeline
+    :param self:
+    :param id: confg node GID id
+    :param custom_property_mappings array of PropertyMapping
+    :param similarity_score_cutoff float required threshold (in range [0,1])
+    :return: deserialized RunEntityMatchingPipelineResponse
+    """
+    sys.excepthook = logger.handle_excepthook
+    try:
+        response = self.stub.RunEntityMatchingPipeline(
+            pb2.RunEntityMatchingPipelineRequest(
+                id=id,
+                custom_property_mappings=custom_property_mappings,
+                similarity_score_cutoff=similarity_score_cutoff
+            )
+        )
+        if not response:
+            return None
+        return RunEntityMatchingPipelineResponse.deserialize(response)
+    except Exception as exception:
+        return logger.logger_error(exception)