diff --git a/warehouse/ingest-configuration/src/main/resources/config/edge-definitions.xml b/warehouse/ingest-configuration/src/main/resources/config/edge-definitions.xml index 455a5be09a2..a9103aff1be 100644 --- a/warehouse/ingest-configuration/src/main/resources/config/edge-definitions.xml +++ b/warehouse/ingest-configuration/src/main/resources/config/edge-definitions.xml @@ -1235,44 +1235,5 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - - - - - - - PERSON - CHARACTER - SHOW - TO - FROM - EQUIVALENCE - USERNAME - USERID - PAGEID - REVISIONID - CONTRIBUTORID - CONTRIBUTOR - COMMENT - USER - PAGE - REDIRECT - REVISION - - - - - - - - - - CSV_METADATA - TVMAZE_METADATA - WIKI_PAGE_METADATA - - - - diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java index 7c70e7235a4..1c36c0ec8da 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/ProtobufEdgeDataTypeHandler.java @@ -103,9 +103,6 @@ public class ProtobufEdgeDataTypeHandler implements Exten public static final String EDGE_SPRING_CONFIG = "protobufedge.spring.config"; - public static final String EDGE_SPRING_RELATIONSHIPS = "protobufedge.table.relationships"; - public static final String EDGE_SPRING_COLLECTIONS = "protobufedge.table.collections"; - public static final String EDGE_SETUP_FAILURE_POLICY = "protobufedge.setup.default.failurepolicy"; public static final String EDGE_PROCESS_FAILURE_POLICY = "protobufedge.process.default.failurepolicy"; @@ -152,9 +149,6 @@ public class ProtobufEdgeDataTypeHandler implements Exten protected EdgeKeyVersioningCache versioningCache = null; - protected HashSet edgeRelationships = new HashSet<>(); - protected HashSet collectionType = new HashSet<>(); - long futureDelta, pastDelta; long newFormatStartDate; @@ -250,27 +244,13 @@ public void setup(Configuration conf) { registry.put(EDGE_DEFAULT_DATA_TYPE, null); - // HashSet edgeRelationships, collectionType; - - if (ctx.containsBean(EDGE_SPRING_RELATIONSHIPS) && ctx.containsBean(EDGE_SPRING_COLLECTIONS)) { - edgeRelationships.addAll((HashSet) ctx.getBean(EDGE_SPRING_RELATIONSHIPS)); - collectionType.addAll((HashSet) ctx.getBean(EDGE_SPRING_COLLECTIONS)); - } else { - log.error("Edge relationships and or collection types are not configured correctly. Cannot build edge definitions"); - if (setUpFailurePolicy == FailurePolicy.FAIL_JOB) { - throw new RuntimeException("Missing some spring configurations"); - } else { - return; // no edges will be created but the ingest job will continue - } - } - for (Entry entry : registry.entrySet()) { if (ctx.containsBean(entry.getKey())) { EdgeDefinitionConfigurationHelper thing = (EdgeDefinitionConfigurationHelper) ctx.getBean(entry.getKey()); - // Always call init first before getting getting edge defs. This performs validation on the config file + // Always call init first before getting edge defs. This performs validation on the config file // and builds the edge pairs/groups - thing.init(edgeRelationships, collectionType); + thing.init(); edges.put(entry.getKey(), thing); if (thing.getEnrichmentTypeMappings() != null) { @@ -430,14 +410,6 @@ public void setEdges(Map edges) { this.edges = edges; } - public Map> getDisallowlistFieldLookup() { - return disallowlistFieldLookup; - } - - public Map> getDisallowlistValueLookup() { - return disallowlistValueLookup; - } - private boolean isDisallowlistField(String dataType, String fieldName) { if (disallowlistFieldLookup.containsKey(dataType)) { return this.disallowlistFieldLookup.get(dataType).contains(fieldName); @@ -1441,8 +1413,4 @@ public RawRecordMetadata getMetadata() { // TODO Auto-generated method stub return null; } - - public void setVersioningCache(EdgeKeyVersioningCache versioningCache) { - this.versioningCache = versioningCache; - } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDefinitionConfigurationHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDefinitionConfigurationHelper.java index 9c5c4bf2729..42332505c92 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDefinitionConfigurationHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/edge/define/EdgeDefinitionConfigurationHelper.java @@ -1,7 +1,6 @@ package datawave.ingest.mapreduce.handler.edge.define; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; @@ -38,7 +37,7 @@ public void setEdges(List edges) { this.edges = edges; } - public void init(HashSet edgeRelationships, HashSet collectionType) { + public void init() { // Sanity check before we continue validateRequiredVariablesSet(); @@ -50,29 +49,16 @@ public void init(HashSet edgeRelationships, HashSet collectionTy int nPieces = edgeNodes.size(); for (int ii = 0; ii < nPieces - 1; ii++) { for (int jj = ii + 1; jj < nPieces; jj++) { - - if (validateEdgeNode(edgeNodes.get(ii), edgeRelationships, collectionType) - && validateEdgeNode(edgeNodes.get(jj), edgeRelationships, collectionType)) { - - EdgeDefinition edgePair = buildEdgePair(edgeDefinition, edgeNodes.get(ii), edgeNodes.get(jj)); - - realEdges.add(edgePair); - } - + EdgeDefinition edgePair = buildEdgePair(edgeDefinition, edgeNodes.get(ii), edgeNodes.get(jj)); + realEdges.add(edgePair); } } } else if (edgeDefinition.getGroupPairs() != null) { for (EdgeNode group1 : edgeDefinition.getGroupPairs().getGroup1()) { for (EdgeNode group2 : edgeDefinition.getGroupPairs().getGroup2()) { - - if (validateEdgeNode(group1, edgeRelationships, collectionType) && validateEdgeNode(group2, edgeRelationships, collectionType)) { - - EdgeDefinition groupPair = buildEdgePair(edgeDefinition, group1, group2); - - realEdges.add(groupPair); - } - + EdgeDefinition groupPair = buildEdgePair(edgeDefinition, group1, group2); + realEdges.add(groupPair); } } } else { @@ -130,16 +116,6 @@ public EdgeDefinition buildEdgePair(EdgeDefinition edgeDefinition, EdgeNode node return edgePair; } - private boolean validateEdgeNode(EdgeNode node, HashSet edgeRelationships, HashSet collectionType) { - if (edgeRelationships.contains(node.getRelationship()) && collectionType.contains(node.getCollection())) { - return true; - } else { - log.error("Edge Definition in config file does not have a matching edge relationship and collection type for Relationship: " - + node.getRelationship() + " and Collection: " + node.getCollection()); - return false; - } - } - // Sanity checks to make sure configuration was set up properly private void validateRequiredVariablesSet() { diff --git a/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfig.xml b/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfig.xml index 39411fd313c..9069aa4c4a7 100644 --- a/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfig.xml +++ b/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfig.xml @@ -31,21 +31,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -56,29 +52,5 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - - - - - - - - TO - FROM - - - - - - - - - - MY_CSV_DATA - UNKNOWN - - - - diff --git a/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigDirection.xml b/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigDirection.xml index 807093803e2..e7b91e4933e 100644 --- a/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigDirection.xml +++ b/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigDirection.xml @@ -32,21 +32,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -78,21 +74,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -125,21 +117,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -172,21 +160,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -195,27 +179,4 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - - - - - - - TO - FROM - - - - - - - - - - MY_CSV_DATA - UNKNOWN - - - - diff --git a/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigPrecon.xml b/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigPrecon.xml index 879f7b91c23..552ad8d1cb6 100644 --- a/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigPrecon.xml +++ b/warehouse/ingest-core/src/test/resources/config/EdgeSpringConfigPrecon.xml @@ -34,21 +34,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -67,21 +63,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -100,21 +92,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -133,21 +121,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -166,21 +150,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -199,21 +179,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -232,21 +208,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -265,21 +237,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -298,21 +266,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -331,21 +295,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -363,21 +323,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -395,21 +351,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -418,28 +370,5 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - - - - - - - TO - FROM - - - - - - - - - - MY_CSV_DATA - UNKNOWN - - - - diff --git a/warehouse/ingest-csv/src/test/resources/config/EdgeSpringConfig.xml b/warehouse/ingest-csv/src/test/resources/config/EdgeSpringConfig.xml index 680f90bb8d9..8bd77444c2d 100644 --- a/warehouse/ingest-csv/src/test/resources/config/EdgeSpringConfig.xml +++ b/warehouse/ingest-csv/src/test/resources/config/EdgeSpringConfig.xml @@ -31,21 +31,17 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - + - + - + - + @@ -54,30 +50,6 @@ http://www.springframework.org/schema/util/spring-util-4.0.xsd"> - - - - - - - - TO - FROM - - - - - - - - - - MY_CSV_DATA - UNKNOWN - - - - diff --git a/warehouse/ingest-json/src/test/resources/config/ingest/tvmaze-edge-definitions.xml b/warehouse/ingest-json/src/test/resources/config/ingest/tvmaze-edge-definitions.xml index a738431e557..b6376a45b8e 100644 --- a/warehouse/ingest-json/src/test/resources/config/ingest/tvmaze-edge-definitions.xml +++ b/warehouse/ingest-json/src/test/resources/config/ingest/tvmaze-edge-definitions.xml @@ -92,45 +92,5 @@ http://www.springframework.org/schema/beans/spring-beans-4.0.xsd"> - - - - - - - ACTOR - STATUS - CHARACTER - SHOW - TO - FROM - EQUIVALENCE - USERNAME - USERID - PAGEID - REVISIONID - CONTRIBUTORID - CONTRIBUTOR - COMMENT - USER - PAGE - REDIRECT - REVISION - - - - - - - - - - CSV_METADATA - TVMAZE_METADATA - WIKI_PAGE_METADATA - - - -