From c992a6773e95be1c78a2302cf567d12797b7a3f2 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Mon, 20 Jan 2025 10:49:12 +0000 Subject: [PATCH 1/5] add new binary operator docs and update delete elements --- docs/administration-guide/delete-elements.md | 237 ------------------ .../binary-operators.md | 1 + .../koryphe-operators.md | 35 +++ docs/reference/operations-guide/core.md | 164 +++++++++++- mkdocs.yml | 1 - 5 files changed, 194 insertions(+), 244 deletions(-) delete mode 100644 docs/administration-guide/delete-elements.md diff --git a/docs/administration-guide/delete-elements.md b/docs/administration-guide/delete-elements.md deleted file mode 100644 index a49cd3650d..0000000000 --- a/docs/administration-guide/delete-elements.md +++ /dev/null @@ -1,237 +0,0 @@ -# Delete Elements Operation - -In Gaffer 2.3.0 a DeleteElements operation was added to allow users -to delete specific elements from their graph instance. - -!!! warning - Deleting elements is not reversible. Use of this operation should be - limited and users should run test a Operation Chain which extracts any elements - to be deleted prior to running any delete operations. - -We will use the following graph to demonstrate how to delete elements from a -Gaffer graph. - -```mermaid -graph LR - A(["Person - - ID: John"]) - -- - "Created - weight: 0.2" - --> - B(["Software - - ID: 1"]) - A - -- - "Created - weight: 0.6" - --> - C(["Software - - ID: 2"]) -``` - -To delete elements from a graph, users can simply add a `DeleteElements.Builder` to their -Operation Chain. This will take as its input the output of any operations run prior to it. - - -!!! example "" - If a user gets the edge between John and Software 2 in a query and follows this - up with a delete, then this edge will be removed leaving the three entities and a single edge. - - === "Java" - - ```java - final OperationChain deleteElementsChain = new OperationChain.Builder() - .first(new GetElements.Builder() - .input(new EdgeSeed("John", "2", DirectedType.EITHER)) - .view(new View.Builder().edge("created").build()) - .build()) - .then(new DeleteElements()) - .build(); - - graph.execute(deleteElementsChain, new User()); - ``` - - === "JSON" - - ```json - { - "class" : "OperationChain", - "operations" : [{ - "class": "GetElements", - "input": [{ - "class": "EdgeSeed", - "source": "John", - "destination": "2", - "directedType": "EITHER" - }], - "view": { - "edges": "created" - } - }, - { - "class" : "DeleteElements" - }] - } - ``` - - === "Python" - - ```python - g.OperationChain( - operations=[ - g.GetElements( - input = [g.EdgeSeed(source="John", destination="2", directedType="Either")], - view = g.View( - edges=[ - g.ElementDefinition( - group="created" - ) - ] - ) - ), - g.DeleteElements() - ] - ) - ``` - - Results: - ```JSON - Entity[vertex="John",group="person"] - Edge[source="John",destination="1",directed=true,matchedVertex=SOURCE,group="created",properties=Properties[weight=0.2]] - Entity[vertex="1",group="software"] - Entity[vertex="2",group="software"] - ``` - -If a user wishes to remove an entity and its associated edges, this is done by -querying for that entity with no filters. - -!!! example "" - If a user gets the John entity in a query and follows this up with a delete, - then the John entity is removed and so are the edges for John -created-> 1 - and John-created-> 2. - - === "Java" - - ```java - final OperationChain deleteElementsChain = new OperationChain.Builder() - .first(new GetElements.Builder() - .input(new EntitySeed("John")) - .build()) - .then(new DeleteElements()) - .build(); - - graph.execute(deleteElementsChain, new User()); - ``` - - === "JSON" - - ```json - { - "class" : "OperationChain", - "operations" : [{ - "class": "GetElements", - "input": [{ - "class": "EntitySeed", - "vertex": "John" - }] - }, - { - "class" : "DeleteElements" - } - ] - } - ``` - - === "Python" - - ```python - g.OperationChain( - operations=[ - g.GetElements( - input = [g.EntitySeed(vertex="John")] - ), - g.DeleteElements() - ] - ) - ``` - - Results: - ```JSON - Entity[vertex="1",group="software"] - Entity[vertex="2",group="software"] - ``` - -If a user wishes to remove an entity but leave any edges associated with the entity's vertex, -this is done by querying for that entity with a filter. - -!!! example "" - If a user gets the John entity in a query but uses a `View` that filters for just entities, then the - delete will leave the associated edges and only delete the John entity. - - === "Java" - - ```java - final OperationChain deleteElementsChain = new OperationChain.Builder() - .first(new GetElements.Builder() - .input(new EntitySeed("John")) - .view(new View.Builder().entity("person").build()) - .build()) - .then(new DeleteElements()) - .build(); - - graph.execute(deleteElementsChain, new User()); - ``` - - === "JSON" - - ```json - { - "class" : "OperationChain", - "operations" : [{ - "class": "GetElements", - "input": [{ - "class": "EntitySeed", - "vertex": "John" - }], - "view": { - "entities": "person" - } - }, - { - "class" : "DeleteElements" - } - ] - } - ``` - - === "Python" - - ```python - g.OperationChain( - operations=[ - g.GetElements( - input=[g.EntitySeed(vertex="John")], - view=g.View( - entities=[ - g.ElementDefinition( - group="person" - ) - ] - ) - ), - g.DeleteElements() - ] - ) - ``` - - Results: - ```JSON - Edge[source="John",destination="1",directed=true,matchedVertex=SOURCE,group="created",properties=Properties[weight=0.2]] - Edge[source="John",destination="2",directed=true,matchedVertex=SOURCE,group="created",properties=Properties[weight=0.6]] - Entity[vertex="1",group="software"] - Entity[vertex="2",group="software"] - ``` diff --git a/docs/reference/binary-operators-guide/binary-operators.md b/docs/reference/binary-operators-guide/binary-operators.md index b7c5d8103b..c97d3b3d85 100644 --- a/docs/reference/binary-operators-guide/binary-operators.md +++ b/docs/reference/binary-operators-guide/binary-operators.md @@ -18,6 +18,7 @@ Binary Operator | Origin [`impl.binaryoperator.And`](koryphe-operators.md#and) | Koryphe [`impl.binaryoperator.CollectionConcat`](koryphe-operators.md#collectionconcat) | Koryphe [`impl.binaryoperator.CollectionIntersect`](koryphe-operators.md#collectionintersect) | Koryphe +[`impl.binaryoperator.IterableMerge`](koryphe-operators.md#iterablemerge) | Koryphe [`impl.binaryoperator.First`](koryphe-operators.md#first) | Koryphe `impl.binaryoperator.Last` | Koryphe [`impl.binaryoperator.Max`](koryphe-operators.md#max) | Koryphe diff --git a/docs/reference/binary-operators-guide/koryphe-operators.md b/docs/reference/binary-operators-guide/koryphe-operators.md index c9c1e52581..7d407afaf8 100644 --- a/docs/reference/binary-operators-guide/koryphe-operators.md +++ b/docs/reference/binary-operators-guide/koryphe-operators.md @@ -348,6 +348,41 @@ Input type: `java.util.Collection` java.util.ArrayList | [test1] and null | java.util.ArrayList | [test1] java.util.HashSet | [a, b] and [b, c] | java.util.HashSet | [b] +## IterableMerge + +Returns a ChainedIterable of two Iterables. [Javadoc]() + +Input type: `java.lang.Iterable` + +??? example "Example IterableMerge" + + === "Java" + + ``` java + final IterableMerge iterableMerge = new IterableMerge(); + ``` + + === "JSON" + + ``` json + { + "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.IterableMerge" + } + ``` + + === "Python" + + ``` python + g.IterableMerge() + ``` + + Example inputs: + + Input Type | Inputs | Result Type | Results + ---------- | ------ | ----------- | ------- + java.util.ArrayList | [test1] and [test2, test3] | uk.gov.gchq.koryphe.util.ChainedIterable | [test1, test2, test3] + java.util.HashSet | [a, b] and [b, c] | uk.gov.gchq.koryphe.util.ChainedIterable | [a, b, b, c] + ## StringConcat Concatenates 2 strings. [Javadoc](https://gchq.github.io/koryphe/uk/gov/gchq/koryphe/impl/binaryoperator/StringConcat.html) diff --git a/docs/reference/operations-guide/core.md b/docs/reference/operations-guide/core.md index 80451db5b8..e67d67ad7d 100644 --- a/docs/reference/operations-guide/core.md +++ b/docs/reference/operations-guide/core.md @@ -108,16 +108,22 @@ Deletes Elements from a graph. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gc Note that this operation does not return any response. +!!! warning + Deleting elements is not reversible. Use of this operation should be + limited and users should run test a Operation Chain which extracts any elements + to be deleted prior to running any delete operations. + ??? example "Example deleting an entity and edge" - Deleting entity '6' and its edge to 5. + Deleting entity '5' and its edge to 2. A user can remove an entity and its associated edges by + simply using a GetElements with an entity seed and no filtering. === "Java" ``` java final OperationChain deleteElementsChain = new OperationChain.Builder() .first(new GetElements.Builder() - .input(new EntitySeed(6)) + .input(new EntitySeed(5)) .build()) .then(new DeleteElements()) .build(); @@ -134,7 +140,7 @@ Note that this operation does not return any response. "class": "GetElements", "input": [{ "class": "EntitySeed", - "vertex": 6, + "vertex": 5, }] }, { @@ -148,13 +154,13 @@ Note that this operation does not return any response. ``` python g.OperationChain( operations=[ - g.GetElements(input=[g.EntitySeed(vertex=6)]), + g.GetElements(input=[g.EntitySeed(vertex=5)]), g.DeleteElements() ] ) ``` - Results: + Graph following the removal of the entity and associated edges: ``` mermaid graph TD @@ -162,7 +168,153 @@ Note that this operation does not return any response. 1 --> 4 2 --> 3 2 --> 4 - 2 --> 5 + 3 --> 4 + ``` + +??? example "Example deleting only an edge" + + Using filtering, users can target a single edge and leave the associated entities + from either end. + + === "Java" + + ``` java + final OperationChain deleteElementsChain = new OperationChain.Builder() + .first(new GetElements.Builder() + .input(new EdgeSeed(2, 5, DirectedType.EITHER)) + .view(new View.Builder().edge("edge").build()) + .build()) + .then(new DeleteElements()) + .build(); + + graph.execute(deleteElementsChain, new User()); + ``` + + === "JSON" + + ``` json + { + "class" : "OperationChain", + "operations" : [{ + "class": "GetElements", + "input": [{ + "class": "EdgeSeed", + "source": 2, + "destination": 5, + "directedType": "EITHER" + }], + "view": { + "edges": { + "edge": {} + } + } + }, + { + "class" : "DeleteElements" + }] + } + ``` + + === "Python" + + ``` python + g.OperationChain( + operations=[ + g.GetElements(input=[g.EdgeSeed(source=2, destination=5, directedType="EITHER")], + view = g.View( + edges=[ + g.ElementDefinition(group="edge") + ] + )), + g.DeleteElements() + ] + ) + ``` + + Graph following the removal of a single edge, leaving all entities: + + ``` mermaid + graph TD + 1 --> 2 + 1 --> 4 + 2 --> 3 + 2 --> 4 + 5 + 3 --> 4 + ``` + +??? example "Example deleting an entity" + + Deleting entity '5' but leaving all edges associated with it. This will leave 'dangling' or 'orphan' + edges where there is no entity associated with the vertex on one end. + + === "Java" + + ``` java + final OperationChain deleteElementsChain = new OperationChain.Builder() + .first(new GetElements.Builder() + .input(new EntitySeed(5)) + .view(new View.Builder().entity("person").build()) + .build()) + .then(new DeleteElements()) + .build(); + + graph.execute(deleteElementsChain, new User()); + ``` + + === "JSON" + + ``` json + { + "class" : "OperationChain", + "operations" : [{ + "class": "GetElements", + "input": [{ + "class": "EntitySeed", + "vertex": 5, + }], + "view": { + "entities": { + "entity": {} + } + } + }, + { + "class" : "DeleteElements" + }] + } + ``` + + === "Python" + + ``` python + g.OperationChain( + operations=[ + g.GetElements( + input=[g.EntitySeed(vertex=5)], + view = g.View( + entities=[ + g.ElementDefinition( + group="entity" + ) + ] + ) + ), + g.DeleteElements() + ] + ) + ``` + + Graph following the removal of an entity. There will be the edge 2 --> 5 but + it will only be a vertex: + + ``` mermaid + graph TD + 1 --> 2 + 1 --> 4 + 2 --> 3 + 2 --> 4 + 2 --> 5(5, vertex-only) 3 --> 4 ``` diff --git a/mkdocs.yml b/mkdocs.yml index 875558a19b..64c812558b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -171,7 +171,6 @@ nav: - 'Named Views': 'administration-guide/named-views.md' - 'Operation Score': 'administration-guide/operation-score.md' - 'Job Tracker': 'administration-guide/job-tracker.md' - - 'Delete Operation': 'administration-guide/delete-elements.md' - Security: - 'Security Guide': 'administration-guide/security/security-guide.md' - 'User Control': 'administration-guide/security/user-control.md' From 93b80c1c8016a3437b4158cdead36f193f512bed Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:55:46 +0000 Subject: [PATCH 2/5] update output --- docs/reference/operations-guide/core.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/reference/operations-guide/core.md b/docs/reference/operations-guide/core.md index e67d67ad7d..a2b171b31f 100644 --- a/docs/reference/operations-guide/core.md +++ b/docs/reference/operations-guide/core.md @@ -106,7 +106,7 @@ Adds elements to a graph. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/ga Deletes Elements from a graph. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/operation/impl/delete/DeleteElements.html) -Note that this operation does not return any response. +This operation will simply return a count of the number of elements deleted. !!! warning Deleting elements is not reversible. Use of this operation should be @@ -171,6 +171,11 @@ Note that this operation does not return any response. 3 --> 4 ``` + Output: + ``` + 2 + ``` + ??? example "Example deleting only an edge" Using filtering, users can target a single edge and leave the associated entities @@ -243,6 +248,11 @@ Note that this operation does not return any response. 3 --> 4 ``` + Output: + ``` + 1 + ``` + ??? example "Example deleting an entity" Deleting entity '5' but leaving all edges associated with it. This will leave 'dangling' or 'orphan' @@ -318,6 +328,11 @@ Note that this operation does not return any response. 3 --> 4 ``` + Output: + ``` + 1 + ``` + ## Aggregate The Aggregate operation would normally be used in an Operation Chain to aggregate the results of a previous operation. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/operation/impl/function/Aggregate.html) From 08962742bc438107f40001c20269e041bbc4b65f Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:51:18 +0000 Subject: [PATCH 3/5] fmt --- docs/administration-guide/gaffer-stores/accumulo-store.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/administration-guide/gaffer-stores/accumulo-store.md b/docs/administration-guide/gaffer-stores/accumulo-store.md index df7ae7760f..5ef289c884 100644 --- a/docs/administration-guide/gaffer-stores/accumulo-store.md +++ b/docs/administration-guide/gaffer-stores/accumulo-store.md @@ -219,6 +219,7 @@ The Accumulo Store also provides a utility [AddUpdateTableIterator](https://gith to help with migrations - updating to new versions of Gaffer or updating your schema. The following changes to your schema are allowed: + - add new groups - add new non-groupBy properties (including visibility and timestamp), but they must go after the other properties - rename properties @@ -227,6 +228,7 @@ The following changes to your schema are allowed: - change descriptions But, you cannot do the following: + - rename groups - remove any properties (groupBy, non-groupBy, visibility or timestamp) - add new groupBy properties @@ -332,7 +334,7 @@ setauths -u root -s vis1,vis2,publicVisibility,privateVisibility,public,private You may notice that sometimes `MatchedVertex` is included on edges when you might not be expecting it. When you seed with a mixture of EdgeSeeds and EntitySeeds, `MatchedVertex` will always be included on edges whether they were matched by a vertex or not. In this case `MatchedVertex` will always equal `SOURCE`. -This is a peculiarity of the Accumulo store. +This is a peculiarity of the Accumulo store. !!! example "Example Query" ``` mermaid @@ -389,4 +391,3 @@ This is a peculiarity of the Accumulo store. ] ``` The 1 -> 2 edge has MatchedVertex=SOURCE even though the source wasn't matched by an EntitySeed. - \ No newline at end of file From 8c1fa1dcacfc8be97d48f87cdde15681c0a834b0 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Thu, 30 Jan 2025 14:23:05 +0000 Subject: [PATCH 4/5] add missing link --- docs/reference/binary-operators-guide/koryphe-operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/binary-operators-guide/koryphe-operators.md b/docs/reference/binary-operators-guide/koryphe-operators.md index 7d407afaf8..44567ff482 100644 --- a/docs/reference/binary-operators-guide/koryphe-operators.md +++ b/docs/reference/binary-operators-guide/koryphe-operators.md @@ -350,7 +350,7 @@ Input type: `java.util.Collection` ## IterableMerge -Returns a ChainedIterable of two Iterables. [Javadoc]() +Returns a ChainedIterable of two Iterables. [Javadoc](https://gchq.github.io/koryphe/uk/gov/gchq/koryphe/impl/binaryoperator/IterableMerge.html) Input type: `java.lang.Iterable` From 2d9c7abc9a02c06323ea1af4b7deb61b83a02e72 Mon Sep 17 00:00:00 2001 From: cn337131 <141730190+cn337131@users.noreply.github.com> Date: Tue, 4 Feb 2025 10:24:43 +0000 Subject: [PATCH 5/5] address comments Co-authored-by: wb36499 <166839644+wb36499@users.noreply.github.com> --- docs/reference/operations-guide/core.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/operations-guide/core.md b/docs/reference/operations-guide/core.md index a2b171b31f..239d68ee13 100644 --- a/docs/reference/operations-guide/core.md +++ b/docs/reference/operations-guide/core.md @@ -110,12 +110,12 @@ This operation will simply return a count of the number of elements deleted. !!! warning Deleting elements is not reversible. Use of this operation should be - limited and users should run test a Operation Chain which extracts any elements + limited and users should run a test Operation Chain which extracts any elements to be deleted prior to running any delete operations. ??? example "Example deleting an entity and edge" - Deleting entity '5' and its edge to 2. A user can remove an entity and its associated edges by + Deleting entity '5' and its edge to entity '2'. A user can remove an entity and its associated edges by simply using a GetElements with an entity seed and no filtering. === "Java"