Skip to content

Commit 2ba6fc5

Browse files
thjaecklethjaeckle
authored
Merge pull request #2076 from beyonnex-io/feature/pre-defined-extra-fields
#2072: add configuration option to configure pre-defined "extraFields"
2 parents fa2b4c9 + a2b35da commit 2ba6fc5

File tree

42 files changed

+2001
-121
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2001
-121
lines changed

‎base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java

+35
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,41 @@ public enum DittoHeaderDefinition implements HeaderDefinition {
527527
JsonObject.class,
528528
false,
529529
true,
530+
HeaderValueValidators.getJsonObjectValidator()),
531+
532+
/**
533+
* Internal header containing the pre-defined configured {@code extraFields} as list of jsonPointers for the
534+
* emitted thing event.
535+
*
536+
* @since 3.7.0
537+
*/
538+
PRE_DEFINED_EXTRA_FIELDS("ditto-pre-defined-extra-fields",
539+
JsonArray.class,
540+
false,
541+
false,
542+
HeaderValueValidators.getJsonArrayValidator()),
543+
544+
/**
545+
* Internal header containing the pre-defined configured {@code extraFields} as keys and the allowed "read subjects"
546+
* as array of stings - defining which "auth subjects" are allowed to read which pre-defined extra field.
547+
*
548+
* @since 3.7.0
549+
*/
550+
PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT("ditto-pre-defined-extra-fields-read-grant",
551+
JsonObject.class,
552+
false,
553+
false,
554+
HeaderValueValidators.getJsonObjectValidator()),
555+
556+
/**
557+
* Internal header containing pre-defined {@code extraFields} as JSON object sent along for emitted thing event.
558+
*
559+
* @since 3.7.0
560+
*/
561+
PRE_DEFINED_EXTRA_FIELDS_OBJECT("ditto-pre-defined-extra-fields-object",
562+
JsonObject.class,
563+
false,
564+
false,
530565
HeaderValueValidators.getJsonObjectValidator());
531566

532567
/**

‎base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java

+27
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,16 @@ public final class ImmutableDittoHeadersTest {
137137
.set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar")
138138
.build();
139139

140+
private static final JsonArray KNOWN_PRE_DEFINED_EXTRA_FIELDS = JsonArray.newBuilder()
141+
.add("foo:bar:123")
142+
.build();
143+
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT = JsonObject.newBuilder()
144+
.set("/definition", "known:subject")
145+
.build();
146+
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT = JsonObject.newBuilder()
147+
.set("definition", "foo:bar:123")
148+
.build();
149+
140150

141151
static {
142152
KNOWN_METADATA_HEADERS = MetadataHeaders.newInstance();
@@ -205,6 +215,12 @@ public void settingAllKnownHeadersWorksAsExpected() {
205215
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION))
206216
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP))
207217
.putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString())
218+
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
219+
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString())
220+
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
221+
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString())
222+
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
223+
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString())
208224
.build();
209225

210226
assertThat(underTest).isEqualTo(expectedHeaderMap);
@@ -535,6 +551,11 @@ public void toJsonReturnsExpected() {
535551
.set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION)
536552
.set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString())
537553
.set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS)
554+
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(), KNOWN_PRE_DEFINED_EXTRA_FIELDS)
555+
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
556+
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT)
557+
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
558+
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT)
538559
.build();
539560

540561
final Map<String, String> allKnownHeaders = createMapContainingAllKnownHeaders();
@@ -774,6 +795,12 @@ private static Map<String, String> createMapContainingAllKnownHeaders() {
774795
result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION));
775796
result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP));
776797
result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString());
798+
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
799+
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString());
800+
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
801+
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString());
802+
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
803+
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString());
777804

778805
return result;
779806
}

‎bom/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
<properties>
2929
<scala.version>2.13</scala.version> <!-- for scala libraries the scala version is used in their artifactId -->
30-
<scala.full.version>2.13.15</scala.full.version>
30+
<scala.full.version>2.13.16</scala.full.version>
3131
<scala-parser-combinators.version>1.1.2</scala-parser-combinators.version>
3232
<scala-java8-compat.version>1.0.2</scala-java8-compat.version>
3333

‎deployment/helm/ditto/Chart.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ description: |
1616
A digital twin is a virtual, cloud based, representation of his real world counterpart
1717
(real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
1818
type: application
19-
version: 3.6.9 # chart version is effectively set by release-job
19+
version: 3.6.10 # chart version is effectively set by release-job
2020
appVersion: 3.6.9
2121
keywords:
2222
- iot-chart

‎deployment/helm/ditto/service-config/things-extension.conf.tpl

+40
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,46 @@ ditto {
4444
"{{$header}}"
4545
{{- end }}
4646
]
47+
pre-defined-extra-fields = [
48+
{{- range $index, $extraFieldConfig := .Values.things.config.event.preDefinedExtraFields }}
49+
{
50+
namespaces = [
51+
{{- range $index, $namespace := $extraFieldConfig.namespaces }}
52+
"{{$namespace}}"
53+
{{- end }}
54+
]
55+
{{- if $extraFieldConfig.condition }}
56+
condition = "{{$extraFieldConfig.condition}}"
57+
{{- end }}
58+
extra-fields = [
59+
{{- range $index, $extraField := $extraFieldConfig.extraFields }}
60+
"{{$extraField}}"
61+
{{- end }}
62+
]
63+
}
64+
{{- end }}
65+
]
66+
}
67+
message {
68+
pre-defined-extra-fields = [
69+
{{- range $index, $extraFieldConfig := .Values.things.config.message.preDefinedExtraFields }}
70+
{
71+
namespaces = [
72+
{{- range $index, $namespace := $extraFieldConfig.namespaces }}
73+
"{{$namespace}}"
74+
{{- end }}
75+
]
76+
{{- if $extraFieldConfig.condition }}
77+
condition = "{{$extraFieldConfig.condition}}"
78+
{{- end }}
79+
extra-fields = [
80+
{{- range $index, $extraField := $extraFieldConfig.extraFields }}
81+
"{{$extraField}}"
82+
{{- end }}
83+
]
84+
}
85+
{{- end }}
86+
]
4787
}
4888
}
4989

‎deployment/helm/ditto/values.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -1059,6 +1059,27 @@ things:
10591059
interval: 15m
10601060
# the threshold after how many changes to a Thing to do a snapshot
10611061
threshold: 50
1062+
# event contains configuration related to e.g. publishing of thing events
1063+
event:
1064+
# preDefinedExtraFields contains pre-defined (configured) extraFields to send along all thing (change) events
1065+
preDefinedExtraFields: []
1066+
# - namespaces:
1067+
# - "org.eclipse.ditto*"
1068+
# condition: "eq(attributes/foo,'RQL condition')"
1069+
# extraFields:
1070+
# - "definition"
1071+
# - "attributes/serial"
1072+
# message contains configuration related to distributing thing messages
1073+
message:
1074+
# preDefinedExtraFields contains pre-defined (configured) extraFields to send along all thing messages
1075+
preDefinedExtraFields: []
1076+
# - namespaces:
1077+
# - "namespace1"
1078+
# - "namespace2"
1079+
# condition: "eq(attributes/foo,'RQL condition')"
1080+
# extra-fields:
1081+
# - "definition"
1082+
# - "attributes/serial"
10621083
# entityCreation by default, Ditto allows anyone to create a new entity (thing in this case) in any namespace.
10631084
# However, this behavior can be customized, and the ability to create new entities can be restricted:
10641085
entityCreation:

‎documentation/src/main/resources/pages/ditto/installation-operating.md

+80
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,86 @@ entities (things/policies) and no-one other:
385385

386386
These system properties would have to be configured for the "things" and "policies" services.
387387

388+
## Configuring pre-defined extra fields
389+
390+
Starting with Ditto 3.7.0, it is possible to configure [enrichment of `extraFields`](basic-enrichment.html) statically
391+
via the configuration of the Ditto "things" service.
392+
393+
The benefit of doing this statically is a Ditto internal optimization to reduce internal traffic between Ditto services.
394+
By default, Ditto will internally do an additional roundtrip from an "edge" service ("gateway" or "connectivity") to the
395+
"things" service in order to retrieve configured `extraFields` (of a [managed connection](basic-connections.html#target-topics-and-enrichment) or
396+
e.g. a [WebSocket session](httpapi-protocol-bindings-websocket.html#enrichment)).
397+
398+
Those retrieved `extraFields` are additionally cached, so also require some memory as well.
399+
400+
If for a Ditto installation the `extraFields` are known upfront and will not change dynamically, it is possible to configure
401+
them in the [things.conf](https://github.com/eclipse/ditto/blob/master/things/service/src/main/resources/things.conf).
402+
403+
This configuration is something for power operators of Ditto, needing to reduce resources and improving resiliency by
404+
reducing internal lookups.
405+
406+
### Pre-defined extra fields configuration
407+
408+
The configuration can be done for:
409+
* events: Thing Events emitted to subscriber
410+
* messages: Thing Messages forwarded by Ditto to message subscribers
411+
412+
Available options:
413+
* `pre-defined-extra-fields`: a list of pre-defined extra fields configurations
414+
* `namespaces`: a list of namespaces for which the configuration applies
415+
* if this list is empty, the configuration applies to all namespaces
416+
* the entries support wildcards (`*` matches any number of characters, `?` matches exactly one character)
417+
* `condition`: a [RQL condition](basic-rql.html) to check if the extra fields should be added
418+
* `extra-fields`: a list of extra fields (as JsonPointers) to proactively add for all matching `namespaces` and `condition` combinations
419+
420+
Example configuration:
421+
```hocon
422+
ditto {
423+
things {
424+
thing {
425+
event {
426+
pre-defined-extra-fields = [
427+
{
428+
namespaces = []
429+
condition = "exists(definition)"
430+
extra-fields = [
431+
"definition"
432+
]
433+
},
434+
{
435+
namespaces = [
436+
"org.eclipse.ditto.lamps"
437+
]
438+
extra-fields = [
439+
"attributes/manufacturer",
440+
"attributes/serial"
441+
]
442+
}
443+
]
444+
}
445+
446+
message {
447+
pre-defined-extra-fields = [
448+
{
449+
namespaces = []
450+
condition = "exists(definition)"
451+
extra-fields = [
452+
"definition"
453+
]
454+
}
455+
]
456+
//...
457+
}
458+
```
459+
460+
The above example configuration would always proactively send the `definition` of all Things (if it exists) in the published events
461+
and all messages.
462+
If a consumer of events or messages is interested in this `extraField`, this would not lead to an additional internal
463+
lookup in Ditto and save an internal roundtrip + caching of the result.
464+
465+
For the namespace `org.eclipse.ditto.lamps` there would even be some defined attributes pre-defined to be available as
466+
`extraFields` without the need for another internal lookup.
467+
388468
## Limiting Indexed Fields
389469

390470
The default behavior of Ditto is to index the complete JSON of a thing, which includes all its attributes and features. This may not be desired behavior for certain use cases:

0 commit comments

Comments
 (0)