#1650 provide WoT TM validation

* restructuring of "ditto-wot" module to enable re-usability of non-pekko/non-Ditto specifics
* adding "validator" concept and first sample implementation (WIP)

Signed-off-by: Thomas Jäckle <[email protected]>

Author: thjaeckle

base/model/src/main/java/org/eclipse/ditto/base/model/signals/commands/streaming/SubscribeForPersistedEvents.java

@@ -384,11 +384,6 @@ protected void appendPayload(final JsonObjectBuilder jsonObjectBuilder,
         getFilter().ifPresent(theFilter -> jsonObjectBuilder.set(JsonFields.FILTER, theFilter));
     }
 
-    @Override
-    public String getTypePrefix() {
-        return TYPE_PREFIX;
-    }
-
     @Override
     public SubscribeForPersistedEvents setDittoHeaders(final DittoHeaders dittoHeaders) {
         return new SubscribeForPersistedEvents(entityId, resourcePath, fromHistoricalRevision, toHistoricalRevision,

base/service/pom.xml

@@ -38,6 +38,10 @@
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-internal-utils-metrics</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.ditto</groupId>
+            <artifactId>ditto-internal-utils-metrics-service</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-internal-utils-tracing</artifactId>

base/service/src/main/java/org/eclipse/ditto/base/service/DittoService.java

@@ -47,7 +47,7 @@
 import org.eclipse.ditto.internal.utils.config.raw.RawConfigSupplier;
 import org.eclipse.ditto.internal.utils.health.status.StatusSupplierActor;
 import org.eclipse.ditto.internal.utils.metrics.config.MetricsConfig;
-import org.eclipse.ditto.internal.utils.metrics.prometheus.PrometheusReporterRoute;
+import org.eclipse.ditto.internal.utils.metrics.service.prometheus.PrometheusReporterRoute;
 import org.eclipse.ditto.internal.utils.tracing.DittoTracing;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

bom/pom.xml

@@ -27,7 +27,7 @@
 
     <properties>
         <scala.version>2.13</scala.version> <!-- for scala libraries the scala version is used in their artifactId -->
-        <scala.full.version>2.13.12</scala.full.version>
+        <scala.full.version>2.13.14</scala.full.version>
         <scala-parser-combinators.version>1.1.2</scala-parser-combinators.version>
         <scala-java8-compat.version>1.0.2</scala-java8-compat.version>
 
@@ -37,6 +37,7 @@
         <!-- ### Compile dependencies versions -->
         <minimal-json.version>0.9.5</minimal-json.version>
         <jackson-bom.version>2.16.1</jackson-bom.version>
+        <json-schema-validator.version>1.4.0</json-schema-validator.version>
         <typesafe-config.version>1.4.3</typesafe-config.version>
         <ssl-config-core.version>0.6.1</ssl-config-core.version>
         <kafka-client.version>3.6.1</kafka-client.version>
@@ -75,7 +76,7 @@
         <janino.version>3.1.11</janino.version>
 
         <!-- ### Metrics and Tracing -->
-        <kamon.version>2.7.0</kamon.version>
+        <kamon.version>2.7.1</kamon.version>
 
         <jsr305.version>3.0.2</jsr305.version>
 
@@ -130,6 +131,22 @@
                 <scope>import</scope>
             </dependency>
 
+            <dependency>
+                <groupId>com.networknt</groupId>
+                <artifactId>json-schema-validator</artifactId>
+                <version>${json-schema-validator.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>com.fasterxml.jackson.dataformat</groupId>
+                        <artifactId>jackson-dataformat-yaml</artifactId>
+                    </exclusion>
+                    <exclusion>
+                        <groupId>org.apache.commons</groupId>
+                        <artifactId>commons-lang3</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>com.typesafe</groupId>
                 <artifactId>config</artifactId>
@@ -538,11 +555,21 @@
                 <artifactId>ditto-rql-query</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.ditto</groupId>
+                <artifactId>ditto-wot-api</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.ditto</groupId>
                 <artifactId>ditto-wot-model</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.ditto</groupId>
+                <artifactId>ditto-wot-validation</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.ditto</groupId>
                 <artifactId>ditto-wot-integration</artifactId>
@@ -616,6 +643,11 @@
                 <artifactId>ditto-internal-utils-http</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.ditto</groupId>
+                <artifactId>ditto-internal-utils-json</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.ditto</groupId>
                 <artifactId>ditto-internal-utils-jwt</artifactId>
@@ -666,6 +698,11 @@
                 <artifactId>ditto-internal-utils-metrics</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.ditto</groupId>
+                <artifactId>ditto-internal-utils-metrics-service</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.eclipse.ditto</groupId>
                 <artifactId>ditto-internal-utils-extension</artifactId>

connectivity/service/pom.xml

@@ -14,12 +14,12 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
     <parent>
         <artifactId>ditto-connectivity</artifactId>
         <groupId>org.eclipse.ditto</groupId>
         <version>${revision}</version>
     </parent>
-    <modelVersion>4.0.0</modelVersion>
 
     <artifactId>ditto-connectivity-service</artifactId>
     <name>Eclipse Ditto :: Connectivity :: Service</name>
@@ -55,6 +55,10 @@
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-wot-model</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.ditto</groupId>
+            <artifactId>ditto-wot-validation</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.eclipse.ditto</groupId>
@@ -69,6 +73,10 @@
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-internal-models-signalenrichment</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.ditto</groupId>
+            <artifactId>ditto-internal-utils-http</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-internal-utils-persistence</artifactId>

connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/DefaultHttpPushConfig.java

@@ -19,10 +19,10 @@
 
 import javax.annotation.concurrent.Immutable;
 
-import org.eclipse.ditto.base.service.config.http.DefaultHttpProxyConfig;
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
 import org.eclipse.ditto.internal.utils.config.ConfigWithFallback;
 import org.eclipse.ditto.internal.utils.config.ScopedConfig;
+import org.eclipse.ditto.internal.utils.http.config.DefaultHttpProxyConfig;
+import org.eclipse.ditto.internal.utils.http.config.HttpProxyConfig;
 
 import com.typesafe.config.Config;
 

connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/config/HttpPushConfig.java

@@ -16,8 +16,8 @@
 import java.util.List;
 import java.util.Map;
 
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
 import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
+import org.eclipse.ditto.internal.utils.http.config.HttpProxyConfig;
 
 import com.typesafe.config.Config;
 

connectivity/service/src/main/java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpPushClientActor.java

@@ -39,7 +39,6 @@
 import org.apache.pekko.stream.javadsl.Sink;
 import org.apache.pekko.stream.javadsl.Source;
 import org.eclipse.ditto.base.model.headers.DittoHeaders;
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
 import org.eclipse.ditto.connectivity.model.Connection;
 import org.eclipse.ditto.connectivity.model.signals.commands.modify.TestConnection;
 import org.eclipse.ditto.connectivity.service.config.HttpPushConfig;
@@ -50,6 +49,7 @@
 import org.eclipse.ditto.connectivity.service.messaging.internal.ssl.SSLContextCreator;
 import org.eclipse.ditto.connectivity.service.messaging.monitoring.ConnectionMonitor;
 import org.eclipse.ditto.connectivity.service.messaging.monitoring.logs.InfoProviderFactory;
+import org.eclipse.ditto.internal.utils.config.http.HttpProxyBaseConfig;
 
 import com.typesafe.config.Config;
 
@@ -215,7 +215,7 @@ private CompletionStage<Status.Status> testSSL(final Connection connection, fina
     }
 
     private CompletionStage<Status.Status> connectViaProxy(final String hostWithoutLookup, final int port) {
-        final HttpProxyConfig httpProxyConfig = this.httpPushConfig.getHttpProxyConfig();
+        final HttpProxyBaseConfig httpProxyConfig = this.httpPushConfig.getHttpProxyConfig();
         try (final Socket proxySocket = new Socket(httpProxyConfig.getHostname(), httpProxyConfig.getPort())) {
             String proxyConnect = "CONNECT " + hostWithoutLookup + ":" + port + " HTTP/1.1\n";
             proxyConnect += "Host: " + hostWithoutLookup + ":" + port;

connectivity/service/src/main/resources/connectivity.conf

@@ -1240,7 +1240,7 @@ pekko-contrib-mongodb-persistence-connection-remember-snapshots {
 
 connection-persistence-dispatcher {
   type = Dispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedForkJoinExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedForkJoinExecutorServiceConfigurator"
   fork-join-executor {
     parallelism-min = 4
     parallelism-factor = 3.0
@@ -1261,7 +1261,7 @@ rabbit-stats-bounded-mailbox {
 
 message-mapping-processor-dispatcher {
   type = Dispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedForkJoinExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedForkJoinExecutorServiceConfigurator"
   fork-join-executor {
     # Min number of threads to cap factor-based parallelism number to
     parallelism-min = 4
@@ -1276,17 +1276,17 @@ message-mapping-processor-dispatcher {
 jms-connection-handling-dispatcher {
   # one thread per actor because the actor blocks.
   type = PinnedDispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
 }
 
 signal-enrichment-cache-dispatcher {
   type = Dispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
 }
 
 http-push-connection-dispatcher {
   type = Dispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
 
   # This executor is meant to be allowed to grow quite big as its limited by the max parallelism of each http connection client.
   # Limit this parallelism here additionally could lead to confusing results regarding througput of some http connections.
@@ -1300,17 +1300,17 @@ http-push-connection-dispatcher {
 
 kafka-consumer-dispatcher {
   type = PinnedDispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
 }
 
 kafka-producer-dispatcher {
   type = PinnedDispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedThreadPoolExecutorServiceConfigurator"
 }
 
 blocked-namespaces-dispatcher {
   type = Dispatcher
-  executor = "org.eclipse.ditto.internal.utils.metrics.executor.InstrumentedForkJoinExecutorServiceConfigurator"
+  executor = "org.eclipse.ditto.internal.utils.metrics.service.executor.InstrumentedForkJoinExecutorServiceConfigurator"
   fork-join-executor {
     # Min number of threads to cap factor-based parallelism number to
     parallelism-min = 4

connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/ConnectivityServiceGlobalErrorRegistryTest.java

@@ -45,6 +45,7 @@
 import org.eclipse.ditto.thingsearch.api.QueryTimeExceededException;
 import org.eclipse.ditto.thingsearch.model.signals.commands.exceptions.InvalidNamespacesException;
 import org.eclipse.ditto.wot.model.WotThingModelInvalidException;
+import org.eclipse.ditto.wot.validation.WotThingModelPayloadValidationException;
 
 public final class ConnectivityServiceGlobalErrorRegistryTest extends GlobalErrorRegistryTestCases {
 
@@ -81,7 +82,8 @@ public ConnectivityServiceGlobalErrorRegistryTest() {
                 JwtInvalidException.class,
                 IllegalAdaptableException.class,
                 WotThingModelInvalidException.class,
-                EdgeServiceTimeoutException.class
+                EdgeServiceTimeoutException.class,
+                WotThingModelPayloadValidationException.class
         );
     }
 

connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/config/DefaultHttpPushConfigTest.java

@@ -21,7 +21,7 @@
 import java.util.Map;
 
 import org.assertj.core.api.JUnitSoftAssertions;
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
+import org.eclipse.ditto.internal.utils.http.config.HttpProxyConfig;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;

connectivity/service/src/test/java/org/eclipse/ditto/connectivity/service/messaging/httppush/HttpPushFactoryTest.java

@@ -30,9 +30,27 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
+import org.apache.pekko.NotUsed;
+import org.apache.pekko.actor.ActorSystem;
+import org.apache.pekko.http.impl.engine.client.ProxyConnectionFailedException;
+import org.apache.pekko.http.javadsl.Http;
+import org.apache.pekko.http.javadsl.ServerBinding;
+import org.apache.pekko.http.javadsl.model.HttpMethods;
+import org.apache.pekko.http.javadsl.model.HttpRequest;
+import org.apache.pekko.http.javadsl.model.HttpResponse;
+import org.apache.pekko.http.javadsl.model.StatusCodes;
+import org.apache.pekko.http.javadsl.model.headers.Authorization;
+import org.apache.pekko.japi.Pair;
+import org.apache.pekko.stream.KillSwitches;
+import org.apache.pekko.stream.OverflowStrategy;
+import org.apache.pekko.stream.javadsl.Flow;
+import org.apache.pekko.stream.javadsl.Keep;
+import org.apache.pekko.stream.javadsl.Sink;
+import org.apache.pekko.stream.javadsl.SinkQueueWithCancel;
+import org.apache.pekko.stream.javadsl.Source;
+import org.apache.pekko.stream.javadsl.SourceQueueWithComplete;
+import org.apache.pekko.testkit.javadsl.TestKit;
 import org.eclipse.ditto.base.service.config.DittoServiceConfig;
-import org.eclipse.ditto.base.service.config.http.DefaultHttpProxyConfig;
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
 import org.eclipse.ditto.connectivity.model.Connection;
 import org.eclipse.ditto.connectivity.model.ConnectionType;
 import org.eclipse.ditto.connectivity.model.ConnectivityModelFactory;
@@ -47,33 +65,15 @@
 import org.eclipse.ditto.connectivity.service.messaging.monitoring.logs.InfoProviderFactory;
 import org.eclipse.ditto.connectivity.service.messaging.tunnel.SshTunnelState;
 import org.eclipse.ditto.internal.utils.config.DefaultScopedConfig;
+import org.eclipse.ditto.internal.utils.http.config.DefaultHttpProxyConfig;
+import org.eclipse.ditto.internal.utils.http.config.HttpProxyConfig;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 
-import org.apache.pekko.NotUsed;
-import org.apache.pekko.actor.ActorSystem;
-import org.apache.pekko.http.impl.engine.client.ProxyConnectionFailedException;
-import org.apache.pekko.http.javadsl.Http;
-import org.apache.pekko.http.javadsl.ServerBinding;
-import org.apache.pekko.http.javadsl.model.HttpMethods;
-import org.apache.pekko.http.javadsl.model.HttpRequest;
-import org.apache.pekko.http.javadsl.model.HttpResponse;
-import org.apache.pekko.http.javadsl.model.StatusCodes;
-import org.apache.pekko.http.javadsl.model.headers.Authorization;
-import org.apache.pekko.japi.Pair;
-import org.apache.pekko.stream.KillSwitches;
-import org.apache.pekko.stream.OverflowStrategy;
-import org.apache.pekko.stream.javadsl.Flow;
-import org.apache.pekko.stream.javadsl.Keep;
-import org.apache.pekko.stream.javadsl.Sink;
-import org.apache.pekko.stream.javadsl.SinkQueueWithCancel;
-import org.apache.pekko.stream.javadsl.Source;
-import org.apache.pekko.stream.javadsl.SourceQueueWithComplete;
-import org.apache.pekko.testkit.javadsl.TestKit;
 import scala.util.Failure;
 import scala.util.Try;
 

gateway/service/pom.xml

@@ -101,6 +101,10 @@
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-wot-model</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.eclipse.ditto</groupId>
+            <artifactId>ditto-wot-validation</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.eclipse.ditto</groupId>
             <artifactId>ditto-internal-models-signalenrichment</artifactId>

gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/AuthenticationConfig.java

@@ -14,8 +14,8 @@
 
 import javax.annotation.concurrent.Immutable;
 
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
 import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
+import org.eclipse.ditto.internal.utils.http.config.HttpProxyConfig;
 
 /**
  * Provides configuration settings for the Gateway authentication.

gateway/service/src/main/java/org/eclipse/ditto/gateway/service/util/config/security/DefaultAuthenticationConfig.java

@@ -17,11 +17,11 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
 
-import org.eclipse.ditto.base.service.config.http.DefaultHttpProxyConfig;
-import org.eclipse.ditto.base.service.config.http.HttpProxyConfig;
 import org.eclipse.ditto.internal.utils.config.ConfigWithFallback;
 import org.eclipse.ditto.internal.utils.config.ScopedConfig;
 import org.eclipse.ditto.internal.utils.config.WithConfigPath;
+import org.eclipse.ditto.internal.utils.http.config.DefaultHttpProxyConfig;
+import org.eclipse.ditto.internal.utils.http.config.HttpProxyConfig;
 
 import com.typesafe.config.Config;