Changes to include local file as a valid fake data sink

.gitignore

@@ -48,3 +48,5 @@ apidocs
 esdata/
 data/
 
+scripts/quickstart/khermes/templates/khermestemplate.scala.html
+scripts/quickstart/khermes/templates/generated-templates/html/khermestemplate.template.scala

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
-## 0.2.0 (upcoming)
+## 0.3.0 (January 2018)
+
+* File as data sink
+* Akka Stream based supervisor
+* Quick start scripts
+* Categoric data generator 
+* Html to support Kafka or file as sinks
+
+## 0.2.0
 
 * Generate songs, including artist, album and genre
 * Email generator
@@ -14,3 +22,5 @@
 * Feature/khermes clustering.
 * Create kafka producer.
 * Implementation for random names.
+
+

README.md

@@ -20,6 +20,91 @@ $ mvn clean package
 
 We have create both shell scripts and docker-compose files to make easier for you to start using khermes. For further info on how to get started please go to the [Getting Started](https://github.com/Stratio/khermes/wiki/Getting-started) section of our Wiki.
 
+## Quickstart
+
+To start fast generating your files data, check the scripts/quickstart directory. There is a khermes.sh script which starts the complete app along with all its dependencies. Follow the intrucions below:
+
+- Be sure that you have **mvn** command in your PATH and **docker-compose**.
+- Execute khermes.sh script. It checks if there is the **khermes artifact** inside the **target** directory, if not it tries to build it with maven. If you execute the script with the **package** option:  **khermes.sh package**, it forces the **jar** generation. 
+- If you don´t set the **LOCALPATH** environment variable, the file will be created in your current directory. If you want to use another location set it: **LOCALPATH=/tmp khermes.sh**. 
+- The scrips starts **khermes** and **zookeeper**. Go to http://localhost:8080/console in your browser a start the configuration.
+- Create the file configuration with the command create **file-config**:
+    ```    
+    file {
+       path = "/tmp/file.csv"
+    }
+    ```
+    It is neccessary to put the file inside the **/tmp** directory. The docker compose mounts this container´s directory.
+- Crate the template with the command **create twirl-template**:
+    ```
+    @import com.stratio.khermes.helpers.faker.generators._
+    @import scala.util.Random
+    @import com.stratio.khermes.helpers.faker.Faker
+    @import com.stratio.khermes.helpers.faker.generators.Positive
+    @import org.joda.time.DateTime
+
+    @(faker: Faker)
+    @defining(faker, List(CategoryFormat("MASTERCARD", "0.5"),CategoryFormat("VISA", "0.5")),List(CategoryFormat("MOVISTAR", "0.5"),CategoryFormat("IUSACELL", "0.5"))){ case (f,s,s2) =>
+    @f.Name.fullName,@f.Categoric.runNext(s),@f.Number.numberInRange(10000,50000),@f.Geo.geolocation.city,@f.Number.numberInRange(1000,10000),@f.Categoric.runNext(s2),@f.Number.numberInRange(1,5000),@f.Datetime.datetime(new DateTime("2000-01-01"), new DateTime("2016-01-01"), Option("yyyy-MM-dd")) }
+    ```
+- Create the generator configuration with the command: **create generator-config**:
+
+    ```
+    khermes {
+      templates-path = "/tmp/khermes/templates"
+      topic = "khermes"
+      template-name = "khermestemplate"
+      i18n = "EN"
+
+      timeout-rules {
+        number-of-events: 20
+        duration: 5 seconds
+      }
+
+      stop-rules {
+        number-of-events: 10000
+      }
+    }
+    ```
+- Check the the actor´s id with the command **ls** and start it. Then you will be asked for the names of the created configurations. Do not include **kafka** and **avro** names, press enter to skip these otherwise **khermes** will try to find them and will crash.
+- Events will be created as many as you put in the **number-of-events** property. Enjoy!
+
+## Tricks
+
+If you want to generate one CSV file, with its headers follow this:
+
+1. Create a generator config which indicates Khermes to emit one single event:
+
+    ```
+    khermes {
+      templates-path = "/tmp/khermes/templates"
+      topic = "khermes"
+      template-name = "khermestemplate"
+      i18n = "EN"
+      stop-rules {
+        number-of-events: 1
+      }
+    }
+    ```
+
+2. Create the following Twirl template:
+
+
+    ```
+    @import com.stratio.khermes.helpers.faker.Faker
+    @(faker: Faker)Element1,Element2
+    @for(p <- 0 to 100000) {@(faker.Gaussian.runNext(5.0, 1.5)),@(faker.Gaussian.runNext(5.0, 1.5))
+    }
+    ```
+
+This will force khermes to generate the file in one step.
+
+If you want to create a JSON Array instead of JSObject lines the following template works:
+
+
+    @(faker: Faker)@for(p <- 0 to 1) {{"Element1":@(faker.Gaussian.runNext(5.0, 1.5)),"Element2":@(faker.Gaussian.runNext(5.0, 1.5))}@(if(p < 1) "," else "")}]
+
+
 ## Licenses.
 Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
 
@@ -31,4 +116,4 @@ For bugs, questions and discussions please use the [GitHub Issues](https://githu
 
 Want to contribute? Great!
 **Khermes is open source and we need you to keep growing.**
-If you want to contribute do not hesitate to create a Pull Request with your changes!
+If you want to contribute do not hesitate to create a Pull Request with your changes!

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.stratio</groupId>
     <artifactId>khermes</artifactId>
-    <version>0.2.0-SNAPSHOT</version>
+    <version>0.3.0-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <name>Stratio Khermes</name>
@@ -43,6 +43,10 @@
         <scala-maven.tests.plugin.version>2.19.1</scala-maven.tests.plugin.version>
         <scala-maven.plugin.version>3.2.2</scala-maven.plugin.version>
         <dropwizard.metrics.version>3.2.2</dropwizard.metrics.version>
+        <breeze.version>0.13.2</breeze.version>
+        <scalaz.version>7.2.16</scalaz.version>
+        <cats.version>0.9.0</cats.version>
+        <akka-stream-kafka.version>0.16</akka-stream-kafka.version>
     </properties>
 
     <developers>
@@ -143,6 +147,21 @@
             <artifactId>config</artifactId>
             <version>${typesafe.config.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.scalaz</groupId>
+            <artifactId>scalaz-core_${scala.binary.version}</artifactId>
+            <version>${scalaz.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.scalaz</groupId>
+            <artifactId>scalaz-concurrent_${scala.binary.version}</artifactId>
+            <version>${scalaz.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.typelevel</groupId>
+            <artifactId>cats-core_${scala.binary.version}</artifactId>
+            <version>${cats.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.typesafe.akka</groupId>
             <artifactId>akka-http-experimental_${scala.binary.version}</artifactId>
@@ -163,11 +182,6 @@
             <artifactId>akka-cluster-tools_${scala.binary.version}</artifactId>
             <version>${akka.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.typesafe.akka</groupId>
-            <artifactId>akka-testkit_${scala.binary.version}</artifactId>
-            <version>${akka.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.typesafe.akka</groupId>
             <artifactId>akka-multi-node-testkit_${scala.binary.version}</artifactId>
@@ -239,10 +253,9 @@
             <version>${dropwizard.metrics.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_${scala.binary.version}</artifactId>
-            <version>${kafka.version}</version>
-            <scope>test</scope>
+            <groupId>org.scalanlp</groupId>
+            <artifactId>breeze_${scala.binary.version}</artifactId>
+            <version>${breeze.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
@@ -264,6 +277,12 @@
             <version>${curator-framework.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.typesafe.akka</groupId>
+            <artifactId>akka-stream-kafka_${scala.binary.version}</artifactId>
+            <version>${akka-stream-kafka.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.binary.version}</artifactId>
@@ -288,7 +307,6 @@
             <version>${scalacheck.version}</version>
             <scope>test</scope>
         </dependency>
-
     </dependencies>
 
     <build>

project/scalastyle_config.xml

@@ -8,20 +8,15 @@
     </check>
     <check level="warning" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
         <parameters>
-            <parameter name="header"><![CDATA[/*
- * Copyright (C) 2016 Stratio (http://stratio.com)
+            <parameter name="header"><![CDATA[/**
+ * © 2017 Stratio Big Data Inc., Sucursal en España.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * This software is licensed under the Apache 2.0.
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the terms of the License for more details.
  *
- *         http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * SPDX-License-Identifier:  Apache-2.0.
  */]]></parameter>
         </parameters>
     </check>

scripts/quickstart/Dockerfile

@@ -0,0 +1,10 @@
+FROM openjdk:8-jre-alpine
+
+COPY khermes-0.3.0-SNAPSHOT-allinone.jar /khermes/khermes-0.3.0-SNAPSHOT-allinone.jar
+COPY seed.sh /khermes/seed.sh
+
+EXPOSE 8080
+
+ENV ZOOKEEPER_HOST_PORT "zookeeper:2181"
+
+ENTRYPOINT ["/khermes/seed.sh"]

scripts/quickstart/compose.yml

@@ -0,0 +1,20 @@
+version: '2'
+services:
+  zookeeper:
+    container_name: zookeeper
+    image: confluent/zookeeper:latest
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  khermes:
+    container_name: khermes
+    image: khermes:latest
+    ports:
+      - "8080:8080"
+    links:
+      - zookeeper
+    volumes:
+    - ${LOCALPATH}:/tmp

scripts/quickstart/khermes.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+#Build khermes image
+
+if [ ! -e ../../target/khermes-0.3.0-SNAPSHOT-allinone.jar ]
+then
+    cd ../..
+    mvn clean package
+    cd scripts/quickstart
+else
+   if [ "$1" = "package" ]
+    then
+      cd ../..
+      mvn clean package
+      cd scripts/quickstart
+   fi
+fi
+
+echo "copying khermes jar..."
+cp ../../target/khermes-0.3.0-SNAPSHOT-allinone.jar .
+
+echo "copying khermes startup script ..."
+cp ../seed ./seed.sh
+
+echo -e "#!/bin/sh\n$(cat seed.sh)" > input
+cat input > seed.sh
+
+sed -i -e 's"../target"/khermes"g' seed.sh
+
+echo "building khermes image..."
+docker build . -t khermes
+
+echo "checking path..."
+if [ -z ${LOCALPATH+x} ]; then
+  echo "LOCALPATH var is not set, using current directory"
+  export LOCALPATH=.
+else
+  export LOCALPATH=$LOCALPATH
+  echo "using ${LOCALPATH} as workdir"
+fi
+
+rm seed
+rm khermes-0.3.0-SNAPSHOT-allinone.jar
+rm input
+rm seed.sh-e
+
+#start services
+echo "Stating docker..."
+docker-compose -f compose.yml up zookeeper khermes

scripts/seed

@@ -1 +1 @@
-java -jar -Dkhermes.ws=true -Dakka.remote.netty.tcp.port=2552 -Dakka.remote.netty.tcp.hostname=localhost -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-seed" ../target/khermes-0.2.0-SNAPSHOT-allinone.jar
+java -jar -Dkhermes.ws=true -Dakka.remote.netty.tcp.port=2552 -Dakka.remote.netty.tcp.hostname=localhost -Dakka.cluster.seed-nodes.0=akka.tcp://khermes@localhost:2552 -Dmetrics.graphite.name="khermes-seed" ../target/khermes-0.3.0-SNAPSHOT-allinone.jar

src/main/resources/application.conf

@@ -35,6 +35,7 @@ akka {
 
 zookeeper {
   connection = "localhost:2181"
+  connection = ${?ZOOKEEPER_HOST_PORT}
   connectionTimeout = 15000
   sessionTimeout = 60000
   retryAttempts = 50