add kafka-manager #15

Author: thangchung

samples/Notifier/appsettings.json

@@ -1,6 +1,6 @@
 {
   "EventBus": {
-    "Brokers": "127.0.0.1:9092"
+    "Brokers": "kafka-server:9092"
   },
   "Logging": {
     "LogLevel": {

samples/SignalRNotifier/Services/Hubs/ProjectHub.cs

@@ -40,21 +40,19 @@ public async Task Handle(Notifications.TaskCreated notification, CancellationTok
       await Clients.All.SendAsync("taskAddedToProjectNotify", notification, cancellationToken);
     }
 
-    protected override Task ExecuteAsync(CancellationToken cancellationToken)
+    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
     {
-      Task.Run(() =>
+      await Task.Run(() =>
       {
         _logger.LogInformation("[NCK] Start to subscribe to project-created...");
         return _eventBus.Subscribe<ProjectCreatedMsg>("project-created");
       }, cancellationToken);
 
-      Task.Run(() =>
+      await Task.Run(() =>
       {
         _logger.LogInformation("[NCK] Start to subscribe to task-created...");
         return _eventBus.Subscribe<TaskCreatedMsg>("task-created");
       }, cancellationToken);
-
-      return Task.CompletedTask;
     }
   }
 }

samples/SignalRNotifier/appsettings.json

@@ -3,7 +3,7 @@
     "BasePath": "/"
   },
   "EventBus": {
-    "Brokers": "PLAINTEXT://kafka-cp-kafka:9092"
+    "Brokers": "kafka-server:9092"
   },
   "Logging": {
     "IncludeScopes": false,

samples/SignalRNotifier/k8s/signalrnotifier-dep.yaml

@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: signalrnotifier-v1
-  namespace: default
+  namespace: kafka
 spec:
   replicas: 1
   template:
@@ -19,7 +19,7 @@ spec:
         - name: Hosts__BasePath
           value: /
         - name: EventBus__Brokers
-          value: PLAINTEXT://kafka-cp-kafka:9092
+          value: kafka-server:9092
         ports:
         - containerPort: 5002
         resources:

samples/SignalRNotifier/k8s/signalrnotifier-svc.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: signalrnotifier
-  namespace: default
+  namespace: kafka
   labels:
     app: signalrnotifier
 spec:

samples/TodoApi/appsettings.json

@@ -44,5 +44,6 @@
         "Microsoft": "Information"
       }
     }
-  }
+  },
+  "AllowedHosts": "*"
 }

samples/TodoApi/k8s/todolist-dep.yaml

@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: todolist-v1
-  namespace: default
+  namespace: kafka
 spec:
   replicas: 1
   template:
@@ -19,15 +19,15 @@ spec:
         - name: Hosts__BasePath
           value: /
         - name: EventBus__Brokers
-          value: PLAINTEXT://kafka-cp-kafka:9092
+          value: kafka-server:9092
         ports:
         - containerPort: 5001
-        #livenessProbe:
-        #  httpGet:
-        #    path: /healthz
-        #    port: 5001
-        #  initialDelaySeconds: 10
-        #  periodSeconds: 5
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 5001
+          initialDelaySeconds: 10
+          periodSeconds: 5
         resources:
           requests:
             memory: "64Mi"

samples/TodoApi/k8s/todolist-svc.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: todolist
-  namespace: default
+  namespace: kafka
   labels:
     app: todolist
 spec:

samples/WebNotifier/k8s/webnotifier-dep.yaml

@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   name: webnotifier-v1
-  namespace: default
+  namespace: kafka
 spec:
   replicas: 1
   template:

samples/WebNotifier/k8s/webnotifier-svc.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: webnotifier
-  namespace: default
+  namespace: kafka
   labels:
     app: webnotifier
 spec:

samples/_kafka/k8s/kafka-topics-ui.yaml renamed to samples/_deploys/kafka-topics-ui/kafka-topics-ui.yaml

@@ -0,0 +1,40 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kafka-manager
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-manager
+  template:
+    metadata:
+      labels:
+        app: kafka-manager
+    spec:
+      containers:
+      - name: kafka-manager
+        image: solsson/kafka-manager@sha256:28b1a0b355f3972a9e3b5ac82abcbfee9a72b66a2bfe86094f6ea2caad9ce3a7
+        ports:
+        - containerPort: 80
+        env:
+        - name: ZK_HOSTS
+          value: "kafka-server.kafka:2181"
+        command:
+        - ./bin/kafka-manager
+        - -Dhttp.port=80
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kafka-manager
+  namespace: kafka
+spec:
+  selector:
+    app: kafka-manager
+  ports:
+  - protocol: TCP
+    port: 9000
+    targetPort: 80
+  type: LoadBalancer

samples/_kafka/k8s/readme.md renamed to samples/_deploys/kafka-topics-ui/readme.md

@@ -0,0 +1,47 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kafka-server
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-server
+  template:
+    metadata:
+      labels:
+        app: kafka-server
+    spec:
+      containers:
+      - name: kafka-server
+        image: spotify/kafka
+        ports:
+        - name: kafka
+          containerPort: 9092
+        - name: zookeeper
+          containerPort: 2181
+        env:
+        - name: ADVERTISED_HOST
+          value: "kafka-server"
+        - name: ADVERTISED_PORT
+          value: "9092"
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kafka-server
+  namespace: kafka
+spec:
+  selector:
+    app: kafka-server
+  ports:
+  - name: kafka
+    protocol: TCP
+    port: 9092
+    targetPort: 9092
+  - name: zookeeper
+    protocol: TCP
+    port: 2181
+    targetPort: 2181
+  

samples/_deploys/kafka/kafka-manager.yml

@@ -1,12 +1,12 @@
 TAG=${TAG:=$(git rev-parse --short HEAD)}
 NAMESPACE=${NAMESPACE:="vndg"}
-Write-Host "${NAMESPACE} and ${TAG}"
+echo "${NAMESPACE} and ${TAG}"
 
-Write-Host "Build TODO API..."
+echo "Build TODO API..."
 docker build -f samples/TodoApi/Dockerfile -t vndg/todoapi:$(git rev-parse --short HEAD) -t vndg/todoapi:latest .
 
-Write-Host "Build SignalR..."
+echo "Build SignalR..."
 docker build -f samples/SignalRNotifier/Dockerfile -t vndg/signalrnotifier:$(git rev-parse --short HEAD) -t vndg/signalrnotifier:latest .
 
-Write-Host "Build Web Notifier..."
+echo "Build Web Notifier..."
 docker build -f samples/WebNotifier/Dockerfile -t vndg/webnotifier:$(git rev-parse --short HEAD) -t vndg/webnotifier:latest .

samples/_deploys/kafka/kafka-server.yaml

@@ -42,7 +42,7 @@ public async Task Publish<TMessage>(TMessage @event, params string[] topics)
       if (topics.Length <= 0) throw new CoreException("[NCK] Publish - Topic to publish should be at least one.");
 
       using (var producer = new Producer<Null, TMessage>(
-        new Dictionary<string, object> {["bootstrap.servers"] = _brokerList},
+        ConstructConfig(_brokerList, true),
         null,
         new ProtoSerializer<TMessage>()))
       {
@@ -180,7 +180,7 @@ private static IDictionary<string, object> ConstructConfig(string brokerList, bo
         //["debug"] = "all",
         ["default.topic.config"] = new Dictionary<string, object>
         {
-          ["auto.offset.reset"] = "latest"
+          ["auto.offset.reset"] = "smallest"
         }
       };
     }

samples/_scripts/apply-k8s.ps1 renamed to samples/_deploys/scripts/apply-k8s.ps1

@@ -2,6 +2,6 @@ namespace NetCoreKit.Infrastructure.Bus.Kafka
 {
   public class KafkaOptions
   {
-    public string Brokers { get; set; } = "127.0.0.1:9092";
+    public string Brokers { get; set; } = "kafka-server:9092";
   }
 }