feat: Docker Compose for running pipeline

Author: dudleycarr

.dockerignore

@@ -0,0 +1,3 @@
+.env
+.git
+data

Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu
+
+RUN apt-get update
+
+# curl, jq, sqlite-utils
+RUN apt-get -y install curl jq python3 pip
+RUN pip install sqlite-utils
+
+# Taskfile
+RUN curl -L https://taskfile.dev/install.sh | sh
+
+# Sourcegraph CLI
+RUN curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /usr/local/bin/src && \
+    chmod +x /usr/local/bin/src
+
+# nsq
+RUN curl -L https://s3.amazonaws.com/bitly-downloads/nsq/nsq-1.2.1.linux-amd64.go1.16.6.tar.gz \
+    -o nsq.tar.gz && \
+    tar -zxf nsq.tar.gz -C /usr/local --strip-components=1 && \
+    rm nsq.tar.gz
+
+WORKDIR /app
+VOLUME /app/data
+COPY Taskfile.yaml .
+
+ENTRYPOINT ["task"]
+CMD ["-l"]

docker-compose.yml

@@ -0,0 +1,58 @@
+services:
+  nsqd:
+    image: nsqio/nsq
+    command: /nsqd -data-path /data -broadcast-address=nsqd -mem-queue-size 0
+    ports:
+      - "4150"
+      - "4151"
+    volumes:
+      - nsqd-data:/data
+
+  nsqadmin:
+    image: nsqio/nsq
+    command: /nsqadmin -nsqd-http-address=nsqd:4151
+    depends_on:
+      - nsqd
+    ports:
+      - "4171:4171"
+
+  # Note: This container expects that the SRC_ACCESS_TOKEN environment variable is set. Check your
+  #  .env file.
+  sourcegraph:
+    build: .
+    command: src:query meta-db github:collect:enqueue
+    env_file: ["./.env"]
+    volumes:
+      - ./data:/app/data
+      - ./npm:/app/npm
+    depends_on:
+      - nsqd
+
+  benthos:
+    image: jeffail/benthos
+    working_dir: /app
+    command:
+      - "-c"
+      - "/app/config/config.yml"
+      - "-r"
+      - "/app/config/resources.yml"
+      - "streams"
+      - "/app/streams/github.yml"
+    volumes:
+      - ./benthos:/app
+      - type: bind
+        source: ./data
+        target: /app/data
+    depends_on:
+      - nsqd
+
+  save-package-json:
+    build: .
+    command: github:collect:to-disk
+    volumes:
+      - ./data:/app/data
+    depends_on:
+      - nsqd
+
+volumes:
+  nsqd-data: