Test clj uberjar (#285)

* Start on testing clj cli tools

* Added test-clj-template to Makefile and CI

* Clean up deps.edn + Dockerfile

* Optimised CI runtime

* First draft of handling deps.edn version bump

* Deleted TODO

Author: anthony-khong

.github/workflows/continuous-integration.yml

@@ -41,14 +41,15 @@ jobs:
         uses: docker://zeroonetechnology/geni:latest
         with:
           entrypoint: scripts/test-geni-cli
-  test-lein-template:
+  test-templates:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: test-lein-template
         uses: docker://zeroonetechnology/geni:latest
         with:
-          entrypoint: scripts/test-lein-template
+          entrypoint: /bin/bash
+          args: -c "scripts/test-lein-template && scripts/test-clj-app-uberjar"
   test-install-geni-cli:
     runs-on: ubuntu-latest
     steps:

CONTRIBUTING.md

@@ -60,8 +60,10 @@ instead of `(complement :slow)`, and start tagging midje facts with `:streaming`
 - Run `make version-bumps` or manually Bump the versions in:
     - `project.clj`.
     - `lein-template`'s `project.clj`;
-    - `lein-template`'s `resources/.../project.clj`; and
-    - `resources/GENI_REPL_RELEASED_VERSION`.
+    - `lein-template`'s `resources/.../project.clj`;
+    - `resources/GENI_REPL_RELEASED_VERSION`;
+    - `docker/deps.edn`; and
+    - `examples/geni-clj-app/deps.edn`.
 - Ensure that the pre-release CI steps pass with `make pre-release`.
 - Deploy the main library with `lein deploy clojars`.
 - Ensure that the post-release CI steps pass with `make post-release`.

Makefile

@@ -51,19 +51,19 @@ test-geni-cli: build
 	docker run --rm -v $(TMP):/root/geni -w /root/geni -t $(DOCKERNAME) \
 		scripts/test-geni-cli
 
-test-lein-template: build
+test-templates: build
 	$(eval TMP := $(shell mktemp -d))
 	cp -r . $(TMP)
 	docker run --rm -v $(TMP):/root/geni -w /root/geni -t $(DOCKERNAME) \
-		scripts/test-lein-template
+		/bin/bash -c "scripts/test-lein-template && scripts/test-clj-app-uberjar"
 
 test-install-geni-cli: build
 	$(eval TMP := $(shell mktemp -d))
 	cp -r . $(TMP)
 	docker run --rm -v $(TMP):/root/geni -w /root/geni -t $(DOCKERNAME) \
 		scripts/test-install-geni-cli
 
-ci: coverage test-install-geni-cli test-geni-cli test-lein-template lint-ancient
+ci: coverage test-install-geni-cli test-geni-cli test-templates lint-ancient
 	echo "CI steps passed!"
 
 version-bumps: build
@@ -73,5 +73,5 @@ version-bumps: build
 pre-release: coverage test-install-geni-cli test-geni-cli lint-ancient
 	echo "Pre-release steps passed!"
 
-post-release: test-lein-template
+post-release: test-templates
 	echo "Post-release steps passed!"

docker/Dockerfile

@@ -6,6 +6,12 @@ ENV HOME /root
 # Install other dependencies
 RUN apt-get update && apt-get install -y libgomp1 libatlas3-base libopenblas-base
 
+# Install Clojure CLI
+RUN apt-get install -y rlwrap
+RUN curl -O https://download.clojure.org/install/linux-install-1.10.1.727.sh
+RUN chmod +x linux-install-1.10.1.727.sh
+RUN ./linux-install-1.10.1.727.sh
+
 # Install clj-kondo and babashka
 RUN bash -c "bash <(curl -s https://raw.githubusercontent.com/borkdude/clj-kondo/master/script/install-clj-kondo)"
 RUN  curl -s https://raw.githubusercontent.com/borkdude/babashka/master/install -o install-babashka
@@ -18,7 +24,11 @@ RUN cd /root && lein with-profile +dev deps
 # Install other dependencies and plugins
 ADD project.clj /root/project.clj
 RUN cd /root && lein with-profile +dev deps
-
 RUN cd /root/ && lein cloverage || true
 
+# Install deps.edn dependencies
+ADD deps.edn /root/deps.edn
+ADD deps-project.clj /root/project.clj
+RUN cd /root && lein with-profile +dev deps
+
 ENTRYPOINT []

docker/deps-project.clj

@@ -0,0 +1,10 @@
+(defproject geni-uberjar-example "0.0.1-SNAPSHOT"
+
+  :source-paths           []
+  :resource-paths         []
+  :main                   user
+  :aot                    [user]
+  :jar-name               "geni-uberjar-example-%s.jar"
+  :plugins                [[lein-tools-deps "0.4.5"]]
+  :middleware             [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn]
+  :lein-tools-deps/config {:config-files [:install :project]})

docker/deps.edn

@@ -0,0 +1,16 @@
+{:deps
+ {io.netty/netty-all                    {:mvn/version "4.1.53.Final"}
+  ;; Spark
+  org.apache.spark/spark-core_2.12      {:mvn/version "3.0.1"}
+  org.apache.spark/spark-mllib_2.12     {:mvn/version "3.0.1"}
+  org.apache.spark/spark-sql_2.12       {:mvn/version "3.0.1"}
+  org.apache.spark/spark-streaming_2.12 {:mvn/version "3.0.1"}
+  ;; Databases
+  mysql/mysql-connector-java            {:mvn/version "8.0.21"}
+  org.postgresql/postgresql             {:mvn/version "42.2.16"}
+  org.xerial/sqlite-jdbc                {:mvn/version "3.32.3.2"}
+  ;; Optional: Spark XGBoost
+  ml.dmlc/xgboost4j-spark_2.12          {:mvn/version "1.0.0"}
+  ml.dmlc/xgboost4j_2.12                {:mvn/version "1.0.0"}
+  ;; Geni
+  zero.one/geni                         {:mvn/version "0.0.34"}}}

examples/geni-clj-app/.gitignore

@@ -0,0 +1 @@
+target

examples/geni-clj-app/deps.edn

@@ -0,0 +1,16 @@
+{:deps
+ {io.netty/netty-all                    {:mvn/version "4.1.53.Final"}
+  ;; Spark
+  org.apache.spark/spark-core_2.12      {:mvn/version "3.0.1"}
+  org.apache.spark/spark-mllib_2.12     {:mvn/version "3.0.1"}
+  org.apache.spark/spark-sql_2.12       {:mvn/version "3.0.1"}
+  org.apache.spark/spark-streaming_2.12 {:mvn/version "3.0.1"}
+  ;; Databases
+  mysql/mysql-connector-java            {:mvn/version "8.0.21"}
+  org.postgresql/postgresql             {:mvn/version "42.2.16"}
+  org.xerial/sqlite-jdbc                {:mvn/version "3.32.3.2"}
+  ;; Optional: Spark XGBoost
+  ml.dmlc/xgboost4j-spark_2.12          {:mvn/version "1.0.0"}
+  ml.dmlc/xgboost4j_2.12                {:mvn/version "1.0.0"}
+  ;; Geni
+  zero.one/geni                         {:mvn/version "0.0.34"}}}

examples/geni-clj-app/project.clj

@@ -0,0 +1,10 @@
+(defproject geni-uberjar-example "0.0.1-SNAPSHOT"
+
+  :source-paths           []
+  :resource-paths         []
+  :main                   user
+  :aot                    [user]
+  :jar-name               "geni-uberjar-example-%s.jar"
+  :plugins                [[lein-tools-deps "0.4.5"]]
+  :middleware             [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn]
+  :lein-tools-deps/config {:config-files [:install :project]})

examples/geni-clj-app/src/user.clj

@@ -0,0 +1,8 @@
+(ns user
+  (:require [zero-one.geni.core :as g]
+            [zero-one.geni.defaults :as defaults])
+  (:gen-class))
+
+(defn -main
+  []
+  (println (g/spark-conf @defaults/spark)))

scripts/test-clj-app-uberjar

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd examples/geni-clj-app
+
+lein uberjar
+java -jar target/geni-uberjar-example-0.0.1-SNAPSHOT-standalone.jar > target/clj-app-stdout
+cat target/clj-app-stdout
+if ! grep -q ":spark.app.name Geni App" target/clj-app-stdout ; then
+    echo "Geni clj app failed to run!"
+    exit 1
+fi
+
+cd -

scripts/version-bumps.clj

@@ -15,6 +15,12 @@
 (def generated-project-clj-path
   "lein-template/resources/leiningen/new/geni/project.clj")
 
+(def docker-deps-edn-path
+  "docker/deps.edn")
+
+(def example-clj-app-deps-edn-path
+  "examples/geni-clj-app/deps.edn")
+
 ;; Bump GENI_RELEASED_VERSION
 (def current-released-version
   (string/replace (slurp released-version-path) #"\n" ""))
@@ -56,8 +62,28 @@
                   (re-pattern (dep-str project-name current-released-version))
                   (dep-str project-name bumped-released-version)))
 
+;; deps.edn
+(defn mvn-str [version]
+  (str ":mvn/version \"" version "\""))
+
+(def current-docker-deps-edn (slurp docker-deps-edn-path))
+
+(def bumped-docker-deps-edn
+  (string/replace current-docker-deps-edn
+                  (re-pattern (mvn-str current-released-version))
+                  (mvn-str bumped-released-version)))
+
+(def current-example-clj-app-deps-edn (slurp example-clj-app-deps-edn-path))
+
+(def bumped-example-clj-app-deps-edn
+  (string/replace current-example-clj-app-deps-edn
+                  (re-pattern (mvn-str current-released-version))
+                  (mvn-str bumped-released-version)))
+
 ;; Write the changes
 (spit project-clj-path bumped-project-clj)
 (spit template-project-clj-path bumped-template-project-clj)
 (spit generated-project-clj-path bumped-generated-project-clj)
 (spit released-version-path bumped-released-version)
+(spit docker-deps-edn-path bumped-docker-deps-edn)
+(spit example-clj-app-deps-edn-path bumped-example-clj-app-deps-edn)