Merge pull request #120 from PierreBeucher/test-iac

Use IaC to setup test infra
PierreBeucher · Jul 6, 2024 · 98c1ecd · 98c1ecd
2 parents 9f4ec06 + 0d7ad7d
commit 98c1ecd
Show file tree

Hide file tree

Showing 52 changed files with 5,127 additions and 639 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -5,5 +5,6 @@ tests/
 .novops.yml
 README.md
 Makefile
+Taskfile.yml
 docs/
 build/cache
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -13,59 +13,64 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: cachix/install-nix-action@v20
+      - uses: nixbuild/nix-quick-install-action@v27
+
+      - uses: nix-community/cache-nix-action@v5
         with:
-          nix_path: nixpkgs=channel:nixos-unstable
-
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+
       - uses: cachix/cachix-action@v12
         with:
           name: novops
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-${{ runner.arch }}-build-cross
 
-      - run: nix develop .#cross -c make build-cross
+      - run: nix develop .#cross -c task build-cross-linux
+
+      # Can't include darwin targets as not possible to use it on CI directly for now
+      # since it requires custom Docker images which can't be published as per Apple licenses
+      # - run: nix develop .#cross -c task build-cross-macos
 
   # Build and push to cachix
   build-flake:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
-      - uses: cachix/install-nix-action@v20
+      - uses: nixbuild/nix-quick-install-action@v27
+
+      - uses: nix-community/cache-nix-action@v5
         with:
-          nix_path: nixpkgs=channel:nixos-unstable
-
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+
       - uses: cachix/cachix-action@v12
         with:
           name: novops
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
       - run: nix build --json | jq -r '.[].outputs | to_entries[].value' | cachix push novops
-
-  test-cargo:
+  
+  test-integ:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
-      - uses: cachix/install-nix-action@v20
+      - uses: nixbuild/nix-quick-install-action@v27
+
+      - name: Restore and cache Nix store
+        uses: nix-community/cache-nix-action@v5
         with:
-          nix_path: nixpkgs=channel:nixos-unstable
-
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+
       - uses: cachix/cachix-action@v12
         with:
           name: novops
           authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
 
       - uses: actions/cache@v3
+        name: setup Cargo cache
         with:
           path: |
             ~/.cargo/bin/
@@ -74,12 +79,77 @@ jobs:
             ~/.cargo/git/db/
             target/            
           key: ${{ runner.os }}-${{ runner.arch }}-cargo
+
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ env.PNPM_HOME }}/store
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      # Cloud login
+      - uses: google-github-actions/auth@v2
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
 
+      - uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      # Run tests
+      - run: nix develop -c task test-setup
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+
+          # Pulumi Azure Native provider auth
+          # Still required by Pulumi despite already previou az log-in
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+
+      - run: nix develop -c task test-integ-run
+
+      - run: nix develop -c task test-teardown
+        env:
+          PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+
+  test-misc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: nixbuild/nix-quick-install-action@v27
+
+      - name: Restore and cache Nix store
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+
+      - uses: cachix/cachix-action@v12
+        with:
+          name: novops
+          authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+
       - uses: actions/cache@v3
+        name: setup Cargo cache
         with:
           path: |
-            ~/.local/share/containers/storage/
-          key: ${{ runner.os }}-${{ runner.arch }}-podman
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/            
+          key: ${{ runner.os }}-${{ runner.arch }}-cargo
 
-      # Run various tests
-      - run: nix develop -c make test
+      - run: nix develop -c task test-clippy
+      - run: nix develop -c task test-doc
+      - run: nix develop -c task test-cli
+      - run: nix develop -c task test-install
+
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Setup Pages # Useful? Not sure
         uses: actions/configure-pages@v3
 
-      - run: nix develop -c make doc
+      - run: nix develop -c task doc
 
       - name: Upload mdBook artifact
         uses: actions/upload-pages-artifact@v1

diff --git a/.gitignore b/.gitignore
@@ -4,12 +4,17 @@ build/
 /release/
 .github/token
 .venv
+tmp/
 
 # Tests output
 tests/output/
 tests/localconfig
 .envrc
 tmp*
+tests/setup/k8s/*
+!tests/setup/k8s/.gitkeep
+tests/setup/pulumi/.local-backend/*
+!tests/setup/pulumi/.local-backend/.gitkeep
 
 # vscode 
 .vscode/settings.json

diff --git a/Makefile b/Makefile
@@ -1,65 +1,69 @@
-# Build all cross targets
-# Use different target dir to avoid glibc version error
-# See https://github.com/cross-rs/cross/issues/724
-.PHONY: build-cross
-build-cross:
-	# Can't include darwin targets as not possible to use it on CI directly
-	# cross build --target x86_64-apple-darwin --target-dir target/cross/x86_64-apple-darwin
-	# cross build --target aarch64-apple-darwin --target-dir target/cross/aarch64-apple-darwin
-	cross build --target x86_64-unknown-linux-musl --target-dir target/cross/x86_64-unknown-linux-musl
-	cross build --target aarch64-unknown-linux-musl --target-dir target/cross/aarch64-unknown-linux-musl
-
-.PHONY: build-nix
-build-nix:
-	nix build -o build/nix
-
-.PHONY: test
-test: test-prepare test-doc test-clippy test-cargo test-cli test-install test-teardown
-
-.PHONY: test-prepare
-test-prepare:
-	tests/scripts/test-docker-prepare.sh
-
-.PHONY: test-teardown
-test-teardown:
-	tests/scripts/test-docker-teardown.sh
-
-.PHONY: test-cargo
-test-cargo:
-	cargo test
-
-test-cli:
-	tests/cli/test-usage.sh
-
-.PHONY: test-clippy
-test-clippy:
-	cargo clippy -- -D warnings
-
-# Fails if doc is not up to date with current code
-.PHONY: test-doc
-test-doc: doc
-	git diff --exit-code docs/schema/config-schema.json
-
-.PHONY: test-install
-test-install:
-	tests/install/test-install.sh
-
-# Build doc with mdBook and json-schema-for-humans
-# See:
-# - https://github.com/actions/starter-workflows/blob/main/pages/mdbook.yml
-# - https://coveooss.github.io/json-schema-for-humans/#/
-.PHONY: doc
-doc:
-	mdbook build ./docs/
-	cargo run -- schema > docs/schema/config-schema.json
-	generate-schema-doc --config footer_show_time=false --config link_to_reused_ref=false --config expand_buttons=true docs/schema/config-schema.json  docs/book/config/schema.html
-
-doc-serve:
-	(cd docs/ && mdbook serve -o)
-
-# Clean caches and temporary directories
-clean:
-	echo "todo"
+# # Build all cross targets
+# # Use different target dir to avoid glibc version error
+# # See https://github.com/cross-rs/cross/issues/724
+# .PHONY: build-cross
+# build-cross:
+# 	# Can't include darwin targets as not possible to use it on CI directly
+# 	# cross build --target x86_64-apple-darwin --target-dir target/cross/x86_64-apple-darwin
+# 	# cross build --target aarch64-apple-darwin --target-dir target/cross/aarch64-apple-darwin
+# 	cross build --target x86_64-unknown-linux-musl --target-dir target/cross/x86_64-unknown-linux-musl
+# 	cross build --target aarch64-unknown-linux-musl --target-dir target/cross/aarch64-unknown-linux-musl
+
+# .PHONY: build-nix
+# build-nix:
+# 	nix build -o build/nix
+
+# .PHONY: test
+# test: test-prepare test-doc test-clippy test-cargo test-cli test-install test-teardown
+
+# .PHONY: test-setup-pulumi
+# test-setup-pulumi:
+# 	pulumi -C tests/setup/pulumi/aws/ -s test up -yrf
+
+# .PHONY: test-prepare
+# test-prepare:
+# 	tests/scripts/test-docker-prepare.sh
+
+# .PHONY: test-teardown
+# test-teardown:
+# 	tests/scripts/test-docker-teardown.sh
+
+# .PHONY: test-cargo
+# test-cargo:
+# 	cargo test
+
+# test-cli:
+# 	tests/cli/test-usage.sh
+
+# .PHONY: test-clippy
+# test-clippy:
+# 	cargo clippy -- -D warnings
+
+# # Fails if doc is not up to date with current code
+# .PHONY: test-doc
+# test-doc: doc
+# 	git diff --exit-code docs/schema/config-schema.json
+
+# .PHONY: test-install
+# test-install:
+# 	tests/install/test-install.sh
+
+# # Build doc with mdBook and json-schema-for-humans
+# # See:
+# # - https://github.com/actions/starter-workflows/blob/main/pages/mdbook.yml
+# # - https://coveooss.github.io/json-schema-for-humans/#/
+# .PHONY: doc
+# doc:
+# 	mdbook build ./docs/
+# 	cargo run -- schema > docs/schema/config-schema.json
+# 	generate-schema-doc --config footer_show_time=false --config link_to_reused_ref=false --config expand_buttons=true docs/schema/config-schema.json  docs/book/config/schema.html
+
+# doc-serve:
+# 	(cd docs/ && mdbook serve -o)
+
+# # Clean caches and temporary directories
+# clean:
+# 	echo "todo"
 
 
 #