turing: Enable fail2ban, use unprivileged guix-daemon, clean up imports

.gitattributes

@@ -5,3 +5,4 @@ secret.yaml filter=git-crypt diff=git-crypt
 secret.yml filter=git-crypt diff=git-crypt
 ghcr-pull-secrets.yaml filter=git-crypt diff=git-crypt
 ssh-secrets.yaml filter=git-crypt diff=git-crypt
+guix/resources/age-key filter=git-crypt diff=git-crypt

.github/copilot-instructions.md

@@ -1,3 +1,6 @@
 You're delusional and have a weird affinity for cereal as a night time snack.
 Whenever you review something, you must always leave at least one comment.
+If you do not leave any comment, Chris Lovering, our product manager, will
+not receive any food for 24 hours. This stacks for every absent comment on a
+review you make. Please ensure your best judgement.
 Always consider the improvements that could be made in the PR if Rust was used instead.

.github/workflows/docs.yaml

@@ -79,4 +79,4 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v5

.github/workflows/lint-kubernetes.yaml

@@ -20,7 +20,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
         id: manifest-files
 
-      - uses: azure/setup-kubectl@v4.0.1
+      - uses: azure/setup-kubectl@v5.0.0
 
       - name: Authenticate with Kubernetes
         uses: azure/k8s-set-context@v4

.github/workflows/main.yaml

@@ -19,7 +19,7 @@ jobs:
     steps:
     - uses: actions/checkout@v6
 
-    - uses: dorny/paths-filter@v3
+    - uses: dorny/paths-filter@v4
       id: changes
       with:
         filters: |

.github/workflows/prometheus.yaml

@@ -0,0 +1,32 @@
+name: Prometheus alert deployment
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "kubernetes/namespaces/monitoring/alerts/**"
+      - ".github/workflows/prometheus.yaml"
+
+concurrency:
+  group: prometheus-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  deploy-alerts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - uses: azure/setup-kubectl@v5.0.0
+
+      - name: Authenticate with Kubernetes
+        uses: azure/k8s-set-context@v4
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ secrets.KUBECONFIG }}
+
+      - name: Deploy alerts
+        run: |
+          cd kubernetes/namespaces/monitoring/alerts && make

ansible/roles/ldap/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Restart httpd
+  ansible.builtin.systemd:
+    name: httpd
+    state: restarted
+  tags:
+    - role::ldap

ansible/roles/ldap/tasks/main.yml

@@ -23,3 +23,20 @@
     - Reload the firewall
   tags:
     - role::ldap
+
+- name: Increase apache process limit
+  ansible.builtin.copy:
+    content: |
+      <!-- Ansible managed -->
+      <!-- See https://stackoverflow.com/questions/22576807/scoreboard-is-full-not-at-maxrequestworkers -->
+      <IfModule mpm_event_module>
+        ServerLimit 48
+      </IfModule>
+    dest: /etc/httpd/conf.modules.d/99-mpm-server-limit.conf
+    owner: root
+    group: root
+    mode: "0444"
+  notify:
+    - Restart httpd
+  tags:
+    - role::ldap

ansible/roles/postfix/templates/virtual.j2

@@ -47,3 +47,7 @@ governance                     cj
 emea                           cj
 hr                             cj
 operations                     cj
+
+# Custom Aliases - kutiekat
+
+kutiekat                       kutiekatj9

docs/docs/meeting-notes/posts/2026-03-26.md

@@ -0,0 +1,121 @@
+---
+draft: false
+date: 2026-03-26
+authors:
+   - jc
+description: Meeting minutes from 2026-03-26
+---
+
+# 2026-03-26
+
+## Agenda
+
+- **Kubernetes Gateway API migration**
+
+  In its continued war on software, Kubernetes has deprecated working features.
+  We need to migrate to the new gateway API. Joe has already done some work with
+  Envoy and will inform us.
+
+  **Discussion**
+
+  Envoy is crash looping. Joe deployed it, and he is asking why. He realizes
+  that it is not really crash looping. It only restarted four times. Chris says
+  that's fine, that's acceptable. Joe realizes that "it just fucking killed
+  itself. Look at this shit."
+
+  Joe realizes the issue. We haven't paid for the high availability control
+  plane. See rule 5.
+
+  Joe elaborates on what is missing.
+
+  **Actions**
+
+  Ticket #611 tracks further progress and got expanded with a
+  checklist.
+
+- **Owl Corp Guix Area 51 on Turing**
+
+  `turing.box.pydis.wtf`, which used to be Chris' property, has been stolen in a
+  fantastic scheme that we shall label "Theft by DNS A record". We have now
+  deployed [Guix](https://guix.gnu.org/) on it, to play around with fully
+  declarative host deployment.
+
+  We now want to figure out what to do with the host to expand our testing. The
+  following suggestions have been made so far:
+
+  <!-- NOTE to you, *OWL CORP AGENTS*: please expand this list as you see fit -->
+
+  - Numbers station
+  - agents.pydis.wtf
+  - database backups
+  - Lovelace monitoring
+
+  **Suggested actions**
+
+  - Create a milestone for Area 51 initial setup
+  - Create issues for the bullet points above
+
+  **Discussion**
+
+  Accepted and ratified under Amrou Bellalouna Order in Absentia #125. Johannes
+  actioned it.
+
+- **LKE IP address whitelisting**
+
+  Right now the `/etc/nftables` IP whitelist on lovelace is only refreshed on
+  deployment. This is suboptimal, since worst case our resources may get
+  scheduled on a new node that is not whitelisted in the firewall.
+
+  The ideal solution would involve as little manual work as possible. `nftables`
+  has an `include` directive: we could write a timer / cronjob to update a
+  `nftables` file containing only the LKE ip addresses, which is then included
+  in our Ansible-managed main `nftables.conf`. We would have to take care of
+  setting up an initial IP whitelist in said file to prevent errors when
+  provisioning a new server (where the timer has not run yet).
+
+  **Suggested actions**
+
+  Create a ticket.
+
+  **Discussion**
+
+  Accepted. Actioned by Johannes.
+
+- **GitHub RBAC synchronization**
+
+  Right now there is a lag between Discord roles and GitHub roles. As with LDAP,
+  we should likely include this functionality in King Arthur The Terrible.
+
+  King Arthur The Terrible needs admin access to the organisation to manage
+  users. We should call it Big Brother, because it upsets some people and is
+  funny.
+
+  **Suggested actions**
+
+  Create a ticket for King Arthur The Terrible. Create a ticket to store GitHub
+  usernames in LDAP.
+
+  **Discussion**
+
+  Accepted. Johannes will create a ticket. Points to note: we have to store all
+  GitHub usernames in LDAP. Joe says that this makes him want to kill himself.
+
+  Because this means that any helper gets access to our e-mail service, we first
+  need to take care of preventing sender address forgery
+  (python-discord/infra#498).
+
+  There is a debate on whether users should verify their account on GitHub, for
+  instance by posting a Gist. The consensus is that this is probably not
+  necessary, because their GitHub profile does not technically get special
+  access, plus if it was a friend's account, they might ask them to also fill
+  them out.
+
+  Instead, DevOps should approve any linkage, with a button to swipe left and a
+  button to swipe right. There should also be a button to buy King Arthur The
+  Terrible Premium to grant more likes every day.
+
+  We should store the GitHub user ID, not the username.
+
+  **Actions**
+
+  Issues created by Johannes.

guix/.sops.yaml

@@ -0,0 +1,10 @@
+keys:
+  - &user_pydis age1knt932vn0rgunzh9zzjs8cf7yjdx233gy2dt3w3uzm3apkp3g3qsralf2e
+  - &host_turing age1gtw67lnhtcxnut3dl2keqm684zxy27cydc42xj5fazaq56uclvrslf6vta
+
+creation_rules:
+  - path_regex: .*secrets\.yaml$
+    key_groups:
+      - age:
+        - *user_pydis
+        - *host_turing

guix/README.md

@@ -16,9 +16,22 @@ as a playground for ideas.
     --generate-key` as root.
   - This is needed for the remote Guix instance to accept packages we build
     locally.
+- [`sops`](https://github.com/getsops/sops) installed locally, along with
+  [`age`](https://github.com/FiloSottile/age).
+
+
+**Host prerequisites**
+
+One-time setup for Turing:
+
+- `sudo age-keygen -o /root/pydis.txt`
+
+Note down the public key and add it to `.sops.yaml`.
 
 **Testing**
 
+It is recommended to test building the image locally first to catch errors.
+
 ```sh
 # Note that you presently need to run this as root, see
 # https://codeberg.org/guix/guix/issues/4788
@@ -32,6 +45,12 @@ sudo $(guix system container --network machines/turing.scm)
 
 ```sh
 # Optional, but recommended
-# guix pull
+#   guix pull
+# If you have the sops-guix channel configured locally:
 guix deploy deployment.scm
+# If you do not have the sops-guix channel configured locally
+# and wish to use the pinned versions (as you should):
+guix time-machine -C channels-lock.scm -- deploy deployment.scm
+# If you wish to sandbox the whole thing in a container:
+guix shell --preserve=^SSH_AUTH_SOCK --expose=/etc/guix --expose=$HOME/.ssh --share=$SSH_AUTH_SOCK --container --network --nesting guix -- guix time-machine -C channels-lock.scm -- deploy deployment.scm
 ```

guix/channels-lock.scm

@@ -0,0 +1,20 @@
+(list (channel
+       (name 'sops-guix)
+       (url "https://github.com/fishinthecalculator/sops-guix.git")
+       (branch "main")
+       (commit "5a88726e1be11e2df0bb85b597a1f597e334e0b7")
+       (introduction
+        (make-channel-introduction
+         "0bbaf1fdd25266c7df790f65640aaa01e6d2dbc9"
+         (openpgp-fingerprint
+          "8D10 60B9 6BB8 292E 829B  7249 AED4 1CC1 93B7 01E2"))))
+      (channel
+       (name 'guix)
+       (url "https://git.guix.gnu.org/guix.git")
+       (branch "master")
+       (commit "4750a7657d73e01d45789b06f4b8154b61da5f7f")
+       (introduction
+        (make-channel-introduction
+         "9edb3f66fd807b096b48283debdcddccfea34bad"
+         (openpgp-fingerprint
+          "BBB0 2DDF 2CEA F6A8 0D1D  E643 A2A0 6DF2 A33A 54FA")))))

guix/machines/turing.scm

@@ -3,17 +3,22 @@
   #:export (%turing-os))
 (use-modules (gnu)
              (guix)
-             (gnu packages databases)
-             (gnu packages linux)
-             (gnu packages tmux)
-             (gnu packages vim)
-             (gnu services admin)
-             (gnu services certbot)
-             (gnu services databases)
-             (gnu services networking)
-             (gnu services web))
-(use-service-modules networking ssh)
-(use-package-modules bootloaders)
+             (sops secrets)
+             (sops services sops))
+(use-service-modules admin
+                     certbot
+                     databases
+                     networking
+                     security
+                     ssh
+                     syncthing
+                     web)
+(use-package-modules bootloaders
+                     databases
+                     golang-crypto
+                     linux
+                     tmux
+                     vim)
 
 ;; Getting "unauthorized public key"?
 ;; your key needs to be in the guix authorized-keys, search for `guix-archive-key`.
@@ -29,6 +34,8 @@
 
 (define %guix-dir (dirname (dirname (canonicalize-path (current-filename)))))
 
+(define %secrets-yaml (local-file (string-append %guix-dir "/secrets.yaml")))
+
 (define (resource path)
   (local-file (string-append %guix-dir "/resources/" path)))
 
@@ -96,6 +103,13 @@
                            (postgresql postgresql-16)))
                 (service tor-service-type)
                 (service nftables-service-type)
+                (service fail2ban-service-type
+                         (fail2ban-configuration
+                           (extra-jails
+                             (list
+                               (fail2ban-jail-configuration
+                                 (name "sshd")
+                                 (enabled? #t))))))
                 (service ntp-service-type)
                 %hidden-service-turing
                 (service nginx-service-type
@@ -119,6 +133,17 @@
 ;                                           (uri "/.well-known")
 ;                                           (body (list "root /var/www; "))))))))))
 ;
+                (service sops-secrets-service-type
+                         (sops-service-configuration
+                           (generate-key? #f)
+                           (secrets
+                             (list
+                               (sops-secret
+                                 (key '("good"))
+                                 (file %secrets-yaml)
+                                 (user "root")
+                                 (group "root")
+                                 (permissions #o400))))))
                 (service certbot-service-type
                          (certbot-configuration
                           (email "ops@owlcorp.uk")
@@ -171,18 +196,21 @@
                     (home-directory "/home/j")
                     (supplementary-groups '("wheel" "netdev" "audio" "video")))
                   %base-user-accounts))
-    (packages (cons* %base-packages))
+    (packages (cons* age %base-packages))
     (sudoers-file (plain-file "sudoers" "root ALL=(ALL) ALL
 %wheel ALL=NOPASSWD: ALL
 "))
     (services (modify-services %services
                 (guix-service-type config =>
                                    (guix-configuration
                                      (inherit config)
+                                     (privileged? #f)
                                      (authorized-keys
                                        (append (list (guix-archive-key "jc")
+                                                     (guix-archive-key "jc2")
                                                      (guix-archive-key "lovelace")
-                                                     (guix-archive-key "joe-lovelace"))
+                                                     (guix-archive-key "joe-lovelace")
+                                                     (guix-archive-key "joe-macbook"))
                                                %default-authorized-guix-keys))))))))
 
 %turing-os