Strict patch

Author: ktsakalozos

.github/workflows/build-snap.yml

@@ -1,12 +1,17 @@
 name: Build MicroK8s snap on PR and push to master
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+  - push
+  - pull_request
+
+### While we work on the strict feature we want the tests to run even if we do put PRs against the master.
+### When this work get merged into master the following should be commented in.
+#  push:
+#    branches:
+#      - master
+#  pull_request:
+#    branches:
+#      - master
 
 jobs:
   build:
@@ -43,24 +48,80 @@ jobs:
       - name: Running upgrade path test
         run: |
           set -x
-          sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=`pwd`/`ls microk8s*.snap` pytest -s ./tests/test-upgrade-path.py
-          sudo snap remove microk8s --purge
-      - name: Running addons tests
+          # Remove the snapd refresh as soon as v2.52 lands
+          sudo snap refresh snapd --channel=latest/edge
+      - name: Check branches
+        run: |
+          set -x
+          (cd tests; pytest -s verify-branches.py)
+      - name: Running addons tests in strict mode
         run: |
           set -x
-          sudo snap install *.snap --classic --dangerous
+          sudo snap install microk8s.snap --dangerous
+          sudo ./tests/connect-all-interfaces.sh
           ./tests/smoke-test.sh
           export UNDER_TIME_PRESSURE="True"
+          export SKIP_OPENEBS="True"
           export SKIP_PROMETHEUS="False"
           (cd tests; pytest -s verify-branches.py)
           sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py"
           sudo microk8s enable community
           sudo -E bash -c "cd /var/snap/microk8s/common/addons/community/tests; pytest -s -ra test-addons.py"
+          grep -Po "Report tarball is at \K.+" |
+          sudo xargs -I {} mv {} inspection-report-strict-${{ strategy.job-index }}.tar.gz
           sudo snap remove microk8s --purge
-      - name: Running upgrade tests
+          sudo rm -rf $HOME/.kube
+          sudo rm -rf $HOME/.config/helm
+          sudo dmesg | grep 'apparmor="DENIED"' > ./denials-${{ strategy.job-index }}.log
+      - name: Upload strict inspect tarball
+        uses: actions/upload-artifact@v2
+        with:
+          name: inspection-report-strict-actions
+          path: ./inspection-report-strict-${{ strategy.job-index }}.tar.gz
+      - name: Upload AppArmor denials
+        uses: actions/upload-artifact@v2
+        with:
+          name: apparmor-denials
+          path: ./denials-${{ strategy.job-index }}.log
+      - name: Running addons tests in devmode
         run: |
           set -x
-          sudo snap install *.snap --classic --dangerous
+          ################ Until devmode of docker-support is fixed we skip this part of the tests #######
+          exit 0
+          sudo snap install microk8s.snap --devmode --dangerous
+          sudo ./tests/connect-all-interfaces.sh
+          ./tests/smoke-test.sh
           export UNDER_TIME_PRESSURE="True"
-          sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/ ; UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=`pwd`/`ls microk8s*.snap` pytest -s ./tests/test-upgrade.py"
+          export SKIP_OPENEBS="False"
+          export SKIP_PROMETHEUS="False"
+          (cd tests; sudo -E pytest -s -ra test-addons.py)
+          sudo microk8s inspect |
+          grep -Po "Report tarball is at \K.+" |
+          sudo xargs -I {} mv {} inspection-report-devmode-${{ strategy.job-index }}.tar.gz
           sudo snap remove microk8s --purge
+      - name: Upload devmode inspect tarball
+        uses: actions/upload-artifact@v2
+        with:
+          name: inspection-report-devmode-actions
+          path: ./inspection-report-devmode-${{ strategy.job-index }}.tar.gz
+      - name: Generate AppArmor on failure
+        run: sudo dmesg | grep 'apparmor="DENIED"' > ./denials-${{ strategy.job-index }}.log
+        if: failure()
+      - name: Upload AppArmor denials failure
+        uses: actions/upload-artifact@v2
+        with:
+          name: apparmor-denials
+          path: ./denials-${{ strategy.job-index }}.log
+        if: failure()
+      - name: Generate inspect tarball
+        run: >
+          sudo microk8s inspect |
+          grep -Po "Report tarball is at \K.+" |
+          sudo xargs -I {} mv {} inspection-report-fail-${{ strategy.job-index }}.tar.gz
+        if: failure()
+      - name: Upload inspect tarball
+        uses: actions/upload-artifact@v2
+        with:
+          name: inspection-report-actions
+          path: ./inspection-report-fail-${{ strategy.job-index }}.tar.gz
+        if: failure()

docs/build.md

@@ -83,9 +83,16 @@ lxc file pull test-build/root/microk8s/microk8s_v1.9.6_amd64.snap .
 After copying it, you can install it with:
 
 ```shell
-snap install microk8s_*_amd64.snap --classic --dangerous
+sudo snap install microk8s_latest_amd64.snap --dangerous
 ```
 
+Finally, you need to connect the interfaces. To this end you can use the `connect-all-interfaces.sh` under the `tests` directory:
+
+```shell
+sudo tests/connect-all-interfaces.sh
+```
+
+
 ## Assembling the Calico CNI manifest
 
 The calico CNI manifest can be found under `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`.

microk8s-resources/default-args/kubelet

@@ -3,6 +3,7 @@
 --client-ca-file=${SNAP_DATA}/certs/ca.crt
 --anonymous-auth=false
 --root-dir=${SNAP_COMMON}/var/lib/kubelet
+--log-dir=${SNAP_COMMON}/var/log
 --fail-swap-on=false
 --feature-gates=DevicePlugins=true
 --eviction-hard="memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi"

microk8s-resources/wrappers/apiservice-kicker

@@ -59,10 +59,10 @@ do
     # every 5 seconds
     sleep 5
     if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] &&
-      getent group microk8s >/dev/null 2>&1
+      getent group snap_microk8s >/dev/null 2>&1
     then
       chmod -R ug+rwX ${SNAP_DATA}/var/kubernetes/backend || true
-      chgrp microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true
+      chgrp snap_microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true
     fi
 
     if ! [ -e "${SNAP_DATA}/var/lock/no-cert-reissue" ] &&
@@ -80,9 +80,8 @@ do
             echo "CSR change detected. Reconfiguring the kube-apiserver"
             rm -rf .srl
             snapctl stop microk8s.daemon-kubelite
-            snapctl stop microk8s.daemon-containerd
-            kill_all_container_shims
-            snapctl start microk8s.daemon-containerd
+            remove_all_containers
+            snapctl restart microk8s.daemon-containerd
             snapctl start microk8s.daemon-kubelite
             start_all_containers
             restart_attempt=$[$restart_attempt+1]

microk8s-resources/wrappers/microk8s-kubectl.wrapper

@@ -30,7 +30,7 @@ fi
 declare -a args="($(cat $SNAP_DATA/args/kubectl))"
 if [ -n "${args[@]-}" ]
 then
-  "${SNAP}/kubectl" "${args[@]}" "$@"
+  EDITOR="${SNAP}/bin/nano" "${SNAP}/kubectl" "${args[@]}" "$@"
 else
-  "${SNAP}/kubectl" "$@"
+  EDITOR="${SNAP}/bin/nano" "${SNAP}/kubectl" "$@"
 fi

microk8s-resources/wrappers/microk8s-leave.wrapper

@@ -23,4 +23,4 @@ then
   exit 1
 fi
 
-run_with_sudo preserve_env ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/leave.py $@
+run_with_sudo ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/leave.py $@

microk8s-resources/wrappers/microk8s-start.wrapper

@@ -52,10 +52,10 @@ then
     exit 1
 else
     start_all_containers
-    if run_with_sudo test -e ${SNAP_DATA}/var/lock/stopped.lock
+    if test -e ${SNAP_DATA}/var/lock/stopped.lock
     then
         # Mark the api server as starting
-        run_with_sudo rm ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null
+        rm ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null
     fi
 fi
 

microk8s-resources/wrappers/microk8s.wrapper

@@ -29,7 +29,7 @@ if [ -f "${SNAP}/microk8s-${APP}.wrapper" ]; then
     "${SNAP}/microk8s-${APP}.wrapper" "$@"
     readonly EXIT="$?"
 elif [ "${APP}" == "inspect" ]; then
-    sudo SNAP_DATA=${SNAP_DATA} ${SNAP}/inspect.sh "$@"
+    SNAP_DATA=${SNAP_DATA} ${SNAP}/inspect.sh "$@"
     readonly EXIT="$?"
 elif [ "${APP}" == "help" ] || [ "${APP}" == "--help" ] || [ "$APP" == "-h" ]; then
     help

microk8s-resources/wrappers/run-flanneld-with-args

@@ -44,6 +44,4 @@ fi
 
 # This is really the only way I could find to get the args passed in correctly.
 declare -a args="($(cat $SNAP_DATA/args/flanneld))"
-export CORE_LD_LIBRARY_PATH="$SNAP/../../core18/current/lib/$ARCH-linux-gnu"
-export LD_LIBRARY_PATH="$CORE_LD_LIBRARY_PATH:$LD_LIBRARY_PATH"
 exec "$SNAP_DATA/opt/cni/bin/flanneld" "${args[@]}"

microk8s-resources/wrappers/run-kubelite-with-args

@@ -79,23 +79,23 @@ then
 fi
 
 #UFW configuration
-if ufw version &> /dev/null
-then
-  ufw=$(ufw status)
-  if echo $ufw | grep -q "Status: active" &&
-     ! [ -e ${SNAP_DATA}/var/lock/skip.ufw ]
-  then
-    # These succeed regardless of whether the rule exists already or not
-    echo "Found enabled UFW: adding rules to allow in/out traffic on 'cali+' and 'vxlan.calico' devices"
-    if ! ufw allow in on vxlan.calico ||
-       ! ufw allow out on vxlan.calico ||
-       ! ufw allow in on cali+ ||
-       ! ufw allow out on cali+
-    then
-      echo "Failed to update UFW rules. You may want to set them manually."
-    fi
-  fi
-fi
+# if ufw version &> /dev/null
+# then
+#   ufw=$(ufw status)
+#   if echo $ufw | grep -q "Status: active" &&
+#      ! [ -e ${SNAP_DATA}/var/lock/skip.ufw ]
+#   then
+#     # These succeed regardless of whether the rule exists already or not
+#     echo "Found enabled UFW: adding rules to allow in/out traffic on 'cali+' and 'vxlan.calico' devices"
+#     if ! ufw allow in on vxlan.calico ||
+#        ! ufw allow out on vxlan.calico ||
+#        ! ufw allow in on cali+ ||
+#        ! ufw allow out on cali+
+#     then
+#       echo "Failed to update UFW rules. You may want to set them manually."
+#     fi
+#   fi
+# fi
 
 # wait for containerd socket
 if grep -e "--address " $SNAP_DATA/args/containerd &> /dev/null

scripts/cluster/common/utils.py

@@ -21,7 +21,7 @@ def try_set_file_permissions(file):
 
     os.chmod(file, 0o660)
     try:
-        shutil.chown(file, group="microk8s")
+        shutil.chown(file, group="snap_microk8s")
     except LookupError:
         # not setting the group means only the current user can access the file
         pass