Merge branch 'master' into deploy

Author: mpe

README.md

@@ -228,3 +228,59 @@ More than one selftest can be run by passing multiple arguments to
 
 From there the bisection can either be run by hand, or fully automated by
 creating a script to build the kernel and run the qemu test.
+
+Using the powerpc debian image
+------------------------------
+
+The debian powerpc image in `root-disks` can be used to test big endian kernels.
+It also exercises COMPAT, which is not tested on ppc64le these days.
+
+The kernel needs virtio drivers as well as 9PFS built-in. For example to get it
+booting with `g5_defconfig`:
+
+```
+$ cd linux
+$ ~/ci-scripts/scripts/misc/apply-configs.py 9p guest_configs cgroups-y
+$ make g5_defconfig vmlinux
+$ ~/ci-scripts/boot/qemu-g5+debian
+```
+
+To do interactive testing, run the boot script with `--interactive`, the login
+is `root/linuxppc`.
+
+Once logged in, to install packages a few steps are needed.
+
+If the network doesn't come up by default:
+```
+dhclient $(basename $(ls -1d /sys/class/net/en*))
+```
+
+If you need to use a http proxy:
+```
+echo 'Acquire::http::Proxy "http://proxy.org:3128";' > /etc/apt/apt.conf.d/00proxy
+```
+
+Tell apt to update package lists while ignoring missing GPG keys:
+```
+apt -o Acquire::AllowInsecureRepositories=true -o Acquire::AllowDowngradeToInsecureRepositories=true update
+```
+
+At that point you should be able to install the updated keyring:
+```
+apt install -y --allow-unauthenticated debian-ports-archive-keyring
+```
+
+And update package lists again:
+```
+apt update
+```
+
+Then you should be able to install packages, eg:
+```
+apt install gcc
+```
+
+If you still can't install packages due to GPG errors, you can disable package authentication with:
+```
+echo 'APT::Get::AllowUnauthenticated "true";' > /etc/apt/apt.conf.d/00allow-unauth
+```

build/scripts/container-build.sh

@@ -179,7 +179,7 @@ else
     # Workaround 303e6218ecec ("selftests: Fix O= and KBUILD_OUTPUT handling for relative paths")
     export abs_objtree=$KBUILD_OUTPUT
 
-    cmd="make -k $quiet -j $JFACTOR -C tools/testing/selftests"
+    cmd="make -k $quiet $verbose -j $JFACTOR -C tools/testing/selftests"
 
     if [[ "$1" == "ppctests" ]]; then
         TARGETS="powerpc"

etc/configs/kasan-y.config

@@ -0,0 +1 @@
+CONFIG_KASAN=y

etc/filters.ini

@@ -1,3 +1,7 @@
+[ignore]
+start=#@@@ ignore warnings @@@#
+stop=#@@@ detect warnings @@@#
+
 [patterns]
 v1=detected stall(s)? on CPU
 v2=WARNING:.*\s(un)?lock(s|ing)?\s

etc/tests.py

@@ -1,4 +1,4 @@
-from ngci import TestSuite, SelftestsBuild, SelftestsConfig, QemuSelftestsConfig, TestConfig, QemuNetTestConfig
+from ngci import *
 from configs import *
 from defaults import *
 from qemu import kvm_present
@@ -359,3 +359,31 @@ def full_compile_and_qemu(args):
     full_compile_test(args, suite)
     qemu_coverage(args, suite)
     return suite
+
+
+def qemu_kasan(args, suite=None):
+    images = std_images(args)
+    if suite is None:
+        suite = TestSuite('qemu-kasan', qemus=args.qemus)
+
+    k = suite.add_kernel
+    b = suite.add_qemu_boot
+
+    for image in images:
+        k('ppc64le_guest_defconfig',  image, merge_config=guest_configs + ['kasan-y'])
+
+        # Just a plain boot
+        b('qemu-pseries+p9+kvm+radix+fedora34', 'ppc64le_guest_defconfig', image,
+          script='qemu-pseries+p9+kvm+fedora34')
+        b('qemu-pseries+p9+kvm+hpt+fedora34', 'ppc64le_guest_defconfig', image,
+          script='qemu-pseries+p9+kvm+fedora34', cmdline='disable_radix')
+
+        # Now boot and test KASAN
+        test = QemuTestConfig('kasan-kunit', ['kasan_kunit'])
+        b('qemu-pseries+p9+kvm+radix+fedora34+kasan', 'ppc64le_guest_defconfig', image,
+          script='qemu-pseries+p9+kvm+fedora34', tests=[test])
+        # FIXME currently broken - some missing kasan_arch_is_ready() or similar
+        #b('qemu-pseries+p9+kvm+hpt+fedora34+kasan', 'ppc64le_guest_defconfig', image,
+        #  script='qemu-pseries+p9+kvm+fedora34', tests=[test], cmdline='disable_radix')
+
+    return suite

lib/ngci.py

@@ -291,6 +291,25 @@ def setup(self, state, boot, test_dir):
             boot.args.append('--net-tests')
 
 
+class QemuTestConfig(TestConfig):
+    def __init__(self, name, callbacks=[]):
+        super().__init__(f'qemu-test-{name}')
+        self.callbacks = callbacks
+
+    def setup(self, state, boot, test_dir):
+        start_marker = f'starting-{self.name}'
+        end_marker = f'end-{self.name}'
+
+        # Script doesn't run the tests, just greps the qemu console.log
+        gen_script(f'{test_dir}/run.sh',
+            f"awk '/{start_marker}/, /{end_marker}/' ../console.log | tee extracted.log")
+
+        boot.callbacks.append(f'sh(# {start_marker})')
+        for callback in self.callbacks:
+            boot.callbacks.append(callback)
+        boot.callbacks.append(f'sh(# {end_marker})')
+
+
 class QemuSelftestsConfig(TestConfig):
     def __init__(self, selftest_build, collection=None, exclude=[], extra_callbacks=[]):
         name = 'qemu-selftests'

lib/pexpect_utils.py

@@ -198,3 +198,11 @@ def xmon_di(p, addr):
 
     return result
 
+
+def ignore_warnings(p, f):
+    p.cmd('echo "#@@@ ignore warnings @@@#"')
+    bug_patterns = p.bug_patterns
+    p.bug_patterns = []
+    f(p)
+    p.bug_patterns = bug_patterns
+    p.cmd('echo "#@@@ detect warnings @@@#"')

lib/qemu_callbacks.py

@@ -1,5 +1,6 @@
 import logging
 from utils import setup_timeout
+from pexpect_utils import ignore_warnings
 
 ########################################
 # Callbacks that can run once the VM has booted
@@ -71,6 +72,12 @@ def run_selftests_nocheck(qconf, p, arg=None):
     return run_selftests(qconf, p, arg, check=False)
 
 
+# KASAN Kunit test, needs modules
+def kasan_kunit(qconf, p):
+    ignore_warnings(p, lambda p: p.cmd('modprobe kasan_test'))
+    return True
+
+
 # Invoke lkdtm via sysfs
 # eg. --callback "lkdtm(BUG WARNING)"
 def lkdtm(qconf, p, arg):

lib/utils.py

@@ -161,6 +161,8 @@ def filter_log_warnings(infile, outfile):
 
     parser = ConfigParser()
     parser.read_file(open(path))
+    ignore_start = parser['ignore']['start']
+    ignore_stop = parser['ignore']['stop']
     suppressions = [t[1] for t in parser.items('suppressions', [])]
     suppression_patterns = [t[1] for t in parser.items('suppression_patterns', [])]
     suppression_patterns = [re.compile(p) for p in suppression_patterns]
@@ -180,11 +182,20 @@ def suppress(line):
         return False
 
     found = False
+    ignoring = False
     while True:
         line = infile.readline()
         if len(line) == 0:
             break
 
+        if ignore_stop in line:
+            ignoring = False
+        elif not ignoring and ignore_start in line:
+            ignoring = True
+
+        if ignoring:
+            continue
+            
         if suppress(line):
             continue
 

scripts/misc/apply-configs.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+#
+# Apply configs from etc/configs/ to the local kernel tree.
+#
+# eg.
+# $ cd ~/linux
+# $ make ppc64le_defconfig
+# $ ~/ci-scripts/scripts/misc/apply-configs.py 4k-pages compat-y
+# $ make olddefconfig
+# 
+# Or a group of configs defined in configs.py:
+#
+# $ ~/ci-scripts/scripts/misc/apply-configs.py guest_configs
+
+ 
+from subprocess import run
+import os, sys
+
+base_dir = os.path.realpath(f'{os.path.dirname(os.path.realpath(sys.argv[0]))}/../..')
+sys.path.append(f'{base_dir}/lib')
+
+import configs
+
+
+def main(args):
+    if len(args) == 0:
+        print('Usage: apply-configs.py (config group|config file)+')
+        return False
+
+    names = []
+    for arg in args:
+        try:
+            group = getattr(configs, arg)
+        except AttributeError:
+            names.append(arg)
+        else:
+            names.extend(group)
+
+    src_dir = os.getcwd()
+    paths = []
+    for name in names:
+        # Look in source tree first, which must be current directory
+        full_path = f'{src_dir}/{name}'
+        if os.path.exists(full_path):
+            paths.append((name, full_path))
+            continue
+
+        full_path = f'{base_dir}/etc/configs/{name}'
+        if not name.endswith('.config'):
+            full_path += '.config'
+
+        if not os.path.exists(full_path):
+            print(f'Error: unable to find {name}')
+            return False
+
+        paths.append((name, full_path))
+
+    kbuild_output = os.environ.get('KBUILD_OUTPUT', None)
+    if kbuild_output:
+        # merge_config.sh writes its TMP files to $PWD, so change into KBUILD_OUTPUT
+        os.chdir(kbuild_output)
+
+    for name, path in paths:
+        print(f'Merging {name} ...')
+        rc = run([f'{src_dir}/scripts/kconfig/merge_config.sh', '-m', '.config', path])
+        if rc.returncode != 0:
+            print(f'Error: failed merging {name}')
+            return False
+
+    return True
+
+
+sys.exit(0 if main(sys.argv[1:]) else 1)