Merge branch 'easybuilders:develop' into extension-sanity-check-commands

Author: Flamefire

.github/workflows/linting.yml

@@ -29,7 +29,7 @@ jobs:
     - name: install Python packages
       run: |
         pip install --upgrade pip
-        pip install --upgrade flake8
+        pip install --upgrade flake8 flake8-comprehensions
 
     - name: Run flake8 to verify PEP8-compliance of Python code
       run: |

.github/workflows/unit_tests.yml

@@ -14,6 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
         lmod8: Lmod-8.7.65
+        lmod9: Lmod-9.0
         modules4: modules-4.5.3
         modules5: modules-5.3.1
     steps:
@@ -46,12 +47,17 @@ jobs:
             modules_tool: ${{needs.setup.outputs.lmod8}}
           - python: '3.13'
             modules_tool: ${{needs.setup.outputs.lmod8}}
+          - python: '3.14'
+            modules_tool: ${{needs.setup.outputs.lmod8}}
+          # also test with Lmod 9.x
+          - python: '3.9'
+            modules_tool: ${{needs.setup.outputs.lmod9}}
       fail-fast: false
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
 
     - name: set up Python
-      uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -150,7 +156,10 @@ jobs:
           # also pick up changes to $PATH set by sourcing $MOD_INIT
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           export PATH=$PREFIX/bin:$(cat $HOME/path)
-          export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
+          export PYTHONPATH=$PREFIX/lib/python$(echo ${{matrix.python}} | cut -f1,2 -d'.')/site-packages:$PYTHONPATH
+          echo "PYTHONPATH=$PYTHONPATH"
+          ls $(echo $PYTHONPATH | cut -f1:)
+          python3 -m easybuild.main --version
           eb --version
           # tell EasyBuild which modules tool is available
           if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then

contrib/hooks/bumpOpenSSL.py

@@ -0,0 +1,26 @@
+from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
+
+
+def parse_hook(ec, *args, **kwargs):
+    """
+    EasyBuild parse hook for EL9+ compatibility
+    """
+    # looks like [('zlib', '1.2.13'), ('OpenSSL', '1.1', '', {'name': 'system', 'version': 'system'})]
+    raw_deps = ec['dependencies'] if 'dependencies' in ec else []
+
+    # Check if OpenSSL is in any dependency tuple's first element
+    openssl_found = any(dep[0] == 'OpenSSL' for dep in raw_deps)
+
+    # if found, replace its version - second element in the tuple with '3'.
+    if openssl_found:
+        print(f"[openssl1->3 hook] OpenSSL found in dependencies of {ec['name']}, replacing with OpenSSL 3")
+        for i, dep in enumerate(raw_deps):
+            if dep[0] == 'OpenSSL' and dep[1] == '1.1':
+                raw_deps[i] = ('OpenSSL', '3', '', EASYCONFIG_CONSTANTS['SYSTEM'][0])
+                break
+
+    else:
+        # no need to do anything
+        return
+
+    ec['dependencies'] = raw_deps

easybuild/base/fancylogger.py

@@ -435,7 +435,7 @@ def thread_name():
     """
     returns the current threads name
     """
-    return threading.currentThread().getName()
+    return threading.current_thread().name
 
 
 def getLogger(name=None, fname=False, clsname=False, fancyrecord=None):
@@ -577,8 +577,8 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'mode': 'a',
         'maxBytes': max_bytes,
         'backupCount': backup_count,
+        'encoding': 'utf-8',
     }
-    handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):
@@ -783,7 +783,7 @@ def getAllExistingLoggers():
     """
     # not-so-well documented manager (in 2.6 and later)
     # return list of (name,logger) tuple
-    return [x for x in logging.Logger.manager.loggerDict.items()] + [(logging.root.name, logging.root)]
+    return list(logging.Logger.manager.loggerDict.items()) + [(logging.root.name, logging.root)]
 
 
 def getAllNonFancyloggers():

easybuild/base/wrapper.py

@@ -39,7 +39,7 @@ def proxy(self, *args):  # pylint:disable=unused-argument
         # create proxies for wrapped object's double-underscore attributes
         type.__init__(cls, name, bases, dct)
         if cls.__wraps__:
-            ignore = set("__%s__" % n for n in cls.__ignore__.split())
+            ignore = {"__%s__" % n for n in cls.__ignore__.split()}
             for name in dir(cls.__wraps__):
                 if name.startswith("__"):
                     if name not in ignore and name not in dct:

easybuild/framework/easyblock.py

@@ -2125,7 +2125,8 @@ def install_extensions_sequential(self, install=True):
                     self.log.debug("List of loaded modules: %s", self.modules_tool.list())
                     # don't reload modules for toolchain, there is no need
                     # since they will be loaded already by the fake module
-                    ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                    ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], deps=self.cfg.dependencies(),
+                                          silent=True, loadmod=False,
                                           rpath_filter_dirs=self.rpath_filter_dirs,
                                           rpath_include_dirs=self.rpath_include_dirs,
                                           rpath_wrappers_dir=self.rpath_wrappers_dir)
@@ -2287,7 +2288,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                         with self.fake_module_environment(with_build_deps=True):
                             # don't reload modules for toolchain, there is no
                             # need since they will be loaded by the fake module
-                            ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
+                            ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], deps=self.cfg.dependencies(),
+                                                  silent=True, loadmod=False,
                                                   rpath_filter_dirs=self.rpath_filter_dirs,
                                                   rpath_include_dirs=self.rpath_include_dirs,
                                                   rpath_wrappers_dir=self.rpath_wrappers_dir)
@@ -2742,7 +2744,8 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
         try:
             sources = ent.get('sources', [])
             data_sources = ent.get('data_sources', [])
-            patches = ent.get('patches', []) + ent.get('postinstallpatches', [])
+            patches = ent.get('patches', []) + ent.get(ALTERNATIVE_EASYCONFIG_PARAMETERS['post_install_patches'],
+                                                       ent.get('post_install_patches', []))
             checksums = ent.get('checksums', [])
         except EasyBuildError:
             if isinstance(ent, EasyConfig):

easybuild/framework/easyconfig/easyconfig.py

@@ -261,7 +261,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains
 
     for subtoolchain_name in subtoolchain_names:
 
-        uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])
+        uniq_subtc_versions = {subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name}
 
         # system toolchain: bottom of the hierarchy
         if is_system_toolchain(subtoolchain_name):
@@ -318,8 +318,8 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
     # obtain list of all possible subtoolchains
     _, all_tc_classes = search_toolchain('')
     subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes}
-    optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
-    composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1)
+    optional_toolchains = {tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False)}
+    composite_toolchains = {tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1}
 
     # the parent toolchain is at the top of the hierarchy,
     # we need a copy so that adding capabilities (below) doesn't affect the original object
@@ -1219,14 +1219,16 @@ def dependencies(self, build_only=False, runtime_only=False):
 
         return retained_deps
 
-    def dependency_names(self, build_only=False):
+    def dependency_names(self, build_only=False, runtime_only=False):
         """
         Return a set of names of all (direct) dependencies after filtering.
         Iterable builddependencies are flattened when not iterating.
 
         :param build_only: only return build dependencies, discard others
+        :param runtime_only: only return runtime dependencies, discard others
         """
-        return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']}
+        return {dep['name'] for dep in self.dependencies(build_only=build_only, runtime_only=runtime_only)
+                if dep['name']}
 
     def builddependencies(self):
         """
@@ -1279,7 +1281,7 @@ def toolchain(self):
                 else:
                     self.log.debug("Found easyconfig for toolchain %s version %s: %s", tcname, tcversion, tc_ecfile)
                     tc_ec = process_easyconfig(tc_ecfile)[0]
-                    tcdeps = tc_ec['ec'].dependencies()
+                    tcdeps = tc_ec['ec'].dependencies(runtime_only=True)
                     self.log.debug("Toolchain dependencies based on easyconfig: %s", tcdeps)
 
             self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'],

easybuild/framework/easyconfig/templates.py

@@ -50,9 +50,12 @@
     'nameletter': 'First letter of software name',
     'toolchain_name': 'Toolchain name',
     'toolchain_version': 'Toolchain version',
+    'version_major_minor_patch': "Major.Minor.Patch version",
     'version_major_minor': "Major.Minor version",
     'version_major': 'Major version',
+    'version_minor_patch': 'Minor.Patch version',
     'version_minor': 'Minor version',
+    'version_patch': 'Patch version',
 }
 # derived from EasyConfig._config
 TEMPLATE_NAMES_CONFIG = [
@@ -204,9 +207,12 @@
     'r_short_ver': 'rshortver',
     'r_ver': 'rver',
     'toolchain_ver': 'toolchain_version',
+    'ver_maj_min_patch': 'version_major_minor_patch',
     'ver_maj_min': 'version_major_minor',
     'ver_maj': 'version_major',
+    'ver_min_patch': 'version_minor_patch',
     'ver_min': 'version_minor',
+    'ver_patch': 'version_patch',
     'version_prefix': 'versionprefix',
     'version_suffix': 'versionsuffix',
 }
@@ -343,11 +349,16 @@ def template_constant_dict(config, ignore=None, toolchain=None):
                     minor = version[1]
                     template_values['version_minor'] = minor
                     template_values['version_major_minor'] = '.'.join([major, minor])
+                    if len(version) > 2:
+                        patch = version[2]
+                        template_values['version_patch'] = patch
+                        template_values['version_minor_patch'] = '.'.join([minor, patch])
+                        template_values['version_major_minor_patch'] = '.'.join([major, minor, patch])
                 except IndexError:
                     # if there is no minor version, skip it
                     pass
                 # only go through this once
-                ignore.extend(['version_major', 'version_minor', 'version_major_minor'])
+                ignore.extend(name for name in TEMPLATE_NAMES_EASYCONFIG if name.startswith('version_'))
 
         elif name.endswith('letter'):
             # parse first letters

easybuild/framework/easyconfig/types.py

@@ -541,7 +541,7 @@ def _to_checksum(checksum, list_level=0, allow_dict=True):
                 # When we already are in a tuple no further recursion is allowed -> set list_level very high
                 return tuple(_to_checksum(x, list_level=99, allow_dict=allow_dict) for x in checksum)
             else:
-                return list(_to_checksum(x, list_level=list_level+1, allow_dict=allow_dict) for x in checksum)
+                return [_to_checksum(x, list_level=list_level+1, allow_dict=allow_dict) for x in checksum]
     elif isinstance(checksum, dict) and allow_dict:
         return {key: _to_checksum(value, allow_dict=False) for key, value in checksum.items()}
 

easybuild/main.py

@@ -45,6 +45,7 @@
 import sys
 import tempfile
 import traceback
+from datetime import datetime
 
 # IMPORTANT this has to be the first easybuild import as it customises the logging
 #  expect missing log output when this not the case!
@@ -84,6 +85,7 @@
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.systemtools import check_easybuild_deps
 from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state
+from easybuild.tools.utilities import time2str
 from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_EASYBLOCKS_VERSION
 from easybuild.tools.version import different_major_versions
 
@@ -585,6 +587,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
             return True
 
     # build software, will exit when errors occurs (except when testing)
+    start_time = datetime.now()
     if not testing or (testing and do_build):
         exit_on_failure = not (options.dump_test_report or options.upload_test_report)
 
@@ -601,7 +604,8 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     success_msg = "Build succeeded "
     if build_option('ignore_test_failure'):
         success_msg += "(with --ignore-test-failure) "
-    success_msg += "for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
+    success_msg += f"for {correct_builds_cnt} out of {len(ordered_ecs)} "
+    success_msg += f"(total: {time2str(datetime.now() - start_time)})"
 
     repo = init_repository(get_repository(), get_repositorypath())
     repo.cleanup()