Added command line parameter --fetch-all

easybuild/framework/easyblock.py

@@ -830,6 +830,23 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
                     git_config=None, no_download=False, download_instructions=None, alt_location=None,
                     warning_only=False):
+        try:
+            return self.obtain_file_raise_on_failure(filename, extension, urls, download_filename, force_download,
+                                                     git_config, no_download, download_instructions, alt_location,
+                                                     warning_only)
+        except Exception:
+            if build_option('fetch_all'):
+                if not urls:
+                    urls = ['NO_URL']
+                print_warning(f"FAILED: File {filename} not found from {urls[0]}. Continuing ...")
+                return 'NO_FILE_FOUND'
+            else:
+                raise
+
+    @_obtain_file_update_progress_bar_on_return
+    def obtain_file_raise_on_failure(self, filename, extension=False, urls=None, download_filename=None,
+                                     force_download=False, git_config=None, no_download=False,
+                                     download_instructions=None, alt_location=None, warning_only=False):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path

easybuild/main.py

@@ -586,7 +586,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
 
     # build software, will exit when errors occurs (except when testing)
     if not testing or (testing and do_build):
-        exit_on_failure = not (options.dump_test_report or options.upload_test_report)
+        exit_on_failure = not (options.dump_test_report or options.upload_test_report or build_option('fetch_all'))
 
         with rich_live_cm():
             run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs])

easybuild/tools/config.py

@@ -318,6 +318,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'experimental',
         'extended_dry_run',
         'fail_on_mod_files_gcccore',
+        'fetch_all',
         'force',
         'generate_devel_module',
         'group_writable_installdir',

easybuild/tools/options.py

@@ -464,6 +464,8 @@ def override_options(self):
                                           False),
             'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, "
                       "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False),
+            'fetch-all': ("Download sources (like --fetch), don't stop when downloading failed for one easyconfig",
+                          None, 'store_true', False),
             'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, "
                             "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)",
                             'strlist', 'extend', None),
@@ -1359,6 +1361,8 @@ def _postprocess_config(self):
             self.options.search_paths = [os.path.abspath(path) for path in self.options.search_paths]
 
         # Fetch option implies stop=fetch, no moduletool and ignore-osdeps
+        if self.options.fetch_all:
+            self.options.fetch = True
         if self.options.fetch:
             self.options.stop = FETCH_STEP
             self.options.ignore_locks = True