diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dd7f65fc37..99cd517777 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -830,6 +830,23 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, git_config=None, no_download=False, download_instructions=None, alt_location=None, warning_only=False): + try: + return self.obtain_file_raise_on_failure(filename, extension, urls, download_filename, force_download, + git_config, no_download, download_instructions, alt_location, + warning_only) + except Exception: + if build_option('fetch_all'): + if not urls: + urls = ['NO_URL'] + print_warning(f"FAILED: File {filename} not found from {urls[0]}. Continuing ...") + return 'NO_FILE_FOUND' + else: + raise + + @_obtain_file_update_progress_bar_on_return + def obtain_file_raise_on_failure(self, filename, extension=False, urls=None, download_filename=None, + force_download=False, git_config=None, no_download=False, + download_instructions=None, alt_location=None, warning_only=False): """ Locate the file with the given name - searches in different subdirectories of source path diff --git a/easybuild/main.py b/easybuild/main.py index 929e26f1a4..0a4b134c81 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -586,7 +586,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): - exit_on_failure = not (options.dump_test_report or options.upload_test_report) + exit_on_failure = not (options.dump_test_report or options.upload_test_report or build_option('fetch_all')) with rich_live_cm(): run_hook(PRE_PREF + BUILD_AND_INSTALL_LOOP, hooks, args=[ordered_ecs]) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index db667d5110..eae163bc80 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -318,6 +318,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'experimental', 'extended_dry_run', 'fail_on_mod_files_gcccore', + 'fetch_all', 'force', 'generate_devel_module', 'group_writable_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4114647dfc..29d7acbf94 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -464,6 +464,8 @@ def override_options(self): False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), + 'fetch-all': ("Download sources (like --fetch), don't stop when downloading failed for one easyconfig", + None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " "because equivalent OS packages are installed. (e.g. --filter-deps=zlib,ncurses)", 'strlist', 'extend', None), @@ -1359,6 +1361,8 @@ def _postprocess_config(self): self.options.search_paths = [os.path.abspath(path) for path in self.options.search_paths] # Fetch option implies stop=fetch, no moduletool and ignore-osdeps + if self.options.fetch_all: + self.options.fetch = True if self.options.fetch: self.options.stop = FETCH_STEP self.options.ignore_locks = True