Skip to content

Fix Azure Batch executor scratch directory support #6872

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
adamrtalbot wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix Azure Batch executor scratch directory support #6872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fdbbe64
Fix Azure Batch executor scratch directory support
adamrtalbot Feb 26, 2026
b347721
Merge branch 'master' into fix/azure-batch-scratch
adamrtalbot Feb 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,7 @@ class AzBatchService implements Closeable {
protected List<ResourceFile> resourceFileUrls(TaskRun task, String sas) {
final cmdRun = (AzPath) task.workDir.resolve(TaskRun.CMD_RUN)
final cmdScript = (AzPath) task.workDir.resolve(TaskRun.CMD_SCRIPT)
final cmdInfile = (AzPath) task.workDir.resolve(TaskRun.CMD_INFILE)

final resFiles = new ArrayList(10)

Expand All @@ -614,7 +615,7 @@ class AzBatchService implements Closeable {

if( task.stdin ) {
resFiles << new ResourceFile()
.setHttpUrl(AzHelper.toHttpUrl(cmdScript, sas))
.setHttpUrl(AzHelper.toHttpUrl(cmdInfile, sas))
.setFilePath(TaskRun.CMD_INFILE)
}

Expand Down
11 changes: 10 additions & 1 deletion plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzFileCopyStrategy.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ class AzFileCopyStrategy extends SimpleFileCopyStrategy {
final result = new StringBuilder()
final copy = environment ? new LinkedHashMap<String,String>(environment) : new LinkedHashMap<String,String>()
copy.remove('PATH')
// save the launch directory so task scripts can be copied to the scratch dir
copy.put('NXF_TASK_LAUNCH_DIR', '$PWD')
copy.put('PATH', '$PWD/.nextflow-bin:$AZ_BATCH_NODE_SHARED_DIR/bin/:$PATH')
copy.put('AZCOPY_LOG_LOCATION', '$PWD/.azcopy_log')
copy.put('AZ_SAS', sasToken)
Expand All @@ -85,7 +87,14 @@ class AzFileCopyStrategy extends SimpleFileCopyStrategy {

@Override
String getStageInputFilesScript(Map<String, Path> inputFiles) {
String result = ( remoteBinDir ? """\
String result = ''
// copy task control files from launch dir to scratch directory
result += 'if [[ "${NXF_SCRATCH:-}" && "${NXF_TASK_LAUNCH_DIR:-}" && "$NXF_TASK_LAUNCH_DIR" != "$PWD" ]]; then\n'
result += ' cp "$NXF_TASK_LAUNCH_DIR"/.command.sh . 2>/dev/null || true\n'
result += ' cp "$NXF_TASK_LAUNCH_DIR"/.command.run . 2>/dev/null || true\n'
result += ' cp "$NXF_TASK_LAUNCH_DIR"/.command.in . 2>/dev/null || true\n'
result += 'fi\n'
result += ( remoteBinDir ? """\
nxf_az_download '${AzHelper.toHttpUrl(remoteBinDir)}' \$PWD/.nextflow-bin
chmod +x \$PWD/.nextflow-bin/* || true
""".stripIndent(true) : '' )
Expand Down
17 changes: 17 additions & 0 deletions plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ class AzFileCopyStrategyTest extends Specification {
then:
binding.stage_inputs == '''\
# stage input files
if [[ "${NXF_SCRATCH:-}" && "${NXF_TASK_LAUNCH_DIR:-}" && "$NXF_TASK_LAUNCH_DIR" != "$PWD" ]]; then
cp "$NXF_TASK_LAUNCH_DIR"/.command.sh . 2>/dev/null || true
cp "$NXF_TASK_LAUNCH_DIR"/.command.run . 2>/dev/null || true
cp "$NXF_TASK_LAUNCH_DIR"/.command.in . 2>/dev/null || true
fi
downloads=(true)

nxf_parallel "${downloads[@]}"
Expand Down Expand Up @@ -218,6 +223,11 @@ class AzFileCopyStrategyTest extends Specification {
then:
binding.stage_inputs == '''\
# stage input files
if [[ "${NXF_SCRATCH:-}" && "${NXF_TASK_LAUNCH_DIR:-}" && "$NXF_TASK_LAUNCH_DIR" != "$PWD" ]]; then
cp "$NXF_TASK_LAUNCH_DIR"/.command.sh . 2>/dev/null || true
cp "$NXF_TASK_LAUNCH_DIR"/.command.run . 2>/dev/null || true
cp "$NXF_TASK_LAUNCH_DIR"/.command.in . 2>/dev/null || true
fi
nxf_az_download 'http://account.blob.core.windows.net/my-data/work/remote/bin' $PWD/.nextflow-bin
chmod +x $PWD/.nextflow-bin/* || true
downloads=(true)
Expand All @@ -228,6 +238,7 @@ class AzFileCopyStrategyTest extends Specification {
binding.task_env == '''\
export FOO="1"
export BAR="any"
export NXF_TASK_LAUNCH_DIR="$PWD"
export PATH="$PWD/.nextflow-bin:$AZ_BATCH_NODE_SHARED_DIR/bin/:$PATH"
export AZCOPY_LOG_LOCATION="$PWD/.azcopy_log"
export AZ_SAS="12345"
Expand Down Expand Up @@ -368,6 +379,11 @@ class AzFileCopyStrategyTest extends Specification {

binding.stage_inputs == '''\
# stage input files
if [[ "${NXF_SCRATCH:-}" && "${NXF_TASK_LAUNCH_DIR:-}" && "$NXF_TASK_LAUNCH_DIR" != "$PWD" ]]; then
cp "$NXF_TASK_LAUNCH_DIR"/.command.sh . 2>/dev/null || true
cp "$NXF_TASK_LAUNCH_DIR"/.command.run . 2>/dev/null || true
cp "$NXF_TASK_LAUNCH_DIR"/.command.in . 2>/dev/null || true
fi
downloads=(true)
rm -f file1.txt
rm -f file2.txt
Expand All @@ -389,6 +405,7 @@ class AzFileCopyStrategyTest extends Specification {
binding.launch_cmd == '/bin/bash .command.run nxf_trace'

binding.task_env == '''\
export NXF_TASK_LAUNCH_DIR="$PWD"
export PATH="$PWD/.nextflow-bin:$AZ_BATCH_NODE_SHARED_DIR/bin/:$PATH"
export AZCOPY_LOG_LOCATION="$PWD/.azcopy_log"
export AZ_SAS="12345"
Expand Down
Loading