Merge branch 'master' into pr/ewels/6044

Author: bentsherman

.gitignore

@@ -8,6 +8,7 @@
 .node-nextflow*
 .devcontainer
 .vscode/*
+.lineage/
 */*/bin/*
 **/build/**
 build/**

VERSION

@@ -1 +1 @@
-25.03.1-edge
+25.04.0

changelog.txt

@@ -1,5 +1,50 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.04.0 - 8 May 2025
+- Add Support for Google Batch network tags (#5951) [d6e4d6c2]
+- Add Trace observer v2 (#6014) [c4437c1a]
+- Add allowed plugins list (#6049) [e22882e3]
+- Add azure.batch.jobMaxWallClockTime config option (#5996) [74963fdc]
+- Add config error checking to `run` command (#5998) [b0191fa4]
+- Add lineage to gitignore [02e1a542]
+- Add plugin create command (#6032) [68865f7f]
+- Add socketTimeoutException to retryable condition [b676e996]
+- Add support for Java 24 [a47ceefe]
+- Add timezone to lineage list timestamp [77736b55]
+- Add verbose AWS Batch job cleanup logging [504bd2df]
+- Add Support downloading plugins from OCI registries (#6016) [b41c65d5]
+- Add Support prefetching plugins metadata from plugins registry (#5887) [37d433d8]
+- Bring topic channels out of preview (#6030) [4b1b008b]
+- Data Lineage quick fixes (#6045) [f3cee76b]
+- Data Lineage workflow outputs fixes (#6041) [2cb49d08]
+- Data Lineage documentation (#5999) [c0bfc092]
+- Data Lineage programmatic API (#6003) [85b9d00d]
+- Fix NPE when params value is null [c7316eaf]
+- Fix `lineage find` command arguments validation (#6034) [375dc17c]
+- Fix concurrent modification error in `lint` command (#6053) [ccf5705e]
+- Fix errors in output fragment (#6020) [f947c9b4]
+- Fix false error in workflow output (#5982) [039ad401]
+- Fix small formatting inconsistencies  (#6024) [c3a47605]
+- Fix task progress percentage with failed tasks (#5868) [7e931e83]
+- Fix timeout in failing test [cf3e1661]
+- Improve plugin template update [d33c5063]
+- Improve process creation logging [0121670c]
+- Make plugins log less verbose [38410fd0]
+- Minor allow null output params and values [53824bb0]
+- Refactor lineage annotations as list (#6022) [9280b59e]
+- Remove test constructors or mark as TestOnly (#5216) [d4fadd42]
+- Replace output `labels` with `label` directive (#6035) [f99bcd33]
+- Separate language and runtime reference docs (#6018) [75ad21d0]
+- Split standard library docs into multiple pages (#5993) [2f82d236]
+- Store lineage meta in root location and default to .lineage path [d1cae3be]
+- Use `channel` namespace to access channel factories (#6029) [26557f0c]
+- Use the same antlr4 dep as Groovy 4 (#5990) [bd33a76e]
+- Bump plugin template v0.1.0 [b54e78f3]
+- Bump repository provider connection timeout to 60 secs [39aaa831]
+- Bump [email protected] [b7aba74d]
+- Bump [email protected] [e8304c1e]
+- Bump [email protected] [c5100f6d]
+
 24.10.6 - 24 Apr 2025
 - Add support for Fusion Snapshots (#5954) [76a997a3]
 - Bump [email protected] [3ddf16fb]

docs/config.md

@@ -206,7 +206,7 @@ process {
 }
 ```
 
-The above configuration snippet sets 2 cpus and 4 GB of memory to the processes annotated with a label `foo` and `bar`.
+The above configuration snippet requests 2 cpus and 4 GB of memory for processes labeled as `foo` or `bar`.
 
 A process selector can be negated prefixing it with the special character `!`. For example:
 

docs/reference/env-vars.md

@@ -145,6 +145,11 @@ The following environment variables control the configuration of the Nextflow ru
 `NXF_PID_FILE`
 : Name of the file where the process PID is saved when Nextflow is launched in background.
 
+`NXF_PLUGINS_ALLOWED`
+: :::{versionadded} 25.04.0
+  :::
+: Comma separated list of plugin IDs that can be used in a workflow executions e.g. `NXF_PLUGINS_ALLOWED=nf-amazon,nf-tower,nf-wave`. Use empty string to disallow all plugins.
+
 `NXF_PLUGINS_DEFAULT`
 : Whether to use the default plugins when no plugins are specified in the Nextflow configuration (default: `true`).
 

modules/nextflow/src/main/groovy/nextflow/config/parser/v2/ConfigParserV2.groovy

@@ -130,20 +130,25 @@ class ConfigParserV2 implements ConfigParser {
             return Bolts.toConfigObject(target)
         }
         catch( CompilationFailedException e ) {
-            final source = compiler.getSource()
-            final errorListener = new StandardErrorListener('full', false)
-            println()
-            errorListener.beforeErrors()
-            for( final message : compiler.getErrors() ) {
-                final cause = message.getCause()
-                final filename = getRelativePath(source, path)
-                errorListener.onError(cause, filename, source)
-            }
-            errorListener.afterErrors()
+            if( path )
+                printErrors(path)
             throw new ConfigParseException("Config parsing failed", e)
         }
     }
 
+    private void printErrors(Path path) {
+        final source = compiler.getSource()
+        final errorListener = new StandardErrorListener('full', false)
+        println()
+        errorListener.beforeErrors()
+        for( final message : compiler.getErrors() ) {
+            final cause = message.getCause()
+            final filename = getRelativePath(source, path)
+            errorListener.onError(cause, filename, source)
+        }
+        errorListener.afterErrors()
+    }
+
     private String getRelativePath(SourceUnit source, Path path) {
         final uri = source.getSource().getURI()
         return path.getParent().relativize(Path.of(uri)).toString()

modules/nextflow/src/main/groovy/nextflow/processor/TaskConfig.groovy

@@ -555,7 +555,6 @@ class TaskConfig extends LazyMap implements Cloneable {
         if( code == null )
             return defValue
 
-        log.warn1 "The `when` process section is deprecated -- use conditional logic in the calling workflow instead"
         String source = null
         try {
             if( code instanceof Closure ) {

modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy

@@ -308,8 +308,6 @@ class TaskProcessor {
         this.ownerScript = script
         this.config = config
         this.taskBody = taskBody
-        if( taskBody.isShell )
-            log.warn1 "The `shell` process section is deprecated -- use the `script` section instead"
         this.name = name
         this.maxForks = config.maxForks && config.maxForks>0 ? config.maxForks as int : 0
         this.forksCount = maxForks ? new LongAdder() : null

modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy

@@ -29,7 +29,7 @@ import groovy.transform.TupleConstructor
 import groovy.util.logging.Slf4j
 import nextflow.cli.HubOptions
 import nextflow.config.Manifest
-import nextflow.config.parser.v1.ConfigParserV1
+import nextflow.config.ConfigParserFactory
 import nextflow.exception.AbortOperationException
 import nextflow.exception.AmbiguousPipelineNameException
 import nextflow.script.ScriptFile
@@ -455,11 +455,11 @@ class AssetManager {
         }
 
         if( text ) try {
-            def config = new ConfigParserV1().setIgnoreIncludes(true).setStrict(false).parse(text)
+            def config = ConfigParserFactory.create().setIgnoreIncludes(true).setStrict(false).parse(text)
             result = (ConfigObject)config.manifest
         }
         catch( Exception e ) {
-            throw new AbortOperationException("Project config file is malformed -- Cause: ${e.message ?: e}", e)
+            log.warn "Cannot read project manifest -- Cause:  ${e.message ?: e}"
         }
 
         // by default return an empty object

modules/nextflow/src/main/groovy/nextflow/scm/RepositoryProvider.groovy

@@ -316,7 +316,6 @@ abstract class RepositoryProvider {
         return bytes ? new String(bytes) : null
     }
 
-
     /**
      * Validate the repository for the specified file.
      * It tries to read the content of the specified file throwing an {@link AbortOperationException} if it does not exist

modules/nextflow/src/main/groovy/nextflow/trace/AnsiLogObserver.groovy

@@ -475,8 +475,8 @@ class AnsiLogObserver implements TraceObserver {
             term.a(Attribute.INTENSITY_FAINT).a(", cached: $stats.cached").reset()
         if( stats.stored )
             term.a(", stored: $stats.stored")
-        if( stats.failed )
-            term.a(", failed: $stats.failed")
+        if( stats.ignored )
+            term.a(", ignored: $stats.ignored")
         if( stats.retries )
             term.a(", retries: $stats.retries")
         // Show red cross ('✘') or green tick ('✔') according to status

modules/nextflow/src/main/groovy/nextflow/trace/ProgressRecord.groovy

@@ -38,7 +38,7 @@ class ProgressRecord implements Cloneable {
     int running     // number of tasks whose execution started
     int succeeded   // number of tasks whose execution completed successfully
     int cached      // number of tasks whose execution
-    int failed
+    int failed      // number of failed tasks -- includes ignored and retried
     int aborted
     int stored
     int ignored
@@ -61,13 +61,15 @@ class ProgressRecord implements Cloneable {
         this.taskName = processName
     }
 
+    // failed tasks that were not retried are included in the total count
     int getTotalCount() {
-        pending+ submitted+ running+
-           succeeded+ failed+ cached+ stored + aborted
+        pending + submitted + running +
+           succeeded + failed - retries + cached + stored + aborted
     }
 
+    // only failed tasks that were ignored are included in the completed count
     int getCompletedCount() {
-        succeeded+ failed+ cached+ stored
+        succeeded + ignored + cached + stored
     }
 
     @Override

modules/nextflow/src/main/groovy/nextflow/trace/WorkflowStats.groovy

@@ -97,6 +97,10 @@ class WorkflowStats implements Cloneable {
         x >= 0 ? x : 0
     }
 
+    int getTotalCount() {
+        gtz(succeededCount + cachedCount + ignoredCount + effectiveFailedCount + abortedCount)
+    }
+
     String getSucceedCountFmt() {
         INTEGER_FMT.format(gtz(succeededCount))
     }
@@ -114,23 +118,23 @@ class WorkflowStats implements Cloneable {
     }
 
     float getSucceedPct() {
-        int tot = gtz(succeededCount + cachedCount + ignoredCount + failedCount)
+        int tot = getTotalCount()
         tot ? Math.round(succeededCount / tot * 10000.0 as float) / 100.0 as float : 0
     }
 
     float getCachedPct() {
-        def tot = gtz(succeededCount + cachedCount + ignoredCount + failedCount)
+        int tot = getTotalCount()
         tot ? Math.round(gtz(cachedCount) / tot * 10000.0 as float) / 100.0 as float : 0
     }
 
     float getIgnoredPct() {
-        def tot = gtz(succeededCount + cachedCount + ignoredCount + failedCount)
+        int tot = getTotalCount()
         tot ? Math.round(gtz(ignoredCount) / tot * 10000.0 as float) / 100.0 as float : 0
     }
 
-    float getFailedPct() {
-        def tot = gtz(succeededCount + cachedCount + ignoredCount + failedCount)
-        tot ? Math.round(gtz(failedCount) / tot * 10000.0 as float) / 100.0 as float : 0
+    float getEffectiveFailedPct() {
+        int tot = getTotalCount()
+        tot ? Math.round(gtz(effectiveFailedCount) / tot * 10000.0 as float) / 100.0 as float : 0
     }
 
     protected Duration makeDuration(long value) {
@@ -164,10 +168,15 @@ class WorkflowStats implements Cloneable {
     int getSucceededCount() { gtz(succeededCount) }
 
     /**
-     * @return Failed tasks count
+     * @return Failed tasks count (includes ignored and retried)
      */
     int getFailedCount() { gtz(failedCount) }
 
+    /**
+     * @return "Effective" failed tasks count (excludes ignored and retried)
+     */
+    int getEffectiveFailedCount() { gtz(failedCount - ignoredCount - retriesCount) }
+
     /**
      * @return Ignored tasks count
      */

modules/nextflow/src/main/resources/META-INF/build-info.properties

@@ -1,4 +1,4 @@
-build=5943
-version=25.03.1-edge
-timestamp=1745425182395
-commitId=c3bb7a513
+build=5944
+version=25.04.0
+timestamp=1746722511993
+commitId=b7aba74da

modules/nextflow/src/main/resources/META-INF/plugins-info.txt

@@ -1,9 +1,9 @@
-nf-amazon@2.14.0
-nf-azure@1.15.0
+nf-amazon@2.15.0
+nf-azure@1.16.0
 [email protected]
 [email protected]
 [email protected]
-nf-google@1.20.0
+nf-google@1.21.0
 [email protected]
 [email protected]
 [email protected]

modules/nextflow/src/main/resources/nextflow/trace/ReportTemplate.html

@@ -145,8 +145,8 @@ <h4>Workflow execution completed unsuccessfully!</h4>
           <div class="progress" style="height: 1.6rem; margin: 1.2rem auto; border-radius: 0.20rem;">
             <div style="width: ${workflow.stats.succeedPct}%" class="progress-bar bg-success" data-toggle="tooltip" data-placement="top" title="$workflow.stats.succeedCount tasks succeeded"><span class="text-truncate">&nbsp; $workflow.stats.succeedCount succeeded &nbsp;</span></div>
             <div style="width: ${workflow.stats.cachedPct}%" class="progress-bar bg-secondary" data-toggle="tooltip" data-placement="top" title="$workflow.stats.cachedCount tasks were cached"><span class="text-truncate">&nbsp; $workflow.stats.cachedCount cached &nbsp;</span></div>
-            <div style="width: ${workflow.stats.ignoredPct}%" class="progress-bar bg-warning" data-toggle="tooltip" data-placement="top" title="$workflow.stats.ignoredCount tasks reported and error and were ignored"><span class="text-truncate">&nbsp; $workflow.stats.ignoredCount ignored &nbsp;</span></div>
-            <div style="width: ${workflow.stats.failedPct}%" class="progress-bar bg-danger" data-toggle="tooltip" data-placement="top" title="$workflow.stats.failedCount tasks failed"><span class="text-truncate">&nbsp; $workflow.stats.failedCount failed &nbsp;</span></div>
+            <div style="width: ${workflow.stats.ignoredPct}%" class="progress-bar bg-warning" data-toggle="tooltip" data-placement="top" title="$workflow.stats.ignoredCount tasks failed and were ignored"><span class="text-truncate">&nbsp; $workflow.stats.ignoredCount ignored &nbsp;</span></div>
+            <div style="width: ${workflow.stats.effectiveFailedPct}%" class="progress-bar bg-danger" data-toggle="tooltip" data-placement="top" title="$workflow.stats.effectiveFailedCount tasks failed"><span class="text-truncate">&nbsp; $workflow.stats.effectiveFailedCount failed ($workflow.stats.retriesCount retries) &nbsp;</span></div>
           </div>
         </dl>
 

modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy

@@ -227,20 +227,19 @@ class CmdLineageTest extends Specification {
                 'this is a script',
                 null,null, null, null, null, [:],[], null)
         lidFile5.text = encoder.encode(entry)
-        final network = """flowchart BT
-    lid://12345/file.bam@{shape: document, label: "lid://12345/file.bam"}
-    lid://123987/file.bam@{shape: document, label: "lid://123987/file.bam"}
-    lid://123987@{shape: process, label: "foo [lid://123987]"}
-    ggal_gut@{shape: document, label: "ggal_gut"}
-    lid://45678/output.txt@{shape: document, label: "lid://45678/output.txt"}
-    lid://45678@{shape: process, label: "bar [lid://45678]"}
-
-    lid://123987/file.bam -->lid://12345/file.bam
-    lid://123987 -->lid://123987/file.bam
-    ggal_gut -->lid://123987
-    lid://45678/output.txt -->lid://123987
-    lid://45678 -->lid://45678/output.txt
-"""
+        final network = """\
+            flowchart TB
+                lid://12345/file.bam["lid://12345/file.bam"]
+                lid://123987/file.bam["lid://123987/file.bam"]
+                lid://123987(["foo [lid://123987]"])
+                ggal_gut["ggal_gut"]
+                lid://45678/output.txt["lid://45678/output.txt"]
+                lid://45678(["bar [lid://45678]"])
+                lid://123987/file.bam --> lid://12345/file.bam
+                lid://123987 --> lid://123987/file.bam
+                ggal_gut --> lid://123987
+                lid://45678/output.txt --> lid://123987
+                lid://45678 --> lid://45678/output.txt""".stripIndent()
         final template = MermaidHtmlRenderer.readTemplate()
         def expectedOutput = template.replace('REPLACE_WITH_NETWORK_DATA', network)
 
@@ -256,7 +255,7 @@ class CmdLineageTest extends Specification {
 
         then:
         stdout.size() == 1
-        stdout[0] == "Linage graph for lid://12345/file.bam rendered in ${outputHtml}"
+        stdout[0] == "Rendered lineage graph for lid://12345/file.bam to ${outputHtml}"
         outputHtml.exists()
         outputHtml.text == expectedOutput