Add conformance tests for the cwltool issue 1330 (eager vs. lazy eval of reqs and hints)

conformance_tests.yaml

@@ -3314,7 +3314,7 @@
     `echo a && echo b > out.txt`, but instead will produce the correct `echo a && echo b`,
     and capture the output correctly.
   tags: [ shell_command, command_line_tool ]
-  
+
 - label: multiple-input-feature-requirement
   output: {
     "hello_world_in_two_lines": [
@@ -3348,3 +3348,5 @@
     Test SchemaDefRequirement with a workflow, with the `$import` under types.
     It is similar to schemadef-wf, but the `$import` is different.
   tags: [ workflow, schema_def ]
+
+- $import: tests/eager-eval-reqs-hints/test-index.yaml

tests/eager-eval-reqs-hints/10-cpus-job.yml

@@ -0,0 +1,3 @@
+{
+  "threads_max": 10
+}

tests/eager-eval-reqs-hints/cmdtool-hints.cwl

@@ -0,0 +1,22 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+hints:
+  ResourceRequirement:
+    coresMax: $(inputs.threads_max)
+
+inputs:
+  threads_max:
+    type: int
+    default: 4
+
+stdout: out.txt
+outputs:
+  out:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self[0].contents)
+
+arguments: ['echo', $(runtime.cores)]

tests/eager-eval-reqs-hints/cmdtool-reqs.cwl

@@ -0,0 +1,22 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+requirements:
+  ResourceRequirement:
+    coresMax: $(inputs.threads_max)
+
+inputs:
+  threads_max:
+    type: int
+    default: 4
+
+stdout: out.txt
+outputs:
+  out:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self[0].contents)
+
+arguments: ['echo', $(runtime.cores)]

tests/eager-eval-reqs-hints/empty-job.yml

@@ -0,0 +1,2 @@
+{}
+

tests/eager-eval-reqs-hints/expr-hints.cwl

@@ -0,0 +1,20 @@
+cwlVersion: v1.2
+class: ExpressionTool
+
+requirements:
+  InlineJavascriptRequirement: {}
+
+hints:
+  ResourceRequirement:
+    coresMax: $(inputs.threads_max)
+
+inputs:
+  threads_max:
+    type: int
+    default: 4
+
+outputs:
+  out: string
+
+expression: |
+  ${ return {"out": runtime.cores }}

tests/eager-eval-reqs-hints/expr-reqs.cwl

@@ -0,0 +1,18 @@
+cwlVersion: v1.2
+class: ExpressionTool
+
+requirements:
+  InlineJavascriptRequirement: {}
+  ResourceRequirement:
+    coresMax: $(inputs.threads_max)
+
+inputs:
+  threads_max:
+    type: int
+    default: 4
+
+outputs:
+  out: string
+
+expression: |
+  ${ return {"out": runtime.cores }}

tests/eager-eval-reqs-hints/test-index.yaml

@@ -0,0 +1,145 @@
+# Run with cwltest --test test-index.yaml --tool cwltool
+
+# --- workflow
+# ----- using requirements
+# ------- with default values
+- job: empty-job.yml
+  tool: wf-reqs.cwl
+  label: eager_reqs_wf_reqs_default_threads
+  doc: Test a workflow with requirements using the default values
+  output:
+    {
+      "out": "4\n"
+    }
+  tags: [ resource, workflow ]
+  id: eager-eval-reqs-hints-1
+# ------- with provided inputs
+- job: 10-cpus-job.yml
+  tool: wf-reqs.cwl
+  label: eager_reqs_wf_reqs_10_threads
+  doc: Test a workflow with requirements using 10 threads
+  output:
+    {
+      "out": "10\n"
+    }
+  tags: [ resource, workflow ]
+  id: eager-eval-reqs-hints-2
+# ----- using hints
+# ------- with default values
+- job: empty-job.yml
+  tool: wf-hints.cwl
+  label: eager_reqs_wf_hints_default_threads
+  doc: Test a workflow with hints using the default values
+  output:
+    {
+      "out": "4\n"
+    }
+  tags: [ resource, workflow ]
+  id: eager-eval-reqs-hints-3
+# ------- with provided inputs
+- job: 10-cpus-job.yml
+  tool: wf-hints.cwl
+  label: eager_reqs_wf_hints_10_threads
+  doc: Test a workflow with hints using 10 threads
+  output:
+    {
+      "out": "10\n"
+    }
+  tags: [ resource, workflow ]
+  id: eager-eval-reqs-hints-4
+
+# --- command-line tool
+# ----- using requirements
+# ------- with default values
+- job: empty-job.yml
+  tool: cmdtool-reqs.cwl
+  label: eager_reqs_cmdtool_reqs_default_threads
+  doc: Test a cmdtool with requirements using the default values
+  output:
+    {
+      "out": "4\n"
+    }
+  tags: [ resource, command_line_tool ]
+  id: eager-eval-reqs-hints-5
+# ------- with provided inputs
+- job: 10-cpus-job.yml
+  tool: cmdtool-reqs.cwl
+  label: eager_reqs_cmdtool_reqs_10_threads
+  doc: Test a cmdtool with requirements using 10 threads
+  output:
+    {
+      "out": "10\n"
+    }
+  tags: [ resource, command_line_tool ]
+  id: eager-eval-reqs-hints-6
+# ----- using hints
+# ------- with default values
+- job: empty-job.yml
+  tool: cmdtool-hints.cwl
+  label: eager_reqs_cmdtool_hints_default_threads
+  doc: Test a cmdtool with hints using the default values
+  output:
+    {
+      "out": "4\n"
+    }
+  tags: [ resource, command_line_tool ]
+  id: eager-eval-reqs-hints-7
+# ------- with provided inputs
+- job: 10-cpus-job.yml
+  tool: cmdtool-hints.cwl
+  label: eager_reqs_cmdtool_hints_10_threads
+  doc: Test a cmdtool with hints using 10 threads
+  output:
+    {
+      "out": "10\n"
+    }
+  tags: [ resource, command_line_tool ]
+  id: eager-eval-reqs-hints-8
+
+# --- expression tool
+# ----- using requirements
+# ------- with default values
+- job: empty-job.yml
+  tool: expr-reqs.cwl
+  label: eager_reqs_expr_reqs_default_threads
+  doc: Test a expr with requirements using the default values
+  output:
+    {
+      "out": 4
+    }
+  tags: [ resource, expression_tool ]
+  id: eager-eval-reqs-hints-9
+# ------- with provided inputs
+- job: 10-cpus-job.yml
+  tool: expr-reqs.cwl
+  label: eager_reqs_expr_reqs_10_threads
+  doc: Test a expr with requirements using 10 threads
+  output:
+    {
+      "out": 10
+    }
+  tags: [ resource, expression_tool ]
+  id: eager-eval-reqs-hints-10
+# ----- using hints
+# ------- with default values
+- job: empty-job.yml
+  tool: expr-hints.cwl
+  label: eager_reqs_expr_hints_default_threads
+  doc: Test a expr with hints using the default values
+  output:
+    {
+      "out": 4
+    }
+  tags: [ resource, expression_tool ]
+  id: eager-eval-reqs-hints-11
+# ------- with provided inputs
+- job: 10-cpus-job.yml
+  tool: expr-hints.cwl
+  label: eager_reqs_expr_hints_10_threads
+  doc: Test a expr with hints using 10 threads
+  output:
+    {
+      "out": 10
+    }
+  tags: [ resource, expression_tool ]
+  id: eager-eval-reqs-hints-12

tests/eager-eval-reqs-hints/wf-hints.cwl

@@ -0,0 +1,38 @@
+cwlVersion: v1.2
+class: Workflow
+
+hints:
+  ResourceRequirement:
+    coresMax: $(inputs.threads_max)
+
+inputs:
+  threads_max:
+    type: int
+    default: 4
+
+steps:
+  one:
+    in: []
+    run:
+      class: CommandLineTool
+      inputs:
+        other_input:
+          type: int
+          default: 8
+      baseCommand: echo
+      arguments: [ $(runtime.cores) ]
+      stdout: out.txt
+      outputs:
+        out:
+          type: string
+          outputBinding:
+            glob: out.txt
+            loadContents: true
+            outputEval: $(self[0].contents)
+    out: [out]
+
+outputs:
+  out:
+    type: string
+    outputSource: one/out
+

tests/eager-eval-reqs-hints/wf-reqs.cwl

@@ -0,0 +1,39 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+#  InlineJavascriptRequirement: {}
+  ResourceRequirement:
+    coresMax: $(inputs.threads_max)
+
+inputs:
+  threads_max:
+    type: int
+    default: 4
+
+steps:
+  one:
+    in: []
+    run:
+      class: CommandLineTool
+      inputs:
+        other_input:
+          type: int
+          default: 8
+      baseCommand: echo
+      arguments: [ $(runtime.cores) ]
+      stdout: out.txt
+      outputs:
+        out:
+          type: string
+          outputBinding:
+            glob: out.txt
+            loadContents: true
+            outputEval: $(self[0].contents)
+    out: [out]
+
+outputs:
+  out:
+    type: string
+    outputSource: one/out
+