Merge branch 'develop'

Author: Janneke van der Zwaan

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Change Log
 
+## 0.4.0
+
+* Generate unique names for steps that are added to the workflow more than once (#31)
+* Pass all outputs from a step, instead of just one (#27)
+* Improve listing of workflow steps
+
 ## 0.3.1
 
 ### Added

README.md

@@ -92,6 +92,14 @@ for existing command line tools written in Python:
 * [argparse2tool](https://github.com/erasche/argparse2tool#cwl-specific-functionality): Generate CWL CommandLineTool wrappers (and/or Galaxy tool descriptions) from Python programs that use argparse. Also supports the [click](http://click.pocoo.org) argument parser.
 * [pypi2cwl](https://github.com/common-workflow-language/pypi2cwl): Automatically run argparse2cwl on any package in PyPi.
 
+## Listing available steps
+
+Steps loaded into the `WorkflowGenerator` can be listed with:
+
+```
+print wf.list_steps()
+```
+
 ## Running workflows
 
 Workflows created with scriptcwl can be run with:

scriptcwl/__init__.py

@@ -1,4 +1,4 @@
 from .workflow import WorkflowGenerator
 from .scriptcwl import load_steps
 
-__version__ = '0.3.1'
+__version__ = '0.4.0'

scriptcwl/step.py

@@ -87,6 +87,9 @@ def set_input(self, name, value):
             raise ValueError('Invalid input "{}"'.format(name))
         self.step_inputs[name] = value
 
+    def _set_name_in_workflow(self, name):
+        self.name_in_workflow = name
+
     def output_to_input(self, name):
         """Convert the name of an output to an input for a next Step.
 
@@ -101,7 +104,7 @@ def output_to_input(self, name):
         """
         if name not in self.output_names:
             raise ValueError('Invalid output "{}"'.format(name))
-        return ''.join([self.name, '/', name])
+        return ''.join([self.name_in_workflow, '/', name])
 
     def _input_optional(self, inp):
         """Returns True if a step input parameter is optional.
@@ -134,7 +137,7 @@ def to_obj(self):
         obj = CommentedMap()
         obj['run'] = self.run
         obj['in'] = self.step_inputs
-        obj['out'] = [self.output_names[0]]
+        obj['out'] = self.output_names
         if self.is_scattered:
             obj['scatter'] = self.scattered_inputs
             obj['scatterMethod'] = self.scatter_method
@@ -143,12 +146,12 @@ def to_obj(self):
 
     def __str__(self):
         if len(self.optional_input_names) > 0:
-            template = '{} = {}({}[, {}])'
+            template = u'{} = wf.{}({}[, {}])'
         else:
-            template = '{} = {}({})'
-        return template.format(', '.join(self.output_names), self.python_name,
-                               ', '.join(self.input_names),
-                               ', '.join(self.optional_input_names))
+            template = u'{} = wf.{}({})'
+        return template.format(u', '.join(self.output_names), self.python_name,
+                               u', '.join(self.input_names),
+                               u', '.join(self.optional_input_names))
 
     def __repr__(self):
         return str(self)

scriptcwl/workflow.py

@@ -139,10 +139,20 @@ def load(self, steps_dir=None, step_file=None):
                 self.steps_library[n] = step
 
     def list_steps(self):
-        """Prints the signature of all steps in the steps library.
+        """Return string with the signature of all steps in the steps library.
         """
+        steps = []
+        workflows = []
+        template = u'  {:.<25} {}'
         for name, step in self.steps_library.iteritems():
-            print 'Step "{}": {}'.format(name, step)
+            if step.is_workflow:
+                workflows.append(template.format(name, step))
+            else:
+                steps.append(template.format(name, step))
+
+        result = [u'Steps\n', u'\n'.join(steps), u'\n\nWorkflows\n',
+                  u'\n'.join(workflows)]
+        return u''.join(result)
 
     def _has_requirements(self):
         """Returns True if the workflow needs a requirements section.
@@ -169,7 +179,7 @@ def _add_step(self, step):
             step (Step): a step from the steps library.
         """
         self.has_workflow_step = self.has_workflow_step or step.is_workflow
-        self.wf_steps[step.name] = step.to_obj()
+        self.wf_steps[step.name_in_workflow] = step.to_obj()
 
     def add_inputs(self, **kwargs):
         """Add workflow inputs.
@@ -247,6 +257,16 @@ def _get_step(self, name, make_copy=True):
             s = copy.deepcopy(s)
         return s
 
+    def _generate_step_name(self, step_name):
+        name = step_name
+        i = 1
+
+        while name in self.wf_steps.keys():
+            name = '{}-{}'.format(step_name, i)
+            i += 1
+
+        return name
+
     def to_obj(self):
         """Return the created workflow as a dict.
 
@@ -357,6 +377,11 @@ def _make_step(self, step, **kwargs):
             self.has_scatter_requirement = True
             step.is_scattered = True
 
+        # Make sure the step has a unique name in the workflow (so command line
+        # tools can be added to the same workflow multiple times).
+        name_in_wf = self._generate_step_name(step.name)
+        step._set_name_in_workflow(name_in_wf)
+
         outputs = []
         for n in step.output_names:
             oname = step.output_to_input(n)

setup.py

@@ -10,7 +10,7 @@
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version='0.3.1',
+    version='0.4.0',
 
     description='Tool to generate and edit CWL workflows',
     long_description="""Tool to generate and edit workflows in common

tests/data/tools/multiple-out-args.cwl

@@ -0,0 +1,37 @@
+#!/usr/bin/env cwlrunner
+cwlVersion: cwl:v1.0
+class: CommandLineTool
+baseCommand: ["python", "-m", "nlppln.commands.extract_annotations"]
+
+inputs:
+  in_files:
+    type:
+      type: array
+      items: File
+    inputBinding:
+      position: 2
+  out_dir:
+    type: Directory?
+    inputBinding:
+      prefix: --out_dir=
+      separate: false
+  counselors:
+    type:
+      type: array
+      items: string
+      inputBinding:
+        prefix: -c
+
+stdout: missing_introductions.json
+
+outputs:
+  out_files:
+    type:
+      type: array
+      items: File
+    outputBinding:
+      glob: "*.txt"
+  meta_out:
+    type: File
+    outputBinding:
+      glob: "missing_introductions.json"

tests/test_step.py

@@ -51,3 +51,26 @@ def test_argument_is_optional(self, step):
 
     def test_argument_is_not_optional(self, step):
         assert not step._input_optional({'type': 'string'})
+
+
+class TestMultipleOutputArgs(object):
+        @pytest.fixture
+        def step(self):
+            return Step('tests/data/tools/multiple-out-args.cwl')
+
+        def test_has_multiple_out_args(self, step):
+            assert len(step.to_obj()['out']) == 2
+
+
+class TestStepNameInWorkflow(object):
+    @pytest.fixture
+    def step(self):
+        return Step('tests/data/tools/echo.cwl')
+
+    def test_no_name_in_workflow(self, step):
+        with pytest.raises(AttributeError):
+            step.name_in_workflow == 'echo'
+
+    def test_set_name_in_workflow(self, step):
+        step._set_name_in_workflow('echo')
+        assert step.name_in_workflow == 'echo'

tests/test_workflow.py

@@ -17,7 +17,7 @@ def test_load(self):
 
         step_keys = wf.steps_library.keys()
         step_keys.sort()
-        assert step_keys == ['echo', 'wc']
+        assert step_keys == ['echo', 'multiple-out-args', 'wc']
 
     def test_save_with_tools(self, tmpdir):
         wf = WorkflowGenerator()
@@ -107,12 +107,12 @@ def test_scatter_method_correct(self):
         scatter_methods = ['dotproduct', 'nested_crossproduct',
                            'flat_crossproduct']
 
-        wf = WorkflowGenerator()
-        wf.load('tests/data/tools')
+        for method in scatter_methods:
+            wf = WorkflowGenerator()
+            wf.load('tests/data/tools')
 
-        msgs = wf.add_inputs(wfmessages='string[]')
+            msgs = wf.add_inputs(wfmessages='string[]')
 
-        for method in scatter_methods:
             echoed = wf.echo(message=msgs, scatter='message', scatter_method=method)
             assert echoed == 'echo/echoed'
 
@@ -129,12 +129,12 @@ def test_scatter_variable_correct(self):
         scatter_methods = ['dotproduct', 'nested_crossproduct',
                            'flat_crossproduct']
 
-        wf = WorkflowGenerator()
-        wf.load('tests/data/tools')
+        for method in scatter_methods:
+            wf = WorkflowGenerator()
+            wf.load('tests/data/tools')
 
-        msgs = wf.add_inputs(wfmessages='string[]')
+            msgs = wf.add_inputs(wfmessages='string[]')
 
-        for method in scatter_methods:
             echoed = wf.echo(message=msgs, scatter='message', scatter_method=method)
             assert echoed == 'echo/echoed'
 
@@ -155,3 +155,23 @@ def test_missing_scatter_method_argument(self):
 
         with pytest.raises(ValueError):
             wf.echo(message=msgs, scatter='message')
+
+
+class TestWorkflowGeneratorWithStepsAddedMultipleTimes(object):
+    def test_generate_step_name(self):
+        wf = WorkflowGenerator()
+        wf.load('tests/data/tools')
+
+        wfmessage = wf.add_inputs(wfmessage='string')
+
+        name = wf._generate_step_name('echo')
+        echoed = wf.echo(message=wfmessage)
+
+        assert name == 'echo'
+        assert name == echoed.split('/')[0]
+
+        name = wf._generate_step_name('echo')
+        echoed2 = wf.echo(message=wfmessage)
+
+        assert name != 'echo'
+        assert name == echoed2.split('/')[0]