Mccalluc/lint

.travis.yml

@@ -3,4 +3,6 @@ python:
   - '3.7'
 cache: pip
 script:
+  - pip install -r requirements.txt
+  - pip install -r requirements-dev.txt
   - ./test.sh

dags/cwl_to_dag.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
 from cwl_airflow import CWLDAG, CWLJobDispatcher, CWLJobGatherer
-dag = CWLDAG(cwl_workflow="/hive/users/blood/workflow/cwl-airflow/Dockstore.cwl", dag_id="dockstore_cwl")
+dag = CWLDAG(
+    cwl_workflow="/hive/users/blood/workflow/cwl-airflow/Dockstore.cwl",
+    dag_id="dockstore_cwl")
 dag.create()
 dag.add(CWLJobDispatcher(dag=dag), to='top')
 dag.add(CWLJobGatherer(dag=dag), to='bottom')
 
 
+# from cwl_airflow import CWLDAG, CWLJobDispatcher, CWLJobGatherer
+# from datetime import timedelta
 
-#from cwl_airflow import CWLDAG, CWLJobDispatcher, CWLJobGatherer
-#from datetime import timedelta
-
-#def cwl_workflow(workflow_file):
+# def cwl_workflow(workflow_file):
 #    dag = CWLDAG(default_args={
 #        'owner': 'airflow',
 #        'email': ['[email protected]'],
@@ -27,6 +28,5 @@
 #    dag.add(CWLJobGatherer(dag=dag), to='bottom')
 #
 #    return dag
-# 
-#cwl_workflow("/hive/users/blood/workflow/cwl-airflow/Dockstore.cwl")
-
+#
+# cwl_workflow("/hive/users/blood/workflow/cwl-airflow/Dockstore.cwl")

dags/dockstore_to_singularity.py

@@ -31,31 +31,35 @@
 }
 
 # Instantiate a DAG
-dag = DAG('dockstore_singularity', default_args = default_args, schedule_interval = None, max_active_runs = 1)
+dag = DAG(
+    'dockstore_singularity',
+    default_args=default_args,
+    schedule_interval=None,
+    max_active_runs=1)
 
 t1 = BashOperator(
-    task_id = 'print_date',
-    bash_command = 'date',
-    dag = dag) 
+    task_id='print_date',
+    bash_command='date',
+    dag=dag)
 
 t2 = BashOperator(
-    task_id = 'rm',
-    bash_command = 'if [ -f ~/helloworld.txt ]; then rm ~/helloworld.txt; fi; if [ -f ~/dockstore-tool-helloworld_1.0.2.sif ]; then rm ~/dockstore-tool-helloworld_1.0.2.sif; fi',
-    dag = dag)
+    task_id='rm',
+    bash_command='if [ -f ~/helloworld.txt ]; then rm ~/helloworld.txt; fi; if [ -f ~/dockstore-tool-helloworld_1.0.2.sif ]; then rm ~/dockstore-tool-helloworld_1.0.2.sif; fi',  # noqa: E501
+    dag=dag)
 
 t3 = BashOperator(
-    task_id = 'singularity_pull',
-    bash_command = 'singularity pull ~/dockstore-tool-helloworld_1.0.2.sif docker://quay.io/ga4gh-dream/dockstore-tool-helloworld:1.0.2',
-    dag = dag)
+    task_id='singularity_pull',
+    bash_command='singularity pull ~/dockstore-tool-helloworld_1.0.2.sif docker://quay.io/ga4gh-dream/dockstore-tool-helloworld:1.0.2',  # noqa: E501
+    dag=dag)
 
 t4 = BashOperator(
-    task_id = 'singularity_exec',
-    bash_command = 'singularity exec ~/dockstore-tool-helloworld_1.0.2.sif hello_world ~/template.txt ~/words.txt; mv helloworld.txt ~/',
-    dag = dag)
+    task_id='singularity_exec',
+    bash_command='singularity exec ~/dockstore-tool-helloworld_1.0.2.sif hello_world ~/template.txt ~/words.txt; mv helloworld.txt ~/',  # noqa: E501
+    dag=dag)
 
 t5 = BashOperator(
-    task_id = 'cat',
-    bash_command = 'cat ~/helloworld.txt',
-    dag = dag)
+    task_id='cat',
+    bash_command='cat ~/helloworld.txt',
+    dag=dag)
 
 t1 >> t2 >> t3 >> t4 >> t5

dags/dockstore_to_singularity2.py

@@ -31,31 +31,35 @@
 }
 
 # Instantiate a DAG
-dag = DAG('dockstore_singularity2', default_args = default_args, schedule_interval = None, max_active_runs = 1)
+dag = DAG(
+    'dockstore_singularity2',
+    default_args=default_args,
+    schedule_interval=None,
+    max_active_runs=1)
 
 t1 = BashOperator(
-    task_id = 'print_date2',
-    bash_command = 'date',
-    dag = dag) 
+    task_id='print_date2',
+    bash_command='date',
+    dag=dag)
 
 t2 = BashOperator(
-    task_id = 'rm2',
-    bash_command = 'if [ -f ~/helloworld.txt ]; then rm ~/helloworld.txt; fi; if [ -f ~/dockstore-tool-helloworld_1.0.2.sif ]; then rm ~/dockstore-tool-helloworld_1.0.2.sif; fi',
-    dag = dag)
+    task_id='rm2',
+    bash_command='if [ -f ~/helloworld.txt ]; then rm ~/helloworld.txt; fi; if [ -f ~/dockstore-tool-helloworld_1.0.2.sif ]; then rm ~/dockstore-tool-helloworld_1.0.2.sif; fi',  # noqa: E501
+    dag=dag)
 
 t3 = BashOperator(
-    task_id = 'singularity_pull2',
-    bash_command = 'singularity pull ~/dockstore-tool-helloworld_1.0.2.sif docker://quay.io/ga4gh-dream/dockstore-tool-helloworld:1.0.2',
-    dag = dag)
+    task_id='singularity_pull2',
+    bash_command='singularity pull ~/dockstore-tool-helloworld_1.0.2.sif docker://quay.io/ga4gh-dream/dockstore-tool-helloworld:1.0.2',  # noqa: E501
+    dag=dag)
 
 t4 = BashOperator(
-    task_id = 'singularity_exec2',
-    bash_command = 'singularity exec ~/dockstore-tool-helloworld_1.0.2.sif hello_world ~/template.txt ~/words.txt; mv helloworld.txt ~/',
-    dag = dag)
+    task_id='singularity_exec2',
+    bash_command='singularity exec ~/dockstore-tool-helloworld_1.0.2.sif hello_world ~/template.txt ~/words.txt; mv helloworld.txt ~/',  # noqa: E501
+    dag=dag)
 
 t5 = BashOperator(
-    task_id = 'cat2',
-    bash_command = 'cat ~/helloworld.txt',
-    dag = dag)
+    task_id='cat2',
+    bash_command='cat ~/helloworld.txt',
+    dag=dag)
 
 t1 >> t2 >> t3 >> t4 >> t5

dags/salmon_rnaseq.py

@@ -25,7 +25,8 @@
 }
 
 # Hardcoded parameters for first Airflow execution
-DATA_DIRECTORY = Path('/hive/hubmap/data/CMU_Tools_Testing_Group/salmon-rnaseq')
+DATA_DIRECTORY = Path(
+    '/hive/hubmap/data/CMU_Tools_Testing_Group/salmon-rnaseq')
 FASTQ_R1 = DATA_DIRECTORY / 'L001_R1_001_r.fastq.gz'
 FASTQ_R2 = DATA_DIRECTORY / 'L001_R2_001_r.fastq.gz'
 THREADS = 6

dags/tutorial.py

@@ -32,23 +32,23 @@
 
 # Instantiate a DAG
 dag = DAG(
-    'test', default_args = default_args, schedule_interval = timedelta(days = 1))
+    'test', default_args=default_args, schedule_interval=timedelta(days=1))
 
 # Tasks are generated when instantiating operator objects
 # By precedence they use:
 # 1. explicitly passed arguments,
 # 2. default_args
-# 3. Operator defaults 
+# 3. Operator defaults
 t1 = BashOperator(
-    task_id = 'print_date',
-    bash_command = 'date',
-    dag = dag)
+    task_id='print_date',
+    bash_command='date',
+    dag=dag)
 
 t2 = BashOperator(
-    task_id = 'sleep',
-    bash_command = 'sleep 5',
-    retries = 3,
-    dag = dag)
+    task_id='sleep',
+    bash_command='sleep 5',
+    retries=3,
+    dag=dag)
 
 # Airflow leverages Jinja templating
 # http://jinja.pocoo.org/docs/dev/
@@ -62,40 +62,36 @@
 """
 
 t3 = BashOperator(
-    task_id = 'templated',
-    bash_command = templated_command,
-    params = {'my_param': 'Parameter I passed in!'},
-    dag = dag)
+    task_id='templated',
+    bash_command=templated_command,
+    params={'my_param': 'Parameter I passed in!'},
+    dag=dag)
 
 t1.set_downstream(t2)
 t1.set_downstream(t3)
 
 # This means that t2 will depend on t1...
 # ...running successfully to run.
-#t1.set_downstream(t2)
+# t1.set_downstream(t2)
 
 # This is equivalent to:
-#t2.set_upstream(t1)
+# t2.set_upstream(t1)
 
 # The bit shift operator can also be
 # used to chain operations:
-#t1 >> t2
+# t1 >> t2
 
 # And the upstream dependency with the
 # bit shift operator:
-#t2 << t1
+# t2 << t1
 
 # Chaining multiple dependencies becomes
 # concise with the bit shift operator:
-#t1 >> t2 >> t3
+# t1 >> t2 >> t3
 
 # A list of tasks can also be set as
 # dependencies. These operations
 # all have the same effect:
-#t1.set_downstream([t2, t3])
-#t1 >> [t2, t3]
-#[t2, t3] << t1
-
-
-
-
+# t1.set_downstream([t2, t3])
+# t1 >> [t2, t3]
+# [t2, t3] << t1

dags/utils.py

@@ -21,6 +21,7 @@
     '{ref}',
 ]
 
+
 def clone_or_update_pipeline(pipeline_name: str, ref: str = 'origin/master'):
     """
     Ensure that a Git clone of a specific pipeline exists inside the

requirements-dev.txt

@@ -0,0 +1,2 @@
+flake8==3.7.9
+autopep8==1.5

setup.cfg

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 100
+ignore =
+  W503 # "line break before binary operator": Prefer operator at start of line so the context is clear.

test.sh

@@ -5,6 +5,8 @@ start() { echo travis_fold':'start:$1; echo $1; }
 end() { echo travis_fold':'end:$1; }
 die() { set +v; echo "$*" 1>&2 ; sleep 1; exit 1; }
 
+pushd . # Need to return to top level after CWL tests.
+
 CWL_NAME=workflow.cwl
 OUTPUT_NAME=test-output-actual
 for CWL_PATH in workflows/cwl/*/workflow.cwl; do
@@ -18,3 +20,9 @@ for CWL_PATH in workflows/cwl/*/workflow.cwl; do
   diff -w -r test-output-expected $OUTPUT_NAME -x .DS_Store | head -n100 | cut -c 1-100
   end $LABEL
 done
+
+popd
+
+start flake8
+flake8 || die 'Try "autopep8 --in-place --aggressive -r ."'
+end flake8