Merge pull request #6 from galaxyproject/dev

keeps up to date with galaxy:dev

.ci/py3_sources.txt

@@ -3,29 +3,40 @@ cron/
 lib/galaxy/actions/
 lib/galaxy/auth/
 lib/galaxy/config.py
+lib/galaxy/dataset_collections/
+lib/galaxy/datatypes/binary.py
+lib/galaxy/datatypes/converters/
+lib/galaxy/datatypes/dataproviders/
+lib/galaxy/datatypes/sequence.py
+lib/galaxy/datatypes/sniff.py
+lib/galaxy/datatypes/tabular.py
 lib/galaxy/dependencies/
 lib/galaxy/eggs/
 lib/galaxy/exceptions/
 lib/galaxy/external_services/
 lib/galaxy/forms/
 lib/galaxy/jobs/
+lib/galaxy/managers/
+lib/galaxy/model/__init__.py
+lib/galaxy/model/migrate/
 lib/galaxy/objectstore/
 lib/galaxy/openid/
 lib/galaxy/quota/
 lib/galaxy/sample_tracking/
+lib/galaxy/security/
 lib/galaxy/tags/
 lib/galaxy/tools/cwl/
-lib/galaxy/tools/parser/
+lib/galaxy/tools/deps/
 lib/galaxy/tools/lint.py
 lib/galaxy/tools/lint_util.py
+lib/galaxy/tools/linters/
 lib/galaxy/tools/loader.py
 lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/deps/
-lib/galaxy/tools/toolbox/
 lib/galaxy/tools/parser/
+lib/galaxy/tools/toolbox/
 lib/galaxy/tours/
 lib/galaxy/util/
+lib/galaxy/visualization/
 lib/galaxy/work/
 lib/galaxy_ext/
 lib/galaxy_utils/
@@ -40,22 +51,28 @@ lib/tool_shed/repository_types/
 lib/tool_shed/tools/
 lib/tool_shed/util/
 lib/tool_shed/utility_containers/
-scripts/api/common.py
-scripts/api/display.py
-scripts/api/workflow_execute_parameters.py
+scripts/api/
 scripts/auth/
+scripts/bootstrap_history.py
+scripts/build_toolbox.py
+scripts/check_eggs.py
+scripts/check_galaxy.py
+scripts/check_python.py
 scripts/cleanup_datasets/admin_cleanup_datasets.py
 scripts/cleanup_datasets/cleanup_datasets.py
-test/api/test_workflows_from_yaml.py
-test/base/
-test/casperjs/
-test/functional/
-test/integration/
-test/manual/
-test/unit/tools/test_actions.py
-test/unit/workflows/test_run_parameters.py
+scripts/cleanup_datasets/pgcleanup.py
+scripts/cleanup_datasets/populate_uuid.py
+scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py
+scripts/cleanup_datasets/rename_purged_datasets.py
+scripts/cleanup_datasets/update_dataset_size.py
+scripts/cleanup_datasets/update_metadata.py
+scripts/data_libraries/build_whoosh_index.py
+scripts/db_shell.py
+scripts/drmaa_external_runner.py
+test/
 tool_list.py
 tools/data_source/
 tools/evolution/
 tools/sr_mapping/
+tools/stats/aggregate_scores_in_intervals.py
 tools/visualization/

Makefile

@@ -12,6 +12,7 @@ VENV?=.venv
 IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
 PROJECT_URL?=https://github.com/galaxyproject/galaxy
 GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
+GRUNT_EXEC?=node_modules/grunt-cli/bin/grunt
 
 all: help
 	@echo "This makefile is primarily used for building Galaxy's JS client. A sensible all target is not yet implemented."
@@ -59,10 +60,13 @@ npm-deps: ## Install NodeJS dependencies.
 	cd client && npm install
 
 grunt: npm-deps ## Calls out to Grunt to build client
-	cd client && node_modules/grunt-cli/bin/grunt
+	cd client && $(GRUNT_EXEC)
 
 style: npm-deps ## Calls the style task of Grunt
-	cd client && node_modules/grunt-cli/bin/grunt style
+	cd client && $(GRUNT_EXEC) style
+
+client-install-libs: npm-deps ## Fetch updated client dependencies using bower.
+	cd client && $(GRUNT_EXEC) install-libs
 
 client: grunt style ## Rebuild all client-side artifacts
 

client/galaxy/scripts/mvc/dataset/data.js

@@ -64,15 +64,17 @@ var TabularDataset = Dataset.extend({
     defaults: _.extend({}, Dataset.prototype.defaults, {
         chunk_url: null,
         first_data_chunk: null,
-        chunk_index: -1,
+        offset: 0,
         at_eof: false
     }),
 
     initialize: function(options) {
         Dataset.prototype.initialize.call(this);
 
         // If first data chunk is available, next chunk is 1.
-        this.attributes.chunk_index = (this.attributes.first_data_chunk ? 1 : 0);
+        if (this.attributes.first_data_chunk){
+            this.attributes.offset = this.attributes.first_data_chunk.offset;
+        }
         this.attributes.chunk_url = Galaxy.root + 'dataset/display?dataset_id=' + this.id;
         this.attributes.url_viz = Galaxy.root + 'visualization';
     },
@@ -90,12 +92,13 @@ var TabularDataset = Dataset.extend({
         var self = this,
             next_chunk = $.Deferred();
         $.getJSON(this.attributes.chunk_url, {
-            chunk: self.attributes.chunk_index++
+            offset: self.attributes.offset
         }).success(function(chunk) {
             var rval;
             if (chunk.ck_data !== '') {
                 // Found chunk.
                 rval = chunk;
+                self.attributes.offset = chunk.offset;
             }
             else {
                 // At EOF.

client/galaxy/scripts/mvc/tool/tool-form-composite.js

@@ -1,8 +1,9 @@
 /** This is the run workflow tool form view. */
-define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view', 'mvc/form/form-data', 'mvc/tool/tool-form-base' ],
-    function( Utils, Deferred, Ui, Form, FormData, ToolFormBase ) {
+define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view', 'mvc/form/form-data', 'mvc/tool/tool-form-base', 'mvc/ui/ui-modal' ],
+    function( Utils, Deferred, Ui, Form, FormData, ToolFormBase, Modal ) {
     var View = Backbone.View.extend({
         initialize: function( options ) {
+            this.modal = parent.Galaxy.modal || new Modal.View();
             this.model = options && options.model || new Backbone.Model( options );
             this.deferred = new Deferred();
             this.setElement( $( '<div/>' ).addClass( 'ui-form-composite' )
@@ -330,7 +331,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
             var self = this;
             var job_def = {
                 new_history_name    : this.history_form.data.create()[ 'new_history|name' ],
-                wf_parm             : this.wp_form ? this.wp_form.data.create() : {},
+                replacement_params  : this.wp_form ? this.wp_form.data.create() : {},
                 inputs              : {}
             };
             var validated = true;
@@ -371,7 +372,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
                 Galaxy.emit.debug( 'tool-form-composite::submit()', 'Validation complete.', job_def );
                 Utils.request({
                     type    : 'POST',
-                    url     : Galaxy.root + 'api/workflows/' + this.workflow_id + '/invocations',
+                    url     : Galaxy.root + 'api/workflows/' + this.model.id + '/run',
                     data    : job_def,
                     success : function( response ) {
                         Galaxy.emit.debug( 'tool-form-composite::submit', 'Submission successful.', response );
@@ -393,13 +394,12 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
                                 }
                             }
                         } else {
-                            var modal = parent.Galaxy.modal;
-                            modal && modal.show({
+                            self.modal.show({
                                 title   : 'Job submission failed',
                                 body    : self._templateError( response && response.err_msg || job_def ),
                                 buttons : {
                                     'Close' : function() {
-                                        modal.hide();
+                                        self.modal.hide();
                                     }
                                 }
                             });

config/galaxy.ini.sample

@@ -418,7 +418,11 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # In use only if activation_grace_period is set.
 #inactivity_box_content = Your account has not been activated yet.  Feel free to browse around and see what's available, but you won't be able to upload data or run jobs until you have verified your email address.
 
-
+# Password expiration period (in days). Users are required to change their
+# password every x days. Users will be redirected to the change password
+# screen when they log in after their password expires. Enter 0 to disable
+# password expiration.
+#password_expiration_period = 0
 
 # Galaxy Session Timeout
 # This provides a timeout (in minutes) after which a user will have to log back in.
@@ -922,7 +926,7 @@ use_interactive = True
 # Allow users to remove their datasets from disk immediately (otherwise,
 # datasets will be removed after a time period specified by an administrator in
 # the cleanup scripts run via cron)
-#allow_user_dataset_purge = False
+#allow_user_dataset_purge = True
 
 # By default, users' data will be public, but setting this to True will cause
 # it to be private.  Does not affect existing users and data, only ones created

config/job_conf.xml.sample_advanced

@@ -174,9 +174,14 @@
             -->
 
             <param id="k8s_config_path">/path/to/kubeconfig</param>
-            <!--- This is the path to the kube config file, which is normally on ~/.kube/config, but that will depend on
+            <!-- This is the path to the kube config file, which is normally on ~/.kube/config, but that will depend on
                  your installation. This is the file that tells the plugin where the k8s cluster is, access credentials,
-                 etc. -->
+                 etc. This parameter is not necessary and ignored if k8s_use_service_account is set to True -->
+
+            <param id="k8s_use_service_account">false</param>
+            <!-- For use when Kubernetes should be accessed from inside a Pod running Galaxy (that is,
+                 galaxy is running inside Kubernetes). If this variable is True, then the previous k8s_config_path is
+                 not required and actually ignored. It is not necessary to set this value if not setting it to true -->
 
             <param id="k8s_persistent_volume_claim_name">galaxy_pvc</param>
             <!-- The name of the Persisten Volume Claim (PVC) to be used, details above, needs to match the PVC's

config/plugins/interactive_environments/jupyter/config/jupyter.xml

@@ -6,6 +6,7 @@
             <model_class>HistoryDatasetAssociation</model_class>
             <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test>
             <test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test>
+            <test type="isinstance" test_attr="datatype" result_type="datatype">binary.Binary</test>
             <to_param param_attr="id">dataset_id</to_param>
         </data_source>
     </data_sources>

lib/galaxy/config.py

@@ -113,6 +113,7 @@ def __init__( self, **kwargs ):
 
         self.expose_user_name = kwargs.get( "expose_user_name", False )
         self.expose_user_email = kwargs.get( "expose_user_email", False )
+        self.password_expiration_period = timedelta( days=int( kwargs.get( "password_expiration_period", 0 ) ) )
 
         # Check for tools defined in the above non-shed tool configs (i.e., tool_conf.xml) tht have
         # been migrated from the Galaxy code distribution to the Tool Shed.
@@ -162,7 +163,7 @@ def __init__( self, **kwargs ):
         self.require_login = string_as_bool( kwargs.get( "require_login", "False" ) )
         self.allow_user_creation = string_as_bool( kwargs.get( "allow_user_creation", "True" ) )
         self.allow_user_deletion = string_as_bool( kwargs.get( "allow_user_deletion", "False" ) )
-        self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "False" ) )
+        self.allow_user_dataset_purge = string_as_bool( kwargs.get( "allow_user_dataset_purge", "True" ) )
         self.allow_user_impersonation = string_as_bool( kwargs.get( "allow_user_impersonation", "False" ) )
         self.new_user_dataset_access_role_default_private = string_as_bool( kwargs.get( "new_user_dataset_access_role_default_private", "False" ) )
         self.collect_outputs_from = [ x.strip() for x in kwargs.get( 'collect_outputs_from', 'new_file_path,job_working_directory' ).lower().split(',') ]

lib/galaxy/dataset_collections/structure.py

@@ -119,12 +119,12 @@ def multiply( self, other_structure ):
         return Tree( new_children, new_collection_type )
 
     def clone( self ):
-        cloned_children = map( lambda (identifier, element): (identifier, element.clone()), self.children )
+        cloned_children = [(_[0], _[1].clone()) for _ in self.children]
         return Tree( cloned_children, self.collection_type_description )
 
 
 def dict_map( func, input_dict ):
-    return dict( [ ( k, func(v) ) for k, v in input_dict.iteritems() ] )
+    return dict( [ ( k, func(v) ) for k, v in input_dict.items() ] )
 
 
 def get_structure( dataset_collection_instance, collection_type_description, leaf_subcollection_type=None ):

lib/galaxy/datatypes/binary.py

@@ -1,4 +1,5 @@
 """Binary classes"""
+from __future__ import print_function
 
 import binascii
 import gzip
@@ -72,7 +73,7 @@ def get_mime( self ):
         """Returns the mime type of the datatype"""
         return 'application/octet-stream'
 
-    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, size=None, offset=None, **kwd):
+    def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
         trans.response.set_content_type(dataset.get_mime())
         trans.log_event( "Display dataset id: %s" % str( dataset.id ) )
         trans.response.headers['Content-Length'] = int( os.stat( dataset.file_name ).st_size )
@@ -234,7 +235,7 @@ def merge(split_files, output_file):
                 shutil.rmtree(tmp_dir)  # clean up
                 raise Exception( "Error merging BAM files: %s" % stderr )
             else:
-                print stderr
+                print(stderr)
         os.unlink(stderr_name)
         os.rmdir(tmp_dir)
 
@@ -322,7 +323,7 @@ def groom_dataset_content( self, file_name ):
                 shutil.rmtree( tmp_dir)  # clean up
                 raise Exception( "Error Grooming BAM file contents: %s" % stderr )
             else:
-                print stderr
+                print(stderr)
         # Move samtools_created_sorted_file_name to our output dataset location
         shutil.move( samtools_created_sorted_file_name, file_name )
         # Remove temp file and empty temporary directory
@@ -365,7 +366,7 @@ def set_meta( self, dataset, overwrite=True, **kwd ):
                 os.unlink( stderr_name )  # clean up
                 raise Exception( "Error Setting BAM Metadata: %s" % stderr )
             else:
-                print stderr
+                print(stderr)
         dataset.metadata.bam_index = index_file
         # Remove temp file
         os.unlink( stderr_name )
@@ -932,7 +933,7 @@ def sniff( self, filename ):
                 c = conn.cursor()
                 tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
                 result = c.execute( tables_query ).fetchall()
-                result = map( lambda x: x[0], result )
+                result = [_[0] for _ in result]
                 for table_name in gemini_table_names:
                     if table_name not in result:
                         return False
@@ -971,7 +972,7 @@ def sniff( self, filename ):
                 c = conn.cursor()
                 tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
                 result = c.execute( tables_query ).fetchall()
-                result = map( lambda x: x[0], result )
+                result = [_[0] for _ in result]
                 for table_name in mz_table_names:
                     if table_name not in result:
                         return False
@@ -1006,7 +1007,7 @@ def sniff( self, filename ):
                 c = conn.cursor()
                 tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
                 result = c.execute( tables_query ).fetchall()
-                result = map( lambda x: x[0], result )
+                result = [_[0] for _ in result]
                 for table_name in mz_table_names:
                     if table_name not in result:
                         return False

lib/galaxy/datatypes/converters/bed_to_gff_converter.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # This code exists in 2 places: ~/datatypes/converters and ~/tools/filters
+from __future__ import print_function
+
 import sys
 
 assert sys.version_info[:2] >= ( 2, 4 )
@@ -69,7 +71,7 @@ def __main__():
     info_msg = "%i lines converted to GFF version 2.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
         info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
-    print info_msg
+    print(info_msg)
 
 if __name__ == "__main__":
     __main__()

lib/galaxy/datatypes/converters/fastq_to_fqtoc.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
+from __future__ import print_function
 
 import sys
+
 from galaxy.util.checkers import is_gzip
 
 
@@ -18,7 +20,7 @@ def main():
     """
     input_fname = sys.argv[1]
     if is_gzip(input_fname):
-        print 'Conversion is only possible for uncompressed files'
+        print('Conversion is only possible for uncompressed files')
         sys.exit(1)
 
     out_file = open(sys.argv[2], 'w')

lib/galaxy/datatypes/converters/gff_to_bed_converter.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import sys
 
 assert sys.version_info[:2] >= ( 2, 4 )
@@ -38,7 +40,7 @@ def __main__():
     info_msg = "%i lines converted to BED.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
         info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
-    print info_msg
+    print(info_msg)
 
 if __name__ == "__main__":
     __main__()