From 6d3c6c027f906389ebec42e247e84ac54ed3784a Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Thu, 16 Nov 2023 10:22:47 +0100 Subject: [PATCH 1/7] Add variables for usegalaxy.star servers --- _plugins/gtn.rb | 9 +++++++++ _plugins/gtn/supported.rb | 7 ++----- _plugins/gtn/usegalaxy.rb | 15 +++++++++++++++ bin/workflow-test.rb | 10 +++++++--- bin/workflows-fetch.rb | 14 +++++++------- faqs/galaxy/account_create.md | 6 +++++- workflows/list.html | 4 +++- 7 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 _plugins/gtn/usegalaxy.rb diff --git a/_plugins/gtn.rb b/_plugins/gtn.rb index e1659b71f29b87..34b5947fc6a1a2 100644 --- a/_plugins/gtn.rb +++ b/_plugins/gtn.rb @@ -10,6 +10,7 @@ require './_plugins/gtn/scholar' require './_plugins/gtn/supported' require './_plugins/gtn/toolshed' +require './_plugins/gtn/usegalaxy' require './_plugins/jekyll-topic-filter' require 'time' @@ -375,6 +376,14 @@ def get_version_number(page) Gtn::ModificationTimes.obtain_modification_count(page['path']) end + def list_usegalaxy_servers(site) + Gtn::Usegalaxy.servers.map{|x| x.transform_keys(&:to_s)} + end + + def list_usegalaxy_servers_shuffle(site) + Gtn::Usegalaxy.servers.map{|x| x.transform_keys(&:to_s)}.shuffle + end + def topic_name_from_page(page, site) if page.key? 'topic_name' site.data[page['topic_name']]['title'] diff --git a/_plugins/gtn/supported.rb b/_plugins/gtn/supported.rb index 0931ca4b5d891b..7d1c4b1754d9b6 100644 --- a/_plugins/gtn/supported.rb +++ b/_plugins/gtn/supported.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +require './_plugins/gtn/usegalaxy' module Gtn # Handle tool support queries @@ -21,11 +22,7 @@ def self.calculate(data, tool_list) if data.nil? || data.empty? || tool_list.empty? || tool_list.nil? return { 'exact' => [], - 'inexact' => [ - { 'name' => 'UseGalaxy.eu', 'url' => 'https://usegalaxy.eu', 'usegalaxy' => true }, - { 'name' => 'UseGalaxy.org', 'url' => 'https://usegalaxy.org', 'usegalaxy' => true }, - { 'name' => 'UseGalaxy.org.au', 'url' => 'https://usegalaxy.org.au', 'usegalaxy' => true } - ], + 'inexact' => Gtn::Usegalaxy.servers.map{|x| x['usegalaxy'] = true }, } end diff --git a/_plugins/gtn/usegalaxy.rb b/_plugins/gtn/usegalaxy.rb new file mode 100644 index 00000000000000..3093fbf415c56e --- /dev/null +++ b/_plugins/gtn/usegalaxy.rb @@ -0,0 +1,15 @@ +module Gtn + module Usegalaxy + @@SERVERS = [ + { name: 'UseGalaxy.eu', url: 'https://usegalaxy.eu', id: 'eu', human: 'Galaxy Europe'}, + { name: 'UseGalaxy.org', url: 'https://usegalaxy.org', id: 'us', human: 'Galaxy Main'}, + { name: 'UseGalaxy.org.au', url: 'https://usegalaxy.org.au', id: 'au', human: 'Galaxy Australia'}, + { name: 'UseGalaxy.fr', url: 'https://usegalaxy.fr', id: 'fr', human: 'Galaxy France'} + ] + + def self.servers + @@SERVERS + end + + end +end diff --git a/bin/workflow-test.rb b/bin/workflow-test.rb index c4fc6f0bbe51ac..cda54e94d8f36e 100755 --- a/bin/workflow-test.rb +++ b/bin/workflow-test.rb @@ -1,12 +1,16 @@ #!/usr/bin/env ruby # frozen_string_literal: true +require './_plugins/gtn/usegalaxy' require 'open3' require 'json' -GALAXIES = { - eu: { url: 'https://usegalaxy.eu', key: ENV.fetch('GALAXY_EU_KEY', 'NONE') }, -} +GALAXIES = Gtn::Usegalaxy.servers.select{|s| s[:id] == "eu"}.map do |server| + [ + server[:id], + { url: server[:url], key: ENV.fetch("GALAXY_#{server[:id].upcase}_KEY", 'NONE') } + ] +end.to_h def test_workflow(workflow_file, galaxy_id) directory = File.dirname(workflow_file) diff --git a/bin/workflows-fetch.rb b/bin/workflows-fetch.rb index e7dbb38b735cbd..c42d9059a24537 100755 --- a/bin/workflows-fetch.rb +++ b/bin/workflows-fetch.rb @@ -3,6 +3,8 @@ require 'net/http' require 'uri' require 'yaml' +require './_plugins/gtn/usegalaxy' + def request(url) uri = URI.parse(url) @@ -86,13 +88,11 @@ def fetch_workflowhub() # Parse the response -workflows_eu = fetch_workflows('https://usegalaxy.eu') -puts "INFO: Fetched #{workflows_eu.length} workflows from EU" -workflows_org = fetch_workflows("https://usegalaxy.org") -puts "INFO: Fetched #{workflows_org.length} workflows from ORG" -workflows_aus = fetch_workflows("https://usegalaxy.org.au") -puts "INFO: Fetched #{workflows_aus.length} workflows from AUS" -workflows = workflows_eu + workflows_org + workflows_aus +workflows = Gtn::Usegalaxy.servers.map {|server| + workflows = fetch_workflows(server[:url]) + puts "INFO: Fetched #{workflows.length} workflows from #{server[:name]}" + workflows +}.flatten # Cleanup the list workflows.filter! do |w| diff --git a/faqs/galaxy/account_create.md b/faqs/galaxy/account_create.md index b15a4690bc61e6..83b297f510b118 100644 --- a/faqs/galaxy/account_create.md +++ b/faqs/galaxy/account_create.md @@ -9,7 +9,11 @@ contributors: [jennaj, bernandez, samanthaanjei, wm75] 1. To create an account at any public Galaxy instance, choose your server from the available [list of Galaxy Platforms](https://galaxyproject.org/use/). - There are 3 main public Galaxy servers: [UseGalaxy.org](https://usegalaxy.org/) (**US**), [UseGalaxy.eu](https://usegalaxy.eu/) (**EU**), and [UseGalaxy.org.au](https://usegalaxy.org.au/) (**AU**). + There are several UseGalaxy servers: + {% assign servers = nil | list_usegalaxy_servers_shuffle %} + {% for server in servers %} + - [{{ server.name }}]({{ server.url }}) (**{{ server.id | upcase }}**) + {% endfor %} 2. Click on **"Login or Register"** in the masthead on the server. diff --git a/workflows/list.html b/workflows/list.html index 2f3b6bb8ae603b..424bc124b05184 100644 --- a/workflows/list.html +++ b/workflows/list.html @@ -9,7 +9,9 @@

This site currently lists {{ site.data['workflows'] | size }} workflows from -UseGalaxy.org, UseGalaxy.eu, UseGalaxy.org.au, and WorkflowHub.eu. It is updated weekly. +{% assign servers = nil | list_usegalaxy_servers %} +{% for server in servers %}{{ server.name }}, {% endfor %} +and WorkflowHub.eu. It is updated weekly.

From 51e254a6a1e9a3ef5d02a52f21e25d8c3130b592 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Thu, 16 Nov 2023 10:23:01 +0100 Subject: [PATCH 2/7] template out other uses --- _layouts/workflow-list.html | 7 ++++--- faqs/galaxy/support_admins.md | 1 + faqs/gtn/contributors_getting_started.md | 2 +- faqs/gtn/contributors_tours.md | 2 +- short/galaxy.html | 7 ++++--- topics/admin/tutorials/introduction/slides.html | 4 +++- topics/galaxy-interface/tutorials/ncbi-sarf/tutorial.md | 3 ++- .../tutorials/galaxy-intro-ngs-data-managment/tutorial.md | 2 +- .../introduction/tutorials/galaxy-intro-short/slides.html | 3 ++- .../tutorials/galaxy-intro-short/slides_CAT_ES.html | 3 ++- .../introduction/tutorials/galaxy-intro-short/tutorial.md | 4 +--- topics/introduction/tutorials/galaxy-reproduce/tutorial.md | 5 +++-- .../tutorials/setup-tiaas-for-training/tutorial.md | 2 +- topics/teaching/tutorials/workshop-intro/slides.html | 7 ++++--- .../tutorials/rna-seq-reads-to-counts/tutorial.md | 4 +++- 15 files changed, 33 insertions(+), 23 deletions(-) diff --git a/_layouts/workflow-list.html b/_layouts/workflow-list.html index df04ecd2b7dcbc..6ba7d4de7cdef0 100644 --- a/_layouts/workflow-list.html +++ b/_layouts/workflow-list.html @@ -48,9 +48,10 @@
diff --git a/faqs/galaxy/support_admins.md b/faqs/galaxy/support_admins.md index b0baf3b3aa50ee..968a544deb4611 100644 --- a/faqs/galaxy/support_admins.md +++ b/faqs/galaxy/support_admins.md @@ -13,4 +13,5 @@ If you suspect there is something wrong with the server, or would like to reques - **Galaxy US:** [Email](mailto:help@galaxyproject.org), [Gitter channel](https://gitter.im/galaxyproject/Lobby) - **Galaxy EU:** [Gitter channel](https://gitter.im/usegalaxy-eu/Lobby), [Request TIaaS](https://usegalaxy.eu/tiaas/) - **Galaxy AU:** [Email](mail:help@genome.edu.au), [Request a tool](https://request.usegalaxy.org.au/), [Request Data Quota](https://docs.google.com/forms/d/e/1FAIpQLSeiw6ajmkezLCwbXc3OFQEU3Ai9hGnBd967u9YbQ8ANPgvatA/viewform) + - **Galaxy FR:** [Request TIaaS](https://usegalaxy.fr/tiaas/) - Other Galaxy servers? Check the homepage for more information. diff --git a/faqs/gtn/contributors_getting_started.md b/faqs/gtn/contributors_getting_started.md index bb8203da69fd1a..654517fd7aa3bc 100644 --- a/faqs/gtn/contributors_getting_started.md +++ b/faqs/gtn/contributors_getting_started.md @@ -15,7 +15,7 @@ Below is a checklist of things to look out for to help you get started. If you f Basic: - **Test** the tutorial on a running Galaxy instance - - For example [Galaxy Main](https://usegalaxy.org), [Galaxy Europe](https://usegalaxy.eu), or [Galaxy Australia](https://usegalaxy.org.au) + - For example {% assign servers = list_usegalaxy_servers_shuffle %}{% for server in servers %}[{{ server.name }}]({{ server.url }}){% if forloop.last %}{% else %}, {% endif %}{% endfor %} - Report any issues you run into - **Language** editing - Fix spelling and grammar mistakes diff --git a/faqs/gtn/contributors_tours.md b/faqs/gtn/contributors_tours.md index 9108dceff0fb12..daee90427075fb 100644 --- a/faqs/gtn/contributors_tours.md +++ b/faqs/gtn/contributors_tours.md @@ -9,7 +9,7 @@ contributors: [shiltemann] Perhaps you've been asked to review an interactive tour, or maybe you just want to try one out. The easiest way to run an interactive tour is to use the [Tour builder](https://tailordev.fr/blog/2017/07/19/the-galaxy-tour-builder-extension/) browser extension. 1. Install the Tour Builder extension to your browser ([Chrome Web Store](https://chrome.google.com/webstore/detail/galaxy-tour-builder/mdfbapknmcpnbmggahhaegehbbbmhmgg), [Firefox add-on](https://addons.mozilla.org/en-US/firefox/addon/galaxy-tour-builder/)). -2. Navigate to a Galaxy instance supporting the tutorial. To find which Galaxy instances support each tutorial, please see the dropdown menu next to the tutorial on the training website. Using one of the usegalaxy.\* instances ([Galaxy Main](https://usegalaxy.org), [Galaxy Europe](https://usegalaxy.eu), or [Galaxy Australia](https://usegalaxy.org.au) +2. Navigate to a Galaxy instance supporting the tutorial. To find which Galaxy instances support each tutorial, please see the dropdown menu next to the tutorial on the training website. Using one of the usegalaxy.\* instances ({% assign servers = list_usegalaxy_servers_shuffle %}{% for server in servers %}[{{ server.name }}]({{ server.url }}){% if forloop.last %}{% else %}, {% endif %}{% endfor %}) ) is usually a good bet. 3. Start the Tour Builder plugin by clicking on the icon in your browser menu bar 4. Copy the contents of the `tour.yaml` file into the Tour builder editor window diff --git a/short/galaxy.html b/short/galaxy.html index ba5c2887bed5a2..dfc434e88dfca5 100644 --- a/short/galaxy.html +++ b/short/galaxy.html @@ -25,9 +25,10 @@

Go!

Select your Preferred Galaxy Server

- UseGalaxy.eu 🌍πŸ‡ͺπŸ‡Ί - UseGalaxy.org πŸŒŽπŸ‡ΊπŸ‡Έ - UseGalaxy.org.au 🌏 + {% assign servers = list_usegalaxy_servers_shuffle %} + {% for server in servers %} + {{ server.human }} ({{ server.id | upcase }}) + {% endfor %}

diff --git a/topics/admin/tutorials/introduction/slides.html b/topics/admin/tutorials/introduction/slides.html index 32a245be77eb3d..fdd9a63a6f419d 100644 --- a/topics/admin/tutorials/introduction/slides.html +++ b/topics/admin/tutorials/introduction/slides.html @@ -33,8 +33,10 @@ # Where can Galaxy run? +{% assign servers = list_usegalaxy_servers_shuffle %} + * Cloud (SaaS) - - A usegalaxy.* site [Main](https://usegalaxy.org) [Europe](https://usegalaxy.eu) [Australia](https://usegalaxy.org.au/) + - A usegalaxy.* site: {% for server in servers %}[{{ server.human }}]({{ server.url }}){% if forloop.last %}{% else %}, {% endif %}{% endfor %} - [Public Galaxy Servers](https://galaxyproject.org/use) - [Amazon EC2 or MS Azure](https://launch.usegalaxy.org/) - Semi-private cloud (e.g.: [NeCTAR](https://nectar.org.au/), [Jetstream](http://jetstream-cloud.org/)) diff --git a/topics/galaxy-interface/tutorials/ncbi-sarf/tutorial.md b/topics/galaxy-interface/tutorials/ncbi-sarf/tutorial.md index 43e27d18afa038..2e25fbca2f6997 100644 --- a/topics/galaxy-interface/tutorials/ncbi-sarf/tutorial.md +++ b/topics/galaxy-interface/tutorials/ncbi-sarf/tutorial.md @@ -104,12 +104,13 @@ We will import the JSON files into Galaxy to query them directory, however the f > {: .comment} +{% assign servers = list_usegalaxy_servers_shuffle %} > Loading SRA Aligned Read Format (SARF) Object Metadata URLs into Galaxy > >This step needs to be repeated at the beginning of an analysis to refresh the metadata to the latest daily version. > -> 1. Go to your Galaxy instance of choice such as one of the [usegalaxy.org](https://usegalaxy.org/), [usegalaxy.eu](https://usegalaxy.eu), [usegalaxy.org.au](https://usegalaxy.org.au) or any other. +> 1. Go to your Galaxy instance of choice such as one of the {% for server in servers %}[{{ server.name }}]({{ server.url }}){% if forloop.last %}{% else %}, {% endif %}{% endfor %} or any other. > > 2. Create a new history > diff --git a/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md b/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md index 9599ece8e5911d..93fad76553ec06 100644 --- a/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md +++ b/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md @@ -448,7 +448,7 @@ Now that we have downloaded this file we can go to a Galaxy instance and start p > Upload `SraRunInfo.csv` file into Galaxy > -> 1. Go to your Galaxy instance of choice such as one of the [usegalaxy.org](https://usegalaxy.org/), [usegalaxy.eu](https://usegalaxy.eu), [usegalaxy.org.au](https://usegalaxy.org.au) or any other. (This tutorial uses usegalaxy.org). +> 1. Go to your Galaxy instance of choice such as one of the [UseGalaxy.* instances](https://galaxyproject.org/usegalaxy/) or any other. (This tutorial uses usegalaxy.org). > 1. Click *Upload Data* button: > ![Data upload button](../../images/upload_button.png) > 1. In the dialog box that would appear click "*Choose local files*" button: diff --git a/topics/introduction/tutorials/galaxy-intro-short/slides.html b/topics/introduction/tutorials/galaxy-intro-short/slides.html index 34b4607757e91a..3fa21260fd0a1a 100644 --- a/topics/introduction/tutorials/galaxy-intro-short/slides.html +++ b/topics/introduction/tutorials/galaxy-intro-short/slides.html @@ -101,7 +101,8 @@ # Find a Galaxy server -- **The Big Three**: Galaxy Main ([usegalaxy.org](https://usegalaxy.org)), Galaxy Europe ([usegalaxy.eu](https://usegalaxy.eu)), Galaxy Australia ([usegalaxy.org.au](https://usegalaxy.org.au)) +{% assign servers = list_usegalaxy_servers_shuffle %} +- **UseGalaxy.\***: {% for server in servers %}{{ server.human }} ([{{ server.name }}]({{ server.url }})){% if forloop.last %}{% else %}, {% endif %}{% endfor %} .image-50[![usegalaxy.* servers]({{site.baseurl}}/shared/images/usegalaxy_servers.png)] diff --git a/topics/introduction/tutorials/galaxy-intro-short/slides_CAT_ES.html b/topics/introduction/tutorials/galaxy-intro-short/slides_CAT_ES.html index 87a6d439783635..4c78d3ffbaf491 100644 --- a/topics/introduction/tutorials/galaxy-intro-short/slides_CAT_ES.html +++ b/topics/introduction/tutorials/galaxy-intro-short/slides_CAT_ES.html @@ -108,7 +108,8 @@ # Encuentra un servidor Galaxy -- ** Los tres grandes **: Galaxy Main ([usegalaxy.org](https://usegalaxy.org)), Galaxy Europe ([usegalaxy.eu](https://usegalaxy.eu)), Galaxy Australia ([usegalaxy.org.au](https://usegalaxy.org.au)) +{% assign servers = list_usegalaxy_servers_shuffle %} +- ** Los tres grandes **: {% for server in servers %}{{ server.human }} ([{{ server.name }}]({{ server.url }})){% if forloop.last %}{% else %}, {% endif %}{% endfor %} .image-75[![usegalaxy.* servidores]({{site.baseurl}}/shared/images/usegalaxy_servers.png)] diff --git a/topics/introduction/tutorials/galaxy-intro-short/tutorial.md b/topics/introduction/tutorials/galaxy-intro-short/tutorial.md index 59bb210593a812..9655e6c24c2af7 100644 --- a/topics/introduction/tutorials/galaxy-intro-short/tutorial.md +++ b/topics/introduction/tutorials/galaxy-intro-short/tutorial.md @@ -62,9 +62,7 @@ contributions: > > Different Galaxy servers > > This is an image of Galaxy Australia, located at [usegalaxy.org.au](https://usegalaxy.org.au/) > > -> > The particular Galaxy server that you are using may look slightly different and have a different web address: -> > - The main Galaxy server is [usegalaxy.org](https://usegalaxy.org/) -> > - The European Galaxy server is [usegalaxy.eu](https://usegalaxy.eu/) +> > The particular Galaxy server that you are using may look slightly different and have a different web address. > > > > You can also find more possible Galaxy servers at the top of this tutorial in **Available on these Galaxies** > {: .comment} diff --git a/topics/introduction/tutorials/galaxy-reproduce/tutorial.md b/topics/introduction/tutorials/galaxy-reproduce/tutorial.md index 4bcba9d099232b..6f527dab87a60b 100644 --- a/topics/introduction/tutorials/galaxy-reproduce/tutorial.md +++ b/topics/introduction/tutorials/galaxy-reproduce/tutorial.md @@ -40,12 +40,13 @@ For more background information about Galaxy, have a look into the Galaxy public # What does Galaxy look like? -Many different Galaxy servers exist. Some are public, some are private, some focus on a specific topic and others like the usegalaxy.\* servers cover a broad range of tools. To reproduce published results it is highly recommended to use the same Galaxy server that was used in the original study. In the case that this was a private server that is not accessible to you, you might want to use one of the main Galaxy servers: [usegalaxy.org](https://usegalaxy.org), [usegalaxy.eu](https://usegalaxy.eu), [usegalaxy.org.au](https://usegalaxy.org.au). To learn more about the different Galaxy servers visit the [slides: options for using Galaxy]({{site.baseurl}}/topics/introduction/tutorials/options-for-using-galaxy/slides.html#1). The particular Galaxy server that you are using may look slightly different than the one shown in this training. Galaxy instance administrators can choose the exact version of Galaxy they would like to offer and can customize its look to some extent. The basic functionality will be rather similar across instances, so don’t worry! In this training we will use the European Galaxy server on which the original analysis was performed and shared. +{% assign servers = list_usegalaxy_servers_shuffle %} +Many different Galaxy servers exist. Some are public, some are private, some focus on a specific topic and others like the usegalaxy.\* servers cover a broad range of tools. To reproduce published results it is highly recommended to use the same Galaxy server that was used in the original study. In the case that this was a private server that is not accessible to you, you might want to use one of the main Galaxy servers: {% for server in servers %}[{{ server.name }}]({{ server.url }}){% if forloop.last %}{% else %}, {% endif %}{% endfor %}. To learn more about the different Galaxy servers visit the [slides: options for using Galaxy]({% link /topics/introduction/tutorials/options-for-using-galaxy/slides.html %}). The particular Galaxy server that you are using may look slightly different than the one shown in this training. Galaxy instance administrators can choose the exact version of Galaxy they would like to offer and can customize its look to some extent. The basic functionality will be rather similar across instances, so don’t worry! In this training we will use the European Galaxy server on which the original analysis was performed and shared. > Log in or register > 1. Open your favorite browser (Chrome/Chromium, Safari or Firefox, but not Internet Explorer/Edge!) -> 2. Browse to the [Galaxy Europe instance](https://usegalaxy.eu/) (recommended) or to a Galaxy instance of your choosing +> 2. Browse to the [Galaxy Europe instance](https://usegalaxy.eu/) or to a Galaxy instance of your choosing > 3. Choose *Login or Register* from the navigation bar at the top of the page > 4. If you have previously registered an account with this particular instance of Galaxy (user accounts are *not* shared between public servers!), proceed by logging in with your registered *public name*, or email address, and your password. > diff --git a/topics/teaching/tutorials/setup-tiaas-for-training/tutorial.md b/topics/teaching/tutorials/setup-tiaas-for-training/tutorial.md index 89248df61e1f5d..3a58aefcc180d3 100644 --- a/topics/teaching/tutorials/setup-tiaas-for-training/tutorial.md +++ b/topics/teaching/tutorials/setup-tiaas-for-training/tutorial.md @@ -12,7 +12,7 @@ objectives: - Identify if it is appropriate - Interact with the service administrators to arrange for infrastructure key_points: - - Infrastructure is available for running Galaxy trainings for free from UseGalaxy.eu, UseGalaxy.org, and UseGalaxy.org.au + - Infrastructure is available for running Galaxy trainings for free from UseGalaxy.* - This can be easier than setting up a local Galaxy and may have more resources available - But have a backup plan contributors: diff --git a/topics/teaching/tutorials/workshop-intro/slides.html b/topics/teaching/tutorials/workshop-intro/slides.html index 9c790a2ddd5677..f0b783cb8a58f0 100644 --- a/topics/teaching/tutorials/workshop-intro/slides.html +++ b/topics/teaching/tutorials/workshop-intro/slides.html @@ -245,9 +245,10 @@ - Training materials: https://training.galaxyproject.org - Galaxy (an instructor will tell you which Galaxy to use) - - Main: https://usegalaxy.org - - EU: https://usegalaxy.eu - - Australia: https://usegalaxy.org.au + {% assign servers = list_usegalaxy_servers_shuffle %} + {% for server in servers %} + - {{ server.name }} {{ server.url }} + {% endfor %} - [Other](https://galaxyproject.org/use/) - Register/log in - You may receive an e-mail with an activation link diff --git a/topics/transcriptomics/tutorials/rna-seq-reads-to-counts/tutorial.md b/topics/transcriptomics/tutorials/rna-seq-reads-to-counts/tutorial.md index 51c67566ae5422..3482ca87d75592 100644 --- a/topics/transcriptomics/tutorials/rna-seq-reads-to-counts/tutorial.md +++ b/topics/transcriptomics/tutorials/rna-seq-reads-to-counts/tutorial.md @@ -82,7 +82,9 @@ This is a Galaxy tutorial based on material from the [COMBINE R RNAseq workshop] ## Import data from URLs -Read sequences are usually stored in compressed (gzipped) FASTQ files. Before the differential expression analysis can proceed, these reads must be aligned to the reference genome and counted into annotated genes. Mapping reads to the genome is a very important task, and many different aligners are available, such as HISAT2 ({% cite kim2015hisat %}), STAR ({% cite dobin2013star %}) and Subread ({% cite Liao2013 %}). Most mapping tasks require larger computers than an average laptop, so usually read mapping is done on a server in a linux-like environment, requiring some programming knowledge. However, Galaxy enables you to do this mapping without needing to know programming and if you don't have access to a server you can try to use one of the publically available Galaxies e.g. [usegalaxy.org](https://usegalaxy.org), [usegalaxy.eu](https://usegalaxy.eu), [usegalaxy.org.au](https://usegalaxy.org.au/). +{% assign servers = list_usegalaxy_servers_shuffle %} + +Read sequences are usually stored in compressed (gzipped) FASTQ files. Before the differential expression analysis can proceed, these reads must be aligned to the reference genome and counted into annotated genes. Mapping reads to the genome is a very important task, and many different aligners are available, such as HISAT2 ({% cite kim2015hisat %}), STAR ({% cite dobin2013star %}) and Subread ({% cite Liao2013 %}). Most mapping tasks require larger computers than an average laptop, so usually read mapping is done on a server in a linux-like environment, requiring some programming knowledge. However, Galaxy enables you to do this mapping without needing to know programming and if you don't have access to a server you can try to use one of the publically available Galaxies e.g. {% for server in servers %}[{{ server.name }}]({{ server.url }}){% if forloop.last %}{% else %}, {% endif %}{% endfor %}. The raw reads used in this tutorial were obtained from SRA from the link in GEO for the the mouse mammary gland dataset (e.g `ftp://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByStudy/sra/SRP%2FSRP045%2FSRP045534`). For the purpose of this tutorial we are going to be working with a small part of the FASTQ files. We are only going to be mapping 1000 reads from each sample to enable running through all the steps quickly. If working with your own data you would use the full data and some results for the full mouse dataset will be shown for comparison. The small FASTQ files are available in [Zenodo](https://zenodo.org/record/4249555) and the links to the FASTQ files are provided below. From 133b97b4f870bcbab4beb05cdb3c60cca6a54270 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 20 Dec 2023 12:57:03 +0100 Subject: [PATCH 3/7] fix heading issues --- .../tutorials/galaxy-intro-ngs-data-managment/tutorial.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md b/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md index 93fad76553ec06..27b8de143277d5 100644 --- a/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md +++ b/topics/introduction/tutorials/galaxy-intro-ngs-data-managment/tutorial.md @@ -28,8 +28,6 @@ contributors: In this section we will look at practical aspects of manipulation of next-generation sequencing data. We will start with the FASTQ format produced by most sequencing machines and will finish with the SAM/BAM format representing mapped reads. The cover image above shows a screen dump of a SAM dataset. -# Introduction to sequencing data - ## FASTQ manipulation and quality control [FASTQ](https://en.wikipedia.org/wiki/FASTQ_format) is not a very well defined format. In the beginning various manufacturers of sequencing instruments were free to interpret FASTQ as they saw fit, resulting in a multitude of FASTQ flavors. This variation stemmed primarily from different ways of encoding quality values as described [on the Wikipedia article for FASTQ](https://en.wikipedia.org/wiki/FASTQ_format) (below you will find an explanation of quality scores and their meaning). Today, the [FASTQ Sanger](https://www.ncbi.nlm.nih.gov/pubmed/20015970) version of the format is considered to be the standard form of FASTQ. Galaxy is using FASTQ Sanger as the only legitimate input for downstream processing tools and provides [a number of utilities for converting FASTQ files](https://www.ncbi.nlm.nih.gov/pubmed/20562416) into this form (see **FASTQ Quality Control** section of Galaxy tools). @@ -73,7 +71,7 @@ It is common to prepare pair-end and mate-pair sequencing libraries. This is hig Thus in both cases (paired-end and mate-pair) a single physical piece of DNA (or RNA in the case of RNA-seq) is sequenced from two ends and so generates two reads. These can be represented as separate files (two FASTQ files with first and second reads) or a single file were reads for each end are interleaved. Here are examples: -#### Two single files +### Two single files File 1 @@ -105,7 +103,7 @@ CACTACCGGGGTATCTAATCCTGTTCGCTCCCCACGCTTTCGTCCATC > Note that read IDs are **identical** in two files and they are listed in **the same** order. In some cases read IDs in the first and second file may be appended with `/1` and `/2` tags, respectively. {: .comment} -#### Interleaved file +### Interleaved file ``` @1/1 From c0c8f53807e54ff547e1912f5bc7c88adf8182fd Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 20 Dec 2023 13:02:05 +0100 Subject: [PATCH 4/7] missing doc --- _plugins/gtn.rb | 8 ++++---- _plugins/gtn/supported.rb | 3 ++- _plugins/gtn/usegalaxy.rb | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/_plugins/gtn.rb b/_plugins/gtn.rb index 34b5947fc6a1a2..6d357b39150d05 100644 --- a/_plugins/gtn.rb +++ b/_plugins/gtn.rb @@ -376,12 +376,12 @@ def get_version_number(page) Gtn::ModificationTimes.obtain_modification_count(page['path']) end - def list_usegalaxy_servers(site) - Gtn::Usegalaxy.servers.map{|x| x.transform_keys(&:to_s)} + def list_usegalaxy_servers(_site) + Gtn::Usegalaxy.servers.map { |x| x.transform_keys(&:to_s) } end - def list_usegalaxy_servers_shuffle(site) - Gtn::Usegalaxy.servers.map{|x| x.transform_keys(&:to_s)}.shuffle + def list_usegalaxy_servers_shuffle(_site) + Gtn::Usegalaxy.servers.map { |x| x.transform_keys(&:to_s) }.shuffle end def topic_name_from_page(page, site) diff --git a/_plugins/gtn/supported.rb b/_plugins/gtn/supported.rb index 7d1c4b1754d9b6..216463f61c2d50 100644 --- a/_plugins/gtn/supported.rb +++ b/_plugins/gtn/supported.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require './_plugins/gtn/usegalaxy' module Gtn @@ -22,7 +23,7 @@ def self.calculate(data, tool_list) if data.nil? || data.empty? || tool_list.empty? || tool_list.nil? return { 'exact' => [], - 'inexact' => Gtn::Usegalaxy.servers.map{|x| x['usegalaxy'] = true }, + 'inexact' => Gtn::Usegalaxy.servers.map { |x| x['usegalaxy'] = true }, } end diff --git a/_plugins/gtn/usegalaxy.rb b/_plugins/gtn/usegalaxy.rb index 3093fbf415c56e..0e92d8efb72acc 100644 --- a/_plugins/gtn/usegalaxy.rb +++ b/_plugins/gtn/usegalaxy.rb @@ -1,15 +1,15 @@ module Gtn + # Data about the UseGalaxy.* servers module Usegalaxy @@SERVERS = [ - { name: 'UseGalaxy.eu', url: 'https://usegalaxy.eu', id: 'eu', human: 'Galaxy Europe'}, - { name: 'UseGalaxy.org', url: 'https://usegalaxy.org', id: 'us', human: 'Galaxy Main'}, - { name: 'UseGalaxy.org.au', url: 'https://usegalaxy.org.au', id: 'au', human: 'Galaxy Australia'}, - { name: 'UseGalaxy.fr', url: 'https://usegalaxy.fr', id: 'fr', human: 'Galaxy France'} + { name: 'UseGalaxy.eu', url: 'https://usegalaxy.eu', id: 'eu', human: 'Galaxy Europe' }, + { name: 'UseGalaxy.org', url: 'https://usegalaxy.org', id: 'us', human: 'Galaxy Main' }, + { name: 'UseGalaxy.org.au', url: 'https://usegalaxy.org.au', id: 'au', human: 'Galaxy Australia' }, + { name: 'UseGalaxy.fr', url: 'https://usegalaxy.fr', id: 'fr', human: 'Galaxy France' } ] def self.servers @@SERVERS end - end end From 067687b526d3282da8e3f6bd2cf538c61db8c000 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 20 Dec 2023 13:06:12 +0100 Subject: [PATCH 5/7] linting cleanups --- bin/fetch-categories.rb | 9 ++------- bin/lint.rb | 4 +++- bin/workflow-test.rb | 5 +++-- bin/workflows-fetch.rb | 41 ++++++++++++++++++----------------------- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/bin/fetch-categories.rb b/bin/fetch-categories.rb index caf11926c67228..2cb3af2886467b 100755 --- a/bin/fetch-categories.rb +++ b/bin/fetch-categories.rb @@ -6,7 +6,7 @@ # Get the list of toolcats def fetch_toolcats(server) - uri = URI.parse("#{server}") + uri = URI.parse(server.to_s) request = Net::HTTP::Get.new(uri) req_options = { use_ssl: uri.scheme == 'https', @@ -45,16 +45,11 @@ def fetch_toolcats(server) values = [eu, org, aus].compact # values = [org].compact - # Majority answer wins # set that value to toolcats[k] # If there is no majority, pick one. # print("#{k} - #{values.length} => #{values.uniq.compact.length}\n") - if values.length.positive? - toolcats[k] = values.max_by { |v| v['count'] } - else - toolcats[k] = nil - end + toolcats[k] = (values.max_by { |v| v['count'] } if values.length.positive?) end # Write the list to a file diff --git a/bin/lint.rb b/bin/lint.rb index 99d25a46df2acf..272b9dbcd3ddb0 100755 --- a/bin/lint.rb +++ b/bin/lint.rb @@ -1110,7 +1110,9 @@ def self.enumerate_symlinks end def self.enumerate_lintable - enumerate_type(/bib$/) + enumerate_type(/md$/) + enumerate_type(/md$/, root_dir: 'faqs') + enumerate_type(/md$/, root_dir: 'news') + enumerate_type(/bib$/) + enumerate_type(/md$/) + enumerate_type(/md$/, + root_dir: 'faqs') + enumerate_type(/md$/, + root_dir: 'news') end def self.enumerate_all diff --git a/bin/workflow-test.rb b/bin/workflow-test.rb index cda54e94d8f36e..871f552fc38b38 100755 --- a/bin/workflow-test.rb +++ b/bin/workflow-test.rb @@ -1,16 +1,17 @@ #!/usr/bin/env ruby # frozen_string_literal: true + require './_plugins/gtn/usegalaxy' require 'open3' require 'json' -GALAXIES = Gtn::Usegalaxy.servers.select{|s| s[:id] == "eu"}.map do |server| +GALAXIES = Gtn::Usegalaxy.servers.select { |s| s[:id] == 'eu' }.to_h do |server| [ server[:id], { url: server[:url], key: ENV.fetch("GALAXY_#{server[:id].upcase}_KEY", 'NONE') } ] -end.to_h +end def test_workflow(workflow_file, galaxy_id) directory = File.dirname(workflow_file) diff --git a/bin/workflows-fetch.rb b/bin/workflows-fetch.rb index c42d9059a24537..c787d6a04caa17 100755 --- a/bin/workflows-fetch.rb +++ b/bin/workflows-fetch.rb @@ -5,7 +5,6 @@ require 'yaml' require './_plugins/gtn/usegalaxy' - def request(url) uri = URI.parse(url) request = Net::HTTP::Get.new(uri) @@ -13,10 +12,9 @@ def request(url) req_options = { use_ssl: uri.scheme == 'https', } - response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http| + Net::HTTP.start(uri.hostname, uri.port, req_options) do |http| http.request(request) end - response end # Get the list of workflows @@ -33,34 +31,32 @@ def fetch_workflows(server) end end -def fetch_workflowhub() - projects = JSON.parse(request("https://workflowhub.eu/projects").body) - project_mapping = projects['data'].map{|p| [p['id'], p['attributes']['title']]}.to_h +def fetch_workflowhub + projects = JSON.parse(request('https://workflowhub.eu/projects').body) + project_mapping = projects['data'].to_h { |p| [p['id'], p['attributes']['title']] } - response = request("https://workflowhub.eu/workflows?filter[workflow_type]=galaxy") + response = request('https://workflowhub.eu/workflows?filter[workflow_type]=galaxy') data = JSON.parse(response.body) if !data['links']['next'].nil? - puts "ERROR: Cannot yet handle multiple pages" + puts 'ERROR: Cannot yet handle multiple pages' exit 42 end puts "INFO: Fetching #{data['data'].length} workflows from WorkflowHub" - data['data'].map.with_index { |w, i| + data['data'].map.with_index do |w, _i| # {"id"=>"14", "type"=>"workflows", "attributes"=>{"title"=>"Cheminformatics - Docking"}, "links"=>{"self"=>"/workflows/14"}} wf_info = JSON.parse(request("https://workflowhub.eu#{w['links']['self']}").body) creator_list = [] creator0 = wf_info['data']['attributes']['creators'][0] - owner = "" - if !creator0.nil? - # Primary - creator_list.push(creator0['given_name'] + " " + creator0['family_name']) - else + if creator0.nil? # Other creators other = wf_info['data']['attributes']['other_creators'] if !other.nil? && other.length.positive? - creator_list.push(wf_info['data']['attributes']['other_creators'].split(',').map{|x| x.strip}) - else + creator_list.push(wf_info['data']['attributes']['other_creators'].split(',').map(&:strip)) end + else + # Primary + creator_list.push("#{creator0['given_name']} #{creator0['family_name']}") end # Projects wf_info['data']['relationships']['projects']['data'].each do |p| @@ -76,23 +72,22 @@ def fetch_workflowhub() 'number_of_steps' => wf_info['data']['attributes']['internals']['steps'].length, 'server' => 'https://workflowhub.eu', 'id' => wf_info['data']['id'], - 'tags' => wf_info['data']['attributes']['tags'].map{|t| t.gsub(/^name:/, '')}, + 'tags' => wf_info['data']['attributes']['tags'].map { |t| t.gsub(/^name:/, '') }, 'update_time' => wf_info['data']['attributes']['updated_at'], } - rescue + rescue StandardError r = nil end r - }.reject{|x| x.nil? } + end.compact end - # Parse the response -workflows = Gtn::Usegalaxy.servers.map {|server| +workflows = Gtn::Usegalaxy.servers.map do |server| workflows = fetch_workflows(server[:url]) puts "INFO: Fetched #{workflows.length} workflows from #{server[:name]}" workflows -}.flatten +end.flatten # Cleanup the list workflows.filter! do |w| @@ -100,7 +95,7 @@ def fetch_workflowhub() end # Add in WFHub workflows -workflows += fetch_workflowhub() +workflows += fetch_workflowhub # Group by name + owner cleaned = workflows.group_by { |w| "#{w['name']}#{w['owner']}" } From 0d552a83690bf6e883231f936dcaf8053abf1d80 Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 20 Dec 2023 14:25:21 +0100 Subject: [PATCH 6/7] fix server url matching --- Makefile | 3 +++ _plugins/gtn/supported.rb | 12 +++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7c256a576aee72..480777a718d58a 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,9 @@ _site/%/slides_CAT_ES.pdf: _site/%/slides_CAT_ES.html video: ## Build all videos bash bin/ari-make.sh +metadata/public-server-tools.json: + python ./bin/supported-fetch.py + annotate: ## annotate the tutorials with usable Galaxy instances ${ACTIVATE_ENV} && \ wget https://github.com/hexylena/toolshed-version-database/raw/main/guid-rev.json -O metadata/toolshed-revisions.json && \ diff --git a/_plugins/gtn/supported.rb b/_plugins/gtn/supported.rb index 216463f61c2d50..34e9779379bbbf 100644 --- a/_plugins/gtn/supported.rb +++ b/_plugins/gtn/supported.rb @@ -23,7 +23,7 @@ def self.calculate(data, tool_list) if data.nil? || data.empty? || tool_list.empty? || tool_list.nil? return { 'exact' => [], - 'inexact' => Gtn::Usegalaxy.servers.map { |x| x['usegalaxy'] = true }, + 'inexact' => Gtn::Usegalaxy.servers.map{|x| x = x.map{|k, v| [k.to_s, v]}.to_h; x['usegalaxy'] = true; x } } end @@ -68,13 +68,19 @@ def self.calculate(data, tool_list) # generate a 'false' value when merging sets. inexact_support -= exact_support + usegalaxy_server_urls = Gtn::Usegalaxy.servers.map{|x| x[:url]} + { 'exact' => (exact_support || []).map do |id| data['servers'][id].update( - { 'usegalaxy' => data['servers'][id]['url'].downcase.include?('usegalaxy.').to_s } + { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url']) } ) end, - 'inexact' => (inexact_support || []).map { |id| data['servers'][id] } + 'inexact' => (inexact_support || []).map { |id| + data['servers'][id].update( + { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url']) } + ) + } } end end From 229669bb5f0c49f153754da60f451c932969fedb Mon Sep 17 00:00:00 2001 From: Helena Rasche Date: Wed, 20 Dec 2023 14:26:05 +0100 Subject: [PATCH 7/7] rubocop --- _plugins/gtn/supported.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/_plugins/gtn/supported.rb b/_plugins/gtn/supported.rb index 34e9779379bbbf..b851010fe0b099 100644 --- a/_plugins/gtn/supported.rb +++ b/_plugins/gtn/supported.rb @@ -23,7 +23,11 @@ def self.calculate(data, tool_list) if data.nil? || data.empty? || tool_list.empty? || tool_list.nil? return { 'exact' => [], - 'inexact' => Gtn::Usegalaxy.servers.map{|x| x = x.map{|k, v| [k.to_s, v]}.to_h; x['usegalaxy'] = true; x } + 'inexact' => Gtn::Usegalaxy.servers.map do |x| + x = x.transform_keys(&:to_s) + x['usegalaxy'] = true + x + end } end @@ -68,7 +72,7 @@ def self.calculate(data, tool_list) # generate a 'false' value when merging sets. inexact_support -= exact_support - usegalaxy_server_urls = Gtn::Usegalaxy.servers.map{|x| x[:url]} + usegalaxy_server_urls = Gtn::Usegalaxy.servers.map { |x| x[:url] } { 'exact' => (exact_support || []).map do |id| @@ -76,11 +80,11 @@ def self.calculate(data, tool_list) { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url']) } ) end, - 'inexact' => (inexact_support || []).map { |id| + 'inexact' => (inexact_support || []).map do |id| data['servers'][id].update( { 'usegalaxy' => usegalaxy_server_urls.include?(data['servers'][id]['url']) } ) - } + end } end end