merging in dev

.ci/flake8_ignorelist.txt

@@ -11,6 +11,7 @@ database
 doc/build
 eggs
 lib/galaxy/web/proxy/js/node_modules
+lib/tool_shed/test/test_data/repos
 static/maps
 static/scripts
 test/functional/tools/cwl_tools/v1.?/

.ci/ignore-spelling.txt

@@ -1 +1,2 @@
 hda
+implementors

.flake8

@@ -7,3 +7,4 @@
 # W503 is line breaks before binary operators, which has been reversed in PEP 8.
 # D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time).
 ignore = B008,E203,E402,E501,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413
+exclude = lib/tool_shed/test/test_data/repos

.github/labeler.yml

@@ -63,7 +63,7 @@ area/toolshed:
   - lib/toolshed/**/*
   - templates/webapps/tool_shed/**/*
 area/UI-UX:
-  - all: ["client/src/**/*", "!client/src/schema/schema.ts"]
+  - all: ["client/src/**/*", "!client/src/api/schema/schema.ts"]
     any: ["templates/**/*"]
 area/util:
   - lib/galaxy/util/**/*

.github/workflows/converter_tests.yaml

@@ -68,7 +68,7 @@ jobs:
       - name: Run tests
         run: |
           mapfile -t TOOL_ARRAY < tool_list.txt
-          planemo test --galaxy_python_version ${{ matrix.python-version }} --galaxy_root 'galaxy root' "${TOOL_ARRAY[@]}"
+          planemo test --biocontainers --galaxy_python_version ${{ matrix.python-version }} --galaxy_root 'galaxy root' "${TOOL_ARRAY[@]}"
       - uses: actions/upload-artifact@v3
         if: failure()
         with:

.github/workflows/dependencies.yaml

@@ -14,12 +14,12 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Cache pip dir
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+          # poetry requires Python >=3.8, but lint requirements currently need
+          # to be generated with `pip freeze`` on the oldest Python version
+          # supported by Galaxy.
+          python-version: |
+            ${{ matrix.python-version }}
+            3.8
       - name: Update dependencies
         run: | 
           python -m venv .venv

.github/workflows/lint.yaml

@@ -20,6 +20,7 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ['3.7', '3.11']
     env:

.github/workflows/lint_openapi_schema.yml

@@ -51,10 +51,13 @@ jobs:
       - name: Build typescript schema
         run: make update-client-api-schema
         working-directory: 'galaxy root'
+      - name: Diff...
+        run: git diff
+        working-directory: 'galaxy root'
       - name: Check for changes
         run: |
           if [[ `git status --porcelain` ]]; then
-            echo "Rebuilding client/src/schema/schema.ts resulted in changes, run 'make update-client-api-schema' and commit results"
+            echo "Rebuilding client/src/api/schema/schema.ts resulted in changes, run 'make update-client-api-schema' and commit results"
             exit 1
           fi
         working-directory: 'galaxy root'

.github/workflows/setup_selenium.yaml

@@ -5,4 +5,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install chromedriver
-        uses: mvdbeek/setup-chromedriver@chromedriver_puppeteer
+        uses: nanasess/setup-chromedriver@v2

.github/workflows/test_galaxy_packages_for_pulsar.yaml

@@ -0,0 +1,44 @@
+name: Test Galaxy packages for Pulsar
+on:
+  push:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+      - 'lib/galaxy_test/selenium/**'
+  pull_request:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+      - 'lib/galaxy_test/selenium/**'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7']
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip dir
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Install Apptainer's singularity
+        uses: eWaterCycle/setup-apptainer@v2
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get -y install ffmpeg
+      - name: Install tox
+        run: pip install tox
+      - name: Run tests
+        run: tox -e test_galaxy_packages_for_pulsar
+        working-directory: 'galaxy root'

.github/workflows/toolshed.yaml

@@ -19,8 +19,11 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ['3.7']
+        python-version: ['3.7', '3.11']
+        shed-api: ['v1', 'v2']
+        test-install-client: ['galaxy_api', 'standalone']
     services:
       postgres:
         image: postgres:13
@@ -34,6 +37,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
@@ -52,11 +60,30 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-toolshed
+      - name: Install dependencies
+        run: ./scripts/common_startup.sh --skip-client-build
+        working-directory: 'galaxy root'
+      - name: Build Frontend
+        run: |
+          . .venv/bin/activate
+          cd lib/tool_shed/webapp/frontend
+          yarn
+          make client
+        working-directory: 'galaxy root'
+      - name: Install playwright
+        run: |
+          . .venv/bin/activate
+          playwright install
+        working-directory: 'galaxy root'
       - name: Run tests
-        run: './run_tests.sh -toolshed'
+        run: ./run_tests.sh -toolshed
+        env:
+          TOOL_SHED_TEST_INSTALL_CLIENT: ${{ matrix.test-install-client }}
+          TOOL_SHED_API_VERSION: ${{ matrix.shed-api }}
+          TOOL_SHED_TEST_BROWSER: ${{ matrix.shed-api == 'v1' && 'twill' || 'playwright' }}
         working-directory: 'galaxy root'
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: Toolshed test results (${{ matrix.python-version }})
+          name: Toolshed test results (${{ matrix.python-version }}, ${{ matrix.shed-api }}, ${{ matrix.test-install-client }})
           path: 'galaxy root/run_toolshed_tests.html'

.isort.cfg

@@ -10,5 +10,5 @@ profile=black
 reverse_relative=true
 skip_gitignore=true
 # Make isort run faster by skipping database
-skip_glob=database/*
+skip_glob=database/*,lib/tool_shed/test/test_data/repos/*
 src_paths=lib

.k8s_ci.Dockerfile

@@ -166,7 +166,7 @@ RUN set -xe; \
 
 # Create Galaxy user, group, directory; chown
 RUN set -xe; \
-      adduser --system --group $GALAXY_USER \
+      adduser --system --group --uid 101 $GALAXY_USER \
       && mkdir -p $SERVER_DIR \
       && chown $GALAXY_USER:$GALAXY_USER $ROOT_DIR -R
 
@@ -179,7 +179,8 @@ COPY --chown=$GALAXY_USER:$GALAXY_USER --from=client_build $SERVER_DIR/static ./
 WORKDIR $SERVER_DIR
 
 # The data in version.json will be displayed in Galaxy's /api/version endpoint
-RUN printf "{\n  \"git_commit\": \"$(cat GITREVISION)\",\n  \"build_date\": \"$BUILD_DATE\",\n  \"image_tag\": \"$IMAGE_TAG\"\n}\n" > version.json
+RUN printf "{\n  \"git_commit\": \"$(cat GITREVISION)\",\n  \"build_date\": \"$BUILD_DATE\",\n  \"image_tag\": \"$IMAGE_TAG\"\n}\n" > version.json \
+    && chown $GALAXY_USER:$GALAXY_USER version.json
 
 EXPOSE 8080
 USER $GALAXY_USER

.redocly.lint-ignore.yaml

@@ -14,3 +14,6 @@ _schema.yaml:
       #/paths/~1api~1histories~1{history_id}~1contents~1{history_content_id}~1metadata_file
     - '#/paths/~1api~1histories~1{history_id}~1contents~1{id}~1validate'
     - '#/paths/~1api~1histories~1{history_id}~1contents~1{type}s~1{id}'
+_shed_schema.yaml:
+  no-empty-servers:
+    - '#/openapi'

.vscode/shed.code-snippets

@@ -0,0 +1,41 @@
+{
+    "shedcomp": {
+        "prefix": "shed_component",
+        "body": [
+            "<script setup lang=\"ts\">",
+            "\t$0",
+            "</script>",
+            "<template>",
+            "</template>"
+        ],
+       "description": "outline of a tool shed component"
+    },
+    "shedpage": {
+        "prefix": "shed_page",
+        "body": [
+            "<script setup lang=\"ts\">",
+            "import PageContainer from \"@/components/PageContainer.vue\"",
+            "</script>",
+            "<template>",
+            "    <page-container>",
+            "        $0",
+            "    </page-container>",
+            "</template>"
+        ],
+       "description": "outline of a tool shed page"
+    },
+    "shedfetcher": {
+        "prefix": "shed_fetcher",
+        "body": [
+            "import { fetcher } from \"@/schema\"",
+            "const fetcher = fetcher.path(\"$1\").method(\"get\").create()"
+        ],
+        "description": "Import shed fetcher and instantiate with a path"
+    },
+    "shedrouter": {
+        "prefix": "shed_router",
+        "body": [
+            "import router from \"@/router\""
+        ]
+    }
+}

Makefile

@@ -139,7 +139,7 @@ release-bootstrap-history: ## bootstrap history for a new release
 update-lint-requirements:
 	./lib/galaxy/dependencies/update_lint_requirements.sh
 
-update-dependencies: update-lint-requirements ## update pinned and dev dependencies
+update-dependencies: update-lint-requirements ## update pinned, dev and typecheck dependencies
 	$(IN_VENV) ./lib/galaxy/dependencies/update.sh
 
 $(CWL_TARGETS):
@@ -182,17 +182,22 @@ endif
 
 build-api-schema:
 	$(IN_VENV) python scripts/dump_openapi_schema.py _schema.yaml
+	$(IN_VENV) python scripts/dump_openapi_schema.py --app shed _shed_schema.yaml
 
 remove-api-schema:
 	rm _schema.yaml
+	rm _shed_schema.yaml
 
 update-client-api-schema: client-node-deps build-api-schema
-	$(IN_VENV) cd client && node openapi_to_schema.mjs ../_schema.yaml > src/schema/schema.ts && npx prettier --write src/schema/schema.ts
+	$(IN_VENV) cd client && node openapi_to_schema.mjs ../_schema.yaml > src/api/schema/schema.ts && npx prettier --write src/api/schema/schema.ts
+	$(IN_VENV) cd client && node openapi_to_schema.mjs ../_shed_schema.yaml > ../lib/tool_shed/webapp/frontend/src/schema/schema.ts && npx prettier --write ../lib/tool_shed/webapp/frontend/src/schema/schema.ts
 	$(MAKE) remove-api-schema
 
 lint-api-schema: build-api-schema
 	$(IN_VENV) npx --yes @redocly/cli lint _schema.yaml
+	$(IN_VENV) npx --yes @redocly/cli lint _shed_schema.yaml
 	$(IN_VENV) codespell -I .ci/ignore-spelling.txt _schema.yaml
+	$(IN_VENV) codespell -I .ci/ignore-spelling.txt _shed_schema.yaml
 	$(MAKE) remove-api-schema
 
 update-navigation-schema: client-node-deps

client/.eslintrc.js

@@ -41,7 +41,15 @@ const baseRules = {
     "vuejs-accessibility/form-control-has-label": "warn",
     "vuejs-accessibility/heading-has-content": "error",
     "vuejs-accessibility/iframe-has-title": "error",
-    "vuejs-accessibility/label-has-for": "warn",
+    "vuejs-accessibility/label-has-for": [
+        "warn",
+        {
+            required: {
+                some: ["nesting", "id"],
+            },
+            allowChildren: true,
+        },
+    ],
     "vuejs-accessibility/mouse-events-have-key-events": "warn",
     "vuejs-accessibility/no-autofocus": "error",
     "vuejs-accessibility/no-static-element-interactions": "warn",
@@ -117,21 +125,23 @@ module.exports = {
             files: ["**/*.ts", "**/*.tsx"],
             extends: [
                 ...baseExtends,
-                "plugin:@typescript-eslint/eslint-recommended",
                 "plugin:@typescript-eslint/recommended",
+                // "plugin:@typescript-eslint/stylistic"  // TODO: work towards this
             ],
             rules: {
                 ...baseRules,
                 "@typescript-eslint/no-throw-literal": "error",
                 "@typescript-eslint/ban-ts-comment": "warn",
+                "@typescript-eslint/no-explicit-any": "warn", // TODO: re-enable this
+                "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "_.+", varsIgnorePattern: "_.+" }],
             },
             parser: "@typescript-eslint/parser",
             parserOptions: {
                 ecmaFeatures: { jsx: true },
                 ecmaVersion: 2020,
                 sourceType: "module",
                 extraFileExtensions: [".vue"],
-                project: "./tsconfig.json",
+                project: true,
             },
             plugins: [...basePlugins, "@typescript-eslint"],
         },