Address issues of missing bucket_name in s3fs paths (#673)

* pin typer and add defaults (api change)

* Adjust S3 config w.r.t. to s3fs requirements

* pin typer>=0.9,<1.0

* remove comment

* revert cli changes

* update docs

* add S3_BUCKET_NAME env variable

* pin typer in setup.cfg

* fix?

* test_cli env. variable fix

* remove hard coded S3_BUCKET_NAME (use env)

* try setting QUETZ_S3_BUCKET_NAME in ci

* add debug statements to find where env variable is lost

* remove commented out code

.github/workflows/ci.yml

@@ -68,12 +68,12 @@ jobs:
           S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
           S3_ENDPOINT: https://s3.sbg.cloud.ovh.net/
           S3_REGION: sbg
+          S3_BUCKET_NAME: quetz
         run: |
           # install dev dependencies
           pip install -e .[all,dev]
           pip install redis rq
           pip install pytest-github-actions-annotate-failures
-
       - name: Testing server
         shell: bash -l -eo pipefail {0}
         env:
@@ -85,6 +85,7 @@ jobs:
           S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
           S3_ENDPOINT: https://s3.sbg.cloud.ovh.net/
           S3_REGION: sbg
+          S3_BUCKET_NAME: quetz
         run: |
           if [ "$TEST_DB_BACKEND" == "postgres" ]; then
             export QUETZ_TEST_DATABASE="postgresql://postgres:mysecretpassword@${POSTGRES_HOST}:${POSTGRES_PORT}/postgres"
@@ -96,6 +97,7 @@ jobs:
           fi
 
           export QUETZ_IS_TEST=1
+
           pytest -v ./quetz/tests/ --cov-config=pyproject.toml --cov=. --cov-report=xml
 
       - name: Test the plugins
@@ -109,6 +111,7 @@ jobs:
           S3_SECRET_KEY: ${{ secrets.s3_secret_key }}
           S3_ENDPOINT: https://s3.sbg.cloud.ovh.net/
           S3_REGION: sbg
+          S3_BUCKET_NAME: quetz
         run: |
           if [ "$TEST_DB_BACKEND" == "postgres" ]; then
             export QUETZ_TEST_DATABASE="postgresql://postgres:mysecretpassword@${POSTGRES_HOST}:${POSTGRES_PORT}/postgres"

.gitignore

@@ -23,3 +23,6 @@ test_quetz
 .env
 .envrc
 .venv
+
+# OS
+.DS_Store

docs/source/deploying/configuration.rst

@@ -156,16 +156,18 @@ Quetz can store package in object cloud storage compatible with S3 interface. To
     [s3]
     access_key = "AKIAIOSFODNN7EXAMPLE"
     secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-    url = "https://..."
+    url = "https://s3.amazonaws.com"
     region = ""
+    bucket_name = ""
     bucket_prefix="..."
     bucket_suffix="..."
 
 
 :access key:
 :secret key: credentials to S3 account, if you use IAM roles, don't set them or set them to ``""``
-:url: set to the S3 endpoint of your provider (for AWS, you can skip it)
+:url: set to the S3 endpoint (excluding bucket name) of your provider (for AWS, you can skip it)
 :region: region of the S3 instance
+:bucket_name: name of the bucket to store the data in
 :bucket_prefix:
 :bucket_suffix: channel directories on S3 are created with the following semantics: ``{bucket_prefix}{channel_name}{bucket_suffix}``
 
@@ -276,5 +278,6 @@ Variable                 description                             values
 ``S3_SECRET_KEY``        secret key to s3 (used in tests)        string                                         
 ``S3_ENDPOINT``          s3 endpoint url                         string                                         
 ``S3_REGION``            s3 region                               string                                         
+``S3_BUCKET_NAME``       s3 bucket name                          string                                         
 =======================  ======================================  ===========================  ===================
 

quetz/config.py

@@ -128,8 +128,9 @@ class Config:
             [
                 ConfigEntry("access_key", str, default=""),
                 ConfigEntry("secret_key", str, default=""),
-                ConfigEntry("url", str, default=""),
+                ConfigEntry("url", str, default="https://s3.amazonaws.com"),
                 ConfigEntry("region", str, default=""),
+                ConfigEntry("bucket_name", str, default=""),
                 ConfigEntry("bucket_prefix", str, default=""),
                 ConfigEntry("bucket_suffix", str, default=""),
             ],
@@ -450,6 +451,7 @@ def get_package_store(self) -> pkgstores.PackageStore:
                     'secret': self.s3_secret_key,
                     'url': self.s3_url,
                     'region': self.s3_region,
+                    'bucket_name': self.s3_bucket_name,
                     'bucket_prefix': self.s3_bucket_prefix,
                     'bucket_suffix': self.s3_bucket_suffix,
                 }

quetz/pkgstores.py

@@ -292,8 +292,11 @@ def __init__(self, config):
         client_kwargs = {}
         url = config.get('url')
         region = config.get("region")
+        self.bucket_name = config.get("bucket_name")
         if url:
             client_kwargs['endpoint_url'] = url
+        if not self.bucket_name:
+            raise ValueError("bucket_name is required in s3 configuration")
         if region:
             client_kwargs["region_name"] = region
 
@@ -326,7 +329,7 @@ def _get_fs(self):
             raise ConfigError(f"{e} - check configured S3 credentials")
 
     def _bucket_map(self, name):
-        return f"{self.bucket_prefix}{name}{self.bucket_suffix}"
+        return f"{self.bucket_name}/{self.bucket_prefix}{name}{self.bucket_suffix}"
 
     def create_channel(self, name):
         """Create the bucket if one doesn't already exist

quetz/tests/test_cli.py

@@ -7,12 +7,12 @@
 from unittest import mock
 from unittest.mock import MagicMock
 
-import pytest
 import sqlalchemy as sa
 from alembic.script import ScriptDirectory
 from pytest_mock.plugin import MockerFixture
 from typer.testing import CliRunner
 
+import pytest
 from quetz import cli
 from quetz.config import Config
 from quetz.db_models import Base, Identity, User
@@ -554,9 +554,12 @@ def database_url_environment_variable(database_url) -> None:
 @pytest.fixture()
 def s3_environment_variable() -> None:
     os.environ["QUETZ_S3_ACCESS_KEY"] = "fake_key"
+    os.environ["QUETZ_S3_BUCKET_NAME"] = "fake_bucket"
     yield
     if "QUETZ_S3_ACCESS_KEY" in os.environ:
         del os.environ["QUETZ_S3_ACCESS_KEY"]
+    if "QUETZ_S3_BUCKET_NAME" in os.environ:
+        del os.environ["QUETZ_S3_BUCKET_NAME"]
 
 
 @pytest.fixture()

quetz/tests/test_config.py

@@ -3,7 +3,6 @@
 import tempfile
 
 import pytest
-
 from quetz.config import Config, ConfigEntry, ConfigSection, configure_logger
 from quetz.dao import Dao
 from quetz.errors import ConfigError

quetz/tests/test_pkg_stores.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 
 import pytest
-
 from quetz.pkgstores import (
     AzureBlobStore,
     GoogleCloudStorageStore,
@@ -23,6 +22,7 @@
     'secret': os.environ.get("S3_SECRET_KEY"),
     'url': os.environ.get("S3_ENDPOINT"),
     'region': os.environ.get("S3_REGION"),
+    'bucket_name': os.environ.get("S3_BUCKET_NAME"),
     'bucket_prefix': "test",
     'bucket_suffix': "",
 }