Merge pull request #640 from nkaretnikov/windows

Windows support

.github/workflows/tests.yaml

@@ -19,13 +19,14 @@ jobs:
     name: "unit-test conda-store-server"
     strategy:
       matrix:
-        # cannot run on windows due to needing fake-chroot for conda-docker
-        os: ["ubuntu", "macos"]
+        os: ["ubuntu", "macos", "windows"]
         include:
           - os: ubuntu
             environment-file: conda-store-server/environment-dev.yaml
           - os: macos
             environment-file: conda-store-server/environment-macos-dev.yaml
+          - os: windows
+            environment-file: conda-store-server/environment-windows-dev.yaml
     runs-on: ${{ matrix.os }}-latest
     defaults:
       run:
@@ -41,6 +42,16 @@ jobs:
             environment-file: ${{ matrix.environment-file }}
             miniforge-version: latest
 
+      # This fixes a "DLL not found" issue importing ctypes from the hatch env
+      - name: Reinstall Python 3.10 on Windows runner
+        uses: nick-fields/[email protected]
+        with:
+          timeout_minutes: 9999
+          max_attempts: 6
+          command:
+            conda install --channel=conda-forge --quiet --yes python=${{ matrix.python }}
+        if: matrix.os == 'windows'
+
       - name: "Linting Checks 🧹"
         run: |
           hatch env run -e dev lint

conda-store-server/conda_store_server/action/base.py

@@ -38,20 +38,23 @@ class ActionContext:
     def __init__(self):
         self.id = str(uuid.uuid4())
         self.stdout = io.StringIO()
+        self.stderr = io.StringIO()
         self.log = logging.getLogger(f"conda_store_server.action.{self.id}")
         self.log.propagate = False
         self.log.addHandler(logging.StreamHandler(stream=self.stdout))
         self.log.setLevel(logging.INFO)
         self.result = None
         self.artifacts = {}
 
-    def run(self, *args, **kwargs):
+    def run(self, *args, redirect_stderr=True, **kwargs):
         result = subprocess.run(
             *args,
             **kwargs,
             stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
+            stderr=subprocess.STDOUT if redirect_stderr else subprocess.PIPE,
             encoding="utf-8",
         )
         self.stdout.write(result.stdout)
+        if not redirect_stderr:
+            self.stderr.write(result.stderr)
         return result

conda-store-server/conda_store_server/action/generate_conda_export.py

@@ -17,5 +17,7 @@ def action_generate_conda_export(
         "--json",
     ]
 
-    result = context.run(command, check=True)
+    result = context.run(command, check=True, redirect_stderr=False)
+    if result.stderr:
+        context.log.warning(f"conda env export stderr: {result.stderr}")
     return json.loads(result.stdout)

conda-store-server/conda_store_server/action/install_lockfile.py

@@ -1,5 +1,6 @@
 import json
 import pathlib
+import sys
 import typing
 
 from conda_store_server import action
@@ -16,7 +17,9 @@ def action_install_lockfile(
         json.dump(conda_lock_spec, f)
 
     command = [
-        "conda-lock",
+        sys.executable,
+        "-m",
+        "conda_lock",
         "install",
         "--validate-platform",
         "--log-level",

conda-store-server/conda_store_server/app.py

@@ -302,21 +302,24 @@ def _default_celery_results_backend(self):
     )
 
     default_uid = Integer(
-        os.getuid(),
+        None if sys.platform == "win32" else os.getuid(),
         help="default uid to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_gid = Integer(
-        os.getgid(),
+        None if sys.platform == "win32" else os.getgid(),
         help="default gid to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_permissions = Unicode(
-        "775",
+        None if sys.platform == "win32" else "775",
         help="default file permissions to assign to built environments",
         config=True,
+        allow_none=True,
     )
 
     default_docker_base_image = Union(

conda-store-server/conda_store_server/dbutil.py

@@ -1,6 +1,5 @@
 import os
 from contextlib import contextmanager
-from subprocess import check_call
 from tempfile import TemporaryDirectory
 
 from alembic import command
@@ -78,6 +77,8 @@ def upgrade(db_url, revision="head"):
     current_table_names = set(inspect(engine).get_table_names())
 
     with _temp_alembic_ini(db_url) as alembic_ini:
+        alembic_cfg = Config(alembic_ini)
+
         if (
             "alembic_version" not in current_table_names
             and len(current_table_names) > 0
@@ -86,10 +87,10 @@ def upgrade(db_url, revision="head"):
             # we stamp the revision at the first one, that introduces the alembic revisions.
             # I chose the leave the revision number hardcoded as it's not something
             # dynamic, not something we want to change, and tightly related to the codebase
-            command.stamp(Config(alembic_ini), "48be4072fe58")
+            command.stamp(alembic_cfg, "48be4072fe58")
             # After this point, whatever is in the database, Alembic will
             # believe it's at the first revision. If there are more upgrades/migrations
             # to run, they'll be at the next step :
 
         # run the upgrade.
-        check_call(["alembic", "-c", alembic_ini, "upgrade", revision])
+        command.upgrade(config=alembic_cfg, revision=revision)

conda-store-server/conda_store_server/orm.py

@@ -513,10 +513,11 @@ def update_packages(self, db, subdirs=None):
                 package_builds[package_key].append(new_package_build_dict)
             logger.info("CondaPackageBuild objects created")
 
-            batch_size = 1000
+            # sqlite3 has a max expression depth of 1000
+            batch_size = 990
             all_package_keys = list(package_builds.keys())
             for i in range(0, len(all_package_keys), batch_size):
-                logger.info(f"handling subset at index {i} (batch size {batch_size}")
+                logger.info(f"handling subset at index {i} (batch size {batch_size})")
                 subset_keys = all_package_keys[i : i + batch_size]
 
                 # retrieve the parent packages for the subset

conda-store-server/conda_store_server/schema.py

@@ -194,20 +194,20 @@ class Settings(BaseModel):
         metadata={"global": True},
     )
 
-    default_uid: int = Field(
-        os.getuid(),
+    default_uid: Optional[int] = Field(
+        None if sys.platform == "win32" else os.getuid(),
         description="default uid to assign to built environments",
         metadata={"global": True},
     )
 
-    default_gid: int = Field(
-        os.getgid(),
+    default_gid: Optional[int] = Field(
+        None if sys.platform == "win32" else os.getgid(),
         description="default gid to assign to built environments",
         metadata={"global": True},
     )
 
-    default_permissions: str = Field(
-        "775",
+    default_permissions: Optional[str] = Field(
+        None if sys.platform == "win32" else "775",
         description="default file permissions to assign to built environments",
         metadata={"global": True},
     )

conda-store-server/conda_store_server/server/app.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import posixpath
 import sys
 
 import conda_store_server
@@ -198,9 +199,9 @@ def trim_slash(url):
         app = FastAPI(
             title="conda-store",
             version=__version__,
-            openapi_url=os.path.join(self.url_prefix, "openapi.json"),
-            docs_url=os.path.join(self.url_prefix, "docs"),
-            redoc_url=os.path.join(self.url_prefix, "redoc"),
+            openapi_url=posixpath.join(self.url_prefix, "openapi.json"),
+            docs_url=posixpath.join(self.url_prefix, "docs"),
+            redoc_url=posixpath.join(self.url_prefix, "redoc"),
             contact={
                 "name": "Quansight",
                 "url": "https://quansight.com",

conda-store-server/conda_store_server/server/auth.py

@@ -436,7 +436,9 @@ async def post_login_method(
             samesite="strict",
             domain=self.cookie_domain,
             # set cookie to expire at same time as jwt
-            max_age=(authentication_token.exp - datetime.datetime.utcnow()).seconds,
+            max_age=int(
+                (authentication_token.exp - datetime.datetime.utcnow()).total_seconds()
+            ),
         )
         return response
 

conda-store-server/conda_store_server/storage.py

@@ -1,5 +1,6 @@
 import io
 import os
+import posixpath
 import shutil
 
 import minio
@@ -223,7 +224,7 @@ def get(self, key):
             return f.read()
 
     def get_url(self, key):
-        return os.path.join(self.storage_url, key)
+        return posixpath.join(self.storage_url, key)
 
     def delete(self, db, build_id, key):
         filename = os.path.join(self.storage_path, key)

conda-store-server/conda_store_server/utils.py

@@ -50,13 +50,44 @@ def chdir(directory: pathlib.Path):
         os.chdir(current_directory)
 
 
+def du(path):
+    """
+    Pure Python equivalent of du -sb
+    Based on https://stackoverflow.com/a/55648984/161801
+    """
+    if os.path.islink(path) or os.path.isfile(path):
+        return os.lstat(path).st_size
+    nbytes = 0
+    seen = set()
+    for dirpath, dirnames, filenames in os.walk(path):
+        nbytes += os.lstat(dirpath).st_size
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            st = os.lstat(fp)
+            if st.st_ino in seen:
+                continue
+            seen.add(st.st_ino)  # adds inode to seen list
+            nbytes += st.st_size  # adds bytes to total
+        for d in dirnames:
+            dp = os.path.join(dirpath, d)
+            if os.path.islink(dp):
+                nbytes += os.lstat(dp).st_size
+    return nbytes
+
+
 def disk_usage(path: pathlib.Path):
     if sys.platform == "darwin":
         cmd = ["du", "-sAB1", str(path)]
-    else:
+    elif sys.platform == "linux":
         cmd = ["du", "-sb", str(path)]
+    else:
+        return str(du(path))
 
-    return subprocess.check_output(cmd, encoding="utf-8").split()[0]
+    output = subprocess.check_output(cmd, encoding="utf-8").split()[0]
+    if sys.platform == "darwin":
+        # mac du does not have the -b option to return bytes
+        output = str(int(output) * 512)
+    return output
 
 
 @contextlib.contextmanager

conda-store-server/conda_store_server/worker/app.py

@@ -71,9 +71,18 @@ def start(self):
         argv = [
             "worker",
             "--loglevel=INFO",
-            "--beat",
         ]
 
+        # The default Celery pool requires this on Windows. See
+        # https://stackoverflow.com/questions/37255548/how-to-run-celery-on-windows
+        if sys.platform == "win32":
+            os.environ.setdefault("FORKED_BY_MULTIPROCESSING", "1")
+        else:
+            # --beat does not work on Windows
+            argv += [
+                "--beat",
+            ]
+
         if self.concurrency:
             argv.append(f"--concurrency={self.concurrency}")
 

conda-store-server/environment-windows-dev.yaml

@@ -0,0 +1,59 @@
+name: conda-store-server-dev
+channels:
+  - conda-forge
+  - microsoft
+  - nodefaults
+dependencies:
+  - python ==3.10
+  # conda builds
+  - conda ==23.5.2
+  - python-docker
+  - conda-pack
+  - conda-lock >=1.0.5
+  - mamba
+  - conda-package-handling
+  # web server
+  - celery
+  - flower
+  - redis-py
+  - sqlalchemy<=1.4.47
+  - psycopg2
+  - pymysql
+  - requests
+  - uvicorn
+  - fastapi
+  - pydantic < 2.0
+  - pyyaml
+  - traitlets
+  - yarl
+  - pyjwt
+  - filelock
+  - itsdangerous
+  - jinja2
+  - python-multipart
+  - alembic
+  # artifact storage
+  - minio
+  # CLI
+  - typer
+
+  # dev dependencies
+  - aiohttp>=3.8.1
+  - hatch
+  - pytest
+  - pytest-celery
+  - pytest-mock
+  - black ==22.3.0
+  - flake8
+  - ruff
+  - sphinx
+  - myst-parser
+  - sphinx-panels
+  - sphinx-copybutton
+  - pydata-sphinx-theme
+  - playwright
+  - docker-compose
+  - pip
+
+  - pip:
+      - pytest-playwright

conda-store-server/hatch_build.py

@@ -61,10 +61,12 @@ def initialize(self, version: str, build_data: Dict[str, Any]) -> None:
             # main.js to enable easy configuration see
             # conda_store_server/server/templates/conda-store-ui.html
             # for global variable set
-            with (source_directory / "main.js").open("r") as source_f:
+            with (source_directory / "main.js").open("r", encoding="utf-8") as source_f:
                 content = source_f.read()
                 content = re.sub(
                     '"MISSING_ENV_VAR"', "GLOBAL_CONDA_STORE_STATE", content
                 )
-                with (destination_directory / "main.js").open("w") as dest_f:
+                with (destination_directory / "main.js").open(
+                    "w", encoding="utf-8"
+                ) as dest_f:
                     dest_f.write(content)