Skip to content

Commit

Permalink
Add extended-length prefix support
Browse files Browse the repository at this point in the history
Fixes #588.
  • Loading branch information
nkaretnikov committed Dec 24, 2023
1 parent a4863e3 commit a605177
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 14 deletions.
15 changes: 14 additions & 1 deletion .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,20 @@ jobs:
- name: "Unit tests ✅"
run: |
pytest tests
pytest -m "not extended_prefix" tests
# https://github.com/actions/runner-images/issues/1052
- name: "Windows extended prefix unit tests ✅"
shell: pwsh
run: |
Set-ItemProperty "HKLM:\System\CurrentControlSet\Control\FileSystem" `
-Name "LongPathsEnabled" `
-Value 0 `
-Type DWord
(Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled
pytest -m "extended_prefix" tests
if: matrix.os == 'windows'


integration-test-conda-store-server:
name: "integration-test conda-store-server"
Expand Down
6 changes: 6 additions & 0 deletions conda-store-server/conda_store_server/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,12 @@ def _check_build_key_version(self, proposal):
except Exception as e:
raise TraitError(f"c.CondaStore.build_key_version: {e}")

win_extended_length_prefix = Bool(
False,
help="Use the extended-length prefix '\\\\?\\' (Windows-only), default: False",
config=True,
)

conda_command = Unicode(
"mamba",
help="conda executable to use for solves",
Expand Down
16 changes: 14 additions & 2 deletions conda-store-server/conda_store_server/orm.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import pathlib
import re
import shutil
import sys

from conda_store_server import conda_utils, schema, utils
from conda_store_server.environment import validate_environment
Expand Down Expand Up @@ -254,7 +255,12 @@ def build_path(self, conda_store):
# https://github.com/conda-incubator/conda-store/issues/649
if len(str(res)) > 255:
raise BuildPathError("build_path too long: must be <= 255 characters")
return res
# Note: cannot use the '/' operator to prepend the extended-length
# prefix
if sys.platform == "win32" and conda_store.win_extended_length_prefix:
return pathlib.Path(f"\\\\?\\{res}")
else:
return res

def environment_path(self, conda_store):
"""Environment path is the path for the symlink to the build
Expand All @@ -264,11 +270,17 @@ def environment_path(self, conda_store):
store_directory = os.path.abspath(conda_store.store_directory)
namespace = self.environment.namespace.name
name = self.specification.name
return pathlib.Path(
res = pathlib.Path(
conda_store.environment_directory.format(
store_directory=store_directory, namespace=namespace, name=name
)
)
# Note: cannot use the '/' operator to prepend the extended-length
# prefix
if sys.platform == "win32" and conda_store.win_extended_length_prefix:
return pathlib.Path(f"\\\\?\\{res}")
else:
return res

@property
def build_key(self):
Expand Down
72 changes: 72 additions & 0 deletions conda-store-server/tests/test_server.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import sys
import time

import pytest
Expand Down Expand Up @@ -396,6 +397,77 @@ def test_create_specification_auth_env_name_too_long(testclient, celery_worker,
assert False, f"failed to update status"


@pytest.fixture
def win_extended_length_prefix(request):
# Overrides the attribute before other fixtures are called
import conda_store_server
conda_store_server.app.CondaStore.win_extended_length_prefix = request.param
yield request.param


@pytest.mark.skipif(sys.platform != "win32", reason="tests a Windows issue")
@pytest.mark.parametrize('win_extended_length_prefix', [True, False], indirect=True)
@pytest.mark.extended_prefix
def test_create_specification_auth_extended_prefix(win_extended_length_prefix, testclient, celery_worker, authenticate):
# Adds padding to cause an error if the extended prefix is not enabled
namespace = "default" + 'A' * 10
environment_name = "pytest"

# The debugpy 1.8.0 package was deliberately chosen because it has long
# paths internally, which causes issues on Windows due to the path length
# limit
response = testclient.post(
"api/v1/specification",
json={
"namespace": namespace,
"specification": json.dumps({
"name": environment_name,
"channels": ["conda-forge"],
"dependencies": ["debugpy==1.8.0"],
"variables": None,
"prefix": None,
"description": "test"
}),
},
timeout=30,
)
response.raise_for_status()

r = schema.APIPostSpecification.parse_obj(response.json())
assert r.status == schema.APIStatus.OK
build_id = r.data.build_id

# Try checking that the status is 'FAILED'
is_updated = False
for _ in range(30):
time.sleep(5)

# check for the given build
response = testclient.get(f"api/v1/build/{build_id}", timeout=30)
response.raise_for_status()

r = schema.APIGetBuild.parse_obj(response.json())
assert r.status == schema.APIStatus.OK
assert r.data.specification.name == environment_name
if r.data.status in ["QUEUED", "BUILDING"]:
continue # checked too fast, try again

if win_extended_length_prefix:
assert r.data.status == "COMPLETED"
else:
assert r.data.status == "FAILED"
response = testclient.get(f"api/v1/build/{build_id}/logs", timeout=30)
response.raise_for_status()
assert "[WinError 206] The filename or extension is too long" in response.text

is_updated = True
break

# If we're here, the task didn't update the status on failure
if not is_updated:
assert False, f"failed to get status"


def test_create_specification_auth(testclient, celery_worker, authenticate):
namespace = "default"
environment_name = "pytest"
Expand Down
53 changes: 42 additions & 11 deletions docusaurus-docs/conda-store/references/faq.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,29 +115,60 @@ c.CondaStore.build_key_version = 2
## Long paths on Windows
conda-store supports Windows in standalone mode. However, when creating
environments with certain packages, you may see errors like
environments with certain packages, you may see errors like:
```bash
ERROR:root:[WinError 206] The filename or extension is too long: 'C:\\...'
```
This error is due to the fact that Windows has a limitation that file paths
cannot be more than 260 characters. The fix is to set the registry key
cannot be more than 260 characters.
See [conda-store issue #588][max-path-issue] for more details.
### Solution 1: Extended-length path prefix (`\\?\`)
If you *don't have administrator privileges*, try using the following config
option:
```python
c.CondaStore.win_extended_length_prefix = True
```
This adds the extended-length path prefix (`\\?\`) to conda-store `build_path`
and `environment_path` methods, which should allow for a maximum total path
length of 32,767 characters when building packages.
See [this Microsoft support article][max-path] for more details on the
extended-length path prefix.
### Solution 2: `LongPathsEnabled`
If you *have administrator privileges*, set the registry key
`Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled
(Type: REG_DWORD)` to `1`, which removes this MAX_PATH limitation. See [this
Microsoft support
article](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation)
for more details on how to set this registry key.
(Type: REG_DWORD)` to `1`, which removes this `MAX_PATH` limitation.
See [this Microsoft support article][max-path] for more details on how to set
this registry key.
### Solution 3: `store_directory`
If it is not possible to set this registry key, for instance, because you do
not have access to administrator privileges, you should configure the
If it is not possible to set the registry key, for instance, because you *do
not have access to administrator privileges*, you should configure the
conda-store `CondaStore.store_directory` to be as close to the filesystem root
as possible, so that the total length of the paths of package files is
minimized.
See [conda-store issue
#588](https://github.com/conda-incubator/conda-store/issues/588) for more
details.
### Solution 4: `build_key_version`
Use the short build key version as explained [above](#build-key-versions):
```python
c.CondaStore.build_key_version = 2
```
[max-path-issue]: https://github.com/conda-incubator/conda-store/issues/588
[max-path]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
## What are the resource requirements for `conda-store-server`
Expand Down

0 comments on commit a605177

Please sign in to comment.