Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unpin huggingface_hub #24667

Merged
merged 6 commits into from
Jul 5, 2023
Merged

Unpin huggingface_hub #24667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2a0b27a
fix
ydshieh Jul 5, 2023
38c91e4
fix
ydshieh Jul 5, 2023
b5b61ed
fix
ydshieh Jul 5, 2023
3d3acf6
[test all] commit
ydshieh Jul 5, 2023
6b26efa
[test all] commit
ydshieh Jul 5, 2023
bc2ae97
[test all] commit
ydshieh Jul 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ jobs:
parallelism: 1
steps:
- checkout
- run: pip install --upgrade pip
- run: pip install -U GitPython
- run: pip install -U .
- run: pip install --upgrade --upgrade-strategy eager pip
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so we can get newer versions when they are available.

- run: pip install -U --upgrade-strategy eager GitPython
- run: pip install -U --upgrade-strategy eager .
- run: mkdir -p test_preparation
- run: python utils/tests_fetcher.py | tee tests_fetched_summary.txt
- store_artifacts:
Expand Down Expand Up @@ -104,9 +104,9 @@ jobs:
parallelism: 1
steps:
- checkout
- run: pip install --upgrade pip
- run: pip install -U GitPython
- run: pip install -U .
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager GitPython
- run: pip install -U --upgrade-strategy eager .
- run: |
mkdir test_preparation
echo -n "tests" > test_preparation/test_list.txt
Expand Down Expand Up @@ -136,20 +136,20 @@ jobs:
- checkout
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}
- v0.6-code-quality
- v0.7-code_quality-{{ checksum "setup.py" }}
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

to include the new packages' versions (installed by this PR)

- v0.7-code-quality
- restore_cache:
keys:
- v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.6-code-quality-site-packages
- run: pip install --upgrade pip
- run: pip install -U .[all,quality]
- v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
- v0.7-code-quality-site-packages
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager .[all,quality]
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}
key: v0.7-code_quality-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- save_cache:
key: v0.6-code_quality-{{ checksum "setup.py" }}-site-packages
key: v0.7-code_quality-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run:
Expand Down Expand Up @@ -177,20 +177,20 @@ jobs:
- checkout
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}
- v0.6-repository_consistency
- v0.7-repository_consistency-{{ checksum "setup.py" }}
- v0.7-repository_consistency
- restore_cache:
keys:
- v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.6-repository_consistency-site-packages
- run: pip install --upgrade pip
- run: pip install -U .[all,quality]
- v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
- v0.7-repository_consistency-site-packages
- run: pip install --upgrade --upgrade-strategy eager pip
- run: pip install -U --upgrade-strategy eager .[all,quality]
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}
key: v0.7-repository_consistency-{{ checksum "setup.py" }}
paths:
- '~/.cache/pip'
- save_cache:
key: v0.6-repository_consistency-{{ checksum "setup.py" }}-site-packages
key: v0.7-repository_consistency-{{ checksum "setup.py" }}-site-packages
paths:
- '~/.pyenv/versions/'
- run:
Expand Down
104 changes: 52 additions & 52 deletions .circleci/create_circleci_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class CircleCIJob:
name: str
additional_env: Dict[str, Any] = None
cache_name: str = None
cache_version: str = "0.6"
cache_version: str = "0.7"
docker_image: List[Dict[str, str]] = None
install_steps: List[str] = None
marker: Optional[str] = None
Expand Down Expand Up @@ -116,8 +116,6 @@ def to_dict(self):
},
]
steps.extend([{"run": l} for l in self.install_steps])
# TODO (ydshieh): Remove this line after the next release (the one after 2023/06/19) of `huggingface_hub`
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the main objective of this PR.

steps.append({"run": {"name": "Split tests", "command": "pip uninstall -y huggingface_hub && pip install -U git+https://github.com/huggingface/huggingface_hub.git@e4a419bf6bbaa95d14704cc781d3e81a49cef413"}})
steps.append(
{
"save_cache": {
Expand Down Expand Up @@ -244,10 +242,10 @@ def job_name(self):
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng git-lfs cmake",
"git lfs install",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U tensorflow_probability",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
marker="is_pt_tf_cross_test",
pytest_options={"rA": None, "durations": 0},
Expand All @@ -259,9 +257,9 @@ def job_name(self):
additional_env={"RUN_PT_FLAX_CROSS_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install -U --upgrade pip",
"pip install -U .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install -U --upgrade-strategy eager --upgrade pip",
"pip install -U --upgrade-strategy eager .[sklearn,flax,torch,testing,sentencepiece,torch-speech,vision]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
marker="is_pt_flax_cross_test",
pytest_options={"rA": None, "durations": 0},
Expand All @@ -272,9 +270,9 @@ def job_name(self):
"torch",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
],
parallelism=1,
pytest_num_workers=3,
Expand All @@ -285,9 +283,9 @@ def job_name(self):
"tf",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install -U tensorflow_probability",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
],
parallelism=1,
pytest_num_workers=6,
Expand All @@ -299,8 +297,8 @@ def job_name(self):
"flax",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[flax,testing,sentencepiece,flax-speech,vision]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece,flax-speech,vision]",
],
parallelism=1,
pytest_options={"rA": None},
Expand All @@ -312,8 +310,8 @@ def job_name(self):
additional_env={"RUN_PIPELINE_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,testing,sentencepiece,torch-speech,vision,timm,video]",
],
pytest_options={"rA": None},
marker="is_pipeline_test",
Expand All @@ -325,9 +323,9 @@ def job_name(self):
additional_env={"RUN_PIPELINE_TESTS": True},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install -U tensorflow_probability",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,testing,sentencepiece,vision]",
"pip install -U --upgrade-strategy eager tensorflow_probability",
],
pytest_options={"rA": None},
marker="is_pipeline_test",
Expand All @@ -349,8 +347,8 @@ def job_name(self):
"sudo cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local\n"
"sudo make install\n",
},
"pip install --upgrade pip",
"pip install -U .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[ja,testing,sentencepiece,jieba,spacy,ftfy,rjieba]",
"python -m unidic download",
],
parallelism=None,
Expand All @@ -368,9 +366,9 @@ def job_name(self):
cache_name="torch_examples",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng",
"pip install --upgrade pip",
"pip install -U .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -U -r examples/pytorch/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,torch,sentencepiece,testing,torch-speech]",
"pip install -U --upgrade-strategy eager -r examples/pytorch/_tests_requirements.txt",
],
)

Expand All @@ -380,9 +378,9 @@ def job_name(self):
cache_name="tensorflow_examples",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -U -r examples/tensorflow/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[sklearn,tensorflow,sentencepiece,testing]",
"pip install -U --upgrade-strategy eager -r examples/tensorflow/_tests_requirements.txt",
],
)

Expand All @@ -391,9 +389,9 @@ def job_name(self):
"examples_flax",
cache_name="flax_examples",
install_steps=[
"pip install --upgrade pip",
"pip install -U .[flax,testing,sentencepiece]",
"pip install -U -r examples/flax/_tests_requirements.txt",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[flax,testing,sentencepiece]",
"pip install -U --upgrade-strategy eager -r examples/flax/_tests_requirements.txt",
],
)

Expand All @@ -404,8 +402,8 @@ def job_name(self):
"sudo apt-get -y update && sudo apt-get install git-lfs",
'git config --global user.email "[email protected]"',
'git config --global user.name "ci"',
"pip install --upgrade pip",
"pip install -U .[torch,sentencepiece,testing]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,sentencepiece,testing]",
],
marker="is_staging_test",
pytest_num_workers=1,
Expand All @@ -416,8 +414,8 @@ def job_name(self):
"onnx",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y cmake",
"pip install --upgrade pip",
"pip install -U .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,tf,testing,sentencepiece,onnxruntime,vision,rjieba]",
],
pytest_options={"k onnx": None},
pytest_num_workers=1,
Expand All @@ -428,14 +426,16 @@ def job_name(self):
"exotic_models",
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev",
"pip install --upgrade pip",
"pip install -U .[torch,testing,vision]",
"pip install -U torchvision",
"pip install -U scipy",
"pip install -U 'git+https://github.com/facebookresearch/detectron2.git'",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[torch,testing,vision]",
"pip install -U --upgrade-strategy eager torchvision",
"pip install -U --upgrade-strategy eager scipy",
"pip install -U --upgrade-strategy eager 'git+https://github.com/facebookresearch/detectron2.git'",
"sudo apt install tesseract-ocr",
"pip install -U pytesseract",
"pip install -U natten",
"pip install -U --upgrade-strategy eager pytesseract",
"pip install -U --upgrade-strategy eager natten",
# TODO (ydshieh): Remove this line once `https://github.com/facebookresearch/detectron2/issues/5010` is resolved
'pip install -U --upgrade-strategy eager "Pillow<10.0.0"',
],
tests_to_run=[
"tests/models/*layoutlmv*",
Expand All @@ -450,8 +450,8 @@ def job_name(self):
repo_utils_job = CircleCIJob(
"repo_utils",
install_steps=[
"pip install --upgrade pip",
"pip install -U .[quality,testing,torch]",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager .[quality,testing,torch]",
],
parallelism=None,
pytest_num_workers=1,
Expand All @@ -471,11 +471,11 @@ def job_name(self):
additional_env={"TRANSFORMERS_VERBOSITY": "error", "DATASETS_VERBOSITY": "error", "SKIP_CUDA_DOCTEST": "1"},
install_steps=[
"sudo apt-get -y update && sudo apt-get install -y libsndfile1-dev espeak-ng time ffmpeg",
"pip install --upgrade pip",
"pip install -U -e .[dev]",
"pip install -U git+https://github.com/huggingface/accelerate",
"pip install --upgrade pytest pytest-sugar",
"pip install -U natten",
"pip install --upgrade --upgrade-strategy eager pip",
"pip install -U --upgrade-strategy eager -e .[dev]",
"pip install -U --upgrade-strategy eager git+https://github.com/huggingface/accelerate",
"pip install --upgrade --upgrade-strategy eager pytest pytest-sugar",
"pip install -U --upgrade-strategy eager natten",
"find -name __pycache__ -delete",
"find . -name \*.pyc -delete",
# Add an empty file to keep the test step running correctly even no file is selected to be tested.
Expand Down