From c04984a6bb1d72a479defcb8db230e9b1e37c44d Mon Sep 17 00:00:00 2001 From: Yunchu Lee Date: Fri, 14 Apr 2023 18:49:23 +0900 Subject: [PATCH] Updates docs for release --- .github/workflows/docs_stable.yml | 34 ++++++++++++ docs/contributing.md => contributing.md | 0 datumaro/cli/util/project.py | 2 +- .../common_semantic_segmentation.py | 26 ++++----- .../datumaro_binary/mapper/__init__.py | 2 +- datumaro/plugins/data_formats/imagenet.py | 55 ++++++++++--------- datumaro/plugins/sampler/random_sampler.py | 2 +- docs/source/conf.py | 3 +- .../docs/level-up/advanced_skills/index.rst | 2 +- .../basic_skills/03_dataset_import_export.rst | 8 +-- .../basic_skills/04_detect_data_format.rst | 6 +- .../docs/level-up/basic_skills/index.rst | 2 +- .../level-up/intermediate_skills/index.rst | 2 +- ...maro.plugins.sampler.algorithm.entropy.rst | 13 +---- .../datumaro.plugins.sampler.algorithm.rst | 1 - docs/source/docs/user-manual/extending.md | 4 +- notebooks/09_encrypt_dataset.ipynb | 2 +- requirements.txt | 5 -- tox.ini | 2 + 19 files changed, 100 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/docs_stable.yml rename docs/contributing.md => contributing.md (100%) diff --git a/.github/workflows/docs_stable.yml b/.github/workflows/docs_stable.yml new file mode 100644 index 0000000000..89a42516bd --- /dev/null +++ b/.github/workflows/docs_stable.yml @@ -0,0 +1,34 @@ +name: Build Docs for releases + +on: + workflow_dispatch: # run on request (no need for PR) + release: + types: [published] + +jobs: + Build-Docs: + runs-on: ubuntu-20.04 + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 # otherwise, you will failed to push refs to dest repo + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: python -m pip install tox + - name: Build-Docs + run: | + echo RELEASE_VERSION=${GITHUB_REF#refs/*/} >> $GITHUB_ENV + tox -e build-doc + # - name: Deploy + # uses: peaceiris/actions-gh-pages@v3 + # with: + # github_token: ${{ secrets.GITHUB_TOKEN }} + # publish_dir: ./public + # destination_dir: ${{ env.RELEASE_VERSION }} + # force_orphan: true diff --git a/docs/contributing.md b/contributing.md similarity index 100% rename from docs/contributing.md rename to contributing.md diff --git a/datumaro/cli/util/project.py b/datumaro/cli/util/project.py index e365209507..ee27ad2b54 100644 --- a/datumaro/cli/util/project.py +++ b/datumaro/cli/util/project.py @@ -153,7 +153,7 @@ def split_local_revpath(revpath: str) -> Tuple[Revision, str]: A local revpath is a path to a revision withing the current project. The syntax is: - - [ : ] [ ] + - [ : ] [ ] At least one part must be present. Returns: (revision, build target) diff --git a/datumaro/plugins/data_formats/common_semantic_segmentation.py b/datumaro/plugins/data_formats/common_semantic_segmentation.py index ff2279a17d..80c0788225 100644 --- a/datumaro/plugins/data_formats/common_semantic_segmentation.py +++ b/datumaro/plugins/data_formats/common_semantic_segmentation.py @@ -165,19 +165,19 @@ def find_sources(cls, path): class CommonSemanticSegmentationWithSubsetDirsImporter(CommonSemanticSegmentationImporter): """It supports the following subset sub-directory structure for CommonSemanticSegmentation. - ``` - Dataset/ - └─ - ├── dataset_meta.json # a list of labels - ├── images/ - │ ├── .png - │ ├── .png - │ └── ... - └── masks/ - ├── .png - ├── .png - └── ... + .. code-block:: + + Dataset/ + └─ + ├── dataset_meta.json # a list of labels + ├── images/ + │ ├── .png + │ ├── .png + │ └── ... + └── masks/ + ├── .png + ├── .png + └── ... Then, the imported dataset will have train, val, ... CommonSemanticSegmentation subsets. - ``` """ diff --git a/datumaro/plugins/data_formats/datumaro_binary/mapper/__init__.py b/datumaro/plugins/data_formats/datumaro_binary/mapper/__init__.py index a473c93539..cefedf4cbd 100644 --- a/datumaro/plugins/data_formats/datumaro_binary/mapper/__init__.py +++ b/datumaro/plugins/data_formats/datumaro_binary/mapper/__init__.py @@ -5,7 +5,7 @@ # ruff: noqa: F405 from .annotation import * -from .common import * +from .common import DictMapper, FloatListMapper, IntListMapper, Mapper, StringMapper from .dataset_item import * from .media import * diff --git a/datumaro/plugins/data_formats/imagenet.py b/datumaro/plugins/data_formats/imagenet.py index 16e0700881..249c2254cf 100644 --- a/datumaro/plugins/data_formats/imagenet.py +++ b/datumaro/plugins/data_formats/imagenet.py @@ -66,12 +66,15 @@ class ImagenetImporter(Importer): """TorchVision's ImageFolder style importer. For example, it imports the following directory structure. - root - ├── label_0 - │ ├── label_0_1.jpg - │ └── label_0_2.jpg - └── label_1 - └── label_1_1.jpg + .. code-block:: text + + root + ├── label_0 + │ ├── label_0_1.jpg + │ └── label_0_2.jpg + └── label_1 + └── label_1_1.jpg + """ @classmethod @@ -106,25 +109,27 @@ class ImagenetWithSubsetDirsImporter(ImagenetImporter): """TorchVision ImageFolder style importer. For example, it imports the following directory structure. - root - ├── train - │ ├── label_0 - │ │ ├── label_0_1.jpg - │ │ └── label_0_2.jpg - │ └── label_1 - │ └── label_1_1.jpg - ├── val - │ ├── label_0 - │ │ ├── label_0_1.jpg - │ │ └── label_0_2.jpg - │ └── label_1 - │ └── label_1_1.jpg - └── test - ├── label_0 - │ ├── label_0_1.jpg - │ └── label_0_2.jpg - └── label_1 - └── label_1_1.jpg + .. code-block:: + + root + ├── train + │ ├── label_0 + │ │ ├── label_0_1.jpg + │ │ └── label_0_2.jpg + │ └── label_1 + │ └── label_1_1.jpg + ├── val + │ ├── label_0 + │ │ ├── label_0_1.jpg + │ │ └── label_0_2.jpg + │ └── label_1 + │ └── label_1_1.jpg + └── test + ├── label_0 + │ ├── label_0_1.jpg + │ └── label_0_2.jpg + └── label_1 + └── label_1_1.jpg Then, it will have three subsets: train, val, and test and they have label_0 and label_1 labels. """ diff --git a/datumaro/plugins/sampler/random_sampler.py b/datumaro/plugins/sampler/random_sampler.py index ed8813ca0b..2a3cd21673 100644 --- a/datumaro/plugins/sampler/random_sampler.py +++ b/datumaro/plugins/sampler/random_sampler.py @@ -15,7 +15,7 @@ class RandomSampler(Transform, CliPlugin): - """ + r""" Sampler that keeps no more than required number of items in the dataset.|n |n Notes:|n diff --git a/docs/source/conf.py b/docs/source/conf.py index f6112f2eff..334e0cf13f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -63,7 +63,8 @@ suppress_warnings = [ # "myst.xref_missing", - "myst.iref_ambiguous" + "myst.iref_ambiguous", + "autosectionlabel.*", ] autosummary_generate = True # Turn on sphinx.ext.autosummary diff --git a/docs/source/docs/level-up/advanced_skills/index.rst b/docs/source/docs/level-up/advanced_skills/index.rst index 02e7fc045f..4e2ce0ec01 100644 --- a/docs/source/docs/level-up/advanced_skills/index.rst +++ b/docs/source/docs/level-up/advanced_skills/index.rst @@ -1,5 +1,5 @@ Advanced Skills -########### +############### .. panels:: diff --git a/docs/source/docs/level-up/basic_skills/03_dataset_import_export.rst b/docs/source/docs/level-up/basic_skills/03_dataset_import_export.rst index c4e67b15dc..0d0fe33c5e 100644 --- a/docs/source/docs/level-up/basic_skills/03_dataset_import_export.rst +++ b/docs/source/docs/level-up/basic_skills/03_dataset_import_export.rst @@ -1,6 +1,6 @@ -============= +=============================== Level 3: Data Import and Export -============= +=============================== Datumaro is a tool that supports public data formats across a wide range of tasks such as classification, detection, segmentation, pose estimation, or visual tracking. @@ -8,7 +8,7 @@ To facilitate this, Datumaro provides assistance with data import and export via This makes it easier for users to work with various data formats using Datumaro. Prepare dataset -============ +=============== For the segmentation task, we here introduce the Cityscapes, which collects road scenes from 50 different cities and contains 5K fine-grained pixel-level annotations and 20K coarse annotations. @@ -16,7 +16,7 @@ More detailed description is given by :ref:`here `. The Cityscapes dataset is available for free `download `_. Convert data format -============ +=================== Users sometimes needs to compare, merge, or manage various kinds of public datasets in a unified system. To achieve this, Datumaro not only has `import` and `export` funcionalities, but also diff --git a/docs/source/docs/level-up/basic_skills/04_detect_data_format.rst b/docs/source/docs/level-up/basic_skills/04_detect_data_format.rst index 5a30683f0f..739ffadde8 100644 --- a/docs/source/docs/level-up/basic_skills/04_detect_data_format.rst +++ b/docs/source/docs/level-up/basic_skills/04_detect_data_format.rst @@ -1,6 +1,6 @@ -============= +=================================================== Level 4: Detect Data Format from an Unknown Dataset -============= +=================================================== Datumaro provides a function to detect the format of a dataset before importing data. This can be useful in cases where information about the original format of the data has been lost or is unclear. @@ -8,7 +8,7 @@ With this function, users can easily identify the format and proceed with approp handling processes. Detect data format -============ +================== .. tabbed:: CLI diff --git a/docs/source/docs/level-up/basic_skills/index.rst b/docs/source/docs/level-up/basic_skills/index.rst index 44a45c6a18..55d9f1c975 100644 --- a/docs/source/docs/level-up/basic_skills/index.rst +++ b/docs/source/docs/level-up/basic_skills/index.rst @@ -1,5 +1,5 @@ Basic Skills -########### +############ .. panels:: diff --git a/docs/source/docs/level-up/intermediate_skills/index.rst b/docs/source/docs/level-up/intermediate_skills/index.rst index c43c1f183c..58301b2620 100644 --- a/docs/source/docs/level-up/intermediate_skills/index.rst +++ b/docs/source/docs/level-up/intermediate_skills/index.rst @@ -1,5 +1,5 @@ Intermediate Skills -########### +################### .. panels:: diff --git a/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.entropy.rst b/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.entropy.rst index afeed79720..5c409a8416 100644 --- a/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.entropy.rst +++ b/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.entropy.rst @@ -2,13 +2,6 @@ Entropy module -------------- .. automodule:: datumaro.plugins.sampler.algorithm.entropy - - .. autoclass:: SampleEntropy - - .. automethod:: __init__ - - .. automethod:: get_sample - - .. automethod:: _get_sample_mixed - - .. automethod:: _rank_images + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.rst b/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.rst index 82897cdf92..af0de90cf3 100644 --- a/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.rst +++ b/docs/source/docs/reference/plugins/sampler/algorithm/datumaro.plugins.sampler.algorithm.rst @@ -5,4 +5,3 @@ Algorithm module :members: :undoc-members: :show-inheritance: - :private-members: diff --git a/docs/source/docs/user-manual/extending.md b/docs/source/docs/user-manual/extending.md index 0ab1a4af36..b40143cdea 100644 --- a/docs/source/docs/user-manual/extending.md +++ b/docs/source/docs/user-manual/extending.md @@ -1,8 +1,8 @@ # Extending There are few ways to extend and customize Datumaro behavior, which is -supported by plugins. Check [our contribution guide](/docs/contributing) for -details on plugin implementation. In general, a plugin is a Python module. +supported by plugins. Check [our contribution guide](https://github.com/openvinotoolkit/datumaro/blob/develop/contributing.md) +for details on plugin implementation. In general, a plugin is a Python module. It must be put into a plugin directory: - `/.datumaro/plugins` for project-specific plugins - `/plugins` for global plugins diff --git a/notebooks/09_encrypt_dataset.ipynb b/notebooks/09_encrypt_dataset.ipynb index 031eeb170c..583eec912b 100644 --- a/notebooks/09_encrypt_dataset.ipynb +++ b/notebooks/09_encrypt_dataset.ipynb @@ -315,7 +315,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Re-export again to any dataset format with no encryption\n", + "## Re-export again to any dataset format with no encryption\n", "\n", "Because the `DatumaroBinary` format is encrypted, it cannot be easily used for your purposes. In this time, we re-export it to any dataset format for the future usage. For example, COCO format is used for the export." ] diff --git a/requirements.txt b/requirements.txt index b92ecb1207..dd2ea9f2b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,3 @@ -r requirements-default.txt opencv-python-headless>=4.1.0.25 - -# docs -markupsafe>=2.0.1 -nbconvert>=7.2.3 -ipython>=8.4.0 diff --git a/tox.ini b/tox.ini index 524c366078..26d64b8777 100644 --- a/tox.ini +++ b/tox.ini @@ -2,10 +2,12 @@ isolated_build = true skip_missing_interpreters = true + [testenv] deps = -r{toxinidir}/requirements.txt + [testenv:pre-commit] basepython = python3 deps =