From 02ac038faa1d8727459a9725dde38fdfd2fdb1bd Mon Sep 17 00:00:00 2001 From: Jordi Smit Date: Mon, 8 Jul 2024 09:48:02 +0200 Subject: [PATCH] refactor: add pre-commit hooks to verify date consistancy (#25) --- .pre-commit-config.yaml | 12 ++ .../index.md | 2 +- .../images/cover.jpg | 0 .../images/demo.gif | 0 .../images/few-shot-example-visual.gif | 0 .../images/few-shot-examples-setting.jpg | 0 .../images/state_diagram.jpg | 0 .../index.md | 0 .../index.md | 2 +- .../index.md | 4 +- requirements.txt | 3 +- scripts/verify_data_matches_folder.py | 105 ++++++++++++++++++ 12 files changed, 123 insertions(+), 5 deletions(-) rename content/blog/{2023-10-05-what-does-it-take-to-add-copilot-to-obsidian => 2023-09-30-what-does-it-take-to-add-copilot-to-obsidian}/images/cover.jpg (100%) rename content/blog/{2023-10-05-what-does-it-take-to-add-copilot-to-obsidian => 2023-09-30-what-does-it-take-to-add-copilot-to-obsidian}/images/demo.gif (100%) rename content/blog/{2023-10-05-what-does-it-take-to-add-copilot-to-obsidian => 2023-09-30-what-does-it-take-to-add-copilot-to-obsidian}/images/few-shot-example-visual.gif (100%) rename content/blog/{2023-10-05-what-does-it-take-to-add-copilot-to-obsidian => 2023-09-30-what-does-it-take-to-add-copilot-to-obsidian}/images/few-shot-examples-setting.jpg (100%) rename content/blog/{2023-10-05-what-does-it-take-to-add-copilot-to-obsidian => 2023-09-30-what-does-it-take-to-add-copilot-to-obsidian}/images/state_diagram.jpg (100%) rename content/blog/{2023-10-05-what-does-it-take-to-add-copilot-to-obsidian => 2023-09-30-what-does-it-take-to-add-copilot-to-obsidian}/index.md (100%) create mode 100644 scripts/verify_data_matches_folder.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31af6a9..15f8ce0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,3 +38,15 @@ repos: - id: markdownlint-fix name: Format Markdown files exclude: ^utils + +- repo: local + hooks: + - id: verify_data_matches_folder + name: verify_data_matches_folder + files: ^content/(blog|til)/.*/index\.md$ + stages: [commit] + language: python + additional_dependencies: + - python-frontmatter==1.1.0 + args: [] + entry: python scripts/verify_data_matches_folder.py diff --git a/content/blog/2022-11-06-autograd-engine-from-scratch/index.md b/content/blog/2022-11-06-autograd-engine-from-scratch/index.md index 6d194a6..56851f9 100644 --- a/content/blog/2022-11-06-autograd-engine-from-scratch/index.md +++ b/content/blog/2022-11-06-autograd-engine-from-scratch/index.md @@ -1,7 +1,7 @@ --- title: "DIY auto-grad Engine: A Step-by-Step Guide to Calculating Derivatives Automatically" description: "Want to understand the magic of Jax, PyTorch, and TensorFlow auto-grad engines? The best to learn is to build your DIY version from scratch in Python." -date: 2022-12-22TT16:37:27+01:00 +date: 2022-11-06T16:37:27+01:00 publishdate: 2022-12-22T16:37:27+01:00 tags: - python diff --git a/content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/cover.jpg b/content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/cover.jpg similarity index 100% rename from content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/cover.jpg rename to content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/cover.jpg diff --git a/content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/demo.gif b/content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/demo.gif similarity index 100% rename from content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/demo.gif rename to content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/demo.gif diff --git a/content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-example-visual.gif b/content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-example-visual.gif similarity index 100% rename from content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-example-visual.gif rename to content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-example-visual.gif diff --git a/content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-examples-setting.jpg b/content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-examples-setting.jpg similarity index 100% rename from content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-examples-setting.jpg rename to content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/few-shot-examples-setting.jpg diff --git a/content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/state_diagram.jpg b/content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/state_diagram.jpg similarity index 100% rename from content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/images/state_diagram.jpg rename to content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/images/state_diagram.jpg diff --git a/content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/index.md b/content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/index.md similarity index 100% rename from content/blog/2023-10-05-what-does-it-take-to-add-copilot-to-obsidian/index.md rename to content/blog/2023-09-30-what-does-it-take-to-add-copilot-to-obsidian/index.md diff --git a/content/blog/2024-03-13-how-to-avoid-orientation-bug-in-computer-vision-labeling/index.md b/content/blog/2024-03-13-how-to-avoid-orientation-bug-in-computer-vision-labeling/index.md index 75ea403..4f88fdb 100644 --- a/content/blog/2024-03-13-how-to-avoid-orientation-bug-in-computer-vision-labeling/index.md +++ b/content/blog/2024-03-13-how-to-avoid-orientation-bug-in-computer-vision-labeling/index.md @@ -1,7 +1,7 @@ --- title: "How to avoid orientation bugs in Computer Vision labeling?" description: "Image orientation metadata can cause strange bugs. This post teaches you how to identify these bugs and how to fix them." -date: 2024-04-13T19:54:44+02:00 +date: 2024-03-13T19:54:44+02:00 publishdate: 2024-03-13T19:54:44+02:00 tags: - computer vision diff --git a/content/blog/2024-04-03-how-to-upload-pre-annotations-to-roboflow/index.md b/content/blog/2024-04-03-how-to-upload-pre-annotations-to-roboflow/index.md index 4349dac..55c6b26 100644 --- a/content/blog/2024-04-03-how-to-upload-pre-annotations-to-roboflow/index.md +++ b/content/blog/2024-04-03-how-to-upload-pre-annotations-to-roboflow/index.md @@ -1,8 +1,8 @@ --- title: "How to Upload (Pre)-Annotations to Roboflow" description: "Learn how to upload annotations and predictions to Roboflow using the Python SDK, allowing you to import pre-existing datasets or create active learning loops programmatically." -date: 2024-06-13T21:44:10+02:00 -publishdate: 2024-06-13T21:44:10+02:00 +date: 2024-04-03T21:44:10+02:00 +publishdate: 2024-04-03T21:44:10+02:00 tags: - computer vision - labeling diff --git a/requirements.txt b/requirements.txt index 73f02e9..0da7819 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -invoke \ No newline at end of file +invoke +python-frontmatter \ No newline at end of file diff --git a/scripts/verify_data_matches_folder.py b/scripts/verify_data_matches_folder.py new file mode 100644 index 0000000..9bfac1d --- /dev/null +++ b/scripts/verify_data_matches_folder.py @@ -0,0 +1,105 @@ +import datetime +from argparse import Namespace, ArgumentParser +from pathlib import Path +import re +import sys + +import frontmatter + + +def main() -> None: + args = parse_args() + paths = args.paths + + errors_per_path = {} + + for path in paths: + if not path.exists() or path.is_dir() or path.suffix != ".md": + continue + + errors = [] + folder_date = _get_folder_data(path) + publish_date = _get_publish_date(path) + creation_data = _get_data(path) + + + + if folder_date is None: + errors.append(f"Folder name does not contain a date: {path.parent.name}") + + if publish_date is None: + errors.append("Publish date not found in metadata") + + if creation_data is None: + errors.append("Creation date not found in metadata") + + if folder_date != creation_data: + errors.append(f"Folder date ({folder_date}) does not match creation date ({creation_data})") + + + if publish_date is not None and creation_data is not None: + if creation_data > publish_date: + errors.append(f"Publish date ({publish_date}) is before creation date ({creation_data})") + + if len(errors) > 0: + errors_per_path[path] = errors + + if len(errors_per_path) > 0: + for path, errors in errors_per_path.items(): + print(f"Errors in {path}") + for error in errors: + print(f"- {error}") + sys.exit(1) + + +def parse_args() -> Namespace: + parser = ArgumentParser() + parser.add_argument("paths", type=Path, nargs='+', help="Paths to the files to verify") + + return parser.parse_args() + +def _get_folder_data(path: Path) -> datetime.date: + folder_name = path.parent.name + + match = re.match(r"(\d{4}-\d{2}-\d{2})", folder_name) + if match is None: + raise None + + return datetime.datetime.strptime(match.group(1), "%Y-%m-%d").date() + +def _get_publish_date(path: Path) -> datetime.date: + assert path.suffix == ".md", "File must be a markdown file" + + with path.open() as file: + meta_data = frontmatter.load(file) + + + if "publishdate" in meta_data: + try: + return _parse_metadata_date(meta_data["publishdate"]).date() + except Exception: + return None + + + raise None + +def _get_data(path: Path) -> datetime.date: + with path.open() as file: + meta_data = frontmatter.load(file) + + if "date" in meta_data: + try: + return _parse_metadata_date(meta_data["date"]).date() + except Exception: + return None + + + raise None + +def _parse_metadata_date(date: str|datetime.datetime) -> datetime.datetime: + if isinstance(date, datetime.datetime): + return date + return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z") + +if __name__ == "__main__": + main()