Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PR: Implement support for fast integer digest using "xxhash". #1149

Merged
merged 2 commits into from
May 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions colour/utilities/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
validate_method,
optional,
slugify,
int_digest,
)
from .verbose import (
ColourWarning,
Expand Down Expand Up @@ -167,6 +168,7 @@
"validate_method",
"optional",
"slugify",
"int_digest",
]
__all__ += [
"ColourWarning",
Expand Down
80 changes: 80 additions & 0 deletions colour/utilities/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
"is_pandas_installed",
"is_tqdm_installed",
"is_trimesh_installed",
"is_xxhash_installed",
"required",
"is_iterable",
"is_string",
Expand All @@ -86,6 +87,7 @@
"validate_method",
"optional",
"slugify",
"int_digest",
]


Expand Down Expand Up @@ -894,6 +896,42 @@ def is_trimesh_installed(raise_exception: bool = False) -> bool:
return False


def is_xxhash_installed(raise_exception: bool = False) -> bool:
"""
Return whether *xxhash* is installed and available.

Parameters
----------
raise_exception
Whether to raise an exception if *xxhash* is unavailable.

Returns
-------
:class:`bool`
Whether *xxhash* is installed.

Raises
------
:class:`ImportError`
If *xxhash* is not installed.
"""

try: # pragma: no cover
# pylint: disable=W0611
import xxhash # noqa: F401

return True
except ImportError as error: # pragma: no cover
if raise_exception:
raise ImportError(
'"xxhash" related API features are not available: '
f'"{error}".\nSee the installation guide for more information: '
"https://www.colour-science.org/installation-guide/"
) from error

return False


_REQUIREMENTS_TO_CALLABLE: CanonicalMapping = CanonicalMapping(
{
"ctlrender": is_ctlrender_installed,
Expand All @@ -905,6 +943,7 @@ def is_trimesh_installed(raise_exception: bool = False) -> bool:
"Pandas": is_pandas_installed,
"tqdm": is_tqdm_installed,
"trimesh": is_trimesh_installed,
"xxhash": is_xxhash_installed,
}
)
"""
Expand All @@ -923,6 +962,7 @@ def required(
"Pandas",
"tqdm",
"trimesh",
"xxhash",
]
) -> Callable:
"""
Expand Down Expand Up @@ -1427,3 +1467,43 @@ def slugify(object_: Any, allow_unicode: bool = False) -> str:
value = re.sub(r"[^\w\s-]", "", value.lower())

return re.sub(r"[-\s]+", "-", value).strip("-_")


if is_xxhash_installed():
import xxhash
from colour.utilities.documentation import is_documentation_building

int_digest = xxhash.xxh3_64_intdigest # pyright: ignore

if is_documentation_building(): # pragma: no cover
import array

def int_digest(
args: str # noqa: ARG001
| bytes
| bytearray
| memoryview
| array.ArrayType[int],
seed: int = 0, # noqa: ARG001
) -> int:
"""
Generate an integer digest for given argument using *xxhash* if
available or falling back to :func:`hash` if not.

Parameters
----------
args
Argument to generate the int digest of.
seed
Seed used to alter result predictably.

Returns
-------
:class:`int`
Integer digest.
"""

return -1

else:
int_digest = hash # pyright: ignore
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this line need #pragma: no cover?

Otherwise I don't see any issues and this looks ready to merge.

27 changes: 26 additions & 1 deletion colour/utilities/tests/test_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from __future__ import annotations

import numpy as np
import platform
import unittest
import unicodedata
from functools import partial
Expand All @@ -26,6 +27,7 @@
validate_method,
optional,
slugify,
int_digest,
)

__author__ = "Colour Developers"
Expand Down Expand Up @@ -505,7 +507,7 @@ class TestSlugify(unittest.TestCase):
"""

def test_slugify(self):
"""Test :func:`colour.utilities.common.optional` definition."""
"""Test :func:`colour.utilities.common.slugify` definition."""

self.assertEqual(
slugify(
Expand Down Expand Up @@ -536,5 +538,28 @@ def test_slugify(self):
self.assertEqual(slugify(123), "123")


class TestIntDigest(unittest.TestCase):
"""
Define :func:`colour.utilities.common.int_digest` definition unit tests
methods.
"""

def test_int_digest(self):
"""Test :func:`colour.utilities.common.int_digest` definition."""

self.assertEqual(int_digest("Foo"), 7467386374397815550)

if platform.system() in ("Windows", "Microsoft"): # pragma: no cover
self.assertEqual(
int_digest(np.array([1, 2, 3]).tobytes()), 7764052002911021640
)
else:
self.assertEqual(
int_digest(np.array([1, 2, 3]).tobytes()), 8964613590703056768
)

self.assertEqual(int_digest(repr((1, 2, 3))), 5069958125469218295)


if __name__ == "__main__":
unittest.main()
15 changes: 8 additions & 7 deletions colour/utilities/verbose.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import traceback
import warnings
from collections import defaultdict
from contextlib import contextmanager
from contextlib import contextmanager, suppress
from itertools import chain
from textwrap import TextWrapper
from warnings import filterwarnings, formatwarning, warn
Expand Down Expand Up @@ -655,18 +655,19 @@ def describe_environment(
"tqdm",
"trimesh",
]:
try:
with suppress(ImportError):
namespace = __import__(package)
environment["Runtime"][package] = namespace.__version__
except ImportError:
continue

# OpenImageIO
try: # pragma: no cover
with suppress(ImportError): # pragma: no cover
namespace = __import__("OpenImageIO")
environment["Runtime"]["OpenImageIO"] = namespace.VERSION_STRING
except ImportError: # pragma: no cover
pass

# xxhash
with suppress(ImportError): # pragma: no cover
namespace = __import__("xxhash")
environment["Runtime"]["xxhash"] = namespace.version.VERSION

environment["Runtime"].update(ANCILLARY_RUNTIME_PACKAGES)

Expand Down
2 changes: 2 additions & 0 deletions docs/colour.utilities.rst
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ Common
is_pandas_installed
is_tqdm_installed
is_trimesh_installed
is_xxhash_installed
required
is_iterable
is_string
Expand All @@ -81,6 +82,7 @@ Common
validate_method
optional
slugify
int_digest

Array
-----
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ pandas = { version = ">= 1.3, < 2", optional = true }
pygraphviz = { version = ">= 1, < 2", optional = true }
tqdm = { version = ">= 4, < 5", optional = true }
trimesh = { version = ">= 3, < 4", optional = true }
xxhash = { version = ">= 3.2, < 4", optional = true }

biblib-simple = { version = "*", optional = true } # Development dependency.
black = { version = "*", optional = true } # Development dependency.
Expand Down Expand Up @@ -124,7 +125,7 @@ development = [
]
graphviz = [ "pygraphviz" ]
meshing = [ "trimesh" ]
optional = [ "networkx", "pandas", "scikit-learn", "tqdm" ]
optional = [ "networkx", "pandas", "scikit-learn", "tqdm", "xxhash" ]
plotting = [ "matplotlib" ]
read-the-docs = [
"matplotlib",
Expand Down