diff --git a/poetry.lock b/poetry.lock index 816305c49..b18b2626e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "apipkg" @@ -698,6 +698,40 @@ files = [ [package.dependencies] colorama = ">=0.4" +[[package]] +name = "huggingface-hub" +version = "0.23.0" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "huggingface_hub-0.23.0-py3-none-any.whl", hash = "sha256:075c30d48ee7db2bba779190dc526d2c11d422aed6f9044c5e2fdc2c432fdb91"}, + {file = "huggingface_hub-0.23.0.tar.gz", hash = "sha256:7126dedd10a4c6fac796ced4d87a8cf004efc722a5125c2c09299017fa366fa9"}, +] + +[package.dependencies] +filelock = "*" +fsspec = ">=2023.5.0" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.3.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.3.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +hf-transfer = ["hf-transfer (>=0.1.4)"] +inference = ["aiohttp", "minijinja (>=1.0)"] +quality = ["mypy (==1.5.1)", "ruff (>=0.3.0)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +tensorflow-testing = ["keras (<3.0)", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["safetensors", "torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] + [[package]] name = "idna" version = "3.7" @@ -2466,7 +2500,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2847,6 +2880,128 @@ files = [ {file = "rpds_py-0.18.1.tar.gz", hash = "sha256:dc48b479d540770c811fbd1eb9ba2bb66951863e448efec2e2c102625328e92f"}, ] +[[package]] +name = "safetensors" +version = "0.4.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "safetensors-0.4.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:dcf5705cab159ce0130cd56057f5f3425023c407e170bca60b4868048bae64fd"}, + {file = "safetensors-0.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bb4f8c5d0358a31e9a08daeebb68f5e161cdd4018855426d3f0c23bb51087055"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70a5319ef409e7f88686a46607cbc3c428271069d8b770076feaf913664a07ac"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fb9c65bd82f9ef3ce4970dc19ee86be5f6f93d032159acf35e663c6bea02b237"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edb5698a7bc282089f64c96c477846950358a46ede85a1c040e0230344fdde10"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:efcc860be094b8d19ac61b452ec635c7acb9afa77beb218b1d7784c6d41fe8ad"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d88b33980222085dd6001ae2cad87c6068e0991d4f5ccf44975d216db3b57376"}, + {file = "safetensors-0.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5fc6775529fb9f0ce2266edd3e5d3f10aab068e49f765e11f6f2a63b5367021d"}, + {file = "safetensors-0.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9c6ad011c1b4e3acff058d6b090f1da8e55a332fbf84695cf3100c649cc452d1"}, + {file = "safetensors-0.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8c496c5401c1b9c46d41a7688e8ff5b0310a3b9bae31ce0f0ae870e1ea2b8caf"}, + {file = "safetensors-0.4.3-cp310-none-win32.whl", hash = "sha256:38e2a8666178224a51cca61d3cb4c88704f696eac8f72a49a598a93bbd8a4af9"}, + {file = "safetensors-0.4.3-cp310-none-win_amd64.whl", hash = "sha256:393e6e391467d1b2b829c77e47d726f3b9b93630e6a045b1d1fca67dc78bf632"}, + {file = "safetensors-0.4.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:22f3b5d65e440cec0de8edaa672efa888030802e11c09b3d6203bff60ebff05a"}, + {file = "safetensors-0.4.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c4fa560ebd4522adddb71dcd25d09bf211b5634003f015a4b815b7647d62ebe"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9afd5358719f1b2cf425fad638fc3c887997d6782da317096877e5b15b2ce93"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d8c5093206ef4b198600ae484230402af6713dab1bd5b8e231905d754022bec7"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0b2104df1579d6ba9052c0ae0e3137c9698b2d85b0645507e6fd1813b70931a"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8cf18888606dad030455d18f6c381720e57fc6a4170ee1966adb7ebc98d4d6a3"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0bf4f9d6323d9f86eef5567eabd88f070691cf031d4c0df27a40d3b4aaee755b"}, + {file = "safetensors-0.4.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:585c9ae13a205807b63bef8a37994f30c917ff800ab8a1ca9c9b5d73024f97ee"}, + {file = "safetensors-0.4.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faefeb3b81bdfb4e5a55b9bbdf3d8d8753f65506e1d67d03f5c851a6c87150e9"}, + {file = "safetensors-0.4.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:befdf0167ad626f22f6aac6163477fcefa342224a22f11fdd05abb3995c1783c"}, + {file = "safetensors-0.4.3-cp311-none-win32.whl", hash = "sha256:a7cef55929dcbef24af3eb40bedec35d82c3c2fa46338bb13ecf3c5720af8a61"}, + {file = "safetensors-0.4.3-cp311-none-win_amd64.whl", hash = "sha256:840b7ac0eff5633e1d053cc9db12fdf56b566e9403b4950b2dc85393d9b88d67"}, + {file = "safetensors-0.4.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:22d21760dc6ebae42e9c058d75aa9907d9f35e38f896e3c69ba0e7b213033856"}, + {file = "safetensors-0.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d22c1a10dff3f64d0d68abb8298a3fd88ccff79f408a3e15b3e7f637ef5c980"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1648568667f820b8c48317c7006221dc40aced1869908c187f493838a1362bc"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:446e9fe52c051aeab12aac63d1017e0f68a02a92a027b901c4f8e931b24e5397"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fef5d70683643618244a4f5221053567ca3e77c2531e42ad48ae05fae909f542"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a1f4430cc0c9d6afa01214a4b3919d0a029637df8e09675ceef1ca3f0dfa0df"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d603846a8585b9432a0fd415db1d4c57c0f860eb4aea21f92559ff9902bae4d"}, + {file = "safetensors-0.4.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a844cdb5d7cbc22f5f16c7e2a0271170750763c4db08381b7f696dbd2c78a361"}, + {file = "safetensors-0.4.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:88887f69f7a00cf02b954cdc3034ffb383b2303bc0ab481d4716e2da51ddc10e"}, + {file = "safetensors-0.4.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ee463219d9ec6c2be1d331ab13a8e0cd50d2f32240a81d498266d77d07b7e71e"}, + {file = "safetensors-0.4.3-cp312-none-win32.whl", hash = "sha256:d0dd4a1db09db2dba0f94d15addc7e7cd3a7b0d393aa4c7518c39ae7374623c3"}, + {file = "safetensors-0.4.3-cp312-none-win_amd64.whl", hash = "sha256:d14d30c25897b2bf19b6fb5ff7e26cc40006ad53fd4a88244fdf26517d852dd7"}, + {file = "safetensors-0.4.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d1456f814655b224d4bf6e7915c51ce74e389b413be791203092b7ff78c936dd"}, + {file = "safetensors-0.4.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:455d538aa1aae4a8b279344a08136d3f16334247907b18a5c3c7fa88ef0d3c46"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf476bca34e1340ee3294ef13e2c625833f83d096cfdf69a5342475602004f95"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02ef3a24face643456020536591fbd3c717c5abaa2737ec428ccbbc86dffa7a4"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7de32d0d34b6623bb56ca278f90db081f85fb9c5d327e3c18fd23ac64f465768"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a0deb16a1d3ea90c244ceb42d2c6c276059616be21a19ac7101aa97da448faf"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59d51f182c729f47e841510b70b967b0752039f79f1de23bcdd86462a9b09ee"}, + {file = "safetensors-0.4.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1f598b713cc1a4eb31d3b3203557ac308acf21c8f41104cdd74bf640c6e538e3"}, + {file = "safetensors-0.4.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5757e4688f20df083e233b47de43845d1adb7e17b6cf7da5f8444416fc53828d"}, + {file = "safetensors-0.4.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fe746d03ed8d193674a26105e4f0fe6c726f5bb602ffc695b409eaf02f04763d"}, + {file = "safetensors-0.4.3-cp37-none-win32.whl", hash = "sha256:0d5ffc6a80f715c30af253e0e288ad1cd97a3d0086c9c87995e5093ebc075e50"}, + {file = "safetensors-0.4.3-cp37-none-win_amd64.whl", hash = "sha256:a11c374eb63a9c16c5ed146457241182f310902bd2a9c18255781bb832b6748b"}, + {file = "safetensors-0.4.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1e31be7945f66be23f4ec1682bb47faa3df34cb89fc68527de6554d3c4258a4"}, + {file = "safetensors-0.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:03a4447c784917c9bf01d8f2ac5080bc15c41692202cd5f406afba16629e84d6"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d244bcafeb1bc06d47cfee71727e775bca88a8efda77a13e7306aae3813fa7e4"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53c4879b9c6bd7cd25d114ee0ef95420e2812e676314300624594940a8d6a91f"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74707624b81f1b7f2b93f5619d4a9f00934d5948005a03f2c1845ffbfff42212"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0d52c958dc210265157573f81d34adf54e255bc2b59ded6218500c9b15a750eb"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f9568f380f513a60139971169c4a358b8731509cc19112369902eddb33faa4d"}, + {file = "safetensors-0.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0d9cd8e1560dfc514b6d7859247dc6a86ad2f83151a62c577428d5102d872721"}, + {file = "safetensors-0.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:89f9f17b0dacb913ed87d57afbc8aad85ea42c1085bd5de2f20d83d13e9fc4b2"}, + {file = "safetensors-0.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1139eb436fd201c133d03c81209d39ac57e129f5e74e34bb9ab60f8d9b726270"}, + {file = "safetensors-0.4.3-cp38-none-win32.whl", hash = "sha256:d9c289f140a9ae4853fc2236a2ffc9a9f2d5eae0cb673167e0f1b8c18c0961ac"}, + {file = "safetensors-0.4.3-cp38-none-win_amd64.whl", hash = "sha256:622afd28968ef3e9786562d352659a37de4481a4070f4ebac883f98c5836563e"}, + {file = "safetensors-0.4.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8651c7299cbd8b4161a36cd6a322fa07d39cd23535b144d02f1c1972d0c62f3c"}, + {file = "safetensors-0.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e375d975159ac534c7161269de24ddcd490df2157b55c1a6eeace6cbb56903f0"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:084fc436e317f83f7071fc6a62ca1c513b2103db325cd09952914b50f51cf78f"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:41a727a7f5e6ad9f1db6951adee21bbdadc632363d79dc434876369a17de6ad6"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7dbbde64b6c534548696808a0e01276d28ea5773bc9a2dfb97a88cd3dffe3df"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bbae3b4b9d997971431c346edbfe6e41e98424a097860ee872721e176040a893"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01e4b22e3284cd866edeabe4f4d896229495da457229408d2e1e4810c5187121"}, + {file = "safetensors-0.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dd37306546b58d3043eb044c8103a02792cc024b51d1dd16bd3dd1f334cb3ed"}, + {file = "safetensors-0.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8815b5e1dac85fc534a97fd339e12404db557878c090f90442247e87c8aeaea"}, + {file = "safetensors-0.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e011cc162503c19f4b1fd63dfcddf73739c7a243a17dac09b78e57a00983ab35"}, + {file = "safetensors-0.4.3-cp39-none-win32.whl", hash = "sha256:01feb3089e5932d7e662eda77c3ecc389f97c0883c4a12b5cfdc32b589a811c3"}, + {file = "safetensors-0.4.3-cp39-none-win_amd64.whl", hash = "sha256:3f9cdca09052f585e62328c1c2923c70f46814715c795be65f0b93f57ec98a02"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1b89381517891a7bb7d1405d828b2bf5d75528299f8231e9346b8eba092227f9"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:cd6fff9e56df398abc5866b19a32124815b656613c1c5ec0f9350906fd798aac"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:840caf38d86aa7014fe37ade5d0d84e23dcfbc798b8078015831996ecbc206a3"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9650713b2cfa9537a2baf7dd9fee458b24a0aaaa6cafcea8bdd5fb2b8efdc34"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e4119532cd10dba04b423e0f86aecb96cfa5a602238c0aa012f70c3a40c44b50"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e066e8861eef6387b7c772344d1fe1f9a72800e04ee9a54239d460c400c72aab"}, + {file = "safetensors-0.4.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:90964917f5b0fa0fa07e9a051fbef100250c04d150b7026ccbf87a34a54012e0"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c41e1893d1206aa7054029681778d9a58b3529d4c807002c156d58426c225173"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae7613a119a71a497d012ccc83775c308b9c1dab454806291427f84397d852fd"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9bac020faba7f5dc481e881b14b6425265feabb5bfc552551d21189c0eddc3"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:420a98f593ff9930f5822560d14c395ccbc57342ddff3b463bc0b3d6b1951550"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:f5e6883af9a68c0028f70a4c19d5a6ab6238a379be36ad300a22318316c00cb0"}, + {file = "safetensors-0.4.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:cdd0a3b5da66e7f377474599814dbf5cbf135ff059cc73694de129b58a5e8a2c"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9bfb92f82574d9e58401d79c70c716985dc049b635fef6eecbb024c79b2c46ad"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3615a96dd2dcc30eb66d82bc76cda2565f4f7bfa89fcb0e31ba3cea8a1a9ecbb"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:868ad1b6fc41209ab6bd12f63923e8baeb1a086814cb2e81a65ed3d497e0cf8f"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7ffba80aa49bd09195145a7fd233a7781173b422eeb995096f2b30591639517"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0acbe31340ab150423347e5b9cc595867d814244ac14218932a5cf1dd38eb39"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:19bbdf95de2cf64f25cd614c5236c8b06eb2cfa47cbf64311f4b5d80224623a3"}, + {file = "safetensors-0.4.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b852e47eb08475c2c1bd8131207b405793bfc20d6f45aff893d3baaad449ed14"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5d07cbca5b99babb692d76d8151bec46f461f8ad8daafbfd96b2fca40cadae65"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1ab6527a20586d94291c96e00a668fa03f86189b8a9defa2cdd34a1a01acc7d5"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02318f01e332cc23ffb4f6716e05a492c5f18b1d13e343c49265149396284a44"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec4b52ce9a396260eb9731eb6aea41a7320de22ed73a1042c2230af0212758ce"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:018b691383026a2436a22b648873ed11444a364324e7088b99cd2503dd828400"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:309b10dbcab63269ecbf0e2ca10ce59223bb756ca5d431ce9c9eeabd446569da"}, + {file = "safetensors-0.4.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b277482120df46e27a58082df06a15aebda4481e30a1c21eefd0921ae7e03f65"}, + {file = "safetensors-0.4.3.tar.gz", hash = "sha256:2f85fc50c4e07a21e95c24e07460fe6f7e2859d0ce88092838352b798ce711c2"}, +] + +[package.extras] +all = ["safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[pinned-tf]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] +dev = ["safetensors[all]"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] +mlx = ["mlx (>=0.0.9)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] +pinned-tf = ["safetensors[numpy]", "tensorflow (==2.11.0)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] +torch = ["safetensors[numpy]", "torch (>=1.10)"] + [[package]] name = "scikit-learn" version = "1.4.2" @@ -3108,6 +3263,123 @@ webencodings = ">=0.4" doc = ["sphinx", "sphinx_rtd_theme"] test = ["pytest", "ruff"] +[[package]] +name = "tokenizers" +version = "0.19.1" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tokenizers-0.19.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:952078130b3d101e05ecfc7fc3640282d74ed26bcf691400f872563fca15ac97"}, + {file = "tokenizers-0.19.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82c8b8063de6c0468f08e82c4e198763e7b97aabfe573fd4cf7b33930ca4df77"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f03727225feaf340ceeb7e00604825addef622d551cbd46b7b775ac834c1e1c4"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:453e4422efdfc9c6b6bf2eae00d5e323f263fff62b29a8c9cd526c5003f3f642"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02e81bf089ebf0e7f4df34fa0207519f07e66d8491d963618252f2e0729e0b46"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b07c538ba956843833fee1190cf769c60dc62e1cf934ed50d77d5502194d63b1"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e28cab1582e0eec38b1f38c1c1fb2e56bce5dc180acb1724574fc5f47da2a4fe"}, + {file = "tokenizers-0.19.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b01afb7193d47439f091cd8f070a1ced347ad0f9144952a30a41836902fe09e"}, + {file = "tokenizers-0.19.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7fb297edec6c6841ab2e4e8f357209519188e4a59b557ea4fafcf4691d1b4c98"}, + {file = "tokenizers-0.19.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2e8a3dd055e515df7054378dc9d6fa8c8c34e1f32777fb9a01fea81496b3f9d3"}, + {file = "tokenizers-0.19.1-cp310-none-win32.whl", hash = "sha256:7ff898780a155ea053f5d934925f3902be2ed1f4d916461e1a93019cc7250837"}, + {file = "tokenizers-0.19.1-cp310-none-win_amd64.whl", hash = "sha256:bea6f9947e9419c2fda21ae6c32871e3d398cba549b93f4a65a2d369662d9403"}, + {file = "tokenizers-0.19.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:5c88d1481f1882c2e53e6bb06491e474e420d9ac7bdff172610c4f9ad3898059"}, + {file = "tokenizers-0.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ddf672ed719b4ed82b51499100f5417d7d9f6fb05a65e232249268f35de5ed14"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:dadc509cc8a9fe460bd274c0e16ac4184d0958117cf026e0ea8b32b438171594"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfedf31824ca4915b511b03441784ff640378191918264268e6923da48104acc"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac11016d0a04aa6487b1513a3a36e7bee7eec0e5d30057c9c0408067345c48d2"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76951121890fea8330d3a0df9a954b3f2a37e3ec20e5b0530e9a0044ca2e11fe"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b342d2ce8fc8d00f376af068e3274e2e8649562e3bc6ae4a67784ded6b99428d"}, + {file = "tokenizers-0.19.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d16ff18907f4909dca9b076b9c2d899114dd6abceeb074eca0c93e2353f943aa"}, + {file = "tokenizers-0.19.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:706a37cc5332f85f26efbe2bdc9ef8a9b372b77e4645331a405073e4b3a8c1c6"}, + {file = "tokenizers-0.19.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:16baac68651701364b0289979ecec728546133e8e8fe38f66fe48ad07996b88b"}, + {file = "tokenizers-0.19.1-cp311-none-win32.whl", hash = "sha256:9ed240c56b4403e22b9584ee37d87b8bfa14865134e3e1c3fb4b2c42fafd3256"}, + {file = "tokenizers-0.19.1-cp311-none-win_amd64.whl", hash = "sha256:ad57d59341710b94a7d9dbea13f5c1e7d76fd8d9bcd944a7a6ab0b0da6e0cc66"}, + {file = "tokenizers-0.19.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:621d670e1b1c281a1c9698ed89451395d318802ff88d1fc1accff0867a06f153"}, + {file = "tokenizers-0.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d924204a3dbe50b75630bd16f821ebda6a5f729928df30f582fb5aade90c818a"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4f3fefdc0446b1a1e6d81cd4c07088ac015665d2e812f6dbba4a06267d1a2c95"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9620b78e0b2d52ef07b0d428323fb34e8ea1219c5eac98c2596311f20f1f9266"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04ce49e82d100594715ac1b2ce87d1a36e61891a91de774755f743babcd0dd52"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5c2ff13d157afe413bf7e25789879dd463e5a4abfb529a2d8f8473d8042e28f"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3174c76efd9d08f836bfccaca7cfec3f4d1c0a4cf3acbc7236ad577cc423c840"}, + {file = "tokenizers-0.19.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c9d5b6c0e7a1e979bec10ff960fae925e947aab95619a6fdb4c1d8ff3708ce3"}, + {file = "tokenizers-0.19.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a179856d1caee06577220ebcfa332af046d576fb73454b8f4d4b0ba8324423ea"}, + {file = "tokenizers-0.19.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:952b80dac1a6492170f8c2429bd11fcaa14377e097d12a1dbe0ef2fb2241e16c"}, + {file = "tokenizers-0.19.1-cp312-none-win32.whl", hash = "sha256:01d62812454c188306755c94755465505836fd616f75067abcae529c35edeb57"}, + {file = "tokenizers-0.19.1-cp312-none-win_amd64.whl", hash = "sha256:b70bfbe3a82d3e3fb2a5e9b22a39f8d1740c96c68b6ace0086b39074f08ab89a"}, + {file = "tokenizers-0.19.1-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:bb9dfe7dae85bc6119d705a76dc068c062b8b575abe3595e3c6276480e67e3f1"}, + {file = "tokenizers-0.19.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:1f0360cbea28ea99944ac089c00de7b2e3e1c58f479fb8613b6d8d511ce98267"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:71e3ec71f0e78780851fef28c2a9babe20270404c921b756d7c532d280349214"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b82931fa619dbad979c0ee8e54dd5278acc418209cc897e42fac041f5366d626"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e8ff5b90eabdcdaa19af697885f70fe0b714ce16709cf43d4952f1f85299e73a"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e742d76ad84acbdb1a8e4694f915fe59ff6edc381c97d6dfdd054954e3478ad4"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d8c5d59d7b59885eab559d5bc082b2985555a54cda04dda4c65528d90ad252ad"}, + {file = "tokenizers-0.19.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b2da5c32ed869bebd990c9420df49813709e953674c0722ff471a116d97b22d"}, + {file = "tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:638e43936cc8b2cbb9f9d8dde0fe5e7e30766a3318d2342999ae27f68fdc9bd6"}, + {file = "tokenizers-0.19.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:78e769eb3b2c79687d9cb0f89ef77223e8e279b75c0a968e637ca7043a84463f"}, + {file = "tokenizers-0.19.1-cp37-none-win32.whl", hash = "sha256:72791f9bb1ca78e3ae525d4782e85272c63faaef9940d92142aa3eb79f3407a3"}, + {file = "tokenizers-0.19.1-cp37-none-win_amd64.whl", hash = "sha256:f3bbb7a0c5fcb692950b041ae11067ac54826204318922da754f908d95619fbc"}, + {file = "tokenizers-0.19.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:07f9295349bbbcedae8cefdbcfa7f686aa420be8aca5d4f7d1ae6016c128c0c5"}, + {file = "tokenizers-0.19.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:10a707cc6c4b6b183ec5dbfc5c34f3064e18cf62b4a938cb41699e33a99e03c1"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6309271f57b397aa0aff0cbbe632ca9d70430839ca3178bf0f06f825924eca22"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ad23d37d68cf00d54af184586d79b84075ada495e7c5c0f601f051b162112dc"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:427c4f0f3df9109314d4f75b8d1f65d9477033e67ffaec4bca53293d3aca286d"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e83a31c9cf181a0a3ef0abad2b5f6b43399faf5da7e696196ddd110d332519ee"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c27b99889bd58b7e301468c0838c5ed75e60c66df0d4db80c08f43462f82e0d3"}, + {file = "tokenizers-0.19.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bac0b0eb952412b0b196ca7a40e7dce4ed6f6926489313414010f2e6b9ec2adf"}, + {file = "tokenizers-0.19.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8a6298bde623725ca31c9035a04bf2ef63208d266acd2bed8c2cb7d2b7d53ce6"}, + {file = "tokenizers-0.19.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:08a44864e42fa6d7d76d7be4bec62c9982f6f6248b4aa42f7302aa01e0abfd26"}, + {file = "tokenizers-0.19.1-cp38-none-win32.whl", hash = "sha256:1de5bc8652252d9357a666e609cb1453d4f8e160eb1fb2830ee369dd658e8975"}, + {file = "tokenizers-0.19.1-cp38-none-win_amd64.whl", hash = "sha256:0bcce02bf1ad9882345b34d5bd25ed4949a480cf0e656bbd468f4d8986f7a3f1"}, + {file = "tokenizers-0.19.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:0b9394bd204842a2a1fd37fe29935353742be4a3460b6ccbaefa93f58a8df43d"}, + {file = "tokenizers-0.19.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4692ab92f91b87769d950ca14dbb61f8a9ef36a62f94bad6c82cc84a51f76f6a"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6258c2ef6f06259f70a682491c78561d492e885adeaf9f64f5389f78aa49a051"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c85cf76561fbd01e0d9ea2d1cbe711a65400092bc52b5242b16cfd22e51f0c58"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:670b802d4d82bbbb832ddb0d41df7015b3e549714c0e77f9bed3e74d42400fbe"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85aa3ab4b03d5e99fdd31660872249df5e855334b6c333e0bc13032ff4469c4a"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbf001afbbed111a79ca47d75941e9e5361297a87d186cbfc11ed45e30b5daba"}, + {file = "tokenizers-0.19.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4c89aa46c269e4e70c4d4f9d6bc644fcc39bb409cb2a81227923404dd6f5227"}, + {file = "tokenizers-0.19.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:39c1ec76ea1027438fafe16ecb0fb84795e62e9d643444c1090179e63808c69d"}, + {file = "tokenizers-0.19.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c2a0d47a89b48d7daa241e004e71fb5a50533718897a4cd6235cb846d511a478"}, + {file = "tokenizers-0.19.1-cp39-none-win32.whl", hash = "sha256:61b7fe8886f2e104d4caf9218b157b106207e0f2a4905c9c7ac98890688aabeb"}, + {file = "tokenizers-0.19.1-cp39-none-win_amd64.whl", hash = "sha256:f97660f6c43efd3e0bfd3f2e3e5615bf215680bad6ee3d469df6454b8c6e8256"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b11853f17b54c2fe47742c56d8a33bf49ce31caf531e87ac0d7d13d327c9334"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d26194ef6c13302f446d39972aaa36a1dda6450bc8949f5eb4c27f51191375bd"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e8d1ed93beda54bbd6131a2cb363a576eac746d5c26ba5b7556bc6f964425594"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca407133536f19bdec44b3da117ef0d12e43f6d4b56ac4c765f37eca501c7bda"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce05fde79d2bc2e46ac08aacbc142bead21614d937aac950be88dc79f9db9022"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:35583cd46d16f07c054efd18b5d46af4a2f070a2dd0a47914e66f3ff5efb2b1e"}, + {file = "tokenizers-0.19.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:43350270bfc16b06ad3f6f07eab21f089adb835544417afda0f83256a8bf8b75"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b4399b59d1af5645bcee2072a463318114c39b8547437a7c2d6a186a1b5a0e2d"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6852c5b2a853b8b0ddc5993cd4f33bfffdca4fcc5d52f89dd4b8eada99379285"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcd266ae85c3d39df2f7e7d0e07f6c41a55e9a3123bb11f854412952deacd828"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecb2651956eea2aa0a2d099434134b1b68f1c31f9a5084d6d53f08ed43d45ff2"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:b279ab506ec4445166ac476fb4d3cc383accde1ea152998509a94d82547c8e2a"}, + {file = "tokenizers-0.19.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:89183e55fb86e61d848ff83753f64cded119f5d6e1f553d14ffee3700d0a4a49"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2edbc75744235eea94d595a8b70fe279dd42f3296f76d5a86dde1d46e35f574"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:0e64bfde9a723274e9a71630c3e9494ed7b4c0f76a1faacf7fe294cd26f7ae7c"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0b5ca92bfa717759c052e345770792d02d1f43b06f9e790ca0a1db62838816f3"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f8a20266e695ec9d7a946a019c1d5ca4eddb6613d4f466888eee04f16eedb85"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63c38f45d8f2a2ec0f3a20073cccb335b9f99f73b3c69483cd52ebc75369d8a1"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dd26e3afe8a7b61422df3176e06664503d3f5973b94f45d5c45987e1cb711876"}, + {file = "tokenizers-0.19.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:eddd5783a4a6309ce23432353cdb36220e25cbb779bfa9122320666508b44b88"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:56ae39d4036b753994476a1b935584071093b55c7a72e3b8288e68c313ca26e7"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f9939ca7e58c2758c01b40324a59c034ce0cebad18e0d4563a9b1beab3018243"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c330c0eb815d212893c67a032e9dc1b38a803eccb32f3e8172c19cc69fbb439"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec11802450a2487cdf0e634b750a04cbdc1c4d066b97d94ce7dd2cb51ebb325b"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2b718f316b596f36e1dae097a7d5b91fc5b85e90bf08b01ff139bd8953b25af"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ed69af290c2b65169f0ba9034d1dc39a5db9459b32f1dd8b5f3f32a3fcf06eab"}, + {file = "tokenizers-0.19.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f8a9c828277133af13f3859d1b6bf1c3cb6e9e1637df0e45312e6b7c2e622b1f"}, + {file = "tokenizers-0.19.1.tar.gz", hash = "sha256:ee59e6680ed0fdbe6b724cf38bd70400a0c1dd623b07ac729087270caeac88e3"}, +] + +[package.dependencies] +huggingface-hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] + [[package]] name = "torch" version = "2.3.0" @@ -3300,6 +3572,26 @@ files = [ {file = "tornado-6.4.tar.gz", hash = "sha256:72291fa6e6bc84e626589f1c29d90a5a6d593ef5ae68052ee2ef000dfd273dee"}, ] +[[package]] +name = "tqdm" +version = "4.66.4" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"}, + {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "traitlets" version = "5.14.3" @@ -3315,6 +3607,74 @@ files = [ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0,<8.2)", "pytest-mock", "pytest-mypy-testing"] +[[package]] +name = "transformers" +version = "4.40.2" +description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "transformers-4.40.2-py3-none-any.whl", hash = "sha256:71cb94301ec211a2e1d4b8c8d18dcfaa902dfa00a089dceca167a8aa265d6f2d"}, + {file = "transformers-4.40.2.tar.gz", hash = "sha256:657b6054a2097671398d976ad46e60836e7e15f9ea9551631a96e33cb9240649"}, +] + +[package.dependencies] +filelock = "*" +huggingface-hub = ">=0.19.3,<1.0" +numpy = ">=1.17" +packaging = ">=20.0" +pyyaml = ">=5.1" +regex = "!=2019.12.17" +requests = "*" +safetensors = ">=0.4.1" +tokenizers = ">=0.19,<0.20" +tqdm = ">=4.27" + +[package.extras] +accelerate = ["accelerate (>=0.21.0)"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision"] +audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +codecarbon = ["codecarbon (==1.2.0)"] +deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.19,<0.20)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.19,<0.20)", "torch", "torchaudio", "torchvision"] +docs-specific = ["hf-doc-builder"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] +flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +ftfy = ["ftfy"] +integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +modelcreation = ["cookiecutter (==1.7.3)"] +natten = ["natten (>=0.14.6,<0.15.0)"] +onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] +onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +optuna = ["optuna"] +quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"] +ray = ["ray[tune] (>=2.7.0)"] +retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sigopt = ["sigopt"] +sklearn = ["scikit-learn"] +speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +timm = ["timm"] +tokenizers = ["tokenizers (>=0.19,<0.20)"] +torch = ["accelerate (>=0.21.0)", "torch"] +torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.19,<0.20)", "torch", "tqdm (>=4.27)"] +video = ["av (==9.2.0)", "decord (==0.6.0)"] +vision = ["Pillow (>=10.0.1,<=15.0)"] + [[package]] name = "triton" version = "2.3.0" @@ -3560,4 +3920,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11,<3.13" -content-hash = "b0b59af0288ab5fe6287fbd4af9311c6f3825971b73410d7337506aeac831b37" +content-hash = "8b37477ecadb4f68cb1ab4f350b5bcc2b0f5addbe244d6bad4f255dfb4db0b55" diff --git a/pyproject.toml b/pyproject.toml index 56f210843..75a9d1c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ statsmodels = "^0.14.1" torch = "^2.3.0" torchvision = "^0.18.0" xxhash = "^3.4.1" +transformers = "^4.40.2" [tool.poetry.group.dev.dependencies] pytest = ">=7.2.1,<9.0.0" diff --git a/src/safeds/data/image/typing/_image_size.py b/src/safeds/data/image/typing/_image_size.py index b23854503..4f20774bf 100644 --- a/src/safeds/data/image/typing/_image_size.py +++ b/src/safeds/data/image/typing/_image_size.py @@ -1,16 +1,7 @@ -from __future__ import annotations +from safeds.ml.nn.typing import ConstantImageSize -import sys -from typing import TYPE_CHECKING -from safeds._utils import _structural_hash -from safeds._validation import _check_bounds, _ClosedBound - -if TYPE_CHECKING: - from safeds.data.image.containers import Image - - -class ImageSize: +class ImageSize(ConstantImageSize): """ A container for image size data. @@ -31,82 +22,5 @@ class ImageSize: if an invalid channel is given """ - def __init__(self, width: int, height: int, channel: int, *, _ignore_invalid_channel: bool = False) -> None: - _check_bounds("width", width, lower_bound=_ClosedBound(1)) - _check_bounds("height", height, lower_bound=_ClosedBound(1)) - if not _ignore_invalid_channel and channel not in (1, 3, 4): - raise ValueError(f"Channel {channel} is not a valid channel option. Use either 1, 3 or 4") - _check_bounds("channel", channel, lower_bound=_ClosedBound(1)) - - self._width = width - self._height = height - self._channel = channel - - @staticmethod - def from_image(image: Image) -> ImageSize: - """ - Create a `ImageSize` of a given image. - - Parameters - ---------- - image: - the given image for the `ImageSize` - - Returns - ------- - image_size: - the calculated `ImageSize` - """ - return ImageSize(image.width, image.height, image.channel) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ImageSize): - return NotImplemented - return (self is other) or ( - self._width == other._width and self._height == other._height and self._channel == other._channel - ) - - def __hash__(self) -> int: - return _structural_hash(self._width, self._height, self._channel) - - def __sizeof__(self) -> int: - return sys.getsizeof(self._width) + sys.getsizeof(self._height) + sys.getsizeof(self._channel) - def __str__(self) -> str: return f"{self._width}x{self._height}x{self._channel} (WxHxC)" - - @property - def width(self) -> int: - """ - Get the width of this `ImageSize` in pixels. - - Returns - ------- - width: - The width of this `ImageSize`. - """ - return self._width - - @property - def height(self) -> int: - """ - Get the height of this `ImageSize` in pixels. - - Returns - ------- - height: - The height of this `ImageSize`. - """ - return self._height - - @property - def channel(self) -> int: - """ - Get the channel of this `ImageSize` in pixels. - - Returns - ------- - channel: - The channel of this `ImageSize`. - """ - return self._channel diff --git a/src/safeds/data/tabular/containers/_column.py b/src/safeds/data/tabular/containers/_column.py index f047a8741..e94c1e2e0 100644 --- a/src/safeds/data/tabular/containers/_column.py +++ b/src/safeds/data/tabular/containers/_column.py @@ -219,7 +219,7 @@ def get_distinct_values( else: series = self._series - return series.unique().sort().to_list() + return series.unique(maintain_order=True).to_list() def get_value(self, index: int) -> T_co: """ diff --git a/src/safeds/exceptions/_ml.py b/src/safeds/exceptions/_ml.py index d443216f7..fcb8460dd 100644 --- a/src/safeds/exceptions/_ml.py +++ b/src/safeds/exceptions/_ml.py @@ -1,4 +1,4 @@ -from safeds.data.image.typing import ImageSize +from safeds.ml.nn.typing import ModelImageSize class DatasetMissesFeaturesError(ValueError): @@ -76,7 +76,7 @@ def __init__(self) -> None: class InputSizeError(Exception): """Raised when the amount of features being passed to a network does not match with its input size.""" - def __init__(self, data_size: int | ImageSize, input_layer_size: int | ImageSize) -> None: + def __init__(self, data_size: int | ModelImageSize, input_layer_size: int | ModelImageSize) -> None: super().__init__( f"The data size being passed to the network({data_size}) does not match with its input size({input_layer_size}). Consider changing the data size of the model or reformatting the data.", ) diff --git a/src/safeds/ml/nn/_model.py b/src/safeds/ml/nn/_model.py index 3954606a3..b5941feb4 100644 --- a/src/safeds/ml/nn/_model.py +++ b/src/safeds/ml/nn/_model.py @@ -7,7 +7,9 @@ from safeds._validation import _check_bounds, _ClosedBound from safeds.data.image.containers import ImageList from safeds.data.labeled.containers import ImageDataset, TabularDataset, TimeSeriesDataset +from safeds.data.labeled.containers._image_dataset import _ColumnAsTensor from safeds.data.tabular.containers import Table +from safeds.data.tabular.transformation import OneHotEncoder from safeds.exceptions import ( FeatureDataMismatchError, InputSizeError, @@ -27,17 +29,18 @@ ForwardLayer, ) from safeds.ml.nn.layers._pooling2d_layer import _Pooling2DLayer +from safeds.ml.nn.typing import ConstantImageSize, ModelImageSize, VariableImageSize if TYPE_CHECKING: from collections.abc import Callable from torch import Tensor, nn + from torch.nn import Module + from transformers.image_processing_utils import BaseImageProcessor - from safeds.data.image.typing import ImageSize from safeds.ml.nn.converters import InputConversion, OutputConversion from safeds.ml.nn.layers import Layer - IFT = TypeVar("IFT", TabularDataset, TimeSeriesDataset, ImageDataset) # InputFitType IPT = TypeVar("IPT", Table, TimeSeriesDataset, ImageList) # InputPredictType OT = TypeVar("OT", TabularDataset, TimeSeriesDataset, ImageDataset) # OutputType @@ -117,6 +120,61 @@ def __init__( self._total_number_of_batches_done = 0 self._total_number_of_epochs_done = 0 + @staticmethod + def load_pretrained_model(huggingface_repo: str) -> NeuralNetworkRegressor: # pragma: no cover + """ + Load a pretrained model from a [Huggingface repository](https://huggingface.co/models/). + + Parameters + ---------- + huggingface_repo: + the name of the huggingface repository + + Returns + ------- + pretrained_model: + the pretrained model as a NeuralNetworkRegressor + """ + from transformers import ( + AutoConfig, + AutoImageProcessor, + AutoModelForImageToImage, + PretrainedConfig, + Swin2SRForImageSuperResolution, + Swin2SRImageProcessor, + ) + + _init_default_device() + + config: PretrainedConfig = AutoConfig.from_pretrained(huggingface_repo) + + if config.model_type != "swin2sr": + raise ValueError("This model is not supported") + + model: Swin2SRForImageSuperResolution = AutoModelForImageToImage.from_pretrained(huggingface_repo) + + image_processor: Swin2SRImageProcessor = AutoImageProcessor.from_pretrained(huggingface_repo) + + if hasattr(config, "num_channels"): + input_size = VariableImageSize(image_processor.pad_size, image_processor.pad_size, config.num_channels) + else: # Should never happen due to model check + raise ValueError("This model is not supported") # pragma: no cover + + in_conversion = InputConversionImage(input_size) + out_conversion = OutputConversionImageToImage() + + network = NeuralNetworkRegressor.__new__(NeuralNetworkRegressor) + network._input_conversion = in_conversion + network._model = model + network._output_conversion = out_conversion + network._input_size = input_size + network._batch_size = 1 + network._is_fitted = True + network._total_number_of_epochs_done = 0 + network._total_number_of_batches_done = 0 + + return network + def fit( self, train_data: IFT, @@ -243,6 +301,10 @@ def predict(self, test_data: IPT) -> OT: with torch.no_grad(): for x in dataloader: elem = self._model(x) + if not isinstance(elem, torch.Tensor) and hasattr(elem, "reconstruction"): + elem = elem.reconstruction # pragma: no cover + elif not isinstance(elem, torch.Tensor): + raise ValueError(f"Output of model has unsupported type: {type(elem)}") # pragma: no cover predictions.append(elem.squeeze(dim=1)) return self._output_conversion._data_conversion( test_data, @@ -255,6 +317,11 @@ def is_fitted(self) -> bool: """Whether the model is fitted.""" return self._is_fitted + @property + def input_size(self) -> int | ModelImageSize: + """The input size of the model.""" + return self._input_size + class NeuralNetworkClassifier(Generic[IFT, IPT, OT]): """ @@ -285,6 +352,13 @@ def __init__( raise InvalidModelStructureError("You need to provide at least one layer to a neural network.") if isinstance(output_conversion, OutputConversionImageToImage): raise InvalidModelStructureError("A NeuralNetworkClassifier cannot be used with images as output.") + if isinstance(input_conversion, InputConversionImage) and isinstance( + input_conversion._input_size, + VariableImageSize, + ): + raise InvalidModelStructureError( + "A NeuralNetworkClassifier cannot be used with a InputConversionImage that uses a VariableImageSize.", + ) elif isinstance(input_conversion, InputConversionImage): if not isinstance(output_conversion, _OutputConversionImage): raise InvalidModelStructureError( @@ -324,7 +398,7 @@ def __init__( self._input_conversion: InputConversion[IFT, IPT] = input_conversion self._model = _create_internal_model(input_conversion, layers, is_for_classification=True) self._output_conversion: OutputConversion[IPT, OT] = output_conversion - self._input_size = self._model.input_size + self._input_size: int | ModelImageSize = self._model.input_size self._batch_size = 1 self._is_fitted = False self._num_of_classes = ( @@ -333,6 +407,77 @@ def __init__( self._total_number_of_batches_done = 0 self._total_number_of_epochs_done = 0 + @staticmethod + def load_pretrained_model(huggingface_repo: str) -> NeuralNetworkClassifier: # pragma: no cover + """ + Load a pretrained model from a [Huggingface repository](https://huggingface.co/models/). + + Parameters + ---------- + huggingface_repo: + the name of the huggingface repository + + Returns + ------- + pretrained_model: + the pretrained model as a NeuralNetworkClassifier + """ + from transformers import AutoConfig, AutoImageProcessor, AutoModelForImageClassification, PretrainedConfig + from transformers.models.auto.modeling_auto import MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES + + _init_default_device() + + config: PretrainedConfig = AutoConfig.from_pretrained(huggingface_repo) + + if config.model_type not in MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES: + raise ValueError("This model is not supported") + + model: Module = AutoModelForImageClassification.from_pretrained(huggingface_repo) + + image_processor: BaseImageProcessor = AutoImageProcessor.from_pretrained(huggingface_repo) + if hasattr(image_processor, "size") and hasattr(config, "num_channels"): + if "shortest_edge" in image_processor.size: + input_size = ConstantImageSize( + image_processor.size.get("shortest_edge"), + image_processor.size.get("shortest_edge"), + config.num_channels, + ) + else: + input_size = ConstantImageSize( + image_processor.size.get("width"), + image_processor.size.get("height"), + config.num_channels, + ) + else: # Should never happen due to model check + raise ValueError("This model is not supported") # pragma: no cover + + label_dict: dict[str, str] = config.id2label + column_name = "label" + labels_table = Table({column_name: [label for _, label in label_dict.items()]}) + one_hot_encoder = OneHotEncoder().fit(labels_table, [column_name]) + + in_conversion = InputConversionImage(input_size) + out_conversion = OutputConversionImageToColumn() + + in_conversion._column_name = column_name + in_conversion._one_hot_encoder = one_hot_encoder + in_conversion._input_size = input_size + in_conversion._output_type = _ColumnAsTensor + num_of_classes = labels_table.row_count + + network = NeuralNetworkClassifier.__new__(NeuralNetworkClassifier) + network._input_conversion = in_conversion + network._model = model + network._output_conversion = out_conversion + network._input_size = input_size + network._batch_size = 1 + network._is_fitted = True + network._num_of_classes = num_of_classes + network._total_number_of_epochs_done = 0 + network._total_number_of_batches_done = 0 + + return network + def fit( self, train_data: IFT, @@ -466,6 +611,10 @@ def predict(self, test_data: IPT) -> OT: with torch.no_grad(): for x in dataloader: elem = self._model(x) + if not isinstance(elem, torch.Tensor) and hasattr(elem, "logits"): + elem = elem.logits # pragma: no cover + elif not isinstance(elem, torch.Tensor): + raise ValueError(f"Output of model has unsupported type: {type(elem)}") # pragma: no cover if self._num_of_classes > 1: predictions.append(torch.argmax(elem, dim=1)) else: @@ -481,6 +630,11 @@ def is_fitted(self) -> bool: """Whether the model is fitted.""" return self._is_fitted + @property + def input_size(self) -> int | ModelImageSize: + """The input size of the model.""" + return self._input_size + def _create_internal_model( input_conversion: InputConversion[IFT, IPT], @@ -518,7 +672,7 @@ def __init__(self, layers: list[Layer], is_for_classification: bool) -> None: self._pytorch_layers = nn.Sequential(*internal_layers) @property - def input_size(self) -> int | ImageSize: + def input_size(self) -> int | ModelImageSize: return self._layer_list[0].input_size def forward(self, x: Tensor) -> Tensor: diff --git a/src/safeds/ml/nn/converters/_input_converter.py b/src/safeds/ml/nn/converters/_input_converter.py index 1ac505c2d..54d45b881 100644 --- a/src/safeds/ml/nn/converters/_input_converter.py +++ b/src/safeds/ml/nn/converters/_input_converter.py @@ -11,7 +11,7 @@ from torch.utils.data import DataLoader from safeds.data.image.containers._single_size_image_list import _SingleSizeImageList - from safeds.data.image.typing import ImageSize + from safeds.ml.nn.typing import ModelImageSize FT = TypeVar("FT", TabularDataset, TimeSeriesDataset, ImageDataset) PT = TypeVar("PT", Table, TimeSeriesDataset, ImageList) @@ -22,7 +22,7 @@ class InputConversion(Generic[FT, PT], ABC): @property @abstractmethod - def _data_size(self) -> int | ImageSize: + def _data_size(self) -> int | ModelImageSize: pass # pragma: no cover @abstractmethod diff --git a/src/safeds/ml/nn/converters/_input_converter_image.py b/src/safeds/ml/nn/converters/_input_converter_image.py index 31a7c3534..bb94fde7c 100644 --- a/src/safeds/ml/nn/converters/_input_converter_image.py +++ b/src/safeds/ml/nn/converters/_input_converter_image.py @@ -12,14 +12,14 @@ from ._input_converter import InputConversion if TYPE_CHECKING: - from safeds.data.image.typing import ImageSize from safeds.data.tabular.transformation import OneHotEncoder + from safeds.ml.nn.typing import ModelImageSize class InputConversionImage(InputConversion[ImageDataset, ImageList]): """The input conversion for a neural network, defines the input parameters for the neural network.""" - def __init__(self, image_size: ImageSize) -> None: + def __init__(self, image_size: ModelImageSize) -> None: """ Define the input parameters for the neural network in the input conversion. @@ -29,14 +29,14 @@ def __init__(self, image_size: ImageSize) -> None: the size of the input images """ self._input_size = image_size - self._output_size: ImageSize | int | None = None + self._output_size: ModelImageSize | int | None = None self._one_hot_encoder: OneHotEncoder | None = None self._column_name: str | None = None self._column_names: list[str] | None = None self._output_type: type | None = None @property - def _data_size(self) -> ImageSize: + def _data_size(self) -> ModelImageSize: return self._input_size def _data_conversion_fit( diff --git a/src/safeds/ml/nn/layers/_convolutional2d_layer.py b/src/safeds/ml/nn/layers/_convolutional2d_layer.py index 8c507c933..db03b22b6 100644 --- a/src/safeds/ml/nn/layers/_convolutional2d_layer.py +++ b/src/safeds/ml/nn/layers/_convolutional2d_layer.py @@ -6,13 +6,14 @@ from safeds._config import _init_default_device from safeds._utils import _structural_hash -from safeds.data.image.typing import ImageSize from ._layer import Layer if TYPE_CHECKING: from torch import Tensor, nn + from safeds.ml.nn.typing import ModelImageSize + def _create_internal_model( input_size: int, @@ -102,8 +103,8 @@ def __init__(self, output_channel: int, kernel_size: int, *, stride: int = 1, pa self._kernel_size = kernel_size self._stride = stride self._padding = padding - self._input_size: ImageSize | None = None - self._output_size: ImageSize | None = None + self._input_size: ModelImageSize | None = None + self._output_size: ModelImageSize | None = None def _get_internal_layer(self, **kwargs: Any) -> nn.Module: if self._input_size is None: @@ -131,7 +132,7 @@ def _get_internal_layer(self, **kwargs: Any) -> nn.Module: ) @property - def input_size(self) -> ImageSize: + def input_size(self) -> ModelImageSize: """ Get the input_size of this layer. @@ -150,7 +151,7 @@ def input_size(self) -> ImageSize: return self._input_size @property - def output_size(self) -> ImageSize: + def output_size(self) -> ModelImageSize: """ Get the output_size of this layer. @@ -175,10 +176,15 @@ def output_size(self) -> ImageSize: new_height = math.ceil( (self._input_size.height + self._padding * 2 - self._kernel_size + 1) / (1.0 * self._stride), ) - self._output_size = ImageSize(new_width, new_height, self._output_channel, _ignore_invalid_channel=True) + self._output_size = self._input_size.__class__( + new_width, + new_height, + self._output_channel, + _ignore_invalid_channel=True, + ) return self._output_size - def _set_input_size(self, input_size: int | ImageSize) -> None: + def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, int): raise TypeError("The input_size of a convolution layer has to be of type ImageSize.") self._input_size = input_size @@ -303,7 +309,7 @@ def _get_internal_layer(self, **kwargs: Any) -> nn.Module: ) @property - def output_size(self) -> ImageSize: + def output_size(self) -> ModelImageSize: if self._input_size is None: raise ValueError( "The input_size is not yet set. The layer cannot compute the output_size if the input_size is not set.", @@ -321,7 +327,12 @@ def output_size(self) -> ImageSize: + self._kernel_size + self._output_padding ) - self._output_size = ImageSize(new_width, new_height, self._output_channel, _ignore_invalid_channel=True) + self._output_size = self._input_size.__class__( + new_width, + new_height, + self._output_channel, + _ignore_invalid_channel=True, + ) return self._output_size def __hash__(self) -> int: diff --git a/src/safeds/ml/nn/layers/_flatten_layer.py b/src/safeds/ml/nn/layers/_flatten_layer.py index 17f72388f..4bbacc995 100644 --- a/src/safeds/ml/nn/layers/_flatten_layer.py +++ b/src/safeds/ml/nn/layers/_flatten_layer.py @@ -5,13 +5,14 @@ from safeds._config import _init_default_device from safeds._utils import _structural_hash +from safeds.ml.nn.typing import ConstantImageSize from ._layer import Layer if TYPE_CHECKING: from torch import Tensor, nn - from safeds.data.image.typing import ImageSize + from safeds.ml.nn.typing import ModelImageSize def _create_internal_model() -> nn.Module: @@ -34,14 +35,14 @@ class FlattenLayer(Layer): """A flatten layer.""" def __init__(self) -> None: - self._input_size: ImageSize | None = None + self._input_size: ModelImageSize | None = None self._output_size: int | None = None def _get_internal_layer(self, **kwargs: Any) -> nn.Module: # noqa: ARG002 return _create_internal_model() @property - def input_size(self) -> ImageSize: + def input_size(self) -> ModelImageSize: """ Get the input_size of this layer. @@ -82,9 +83,11 @@ def output_size(self) -> int: self._output_size = self._input_size.width * self._input_size.height * self._input_size.channel return self._output_size - def _set_input_size(self, input_size: int | ImageSize) -> None: + def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, int): raise TypeError("The input_size of a flatten layer has to be of type ImageSize.") + if not isinstance(input_size, ConstantImageSize): + raise TypeError("The input_size of a flatten layer has to be a ConstantImageSize.") self._input_size = input_size self._output_size = None diff --git a/src/safeds/ml/nn/layers/_forward_layer.py b/src/safeds/ml/nn/layers/_forward_layer.py index 741668b69..7b50f85e3 100644 --- a/src/safeds/ml/nn/layers/_forward_layer.py +++ b/src/safeds/ml/nn/layers/_forward_layer.py @@ -5,7 +5,7 @@ from safeds._config import _init_default_device from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound -from safeds.data.image.typing import ImageSize +from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer @@ -72,8 +72,8 @@ def output_size(self) -> int: """ return self._output_size - def _set_input_size(self, input_size: int | ImageSize) -> None: - if isinstance(input_size, ImageSize): + def _set_input_size(self, input_size: int | ModelImageSize) -> None: + if isinstance(input_size, ModelImageSize): raise TypeError("The input_size of a forward layer has to be of type int.") _check_bounds("input_size", input_size, lower_bound=_ClosedBound(1)) diff --git a/src/safeds/ml/nn/layers/_layer.py b/src/safeds/ml/nn/layers/_layer.py index c1c305c21..058036688 100644 --- a/src/safeds/ml/nn/layers/_layer.py +++ b/src/safeds/ml/nn/layers/_layer.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from torch import nn - from safeds.data.image.typing import ImageSize + from safeds.ml.nn.typing import ModelImageSize class Layer(ABC): @@ -20,16 +20,16 @@ def _get_internal_layer(self, **kwargs: Any) -> nn.Module: @property @abstractmethod - def input_size(self) -> int | ImageSize: + def input_size(self) -> int | ModelImageSize: pass # pragma: no cover @property @abstractmethod - def output_size(self) -> int | ImageSize: + def output_size(self) -> int | ModelImageSize: pass # pragma: no cover @abstractmethod - def _set_input_size(self, input_size: int | ImageSize) -> None: + def _set_input_size(self, input_size: int | ModelImageSize) -> None: pass # pragma: no cover @abstractmethod diff --git a/src/safeds/ml/nn/layers/_lstm_layer.py b/src/safeds/ml/nn/layers/_lstm_layer.py index 0e15149cb..0637db453 100644 --- a/src/safeds/ml/nn/layers/_lstm_layer.py +++ b/src/safeds/ml/nn/layers/_lstm_layer.py @@ -6,7 +6,7 @@ from safeds._config import _init_default_device from safeds._utils import _structural_hash from safeds._validation import _check_bounds, _ClosedBound -from safeds.data.image.typing import ImageSize +from safeds.ml.nn.typing import ModelImageSize from ._layer import Layer @@ -73,8 +73,8 @@ def output_size(self) -> int: """ return self._output_size - def _set_input_size(self, input_size: int | ImageSize) -> None: - if isinstance(input_size, ImageSize): + def _set_input_size(self, input_size: int | ModelImageSize) -> None: + if isinstance(input_size, ModelImageSize): raise TypeError("The input_size of a forward layer has to be of type int.") _check_bounds("input_size", input_size, lower_bound=_ClosedBound(1)) diff --git a/src/safeds/ml/nn/layers/_pooling2d_layer.py b/src/safeds/ml/nn/layers/_pooling2d_layer.py index 1e2170b14..3b49c1356 100644 --- a/src/safeds/ml/nn/layers/_pooling2d_layer.py +++ b/src/safeds/ml/nn/layers/_pooling2d_layer.py @@ -6,13 +6,14 @@ from safeds._config import _init_default_device from safeds._utils import _structural_hash -from safeds.data.image.typing import ImageSize from ._layer import Layer if TYPE_CHECKING: from torch import Tensor, nn + from safeds.ml.nn.typing import ModelImageSize + class _Pooling2DLayer(Layer): """ @@ -35,14 +36,14 @@ def __init__(self, strategy: Literal["max", "avg"], kernel_size: int, *, stride: self._kernel_size = kernel_size self._stride = stride if stride != -1 else kernel_size self._padding = padding - self._input_size: ImageSize | None = None - self._output_size: ImageSize | None = None + self._input_size: ModelImageSize | None = None + self._output_size: ModelImageSize | None = None def _get_internal_layer(self, **kwargs: Any) -> nn.Module: # noqa: ARG002 return _create_internal_model(self._strategy, self._kernel_size, self._padding, self._stride) @property - def input_size(self) -> ImageSize: + def input_size(self) -> ModelImageSize: """ Get the input_size of this layer. @@ -61,7 +62,7 @@ def input_size(self) -> ImageSize: return self._input_size @property - def output_size(self) -> ImageSize: + def output_size(self) -> ModelImageSize: """ Get the output_size of this layer. @@ -86,10 +87,15 @@ def output_size(self) -> ImageSize: new_height = math.ceil( (self.input_size.height + self._padding * 2 - self._kernel_size + 1) / (1.0 * self._stride), ) - self._output_size = ImageSize(new_width, new_height, self._input_size.channel, _ignore_invalid_channel=True) + self._output_size = self._input_size.__class__( + new_width, + new_height, + self._input_size.channel, + _ignore_invalid_channel=True, + ) return self._output_size - def _set_input_size(self, input_size: int | ImageSize) -> None: + def _set_input_size(self, input_size: int | ModelImageSize) -> None: if isinstance(input_size, int): raise TypeError("The input_size of a pooling layer has to be of type ImageSize.") self._input_size = input_size diff --git a/src/safeds/ml/nn/typing/__init__.py b/src/safeds/ml/nn/typing/__init__.py new file mode 100644 index 000000000..913ad6f65 --- /dev/null +++ b/src/safeds/ml/nn/typing/__init__.py @@ -0,0 +1,23 @@ +"""Types used to define neural networks and related attributes.""" + +from typing import TYPE_CHECKING + +import apipkg + +if TYPE_CHECKING: + from ._model_image_size import ConstantImageSize, ModelImageSize, VariableImageSize + +apipkg.initpkg( + __name__, + { + "ConstantImageSize": "._model_image_size:ConstantImageSize", + "ModelImageSize": "._model_image_size:ModelImageSize", + "VariableImageSize": "._model_image_size:VariableImageSize", + }, +) + +__all__ = [ + "ConstantImageSize", + "ModelImageSize", + "VariableImageSize", +] diff --git a/src/safeds/ml/nn/typing/_model_image_size.py b/src/safeds/ml/nn/typing/_model_image_size.py new file mode 100644 index 000000000..b28ea5ed1 --- /dev/null +++ b/src/safeds/ml/nn/typing/_model_image_size.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import sys +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Self + +from safeds._utils import _structural_hash +from safeds._validation import _check_bounds, _ClosedBound + +if TYPE_CHECKING: + from safeds.data.image.containers import Image + + +class ModelImageSize(ABC): + """ + A container for image size in neural networks. + + Parameters + ---------- + width: + the width of the image + height: + the height of the image + channel: + the channel of the image + + Raises + ------ + OutOfBoundsError: + if width or height are below 1 + ValueError + if an invalid channel is given + """ + + def __init__(self, width: int, height: int, channel: int, *, _ignore_invalid_channel: bool = False) -> None: + _check_bounds("width", width, lower_bound=_ClosedBound(1)) + _check_bounds("height", height, lower_bound=_ClosedBound(1)) + if not _ignore_invalid_channel and channel not in (1, 3, 4): + raise ValueError(f"Channel {channel} is not a valid channel option. Use either 1, 3 or 4") + _check_bounds("channel", channel, lower_bound=_ClosedBound(1)) + + self._width = width + self._height = height + self._channel = channel + + @classmethod + def from_image(cls: type[Self], image: Image) -> Self: + """ + Create a `ModelImageSize` of a given image. + + Parameters + ---------- + image: + the given image for the `ModelImageSize` + + Returns + ------- + image_size: + the calculated `ModelImageSize` + """ + return cls(image.width, image.height, image.channel) + + @classmethod + def from_image_size(cls: type[Self], image_size: ModelImageSize) -> Self: + """ + Create a `ModelImageSize` of a given image size. + + Parameters + ---------- + image_size: + the given image size for the `ModelImageSize` + + Returns + ------- + image_size: + the new `ModelImageSize` + """ + return cls(image_size.width, image_size.height, image_size.channel) + + @abstractmethod + def __eq__(self, other: object) -> bool: + if not isinstance(other, ModelImageSize): + return NotImplemented + return (self is other) or ( + self._width == other._width and self._height == other._height and self._channel == other._channel + ) + + @abstractmethod + def __hash__(self) -> int: + return _structural_hash(self.__class__.__name__, self._width, self._height, self._channel) + + def __sizeof__(self) -> int: + return sys.getsizeof(self._width) + sys.getsizeof(self._height) + sys.getsizeof(self._channel) + + def __str__(self) -> str: + return f"{self._width}x{self._height}x{self._channel} (WxHxC)" + + @property + def width(self) -> int: + """ + Get the width of this `ImageSize` in pixels. + + Returns + ------- + width: + The width of this `ImageSize`. + """ + return self._width + + @property + def height(self) -> int: + """ + Get the height of this `ImageSize` in pixels. + + Returns + ------- + height: + The height of this `ImageSize`. + """ + return self._height + + @property + def channel(self) -> int: + """ + Get the channel of this `ImageSize` in pixels. + + Returns + ------- + channel: + The channel of this `ImageSize`. + """ + return self._channel + + +class ConstantImageSize(ModelImageSize): + """ + A container for constant image size in neural networks. + + Parameters + ---------- + width: + the width of the image + height: + the height of the image + channel: + the channel of the image + + Raises + ------ + OutOfBoundsError: + if width or height are below 1 + ValueError + if an invalid channel is given + """ + + def __eq__(self, other: object) -> bool: + if isinstance(other, VariableImageSize): + return other.__eq__(self) + else: + return super().__eq__(other) + + def __hash__(self) -> int: + return super().__hash__() + + def __str__(self) -> str: + return f"ConstantImageSize | {super().__str__()}" + + +class VariableImageSize(ModelImageSize): + """ + A container for variable image size in neural networks. + + With a `VariableImageSize`, all image sizes that are a multiple of `width` and `height` are valid. + + Parameters + ---------- + width: + the width of the image + height: + the height of the image + channel: + the channel of the image + + Raises + ------ + OutOfBoundsError: + if width or height are below 1 + ValueError + if an invalid channel is given + """ + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ModelImageSize): + return NotImplemented + return (self is other) or ( + not (self._width % other._width and other._width % self._width) + and not (self._height % other._height and other._height % self._height) + and self._channel == other._channel + ) + + def __hash__(self) -> int: + return super().__hash__() + + def __str__(self) -> str: + return f"VariableImageSize | {super().__str__()}" diff --git a/tests/safeds/data/tabular/containers/_column/test_get_distinct_values.py b/tests/safeds/data/tabular/containers/_column/test_get_distinct_values.py index 07bc9bdbd..a2d70cdb4 100644 --- a/tests/safeds/data/tabular/containers/_column/test_get_distinct_values.py +++ b/tests/safeds/data/tabular/containers/_column/test_get_distinct_values.py @@ -36,7 +36,7 @@ def test_should_get_unique_values_ignoring_missing_values(values: list[Any], exp @pytest.mark.parametrize( ("values", "expected"), [ - ([1, None, None], [None, 1]), + ([1, None, None], [1, None]), ([None, None, None], [None]), ], ids=[ diff --git a/tests/safeds/ml/nn/layers/test_flatten_layer.py b/tests/safeds/ml/nn/layers/test_flatten_layer.py index b034a7620..64db6127c 100644 --- a/tests/safeds/ml/nn/layers/test_flatten_layer.py +++ b/tests/safeds/ml/nn/layers/test_flatten_layer.py @@ -4,6 +4,7 @@ from safeds.data.image.typing import ImageSize from safeds.data.tabular.containers import Table from safeds.ml.nn.layers import FlattenLayer +from safeds.ml.nn.typing import VariableImageSize from torch import nn @@ -31,6 +32,11 @@ def test_should_raise_if_input_size_is_set_with_int(self) -> None: with pytest.raises(TypeError, match=r"The input_size of a flatten layer has to be of type ImageSize."): layer._set_input_size(1) + def test_should_raise_if_input_size_is_set_with_variable_image_size(self) -> None: + layer = FlattenLayer() + with pytest.raises(TypeError, match=r"The input_size of a flatten layer has to be a ConstantImageSize."): + layer._set_input_size(VariableImageSize(1, 2, 3)) + class TestEq: def test_should_be_equal(self) -> None: assert FlattenLayer() == FlattenLayer() diff --git a/tests/safeds/ml/nn/test_cnn_workflow.py b/tests/safeds/ml/nn/test_cnn_workflow.py index f927cd88b..f42adad37 100644 --- a/tests/safeds/ml/nn/test_cnn_workflow.py +++ b/tests/safeds/ml/nn/test_cnn_workflow.py @@ -5,6 +5,7 @@ import torch from safeds._config import _get_device from safeds.data.image.containers import ImageList +from safeds.data.image.containers._single_size_image_list import _SingleSizeImageList from safeds.data.labeled.containers import ImageDataset from safeds.data.tabular.containers import Column, Table from safeds.data.tabular.transformation import OneHotEncoder @@ -26,7 +27,7 @@ ForwardLayer, MaxPooling2DLayer, ) -from syrupy import SnapshotAssertion +from safeds.ml.nn.typing import VariableImageSize from torch.types import Device from tests.helpers import configure_test_with_device, device_cpu, device_cuda, images_all, resolve_resource_path @@ -42,22 +43,22 @@ class TestImageToTableClassifier: ( 1234, device_cuda, - ["white_square"] * 7, + ["grayscale"] * 7, ), ( 4711, device_cuda, - ["rgba"] * 7, + ["white_square"] * 7, ), ( 1234, device_cpu, - ["white_square"] * 7, + ["grayscale"] * 7, ), ( 4711, device_cpu, - ["rgba"] * 7, + ["white_square"] * 7, ), ], ids=["seed-1234-cuda", "seed-4711-cuda", "seed-1234-cpu", "seed-4711-cpu"], @@ -109,22 +110,22 @@ class TestImageToColumnClassifier: ( 1234, device_cuda, - ["white_square"] * 7, + ["grayscale"] * 7, ), ( 4711, device_cuda, - ["rgba"] * 7, + ["white_square"] * 7, ), ( 1234, device_cpu, - ["white_square"] * 7, + ["grayscale"] * 7, ), ( 4711, device_cpu, - ["rgba"] * 7, + ["white_square"] * 7, ), ], ids=["seed-1234-cuda", "seed-4711-cuda", "seed-1234-cpu", "seed-4711-cpu"], @@ -182,7 +183,6 @@ class TestImageToImageRegressor: def test_should_train_and_predict_model( self, seed: int, - snapshot_png_image_list: SnapshotAssertion, device: Device, ) -> None: configure_test_with_device(device) @@ -215,3 +215,61 @@ def test_should_train_and_predict_model( prediction = nn.predict(image_dataset.get_input()) assert isinstance(prediction.get_output(), ImageList) assert prediction._output._tensor.device == _get_device() + + @pytest.mark.parametrize( + ("seed", "device"), + [ + (4711, device_cuda), + (4711, device_cpu), + ], + ids=["seed-4711-cuda", "seed-4711-cpu"], + ) + @pytest.mark.parametrize("multi_width", [1, 2, 3]) + @pytest.mark.parametrize("multi_height", [1, 2, 3]) + def test_should_train_and_predict_model_variable_image_size( + self, + seed: int, + device: Device, + multi_width: int, + multi_height: int, + ) -> None: + configure_test_with_device(device) + torch.manual_seed(seed) + + image_list = ImageList.from_files(resolve_resource_path(images_all())) + image_list = image_list.resize(20, 20) + image_list_grayscale = image_list.convert_to_grayscale() + image_dataset = ImageDataset(image_list, image_list_grayscale) + + layers: list[Layer] = [ + Convolutional2DLayer(6, 2), + Convolutional2DLayer(12, 2), + ConvolutionalTranspose2DLayer(6, 2), + ConvolutionalTranspose2DLayer(4, 2), + ] + nn_original = NeuralNetworkRegressor( + InputConversionImage(VariableImageSize.from_image_size(image_dataset.input_size)), + layers, + OutputConversionImageToImage(), + ) + nn = nn_original.fit(image_dataset, epoch_size=20) + assert str(nn_original._model.state_dict().values()) != str(nn._model.state_dict().values()) + assert not torch.all( + torch.eq( + nn_original._model.state_dict()["_pytorch_layers.3._layer.bias"], + nn._model.state_dict()["_pytorch_layers.3._layer.bias"], + ), + ).item() + prediction = nn.predict( + image_dataset.get_input().resize( + image_dataset.input_size.width * multi_width, + image_dataset.input_size.height * multi_height, + ), + ) + pred_output = prediction.get_output() + assert isinstance(pred_output, ImageList) + if isinstance(pred_output, _SingleSizeImageList): + assert pred_output.widths[0] == image_dataset.input_size.width * multi_width + assert pred_output.heights[0] == image_dataset.input_size.height * multi_height + assert prediction.input_size.height == image_dataset.input_size.height * multi_height + assert prediction._output._tensor.device == _get_device() diff --git a/tests/safeds/ml/nn/test_model.py b/tests/safeds/ml/nn/test_model.py index c83172c09..394e34dad 100644 --- a/tests/safeds/ml/nn/test_model.py +++ b/tests/safeds/ml/nn/test_model.py @@ -33,6 +33,7 @@ LSTMLayer, MaxPooling2DLayer, ) +from safeds.ml.nn.typing import VariableImageSize from torch.types import Device from tests.helpers import configure_test_with_device, get_devices, get_devices_ids @@ -40,6 +41,24 @@ @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) class TestClassificationModel: + + @pytest.mark.parametrize( + "input_size", + [ + 1, + ], + ) + def test_should_return_input_size(self, input_size: int, device: Device) -> None: + configure_test_with_device(device) + assert ( + NeuralNetworkClassifier( + InputConversionTable(), + [ForwardLayer(1, input_size)], + OutputConversionTable(), + ).input_size + == input_size + ) + @pytest.mark.parametrize( "epoch_size", [ @@ -225,13 +244,13 @@ def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: [ForwardLayer(input_size=1, output_size=1), ForwardLayer(output_size=1)], OutputConversionTable(), ) + learned_model = model.fit( + Table.from_dict({"a": [0.1, 0, 0.2], "b": [0, 0.15, 0.5]}).to_tabular_dataset("b"), + ) with pytest.raises( FeatureDataMismatchError, match="The features in the given table do not match with the specified feature columns names of the neural network.", ): - learned_model = model.fit( - Table.from_dict({"a": [0.1, 0, 0.2], "b": [0, 0.15, 0.5]}).to_tabular_dataset("b"), - ) learned_model.fit(Table.from_dict({"k": [0.1, 0, 0.2], "l": [0, 0.15, 0.5]}).to_tabular_dataset("k")) def test_should_raise_if_table_size_and_input_size_mismatch(self, device: Device) -> None: @@ -483,6 +502,12 @@ def callback_was_called(self) -> bool: OutputConversionImageToColumn(), r"You need to provide at least one layer to a neural network.", ), + ( + InputConversionImage(VariableImageSize(1, 1, 1)), + [FlattenLayer()], + OutputConversionImageToColumn(), + r"A NeuralNetworkClassifier cannot be used with a InputConversionImage that uses a VariableImageSize.", + ), ], ) def test_should_raise_if_model_has_invalid_structure( @@ -500,6 +525,24 @@ def test_should_raise_if_model_has_invalid_structure( @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) class TestRegressionModel: + + @pytest.mark.parametrize( + "input_size", + [ + 1, + ], + ) + def test_should_return_input_size(self, input_size: int, device: Device) -> None: + configure_test_with_device(device) + assert ( + NeuralNetworkRegressor( + InputConversionTable(), + [ForwardLayer(1, input_size)], + OutputConversionTable(), + ).input_size + == input_size + ) + @pytest.mark.parametrize( "epoch_size", [ @@ -628,13 +671,13 @@ def test_should_raise_if_train_features_mismatch(self, device: Device) -> None: [ForwardLayer(input_size=1, output_size=1)], OutputConversionTable(), ) + trained_model = model.fit( + Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("b"), + ) with pytest.raises( FeatureDataMismatchError, match="The features in the given table do not match with the specified feature columns names of the neural network.", ): - trained_model = model.fit( - Table.from_dict({"a": [1, 0, 2], "b": [0, 15, 5]}).to_tabular_dataset("b"), - ) trained_model.fit( Table.from_dict({"k": [1, 0, 2], "l": [0, 15, 5]}).to_tabular_dataset("l"), ) diff --git a/tests/safeds/ml/nn/typing/__init__.py b/tests/safeds/ml/nn/typing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/safeds/ml/nn/typing/test_model_image_size.py b/tests/safeds/ml/nn/typing/test_model_image_size.py new file mode 100644 index 000000000..eb3a6b0ea --- /dev/null +++ b/tests/safeds/ml/nn/typing/test_model_image_size.py @@ -0,0 +1,259 @@ +import sys +from typing import Any + +import pytest +from safeds.data.image.containers import Image +from safeds.exceptions import OutOfBoundsError +from safeds.ml.nn.typing import ConstantImageSize, ModelImageSize, VariableImageSize +from torch.types import Device + +from tests.helpers import ( + configure_test_with_device, + get_devices, + get_devices_ids, + images_all, + images_all_ids, + plane_png_path, + resolve_resource_path, +) + + +class TestFromImage: + @pytest.mark.parametrize("device", get_devices(), ids=get_devices_ids()) + @pytest.mark.parametrize("resource_path", images_all(), ids=images_all_ids()) + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_should_create(self, resource_path: str, device: Device, image_size_class: type[ModelImageSize]) -> None: + configure_test_with_device(device) + image = Image.from_file(resolve_resource_path(resource_path)) + expected_image_size = image_size_class(image.width, image.height, image.channel) + assert image_size_class.from_image(image) == expected_image_size + + +class TestFromImageSize: + + def test_should_create(self) -> None: + constant_image_size = ConstantImageSize(1, 2, 3) + variable_image_size = VariableImageSize(1, 2, 3) + assert isinstance(VariableImageSize.from_image_size(constant_image_size), VariableImageSize) + assert isinstance(ConstantImageSize.from_image_size(variable_image_size), ConstantImageSize) + + +class TestEq: + @pytest.mark.parametrize( + ("image_size", "width", "height", "channel"), + [ + (ConstantImageSize(1, 2, 3), 1, 2, 3), + (VariableImageSize(1, 2, 3), 1, 2, 3), + ], + ) + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_should_be_equal( + self, + image_size: ModelImageSize, + width: int, + height: int, + channel: int, + image_size_class: type[ModelImageSize], + ) -> None: + assert image_size == image_size_class(width, height, channel) + + @pytest.mark.parametrize( + ("image_size", "width", "height", "channel"), + [ + (ConstantImageSize(1, 2, 3), 3, 2, 1), + (VariableImageSize(1, 2, 3), 3, 2, 1), + ], + ) + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_should_not_be_equal( + self, + image_size: ModelImageSize, + width: int, + height: int, + channel: int, + image_size_class: type[ModelImageSize], + ) -> None: + assert image_size != image_size_class(width, height, channel) + + @pytest.mark.parametrize( + ("image_size", "other"), + [ + (ConstantImageSize(1, 2, 3), None), + (ConstantImageSize(1, 2, 3), Image.from_file(resolve_resource_path(plane_png_path))), + (VariableImageSize(1, 2, 3), None), + (VariableImageSize(1, 2, 3), Image.from_file(resolve_resource_path(plane_png_path))), + ], + ids=["ConstantImageSize-None", "ConstantImageSize-Image", "VariableImageSize-None", "VariableImageSize-Image"], + ) + def test_should_be_not_implemented(self, image_size: ModelImageSize, other: Any) -> None: + assert image_size.__eq__(other) is NotImplemented + + +class TestHash: + @pytest.mark.parametrize( + "resource_path", + images_all(), + ids=images_all_ids(), + ) + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_hash_should_be_equal(self, resource_path: str, image_size_class: type[ModelImageSize]) -> None: + image = Image.from_file(resolve_resource_path(resource_path)) + image2 = Image.from_file(resolve_resource_path(resource_path)) + assert hash(image_size_class.from_image(image)) == hash(image_size_class.from_image(image2)) + + @pytest.mark.parametrize( + "image_size_class1", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + @pytest.mark.parametrize( + "image_size_class2", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_hash_should_not_be_equal( + self, + image_size_class1: type[ModelImageSize], + image_size_class2: type[ModelImageSize], + ) -> None: + assert hash(image_size_class1(1, 2, 3)) != hash(image_size_class2(3, 2, 1)) + + def test_hash_should_not_be_equal_different_model_image_sizes(self) -> None: + assert hash(ConstantImageSize(1, 2, 3)) != hash(VariableImageSize(1, 2, 3)) + + +class TestSizeOf: + + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_should_size_be_greater_than_normal_object(self, image_size_class: type[ModelImageSize]) -> None: + assert sys.getsizeof(image_size_class(1, 2, 3)) >= sys.getsizeof(0) * 3 + + +class TestStr: + + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_string(self, image_size_class: type[ModelImageSize]) -> None: + image_size = image_size_class(1, 2, 3) + assert ( + str(image_size) + == f"{image_size_class.__name__} | {image_size.width}x{image_size.height}x{image_size.channel} (WxHxC)" + ) + + +class TestProperties: + @pytest.mark.parametrize("width", list(range(1, 5))) + @pytest.mark.parametrize("height", list(range(1, 5))) + @pytest.mark.parametrize("channel", [1, 3, 4]) + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_width_height_channel( + self, + width: int, + height: int, + channel: int, + image_size_class: type[ModelImageSize], + ) -> None: + image_size = image_size_class(width, height, channel) + assert image_size.width == width + assert image_size.height == height + assert image_size.channel == channel + + @pytest.mark.parametrize("channel", [2, 5, 6]) + @pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], + ) + def test_should_ignore_invalid_channel(self, channel: int, image_size_class: type[ModelImageSize]) -> None: + assert image_size_class(1, 1, channel, _ignore_invalid_channel=True).channel == channel + + +@pytest.mark.parametrize( + "image_size_class", + [ + ConstantImageSize, + VariableImageSize, + ], + ids=["ConstantImageSize", "VariableImageSize"], +) +class TestErrors: + @pytest.mark.parametrize("width", [-1, 0]) + def test_should_raise_invalid_width(self, width: int, image_size_class: type[ModelImageSize]) -> None: + with pytest.raises(OutOfBoundsError): + image_size_class(width, 1, 1) + + @pytest.mark.parametrize("height", [-1, 0]) + def test_should_raise_invalid_height(self, height: int, image_size_class: type[ModelImageSize]) -> None: + with pytest.raises(OutOfBoundsError): + image_size_class(1, height, 1) + + @pytest.mark.parametrize("channel", [-1, 0, 2, 5]) + def test_should_raise_invalid_channel(self, channel: int, image_size_class: type[ModelImageSize]) -> None: + with pytest.raises(ValueError, match=rf"Channel {channel} is not a valid channel option. Use either 1, 3 or 4"): + image_size_class(1, 1, channel) + + @pytest.mark.parametrize("channel", [-1, 0]) + def test_should_raise_negative_channel_ignore_invalid_channel( + self, + channel: int, + image_size_class: type[ModelImageSize], + ) -> None: + with pytest.raises(OutOfBoundsError): + image_size_class(1, 1, channel, _ignore_invalid_channel=True)