From 56ed05d817a62b26163f20952a7ddb1770628cde Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Sun, 28 Jan 2024 17:47:54 +0100 Subject: [PATCH] Clean import statements The import statements were sorted with `isort .` and unused import statements were removed with `fix8`. Signed-off-by: Stefan Weil --- docs/conf.py | 5 +-- kraken/align.py | 13 +++---- kraken/binarization.py | 12 +++--- kraken/blla.py | 34 ++++++++--------- kraken/containers.py | 11 +++--- kraken/contrib/baselineset_overlay.py | 6 ++- kraken/contrib/extract_lines.py | 10 +++-- kraken/contrib/forced_alignment_overlay.py | 11 +++--- kraken/contrib/generate_scripts.py | 3 +- kraken/contrib/heatmap_overlay.py | 8 ++-- .../hyperparameters/tune_pretraining.py | 14 +++---- .../contrib/hyperparameters/tune_training.py | 11 +++--- kraken/contrib/print_word_spreader.py | 7 +++- kraken/contrib/recognition_boxes.py | 6 +-- kraken/contrib/repolygonize.py | 7 ++-- kraken/contrib/segmentation_overlay.py | 10 ++--- kraken/contrib/set_seg_options.py | 3 +- kraken/ketos/__init__.py | 8 ++-- kraken/ketos/dataset.py | 3 +- kraken/ketos/linegen.py | 13 +++---- kraken/ketos/pretrain.py | 11 ++++-- kraken/ketos/recognition.py | 29 ++++++++------- kraken/ketos/repo.py | 5 ++- kraken/ketos/ro.py | 8 ++-- kraken/ketos/segmentation.py | 26 ++++++------- kraken/ketos/transcription.py | 20 ++++------ kraken/ketos/util.py | 6 +-- kraken/kraken.py | 26 ++++++------- kraken/lib/arrow_dataset.py | 17 +++++---- kraken/lib/codec.py | 9 +++-- kraken/lib/ctc_decoder.py | 9 ++--- kraken/lib/dataset/__init__.py | 8 ++-- kraken/lib/dataset/recognition.py | 31 ++++++++-------- kraken/lib/dataset/ro.py | 11 +++--- kraken/lib/dataset/segmentation.py | 26 ++++++------- kraken/lib/dataset/utils.py | 13 +++---- kraken/lib/functional_im_transforms.py | 15 ++++---- kraken/lib/layers.py | 9 +++-- kraken/lib/lineest.py | 7 ++-- kraken/lib/models.py | 14 +++---- kraken/lib/morph.py | 5 +-- kraken/lib/pretrain/__init__.py | 2 +- kraken/lib/pretrain/layers.py | 6 +-- kraken/lib/pretrain/model.py | 37 +++++++++---------- kraken/lib/pretrain/util.py | 6 +-- kraken/lib/progress.py | 13 ++++--- kraken/lib/ro/__init__.py | 2 +- kraken/lib/ro/layers.py | 4 +- kraken/lib/ro/model.py | 24 ++++++------ kraken/lib/ro/util.py | 4 +- kraken/lib/segmentation.py | 27 +++++++------- kraken/lib/train.py | 34 +++++++++-------- kraken/lib/util.py | 11 +++--- kraken/lib/vgsl.py | 18 ++++----- kraken/lib/xml.py | 12 +++--- kraken/linegen.py | 17 ++++----- kraken/pageseg.py | 17 ++++----- kraken/repo.py | 10 ++--- kraken/rpred.py | 15 ++++---- kraken/serialization.py | 18 ++++----- kraken/transcribe.py | 10 ++--- tests/test_align.py | 3 +- tests/test_arrow_dataset.py | 7 ++-- tests/test_binarization.py | 5 +-- tests/test_cli.py | 14 +++---- tests/test_codec.py | 4 +- tests/test_dataset.py | 7 ++-- tests/test_layers.py | 1 + tests/test_lineest.py | 2 +- tests/test_merging.py | 8 ++-- tests/test_models.py | 4 +- tests/test_pageseg.py | 4 +- tests/test_readingorder.py | 4 +- tests/test_repo.py | 3 +- tests/test_rpred.py | 15 ++++---- tests/test_serialization.py | 12 +++--- tests/test_train.py | 9 ++--- tests/test_transcribe.py | 8 ++-- tests/test_vgsl.py | 9 +++-- tests/test_xml.py | 6 +-- 80 files changed, 456 insertions(+), 456 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3dbcc709d..22c7d6614 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,11 +12,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os -import shlex +from subprocess import PIPE, Popen -from subprocess import Popen, PIPE # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. diff --git a/kraken/align.py b/kraken/align.py index b496821ed..a3fb7c651 100644 --- a/kraken/align.py +++ b/kraken/align.py @@ -21,18 +21,17 @@ A character alignment module using a network output lattice and ground truth to accuractely determine grapheme locations in input data. """ -import torch -import logging import dataclasses +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING, Literal, Optional -from PIL import Image +import torch from bidi.algorithm import get_display - -from dataclasses import dataclass -from typing import Optional, Literal, TYPE_CHECKING +from PIL import Image from kraken import rpred -from kraken.containers import Segmentation, BaselineOCRRecord +from kraken.containers import BaselineOCRRecord, Segmentation if TYPE_CHECKING: from kraken.lib.models import TorchSeqRecognizer diff --git a/kraken/binarization.py b/kraken/binarization.py index db3adacc7..b721c402b 100644 --- a/kraken/binarization.py +++ b/kraken/binarization.py @@ -19,17 +19,17 @@ An adaptive binarization algorithm. """ -import warnings import logging -import numpy as np +import warnings +from typing import TYPE_CHECKING -from kraken.lib.util import pil2array, array2pil, is_bitonal, get_im_str -from scipy.ndimage import affine_transform, percentile_filter, gaussian_filter, binary_dilation +import numpy as np +from scipy.ndimage import (affine_transform, binary_dilation, gaussian_filter, + percentile_filter) from scipy.ndimage import zoom as _zoom -from typing import TYPE_CHECKING - from kraken.lib.exceptions import KrakenInputException +from kraken.lib.util import array2pil, get_im_str, is_bitonal, pil2array if TYPE_CHECKING: from PIL import Image diff --git a/kraken/blla.py b/kraken/blla.py index d04682ef1..3d0e961f4 100644 --- a/kraken/blla.py +++ b/kraken/blla.py @@ -20,32 +20,30 @@ line recognizer uses the baseline paradigm. """ -import PIL -import uuid -import torch import logging -import numpy as np +import uuid +from typing import Any, Callable, Dict, List, Literal, Optional, Union + import importlib_resources +import numpy as np +import PIL import shapely.geometry as geom +import torch import torch.nn.functional as F import torchvision.transforms as tf - -from typing import Optional, Dict, Callable, Union, List, Any, Literal - from scipy.ndimage import gaussian_filter from skimage.filters import sobel -from kraken.lib import vgsl, dataset -from kraken.containers import Region, Segmentation, BaselineLine -from kraken.lib.util import is_bitonal, get_im_str -from kraken.lib.exceptions import KrakenInputException, KrakenInvalidModelException -from kraken.lib.segmentation import (polygonal_reading_order, - neural_reading_order, - vectorize_lines, vectorize_regions, - scale_polygonal_lines, - calculate_polygonal_environment, - is_in_region, - scale_regions) +from kraken.containers import BaselineLine, Region, Segmentation +from kraken.lib import dataset, vgsl +from kraken.lib.exceptions import (KrakenInputException, + KrakenInvalidModelException) +from kraken.lib.segmentation import (calculate_polygonal_environment, + is_in_region, neural_reading_order, + polygonal_reading_order, + scale_polygonal_lines, scale_regions, + vectorize_lines, vectorize_regions) +from kraken.lib.util import get_im_str, is_bitonal __all__ = ['segment'] diff --git a/kraken/containers.py b/kraken/containers.py index fdd9fd283..b649a5ced 100644 --- a/kraken/containers.py +++ b/kraken/containers.py @@ -19,12 +19,13 @@ Container classes replacing the old dictionaries returned by kraken's functional blocks. """ -import numpy as np -import bidi.algorithm as bd - -from typing import Literal, List, Dict, Union, Optional, Tuple, Any, TYPE_CHECKING -from dataclasses import dataclass, asdict from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass +from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, + Union) + +import bidi.algorithm as bd +import numpy as np from kraken.lib.segmentation import compute_polygon_section diff --git a/kraken/contrib/baselineset_overlay.py b/kraken/contrib/baselineset_overlay.py index 3f466ae50..82a83019e 100755 --- a/kraken/contrib/baselineset_overlay.py +++ b/kraken/contrib/baselineset_overlay.py @@ -9,10 +9,12 @@ @click.argument('files', nargs=-1) def cli(files): - import torch - from PIL import Image from os.path import splitext + + import torch import torchvision.transforms as tf + from PIL import Image + from kraken.lib import dataset batch, channels, height, width = 1, 3, 1200, 0 diff --git a/kraken/contrib/extract_lines.py b/kraken/contrib/extract_lines.py index 3dec3bc51..95233263c 100755 --- a/kraken/contrib/extract_lines.py +++ b/kraken/contrib/extract_lines.py @@ -20,13 +20,15 @@ def cli(format_type, model, files): click.echo(ctx.get_help()) ctx.exit() - from PIL import Image - from os.path import splitext - from kraken import blla - from kraken.lib import segmentation, vgsl, xml import io import json + from os.path import splitext + import pyarrow as pa + from PIL import Image + + from kraken import blla + from kraken.lib import segmentation, vgsl, xml if model is None: for doc in files: diff --git a/kraken/contrib/forced_alignment_overlay.py b/kraken/contrib/forced_alignment_overlay.py index dae491e2f..2610d84c4 100755 --- a/kraken/contrib/forced_alignment_overlay.py +++ b/kraken/contrib/forced_alignment_overlay.py @@ -3,14 +3,15 @@ Draws a transparent overlay of the forced alignment output over the input image. """ -import re import os -import click +import re import unicodedata -from lxml import etree from itertools import cycle from unicodedata import normalize +import click +from lxml import etree + cmap = cycle([(230, 25, 75, 127), (60, 180, 75, 127), (255, 225, 25, 127), @@ -100,9 +101,9 @@ def cli(format_type, model, normalization, output, files): from PIL import Image, ImageDraw - from kraken.lib.xml import XMLPage - from kraken.lib import models from kraken import align + from kraken.lib import models + from kraken.lib.xml import XMLPage if format_type == 'alto': repl_fn = _repl_alto diff --git a/kraken/contrib/generate_scripts.py b/kraken/contrib/generate_scripts.py index 7b328d4c7..aff36580d 100755 --- a/kraken/contrib/generate_scripts.py +++ b/kraken/contrib/generate_scripts.py @@ -2,8 +2,9 @@ """ Script fetching the latest unicode Scripts.txt and dumping it as json. """ -from urllib import request import json +from urllib import request + import regex uri = 'http://www.unicode.org/Public/UNIDATA/Scripts.txt' diff --git a/kraken/contrib/heatmap_overlay.py b/kraken/contrib/heatmap_overlay.py index 8e2c7236b..fc51f1e59 100755 --- a/kraken/contrib/heatmap_overlay.py +++ b/kraken/contrib/heatmap_overlay.py @@ -14,12 +14,14 @@ def cli(model, files): Applies a BLLA baseline segmentation model and outputs the raw heatmaps of the first baseline class. """ + from os.path import splitext + import torch - from PIL import Image - from kraken.lib import vgsl, dataset import torch.nn.functional as F - from os.path import splitext import torchvision.transforms as tf + from PIL import Image + + from kraken.lib import dataset, vgsl model = vgsl.TorchVGSLModel.load_model(model) model.eval() diff --git a/kraken/contrib/hyperparameters/tune_pretraining.py b/kraken/contrib/hyperparameters/tune_pretraining.py index 1348280c1..5564b521d 100644 --- a/kraken/contrib/hyperparameters/tune_pretraining.py +++ b/kraken/contrib/hyperparameters/tune_pretraining.py @@ -2,19 +2,19 @@ """ A script for a grid search over pretraining hyperparameters. """ -import click from functools import partial +import click +import pytorch_lightning as pl +from pytorch_lightning import seed_everything from ray import tune - from ray.tune.integration.pytorch_lightning import TuneReportCallback -from kraken.lib.default_specs import RECOGNITION_PRETRAIN_HYPER_PARAMS, RECOGNITION_SPEC -from kraken.lib.pretrain.model import PretrainDataModule, RecognitionPretrainModel from kraken.ketos.util import _validate_manifests - -import pytorch_lightning as pl -from pytorch_lightning import seed_everything +from kraken.lib.default_specs import (RECOGNITION_PRETRAIN_HYPER_PARAMS, + RECOGNITION_SPEC) +from kraken.lib.pretrain.model import (PretrainDataModule, + RecognitionPretrainModel) config = {'lrate': tune.loguniform(1e-8, 1e-2), 'num_negatives': tune.qrandint(1, 4, 1), diff --git a/kraken/contrib/hyperparameters/tune_training.py b/kraken/contrib/hyperparameters/tune_training.py index 98f3781b4..31275c4eb 100644 --- a/kraken/contrib/hyperparameters/tune_training.py +++ b/kraken/contrib/hyperparameters/tune_training.py @@ -5,15 +5,14 @@ import sys from functools import partial +import pytorch_lightning as pl from ray import tune - from ray.tune.integration.pytorch_lightning import TuneReportCallback -from kraken.lib.default_spec import RECOGNITION_PRETRAIN_HYPER_PARAMS, RECOGNITION_SPEC -from kraken.lib.pretrain.model import PretrainDataModule, RecognitionPretrainModel - -import pytorch_lightning as pl - +from kraken.lib.default_spec import (RECOGNITION_PRETRAIN_HYPER_PARAMS, + RECOGNITION_SPEC) +from kraken.lib.pretrain.model import (PretrainDataModule, + RecognitionPretrainModel) config = {'lrate': tune.loguniform(1e-8, 1e-2), 'num_negatives': tune.qrandint(2, 100, 8), diff --git a/kraken/contrib/print_word_spreader.py b/kraken/contrib/print_word_spreader.py index 616e2774e..6f2e28eb1 100755 --- a/kraken/contrib/print_word_spreader.py +++ b/kraken/contrib/print_word_spreader.py @@ -1,11 +1,16 @@ #!/usr/bin/env python #2020, Bruce Robertson #Master file at https://github.com/brobertson/Lace2-tools/blob/master/normalize_hocr.py -import html, os, sys, argparse +import argparse +import html +import os +import sys from statistics import mean + from lxml import etree from PIL import Image + #a custom exception to indicate when a page or other element doesn't #have a bounding box where we would expect it. class BboxError(Exception): diff --git a/kraken/contrib/recognition_boxes.py b/kraken/contrib/recognition_boxes.py index 1cd5dec6f..d28dcdf46 100755 --- a/kraken/contrib/recognition_boxes.py +++ b/kraken/contrib/recognition_boxes.py @@ -6,14 +6,14 @@ import os import sys +from itertools import cycle from PIL import Image, ImageDraw -from kraken.pageseg import segment from kraken.binarization import nlbin -from kraken.rpred import rpred -from itertools import cycle from kraken.lib import models +from kraken.pageseg import segment +from kraken.rpred import rpred cmap = cycle([(230, 25, 75, 127), (60, 180, 75, 127), diff --git a/kraken/contrib/repolygonize.py b/kraken/contrib/repolygonize.py index d3f498c57..36b4e3200 100755 --- a/kraken/contrib/repolygonize.py +++ b/kraken/contrib/repolygonize.py @@ -28,12 +28,13 @@ def cli(format_type, topline, files): click.echo(ctx.get_help()) ctx.exit() - from lxml import etree + from itertools import groupby from os.path import splitext - from itertools import groupby - from kraken.lib import xml + from lxml import etree from PIL import Image + + from kraken.lib import xml from kraken.lib.segmentation import calculate_polygonal_environment def _repl_alto(fname, polygons): diff --git a/kraken/contrib/segmentation_overlay.py b/kraken/contrib/segmentation_overlay.py index 181a92ddc..a7370680d 100755 --- a/kraken/contrib/segmentation_overlay.py +++ b/kraken/contrib/segmentation_overlay.py @@ -3,14 +3,14 @@ Draws a transparent overlay of baseline segmenter output over a list of image files. """ -import re +import dataclasses import os -import click +import re import unicodedata -import dataclasses -from itertools import cycle from collections import defaultdict +from itertools import cycle +import click cmap = cycle([(230, 25, 75, 127), (60, 180, 75, 127)]) @@ -58,8 +58,8 @@ def cli(model, text_direction, repolygonize, files): from PIL import Image, ImageDraw - from kraken.lib import vgsl, xml, segmentation from kraken import blla + from kraken.lib import segmentation, vgsl, xml if model is None: for doc in files: diff --git a/kraken/contrib/set_seg_options.py b/kraken/contrib/set_seg_options.py index 9cab733c7..a59000689 100755 --- a/kraken/contrib/set_seg_options.py +++ b/kraken/contrib/set_seg_options.py @@ -2,9 +2,10 @@ """ A script setting the metadata of segmentation models. """ -import click import shutil +import click + @click.command() @click.option('-b', '--bounding-region', multiple=True, help='Sets region identifiers which bound line bounding polygons') diff --git a/kraken/ketos/__init__.py b/kraken/ketos/__init__.py index a8ddfe9a2..9edb8d005 100644 --- a/kraken/ketos/__init__.py +++ b/kraken/ketos/__init__.py @@ -19,9 +19,9 @@ Command line drivers for training functionality. """ -import click import logging +import click from PIL import Image from rich.traceback import install @@ -30,11 +30,11 @@ from .dataset import compile from .linegen import line_generator from .pretrain import pretrain -from .recognition import train, test +from .recognition import test, train from .repo import publish -from .segmentation import segtrain, segtest +from .ro import roadd, rotrain +from .segmentation import segtest, segtrain from .transcription import extract, transcription -from .ro import rotrain, roadd APP_NAME = 'kraken' diff --git a/kraken/ketos/dataset.py b/kraken/ketos/dataset.py index cfd267b67..a4df23400 100644 --- a/kraken/ketos/dataset.py +++ b/kraken/ketos/dataset.py @@ -61,9 +61,10 @@ def compile(ctx, output, workers, format_type, files, random_split, force_type, """ Precompiles a binary dataset from a collection of XML files. """ - from .util import message from kraken.lib.progress import KrakenProgressBar + from .util import message + ground_truth = list(ground_truth) if files: diff --git a/kraken/ketos/linegen.py b/kraken/ketos/linegen.py index e91018f03..70480355f 100644 --- a/kraken/ketos/linegen.py +++ b/kraken/ketos/linegen.py @@ -69,23 +69,22 @@ def line_generator(ctx, font, maxlines, encoding, normalization, renormalize, """ Generates artificial text line training data. """ - import os import errno import logging - import numpy as np + import os import unicodedata - from typing import Set + + import numpy as np from bidi.algorithm import get_display - from kraken.lib.progress import KrakenProgressBar + from kraken import linegen from kraken.lib.exceptions import KrakenCairoSurfaceException + from kraken.lib.progress import KrakenProgressBar + from kraken.lib.util import make_printable from .util import message - from kraken import linegen - from kraken.lib.util import make_printable - logging.captureWarnings(True) logger = logging.getLogger('kraken') diff --git a/kraken/ketos/pretrain.py b/kraken/ketos/pretrain.py index f323cd9f0..a8c404dac 100644 --- a/kraken/ketos/pretrain.py +++ b/kraken/ketos/pretrain.py @@ -18,15 +18,15 @@ Command line driver for unsupervised recognition pretraining """ -import click import logging +import click from PIL import Image from kraken.lib.default_specs import (RECOGNITION_PRETRAIN_HYPER_PARAMS, RECOGNITION_SPEC) -from .util import _validate_manifests, _expand_gt, message, to_ptl_device +from .util import _expand_gt, _validate_manifests, message, to_ptl_device logging.captureWarnings(True) logger = logging.getLogger('kraken') @@ -195,14 +195,17 @@ def pretrain(ctx, batch_size, pad, output, spec, load, freq, quit, epochs, if augment: try: - import albumentations # NOQA + import albumentations # NOQA except ImportError: raise click.BadOptionUsage('augment', 'augmentation needs the `albumentations` package installed.') import shutil + from threadpoolctl import threadpool_limits + + from kraken.lib.pretrain import (PretrainDataModule, + RecognitionPretrainModel) from kraken.lib.train import KrakenTrainer - from kraken.lib.pretrain import PretrainDataModule, RecognitionPretrainModel hyper_params = RECOGNITION_PRETRAIN_HYPER_PARAMS.copy() hyper_params.update({'freq': freq, diff --git a/kraken/ketos/recognition.py b/kraken/ketos/recognition.py index ec17372c6..248402b3a 100644 --- a/kraken/ketos/recognition.py +++ b/kraken/ketos/recognition.py @@ -18,16 +18,17 @@ Command line driver for recognition training and evaluation. """ -import click import logging import pathlib - from typing import List + +import click from threadpoolctl import threadpool_limits -from kraken.lib.exceptions import KrakenInputException from kraken.lib.default_specs import RECOGNITION_HYPER_PARAMS, RECOGNITION_SPEC -from .util import _validate_manifests, _expand_gt, message, to_ptl_device +from kraken.lib.exceptions import KrakenInputException + +from .util import _expand_gt, _validate_manifests, message, to_ptl_device logging.captureWarnings(True) logger = logging.getLogger('kraken') @@ -211,13 +212,13 @@ def train(ctx, batch_size, pad, output, spec, append, load, freq, quit, epochs, if augment: try: - import albumentations # NOQA + import albumentations # NOQA except ImportError: raise click.BadOptionUsage('augment', 'augmentation needs the `albumentations` package installed.') if pl_logger == 'tensorboard': try: - import tensorboard # NOQA + import tensorboard # NOQA except ImportError: raise click.BadOptionUsage('logger', 'tensorboard logger needs the `tensorboard` package installed.') @@ -226,7 +227,8 @@ def train(ctx, batch_size, pad, output, spec, append, load, freq, quit, epochs, import json import shutil - from kraken.lib.train import RecognitionModel, KrakenTrainer + + from kraken.lib.train import KrakenTrainer, RecognitionModel hyper_params = RECOGNITION_HYPER_PARAMS.copy() hyper_params.update({'freq': freq, @@ -395,15 +397,14 @@ def test(ctx, batch_size, model, evaluation_files, device, pad, workers, import numpy as np from torch.utils.data import DataLoader - from kraken.serialization import render_report from kraken.lib import models, util - from kraken.lib.xml import XMLPage - from kraken.lib.dataset import (global_align, compute_confusions, - PolygonGTDataset, GroundTruthDataset, - ImageInputTransforms, - ArrowIPCRecognitionDataset, - collate_sequences) + from kraken.lib.dataset import (ArrowIPCRecognitionDataset, + GroundTruthDataset, ImageInputTransforms, + PolygonGTDataset, collate_sequences, + compute_confusions, global_align) from kraken.lib.progress import KrakenProgressBar + from kraken.lib.xml import XMLPage + from kraken.serialization import render_report logger.info('Building test set from {} line images'.format(len(test_set) + len(evaluation_files))) diff --git a/kraken/ketos/repo.py b/kraken/ketos/repo.py index d977d947d..043a51087 100644 --- a/kraken/ketos/repo.py +++ b/kraken/ketos/repo.py @@ -18,9 +18,10 @@ Command line driver for publishing models to the model repository. """ +import logging import os + import click -import logging from .util import message @@ -43,8 +44,8 @@ def publish(ctx, metadata, access_token, private, model): Publishes a model on the zenodo model repository. """ import json - import importlib_resources + import importlib_resources from jsonschema import validate from jsonschema.exceptions import ValidationError diff --git a/kraken/ketos/ro.py b/kraken/ketos/ro.py index dceee4d7f..9ff26d57e 100644 --- a/kraken/ketos/ro.py +++ b/kraken/ketos/ro.py @@ -18,16 +18,16 @@ Command line driver for reading order training, evaluation, and handling. """ -import click -import pathlib import logging +import pathlib +import click from PIL import Image +from kraken.ketos.util import (_expand_gt, _validate_manifests, message, + to_ptl_device) from kraken.lib.default_specs import READING_ORDER_HYPER_PARAMS -from kraken.ketos.util import _validate_manifests, _expand_gt, message, to_ptl_device - logging.captureWarnings(True) logger = logging.getLogger('kraken') diff --git a/kraken/ketos/segmentation.py b/kraken/ketos/segmentation.py index addd5ebbf..4d6cdfaeb 100644 --- a/kraken/ketos/segmentation.py +++ b/kraken/ketos/segmentation.py @@ -18,19 +18,19 @@ Command line driver for segmentation training and evaluation. """ -import shlex -import click -import pathlib import logging +import pathlib +import shlex +from typing import Dict +import click from PIL import Image -from typing import Dict - +from kraken.ketos.util import (_expand_gt, _validate_manifests, message, + to_ptl_device) +from kraken.lib.default_specs import (SEGMENTATION_HYPER_PARAMS, + SEGMENTATION_SPEC) from kraken.lib.exceptions import KrakenInputException -from kraken.lib.default_specs import SEGMENTATION_HYPER_PARAMS, SEGMENTATION_SPEC - -from kraken.ketos.util import _validate_manifests, _expand_gt, message, to_ptl_device logging.captureWarnings(True) logger = logging.getLogger('kraken') @@ -239,7 +239,7 @@ def segtrain(ctx, output, spec, line_width, pad, load, freq, quit, epochs, from threadpoolctl import threadpool_limits - from kraken.lib.train import SegmentationModel, KrakenTrainer + from kraken.lib.train import KrakenTrainer, SegmentationModel if resize != 'fail' and not load: raise click.BadOptionUsage('resize', 'resize option requires loading an existing model') @@ -249,13 +249,13 @@ def segtrain(ctx, output, spec, line_width, pad, load, freq, quit, epochs, if augment: try: - import albumentations # NOQA + import albumentations # NOQA except ImportError: raise click.BadOptionUsage('augment', 'augmentation needs the `albumentations` package installed.') if pl_logger == 'tensorboard': try: - import tensorboard # NOQA + import tensorboard # NOQA except ImportError: raise click.BadOptionUsage('logger', 'tensorboard logger needs the `tensorboard` package installed.') @@ -433,10 +433,10 @@ def segtest(ctx, model, evaluation_files, device, workers, threads, threshold, if not model: raise click.UsageError('No model to evaluate given.') - from threadpoolctl import threadpool_limits - from torch.utils.data import DataLoader import torch import torch.nn.functional as F + from threadpoolctl import threadpool_limits + from torch.utils.data import DataLoader from kraken.lib.progress import KrakenProgressBar from kraken.lib.train import BaselineSet, ImageInputTransforms diff --git a/kraken/ketos/transcription.py b/kraken/ketos/transcription.py index 2246c3f5c..e8a8fc11d 100644 --- a/kraken/ketos/transcription.py +++ b/kraken/ketos/transcription.py @@ -18,13 +18,13 @@ Legacy command line drivers for recognition training data annotation. """ -import os -import uuid -import click import logging +import os import unicodedata - +import uuid from typing import IO, Any, cast + +import click from bidi.algorithm import get_display from .util import message @@ -59,12 +59,12 @@ def extract(ctx, binarize, normalization, normalize_whitespace, reorder, Extracts image-text pairs from a transcription environment created using ``ketos transcribe``. """ - import regex import base64 - from io import BytesIO + + import regex + from lxml import etree, html from PIL import Image - from lxml import html, etree from kraken import binarization from kraken.lib.progress import KrakenProgressBar @@ -166,11 +166,7 @@ def transcription(ctx, text_direction, scale, bw, maxcolseps, from PIL import Image - from kraken import rpred - from kraken import pageseg - from kraken import transcribe - from kraken import binarization - + from kraken import binarization, pageseg, rpred, transcribe from kraken.lib import models from kraken.lib.progress import KrakenProgressBar diff --git a/kraken/ketos/util.py b/kraken/ketos/util.py index b37b298b2..923d51da9 100644 --- a/kraken/ketos/util.py +++ b/kraken/ketos/util.py @@ -18,13 +18,13 @@ Command line driver helpers """ -import os import glob -import click import logging - +import os from typing import List, Optional, Tuple +import click + logging.captureWarnings(True) logger = logging.getLogger('kraken') diff --git a/kraken/kraken.py b/kraken/kraken.py index ccb98e8a2..fc5d65faa 100644 --- a/kraken/kraken.py +++ b/kraken/kraken.py @@ -18,21 +18,19 @@ Command line drivers for recognition functionality. """ +import dataclasses +import logging import os import shlex import warnings -import logging -import dataclasses -import importlib_resources - -from PIL import Image -from pathlib import Path - from functools import partial -from rich.traceback import install -from typing import Dict, cast, Any, IO, Callable, Union, List +from pathlib import Path +from typing import IO, Any, Callable, Dict, List, Union, cast import click +import importlib_resources +from PIL import Image +from rich.traceback import install from kraken.lib import log @@ -179,13 +177,12 @@ def segmenter(legacy, model, text_direction, scale, maxcolseps, black_colseps, def recognizer(model, pad, no_segmentation, bidi_reordering, tags_ignore, input, output) -> None: + import dataclasses import json import uuid - import dataclasses from kraken import rpred - from kraken.containers import Segmentation, BBoxLine - + from kraken.containers import BBoxLine, Segmentation from kraken.lib.progress import KrakenProgressBar ctx = click.get_current_context() @@ -347,10 +344,11 @@ def process_pipeline(subcommands, input, batch_input, suffix, verbose, format_ty placing their respective outputs in temporary files. """ import glob - import uuid import tempfile + import uuid from threadpoolctl import threadpool_limits + from kraken.lib.progress import KrakenProgressBar ctx = click.get_current_context() @@ -658,7 +656,7 @@ def show(ctx, model_id): Retrieves model metadata from the repository. """ from kraken import repo - from kraken.lib.util import make_printable, is_printable + from kraken.lib.util import is_printable, make_printable desc = repo.get_description(model_id) diff --git a/kraken/lib/arrow_dataset.py b/kraken/lib/arrow_dataset.py index e945d123f..c9159a916 100755 --- a/kraken/lib/arrow_dataset.py +++ b/kraken/lib/arrow_dataset.py @@ -19,21 +19,22 @@ import io import json -import numpy as np -import pyarrow as pa import tempfile - -from PIL import Image, UnidentifiedImageError -from functools import partial from collections import Counter -from typing import Optional, List, Union, Callable, Tuple, Dict, TYPE_CHECKING +from functools import partial from multiprocessing import Pool +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import pyarrow as pa +from PIL import Image, UnidentifiedImageError + from kraken.containers import Segmentation from kraken.lib import functional_im_transforms as F_t +from kraken.lib.exceptions import KrakenInputException from kraken.lib.segmentation import extract_polygons -from kraken.lib.xml import XMLPage from kraken.lib.util import is_bitonal, make_printable -from kraken.lib.exceptions import KrakenInputException +from kraken.lib.xml import XMLPage if TYPE_CHECKING: from os import PathLike diff --git a/kraken/lib/codec.py b/kraken/lib/codec.py index 8eb911441..ae013a869 100644 --- a/kraken/lib/codec.py +++ b/kraken/lib/codec.py @@ -18,12 +18,13 @@ graphemes. """ import logging -import numpy as np - from collections import Counter -from typing import List, Tuple, Set, Union, Dict, Sequence +from typing import Dict, List, Sequence, Set, Tuple, Union + +import numpy as np from torch import IntTensor -from kraken.lib.exceptions import KrakenEncodeException, KrakenCodecException + +from kraken.lib.exceptions import KrakenCodecException, KrakenEncodeException __all__ = ['PytorchCodec'] diff --git a/kraken/lib/ctc_decoder.py b/kraken/lib/ctc_decoder.py index f1736c74d..a2bb91a4c 100644 --- a/kraken/lib/ctc_decoder.py +++ b/kraken/lib/ctc_decoder.py @@ -24,13 +24,12 @@ """ import collections -import numpy as np - +from itertools import groupby from typing import List, Tuple -from scipy.special import logsumexp -from scipy.ndimage import measurements -from itertools import groupby +import numpy as np +from scipy.ndimage import measurements +from scipy.special import logsumexp __all__ = ['beam_decoder', 'greedy_decoder', 'blank_threshold_decoder'] diff --git a/kraken/lib/dataset/__init__.py b/kraken/lib/dataset/__init__.py index 95cf91a84..861fd0587 100644 --- a/kraken/lib/dataset/__init__.py +++ b/kraken/lib/dataset/__init__.py @@ -15,7 +15,9 @@ """ Top-level module containing datasets for recognition and segmentation training. """ -from .recognition import ArrowIPCRecognitionDataset, PolygonGTDataset, GroundTruthDataset # NOQA +from .recognition import (ArrowIPCRecognitionDataset, # NOQA + GroundTruthDataset, PolygonGTDataset) +from .ro import PageWiseROSet, PairWiseROSet # NOQA from .segmentation import BaselineSet # NOQA -from .ro import PairWiseROSet, PageWiseROSet # NOQA -from .utils import ImageInputTransforms, collate_sequences, global_align, compute_confusions # NOQA +from .utils import (ImageInputTransforms, collate_sequences, # NOQA + compute_confusions, global_align) diff --git a/kraken/lib/dataset/recognition.py b/kraken/lib/dataset/recognition.py index 16d00fa0e..a80f6cd89 100644 --- a/kraken/lib/dataset/recognition.py +++ b/kraken/lib/dataset/recognition.py @@ -15,28 +15,28 @@ """ Utility functions for data loading and training of VGSL networks. """ +import dataclasses import io import json -import torch import traceback -import dataclasses +from collections import Counter +from functools import partial +from typing import (TYPE_CHECKING, Any, Callable, List, Literal, Optional, + Tuple, Union) + import numpy as np import pyarrow as pa - +import torch from PIL import Image -from functools import partial -from torchvision import transforms -from collections import Counter from torch.utils.data import Dataset -from typing import List, Tuple, Callable, Optional, Any, Union, Literal, TYPE_CHECKING +from torchvision import transforms from kraken.containers import BaselineLine, BBoxLine, Segmentation -from kraken.lib.util import is_bitonal +from kraken.lib import functional_im_transforms as F_t from kraken.lib.codec import PytorchCodec +from kraken.lib.exceptions import KrakenEncodeException, KrakenInputException from kraken.lib.segmentation import extract_polygons -from kraken.lib.exceptions import KrakenInputException, KrakenEncodeException - -from kraken.lib import functional_im_transforms as F_t +from kraken.lib.util import is_bitonal if TYPE_CHECKING: from os import PathLike @@ -55,11 +55,10 @@ class DefaultAugmenter(): def __init__(self): import cv2 cv2.setNumThreads(0) - from albumentations import ( - Compose, ToFloat, OneOf, MotionBlur, MedianBlur, Blur, - ShiftScaleRotate, OpticalDistortion, ElasticTransform, - PixelDropout - ) + from albumentations import (Blur, Compose, ElasticTransform, + MedianBlur, MotionBlur, OneOf, + OpticalDistortion, PixelDropout, + ShiftScaleRotate, ToFloat) self._transforms = Compose([ ToFloat(), diff --git a/kraken/lib/dataset/ro.py b/kraken/lib/dataset/ro.py index 65638da8f..d407fe660 100644 --- a/kraken/lib/dataset/ro.py +++ b/kraken/lib/dataset/ro.py @@ -15,16 +15,15 @@ """ Utility functions for data loading and training of VGSL networks. """ -import torch -import numpy as np - from math import factorial -from torch.utils.data import Dataset -from typing import Dict, Sequence, Union, Literal, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Literal, Optional, Sequence, Union -from kraken.lib.xml import XMLPage +import numpy as np +import torch +from torch.utils.data import Dataset from kraken.lib.exceptions import KrakenInputException +from kraken.lib.xml import XMLPage if TYPE_CHECKING: from os import PathLike diff --git a/kraken/lib/dataset/segmentation.py b/kraken/lib/dataset/segmentation.py index 61a77e4eb..ede985da0 100644 --- a/kraken/lib/dataset/segmentation.py +++ b/kraken/lib/dataset/segmentation.py @@ -15,21 +15,20 @@ """ Utility functions for data loading and training of VGSL networks. """ -import torch import traceback +from collections import defaultdict +from itertools import groupby +from typing import TYPE_CHECKING, Any, Callable, Dict, Sequence, Tuple + import numpy as np -import torch.nn.functional as F import shapely.geometry as geom - +import torch +import torch.nn.functional as F from PIL import Image from shapely.ops import split -from itertools import groupby -from torchvision import transforms -from collections import defaultdict -from torch.utils.data import Dataset -from typing import Dict, Tuple, Sequence, Callable, Any, TYPE_CHECKING - from skimage.draw import polygon +from torch.utils.data import Dataset +from torchvision import transforms from kraken.lib.segmentation import scale_regions @@ -97,11 +96,10 @@ def __init__(self, if augmentation: import cv2 cv2.setNumThreads(0) - from albumentations import ( - Compose, ToFloat, OneOf, MotionBlur, MedianBlur, Blur, - ShiftScaleRotate, OpticalDistortion, ElasticTransform, - HueSaturationValue, - ) + from albumentations import (Blur, Compose, ElasticTransform, + HueSaturationValue, MedianBlur, + MotionBlur, OneOf, OpticalDistortion, + ShiftScaleRotate, ToFloat) self.aug = Compose([ ToFloat(), diff --git a/kraken/lib/dataset/utils.py b/kraken/lib/dataset/utils.py index 98defd9d9..879ac4592 100644 --- a/kraken/lib/dataset/utils.py +++ b/kraken/lib/dataset/utils.py @@ -16,21 +16,20 @@ Utility functions for data loading and training of VGSL networks. """ import json -import torch import numbers +from collections import Counter +from functools import partial +from typing import Any, Dict, List, Sequence, Tuple, Union + import importlib_resources +import torch import torch.nn.functional as F - -from functools import partial from torchvision import transforms -from collections import Counter -from typing import Dict, List, Tuple, Sequence, Any, Union +from kraken.lib import functional_im_transforms as F_t from kraken.lib.exceptions import KrakenInputException from kraken.lib.lineest import CenterNormalizer -from kraken.lib import functional_im_transforms as F_t - __all__ = ['ImageInputTransforms', 'collate_sequences'] diff --git a/kraken/lib/functional_im_transforms.py b/kraken/lib/functional_im_transforms.py index 014389617..43a578e35 100644 --- a/kraken/lib/functional_im_transforms.py +++ b/kraken/lib/functional_im_transforms.py @@ -16,20 +16,21 @@ Named functions for all the transforms that were lambdas in the past to facilitate pickling. """ -import torch -import regex import unicodedata -import bidi.algorithm as bd - from pathlib import Path -from PIL.Image import Resampling +from typing import (TYPE_CHECKING, Any, Callable, Literal, Optional, Tuple, + Union) -from typing import Literal, Tuple, Optional, Callable, Any, Union, TYPE_CHECKING +import bidi.algorithm as bd +import regex +import torch +from PIL.Image import Resampling -from kraken.lib.lineest import dewarp, CenterNormalizer +from kraken.lib.lineest import CenterNormalizer, dewarp if TYPE_CHECKING: from os import PathLike + from PIL import Image diff --git a/kraken/lib/layers.py b/kraken/lib/layers.py index 38b25aa1b..e1a23b8e2 100644 --- a/kraken/lib/layers.py +++ b/kraken/lib/layers.py @@ -1,18 +1,19 @@ """ Layers for VGSL models """ -import torch import logging -import numpy as np +from typing import Iterable, List, Optional, Tuple -from typing import List, Tuple, Optional, Iterable +import numpy as np +import torch from torch.nn import Module, Sequential from torch.nn import functional as F -from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence +from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence logger = logging.getLogger('coremltools') logger.setLevel(logging.ERROR) from coremltools.proto import NeuralNetwork_pb2 # NOQA + logger.setLevel(logging.WARNING) # all tensors are ordered NCHW, the "feature" dimension is C, so the output of diff --git a/kraken/lib/lineest.py b/kraken/lib/lineest.py index 948559bdb..038a71d08 100644 --- a/kraken/lib/lineest.py +++ b/kraken/lib/lineest.py @@ -1,10 +1,9 @@ import warnings -import numpy as np +from typing import TYPE_CHECKING +import numpy as np from scipy.ndimage import affine_transform, gaussian_filter, uniform_filter -from typing import TYPE_CHECKING - if TYPE_CHECKING: from PIL import Image @@ -78,7 +77,7 @@ def dewarp(normalizer: CenterNormalizer, im: 'Image.Image') -> 'Image.Image': Returns: PIL.Image.Image containing the dewarped image. """ - from kraken.lib.util import pil2array, array2pil + from kraken.lib.util import array2pil, pil2array line = pil2array(im) temp = np.amax(line)-line diff --git a/kraken/lib/models.py b/kraken/lib/models.py index ed0f9c173..fbc1c32bb 100644 --- a/kraken/lib/models.py +++ b/kraken/lib/models.py @@ -5,17 +5,17 @@ Wrapper around TorchVGSLModel including a variety of forward pass helpers for sequence classification. """ -from os.path import expandvars, expanduser, abspath +from os.path import abspath, expanduser, expandvars +from typing import TYPE_CHECKING, List, Optional, Tuple, Union -import torch import numpy as np -import kraken.lib.lineest -import kraken.lib.ctc_decoder - -from typing import List, Tuple, Optional, Union, TYPE_CHECKING +import torch +import kraken.lib.ctc_decoder +import kraken.lib.lineest +from kraken.lib.exceptions import (KrakenInputException, + KrakenInvalidModelException) from kraken.lib.vgsl import TorchVGSLModel -from kraken.lib.exceptions import KrakenInvalidModelException, KrakenInputException if TYPE_CHECKING: from os import PathLike diff --git a/kraken/lib/morph.py b/kraken/lib/morph.py index 69d0e2f59..dbcb4f224 100644 --- a/kraken/lib/morph.py +++ b/kraken/lib/morph.py @@ -2,10 +2,9 @@ Various add-ons to the SciPy morphology package """ import numpy as np -from scipy.ndimage import label as _label -from scipy.ndimage import distance_transform_edt +from scipy.ndimage import distance_transform_edt, filters from scipy.ndimage import find_objects as _find_objects -from scipy.ndimage import filters +from scipy.ndimage import label as _label def label(image: np.ndarray, **kw) -> np.ndarray: diff --git a/kraken/lib/pretrain/__init__.py b/kraken/lib/pretrain/__init__.py index 71e08ed50..c10c0b650 100644 --- a/kraken/lib/pretrain/__init__.py +++ b/kraken/lib/pretrain/__init__.py @@ -16,4 +16,4 @@ Tools for unsupervised pretraining of recognition models. """ -from .model import PretrainDataModule, RecognitionPretrainModel # NOQA +from .model import PretrainDataModule, RecognitionPretrainModel # NOQA diff --git a/kraken/lib/pretrain/layers.py b/kraken/lib/pretrain/layers.py index 009ce72fd..55ef3e756 100644 --- a/kraken/lib/pretrain/layers.py +++ b/kraken/lib/pretrain/layers.py @@ -1,10 +1,10 @@ """ Layers for VGSL models """ -import torch +from typing import TYPE_CHECKING, Optional, Tuple -from typing import Tuple, Optional, TYPE_CHECKING -from torch.nn import Module, Embedding, Linear +import torch +from torch.nn import Embedding, Linear, Module from kraken.lib.pretrain.util import compute_mask_indices, sample_negatives diff --git a/kraken/lib/pretrain/model.py b/kraken/lib/pretrain/model.py index 92ab35684..4685e1851 100644 --- a/kraken/lib/pretrain/model.py +++ b/kraken/lib/pretrain/model.py @@ -28,34 +28,33 @@ for Low-Resource Historical Document Transcription." arXiv preprint arXiv:2112.08692 (2021). """ -import re -import math -import torch import logging +import math +import re +from itertools import chain +from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Union + import numpy as np -import torch.nn.functional as F import pytorch_lightning as pl - -from itertools import chain -from torch.optim import lr_scheduler -from typing import Dict, Optional, Sequence, Union, Any, TYPE_CHECKING +import torch +import torch.nn.functional as F from pytorch_lightning.callbacks import EarlyStopping -from pytorch_lightning.utilities.memory import is_oom_error, garbage_collection_cuda +from pytorch_lightning.utilities.memory import (garbage_collection_cuda, + is_oom_error) +from torch.optim import lr_scheduler +from torch.utils.data import DataLoader, Subset, random_split from kraken.containers import Segmentation - -from kraken.lib import vgsl, default_specs, layers -from kraken.lib.xml import XMLPage -from kraken.lib.util import parse_gt_path +from kraken.lib import default_specs, layers, vgsl from kraken.lib.codec import PytorchCodec -from kraken.lib.dataset import (ArrowIPCRecognitionDataset, - GroundTruthDataset, PolygonGTDataset, - ImageInputTransforms, collate_sequences) +from kraken.lib.dataset import (ArrowIPCRecognitionDataset, GroundTruthDataset, + ImageInputTransforms, PolygonGTDataset, + collate_sequences) from kraken.lib.exceptions import KrakenInputException -from kraken.lib.train import _configure_optimizer_and_lr_scheduler from kraken.lib.pretrain.layers import Wav2Vec2Mask - -from torch.utils.data import DataLoader, random_split, Subset +from kraken.lib.train import _configure_optimizer_and_lr_scheduler +from kraken.lib.util import parse_gt_path +from kraken.lib.xml import XMLPage if TYPE_CHECKING: from os import PathLike diff --git a/kraken/lib/pretrain/util.py b/kraken/lib/pretrain/util.py index 31df04978..7bf3760a8 100644 --- a/kraken/lib/pretrain/util.py +++ b/kraken/lib/pretrain/util.py @@ -3,11 +3,11 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from typing import Sequence, Union, Tuple - -import torch import random +from typing import Sequence, Tuple, Union + import numpy as np +import torch def positive_integers_with_sum(n, total): diff --git a/kraken/lib/progress.py b/kraken/lib/progress.py index e9c8e6eb5..25201a9be 100644 --- a/kraken/lib/progress.py +++ b/kraken/lib/progress.py @@ -15,19 +15,20 @@ """ Handlers for rich-based progress bars. """ -from typing import Union, TYPE_CHECKING from dataclasses import dataclass +from typing import TYPE_CHECKING, Union -from pytorch_lightning.callbacks.progress.rich_progress import CustomProgress, RichProgressBar, MetricsTextColumn - +from pytorch_lightning.callbacks.progress.rich_progress import ( + CustomProgress, MetricsTextColumn, RichProgressBar) from rich import get_console, reconfigure -from rich.progress import BarColumn, Progress, ProgressColumn, TextColumn, TimeRemainingColumn, TimeElapsedColumn, DownloadColumn -from rich.text import Text from rich.default_styles import DEFAULT_STYLES +from rich.progress import (BarColumn, DownloadColumn, Progress, ProgressColumn, + TextColumn, TimeElapsedColumn, TimeRemainingColumn) +from rich.text import Text if TYPE_CHECKING: - from rich.style import Style from rich.console import RenderableType + from rich.style import Style __all__ = ['KrakenProgressBar', 'KrakenDownloadProgressBar', 'KrakenTrainProgressBar'] diff --git a/kraken/lib/ro/__init__.py b/kraken/lib/ro/__init__.py index 4e370b855..08eb56587 100644 --- a/kraken/lib/ro/__init__.py +++ b/kraken/lib/ro/__init__.py @@ -16,4 +16,4 @@ Tools for trainable reading order. """ -from .model import ROModel # NOQA +from .model import ROModel # NOQA diff --git a/kraken/lib/ro/layers.py b/kraken/lib/ro/layers.py index a90315b6e..14ef6fc5e 100644 --- a/kraken/lib/ro/layers.py +++ b/kraken/lib/ro/layers.py @@ -1,11 +1,11 @@ """ Layers for VGSL models """ +from typing import TYPE_CHECKING, Tuple + import torch from torch import nn -from typing import Tuple, TYPE_CHECKING - if TYPE_CHECKING: from kraken.lib.vgsl import VGSLBlock diff --git a/kraken/lib/ro/model.py b/kraken/lib/ro/model.py index 4dbb762de..cba78bd13 100644 --- a/kraken/lib/ro/model.py +++ b/kraken/lib/ro/model.py @@ -17,28 +17,28 @@ Adapted from: """ -import torch import logging -import numpy as np -import torch.nn.functional as F -import pytorch_lightning as pl - -from torch.optim import lr_scheduler from dataclasses import dataclass, field -from typing import Dict, Optional, Sequence, Union, Any, Literal, List, TYPE_CHECKING +from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, + Sequence, Union) +import numpy as np +import pytorch_lightning as pl +import torch +import torch.nn.functional as F from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor +from torch.optim import lr_scheduler +from torch.utils.data import DataLoader, Subset from kraken.lib import default_specs -from kraken.lib.dataset import PairWiseROSet, PageWiseROSet -from kraken.lib.train import _configure_optimizer_and_lr_scheduler -from kraken.lib.segmentation import _greedy_order_decoder +from kraken.lib.dataset import PageWiseROSet, PairWiseROSet from kraken.lib.ro.layers import MLP - -from torch.utils.data import DataLoader, Subset +from kraken.lib.segmentation import _greedy_order_decoder +from kraken.lib.train import _configure_optimizer_and_lr_scheduler if TYPE_CHECKING: from os import PathLike + from torch.nn import Module logger = logging.getLogger(__name__) diff --git a/kraken/lib/ro/util.py b/kraken/lib/ro/util.py index 57fea354b..c722e595e 100644 --- a/kraken/lib/ro/util.py +++ b/kraken/lib/ro/util.py @@ -3,11 +3,11 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import random from typing import Sequence, Union -import torch -import random import numpy as np +import torch def positive_integers_with_sum(n, total): diff --git a/kraken/lib/segmentation.py b/kraken/lib/segmentation.py index 03edd33d6..2355443be 100644 --- a/kraken/lib/segmentation.py +++ b/kraken/lib/segmentation.py @@ -15,38 +15,37 @@ """ Processing for baseline segmenter output """ -import torch import logging +from collections import defaultdict +from typing import (TYPE_CHECKING, Dict, List, Literal, Optional, Sequence, + Tuple, Union) + import numpy as np import shapely.geometry as geom +import torch import torch.nn.functional as F - -from collections import defaultdict - from PIL import Image - +from scipy.ndimage import (binary_erosion, distance_transform_cdt, + gaussian_filter, maximum_filter) from scipy.signal import convolve2d -from scipy.ndimage import maximum_filter, binary_erosion, gaussian_filter, distance_transform_cdt from scipy.spatial.distance import pdist, squareform - from shapely.ops import nearest_points, unary_union from shapely.validation import explain_validity - from skimage import draw, filters -from skimage.graph import MCP_Connect from skimage.filters import sobel -from skimage.measure import approximate_polygon, subdivide_polygon, regionprops, label +from skimage.graph import MCP_Connect +from skimage.measure import (approximate_polygon, label, regionprops, + subdivide_polygon) from skimage.morphology import skeletonize -from skimage.transform import PiecewiseAffineTransform, SimilarityTransform, AffineTransform, warp - -from typing import List, Tuple, Union, Dict, Sequence, Optional, Literal, TYPE_CHECKING +from skimage.transform import (AffineTransform, PiecewiseAffineTransform, + SimilarityTransform, warp) from kraken.lib import default_specs from kraken.lib.exceptions import KrakenInputException if TYPE_CHECKING: - from kraken.lib.vgsl import TorchVGSLModel from kraken.containers import Segmentation + from kraken.lib.vgsl import TorchVGSLModel logger = logging.getLogger('kraken') diff --git a/kraken/lib/train.py b/kraken/lib/train.py index a05b563c5..62cdafbec 100644 --- a/kraken/lib/train.py +++ b/kraken/lib/train.py @@ -15,32 +15,34 @@ """ Training loop interception helpers """ -import re -import torch import logging +import re import warnings +from typing import (TYPE_CHECKING, Any, Callable, Dict, Literal, Optional, + Sequence, Union) + import numpy as np -import torch.nn.functional as F import pytorch_lightning as pl - -from torchmetrics.classification import MultilabelAccuracy, MultilabelJaccardIndex -from torchmetrics.text import CharErrorRate, WordErrorRate +import torch +import torch.nn.functional as F +from pytorch_lightning.callbacks import (BaseFinetuning, Callback, + EarlyStopping, LearningRateMonitor) from torch.optim import lr_scheduler -from typing import Callable, Dict, Optional, Sequence, Union, Any, Literal, TYPE_CHECKING -from pytorch_lightning.callbacks import Callback, EarlyStopping, BaseFinetuning, LearningRateMonitor +from torch.utils.data import DataLoader, Subset, random_split +from torchmetrics.classification import (MultilabelAccuracy, + MultilabelJaccardIndex) +from torchmetrics.text import CharErrorRate, WordErrorRate from kraken.containers import Segmentation -from kraken.lib import models, vgsl, default_specs, progress -from kraken.lib.xml import XMLPage -from kraken.lib.util import make_printable, parse_gt_path +from kraken.lib import default_specs, models, progress, vgsl from kraken.lib.codec import PytorchCodec from kraken.lib.dataset import (ArrowIPCRecognitionDataset, BaselineSet, - GroundTruthDataset, PolygonGTDataset, - ImageInputTransforms, collate_sequences) + GroundTruthDataset, ImageInputTransforms, + PolygonGTDataset, collate_sequences) +from kraken.lib.exceptions import KrakenEncodeException, KrakenInputException from kraken.lib.models import validate_hyper_parameters -from kraken.lib.exceptions import KrakenInputException, KrakenEncodeException - -from torch.utils.data import DataLoader, random_split, Subset +from kraken.lib.util import make_printable, parse_gt_path +from kraken.lib.xml import XMLPage if TYPE_CHECKING: from os import PathLike diff --git a/kraken/lib/util.py b/kraken/lib/util.py index 6eee8ab92..609e924fe 100644 --- a/kraken/lib/util.py +++ b/kraken/lib/util.py @@ -2,17 +2,16 @@ Ocropus's magic PIL-numpy array conversion routines. They express slightly different behavior from PIL.Image.toarray(). """ -import uuid -import torch import unicodedata -import numpy as np +import uuid +from typing import TYPE_CHECKING, Callable, Literal, Optional, Union +import numpy as np +import torch from PIL import Image -from typing import Union, Callable, Optional, Literal, TYPE_CHECKING - -from kraken.lib import functional_im_transforms as F_t from kraken.containers import BBoxLine +from kraken.lib import functional_im_transforms as F_t from kraken.lib.exceptions import KrakenInputException if TYPE_CHECKING: diff --git a/kraken/lib/vgsl.py b/kraken/lib/vgsl.py index 4335fe7e4..43bf6281e 100644 --- a/kraken/lib/vgsl.py +++ b/kraken/lib/vgsl.py @@ -1,25 +1,23 @@ """ VGSL plumbing """ -import re import json -import torch import logging +import re +from os import PathLike +from typing import (Any, Callable, Dict, Iterable, List, Optional, Sequence, + Tuple, Union) +import torch +from coremltools.models import MLModel, datatypes +from coremltools.models.neural_network import NeuralNetworkBuilder +from google.protobuf.message import DecodeError from torch import nn -from os import PathLike -from typing import Sequence, List, Tuple, Union, Optional, Iterable, Callable, Dict, Any from kraken.lib import layers from kraken.lib.codec import PytorchCodec from kraken.lib.exceptions import KrakenInvalidModelException -from coremltools.models import MLModel -from coremltools.models import datatypes -from coremltools.models.neural_network import NeuralNetworkBuilder - -from google.protobuf.message import DecodeError - # all tensors are ordered NCHW, the "feature" dimension is C, so the output of # an LSTM will be put into C same as the filters of a CNN. diff --git a/kraken/lib/xml.py b/kraken/lib/xml.py index acd38d24b..d367072a4 100644 --- a/kraken/lib/xml.py +++ b/kraken/lib/xml.py @@ -15,17 +15,17 @@ """ ALTO/Page data loaders for segmentation training """ -import re import logging - +import re +from collections import defaultdict +from itertools import groupby from pathlib import Path +from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, + Sequence, Tuple, Union) -from itertools import groupby from lxml import etree -from typing import Union, Dict, Any, Sequence, Tuple, Literal, Optional, List, TYPE_CHECKING -from collections import defaultdict -from kraken.containers import Segmentation, BaselineLine, Region +from kraken.containers import BaselineLine, Region, Segmentation logger = logging.getLogger(__name__) diff --git a/kraken/linegen.py b/kraken/linegen.py index 6ea15aa8d..67e99f60b 100644 --- a/kraken/linegen.py +++ b/kraken/linegen.py @@ -31,20 +31,19 @@ """ -from scipy.ndimage.filters import gaussian_filter -from scipy.ndimage.measurements import find_objects -from scipy.ndimage.morphology import distance_transform_cdt, binary_closing - -from scipy.ndimage.interpolation import affine_transform, geometric_transform -from PIL import Image, ImageOps - -import logging import ctypes import ctypes.util +import logging + import numpy as np +from PIL import Image, ImageOps +from scipy.ndimage.filters import gaussian_filter +from scipy.ndimage.interpolation import affine_transform, geometric_transform +from scipy.ndimage.measurements import find_objects +from scipy.ndimage.morphology import binary_closing, distance_transform_cdt from kraken.lib.exceptions import KrakenCairoSurfaceException -from kraken.lib.util import pil2array, array2pil +from kraken.lib.util import array2pil, pil2array logger = logging.getLogger(__name__) diff --git a/kraken/pageseg.py b/kraken/pageseg.py index 82ab81c0d..c29742620 100644 --- a/kraken/pageseg.py +++ b/kraken/pageseg.py @@ -19,21 +19,20 @@ Layout analysis methods. """ -import PIL -import uuid import logging -import numpy as np - -from typing import Tuple, List, Callable, Optional, Union -from scipy.ndimage.filters import (gaussian_filter, uniform_filter, - maximum_filter) +import uuid +from typing import Callable, List, Optional, Tuple, Union -from kraken.containers import Segmentation, BBoxLine +import numpy as np +import PIL +from scipy.ndimage.filters import (gaussian_filter, maximum_filter, + uniform_filter) +from kraken.containers import BBoxLine, Segmentation from kraken.lib import morph, sl -from kraken.lib.util import pil2array, is_bitonal, get_im_str from kraken.lib.exceptions import KrakenInputException from kraken.lib.segmentation import reading_order, topsort +from kraken.lib.util import get_im_str, is_bitonal, pil2array __all__ = ['segment'] diff --git a/kraken/repo.py b/kraken/repo.py index 68aee05d7..628c327fb 100644 --- a/kraken/repo.py +++ b/kraken/repo.py @@ -15,14 +15,14 @@ """ Accessors to the model repository on zenodo. """ -import os import json import logging -import requests - -from pathlib import Path +import os from contextlib import closing -from typing import Callable, Any, TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable + +import requests from kraken.lib.exceptions import KrakenRepoException diff --git a/kraken/rpred.py b/kraken/rpred.py index 9bedcd0d9..96dff4fdd 100644 --- a/kraken/rpred.py +++ b/kraken/rpred.py @@ -18,21 +18,22 @@ Generators for recognition on lines images. """ -import logging import dataclasses - -from functools import partial +import logging from collections import defaultdict -from typing import List, Tuple, Optional, Generator, Union, Dict, Sequence, TYPE_CHECKING +from functools import partial +from typing import (TYPE_CHECKING, Dict, Generator, List, Optional, Sequence, + Tuple, Union) from kraken.containers import BaselineOCRRecord, BBoxOCRRecord, ocr_record -from kraken.lib.util import get_im_str, is_bitonal -from kraken.lib.segmentation import extract_polygons -from kraken.lib.exceptions import KrakenInputException from kraken.lib.dataset import ImageInputTransforms +from kraken.lib.exceptions import KrakenInputException +from kraken.lib.segmentation import extract_polygons +from kraken.lib.util import get_im_str, is_bitonal if TYPE_CHECKING: from PIL import Image + from kraken.containers import Segmentation from kraken.lib.models import TorchSeqRecognizer diff --git a/kraken/serialization.py b/kraken/serialization.py index e1392feb0..fb2b8ef69 100644 --- a/kraken/serialization.py +++ b/kraken/serialization.py @@ -12,22 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. -from jinja2 import Environment, PackageLoader, FunctionLoader - -import regex -import logging import datetime - import importlib.metadata +import logging +from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Literal, + Optional, Sequence, Tuple) -from kraken.lib.util import make_printable +import regex +from jinja2 import Environment, FunctionLoader, PackageLoader -from typing import List, Tuple, Iterable, Optional, Sequence, Literal, TYPE_CHECKING, Dict, Any +from kraken.lib.util import make_printable if TYPE_CHECKING: - from os import PathLike from collections import Counter - from kraken.containers import Segmentation, ProcessingStep + from os import PathLike + + from kraken.containers import ProcessingStep, Segmentation logger = logging.getLogger(__name__) diff --git a/kraken/transcribe.py b/kraken/transcribe.py index 6a6ba0a68..9e355da93 100644 --- a/kraken/transcribe.py +++ b/kraken/transcribe.py @@ -15,13 +15,13 @@ """ Utility functions for ground truth transcription. """ -from io import BytesIO -from typing import List, Dict, Any -from jinja2 import Environment, PackageLoader - -import uuid import base64 import logging +import uuid +from io import BytesIO +from typing import Any, Dict, List + +from jinja2 import Environment, PackageLoader from kraken.lib.exceptions import KrakenInputException from kraken.lib.util import get_im_str diff --git a/tests/test_align.py b/tests/test_align.py index 69bc297f0..0836b2bde 100644 --- a/tests/test_align.py +++ b/tests/test_align.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- import unittest - from pathlib import Path -from kraken.lib import xml from kraken.align import forced_align +from kraken.lib import xml thisfile = Path(__file__).resolve().parent resources = thisfile / 'resources' diff --git a/tests/test_arrow_dataset.py b/tests/test_arrow_dataset.py index fff086c12..4ce06031a 100644 --- a/tests/test_arrow_dataset.py +++ b/tests/test_arrow_dataset.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- -import unittest import json - -import kraken +import unittest +from pathlib import Path from pytest import raises -from pathlib import Path +import kraken from kraken.lib import xml from kraken.lib.arrow_dataset import build_binary_dataset diff --git a/tests/test_binarization.py b/tests/test_binarization.py index 2662b56ba..977193d33 100644 --- a/tests/test_binarization.py +++ b/tests/test_binarization.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- import unittest +from pathlib import Path +from PIL import Image from pytest import raises -from PIL import Image -from pathlib import Path from kraken.binarization import nlbin - from kraken.lib.exceptions import KrakenInputException thisfile = Path(__file__).resolve().parent diff --git a/tests/test_cli.py b/tests/test_cli.py index e555d6f8b..58dc61927 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,17 +1,17 @@ # -*- coding: utf-8 -*- import os -import click -import unittest import tempfile -import numpy as np - -from PIL import Image +import unittest from pathlib import Path -from click.testing import CliRunner -from kraken.kraken import cli +import click +import numpy as np +from click.testing import CliRunner +from PIL import Image from pytest import raises +from kraken.kraken import cli + thisfile = Path(__file__).resolve().parent resources = thisfile / 'resources' diff --git a/tests/test_codec.py b/tests/test_codec.py index 67f5b9f23..2d732ec9a 100644 --- a/tests/test_codec.py +++ b/tests/test_codec.py @@ -2,11 +2,11 @@ import unittest from pytest import raises - from torch import IntTensor from kraken.lib import codec -from kraken.lib.exceptions import KrakenEncodeException, KrakenCodecException +from kraken.lib.exceptions import KrakenCodecException, KrakenEncodeException + class TestCodec(unittest.TestCase): diff --git a/tests/test_dataset.py b/tests/test_dataset.py index d129adc19..6b7c3d110 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,15 +1,14 @@ # -*- coding: utf-8 -*- import unittest - from pathlib import Path -from pytest import raises from PIL import Image -from kraken.lib.dataset import ImageInputTransforms, BaselineSet +from pytest import raises from kraken.lib import xml -from kraken.lib.util import is_bitonal +from kraken.lib.dataset import BaselineSet, ImageInputTransforms from kraken.lib.exceptions import KrakenInputException +from kraken.lib.util import is_bitonal thisfile = Path(__file__).resolve().parent resources = thisfile / 'resources' diff --git a/tests/test_layers.py b/tests/test_layers.py index b8adbb6ff..78f2969a1 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -2,6 +2,7 @@ import unittest import torch + from kraken.lib import layers diff --git a/tests/test_lineest.py b/tests/test_lineest.py index 776246bc0..6d9a07847 100644 --- a/tests/test_lineest.py +++ b/tests/test_lineest.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import unittest +from pathlib import Path from PIL import Image -from pathlib import Path from pytest import raises from kraken.lib import lineest diff --git a/tests/test_merging.py b/tests/test_merging.py index 9cc464761..a9a00631e 100644 --- a/tests/test_merging.py +++ b/tests/test_merging.py @@ -1,11 +1,11 @@ -from kraken.lib.train import RecognitionModel -from kraken.lib.exceptions import KrakenInputException -from kraken.lib.default_specs import RECOGNITION_HYPER_PARAMS #from kraken.ketos.util import _expand_gt from pathlib import Path -from unittest import TestCase from unicodedata import normalize +from unittest import TestCase +from kraken.lib.default_specs import RECOGNITION_HYPER_PARAMS +from kraken.lib.exceptions import KrakenInputException +from kraken.lib.train import RecognitionModel _here = Path(__file__).parent base_model = _here.joinpath(Path("./resources/merge_tests/merge_codec_nfd.mlmodel")) diff --git a/tests/test_models.py b/tests/test_models.py index f928d4fcb..f8e0341c8 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- import os import pickle -import unittest import tempfile +import unittest +from pathlib import Path from pytest import raises -from pathlib import Path from kraken.lib import models from kraken.lib.exceptions import KrakenInvalidModelException diff --git a/tests/test_pageseg.py b/tests/test_pageseg.py index ad23c89f0..071213512 100644 --- a/tests/test_pageseg.py +++ b/tests/test_pageseg.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- import unittest +from pathlib import Path from PIL import Image from pytest import raises -from pathlib import Path -from kraken.pageseg import segment from kraken.lib.exceptions import KrakenInputException +from kraken.pageseg import segment thisfile = Path(__file__).resolve().parent resources = thisfile / 'resources' diff --git a/tests/test_readingorder.py b/tests/test_readingorder.py index 8b91ae7d9..e5230ccb9 100644 --- a/tests/test_readingorder.py +++ b/tests/test_readingorder.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- -import pytest import unittest from typing import Sequence, Tuple -import shapely.geometry as geom import numpy as np +import pytest +import shapely.geometry as geom from kraken.lib.segmentation import is_in_region, reading_order, topsort diff --git a/tests/test_repo.py b/tests/test_repo.py index 94a6407c0..06a97fc28 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import shutil -import unittest import tempfile - +import unittest from pathlib import Path from kraken import repo diff --git a/tests/test_rpred.py b/tests/test_rpred.py index b0486b83e..fdf2e0cf1 100644 --- a/tests/test_rpred.py +++ b/tests/test_rpred.py @@ -1,18 +1,19 @@ # -*- coding: utf-8 -*- -import os import json -import pytest +import os import unittest +from collections import defaultdict +from pathlib import Path +import pytest from PIL import Image from pytest import raises -from pathlib import Path -from collections import defaultdict -from kraken.containers import Segmentation, BBoxOCRRecord, BaselineOCRRecord, BaselineLine, BBoxLine -from kraken.lib.models import load_any -from kraken.rpred import rpred, mm_rpred +from kraken.containers import (BaselineLine, BaselineOCRRecord, BBoxLine, + BBoxOCRRecord, Segmentation) from kraken.lib.exceptions import KrakenInputException +from kraken.lib.models import load_any +from kraken.rpred import mm_rpred, rpred thisfile = Path(__file__).resolve().parent resources = thisfile / 'resources' diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 7b7f9eb3e..27febfa7d 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- import json -import uuid -import unittest import tempfile -import numpy as np - -from lxml import etree +import unittest +import uuid +from collections import Counter from io import StringIO from pathlib import Path + +import numpy as np from hocr_spec import HocrValidator -from collections import Counter +from lxml import etree from kraken import containers, serialization from kraken.lib import xml diff --git a/tests/test_train.py b/tests/test_train.py index b3bc14454..3d651d927 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- -import unittest import json - -import kraken +import unittest +from pathlib import Path from pytest import raises -from pathlib import Path +import kraken from kraken.lib import xml -from kraken.lib.train import KrakenTrainer, RecognitionModel, SegmentationModel from kraken.lib.exceptions import KrakenInputException +from kraken.lib.train import KrakenTrainer, RecognitionModel, SegmentationModel thisfile = Path(__file__).resolve().parent resources = thisfile / 'resources' diff --git a/tests/test_transcribe.py b/tests/test_transcribe.py index 3b91797df..044779813 100644 --- a/tests/test_transcribe.py +++ b/tests/test_transcribe.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- -import os import json +import os import unittest - -from PIL import Image from io import BytesIO -from lxml import etree from pathlib import Path +from lxml import etree +from PIL import Image + from kraken.transcribe import TranscriptionInterface thisfile = Path(__file__).resolve().parent diff --git a/tests/test_vgsl.py b/tests/test_vgsl.py index 56d85567d..b768fd8cd 100644 --- a/tests/test_vgsl.py +++ b/tests/test_vgsl.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- +import os +import tempfile import unittest -from pytest import raises -import os import torch -import tempfile -from kraken.lib import vgsl, layers +from pytest import raises + +from kraken.lib import layers, vgsl class TestVGSL(unittest.TestCase): diff --git a/tests/test_xml.py b/tests/test_xml.py index dbb876012..0d0369178 100644 --- a/tests/test_xml.py +++ b/tests/test_xml.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import json -import unittest import tempfile -import numpy as np - +import unittest from pathlib import Path + +import numpy as np from pytest import raises from kraken.lib import xml