From db286ecb6d6b97d950898d2151e6061e81f18280 Mon Sep 17 00:00:00 2001 From: Benjamin Kiessling Date: Wed, 31 Jan 2024 00:06:01 +0100 Subject: [PATCH] More linting --- .../contrib/hyperparameters/tune_training.py | 24 ++++++----- kraken/contrib/print_word_spreader.py | 41 +++++++++---------- kraken/lib/layers.py | 3 +- kraken/lib/train.py | 12 +++--- kraken/lib/vgsl.py | 36 ++++++++++------ 5 files changed, 65 insertions(+), 51 deletions(-) diff --git a/kraken/contrib/hyperparameters/tune_training.py b/kraken/contrib/hyperparameters/tune_training.py index 31275c4eb..e123c3754 100644 --- a/kraken/contrib/hyperparameters/tune_training.py +++ b/kraken/contrib/hyperparameters/tune_training.py @@ -31,15 +31,15 @@ def train_tune(config, training_data=None, epochs=100): output='model', spec=RECOGNITION_SPEC) - data_module = PretrainDataModule(batch_size=hyper_params.pop('batch_size'), - pad=hyper_params.pop('pad'), - augment=hyper_params.pop('augment'), - training_data=training_data, - num_workers=resources_per_trial['cpu'], - height=model.height, - width=model.width, - channels=model.channels, - format_type='binary') + _ = PretrainDataModule(batch_size=hyper_params.pop('batch_size'), + pad=hyper_params.pop('pad'), + augment=hyper_params.pop('augment'), + training_data=training_data, + num_workers=resources_per_trial['cpu'], + height=model.height, + width=model.width, + channels=model.channels, + format_type='binary') callback = TuneReportCallback({'loss': 'CE'}, on='validation_end') trainer = pl.Trainer(max_epochs=epochs, @@ -49,6 +49,10 @@ def train_tune(config, training_data=None, epochs=100): trainer.fit(model) -analysis = tune.run(partial(train_tune, training_data=sys.argv[2:]), local_dir=sys.argv[1], num_samples=100, resources_per_trial=resources_per_trial, config=config) +analysis = tune.run(partial(train_tune, training_data=sys.argv[2:]), + local_dir=sys.argv[1], + num_samples=100, + resources_per_trial=resources_per_trial, + config=config) print("Best hyperparameters found were: ", analysis.get_best_config(metric='accuracy', mode='max')) diff --git a/kraken/contrib/print_word_spreader.py b/kraken/contrib/print_word_spreader.py index ae0ab65e9..dd95cd3e0 100755 --- a/kraken/contrib/print_word_spreader.py +++ b/kraken/contrib/print_word_spreader.py @@ -26,24 +26,21 @@ class BboxError(Exception): parser.add_argument('--inputDir', help='Path to directory where source files are found', required=True) parser.add_argument('--outputDir', help='Path to directory where output is stored', required=True) parser.add_argument('--imageDir', help='Path to directory where images corresponding to the html files are stored.', required=False) -parser.add_argument( - '-c', - '--confidenceSummary', - default=False, - action="store_true", - help="store summaries of word confidence in xhtml data- attributes and cut all material after the first ; from the word span title attribute, making their mouseover popups less obtrusive.") -parser.add_argument( - '-f', - '--fixBigWordSpans', - default=False, - action="store_true", - help="fix word_span elements whose bbox area is greater than a sixth of the whole page area by assigning them the bbox of the previous word.") -parser.add_argument( - '-s', - '--shareSpaceSpans', - default=False, - action="store_true", - help="normalize hocr output from kraken, which assigns a word to every space and gives it a bbox. This removes those space words and assigns their area to the words on either side, with some space in between, generating output more like Ocropus and tesseract.") +parser.add_argument('-c', '--confidenceSummary', default=False, action="store_true", + help="store summaries of word confidence in xhtml data- " + "attributes and cut all material after the first ; from the " + "word span title attribute, making their mouseover popups " + "less obtrusive.") +parser.add_argument('-f', '--fixBigWordSpans', default=False, action="store_true", + help="fix word_span elements whose bbox area is greater " + "than a sixth of the whole page area by assigning them the " + "bbox of the previous word.") +parser.add_argument('-s', '--shareSpaceSpans', default=False, action="store_true", + help="normalize hocr output from kraken, which assigns a " + "word to every space and gives it a bbox. This removes those " + "space words and assigns their area to the words on either " + "side, with some space in between, generating output more " + "like Ocropus and tesseract.") parser.add_argument("-v", "--verbose", help="increase output verbosity", default=False, action="store_true") args = parser.parse_args() @@ -76,7 +73,7 @@ def get_bbox_area(span): if (args.verbose): print("this element's area is " + str(area)) return area - except Exception as e: + except Exception: # print("Exception getting area on span {}".format(etree.tostring(span))) raise @@ -84,7 +81,7 @@ def get_bbox_area(span): def set_bbox_value(span, position, val): try: parts = span.get('title').split(';') - except Exception as e: + except Exception: print("Exception getting title element on span id {}.".format(span.get('id'))) raise BboxError bbox_parts = parts[0].split(' ') @@ -114,7 +111,7 @@ def share_space_spans(treeIn): print(e) raise # check that we have both - if ((not previous_span is None) and (not next_span is None)): + if previous_span and next_span: # this means that there is both a previous and a next if (args.verbose): print("***") @@ -155,7 +152,7 @@ def confidence_summary(treeIn): word_span.set('data-min-confidence', str(minimum)) word_span.set('data-average-confidence', str(average)) word_span.set('title', bbox_only) - except Exception as e: + except Exception: # there's not much to do if this goes wrong pass diff --git a/kraken/lib/layers.py b/kraken/lib/layers.py index e1a23b8e2..a937fdd32 100644 --- a/kraken/lib/layers.py +++ b/kraken/lib/layers.py @@ -865,7 +865,8 @@ def get_shape(self, input: Tuple[int, int, int, int], target_shape: Optional[Tup self.out_channels, int(max(np.floor((input[2]+2*self.padding[0]-self.dilation[0]*(self.kernel_size[0]-1)-1) / self.stride[0]+1), 1) if input[2] != 0 else 0), - int(max(np.floor((input[3]+2*self.padding[1]-self.dilation[1]*(self.kernel_size[1]-1)-1)/self.stride[1]+1), 1) if input[3] != 0 else 0)) + int(max(np.floor((input[3]+2*self.padding[1]-self.dilation[1]*(self.kernel_size[1]-1)-1) / + self.stride[1]+1), 1) if input[3] != 0 else 0)) return self.output_shape def deserialize(self, name: str, spec) -> None: diff --git a/kraken/lib/train.py b/kraken/lib/train.py index 62cdafbec..fa7dd5450 100644 --- a/kraken/lib/train.py +++ b/kraken/lib/train.py @@ -946,8 +946,8 @@ def setup(self, stage: Optional[str] = None): elif self.resize == 'union': new_bls = self.train_set.dataset.class_mapping['baselines'].keys() - self.nn.user_metadata['class_mapping']['baselines'].keys() new_regions = self.train_set.dataset.class_mapping['regions'].keys() - self.nn.user_metadata['class_mapping']['regions'].keys() - cls_idx = max(max(self.nn.user_metadata['class_mapping']['baselines'].values()) if self.nn.user_metadata['class_mapping']['baselines'] else -1, - max(self.nn.user_metadata['class_mapping']['regions'].values()) if self.nn.user_metadata['class_mapping']['regions'] else -1) + cls_idx = max(max(self.nn.user_metadata['class_mapping']['baselines'].values()) if self.nn.user_metadata['class_mapping']['baselines'] else -1, # noqa + max(self.nn.user_metadata['class_mapping']['regions'].values()) if self.nn.user_metadata['class_mapping']['regions'] else -1) # noqa logger.info(f'Adding {len(new_bls) + len(new_regions)} missing types to network output layer.') self.nn.resize_output(cls_idx + len(new_bls) + len(new_regions) + 1) for c in new_bls: @@ -965,8 +965,8 @@ def setup(self, stage: Optional[str] = None): logger.info(f'Adding {len(new_bls) + len(new_regions)} missing ' f'types and removing {len(del_bls) + len(del_regions)} to network output layer ') - cls_idx = max(max(self.nn.user_metadata['class_mapping']['baselines'].values()) if self.nn.user_metadata['class_mapping']['baselines'] else -1, - max(self.nn.user_metadata['class_mapping']['regions'].values()) if self.nn.user_metadata['class_mapping']['regions'] else -1) + cls_idx = max(max(self.nn.user_metadata['class_mapping']['baselines'].values()) if self.nn.user_metadata['class_mapping']['baselines'] else -1, # noqa + max(self.nn.user_metadata['class_mapping']['regions'].values()) if self.nn.user_metadata['class_mapping']['regions'] else -1) # noqa del_indices = [self.nn.user_metadata['class_mapping']['baselines'][x] for x in del_bls] del_indices.extend(self.nn.user_metadata['class_mapping']['regions'][x] for x in del_regions) @@ -974,8 +974,8 @@ def setup(self, stage: Optional[str] = None): len(del_bls) - len(del_regions) + 1, del_indices) # delete old baseline/region types - cls_idx = min(min(self.nn.user_metadata['class_mapping']['baselines'].values()) if self.nn.user_metadata['class_mapping']['baselines'] else np.inf, - min(self.nn.user_metadata['class_mapping']['regions'].values()) if self.nn.user_metadata['class_mapping']['regions'] else np.inf) + cls_idx = min(min(self.nn.user_metadata['class_mapping']['baselines'].values()) if self.nn.user_metadata['class_mapping']['baselines'] else np.inf, # noqa + min(self.nn.user_metadata['class_mapping']['regions'].values()) if self.nn.user_metadata['class_mapping']['regions'] else np.inf) # noqa bls = {} for k, v in sorted(self.nn.user_metadata['class_mapping']['baselines'].items(), key=lambda item: item[1]): diff --git a/kraken/lib/vgsl.py b/kraken/lib/vgsl.py index cdbc6c7a2..9d1cfc6cc 100644 --- a/kraken/lib/vgsl.py +++ b/kraken/lib/vgsl.py @@ -469,7 +469,8 @@ def build_rnn(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds an LSTM/GRU layer returning number of outputs and layer. """ @@ -497,7 +498,8 @@ def build_dropout(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: pattern = re.compile(r'(?PDo)(?P{\w+})?(?P

(\d+(\.\d*)?|\.\d+))?(,(?P\d+))?') m = pattern.match(blocks[idx]) if not m: @@ -513,7 +515,8 @@ def build_addition(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: pattern = re.compile(r'(?PA)(?P{\w+})?(?P\d+),(?P\d+)') m = pattern.match(blocks[idx]) if not m: @@ -533,7 +536,8 @@ def build_identity(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: pattern = re.compile(r'(?PI)(?P{\w+})?') m = pattern.match(blocks[idx]) if not m: @@ -547,7 +551,8 @@ def build_groupnorm(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: pattern = re.compile(r'(?PGn)(?P{\w+})?(?P\d+)') m = pattern.match(blocks[idx]) if not m: @@ -562,7 +567,8 @@ def build_wav2vec2(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds a Wav2Vec2 masking layer. """ @@ -604,7 +610,8 @@ def build_conv(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds a 2D convolution layer. """ @@ -629,7 +636,8 @@ def build_maxpool(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds a maxpool layer. """ @@ -649,7 +657,8 @@ def build_reshape(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds a reshape layer """ @@ -688,7 +697,8 @@ def build_output(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds an output layer. """ @@ -755,7 +765,8 @@ def build_series(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds a serial block of layers. """ @@ -784,7 +795,8 @@ def build_parallel(self, input: Tuple[int, int, int, int], blocks: List[str], idx: int, - target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], Tuple[Tuple[int, int, int, int], str, Callable]]: + target_output_shape: Optional[Tuple[int, int, int, int]] = None) -> Union[Tuple[None, None, None], + Tuple[Tuple[int, int, int, int], str, Callable]]: """ Builds a block of parallel layers. """