diff --git a/src/api/hocrrenderer.cpp b/src/api/hocrrenderer.cpp index 3a5cb986c3..b54445dddd 100644 --- a/src/api/hocrrenderer.cpp +++ b/src/api/hocrrenderer.cpp @@ -213,13 +213,17 @@ char* TessBaseAPI::GetHOCRText(ETEXT_DESC* monitor, int page_number) { } // Now, process the word... - std::vector>>* confidencemap = + std::vector>>* rawTimestepMap = + nullptr; + std::vector>>* choiceMap = nullptr; std::vector>>>* symbolMap = nullptr; if (tesseract_->lstm_choice_mode) { - confidencemap = res_it->GetBestLSTMSymbolChoices(); - symbolMap = res_it->GetBestSegmentedLSTMSymbolChoices(); + + choiceMap = res_it->GetBestLSTMSymbolChoices(); + symbolMap = res_it->GetSegmentedLSTMTimesteps(); + rawTimestepMap = res_it->GetRawLSTMTimesteps(); } hocr_str << "\n "; std::vector> timestep = - (*confidencemap)[i]; + (*rawTimestepMap)[i]; for (std::pair conf : timestep) { hocr_str << ""; - for (size_t j = 1; j < timestep.size(); j++) { + << "'>"; + for (size_t j = 0; j < timestep.size(); j++) { hocr_str << "" - << timesteps[0][0].first; - for (size_t i = 1; i < timesteps.size(); i++) { + << "symbol_" << page_id << "_" << wcnt << "_" << scnt + << "'>"; + for (size_t i = 0; i < timesteps.size(); i++) { hocr_str << "\n data()->certainty(); - if (confidence < 0.0f) confidence = 0.0f; - if (confidence > 100.0f) confidence = 100.0f; - return confidence; + if (LSTM_mode_) { + std::pair choice = *LSTM_choice_it_; + return choice.second; + } else { + if (choice_it_ == nullptr) return 0.0f; + float confidence = 100 + 5 * choice_it_->data()->certainty(); + if (confidence < 0.0f) confidence = 0.0f; + if (confidence > 100.0f) confidence = 100.0f; + return confidence; + } } +// Returns the set of timesteps which belong to the current symbol std::vector>>* ChoiceIterator::Timesteps() const { - if (&word_res_->symbol_steps == nullptr) return nullptr; - return &*symbol_step_it_; + if (&word_res_->symbol_steps == nullptr || !LSTM_mode_) return nullptr; + if (word_res_->leadingSpace) { + return &word_res_->symbol_steps[*(tstep_index_) + 1]; + } else { + return &word_res_->symbol_steps[*tstep_index_]; + } +} + +void ChoiceIterator::filterSpaces() { + if (LSTM_choices_->empty()) return; + std::vector>::iterator it = + LSTM_choices_->begin(); + bool found_space = false; + float sum = 0; + for (it; it != LSTM_choices_->end();) { + if (!strcmp(it->first, " ")) { + it = LSTM_choices_->erase(it); + found_space = true; + } else { + sum += it->second; + ++it; + } + } + if (found_space) { + for (it = LSTM_choices_->begin(); it != LSTM_choices_->end(); ++it) { + it->second /= sum; + } + } } } // namespace tesseract. diff --git a/src/ccmain/ltrresultiterator.h b/src/ccmain/ltrresultiterator.h index 98b42da465..5bd8d5f00d 100644 --- a/src/ccmain/ltrresultiterator.h +++ b/src/ccmain/ltrresultiterator.h @@ -208,25 +208,36 @@ class ChoiceIterator { // internal structure and should NOT be delete[]ed to free after use. const char* GetUTF8Text() const; - // Returns the confidence of the current choice. - // The number should be interpreted as a percent probability. (0.0f-100.0f) + // Returns the confidence of the current choice depending on the used language + // data. If only LSTM traineddata is used the value range is 0.0f - 1.0f. All + // choices for one symbol should roughly add up to 1.0f. + // If only traineddata of the legacy engine is used, the number should be + // interpreted as a percent probability. (0.0f-100.0f) In this case + // probabilities won't add up to 100. Each one stands on its own. float Confidence() const; // Returns a vector containing all timesteps, which belong to the currently // selected symbol. A timestep is a vector containing pairs of symbols and // floating point numbers. The number states the probability for the // corresponding symbol. - std::vector>>* - Timesteps() const; + std::vector>>* Timesteps() const; private: + //clears the remaining spaces out of the results and adapt the probabilities + void filterSpaces(); // Pointer to the WERD_RES object owned by the API. WERD_RES* word_res_; // Iterator over the blob choices. BLOB_CHOICE_IT* choice_it_; - //Iterator over the symbol steps. - std::vector>>>::iterator - symbol_step_it_; + std::vector>* LSTM_choices_ = nullptr; + std::vector>::iterator LSTM_choice_it_; + + const int* tstep_index_; + bool LSTM_mode_ = false; + //true when there is lstm engine related trained data + bool oemLSTM_; + // true when there is legacy engine related trained data + bool oemLegacy_; }; } // namespace tesseract. diff --git a/src/ccmain/resultiterator.cpp b/src/ccmain/resultiterator.cpp index b5814190d1..514f52a98c 100644 --- a/src/ccmain/resultiterator.cpp +++ b/src/ccmain/resultiterator.cpp @@ -604,18 +604,26 @@ char* ResultIterator::GetUTF8Text(PageIteratorLevel level) const { strncpy(result, text.string(), length); return result; } +std::vector>>* +ResultIterator::GetRawLSTMTimesteps() const { + if (it_->word() != nullptr) { + return &it_->word()->raw_timesteps; + } else { + return nullptr; + } +} std::vector>>* ResultIterator::GetBestLSTMSymbolChoices() const { if (it_->word() != nullptr) { - return &it_->word()->timesteps; + return &it_->word()->accumulated_timesteps; } else { return nullptr; } } std::vector>>>* - ResultIterator::GetBestSegmentedLSTMSymbolChoices() const { + ResultIterator::GetSegmentedLSTMTimesteps() const { if (it_->word() != nullptr) { return &it_->word()->symbol_steps; } else { diff --git a/src/ccmain/resultiterator.h b/src/ccmain/resultiterator.h index de45bc1262..817659951c 100644 --- a/src/ccmain/resultiterator.h +++ b/src/ccmain/resultiterator.h @@ -100,10 +100,12 @@ class TESS_API ResultIterator : public LTRResultIterator { /** * Returns the LSTM choices for every LSTM timestep for the current word. */ + virtual std::vector>>* + GetRawLSTMTimesteps() const; virtual std::vector>>* GetBestLSTMSymbolChoices() const; virtual std::vector>>>* - GetBestSegmentedLSTMSymbolChoices() const; + GetSegmentedLSTMTimesteps() const; /** * Return whether the current paragraph's dominant reading direction diff --git a/src/ccmain/tesseractclass.cpp b/src/ccmain/tesseractclass.cpp index 5ee5ccb2c3..e837fa9095 100644 --- a/src/ccmain/tesseractclass.cpp +++ b/src/ccmain/tesseractclass.cpp @@ -524,11 +524,12 @@ Tesseract::Tesseract() this->params()), INT_MEMBER(lstm_choice_mode, 0, "Allows to include alternative symbols choices in the hOCR output. " - "Valid input values are 0, 1 and 2. 0 is the default value. " + "Valid input values are 0, 1, 2 and 3. 0 is the default value. " "With 1 the alternative symbol choices per timestep are included. " - "With 2 the alternative symbol choices are accumulated per character." - "With 3 the alternative symbol choices per timestep are included and " - "separated by the suggested segmentation of Tesseract", + "With 2 the alternative symbol choices are accumulated per " + "character. " + "With 3 the alternative symbol choices per timestep are included " + "and separated by the suggested segmentation of Tesseract", this->params()), backup_config_file_(nullptr), diff --git a/src/ccmain/tesseractclass.h b/src/ccmain/tesseractclass.h index f937cc69a9..95b7c04a58 100644 --- a/src/ccmain/tesseractclass.h +++ b/src/ccmain/tesseractclass.h @@ -1124,12 +1124,14 @@ class Tesseract : public Wordrec { STRING_VAR_H(page_separator, "\f", "Page separator (default is form feed control character)"); INT_VAR_H(lstm_choice_mode, 0, - "Allows to include alternative symbols choices in the hOCR output. " - "Valid input values are 0, 1 and 2. 0 is the default value. " + "Allows to include alternative symbols choices in the hOCR " + "output. " + "Valid input values are 0, 1, 2 and 3. 0 is the default value. " "With 1 the alternative symbol choices per timestep are included. " - "With 2 the alternative symbol choices are accumulated per character." - "With 3 the alternative symbol choices per timestep are included and " - "separated by the suggested segmentation of Tesseract"); + "With 2 the alternative symbol choices are accumulated per " + "character. " + "With 3 the alternative symbol choices per timestep are included " + "and separated by the suggested segmentation of Tesseract"); //// ambigsrecog.cpp ///////////////////////////////////////////////////////// FILE *init_recog_training(const STRING &fname); diff --git a/src/ccstruct/pageres.h b/src/ccstruct/pageres.h index 55997839d4..b41e4c359a 100644 --- a/src/ccstruct/pageres.h +++ b/src/ccstruct/pageres.h @@ -221,9 +221,12 @@ class WERD_RES : public ELIST_LINK { // blob i and blob i+1. GenericVector blob_gaps; // Stores the lstm choices of every timestep - std::vector>> timesteps; + std::vector>> raw_timesteps; + std::vector>> accumulated_timesteps; std::vector>>> symbol_steps; + //Stores if the timestep vector starts with a space + bool leadingSpace = false; // Ratings matrix contains classifier choices for each classified combination // of blobs. The dimension is the same as the number of blobs in chopped_word // and the leading diagonal corresponds to classifier results of the blobs diff --git a/src/lstm/recodebeam.cpp b/src/lstm/recodebeam.cpp index 06c240275a..56b5f92924 100644 --- a/src/lstm/recodebeam.cpp +++ b/src/lstm/recodebeam.cpp @@ -188,7 +188,8 @@ void RecodeBeamSearch::ExtractBestPathAsWords(const TBOX& line_box, GenericVector xcoords; GenericVector best_nodes; GenericVector second_nodes; - std::deque> best_choices; + std::deque> best_choices; + std::deque> best_choices_acc; ExtractBestPaths(&best_nodes, &second_nodes); if (debug) { DebugPath(unicharset, best_nodes); @@ -198,18 +199,18 @@ void RecodeBeamSearch::ExtractBestPathAsWords(const TBOX& line_box, DebugUnicharPath(unicharset, second_nodes, unichar_ids, certs, ratings, xcoords); } - int current_char = 0; + int timestepEndRaw = 0; int timestepEnd = 0; + int timestepEnd_acc = 0; //if lstm choice mode is required in granularity level 2 it stores the x //Coordinates of every chosen character to match the alternative choices to it - if (lstm_choice_mode == 2 || lstm_choice_mode == 3) { + if (lstm_choice_mode) { ExtractPathAsUnicharIds(best_nodes, &unichar_ids, &certs, &ratings, - &xcoords, &best_choices); + &xcoords, &best_choices, &best_choices_acc); if (best_choices.size() > 0) { - current_char = std::get<0>(best_choices.front()); timestepEnd = std::get<1>(best_choices.front()); - if(lstm_choice_mode == 2) - best_choices.pop_front(); + timestepEnd_acc = std::get<1>(best_choices_acc.front()); + best_choices_acc.pop_front(); } } else { ExtractPathAsUnicharIds(best_nodes, &unichar_ids, &certs, &ratings, @@ -245,31 +246,28 @@ void RecodeBeamSearch::ExtractBestPathAsWords(const TBOX& line_box, WERD_RES* word_res = InitializeWord( leading_space, line_box, word_start, word_end, std::min(space_cert, prev_space_cert), unicharset, xcoords, scale_factor); - if (lstm_choice_mode == 1) { - for (size_t i = timestepEnd; i < xcoords[word_end]; i++) { - word_res->timesteps.push_back(timesteps[i]); + if (lstm_choice_mode) { + for (size_t i = timestepEndRaw; i < xcoords[word_end]; i++) { + word_res->raw_timesteps.push_back(timesteps[i]); } - timestepEnd = xcoords[word_end]; - } else if (lstm_choice_mode == 2) { + timestepEndRaw = xcoords[word_end]; + // Accumulated Timesteps (choice mode 2 processing) float sum = 0; std::vector> choice_pairs; - for (size_t i = timestepEnd; i < xcoords[word_end]; i++) { + for (size_t i = timestepEnd_acc; i < xcoords[word_end]; i++) { for (std::pair choice : timesteps[i]) { - if (std::strcmp(choice.first, "") != 0) { + if (std::strcmp(choice.first, "")) { sum += choice.second; choice_pairs.push_back(choice); } } - if ((best_choices.size() > 0 && i == std::get<1>(best_choices.front()) - 1) + if ((best_choices_acc.size() > 0 && i == std::get<1>(best_choices_acc.front()) - 1) || i == xcoords[word_end]-1) { std::map summed_propabilities; for (auto it = choice_pairs.begin(); it != choice_pairs.end(); ++it) { summed_propabilities[it->first] += it->second; } std::vector> accumulated_timestep; - accumulated_timestep.push_back(std::pair - (unicharset->id_to_unichar_ext - (current_char), 2.0)); int pos; for (auto it = summed_propabilities.begin(); it != summed_propabilities.end(); ++it) { @@ -284,17 +282,16 @@ void RecodeBeamSearch::ExtractBestPathAsWords(const TBOX& line_box, std::pair(it->first, it->second)); } - if (best_choices.size() > 0) { - current_char = std::get<0>(best_choices.front()); - best_choices.pop_front(); + if (best_choices_acc.size() > 0) { + best_choices_acc.pop_front(); } choice_pairs.clear(); - word_res->timesteps.push_back(accumulated_timestep); + word_res->accumulated_timesteps.push_back(accumulated_timestep); sum = 0; } } - timestepEnd = xcoords[word_end]; - } else if (lstm_choice_mode == 3) { + timestepEnd_acc = xcoords[word_end]; + //Symbol Step (choice mode 3 processing) std::vector>> currentSymbol; for (size_t i = timestepEnd; i < xcoords[word_end]; i++) { if (i == std::get<1>(best_choices.front())) { @@ -302,11 +299,10 @@ void RecodeBeamSearch::ExtractBestPathAsWords(const TBOX& line_box, word_res->symbol_steps.push_back(currentSymbol); currentSymbol.clear(); } - std::vector> choice_Header; - choice_Header.push_back(std::pair( - unicharset->id_to_unichar_ext(std::get<0>(best_choices.front())), - 2.0)); - currentSymbol.push_back(choice_Header); + const char* leadCharacter = + unicharset->id_to_unichar_ext(std::get<0>(best_choices.front())); + if (!strcmp(leadCharacter, " ")) + word_res->leadingSpace = true; if(best_choices.size()>1) best_choices.pop_front(); } currentSymbol.push_back(timesteps[i]); @@ -387,7 +383,8 @@ void RecodeBeamSearch::ExtractPathAsUnicharIds( const GenericVector& best_nodes, GenericVector* unichar_ids, GenericVector* certs, GenericVector* ratings, GenericVector* xcoords, - std::deque>* best_choices) { + std::deque>* best_choices, + std::deque>* best_choices_acc) { unichar_ids->truncate(0); certs->truncate(0); ratings->truncate(0); @@ -440,7 +437,9 @@ void RecodeBeamSearch::ExtractPathAsUnicharIds( } if (best_choices != nullptr) { best_choices->push_back( - std::tuple(id, tposition, rating)); + std::tuple(id, tposition)); + best_choices_acc->push_back( + std::tuple(id, tposition)); } } xcoords->push_back(width); diff --git a/src/lstm/recodebeam.h b/src/lstm/recodebeam.h index 7aa0d7b32d..921eeb6254 100644 --- a/src/lstm/recodebeam.h +++ b/src/lstm/recodebeam.h @@ -282,7 +282,8 @@ class RecodeBeamSearch { const GenericVector& best_nodes, GenericVector* unichar_ids, GenericVector* certs, GenericVector* ratings, GenericVector* xcoords, - std::deque>* best_choices = nullptr); + std::deque>* best_choices = nullptr, + std::deque>* best_choices_acc = nullptr); // Sets up a word with the ratings matrix and fake blobs with boxes in the // right places.