fixed a problem comparable to proycon/python-ucto#16 (for XmlInput)

LanguageMachines · Apr 21, 2023 · 4410794 · 4410794
1 parent 98e308c
commit 4410794
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/tokenize.cxx b/src/tokenize.cxx
@@ -1847,6 +1847,7 @@ namespace Tokenizer {
   }
 
   folia::Document *TokenizerClass::tokenize_folia( const string& infile_name ){
+    reset(); // when starting a new inputfile, we must reset provenance et.al.
     if ( inputclass == outputclass
 	 && !doWordCorrection ){
       DBG << "ucto: --filter=NO is automatically set. inputclass equals outputclass!"