2025-01-14 18:17:48,179 WARN org.apache.flink.runtime.taskmanager.Task [] - split_reader (2/5)#0 (19d504a6ac568a1412b59b4ac9ca94ce) switched from RUNNING to FAILED with failure cause: java.io.FileNotFoundException: File does not exist: hdfs://node1/hudi_t1_cow/00000000-e4fe-4a87-b05b-9ee7182da049-0_0-10-0_20250114171356793.parquet at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1749) at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1742) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1757) at org.apache.parquet.hadoop.util.HadoopInputFile.fromPath(HadoopInputFile.java:39) at org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:469) at org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader.(ParquetColumnarRowSplitReader.java:134) at org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.genPartColumnarRowReader(ParquetSplitReaderUtil.java:153) at org.apache.hudi.table.format.RecordIterators.getParquetRecordIterator(RecordIterators.java:78) at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.getBaseFileIterator(MergeOnReadInputFormat.java:333) at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.getBaseFileIterator(MergeOnReadInputFormat.java:320) at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.initIterator(MergeOnReadInputFormat.java:204) at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.open(MergeOnReadInputFormat.java:189) at org.apache.hudi.source.StreamReadOperator.processSplits(StreamReadOperator.java:169) at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:50) at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:90) at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMailsNonBlocking(MailboxProcessor.java:359) at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:323) at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:202) at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:681) at org.apache.flink.streaming.runtime.tasks.StreamTask.executeInvoke(StreamTask.java:636) at org.apache.flink.streaming.runtime.tasks.StreamTask.runWithCleanUpOnFail(StreamTask.java:647) at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:620) at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:779) at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) at java.lang.Thread.run(Thread.java:748) 2025-01-14 18:18:48,627 INFO org.apache.hudi.common.table.HoodieTableMetaClient [] - Loading HoodieTableMetaClient from hdfs://node1/hudi_t1_cow/ 2025-01-14 18:18:48,592 WARN org.apache.flink.streaming.api.operators.BackendRestorerProcedure [] - Exception while restoring operator state backend for StreamReadOperator_a8d133e381dec8afcbd7ef8805eaed60_(1/5) from alternative (1/1), will retry while more alternatives are available. org.apache.flink.runtime.state.BackendBuildingException: Failed when trying to restore operator state backend at org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder.build(DefaultOperatorStateBackendBuilder.java:83) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend.createOperatorStateBackend(EmbeddedRocksDBStateBackend.java:485) ~[flink-dist_2.12-1.13.2-csa1.5.1.0.jar:1.13.2-csa1.5.1.0] at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createOperatorStateBackend(RocksDBStateBackend.java:367) ~[flink-dist_2.12-1.13.2-csa1.5.1.0.jar:1.13.2-csa1.5.1.0] at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$operatorStateBackend$0(StreamTaskStateInitializerImpl.java:276) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:168) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.operatorStateBackend(StreamTaskStateInitializerImpl.java:285) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:173) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:272) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.OperatorChain.initializeStateAndOpenOperators(OperatorChain.java:441) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.restoreGates(StreamTask.java:582) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.call(StreamTaskActionExecutor.java:55) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.executeRestore(StreamTask.java:562) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.runWithCleanUpOnFail(StreamTask.java:647) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.restore(StreamTask.java:537) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:759) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_232] Caused by: java.io.IOException: Stream closed at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:758) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:840) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:695) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:83) ~[?:1.8.0_232] at org.apache.flink.runtime.fs.hdfs.HadoopDataInputStream.read(HadoopDataInputStream.java:86) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.core.fs.FSDataInputStreamWrapper.read(FSDataInputStreamWrapper.java:50) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.io.DataInputStream.readUnsignedByte(DataInputStream.java:288) ~[?:1.8.0_232] at org.apache.flink.api.java.typeutils.runtime.DataInputViewStream.read(DataInputViewStream.java:45) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.io.ObjectInputStream$PeekInputStream.peek(ObjectInputStream.java:2641) ~[?:1.8.0_232] at java.io.ObjectInputStream$BlockDataInputStream.peek(ObjectInputStream.java:2948) ~[?:1.8.0_232] at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2958) ~[?:1.8.0_232] at java.io.ObjectInputStream.readTypeString(ObjectInputStream.java:1644) ~[?:1.8.0_232] at java.io.ObjectStreamClass.readNonProxy(ObjectStreamClass.java:803) ~[?:1.8.0_232] at java.io.ObjectInputStream.readClassDescriptor(ObjectInputStream.java:891) ~[?:1.8.0_232] at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1857) ~[?:1.8.0_232] at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1751) ~[?:1.8.0_232] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2042) ~[?:1.8.0_232] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573) ~[?:1.8.0_232] at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431) ~[?:1.8.0_232] at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:615) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:593) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.JavaSerializer.deserialize(JavaSerializer.java:85) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.JavaSerializer.deserialize(JavaSerializer.java:40) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.OperatorStateRestoreOperation.deserializeOperatorStateValues(OperatorStateRestoreOperation.java:217) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.OperatorStateRestoreOperation.restore(OperatorStateRestoreOperation.java:188) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder.build(DefaultOperatorStateBackendBuilder.java:80) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] ... 17 more 2025-01-14 18:18:48,627 INFO org.apache.hudi.common.table.HoodieTableMetaClient [] - Loading HoodieTableMetaClient from hdfs://node1/hudi_t1_cow/ 2025-01-14 18:18:48,592 WARN org.apache.flink.streaming.api.operators.BackendRestorerProcedure [] - Exception while restoring operator state backend for StreamReadOperator_a8d133e381dec8afcbd7ef8805eaed60_(1/5) from alternative (1/1), will retry while more alternatives are available. org.apache.flink.runtime.state.BackendBuildingException: Failed when trying to restore operator state backend at org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder.build(DefaultOperatorStateBackendBuilder.java:83) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.contrib.streaming.state.EmbeddedRocksDBStateBackend.createOperatorStateBackend(EmbeddedRocksDBStateBackend.java:485) ~[flink-dist_2.12-1.13.2-csa1.5.1.0.jar:1.13.2-csa1.5.1.0] at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createOperatorStateBackend(RocksDBStateBackend.java:367) ~[flink-dist_2.12-1.13.2-csa1.5.1.0.jar:1.13.2-csa1.5.1.0] at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$operatorStateBackend$0(StreamTaskStateInitializerImpl.java:276) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:168) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.operatorStateBackend(StreamTaskStateInitializerImpl.java:285) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:173) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:272) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.OperatorChain.initializeStateAndOpenOperators(OperatorChain.java:441) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.restoreGates(StreamTask.java:582) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.call(StreamTaskActionExecutor.java:55) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.executeRestore(StreamTask.java:562) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.runWithCleanUpOnFail(StreamTask.java:647) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.restore(StreamTask.java:537) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:759) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_232] Caused by: java.io.IOException: Stream closed at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:758) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:840) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:695) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:83) ~[?:1.8.0_232] at org.apache.flink.runtime.fs.hdfs.HadoopDataInputStream.read(HadoopDataInputStream.java:86) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.core.fs.FSDataInputStreamWrapper.read(FSDataInputStreamWrapper.java:50) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.io.DataInputStream.readUnsignedByte(DataInputStream.java:288) ~[?:1.8.0_232] at org.apache.flink.api.java.typeutils.runtime.DataInputViewStream.read(DataInputViewStream.java:45) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.io.ObjectInputStream$PeekInputStream.peek(ObjectInputStream.java:2641) ~[?:1.8.0_232] at java.io.ObjectInputStream$BlockDataInputStream.peek(ObjectInputStream.java:2948) ~[?:1.8.0_232] at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2958) ~[?:1.8.0_232] at java.io.ObjectInputStream.readTypeString(ObjectInputStream.java:1644) ~[?:1.8.0_232] at java.io.ObjectStreamClass.readNonProxy(ObjectStreamClass.java:803) ~[?:1.8.0_232] at java.io.ObjectInputStream.readClassDescriptor(ObjectInputStream.java:891) ~[?:1.8.0_232] at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1857) ~[?:1.8.0_232] at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1751) ~[?:1.8.0_232] at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2042) ~[?:1.8.0_232] at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1573) ~[?:1.8.0_232] at java.io.ObjectInputStream.readObject(ObjectInputStream.java:431) ~[?:1.8.0_232] at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:615) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:593) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.JavaSerializer.deserialize(JavaSerializer.java:85) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.JavaSerializer.deserialize(JavaSerializer.java:40) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.OperatorStateRestoreOperation.deserializeOperatorStateValues(OperatorStateRestoreOperation.java:217) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.OperatorStateRestoreOperation.restore(OperatorStateRestoreOperation.java:188) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.state.DefaultOperatorStateBackendBuilder.build(DefaultOperatorStateBackendBuilder.java:80) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] ... 17 more 2025-01-14 18:17:48,234 INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - split_reader (2/5) (19d504a6ac568a1412b59b4ac9ca94ce) switched from RUNNING to FAILED on container_e38_1734494154374_0935_01_000006 @ bobcfc-sdcdh-dn01.server.com (dataPort=17557). java.io.FileNotFoundException: File does not exist: hdfs://node1/hudi_t1_cow/00000000-e4fe-4a87-b05b-9ee7182da049-0_0-10-0_20250114171356793.parquet at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1749) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1742) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1757) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.parquet.hadoop.util.HadoopInputFile.fromPath(HadoopInputFile.java:39) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:469) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader.(ParquetColumnarRowSplitReader.java:134) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.genPartColumnarRowReader(ParquetSplitReaderUtil.java:153) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.RecordIterators.getParquetRecordIterator(RecordIterators.java:78) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.getBaseFileIterator(MergeOnReadInputFormat.java:333) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.getBaseFileIterator(MergeOnReadInputFormat.java:320) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.initIterator(MergeOnReadInputFormat.java:204) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.open(MergeOnReadInputFormat.java:189) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.source.StreamReadOperator.processSplits(StreamReadOperator.java:169) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:50) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:90) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMailsNonBlocking(MailboxProcessor.java:359) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:323) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:202) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:681) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.executeInvoke(StreamTask.java:636) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.runWithCleanUpOnFail(StreamTask.java:647) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:620) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:779) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_232] org.apache.flink.runtime.JobException: Recovery is suppressed by FixedDelayRestartBackoffTimeStrategy(maxNumberRestartAttempts=3, backoffTimeMS=60000) at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:138) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:82) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:216) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:206) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:197) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:682) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.scheduler.SchedulerNG.updateTaskExecutionState(SchedulerNG.java:79) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:435) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_232] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_232] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:305) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:212) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:77) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:158) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.actor.Actor.aroundReceive(Actor.scala:517) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.actor.Actor.aroundReceive$(Actor.scala:515) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.actor.ActorCell.receiveMessage(ActorCell.scala:592) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.actor.ActorCell.invoke(ActorCell.scala:561) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.Mailbox.run(Mailbox.scala:225) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.Mailbox.exec(Mailbox.scala:235) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] Caused by: java.io.FileNotFoundException: File does not exist: hdfs://node1/hudi_t1_cow/00000006-9767-4c9e-aaab-40e9c4bb9cfb-0_6-10-0_20250114171356793.parquet at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1749) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1742) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1757) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.parquet.hadoop.util.HadoopInputFile.fromPath(HadoopInputFile.java:39) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:469) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.cow.vector.reader.ParquetColumnarRowSplitReader.(ParquetColumnarRowSplitReader.java:134) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.genPartColumnarRowReader(ParquetSplitReaderUtil.java:153) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.RecordIterators.getParquetRecordIterator(RecordIterators.java:78) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.getBaseFileIterator(MergeOnReadInputFormat.java:333) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.getBaseFileIterator(MergeOnReadInputFormat.java:320) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.initIterator(MergeOnReadInputFormat.java:204) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.table.format.mor.MergeOnReadInputFormat.open(MergeOnReadInputFormat.java:189) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.hudi.source.StreamReadOperator.processSplits(StreamReadOperator.java:169) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.runThrowing(StreamTaskActionExecutor.java:50) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:90) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMailsNonBlocking(MailboxProcessor.java:359) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:323) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:202) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.executeRestore(StreamTask.java:565) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.runWithCleanUpOnFail(StreamTask.java:647) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.streaming.runtime.tasks.StreamTask.restore(StreamTask.java:537) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:759) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) ~[flink-1.0-SNAPSHOT-jar-with-dependencies_kudu_5.jar:?] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_232]