[Enhancement] Log the content of journal entity on replay failure (ba…

…ckport #46011) (#46028) Signed-off-by: yiming <[email protected]> Co-authored-by: yiming <[email protected]>
StarRocks · May 23, 2024 · dddc16f · dddc16f
1 parent 3effd9e
commit dddc16f
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 6 deletions.
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockReader.java b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockReader.java
@@ -110,7 +110,7 @@ public void close() throws IOException, SRMetaBlockException {
             LOG.warn("Meta block for {} read {} json < total {} json, will skip the rest {} json",
                     header.getSrMetaBlockID(), numJsonRead, header.getNumJson(), rest);
             for (int i = 0; i != rest; ++i) {
-                LOG.warn("skip {} json: {}", i, Text.readStringWithChecksum(checkedInputStream));
+                LOG.warn("skip {}th json: {}", i, Text.readStringWithChecksum(checkedInputStream));
             }
         }
 

diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
@@ -2199,17 +2199,16 @@ protected boolean replayJournalInner(JournalCursor cursor, boolean flowControl)
                 EditLog.loadJournal(this, entity);
             } catch (Throwable e) {
                 if (canSkipBadReplayedJournal()) {
-                    LOG.error("!!! DANGER: SKIP JOURNAL {}: {} !!!",
-                            replayedJournalId.incrementAndGet(),
-                            entity == null ? null : entity.getData(),
-                            e);
+                    LOG.error("!!! DANGER: SKIP JOURNAL, id: {}, data: {} !!!",
+                            replayedJournalId.incrementAndGet(), journalEntityToReadableString(entity), e);
                     if (!readSucc) {
                         cursor.skipNext();
                     }
                     continue;
                 }
                 // handled in outer loop
-                LOG.warn("catch exception when replaying {},", replayedJournalId.get() + 1, e);
+                LOG.warn("catch exception when replaying journal, id: {}, data: {},",
+                        replayedJournalId.get() + 1, journalEntityToReadableString(entity), e);
                 throw e;
             }
 
@@ -2247,6 +2246,19 @@ protected boolean replayJournalInner(JournalCursor cursor, boolean flowControl)
         return false;
     }
 
+    private String journalEntityToReadableString(JournalEntity entity) {
+        if (entity == null) {
+            return null;
+        }
+        Writable data = entity.getData();
+        try {
+            return GsonUtils.GSON.toJson(data);
+        } catch (Exception e) {
+            // In older version, data may not be json, here we just return the binary data.
+            return data.toString();
+        }
+    }
+
     private boolean canSkipBadReplayedJournal() {
         try {
             for (String idStr : Config.metadata_journal_skip_bad_journal_ids.split(",")) {