Adding op_get_list_status_result_size metrics to branch-2.2.x (#1226)

* adding list_status_result_size * Adding Integration Test
GoogleCloudDataproc · Jul 18, 2024 · 158994c · 158994c
1 parent 080cd79
commit 158994c
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 0 deletions.
diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java
@@ -75,6 +75,8 @@ public enum GhfsStatistic {
   INVOCATION_HSYNC("op_hsync", "Calls of hsync()", TYPE_DURATION),
   INVOCATION_LIST_FILES("op_list_files", "Calls of listFiles()", TYPE_COUNTER),
   INVOCATION_LIST_STATUS("op_list_status", "Calls of listStatus()", TYPE_DURATION),
+  INVOCATION_LIST_STATUS_RESULT_SIZE(
+      "op_get_list_status_result_size", "Number of files returned from list call", TYPE_COUNTER),
   INVOCATION_MKDIRS("op_mkdirs", "Calls of mkdirs()", TYPE_DURATION),
   INVOCATION_OPEN("op_open", "Calls of open()", TYPE_DURATION),
   INVOCATION_RENAME("op_rename", "Calls of rename()", TYPE_DURATION),

diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemBase.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemBase.java
@@ -975,6 +975,8 @@ public FileStatus[] listStatus(Path hadoopPath) throws IOException {
                             hadoopPath, gcsPath))
                     .initCause(fnfe);
           }
+          storageStatistics.incrementCounter(
+              GhfsStatistic.INVOCATION_LIST_STATUS_RESULT_SIZE, status.size());
           return status.toArray(new FileStatus[0]);
         });
   }

diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java
@@ -20,6 +20,7 @@
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_DELETE;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_GET_FILE_STATUS;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_GLOB_STATUS;
+import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_STATUS_RESULT_SIZE;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_MKDIRS;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_OPEN;
 import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_RENAME;
@@ -1635,6 +1636,43 @@ public void testHnBucketDeleteOperationOnNonExistingFolder() throws Exception {
     }
   }
 
+  @Test
+  public void statistics_check_get_list_status_result_size() throws IOException {
+
+    // first filesystem object
+    GoogleHadoopFileSystem myGhfs1 = createInMemoryGoogleHadoopFileSystem();
+    StorageStatistics stats = TestUtils.getStorageStatistics();
+
+    Path testRoot = new Path("/directory1/");
+
+    // first file created in ghfs1
+    myGhfs1.mkdirs(testRoot);
+    FSDataOutputStream fout = myGhfs1.create(new Path("/directory1/file1"));
+    fout.writeBytes("data");
+    fout.close();
+    myGhfs1.listStatus(testRoot);
+    TestUtils.verifyCounter((GhfsStorageStatistics) stats, INVOCATION_LIST_STATUS_RESULT_SIZE, 1);
+    assertThat(myGhfs1.delete(testRoot, /* recursive= */ true)).isTrue();
+
+    // create another FileSystem Object
+    GoogleHadoopFileSystem myGhfs2 = createInMemoryGoogleHadoopFileSystem();
+
+    // first file created in ghfs2
+    fout = myGhfs2.create(new Path("/directory1/file1"));
+    fout.writeBytes("data");
+    fout.close();
+
+    // first file created in ghfs3
+    fout = myGhfs2.create(new Path("/directory1/file2"));
+    fout.writeBytes("data");
+    fout.close();
+
+    myGhfs2.listStatus(testRoot);
+    TestUtils.verifyCounter((GhfsStorageStatistics) stats, INVOCATION_LIST_STATUS_RESULT_SIZE, 3);
+
+    assertThat(myGhfs2.delete(testRoot, /* recursive= */ true)).isTrue();
+  }
+
   private void createFile(GoogleHadoopFileSystem googleHadoopFileSystem, Path path)
       throws Exception {
     try (FSDataOutputStream fout = googleHadoopFileSystem.create(path)) {