Have 'firebase firestore:delete' retry on bandwidth RESOURCE_EXHAUSTE…

…D errors (#7863)

* Add retry mechanisms for when firestore deletes are failing due to bandwidth limits

* Alias Commit

* Run the formatter

CHANGELOG.md

@@ -1 +1,2 @@
 - Add support for deploying new blocking triggers. (#6384)
+- Have the firestore:delete command retry on bandwidth exceeded errors. (#7845)

src/firestore/delete.ts

@@ -424,6 +424,24 @@ export class FirestoreDelete {
               this.setDeleteBatchSize(newBatchSize);
             }
 
+            // Retry this batch
+            queue.unshift(...toDelete);
+          } else if (
+            e.status === 429 &&
+            this.deleteBatchSize >= 10 &&
+            e.message.includes("database has exceeded their maximum bandwidth")
+          ) {
+            logger.debug("Database has exceeded maximum write bandwidth", e);
+            const newBatchSize = Math.floor(toDelete.length / 2);
+
+            if (newBatchSize < this.deleteBatchSize) {
+              utils.logLabeledWarning(
+                "firestore",
+                `delete rate exceeding maximum bandwidth, reducing batch size from ${this.deleteBatchSize} to ${newBatchSize}`,
+              );
+              this.setDeleteBatchSize(newBatchSize);
+            }
+
             // Retry this batch
             queue.unshift(...toDelete);
           } else if (e.status >= 500 && e.status < 600) {

src/gcp/firestore.ts

@@ -153,7 +153,11 @@ export async function deleteDocuments(
   });
   const data = { writes };
 
-  const res = await apiClient.post<any, { writeResults: any[] }>(url, data);
+  const res = await apiClient.post<any, { writeResults: any[] }>(url, data, {
+    retries: 10,
+    retryCodes: [429, 409, 503],
+    retryMaxTimeout: 20 * 1000,
+  });
   return res.body.writeResults.length;
 }