diff --git a/CHANGELOG.md b/CHANGELOG.md index e2b9903f655..2c2316f58c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1 +1,2 @@ - Add support for deploying new blocking triggers. (#6384) +- Have the firestore:delete command retry on bandwidth exceeded errors. (#7845) diff --git a/src/firestore/delete.ts b/src/firestore/delete.ts index aadaa9584af..d739fa222ac 100644 --- a/src/firestore/delete.ts +++ b/src/firestore/delete.ts @@ -424,6 +424,24 @@ export class FirestoreDelete { this.setDeleteBatchSize(newBatchSize); } + // Retry this batch + queue.unshift(...toDelete); + } else if ( + e.status === 429 && + this.deleteBatchSize >= 10 && + e.message.includes("database has exceeded their maximum bandwidth") + ) { + logger.debug("Database has exceeded maximum write bandwidth", e); + const newBatchSize = Math.floor(toDelete.length / 2); + + if (newBatchSize < this.deleteBatchSize) { + utils.logLabeledWarning( + "firestore", + `delete rate exceeding maximum bandwidth, reducing batch size from ${this.deleteBatchSize} to ${newBatchSize}`, + ); + this.setDeleteBatchSize(newBatchSize); + } + // Retry this batch queue.unshift(...toDelete); } else if (e.status >= 500 && e.status < 600) { diff --git a/src/gcp/firestore.ts b/src/gcp/firestore.ts index eb500f38e83..d342c35e914 100644 --- a/src/gcp/firestore.ts +++ b/src/gcp/firestore.ts @@ -153,7 +153,11 @@ export async function deleteDocuments( }); const data = { writes }; - const res = await apiClient.post(url, data); + const res = await apiClient.post(url, data, { + retries: 10, + retryCodes: [429, 409, 503], + retryMaxTimeout: 20 * 1000, + }); return res.body.writeResults.length; }