Skip to content

Commit

Permalink
feat(gs): use parallel requests for s3#rmtree
Browse files Browse the repository at this point in the history
from 8-10 per sec to 110-125 per sec
  • Loading branch information
fbeauchamp committed Jan 20, 2022
1 parent 893a4b4 commit 5af2f5b
Showing 1 changed file with 14 additions and 6 deletions.
20 changes: 14 additions & 6 deletions @xen-orchestra/fs/src/s3.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { decorateWith } from '@vates/decorate-with'
import { parse } from 'xo-remote-parser'

import RemoteHandlerAbstract from './abstract'
import { asyncEach } from '@vates/async-each'

// endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html

Expand Down Expand Up @@ -278,14 +279,21 @@ export default class S3Handler extends RemoteHandlerAbstract {
ContinuationToken: NextContinuationToken,
})
NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined
for (const { Key } of result.Contents) {
await asyncEach(
result.Contents,
async ({Key})=>{
// _unlink will add the prefix, but Key contains everything
// also we don't need to check if we delete a directory, since the list only return files
await this._s3.deleteObject({
Bucket: this._bucket,
Key,
})
}
await this._s3.deleteObject({
Bucket: this._bucket,
Key,
})
},
{
concurrency: 16
}
)

} while (NextContinuationToken !== undefined)
}

Expand Down

0 comments on commit 5af2f5b

Please sign in to comment.