Skip to content

Commit

Permalink
Replaces the old xml parser with a new, faster one (#861)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebozduman
ebozduman authored Nov 5, 2020
1 parent ffd3fdb commit fdbc801
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 82 deletions.
8 changes: 4 additions & 4 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@
"querystring": "0.2.0",
"through2": "^3.0.1",
"xml": "^1.0.0",
"xml2js": "^0.4.15"
"fast-xml-parser": "^3.17.4"
},
"devDependencies": {
"@babel/core": "^7.5.5",
"@babel/preset-env": "^7.5.5",
"@babel/core": "^7.11.0",
"@babel/preset-env": "^7.11.0",
"babelify": "^10.0.0",
"browserify": "^16.3.0",
"browserify": "^16.5.1",
"chai": "^4.2.0",
"eslint": "^6.1.0",
"gulp": "^4.0.2",
Expand Down
4 changes: 2 additions & 2 deletions src/main/minio.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import Https from 'https'
import Stream from 'stream'
import BlockStream2 from 'block-stream2'
import Xml from 'xml'
import xml2js from 'xml2js'
import fxp from 'fast-xml-parser'
import async from 'async'
import querystring from 'querystring'
import mkdirp from 'mkdirp'
Expand Down Expand Up @@ -2066,7 +2066,7 @@ export class Client {
}
var method = 'PUT'
var query = 'notification'
var builder = new xml2js.Builder({rootName:'NotificationConfiguration', renderOpts:{'pretty':false}, headless:true})
var builder = new fxp.Builder({rootName:'NotificationConfiguration', renderOpts:{'pretty':false}, headless:true})
var payload = builder.buildObject(config)
this.makeRequest({method, bucketName, query}, payload, 200, '', false, cb)
}
Expand Down
187 changes: 111 additions & 76 deletions src/main/xml-parsers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,35 @@
* limitations under the License.
*/

import xml2js from 'xml2js'
import fxp from 'fast-xml-parser'
import _ from 'lodash'
import * as errors from './errors.js'

var options = { // options passed to xml2js parser
explicitRoot: false, // return the root node in the resulting object?
ignoreAttrs: true, // ignore attributes, only create text nodes
}

var parseXml = (xml) => {
var result = null
var error = null

var parser = new xml2js.Parser(options)
parser.parseString(xml, function (e, r) {
error = e
result = r
})

if (error) {
throw new Error('XML parse error')
result = fxp.parse(xml)
if (result.Error) {
throw result.Error
}

return result
}

// Parse XML and return information as Javascript types

// parse error XML response
export function parseError(xml, headerInfo) {
var xmlError = {}
var xmlobj = parseXml(xml)
var message
_.each(xmlobj, (n, key) => {
if (key === 'Message') {
message = xmlobj[key][0]
return
}
xmlError[key.toLowerCase()] = xmlobj[key][0]
})
var e = new errors.S3Error(message)
_.each(xmlError, (value, key) => {
e[key] = value
var xmlErr = {}
var xmlObj = fxp.parse(xml)
if (xmlObj.Error) {
xmlErr = xmlObj.Error
}

var e = new errors.S3Error()
_.each(xmlErr, (value, key) => {
e[key.toLowerCase()] = value
})

_.each(headerInfo, (value, key) => {
e[key] = value
})
Expand All @@ -69,11 +55,16 @@ export function parseCopyObject(xml) {
etag: "",
lastModified: ""
}

var xmlobj = parseXml(xml)
if (xmlobj.ETag) result.etag = xmlobj.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
if (!xmlobj.CopyObjectResult) {
throw new errors.InvalidXMLError('Missing tag: "CopyObjectResult"')
}
xmlobj = xmlobj.CopyObjectResult
if (xmlobj.ETag) result.etag = xmlobj.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/^"$/g, '')
if (xmlobj.LastModified) result.lastModified = new Date(xmlobj.LastModified[0])
.replace(/^"/g, '').replace(/"$/g, '')
if (xmlobj.LastModified) result.lastModified = new Date(xmlobj.LastModified)

return result
}
Expand All @@ -85,32 +76,54 @@ export function parseListMultipart(xml) {
prefixes: [],
isTruncated: false
}
var xmlobj = parseXml(xml)
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
if (xmlobj.NextKeyMarker) result.nextKeyMarker = xmlobj.NextKeyMarker[0]

var xmlobj = parseXml(xml)

if (!xmlobj.ListMultipartUploadsResult) {
throw new errors.InvalidXMLError('Missing tag: "ListMultipartUploadsResult"')
}
xmlobj = xmlobj.ListMultipartUploadsResult
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
if (xmlobj.NextKeyMarker) result.nextKeyMarker = xmlobj.NextKeyMarker
if (xmlobj.NextUploadIdMarker) result.nextUploadIdMarker = xmlobj.NextUploadIdMarker[0]
if (xmlobj.CommonPrefixes) xmlobj.CommonPrefixes.forEach(prefix => {
result.prefixes.push({prefix: prefix[0]})
})
if (xmlobj.Upload) xmlobj.Upload.forEach(upload => {
result.uploads.push({
key: upload.Key[0],
uploadId: upload.UploadId[0],
initiated: new Date(upload.Initiated[0])
if (xmlobj.Upload) {
if (!Array.isArray(xmlobj.Upload)) {
xmlobj.Upload = Array(xmlobj.Upload)
}
xmlobj.Upload.forEach(upload => {
var key = upload.Key
var uploadId = upload.UploadId
var initiator = {id: upload.Initiator.ID, displayName: upload.Initiator.DisplayName}
var owner = {id: upload.Owner.ID, displayName: upload.Owner.DisplayName}
var storageClass = upload.StorageClass
var initiated = new Date(upload.Initiated)
result.uploads.push({key, uploadId, initiator, owner, storageClass, initiated})
})
})
}
return result
}

// parse XML response to list all the owned buckets
export function parseListBucket(xml) {
var result = []
var xmlobj = parseXml(xml)

if (!xmlobj.ListAllMyBucketsResult) {
throw new errors.InvalidXMLError('Missing tag: "ListAllMyBucketsResult"')
}
xmlobj = xmlobj.ListAllMyBucketsResult

if (xmlobj.Buckets) {
if (xmlobj.Buckets[0].Bucket) {
xmlobj.Buckets[0].Bucket.forEach(bucket => {
var name = bucket.Name[0]
var creationDate = new Date(bucket.CreationDate[0])
if (xmlobj.Buckets.Bucket) {
if (!Array.isArray(xmlobj.Buckets.Bucket)) {
xmlobj.Buckets.Bucket = Array(xmlobj.Buckets.Bucket)
}
xmlobj.Buckets.Bucket.forEach(bucket => {
var name = bucket.Name
var creationDate = new Date(bucket.CreationDate)
result.push({name, creationDate})
})
}
Expand Down Expand Up @@ -149,7 +162,6 @@ export function parseBucketNotification(xml) {
}

var xmlobj = parseXml(xml)

// Parse all topic configurations in the xml
if (xmlobj.TopicConfiguration) {
xmlobj.TopicConfiguration.forEach(config => {
Expand Down Expand Up @@ -186,7 +198,8 @@ export function parseBucketNotification(xml) {

// parse XML response for bucket region
export function parseBucketRegion(xml) {
return parseXml(xml)
// return region information
return parseXml(xml).LocationConstraint
}

// parse XML response for list parts of an in progress multipart upload
Expand All @@ -197,15 +210,15 @@ export function parseListParts(xml) {
parts: [],
marker: undefined
}
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
if (xmlobj.NextPartNumberMarker) result.marker = +xmlobj.NextPartNumberMarker[0]
if (xmlobj.Part) {
xmlobj.Part.forEach(p => {
var part = +p.PartNumber[0]
var lastModified = new Date(p.LastModified[0])
var etag = p.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
var lastModified = new Date(p.LastModified)
var etag = p.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/^"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
result.parts.push({part, lastModified, etag})
})
}
Expand All @@ -214,21 +227,21 @@ export function parseListParts(xml) {

// parse XML response when a new multipart upload is initiated
export function parseInitiateMultipart(xml) {
var xmlobj = parseXml(xml)
if (xmlobj.UploadId) return xmlobj.UploadId[0]
throw new errors.InvalidXMLError('UploadId missing in XML')
var xmlobj = parseXml(xml).InitiateMultipartUploadResult
if (xmlobj.UploadId) return xmlobj.UploadId
throw new errors.InvalidXMLError('Missing tag: "UploadId"')
}

// parse XML response when a multipart upload is completed
export function parseCompleteMultipart(xml) {
var xmlobj = parseXml(xml)
var xmlobj = parseXml(xml).CompleteMultipartUploadResult
if (xmlobj.Location) {
var location = xmlobj.Location[0]
var bucket = xmlobj.Bucket[0]
var key = xmlobj.Key[0]
var etag = xmlobj.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
var key = xmlobj.Key
var etag = xmlobj.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/^"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')

return {location, bucket, key, etag}
}
Expand All @@ -248,15 +261,23 @@ export function parseListObjects(xml) {
}
var nextMarker
var xmlobj = parseXml(xml)
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true

if (!xmlobj.ListBucketResult) {
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
}
xmlobj = xmlobj.ListBucketResult
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') xmlobj.isTruncated = true
if (xmlobj.Contents) {
if (!Array.isArray(xmlobj.Contents)) {
xmlobj.Contents = Array(xmlobj.Contents)
}
xmlobj.Contents.forEach(content => {
var name = content.Key[0]
var lastModified = new Date(content.LastModified[0])
var etag = content.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/^"$/g, '')
var size = +content.Size[0]
.replace(/^"/g, '').replace(/"$/g, '')
var size = +content.Size
result.objects.push({name, lastModified, etag, size})
nextMarker = name
})
Expand All @@ -281,17 +302,24 @@ export function parseListObjectsV2(xml) {
isTruncated: false
}
var xmlobj = parseXml(xml)
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
if (!xmlobj.ListBucketResult) {
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
}
xmlobj = xmlobj.ListBucketResult
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
if (xmlobj.NextContinuationToken) result.nextContinuationToken = xmlobj.NextContinuationToken[0]

if (xmlobj.Contents) {
if (!Array.isArray(xmlobj.Contents)) {
xmlobj.Contents = Array(xmlobj.Contents)
}
xmlobj.Contents.forEach(content => {
var name = content.Key[0]
var lastModified = new Date(content.LastModified[0])
var etag = content.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/^"$/g, '')
var size = +content.Size[0]
.replace(/^"/g, '').replace(/"$/g, '')
var size = +content.Size
result.objects.push({name, lastModified, etag, size})
})
}
Expand All @@ -312,17 +340,24 @@ export function parseListObjectsV2WithMetadata(xml) {
isTruncated: false
}
var xmlobj = parseXml(xml)
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
if (!xmlobj.ListBucketResult) {
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
}
xmlobj = xmlobj.ListBucketResult
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
if (xmlobj.NextContinuationToken) result.nextContinuationToken = xmlobj.NextContinuationToken[0]

if (xmlobj.Contents) {
if (!Array.isArray(xmlobj.Contents)) {
xmlobj.Contents = Array(xmlobj.Contents)
}
xmlobj.Contents.forEach(content => {
var name = content.Key[0]
var lastModified = new Date(content.LastModified[0])
var etag = content.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
var name = content.Key
var lastModified = new Date(content.LastModified)
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/"$/g, '')
.replace(/^"/g, '').replace(/^"$/g, '')
var size = +content.Size[0]
.replace(/^"/g, '').replace(/"$/g, '')
var size = +content.Size
var metadata
if (content.UserMetadata != null) {
metadata = content.UserMetadata[0]
Expand Down
1 change: 1 addition & 0 deletions src/test/functional/functional-tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1068,6 +1068,7 @@ describe('functional tests', function() {
})

step(`listObjects(bucketName, prefix, recursive)_bucketName:${bucketName}, recursive:false_`, done => {
listArray = []
client.listObjects(bucketName, '', false)
.on('error', done)
.on('end', () => {
Expand Down

0 comments on commit fdbc801

Please sign in to comment.