From 3568969ae98ec0cf2a93e0a0643750a91d889c66 Mon Sep 17 00:00:00 2001 From: airosa Date: Mon, 2 Jan 2017 22:25:52 +0100 Subject: [PATCH] feat(-interface): Add support for measure dimension when returning data to Tableau Closes #59 --- src/util/wdc-interface.coffee | 133 +++++++++++++++++++++------- test/fixtures/ECB_EXR1_USD_GBP.json | 1 + test/util/wdc-interface.test.coffee | 65 ++++++++++++++ 3 files changed, 169 insertions(+), 30 deletions(-) create mode 100644 test/fixtures/ECB_EXR1_USD_GBP.json diff --git a/src/util/wdc-interface.coffee b/src/util/wdc-interface.coffee index 0b37105..8fe47cd 100644 --- a/src/util/wdc-interface.coffee +++ b/src/util/wdc-interface.coffee @@ -4,12 +4,16 @@ chunkSize = 1000 fieldNames = [] fieldTypes = [] dataToReturn = [] +measureDimensionIndex = null #------------------------------------------------------------------------------- -submit = (url, dimension) -> - tableau.connectionData = JSON.stringify { url: url, dimension: dimension } - tableau.connectionName = 'SDMX-REST Connection' +submit = (url, index) -> + connectionData = { url: url, measureDimIndex: -1 } + connectionData.measureDimIndex = index unless isNaN(index) or + index is null or index is undefined + tableau.connectionData = JSON.stringify connectionData + tableau.connectionName = "SDMX-REST Connection" tableau.submit() #------------------------------------------------------------------------------- @@ -25,15 +29,15 @@ getComponents = (structure) -> if type is 'attributes' prefix = "M" else + if not c.keyPosition? then c.keyPosition = dimPos dimPos += 1 - if c.keyPosition? - keyPos = c.keyPosition + 1 # keyPosition is zero-based - else - keyPos = dimPos + keyPos = c.keyPosition + 1 # keyPosition is zero-based # assumes there will always be less than 99 dimensions prefix = if keyPos < 10 then "0#{keyPos}" else "#{keyPos}" c.name = "#{prefix} - #{c.name}" c.type = type + c.isDimension = (type is 'dimensions') + c.isAttribute = (type is 'attributes') c.level = level c @@ -46,21 +50,21 @@ formatDT = (d) -> (new Date(d)).toISOString()[0..18].replace('T',' ') processResponse = (response) -> msg = JSON.parse response components = getComponents msg.structure + mDimPos = -1 - columnNames = for c in components - switch + for c, i in components + c.index = i + if 0 <= measureDimensionIndex + mDimPos = i if +c.keyPosition is measureDimensionIndex + c.columnNames = switch when c.values[0]?.start? then [c.name, "#{c.name} start", "#{c.name} end"] when c.values[0]?.id? then [c.name, "#{c.name} ID"] else c.name - - columnTypes = for c in components - switch + c.columnTypes = switch when c.values[0]?.start? then ['string', 'datetime', 'datetime'] when c.values[0]?.id? then ['string', 'string'] else 'string' - - columnValues = for c in components - switch + c.columnValues = switch when c.values[0]?.start? ([v.name, formatDT(v.start), formatDT(v.end)] for v in c.values) when c.values[0]?.id? @@ -68,38 +72,107 @@ processResponse = (response) -> else ([v.name] for v in c.values) - columnNames.push 'Observation Value' - columnTypes.push 'float' - - fieldNames = [].concat.apply [], columnNames - fieldTypes = [].concat.apply [], columnTypes - dataToReturn = getTableData msg, columnValues - - -getTableData = (msg, columnValues) -> + fieldNames = [].concat.apply [], + (c.columnNames for c, i in components when i isnt mDimPos) + fieldTypes = [].concat.apply [], + (c.columnTypes for c, i in components when i isnt mDimPos) + + if mDimPos < 0 + fieldNames.push 'Observation Value' + fieldTypes.push 'float' + dataToReturn = getTableData msg, components + else + for v in components[mDimPos].values + fieldNames.push v.name + fieldTypes.push 'float' + dataToReturn = getTableDataPivoted msg, components, mDimPos + +# returns observations in a flattened array: +# [ds dims, ser dims, obs dims, ds attrs, ser attrs, obs attrs, obs value] +flattenObservations = (msg) -> dsDims = msg.structure.dimensions.dataSet?.map (d) -> 0 dsAttrs = msg.structure.attributes?.dataSet?.map (d) -> 0 dsDims ?= [] dsAttrs ?= [] results = [] - mapToValue = (i, j) -> if i? then columnValues[j][i] else i - for dataset in msg.dataSets for seriesKey, series of dataset.series seriesDims = dsDims.concat seriesKey.split(':').map( (d) -> +d ) seriesAttrs = dsAttrs.concat series.attributes for obsKey, obs of series.observations obsRow = seriesDims.concat(+obsKey, seriesAttrs, obs[1..]) - obsRow = [].concat.apply([], obsRow.map(mapToValue)) obsRow.push obs[0] results.push obsRow return results + +getTableData = (msg, components) -> + obsArray = flattenObservations msg + columnValues = (c.columnValues for c in components) + results = [] + + mapToValue = (i, j) -> + return i unless i? + return i if (columnValues.length - 1) < j + return columnValues[j][i] + + for obsRow in obsArray + results.push [].concat.apply([], obsRow.map(mapToValue)) + + return results + + +getTableDataPivoted = (msg, components, mDimPos) -> + obsArray = flattenObservations msg + obsMap = {} + results = [] + dimCount = 0 + dimCount += 1 for c in components when c.isDimension + lastDim = dimCount - 1 + firstAttr = lastDim + 1 + attrCount = 0 + attrCount += 1 for c in components when c.isAttribute + lastAttr = firstAttr + attrCount - 1 + lastValue = components[mDimPos].values.length - 1 + columnValues = (c.columnValues for c, i in components when i isnt mDimPos) + + # group observations by the key + for obs in obsArray + key = obs[0..lastDim] + key.splice(mDimPos, 1) # Remove the measure dimension from the key + key = key.join(':') # Join dimensions for the groupping key + obsMap[key] ?= (null for [0..lastValue]) # Reserve a slot for each measure + obsMap[key][obs[mDimPos]] = obs # Allocate the slot for the measure + + obsArray = [] + for key, value of obsMap + obs = key.split ':' + + obsValues = (null for [0..lastValue]) + for obs2, i in value when obs2? + attributes = obs2[firstAttr..lastAttr] # Assume attributes are same + obsValues[i] = obs2[obs2.length - 1] # Take the obs value and allocate + + obs = obs.concat attributes + obsArray.push obs.concat obsValues + + mapToValue = (i, j) -> + return i unless i? + return i if (columnValues.length - 1) < j + return columnValues[j][i] + + for obsRow in obsArray + results.push [].concat.apply([], obsRow.map(mapToValue)) + + return results + #------------------------------------------------------------------------------- -makeRequest = (url, callback) -> +makeRequest = (url, measureDimIndex, callback) -> + measureDimensionIndex = measureDimIndex + errorHandler = (error) -> console.log error tableau.abortWithError "#{error}" @@ -123,9 +196,9 @@ registerConnector = () -> connector = tableau.makeConnector() connector.getColumnHeaders = -> - connectionData = JSON.parse tableau.connectionData + connData = JSON.parse tableau.connectionData callback = () -> tableau.headersCallback fieldNames, fieldTypes - makeRequest connectionData.url, callback + makeRequest connData.url, connData.measureDimIndex, callback connector.getTableData = (lastRecordToken) -> if lastRecordToken.length is 0 diff --git a/test/fixtures/ECB_EXR1_USD_GBP.json b/test/fixtures/ECB_EXR1_USD_GBP.json new file mode 100644 index 0000000..f57f41a --- /dev/null +++ b/test/fixtures/ECB_EXR1_USD_GBP.json @@ -0,0 +1 @@ +{"header":{"id":"1ae59482-9b8d-48b1-b85c-0bab5f4da022","test":false,"prepared":"2016-10-28T22:49:33.000+02:00","sender":{"id":"ECB"}},"dataSets":[{"action":"Replace","validFrom":"2016-10-28T22:49:33.000+02:00","series":{"0:0:0:0:0":{"attributes":[null,null,0,null,null,null,null,null,null,null,0,null,0,null,0,0,0,0],"observations":{"0":[0.790493636363636,0,null,null,null],"1":[0.841057619047619,0,null,null,null],"2":[0.855208695652174,0,null,null,null],"3":[0.852276818181818,0,null,null,null]}},"0:1:0:0:0":{"attributes":[null,null,0,null,null,null,null,null,null,null,1,null,0,null,1,1,1,0],"observations":{"0":[1.122890909090909,0,null,null,null],"1":[1.106852380952381,0,null,null,null],"2":[1.121173913043479,0,null,null,null],"3":[1.121209090909091,0,null,null,null]}}}}],"structure":{"links":[{"title":"Exchange Rates","rel":"dataflow","href":"http://a-sdw-wsrest.ecb.europa.eu:80null/service/dataflow/ECB/EXR/1.0"}],"name":"Exchange Rates","dimensions":{"series":[{"id":"FREQ","name":"Frequency","values":[{"id":"M","name":"Monthly"}]},{"id":"CURRENCY","name":"Currency","values":[{"id":"GBP","name":"UK pound sterling"},{"id":"USD","name":"US dollar"}]},{"id":"CURRENCY_DENOM","name":"Currency denominator","values":[{"id":"EUR","name":"Euro"}]},{"id":"EXR_TYPE","name":"Exchange rate type","values":[{"id":"SP00","name":"Spot"}]},{"id":"EXR_SUFFIX","name":"Series variation - EXR context","values":[{"id":"A","name":"Average"}]}],"observation":[{"id":"TIME_PERIOD","name":"Time period or range","role":"time","values":[{"id":"2016-06","name":"2016-06","start":"2016-06-01T00:00:00.000+02:00","end":"2016-06-30T23:59:59.999+02:00"},{"id":"2016-07","name":"2016-07","start":"2016-07-01T00:00:00.000+02:00","end":"2016-07-31T23:59:59.999+02:00"},{"id":"2016-08","name":"2016-08","start":"2016-08-01T00:00:00.000+02:00","end":"2016-08-31T23:59:59.999+02:00"},{"id":"2016-09","name":"2016-09","start":"2016-09-01T00:00:00.000+02:00","end":"2016-09-30T23:59:59.999+02:00"}]}]},"attributes":{"series":[{"id":"TIME_FORMAT","name":"Time format code","values":[]},{"id":"BREAKS","name":"Breaks","values":[]},{"id":"COLLECTION","name":"Collection indicator","values":[{"id":"A","name":"Average of observations through period"}]},{"id":"DOM_SER_IDS","name":"Domestic series ids","values":[]},{"id":"PUBL_ECB","name":"Source publication (ECB only)","values":[]},{"id":"PUBL_MU","name":"Source publication (Euro area only)","values":[]},{"id":"PUBL_PUBLIC","name":"Source publication (public)","values":[]},{"id":"UNIT_INDEX_BASE","name":"Unit index base","values":[]},{"id":"COMPILATION","name":"Compilation","values":[]},{"id":"COVERAGE","name":"Coverage","values":[]},{"id":"DECIMALS","name":"Decimals","values":[{"id":"5","name":"Five"},{"id":"4","name":"Four"}]},{"id":"NAT_TITLE","name":"National language title","values":[]},{"id":"SOURCE_AGENCY","name":"Source agency","values":[{"id":"4F0","name":"European Central Bank (ECB)"}]},{"id":"SOURCE_PUB","name":"Publication source","values":[]},{"id":"TITLE","name":"Title","values":[{"name":"UK pound sterling/Euro"},{"name":"US dollar/Euro"}]},{"id":"TITLE_COMPL","name":"Title complement","values":[{"name":"ECB reference exchange rate, UK pound sterling/Euro, 2:15 pm (C.E.T.)"},{"name":"ECB reference exchange rate, US dollar/Euro, 2:15 pm (C.E.T.)"}]},{"id":"UNIT","name":"Unit","values":[{"id":"GBP","name":"UK pound sterling"},{"id":"USD","name":"US dollar"}]},{"id":"UNIT_MULT","name":"Unit multiplier","values":[{"id":"0","name":"Units"}]}],"observation":[{"id":"OBS_STATUS","name":"Observation status","values":[{"id":"A","name":"Normal value"}]},{"id":"OBS_CONF","name":"Observation confidentiality","values":[]},{"id":"OBS_PRE_BREAK","name":"Pre-break observation value","values":[]},{"id":"OBS_COM","name":"Observation comment","values":[]}]}}} \ No newline at end of file diff --git a/test/util/wdc-interface.test.coffee b/test/util/wdc-interface.test.coffee index 67580b2..17de0f4 100644 --- a/test/util/wdc-interface.test.coffee +++ b/test/util/wdc-interface.test.coffee @@ -14,12 +14,21 @@ describe 'Web Data Connector Interface', -> response = wdcInterface.response() response.fieldNames.should.be.an('array').with.lengthOf(42) response.fieldNames.should.include('02 - Currency') + response.fieldNames[0].should.equal('01 - Frequency') + response.fieldNames[1].should.equal('01 - Frequency ID') + response.fieldNames[41].should.equal('Observation Value') response.fieldTypes.should.be.an('array').with.lengthOf(42) response.fieldTypes.should.have.members(['string', 'datetime', 'float']) + response.fieldTypes[12].should.equal('datetime') + response.fieldTypes[41].should.equal('float') response.dataToReturn.should.be.an('array').with.lengthOf(4) + response.dataToReturn[0].should.be.an('array').with.lengthOf(42) + response.dataToReturn[0][0].should.equal('Monthly') + response.dataToReturn[0][41].should.equal(1.085965) done() wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/EXR', + undefined, checkResult return @@ -35,6 +44,62 @@ describe 'Web Data Connector Interface', -> done() wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/MNA', + undefined, + checkResult + + return + + it 'it supports setting measure dimension', (done) -> + query = nock('http://sdw-wsrest.ecb.europa.eu') + .get((uri) -> uri.indexOf('EXR') > -1) + .replyWithFile(200, __dirname + '/../fixtures/ECB_EXR.json') + + checkResult = () -> + response = wdcInterface.response() + response.fieldNames.should.be.an('array').with.lengthOf(40) + response.fieldNames.should.not.include('02 - Currency') + response.fieldNames[0].should.equal('01 - Frequency') + response.fieldNames[39].should.equal('US dollar') + response.fieldTypes.should.be.an('array').with.lengthOf(40) + response.fieldTypes.should.have.members(['string', 'datetime', 'float']) + response.fieldTypes[10].should.equal('datetime') + response.fieldTypes[39].should.equal('float') + response.dataToReturn.should.be.an('array').with.lengthOf(4) + response.dataToReturn[0].should.be.an('array').with.lengthOf(40) + response.dataToReturn[0][0].should.equal('Monthly') + response.dataToReturn[0][39].should.equal(1.085965) + done() + + wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/EXR', 1, + checkResult + + return + + it 'it supports measure dimension with multiple values', (done) -> + query = nock('http://sdw-wsrest.ecb.europa.eu') + .get((uri) -> uri.indexOf('EXR') > -1) + .replyWithFile(200, __dirname + '/../fixtures/ECB_EXR1_USD_GBP.json') + + checkResult = () -> + response = wdcInterface.response() + response.fieldNames.should.be.an('array').with.lengthOf(41) + response.fieldNames.should.not.include('02 - Currency') + response.fieldNames[0].should.equal('01 - Frequency') + response.fieldNames[39].should.equal('UK pound sterling') + response.fieldNames[40].should.equal('US dollar') + response.fieldTypes.should.be.an('array').with.lengthOf(41) + response.fieldTypes.should.have.members(['string', 'datetime', 'float']) + response.fieldTypes[10].should.equal('datetime') + response.fieldTypes[39].should.equal('float') + response.fieldTypes[40].should.equal('float') + response.dataToReturn.should.be.an('array').with.lengthOf(4) + response.dataToReturn[0].should.be.an('array').with.lengthOf(41) + response.dataToReturn[0][0].should.equal('Monthly') + response.dataToReturn[0][39].should.equal(0.790493636363636) + response.dataToReturn[0][40].should.equal(1.122890909090909) + done() + + wdcInterface.makeRequest 'http://sdw-wsrest.ecb.europa.eu/service/EXR', 1, checkResult return