Merge pull request #593 from JuliaRobotics/feat/20Q3/in_mem_store

add InMemoryBlobStore, more tests, and small bugfixes

src/CloudGraphsDFG/services/CloudGraphsDFG.jl

@@ -239,9 +239,9 @@ function getVariable(dfg::CloudGraphsDFG, label::Union{Symbol, String})
     for solveKey in listVariableSolverData(dfg, label)
         variable.solverDataDict[solveKey] = getVariableSolverData(dfg, label, solveKey)
     end
-    dataDict = getDataEntries(dfg, label)
-    for (k,v) in dataDict
-        variable.dataDict[k] = v
+    dataSet = getDataEntries(dfg, label)
+    for v in dataSet
+        variable.dataDict[v.label] = v
     end
 
     return variable
@@ -258,7 +258,7 @@ function mergeVariableData!(dfg::CloudGraphsDFG, sourceVariable::DFGVariable; cu
         updateVariableSolverData!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
     end
     for (k,v) in sourceVariable.dataDict
-        updateDatEntry!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
+        updateDataEntry!(dfg, getLabel(sourceVariable), v, currentTransaction=currentTransaction)
     end
     return sourceVariable
 end
@@ -643,11 +643,9 @@ end
 function getDataEntries(dfg::CloudGraphsDFG, label::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
     entries = Dict{Symbol, BlobStoreEntry}()
     # TODO: Optimize if necessary.
-    for key in listDataEntries(dfg, label, currentTransaction=currentTransaction)
-        entry = getDataEntry(dfg, label, key, currentTransaction=currentTransaction)
-        entries[entry.label] = entry
-    end
-    return entries
+    delist = listDataEntries(dfg, label, currentTransaction=currentTransaction)
+    return getDataEntry.(dfg, label, delist; currentTransaction=currentTransaction)
+
 end
 
 function listDataEntries(dfg::CloudGraphsDFG, label::Symbol; currentTransaction::Union{Nothing, Neo4j.Transaction}=nothing)
@@ -665,6 +663,10 @@ function getDataEntry(dfg::CloudGraphsDFG, label::Symbol, key::Symbol; currentTr
         BlobStoreEntry, 
         key; 
         currentTransaction=currentTransaction)
+
+    #FIXME
+    properties["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(properties["createdTimestamp"])
+
     return Unmarshal.unmarshal(
         BlobStoreEntry,
         properties)
@@ -681,6 +683,10 @@ function addDataEntry!(dfg::CloudGraphsDFG, label::Symbol, bde::BlobStoreEntry;
         bde,
         :DATA,
         currentTransaction=currentTransaction)
+
+    #FIXME
+    packed["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(packed["createdTimestamp"])
+
     return Unmarshal.unmarshal(
             BlobStoreEntry,
             packed)
@@ -697,6 +703,10 @@ function updateDataEntry!(dfg::CloudGraphsDFG, label::Symbol,  bde::BlobStoreEnt
         bde,
         :DATA,
         currentTransaction=currentTransaction)
+
+    #FIXME
+    packed["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(packed["createdTimestamp"])
+
     return Unmarshal.unmarshal(
             BlobStoreEntry,
             packed)
@@ -710,6 +720,10 @@ function deleteDataEntry!(dfg::CloudGraphsDFG, label::Symbol, key::Symbol; curre
         _getLabelsForType(dfg, BlobStoreEntry, parentKey=label),
         key,
         currentTransaction=currentTransaction)
+
+    #FIXME
+    props["createdTimestamp"] = DistributedFactorGraphs.getStandardZDTString(props["createdTimestamp"])
+
     return Unmarshal.unmarshal(
         BlobStoreEntry,
         props)

src/DataBlobs/services/BlobStores.jl

@@ -109,7 +109,7 @@ addData!(dfg::AbstractDFG, blobstorekey::Symbol, label::Symbol, key::Symbol, blo
                                                            kwargs...)
 
 function addData!(dfg::AbstractDFG, blobstore::AbstractBlobStore, label::Symbol, key::Symbol,
-                  blob::Vector{UInt8}, timestamp=now(localzone()); description="", mimeType = "", id::UUID = uuid4(), hashfunction = sha256)
+                  blob::Vector{UInt8}, timestamp=now(localzone()); description="", mimeType = "application/octet-stream", id::UUID = uuid4(), hashfunction = sha256)
 
 
     entry = BlobStoreEntry(key, id, blobstore.key, bytes2hex(hashfunction(blob)),
@@ -196,9 +196,43 @@ end
 function deleteDataBlob!(store::FolderStore{T}, entry::BlobStoreEntry) where T
     blobfilename = joinpath(store.folder,"$(entry.id).dat")
     entryfilename = joinpath(store.folder,"$(entry.id).json")
-
+    
     data = getDataBlob(store, entry)
     rm(blobfilename)
     rm(entryfilename)
     return data
 end
+
+##==============================================================================
+## InMemoryBlobStore
+##==============================================================================
+export InMemoryBlobStore
+struct InMemoryBlobStore{T} <: AbstractBlobStore{T}
+    key::Symbol
+    blobs::Dict{UUID, T}
+end
+
+InMemoryBlobStore{T}(storeKey::Symbol) where T = InMemoryBlobStore{Vector{UInt8}}(storeKey, Dict{UUID, T}())
+InMemoryBlobStore(storeKey::Symbol=:default_inmemory_store) = InMemoryBlobStore{Vector{UInt8}}(storeKey)
+
+function getDataBlob(store::InMemoryBlobStore{T}, entry::BlobStoreEntry) where T
+    return store.blobs[entry.id]
+end
+
+function addDataBlob!(store::InMemoryBlobStore{T}, entry::BlobStoreEntry, data::T) where T
+    if haskey(store.blobs, entry.id)
+        error("Key '$(entry.id)' blob already exists.")
+    end
+    return store.blobs[entry.id] = data
+end
+
+function updateDataBlob!(store::InMemoryBlobStore{T},  entry::BlobStoreEntry, data::T) where T
+    if haskey(store.blobs, entry.id)
+        @warn "Key '$(entry.id)' doesn't exist."
+    end
+    return store.blobs[entry.id] = data
+end
+
+function deleteDataBlob!(store::InMemoryBlobStore{T}, entry::BlobStoreEntry) where T
+    return pop!(store.blobs, entry.id)
+end

test/consol_DataEntryBlobTests.jl

@@ -97,6 +97,29 @@ dfs = FolderStore("/tmp/defaultfolderstore")
 @test dfs.key == :default_folder_store
 @test dfs isa FolderStore{Vector{UInt8}}
 
+##==============================================================================
+## InMemoryBlobStore
+##==============================================================================
+
+# Create a data store and add it to DFG
+ds = InMemoryBlobStore()
+addBlobStore!(dfg, ds)
+
+ade,adb = addData!(dfg, :default_inmemory_store, :x1, :random, dataset1)
+gde,gdb = getData(dfg, :x1, :random)
+dde,ddb = deleteData!(dfg, :x1, :random)
+
+@test ade == gde == dde
+@test adb == gdb == ddb
+
+ade2,adb2 = addData!(dfg, :x2, deepcopy(ade), dataset1)
+# ade3,adb3 = updateData!(dfg, :x2, deepcopy(ade), dataset1)
+
+@test ade == ade2# == ade3
+@test adb == adb2# == adb3
+
+deleteData!(dfg, :x2, :random)
+
 ##==============================================================================
 ## Unimplemented store
 ##==============================================================================

test/interfaceTests.jl

@@ -115,12 +115,10 @@ end
 @testset "Data Entries and Blobs" begin
     if typeof(fg1) <: InMemoryDFGTypes
         DataEntriesTestBlock!(fg1, var2)
-    else
-        @test_skip DataEntriesTestBlock!(fg1, var2)
     end
-
-    # New data blob API
-    blobsTestBlock!(fg1, var2)
+    @testset "Data blob tests" begin
+        blobsStoresTestBlock!(fg1)
+    end
 end
 
 @testset "TODO Sorteer groep" begin

test/testBlocks.jl

@@ -865,46 +865,91 @@ function  DataEntriesTestBlock!(fg, v2)
     #delete from dfg
     @test deleteDataEntry!(fg, :a, :key2) == de2_update
     @test listDataEntries(v1) == Symbol[]
+    deleteDataEntry!(fg, :b, :key2)
 end
 
-function blobsTestBlock!(fg, v1)
-    @testset "Data blob tests" begin
-        # Blobstore functions
-        fs = FolderStore("/tmp/$(string(uuid4())[1:8])")
-        # Adding
-        addBlobStore!(fg, fs)
-        # Listing
-        @test listBlobStores(fg) == [fs.key]
-        # Getting
-        @test getBlobStore(fg, fs.key) == fs
-        # Deleting
-        @test deleteBlobStore!(fg, fs.key) == fs
-        # Updating
-        updateBlobStore!(fg, fs)
-        @test listBlobStores(fg) == [fs.key]
-        # Emptying
-        emptyBlobStore!(fg)
-        @test listBlobStores(fg) == []
-        # Add it back
-        addBlobStore!(fg, fs)
-
-        # Data functions
-        testData = rand(UInt8, 50)
-        # Adding 
-        newData = addData!(fg, fs.key, getLabel(v1), :testing, testData)
-        # Listing
-        @test :testing in listDataEntries(fg, getLabel(v1))
-        # Getting
-        data = getData(fg, fs, getLabel(v1), :testing)
-        @test data[1].hash == newData[1].hash
-        @test data[2] == newData[2]
-        # Updating
-        updateData = updateData!(fg, fs, getLabel(v1), newData[1], rand(UInt8, 50))
-        @test updateData[1].hash != data[1].hash
-        @test updateData[2] != data[2]
-        # Deleting
-        retData = deleteData!(fg, getLabel(v1), :testing)
-    end
+function blobsStoresTestBlock!(fg)
+
+    de1 = BlobStoreEntry(:label1,uuid4(), :store1, "AAAA","origin1","description1","mimetype1",now(localzone()))
+    de2 = BlobStoreEntry(:label2,uuid4(), :store2, "FFFF","origin2","description2","mimetype2",ZonedDateTime("2020-08-12T12:00:00.000+00:00"))
+    de2_update = BlobStoreEntry(:label2,uuid4(), :store2, "0123","origin2","description2","mimetype2",ZonedDateTime("2020-08-12T12:00:01.000+00:00"))
+    @test getLabel(de1) == de1.label
+    @test getId(de1) == de1.id
+    @test getHash(de1) == hex2bytes(de1.hash)
+    @test getCreatedTimestamp(de1) == de1.createdTimestamp
+
+    #add
+    var1 = getVariable(fg, :a)
+    @test addDataEntry!(var1, de1) == de1
+    updateVariable!(fg, var1)
+    @test addDataEntry!(fg, :a, de2) == de2
+    @test_throws ErrorException addDataEntry!(var1, de1)
+    @test de2 in getDataEntries(fg, var1.label)
+
+    #get
+    @test deepcopy(de1) == getDataEntry(var1, :label1)
+    @test deepcopy(de2) == getDataEntry(fg, :a, :label2)
+    @test_throws ErrorException getDataEntry(v2, :label1)
+    @test_throws ErrorException getDataEntry(fg, :b, :label1)
+
+    #update
+    @test updateDataEntry!(fg, :a, de2_update) == de2_update
+    @test deepcopy(de2_update) == getDataEntry(fg, :a, :label2)
+    @test @test_logs (:warn, r"does not exist") updateDataEntry!(fg, :b, de2_update) == de2_update
+
+    #list
+    entries = getDataEntries(fg, :a)
+    @test length(entries) == 2
+    @test issetequal(map(e->e.label, entries), [:label1, :label2])
+    @test length(getDataEntries(fg, :b)) == 1
+
+    @test issetequal(listDataEntries(fg, :a), [:label1, :label2])
+    @test listDataEntries(fg, :b) == Symbol[:label2]
+
+    #delete
+    @test deleteDataEntry!(fg, var1.label, de1.label) == de1
+    @test listDataEntries(fg, var1.label) == Symbol[:label2]
+    #delete from dfg
+    @test deleteDataEntry!(fg, :a, :label2) == de2_update
+    var1 = getVariable(fg, :a)
+    @test listDataEntries(var1) == Symbol[]
+
+    # Blobstore functions
+    fs = FolderStore("/tmp/$(string(uuid4())[1:8])")
+    # Adding
+    addBlobStore!(fg, fs)
+    # Listing
+    @test listBlobStores(fg) == [fs.key]
+    # Getting
+    @test getBlobStore(fg, fs.key) == fs
+    # Deleting
+    @test deleteBlobStore!(fg, fs.key) == fs
+    # Updating
+    updateBlobStore!(fg, fs)
+    @test listBlobStores(fg) == [fs.key]
+    # Emptying
+    emptyBlobStore!(fg)
+    @test listBlobStores(fg) == []
+    # Add it back
+    addBlobStore!(fg, fs)
+
+    # Data functions
+    testData = rand(UInt8, 50)
+    # Adding 
+    newData = addData!(fg, fs.key, :a, :testing, testData)
+    # Listing
+    @test :testing in listDataEntries(fg, :a)
+    # Getting
+    data = getData(fg, fs, :a, :testing)
+    @test data[1].hash == newData[1].hash
+    @test data[2] == newData[2]
+    # Updating
+    updateData = updateData!(fg, fs, :a, newData[1], rand(UInt8, 50))
+    @test updateData[1].hash != data[1].hash
+    @test updateData[2] != data[2]
+    # Deleting
+    retData = deleteData!(fg, :a, :testing)
+
 end