Merge pull request #477 from JuliaDataCubes/fg/open_mfdataset2

Fix open_mfdataset

Project.toml

@@ -1,7 +1,7 @@
 name = "YAXArrays"
 uuid = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 authors = ["Fabian Gans <[email protected]>"]
-version = "0.5.14"
+version = "0.5.15"
 
 [deps]
 CFTime = "179af706-886a-5703-950a-314cd64e0468"

src/DatasetAPI/Datasets.jl

@@ -8,8 +8,8 @@ using IntervalSets: Interval, (..)
 using CFTime: timedecode, timeencode, DateTimeNoLeap, DateTime360Day, DateTimeAllLeap
 using YAXArrayBase
 using YAXArrayBase: iscontdimval, add_var
-using DiskArrayTools: CFDiskArray, ConcatDiskArray
-using DiskArrays: DiskArrays, GridChunks
+using DiskArrayTools: CFDiskArray, diskstack
+using DiskArrays: DiskArrays, GridChunks, ConcatDiskArray
 using Glob: glob
 using DimensionalData: DimensionalData as DD
 
@@ -331,7 +331,17 @@ testrange(x::AbstractArray{<:TimeType}) = x
 
 testrange(x::AbstractArray{<:AbstractString}) = x
 
-_glob(x) = startswith(x, "/") ? glob(x[2:end], "/") : glob(x)
+
+# This is a bit unfortunate since it will disallow globbing hierarchies of directories, 
+# but necessary to have it work on both windows and Unix systems
+function _glob(x) 
+    if isabspath(x)
+        p, rest = splitdir(x)
+        glob(rest,p)
+    else
+        glob(x)
+    end
+end
 
 open_mfdataset(g::AbstractString; kwargs...) = open_mfdataset(_glob(g); kwargs...)
 open_mfdataset(g::Vector{<:AbstractString}; kwargs...) =
@@ -341,7 +351,7 @@ function merge_new_axis(alldatasets, firstcube,var,mergedim)
     newdim = DD.rebuild(mergedim,1:length(alldatasets))
     alldiskarrays = map(ds->ds.cubes[var].data,alldatasets).data
     newda = diskstack(alldiskarrays)
-    newdims = (DD.dim(firstcube)...,newdim)
+    newdims = (DD.dims(firstcube)...,newdim)
     YAXArray(newdims,newda,deepcopy(firstcube.properties))
 end
 function merge_existing_axis(alldatasets,firstcube,var,mergedim)

test/Datasets/datasets.jl

@@ -531,3 +531,36 @@ end
     ds = Dataset(a=a1, b=a2, c=a3, d=a4, e=a5)
     @test_throws ArgumentError Cube(ds)
 end
+
+@testset "Open_mfdataset" begin
+    import NetCDF
+    d1 = DD.X(1:20)
+    d2 = DD.Y(1:10)
+    a1 = rand(20,10)
+    a2 = rand(20,10)
+    td = mktempdir()
+    f1, f2 = joinpath.(td,("file_1.nc","file_2.nc"))
+
+    array1 = YAXArray((d1,d2),a1)
+    array2 = YAXArray((d1,d2),a2)
+
+    savecube(array1,f1)
+    savecube(array2,f2)
+
+    ds = open_mfdataset(DD.DimArray([f1,f2],(DD.Ti(1:2),)))
+
+    @test ds.layer.data[:,:,1] == array1
+    @test ds.layer.data[:,:,2] == array2
+
+    td = mktempdir()
+    f1, f2 = joinpath.(td,("file_1.nc","file_2.nc"))
+    td1, td2 = DD.Ti(1:2), DD.Ti(3:4)
+    a1,a2 = rand(20,10,2), rand(20,10,2)
+    array1,array2 = YAXArray((d1,d2,td1),a1),  YAXArray((d1,d2,td2),a2)
+    savecube(array1,f1)
+    savecube(array2,f2)
+
+    ds = open_mfdataset(joinpath(td,"*.nc"))
+    @test ds.layer.data[:,:,1:2] == array1
+    @test ds.layer.data[:,:,3:4] == array2
+end