Solved the following limitation of MPIPETScDistributedVector:

	  "If T<:AbstractVector{<:Number}, length for all entries of MPIPETScDistributedVector must match."

src/DistributedFESpaces.jl

@@ -143,10 +143,9 @@ function DistributedFESpace(::Type{V}; model::DistributedDiscreteModel,kwargs...
       ngids=lngids
   end
 
-  num_dofs_x_cell = 0
-  do_on_parts(comm,spaces) do part, lspace
+  num_dofs_x_cell=DistributedData(comm,spaces) do part, lspace
     cell_dofs=get_cell_dofs(lspace)
-    num_dofs_x_cell=length(cell_dofs[1])
+    [length(cell_dofs[i]) for i=1:length(cell_dofs)]
   end
 
   function init_cell_to_owners(part,cell_to_owners,lspace,lid_to_owner)

src/MPIPETScDistributedVectors.jl

@@ -80,6 +80,26 @@ function DistributedVector{T}(
   MPIPETScDistributedVector(part,indices,vecghost)
 end
 
+function DistributedVector{T}(
+  indices::MPIPETScDistributedIndexSet, length_entries :: MPIPETScDistributedData ) where T <: AbstractVector{<:Number}
+  num_entries   = length(indices.parts.part.lid_to_owner)
+  @assert num_entries == length(length_entries.part)
+  block_indices = indices
+  indices,vecghost = _create_eltype_vector_number_variable_length_indices_ghost(T,length_entries.part,indices)
+  lvecghost = PETSc.LocalVector(vecghost, length(indices.parts.part.lid_to_owner))
+  a_reint=reinterpret(eltype(T), lvecghost.a)
+  ptrs=Vector{Int32}(undef,num_entries+1)
+  ptrs[1]=1
+  for i=1:num_entries
+    ptrs[i+1]=ptrs[i]+length_entries.part[i]
+  end
+  TSUB=SubArray{eltype(T),1,typeof(a_reint),Tuple{UnitRange{Int64}},true}
+  part=TSUB[ view(a_reint,ptrs[i]:ptrs[i+1]-1) for i=1:num_entries ]
+  part=reindex(part,block_indices.app_to_petsc_locidx)
+  PETSc.restore(lvecghost)
+  MPIPETScDistributedVector(part,indices,vecghost)
+end
+
 
 function _create_eltype_number_indices_ghost(
   eltype::Type{T},
@@ -121,6 +141,69 @@ vecghost = create_ghost_vector(indices)
 indices, vecghost
 end
 
+function _create_eltype_vector_number_variable_length_indices_ghost(
+  eltype::Type{T},
+  length_entries::AbstractVector{<:Integer},
+  indices::MPIPETScDistributedIndexSet,
+) where T<:AbstractVector{<:Number}
+
+#println(T)
+#println(eltype(T))
+#@assert sizeof(eltype(T)) == sizeof(Float64)
+comm      = get_comm(indices)
+part      = MPI.Comm_rank(comm.comm)+1
+num_owned_entries = 0
+num_local_entries = 0
+lid_to_owner = indices.parts.part.lid_to_owner
+for i = 1:length(lid_to_owner)
+   if (lid_to_owner[i] == part)
+     num_owned_entries = num_owned_entries + length_entries[i]
+   end
+   num_local_entries = num_local_entries + length_entries[i]
+end
+
+sndbuf = Ref{Int64}(num_owned_entries)
+rcvbuf = Ref{Int64}()
+
+MPI.Exscan!(sndbuf, rcvbuf, 1, +, comm.comm)
+(part == 1) ? (offset = 1) : (offset = rcvbuf[] + 1)
+
+sendrecvbuf = Ref{Int64}(num_owned_entries)
+MPI.Allreduce!(sendrecvbuf, +, comm.comm)
+n = sendrecvbuf[]
+
+offsets = DistributedVector{Int}(indices)
+do_on_parts(offsets, indices) do part, offsets, indices
+  for i=1:length(indices.lid_to_owner)
+     if (indices.lid_to_owner[i] == part)
+       offsets[i] = offset
+       for j=1:length_entries[i]
+         offset=offset+1
+       end
+     end
+  end
+end
+exchange!(offsets)
+
+indices = DistributedIndexSet(get_comm(indices),n,indices,offsets) do part, indices, offsets
+  lid_to_gid   = Vector{Int}(undef, num_local_entries)
+  lid_to_owner = Vector{Int}(undef, num_local_entries)
+  k=1
+  for i=1:length(indices.lid_to_owner)
+     offset=offsets[i]
+     for j=1:length_entries[i]
+        lid_to_gid[k]   = offset
+        lid_to_owner[k] = indices.lid_to_owner[i]
+        k=k+1
+        offset=offset+1
+     end
+  end
+  IndexSet(n, lid_to_gid, lid_to_owner)
+end
+vecghost = create_ghost_vector(indices)
+indices, vecghost
+end
+
 function unpack_all_entries!(a::MPIPETScDistributedVector{T}) where T
   num_local = length(a.indices.app_to_petsc_locidx)
   lvec = PETSc.LocalVector(a.vecghost,num_local)

src/SequentialDistributedVectors.jl

@@ -52,6 +52,16 @@ function DistributedVector{T}(
   SequentialDistributedVector(parts,indices)
 end
 
+function DistributedVector{T}(
+  indices::SequentialDistributedIndexSet, length_entries :: SequentialDistributedData ) where T <: AbstractVector{<:Number}
+  comm = get_comm(indices)
+  data = DistributedData(comm,indices,length_entries) do part, lindices, length_entries
+    Vector{eltype(T)}[ Vector{eltype(T)}(undef,length_entries[i]) for i=1:length(lindices.lid_to_owner) ]
+  end
+  parts = data.parts
+  SequentialDistributedVector(parts,indices)
+end
+
 
 
 

test/MPIPETScDistributedVectors.jl

@@ -29,28 +29,41 @@ exchange!(vec)
 @test vec.part == indices.parts.part.lid_to_owner
 
 vec = DistributedVector{Vector{Int64}}(indices,4)
-do_on_parts(vec,indices) do part, local_part, indices
-  for i=1:length(local_part)
-    if (indices.lid_to_owner[i] == part)
-      for j=1:length(local_part[i])
-        local_part[i][j] = part
-      end
-    else
-      for j=1:length(local_part[i])
-        local_part[i][j] = 0
+function init_vec()
+  do_on_parts(vec, indices) do part, local_part, indices
+    for i = 1:length(local_part)
+      if (indices.lid_to_owner[i] == part)
+        for j = 1:length(local_part[i])
+          local_part[i][j] = part
+        end
+      else
+        for j = 1:length(local_part[i])
+          local_part[i][j] = 0
+        end
       end
     end
   end
 end
+init_vec()
 exchange!(vec)
-test_result = true
-for i = 1:length(vec.part)
-  for j = 1:length(vec.part[i])
-     global test_result
-     test_result = test_result && (vec.part[i][j] == indices.parts.part.lid_to_owner[i])
+function test_result()
+  test_result = true
+  for i = 1:length(vec.part)
+    for j = 1:length(vec.part[i])
+      test_result =
+        test_result && (vec.part[i][j] == indices.parts.part.lid_to_owner[i])
+    end
   end
+  @test test_result
 end
+test_result()
 
-@test test_result
+length_entries = DistributedData(comm, indices) do part, indices
+  [ 4 for i=1:length(indices.lid_to_owner) ]
+end
+vec = DistributedVector{Vector{Int64}}(indices,length_entries)
+init_vec()
+exchange!(vec)
+test_result()
 
 end # module