Save network to a binary buffer

cpp/src/bindings.cpp

@@ -326,10 +326,13 @@ PYBIND11_MODULE(_pypowsybl, m) {
     m.def("load_network_from_binary_buffers", &pypowsybl::loadNetworkFromBinaryBuffers, "Load a network from a list of binary buffer", py::call_guard<py::gil_scoped_release>(),
               py::arg("buffers"), py::arg("parameters"), py::arg("reporter"));
 
-    m.def("dump_network", &pypowsybl::dumpNetwork, "Dump network to a file in a given format", py::call_guard<py::gil_scoped_release>(),
+    m.def("save_network", &pypowsybl::saveNetwork, "Save network to a file in a given format", py::call_guard<py::gil_scoped_release>(),
           py::arg("network"), py::arg("file"),py::arg("format"), py::arg("parameters"), py::arg("reporter"));
 
-    m.def("dump_network_to_string", &pypowsybl::dumpNetworkToString, "Dump network in a given format", py::call_guard<py::gil_scoped_release>(),
+    m.def("save_network_to_string", &pypowsybl::saveNetworkToString, "Save network in a given format to a string", py::call_guard<py::gil_scoped_release>(),
+          py::arg("network"), py::arg("format"), py::arg("parameters"), py::arg("reporter"));
+
+    m.def("save_network_to_binary_buffer", &pypowsybl::saveNetworkToBinaryBuffer, "Save network in a given format to a binary byffer", py::call_guard<py::gil_scoped_release>(),
           py::arg("network"), py::arg("format"), py::arg("parameters"), py::arg("reporter"));
 
     m.def("reduce_network", &pypowsybl::reduceNetwork, "Reduce network", py::call_guard<py::gil_scoped_release>(),

cpp/src/pypowsybl.cpp

@@ -664,7 +664,7 @@ JavaHandle loadNetworkFromBinaryBuffers(std::vector<py::buffer> byteBuffers, con
     return networkHandle;
 }
 
-void dumpNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
+void saveNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
     std::vector<std::string> parameterNames;
     std::vector<std::string> parameterValues;
     parameterNames.reserve(parameters.size());
@@ -675,11 +675,11 @@ void dumpNetwork(const JavaHandle& network, const std::string& file, const std::
     }
     ToCharPtrPtr parameterNamesPtr(parameterNames);
     ToCharPtrPtr parameterValuesPtr(parameterValues);
-    callJava(::dumpNetwork, network, (char*) file.data(), (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
+    callJava(::saveNetwork, network, (char*) file.data(), (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
                 parameterValuesPtr.get(), parameterValues.size(), (reporter == nullptr) ? nullptr : *reporter);
 }
 
-std::string dumpNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
+std::string saveNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
     std::vector<std::string> parameterNames;
     std::vector<std::string> parameterValues;
     parameterNames.reserve(parameters.size());
@@ -690,10 +690,28 @@ std::string dumpNetworkToString(const JavaHandle& network, const std::string& fo
     }
     ToCharPtrPtr parameterNamesPtr(parameterNames);
     ToCharPtrPtr parameterValuesPtr(parameterValues);
-    return toString(callJava<char*>(::dumpNetworkToString, network, (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
+    return toString(callJava<char*>(::saveNetworkToString, network, (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
              parameterValuesPtr.get(), parameterValues.size(), (reporter == nullptr) ? nullptr : *reporter));
 }
 
+py::bytes saveNetworkToBinaryBuffer(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter) {
+    std::vector<std::string> parameterNames;
+    std::vector<std::string> parameterValues;
+    parameterNames.reserve(parameters.size());
+    parameterValues.reserve(parameters.size());
+    for (std::pair<std::string, std::string> p : parameters) {
+        parameterNames.push_back(p.first);
+        parameterValues.push_back(p.second);
+    }
+    ToCharPtrPtr parameterNamesPtr(parameterNames);
+    ToCharPtrPtr parameterValuesPtr(parameterValues);
+    array* byteArray = callJava<array*>(::saveNetworkToBinaryBuffer, network, (char*) format.data(), parameterNamesPtr.get(), parameterNames.size(),
+                     parameterValuesPtr.get(), parameterValues.size(), reporter == nullptr ? nullptr : *reporter);
+    py::bytes bytes((char*) byteArray->ptr, byteArray->length);
+    callJava<>(::freeNetworkBinaryBuffer, byteArray);
+    return bytes;
+}
+
 void reduceNetwork(const JavaHandle& network, double v_min, double v_max, const std::vector<std::string>& ids,
                    const std::vector<std::string>& vls, const std::vector<int>& depths, bool withDangLingLines) {
     ToCharPtrPtr elementIdPtr(ids);

cpp/src/pypowsybl.h

@@ -346,7 +346,7 @@ JavaHandle loadNetworkFromString(const std::string& fileName, const std::string&
 
 JavaHandle loadNetworkFromBinaryBuffers(std::vector<py::buffer> byteBuffer, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
 
-void dumpNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
+void saveNetwork(const JavaHandle& network, const std::string& file, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
 
 LoadFlowParameters* createLoadFlowParameters();
 
@@ -364,7 +364,9 @@ SensitivityAnalysisParameters* createSensitivityAnalysisParameters();
 
 std::vector<std::string> getSensitivityAnalysisProviderParametersNames(const std::string& sensitivityAnalysisProvider);
 
-std::string dumpNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
+std::string saveNetworkToString(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
+
+py::bytes saveNetworkToBinaryBuffer(const JavaHandle& network, const std::string& format, const std::map<std::string, std::string>& parameters, JavaHandle* reporter);
 
 void reduceNetwork(const JavaHandle& network, const double v_min, const double v_max, const std::vector<std::string>& ids, const std::vector<std::string>& vls, const std::vector<int>& depths, bool withDangLingLines);
 

docs/user_guide/network.rst

@@ -68,13 +68,19 @@ Networks can be written to the filesystem, using one of the available export for
 
 .. code-block:: python
 
-   network.dump('network.xiidm', format='XIIDM')
+   network.save('network.xiidm', format='XIIDM')
 
 You can also serialize networks to a string:
 
 .. code-block:: python
 
-   xiidm_str = network.dump_to_string('XIIDM')
+   xiidm_str = network.save_to_string('XIIDM')
+
+And also to a zip file as a (io.BytesIO) binary buffer.
+
+.. code-block:: python
+
+   zipped_xiidm = network.save_to_binary_buffer('XIIDM')
 
 The supported formats are:
 

java/src/main/java/com/powsybl/python/commons/CTypeUtil.java

@@ -38,6 +38,10 @@ public static CCharPointer toCharPtr(String str) {
         }
         // pybind11 convert std::string and char* to python utf-8 string
         byte[] bytes = str.getBytes(StandardCharsets.UTF_8);
+        return toCharPtr(bytes);
+    }
+
+    public static CCharPointer toCharPtr(byte[] bytes) {
         CCharPointer charPtr = UnmanagedMemory.calloc((bytes.length + 1) * SizeOf.get(CCharPointer.class));
         for (int i = 0; i < bytes.length; ++i) {
             charPtr.write(i, bytes[i]);

java/src/main/java/com/powsybl/python/commons/Util.java

@@ -21,6 +21,7 @@
 import com.powsybl.python.dataframe.CDataframeHandler;
 import org.graalvm.nativeimage.UnmanagedMemory;
 import org.graalvm.nativeimage.c.struct.SizeOf;
+import org.graalvm.nativeimage.c.type.CCharPointer;
 import org.graalvm.nativeimage.c.type.CCharPointerPointer;
 import org.graalvm.nativeimage.c.type.CDoublePointer;
 import org.graalvm.nativeimage.c.type.CIntPointer;
@@ -156,6 +157,10 @@ public static ArrayPointer<CIntPointer> createIntegerArray(List<Integer> integer
         return allocArrayPointer(intListPtr, integerList.size());
     }
 
+    public static ArrayPointer<CCharPointer> createByteArray(byte[] bytes) {
+        return allocArrayPointer(CTypeUtil.toCharPtr(bytes), bytes.length);
+    }
+
     public static int convert(SeriesDataType type) {
         return switch (type) {
             case STRING -> CDataframeHandler.STRING_SERIES_TYPE;

java/src/main/java/com/powsybl/python/network/NetworkCFunctions.java

@@ -60,6 +60,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.zip.ZipOutputStream;
 
 import static com.powsybl.python.commons.CTypeUtil.toStringList;
 import static com.powsybl.python.commons.PyPowsyblApiHeader.*;
@@ -210,8 +211,8 @@ public static ObjectHandle loadNetworkFromBinaryBuffers(IsolateThread thread, CC
         });
     }
 
-    @CEntryPoint(name = "dumpNetwork")
-    public static void dumpNetwork(IsolateThread thread, ObjectHandle networkHandle, CCharPointer file, CCharPointer format,
+    @CEntryPoint(name = "saveNetwork")
+    public static void saveNetwork(IsolateThread thread, ObjectHandle networkHandle, CCharPointer file, CCharPointer format,
                                    CCharPointerPointer parameterNamesPtrPtr, int parameterNamesCount,
                                    CCharPointerPointer parameterValuesPtrPtr, int parameterValuesCount,
                                    ObjectHandle reporterHandle, ExceptionHandlerPointer exceptionHandlerPtr) {
@@ -228,8 +229,8 @@ public static void dumpNetwork(IsolateThread thread, ObjectHandle networkHandle,
         });
     }
 
-    @CEntryPoint(name = "dumpNetworkToString")
-    public static CCharPointer dumpNetworkToString(IsolateThread thread, ObjectHandle networkHandle, CCharPointer format,
+    @CEntryPoint(name = "saveNetworkToString")
+    public static CCharPointer saveNetworkToString(IsolateThread thread, ObjectHandle networkHandle, CCharPointer format,
                                                    CCharPointerPointer parameterNamesPtrPtr, int parameterNamesCount,
                                                    CCharPointerPointer parameterValuesPtrPtr, int parameterValuesCount,
                                                    ObjectHandle reporterHandle, ExceptionHandlerPointer exceptionHandlerPtr) {
@@ -260,6 +261,40 @@ public static CCharPointer dumpNetworkToString(IsolateThread thread, ObjectHandl
         });
     }
 
+    @CEntryPoint(name = "saveNetworkToBinaryBuffer")
+    public static ArrayPointer<CCharPointer> saveNetworkToBinaryBuffer(IsolateThread thread, ObjectHandle networkHandle, CCharPointer format,
+                                                             CCharPointerPointer parameterNamesPtrPtr, int parameterNamesCount,
+                                                             CCharPointerPointer parameterValuesPtrPtr, int parameterValuesCount,
+                                                             ObjectHandle reporterHandle, ExceptionHandlerPointer exceptionHandlerPtr) {
+        return doCatch(exceptionHandlerPtr, () -> {
+            Network network = ObjectHandles.getGlobal().get(networkHandle);
+            String formatStr = CTypeUtil.toString(format);
+            Properties parameters = createParameters(parameterNamesPtrPtr, parameterNamesCount, parameterValuesPtrPtr, parameterValuesCount);
+            var exporter = Exporter.find(formatStr);
+            if (exporter == null) {
+                throw new PowsyblException("No exporter found for '" + formatStr + "' to export as a string");
+            }
+            Reporter reporter = ReportCUtils.getReporter(reporterHandle);
+            // to support all kind of export: simple file or multiple to an archive,
+            // best is to write to a zip file
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            try (ZipOutputStream zos = new ZipOutputStream(bos)) {
+                DataSource dataSource = new ZipMemDataSource("file", zos);
+                exporter.export(network, parameters, dataSource, reporter);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+            byte[] bytes = bos.toByteArray();
+            return Util.createByteArray(bytes);
+        });
+    }
+
+    @CEntryPoint(name = "freeNetworkBinaryBuffer")
+    public static void freeNetworkBinaryBuffer(IsolateThread thread, PyPowsyblApiHeader.ArrayPointer<CCharPointer> byteArrayPtr,
+                                               PyPowsyblApiHeader.ExceptionHandlerPointer exceptionHandlerPtr) {
+        doCatch(exceptionHandlerPtr, () -> freeArrayPointer(byteArrayPtr));
+    }
+
     @CEntryPoint(name = "reduceNetwork")
     public static void reduceNetwork(IsolateThread thread, ObjectHandle networkHandle,
                                      double vMin, double vMax,

java/src/main/java/com/powsybl/python/network/ZipMemDataSource.java

@@ -0,0 +1,85 @@
+/**
+ * Copyright (c) 2023, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+package com.powsybl.python.network;
+
+import com.powsybl.commons.datasource.DataSource;
+import com.powsybl.commons.datasource.DataSourceUtil;
+import com.powsybl.commons.io.ForwardingOutputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Objects;
+import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * @author Geoffroy Jamgotchian {@literal <geoffroy.jamgotchian at rte-france.com>}
+ */
+public class ZipMemDataSource implements DataSource {
+
+    private final String baseName;
+
+    private final ZipOutputStream zos;
+
+    public ZipMemDataSource(String baseName, ZipOutputStream zos) {
+        this.baseName = Objects.requireNonNull(baseName);
+        this.zos = Objects.requireNonNull(zos);
+    }
+
+    @Override
+    public OutputStream newOutputStream(String fileName, boolean append) throws IOException {
+        Objects.requireNonNull(fileName);
+        if (append) {
+            throw new UnsupportedOperationException("append not supported in zip file data source");
+        }
+        zos.putNextEntry(new ZipEntry(fileName));
+        return new ForwardingOutputStream<OutputStream>(zos) {
+            @Override
+            public void close() throws IOException {
+                zos.closeEntry();
+            }
+        };
+    }
+
+    @Override
+    public OutputStream newOutputStream(String suffix, String ext, boolean append) throws IOException {
+        return newOutputStream(DataSourceUtil.getFileName(baseName, suffix, ext), append);
+    }
+
+    @Override
+    public String getBaseName() {
+        return baseName;
+    }
+
+    @Override
+    public boolean exists(String suffix, String ext) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean exists(String fileName) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public InputStream newInputStream(String suffix, String ext) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public InputStream newInputStream(String fileName) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Set<String> listNames(String regex) {
+        throw new UnsupportedOperationException();
+    }
+}

pypowsybl/_pypowsybl.pyi

@@ -553,8 +553,9 @@ def get_extensions_names() -> List[str]: ...
 def get_extensions_information() -> SeriesArray: ...
 def create_security_analysis() -> JavaHandle: ...
 def create_sensitivity_analysis() -> JavaHandle: ...
-def dump_network(network: JavaHandle, file: str, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> None: ...
-def dump_network_to_string(network: JavaHandle, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> str: ...
+def save_network(network: JavaHandle, file: str, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> None: ...
+def save_network_to_string(network: JavaHandle, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> str: ...
+def save_network_to_binary_buffer(network: JavaHandle, format: str, parameters: Dict[str,str], report: Optional[JavaHandle]) -> bytes: ...
 def get_branch_flows_sensitivity_matrix(sensitivity_analysis_result_context: JavaHandle, matrix_id: str, contingency_id: str) -> Matrix: ...
 def get_branch_results(result: JavaHandle) -> SeriesArray: ...
 def get_bus_breaker_view_buses(network: JavaHandle, voltage_level: str) -> SeriesArray: ...