Merge remote-tracking branch 'upstream/main' into blocking-read-buffer

.github/workflows/bindings_go.yml

@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Bindings Go CI
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "core/**"
+      - "bindings/c/**"
+      - "bindings/go/**"
+      - ".github/workflows/bindings_go.yml"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20'
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+
+      - name: Build c binding
+        working-directory: bindings/c
+        run: make build
+
+      - name: Check diff
+        run: git diff --exit-code
+
+      - name: Generate pkg-config file
+        run: |
+          echo "libdir=$(pwd)/target/debug/" >> opendal_c.pc
+          echo "includedir=$(pwd)/bindings/c/include/" >> opendal_c.pc
+          echo "Name: opendal_c" >> opendal_c.pc
+          echo "Description: opendal c binding" >> opendal_c.pc
+          echo "Version: 0.0.1" >> opendal_c.pc
+          echo "Libs: -L\${libdir} -lopendal_c" >> opendal_c.pc
+          echo "Cflags: -I\${includedir}" >> opendal_c.pc
+
+          echo "PKG_CONFIG_PATH=$(pwd)" >> $GITHUB_ENV
+          echo "LD_LIBRARY_PATH=$(pwd)/target/debug" >> $GITHUB_ENV
+
+      - name: Run tests
+        working-directory: bindings/go
+        run: go test -tags dynamic .

bindings/python/python/opendal/__init__.pyi

@@ -23,7 +23,16 @@ class Operator:
     def __init__(self, scheme: str, **kwargs): ...
     def read(self, path: str) -> bytes: ...
     def open_reader(self, path: str) -> Reader: ...
-    def write(self, path: str, bs: bytes): ...
+    def write(
+        self,
+        path: str,
+        bs: bytes,
+        append: bool = None,
+        buffer: int = None,
+        content_type: str = None,
+        content_disposition: str = None,
+        cache_control: str = None,
+    ): ...
     def stat(self, path: str) -> Metadata: ...
     def create_dir(self, path: str): ...
     def delete(self, path: str): ...
@@ -34,15 +43,26 @@ class AsyncOperator:
     def __init__(self, scheme: str, **kwargs): ...
     async def read(self, path: str) -> bytes: ...
     def open_reader(self, path: str) -> AsyncReader: ...
-    async def write(self, path: str, bs: bytes): ...
+    async def write(
+        self,
+        path: str,
+        bs: bytes,
+        append: bool = None,
+        buffer: int = None,
+        content_type: str = None,
+        content_disposition: str = None,
+        cache_control: str = None,
+    ): ...
     async def stat(self, path: str) -> Metadata: ...
     async def create_dir(self, path: str): ...
     async def delete(self, path: str): ...
     async def list(self, path: str) -> AsyncIterable[Entry]: ...
     async def scan(self, path: str) -> AsyncIterable[Entry]: ...
     async def presign_stat(self, path: str, expire_second: int) -> PresignedRequest: ...
     async def presign_read(self, path: str, expire_second: int) -> PresignedRequest: ...
-    async def presign_write(self, path: str, expire_second: int) -> PresignedRequest: ...
+    async def presign_write(
+        self, path: str, expire_second: int
+    ) -> PresignedRequest: ...
 
 class Reader:
     def read(self, size: Optional[int] = None) -> bytes: ...
@@ -86,4 +106,4 @@ class PresignedRequest:
     @property
     def method(self) -> str: ...
     @property
-    def headers(self) -> dict[str, str]: ...
+    def headers(self) -> dict[str, str]: ...

bindings/python/src/asyncio.rs

@@ -36,6 +36,7 @@ use tokio::io::AsyncSeekExt;
 use tokio::sync::Mutex;
 
 use crate::build_operator;
+use crate::build_opwrite;
 use crate::format_pyerr;
 use crate::layers;
 use crate::Entry;
@@ -87,11 +88,32 @@ impl AsyncOperator {
     }
 
     /// Write bytes into given path.
-    pub fn write<'p>(&'p self, py: Python<'p>, path: String, bs: &PyBytes) -> PyResult<&'p PyAny> {
+    #[pyo3(signature = (path, bs, **kwargs))]
+    pub fn write<'p>(
+        &'p self,
+        py: Python<'p>,
+        path: String,
+        bs: &PyBytes,
+        kwargs: Option<&PyDict>,
+    ) -> PyResult<&'p PyAny> {
+        let opwrite = build_opwrite(kwargs)?;
         let this = self.0.clone();
         let bs = bs.as_bytes().to_vec();
         future_into_py(py, async move {
-            this.write(&path, bs).await.map_err(format_pyerr)
+            let mut write = this.write_with(&path, bs).append(opwrite.append());
+            if let Some(buffer) = opwrite.buffer() {
+                write = write.buffer(buffer);
+            }
+            if let Some(content_type) = opwrite.content_type() {
+                write = write.content_type(content_type);
+            }
+            if let Some(content_disposition) = opwrite.content_disposition() {
+                write = write.content_disposition(content_disposition);
+            }
+            if let Some(cache_control) = opwrite.cache_control() {
+                write = write.cache_control(cache_control);
+            }
+            write.await.map_err(format_pyerr)
         })
     }
 

bindings/python/src/lib.rs

@@ -120,8 +120,24 @@ impl Operator {
     }
 
     /// Write bytes into given path.
-    pub fn write(&self, path: &str, bs: Vec<u8>) -> PyResult<()> {
-        self.0.write(path, bs).map_err(format_pyerr)
+    #[pyo3(signature = (path, bs, **kwargs))]
+    pub fn write(&self, path: &str, bs: Vec<u8>, kwargs: Option<&PyDict>) -> PyResult<()> {
+        let opwrite = build_opwrite(kwargs)?;
+        let mut write = self.0.write_with(path, bs).append(opwrite.append());
+        if let Some(buffer) = opwrite.buffer() {
+            write = write.buffer(buffer);
+        }
+        if let Some(content_type) = opwrite.content_type() {
+            write = write.content_type(content_type);
+        }
+        if let Some(content_disposition) = opwrite.content_disposition() {
+            write = write.content_disposition(content_disposition);
+        }
+        if let Some(cache_control) = opwrite.cache_control() {
+            write = write.cache_control(cache_control);
+        }
+
+        write.call().map_err(format_pyerr)
     }
 
     /// Get current path's metadata **without cache** directly.
@@ -421,6 +437,55 @@ fn format_pyerr(err: od::Error) -> PyErr {
     }
 }
 
+/// recognize OpWrite-equivalent options passed as python dict
+pub(crate) fn build_opwrite(kwargs: Option<&PyDict>) -> PyResult<od::raw::OpWrite> {
+    use od::raw::OpWrite;
+    let mut op = OpWrite::new();
+
+    let dict = if let Some(kwargs) = kwargs {
+        kwargs
+    } else {
+        return Ok(op);
+    };
+
+    if let Some(append) = dict.get_item("append") {
+        let v = append
+            .extract::<bool>()
+            .map_err(|err| PyValueError::new_err(format!("append must be bool, got {}", err)))?;
+        op = op.with_append(v);
+    }
+
+    if let Some(buffer) = dict.get_item("buffer") {
+        let v = buffer
+            .extract::<usize>()
+            .map_err(|err| PyValueError::new_err(format!("buffer must be usize, got {}", err)))?;
+        op = op.with_buffer(v);
+    }
+
+    if let Some(content_type) = dict.get_item("content_type") {
+        let v = content_type.extract::<String>().map_err(|err| {
+            PyValueError::new_err(format!("content_type must be str, got {}", err))
+        })?;
+        op = op.with_content_type(v.as_str());
+    }
+
+    if let Some(content_disposition) = dict.get_item("content_disposition") {
+        let v = content_disposition.extract::<String>().map_err(|err| {
+            PyValueError::new_err(format!("content_disposition must be str, got {}", err))
+        })?;
+        op = op.with_content_disposition(v.as_str());
+    }
+
+    if let Some(cache_control) = dict.get_item("cache_control") {
+        let v = cache_control.extract::<String>().map_err(|err| {
+            PyValueError::new_err(format!("cache_control must be str, got {}", err))
+        })?;
+        op = op.with_cache_control(v.as_str());
+    }
+
+    Ok(op)
+}
+
 /// OpenDAL Python binding
 ///
 /// ## Installation