Allow migration to V1 (#85)

.github/workflows/compatibility.yml

@@ -0,0 +1,181 @@
+# Compatbility workflow.
+#
+# Ensure the compatibility between cshelve version, Python version and OS.
+
+name: Compatibility
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  WHEEL: cshelve-1.0.0-py3-none-any.whl
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.9"]
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    environment: azure
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      run: uv python install ${{ matrix.python-version }}
+
+    - name: Install the project
+      run: uv build
+
+    - name: Upload the package
+      uses: actions/upload-artifact@v4
+      with:
+        name: cshelve-${{ github.sha }}
+        path: dist/*.whl
+        retention-days: 10
+        if-no-files-found: error
+        overwrite: false
+
+  write:
+    strategy:
+      matrix:
+        os: [windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        cshelve-version: ["0.9.0"]
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    environment: azure
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    # An Azure storage account is used for reality.
+    - name: 'Az CLI login'
+      uses: azure/login@v1
+      with:
+        client-id: ${{ secrets.AZURE_CLIENT_ID }}
+        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+    - name: Install package.
+      working-directory: compatibility
+      run: |
+        pip install "cshelve[azure-blob] == ${{ matrix.cshelve-version }}"
+
+    - name: Write
+      working-directory: compatibility
+      run: |
+        python3 write.py "${{ matrix.cshelve-version }}" "${{ matrix.python-version }}"
+
+  versions:
+    needs: write
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        cshelve-version: ["0.9.0"]
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    environment: azure
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    # An Azure storage account is used for reality.
+    - name: 'Az CLI login'
+      uses: azure/login@v1
+      with:
+        client-id: ${{ secrets.AZURE_CLIENT_ID }}
+        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+    - name: Install package on Unix like.
+      if: runner.os != 'Windows'
+      working-directory: compatibility
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        pip install "cshelve[azure-blob] == ${{ matrix.cshelve-version }}"
+        python read.py "${{ matrix.cshelve-version }}" "3.9"
+        python read.py "${{ matrix.cshelve-version }}" "3.10"
+        python read.py "${{ matrix.cshelve-version }}" "3.11"
+        python read.py "${{ matrix.cshelve-version }}" "3.12"
+        python read.py "${{ matrix.cshelve-version }}" "3.13"
+
+    - name: Install package on Windows.
+      if: runner.os == 'Windows'
+      working-directory: compatibility
+      run: |
+        pip install "cshelve[azure-blob] == ${{ matrix.cshelve-version }}"
+        python read.py "${{ matrix.cshelve-version }}" "3.9"
+        python read.py "${{ matrix.cshelve-version }}" "3.10"
+        python read.py "${{ matrix.cshelve-version }}" "3.11"
+        python read.py "${{ matrix.cshelve-version }}" "3.12"
+        python read.py "${{ matrix.cshelve-version }}" "3.13"
+
+  latest:
+    needs: build
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        cshelve-version: ["0.9.0"]
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    environment: azure
+
+    steps:
+    # An Azure storage account is used for reality.
+    - name: 'Az CLI login'
+      uses: azure/login@v1
+      with:
+        client-id: ${{ secrets.AZURE_CLIENT_ID }}
+        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Download the wheel.
+      uses: actions/download-artifact@v4
+      with:
+        name: cshelve-${{ github.sha }}
+        path: compatibility/dist
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install package.
+      working-directory: compatibility
+      run: |
+        pip install ./dist/${{ env.WHEEL }}[azure-blob]
+
+    - name: Read
+      working-directory: compatibility
+      run: |
+        python3 read.py "${{ matrix.cshelve-version }}" "3.9"
+        python3 read.py "${{ matrix.cshelve-version }}" "3.10"
+        python3 read.py "${{ matrix.cshelve-version }}" "3.11"
+        python3 read.py "${{ matrix.cshelve-version }}" "3.12"
+        python3 read.py "${{ matrix.cshelve-version }}" "3.13"

.github/workflows/quality.yml

@@ -20,7 +20,7 @@ permissions:
   contents: read
 
 env:
-  WHEEL: cshelve-0.9.0-py3-none-any.whl
+  WHEEL: cshelve-1.0.0-py3-none-any.whl
 
 jobs:
   build:

.github/workflows/release.yml

@@ -9,7 +9,7 @@ permissions:
   contents: read
 
 env:
-  WHEEL: cshelve-0.9.0-py3-none-any.whl
+  WHEEL: cshelve-1.0.0-py3-none-any.whl
 
 jobs:
   build:

CHANGELOG.md

@@ -1,8 +1,10 @@
 # Changelog
 
-## [0.10.0] - 2024-
-### Breaking change
+## [0.10.0] - 2024-01-31
+### Added
 - Add metadata to object to improve compatibility between versions.
+- Allow transparent migration from older version to 1.0.0.
+- Tests for compatibility between versions, os and python versions.
 
 ## [0.9.0] - 2024-12-22
 ### Added

compatibility/azure-passwordless.ini

@@ -0,0 +1,5 @@
+[default]
+provider        = azure-blob
+account_url     = https://dscccccccccccccc.blob.core.windows.net
+auth_type       = passwordless
+container_name  = compatibility

compatibility/read.py

@@ -0,0 +1,13 @@
+import sys
+
+import cshelve
+
+
+cshelve_version = sys.argv[1]
+python_version = sys.argv[2]
+
+with cshelve.open("./azure-passwordless.ini") as db:
+    assert (
+        db[f"compatibility-{cshelve_version}-{python_version}"]
+        == f"my complex data from cshelve version {cshelve_version} and python {python_version}"
+    )

compatibility/write.py

@@ -0,0 +1,12 @@
+import sys
+
+import cshelve
+
+
+cshelve_version = sys.argv[1]
+python_version = sys.argv[2]
+
+with cshelve.open("./azure-passwordless.ini") as db:
+    db[
+        f"compatibility-{cshelve_version}-{python_version}"
+    ] = f"my complex data from cshelve version {cshelve_version} and python {python_version}"

cshelve/_data_processing.py

@@ -21,6 +21,8 @@
 
 # Algorithm signatures to applied to the data.
 SIGNATURES = {"COMPRESSION": b"c", "ENCRYPTION": b"e"}
+# Default signature, when no processing is applied.
+EMPTY_SIGNATURE = b""
 
 
 class DataProcessing:
@@ -37,7 +39,7 @@ def __init__(self, logger):
         self.post_processing: List[Callable[[bytes], bytes]] = []
         # The signature of the data processing.
         # It is used to ensure the data processing is applied in the correct order.
-        self.signature = b""
+        self.signature = EMPTY_SIGNATURE
 
     def add(
         self,
@@ -62,22 +64,7 @@ def apply_pre_processing(self, data: bytes) -> bytes:
         for fct in self.pre_processing:
             data = fct(data)
 
-        len_data_proc_signature = len(self.signature)
-        len_data = len(data)
-
-        data_processing = struct.pack(
-            f"<{len_data_proc_signature}s{len_data}s",
-            self.signature,
-            data,
-        )
-        # We are using unsigned long long due to the potential size of the data.
-        metadata = struct.pack(
-            f"<BQ{len_data_proc_signature + len_data}s",
-            len_data_proc_signature,
-            len_data,
-            data_processing,
-        )
-        return metadata
+        return self.encapsulate(data, self.signature)
 
     def apply_post_processing(self, data: bytes) -> bytes:
         """
@@ -93,15 +80,52 @@ def apply_post_processing(self, data: bytes) -> bytes:
             )
         )
 
-        if data_processing.signature != self.signature:
-            self.logger.error(
-                "Data processing signature: %s, expected: %s",
-                data_processing.signature,
-                self.signature,
-            )
-            raise DataProcessingSignatureError("Wrong data processing signature.")
+        result = data_processing.data
+        signature = data_processing.signature
 
-        data = data_processing.data
-        for fct in self.post_processing:
-            data = fct(data)
-        return data
+        # Apply all signatures known from the current signature if possible.
+        for idx, s in enumerate(self.signature):
+            if signature == EMPTY_SIGNATURE:
+                # The signature of the object can be shorter then the signature of the incoming object.
+                break
+            if signature[0] == s:
+                # The transformation must be applied.
+                result = self.post_processing[idx](result)
+                signature = signature[1:]
+        else:
+            # If the signature is not empty, it means at least one transformation of the incoming object
+            # is unknowned of the current process and so the incoming object can't be retrieved.
+            if signature != EMPTY_SIGNATURE:
+                self.logger.error(
+                    "Data processing signature: %s is incompatible with: %s",
+                    data_processing.signature,
+                    self.signature,
+                )
+                raise DataProcessingSignatureError(
+                    f"Following transformation can't be applied: {signature}."
+                )
+
+        return result
+
+    @classmethod
+    def encapsulate(cls, data: bytes, signature: bytes = EMPTY_SIGNATURE) -> bytes:
+        """
+        Wraps the data with the processing metadata.
+        """
+        len_data = len(data)
+        len_data_proc_signature = len(signature)
+
+        data_processing = struct.pack(
+            f"<{len_data_proc_signature}s{len_data}s",
+            signature,
+            data,
+        )
+        # We are using unsigned long long due to the potential size of the data.
+        metadata = struct.pack(
+            f"<BQ{len_data_proc_signature + len_data}s",
+            len_data_proc_signature,
+            len_data,
+            data_processing,
+        )
+
+        return metadata

cshelve/_database.py

@@ -53,9 +53,15 @@ def __getitem__(self, key: bytes) -> bytes:
         """
         value = self.db.get(key)
         record = _Record._make(struct.unpack(f"<B{len(value) - 1}s", value))
-        if record.version != VERSION:
-            self.logger.critical(f"Version mismatch: {record.version} != {VERSION}")
-            raise VersionMismatch("Version mismatch.")
+
+        if record.version > VERSION:
+            # If the version is greater than the current version, its a raw pickle from earlier cshelve versions.
+            self.logger.warning(
+                f"Version mismatch: {record.version} != {VERSION}. Migrating..."
+            )
+            value = DataProcessing.encapsulate(value)
+            record = _Record(VERSION, value)
+            self.logger.warning(f"Migration successful.")
         return self.data_processing.apply_post_processing(record.data)
 
     @can_write

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cshelve"
-version = "0.9.0"
+version = "1.0.0"
 description = "Propulsing the shelve module to the cloud"
 readme = "README.md"
 requires-python = ">=3.9"

tests/configurations/in-memory/compatibility/compressed.ini

@@ -0,0 +1,8 @@
+[default]
+provider        = in-memory
+persist-key     = persisted-to-compression
+exists          = true
+
+[compression]
+algorithm   = zlib
+level       = 1

tests/configurations/in-memory/compatibility/init.ini

@@ -0,0 +1,4 @@
+[default]
+provider        = in-memory
+persist-key     = persisted-to-compression
+exists          = true