Merge pull request #145 from redis-collections/atomic-synchronization

Make Syncable collections sync atomically

docs/changelog.rst

@@ -6,6 +6,9 @@ Changelog
 Releases
 --------
 
+- 0.10.0:
+    - **Bending change**: The Syncable collections (like ``SyncableDict``) have been updated such that their ``.sync()`` methods are atomic.
+      This requires additional storage on Redis when syncing, so be aware of space constraints.
 - 0.9.1:
     - **Version compatibility** - The collection classes now accept a ``hmset_command`` keyword argument. This is set to ``hmset`` for compatibility with Redis server versions before 4.0.0. Set it to ``hset`` to avoid a ``DeprecationWarning`` from `redis-py`.
 - 0.9.0:
@@ -62,8 +65,8 @@ Versioning
 A 1.0 release is planned. Before that happens:
 
 - Releases with significant new features or breaking changes will be tagged as
-  0.10.x, 0.11.x, etc.
-- Bug fix releases will be tagged as 0.8.x
+  0.11.x, 0.12.x, etc.
+- Bug fix releases will be tagged as 0.10.x
 
 After 1.0 is released:
 

redis_collections/__init__.py

@@ -1,5 +1,5 @@
 __title__ = 'redis-collections'
-__version__ = '0.9.1'
+__version__ = '0.10.0'
 __author__ = 'Honza Javorek'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2013-? Honza Javorek'

redis_collections/syncable.py

@@ -29,6 +29,12 @@
 The underlying :class:`RedisCollection` instance can be accessed from the
 ``persistence`` attribute.
 
+.. note::
+    The synchronization process will transfer all local items to a new
+    key in Redis and then overwrite the original key with the
+    `RENAME <https://redis.io/commands/rename>`__ operation.
+    This might not be appropriate for very large collections.
+
 """
 import collections.abc as collections_abc
 import collections
@@ -53,6 +59,10 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         self.sync()
 
+    def sync(self):
+        temp_collection = self.persistence_cls(redis=self.redis, data=self)
+        self.redis.rename(temp_collection.key, self.key)
+
 
 class SyncableDict(_SyncableBase, dict):
     """
@@ -63,16 +73,13 @@ class SyncableDict(_SyncableBase, dict):
     details.
     """
 
+    persistence_cls = Dict
+
     def __init__(self, **kwargs):
         self.persistence = Dict(**kwargs)
-
         super().__init__()
         self.update(self.persistence)
 
-    def sync(self):
-        self.persistence.clear()
-        self.persistence.update(self)
-
 
 class SyncableCounter(_SyncableBase, collections.Counter):
     """
@@ -84,16 +91,13 @@ class SyncableCounter(_SyncableBase, collections.Counter):
     for details.
     """
 
+    persistence_cls = Counter
+
     def __init__(self, **kwargs):
         self.persistence = Counter(**kwargs)
-
         super().__init__()
         self.update(self.persistence)
 
-    def sync(self):
-        self.persistence.clear()
-        self.persistence.update(self)
-
 
 class SyncableDefaultDict(_SyncableBase, collections.defaultdict):
     """
@@ -105,16 +109,13 @@ class SyncableDefaultDict(_SyncableBase, collections.defaultdict):
     #collections.defaultdict>`_ for details.
     """
 
+    persistence_cls = DefaultDict
+
     def __init__(self, *args, **kwargs):
         self.persistence = DefaultDict(*args, **kwargs)
-
         super().__init__(args[0] if args else None)
         self.update(self.persistence)
 
-    def sync(self):
-        self.persistence.clear()
-        self.persistence.update(self)
-
 
 class SyncableList(_SyncableBase, list):
     """
@@ -125,16 +126,13 @@ class SyncableList(_SyncableBase, list):
     #sequence-types-list-tuple-range>`__ for details.
     """
 
+    persistence_cls = List
+
     def __init__(self, **kwargs):
         self.persistence = List(**kwargs)
-
         super().__init__()
         self.extend(self.persistence)
 
-    def sync(self):
-        self.persistence.clear()
-        self.persistence.extend(self)
-
 
 class SyncableDeque(_SyncableBase, collections.deque):
     """
@@ -145,16 +143,13 @@ class SyncableDeque(_SyncableBase, collections.deque):
     for details.
     """
 
+    persistence_cls = Deque
+
     def __init__(self, iterable=None, maxlen=None, **kwargs):
         self.persistence = Deque(iterable=iterable, maxlen=maxlen, **kwargs)
-
         super().__init__(maxlen=self.persistence.maxlen)
         self.extend(self.persistence)
 
-    def sync(self):
-        self.persistence.clear()
-        self.persistence.extend(self)
-
 
 class SyncableSet(_SyncableBase, set):
     """
@@ -165,16 +160,13 @@ class SyncableSet(_SyncableBase, set):
     #set-types-set-frozenset>`__ for details.
     """
 
+    persistence_cls = Set
+
     def __init__(self, **kwargs):
         self.persistence = Set(**kwargs)
-
         super().__init__()
         self.update(self.persistence)
 
-    def sync(self):
-        self.persistence.clear()
-        self.persistence.update(self)
-
 
 class LRUDict(_SyncableBase, collections_abc.MutableMapping):
     """

tests/test_syncable.py

@@ -107,6 +107,7 @@ def test_defaultdict(self):
         del ddict_1['A']
         ddict_1.sync()
         self.assertNotIn('A', ddict_1.persistence)
+        self.assertEqual(ddict_1['A'], 0)
 
         # The default_factory can be changed between synchronizations
         key_1 = ddict_1.key