Use msgpack to serialize queue in ingestion-edge

fixes #103
mozilla · Dec 31, 2018 · 0f22ec8 · 0f22ec8
1 parent 9843c54
commit 0f22ec8
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 1 deletion.
diff --git a/ingestion-edge/ingestion_edge/config.py b/ingestion-edge/ingestion_edge/config.py
@@ -13,6 +13,7 @@
 from os import environ
 from typing import Tuple
 import json
+import persistqueue.serializers.msgpack
 
 dictConfig(
     {
@@ -48,6 +49,8 @@ class Route:
 
 QUEUE_PATH = environ.get("QUEUE_PATH", "queue")
 
+QUEUE_SERIALIZER = persistqueue.serializers.msgpack
+
 MINIMUM_DISK_FREE_BYTES = int(environ.get("MINIMUM_DISK_FREE_BYTES", 0)) or None
 
 DEFAULT_METADATA_HEADERS = [

diff --git a/ingestion-edge/requirements.txt b/ingestion-edge/requirements.txt
@@ -2,6 +2,7 @@
 aiohttp==3.5.1 --hash=sha256:359baeea2ca640e0dde31a03c3bf3d3008bcbd136c6b1768b58a3499a46a6cc2 --hash=sha256:55355947c4fe4b37d2a51b8f1d3f36f7fca541cf012031225be836d1f743c011 --hash=sha256:6739494376c90806cbb88e7ea2c9e2c35949e6c7089507d19e8f489170a26156 --hash=sha256:c642901f6c53b965785e57a597229dd87910991b3e2d8aecf552da7d48cfe170 --hash=sha256:3011371a48fdef061a8669b6636306b33cf2bf621e1960513c6ce70449f7cd3d --hash=sha256:dd07976a2f2615d4f2ed3654b24e53fe837708602c00934ce1e963690c91c933 --hash=sha256:b9def7acd7c84ca86d0c3247e83180782c423d0e8a68254718fcc69e521570da --hash=sha256:5691c630435fd6bd09a789de9ffd5a61b812445dfd515525c738a97d4f9b550a --hash=sha256:310c95f1da5f92e937b136e55c2013e4bccd1b53bc88780256ba8ed75699dbdb --hash=sha256:bb96d5e0a82f67a04cde32f970ca837fbcf7ef44124170bc5e34f26c0ed92f7d --hash=sha256:c9b47b2ee669b2f01824e0f3b364a8cdfab8d40df1b5987c7c2103d3e13ec9e9 --hash=sha256:0bbaec0b171b1ea77d34bc7c49db71a15e511ef34c45065fd2c7fad8daf1483f --hash=sha256:f73d6a3e711f26be58bfa13a65a425638fa9d3f4a081eebff0eb70e42fee40a8 --hash=sha256:e3b29248c9180fd6a30619b2714c534e3165e523a568296250337fe8952d39b8 --hash=sha256:168f0ecc91200784467479765eb26a80d6d9cf0025b8a9cc5e501413812d32e7 --hash=sha256:53fc0ad2e8d8f2f0c87bdc3009784de61f5dd9a4259f67301b317525eedc3ed5 --hash=sha256:f438eab30868997407b73814ba097b80862d6d5bc5f7f2fda384e60df769777b --hash=sha256:af664f067d3c905f4f44d724e65406ed95dd2b4adfcc3d23a9203320ce497950 --hash=sha256:5202ac2d00226f0b2990af9f3301c1ba5eebb673ae0a0acfe499eaea8a1b23ad --hash=sha256:ed65392135299698b0ebff4ee53ccf19d5c7c12077652a7faab05db369eb3996 --hash=sha256:a68232a60b8c1a822c4ac4096bfb42b4f873ac7dcef265642223690220b5af4f --hash=sha256:c115744b2a0bf666fd8cde52a6d3e9319ffeb486009579743f5adfdcf0bf0773
 google-cloud-pubsub==0.39.1 --hash=sha256:32dbaf9b8c16d4a14b7e1eca2805689f82f69f542a68bf1516b9c77a3ef54b73 --hash=sha256:4186386aec02752e982eeb1e399d76f1cf70eed56312934df04bfa68d8cfabf0
 gunicorn==19.9.0 --hash=sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471
+msgpack==6.0 --hash=sha256:b688721df31c4bad6f508fb262719eb7e4a3532024c66d3c44ad6a4704519dda
 persist-queue==0.4.1 --hash=sha256:5411a0b156f754aacc75dd52c3dc3ebeac83a0b3d11f874a69d91b5bf1a21859 --hash=sha256:16604b8ca4f51e1d8567e2847cea08dffd50d35e1a65ed3f14f02ed38bcfc55c
 pytest-mypy==0.3.2 --hash=sha256:8f6436eed8118afd6c10a82b3b60fb537336736b0fd7a29262a656ac42ce01ac
 pytest==4.0.2 --hash=sha256:f812ea39a0153566be53d88f8de94839db1e8a05352ed8a49525d7d7f37861e9 --hash=sha256:f689bf2fc18c4585403348dd56f47d87780bf217c53ed9ae7a3e2d7faa45f8e9

diff --git a/ingestion-edge/tests/unit/conftest.py b/ingestion-edge/tests/unit/conftest.py
@@ -7,6 +7,7 @@
 from sanic import Sanic
 import grpc
 import os
+import persistqueue.serializers.msgpack
 import pytest
 
 
@@ -25,4 +26,4 @@ def client() -> PublisherClient:
 
 @pytest.fixture
 def q() -> SQLiteAckQueue:
-    return SQLiteAckQueue(":memory:")
+    return SQLiteAckQueue(path=":memory:", serializer=persisqueue.serializers.msgpack)