Add support for bypassing pre-packed data

Author: marcoffee

msgpack/__init__.py

@@ -2,7 +2,7 @@
 import os
 
 from .exceptions import *  # noqa: F403
-from .ext import ExtType, Timestamp
+from .ext import Bypass, ExtType, Timestamp
 
 version = (1, 2, 1)
 __version__ = "1.2.1"

msgpack/_packer.pyx

@@ -8,7 +8,7 @@ from cpython.datetime cimport (
 cdef ExtType
 cdef Timestamp
 
-from .ext import ExtType, Timestamp
+from .ext import ExtType, Timestamp, Bypass
 
 
 cdef extern from "Python.h":
@@ -222,6 +222,8 @@ cdef class Packer:
             llval = o.seconds
             ulval = o.nanoseconds
             msgpack_pack_timestamp(&self.pk, llval, ulval)
+        elif type(o) is Bypass:
+            msgpack_pack_raw_body(&self.pk, o.data, len(o.data))
         elif PyList_CheckExact(o) if strict else (PyTuple_Check(o) or PyList_Check(o)):
             L = Py_SIZE(o)
             if L > ITEM_LIMIT:

msgpack/ext.py

@@ -18,6 +18,19 @@ def __new__(cls, code, data):
         return super().__new__(cls, code, data)
 
 
+class Bypass:
+    """Bypass is a placeholder class to skip serialization and pass the bytes value as is."""
+
+    __slots__ = ["data"]
+
+    def __init__(self, data):
+        if isinstance(data, bytes):
+            self.data = data
+
+        else:
+            self.data = memoryview(data).tobytes()
+
+
 class Timestamp:
     """Timestamp represents the Timestamp extension type in msgpack.
 

msgpack/fallback.py

@@ -38,7 +38,7 @@ def newlist_hint(size):
 
 
 from .exceptions import BufferFull, ExtraData, FormatError, OutOfData, StackError
-from .ext import ExtType, Timestamp
+from .ext import Bypass, ExtType, Timestamp
 
 EX_SKIP = 0
 EX_CONSTRUCT = 1
@@ -779,6 +779,9 @@ def _pack(
                 self._buffer.write(struct.pack("b", code))
                 self._buffer.write(data)
                 return
+            if check(obj, Bypass):
+                self._buffer.write(obj.data)
+                return
             if check(obj, list_types):
                 n = len(obj)
                 self._pack_array_header(n)

test/test_bypass.py

@@ -0,0 +1,57 @@
+from msgpack import Bypass, packb, unpackb
+
+
+def test_roundtrip_bypassed_bytes():
+    binary = b"\x00\x01\x02\x03\x04\x05"
+
+    binary_packed = packb(binary)
+    binary_bypassed_packed = packb(Bypass(binary_packed))
+
+    assert binary_bypassed_packed == binary_packed
+    assert unpackb(binary_packed) == binary
+
+
+def test_roundtrip_bypassed_object():
+    obj = {"key0": {"key1": {"key2": {"key3": [1, 2, 3]}}}}
+
+    obj_packed = packb(obj)
+    obj_bypassed_packed = packb(Bypass(obj_packed))
+
+    assert obj_bypassed_packed == obj_packed
+    assert unpackb(obj_bypassed_packed) == obj
+
+
+def test_roundtrip_deep_bypassed_object():
+    obj = {"key0": {"key1": {"key2": {"key3": [1, 2, 3]}}}}
+    obj_bypassed = {"key0": {"key1": {"key2": {"key3": [1, 2, Bypass(packb(3))]}}}}
+
+    assert packb(obj) == packb(obj_bypassed)
+    assert unpackb(packb(obj)) == obj
+
+
+def test_roundtrip_bypassed_object_with_cache():
+    class SimpleCache:
+        def __init__(self, obj):
+            self.obj = obj
+            self.packed = None
+
+        def get_packed(self):
+            if self.packed is None:
+                self.packed = packb(self.obj)
+
+            return Bypass(self.packed)
+
+    def default(obj):
+        if isinstance(obj, SimpleCache):
+            return obj.get_packed()
+        return obj
+
+    obj = {"key0": {"key1": {"key2": {"key3": [1, 2, 3]}}}}
+    cache = SimpleCache(obj)
+
+    obj_packed = packb(cache.obj, default=default)
+    cache.packed = obj_packed
+    obj_bypassed_packed = packb(Bypass(cache.packed))
+
+    assert obj_bypassed_packed == obj_packed
+    assert unpackb(obj_bypassed_packed) == cache.obj