From be3d1dd244c9cc49463693d5728bc950284f9333 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Fri, 8 May 2026 20:54:57 +0000 Subject: [PATCH 1/4] fix: size _incr_decr key in bytes, not codepoints. The struct format spec for incr/decr was sized as COMMANDS[command]['struct'] % len(key), but the value packed into that slot is keybytes (UTF-8). For a non-ASCII key the byte length exceeds the codepoint count, the format spec under-sizes the field, and struct.pack silently truncates the encoded key. The header keylen advertises the full byte length, so the resulting wire packet is shorter than the server expects -- the server blocks reading the "missing" key bytes while the client blocks waiting for a response, deadlocking the connection. --- bmemcached/protocol.py | 2 +- test/test_simple_functions.py | 42 +++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/bmemcached/protocol.py b/bmemcached/protocol.py index b1132fb..bba99a5 100644 --- a/bmemcached/protocol.py +++ b/bmemcached/protocol.py @@ -873,7 +873,7 @@ def _incr_decr(self, command, key, value, default, time): keybytes = str_to_bytes(key) time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME self._send(struct.pack(self.HEADER_STRUCT + - self.COMMANDS[command]['struct'] % len(key), + self.COMMANDS[command]['struct'] % len(keybytes), self.MAGIC['request'], self.COMMANDS[command]['command'], len(keybytes), diff --git a/test/test_simple_functions.py b/test/test_simple_functions.py index ae3c024..1497d3f 100644 --- a/test/test_simple_functions.py +++ b/test/test_simple_functions.py @@ -385,6 +385,48 @@ def testDecrementInitialize(self): self.assertEqual(10, self.client.decr('test_key', 1, default=10)) self.assertEqual(9, self.client.decr('test_key', 1, default=10)) + def testNonAsciiKeySingle(self): + key = u'シシ' + try: + self.assertEqual(0, self.client.incr(key, 1)) + self.assertEqual(1, self.client.incr(key, 1)) + self.assertEqual(0, self.client.decr(key, 1)) + self.client.delete(key) + + self.assertTrue(self.client.set(key, 'v1')) + self.assertEqual('v1', self.client.get(key)) + + self.assertFalse(self.client.add(key, 'v2')) + self.assertTrue(self.client.replace(key, 'v3')) + self.assertEqual('v3', self.client.get(key)) + + value, cas = self.client.gets(key) + self.assertEqual('v3', value) + self.assertTrue(self.client.cas(key, 'v4', cas)) + self.assertEqual('v4', self.client.get(key)) + + self.assertTrue(self.client.delete(key)) + self.assertEqual(None, self.client.get(key)) + finally: + self.client.delete(key) + + def testNonAsciiKeyBulk(self): + keys = [u'café', u'日本語'] + try: + self.assertEqual([], self.client.set_multi({k: 'v' for k in keys})) + self.assertEqual({k: 'v' for k in keys}, self.client.get_multi(keys)) + + self.client.delete_multi(keys) + self.assertEqual({}, self.client.get_multi(keys)) + + result = self.client.set_multi_cas({k: 'w' for k in keys}) + for k in keys: + self.assertTrue(result[k] is not None) + self.assertEqual({k: 'w' for k in keys}, self.client.get_multi(keys)) + finally: + for k in keys: + self.client.delete(k) + def testFlush(self): self.client.set('test_key', 'test') self.assertTrue(self.client.flush_all()) From bf039fc46fba9583ed233802bfc709dc7657ae02 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Sun, 10 May 2026 01:49:39 +0000 Subject: [PATCH 2/4] fix: ensure serialize() always returns bytes. For int/long values serialize() did value = str(value), producing a Python 3 str. When that string is large enough to cross the COMPRESSION_THRESHOLD it falls into self.compression.compress(value), which rejects str with "TypeError: a bytes-like object is required". Encode to bytes inline in the int/long branches so every assignment to value in the type-dispatch already produces bytes; the binary, text, and pickler branches all already do. Reachable today via e.g. set('k', 10 ** 200): str(10 ** 200) is 201 chars, exceeds the 128-byte threshold, hits the compress branch, and raises before any caller-side coercion runs. The downstream text_type-to-bytes guards in _set_add_replace, set_multi, and set_multi_cas only cover the small-value path where compression is skipped; they cannot save the compress branch because the crash is inside serialize() itself. Drop the now-redundant guard in _set_add_replace. The function's :rtype: str was already inaccurate -- the text, binary, and pickler paths returned bytes; only int/long returned str. Update it to bytes now that the contract actually holds across every branch. --- bmemcached/protocol.py | 8 +++----- test/test_simple_functions.py | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bmemcached/protocol.py b/bmemcached/protocol.py index bba99a5..3863fc5 100644 --- a/bmemcached/protocol.py +++ b/bmemcached/protocol.py @@ -357,7 +357,7 @@ def serialize(self, value, compress_level=-1): -1 = default compression level. :type compress_level: int :return: Serialized type - :rtype: str + :rtype: bytes """ flags = 0 if isinstance(value, binary_type): @@ -366,10 +366,10 @@ def serialize(self, value, compress_level=-1): value = value.encode('utf8') elif isinstance(value, int) and isinstance(value, bool) is False: flags |= self.FLAGS['integer'] - value = str(value) + value = str(value).encode() elif isinstance(value, long) and isinstance(value, bool) is False: flags |= self.FLAGS['long'] - value = str(value) + value = str(value).encode() else: flags |= self.FLAGS['object'] buf = BytesIO() @@ -581,8 +581,6 @@ def _set_add_replace(self, command, key, value, time, cas=0, compress_level=-1): logger.debug('Setting/adding/replacing key %s.', key) flags, value = self.serialize(value, compress_level=compress_level) logger.debug('Value bytes %s.', len(value)) - if isinstance(value, text_type): - value = value.encode('utf8') keybytes = str_to_bytes(key) self._send(struct.pack(self.HEADER_STRUCT + diff --git a/test/test_simple_functions.py b/test/test_simple_functions.py index 1497d3f..f155af1 100644 --- a/test/test_simple_functions.py +++ b/test/test_simple_functions.py @@ -410,6 +410,11 @@ def testNonAsciiKeySingle(self): finally: self.client.delete(key) + def testSetLargeNumeric(self): + big = 10 ** 200 + self.client.set('test_key', big) + self.assertEqual(big, self.client.get('test_key')) + def testNonAsciiKeyBulk(self): keys = [u'café', u'日本語'] try: From 8d193b49e9f3620b5331d26244afad16e2ee73ef Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Wed, 29 Apr 2026 16:30:42 +0000 Subject: [PATCH 3/4] perf: replace COMMANDS struct format strings with precompiled packers. Each entry in the COMMANDS table now carries a precompiled struct.Struct for the fixed-size prefix of its wire format (HEADER_STRUCT plus any leading "extras" bytes); variable-length tails (key, value, auth payloads) are concatenated as bytes after packer.pack(...). Previously each call site built a fresh format string from HEADER_STRUCT + the per-command 'struct' suffix and substituted in the per-call lengths via "%". This costs a string concat plus a % format on every call, and -- because the resulting format string embeds the per-call lengths ('17s', '23s', ...) -- defeats the LRU cache that the struct module keeps over compiled formats. Once the working set of distinct lengths exceeds that cache (100 entries by default in CPython), every call recompiles its format from scratch. The hot build loops in get_multi, set_multi, and set_multi_cas paid this on every key; additionally bind packer.pack and the relevant COMMANDS / MAGIC / STATUS lookups to locals. Microbench (500-key get_multi request build, no network): 1181us -> 441us, ~2.7x. --- bmemcached/protocol.py | 266 ++++++++++++++++++---------------- test/test_simple_functions.py | 17 +++ 2 files changed, 159 insertions(+), 124 deletions(-) diff --git a/bmemcached/protocol.py b/bmemcached/protocol.py index 3863fc5..2a90cd0 100644 --- a/bmemcached/protocol.py +++ b/bmemcached/protocol.py @@ -54,24 +54,27 @@ class Protocol(threading.local): 'response': 0x81 } - # All structures will be appended to HEADER_STRUCT + # 'packer' is a struct.Struct compiled from HEADER_STRUCT plus the + # fixed-size leading "extras" bytes for that command. Variable-length + # tails (key, value, auth payloads) are concatenated as bytes after + # packer.pack(...). COMMANDS = { - 'get': {'command': 0x00, 'struct': '%ds'}, - 'getk': {'command': 0x0C, 'struct': '%ds'}, - 'getkq': {'command': 0x0D, 'struct': '%ds'}, - 'set': {'command': 0x01, 'struct': 'LL%ds%ds'}, - 'setq': {'command': 0x11, 'struct': 'LL%ds%ds'}, - 'add': {'command': 0x02, 'struct': 'LL%ds%ds'}, - 'addq': {'command': 0x12, 'struct': 'LL%ds%ds'}, - 'replace': {'command': 0x03, 'struct': 'LL%ds%ds'}, - 'delete': {'command': 0x04, 'struct': '%ds'}, - 'incr': {'command': 0x05, 'struct': 'QQL%ds'}, - 'decr': {'command': 0x06, 'struct': 'QQL%ds'}, - 'flush': {'command': 0x08, 'struct': 'I'}, - 'noop': {'command': 0x0a, 'struct': ''}, - 'stat': {'command': 0x10}, - 'auth_negotiation': {'command': 0x20}, - 'auth_request': {'command': 0x21, 'struct': '%ds%ds'}, + 'get': {'command': 0x00, 'packer': struct.Struct(HEADER_STRUCT)}, + 'getk': {'command': 0x0C, 'packer': struct.Struct(HEADER_STRUCT)}, + 'getkq': {'command': 0x0D, 'packer': struct.Struct(HEADER_STRUCT)}, + 'set': {'command': 0x01, 'packer': struct.Struct(HEADER_STRUCT + 'LL')}, + 'setq': {'command': 0x11, 'packer': struct.Struct(HEADER_STRUCT + 'LL')}, + 'add': {'command': 0x02, 'packer': struct.Struct(HEADER_STRUCT + 'LL')}, + 'addq': {'command': 0x12, 'packer': struct.Struct(HEADER_STRUCT + 'LL')}, + 'replace': {'command': 0x03, 'packer': struct.Struct(HEADER_STRUCT + 'LL')}, + 'delete': {'command': 0x04, 'packer': struct.Struct(HEADER_STRUCT)}, + 'incr': {'command': 0x05, 'packer': struct.Struct(HEADER_STRUCT + 'QQL')}, + 'decr': {'command': 0x06, 'packer': struct.Struct(HEADER_STRUCT + 'QQL')}, + 'flush': {'command': 0x08, 'packer': struct.Struct(HEADER_STRUCT + 'I')}, + 'noop': {'command': 0x0a, 'packer': struct.Struct(HEADER_STRUCT)}, + 'stat': {'command': 0x10, 'packer': struct.Struct(HEADER_STRUCT)}, + 'auth_negotiation': {'command': 0x20, 'packer': struct.Struct(HEADER_STRUCT)}, + 'auth_request': {'command': 0x21, 'packer': struct.Struct(HEADER_STRUCT)}, } STATUS = { @@ -297,10 +300,10 @@ def _send_authentication(self): return False logger.debug('Authenticating as %s', self._username) - self._send(struct.pack(self.HEADER_STRUCT, - self.MAGIC['request'], - self.COMMANDS['auth_negotiation']['command'], - 0, 0, 0, 0, 0, 0, 0)) + cmd = self.COMMANDS['auth_negotiation'] + self._send(cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + 0, 0, 0, 0, 0, 0, 0)) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() @@ -324,10 +327,10 @@ def _send_authentication(self): if isinstance(auth, text_type): auth = auth.encode() - self._send(struct.pack(self.HEADER_STRUCT + - self.COMMANDS['auth_request']['struct'] % (len(method), len(auth)), - self.MAGIC['request'], self.COMMANDS['auth_request']['command'], - len(method), 0, 0, 0, len(method) + len(auth), 0, 0, method, auth)) + cmd = self.COMMANDS['auth_request'] + self._send(cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + len(method), 0, 0, 0, len(method) + len(auth), 0, 0) + method + auth) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() @@ -447,11 +450,11 @@ def get(self, key): """ logger.debug('Getting key %s', key) keybytes = str_to_bytes(key) - data = struct.pack(self.HEADER_STRUCT + - self.COMMANDS['get']['struct'] % (len(keybytes),), - self.MAGIC['request'], - self.COMMANDS['get']['command'], - len(keybytes), 0, 0, 0, len(keybytes), 0, 0, keybytes) + cmd = self.COMMANDS['get'] + klen = len(keybytes) + data = cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + klen, 0, 0, 0, klen, 0, 0) + keybytes self._send(data) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, @@ -482,11 +485,10 @@ def noop(self): :rtype: int """ logger.debug('Sending NOOP') - data = struct.pack(self.HEADER_STRUCT + - self.COMMANDS['noop']['struct'], - self.MAGIC['request'], - self.COMMANDS['noop']['command'], - 0, 0, 0, 0, 0, 0, 0) + cmd = self.COMMANDS['noop'] + data = cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + 0, 0, 0, 0, 0, 0, 0) self._send(data) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, @@ -521,38 +523,45 @@ def get_multi(self, keys): if n == 0: return {} + MAGIC_REQ = self.MAGIC['request'] + getkq = self.COMMANDS['getkq'] + GETKQ_CMD = getkq['command'] + pack_header = getkq['packer'].pack # same packer for getk and getkq + GETK_CMD = self.COMMANDS['getk']['command'] + msg = bytearray() - for i, key in enumerate(keys): - keybytes = str_to_bytes(key) - command = self.COMMANDS['getk' if i == n - 1 else 'getkq'] - msg += struct.pack(self.HEADER_STRUCT + - command['struct'] % (len(keybytes),), - self.MAGIC['request'], - command['command'], - len(keybytes), 0, 0, 0, len(keybytes), 0, 0, keybytes) + keybytes_list = [str_to_bytes(k) for k in keys] + last = n - 1 + for i, keybytes in enumerate(keybytes_list): + klen = len(keybytes) + opcode = GETK_CMD if i == last else GETKQ_CMD + msg += pack_header(MAGIC_REQ, opcode, klen, 0, 0, 0, klen, 0, 0) + msg += keybytes self._send(msg) d = {} + SUCCESS = self.STATUS['success'] + DISCONNECTED = self.STATUS['server_disconnected'] + NOT_FOUND = self.STATUS['key_not_found'] opcode = -1 - while opcode != self.COMMANDS['getk']['command']: + while opcode != GETK_CMD: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() - if status == self.STATUS['success']: + if status == SUCCESS: flags, key, value = struct.unpack('!L%ds%ds' % (keylen, bodylen - keylen - 4), extra_content) d[key] = self.deserialize(value, flags), cas - elif status == self.STATUS['server_disconnected']: + elif status == DISCONNECTED: break - elif status != self.STATUS['key_not_found']: + elif status != NOT_FOUND: raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status) ret = {} - for key in keys: - keybytes = str_to_bytes(key) + for key, keybytes in zip(keys, keybytes_list): if keybytes in d: ret[key] = d[keybytes] return ret @@ -583,12 +592,13 @@ def _set_add_replace(self, command, key, value, time, cas=0, compress_level=-1): logger.debug('Value bytes %s.', len(value)) keybytes = str_to_bytes(key) - self._send(struct.pack(self.HEADER_STRUCT + - self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), - self.MAGIC['request'], - self.COMMANDS[command]['command'], - len(keybytes), 8, 0, 0, len(keybytes) + len(value) + 8, 0, cas, flags, - time, keybytes, value)) + cmd = self.COMMANDS[command] + klen = len(keybytes) + vlen = len(value) + self._send(cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + klen, 8, 0, 0, klen + vlen + 8, 0, cas, + flags, time) + keybytes + value) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() @@ -739,6 +749,12 @@ def set_multi(self, mappings, time=100, compress_level=-1): mappings = list(mappings.items()) msg = bytearray() + MAGIC_REQ = self.MAGIC['request'] + addq = self.COMMANDS['addq'] + ADDQ_CMD = addq['command'] + pack_set_prefix = addq['packer'].pack # same packer for setq/addq + SETQ_CMD = self.COMMANDS['setq']['command'] + for opaque, (key, value) in enumerate(mappings): if isinstance(key, tuple): key, cas = key @@ -748,37 +764,38 @@ def set_multi(self, mappings, time=100, compress_level=-1): if cas == 0: # Like cas(), if the cas value is 0, treat it as compare-and-set against not # existing. - command = 'addq' + opcode = ADDQ_CMD else: - command = 'setq' + opcode = SETQ_CMD keybytes = str_to_bytes(key) flags, value = self.serialize(value, compress_level=compress_level) - msg += struct.pack(self.HEADER_STRUCT + - self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), - self.MAGIC['request'], - self.COMMANDS[command]['command'], - len(keybytes), - 8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0, - flags, time, keybytes, value) - - msg += struct.pack(self.HEADER_STRUCT + - self.COMMANDS['noop']['struct'], - self.MAGIC['request'], - self.COMMANDS['noop']['command'], - 0, 0, 0, 0, 0, 0, 0) + klen = len(keybytes) + vlen = len(value) + msg += pack_set_prefix(MAGIC_REQ, opcode, klen, + 8, 0, 0, klen + vlen + 8, opaque, cas or 0, + flags, time) + msg += keybytes + msg += value + + noop = self.COMMANDS['noop'] + NOOP_CMD = noop['command'] + msg += noop['packer'].pack(MAGIC_REQ, NOOP_CMD, + 0, 0, 0, 0, 0, 0, 0) self._send(msg) opcode = -1 failed = [] - while opcode != self.COMMANDS['noop']['command']: + DISCONNECTED = self.STATUS['server_disconnected'] + SUCCESS = self.STATUS['success'] + while opcode != NOOP_CMD: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() - if status == self.STATUS['server_disconnected']: + if status == DISCONNECTED: # Assume that the entire operation failed. return list(key for key, value in mappings) - if status != self.STATUS['success']: + if status != SUCCESS: key, value = mappings[opaque] if isinstance(key, tuple): failed.append((key[0], cas)) @@ -815,6 +832,12 @@ def set_multi_cas(self, mappings, time=100, compress_level=-1): msg = bytearray() result = {} + MAGIC_REQ = self.MAGIC['request'] + add = self.COMMANDS['add'] + ADD_CMD = add['command'] + pack_set_prefix = add['packer'].pack # same packer for set/add + SET_CMD = self.COMMANDS['set']['command'] + for opaque, (key, value) in enumerate(mappings): if isinstance(key, tuple): str_key, cas = key @@ -823,30 +846,32 @@ def set_multi_cas(self, mappings, time=100, compress_level=-1): result[str_key] = None if cas == 0: - command = 'add' + opcode = ADD_CMD else: - command = 'set' + opcode = SET_CMD keybytes = str_to_bytes(str_key) flags, value = self.serialize(value, compress_level=compress_level) - msg += struct.pack(self.HEADER_STRUCT + - self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), - self.MAGIC['request'], - self.COMMANDS[command]['command'], - len(keybytes), - 8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0, - flags, time, keybytes, value) + klen = len(keybytes) + vlen = len(value) + msg += pack_set_prefix(MAGIC_REQ, opcode, klen, + 8, 0, 0, klen + vlen + 8, opaque, cas or 0, + flags, time) + msg += keybytes + msg += value self._send(msg) # Non-quiet set/add return exactly one response per request, so we can # read a fixed count rather than relying on a trailing noop sentinel. + DISCONNECTED = self.STATUS['server_disconnected'] + SUCCESS = self.STATUS['success'] for _ in range(len(mappings)): (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() - if status == self.STATUS['server_disconnected']: + if status == DISCONNECTED: return result - if status == self.STATUS['success']: + if status == SUCCESS: key, value = mappings[opaque] str_key = key[0] if isinstance(key, tuple) else key result[str_key] = cas @@ -870,13 +895,12 @@ def _incr_decr(self, command, key, value, default, time): """ keybytes = str_to_bytes(key) time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME - self._send(struct.pack(self.HEADER_STRUCT + - self.COMMANDS[command]['struct'] % len(keybytes), - self.MAGIC['request'], - self.COMMANDS[command]['command'], - len(keybytes), - 20, 0, 0, len(keybytes) + 20, 0, 0, value, - default, time, keybytes)) + cmd = self.COMMANDS[command] + klen = len(keybytes) + self._send(cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + klen, 20, 0, 0, klen + 20, 0, 0, + value, default, time) + keybytes) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() @@ -936,11 +960,11 @@ def delete(self, key, cas=0): """ logger.debug('Deleting key %s', key) keybytes = str_to_bytes(key) - self._send(struct.pack(self.HEADER_STRUCT + - self.COMMANDS['delete']['struct'] % (len(keybytes),), - self.MAGIC['request'], - self.COMMANDS['delete']['command'], - len(keybytes), 0, 0, 0, len(keybytes), 0, cas, keybytes)) + cmd = self.COMMANDS['delete'] + klen = len(keybytes) + self._send(cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + klen, 0, 0, 0, klen, 0, cas) + keybytes) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() @@ -964,27 +988,25 @@ def delete_multi(self, keys): """ logger.debug('Deleting keys %r', keys) msg = bytearray() + delete = self.COMMANDS['delete'] + DELETE_CMD = delete['command'] + pack_header = delete['packer'].pack # same packer as noop + MAGIC_REQ = self.MAGIC['request'] for key in keys: keybytes = str_to_bytes(key) - msg += struct.pack( - self.HEADER_STRUCT + - self.COMMANDS['delete']['struct'] % (len(keybytes),), - self.MAGIC['request'], - self.COMMANDS['delete']['command'], - len(keybytes), 0, 0, 0, len(keybytes), 0, 0, keybytes) - - msg += struct.pack( - self.HEADER_STRUCT + - self.COMMANDS['noop']['struct'], - self.MAGIC['request'], - self.COMMANDS['noop']['command'], - 0, 0, 0, 0, 0, 0, 0) + klen = len(keybytes) + msg += pack_header(MAGIC_REQ, DELETE_CMD, klen, 0, 0, 0, klen, 0, 0) + msg += keybytes + + noop = self.COMMANDS['noop'] + NOOP_CMD = noop['command'] + msg += noop['packer'].pack(MAGIC_REQ, NOOP_CMD, 0, 0, 0, 0, 0, 0, 0) self._send(msg) opcode = -1 retval = True - while opcode != self.COMMANDS['noop']['command']: + while opcode != NOOP_CMD: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status != self.STATUS['success']: @@ -1004,11 +1026,10 @@ def flush_all(self, time): :rtype: bool """ logger.info('Flushing memcached') - self._send(struct.pack(self.HEADER_STRUCT + - self.COMMANDS['flush']['struct'], - self.MAGIC['request'], - self.COMMANDS['flush']['command'], - 0, 4, 0, 0, 4, 0, 0, time)) + cmd = self.COMMANDS['flush'] + self._send(cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + 0, 4, 0, 0, 4, 0, 0, time)) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() @@ -1029,20 +1050,17 @@ def stats(self, key=None): :rtype: dict """ # TODO: Stats with key is not working. + cmd = self.COMMANDS['stat'] if key is not None: if isinstance(key, text_type): key = str_to_bytes(key) keylen = len(key) - packed = struct.pack( - self.HEADER_STRUCT + '%ds' % keylen, - self.MAGIC['request'], - self.COMMANDS['stat']['command'], - keylen, 0, 0, 0, keylen, 0, 0, key) + packed = cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], + keylen, 0, 0, 0, keylen, 0, 0) + key else: - packed = struct.pack( - self.HEADER_STRUCT, - self.MAGIC['request'], - self.COMMANDS['stat']['command'], + packed = cmd['packer'].pack( + self.MAGIC['request'], cmd['command'], 0, 0, 0, 0, 0, 0, 0) self._send(packed) diff --git a/test/test_simple_functions.py b/test/test_simple_functions.py index f155af1..0135798 100644 --- a/test/test_simple_functions.py +++ b/test/test_simple_functions.py @@ -42,6 +42,23 @@ def testSetMultiBigData(self): self.client.set_multi( dict((unicode(k), b'value') for k in range(32767))) + def testSetMultiNumericValues(self): + six.assertCountEqual(self, self.client.set_multi({ + 'test_key': 42, + 'test_key2': long(2 ** 40), + }), []) + self.assertEqual(self.client.get('test_key'), 42) + self.assertEqual(self.client.get('test_key2'), 2 ** 40) + + result = self.client.set_multi_cas({ + 'test_key': 7, + 'test_key2': long(2 ** 40 + 1), + }) + self.assertTrue(result['test_key'] is not None) + self.assertTrue(result['test_key2'] is not None) + self.assertEqual(self.client.get('test_key'), 7) + self.assertEqual(self.client.get('test_key2'), 2 ** 40 + 1) + def testGetSimple(self): self.client.set('test_key', 'test') self.assertEqual('test', self.client.get('test_key')) From 1bb563c1e5f8f2f9f3c4a2a150dc7b341e4e2605 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Wed, 29 Apr 2026 16:17:09 +0000 Subject: [PATCH 4/4] perf: use pickle.loads / pickle.dumps when self.(un)pickler is the default. The serialize / deserialize paths construct a BytesIO and a Pickler or Unpickler instance per call to support a user-overridable pickler class. When self.(un)pickler is pickle.(Un)Pickler -- the default -- this is equivalent to pickle.dumps / pickle.loads, which are implemented in C (_pickle) and skip the Python-level allocation. Microbench (round-trip a small dict): BytesIO+Unpickler.load 3.08us vs pickle.loads 1.59us, ~1.94x. --- bmemcached/protocol.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bmemcached/protocol.py b/bmemcached/protocol.py index 2a90cd0..59cc767 100644 --- a/bmemcached/protocol.py +++ b/bmemcached/protocol.py @@ -14,7 +14,7 @@ import six from six import binary_type, text_type -from bmemcached.compat import long +from bmemcached.compat import long, pickle from bmemcached.exceptions import AuthenticationNotSupported, InvalidCredentials, MemcachedException from bmemcached.utils import str_to_bytes @@ -375,10 +375,13 @@ def serialize(self, value, compress_level=-1): value = str(value).encode() else: flags |= self.FLAGS['object'] - buf = BytesIO() - pickler = self.pickler(buf, self.pickle_protocol) - pickler.dump(value) - value = buf.getvalue() + if self.pickler is None or self.pickler is pickle.Pickler: + value = pickle.dumps(value, self.pickle_protocol) + else: + buf = BytesIO() + pickler = self.pickler(buf, self.pickle_protocol) + pickler.dump(value) + value = buf.getvalue() if compress_level != 0 and len(value) > self.COMPRESSION_THRESHOLD: if compress_level is not None and compress_level > 0: @@ -418,9 +421,9 @@ def deserialize(self, value, flags): elif flags & FLAGS['long']: return long(value) elif flags & FLAGS['object']: - buf = BytesIO(value) - unpickler = self.unpickler(buf) - return unpickler.load() + if self.unpickler is None or self.unpickler is pickle.Unpickler: + return pickle.loads(value) + return self.unpickler(BytesIO(value)).load() if six.PY3: return value.decode('utf8')