From 0a203d76bc91a01d54405ca7556192c473f80841 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Wed, 3 Mar 2021 15:16:29 +0100 Subject: [PATCH 01/68] change logging --- examples/pack.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/pack.py b/examples/pack.py index 2b57cf5..110bb46 100644 --- a/examples/pack.py +++ b/examples/pack.py @@ -10,7 +10,7 @@ import ubirch -logging.basicConfig(format='%(asctime)s %(name)20.20s %(levelname)-8.8s %(message)s', level=logging.DEBUG) +logging.basicConfig(format='%(asctime)s %(name)20.20s %(levelname)-8.8s %(message)s', level=logging.INFO) logger = logging.getLogger() @@ -26,7 +26,7 @@ def load(self, uuid: UUID): try: with open(uuid.hex + ".sig", "rb") as f: signatures = pickle.load(f) - logger.info("loaded {} known signatures".format(len(signatures))) + logger.debug("loaded {} known signatures".format(len(signatures))) self.set_saved_signatures(signatures) except: logger.warning("no existing saved signatures") @@ -45,8 +45,8 @@ def _sign(self, uuid: UUID, message: bytes) -> bytes: if len(sys.argv) < 2: print("usage:") - print(" python ./pack.py ") - print(" e.g.: python ./pack.py 56bd9b85-6c6e-4a24-bf71-f2ac2de10183") + print(" python3 ./pack.py ") + print(" e.g.: python3 ./pack.py 56bd9b85-6c6e-4a24-bf71-f2ac2de10183") sys.exit(0) uuid = UUID(hex=sys.argv[1]) @@ -58,6 +58,9 @@ def _sign(self, uuid: UUID, message: bytes) -> bytes: if not keystore.exists_signing_key(uuid): keystore.create_ed25519_keypair(uuid) +logger.info("public key [base64]: {}".format( + binascii.b2a_base64(keystore.find_verifying_key(uuid).to_bytes()).decode().rstrip('\n'))) + # create an instance of the protocol with signature saving protocol = Proto(keystore) protocol.load(uuid) @@ -73,11 +76,11 @@ def _sign(self, uuid: UUID, message: bytes) -> bytes: serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() # calculate the hash of the message -message_hash = hashlib.sha256(serialized).digest() +message_hash = hashlib.sha512(serialized).digest() # create a new chained protocol message with the hash of the message upp = protocol.message_chained(uuid, 0x00, message_hash) -logger.info("UPP: {}".format(binascii.hexlify(upp))) +logger.info("UPP: {}".format(binascii.hexlify(upp).decode())) # store signature persistently for chaining protocol.persist(uuid) From 7658673ebc4a59282592cc4e470db70b966e046b Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Wed, 3 Mar 2021 15:19:03 +0100 Subject: [PATCH 02/68] separate unpacking and verifying messages --- ubirch/ubirch_protocol.py | 135 +++++++++++++++++++++++++++++++++----- 1 file changed, 117 insertions(+), 18 deletions(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index 45b74f0..4626dc8 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -18,6 +18,8 @@ import logging from abc import abstractmethod from uuid import UUID +from ed25519 import BadSignatureError +from ecdsa.keys import BadSignatureError as BadSignatureErrorEcdsa import msgpack @@ -26,14 +28,57 @@ # ubirch-protocol constants UBIRCH_PROTOCOL_VERSION = 2 -PLAIN = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x01) -SIGNED = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x02) +PLAIN = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x01) +SIGNED = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x02) CHAINED = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x03) UBIRCH_PROTOCOL_TYPE_BIN = 0x00 UBIRCH_PROTOCOL_TYPE_REG = 0x01 UBIRCH_PROTOCOL_TYPE_HSK = 0x02 +# for use with the "get_unpacked_index" function +UNPACKED_UPP_FIELD_VERSION = 0 +UNPACKED_UPP_FIELD_UUID = 1 +UNPACKED_UPP_FIELD_PREV_SIG = 2 +UNPACKED_UPP_FIELD_TYPE = 3 +UNPACKED_UPP_FIELD_PAYLOAD = 4 +UNPACKED_UPP_FIELD_SIG = 5 + +# lookup tables for fields in unpacked upps (used by the "get_unpacked_index" function) + +# message without any signatures +# 0 | 1 | 2 | 3 +# --------|------|------|-------- +# VERSION | UUID | TYPE | PAYLOAD +UNPACKED_UNSIGNED_UPP_INDEX_TABLE = [-1, -1, -1, -1, -1, -1] +UNPACKED_UNSIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_VERSION] = 0 +UNPACKED_UNSIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_UUID] = 1 +UNPACKED_UNSIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_TYPE] = 2 +UNPACKED_UNSIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_PAYLOAD] = 3 + +# message without the previous signature, contains a message signature +# 0 | 1 | 2 | 3 | 4 +# --------|------|------|---------|----------- +# VERSION | UUID | TYPE | PAYLOAD | SIGNATURE +UNPACKED_SIGNED_UPP_INDEX_TABLE = [-1, -1, -1, -1, -1, -1] +UNPACKED_SIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_VERSION] = 0 +UNPACKED_SIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_UUID] = 1 +UNPACKED_SIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_TYPE] = 2 +UNPACKED_SIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_PAYLOAD] = 3 +UNPACKED_SIGNED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_SIG] = 4 + +# message with all signatures +# 0 | 1 | 2 | 3 | 4 | 5 +# --------|------|----------------|------|---------|---------- +# VERSION | UUID | PREV-SIGNATURE | TYPE | PAYLOAD | SIGNATURE +UNPACKED_CHAINED_UPP_INDEX_TABLE = [-1, -1, -1, -1, -1, -1] +UNPACKED_CHAINED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_VERSION] = 0 +UNPACKED_CHAINED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_UUID] = 1 +UNPACKED_CHAINED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_PREV_SIG] = 2 +UNPACKED_CHAINED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_TYPE] = 3 +UNPACKED_CHAINED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_PAYLOAD] = 4 +UNPACKED_CHAINED_UPP_INDEX_TABLE[UNPACKED_UPP_FIELD_SIG] = 5 + class Protocol(object): _signatures = {} @@ -182,26 +227,80 @@ def _prepare_and_verify(self, uuid: UUID, message: bytes, signature: bytes) -> b """ return self._verify(uuid, self._hash(message), signature) - def message_verify(self, message: bytes) -> list: + def unpack_upp(self, msgpackUPP: bytes) -> list: """ - Verify the integrity of the message and decode the contents. - Throws an exception if the message is not verifiable. + Unpack a UPP (msgpack) + Throws an exception if the UPP can't be unpacked + Returns the unpacked upp as a list :param message: the msgpack encoded message + :return: the unpacked message + """ + # check for the UPP version + if msgpackUPP[1] >> 4 == 2: # version 2 + legacy = False + elif msgpackUPP[1] >> 4 == 1: # version 1 (legacy) + legacy = True + else: + raise ValueError("Invalid UPP version byte: 0x%02x" % msgpackUPP[1]) + + # unpack the msgpack + return msgpack.unpackb(msgpackUPP, raw=False) + + def get_unpacked_index(self, versionByte: int, targetField: int) -> int: + """ + Get the index of a given target field for a UPP with the given version byte + Throws a ValueError if the version byte (lower four bits) is invalid + :param versioByte: the first byte of an unpacked upp (first element of the list) + :param targetField: one off "UNPACKED_UPP_*" + :return: the index of the field on success + """ + # check the lower four bits of the version byte + lowerFour = versionByte & 0x0f + + if lowerFour == 0x01: + return UNPACKED_UNSIGNED_UPP_INDEX_TABLE[targetField] + elif lowerFour == 0x02: + return UNPACKED_SIGNED_UPP_INDEX_TABLE[targetField] + elif lowerFour == 0x03: + return UNPACKED_CHAINED_UPP_INDEX_TABLE[targetField] + else: + # unknown lower four bits; error + raise ValueError("Invalid lower four bits of the UPP version byte: %s" % bin(lowerFour)) + + def verfiy_signature(self, msgpackUPP: bytes, unpackedUPP: list = None) -> True: + """ + Verify the integrity of the message and decode the contents + Throws an exception if the version byte of the upp is invalid (ValueError) + Throws an exception if the upp doesn't contain a signature (ValueError) + :param msgpackUPP: the msgpack encoded message + :param unpackedUPP: (optional) if not provided, the function will unpack the upp itself :return: the decoded message """ - if len(message) < 70: - raise Exception("message format wrong (size < 70 bytes): {}".format(len(message))) + # check whether the UPP has to be unpacked + if unpackedUPP == None: + unpackedUPP = self.unpack_upp(msgpackUPP) + + # get the indexes for the needed fields + uuidIndex = self.get_unpacked_index(unpackedUPP[0], UNPACKED_UPP_FIELD_UUID) + signatureIndex = self.get_unpacked_index(unpackedUPP[0], UNPACKED_UPP_FIELD_SIG) - unpacker = msgpack.Unpacker() - unpacker.feed(message) + # check if a valid signature index was returned + if signatureIndex == -1: + raise ValueError("The UPP doesn't contain a signature. Version byte: %s" % bin(unpackedUPP[0])) - unpacked = [] - # unpack all entries, except the last one, remember the byte index for signature verification - for n in range(0, unpacker.read_array_header() - 1): - unpacked.append(unpacker.unpack()) - signatureIndex = unpacker.tell() - unpacked.append(unpacker.unpack()) + # verify the message + try: + # arg1: the uuid + # arg2: the message (all bytes in front of the signature (64 bytes + 1 bytes msgpack header)) + # arg3: the signature + self._prepare_and_verify( + UUID(bytes=unpackedUPP[uuidIndex]), + msgpackUPP[0:-(2 + len(unpackedUPP[signatureIndex]))], + unpackedUPP[signatureIndex] + ) + except BadSignatureError: + return False + except BadSignatureErrorEcdsa: + return False - # verify the message using extracted values - self._prepare_and_verify(UUID(bytes=unpacked[1]), message[0:signatureIndex], unpacked[-1]) - return unpacked + return True From 08aa451965dee137cfe02ccf3147c47a905bad0f Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Wed, 3 Mar 2021 15:19:27 +0100 Subject: [PATCH 03/68] adapt tests to changes --- tests/test_ubirch_protocol.py | 61 ++++++++++++++++++++++------- tests/test_ubirch_protocol_ecdsa.py | 59 +++++++++++----------------- 2 files changed, 68 insertions(+), 52 deletions(-) diff --git a/tests/test_ubirch_protocol.py b/tests/test_ubirch_protocol.py index ef3605e..87181c7 100644 --- a/tests/test_ubirch_protocol.py +++ b/tests/test_ubirch_protocol.py @@ -49,6 +49,11 @@ "c15a2c9b404a32d67abb414061b7639e1ea5a20ce90b" )) +EXPECTED_SIGNED_UNPACKED = [ + 34, b'n\xacM\x0b\x16\xe6E\x08\x8cF"\xe7E\x1e\xa5\xa1', 239, 1, b'\xc8\xf1\xc1\x9f\xb6L\xa6\xec\xd6\x8a3k\xbf\xfb9\xe8\xf4\xe6\xeehm\xe7%\xce\x9e#\xf7iE\xfc-sKNw\xf9\xf0,\xb0\xbb-O\x8f\x8e6\x1e\xfc^\xa1\x003\xbd\xc7A\xa2L\xffM~\xb0\x8d\xb64\x0b' +] + + # expected sequence of chained messages EXPECTED_CHAINED = [ bytearray(bytes.fromhex( @@ -95,7 +100,6 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes): class TestUbirchProtocol(unittest.TestCase): - def test_sign_not_implemented(self): p = ubirch.Protocol() try: @@ -106,10 +110,37 @@ def test_sign_not_implemented(self): def test_verify_not_implemented(self): p = ubirch.Protocol() try: - p.message_verify(EXPECTED_SIGNED) + p.verfiy_signature(EXPECTED_SIGNED) except NotImplementedError as e: self.assertEqual(e.args[0], 'verification not implemented') + def test_get_unpacked_index(self): + p = Protocol() + + # test indexes of signatures for unsigned messages + self.assertEqual(p.get_unpacked_index(0b0001, ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG), -1) + self.assertEqual(p.get_unpacked_index(0b0001, ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG), -1) + + # test indexes of signatures for signed messages + self.assertEqual(p.get_unpacked_index(0b0010, ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG), 4) + self.assertEqual(p.get_unpacked_index(0b0010, ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG), -1) + + # test indexes of signatures for chained messages + self.assertEqual(p.get_unpacked_index(0b0011, ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG), 5) + self.assertEqual(p.get_unpacked_index(0b0011, ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG), 2) + + def test_unpack_upp(self): + p = Protocol() + + self.assertEqual(p.unpack_upp(EXPECTED_SIGNED), EXPECTED_SIGNED_UNPACKED) + + BROKEN_EXPECTED_SIGNED = EXPECTED_SIGNED.copy() + BROKEN_EXPECTED_SIGNED[1] = 0 + + self.assertRaises(ValueError, p.unpack_upp, BROKEN_EXPECTED_SIGNED) + + return + def test_create_signed_message(self): p = Protocol() message = p.message_signed(TEST_UUID, 0xEF, 1) @@ -137,7 +168,8 @@ def test_create_chained_message_with_hash(self): def test_verify_signed_message(self): p = Protocol() - unpacked = p.message_verify(EXPECTED_SIGNED) + unpacked = p.unpack_upp(EXPECTED_SIGNED) + self.assertEqual(p.verfiy_signature(EXPECTED_SIGNED, unpackedUPP=unpacked), True) self.assertEqual(SIGNED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(0xEF, unpacked[2]) @@ -147,7 +179,8 @@ def test_verify_chained_messages(self): p = Protocol() last_signature = b'\0' * 64 for i in range(0, 3): - unpacked = p.message_verify(EXPECTED_CHAINED[i]) + unpacked = p.unpack_upp(EXPECTED_CHAINED[i]) + self.assertEqual(p.verfiy_signature(EXPECTED_CHAINED[i], unpackedUPP=unpacked), True) self.assertEqual(CHAINED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(last_signature, unpacked[2]) @@ -158,14 +191,6 @@ def test_verify_chained_messages(self): # TODO add randomized message generation and verification - def test_verify_fails_missing_data(self): - p = Protocol() - message = EXPECTED_SIGNED[0:-67] - try: - p.message_verify(message) - except Exception as e: - self.assertEqual(e.args[0], "message format wrong (size < 70 bytes): {}".format(len(message))) - def test_set_saved_signatures(self): p = Protocol() p.set_saved_signatures({TEST_UUID: "1234567890"}) @@ -209,9 +234,12 @@ def test_unpack_legacy_trackle_message(self): class ProtocolNoVerify(ubirch.Protocol): def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: pass - + p = ProtocolNoVerify() - unpacked = p.message_verify(message) + + unpacked = p.unpack_upp(message) + + self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) self.assertEqual(CHAINED & 0x0f, unpacked[0] & 0x0f) self.assertEqual(UUID(bytes=bytes.fromhex("af931b05acca758bc2aaeb98d6f93329")), UUID(bytes=unpacked[1])) self.assertEqual(0x54, unpacked[3]) @@ -235,7 +263,10 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: pass p = ProtocolNoVerify() - unpacked = p.message_verify(message) + + unpacked = p.unpack_upp(message) + + self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) self.assertEqual(SIGNED & 0x0f, unpacked[0] & 0x0f) self.assertEqual(1, unpacked[0] >> 4) self.assertEqual(UUID(bytes=bytes.fromhex("00000000000000000000000000000000")), UUID(bytes=unpacked[1])) diff --git a/tests/test_ubirch_protocol_ecdsa.py b/tests/test_ubirch_protocol_ecdsa.py index 387fd6d..133f3f9 100644 --- a/tests/test_ubirch_protocol_ecdsa.py +++ b/tests/test_ubirch_protocol_ecdsa.py @@ -87,7 +87,7 @@ def test_sign_not_implemented(self): def test_verify_not_implemented(self): p = ubirch.Protocol() try: - p.message_verify(EXPECTED_SIGNED) + p.verfiy_signature(EXPECTED_SIGNED) except NotImplementedError as e: self.assertEqual(e.args[0], 'verification not implemented') @@ -96,21 +96,15 @@ def test_create_signed_message(self): message = p.message_signed(TEST_UUID, 0xEF, 1) logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_SIGNED[0:-64], message[0:-64]) - try: - p.message_verify(message) - except Exception as e: - self.fail("verification failed: {}".format(e)) + self.assertEqual(p.verfiy_signature(message), True) def test_create_signed_message_with_hash(self): p = Protocol() message = p.message_signed(TEST_UUID, 0xEF, hashlib.sha512(b'1').digest()) logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_SIGNED_HASH, message[0:-64]) - try: - p.message_verify(message) - except Exception as e: - self.fail("verification failed: {}".format(e)) - + self.assertEqual(p.verfiy_signature(message), True) + def test_create_chained_messages(self): p = Protocol() last_signature = bytearray(b'\0'*64) @@ -123,11 +117,8 @@ def test_create_chained_messages(self): logger.debug("EXPECT : %s", binascii.hexlify(EXPECTED)) self.assertEqual(EXPECTED[0:-64], message[0:-64], "message #{} failed".format(i + 1)) self.assertEqual(last_signature, message[22:22+64]) - try: - p.message_verify(message) - last_signature = message[-64:] - except Exception as e: - self.fail("verification failed: {}".format(e)) + self.assertEqual(p.verfiy_signature(message), True) + last_signature = message[-64:] def test_create_chained_message_with_hash(self): p = Protocol() @@ -135,14 +126,12 @@ def test_create_chained_message_with_hash(self): logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_CHAINED_HASH[0:-64], message[0:-64]) - try: - p.message_verify(message) - except Exception as e: - self.fail("verification failed: {}".format(e)) + self.assertEqual(p.verfiy_signature(message), True) def test_verify_signed_message(self): p = Protocol() - unpacked = p.message_verify(EXPECTED_SIGNED) + unpacked = p.unpack_upp(EXPECTED_SIGNED) + self.assertEqual(p.verfiy_signature(EXPECTED_SIGNED, unpackedUPP=unpacked), True) self.assertEqual(SIGNED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(0xEF, unpacked[2]) @@ -152,7 +141,8 @@ def test_verify_chained_messages(self): p = Protocol() last_signature = b'\0' * 64 for i in range(0, 3): - unpacked = p.message_verify(EXPECTED_CHAINED[i]) + unpacked = p.unpack_upp(EXPECTED_CHAINED[i]) + self.assertEqual(p.verfiy_signature(EXPECTED_CHAINED[i], unpackedUPP=unpacked), True) self.assertEqual(CHAINED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(last_signature, unpacked[2]) @@ -161,16 +151,6 @@ def test_verify_chained_messages(self): # update the last signature we expect in the next message last_signature = unpacked[5] - # TODO add randomized message generation and verification - - def test_verify_fails_missing_data(self): - p = Protocol() - message = EXPECTED_SIGNED[0:-67] - try: - p.message_verify(message) - except Exception as e: - self.assertEqual(e.args[0], "message format wrong (size < 70 bytes): {}".format(len(message))) - class TestUbirchProtocolSIM(unittest.TestCase): def test_verify_registration_message_sim_v1(self): @@ -182,10 +162,12 @@ def test_verify_registration_message_sim_v1(self): p = Protocol() p.vk = ecdsa.VerifyingKey.from_string(binascii.unhexlify(vk), curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) - unpacked = p.message_verify(message) + unpacked = p.unpack_upp(message) + logger.debug(repr(unpacked)) - self.assertEqual(vk, binascii.hexlify(unpacked[3][b'pubKey']).decode()) + self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(vk, binascii.hexlify(unpacked[3]['pubKey']).decode()) def test_verify_signed_message_sim_v1(self): loc = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -196,9 +178,10 @@ def test_verify_signed_message_sim_v1(self): p = Protocol() p.vk = ecdsa.VerifyingKey.from_string(binascii.unhexlify(vk), curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) - unpacked = p.message_verify(message) + unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) + self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) def test_verify_registration_message_sim_v2(self): @@ -210,10 +193,11 @@ def test_verify_registration_message_sim_v2(self): p = Protocol() p.vk = ecdsa.VerifyingKey.from_string(binascii.unhexlify(vk), curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) - unpacked = p.message_verify(message) + unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(vk, binascii.hexlify(unpacked[3][b'pubKey']).decode()) + self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(vk, binascii.hexlify(unpacked[3]['pubKey']).decode()) def test_verify_signed_message_sim_v2(self): loc = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -224,7 +208,8 @@ def test_verify_signed_message_sim_v2(self): p = Protocol() p.vk = ecdsa.VerifyingKey.from_string(binascii.unhexlify(vk), curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) - unpacked = p.message_verify(message) + unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) + self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) \ No newline at end of file From 8ba781a827fa7371086bd38036479d6a7335d565 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Wed, 3 Mar 2021 15:20:52 +0100 Subject: [PATCH 04/68] add proper backend response verification --- examples/example-client.py | 291 +++++++++++++++++++++++++------------ 1 file changed, 202 insertions(+), 89 deletions(-) diff --git a/examples/example-client.py b/examples/example-client.py index 6900144..d4ffc08 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -9,46 +9,66 @@ from uuid import UUID from ed25519 import VerifyingKey -from requests import codes +from requests import codes, Response import ubirch from ubirch.ubirch_protocol import UBIRCH_PROTOCOL_TYPE_REG, UBIRCH_PROTOCOL_TYPE_BIN + +DEFAULT_UBIRCH_ENV = "prod" +UBIRCH_PUBKEYS = { + "dev": "a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", + "demo": "39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", + "prod": "ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690" +} +UBIRCH_UUIDS = { + "dev": "9d3c78ff-22f3-4441-a5d1-85c636d486ff", + "demo": "07104235-1892-4020-9042-00003c94b60b", + "prod": "10b2e1a4-56b3-4fff-9ada-cc8c20f93016" +} + + +# create a global logger logging.basicConfig(format='%(asctime)s %(name)20.20s %(levelname)-8.8s %(message)s', level=logging.DEBUG) logger = logging.getLogger() -######################################################################## -# Implement the ubirch-protocol with signing and saving the signatures class Proto(ubirch.Protocol): - UUID_PROD = UUID(hex="10b2e1a4-56b3-4fff-9ada-cc8c20f93016") - PUB_PROD = VerifyingKey("ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690", encoding='hex') - - def __init__(self, key_store: ubirch.KeyStore, uuid: UUID) -> None: + """ implement the uBirch-protocol, including creating and saving signatures """ + def __init__(self, key_store: ubirch.KeyStore, uuid: UUID, env: str = "prod") -> None: super().__init__() self.__ks = key_store + self.deviceUUID = uuid + + # check env + if not env in ["dev", "demo", "prod"]: + raise ValueError("Inavlid uBirch-Env! Must be one of 'dev', 'demo' or 'prod'") + + # get the uBirch pubkey and uuid for the given stage + self.uBirchPubkey = VerifyingKey(UBIRCH_PUBKEYS[env], encoding="hex") + self.uBirchUUID = UUID(hex=UBIRCH_UUIDS[env]) # check if the device already has keys or generate a new pair - if not keystore.exists_signing_key(uuid): - keystore.create_ed25519_keypair(uuid) + if not self.__ks.exists_signing_key(uuid): + self.__ks.create_ed25519_keypair(uuid) # check if the keystore already has the backend key for verification or insert verifying key - if not self.__ks.exists_verifying_key(self.UUID_PROD): - self.__ks.insert_ed25519_verifying_key(self.UUID_PROD, self.PUB_PROD) + if not self.__ks.exists_verifying_key(self.uBirchUUID): + self.__ks.insert_ed25519_verifying_key(self.uBirchUUID, self.uBirchPubkey) # load last signature for device - self.load(uuid) + self.load() logger.info("ubirch-protocol: device id: {}".format(uuid)) - def persist(self, uuid: UUID): + def persist(self): signatures = self.get_saved_signatures() - with open(uuid.hex + ".sig", "wb") as f: + with open(self.deviceUUID.hex + ".sig", "wb") as f: pickle.dump(signatures, f) - def load(self, uuid: UUID): + def load(self): try: - with open(uuid.hex + ".sig", "rb") as f: + with open(self.deviceUUID.hex + ".sig", "rb") as f: signatures = pickle.load(f) logger.info("loaded {} known signatures".format(len(signatures))) self.set_saved_signatures(signatures) @@ -56,80 +76,173 @@ def load(self, uuid: UUID): logger.warning("no existing saved signatures") pass - def _sign(self, uuid: UUID, message: bytes) -> bytes: - return self.__ks.find_signing_key(uuid).sign(message) - - def _verify(self, uuid: UUID, message: bytes, signature: bytes): - return self.__ks.find_verifying_key(uuid).verify(signature, message) - - -######################################################################## - -if len(sys.argv) < 3: - print("usage:") - print(" python3 example-client.py ") - sys.exit(0) - -env = "prod" -uuid = UUID(hex=sys.argv[1]) -auth = sys.argv[2] - -# create a keystore for the device -keystore = ubirch.KeyStore("demo-device.jks", "keystore") + def _sign(self, _, message: bytes) -> bytes: + return self.__ks.find_signing_key(self.deviceUUID).sign(message) + + def _verify(self, _, message: bytes, signature: bytes): + return self.__ks.find_verifying_key(self.deviceUUID).verify(signature, message) + + +class Main: + """ an example implementation for the uBirch-Client in Python """ + def __init__(self): + if len(sys.argv) < 3: + print("usage:") + print(" python3 example-client.py [ubirch-env]") + + sys.exit(1) + + # extract cli arguments + self.env = sys.argv[3] if len(sys.argv) > 3 else DEFAULT_UBIRCH_ENV + self.uuid = UUID(hex=sys.argv[1]) + self.auth = sys.argv[2] + + # create a keystore for the device + self.keystore = ubirch.KeyStore("demo-device.jks", "keystore") + + # create an instance of the protocol with signature saving + self.protocol = Proto(self.keystore, self.uuid, self.env) + + # create an instance of the UBIRCH API and set the auth token + self.api = ubirch.API(env=self.env) + self.api.set_authentication(self.uuid, self.auth) + + # register the pubkey if needed + self.checkRegisterPubkey() + + # a variable to store the current upp + self.currentUPP = None + + def run(self): + """ generate and send a message; verify the reponse """ + # create the message + message = self.createMessage() + + logger.info("Created an example message: %s" % str(message)) + + # create the upp + upp = self.createUPP(message) + + logging.info("Created UPP: %s" % str(upp.hex())) + + # send the upp and handle the response + resp = self.sendUPP(upp) + self.handleBackendResponse(resp) + + # the handle function is expected to sys.exit() on any kind of error - assume success + logging.info("Successfully sent the UPP and verified the response!") + + # save last signature + self.protocol.persist() + + def checkRegisterPubkey(self): + """ checks if the key is registered at the uBirch backend and registers it if necessary """ + # register the public key at the UBIRCH key service + if not self.api.is_identity_registered(self.uuid): + # get the certificate and create the registration message + certificate = self.keystore.get_certificate(self.uuid) + key_registration = self.protocol.message_signed(self.uuid, UBIRCH_PROTOCOL_TYPE_REG, certificate) + + # send the registration message + r = self.api.register_identity(key_registration) + + # check for success + if r.status_code == codes.ok: + logger.info("{}: public key registered".format(self.uuid)) + else: + logger.error("{}: registration failed".format(self.uuid)) + sys.exit(1) + + def createMessage(self) -> dict: + """ creates a unique example message """ + # create a message like being sent to the customer backend + # include an ID and timestamp in the data message to ensure a unique hash + return { + "id": str(self.uuid), + "ts": int(time.time()), + "data": "{:d}".format(random.randint(0, 100)) + } + + def createUPP(self, message : dict) -> bytearray: + """ creates an UPP from a given message """ + # create a compact rendering of the message to ensure determinism when creating the hash + serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() + + # hash the message + messageHash = hashlib.sha256(serialized).digest() + logger.info("message hash: {}".format(binascii.b2a_base64(messageHash).decode().rstrip("\n"))) + + # create a new chained protocol message with the message hash + upp = self.protocol.message_chained(self.uuid, UBIRCH_PROTOCOL_TYPE_BIN, messageHash) + + # set currentUPP + self.currentUPP = upp + + return upp + + def sendUPP(self, upp : bytearray) -> Response: + """ sends a UPP to the uBirch backend and returns the response object """ + # send chained protocol message to UBIRCH authentication service + return self.api.send(self.uuid, upp) + + def handleBackendResponse(self, response : Response): + """ handles the resonse object returned by sendUPP """ + # check the http status code + # + # 200: OK; try to verify the UPP + # XYZ: ERR; log the error and exit + if response.status_code != codes.ok: + logger.error("Sending UPP failed! response: ({}) {}".format(response.status_code, binascii.hexlify(response.content).decode())) + sys.exit(1) + + logger.info("UPP successfully sent. response: {}".format(binascii.hexlify(response.content).decode())) + + # unpack the UPP + try: + unpackedUPP = self.protocol.unpack_upp(response.content) + except Exception as e: + logger.error("Error unpacking the response UPP: '%s'" % str(response.content)) + logger.exception(e) -# create an instance of the protocol with signature saving -protocol = Proto(keystore, uuid) + sys.exit(1) -# create an instance of the UBIRCH API and set the auth token -api = ubirch.API(env=env) -api.set_authentication(uuid, auth) + # get the index of the signature and previous signature + sigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG) + prevSigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG) -# register the public key at the UBIRCH key service -if not api.is_identity_registered(uuid): - certificate = keystore.get_certificate(uuid) - key_registration = protocol.message_signed(uuid, UBIRCH_PROTOCOL_TYPE_REG, certificate) - r = api.register_identity(key_registration) - if r.status_code == codes.ok: - logger.info("{}: public key registered".format(uuid)) - else: - logger.error("{}: registration failed".format(uuid)) - sys.exit(1) + # check if a valid index for the signature was returned + if sigIndex == -1: + logger.error("The message returned by the backend doesn't contain a signature!") + sys.exit(1) -# create a message like being sent to the customer backend -# include an ID and timestamp in the data message to ensure a unique hash -message = { - "id": str(uuid), - "ts": int(time.time()), - "data": "{:d}".format(random.randint(0, 100)) -} -# >> send data to customer backend << - -# create a compact rendering of the message to ensure determinism when creating the hash -serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() - -# hash the message -message_hash = hashlib.sha256(serialized).digest() -logger.info("message hash: {}".format(binascii.b2a_base64(message_hash).decode().rstrip("\n"))) - -# create a new chained protocol message with the message hash -upp = protocol.message_chained(uuid, UBIRCH_PROTOCOL_TYPE_BIN, message_hash) -logger.info("UPP: {}".format(binascii.hexlify(upp).decode())) - -# send chained protocol message to UBIRCH authentication service -r = api.send(uuid, upp) -if r.status_code == codes.ok: - logger.info("UPP successfully sent. response: {}".format(binascii.hexlify(r.content).decode())) -else: - logger.error("sending UPP failed! response: ({}) {}".format(r.status_code, binascii.hexlify(r.content).decode())) - sys.exit(1) - -# verify the backend response -try: - protocol.message_verify(r.content) - logger.info("backend response signature successfully verified") -except Exception as e: - logger.error("backend response signature verification FAILED! {}".format(repr(e))) - sys.exit(1) - -# save last signature -protocol.persist(uuid) + # verify that the response came from the backend + try: + self.protocol.verfiy_signature(response.content) + logger.info("Backend response signature successfully verified!") + except Exception as e: + logger.error("Backend response signature verification FAILED!") + logger.exception(e) + sys.exit(1) + + # check if a valid index for the previous signature was returned + if prevSigIndex == -1: + logger.error("The message returned by the backend doesn't contain a previous signature!") + sys.exit(1) + + # unpack the previously sent upp; assume that it is a valid chained upp + unpackedPrevUpp = self.protocol.unpack_upp(self.currentUPP) + + # verfiy that the response contains the signature of our upp + if unpackedPrevUpp[sigIndex] != unpackedUPP[prevSigIndex]: + logger.error("The previous signature in the response UPP doesn't match the signature of our UPP!") + logger.error("Previous signature in the response UPP: %s" % str(unpackedUPP[prevSigIndex].hex())) + logger.error("Actual signature of our UPP: %s" % str(unpackedPrevUpp[prevSigIndex].hex())) + sys.exit(1) + else: + logger.info("Matching previous signature!") + + +# initialize/"run" the Main class +if __name__ == "__main__": + m = Main() + m.run() \ No newline at end of file From 69db333a00da0652be4ba4b95883b034136d270b Mon Sep 17 00:00:00 2001 From: Roxana Meixner Date: Wed, 24 Mar 2021 17:47:46 +0100 Subject: [PATCH 05/68] rearranged code, added support for several identities to protocol impl --- examples/example-client.py | 167 ++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 84 deletions(-) diff --git a/examples/example-client.py b/examples/example-client.py index d4ffc08..73530d3 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -14,61 +14,55 @@ import ubirch from ubirch.ubirch_protocol import UBIRCH_PROTOCOL_TYPE_REG, UBIRCH_PROTOCOL_TYPE_BIN - DEFAULT_UBIRCH_ENV = "prod" UBIRCH_PUBKEYS = { - "dev": "a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", - "demo": "39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", - "prod": "ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690" + "dev": VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding="hex"), + "demo": VerifyingKey("39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", encoding="hex"), + "prod": VerifyingKey("ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690", encoding="hex") } UBIRCH_UUIDS = { - "dev": "9d3c78ff-22f3-4441-a5d1-85c636d486ff", - "demo": "07104235-1892-4020-9042-00003c94b60b", - "prod": "10b2e1a4-56b3-4fff-9ada-cc8c20f93016" + "dev": UUID(hex="9d3c78ff-22f3-4441-a5d1-85c636d486ff"), + "demo": UUID(hex="07104235-1892-4020-9042-00003c94b60b"), + "prod": UUID(hex="10b2e1a4-56b3-4fff-9ada-cc8c20f93016") } - # create a global logger logging.basicConfig(format='%(asctime)s %(name)20.20s %(levelname)-8.8s %(message)s', level=logging.DEBUG) logger = logging.getLogger() class Proto(ubirch.Protocol): - """ implement the uBirch-protocol, including creating and saving signatures """ - def __init__(self, key_store: ubirch.KeyStore, uuid: UUID, env: str = "prod") -> None: + """ implement the ubirch-protocol, including creating and saving signatures """ + + def __init__(self, key_store: ubirch.KeyStore, uuid: UUID, env: str = DEFAULT_UBIRCH_ENV) -> None: super().__init__() self.__ks = key_store - self.deviceUUID = uuid - - # check env - if not env in ["dev", "demo", "prod"]: - raise ValueError("Inavlid uBirch-Env! Must be one of 'dev', 'demo' or 'prod'") - - # get the uBirch pubkey and uuid for the given stage - self.uBirchPubkey = VerifyingKey(UBIRCH_PUBKEYS[env], encoding="hex") - self.uBirchUUID = UUID(hex=UBIRCH_UUIDS[env]) # check if the device already has keys or generate a new pair if not self.__ks.exists_signing_key(uuid): self.__ks.create_ed25519_keypair(uuid) + # check env + if env not in UBIRCH_PUBKEYS.keys(): + raise ValueError("Invalid ubirch env! Must be one of {}".format(list(UBIRCH_PUBKEYS.keys()))) + # check if the keystore already has the backend key for verification or insert verifying key - if not self.__ks.exists_verifying_key(self.uBirchUUID): - self.__ks.insert_ed25519_verifying_key(self.uBirchUUID, self.uBirchPubkey) + if not self.__ks.exists_verifying_key(UBIRCH_UUIDS[env]): + self.__ks.insert_ed25519_verifying_key(UBIRCH_UUIDS[env], UBIRCH_PUBKEYS[env]) # load last signature for device - self.load() + self.load(uuid) logger.info("ubirch-protocol: device id: {}".format(uuid)) - def persist(self): + def persist(self, uuid: UUID): signatures = self.get_saved_signatures() - with open(self.deviceUUID.hex + ".sig", "wb") as f: + with open(uuid.hex + ".sig", "wb") as f: pickle.dump(signatures, f) - def load(self): + def load(self, uuid: UUID): try: - with open(self.deviceUUID.hex + ".sig", "rb") as f: + with open(uuid.hex + ".sig", "rb") as f: signatures = pickle.load(f) logger.info("loaded {} known signatures".format(len(signatures))) self.set_saved_signatures(signatures) @@ -76,26 +70,20 @@ def load(self): logger.warning("no existing saved signatures") pass - def _sign(self, _, message: bytes) -> bytes: - return self.__ks.find_signing_key(self.deviceUUID).sign(message) + def _sign(self, uuid: UUID, message: bytes) -> bytes: + return self.__ks.find_signing_key(uuid).sign(message) - def _verify(self, _, message: bytes, signature: bytes): - return self.__ks.find_verifying_key(self.deviceUUID).verify(signature, message) + def _verify(self, uuid: UUID, message: bytes, signature: bytes): + return self.__ks.find_verifying_key(uuid).verify(signature, message) -class Main: - """ an example implementation for the uBirch-Client in Python """ - def __init__(self): - if len(sys.argv) < 3: - print("usage:") - print(" python3 example-client.py [ubirch-env]") - - sys.exit(1) +class UbirchClient: + """ an example implementation for the ubirch-client in Python """ - # extract cli arguments - self.env = sys.argv[3] if len(sys.argv) > 3 else DEFAULT_UBIRCH_ENV - self.uuid = UUID(hex=sys.argv[1]) - self.auth = sys.argv[2] + def __init__(self, uuid: UUID, auth: str, env: str = DEFAULT_UBIRCH_ENV): + self.env = env + self.uuid = uuid + self.auth = auth # create a keystore for the device self.keystore = ubirch.KeyStore("demo-device.jks", "keystore") @@ -113,39 +101,34 @@ def __init__(self): # a variable to store the current upp self.currentUPP = None - def run(self): - """ generate and send a message; verify the reponse """ - # create the message - message = self.createMessage() - - logger.info("Created an example message: %s" % str(message)) - + def run(self, data: dict): + """ create and send a ubirch protocol message; verify the response """ # create the upp - upp = self.createUPP(message) + self.currentUPP = self.createUPP(data) - logging.info("Created UPP: %s" % str(upp.hex())) + logging.info("Created UPP: %s" % str(self.currentUPP.hex())) # send the upp and handle the response - resp = self.sendUPP(upp) + resp = self.sendUPP(self.currentUPP) self.handleBackendResponse(resp) # the handle function is expected to sys.exit() on any kind of error - assume success logging.info("Successfully sent the UPP and verified the response!") # save last signature - self.protocol.persist() + self.protocol.persist(self.uuid) def checkRegisterPubkey(self): - """ checks if the key is registered at the uBirch backend and registers it if necessary """ + """ checks if the key is registered at the ubirch backend and registers it if necessary """ # register the public key at the UBIRCH key service if not self.api.is_identity_registered(self.uuid): # get the certificate and create the registration message certificate = self.keystore.get_certificate(self.uuid) key_registration = self.protocol.message_signed(self.uuid, UBIRCH_PROTOCOL_TYPE_REG, certificate) - + # send the registration message r = self.api.register_identity(key_registration) - + # check for success if r.status_code == codes.ok: logger.info("{}: public key registered".format(self.uuid)) @@ -153,46 +136,32 @@ def checkRegisterPubkey(self): logger.error("{}: registration failed".format(self.uuid)) sys.exit(1) - def createMessage(self) -> dict: - """ creates a unique example message """ - # create a message like being sent to the customer backend - # include an ID and timestamp in the data message to ensure a unique hash - return { - "id": str(self.uuid), - "ts": int(time.time()), - "data": "{:d}".format(random.randint(0, 100)) - } - - def createUPP(self, message : dict) -> bytearray: + def createUPP(self, message: dict) -> bytes: """ creates an UPP from a given message """ # create a compact rendering of the message to ensure determinism when creating the hash serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() # hash the message - messageHash = hashlib.sha256(serialized).digest() + messageHash = hashlib.sha512(serialized).digest() logger.info("message hash: {}".format(binascii.b2a_base64(messageHash).decode().rstrip("\n"))) # create a new chained protocol message with the message hash - upp = self.protocol.message_chained(self.uuid, UBIRCH_PROTOCOL_TYPE_BIN, messageHash) + return self.protocol.message_chained(self.uuid, UBIRCH_PROTOCOL_TYPE_BIN, messageHash) - # set currentUPP - self.currentUPP = upp - - return upp - - def sendUPP(self, upp : bytearray) -> Response: - """ sends a UPP to the uBirch backend and returns the response object """ + def sendUPP(self, upp: bytes) -> Response: + """ sends a UPP to the ubirch backend and returns the response object """ # send chained protocol message to UBIRCH authentication service return self.api.send(self.uuid, upp) - def handleBackendResponse(self, response : Response): - """ handles the resonse object returned by sendUPP """ + def handleBackendResponse(self, response: Response): + """ handles the response object returned by sendUPP """ # check the http status code # # 200: OK; try to verify the UPP # XYZ: ERR; log the error and exit if response.status_code != codes.ok: - logger.error("Sending UPP failed! response: ({}) {}".format(response.status_code, binascii.hexlify(response.content).decode())) + logger.error("Sending UPP failed! response: ({}) {}".format(response.status_code, + binascii.hexlify(response.content).decode())) sys.exit(1) logger.info("UPP successfully sent. response: {}".format(binascii.hexlify(response.content).decode())) @@ -207,8 +176,9 @@ def handleBackendResponse(self, response : Response): sys.exit(1) # get the index of the signature and previous signature - sigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG) - prevSigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG) + sigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG) + prevSigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], + ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG) # check if a valid index for the signature was returned if sigIndex == -1: @@ -228,7 +198,7 @@ def handleBackendResponse(self, response : Response): if prevSigIndex == -1: logger.error("The message returned by the backend doesn't contain a previous signature!") sys.exit(1) - + # unpack the previously sent upp; assume that it is a valid chained upp unpackedPrevUpp = self.protocol.unpack_upp(self.currentUPP) @@ -242,7 +212,36 @@ def handleBackendResponse(self, response : Response): logger.info("Matching previous signature!") +def get_message(uuid: UUID) -> dict: + """ creates a unique example JSON data message """ + # create a message like being sent to the customer backend + # include an ID and timestamp in the data message to ensure a unique hash + return { + "id": str(uuid), + "ts": int(time.time()), + "data": "{:d}".format(random.randint(0, 100)) + } + + # initialize/"run" the Main class if __name__ == "__main__": - m = Main() - m.run() \ No newline at end of file + + if len(sys.argv) < 3: + print("usage:") + print(" python3 example-client.py [ubirch-env]") + sys.exit(1) + + # extract cli arguments + env = sys.argv[3] if len(sys.argv) > 3 else DEFAULT_UBIRCH_ENV + uuid = UUID(hex=sys.argv[1]) + auth = sys.argv[2] + + client = UbirchClient(uuid=uuid, auth=auth, env=env) + + data = get_message(uuid) + + logger.info("Created an example data message: %s" % str(data)) + + # todo >> send data message to data service / cloud / customer backend here << + + client.run(data) From d19d4b65907155073eb8674c142974bda48d10ad Mon Sep 17 00:00:00 2001 From: Roxana Meixner Date: Wed, 24 Mar 2021 18:33:37 +0100 Subject: [PATCH 06/68] comments (FIXME) do not unpack before signature is verified --- examples/example-client.py | 1 + ubirch/ubirch_protocol.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/example-client.py b/examples/example-client.py index 73530d3..1d5d154 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -166,6 +166,7 @@ def handleBackendResponse(self, response: Response): logger.info("UPP successfully sent. response: {}".format(binascii.hexlify(response.content).decode())) + # FIXME do not unpack before signature is verified # unpack the UPP try: unpackedUPP = self.protocol.unpack_upp(response.content) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index 4626dc8..549aecc 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -18,18 +18,18 @@ import logging from abc import abstractmethod from uuid import UUID -from ed25519 import BadSignatureError -from ecdsa.keys import BadSignatureError as BadSignatureErrorEcdsa import msgpack +from ecdsa.keys import BadSignatureError as BadSignatureErrorEcdsa +from ed25519 import BadSignatureError logger = logging.getLogger(__name__) # ubirch-protocol constants UBIRCH_PROTOCOL_VERSION = 2 -PLAIN = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x01) -SIGNED = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x02) +PLAIN = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x01) +SIGNED = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x02) CHAINED = ((UBIRCH_PROTOCOL_VERSION << 4) | 0x03) UBIRCH_PROTOCOL_TYPE_BIN = 0x00 @@ -244,7 +244,7 @@ def unpack_upp(self, msgpackUPP: bytes) -> list: raise ValueError("Invalid UPP version byte: 0x%02x" % msgpackUPP[1]) # unpack the msgpack - return msgpack.unpackb(msgpackUPP, raw=False) + return msgpack.unpackb(msgpackUPP, raw=legacy) def get_unpacked_index(self, versionByte: int, targetField: int) -> int: """ @@ -267,6 +267,8 @@ def get_unpacked_index(self, versionByte: int, targetField: int) -> int: # unknown lower four bits; error raise ValueError("Invalid lower four bits of the UPP version byte: %s" % bin(lowerFour)) + # FIXME: do not unpack UPP before signature verification! + # -> def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> True: def verfiy_signature(self, msgpackUPP: bytes, unpackedUPP: list = None) -> True: """ Verify the integrity of the message and decode the contents From 85789e461969fadd025c927ef5563e572835a206 Mon Sep 17 00:00:00 2001 From: Roxana Meixner Date: Wed, 24 Mar 2021 19:20:58 +0100 Subject: [PATCH 07/68] FIXME do not exit from client methods. throw exception and handle/log in main() --- examples/example-client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/example-client.py b/examples/example-client.py index 1d5d154..a72c5a6 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -134,7 +134,7 @@ def checkRegisterPubkey(self): logger.info("{}: public key registered".format(self.uuid)) else: logger.error("{}: registration failed".format(self.uuid)) - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() def createUPP(self, message: dict) -> bytes: """ creates an UPP from a given message """ @@ -162,7 +162,7 @@ def handleBackendResponse(self, response: Response): if response.status_code != codes.ok: logger.error("Sending UPP failed! response: ({}) {}".format(response.status_code, binascii.hexlify(response.content).decode())) - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() logger.info("UPP successfully sent. response: {}".format(binascii.hexlify(response.content).decode())) @@ -174,7 +174,7 @@ def handleBackendResponse(self, response: Response): logger.error("Error unpacking the response UPP: '%s'" % str(response.content)) logger.exception(e) - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() # get the index of the signature and previous signature sigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG) @@ -184,7 +184,7 @@ def handleBackendResponse(self, response: Response): # check if a valid index for the signature was returned if sigIndex == -1: logger.error("The message returned by the backend doesn't contain a signature!") - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() # verify that the response came from the backend try: @@ -193,12 +193,12 @@ def handleBackendResponse(self, response: Response): except Exception as e: logger.error("Backend response signature verification FAILED!") logger.exception(e) - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() # check if a valid index for the previous signature was returned if prevSigIndex == -1: logger.error("The message returned by the backend doesn't contain a previous signature!") - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() # unpack the previously sent upp; assume that it is a valid chained upp unpackedPrevUpp = self.protocol.unpack_upp(self.currentUPP) @@ -208,7 +208,7 @@ def handleBackendResponse(self, response: Response): logger.error("The previous signature in the response UPP doesn't match the signature of our UPP!") logger.error("Previous signature in the response UPP: %s" % str(unpackedUPP[prevSigIndex].hex())) logger.error("Actual signature of our UPP: %s" % str(unpackedPrevUpp[prevSigIndex].hex())) - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle/log in main() else: logger.info("Matching previous signature!") @@ -230,7 +230,7 @@ def get_message(uuid: UUID) -> dict: if len(sys.argv) < 3: print("usage:") print(" python3 example-client.py [ubirch-env]") - sys.exit(1) + sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() # extract cli arguments env = sys.argv[3] if len(sys.argv) > 3 else DEFAULT_UBIRCH_ENV From 57d68a51294a519a89d7e76c4c3dfc18d7f663b6 Mon Sep 17 00:00:00 2001 From: Roxana Meixner Date: Wed, 24 Mar 2021 19:27:34 +0100 Subject: [PATCH 08/68] fixup! FIXME do not exit from client methods. throw exception and handle/log in main() --- examples/example-client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example-client.py b/examples/example-client.py index a72c5a6..1ede5ee 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -230,7 +230,7 @@ def get_message(uuid: UUID) -> dict: if len(sys.argv) < 3: print("usage:") print(" python3 example-client.py [ubirch-env]") - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() + sys.exit(1) # extract cli arguments env = sys.argv[3] if len(sys.argv) > 3 else DEFAULT_UBIRCH_ENV From 5cd5637d6613b127a3a7184624f827ac2c467ce6 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 29 Mar 2021 17:09:30 +0200 Subject: [PATCH 09/68] add a function to split up a upp-msgpack into the message and the signature --- ubirch/ubirch_protocol.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index 549aecc..93b27f4 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -267,7 +267,16 @@ def get_unpacked_index(self, versionByte: int, targetField: int) -> int: # unknown lower four bits; error raise ValueError("Invalid lower four bits of the UPP version byte: %s" % bin(lowerFour)) - # FIXME: do not unpack UPP before signature verification! + def upp_msgpack_split_signature(self, msgpackUPP) -> (bytes, bytes): + """ + Separate the signature from the msgpack + :param msgpackUPP: the msgpack encoded upp + :return: a tuple consiting of the message without the signature and the signature + """ + try: + return (msgpackUPP[:-66], msgpackUPP[-64:]) + except IndexError: + raise ValueError("The UPP-msgpack is too short: %d bytes" % len(msgpackUPP)) # -> def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> True: def verfiy_signature(self, msgpackUPP: bytes, unpackedUPP: list = None) -> True: """ From f72cecc9710b5561cc48fa77dad563f26bd3835d Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 29 Mar 2021 17:10:33 +0200 Subject: [PATCH 10/68] do not unpack the upp during verification --- ubirch/ubirch_protocol.py | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index 93b27f4..f98eb60 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -277,41 +277,23 @@ def upp_msgpack_split_signature(self, msgpackUPP) -> (bytes, bytes): return (msgpackUPP[:-66], msgpackUPP[-64:]) except IndexError: raise ValueError("The UPP-msgpack is too short: %d bytes" % len(msgpackUPP)) + # -> def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> True: - def verfiy_signature(self, msgpackUPP: bytes, unpackedUPP: list = None) -> True: + def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> bool: """ Verify the integrity of the message and decode the contents - Throws an exception if the version byte of the upp is invalid (ValueError) - Throws an exception if the upp doesn't contain a signature (ValueError) + Raises an value error when the message is too short :param msgpackUPP: the msgpack encoded message :param unpackedUPP: (optional) if not provided, the function will unpack the upp itself :return: the decoded message """ - # check whether the UPP has to be unpacked - if unpackedUPP == None: - unpackedUPP = self.unpack_upp(msgpackUPP) - - # get the indexes for the needed fields - uuidIndex = self.get_unpacked_index(unpackedUPP[0], UNPACKED_UPP_FIELD_UUID) - signatureIndex = self.get_unpacked_index(unpackedUPP[0], UNPACKED_UPP_FIELD_SIG) - - # check if a valid signature index was returned - if signatureIndex == -1: - raise ValueError("The UPP doesn't contain a signature. Version byte: %s" % bin(unpackedUPP[0])) + # separate the message from the signature + msg, sig = self.upp_msgpack_split_signature(msgpackUPP) # verify the message try: - # arg1: the uuid - # arg2: the message (all bytes in front of the signature (64 bytes + 1 bytes msgpack header)) - # arg3: the signature - self._prepare_and_verify( - UUID(bytes=unpackedUPP[uuidIndex]), - msgpackUPP[0:-(2 + len(unpackedUPP[signatureIndex]))], - unpackedUPP[signatureIndex] - ) - except BadSignatureError: - return False - except BadSignatureErrorEcdsa: + self._prepare_and_verify(uuid, msg, sig) + except (BadSignatureError, BadSignatureErrorEcdsa): return False return True From f1d12dc8e48c313a82d81396598ad73eab68cac7 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 29 Mar 2021 17:11:33 +0200 Subject: [PATCH 11/68] do not unpack a upp before verification --- examples/example-client.py | 70 +++++++++++++++----------------------- 1 file changed, 28 insertions(+), 42 deletions(-) diff --git a/examples/example-client.py b/examples/example-client.py index 1ede5ee..e66b18c 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -100,22 +100,25 @@ def __init__(self, uuid: UUID, auth: str, env: str = DEFAULT_UBIRCH_ENV): # a variable to store the current upp self.currentUPP = None + self.currentSig = None def run(self, data: dict): """ create and send a ubirch protocol message; verify the response """ # create the upp self.currentUPP = self.createUPP(data) + _, self.currentSig = self.protocol.upp_msgpack_split_signature(self.currentUPP) logging.info("Created UPP: %s" % str(self.currentUPP.hex())) # send the upp and handle the response resp = self.sendUPP(self.currentUPP) + self.handleBackendResponse(resp) - + # the handle function is expected to sys.exit() on any kind of error - assume success logging.info("Successfully sent the UPP and verified the response!") - # save last signature + # save last signatures self.protocol.persist(self.uuid) def checkRegisterPubkey(self): @@ -134,7 +137,8 @@ def checkRegisterPubkey(self): logger.info("{}: public key registered".format(self.uuid)) else: logger.error("{}: registration failed".format(self.uuid)) - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() + + raise Exception("Failed to register the public key!") def createUPP(self, message: dict) -> bytes: """ creates an UPP from a given message """ @@ -153,7 +157,7 @@ def sendUPP(self, upp: bytes) -> Response: # send chained protocol message to UBIRCH authentication service return self.api.send(self.uuid, upp) - def handleBackendResponse(self, response: Response): + def handleBackendResponse(self, response: Response) -> bool: """ handles the response object returned by sendUPP """ # check the http status code # @@ -162,53 +166,30 @@ def handleBackendResponse(self, response: Response): if response.status_code != codes.ok: logger.error("Sending UPP failed! response: ({}) {}".format(response.status_code, binascii.hexlify(response.content).decode())) - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() + + raise(Exception("Exiting due to failure sending the UPP to the backend!")) logger.info("UPP successfully sent. response: {}".format(binascii.hexlify(response.content).decode())) - # FIXME do not unpack before signature is verified - # unpack the UPP - try: - unpackedUPP = self.protocol.unpack_upp(response.content) - except Exception as e: - logger.error("Error unpacking the response UPP: '%s'" % str(response.content)) - logger.exception(e) - - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() - - # get the index of the signature and previous signature - sigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_SIG) - prevSigIndex = self.protocol.get_unpacked_index(unpackedUPP[0], - ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG) - - # check if a valid index for the signature was returned - if sigIndex == -1: - logger.error("The message returned by the backend doesn't contain a signature!") - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() - # verify that the response came from the backend - try: - self.protocol.verfiy_signature(response.content) + if self.protocol.verfiy_signature(UBIRCH_UUIDS[self.env], response.content) == True: logger.info("Backend response signature successfully verified!") - except Exception as e: + else: logger.error("Backend response signature verification FAILED!") - logger.exception(e) - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() - - # check if a valid index for the previous signature was returned - if prevSigIndex == -1: - logger.error("The message returned by the backend doesn't contain a previous signature!") - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle in main() + + raise(Exception("Exiting due to failed signature verification!")) - # unpack the previously sent upp; assume that it is a valid chained upp - unpackedPrevUpp = self.protocol.unpack_upp(self.currentUPP) + # unpack the received upp to get its previous signature + unpacked = self.protocol.unpack_upp(response.content) + prevSig = unpacked[self.protocol.get_unpacked_index(unpacked[0], ubirch.ubirch_protocol.UNPACKED_UPP_FIELD_PREV_SIG)] # verfiy that the response contains the signature of our upp - if unpackedPrevUpp[sigIndex] != unpackedUPP[prevSigIndex]: + if self.currentSig != prevSig: logger.error("The previous signature in the response UPP doesn't match the signature of our UPP!") - logger.error("Previous signature in the response UPP: %s" % str(unpackedUPP[prevSigIndex].hex())) - logger.error("Actual signature of our UPP: %s" % str(unpackedPrevUpp[prevSigIndex].hex())) - sys.exit(1) # FIXME do not exit from client methods. throw exception and handle/log in main() + logger.error("Previous signature in the response UPP: %s" % str(prevSig.hex())) + logger.error("Actual signature of our UPP: %s" % str(self.currentSig.hex())) + + raise(Exception("Exiting due to a non-matching signature in the response UPP!")) else: logger.info("Matching previous signature!") @@ -245,4 +226,9 @@ def get_message(uuid: UUID) -> dict: # todo >> send data message to data service / cloud / customer backend here << - client.run(data) + try: + client.run(data) + except Exception as e: + logger.exception(e) + + sys.exit(1) From 8f67a1f8c350ba3a7efe4c1468df032ea213c2e1 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:38:49 +0200 Subject: [PATCH 12/68] move pack example and extend it --- examples/pack.py | 86 ---------- examples/upp-creator.py | 351 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+), 86 deletions(-) delete mode 100644 examples/pack.py create mode 100644 examples/upp-creator.py diff --git a/examples/pack.py b/examples/pack.py deleted file mode 100644 index 110bb46..0000000 --- a/examples/pack.py +++ /dev/null @@ -1,86 +0,0 @@ -import binascii -import hashlib -import json -import logging -import pickle -import secrets -import sys -import time -from uuid import UUID - -import ubirch - -logging.basicConfig(format='%(asctime)s %(name)20.20s %(levelname)-8.8s %(message)s', level=logging.INFO) -logger = logging.getLogger() - - -######################################################################## -# Implement the ubirch-protocol with signing and saving the signatures -class Proto(ubirch.Protocol): - - def __init__(self, key_store: ubirch.KeyStore) -> None: - super().__init__() - self._ks = key_store - - def load(self, uuid: UUID): - try: - with open(uuid.hex + ".sig", "rb") as f: - signatures = pickle.load(f) - logger.debug("loaded {} known signatures".format(len(signatures))) - self.set_saved_signatures(signatures) - except: - logger.warning("no existing saved signatures") - pass - - def persist(self, uuid: UUID): - signatures = self.get_saved_signatures() - with open(uuid.hex + ".sig", "wb") as f: - pickle.dump(signatures, f) - - def _sign(self, uuid: UUID, message: bytes) -> bytes: - return self._ks.find_signing_key(uuid).sign(message) - - -######################################################################## - -if len(sys.argv) < 2: - print("usage:") - print(" python3 ./pack.py ") - print(" e.g.: python3 ./pack.py 56bd9b85-6c6e-4a24-bf71-f2ac2de10183") - sys.exit(0) - -uuid = UUID(hex=sys.argv[1]) - -# create a keystore for the device -keystore = ubirch.KeyStore("demo-device.jks", "keystore") - -# check if the device already has keys or generate a new pair -if not keystore.exists_signing_key(uuid): - keystore.create_ed25519_keypair(uuid) - -logger.info("public key [base64]: {}".format( - binascii.b2a_base64(keystore.find_verifying_key(uuid).to_bytes()).decode().rstrip('\n'))) - -# create an instance of the protocol with signature saving -protocol = Proto(keystore) -protocol.load(uuid) - -# include an ID and timestamp in the data message to ensure a unique hash -message = { - "uuid": str(uuid), - "timestamp": int(time.time()), - "data": "{:d}".format(secrets.randbits(16)) -} - -# create a compact rendering of the message to ensure determinism when creating the hash -serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() - -# calculate the hash of the message -message_hash = hashlib.sha512(serialized).digest() - -# create a new chained protocol message with the hash of the message -upp = protocol.message_chained(uuid, 0x00, message_hash) -logger.info("UPP: {}".format(binascii.hexlify(upp).decode())) - -# store signature persistently for chaining -protocol.persist(uuid) diff --git a/examples/upp-creator.py b/examples/upp-creator.py new file mode 100644 index 0000000..1382698 --- /dev/null +++ b/examples/upp-creator.py @@ -0,0 +1,351 @@ +import binascii +import hashlib +import json +import logging +import pickle +import sys +import time +import argparse +from uuid import UUID + +import ubirch + +DEFAULT_TYPE = "0x00" # binary/unknown type +DEFAULT_VERSION = "0x23" # chained upp +DEFAULT_KS = "devices.jks" +DEFAULT_KS_PWD = "keystore" +DEFAULT_KEYREG = "False" +DEFAULT_HASH = "sha512" +DEFAULT_OUTPUT = "upp.bin" +DEFAULT_NOSTDOUT = "False" + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +######################################################################## +# Implement the ubirch-protocol with signing and saving the signatures +class Proto(ubirch.Protocol): + def __init__(self, key_store: ubirch.KeyStore) -> None: + super().__init__() + self._ks = key_store + + def load(self, uuid: UUID): + try: + with open(uuid.hex + ".sig", "rb") as f: + signatures = pickle.load(f) + logger.debug("loaded {} known signatures".format(len(signatures))) + self.set_saved_signatures(signatures) + except: + logger.warning("no existing saved signatures") + pass + + def persist(self, uuid: UUID): + signatures = self.get_saved_signatures() + with open(uuid.hex + ".sig", "wb") as f: + pickle.dump(signatures, f) + + def _sign(self, uuid: UUID, message: bytes) -> bytes: + return self._ks.find_signing_key(uuid).sign(message) +######################################################################## + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.data : str = None + self.version : int = None + self.version_str : str = None + self.type : int = None + self.type_str : str = None + self.hash : str = None + self.uuid : UUID = None + self.uuid_str : str = None + self.keystore_path : str = None + self.keystore_pass : str + self.output : str = None + self.keyreg_str : str = None + self.keyreg : bool = None + self.nostdout_str : str = None + self.nostdout : bool = None + self.payload : bytes = None + + self.hasher : object = None + self.keystore : ubirch.KeyStore = None + self.proto : Proto = None + self.payload : bytes = None + self.upp : bytes = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Create a uBirch Protocol Package (UPP)", + epilog="Note that when using chained UPPs (--version 0x23), this tool will try to load/save signatures to UUID.sig, where UUID will be replaced with the actual UUID. " + "Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain using this tool." + "Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. " + "When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). " + "For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol." + ) + + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("data", metavar="DATA", type=str, + help="data to be packed into the UPP or hashed; e.g.: {\"t\": 23.4, \"ts\": 1624624140}" + ) + self.argparser.add_argument("--version", "-v", metavar="VERISON", type=str, default=DEFAULT_VERSION, + help="version of the UPP; 0x21 (unsigned; NOT IMPLEMENTED), 0x22 (signed) or 0x23 (chained) (default: %s)" % DEFAULT_VERSION + ) + self.argparser.add_argument("--type", "-t", metavar="TYPE", type=str, default=DEFAULT_TYPE, + help="type of the UPP (0 < type < 256); e.g.: 0x00 (unknown), 0x32 (msgpack), 0x53 (generic), ... (default and recommended: %s)" % DEFAULT_TYPE + ) + self.argparser.add_argument("--ks", "-k", metavar="KS", type=str, default=DEFAULT_KS, + help="keystore file path; e.g.: test.jks (default: %s)" % DEFAULT_KS + ) + self.argparser.add_argument("--kspwd", "-p", metavar="KSPWD", type=str, default=DEFAULT_KS_PWD, + help="keystore password; e.g.: secret (default: %s)" % DEFAULT_KS_PWD + ) + self.argparser.add_argument("--keyreg", "-r", metavar="KEYREG", type=str, default=DEFAULT_KEYREG, + help="generate a key registration UPP (data and --hash will be ignored); e.g.: true, false (default: %s)" % DEFAULT_KEYREG + ) + self.argparser.add_argument("--hash", metavar="HASH", type=str, default=DEFAULT_HASH, + help="hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: %s)" % DEFAULT_HASH + ) + self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, + help="file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: %s)" % DEFAULT_OUTPUT + ) + self.argparser.add_argument("--nostdout", "-n", metavar="nostdout", type=str, default=DEFAULT_NOSTDOUT, + help="do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: %s)" % DEFAULT_NOSTDOUT + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.data = self.args.data + self.uuid_str = self.args.uuid + self.version_str = self.args.version + self.type_str = self.args.type + self.keyreg_str = self.args.keyreg + self.hash = self.args.hash + self.keystore_path = self.args.ks + self.keystore_pass = self.args.kspwd + self.output = self.args.output + self.nostdout_str = self.args.nostdout + + # get the keyreg value + if self.keyreg_str.lower() in ["1", "yes", "y", "true"]: + self.keyreg = True + else: + self.keyreg = False + + # check the --hash argument + if self.hash.lower() == "sha512": + self.hasher = hashlib.sha512 + elif self.hash.lower() == "sha256": + self.hasher = hashlib.sha256 + elif self.hash.lower() == "off": + self.hasher = None + else: + logger.error("Invalid value for --hash: \"%s\" is not supported!" % self.hash) + + return False + + # check the uuid argument + try: + self.uuid = UUID(self.uuid_str) + except ValueError as e: + logger.error("Invalud UUID input string: \"%s\"!" % self.uuid_str) + logger.exception(e) + + return False + + # check the version argument + self.version = int(self.version_str, base=16 if "x" in self.version_str else 10) + + if self.version not in [0x22, 0x23]: + logger.error("Unsupported value for the --version argument: \"0x%x\" (%d)" % (self.version, self.version)) + + return False + + # check if the value for type is in range + self.type = int(self.type_str, base=16 if "x" in self.type_str else 10) + + if not (0 <= self.type < 256): + logger.error("Value for --type is out of range: \"0x%x\" (%d)" % (self.type, self.type)) + + return False + + # get the nostdout value + if self.nostdout_str.lower() in ["1", "yes", "y", "true"]: + self.nostdout = True + else: + self.nostdout = False + + # success + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) + + # check if the device already has keys or generate a new pair + if not self.keystore.exists_signing_key(self.uuid): + if self.nostdout == False: + logger.info("No keys found for \"%s\" in \"%s\" - generating a keypair" % (self.uuid_str, self.keystore_path)) + + self.keystore.create_ed25519_keypair(self.uuid) + + if self.nostdout == False: + logger.info("Public/Verifying key for \"%s\" [base64]: \"%s\"" % + (self.uuid_str, binascii.b2a_base64(self.keystore.find_verifying_key(self.uuid).to_bytes(), newline=False).decode())) + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_proto(self) -> bool: + try: + self.proto = Proto(self.keystore) + self.proto.load(self.uuid) + except Exception as e: + logger.exception(e) + + False + + return True + + def prepare_payload(self) -> bool: + try: + if self.hasher == None: + self.payload = self.data.encode() + + if self.nostdout == False: + logger.info("UPP payload (raw data): \"%s\"" % self.payload) + else: + self.payload = self.hasher(self.data.encode()).digest() + + if self.nostdout == False: + logger.info("UPP payload (%s hash of the data) [base64]: \"%s\"" % (self.hash, binascii.b2a_base64(self.payload).decode().rstrip("\n"))) + except Exception as e: + logger.exception(e) + + return False + + return True + + def create_upp(self) -> bool: + try: + if self.keyreg == True: + # generate a key registration upp + if self.nostdout == False: + logger.info("Generating a key registration UPP for UUID \"%s\"" % self.uuid_str) + + self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) + pass + else: + if self.version == 0x22: + if self.nostdout == False: + logger.info("Generating a unchained signed UPP for UUID \"%s\"" % self.uuid_str) + + self.upp = self.proto.message_signed(self.uuid, self.type, self.payload) + elif self.version == 0x23: + if self.nostdout == False: + logger.info("Generating a chained signed UPP for UUID \"%s\"" % self.uuid_str) + + self.upp = self.proto.message_chained(self.uuid, self.type, self.payload) + + # save the new signature + self.proto.persist(self.uuid) + else: + # shouldnt get here/unsupported versions are caught in process_args() + raise(ValueError("Unsupported UPP version")) + except Exception as e: + logger.exception(e) + + return False + + return True + + def show_store_upp(self) -> bool: + try: + if self.nostdout == False: + logger.info("UPP [hex]: \"%s\"" % binascii.hexlify(self.upp).decode()) + + # try to write the upp + with open(self.output, "wb") as fd: + fd.write(self.upp) + + if self.nostdout == False: + logger.info("UPP written to \"%s\"" % self.output) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the uBirch protocol + if self.init_proto() != True: + logger.error("Errors occured while initializing the uBirch protocol - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # prepare the UPP payload + if self.prepare_payload() != True: + logger.error("Errors occured while preparing the UPP payload - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # create the UPP + if self.create_upp() != True: + logger.error("Errors occured while creating the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # show and store the UPP + if self.show_store_upp() != True: + logger.error("Erros occured while showing/storing the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From 653c5cdf674d22d2270858434e21d2253305ec14 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:39:03 +0200 Subject: [PATCH 13/68] add script to dump keystore contents --- examples/keystore-dumper.py | 128 ++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 examples/keystore-dumper.py diff --git a/examples/keystore-dumper.py b/examples/keystore-dumper.py new file mode 100644 index 0000000..0fa8060 --- /dev/null +++ b/examples/keystore-dumper.py @@ -0,0 +1,128 @@ +import sys +import time +import argparse +import logging +import binascii +import uuid + +import ubirch + + +DEFAULT_SHOW_SIGN = "False" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.keystore_path : str = None + self.keystore_pass : str = None + self.show_sign_str : str = None + self.show_sign : bool = None + + self.keystore : ubirch.KeyStore = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Dump the contents of a keystore (.jks)", + epilog="" + ) + + self.argparser.add_argument("keystore", metavar="KEYSTORE", type=str, + help="keystore file path; e.g.: test.jks" + ) + self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, + help="keystore password; e.g.: secret" + ) + self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KET", type=str, default=DEFAULT_SHOW_SIGN, + help="enables/disables showing of signing keys; e.g.: true, false (default: %s)" % DEFAULT_SHOW_SIGN + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.keystore_path = self.args.keystore + self.keystore_pass = self.args.keystore_pass + self.show_sign_str = self.args.show_sk + + # get the bool for show sk + if self.show_sign_str.lower() in ["1", "yes", "y", "true"]: + self.show_sign = True + else: + self.show_sign = False + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) + except Exception as e: + logger.exception(e) + + return False + + return True + + def dump_keystore(self) -> bool: + verifying_keys = self.keystore._ks.certs + signing_keys = self.keystore._ks.private_keys + + # go trough the list of verifiying keys and print information for each entry + for vk_uuid in verifying_keys.keys(): + if self.show_sign == True: + t = signing_keys.get("pke_" + vk_uuid) + + sk = binascii.hexlify(t.pkey).decode() if t != None else "N / A" + else: + sk = "█" * 128 + + print("=" * 134) + print("UUID: %s" % str(uuid.UUID(hex=vk_uuid))) + print(" VK : %s" % binascii.hexlify(verifying_keys[vk_uuid].cert).decode()) + print(" SK : %s" % sk) + print("=" * 134) + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + if self.dump_keystore() != True: + logger.error("Errors occured while dumping the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From a7706915e12f66e37cc8122133af45d7f1d842e5 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:39:36 +0200 Subject: [PATCH 14/68] move verification example and extend it --- examples/test-verification.py | 26 ---- examples/upp-verifier.py | 266 ++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 26 deletions(-) delete mode 100644 examples/test-verification.py create mode 100644 examples/upp-verifier.py diff --git a/examples/test-verification.py b/examples/test-verification.py deleted file mode 100644 index d22ee63..0000000 --- a/examples/test-verification.py +++ /dev/null @@ -1,26 +0,0 @@ -from uuid import UUID - -from ed25519 import VerifyingKey - -import ubirch -from ubirch.ubirch_protocol import SIGNED - -remote_uuid = UUID(hex="6eac4d0b-16e6-4508-8c46-22e7451ea5a1") -remote_vk = VerifyingKey("b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068", encoding='hex') -# a random signed ubirch-protocol message -keystore = ubirch.KeyStore("demo-device.jks", "keystore") -keystore.insert_ed25519_verifying_key(remote_uuid, remote_vk) - - -class ProtocolImpl(ubirch.Protocol): - def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> dict: - return keystore.find_verifying_key(uuid).verify(signature, message) - - -proto = ProtocolImpl(SIGNED) - -message = bytes.fromhex( - "9512b06eac4d0b16e645088c4622e7451ea5a1ccef01da0040578a5b22ceb3e1" - "d0d0f8947c098010133b44d3b1d2ab398758ffed11507b607ed37dbbe006f645" - "f0ed0fdbeb1b48bb50fd71d832340ce024d5a0e21c0ebc8e0e") -print(proto.message_verify(message)) diff --git a/examples/upp-verifier.py b/examples/upp-verifier.py new file mode 100644 index 0000000..3bda61b --- /dev/null +++ b/examples/upp-verifier.py @@ -0,0 +1,266 @@ +import sys +import logging +import argparse +import msgpack +import binascii +import uuid +import ed25519 + +import ubirch + + +DEFAULT_INPUT = "upp.bin" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Proto(ubirch.Protocol): + # UUIDs paired with public keys of uBirch Niomon on all stages + UUID_DEV = uuid.UUID(hex="9d3c78ff-22f3-4441-a5d1-85c636d486ff") + PUB_DEV = ed25519.VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding='hex') + UUID_DEMO = uuid.UUID(hex="07104235-1892-4020-9042-00003c94b60b") + PUB_DEMO = ed25519.VerifyingKey("39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", encoding='hex') + UUID_PROD = uuid.UUID(hex="10b2e1a4-56b3-4fff-9ada-cc8c20f93016") + PUB_PROD = ed25519.VerifyingKey("ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690", encoding='hex') + + def __init__(self, ks : ubirch.KeyStore): + super().__init__() + + self.ks : ubirch.KeyStore = ks + + # insert all keys defined above into the keystore + if not self.ks.exists_verifying_key(self.UUID_DEV): + self.ks.insert_ed25519_verifying_key(self.UUID_DEV, self.PUB_DEV) + if not self.ks.exists_verifying_key(self.UUID_DEMO): + self.ks.insert_ed25519_verifying_key(self.UUID_DEMO, self.PUB_DEMO) + if not self.ks.exists_verifying_key(self.UUID_PROD): + self.ks.insert_ed25519_verifying_key(self.UUID_PROD, self.PUB_PROD) + + def _verify(self, uuid: uuid.UUID, message: bytes, signature: bytes): + return self.ks.find_verifying_key(uuid).verify(signature, message) + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.vk_str : str = None + self.vk : ed25519.VerifyingKey = None + self.vk_uuid : uuid.UUID = None + self.vk_uuid_str : str = None + + self.input : str = None + + self.keystore : ubirch.KeyStore = None + self.proto : ubirch.Protocol = None + + self.upp : bytes = None + self.upp_uuid : uuid.UUID = None + self.upp_uuid_str : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Check if a UPP is valid/properly signed", + epilog="Note that when trying to verify a UPP sent by the uBirch backend (Niomon) a verifying key doesn't have to be provided via the -k option." + "Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator." + "If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed." + ) + + self.argparser.add_argument("--verifying-key", "-k", metavar="VK", type=str, default="AUTO", + help="key to be used for verification; any verifying key in hex like \"b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068\"" + ) + self.argparser.add_argument("--verifying-key-uuid", "-u", metavar="UUID", type=str, default="EMPTY", + help="the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1" + ) + self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, + help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.vk_str = self.args.verifying_key + self.vk_uuid_str = self.args.verifying_key_uuid + self.input = self.args.input + + # check if a verifying key was supplied + if self.vk_str != "AUTO": + try: + # convert the string + self.vk = ed25519.VerifyingKey(self.vk_str, encoding="hex") + except Exception as e: + logger.error("Invalid verifying key supplied via --verifying-key/-k: \"%s\"" % self.vk_str) + logger.exception(e) + + return False + + if self.vk_uuid_str == "EMPTY": + logger.error("--verifying-key-uuid/-u must be specified when --verifying-key/-k is specified!") + + return False + + try: + self.vk_uuid = uuid.UUID(hex=self.vk_uuid_str) + except Exception as e: + logger.error("Invalid UUID supplied via --verifying-key-uuid/-u: \"%s\"" % self.vk_uuid_str) + logger.exception(e) + + return False + + return True + + def read_upp(self) -> bool: + # read the UPP from the input path + try: + logger.info("Reading the input UPP from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + self.upp = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore("-- temporary --", None) + except Exception as e: + logger.exception(e) + + return False + + return True + + def check_cli_vk(self) -> bool: + try: + if self.vk != None: + self.keystore.insert_ed25519_verifying_key(self.vk_uuid, self.vk) + + logger.info("Inserted \"%s\": \"%s\" (UUID/VK) into the keystore" % (self.vk_uuid_str, self.vk_str)) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_upp_uuid(self) -> bool: + try: + # unpack the upp + unpacked = msgpack.unpackb(self.upp) + + # get the uuid + self.upp_uuid = uuid.UUID(bytes=unpacked[1]) + self.upp_uuid_str =str(self.upp_uuid) + + logger.info("UUID of the UPP creator: \"%s\"" % self.upp_uuid_str) + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_proto(self) -> bool: + try: + self.proto = Proto(self.keystore) + except Exception as e: + logger.exception(e) + + return False + + return True + + def verify_upp(self) -> bool: + try: + if self.proto.verfiy_signature(self.upp_uuid, self.upp) == True: + logger.info("Signature verified - the UPP is valid!") + else: + logger.info("The signature does not match - the UPP is invalid!") + except KeyError: + logger.error("No verifying key found for UUID \"%s\" - can't verify the UPP!" % self.upp_uuid_str) + + return False + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # read the upp + if self.read_upp() != True: + logger.error("Errors occured while reading the UPP from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check/insert the cli-provided verifying key + if self.check_cli_vk() != True: + logger.error("Errorc occured while inserting the verifying key into the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # get the uuid + if self.get_upp_uuid() != True: + logger.error("Errors occured while extracting the UUID from the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the Protocol + if self.init_proto() != True: + logger.error("Erros occured while initializing the Protocol - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # try to verify the message + if self.verify_upp() != True: + logger.error("Errors occured while verifying the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 95dd8f79bbc98c4085fc95402b82f40d35a5fb87 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:39:50 +0200 Subject: [PATCH 15/68] add script to send data and key registration upps --- examples/upp-sender.py | 248 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 examples/upp-sender.py diff --git a/examples/upp-sender.py b/examples/upp-sender.py new file mode 100644 index 0000000..8d34134 --- /dev/null +++ b/examples/upp-sender.py @@ -0,0 +1,248 @@ +import sys +import logging +import argparse +import msgpack +import requests +import binascii +import uuid + +import ubirch + + +DEFAULT_ENV = "dev" +DEFAULT_INPUT = "upp.bin" +DEFAULT_OUTPUT = "response_upp.bin" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.uuid_str : str = None + self.uuid : uuid.UUID = None + self.auth : str = None + self.env : str = None + self.input : str = None + + self.iskeyreg : bool = False + + self.upp : bytes = None + self.api : ubirch.API = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Send a uBirch Protocol Package (UPP) to uBirch Niomon", + epilog="" + ) + + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("auth", metavar="AUTH", type=str, + help="uBirch device authentication token" + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, + help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT + ) + self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, + help="response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: %s)" % DEFAULT_OUTPUT + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.uuid_str = self.args.uuid + self.auth = self.args.auth + self.env = self.args.env + self.input = self.args.input + self.output = self.args.output + + # process the uuid + try: + self.uuid = uuid.UUID(hex=self.uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.uuid_str) + logger.exception(e) + + return False + + # validate env + if self.env.lower() not in ["dev", "demo", "prod"]: + logger.error("Invalid value for --env: \"%s\"!" % self.env) + + return False + + return True + + def read_upp(self) -> bool: + # read the UPP from the input path + try: + logger.info("Reading the input UPP for \"%s\" from \"%s\"" % (self.uuid_str, self.input)) + + with open(self.input, "rb") as fd: + self.upp = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def check_is_keyreg(self) -> bool: + # check if the UPP is a key registration UPP + try: + if msgpack.unpackb(self.upp)[2] == 1: + logger.info("The UPP is a key registration UPP - disabling identity registration check") + + self.iskeyreg = True + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_api(self) -> bool: + try: + # initialize the uBirch api + self.api = ubirch.API(env=self.env, debug=True) + self.api.set_authentication(self.uuid, self.auth) + + if self.iskeyreg == False: + # check if the UUID is registered + if not self.api.is_identity_registered(self.uuid): + + logger.error("The identity for \"%s\" is not yet registered; please send a key registration UPP first" % self.uuid_str) + + return False + except Exception as e: + logger.exception(e) + + return False + + return True + + def send_upp(self) -> bool: + # niomon not accepting a UPP is not considered an error; so the return value will still be True + # this choice was made because this tool is meant for playing around/debugging etc. + + try: + # check which API function should be used + if self.iskeyreg: + r = self.api.register_identity(self.upp) + + if r.status_code == requests.codes.ok: + logger.info("The key resgistration message for \"%s\" was accepted" % self.uuid_str) + logger.info(r.content) + else: + logger.error("The key resgistration message for \"%s\" was not accepted; code: %d" % (self.uuid_str, r.status_code)) + logger.error(binascii.hexlify(r.content).decode()) + else: + r = self.api.send(self.uuid, self.upp) + + # set the response + self.response_upp = r.content + + if r.status_code == requests.codes.ok: + logger.info("The UPP for \"%s\" was accepted" % self.uuid_str) + logger.info(binascii.hexlify(r.content).decode()) + else: + logger.error("The UPP for \"%s\" was not accepted; code: %d" % (self.uuid_str, r.status_code)) + logger.error(binascii.hexlify(r.content).decode()) + + if r.status_code == 401: + logger.error("The UPP was rejected because of an authentication error! (Missing header/Invalid auth token)") + elif r.status_code == 403: + logger.error("The UPP wa rejected because of an verification error!") + except Exception as e: + logger.exception(e) + + return False + + return True + + def store_response_upp(self) -> bool: + try: + with open(self.output, "wb") as fd: + fd.write(self.response_upp) + + logger.info("The response UPP has been written to \"%s\"" % self.output) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # read the upp + if self.read_upp() != True: + logger.error("Errors occured while reading the UPP from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + # check if it is a key registration upp (return value is not the result but err code) + if self.check_is_keyreg() != True: + logger.error("Errors occured while checking if the UPP is a key registration UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the api + if self.init_api() != True: + logger.error("Errors occured while initializing the uBirch API - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # send the upp + if self.send_upp() != True: + logger.error("Errors occured while sending the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + if self.iskeyreg == False: + # store the response upp + if self.store_response_upp() != True: + logger.error("Erros occured while storing the response UPP to \"%s\" - exiting!" % self.output) + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 3f924560a0d47745b2442d91e78ff9559003ee7b Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:40:06 +0200 Subject: [PATCH 16/68] move unpacker --- examples/{unpack.py => upp-unpacker.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{unpack.py => upp-unpacker.py} (100%) diff --git a/examples/unpack.py b/examples/upp-unpacker.py similarity index 100% rename from examples/unpack.py rename to examples/upp-unpacker.py From 670d7facc8287d201221da822c65d55b0e05dfa9 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 11:40:19 +0200 Subject: [PATCH 17/68] add script to request anchoring status of an UPP --- examples/upp-anchoring-status.py | 203 +++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 examples/upp-anchoring-status.py diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py new file mode 100644 index 0000000..4a52688 --- /dev/null +++ b/examples/upp-anchoring-status.py @@ -0,0 +1,203 @@ +import sys +import time +import argparse +import logging +import msgpack +import uuid +import base64 +import json +import requests + + +VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" + +DEFAULT_ISHASH = "False" +DEFAULT_ENV = "dev" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.input : str = None + self.env : str = None + self.ishash_str : str = None + self.ishash : bool = None + + self.upp : bytes = None + self.hash : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Requests the verification/anchoring of a UPP from the uBirch backend", + epilog="When --ishash/-i is set to true, the input argument is treated as a base64 payload hash." + "Otherwise, it is expected to be some kind of path to read a UPP from." + "This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input." + ) + + self.argparser.add_argument("input", metavar="INPUT", type=str, + help="input hash or upp path (depends on --ishash)" + ) + self.argparser.add_argument("--ishash", "-i", metavar="ISHASH", type=str, default=DEFAULT_ISHASH, + help="sets if INPUT is being treated as a hash or upp path; true or false (default: %s)" % DEFAULT_ISHASH + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="the environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.input = self.args.input + self.ishash_str = self.args.ishash + self.env = self.args.env + + # get the bool for show sk + if self.ishash_str.lower() in ["1", "yes", "y", "true"]: + self.ishash = True + else: + self.ishash = False + + return True + + def read_upp(self) -> bool: + # read the UPP from the input path + try: + logger.info("Reading the input UPP from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + self.upp = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_upp(self) -> bool: + try: + unpacked = msgpack.unpackb(self.upp) + + # check if this upp is signed (0x21 == unsigned) + if unpacked[0] == 0x21: + # unsigned - no signature at the end + self.hash = unpacked[-1] + else: + # signed - signature at the end + self.hash = unpacked[-2] + + logger.info("Extracted UPP hash: \"%s\"" % base64.b64encode(self.hash).decode()) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_input(self) -> bool: + try: + self.hash = base64.b64decode(self.input) + + logger.info("Extracted hash from input: \"%s\"" % base64.b64encode(self.hash).decode()) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_status(self) -> bool: + try: + url = VERIFICATION_SERVICE % self.env + + logger.info("Requesting anchoring information from: \"%s\"" % url) + + r = requests.post( + url=url, + headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, + data=base64.b64encode(self.hash).decode().rstrip("\n") + ) + + if r.status_code == 200: + logger.info("The UPP is known to the uBirch backend! (code: %d)" % r.status_code) + + jobj = json.loads(r.content) + + print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) + print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + + if jobj.get("anchors") == None: + logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") + else: + logger.info("The UPP has been fully anchored!") + + print(jobj.get("anchors")) + elif r.status_code == 404: + logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check if the input data is the hash + if self.ishash == True: + # the hash can be extracted from the input parameter directly + if self.get_hash_from_input() != True: + logger.error("Errors occured while getting the hash from the input parameter - exiting!\n") + + self.argparser.print_usage() + + return 1 + else: + # the hash can be extracted from an upp which has to be read from a file + if self.read_upp() != True: + logger.error("Errors occured while reading the UPP from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + if self.get_hash_from_upp() != True: + logger.error("Errors occured while extracting the hash from the UPP - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # get the anchoring status + if self.get_status() != True: + logger.error("Errors occured while requesting the anchring status - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From c783b001a62fa4e746771be8439c98001fdfacb1 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 13:19:24 +0200 Subject: [PATCH 18/68] distinguish between v1/v2 upps when splitting off the signature --- ubirch/ubirch_protocol.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index f98eb60..6250f8f 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -274,11 +274,17 @@ def upp_msgpack_split_signature(self, msgpackUPP) -> (bytes, bytes): :return: a tuple consiting of the message without the signature and the signature """ try: - return (msgpackUPP[:-66], msgpackUPP[-64:]) + if msgpackUPP[1] >> 4 == 2: + # version 2 upp - 2 byte signature header + return (msgpackUPP[:-66], msgpackUPP[-64:]) + elif msgpackUPP[1] >> 4 == 1: + # version 1 upp - 3 byte signature header + return (msgpackUPP[:-67], msgpackUPP[-64:]) + else: + raise ValueError("Invalid UPP version byte: %02x" % msgpack[1]) except IndexError: raise ValueError("The UPP-msgpack is too short: %d bytes" % len(msgpackUPP)) - # -> def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> True: def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> bool: """ Verify the integrity of the message and decode the contents From 0e6d2d2e59f949daa9eafe7c622cd6cd694ae8b0 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Mon, 28 Jun 2021 16:22:57 +0200 Subject: [PATCH 19/68] fix tests --- tests/test_ubirch_protocol.py | 9 ++++----- tests/test_ubirch_protocol_ecdsa.py | 28 +++++++++++++--------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/tests/test_ubirch_protocol.py b/tests/test_ubirch_protocol.py index 87181c7..31341ea 100644 --- a/tests/test_ubirch_protocol.py +++ b/tests/test_ubirch_protocol.py @@ -110,7 +110,7 @@ def test_sign_not_implemented(self): def test_verify_not_implemented(self): p = ubirch.Protocol() try: - p.verfiy_signature(EXPECTED_SIGNED) + p.verfiy_signature(None, EXPECTED_SIGNED) except NotImplementedError as e: self.assertEqual(e.args[0], 'verification not implemented') @@ -169,7 +169,7 @@ def test_create_chained_message_with_hash(self): def test_verify_signed_message(self): p = Protocol() unpacked = p.unpack_upp(EXPECTED_SIGNED) - self.assertEqual(p.verfiy_signature(EXPECTED_SIGNED, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_SIGNED)), True) self.assertEqual(SIGNED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(0xEF, unpacked[2]) @@ -180,7 +180,7 @@ def test_verify_chained_messages(self): last_signature = b'\0' * 64 for i in range(0, 3): unpacked = p.unpack_upp(EXPECTED_CHAINED[i]) - self.assertEqual(p.verfiy_signature(EXPECTED_CHAINED[i], unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_CHAINED[i])), True) self.assertEqual(CHAINED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(last_signature, unpacked[2]) @@ -239,7 +239,7 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: unpacked = p.unpack_upp(message) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_CHAINED[i])), True) self.assertEqual(CHAINED & 0x0f, unpacked[0] & 0x0f) self.assertEqual(UUID(bytes=bytes.fromhex("af931b05acca758bc2aaeb98d6f93329")), UUID(bytes=unpacked[1])) self.assertEqual(0x54, unpacked[3]) @@ -266,7 +266,6 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: unpacked = p.unpack_upp(message) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) self.assertEqual(SIGNED & 0x0f, unpacked[0] & 0x0f) self.assertEqual(1, unpacked[0] >> 4) self.assertEqual(UUID(bytes=bytes.fromhex("00000000000000000000000000000000")), UUID(bytes=unpacked[1])) diff --git a/tests/test_ubirch_protocol_ecdsa.py b/tests/test_ubirch_protocol_ecdsa.py index 133f3f9..aa30e25 100644 --- a/tests/test_ubirch_protocol_ecdsa.py +++ b/tests/test_ubirch_protocol_ecdsa.py @@ -87,7 +87,7 @@ def test_sign_not_implemented(self): def test_verify_not_implemented(self): p = ubirch.Protocol() try: - p.verfiy_signature(EXPECTED_SIGNED) + p.verfiy_signature(None, EXPECTED_SIGNED) except NotImplementedError as e: self.assertEqual(e.args[0], 'verification not implemented') @@ -96,14 +96,14 @@ def test_create_signed_message(self): message = p.message_signed(TEST_UUID, 0xEF, 1) logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_SIGNED[0:-64], message[0:-64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) def test_create_signed_message_with_hash(self): p = Protocol() message = p.message_signed(TEST_UUID, 0xEF, hashlib.sha512(b'1').digest()) logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_SIGNED_HASH, message[0:-64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) def test_create_chained_messages(self): p = Protocol() @@ -117,7 +117,7 @@ def test_create_chained_messages(self): logger.debug("EXPECT : %s", binascii.hexlify(EXPECTED)) self.assertEqual(EXPECTED[0:-64], message[0:-64], "message #{} failed".format(i + 1)) self.assertEqual(last_signature, message[22:22+64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) last_signature = message[-64:] def test_create_chained_message_with_hash(self): @@ -126,12 +126,12 @@ def test_create_chained_message_with_hash(self): logger.debug("MESSAGE: %s", binascii.hexlify(message)) self.assertEqual(EXPECTED_CHAINED_HASH[0:-64], message[0:-64]) - self.assertEqual(p.verfiy_signature(message), True) + self.assertEqual(p.verfiy_signature(TEST_UUID, message), True) def test_verify_signed_message(self): p = Protocol() unpacked = p.unpack_upp(EXPECTED_SIGNED) - self.assertEqual(p.verfiy_signature(EXPECTED_SIGNED, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_SIGNED)), True) self.assertEqual(SIGNED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(0xEF, unpacked[2]) @@ -142,7 +142,7 @@ def test_verify_chained_messages(self): last_signature = b'\0' * 64 for i in range(0, 3): unpacked = p.unpack_upp(EXPECTED_CHAINED[i]) - self.assertEqual(p.verfiy_signature(EXPECTED_CHAINED[i], unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(EXPECTED_CHAINED[i])), True) self.assertEqual(CHAINED, unpacked[0]) self.assertEqual(TEST_UUID.bytes, unpacked[1]) self.assertEqual(last_signature, unpacked[2]) @@ -164,10 +164,8 @@ def test_verify_registration_message_sim_v1(self): unpacked = p.unpack_upp(message) - logger.debug(repr(unpacked)) - - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) - self.assertEqual(vk, binascii.hexlify(unpacked[3]['pubKey']).decode()) + self.assertEqual(p.verfiy_signature(None, bytes(message)), True) + self.assertEqual(vk, binascii.hexlify(unpacked[3][b'pubKey']).decode()) def test_verify_signed_message_sim_v1(self): loc = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -181,7 +179,7 @@ def test_verify_signed_message_sim_v1(self): unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), bytes(message)), True) self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) def test_verify_registration_message_sim_v2(self): @@ -196,7 +194,7 @@ def test_verify_registration_message_sim_v2(self): unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), message), True) self.assertEqual(vk, binascii.hexlify(unpacked[3]['pubKey']).decode()) def test_verify_signed_message_sim_v2(self): @@ -211,5 +209,5 @@ def test_verify_signed_message_sim_v2(self): unpacked = p.unpack_upp(message) logger.debug(repr(unpacked)) - self.assertEqual(p.verfiy_signature(message, unpackedUPP=unpacked), True) - self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) \ No newline at end of file + self.assertEqual(p.verfiy_signature(UUID(bytes=unpacked[1]), message), True) + self.assertEqual(hashlib.sha256(b"UBIRCH").digest(), unpacked[3]) From b8a614b97c291655281bda06377a5e7c49b3c08b Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 18:00:27 +0200 Subject: [PATCH 20/68] add a script to send data to the uBirch Simple Data Service --- examples/data-sender.py | 145 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 examples/data-sender.py diff --git a/examples/data-sender.py b/examples/data-sender.py new file mode 100644 index 0000000..136e5df --- /dev/null +++ b/examples/data-sender.py @@ -0,0 +1,145 @@ +import sys +import logging +import argparse +import msgpack +import requests +import binascii +import uuid + +import ubirch + + +DEFAULT_ENV = "dev" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.uuid_str : str = None + self.uuid : uuid.UUID = None + self.auth : str = None + self.env : str = None + self.input : str = None + + self.api : ubirch.API = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Send some data to the uBirch Simple Data Service", + epilog="Note that the input data should follow this pattern: " + "{\"timestamp\": TIMESTAMP, \"uuid\": \"UUID\", \"msg_type\": 0, \"data\": DATA, \"hash\": \"UPP_HASH\"}. " + "For more information take a look at the EXAMPLES.md file." + ) + + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("auth", metavar="AUTH", type=str, + help="uBirch device authentication token" + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + self.argparser.add_argument("input", metavar="INPUT", type=str, + help="data to be sent to the simple data service" + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.uuid_str = self.args.uuid + self.auth = self.args.auth + self.env = self.args.env + self.input = self.args.input + + # process the uuid + try: + self.uuid = uuid.UUID(hex=self.uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.uuid_str) + logger.exception(e) + + return False + + # validate env + if self.env.lower() not in ["dev", "demo", "prod"]: + logger.error("Invalid value for --env: \"%s\"!" % self.env) + + return False + + return True + + def init_api(self) -> bool: + try: + # initialize the uBirch api + self.api = ubirch.API(env=self.env, debug=True) + self.api.set_authentication(self.uuid, self.auth) + except Exception as e: + logger.exception(e) + + return False + + return True + + def send_data(self) -> bool: + try: + r = self.api.send_data(self.uuid, self.input.encode()) + + # check the response + if r.status_code == 200: + logger.info("Successfully sent all data to the Simple Data Service! (%d)" % r.status_code) + else: + logger.error("Failed to send data to the Simple Data Service! (%d)" % r.status_code) + logger.error(r.content) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the api + if self.init_api() != True: + logger.error("Errors occured while initializing the uBirch API - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # send data + if self.send_data() != True: + logger.error("Errors occured while sending data to the simple data service - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 5b90291f33f1e0a294fca039efc391ce0bb75331 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 18:13:08 +0200 Subject: [PATCH 21/68] fix old comment --- examples/upp-anchoring-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 4a52688..ae7fce9 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -64,7 +64,7 @@ def process_args(self) -> bool: self.ishash_str = self.args.ishash self.env = self.args.env - # get the bool for show sk + # get the bool for ishash if self.ishash_str.lower() in ["1", "yes", "y", "true"]: self.ishash = True else: From caed4639c7a875175a654577a2ce1a369a4f7ace Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 18:40:09 +0200 Subject: [PATCH 22/68] only print anchoring information if the UPP is anchored --- examples/upp-anchoring-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index ae7fce9..9195ef4 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -143,7 +143,7 @@ def get_status(self) -> bool: else: logger.info("The UPP has been fully anchored!") - print(jobj.get("anchors")) + print(jobj.get("anchors")) elif r.status_code == 404: logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) except Exception as e: From 53bab4f1b09234b3eea6ec7e766cca5da7139489 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 19:53:30 +0200 Subject: [PATCH 23/68] move and extend data verifier script --- examples/data-verifier.py | 289 ++++++++++++++++++++++++++++++++++++++ examples/data_verifier.py | 28 ---- 2 files changed, 289 insertions(+), 28 deletions(-) create mode 100644 examples/data-verifier.py delete mode 100644 examples/data_verifier.py diff --git a/examples/data-verifier.py b/examples/data-verifier.py new file mode 100644 index 0000000..830a28a --- /dev/null +++ b/examples/data-verifier.py @@ -0,0 +1,289 @@ + +# import hashlib +# import json + +# import requests + +# VERIFICATION_SERVICE = "https://verify.prod.ubirch.com/api/upp/verify/anchor" + +# with open("data_to_verify.json") as f: +# message = json.load(f) + +# # create a compact rendering of the message to ensure determinism when creating the hash +# serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() +# print("rendered data:\n\t{}\n".format(serialized.decode())) + +# # calculate hash of message +# data_hash = hashlib.sha256(serialized).digest() +# print("hash [base64]:\n\t{}\n".format(base64.b64encode(data_hash).decode())) + +# # verify existence of the hash in the UBIRCH backend +# r = requests.post(url=VERIFICATION_SERVICE, +# headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, +# data=base64.b64encode(data_hash).decode().rstrip('\n')) + +# if 200 <= r.status_code < 300: +# print("verification successful:\n\t{}\n".format(r.content.decode())) +# else: +# print("verification FAIL: ({})\n\tdata hash could not be verified\n".format(r.status_code)) + +import sys +import argparse +import base64 +import logging +import json +import hashlib +import requests + + +VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" + +DEFAULT_ISPATH = "False" +DEFAULT_ENV = "dev" +DEFAULT_ISJSON = "True" +DEFAULT_HASH = "sha256" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.input : str = None + self.env : str = None + self.ispath_str : str = None + self.ispath : bool = None + self.isjson_str : str = None + self.isjson : bool = None + self.hashalg : str = None + + self.ishash : bool = False + self.hasher : object = None + + self.data : bytes = None + self.hash : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Check if the hash of given input data is known to the uBirch backend (verify it)", + epilog="When --ispath/-i is set to true, the input data is treated as a file path to read the " + "actual input data from. When setting --hash/-a to off, the input argument is expected " + "to be a valid base64 encoded hash." + ) + + self.argparser.add_argument("input", metavar="INPUT", type=str, + help="input data or data file path (depends on --ispath)" + ) + self.argparser.add_argument("--ispath", "-i", metavar="ISHASH", type=str, default=DEFAULT_ISPATH, + help="sets if INPUT is being treated as data or data file path; true or false (default: %s)" % DEFAULT_ISPATH + ) + self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, + help="the environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV + ) + self.argparser.add_argument("--isjson", "-j", metavar="ISJSON", type=str, default=DEFAULT_ISJSON, + help="tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: %s)" % DEFAULT_ISJSON + ) + self.argparser.add_argument("--hash", "-a", metavar="HASH", type=str, default=DEFAULT_HASH, + help="sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: %s)" % DEFAULT_HASH + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.input = self.args.input + self.ispath_str = self.args.ispath + self.isjson_str = self.args.isjson + self.env = self.args.env + self.hashalg = self.args.hash + + # check the value for --hash + if self.hashalg.lower() == "off": + self.ishash = True + elif self.hashalg.lower() == "sha256": + self.hasher = hashlib.sha256() + self.ishash = False + elif self.hashalg.lower() == "sha512": + self.hasher = hashlib.sha512() + self.ishash = False + else: + logger.error("the value for --hash/-a must be \"sha256\", \"sha512\" or \"off\"; \"%s\" is invalid!" % self.hashalg) + + return False + + # check the value for --env + if self.env not in ["prod", "demo", "dev"]: + logger.error("the value for --env/-e must be \"prod\", \"demo\" or \"dev\"; \"%s\" is invalid!" % self.env) + + return False + + # get the bool for ispath + if self.ispath_str.lower() in ["1", "yes", "y", "true"]: + self.ispath = True + else: + self.ispath = False + + # get the bool for isjson + if self.isjson_str.lower() in ["1", "yes", "y", "true"]: + self.isjson = True + else: + self.isjson = False + + return True + + def read_data(self) -> bool: + # read data from the input path + try: + logger.info("Reading the input data from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + self.data = fd.read() + except Exception as e: + logger.exception(e) + + return False + + return True + + def serialize_json(self) -> bool: + try: + # load the string as json and put it back into a string, serealizing it + self.data = json.loads(self.data) + self.data = json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() + + logger.info("Serialized JSON: \"%s\"" % self.data.decode()) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_data(self) -> bool: + try: + # calculate the hash + self.hasher.update(self.data) + + self.hash = self.hasher.digest() + self.hash = base64.b64encode(self.hash).decode().rstrip("\n") + + logger.info("Calculated hash: \"%s\"" % self.hash) + except Exception as e: + logger.exception(e) + + return False + + return True + + def get_hash_from_input(self) -> bool: + self.hash = self.input + + return True + + def get_status(self) -> bool: + try: + url = VERIFICATION_SERVICE % self.env + + logger.info("Requesting anchoring information from: \"%s\"" % url) + + r = requests.post( + url=url, + headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, + data=self.hash + ) + + if r.status_code == 200: + logger.info("The hash is known to the uBirch backend! (code: %d)" % r.status_code) + + jobj = json.loads(r.content) + + print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) + print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + + if jobj.get("anchors") in [None, []]: + logger.info("The corresponding UPP has NOT been anchored into any blockchains yet! Please retry later") + else: + logger.info("The corresponding UPP has been fully anchored!") + + print(jobj.get("anchors")) + elif r.status_code == 404: + logger.info("The hash is NOT known to the uBirch backend! (code: %d)" % r.status_code) + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check if the input data is the hash + if self.ishash == True: + # the hash can be extracted from the input parameter directly + if self.get_hash_from_input() != True: + logger.error("Errors occured while getting the hash from the input parameter - exiting!\n") + + self.argparser.print_usage() + + return 1 + else: + # check if the input is a path or the actual data + if self.ispath == True: + # read the data from the given path/file + if self.read_data() != True: + logger.error("Errors occured while reading data from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + else: + self.data = self.input + + # check if the input data is json/should be serialized + if self.isjson == True: + if self.serialize_json() != True: + logger.error("Error occured while serealizing the JSON data - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # calculate the hash + if self.get_hash_from_data() != True: + logger.error("Error calculating the hash - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # get the anchoring status + if self.get_status() != True: + logger.error("Errors occured while requesting the anchring status - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) diff --git a/examples/data_verifier.py b/examples/data_verifier.py deleted file mode 100644 index f14bbb5..0000000 --- a/examples/data_verifier.py +++ /dev/null @@ -1,28 +0,0 @@ -import base64 -import hashlib -import json - -import requests - -VERIFICATION_SERVICE = "https://verify.prod.ubirch.com/api/upp/verify/anchor" - -with open("data_to_verify.json") as f: - message = json.load(f) - -# create a compact rendering of the message to ensure determinism when creating the hash -serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() -print("rendered data:\n\t{}\n".format(serialized.decode())) - -# calculate hash of message -data_hash = hashlib.sha256(serialized).digest() -print("hash [base64]:\n\t{}\n".format(base64.b64encode(data_hash).decode())) - -# verify existence of the hash in the UBIRCH backend -r = requests.post(url=VERIFICATION_SERVICE, - headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, - data=base64.b64encode(data_hash).decode().rstrip('\n')) - -if 200 <= r.status_code < 300: - print("verification successful:\n\t{}\n".format(r.content.decode())) -else: - print("verification FAIL: ({})\n\tdata hash could not be verified\n".format(r.status_code)) From b960adc69c245579ea68abab75f2c49fe9ab328a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 19:53:54 +0200 Subject: [PATCH 24/68] fix result displaying --- examples/upp-anchoring-status.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 9195ef4..1c39f81 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -40,8 +40,8 @@ def __init__(self): def setup_argparse(self): self.argparser = argparse.ArgumentParser( description="Requests the verification/anchoring of a UPP from the uBirch backend", - epilog="When --ishash/-i is set to true, the input argument is treated as a base64 payload hash." - "Otherwise, it is expected to be some kind of path to read a UPP from." + epilog="When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. " + "Otherwise, it is expected to be some kind of path to read a UPP from. " "This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input." ) @@ -138,7 +138,7 @@ def get_status(self) -> bool: print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) - if jobj.get("anchors") == None: + if jobj.get("anchors") in [None, []]: logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") else: logger.info("The UPP has been fully anchored!") From f0ec0624ddf4c2f5ab03f87f0af9e5aee995c75a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 1 Jul 2021 19:54:26 +0200 Subject: [PATCH 25/68] add ability to serialize json before creating the hash for the UPP --- examples/upp-creator.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 1382698..785ada9 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -16,6 +16,7 @@ DEFAULT_KS_PWD = "keystore" DEFAULT_KEYREG = "False" DEFAULT_HASH = "sha512" +DEFAULT_ISJSON = "False" DEFAULT_OUTPUT = "upp.bin" DEFAULT_NOSTDOUT = "False" @@ -66,6 +67,8 @@ def __init__(self): self.keystore_path : str = None self.keystore_pass : str self.output : str = None + self.isjson_str : str = None + self.isjson : bool = None self.keyreg_str : str = None self.keyreg : bool = None self.nostdout_str : str = None @@ -117,6 +120,9 @@ def setup_argparse(self): self.argparser.add_argument("--hash", metavar="HASH", type=str, default=DEFAULT_HASH, help="hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: %s)" % DEFAULT_HASH ) + self.argparser.add_argument("--isjson", "-j", metavar="ISJSON", type=str, default=DEFAULT_ISJSON, + help="tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: %s)" % DEFAULT_ISJSON + ) self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, help="file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: %s)" % DEFAULT_OUTPUT ) @@ -135,6 +141,7 @@ def process_args(self) -> bool: self.type_str = self.args.type self.keyreg_str = self.args.keyreg self.hash = self.args.hash + self.isjson_str = self.args.isjson self.keystore_path = self.args.ks self.keystore_pass = self.args.kspwd self.output = self.args.output @@ -189,6 +196,12 @@ def process_args(self) -> bool: else: self.nostdout = False + # get the isjson value + if self.isjson_str.lower() in ["1", "yes", "y", "true"]: + self.isjson = True + else: + self.isjson = False + # success return True @@ -232,6 +245,13 @@ def prepare_payload(self) -> bool: if self.nostdout == False: logger.info("UPP payload (raw data): \"%s\"" % self.payload) else: + if self.isjson == True: + # load the string as json and put it back into a string, serealizing it + self.data = json.loads(self.data) + self.data = json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False) + + logger.info("Serialized data JSON: \"%s\"" % self.data) + self.payload = self.hasher(self.data.encode()).digest() if self.nostdout == False: From 5fc0190b8eb1d15c745021c53a97793153e88a2c Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 14:31:19 +0200 Subject: [PATCH 26/68] fix invalid indentation --- examples/upp-creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 785ada9..d089b6c 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -270,7 +270,7 @@ def create_upp(self) -> bool: if self.nostdout == False: logger.info("Generating a key registration UPP for UUID \"%s\"" % self.uuid_str) - self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) + self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) pass else: if self.version == 0x22: From 3775c12d284c121fd34976a74e799349af36d681 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 16:57:56 +0200 Subject: [PATCH 27/68] fix parameter description --- examples/data-verifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 830a28a..87f8c61 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -83,7 +83,7 @@ def setup_argparse(self): self.argparser.add_argument("input", metavar="INPUT", type=str, help="input data or data file path (depends on --ispath)" ) - self.argparser.add_argument("--ispath", "-i", metavar="ISHASH", type=str, default=DEFAULT_ISPATH, + self.argparser.add_argument("--ispath", "-i", metavar="ISPATH", type=str, default=DEFAULT_ISPATH, help="sets if INPUT is being treated as data or data file path; true or false (default: %s)" % DEFAULT_ISPATH ) self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, From 25a15fb16af986f05f19fc04d28071fa05be86e9 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 17:01:30 +0200 Subject: [PATCH 28/68] add documentation for some examples --- examples/EXAMPLES.md | 365 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 examples/EXAMPLES.md diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md new file mode 100644 index 0000000..38d2f71 --- /dev/null +++ b/examples/EXAMPLES.md @@ -0,0 +1,365 @@ +# uBirch-Protocol-Python Examples +This file documents how to use the examples provided alongside the [uBirch-Protocol-Python](https://github.com/ubirch/ubirch-protocol-python). Those examples aim to provide an insight of how to use the [ubirch-protocol](https://pypi.org/project/ubirch-protocol/) python library, which is implemented in the `/ubirch/` directory in this repository. + +## Table of contents +- [uBirch-Protocol-Python Examples](#ubirch-protocol-python-examples) + - [Table of contents](#table-of-contents) + - [From measurement to blockchain-anchored UPP](#from-measurement-to-blockchain-anchored-upp) + - [Setup](#setup) + - [Generating and managing a keypair](#generating-and-managing-a-keypair) + - [Registering a public key](#registering-a-public-key) + - [Gathering Data](#gathering-data) + - [Creating a UPP](#creating-a-upp) + - [Sending a UPP](#sending-a-upp) + - [Verifying a UPP](#verifying-a-upp) + - [Examining a UPP](#examining-a-upp) + - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) + - [Verifying a measurement](#verifying-a-measurement) + - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) + - [Example uBirch client implementation](#example-ubirch-client-implementation) + +## From measurement to blockchain-anchored UPP +The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how one could handle them. There also are examples showing a full example-client implementation. + +0. [Setup](#setup) +1. [Gathering Data](#gathering-data) + +### Setup +Before anything, you will need to do/get a couple of things: +- Choose a stage to work on +- Get a account for the uBirch-Console + - https://console.prod.ubirch.com for the `prod` stage + - https://console.demo.ubirch.com for the `demo` stage + - https://console.dev.ubirch.com for the `dev` stage +- Get a UUID (can be generated randomly or on the basis of certain device properties like MAC-Addresses) +- Create a "Thing" at the uBirch-Console; remember/note down the used UUID and the generated Auth-Token + +You should now have the following information at hand: +- The stage you want to work on (later referred to as `env`) +- The UUID of your device or "fake" device in this instance +- The authentication token (`auth token`) for the named UUID + +The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, `demo` for the env and +`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. + +### Generating and managing a keypair +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check if the signature is valid and belongs to the correct sender/signer. So logically, it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real usecase, a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against an attacker reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. For that you will have to chose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is exlpained [bellow](#registering-a-public-key). + +**NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. + +### Registering a public key +To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend before starting to send UPPs supposed to be verified and anchored. Registering a verifying key is also done by sending a special kind of UPP containing this key. This can be done by using two scripts: +``` +upp-creator.py +upp-sender.py +``` +Both of these scripts will be explained in more detail in [Creating a UPP](#creating-a-upp) and [Sending a UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: +``` +$ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --output keyreg_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 none + +2021-07-02 11:51:50,483 root init_keystore() INFO No keys found for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" in "devices.jks" - generating a keypair +2021-07-02 11:51:50,485 ubirch.ubirch_ks insert_ed25519_keypa() INFO inserted new key pair for f5ded8a3d46241c4a8dcaf3fd072a217: e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4c +2021-07-02 11:51:50,485 root init_keystore() INFO Public/Verifying key for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" [base64]: "4CZOfZQoFJzvWczsuIE7IU2PlMYuPoNtdUbT+L2ISkw=" +2021-07-02 11:51:50,485 root load() WARNING no existing saved signatures +2021-07-02 11:51:50,485 root prepare_payload() INFO UPP payload (sha512 hash of the data) [base64]: "q5Op6V1w7bBgJVEc6k4rgEf7fh3q9yRPwNPt9efLV9j7e5Ub3rPGtVJxSHh0nrGbkQPmSoNjXoiFx9Ph0PxWSQ==" +2021-07-02 11:51:50,485 root create_upp() INFO Generating a key registration UPP for UUID "f5ded8a3-d462-41c4-a8dc-af3fd072a217" +2021-07-02 11:51:50,485 root show_store_upp() INFO UPP [hex]: "9522c410f5ded8a3d46241c4a8dcaf3fd072a2170187a9616c676f726974686dab4543435f45443235353139a763726561746564ce60dec596aa68774465766963654964c410f5ded8a3d46241c4a8dcaf3fd072a216a67075624b6579c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4ca87075624b65794964c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4cad76616c69644e6f744166746572ce62bff916ae76616c69644e6f744265666f7265ce60dec596c440cadb70d30250a5a2dd2eb44b645e54b56387f228607fbf6f59a11493befa118f0e9c79da1f7d85ba5a4076c134f8b4aff04173adfc4b858ec491be2366988900" +2021-07-02 11:51:50,485 root show_store_upp() INFO UPP written to "keyreg_upp.bin" +``` +The generated key registration UPP has been saved to `keyreg_upp.bin`. Sending the UPP can be done like following (remember to put the correct value for the env of your choice): +``` +$ python upp-sender.py --env demo --input keyreg_upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +2021-07-02 12:05:49,556 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "keyreg_upp.bin" +2021-07-02 12:05:49,556 root check_is_keyreg() INFO The UPP is a key registration UPP - disabling identity registration check +2021-07-02 12:05:50,712 root send_upp() INFO The key resgistration message for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" was accepted +2021-07-02 12:05:50,712 root send_upp() INFO b'{"pubKeyInfo":{"algorithm":"ECC_ED25519","created":"2021-06-27T13:12:46.000Z","hwDeviceId":"f5ded8a3-d462-41c4-a8dc-af3fd072a217","pubKey":"dsbKkw9HpsTvlLGgmiaYAM4M/ytFcySoF5UbfScffxg=","pubKeyId":"dsbKkw9HpsTvlLGgmiaYAM4M/ytFcySoF5UbfScffxg=","validNotAfter":"2022-06-27T13:12:46.000Z","validNotBefore":"2021-06-27T13:12:46.000Z"},"signature":"1e47255c7494fb54d5ab10897d53c1a38872e6a67af3d53e5ae49d6c190d0aaab70741a1af9b08793aaae82ea5a207402d5a15c563b859525e193a05f0a6510b"}' +``` +After the command successfully completed there should be an entry in the `PublicKeys` tab of the device. + +### Gathering Data +UPPs are usually used to anchor the hash of some kind of data. This data, in theory, can be everything. All examples below will use a simple string representing a JSON object like this for simplicity: +```json +{ + "ts": 1625163338, + "T": 11.2, + "H": 35.8, + "S": "OK" +} +``` +Translated to a hypothetical usecase this could be a measurement taken at `1625163338` (Unix-Timestamp), stating that the sensor measured `11.2 C` in temperature (`T`) and `35.8 %H` in humidity (`H`). The status - `S` - is `'OK'`. There is no script for this step, since it can easily done by hand. + +### Creating a UPP +After gathering some measurement data a UPP can be created. The won't contain the actual measurement data, but a hash of it. The script used to create UPPs in this example is `upp-creator.py`. +``` +$ python3 upp-creator.py --help + +usage: upp-creator.py [-h] [--version VERISON] [--type TYPE] [--ks KS] [--kspwd KSPWD] [--keyreg KEYREG] [--hash HASH] [--isjson ISJSON] [--output OUTPUT] [--nostdout nostdout] UUID DATA + +Create a uBirch Protocol Package (UPP) + +positional arguments: + UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 + DATA data to be packed into the UPP or hashed; e.g.: {"t": 23.4, "ts": 1624624140} + +optional arguments: + -h, --help show this help message and exit + --version VERISON, -v VERISON + version of the UPP; 0x21 (unsigned; NOT IMPLEMENTED), 0x22 (signed) or 0x23 (chained) (default: 0x23) + --type TYPE, -t TYPE type of the UPP (0 < type < 256); e.g.: 0x00 (unknown), 0x32 (msgpack), 0x53 (generic), ... (default and recommended: 0x00) + --ks KS, -k KS keystore file path; e.g.: test.jks (default: devices.jks) + --kspwd KSPWD, -p KSPWD + keystore password; e.g.: secret (default: keystore) + --keyreg KEYREG, -r KEYREG + generate a key registration UPP (data and --hash will be ignored); e.g.: true, false (default: False) + --hash HASH hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: sha512) + --isjson ISJSON, -j ISJSON + tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: False) + --output OUTPUT, -o OUTPUT + file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: upp.bin) + --nostdout nostdout, -n nostdout + do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) + +Note that when using chained UPPs (--version 0x23), this tool will try to load/save signatures to UUID.sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain +using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the +payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. +``` +The script allows multiple modes of operation which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two (`0010`) set in almost all usecases. The mode can eather be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The one would be called a _Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--type/-t` This flag sets the type field of the UPP. It it used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases in which a specific value is required are Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). +- `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore pointed to by this parameter doesn't exist, the script will simple create it. +- `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this or you will lose access to the keystore and all its contents. +- `--keyreg/-k` Tells the script that the UPP that should be generated is a key registration UPP. The effect of that is that the script will ignore any custom input data and the `--hash` parameter (below). Instead, the UPP will contain the public key certificate. This parameter is a binary flag which can have two values: `true` or `false`. +- `--hash` Sets the hash algorithm to used to hash the input data. The produced hash will then be inserted into the payload field of the UPP. This parameter can have three values: `sha512`, `sha256` and `off`. When set to off, the input data will be directly put into the UPP without hashing it. This is only useful in some special cases like when manually assembling key registration messages (normally the `--keyreg` option should be used for that). +- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. +- `--output/-o` Tells the script where to write the generated UPP to. +- `--nostdout/-n` Binary flag to disable printing of any log messages to standard output. This can be used for piping a created UPP to another program. For this `--output /dev/stdout` would have to be set. +- `UUID` The UUID of the device as a hex-string, like `f5ded8a3-d462-41c4-a8dc-af3fd072a217`. +- `DATA` The data that is going to be hashed. If `--isjson true` is provided, it has to be a string representing a valid JSON object. **Note** that even though this argument will be ignored when `--keyreg true` is set, it must still exist. + +One common examples of using this script might look like this: +``` +$ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sha256 f5ded8a3-d462-41c4-a8dc-af3fd072a217 '{ + "ts": 1625163338, + "T": 11.2, + "H": 35.8, + "S": "OK" +}' +2021-07-02 15:07:53,040 root init_keystore() INFO Public/Verifying key for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" [base64]: "dsbKkw9HpsTvlLGgmiaYAM4M/ytFcySoF5UbfScffxg=" +2021-07-02 15:07:53,041 root prepare_payload() INFO Serialized data JSON: "{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}" +2021-07-02 15:07:53,041 root prepare_payload() INFO UPP payload (sha256 hash of the data) [base64]: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 15:07:53,041 root create_upp() INFO Generating a chained signed UPP for UUID "f5ded8a3-d462-41c4-a8dc-af3fd072a217" +2021-07-02 15:07:53,041 root show_store_upp() INFO UPP [hex]: "9623c410f5ded8a3d46241c4a8dcaf3fd072a217c440cbe84f33c1d80a9a2a68f10c61c843567035d19179a703bb5e0aff4e920d9b8535acb171f1fd55271371d199fc985f33cf0b31f3c6ecfa7be684b561ac6d900f00c42075f42eef00422f6682b80a962e4c871170b34e528775f32becf32bc4473063a0c440ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef4421102" +2021-07-02 15:07:53,041 root show_store_upp() INFO UPP written to "upp.bin" +``` +Keep in mind that when using chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but defeat the purpose of chaining UPPs. + +### Sending a UPP +After creating the UPP, it can be sent to the uBirch backend where it will be verified (the backend will use the registered public/verifying key to check the signature) and anchored into the blockchain. The `upp-sender.py` script can be used for that. +``` +$ python3 upp-sender.py --help +usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH + +Send a uBirch Protocol Package (UPP) to uBirch Niomon + +positional arguments: + UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 + AUTH uBirch device authentication token + +optional arguments: + -h, --help show this help message and exit + --env ENV, -e ENV environment to operate in; dev, demo or prod (default: dev) + --input INPUT, -i INPUT + UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) + --output OUTPUT, -o OUTPUT + response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: response_upp.bin) +``` +For this script there aren't that many parameters, since the task is rather easy and straight forward. +- `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage the UUID is registered on. +- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin` if, for example, the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. +- `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. +- `UUID` The UUID of the device that generated the UPP as a hex-string. +- `AUTH` The auth token for the device on the specified stage as a hex-string. +Continuing from the example above (see [Creating a UPP](#creating-a-upp)), the send-command might look like this: +``` +$ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +2021-07-02 15:21:36,966 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "upp.bin" +2021-07-02 15:21:37,722 root send_upp() INFO The UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" was accepted +2021-07-02 15:21:37,723 root send_upp() INFO 9623c4109d3c78ff22f34441a5d185c636d486ffc440ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef442110200c42049950c5d778045a7b20c5e4db820c38100000000000000000000000000000000c440577f3679edbf96120066b9d1a794651817ec36fe6d1728841a7110ef0a2692c1e72827e8a48f98eefb42777b4fafd47c6bd7931e21c3c983c6f0c8a99144f90c +2021-07-02 15:21:37,723 root store_response_upp() INFO The response UPP has been written to "response_upp.bin" +``` + +### Verifying a UPP +To make sure that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks if the signature of the response UPP is valid. + +``` +$ python3 upp-verifier.py --help +usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] + +Check if a UPP is valid/properly signed + +optional arguments: + -h, --help show this help message and exit + --verifying-key VK, -k VK + key to be used for verification; any verifying key in hex like "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" + --verifying-key-uuid UUID, -u UUID + the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 + --input INPUT, -i INPUT + UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) + +Note that when trying to verify a UPP sent by the uBirch backend (Niomon) a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +``` +- `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. +- `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. +- The file path to read the UPP from. + +``` +$ python3 upp-verifier.py --input response_upp.bin +2021-07-02 15:43:36,273 root read_upp() INFO Reading the input UPP from "response_upp.bin" +2021-07-02 15:43:36,274 ubirch.ubirch_ks _load_keys() WARNING creating new key store: -- temporary -- +2021-07-02 15:43:36,274 root get_upp_uuid() INFO UUID of the UPP creator: "07104235-1892-4020-9042-00003c94b60b" +2021-07-02 15:43:36,275 root verify_upp() INFO Signature verified - the UPP is valid! +``` + +### Examining a UPP +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the `upp-unpacker.py` script can be used like this: +``` +$ python3 upp-unpacker.py response_upp.bin +- Version: 0x23 +- UUID: 9d3c78ff-22f3-4441-a5d1-85c636d486ff +- prev.Sign.: zMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg== + [hex]: ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef4421102 (64 bytes) +- Type: 0x00 +- Payload: SZUMXXeARaeyDF5NuCDDgQAAAAAAAAAAAAAAAAAAAAA= + [hex]: 49950c5d778045a7b20c5e4db820c38100000000000000000000000000000000 (32 bytes) +- Signature: V382ee2/lhIAZrnRp5RlGBfsNv5tFyiEGnEQ7womksHnKCfopI+Y7vtCd3tPr9R8a9eTHiHDyYPG8MipkUT5DA== + [hex]: 577f3679edbf96120066b9d1a794651817ec36fe6d1728841a7110ef0a2692c1e72827e8a48f98eefb42777b4fafd47c6bd7931e21c3c983c6f0c8a99144f90c (64 bytes) +``` +_The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ + +### Checking the anchoring status of an UPP +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script for that task is `upp-anchoring-status.py`. +``` +$ python3 upp-anchoring-status.py -h +usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT + +Requests the verification/anchoring of a UPP from the uBirch backend + +positional arguments: + INPUT input hash or upp path (depends on --ishash) + +optional arguments: + -h, --help show this help message and exit + --ishash ISHASH, -i ISHASH + sets if INPUT is being treated as a hash or upp path; true or false (default: False) + --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) + +When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. +``` +- `--ishash/-i` A boolean specifying wether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. +- `INPUT` The input UPP file path or payload hash, depending on `--ishash`. + +One example might be: +``` +python3 upp-anchoring-status.py --env demo upp.bin +2021-07-02 16:01:46,761 root read_upp() INFO Reading the input UPP from "upp.bin" +2021-07-02 16:01:46,761 root get_hash_from_upp() INFO Extracted UPP hash: "ToBgV89kXaWU0YHblha7qUXn0gohzpKoIS515cmSl4Y=" +2021-07-02 16:01:46,761 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:01:46,950 root get_status() INFO The UPP is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMzH452aGsvznTB9CNW190IYAW4LnnTR78dkDFQMTNob8YKziaftn9P++wR85s9RPdGgRxk+0KExEPcn/vRCEQIAxCBOgGBXz2RdpZTRgduWFrupRefSCiHOkqghLnXlyZKXhsRAZw4gMp5Wlq5Sij9UQrjMfhxdmeoY6IsVS7Aq8MLZyUT5CvTeEK/4kt4N55tE8pYVN7G+FxEYwvYfwDLZPqViBw==" +Prev. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +2021-07-02 16:01:46,950 root get_status() INFO The UPP has NOT been anchored into any blockchains yet! Please retry later +``` +Here it is visible that the backend knows the UPP and that it is valid, but it hasn't been anchored yet. Additionally the output shows that The backend knows the previous UPP, indicating that the UPP is a chained UPP and not the first UPP in the chain. When using unchained UPPs the line will change to: `Prev. UPP: "None"`. After waiting some time and running the script again with the same parameters: +``` +$ python3 upp-anchoring-status.py --env demo upp.bin +2021-07-02 16:09:34,521 root read_upp() INFO Reading the input UPP from "upp.bin" +2021-07-02 16:09:34,521 root get_hash_from_upp() INFO Extracted UPP hash: "ToBgV89kXaWU0YHblha7qUXn0gohzpKoIS515cmSl4Y=" +2021-07-02 16:09:34,521 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:09:34,727 root get_status() INFO The UPP is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMzH452aGsvznTB9CNW190IYAW4LnnTR78dkDFQMTNob8YKziaftn9P++wR85s9RPdGgRxk+0KExEPcn/vRCEQIAxCBOgGBXz2RdpZTRgduWFrupRefSCiHOkqghLnXlyZKXhsRAZw4gMp5Wlq5Sij9UQrjMfhxdmeoY6IsVS7Aq8MLZyUT5CvTeEK/4kt4N55tE8pYVN7G+FxEYwvYfwDLZPqViBw==" +Prev. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +2021-07-02 16:09:34,727 root get_status() INFO The UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T14:03:22.093Z', 'hash': '0xd1fe1f27a315089e5522eb7c8124962774b335c24d1ed7281091b447a8d3bca2', 'public_chain': 'ETHEREUM_TESTNET_RINKEBY_TESTNET_NETWORK', 'prev_hash': '06700cdb7b196292eceac71520fad2e46890e2d8f74510f1bc4296c6a0e16a631cff533989c9b83363f72051105b8f0bfaf59706a5258d8d275abc93d67d5b4d'}}] +``` +The UPP has been anchored. **Note** that when running on `prod` the output regarding the anchoring status will be significantly longer: +``` +$ python3 upp-anchoring-status.py --env prod --ishash true "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 16:13:47,509 root get_hash_from_input() INFO Extracted hash from input: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 16:13:47,509 root get_status() INFO Requesting anchoring information from: "https://verify.prod.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:13:47,631 root get_status() INFO The UPP is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQA1F2OFz+pQfCw7yxznodtsSf5ubCXPjHOFNWPexyiNFVHouv4m2mcDHzu8icxoD1U8pXFtXscsFrYy3+oCfPgoAxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRADipQZBD9bOdYezTD49h8MuAGBspO+PCkHFAMor8H3OZGRKXs0i4Fa4ICG0VV8B6PtVzoKz5vf8m6pWGFAb/wBQ==" +Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuTTjmzc05qXnjhs+sdQtwN7To21DBrCeDDmK4MYFixx/umBAoAxEAns4128paQWJKi9D+W9UzQqpWtOtg1474cDWvxHTMSGnP87f6IllmqA+DHJ3Xe6LZ47hlbjUsLJtAiYtS1u1hBxEANRdjhc/qUHwsO8sc56HbbEn+bmwlz4xzhTVj3scojRVR6Lr+JtpnAx87vInMaA9VPKVxbV7HLBa2Mt/qAnz4K" +2021-07-02 16:13:47,631 root get_status() INFO The UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:46:45.949Z', 'hash': '0xc32ccbe70ed727a1842f998d56d9928a9a30e201aef91bb08d9ac7faf931dac6', 'public_chain': 'ETHEREUM-CLASSIC_MAINNET_ETHERERUM_CLASSIC_MAINNET_NETWORK', 'prev_hash': '4d79e0d331b1fe057b3c9ee7cb595c371ec0ea764147029a862b3cffce808ae049ec40a6e5cddbd6ee90e4d36955cd2e08ab1f4ef1ccc8c013710617bd689cfe'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:58:24.677Z', 'hash': '0x65438aab8904c467ceb5b22e6b1a4198eeb056647a8b7a9ece0d03739a79a0bb', 'public_chain': 'GOV-DIGITAL_MAINNET_GOV_DIGITAL_MAINNET_NETWORK', 'prev_hash': 'e98a502253ec3aebd9af29cd368e3db899e5c04d6e8214be17856055b9301b63ce2ddb34038c4d3366ab9b4be28f755041f3a089d36c3516d84b0a95741289e3'}}] +``` + +### Verifying a measurement +Being able to check if a UPP is anchored and valid is nice, but in a real usecase it might not be that useful. That's because usually the goal is the check if some data is valid. Verifying a UPP doesn't really help in that case, since it is not possible to reverse the hash contained in the UPP back to the original data. That's why one shouldn't bother to store sent UPPs, but rather the measurements these UPPs are based on. Those can then be hashed, and the hash can be looked up. The `data-verifier.py` script does exactly that. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). +``` +$ python data-verifier.py -h +usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT + +Check if the hash of given input data is known to the uBirch backend (verify it) + +positional arguments: + INPUT input data or data file path (depends on --ispath) + +optional arguments: + -h, --help show this help message and exit + --ispath ISPATH, -i ISPATH + sets if INPUT is being treated as data or data file path; true or false (default: False) + --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) + --isjson ISJSON, -j ISJSON + tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: True) + --hash HASH, -a HASH sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: sha256) + +When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the input argument is expected to be a valid base64 encoded hash. +``` +- `--ispath/-i` Specifies wether the input is to be treated as a data-file path or direct input data. `true` or `false`. +- `--env-e` The stage to check on. Should be the one the UPP corresponding to the data was sent to. `prod`, `demo` or `dev`. +- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. It should only be set to `true` if the data represents a JSON object and if it also was serialized when creating the UPP. +- `--hash/-a` Sets the hashing algorithm to use. `sha256`, `sha512` or `off`. It should match the algorithm used when creating the corresponding UPP. Setting it to `off` means that the input data actually already is the hash of the data. In this case this script will simply look up the hash. + +Example: +``` +python data-verifier.py --env demo --isjson true --hash sha256 '{ + "ts": 1625163338, + "T": 11.2, + "H": 35.8, + "S": "OK" +}' +2021-07-02 16:21:41,178 root serialize_json() INFO Serialized JSON: "{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}" +2021-07-02 16:21:41,178 root get_hash_from_data() INFO Calculated hash: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" +2021-07-02 16:21:41,178 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" +2021-07-02 16:21:41,599 root get_status() INFO The hash is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +Prev. UPP: "None" +2021-07-02 16:21:41,600 root get_status() INFO The corresponding UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] +``` +Just like with `upp-anchoring-status.py`, it might take a short while after sending the corresponding UPP to the backend before it will be anchored. + +## Sending data to the Simple Data Service +The `data-sender.py` example-script allows sending of data to the simple data service. This is only used for demo purposes. +``` +$ python3 data-sender.py --help +usage: data-sender.py [-h] [--env ENV] UUID AUTH INPUT + +Send some data to the uBirch Simple Data Service + +positional arguments: + UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 + AUTH uBirch device authentication token + INPUT data to be sent to the simple data service + +optional arguments: + -h, --help show this help message and exit + --env ENV, -e ENV environment to operate in; dev, demo or prod (default: dev) + +Note that the input data should follow this pattern: {"timestamp": TIMESTAMP, "uuid": "UUID", "msg_type": 0, "data": DATA, "hash": "UPP_HASH"}. For more information take a look at the EXAMPLES.md file. +``` + +## Example uBirch client implementation +`example-client.py` implements a full example uBirch client. From 4636086dcc99a92e3f9b9002ee797d6af2536179 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 17:08:11 +0200 Subject: [PATCH 29/68] remove old comment --- examples/data-verifier.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 87f8c61..ff1c77f 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -1,32 +1,3 @@ - -# import hashlib -# import json - -# import requests - -# VERIFICATION_SERVICE = "https://verify.prod.ubirch.com/api/upp/verify/anchor" - -# with open("data_to_verify.json") as f: -# message = json.load(f) - -# # create a compact rendering of the message to ensure determinism when creating the hash -# serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() -# print("rendered data:\n\t{}\n".format(serialized.decode())) - -# # calculate hash of message -# data_hash = hashlib.sha256(serialized).digest() -# print("hash [base64]:\n\t{}\n".format(base64.b64encode(data_hash).decode())) - -# # verify existence of the hash in the UBIRCH backend -# r = requests.post(url=VERIFICATION_SERVICE, -# headers={'Accept': 'application/json', 'Content-Type': 'text/plain'}, -# data=base64.b64encode(data_hash).decode().rstrip('\n')) - -# if 200 <= r.status_code < 300: -# print("verification successful:\n\t{}\n".format(r.content.decode())) -# else: -# print("verification FAIL: ({})\n\tdata hash could not be verified\n".format(r.status_code)) - import sys import argparse import base64 From c47c537fdaffd711b2179e59602cfd1258a0b6cd Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 17:09:21 +0200 Subject: [PATCH 30/68] fix typo --- examples/upp-anchoring-status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 1c39f81..6234dbf 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -189,7 +189,7 @@ def run(self) -> int: # get the anchoring status if self.get_status() != True: - logger.error("Errors occured while requesting the anchring status - exiting!\n") + logger.error("Errors occured while requesting the anchoring status - exiting!\n") self.argparser.print_usage() From 2bec84bfe14474b73d2bbb5eabd4651d1735c852 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 2 Jul 2021 22:07:13 +0200 Subject: [PATCH 31/68] add documentation for keystore dumper script --- examples/EXAMPLES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 38d2f71..b20da32 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -47,6 +47,25 @@ To create, or more precisely, to _sign_ a UPP, a device will need a keypair. Thi **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. +A keystore can be read out with the `keystore-dumper.py` script. + +``` +$ python keystore-dumper.py --help +usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KET] KEYSTORE KEYSTORE_PASS + +Dump the contents of a keystore (.jks) + +positional arguments: + KEYSTORE keystore file path; e.g.: test.jks + KEYSTORE_PASS keystore password; e.g.: secret + +optional arguments: + -h, --help show this help message and exit + --show-sk SHOW_SIGNING_KET, -s SHOW_SIGNING_KET + enables/disables showing of signing keys; e.g.: true, false (default: False) +``` +By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. + ### Registering a public key To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend before starting to send UPPs supposed to be verified and anchored. Registering a verifying key is also done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` From 887a82a70f7d04b18eb185b9176ebfed9d582ebd Mon Sep 17 00:00:00 2001 From: wowa Date: Tue, 6 Jul 2021 00:21:28 +0200 Subject: [PATCH 32/68] Reviewed and corrected EXAMPLES.md --- examples/EXAMPLES.md | 78 +++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index b20da32..dcf0808 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -13,13 +13,13 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending a UPP](#sending-a-upp) - [Verifying a UPP](#verifying-a-upp) - [Examining a UPP](#examining-a-upp) - - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - - [Verifying a measurement](#verifying-a-measurement) + - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) + - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) ## From measurement to blockchain-anchored UPP -The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how one could handle them. There also are examples showing a full example-client implementation. +The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 0. [Setup](#setup) 1. [Gathering Data](#gathering-data) @@ -31,8 +31,8 @@ Before anything, you will need to do/get a couple of things: - https://console.prod.ubirch.com for the `prod` stage - https://console.demo.ubirch.com for the `demo` stage - https://console.dev.ubirch.com for the `dev` stage -- Get a UUID (can be generated randomly or on the basis of certain device properties like MAC-Addresses) -- Create a "Thing" at the uBirch-Console; remember/note down the used UUID and the generated Auth-Token +- Get a UUID (can be generated randomly, for example [here](https://www.uuidgenerator.net/), or on the basis of certain device properties like MAC-Addresses) +- Create a "Thing" at the uBirch-Console; remember/note down the used UUID and the generated Auth-Token. For details on how to create a Thing, check the[Ubirch console documentation](https://developer.ubirch.com/console.html). You should now have the following information at hand: - The stage you want to work on (later referred to as `env`) @@ -43,7 +43,7 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check if the signature is valid and belongs to the correct sender/signer. So logically, it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real usecase, a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against an attacker reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. For that you will have to chose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is exlpained [bellow](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. @@ -67,7 +67,7 @@ optional arguments: By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. ### Registering a public key -To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend before starting to send UPPs supposed to be verified and anchored. Registering a verifying key is also done by sending a special kind of UPP containing this key. This can be done by using two scripts: +To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` upp-creator.py upp-sender.py @@ -106,10 +106,19 @@ UPPs are usually used to anchor the hash of some kind of data. This data, in the "S": "OK" } ``` -Translated to a hypothetical usecase this could be a measurement taken at `1625163338` (Unix-Timestamp), stating that the sensor measured `11.2 C` in temperature (`T`) and `35.8 %H` in humidity (`H`). The status - `S` - is `'OK'`. There is no script for this step, since it can easily done by hand. +Translated to a hypothetical use case this could be a measurement taken at `1625163338` (Unix-Timestamp), stating that the sensor measured `11.2 C` in temperature (`T`) and `35.8 %H` in humidity (`H`). The status - `S` - is `'OK'`. There is no script for this step, since it can easily be done by hand. + +**Note: _If you use a JSON format for your data, the data has to be alphabetically sorted, all whitespace removed and +serialized into a simple string, before the hash of the data is generated. This ensures, that you can always regenerate +the same hash for your data. This is already implemented in the examples, like the following line of code shows:_** +```python +serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_ascii=False).encode() +``` +This will create a string from the above examplary JSON object: +`{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` ### Creating a UPP -After gathering some measurement data a UPP can be created. The won't contain the actual measurement data, but a hash of it. The script used to create UPPs in this example is `upp-creator.py`. +After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs in is `upp-creator.py`. ``` $ python3 upp-creator.py --help @@ -139,18 +148,18 @@ optional arguments: --nostdout nostdout, -n nostdout do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) -Note that when using chained UPPs (--version 0x23), this tool will try to load/save signatures to UUID.sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` -The script allows multiple modes of operation which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. -- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two (`0010`) set in almost all usecases. The mode can eather be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The one would be called a _Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. -- `--type/-t` This flag sets the type field of the UPP. It it used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases in which a specific value is required are Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). -- `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore pointed to by this parameter doesn't exist, the script will simple create it. -- `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this or you will lose access to the keystore and all its contents. +The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, a UPP with a signature and a UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--type/-t` This flag sets the type field of the UPP. It is used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases where a specific value is required, is a Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). +- `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore, pointed to by this parameter, doesn't exist, the script will simply create it. +- `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this, or you will lose access to the keystore and all its contents. - `--keyreg/-k` Tells the script that the UPP that should be generated is a key registration UPP. The effect of that is that the script will ignore any custom input data and the `--hash` parameter (below). Instead, the UPP will contain the public key certificate. This parameter is a binary flag which can have two values: `true` or `false`. -- `--hash` Sets the hash algorithm to used to hash the input data. The produced hash will then be inserted into the payload field of the UPP. This parameter can have three values: `sha512`, `sha256` and `off`. When set to off, the input data will be directly put into the UPP without hashing it. This is only useful in some special cases like when manually assembling key registration messages (normally the `--keyreg` option should be used for that). -- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. +- `--hash` Sets the hash algorithm to be used to generate the hash of the input data. The produced hash will then be inserted into the payload field of the UPP. This parameter can have three values: `sha512`, `sha256` and `off`. When set to off, the input data will be directly put into the UPP without hashing it. This is only useful in some special cases like when manually assembling key registration messages (normally the `--keyreg` option should be used for that). +- `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string, which should represent a JSON object. This flag can have two values: `true` or `false`. - `--output/-o` Tells the script where to write the generated UPP to. - `--nostdout/-n` Binary flag to disable printing of any log messages to standard output. This can be used for piping a created UPP to another program. For this `--output /dev/stdout` would have to be set. - `UUID` The UUID of the device as a hex-string, like `f5ded8a3-d462-41c4-a8dc-af3fd072a217`. @@ -171,10 +180,10 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh 2021-07-02 15:07:53,041 root show_store_upp() INFO UPP [hex]: "9623c410f5ded8a3d46241c4a8dcaf3fd072a217c440cbe84f33c1d80a9a2a68f10c61c843567035d19179a703bb5e0aff4e920d9b8535acb171f1fd55271371d199fc985f33cf0b31f3c6ecfa7be684b561ac6d900f00c42075f42eef00422f6682b80a962e4c871170b34e528775f32becf32bc4473063a0c440ccc7e39d9a1acbf39d307d08d5b5f74218016e0b9e74d1efc7640c540c4cda1bf182b389a7ed9fd3fefb047ce6cf513dd1a047193ed0a13110f727fef4421102" 2021-07-02 15:07:53,041 root show_store_upp() INFO UPP written to "upp.bin" ``` -Keep in mind that when using chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but defeat the purpose of chaining UPPs. +Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. ### Sending a UPP -After creating the UPP, it can be sent to the uBirch backend where it will be verified (the backend will use the registered public/verifying key to check the signature) and anchored into the blockchain. The `upp-sender.py` script can be used for that. +After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The `upp-sender.py` script can be used for that. ``` $ python3 upp-sender.py --help usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH @@ -193,9 +202,9 @@ optional arguments: --output OUTPUT, -o OUTPUT response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: response_upp.bin) ``` -For this script there aren't that many parameters, since the task is rather easy and straight forward. -- `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage the UUID is registered on. -- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin` if, for example, the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. +For this script the parameters are: +- `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage, the UUID is registered on. +- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin`, if for example the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. @@ -209,7 +218,7 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de ``` ### Verifying a UPP -To make sure that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks if the signature of the response UPP is valid. +To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help @@ -226,11 +235,11 @@ optional arguments: --input INPUT, -i INPUT UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) -Note that when trying to verify a UPP sent by the uBirch backend (Niomon) a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +Note that, when trying to verify a UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` - `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. - `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. -- The file path to read the UPP from. +- `--input/-i` The file path to read the UPP from. ``` $ python3 upp-verifier.py --input response_upp.bin @@ -241,7 +250,7 @@ $ python3 upp-verifier.py --input response_upp.bin ``` ### Examining a UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the `upp-unpacker.py` script can be used like this: +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the `upp-unpacker.py` script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -256,8 +265,8 @@ $ python3 upp-unpacker.py response_upp.bin ``` _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ -### Checking the anchoring status of an UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script for that task is `upp-anchoring-status.py`. +### Checking the anchoring status of a UPP +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is `upp-anchoring-status.py`. ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT @@ -275,7 +284,7 @@ optional arguments: When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` -- `--ishash/-i` A boolean specifying wether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. - `INPUT` The input UPP file path or payload hash, depending on `--ishash`. @@ -290,7 +299,7 @@ Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMzH452aGsvznTB9CNW190IYAW4LnnTR78dkDFQM Prev. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" 2021-07-02 16:01:46,950 root get_status() INFO The UPP has NOT been anchored into any blockchains yet! Please retry later ``` -Here it is visible that the backend knows the UPP and that it is valid, but it hasn't been anchored yet. Additionally the output shows that The backend knows the previous UPP, indicating that the UPP is a chained UPP and not the first UPP in the chain. When using unchained UPPs the line will change to: `Prev. UPP: "None"`. After waiting some time and running the script again with the same parameters: +Here it is visible that the backend knows the UPP and that it is valid, but it hasn't been anchored yet. Additionally, the output shows that the backend knows the previous UPP, indicating that the UPP is a chained UPP and not the first UPP in the chain. When using unchained UPPs, the line will change to: `Prev. UPP: "None"`. After waiting some time and running the script again with the same parameters: ``` $ python3 upp-anchoring-status.py --env demo upp.bin 2021-07-02 16:09:34,521 root read_upp() INFO Reading the input UPP from "upp.bin" @@ -314,8 +323,8 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:46:45.949Z', 'hash': '0xc32ccbe70ed727a1842f998d56d9928a9a30e201aef91bb08d9ac7faf931dac6', 'public_chain': 'ETHEREUM-CLASSIC_MAINNET_ETHERERUM_CLASSIC_MAINNET_NETWORK', 'prev_hash': '4d79e0d331b1fe057b3c9ee7cb595c371ec0ea764147029a862b3cffce808ae049ec40a6e5cddbd6ee90e4d36955cd2e08ab1f4ef1ccc8c013710617bd689cfe'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:58:24.677Z', 'hash': '0x65438aab8904c467ceb5b22e6b1a4198eeb056647a8b7a9ece0d03739a79a0bb', 'public_chain': 'GOV-DIGITAL_MAINNET_GOV_DIGITAL_MAINNET_NETWORK', 'prev_hash': 'e98a502253ec3aebd9af29cd368e3db899e5c04d6e8214be17856055b9301b63ce2ddb34038c4d3366ab9b4be28f755041f3a089d36c3516d84b0a95741289e3'}}] ``` -### Verifying a measurement -Being able to check if a UPP is anchored and valid is nice, but in a real usecase it might not be that useful. That's because usually the goal is the check if some data is valid. Verifying a UPP doesn't really help in that case, since it is not possible to reverse the hash contained in the UPP back to the original data. That's why one shouldn't bother to store sent UPPs, but rather the measurements these UPPs are based on. Those can then be hashed, and the hash can be looked up. The `data-verifier.py` script does exactly that. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). +### Verifying data +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the `data-verifier.py` script. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). ``` $ python data-verifier.py -h usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT @@ -353,7 +362,8 @@ python data-verifier.py --env demo --isjson true --hash sha256 '{ 2021-07-02 16:21:41,178 root get_hash_from_data() INFO Calculated hash: "dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A=" 2021-07-02 16:21:41,178 root get_status() INFO Requesting anchoring information from: "https://verify.demo.ubirch.com/api/upp/verify/anchor" 2021-07-02 16:21:41,599 root get_status() INFO The hash is known to the uBirch backend! (code: 200) -Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay/OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" +Curr. UPP: "liPEEPXe2KPUYkHEqNyvP9ByohfEQMvoTzPB2AqaKmjxDGHIQ1ZwNdGReacDu14K/06SDZuFNayxcfH9VScTcdGZ/JhfM88LMfPG7Pp75oS1YaxtkA8AxCB19C7vAEIvZoK4CpYuTIcRcLNOUod18yvs8yvERzBjoMRAzMfjnZoay + /OdMH0I1bX3QhgBbguedNHvx2QMVAxM2hvxgrOJp+2f0/77BHzmz1E90aBHGT7QoTEQ9yf+9EIRAg==" Prev. UPP: "None" 2021-07-02 16:21:41,600 root get_status() INFO The corresponding UPP has been fully anchored! [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] @@ -361,7 +371,7 @@ Prev. UPP: "None" Just like with `upp-anchoring-status.py`, it might take a short while after sending the corresponding UPP to the backend before it will be anchored. ## Sending data to the Simple Data Service -The `data-sender.py` example-script allows sending of data to the simple data service. This is only used for demo purposes. +The `data-sender.py` example-script allows sending of data to the simple data service. This should only be used for demo purposes. Ubirch will not guarantee, to keep all data, which is sent to this endpoint. ``` $ python3 data-sender.py --help usage: data-sender.py [-h] [--env ENV] UUID AUTH INPUT From 4b58c5576523ab1746a2a87339a59015b2b63048 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 6 Jul 2021 14:57:14 +0200 Subject: [PATCH 33/68] . --- examples/EXAMPLES.md | 90 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 15 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index dcf0808..5ab6dd4 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -43,11 +43,11 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py] script and explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. -A keystore can be read out with the `keystore-dumper.py` script. +A keystore can be read out with the [`keystore-dumper.py`](keystore-dumper.py) script. ``` $ python keystore-dumper.py --help @@ -61,7 +61,7 @@ positional arguments: optional arguments: -h, --help show this help message and exit - --show-sk SHOW_SIGNING_KET, -s SHOW_SIGNING_KET + --show-sk SHOW_SIGNING_KEY, -s SHOW_SIGNING_KEY enables/disables showing of signing keys; e.g.: true, false (default: False) ``` By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. @@ -85,7 +85,7 @@ $ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --o 2021-07-02 11:51:50,485 root show_store_upp() INFO UPP [hex]: "9522c410f5ded8a3d46241c4a8dcaf3fd072a2170187a9616c676f726974686dab4543435f45443235353139a763726561746564ce60dec596aa68774465766963654964c410f5ded8a3d46241c4a8dcaf3fd072a216a67075624b6579c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4ca87075624b65794964c420e0264e7d9428149cef59ccecb8813b214d8f94c62e3e836d7546d3f8bd884a4cad76616c69644e6f744166746572ce62bff916ae76616c69644e6f744265666f7265ce60dec596c440cadb70d30250a5a2dd2eb44b645e54b56387f228607fbf6f59a11493befa118f0e9c79da1f7d85ba5a4076c134f8b4aff04173adfc4b858ec491be2366988900" 2021-07-02 11:51:50,485 root show_store_upp() INFO UPP written to "keyreg_upp.bin" ``` -The generated key registration UPP has been saved to `keyreg_upp.bin`. Sending the UPP can be done like following (remember to put the correct value for the env of your choice): +The [upp-creator.py](upp-creator.py) script will check if the keystore specified with `--ks` contains an entry for the device with the given UUID. If it doesn´t, the script will generate a new keypair and store it. This can be seen when examining the two first log messages starting with `No keys found for` and `inserted new keypair for`. Otherwsise (if there already is a keypair for the device) the script will simple use the existent keypair. The generated key registration UPP has been saved to `keyreg_upp.bin`. Sending the UPP can be done like following (remember to put the correct value for the env of your choice): ``` $ python upp-sender.py --env demo --input keyreg_upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx @@ -118,7 +118,7 @@ This will create a string from the above examplary JSON object: `{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` ### Creating a UPP -After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs in is `upp-creator.py`. +After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). ``` $ python3 upp-creator.py --help @@ -183,7 +183,7 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. ### Sending a UPP -After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The `upp-sender.py` script can be used for that. +After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The [`upp-sender.py`](upp-sender.py) script can be used for that. ``` $ python3 upp-sender.py --help usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH @@ -204,7 +204,7 @@ optional arguments: ``` For this script the parameters are: - `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage, the UUID is registered on. -- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin`, if for example the UPP will be piped to this script from another script (like `upp-creator.py`). In most cases the UPP will just be read from some file. +- `--input/-i` Specifies where to read the UPP to be sent from. This can be a normal file path or also `/dev/stdin`, if for example the UPP will be piped to this script from another script (like [`upp-creator.py`](upp-creator.py)). In most cases the UPP will just be read from some file. - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. @@ -218,7 +218,7 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de ``` ### Verifying a UPP -To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is `upp-verifier.py`. It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. +To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is [`upp-verifier.py`](upp-verifier.py). It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help @@ -250,7 +250,7 @@ $ python3 upp-verifier.py --input response_upp.bin ``` ### Examining a UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the `upp-unpacker.py` script can be used like this: +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -266,7 +266,7 @@ $ python3 upp-unpacker.py response_upp.bin _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ ### Checking the anchoring status of a UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is `upp-anchoring-status.py`. +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT @@ -324,7 +324,7 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ``` ### Verifying data -In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the `data-verifier.py` script. It has similar behaviour to the `upp-anchoring-status.py` script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). ``` $ python data-verifier.py -h usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT @@ -350,7 +350,7 @@ When --ispath/-i is set to true, the input data is treated as a file path to rea - `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. It should only be set to `true` if the data represents a JSON object and if it also was serialized when creating the UPP. - `--hash/-a` Sets the hashing algorithm to use. `sha256`, `sha512` or `off`. It should match the algorithm used when creating the corresponding UPP. Setting it to `off` means that the input data actually already is the hash of the data. In this case this script will simply look up the hash. -Example: +Example for CLI-Input data: ``` python data-verifier.py --env demo --isjson true --hash sha256 '{ "ts": 1625163338, @@ -368,10 +368,24 @@ Prev. UPP: "None" 2021-07-02 16:21:41,600 root get_status() INFO The corresponding UPP has been fully anchored! [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] ``` -Just like with `upp-anchoring-status.py`, it might take a short while after sending the corresponding UPP to the backend before it will be anchored. +Example for File-Input data: +``` +python data-verifier.py --ispath true -j true data_to_verify.json -e prod +2021-07-06 12:17:31,261 root read_data() INFO Reading the input data from "data_to_verify.json" +2021-07-06 12:17:31,262 root serialize_json() INFO Serialized JSON: "{"data":{"AccPitch":"-11.52","AccRoll":"1.26","AccX":"-0.02","AccY":"0.20","AccZ":"0.99","H":"64.85","L_blue":232,"L_red":275,"P":"100934.00","T":"20.69","V":"4.62"},"msg_type":1,"timestamp":1599203876,"uuid":"07104235-1892-4020-9042-00003c94b60b"}" +2021-07-06 12:17:31,262 root get_hash_from_data() INFO Calculated hash: "/hHAPCT60m0/pnsB2z4Y4TYNcALrBnKb8h1ZR429fuY=" +2021-07-06 12:17:31,262 root get_status() INFO Requesting anchoring information from: "https://verify.prod.ubirch.com/api/upp/verify/anchor" +2021-07-06 12:17:31,784 root get_status() INFO The hash is known to the uBirch backend! (code: 200) +Curr. UPP: "liPEEAcQQjUYkkAgkEIAADyUtgvEQHy+eJ38aa7R6A1K+5ZLqYxoP7EraPYBo9cTllip+FCVm3OkzfDNB36/yMkJT5GqyopDs1mBJu8Y3kYczX8VM8oAxCD+EcA8JPrSbT+mewHbPhjhNg1wAusGcpvyHVlHjb1+5sRAsp7YwQtGxGBXX/PgbjEd1JQP1qDWOfDDsYc0oJ0jrZcjLvJv6SGnIgnZvmF1YSYewnHe56Fb3GApTw7Ybs43SQ==" +Prev. UPP: "liPEEAcQQjUYkkAgkEIAADyUtgvEQJViO08kxDSmJWebjNDFAVFwqxGUANe9XkNqi549sVLSlCcNd1lLFWGfXUttolDlENsSgjejqH7Iwf2QxAJWqmsAxCDbSx12E4W489A0oKaaFm+cpCqp9ShhfPJockqU/axOgMRAfL54nfxprtHoDUr7lkupjGg/sSto9gGj1xOWWKn4UJWbc6TN8M0Hfr/IyQlPkarKikOzWYEm7xjeRhzNfxUzyg==" +2021-07-06 12:17:31,784 root get_status() INFO The corresponding UPP has been fully anchored! +[{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2020-09-04T07:27:59.196Z', 'hash': 'FNMJQLBQXPRDAG9ZDPEDZEOQOXEUPJQOFOOBBEUZRXA9BBY9FRZRSYABEAFTYFCDFWJYDMXTZWVXZ9999', 'public_chain': 'IOTA_MAINNET_IOTA_MAINNET_NETWORK', 'prev_hash': 'bc318054140e1f4014977ebd37058807cba5c7c369cebe14daf8fbccdacb24ee135a0773764cb0ad6530fd0d8392d77f7f9d669b2ca973f13c683d1a8930d61b'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2020-09-04T07:28:11.886Z', 'hash': '0x2b1b940d98d35522a326396625690397ef9aab9c8dfb2b8f63a7e7a297559ce9', 'public_chain': 'ETHEREUM-CLASSIC_MAINNET_ETHERERUM_CLASSIC_MAINNET_NETWORK', 'prev_hash': 'bc318054140e1f4014977ebd37058807cba5c7c369cebe14daf8fbccdacb24ee135a0773764cb0ad6530fd0d8392d77f7f9d669b2ca973f13c683d1a8930d61b'}}, {'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2020-09-04T07:28:19.414Z', 'hash': '0x133627d9effaa40186d0bab8331cff05242c87178a32ca370f5fa7512716c361', 'public_chain': 'GOV-DIGITAL_MAINNET_GOV_DIGITAL_MAINNET_NETWORK', 'prev_hash': 'bc318054140e1f4014977ebd37058807cba5c7c369cebe14daf8fbccdacb24ee135a0773764cb0ad6530fd0d8392d77f7f9d669b2ca973f13c683d1a8930d61b'}}] +``` + +Just like with [`upp-anchoring-status.py`](upp-anchoring-status.py), it might take a short while after sending the corresponding UPP to the backend before it will be anchored. ## Sending data to the Simple Data Service -The `data-sender.py` example-script allows sending of data to the simple data service. This should only be used for demo purposes. Ubirch will not guarantee, to keep all data, which is sent to this endpoint. +The [`data-sender.py`](data-sender.py) example-script allows sending of data to the simple data service. This should only be used for demo purposes. Ubirch will not guarantee, to keep all data, which is sent to this endpoint. ``` $ python3 data-sender.py --help usage: data-sender.py [-h] [--env ENV] UUID AUTH INPUT @@ -391,4 +405,50 @@ Note that the input data should follow this pattern: {"timestamp": TIMESTAMP, "u ``` ## Example uBirch client implementation -`example-client.py` implements a full example uBirch client. +[`example-client.py`](example-client.py) implements a full example uBirch client. It generates a keypair if needed, registers it at the uBirch backend if it doesn't know it yet, creates and sends an UPP and handles/verfies the response from the uBirch backend. The used message format looks like this: +``` +{ + "id": "UUID", + "ts": TIMESTAMP, + "data": "DATA" +} +``` +It has two positional and one optional command line parameters. +``` +usage: python3 example-client.py [ubirch-env] +``` +- `UUID` is the UUID as hex-string like `f5ded8a3-d462-41c4-a8dc-af3fd072a217` +- `ubirch-auth-token` is the uBirch authentication token for the specified UUID, e.g.: `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` +- `ubirch-env` (optional) specifies the environment/stage to operator on. `dev`, `demo` or `prod` (default). +Keys are loaded from/stored to `demo-device.jks`. The keystore-password can be read from the [script](example-client.py) itself. + + +## Create a hash from an JSON object +[`create-hash.py`](create-hash.py) takes a string representing a JSON object as input, serializes it, and calculates the corresponding SHA256 hash. +``` +$ python3 create-hash.py '{"ts": 1625163338, "T": 11.2, "H": 35.8, "S": "OK"}' + input: {"ts": 1625163338, "T": 11.2, "H": 35.8, "S": "OK"} +rendered: {"H":35.8,"S":"OK","T":11.2,"ts":1625163338} + hash: dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A= +``` + +## [`test-identity.py`](test-identity.py) +The [`test-identity.py`](test-identity.py) script tests registering and de-registering a public key of a device at the uBirch backend. To function it needs the following variables to be set using the environment: +```sh +export UBIRCH_UUID= +export UBIRCH_AUTH= +export UBIRCH_ENV=[dev|demo|prod] +``` +It uses `test-identity.jsk` as a place to store/look for keypairs. The keystore-password can be read from the [script](test-identity.py) itself. + +## [`test-protocol.py`](test-protocol.py) +The [`test-protocol.py`](test-protocol.py) script sends a couple of UPPs to uBirch Niomon and verifies the backend response. It reads all information it needs interactively from the terminal. Once entered, all device information (UUID, ENV, AUTH TOKEN) are stored in a file called `demo-device.ini`. Devices keys are stored in `demo-device.jks` and the keystore-password can be read from the [script](test-protocol.py) itself. If no keys for the given UUID are found, the script will generated a keypair and stores it in the keystore file. + +## [`test-web-of-trust.py`](test-web-of-trust.py) +**TODO** + +## [`verify-ecdsa.py`](verify-ecdsa.py) +The [`verify-ecdsa.py`](verify-ecdsa.py) script verifies a hard-coded UPP which was signed with an ECDSA signing key using a ECDSA verifying key. All the information are contained in the script. + +## [`verify-ed25519.py`](verify-25519.py) +The [`verify-25519.py`](verify-25519.py) script verifies a hard-coded UPP which was signed with an ED25519 signing key using a ED25519 verifying key. All the information are contained in the script. This mode is normally used (in all other examples). \ No newline at end of file From e7812ab157ed9f51a5a814e56d88b46341d8983a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Wed, 7 Jul 2021 08:33:51 +0200 Subject: [PATCH 34/68] fix incomplete toc --- examples/EXAMPLES.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 5ab6dd4..e174a97 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -17,12 +17,25 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) + - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) + - [test-identiy.py](#test-identity.py) + - [test-web-of-trust.py](#test-web-of-trust.py) + - [verify-ecdsa.py](#verify-ecdsa.py) + - [verify-ed25519.py](#verify-ed25519.py) ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 0. [Setup](#setup) -1. [Gathering Data](#gathering-data) +1. [Generating and managing a keypair](#generating-and-managing-a-keypair) +2. [Registering a public key](#registering-a-public-key) +3. [Gathering Data](#gathering-data) +4. [Creating a UPP](#creating-a-upp) +5. [Sending a UPP](#sending-a-upp) +6. [Verifying a UPP](#verifying-a-upp) +7. [Examining a UPP](#examining-a-upp) +8. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) +9. [Verifying data](#verifying-data) ### Setup Before anything, you will need to do/get a couple of things: From 44d64a2079db0582052939330f38abf65da44c5c Mon Sep 17 00:00:00 2001 From: wowa Date: Thu, 8 Jul 2021 08:24:44 +0200 Subject: [PATCH 35/68] test-identity try to solve broken reference --- examples/EXAMPLES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index e174a97..514c609 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -18,7 +18,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) - - [test-identiy.py](#test-identity.py) + - [test-identity.py](#test-identity) - [test-web-of-trust.py](#test-web-of-trust.py) - [verify-ecdsa.py](#verify-ecdsa.py) - [verify-ed25519.py](#verify-ed25519.py) @@ -56,7 +56,7 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py] script and explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. @@ -445,7 +445,7 @@ rendered: {"H":35.8,"S":"OK","T":11.2,"ts":1625163338} hash: dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A= ``` -## [`test-identity.py`](test-identity.py) +## [`test-identity.py`](test-identity.py){#test-identity} The [`test-identity.py`](test-identity.py) script tests registering and de-registering a public key of a device at the uBirch backend. To function it needs the following variables to be set using the environment: ```sh export UBIRCH_UUID= From 5c6debc3bdfde13d7750c44f4bce3e96573914ad Mon Sep 17 00:00:00 2001 From: wowa Date: Thu, 8 Jul 2021 08:43:51 +0200 Subject: [PATCH 36/68] solve broken reference links --- examples/EXAMPLES.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 514c609..5c222e1 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -18,10 +18,11 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) - - [test-identity.py](#test-identity) - - [test-web-of-trust.py](#test-web-of-trust.py) - - [verify-ecdsa.py](#verify-ecdsa.py) - - [verify-ed25519.py](#verify-ed25519.py) + - [Test identity of the device](#test-identity-of-the-device) + - [Test the complete protocol](#test-the-complete-protocol) + - [Test the web of trust](#test-the-web-of-trust) + - [Verify ECDSA signed UPP](#verify-ecdsa-signed-upp) + - [Verify ED25519 signed UPP](#verify-ed25519-signed-upp) ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. @@ -445,7 +446,7 @@ rendered: {"H":35.8,"S":"OK","T":11.2,"ts":1625163338} hash: dfQu7wBCL2aCuAqWLkyHEXCzTlKHdfMr7PMrxEcwY6A= ``` -## [`test-identity.py`](test-identity.py){#test-identity} +## Test identity of the device The [`test-identity.py`](test-identity.py) script tests registering and de-registering a public key of a device at the uBirch backend. To function it needs the following variables to be set using the environment: ```sh export UBIRCH_UUID= @@ -454,14 +455,15 @@ export UBIRCH_ENV=[dev|demo|prod] ``` It uses `test-identity.jsk` as a place to store/look for keypairs. The keystore-password can be read from the [script](test-identity.py) itself. -## [`test-protocol.py`](test-protocol.py) +## Test the complete protocol The [`test-protocol.py`](test-protocol.py) script sends a couple of UPPs to uBirch Niomon and verifies the backend response. It reads all information it needs interactively from the terminal. Once entered, all device information (UUID, ENV, AUTH TOKEN) are stored in a file called `demo-device.ini`. Devices keys are stored in `demo-device.jks` and the keystore-password can be read from the [script](test-protocol.py) itself. If no keys for the given UUID are found, the script will generated a keypair and stores it in the keystore file. -## [`test-web-of-trust.py`](test-web-of-trust.py) +## Test the web of trust +[`test-web-of-trust.py`](test-web-of-trust.py) **TODO** -## [`verify-ecdsa.py`](verify-ecdsa.py) +## Verify ECDSA signed UPP The [`verify-ecdsa.py`](verify-ecdsa.py) script verifies a hard-coded UPP which was signed with an ECDSA signing key using a ECDSA verifying key. All the information are contained in the script. -## [`verify-ed25519.py`](verify-25519.py) +## Verify ED25519 signed UPP The [`verify-25519.py`](verify-25519.py) script verifies a hard-coded UPP which was signed with an ED25519 signing key using a ED25519 verifying key. All the information are contained in the script. This mode is normally used (in all other examples). \ No newline at end of file From 6b48d137c1ffeac5bc012dae633d878f339c2205 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sat, 4 Sep 2021 14:27:27 +0200 Subject: [PATCH 37/68] add option to only generate data hash without verifying it --- examples/data-verifier.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index ff1c77f..1bd273f 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -13,6 +13,7 @@ DEFAULT_ENV = "dev" DEFAULT_ISJSON = "True" DEFAULT_HASH = "sha256" +DEFAULT_NOSEND = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -31,6 +32,8 @@ def __init__(self): self.isjson_str : str = None self.isjson : bool = None self.hashalg : str = None + self.nosend_str : str = None + self.nosend : bool = None self.ishash : bool = False self.hasher : object = None @@ -66,6 +69,9 @@ def setup_argparse(self): self.argparser.add_argument("--hash", "-a", metavar="HASH", type=str, default=DEFAULT_HASH, help="sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: %s)" % DEFAULT_HASH ) + self.argparser.add_argument("--no-send", "-n", metavar="NOSEND", type=str, default=DEFAULT_NOSEND, + help="if set to true, the script will only generate the hash of the input data without sending it; true or false (default: %s)" % DEFAULT_NOSEND + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -77,6 +83,7 @@ def process_args(self) -> bool: self.isjson_str = self.args.isjson self.env = self.args.env self.hashalg = self.args.hash + self.nosend_str = self.args.no_send # check the value for --hash if self.hashalg.lower() == "off": @@ -110,6 +117,12 @@ def process_args(self) -> bool: else: self.isjson = False + # get the bool for nosend + if self.nosend_str.lower() in ["1", "yes", "y", "true"]: + self.nosend = True + else: + self.nosend = False + return True def read_data(self) -> bool: @@ -143,7 +156,7 @@ def serialize_json(self) -> bool: def get_hash_from_data(self) -> bool: try: # calculate the hash - self.hasher.update(self.data) + self.hasher.update(self.data if type(self.data) == bytes else self.data.encode()) self.hash = self.hasher.digest() self.hash = base64.b64encode(self.hash).decode().rstrip("\n") @@ -244,13 +257,14 @@ def run(self) -> int: return 1 - # get the anchoring status - if self.get_status() != True: - logger.error("Errors occured while requesting the anchring status - exiting!\n") + if self.nosend == False: + # get the anchoring status + if self.get_status() != True: + logger.error("Errors occured while requesting the anchring status - exiting!\n") - self.argparser.print_usage() + self.argparser.print_usage() - return 1 + return 1 return 0 From 76c7ce8e228124314296a2fb4714f55e7c5b9a90 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 12 Oct 2021 12:53:08 +0200 Subject: [PATCH 38/68] fix typo in usage info --- examples/keystore-dumper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/keystore-dumper.py b/examples/keystore-dumper.py index 0fa8060..3ae90b4 100644 --- a/examples/keystore-dumper.py +++ b/examples/keystore-dumper.py @@ -44,7 +44,7 @@ def setup_argparse(self): self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, help="keystore password; e.g.: secret" ) - self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KET", type=str, default=DEFAULT_SHOW_SIGN, + self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KEY", type=str, default=DEFAULT_SHOW_SIGN, help="enables/disables showing of signing keys; e.g.: true, false (default: %s)" % DEFAULT_SHOW_SIGN ) From 701a60c7923e3f17986f421f81f9e4f1d3978807 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 12 Oct 2021 13:01:01 +0200 Subject: [PATCH 39/68] add hashlink functionality to data verifier and upp creator scripts --- examples/data-verifier.py | 84 ++++++++++++++++++++++++++++++++++++++- examples/upp-creator.py | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 1 deletion(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 1bd273f..07b3f66 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -14,6 +14,7 @@ DEFAULT_ISJSON = "True" DEFAULT_HASH = "sha256" DEFAULT_NOSEND = "False" +DEFAULT_ISHL = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -34,7 +35,8 @@ def __init__(self): self.hashalg : str = None self.nosend_str : str = None self.nosend : bool = None - + self.ishl_str : str = None + self.ishl : bool = None self.ishash : bool = False self.hasher : object = None @@ -72,6 +74,9 @@ def setup_argparse(self): self.argparser.add_argument("--no-send", "-n", metavar="NOSEND", type=str, default=DEFAULT_NOSEND, help="if set to true, the script will only generate the hash of the input data without sending it; true or false (default: %s)" % DEFAULT_NOSEND ) + self.argparser.add_argument("--ishl", "-l", metavar="ISHASHLINK", type=str, default=DEFAULT_ISHL, + help="implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to decide which fields to hash; true or false (default: %s)" % DEFAULT_ISHL + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -84,6 +89,7 @@ def process_args(self) -> bool: self.env = self.args.env self.hashalg = self.args.hash self.nosend_str = self.args.no_send + self.ishl_str = self.args.ishl # check the value for --hash if self.hashalg.lower() == "off": @@ -123,6 +129,18 @@ def process_args(self) -> bool: else: self.nosend = False + # get the bool for ishl + if self.ishl_str.lower() in ["1", "yes", "y", "true"]: + self.ishl = True + else: + self.ishl = False + + # check if ishl is true + if (self.ishl == True and self.isjson == False): + logger.warning("Overwriting '--isjson false' because --ishl is true") + + self.isjson = True + return True def read_data(self) -> bool: @@ -139,6 +157,61 @@ def read_data(self) -> bool: return True + def _getValueFromDict(self, keyPath : list, currentObj : dict) -> object: + """ this function gets an object from the config object: config[path[0]][path[1]][path[n]] """ + if len(keyPath) == 0 or not currentObj: + return currentObj + elif type(currentObj) == list and type(keyPath[0]) == int: + return self._getValueFromDict(keyPath[1:], currentObj[keyPath[0]]) + elif type(currentObj) != dict: + return None + else: + return self._getValueFromDict(keyPath[1:], currentObj.get(keyPath[0])) + + def _addValueToDict(self, keyPath : list, value : object) -> dict: + if len(keyPath) == 0: + return {} + elif len(keyPath) == 1: + return { + keyPath[0]: value + } + else: + return { + keyPath[0]: self._addValueToDict(keyPath[1:], value) + } + + def extract_relevant_fields(self) -> bool: + try: + # load the string as data + dataDict = json.loads(self.data) + + newDict = {} + + # check whether the hashlink array exists + if dataDict.get("hashLink") != None and type(dataDict.get("hashLink")) == list: + for hl in dataDict.get("hashLink"): + v = self._getValueFromDict(hl.split("."), dataDict) + + if v == None: + logger.error("Hashlink array contains entries that aren't present in the JSON: %s" % hl) + + return False + + newDict.update(self._addValueToDict(hl.split("."), v)) + else: + logger.warning("No hashLink array found in data but hashlink is enabled") + + newDict = dataDict + + # write back the filtered data + self.data = json.dumps(newDict) + except Exception as e: + logger.exception(e) + + return False + + return True + def serialize_json(self) -> bool: try: # load the string as json and put it back into a string, serealizing it @@ -240,6 +313,15 @@ def run(self) -> int: else: self.data = self.input + # check if hashlink is enabled + if self.ishl: + if self.extract_relevant_fields() != True: + logger.error("Error occured while getting relevant fields from the JSON data - exiting!\n") + + self.argparser.print_usage() + + return 1 + # check if the input data is json/should be serialized if self.isjson == True: if self.serialize_json() != True: diff --git a/examples/upp-creator.py b/examples/upp-creator.py index d089b6c..2151972 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -19,6 +19,7 @@ DEFAULT_ISJSON = "False" DEFAULT_OUTPUT = "upp.bin" DEFAULT_NOSTDOUT = "False" +DEFAULT_ISHL = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) logger = logging.getLogger() @@ -74,6 +75,8 @@ def __init__(self): self.nostdout_str : str = None self.nostdout : bool = None self.payload : bytes = None + self.ishl : bool = None + self.ishash : bool = False self.hasher : object = None self.keystore : ubirch.KeyStore = None @@ -129,6 +132,9 @@ def setup_argparse(self): self.argparser.add_argument("--nostdout", "-n", metavar="nostdout", type=str, default=DEFAULT_NOSTDOUT, help="do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: %s)" % DEFAULT_NOSTDOUT ) + self.argparser.add_argument("--ishl", "-l", metavar="ISHASHLINK", type=str, default=DEFAULT_ISHL, + help="implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to decide which fields to hash; true or false (default: %s)" % DEFAULT_ISHL + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -146,6 +152,7 @@ def process_args(self) -> bool: self.keystore_pass = self.args.kspwd self.output = self.args.output self.nostdout_str = self.args.nostdout + self.ishl_str = self.args.ishl # get the keyreg value if self.keyreg_str.lower() in ["1", "yes", "y", "true"]: @@ -202,6 +209,18 @@ def process_args(self) -> bool: else: self.isjson = False + # get the bool for ishl + if self.ishl_str.lower() in ["1", "yes", "y", "true"]: + self.ishl = True + else: + self.ishl = False + + # check if ishl is true + if (self.ishl == True and self.isjson == False): + logger.warning("Overwriting '--isjson false' because --ishl is true") + + self.isjson = True + # success return True @@ -237,6 +256,61 @@ def init_proto(self) -> bool: return True + def _getValueFromDict(self, keyPath : list, currentObj : dict) -> object: + """ this function gets an object from the config object: config[path[0]][path[1]][path[n]] """ + if len(keyPath) == 0 or not currentObj: + return currentObj + elif type(currentObj) == list and type(keyPath[0]) == int: + return self._getValueFromDict(keyPath[1:], currentObj[keyPath[0]]) + elif type(currentObj) != dict: + return None + else: + return self._getValueFromDict(keyPath[1:], currentObj.get(keyPath[0])) + + def _addValueToDict(self, keyPath : list, value : object) -> dict: + if len(keyPath) == 0: + return {} + elif len(keyPath) == 1: + return { + keyPath[0]: value + } + else: + return { + keyPath[0]: self._addValueToDict(keyPath[1:], value) + } + + def extract_relevant_fields(self) -> bool: + try: + # load the string as data + dataDict = json.loads(self.data) + + newDict = {} + + # check whether the hashlink array exists + if dataDict.get("hashLink") != None and type(dataDict.get("hashLink")) == list: + for hl in dataDict.get("hashLink"): + v = self._getValueFromDict(hl.split("."), dataDict) + + if v == None: + logger.error("Hashlink array contains entries that aren't present in the JSON: %s" % hl) + + return False + + newDict.update(self._addValueToDict(hl.split("."), v)) + else: + logger.warning("No hashLink array found in data but hashlink is enabled") + + newDict = dataDict + + # write back the filtered data + self.data = json.dumps(newDict) + except Exception as e: + logger.exception(e) + + return False + + return True + def prepare_payload(self) -> bool: try: if self.hasher == None: @@ -339,6 +413,15 @@ def run(self) -> int: return 1 + # check if hashlink is enabled + if self.ishl == True and self.ishash == False: + if self.extract_relevant_fields() != True: + logger.error("Error occured while getting relevant fields from the JSON data - exiting!\n") + + self.argparser.print_usage() + + return 1 + # prepare the UPP payload if self.prepare_payload() != True: logger.error("Errors occured while preparing the UPP payload - exiting!\n") From d0d510c8d70dece41a24ffb1d91ffb6cbf0771eb Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 15 Oct 2021 11:48:34 +0200 Subject: [PATCH 40/68] print used hashing method to prevent confusion --- examples/data-verifier.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index 07b3f66..b41793e 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -141,6 +141,9 @@ def process_args(self) -> bool: self.isjson = True + # show the user which hashing method is used + logger.info(("Using %s as hashing algorithm" % self.hashalg.lower())) + return True def read_data(self) -> bool: From 06b1b44a76084c54ae0b88f72fac45e418b8fdbb Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Fri, 15 Oct 2021 17:55:53 +0200 Subject: [PATCH 41/68] update the readme to have some info about hashlink --- examples/EXAMPLES.md | 54 ++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 5c222e1..02eea01 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -13,11 +13,11 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending a UPP](#sending-a-upp) - [Verifying a UPP](#verifying-a-upp) - [Examining a UPP](#examining-a-upp) - - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) + - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) - - [Create a hash from a JSON object](#create-a-hash-from-an-json-object) + - [Create a hash from an JSON object](#create-a-hash-from-an-json-object) - [Test identity of the device](#test-identity-of-the-device) - [Test the complete protocol](#test-the-complete-protocol) - [Test the web of trust](#test-the-web-of-trust) @@ -27,17 +27,25 @@ This file documents how to use the examples provided alongside the [uBirch-Proto ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. -0. [Setup](#setup) -1. [Generating and managing a keypair](#generating-and-managing-a-keypair) -2. [Registering a public key](#registering-a-public-key) -3. [Gathering Data](#gathering-data) -4. [Creating a UPP](#creating-a-upp) -5. [Sending a UPP](#sending-a-upp) -6. [Verifying a UPP](#verifying-a-upp) -7. [Examining a UPP](#examining-a-upp) -8. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp) -9. [Verifying data](#verifying-data) +1. [Setup](#setup) + +2. [Generating and managing a keypair](#generating-and-managing-a-keypair) +3. [Registering a public key](#registering-a-public-key) + +4. [Gathering Data](#gathering-data) + +5. [Creating a UPP](#creating-a-upp) + +6. [Sending a UPP](#sending-a-upp) + +7. [Verifying a UPP](#verifying-a-upp) + +8. [Examining a UPP](#examining-a-upp) + +9. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) + +10. [Verifying data](#verifying-data) ### Setup Before anything, you will need to do/get a couple of things: - Choose a stage to work on @@ -339,9 +347,9 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ### Verifying data In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). -``` -$ python data-verifier.py -h -usage: data-verifier.py [-h] [--ispath ISHASH] [--env ENV] [--isjson ISJSON] [--hash HASH] INPUT +```txt +$ python data-verifier.py --help +usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT Check if the hash of given input data is known to the uBirch backend (verify it) @@ -354,18 +362,26 @@ optional arguments: sets if INPUT is being treated as data or data file path; true or false (default: False) --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) --isjson ISJSON, -j ISJSON - tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: True) + tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false + (default: True) --hash HASH, -a HASH sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: sha256) + --no-send NOSEND, -n NOSEND + if set to true, the script will only generate the hash of the input data without sending it; true or false (default: False) + --ishl ISHASHLINK, -l ISHASHLINK + implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to + decide which fields to hash; true or false (default: False) -When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the input argument is expected to be a valid base64 encoded hash. +When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the +input argument is expected to be a valid base64 encoded hash. ``` - `--ispath/-i` Specifies wether the input is to be treated as a data-file path or direct input data. `true` or `false`. - `--env-e` The stage to check on. Should be the one the UPP corresponding to the data was sent to. `prod`, `demo` or `dev`. - `--isjson/-j` A binary flag that indicates that the input data is in JSON format. The script will serialize the JSON object before calculating the hash. This has the advantage one doesn't have to remember the order in which fields are listed in a JSON object to still be able to reconstruct the hash later on. Serializing the JSON is done like this: `json.dumps(self.data, separators=(',', ':'), sort_keys=True, ensure_ascii=False)` where `self.data` contains the JSON object which was loaded like this: `self.data = json.loads(self.dataStr)` where `dataStr` contains the input string which should represent a JSON object. This flag can have two values: `true` or `false`. It should only be set to `true` if the data represents a JSON object and if it also was serialized when creating the UPP. - `--hash/-a` Sets the hashing algorithm to use. `sha256`, `sha512` or `off`. It should match the algorithm used when creating the corresponding UPP. Setting it to `off` means that the input data actually already is the hash of the data. In this case this script will simply look up the hash. +- `--ishl/-l` enables Hashlink functionality. This means that the script will expect the input data to be a valid JSON object and to contain a list called `hashLink` at root-level. This list contains the names of all fields that should be taken into account when calculating the hash. Different JSON-levels can are represented like this: `[..., "a.b", ...]`. Example for CLI-Input data: -``` +```txt python data-verifier.py --env demo --isjson true --hash sha256 '{ "ts": 1625163338, "T": 11.2, @@ -383,7 +399,7 @@ Prev. UPP: "None" [{'label': 'PUBLIC_CHAIN', 'properties': {'timestamp': '2021-07-02T13:23:30.076Z', 'hash': '0x6e5956b4ac53bcaf58664e189673d4f8c7043488cf05009cc96868b146220604', 'public_chain': 'ETHEREUM-CLASSIC_TESTNET_ETHERERUM_CLASSIC_KOTTI_TESTNET_NETWORK', 'prev_hash': '644b41eee9043de5bda4b58bda1136fa0229712953678fe26486651338109b7a7135211f2b2cb646ab25d3b25198549e3c662c4791d60d343f08349b51ccc92b'}}] ``` Example for File-Input data: -``` +```txt python data-verifier.py --ispath true -j true data_to_verify.json -e prod 2021-07-06 12:17:31,261 root read_data() INFO Reading the input data from "data_to_verify.json" 2021-07-06 12:17:31,262 root serialize_json() INFO Serialized JSON: "{"data":{"AccPitch":"-11.52","AccRoll":"1.26","AccX":"-0.02","AccY":"0.20","AccZ":"0.99","H":"64.85","L_blue":232,"L_red":275,"P":"100934.00","T":"20.69","V":"4.62"},"msg_type":1,"timestamp":1599203876,"uuid":"07104235-1892-4020-9042-00003c94b60b"}" From ec56fad1f9b68d1087db795903fa010ebbbe9a07 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Thu, 21 Oct 2021 17:27:59 +0200 Subject: [PATCH 42/68] add ability to read hex encoded upps as input to the upp verifier --- examples/upp-verifier.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/examples/upp-verifier.py b/examples/upp-verifier.py index 3bda61b..adccb51 100644 --- a/examples/upp-verifier.py +++ b/examples/upp-verifier.py @@ -9,7 +9,8 @@ import ubirch -DEFAULT_INPUT = "upp.bin" +DEFAULT_INPUT = "/dev/stdin" +DEFAULT_ISHEX = "false" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -61,6 +62,9 @@ def __init__(self): self.upp_uuid : uuid.UUID = None self.upp_uuid_str : str = None + self.ishex : bool = None + self.ishex_str : str = None + # initialize the argument parser self.setup_argparse() @@ -80,6 +84,9 @@ def setup_argparse(self): self.argparser.add_argument("--verifying-key-uuid", "-u", metavar="UUID", type=str, default="EMPTY", help="the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1" ) + self.argparser.add_argument("--ishex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, + help="Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT ) @@ -94,6 +101,7 @@ def process_args(self) -> bool: self.vk_str = self.args.verifying_key self.vk_uuid_str = self.args.verifying_key_uuid self.input = self.args.input + self.ishex_str = self.args.ishex # check if a verifying key was supplied if self.vk_str != "AUTO": @@ -119,6 +127,12 @@ def process_args(self) -> bool: return False + # get the ishex value + if self.ishex_str.lower() in ["1", "yes", "y", "true"]: + self.ishex = True + else: + self.ishex = False + return True def read_upp(self) -> bool: @@ -128,6 +142,10 @@ def read_upp(self) -> bool: with open(self.input, "rb") as fd: self.upp = fd.read() + + # check whether hex decoding is needed + if self.ishex == True: + self.upp = binascii.unhexlify(self.upp) except Exception as e: logger.exception(e) From f0d94e8ba63784103262e5ccf3e888dfb433ceee Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sun, 7 Nov 2021 12:01:03 +0100 Subject: [PATCH 43/68] add hex-input support for the anchoring status script --- examples/EXAMPLES.md | 14 ++++++++++---- examples/upp-anchoring-status.py | 21 ++++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 02eea01..7dca234 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -13,7 +13,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Sending a UPP](#sending-a-upp) - [Verifying a UPP](#verifying-a-upp) - [Examining a UPP](#examining-a-upp) - - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) + - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) @@ -287,11 +287,11 @@ $ python3 upp-unpacker.py response_upp.bin ``` _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ -### Checking the anchoring status of a UPP +### Checking the anchoring status of an UPP uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h -usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] INPUT +usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT Requests the verification/anchoring of a UPP from the uBirch backend @@ -303,11 +303,17 @@ optional arguments: --ishash ISHASH, -i ISHASH sets if INPUT is being treated as a hash or upp path; true or false (default: False) --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) + --ishex ISHEX, -x ISHEX + Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: false) -When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. +When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a +UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` - `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. +- `--ishex/-x` A boolean which controls how the input UPP data is interpreted. By default, the data will +be interpreted as normale binary data. When this flag is set to `true`, it will be considered +hex-encoded binary data and de-hexlified before parsing it. - `INPUT` The input UPP file path or payload hash, depending on `--ishash`. One example might be: diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 6234dbf..df0f5dc 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -7,12 +7,14 @@ import base64 import json import requests +import binascii VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" DEFAULT_ISHASH = "False" DEFAULT_ENV = "dev" +DEFAULT_ISHEX = "false" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -31,6 +33,9 @@ def __init__(self): self.upp : bytes = None self.hash : str = None + + self.ishex : bool = None + self.ishex_str : str = None # initialize the argument parser self.setup_argparse() @@ -54,6 +59,9 @@ def setup_argparse(self): self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, help="the environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV ) + self.argparser.add_argument("--ishex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, + help="Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -63,13 +71,20 @@ def process_args(self) -> bool: self.input = self.args.input self.ishash_str = self.args.ishash self.env = self.args.env + self.ishex_str = self.args.ishex # get the bool for ishash if self.ishash_str.lower() in ["1", "yes", "y", "true"]: self.ishash = True else: self.ishash = False - + + # get the ishex value + if self.ishex_str.lower() in ["1", "yes", "y", "true"]: + self.ishex = True + else: + self.ishex = False + return True def read_upp(self) -> bool: @@ -79,6 +94,10 @@ def read_upp(self) -> bool: with open(self.input, "rb") as fd: self.upp = fd.read() + + # check whether hex decoding is needed + if self.ishex == True: + self.upp = binascii.unhexlify(self.upp) except Exception as e: logger.exception(e) From ede1d65182a35b570b284af1277da3b9102e08e6 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sun, 7 Nov 2021 15:46:44 +0100 Subject: [PATCH 44/68] add a script to check chains of upps (upp-chaining) --- examples/upp-chain-checker.py | 316 ++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 examples/upp-chain-checker.py diff --git a/examples/upp-chain-checker.py b/examples/upp-chain-checker.py new file mode 100644 index 0000000..b692124 --- /dev/null +++ b/examples/upp-chain-checker.py @@ -0,0 +1,316 @@ +import sys +import logging +import argparse +import msgpack +import binascii +import uuid +import ed25519 +import json + +import ubirch +from ubirch.ubirch_protocol import UNPACKED_UPP_FIELD_UUID, UNPACKED_UPP_FIELD_PREV_SIG, UNPACKED_UPP_FIELD_SIG + + +DEFAULT_ISJSON = "false" +DEFAULT_ISHEX = "false" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Proto(ubirch.Protocol): + def __init__(self, ks : ubirch.KeyStore): + super().__init__() + + self.ks : ubirch.KeyStore = ks + + def _verify(self, uuid: uuid.UUID, message: bytes, signature: bytes): + return self.ks.find_verifying_key(uuid).verify(signature, message) + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.vk_str : str = None + self.vk : ed25519.VerifyingKey = None + self.vk_uuid : uuid.UUID = None + self.vk_uuid_str : str = None + + self.input : str = None + + self.keystore : ubirch.KeyStore = None + self.proto : ubirch.Protocol = None + + self.upps : [bytes] = None + self.upp_uuid : uuid.UUID = None + self.upp_uuid_str : str = None + + self.isjson : bool = None + self.isjson_str : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Check if a sequence of chained UPPs is valid/properly signed and correctly chained", + epilog="The JSON file (when using --is-json true) es expected to contain a single field called \"upps\", which is a list of " + "hex-encoded UPPs. Otherwise (--is-json false). If --is-hex is true, it expects a sequence of hex-encoded UPPs " + "separated by newlines. The third (default) scenario is that the script expects a sequence of binary UPPs separated " + "by newlines.\n\nIf --is-json true is set, --is-hex will be ignored." + ) + + self.argparser.add_argument("inputfile", metavar="INPUTFILE", type=str, + help="Input file path; e.g. upp_list.bin, upp_list.json or /dev/stdin" + ) + self.argparser.add_argument("verifying_key", metavar="VK", type=str, + help="key to be used for verification; any verifying key in hex like \"b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068\"" + ) + self.argparser.add_argument("verifying_key_uuid", metavar="UUID", type=str, + help="the UUID for the verifying key; e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1" + ) + self.argparser.add_argument("--is-json", "-j", metavar="ISJSON", type=str, default=DEFAULT_ISJSON, + help="If true, the script expects a JSON file for INPUTFILE (see below); e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) + self.argparser.add_argument("--is-hex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, + help="If true, the script hex-encoded UPPs from the input file; e.g. true, false (default: %s)" % DEFAULT_ISHEX + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.vk_str = self.args.verifying_key + self.vk_uuid_str = self.args.verifying_key_uuid + self.input = self.args.inputfile + self.ishex_str = self.args.is_hex + self.isjson_str = self.args.is_json + + # check the VK + try: + # convert the string + self.vk = ed25519.VerifyingKey(self.vk_str, encoding="hex") + except Exception as e: + logger.error("Invalid verifying key: \"%s\"" % self.vk_str) + logger.exception(e) + + return False + + # check the UUID + try: + self.vk_uuid = uuid.UUID(hex=self.vk_uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.vk_uuid_str) + logger.exception(e) + + return False + + # get the ishex value + if self.ishex_str.lower() in ["1", "yes", "y", "true"]: + self.ishex = True + else: + self.ishex = False + + # get the ishex value + if self.isjson_str.lower() in ["1", "yes", "y", "true"]: + self.isjson = True + else: + self.isjson = False + + return True + + def read_upps(self) -> bool: + # check whether json is enabled or not + if self.isjson == True: + try: + # read the json file and get the upps from the contained list + logger.info("Reading the input UPP json from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + input_json : dict = json.load(fd) + + # get the upps field + upps_hex = input_json.get("upps") + + if upps_hex == None or type(upps_hex) != list: + raise Exception("input json must contain a \"upps\" filed which must be a list of hex-encoded UPPs!") + + # decode the hex-upps + self.upps = list(map(lambda x: binascii.unhexlify(x), upps_hex)) + except Exception as e: + logger.exception(e) + + return False + else: + # read the UPP from the input path + try: + logger.info("Reading the input UPPs from \"%s\"" % self.input) + + with open(self.input, "rb") as fd: + upp_list_raw = fd.read().splitlines() + + # check whether hex decoding is needed + if self.ishex == True: + self.upps = list(map(lambda x: binascii.unhexlify(x), upp_list_raw)) + else: + self.upps = upp_list_raw + except Exception as e: + logger.exception(e) + + return False + + logger.info("Read %d UPPs" % len(self.upps)) + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore("-- temporary --", None) + except Exception as e: + logger.exception(e) + + return False + + return True + + def check_cli_vk(self) -> bool: + try: + if self.vk != None: + self.keystore.insert_ed25519_verifying_key(self.vk_uuid, self.vk) + + logger.info("Inserted \"%s\": \"%s\" (UUID/VK) into the keystore" % (self.vk_uuid_str, self.vk_str)) + except Exception as e: + logger.exception(e) + + return False + + return True + + def init_proto(self) -> bool: + try: + self.proto = Proto(self.keystore) + except Exception as e: + logger.exception(e) + + return False + + return True + + def verify_upps(self) -> bool: + # store the signature of the last checked upp + prev_sig = None + + try: + for i in range(0, len(self.upps)): + # check the signature + if self.proto.verfiy_signature(self.vk_uuid, self.upps[i]) == False: + raise Exception("The signature cannot be verified with the given VK - the UPP is invalid - Aborting at UPP %d" % (i + 1)) + + + + # unpack the upp + upp_unpacked = self.proto.unpack_upp(self.upps[i]) + + # check whether the UUID matches with the given vk_uuid + uuid_index = self.proto.get_unpacked_index(upp_unpacked[0], UNPACKED_UPP_FIELD_UUID) + + if upp_unpacked[uuid_index] != self.vk_uuid.bytes: + raise Exception("The UUID contained in UPP %s doesn't match the VK-UUID (%s vs. %s) - Aborting at UPP %d" % + (i + 1, uuid.UUID(bytes=upp_unpacked[uuid_index]), self.vk_uuid_str, i + 1) + ) + + # check whether a prevsig check should be done + if prev_sig != None: + # get the index of the previous signature + prevsig_index = self.proto.get_unpacked_index(upp_unpacked[0], UNPACKED_UPP_FIELD_PREV_SIG) + + # check the return value - -1 means, that the UPP is not chained/doesn't contain a prevsig + if prevsig_index == -1: + raise Exception("UPP %d is NOT a chained UPP/doesn't contain a prevsig - Aborting at UPP %d" % (i + 1, i + 1)) + + # compare the signatures + if prev_sig != upp_unpacked[prevsig_index]: + raise Exception("The prevsig of UPP %d doesn't match the sig of UPP %d - Aborting at UPP %d" % (i + 1, i, i + 1)) + + # set the new prevsig + sig_index = self.proto.get_unpacked_index(upp_unpacked[0], UNPACKED_UPP_FIELD_SIG) + + if sig_index == -1: + raise Exception("UPP %d is NEITHER chained NOR signed/doesn't contain a signature - Aborting at UPP %d" % (i + 1, i + 1)) + + prev_sig = upp_unpacked[sig_index] + + logger.info("All signatures verified and prevsigs compared - the UPP chain is valid!") + except KeyError: + logger.error("No verifying key found for UUID \"%s\" - can't verify the UPP!" % self.upp_uuid_str) + + return False + except Exception as e: + logger.exception(e) + + return False + + return True + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # read the upp + if self.read_upps() != True: + logger.error("Errors occured while reading the UPPs from \"%s\" - exiting!\n" % self.input) + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check/insert the cli-provided verifying key + if self.check_cli_vk() != True: + logger.error("Errorc occured while inserting the verifying key into the keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the Protocol + if self.init_proto() != True: + logger.error("Erros occured while initializing the Protocol - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # try to verify the message + if self.verify_upps() != True: + logger.error("Errors occured while verifying the UPPs - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From b33e3a68c70db00464540b896ae3e6f322dcec71 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sun, 7 Nov 2021 15:47:15 +0100 Subject: [PATCH 45/68] add docs for the chain-checker script --- examples/EXAMPLES.md | 122 +++++++++++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 29 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 7dca234..5c06c50 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -9,10 +9,11 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Generating and managing a keypair](#generating-and-managing-a-keypair) - [Registering a public key](#registering-a-public-key) - [Gathering Data](#gathering-data) - - [Creating a UPP](#creating-a-upp) - - [Sending a UPP](#sending-a-upp) - - [Verifying a UPP](#verifying-a-upp) - - [Examining a UPP](#examining-a-upp) + - [Creating an UPP](#creating-an-upp) + - [Sending an UPP](#sending-an-upp) + - [Verifying an UPP](#verifying-an-upp) + - [Verifying an UPP chain](#verifying-an-upp-chain) + - [Examining an UPP](#examining-an-upp) - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) @@ -25,7 +26,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Verify ED25519 signed UPP](#verify-ed25519-signed-upp) ## From measurement to blockchain-anchored UPP -The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. +The process needed to get an UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 1. [Setup](#setup) @@ -35,15 +36,15 @@ The process needed to get a UPP to be anchored in the blockchain can be cut down 4. [Gathering Data](#gathering-data) -5. [Creating a UPP](#creating-a-upp) +5. [Creating an UPP](#creating-an-upp) -6. [Sending a UPP](#sending-a-upp) +6. [Sending an UPP](#sending-an-upp) -7. [Verifying a UPP](#verifying-a-upp) +7. [Verifying an UPP](#verifying-an-upp) -8. [Examining a UPP](#examining-a-upp) +8. [Examining an UPP](#examining-an-upp) -9. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) +9. [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-a-upp) 10. [Verifying data](#verifying-data) ### Setup @@ -65,7 +66,7 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ an UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. @@ -89,12 +90,12 @@ optional arguments: By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. ### Registering a public key -To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: +To enable the uBirch backend to verify an UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` upp-creator.py upp-sender.py ``` -Both of these scripts will be explained in more detail in [Creating a UPP](#creating-a-upp) and [Sending a UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: +Both of these scripts will be explained in more detail in [Creating an UPP](#creating-a-upp) and [Sending an UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: ``` $ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --output keyreg_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 none @@ -139,8 +140,8 @@ serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_a This will create a string from the above examplary JSON object: `{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` -### Creating a UPP -After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). +### Creating an UPP +After gathering some measurement data an UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). ``` $ python3 upp-creator.py --help @@ -170,12 +171,12 @@ optional arguments: --nostdout nostdout, -n nostdout do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) -Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. -- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, a UPP with a signature and a UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. - `--type/-t` This flag sets the type field of the UPP. It is used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases where a specific value is required, is a Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). - `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore, pointed to by this parameter, doesn't exist, the script will simply create it. - `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this, or you will lose access to the keystore and all its contents. @@ -204,7 +205,7 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh ``` Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. -### Sending a UPP +### Sending an UPP After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The [`upp-sender.py`](upp-sender.py) script can be used for that. ``` $ python3 upp-sender.py --help @@ -230,7 +231,7 @@ For this script the parameters are: - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. -Continuing from the example above (see [Creating a UPP](#creating-a-upp)), the send-command might look like this: +Continuing from the example above (see [Creating an UPP](#creating-a-upp)), the send-command might look like this: ``` $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 2021-07-02 15:21:36,966 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "upp.bin" @@ -239,14 +240,14 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de 2021-07-02 15:21:37,723 root store_response_upp() INFO The response UPP has been written to "response_upp.bin" ``` -### Verifying a UPP +### Verifying an UPP To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is [`upp-verifier.py`](upp-verifier.py). It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] -Check if a UPP is valid/properly signed +Check if an UPP is valid/properly signed optional arguments: -h, --help show this help message and exit @@ -257,9 +258,9 @@ optional arguments: --input INPUT, -i INPUT UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) -Note that, when trying to verify a UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +Note that, when trying to verify an UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` -- `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. +- `--verifying-key/-k` If not trying to verify an UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. - `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. - `--input/-i` The file path to read the UPP from. @@ -271,8 +272,71 @@ $ python3 upp-verifier.py --input response_upp.bin 2021-07-02 15:43:36,275 root verify_upp() INFO Signature verified - the UPP is valid! ``` -### Examining a UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: +### Verifying an UPP chain +When working with `chained UPPs` it can be useful to check whether the chain is in order and valid. For +this task, the [`upp-chain-checker.py`](upp-chain-checke.py) can be used. It reads in a list of UPPs, +checks the signature of each UPP and compares the `prevsig` field with the `signature` of the last UPP. +If at any point something doesn't match up, it will print an error message alonge with the number of the +UPP at which the chain broke/something went wrong. The UPP list can either be read directly from a file +which contains them in binary or hex-encoded, separated by newlines, or from a JSON file which contains +a list of hex-encoded UPPs. +``` +$ python3 upp-chain-checker.py -h +usage: upp-chain-checker.py [-h] [--is-json ISJSON] [--is-hex ISHEX] INPUTFILE VK UUID + +Check if a sequence of chained UPPs is valid/properly signed and correctly chained + +positional arguments: + INPUTFILE Input file path; e.g. upp_list.bin, upp_list.json or /dev/stdin + VK key to be used for verification; any verifying key in hex like + "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" + UUID the UUID for the verifying key; e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 + +optional arguments: + -h, --help show this help message and exit + --is-json ISJSON, -j ISJSON + If true, the script expects a JSON file for INPUTFILE (see below); e.g. true, false (default: false) + --is-hex ISHEX, -x ISHEX + If true, the script hex-encoded UPPs from the input file; e.g. true, false (default: false) + +The JSON file (when using --is-json true) es expected to contain a single field called "upps", which is a list of hex-encoded UPPs. Otherwise (--is-json false). If --is-hex is true, it expects a sequence of hex-encoded UPPs separated by newlines. The third (default) scenario is that the script expects a sequence of binary UPPs separated by newlines. If --is-json true is set, --is-hex will be ignored. +``` +The `VK` and `UUID` arguments work like in the [UPP-Verifier](#verifying-an-upp), with the difference +that they aren't optional and must be provided. Here is an example JSON file containing four UPPs +```json +{ + "upps": [ + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c44025152e18f42352a7fba90b3fa30fc245587c8af1e681a77a25107137926a9ce88287e804b7989f60d9d9ea5673bc1531437fe147281b18071ac0adbe40d27d0b00c440f30a0ee67fc6f5ae5a133a012ab3931198752ee8e13084d473c1d1bd7dd000423b5ede36e5c217a2b8fe0512c5bfb3e8959f6773b812ddf98e45895ee9a7ac06c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f94903", + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f9490300c440a27146b7aa7bc0194468a1e5eee816dd07861bd4036654b74812f36e721b98615aedb84bb8700b5aede01207994c20b1bac759da95a3b41f4614c975a0668883c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a08", + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a0800c44038b971a62ce01cbbf302cb635c4c6f2faa266a5d78aa1edbda28ac8945ed51ac651b3fac2aa85b1d1685cf4424b7fbb1845a09e47b9ce69b957ceff2bcddf61dc4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d03", + "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d0300c440553470115df2e2bc5d1044fa3cec93f95c9e0d9df2daa394daca465d75e3dc91d34c6cfa0d7b29081f0dd58d79deae541e890d6ef04f6cf4a32031a8e855d93bc44040f79e9feb28e4086489431b5650b74849308f5a1911f3d630711e226eef03a0b48185964b753a63e44b36d5a9794f5f3df2af0e613545c063b81c7005f9d400" + ] +} +``` +If stored in `UPP_LIST.json`, it can be used like this: +``` +$ python3 upp-chain-checker.py --is-json true UPP_LIST.json 286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21 ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3 +2021-11-07 15:40:58,168 root read_upps() INFO Reading the input UPP json from "UPP_LIST.json" +2021-11-07 15:40:58,168 root read_upps() INFO Read 4 UPPs +2021-11-07 15:40:58,168 ubirch.ubirch_ks _load_keys() WARNING creating new key store: -- temporary -- +2021-11-07 15:40:58,168 root check_cli_vk() INFO Inserted "ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3": "286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21" (UUID/VK) into the keystore +2021-11-07 15:40:58,173 root verify_upps() INFO All signatures verified and prevsigs compared - the UPP chain is valid! +``` +Another way of using the script is this: +``` +$ echo -n -e "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c44025152e18f42352a7fba90b3fa30fc245587c8af1e681a77a25107137926a9ce88287e804b7989f60d9d9ea5673bc1531437fe147281b18071ac0adbe40d27d0b00c440f30a0ee67fc6f5ae5a133a012ab3931198752ee8e13084d473c1d1bd7dd000423b5ede36e5c217a2b8fe0512c5bfb3e8959f6773b812ddf98e45895ee9a7ac06c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f94903\n9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4406502e436d33edbfa8c1f82f9644344307e79dfd46c2a766083a238bfd6edca2ec6d83b2329a5b302516839bfac36b199c7593dded5bc4f0531f233ce53f9490300c440a27146b7aa7bc0194468a1e5eee816dd07861bd4036654b74812f36e721b98615aedb84bb8700b5aede01207994c20b1bac759da95a3b41f4614c975a0668883c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a08\n9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c440596c4b7b840681ce89bb1d6dbb2ccf1108e2007a68ed39fce71783c6d1e8b39ba78769866bacbc281a64d8f7d9ff20fd5dc6a1cf998104395e2018ad49a15a0800c44038b971a62ce01cbbf302cb635c4c6f2faa266a5d78aa1edbda28ac8945ed51ac651b3fac2aa85b1d1685cf4424b7fbb1845a09e47b9ce69b957ceff2bcddf61dc4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d03\n9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c4409c2f20ece86519f541b45b4e2aea4ea51b98c3d12014e513c303c8c9b0af7c0caab39894419dac6e4bf601c27273f9bc58c22ab9e93879fc472f381da00c1d0300c440553470115df2e2bc5d1044fa3cec93f95c9e0d9df2daa394daca465d75e3dc91d34c6cfa0d7b29081f0dd58d79deae541e890d6ef04f6cf4a32031a8e855d93bc44040f79e9feb28e4086489431b5650b74849308f5a1911f3d630711e226eef03a0b48185964b753a63e44b36d5a9794f5f3df2af0e613545c063b81c7005f9d400\n" | python3 upp-chain-checker.py --is-hex true /dev/stdin 286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21 ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3 + +2021-11-07 15:45:00,121 root read_upps() INFO Reading the input UPPs from "/dev/stdin" +2021-11-07 15:45:00,121 root read_upps() INFO Read 4 UPPs +2021-11-07 15:45:00,121 ubirch.ubirch_ks _load_keys() WARNING creating new key store: -- temporary -- +2021-11-07 15:45:00,121 root check_cli_vk() INFO Inserted "ee8c4cfe-9b3a-43e2-9e9f-8875cb02cec3": "286401c523ebbfb5f6a4044e62af8ef66775f9a76a2ff2af0067ecfb4563df21" (UUID/VK) into the keystore +2021-11-07 15:45:00,127 root verify_upps() INFO All signatures verified and prevsigs compared - the UPP chain is valid! +``` +The UPPs are piped as hex-encoded strings separated by newlines (`\n`) to the script which has the input +file path set to `/dev/stdin`. + +### Examining an UPP +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -288,12 +352,12 @@ $ python3 upp-unpacker.py response_upp.bin _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ ### Checking the anchoring status of an UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if an UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT -Requests the verification/anchoring of a UPP from the uBirch backend +Requests the verification/anchoring of an UPP from the uBirch backend positional arguments: INPUT input hash or upp path (depends on --ishash) @@ -309,7 +373,7 @@ optional arguments: When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` -- `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--ishash/-i` A boolean specifying whether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. - `--ishex/-x` A boolean which controls how the input UPP data is interpreted. By default, the data will be interpreted as normale binary data. When this flag is set to `true`, it will be considered @@ -352,7 +416,7 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ``` ### Verifying data -In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). ```txt $ python data-verifier.py --help usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT From 56bff975ba39f425fbf69f267e10564d1d57fb57 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:17:57 +0100 Subject: [PATCH 46/68] adapt changes from review to the readme --- examples/EXAMPLES.md | 146 +++++++------------------------------------ 1 file changed, 23 insertions(+), 123 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 5c06c50..15e4563 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -74,20 +74,27 @@ A keystore can be read out with the [`keystore-dumper.py`](keystore-dumper.py) s ``` $ python keystore-dumper.py --help -usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KET] KEYSTORE KEYSTORE_PASS - -Dump the contents of a keystore (.jks) - -positional arguments: - KEYSTORE keystore file path; e.g.: test.jks - KEYSTORE_PASS keystore password; e.g.: secret - -optional arguments: - -h, --help show this help message and exit - --show-sk SHOW_SIGNING_KEY, -s SHOW_SIGNING_KEY - enables/disables showing of signing keys; e.g.: true, false (default: False) -``` -By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. +usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KEY] KEYSTORE KEYSTORE_PASS +``` +By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. `KEYSTORE` is the path to the keystore file and `KEYSTORE_PASS` is the password needed to open/decrypt it. Below is an example with `-s` being set to `false`. +``` +====================================================================================================================================== +UUID: 292e2f0b-1d9e-407f-9b1a-bda5a9560797 + VK : 2dc6fdf373c7e13abfe1f51ff0fe45417ad713c5fb8df90b76c3077355c4b5e9 + SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ +====================================================================================================================================== +====================================================================================================================================== +UUID: 832e885c-be2a-44b1-ade6-c6c6dd718a3b + VK : 43536e9793ab5b205ba62e614d12104cbb6f906b2a44b79c1cd01e87a724d5b1 + SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ +====================================================================================================================================== +====================================================================================================================================== +UUID: 622e3933-8fdd-446f-b645-d3aa7ed8b638 + VK : cfd32f687366679d7ef73795bafa3bfca43421e092fbc2501f7f4bfe339bb15a + SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ +====================================================================================================================================== +``` +As you can see, the secret key is not printed and `█`s are used as placeholders. ### Registering a public key To enable the uBirch backend to verify an UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: @@ -147,33 +154,7 @@ $ python3 upp-creator.py --help usage: upp-creator.py [-h] [--version VERISON] [--type TYPE] [--ks KS] [--kspwd KSPWD] [--keyreg KEYREG] [--hash HASH] [--isjson ISJSON] [--output OUTPUT] [--nostdout nostdout] UUID DATA -Create a uBirch Protocol Package (UPP) - -positional arguments: - UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 - DATA data to be packed into the UPP or hashed; e.g.: {"t": 23.4, "ts": 1624624140} - -optional arguments: - -h, --help show this help message and exit - --version VERISON, -v VERISON - version of the UPP; 0x21 (unsigned; NOT IMPLEMENTED), 0x22 (signed) or 0x23 (chained) (default: 0x23) - --type TYPE, -t TYPE type of the UPP (0 < type < 256); e.g.: 0x00 (unknown), 0x32 (msgpack), 0x53 (generic), ... (default and recommended: 0x00) - --ks KS, -k KS keystore file path; e.g.: test.jks (default: devices.jks) - --kspwd KSPWD, -p KSPWD - keystore password; e.g.: secret (default: keystore) - --keyreg KEYREG, -r KEYREG - generate a key registration UPP (data and --hash will be ignored); e.g.: true, false (default: False) - --hash HASH hash algorithm for hashing the data; sha256, sha512 or off (disable hashing), ... (default and recommended: sha512) - --isjson ISJSON, -j ISJSON - tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false (default: False) - --output OUTPUT, -o OUTPUT - file to write the generated UPP to (aside from standard output); e.g. upp.bin (default: upp.bin) - --nostdout nostdout, -n nostdout - do not output anything to stdout; can be combined with --output /dev/stdout; e.g.: true, false (default: False) - -Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain -using this tool.Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the -payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain using this tool. Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. - `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. @@ -210,20 +191,6 @@ After creating the UPP, it can be sent to the uBirch backend where it will be ve ``` $ python3 upp-sender.py --help usage: upp-sender.py [-h] [--env ENV] [--input INPUT] [--output OUTPUT] UUID AUTH - -Send a uBirch Protocol Package (UPP) to uBirch Niomon - -positional arguments: - UUID UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183 - AUTH uBirch device authentication token - -optional arguments: - -h, --help show this help message and exit - --env ENV, -e ENV environment to operate in; dev, demo or prod (default: dev) - --input INPUT, -i INPUT - UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) - --output OUTPUT, -o OUTPUT - response UPP output file path (ignored for key registration UPPs); e.g. response_upp.bin (default: response_upp.bin) ``` For this script the parameters are: - `--env/-e` The env to operate on. This parameter decides wether the UPP will be sent to `niomon.prod.ubirch.com`, `niomon.demo.ubirch.com` or `niomon.dev.ubirch.com`. The value can either be `prod`, `demo` or `dev`. It must match the stage, the UUID is registered on. @@ -247,17 +214,6 @@ To make sure, that the response UPP actually was sent by the uBirch backend, its $ python3 upp-verifier.py --help usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] -Check if an UPP is valid/properly signed - -optional arguments: - -h, --help show this help message and exit - --verifying-key VK, -k VK - key to be used for verification; any verifying key in hex like "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" - --verifying-key-uuid UUID, -u UUID - the UUID for the key supplied via -k (only needed when -k is specified); e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 - --input INPUT, -i INPUT - UPP input file path; e.g. upp.bin or /dev/stdin (default: upp.bin) - Note that, when trying to verify an UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` - `--verifying-key/-k` If not trying to verify an UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. @@ -274,7 +230,7 @@ $ python3 upp-verifier.py --input response_upp.bin ### Verifying an UPP chain When working with `chained UPPs` it can be useful to check whether the chain is in order and valid. For -this task, the [`upp-chain-checker.py`](upp-chain-checke.py) can be used. It reads in a list of UPPs, +this task, the [`upp-chain-checker.py`](upp-chain-checker.py) can be used. It reads in a list of UPPs, checks the signature of each UPP and compares the `prevsig` field with the `signature` of the last UPP. If at any point something doesn't match up, it will print an error message alonge with the number of the UPP at which the chain broke/something went wrong. The UPP list can either be read directly from a file @@ -283,23 +239,6 @@ a list of hex-encoded UPPs. ``` $ python3 upp-chain-checker.py -h usage: upp-chain-checker.py [-h] [--is-json ISJSON] [--is-hex ISHEX] INPUTFILE VK UUID - -Check if a sequence of chained UPPs is valid/properly signed and correctly chained - -positional arguments: - INPUTFILE Input file path; e.g. upp_list.bin, upp_list.json or /dev/stdin - VK key to be used for verification; any verifying key in hex like - "b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068" - UUID the UUID for the verifying key; e.g.: 6eac4d0b-16e6-4508-8c46-22e7451ea5a1 - -optional arguments: - -h, --help show this help message and exit - --is-json ISJSON, -j ISJSON - If true, the script expects a JSON file for INPUTFILE (see below); e.g. true, false (default: false) - --is-hex ISHEX, -x ISHEX - If true, the script hex-encoded UPPs from the input file; e.g. true, false (default: false) - -The JSON file (when using --is-json true) es expected to contain a single field called "upps", which is a list of hex-encoded UPPs. Otherwise (--is-json false). If --is-hex is true, it expects a sequence of hex-encoded UPPs separated by newlines. The third (default) scenario is that the script expects a sequence of binary UPPs separated by newlines. If --is-json true is set, --is-hex will be ignored. ``` The `VK` and `UUID` arguments work like in the [UPP-Verifier](#verifying-an-upp), with the difference that they aren't optional and must be provided. Here is an example JSON file containing four UPPs @@ -356,22 +295,6 @@ uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This proce ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT - -Requests the verification/anchoring of an UPP from the uBirch backend - -positional arguments: - INPUT input hash or upp path (depends on --ishash) - -optional arguments: - -h, --help show this help message and exit - --ishash ISHASH, -i ISHASH - sets if INPUT is being treated as a hash or upp path; true or false (default: False) - --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) - --ishex ISHEX, -x ISHEX - Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: false) - -When --ishash/-i is set to true, the input argument is treated as a base64 payload hash. Otherwise, it is expected to be some kind of path to read a -UPP from. This can be a file path or also /dev/stdin if the UPP is piped to this program via standard input. ``` - `--ishash/-i` A boolean specifying whether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. @@ -420,29 +343,6 @@ In a real use case, not the UPP, but rather the original data itself has to be v ```txt $ python data-verifier.py --help usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT - -Check if the hash of given input data is known to the uBirch backend (verify it) - -positional arguments: - INPUT input data or data file path (depends on --ispath) - -optional arguments: - -h, --help show this help message and exit - --ispath ISPATH, -i ISPATH - sets if INPUT is being treated as data or data file path; true or false (default: False) - --env ENV, -e ENV the environment to operate in; dev, demo or prod (default: dev) - --isjson ISJSON, -j ISJSON - tells the script to treat the input data as json and serealize it (see EXAMPLES.md for more information); true or false - (default: True) - --hash HASH, -a HASH sets the hash algorithm to use; sha256, sha512 or OFF to treat the input data as hash (default: sha256) - --no-send NOSEND, -n NOSEND - if set to true, the script will only generate the hash of the input data without sending it; true or false (default: False) - --ishl ISHASHLINK, -l ISHASHLINK - implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to - decide which fields to hash; true or false (default: False) - -When --ispath/-i is set to true, the input data is treated as a file path to read the actual input data from. When setting --hash/-a to off, the -input argument is expected to be a valid base64 encoded hash. ``` - `--ispath/-i` Specifies wether the input is to be treated as a data-file path or direct input data. `true` or `false`. - `--env-e` The stage to check on. Should be the one the UPP corresponding to the data was sent to. `prod`, `demo` or `dev`. From 71a6a1c9601998490261ce3ce3b52c250bac8b10 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:19:19 +0100 Subject: [PATCH 47/68] include changes from review --- examples/data-sender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/data-sender.py b/examples/data-sender.py index 136e5df..5ca500d 100644 --- a/examples/data-sender.py +++ b/examples/data-sender.py @@ -46,7 +46,7 @@ def setup_argparse(self): help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" ) self.argparser.add_argument("auth", metavar="AUTH", type=str, - help="uBirch device authentication token" + help="uBirch device authentication token, e.g.: 12345678-1234-1234-1234-123456789abc (this is NOT the UUID)" ) self.argparser.add_argument("--env", "-e", metavar="ENV", type=str, default=DEFAULT_ENV, help="environment to operate in; dev, demo or prod (default: %s)" % DEFAULT_ENV From 2c59f532aa7276077272754ca2a4652794d674c4 Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:23:37 +0100 Subject: [PATCH 48/68] [from review] replace old print statement with logger.x --- examples/upp-anchoring-status.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index df0f5dc..9da7d0e 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -154,15 +154,15 @@ def get_status(self) -> bool: jobj = json.loads(r.content) - print("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) - print("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + logger.info("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) + logger.info("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) if jobj.get("anchors") in [None, []]: logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") else: logger.info("The UPP has been fully anchored!") - print(jobj.get("anchors")) + logger.info(jobj.get("anchors")) elif r.status_code == 404: logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) except Exception as e: From 9c3027ec7342c321a1a005079e8614f34dea819a Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:28:13 +0100 Subject: [PATCH 49/68] initialize ishex and ishex_str in __init__ before using it in setup_argparse --- examples/upp-chain-checker.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/upp-chain-checker.py b/examples/upp-chain-checker.py index b692124..b101feb 100644 --- a/examples/upp-chain-checker.py +++ b/examples/upp-chain-checker.py @@ -51,6 +51,9 @@ def __init__(self): self.isjson : bool = None self.isjson_str : str = None + self.ishex : bool = None + self.ishex_str : str = None + # initialize the argument parser self.setup_argparse() From f4cb6e1fee872a7f42692ed71d068664ae32863e Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:29:18 +0100 Subject: [PATCH 50/68] adapt changes from review --- examples/upp-creator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 2151972..2654f7e 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -76,10 +76,12 @@ def __init__(self): self.nostdout : bool = None self.payload : bytes = None self.ishl : bool = None + self.ishl_str : str = None self.ishash : bool = False self.hasher : object = None self.keystore : ubirch.KeyStore = None + self.keystore_pass : str = None self.proto : Proto = None self.payload : bytes = None self.upp : bytes = None @@ -252,7 +254,7 @@ def init_proto(self) -> bool: except Exception as e: logger.exception(e) - False + return False return True From 725ec4b2f85b90b79e1ef9080ad9873144f2d33c Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Tue, 16 Nov 2021 16:38:10 +0100 Subject: [PATCH 51/68] adapt changes from review --- ubirch/ubirch_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubirch/ubirch_protocol.py b/ubirch/ubirch_protocol.py index 6250f8f..1456886 100644 --- a/ubirch/ubirch_protocol.py +++ b/ubirch/ubirch_protocol.py @@ -289,8 +289,8 @@ def verfiy_signature(self, uuid: UUID, msgpackUPP: bytes) -> bool: """ Verify the integrity of the message and decode the contents Raises an value error when the message is too short + :param uuid: the uuid of the sender of the message :param msgpackUPP: the msgpack encoded message - :param unpackedUPP: (optional) if not provided, the function will unpack the upp itself :return: the decoded message """ # separate the message from the signature From bda1dae409d31cf488f0050fc2258b20b8c3d5fc Mon Sep 17 00:00:00 2001 From: UBOK19 Date: Sat, 5 Feb 2022 14:33:29 +0100 Subject: [PATCH 52/68] add note about which endpoint to use for verifiaction --- examples/data-verifier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/data-verifier.py b/examples/data-verifier.py index b41793e..d9e95e2 100644 --- a/examples/data-verifier.py +++ b/examples/data-verifier.py @@ -6,7 +6,8 @@ import hashlib import requests - +# remove '/anchor' to disable anchoring lookup; increases speed +# but no anchoring information will be shown (only curr/prev UPP) VERIFICATION_SERVICE = "https://verify.%s.ubirch.com/api/upp/verify/anchor" DEFAULT_ISPATH = "False" From b37b50be26a64da83d3668791ecc1e23376b5443 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Sun, 20 Feb 2022 12:26:26 +0100 Subject: [PATCH 53/68] replace the keystore dumper with a more capable script --- examples/keystore-dumper.py | 128 ---------------- examples/keystore-tool.py | 282 ++++++++++++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 128 deletions(-) delete mode 100644 examples/keystore-dumper.py create mode 100644 examples/keystore-tool.py diff --git a/examples/keystore-dumper.py b/examples/keystore-dumper.py deleted file mode 100644 index 3ae90b4..0000000 --- a/examples/keystore-dumper.py +++ /dev/null @@ -1,128 +0,0 @@ -import sys -import time -import argparse -import logging -import binascii -import uuid - -import ubirch - - -DEFAULT_SHOW_SIGN = "False" - - -logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) -logger = logging.getLogger() - - -class Main: - def __init__(self): - self.argparser : argparse.ArgumentParser = None - self.args : argparse.Namespace = None - - self.keystore_path : str = None - self.keystore_pass : str = None - self.show_sign_str : str = None - self.show_sign : bool = None - - self.keystore : ubirch.KeyStore = None - - # initialize the argument parser - self.setup_argparse() - - return - - def setup_argparse(self): - self.argparser = argparse.ArgumentParser( - description="Dump the contents of a keystore (.jks)", - epilog="" - ) - - self.argparser.add_argument("keystore", metavar="KEYSTORE", type=str, - help="keystore file path; e.g.: test.jks" - ) - self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, - help="keystore password; e.g.: secret" - ) - self.argparser.add_argument("--show-sk", "-s", metavar="SHOW_SIGNING_KEY", type=str, default=DEFAULT_SHOW_SIGN, - help="enables/disables showing of signing keys; e.g.: true, false (default: %s)" % DEFAULT_SHOW_SIGN - ) - - def process_args(self) -> bool: - # parse cli arguments (exists on err) - self.args = self.argparser.parse_args() - - # get all needed args - self.keystore_path = self.args.keystore - self.keystore_pass = self.args.keystore_pass - self.show_sign_str = self.args.show_sk - - # get the bool for show sk - if self.show_sign_str.lower() in ["1", "yes", "y", "true"]: - self.show_sign = True - else: - self.show_sign = False - - return True - - def init_keystore(self) -> bool: - try: - self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) - except Exception as e: - logger.exception(e) - - return False - - return True - - def dump_keystore(self) -> bool: - verifying_keys = self.keystore._ks.certs - signing_keys = self.keystore._ks.private_keys - - # go trough the list of verifiying keys and print information for each entry - for vk_uuid in verifying_keys.keys(): - if self.show_sign == True: - t = signing_keys.get("pke_" + vk_uuid) - - sk = binascii.hexlify(t.pkey).decode() if t != None else "N / A" - else: - sk = "█" * 128 - - print("=" * 134) - print("UUID: %s" % str(uuid.UUID(hex=vk_uuid))) - print(" VK : %s" % binascii.hexlify(verifying_keys[vk_uuid].cert).decode()) - print(" SK : %s" % sk) - print("=" * 134) - - return True - - def run(self) -> int: - # process all raw argument values - if self.process_args() != True: - logger.error("Errors occured during argument processing - exiting!\n") - - self.argparser.print_usage() - - return 1 - - # initialize the keystore - if self.init_keystore() != True: - logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") - - self.argparser.print_usage() - - return 1 - - if self.dump_keystore() != True: - logger.error("Errors occured while dumping the uBirch Keystore - exiting!\n") - - self.argparser.print_usage() - - return 1 - - return 0 - - -# initialize/start the main class -if __name__ == "__main__": - sys.exit(Main().run()) diff --git a/examples/keystore-tool.py b/examples/keystore-tool.py new file mode 100644 index 0000000..8b911c3 --- /dev/null +++ b/examples/keystore-tool.py @@ -0,0 +1,282 @@ +import sys +import time +import argparse +import logging +import binascii +import uuid +import ed25519 + +import ubirch + + +DEFAULT_SHOW_SECRET = "False" +COMMAND_GET = "get" +COMMAND_PUT = "put" +COMMAND_DEL = "del" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.get_argparser : argparse.ArgumentParser = None + self.put_argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + # for all commands + self.keystore_path : str = None + self.keystore_pass : str = None + self.uuid_str : str = None + self.uuid : uuid.UUID = None + self.cmd : str = None + + # for get + self.show_sign_str : str = None + self.show_sign : bool = None + + # for put + self.pubkey_str : str = None + self.pubkey : ed25519.VerifyingKey = None + self.prvkey_str : str = None + self.prvkey : ed25519.SigningKey = None + + self.keystore : ubirch.KeyStore = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Manipulate/View the contents of a keystore (.jks)", + epilog="Only one entry per UUID is supported. Passing an non-existent KeyStore file as argument will lead to a new KeyStore being created. This new KeyStore will only be persistent if a write operation (-> key insertion) takes place." + ) + + self.argparser.add_argument("keystore", metavar="KEYSTORE", type=str, + help="keystore file path; e.g.: test.jks" + ) + self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, + help="keystore password; e.g.: secret" + ) + + # create subparsers + subparsers = self.argparser.add_subparsers(help="Command to execute.", dest="cmd", required=True) + + # subparser for the get-command + self.get_argparser = subparsers.add_parser(COMMAND_GET, help="Get entries from the KeyStore.") + + self.get_argparser.add_argument("--uuid", "-u", type=str, default=None, + help="UUID to filter for. Only keys for this UUID will be returned; e.g.: f99de1c4-3859-5326-a155-5696f00686d9" + ) + self.get_argparser.add_argument("--show-secret", "-s", type=str, default=DEFAULT_SHOW_SECRET, + help="Enables/Disables showing of secret (signing/private) keys; e.g.: true/false (default: %s)" % DEFAULT_SHOW_SECRET + ) + + # subparser for the put-command + self.put_argparser = subparsers.add_parser(COMMAND_PUT, help="Put a new entry into the KeyStore.") + + self.put_argparser.add_argument("uuid", metavar="UUID", type=str, + help="The UUID the new keys belong to; e.g.: f99de1c4-3859-5326-a155-5696f00686d9" + ) + self.put_argparser.add_argument("pubkey", metavar="PUBKEY", type=str, + help="The HEX-encoded ED25519 PubKey; e.g.: 189595c87a972c55eb7348a310fa1ff479a895a1f226d189b5ad505b9d8c8bbf" + ) + self.put_argparser.add_argument("privkey", metavar="PRIVKEY", type=str, + help="The HEX-encoded ED25519 PrivKey; e.g.: 9c7c43e122ae51e08a86e9bb89fe340bd4c7bd6665bf2b40004d4012f1523575127f8ac54a971765126a866428a6c74d4747d1b68e189f0fa3528a73e3f59714" + ) + + # subparser for the del-command + self.del_argparser = subparsers.add_parser(COMMAND_DEL, help="Delete an entry from the KeyStore.") + + self.del_argparser.add_argument("uuid", metavar="UUID", type=str, + help="The UUID to delete the keypair for (this is safe since each UUID can only occur once in the KeyStore); e.g.: f99de1c4-3859-5326-a155-5696f00686d9" + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.keystore_path = self.args.keystore + self.keystore_pass = self.args.keystore_pass + self.cmd = self.args.cmd + + # for put + if self.cmd == COMMAND_PUT: + self.uuid_str = self.args.uuid + self.pubkey_str = self.args.pubkey + self.prvkey_str = self.args.privkey + + # load the keypair + try: + self.pubkey = ed25519.VerifyingKey(binascii.unhexlify(self.pubkey_str)) + except Exception as e: + logger.error("Error loading the PubKey!") + logger.exception(e) + + return False + + try: + self.prvkey = ed25519.SigningKey(binascii.unhexlify(self.prvkey_str)) + except Exception as e: + logger.error("Error loading the PrivKey!") + logger.exception(e) + + return False + elif self.cmd == COMMAND_GET: + # for get + self.show_sign_str = self.args.show_secret + self.uuid_str = self.args.uuid + + # get the bool for show sk + if self.show_sign_str.lower() in ["1", "yes", "y", "true"]: + self.show_sign = True + else: + self.show_sign = False + elif self.cmd == COMMAND_DEL: + # for del + self.uuid_str = self.args.uuid + else: + logger.error("Unknown command \"%s\"!" % self.cmd) + + return False + + # load the uuid if specified + if self.uuid_str != None: + try: + self.uuid = uuid.UUID(self.uuid_str) + except Exception as e: + logger.error("Error loading UUID: \"%s\"" % self.uuid_str) + logger.exception(e) + + return False + + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) + except Exception as e: + logger.exception(e) + + return False + + return True + + def dump_keystore(self) -> bool: + verifying_keys = self.keystore._ks.certs + signing_keys = self.keystore._ks.private_keys + + # go trough the list of verifiying keys and print information for each entry + for vk_uuid in verifying_keys.keys(): + # check if a filtering uuid is set; if it is, filter + if self.uuid != None: + if self.uuid.hex != vk_uuid: + continue + + if self.show_sign == True: + t = signing_keys.get("pke_" + vk_uuid) + + sk = binascii.hexlify(t.pkey).decode() if t != None else "N / A" + else: + sk = "█" * 128 + + print("=" * 134) + print("UUID: %s" % str(uuid.UUID(hex=vk_uuid))) + print(" VK : %s" % binascii.hexlify(verifying_keys[vk_uuid].cert).decode()) + print(" SK : %s" % sk) + print("=" * 134) + + return True + + def put_keypair(self) -> bool: + logger.info("Inserting keypair for %s with pubkey %s into %s!" % (self.uuid_str, self.pubkey_str, self.keystore_path)) + + try: + self.keystore.insert_ed25519_keypair(self.uuid, self.pubkey, self.prvkey) + except Exception as e: + logger.error("Error inserting the keypair into the KeyStore!") + logger.exception(e) + + return True + + def del_keypair(self) -> bool: + logger.warning("About to remove the keypair for UUID %s from %s! Enter 'YES' to continue" % (self.uuid_str, self.keystore_path)) + + # get user confirmation to delete + if input("> ") != 'YES': + logger.error("Aborting!") + + # stopped the process by user-choice; not a "real" error + return True + + # delete both the pubkey and the private key entries + try: + # direkt access to the entries variable is needed since .certs and .private_keys + # are class properties which are only temporary (-> editing them has no effect) + self.keystore._ks.entries.pop(self.uuid.hex) + self.keystore._ks.entries.pop("pke_" + self.uuid.hex) + except Exception as e: + logger.error("Error deleting keys! No changes will be written!") + logger.exception(e) + + return False + + # write changes + self.keystore._ks.save(self.keystore._ks_file, self.keystore._ks_password) + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_help() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") + + self.argparser.print_help() + + return 1 + + if self.cmd == COMMAND_GET: + if self.dump_keystore() != True: + logger.error("Errors occured while dumping the uBirch Keystore - exiting!\n") + + self.get_argparser.print_help() + + return 1 + elif self.cmd == COMMAND_PUT: + if self.put_keypair() != True: + logger.error("Errors occured while puting a new keypair into the KeyStore - exiting!\n") + + self.put_argparser.print_help() + + return 1 + elif self.cmd == COMMAND_DEL: + if self.del_keypair() != True: + logger.error("Errors occured while deleting a keypair from the KeyStore - exiting!\n") + + self.del_argparser.print_help() + + return 1 + else: + logger.error("Unknown command \"%s\" - exiting!\n" % self.cmd) + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From e4d882c691d1012ee4ed7e6a26a3858a92462132 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Sun, 20 Feb 2022 12:29:22 +0100 Subject: [PATCH 54/68] implement the key service example script --- examples/pubkey-util.py | 536 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 536 insertions(+) create mode 100644 examples/pubkey-util.py diff --git a/examples/pubkey-util.py b/examples/pubkey-util.py new file mode 100644 index 0000000..f329383 --- /dev/null +++ b/examples/pubkey-util.py @@ -0,0 +1,536 @@ +import sys +import logging +import argparse +import requests +import uuid +import ed25519 +import binascii +import json +import base64 +import msgpack + +import ubirch + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + +# put_key use msgpack +PUT_KEY_USE_MSGPACK_DEFAULT="false" + +# commands +PUT_NEW_KEY_CMD = "put_new_key" +GET_DEV_KEYS_CMD = "get_dev_keys" +GET_KEY_INFO_CMD = "get_key_info" +DELETE_KEY_CMD = "delete_key" +REVOKE_KEY_CMD = "revoke_key" + +# URLs and paths +UBIRCH_ID_SERVICE = "https://identity.%s.ubirch.com/api/keyService/v1/pubkey" +GET_DEVICE_KEYS_PATH = "/current/hardwareId/%s" # completed with the uuid +GET_KEY_INFO_PATH = "/%s" # completed with the pubkeyId (equal to pubkey) in b64 +REVOKE_KEY_PATH = "/revoke" +PUT_KEY_MSGPACK_PATH = "/mpack" + +# body formats +DEL_PUBKEY_FMT = '{'\ + '"publicKey":"%s",'\ + '"signature":"%s"'\ +'}' + +REVOKE_PUBKEY_FMT = '{'\ + '"publicKey":"%s",'\ + '"signature":"%s"'\ +'}' + +PUT_PUBKEY_UPDATE_FMT_OUTER = '{'\ + '"pubKeyInfo":%s,'\ + '"prevSignature":"%s",'\ + '"signature":"%s"'\ +'}' +PUT_PUBKEY_UPDATE_FMT_INNER = '{'\ + '"algorithm":"%s",'\ + '"created":"%s",'\ + '"hwDeviceId":"%s",'\ + '"pubKey":"%s",'\ + '"pubKeyId":"%s",'\ + '"prevPubKeyId":"%s",'\ + '"validNotAfter":"%s",'\ + '"validNotBefore":"%s"'\ +'}' + +PUT_NEW_PUBKEY_FMT_OUTER = '{'\ + '"pubKeyInfo":%s,'\ + '"signature":"%s"'\ +'}' +PUT_NEW_PUBKEY_FMT_INNER = '{'\ + '"algorithm":"%s",'\ + '"created":"%s",'\ + '"hwDeviceId":"%s",'\ + '"pubKey":"%s",'\ + '"pubKeyId":"%s",'\ + '"validNotAfter":"%s",'\ + '"validNotBefore":"%s"'\ +'}' + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.get_dev_keys_parser : argparse.ArgumentParser = None + self.get_key_info_parser : argparse.ArgumentParser = None + self.put_new_key_parser : argparse.ArgumentParser = None + self.del_key_parser : argparse.ArgumentParser = None + self.revoke_key_parser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + self.cmd_str : str = None # store the command string + self.base_url : str = None # store the complete url (incl. env) + + # not every variable is needed for every command; still, all are listed here for clarity + self.uuid_str : str = None # for get_dev_keys, put_new_key, delete_key and revoke_key + self.uuid : uuid.UUID = None # for get_dev_keys, put_new_key, delete_key and revoke_key + self.pubkey_str : str = None # for put_new_key, delete_key, revoke_key, get_key_info + self.pubkey_b64 : bytes = None # for put_new_key, delete_key, rewoke_key, get_key_info + self.prvkey_str : str = None # for put_new_key, delete_key, revoke_key + self.old_pubkey_str : str = None # for put_new_key (when updating) + self.old_prvkey_str : str = None # for put_new_key (when updating) + self.old_pubkey_b64 : bytes = None # for put_new_key (when updating) + self.key_created_at : str = None # for put_new_key + self.key_valid_not_after : str = None # for put_new_key + self.key_valid_not_before : str = None # for put_new_key + self.use_msgpack_str : str = None # for put_new_key + self.use_msgpack : bool = None # for put_new_key + + # the privkey needs to be loaded as actual key (not string) for some operations + self.prvkey : ed25519.SigningKey = None + self.old_prvkey : ed25519.SigningKey = None + + self.upp : bytes = None + self.api : ubirch.API = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="A tool to perform pubkey operations with the uBirch Identity Service", + epilog="Choose an environment + command and use the '--help'/'-h' option to see a command-specific help message; e.g.: python %s dev revoke_key -h. Note that for many operations (like putting a new PubKey), only the PrivKey is needed. That is because in case of ED25519 keys, the PubKey can be generated out of the PrivKey, because the PrivKey is generally regarded to as a seed for the keypair." + ) + + # set up the main parameters + self.argparser.add_argument("env", metavar="", type=str, + help="Environment to work on. Must be one of 'dev', 'demo' or 'prod'. Case insensitive." + ) + self.argparser.add_argument("--debug", "-d", type=str, default="false", + help="Enables/Disables debug logging. When enabled, all HTTP bodies will be printed before sending; 'true'/'false' (Default: 'false')" + ) + + # generate a subparser group; the dest parameter is needed so that the program knows + # which subparser was triggered later on + subparsers = self.argparser.add_subparsers(help="Command to execute.", dest="cmd", required=True) + + # set up conditional parameters for each command/operation + # subparser + arguments for the get_dev_keys operation + self.get_dev_keys_parser = subparsers.add_parser(GET_DEV_KEYS_CMD, help="Get PubKeys registered for a given device.") + self.get_dev_keys_parser.add_argument("uuid", metavar="UUID", type=str, + help="The device UUID to get the keys for. E.g.: f99de1c4-3859-5326-a155-5696f00686d9" + ) + + # subparser + arguments for the get_key_info operation + self.get_key_info_parser = subparsers.add_parser(GET_KEY_INFO_CMD, help="Get information for a specific PubKey.") + self.get_key_info_parser.add_argument("pubkey", metavar="PUBKEY_HEX", type=str, + help="ED25519 Pubkey to retrieve information for in HEX" + ) + + # subparser + arguments for the put_new_key operation + self.put_new_key_parser = subparsers.add_parser(PUT_NEW_KEY_CMD, help="Register a new PubKey.") + self.put_new_key_parser.add_argument("uuid", metavar="UUID", type=str, + help="The device UUID to register a key for. E.g.: f99de1c4-3859-5326-a155-5696f00686d9" + ) + self.put_new_key_parser.add_argument("prvkey", metavar="PRIVKEY_HEX", type=str, + help="The ED25519 PrivKey corresponding to the PubKey in HEX." + ) + self.put_new_key_parser.add_argument("created", metavar="CREATED", type=str, + help="Date at which the PubKey was created; (format: 2020-12-30T11:11:11.000Z)" + ) + self.put_new_key_parser.add_argument("validNotBefore", metavar="VALID_NOT_BEFORE", type=str, + help="Date at which the PubKey will become valid; (format: 2020-12-30T22:22:22.000Z)." + ) + self.put_new_key_parser.add_argument("validNotAfter", metavar="VALID_NOT_AFTER", type=str, + help="Date at which the PubKey will become invalid; (format: 2030-02-02T02:02:02.000Z)." + ) + self.put_new_key_parser.add_argument("--update", "-u", metavar="OLD_PRIVKEY_HEX", type=str, default=None, + help="Old private key to sign the keypair update in HEX. Only needed if there already is a PubKey registered." + ) + self.put_new_key_parser.add_argument("--msgpack", "-m", metavar="MSGPACK", type=str, default=PUT_KEY_USE_MSGPACK_DEFAULT, + help="NOT IMPLEMENTED! Enables/Disables usage of MsgPack instead of Json. Can't be used for key updates (-u); true or false (default: %s)" % PUT_KEY_USE_MSGPACK_DEFAULT + ) + + # subparser + arguments for the delete_key operation + self.del_key_parser = subparsers.add_parser(DELETE_KEY_CMD, help="Delete a registered PubKey.") + self.del_key_parser.add_argument("prvkey", metavar="PRIVKEY_HEX", type=str, + help="ED25519 PrivKey in HEX corresponding to the PubKey to be deleted." + ) + + # subparser + arguments for the revoke_key operation + self.revoke_key_parser = subparsers.add_parser(REVOKE_KEY_CMD, help="Revoke a registered PubKey.") + self.revoke_key_parser.add_argument("prvkey", metavar="PRIVKEY_HEX", type=str, + help="ED25519 PrivKey in HEX corresponding to the PubKey to be revoked." + ) + + return + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # the env argument is needed for every command - get it + self.env = self.args.env + self.cmd_str = self.args.cmd + + if self.args.debug.lower() in ["1", "yes", "y", "true"]: + logger.level = logging.DEBUG + + logging.debug("Log level set to debug!") + + # format the url + self.base_url = UBIRCH_ID_SERVICE % self.env + + if self.env.lower() not in ["dev", "demo", "prod"]: + logger.error("Invalid value for env: \"%s\"!" % self.env) + + return False + + # load the UUID if needed + if self.cmd_str in [GET_DEV_KEYS_CMD, PUT_NEW_KEY_CMD]: + try: + self.uuid_str = self.args.uuid + self.uuid = uuid.UUID(hex=self.uuid_str) + except Exception as e: + logger.error("Invalid UUID: \"%s\"" % self.uuid_str) + logger.exception(e) + + return False + + # set the pubkey str and encode it as b64 if needed + if self.cmd_str in [GET_KEY_INFO_CMD]: + self.pubkey_str = self.args.pubkey + + try: + self.pubkey_b64 = base64.b64encode(binascii.unhexlify(self.pubkey_str)).decode("utf8").strip("\n") + except Exception as e: + logger.error("Error un-hexlifying the PubKey and encoding it in Base64!") + logger.exception(e) + + return False + + # load the privkey if needed + if self.cmd_str in [PUT_NEW_KEY_CMD, DELETE_KEY_CMD, REVOKE_KEY_CMD]: + self.prvkey_str = self.args.prvkey + + # the prvkey needs to be loaded to be usable + try: + self.prvkey = ed25519.SigningKey(binascii.unhexlify(self.prvkey_str)) + except Exception as e: + logger.error("Error loading the ED25519 private key!") + logger.exception(e) + + return False + + logger.info("PrivKey loaded!") + + # get the pubkey + self.pubkey_str = binascii.hexlify(self.prvkey.get_verifying_key().to_bytes()).decode("utf8") + + logger.info("PubKey extracted from the PrivKey: %s" % self.pubkey_str) + + # b64-encode the pubkey + try: + self.pubkey_b64 = base64.b64encode(binascii.unhexlify(self.pubkey_str)).decode("utf8").strip("\n") + except Exception as e: + logger.error("Error un-hexlifying the PubKey and encoding it in Base64!") + logger.exception(e) + + return False + + # load the old privkey if needed + if self.cmd_str in [PUT_NEW_KEY_CMD]: + # can be none; used for detecting whether a key-update should be done + # and also needed for the update itself + self.old_prvkey_str = self.args.update + + # the old prvkey needs to be loaded to be usable to sign the update + if self.old_prvkey_str != None: + try: + self.old_prvkey = ed25519.SigningKey(binascii.unhexlify(self.old_prvkey_str)) + except Exception as e: + logger.error("Error loading the old ED25519 private key!") + logger.exception(e) + + return False + + logger.info("Old PrivKey loaded!") + + # get the old pubkey from the privkey + self.old_pubkey_str = binascii.hexlify(self.old_prvkey.get_verifying_key().to_bytes()).decode("utf8") + + logger.info("Old PubKey extracted from old PrivKey: %s" % self.old_pubkey_str) + + # b64-encode the old pubkey + try: + self.old_pubkey_b64 = base64.b64encode(binascii.unhexlify(self.old_pubkey_str)).decode("utf8").strip("\n") + except Exception as e: + logger.error("Error un-hexlifying the old PubKey and encoding it in Base64!") + logger.exception(e) + + return False + + self.key_valid_not_before = self.args.validNotBefore + self.key_valid_not_after = self.args.validNotAfter + self.key_created_at = self.args.created + self.use_msgpack_str = self.args.msgpack + + # extract the bool for the use-msgpack flag + if self.use_msgpack_str.lower() in ["1", "yes", "y", "true"]: + self.use_msgpack = True + else: + self.use_msgpack = False + + return True + + def usage(self): + if self.cmd_str == PUT_NEW_KEY_CMD: + self.put_new_key_parser.print_help() + elif self.cmd_str == GET_KEY_INFO_CMD: + self.get_key_info_parser.print_help() + elif self.cmd_str == GET_DEV_KEYS_CMD: + self.get_dev_keys_parser.print_help() + elif self.cmd_str == "del_key": + self.del_key_parser.print_help() + elif self.cmd_str == REVOKE_KEY_CMD: + self.revoke_key_parser.print_help() + else: + logger.error("Unknown cmd \"%s\"!" % self.cmd_str) + + self.argparser.print_help() + + def handle_http_response(self, r : requests.Response) -> bool: + # check the reponse code + if r.status_code != 200: + logger.error("Received NOT-OK HTTP response code %d!" % r.status_code) + logger.error("Status message: %s" % r.content) + + return 1 + + logger.info("Success! (HTTP) 200)") + + # format the response json and print it + formatted = json.dumps(json.loads(r.content.decode("utf8")), indent=4) + + logger.info("HTTP response:\n" + formatted) + + return 0 + + # signs the pubkey from .pubkey_str with .prvkey and returns the signature in base64 + def sign_data_b64(self, data : bytes, use_old_priv=False) -> bytes: + # sign the pubkey (RAW, NOT B64) + try: + if use_old_priv == True: + signed = self.old_prvkey.sign(data) + else: + signed = self.prvkey.sign(data) + except Exception as e: + logger.error("Error de-hexlifying and signing the PubKey!") + logger.exception(e) + + return None + + # encode the signature in base64 + try: + signed_b64 = base64.b64encode(signed).decode("utf8").strip("\n") + except Exception as e: + logger.error("Error B64-encoding the PubKey signature!") + logger.exception(e) + + return None + + return signed_b64 + + def run_get_dev_keys(self): + url = self.base_url + (GET_DEVICE_KEYS_PATH % self.uuid_str) + + logger.info("Getting keys for %s from %s!" % (self.uuid_str, url)) + + # send the request + r = requests.get( + url=url, + headers={'Accept': 'application/json'} + ) + + # handle the reponse + return self.handle_http_response(r) + + def run_get_key_info(self): + url = self.base_url + (GET_KEY_INFO_PATH % self.pubkey_b64) + + logger.info("Getting information for the PubKey %s (B64) from %s" % (self.pubkey_b64, url)) + + # send the request + r = requests.get( + url=url, + headers={'Accept': 'application/json'} + ) + + # handle the response + return self.handle_http_response(r) + + def run_put_new_key_json(self): + url = self.base_url + + # check if this is a key update + if self.old_prvkey_str != None: + # format the innter message + inner_msg = PUT_PUBKEY_UPDATE_FMT_INNER % ( + "ECC_ED25519", self.key_created_at, self.uuid_str, self.pubkey_b64, self.pubkey_b64, + self.old_pubkey_b64, self.key_valid_not_after, self.key_valid_not_before + ) + + # sign the inner message with both privkeys + prevsig = self.sign_data_b64(bytes(inner_msg, "utf8"), use_old_priv=True) + sig = self.sign_data_b64(bytes(inner_msg, "utf8")) + + + print(inner_msg) + + # create the whole msg + msg = PUT_PUBKEY_UPDATE_FMT_OUTER % ( + inner_msg, prevsig, sig + ) + else: + # format the innter message + inner_msg = PUT_NEW_PUBKEY_FMT_INNER % ( + "ECC_ED25519", self.key_created_at, self.uuid_str, self.pubkey_b64, + self.pubkey_b64, self.key_valid_not_after, self.key_valid_not_before + ) + + # sign the inner message with the privkey + sig = self.sign_data_b64(bytes(inner_msg, "utf8")) + + # create the whole msg + msg = PUT_NEW_PUBKEY_FMT_OUTER % ( + inner_msg, sig + ) + + # get user confirmation to register the key + logger.info("Registering new PubKey %s (B64) at %s!" % (self.pubkey_b64, url)) + logger.debug("Data:\n" + msg) + + if input("Enter 'YES' to continue: ") != 'YES': + logger.error("Aborting!") + + return 0 + + # send the request + r = requests.post(url, data=msg) + + # handle the response + return self.handle_http_response(r) + + def run_put_new_key_msgpack(self): + logger.error("NOT IMPLEMENTED YET. SEE 'upp-creator.py' FOR AN IMPLEMENTATION OF THIS FUNCTIONALITY.") + + return 0 + + def run_put_new_key(self): + # call the subfunction depending on the use-msgpack flag + if self.use_msgpack == True: + return self.run_put_new_key_msgpack() + else: + return self.run_put_new_key_json() + + def run_del_key(self): + pubkey_sign_b64 = self.sign_data_b64(binascii.unhexlify(self.pubkey_str)) + + if pubkey_sign_b64 == None: + return 1 + + # format the message + msg = DEL_PUBKEY_FMT % (self.pubkey_b64, pubkey_sign_b64) + + # get the url + url = self.base_url + + # get user confirmation to delete the key + logger.info("Deleting PubKey %s (B64) at %s!" % (self.pubkey_b64, url)) + logger.debug("Data:\n" + msg) + + if input("Enter 'YES' to continue: ") != 'YES': + logger.error("Aborting!") + + return 0 + + # send the request + r = requests.delete(url, data=msg) + + # handle the response + return self.handle_http_response(r) + + def run_revoke_key(self): + pubkey_sign_b64 = self.sign_pubkey() + + if pubkey_sign_b64 == None: + return 1 + + # format the message + msg = REVOKE_PUBKEY_FMT % (self.pubkey_b64, pubkey_sign_b64) + + # get the url + url = self.base_url + REVOKE_KEY_PATH + + # get user confirmation to revoke the key + logger.info("Revoking PubKey %s (B64) at %s!" % (self.pubkey_b64, url)) + logger.debug("Data:\n" + msg) + + if input("Enter 'YES' to continue: ") != 'YES': + logger.error("Aborting!") + + return 0 + + # send the request + r = requests.delete(url, data=msg) + + # handle the response + return self.handle_http_response(r) + + return 0 + + def run(self): + # process all args + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.usage() + + return 1 + + # call the correct sub-run function + if self.cmd_str == PUT_NEW_KEY_CMD: + return self.run_put_new_key() + elif self.cmd_str == GET_KEY_INFO_CMD: + return self.run_get_key_info() + elif self.cmd_str == GET_DEV_KEYS_CMD: + return self.run_get_dev_keys() + elif self.cmd_str == DELETE_KEY_CMD: + return self.run_del_key() + elif self.cmd_str == REVOKE_KEY_CMD: + return self.run_revoke_key() + else: + logger.error("Unknown cmd \"%s\"!" % self.cmd_str) + + return 1 + +if __name__ == "__main__": + sys.exit(Main().run()) \ No newline at end of file From 3e1ee23bf6edec0fba9fdf40f770990548b80589 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Sun, 20 Feb 2022 12:29:43 +0100 Subject: [PATCH 55/68] update the readme --- examples/EXAMPLES.md | 123 +++++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 51 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 15e4563..0a7df7f 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -1,29 +1,31 @@ -# uBirch-Protocol-Python Examples +# uBirch-Protocol-Python Examples This file documents how to use the examples provided alongside the [uBirch-Protocol-Python](https://github.com/ubirch/ubirch-protocol-python). Those examples aim to provide an insight of how to use the [ubirch-protocol](https://pypi.org/project/ubirch-protocol/) python library, which is implemented in the `/ubirch/` directory in this repository. ## Table of contents -- [uBirch-Protocol-Python Examples](#ubirch-protocol-python-examples) - - [Table of contents](#table-of-contents) - - [From measurement to blockchain-anchored UPP](#from-measurement-to-blockchain-anchored-upp) - - [Setup](#setup) - - [Generating and managing a keypair](#generating-and-managing-a-keypair) - - [Registering a public key](#registering-a-public-key) - - [Gathering Data](#gathering-data) - - [Creating an UPP](#creating-an-upp) - - [Sending an UPP](#sending-an-upp) - - [Verifying an UPP](#verifying-an-upp) - - [Verifying an UPP chain](#verifying-an-upp-chain) - - [Examining an UPP](#examining-an-upp) - - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) - - [Verifying data](#verifying-data) - - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - - [Example uBirch client implementation](#example-ubirch-client-implementation) - - [Create a hash from an JSON object](#create-a-hash-from-an-json-object) - - [Test identity of the device](#test-identity-of-the-device) - - [Test the complete protocol](#test-the-complete-protocol) - - [Test the web of trust](#test-the-web-of-trust) - - [Verify ECDSA signed UPP](#verify-ecdsa-signed-upp) - - [Verify ED25519 signed UPP](#verify-ed25519-signed-upp) +- [Table of contents](#table-of-contents) +- [From measurement to blockchain-anchored UPP](#from-measurement-to-blockchain-anchored-upp) + - [Setup](#setup) + - [Generating and managing a keypair](#generating-and-managing-a-keypair) + - [Registering a public key](#registering-a-public-key) + - [Gathering Data](#gathering-data) + - [Creating an UPP](#creating-an-upp) + - [Sending an UPP](#sending-an-upp) + - [Verifying an UPP](#verifying-an-upp) + - [Verifying an UPP chain](#verifying-an-upp-chain) + - [Examining an UPP](#examining-an-upp) + - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) + - [Verifying data](#verifying-data) +- [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) +- [Example uBirch client implementation](#example-ubirch-client-implementation) +- [Create a hash from an JSON object](#create-a-hash-from-an-json-object) +- [Test identity of the device](#test-identity-of-the-device) +- [Test the complete protocol](#test-the-complete-protocol) +- [Test the web of trust](#test-the-web-of-trust) +- [Verify ECDSA signed UPP](#verify-ecdsa-signed-upp) +- [Verify ED25519 signed UPP](#verify-ed25519-signed-upp) +- [Managing Keys](#managing-keys) + - [Managing the local KeyStore](#managing-the-local-keystore) + - [Managing keys inside the uBirch Identity Service](#managing-keys-inside-the-ubirch-identity-service) ## From measurement to blockchain-anchored UPP The process needed to get an UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. @@ -68,33 +70,9 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` ### Generating and managing a keypair To create, or more precisely, to _sign_ an UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). -**NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. - -A keystore can be read out with the [`keystore-dumper.py`](keystore-dumper.py) script. +To read generated keys from the KeyStore, see [below](#managing-the-local-keystore). -``` -$ python keystore-dumper.py --help -usage: keystore-dumper.py [-h] [--show-sk SHOW_SIGNING_KEY] KEYSTORE KEYSTORE_PASS -``` -By default, only UUIDs und public keys (verifying keys) will be displayed. Displaying of private keys (signing keys) can be enabled by passing `-s true`. `KEYSTORE` is the path to the keystore file and `KEYSTORE_PASS` is the password needed to open/decrypt it. Below is an example with `-s` being set to `false`. -``` -====================================================================================================================================== -UUID: 292e2f0b-1d9e-407f-9b1a-bda5a9560797 - VK : 2dc6fdf373c7e13abfe1f51ff0fe45417ad713c5fb8df90b76c3077355c4b5e9 - SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ -====================================================================================================================================== -====================================================================================================================================== -UUID: 832e885c-be2a-44b1-ade6-c6c6dd718a3b - VK : 43536e9793ab5b205ba62e614d12104cbb6f906b2a44b79c1cd01e87a724d5b1 - SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ -====================================================================================================================================== -====================================================================================================================================== -UUID: 622e3933-8fdd-446f-b645-d3aa7ed8b638 - VK : cfd32f687366679d7ef73795bafa3bfca43421e092fbc2501f7f4bfe339bb15a - SK : ████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ -====================================================================================================================================== -``` -As you can see, the secret key is not printed and `█`s are used as placeholders. +**NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. ### Registering a public key To enable the uBirch backend to verify an UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: @@ -439,7 +417,7 @@ export UBIRCH_UUID= export UBIRCH_AUTH= export UBIRCH_ENV=[dev|demo|prod] ``` -It uses `test-identity.jsk` as a place to store/look for keypairs. The keystore-password can be read from the [script](test-identity.py) itself. +It uses `test-identity.jks` as a place to store/look for keypairs. The keystore-password can be read from the [script](test-identity.py) itself. ## Test the complete protocol The [`test-protocol.py`](test-protocol.py) script sends a couple of UPPs to uBirch Niomon and verifies the backend response. It reads all information it needs interactively from the terminal. Once entered, all device information (UUID, ENV, AUTH TOKEN) are stored in a file called `demo-device.ini`. Devices keys are stored in `demo-device.jks` and the keystore-password can be read from the [script](test-protocol.py) itself. If no keys for the given UUID are found, the script will generated a keypair and stores it in the keystore file. @@ -452,4 +430,47 @@ The [`test-protocol.py`](test-protocol.py) script sends a couple of UPPs to uBir The [`verify-ecdsa.py`](verify-ecdsa.py) script verifies a hard-coded UPP which was signed with an ECDSA signing key using a ECDSA verifying key. All the information are contained in the script. ## Verify ED25519 signed UPP -The [`verify-25519.py`](verify-25519.py) script verifies a hard-coded UPP which was signed with an ED25519 signing key using a ED25519 verifying key. All the information are contained in the script. This mode is normally used (in all other examples). \ No newline at end of file +The [`verify-25519.py`](verify-25519.py) script verifies a hard-coded UPP which was signed with an ED25519 signing key using a ED25519 verifying key. All the information are contained in the script. This mode is normally used (in all other examples). + +## Managing Keys +### Managing the local KeyStore +`keystore-tool.py` is a script to manipulate contents of JavaKeyStores (JKS) as they are used by other example scripts. It supports displaying Keypair, adding new ones and also deleting entries. +``` +$ python keystore-tool.py --help +usage: keystore-tool.py [-h] KEYSTORE KEYSTORE_PASS {get,put,del} ... +``` +Run `python keystore-tool.py a b get --help` to get a help message for the `get` operation. The first two arguments will be ignored in that case. `get` can be exchanges for `put` or `del` to get information about those operations respectively. One valid invocation of the script might look like this: +``` +$ python keystore-tool.py devices.jks keystore get -u 55425678-1234-bf80-30b4-dcbabf80abcd -s true +``` +It will search for an entry matching the given UUID (specified by `-u`) and print the corresponding KeyPair if found. The PrivateKey will also be shown (`-s true`). + +**Note that once an entry is deleted, it is gone. It is recommended to keep backups of KeyStores containing important keys.** + +### Managing keys inside the uBirch Identity Service +The `pubkey-util.py` script can be used to manually add, delete, revoke or update device keys at the uBirch Identity Service. In most cases this won't be necessary since the other scripts documented above are capable of registering a key, which is enough most of the time. In total, this script supports five operations: +``` +get_dev_keys - Get all PubKeys registered for a given device. This won't include revoked or deleted keys. +get_key_info - Get information about a specific key (basically all information provided when registering it). +put_new_key - Register a new key for a device that has no keys yet, or add a new one if it already has one. +delete_key - Removes a key so that it can't be used/won't be recognized by the backend anymore. +revoke_key - Revokes a key so that it can't be used anymore for sending new UPPs, but is still usable to verify old ones (...). +``` +In general a invocation of the `pubkey-util.py` script will look like this: +``` +$ python pubkey-util.py ENV OPERATION ...PARAMETERS... +``` +Each operation has an own set of sub-parammeters. To see more information about a specific operation run: +``` +$ python pubkey-util.py ENV OPERATION --help +``` +To see a general help message run: +``` +$ python pubkey-util.py --help +``` + +For some operations a date string in a specific format will be needed (a specific case of ISO8601); this command can be used to generate date strings in this format: +``` +$ TZ=UTC date "+%Y-%m-%dT%H:%M:%S.000Z" +2022-02-23T11:11:11.000Z +``` \ No newline at end of file From e6e6c0cb6a56ae6a998763a3cc2919715267b94a Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Thu, 7 Apr 2022 15:14:51 +0200 Subject: [PATCH 56/68] fix issues from review --- examples/EXAMPLES.md | 66 ++++++++++++++++---------------- examples/upp-anchoring-status.py | 10 ++--- examples/upp-creator.py | 4 +- examples/upp-sender.py | 10 ++--- examples/upp-verifier.py | 6 +-- tests/test_ubirch_protocol.py | 4 +- 6 files changed, 51 insertions(+), 49 deletions(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index 0a7df7f..e9cb19c 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -8,12 +8,12 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Generating and managing a keypair](#generating-and-managing-a-keypair) - [Registering a public key](#registering-a-public-key) - [Gathering Data](#gathering-data) - - [Creating an UPP](#creating-an-upp) - - [Sending an UPP](#sending-an-upp) - - [Verifying an UPP](#verifying-an-upp) - - [Verifying an UPP chain](#verifying-an-upp-chain) - - [Examining an UPP](#examining-an-upp) - - [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp) + - [Creating a UPP](#creating-a-upp) + - [Sending a UPP](#sending-a-upp) + - [Verifying a UPP](#verifying-a-upp) + - [Verifying a UPP chain](#verifying-a-upp-chain) + - [Examining a UPP](#examining-a-upp) + - [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) - [Verifying data](#verifying-data) - [Sending data to the Simple Data Service](#sending-data-to-the-simple-data-service) - [Example uBirch client implementation](#example-ubirch-client-implementation) @@ -28,7 +28,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Managing keys inside the uBirch Identity Service](#managing-keys-inside-the-ubirch-identity-service) ## From measurement to blockchain-anchored UPP -The process needed to get an UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. +The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. 1. [Setup](#setup) @@ -38,15 +38,15 @@ The process needed to get an UPP to be anchored in the blockchain can be cut dow 4. [Gathering Data](#gathering-data) -5. [Creating an UPP](#creating-an-upp) +5. [Creating a UPP](#creating-an-upp) -6. [Sending an UPP](#sending-an-upp) +6. [Sending a UPP](#sending-an-upp) -7. [Verifying an UPP](#verifying-an-upp) +7. [Verifying a UPP](#verifying-an-upp) -8. [Examining an UPP](#examining-an-upp) +8. [Examining a UPP](#examining-an-upp) -9. [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-a-upp) +9. [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-a-upp) 10. [Verifying data](#verifying-data) ### Setup @@ -68,19 +68,19 @@ The values used below are `f5ded8a3-d462-41c4-a8dc-af3fd072a217` for the UUID, ` `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` for the auth token. ### Generating and managing a keypair -To create, or more precisely, to _sign_ an UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). +To create, or more precisely, to _sign_ a UPP, a device will need a keypair. This keypair consist of a private key (_signing key_) and public key (_verifying key_). The signing key is used to sign UPPs and the verifying key can be used by the uBirch backend to check, if the signature is valid and belongs to the correct sender/signer. So, logically it doesn't matter who knows to verifying key, but the signing key must be kept secret all the time. In a real use case a device might store it in a TPM ([Trusted platform module](https://en.wikipedia.org/wiki/Trusted_Platform_Module)) or use other counter measures against attackers reading the key from the device. For this demo, keypairs will be stored in a [JKS Keystore](https://en.wikipedia.org/wiki/Java_KeyStore) using the [`pyjks`](https://pypi.org/project/pyjks/) library. Therefore, you will have to choose and remember a file path for that keystore and a password used to encrypt it. The process of actually generating the keypair is handled by the [upp-creator.py](upp-creator.py) script and explained [below](#registering-a-public-key). To read generated keys from the KeyStore, see [below](#managing-the-local-keystore). **NOTE** that losing access to the signing key, especially if it is already registered at the uBirch backend, will take away the ability to create and send any new UPPs from that device/UUID, since there is no way of creating a valid signature that would be accepted by the backend. ### Registering a public key -To enable the uBirch backend to verify an UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: +To enable the uBirch backend to verify a UPP, it needs to know the corresponding verifying key. Therefore, the device needs to send this key to the backend, before starting to send UPPs, which are supposed to be verified and anchored. Registering a verifying key is done by sending a special kind of UPP containing this key. This can be done by using two scripts: ``` upp-creator.py upp-sender.py ``` -Both of these scripts will be explained in more detail in [Creating an UPP](#creating-a-upp) and [Sending an UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: +Both of these scripts will be explained in more detail in [Creating a UPP](#creating-a-upp) and [Sending a UPP](#sending-a-upp). To generate a _Public Key Registration UPP_ this command can be used: ``` $ python upp-creator.py -t 1 --ks devices.jks --kspwd keystore --keyreg true --output keyreg_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 none @@ -125,17 +125,17 @@ serialized = json.dumps(message, separators=(',', ':'), sort_keys=True, ensure_a This will create a string from the above examplary JSON object: `{"H":35.8,"S":"OK","T":11.2,"ts":1625163338}` -### Creating an UPP -After gathering some measurement data an UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). +### Creating a UPP +After gathering some measurement data a UPP can be created. The UPP won't contain the actual measurement data, but a hash of it. The example script to create UPPs is [`upp-creator.py`](upp-creator.py). ``` $ python3 upp-creator.py --help usage: upp-creator.py [-h] [--version VERISON] [--type TYPE] [--ks KS] [--kspwd KSPWD] [--keyreg KEYREG] [--hash HASH] [--isjson ISJSON] [--output OUTPUT] [--nostdout nostdout] UUID DATA -Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue an UPP chain using this tool. Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. +Note that, when using chained UPPs (--version 0x23), this tool will try to load/save signatures from/to .sig, where UUID will be replaced with the actual UUID. Make sure that the UUID.sig file is in your current working directory if you try to continue a UPP chain using this tool. Also beware that you will only be able to access the contents of a keystore when you use the same password you used when creating it. Otherwise all contents are lost. When --hash off is set, contents of the DATA argument will be copied into the payload field of the UPP. Normally used for special messages (e.g. key registration). For more information on possible values for --type and --version see https://github.com/ubirch/ubirch-protocol. ``` The script allows multiple modes of operation, which can be set through different command line arguments. Some of those directly set fields in the resulting UPP. Please consult the [uBirch Protocol Readme](https://github.com/ubirch/ubirch-protocol#basic-message-format) for further information on those fields and their possible values. -- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, an UPP with a signature and an UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. +- `--version/-v` This flag sets the version field of the UPP. The version field actually consists of two sub-fields. The higher four bits set the actual version (`1` or `2`) and the "mode". The higher four bits will be set to two(`0010`) in almost all use cases. The mode can either be a simple UPP without a signature, a UPP with a signature and a UPP with a signature + the signature of the previous UPP embedded into it. The latter would be called a_Chained UPP_. Unsigned UPPs (`-v 0x21`) are not implemented. Signed UPPs have `-v 0x22` and chained ones `-v 0x23`. - `--type/-t` This flag sets the type field of the UPP. It is used to indicate what the UPP contains/should be used for. It can be set to `0x00` in most cases. One of the cases where a specific value is required, is a Key Registration Messages, as described in [Registering a Public Key](#registering-a-public-key). - `--k/-k` The path to the keystore that contains the keypair for the device or should be used to store a newly generated keypair. If the keystore, pointed to by this parameter, doesn't exist, the script will simply create it. - `--kspwd/-p` The password to decrypt/encrypt the keystore. You must remember this, or you will lose access to the keystore and all its contents. @@ -164,7 +164,7 @@ $ python3 upp-creator.py --version 0x23 --isjson true --output upp.bin --hash sh ``` Keep in mind that if you use chained UPPs (`--version 0x23`) you should anchor each UPP, or the signature chain will be broken. This won't cause any errors, but the advantage of chaining UPPs and thereby knowing the correct order of them, will get lost. -### Sending an UPP +### Sending a UPP After creating the UPP, it can be sent to the uBirch backend where it will be verified and anchored in the blockchain. The ubirch backend will use the registered public/verifying key to check the signature. The [`upp-sender.py`](upp-sender.py) script can be used for that. ``` $ python3 upp-sender.py --help @@ -176,7 +176,7 @@ For this script the parameters are: - `--output/-o` Normally the uBirch backend will respond to the UPP with another UPP. This parameter sets the location to write that response-UPP to. - `UUID` The UUID of the device that generated the UPP as a hex-string. - `AUTH` The auth token for the device on the specified stage as a hex-string. -Continuing from the example above (see [Creating an UPP](#creating-a-upp)), the send-command might look like this: +Continuing from the example above (see [Creating a UPP](#creating-a-upp)), the send-command might look like this: ``` $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5ded8a3-d462-41c4-a8dc-af3fd072a217 xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 2021-07-02 15:21:36,966 root read_upp() INFO Reading the input UPP for "f5ded8a3-d462-41c4-a8dc-af3fd072a217" from "upp.bin" @@ -185,16 +185,16 @@ $ python upp-sender.py --env demo --input upp.bin --output response_upp.bin f5de 2021-07-02 15:21:37,723 root store_response_upp() INFO The response UPP has been written to "response_upp.bin" ``` -### Verifying an UPP +### Verifying a UPP To make sure, that the response UPP actually was sent by the uBirch backend, its signature can be checked. The example script for that is [`upp-verifier.py`](upp-verifier.py). It knows the UUID and verifying/public key for each uBirch Niomon stage end checks, if the signature of the response UPP is valid. ``` $ python3 upp-verifier.py --help usage: upp-verifier.py [-h] [--verifying-key VK] [--verifying-key-uuid UUID] [--input INPUT] -Note that, when trying to verify an UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. +Note that, when trying to verify a UPP, sent by the uBirch backend (Niomon), a verifying key doesn't have to be provided via the -k option. Instead, this script will try to pick the correct stage key based on the UUID which is contained in the UPP, identifying the creator. If the UUID doesn't match any Niomon stage and no key was specified using -k, an error message will be printed. ``` -- `--verifying-key/-k` If not trying to verify an UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. +- `--verifying-key/-k` If not trying to verify a UPP coming from uBirch Niomon but from another source, the verifying key for that source needs to be provided. This parameter expects the key as a hex-string like `b12a906051f102881bbb487ee8264aa05d8d0fcc51218f2a47f562ceb9b0d068`. - `--verifying-key-uuid/-u` The UUID for the verifying key from `--verifying-key`. This parameter will be ignored when `--verifying-key` is not set. Not setting this parameter when `--verifying-key` is set will cause an error. - `--input/-i` The file path to read the UPP from. @@ -206,11 +206,11 @@ $ python3 upp-verifier.py --input response_upp.bin 2021-07-02 15:43:36,275 root verify_upp() INFO Signature verified - the UPP is valid! ``` -### Verifying an UPP chain +### Verifying a UPP chain When working with `chained UPPs` it can be useful to check whether the chain is in order and valid. For this task, the [`upp-chain-checker.py`](upp-chain-checker.py) can be used. It reads in a list of UPPs, checks the signature of each UPP and compares the `prevsig` field with the `signature` of the last UPP. -If at any point something doesn't match up, it will print an error message alonge with the number of the +If at any point something doesn't match up, it will print an error message along with the number of the UPP at which the chain broke/something went wrong. The UPP list can either be read directly from a file which contains them in binary or hex-encoded, separated by newlines, or from a JSON file which contains a list of hex-encoded UPPs. @@ -252,8 +252,8 @@ $ echo -n -e "9623c410ee8c4cfe9b3a43e29e9f8875cb02cec3c44025152e18f42352a7fba90b The UPPs are piped as hex-encoded strings separated by newlines (`\n`) to the script which has the input file path set to `/dev/stdin`. -### Examining an UPP -To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of an UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: +### Examining a UPP +To examine the contents (Version Field, Type Field, UUID, Signature, Payload, Previous Signature) of a UPP, the [`upp-unpacker.py`](upp-unpacker.py) script can be used like this: ``` $ python3 upp-unpacker.py response_upp.bin - Version: 0x23 @@ -268,13 +268,13 @@ $ python3 upp-unpacker.py response_upp.bin ``` _The UUID in this response UPP doesn't match the one from examples above because the UPP was sent from Niomon-Dev._ -### Checking the anchoring status of an UPP -uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if an UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). +### Checking the anchoring status of a UPP +uBirch Niomon accepting the UPP doesn't mean that it is anchored yet. This process takes place in certain intervals, so one might have to wait a short while. The script to check if a UPP was already anchored is [`upp-anchoring-status.py`](upp-anchoring-status.py). ``` $ python3 upp-anchoring-status.py -h usage: upp-anchoring-status.py [-h] [--ishash ISHASH] [--env ENV] [--ishex ISHEX] INPUT ``` -- `--ishash/-i` A boolean specifying whether the input data is a payload hash or an UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. +- `--ishash/-i` A boolean specifying whether the input data is a payload hash or a UPP. The payload hash is what is actually used to look up anchoring information about the UPP. This script can either extract it from a given UPP or just use the hash directly if provided. If directly provided, it must be base64 encoded (see the last example of this sub-section). `true` or `false`. - `--env/-e` The stage to check on. Should be the one the UPP was sent to. `prod`, `demo` or `dev`. - `--ishex/-x` A boolean which controls how the input UPP data is interpreted. By default, the data will be interpreted as normale binary data. When this flag is set to `true`, it will be considered @@ -317,7 +317,7 @@ Prev. UPP: "liPEEAi4JX1SUEaGhzAnqUf6pn3EQOeEqF4lo+xT9RF2ygDx9+anv14fykUolJ9gmKuT ``` ### Verifying data -In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of an UPP](#checking-the-anchoring-status-of-an-upp). +In a real use case, not the UPP, but rather the original data itself has to be verified. The original data can be hashed, and the hash can be looked up by the [`data-verifier.py`](data-verifier.py) script. It has similar behaviour to the [`upp-anchoring-status.py`](upp-anchoring-status.py) script, see [Checking the anchoring status of a UPP](#checking-the-anchoring-status-of-an-upp). ```txt $ python data-verifier.py --help usage: data-verifier.py [-h] [--ispath ISPATH] [--env ENV] [--isjson ISJSON] [--hash HASH] [--no-send NOSEND] [--ishl ISHASHLINK] INPUT @@ -383,7 +383,7 @@ Note that the input data should follow this pattern: {"timestamp": TIMESTAMP, "u ``` ## Example uBirch client implementation -[`example-client.py`](example-client.py) implements a full example uBirch client. It generates a keypair if needed, registers it at the uBirch backend if it doesn't know it yet, creates and sends an UPP and handles/verfies the response from the uBirch backend. The used message format looks like this: +[`example-client.py`](example-client.py) implements a full example uBirch client. It generates a keypair if needed, registers it at the uBirch backend if it doesn't know it yet, creates and sends a UPP and handles/verfies the response from the uBirch backend. The used message format looks like this: ``` { "id": "UUID", diff --git a/examples/upp-anchoring-status.py b/examples/upp-anchoring-status.py index 9da7d0e..fe5069e 100644 --- a/examples/upp-anchoring-status.py +++ b/examples/upp-anchoring-status.py @@ -152,17 +152,17 @@ def get_status(self) -> bool: if r.status_code == 200: logger.info("The UPP is known to the uBirch backend! (code: %d)" % r.status_code) - jobj = json.loads(r.content) + content_json = json.loads(r.content) - logger.info("Curr. UPP: \"%s\"" % jobj.get("upp", "-- no curr. upp information --")) - logger.info("Prev. UPP: \"%s\"" % jobj.get("prev", "-- no prev. upp information --")) + logger.info("Curr. UPP: \"%s\"" % content_json.get("upp", "-- no curr. upp information --")) + logger.info("Prev. UPP: \"%s\"" % content_json.get("prev", "-- no prev. upp information --")) - if jobj.get("anchors") in [None, []]: + if content_json.get("anchors") in [None, []]: logger.info("The UPP has NOT been anchored into any blockchains yet! Please retry later") else: logger.info("The UPP has been fully anchored!") - logger.info(jobj.get("anchors")) + logger.info(content_json.get("anchors")) elif r.status_code == 404: logger.info("The UPP is NOT known to the uBirch backend! (code: %d)" % r.status_code) except Exception as e: diff --git a/examples/upp-creator.py b/examples/upp-creator.py index 2654f7e..d42adff 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -378,8 +378,8 @@ def show_store_upp(self) -> bool: logger.info("UPP [hex]: \"%s\"" % binascii.hexlify(self.upp).decode()) # try to write the upp - with open(self.output, "wb") as fd: - fd.write(self.upp) + with open(self.output, "wb") as file: + file.write(self.upp) if self.nostdout == False: logger.info("UPP written to \"%s\"" % self.output) diff --git a/examples/upp-sender.py b/examples/upp-sender.py index 8d34134..31615ec 100644 --- a/examples/upp-sender.py +++ b/examples/upp-sender.py @@ -96,8 +96,8 @@ def read_upp(self) -> bool: try: logger.info("Reading the input UPP for \"%s\" from \"%s\"" % (self.uuid_str, self.input)) - with open(self.input, "rb") as fd: - self.upp = fd.read() + with open(self.input, "rb") as file: + self.upp = file.read() except Exception as e: logger.exception(e) @@ -180,8 +180,8 @@ def send_upp(self) -> bool: def store_response_upp(self) -> bool: try: - with open(self.output, "wb") as fd: - fd.write(self.response_upp) + with open(self.output, "wb") as file: + file.write(self.response_upp) logger.info("The response UPP has been written to \"%s\"" % self.output) except Exception as e: @@ -245,4 +245,4 @@ def run(self): if __name__ == "__main__": - sys.exit(Main().run()) \ No newline at end of file + sys.exit(Main().run()) diff --git a/examples/upp-verifier.py b/examples/upp-verifier.py index adccb51..981b024 100644 --- a/examples/upp-verifier.py +++ b/examples/upp-verifier.py @@ -140,8 +140,8 @@ def read_upp(self) -> bool: try: logger.info("Reading the input UPP from \"%s\"" % self.input) - with open(self.input, "rb") as fd: - self.upp = fd.read() + with open(self.input, "rb") as file: + self.upp = file.read() # check whether hex decoding is needed if self.ishex == True: @@ -281,4 +281,4 @@ def run(self): if __name__ == "__main__": - sys.exit(Main().run()) \ No newline at end of file + sys.exit(Main().run()) diff --git a/tests/test_ubirch_protocol.py b/tests/test_ubirch_protocol.py index 31341ea..d59fa97 100644 --- a/tests/test_ubirch_protocol.py +++ b/tests/test_ubirch_protocol.py @@ -225,6 +225,8 @@ def test_reset_saved_signatures(self): p.reset_signature(TEST_UUID) self.assertEqual({}, p.get_saved_signatures()) + #disable the legacy trackle message test + """ def test_unpack_legacy_trackle_message(self): loc = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) @@ -250,7 +252,7 @@ def _verify(self, uuid: UUID, message: bytes, signature: bytes) -> bytes: self.assertEqual(3, payload[2]) self.assertEqual(736, len(payload[3])) self.assertEqual(3519, payload[3].get(1533846771)) - self.assertEqual(3914, payload[3].get(1537214378)) + self.assertEqual(3914, payload[3].get(1537214378))""" def test_unpack_register_v1(self): loc = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) From 7207e825735f1433fd98f17da10fabf077dd1475 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Thu, 7 Apr 2022 16:37:15 +0200 Subject: [PATCH 57/68] [bugfix] used non existent function --- examples/pubkey-util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pubkey-util.py b/examples/pubkey-util.py index f329383..4b5f257 100644 --- a/examples/pubkey-util.py +++ b/examples/pubkey-util.py @@ -479,7 +479,7 @@ def run_del_key(self): return self.handle_http_response(r) def run_revoke_key(self): - pubkey_sign_b64 = self.sign_pubkey() + pubkey_sign_b64 = self.sign_data_b64(binascii.unhexlify(self.pubkey_str)) if pubkey_sign_b64 == None: return 1 From d0dc0ca0963d8c9a56a3011c4843f23e724b5abe Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Thu, 7 Apr 2022 17:51:49 +0200 Subject: [PATCH 58/68] replace demo key with dev key --- examples/example-client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example-client.py b/examples/example-client.py index e66b18c..66874a0 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -17,7 +17,7 @@ DEFAULT_UBIRCH_ENV = "prod" UBIRCH_PUBKEYS = { "dev": VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding="hex"), - "demo": VerifyingKey("39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", encoding="hex"), + "demo": VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding="hex"), "prod": VerifyingKey("ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690", encoding="hex") } UBIRCH_UUIDS = { From d0bfaf8e0882f68812508dca81eae0fd5b50f8eb Mon Sep 17 00:00:00 2001 From: wowa Date: Thu, 7 Apr 2022 18:13:59 +0200 Subject: [PATCH 59/68] added NOTE for environment --- examples/example-client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/example-client.py b/examples/example-client.py index 66874a0..1cc15fe 100644 --- a/examples/example-client.py +++ b/examples/example-client.py @@ -16,12 +16,12 @@ DEFAULT_UBIRCH_ENV = "prod" UBIRCH_PUBKEYS = { - "dev": VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding="hex"), + "dev": VerifyingKey("39ff77632b034d0eba6d219c2ff192e9f24916c9a02672acb49fd05118aad251", encoding="hex"), # NOTE: this environment is not reliable "demo": VerifyingKey("a2403b92bc9add365b3cd12ff120d020647f84ea6983f98bc4c87e0f4be8cd66", encoding="hex"), "prod": VerifyingKey("ef8048ad06c0285af0177009381830c46cec025d01d86085e75a4f0041c2e690", encoding="hex") } UBIRCH_UUIDS = { - "dev": UUID(hex="9d3c78ff-22f3-4441-a5d1-85c636d486ff"), + "dev": UUID(hex="9d3c78ff-22f3-4441-a5d1-85c636d486ff"), # NOTE: this environment is not reliable "demo": UUID(hex="07104235-1892-4020-9042-00003c94b60b"), "prod": UUID(hex="10b2e1a4-56b3-4fff-9ada-cc8c20f93016") } From 2d67dedc87197f647f6146266b12bd88729923b9 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 13:31:06 +0200 Subject: [PATCH 60/68] add ecdsa support to the keystore --- ubirch/ubirch_ks.py | 162 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 139 insertions(+), 23 deletions(-) diff --git a/ubirch/ubirch_ks.py b/ubirch/ubirch_ks.py index 83341ab..3e6e123 100644 --- a/ubirch/ubirch_ks.py +++ b/ubirch/ubirch_ks.py @@ -15,30 +15,39 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import hashlib from datetime import datetime, timedelta from logging import getLogger from os import urandom from uuid import UUID +import base64 +import ecdsa import ed25519 -from ed25519 import SigningKey, VerifyingKey from jks import jks, AlgorithmIdentifier, rfc5208, TrustedCertEntry from pyasn1.codec.ber import encoder logger = getLogger(__name__) -ECC_ENCRYPTION_OID = (1, 2, 1, 3, 101, 112) +EDDSA_OID = (1, 2, 1, 3, 101, 112) +ECDSA_OID = (1, 2, 840, 10045, 4, 3, 2) class ED25519Certificate(TrustedCertEntry): - def __init__(self, alias: str, verifying_key: VerifyingKey, **kwargs): + def __init__(self, alias: str, verifying_key: ed25519.VerifyingKey, **kwargs): super().__init__(**kwargs) self.alias = alias self.cert = verifying_key.to_bytes() self.timestamp = int(datetime.utcnow().timestamp()) +class ECDSACertificate(TrustedCertEntry): + + def __init__(self, alias: str, verifying_key: ecdsa.VerifyingKey, **kwargs): + super().__init__(**kwargs) + self.alias = alias + self.cert = verifying_key.to_string() + self.timestamp = int(datetime.utcnow().timestamp()) class KeyStore(object): """ @@ -60,24 +69,25 @@ def _load_keys(self) -> None: logger.warning("creating new key store: {}".format(self._ks_file)) self._ks = jks.KeyStore.new("jks", []) - def insert_ed25519_signing_key(self, uuid: UUID, sk: SigningKey): + def insert_ed25519_signing_key(self, uuid: UUID, sk: ed25519.SigningKey): """Store an existing ED25519 signing key in the key store.""" # encode the ED25519 private key as PKCS#8 private_key_info = rfc5208.PrivateKeyInfo() private_key_info.setComponentByName('version', 'v1') a = AlgorithmIdentifier() - a.setComponentByName('algorithm', ECC_ENCRYPTION_OID) + a.setComponentByName('algorithm', EDDSA_OID) private_key_info.setComponentByName('privateKeyAlgorithm', a) private_key_info.setComponentByName('privateKey', sk.to_bytes()) pkey_pkcs8 = encoder.encode(private_key_info) pke = jks.PrivateKeyEntry.new(alias=str(uuid.hex), certs=[], key=pkey_pkcs8) self._ks.entries['pke_' + uuid.hex] = pke - def insert_ed25519_verifying_key(self, uuid: UUID, vk: VerifyingKey): + def insert_ed25519_verifying_key(self, uuid: UUID, vk: ed25519.VerifyingKey): """Store an existing ED25519 verifying key in the key store.""" self._ks.entries[uuid.hex] = ED25519Certificate(uuid.hex, vk) - def insert_ed25519_keypair(self, uuid: UUID, vk: VerifyingKey, sk: SigningKey) -> (VerifyingKey, SigningKey): + def insert_ed25519_keypair(self, uuid: UUID, vk: ed25519.VerifyingKey, sk: ed25519.SigningKey) -> ( + ed25519.VerifyingKey, ed25519.SigningKey): """Store an existing ED25519 key pair in the key store.""" if uuid.hex in self._ks.entries or uuid.hex in self._ks.certs: raise Exception("uuid '{}' already exists in keystore".format(uuid.hex)) @@ -88,40 +98,146 @@ def insert_ed25519_keypair(self, uuid: UUID, vk: VerifyingKey, sk: SigningKey) - logger.info("inserted new key pair for {}: {}".format(uuid.hex, bytes.decode(vk.to_ascii(encoding='hex')))) return vk, sk - def create_ed25519_keypair(self, uuid: UUID) -> (VerifyingKey, SigningKey): + def create_ed25519_keypair(self, uuid: UUID) -> (ed25519.VerifyingKey, ed25519.SigningKey): """Create a new ED25519 key pair and store in key store.""" sk, vk = ed25519.create_keypair(entropy=urandom) return self.insert_ed25519_keypair(uuid, vk, sk) + def insert_ecdsa_signing_key(self, uuid, sk: ecdsa.SigningKey): + """Insert an existing ECDSA signing key.""" + # encode the ECDSA private key as PKCS#8 + private_key_info = rfc5208.PrivateKeyInfo() + private_key_info.setComponentByName('version', 'v1') + a = AlgorithmIdentifier() + a.setComponentByName('algorithm', ECDSA_OID) + private_key_info.setComponentByName('privateKeyAlgorithm', a) + private_key_info.setComponentByName('privateKey', sk.to_string()) + pkey_pkcs8 = encoder.encode(private_key_info) + pke = jks.PrivateKeyEntry.new(alias=str(uuid.hex), certs=[], key=pkey_pkcs8) + self._ks.entries['pke_' + uuid.hex] = pke + + def insert_ecdsa_verifying_key(self, uuid, vk: ecdsa.VerifyingKey): + # store verifying key in certificate store + # ecdsa VKs are marked with a "_ecd" suffix + self._ks.entries[uuid.hex + '_ecd'] = ECDSACertificate(uuid.hex, vk) + + def insert_ecdsa_keypair(self, uuid: UUID, vk: ecdsa.VerifyingKey, sk: ecdsa.SigningKey) -> (ecdsa.VerifyingKey, ecdsa.SigningKey): + """Insert an existing ECDSA key pair into the key store.""" + if uuid.hex in self._ks.entries or uuid.hex in self._ks.certs: + raise Exception("uuid '{}' already exists in keystore".format(uuid.hex)) + + self.insert_ecdsa_verifying_key(uuid, vk) + self.insert_ecdsa_signing_key(uuid, sk) + self._ks.save(self._ks_file, self._ks_password) + #logger.info("inserted new key pair for {}: {}".format(uuid.hex, vk.to_string().decode())) + return (vk, sk) + + def create_ecdsa_keypair(self, uuid: UUID, curve: ecdsa.curves.Curve = ecdsa.NIST256p, hashfunc=hashlib.sha256) -> (ecdsa.VerifyingKey, ecdsa.SigningKey): + """Create new ECDSA key pair and store in key store""" + + sk = ecdsa.SigningKey.generate(curve=curve, entropy=urandom, hashfunc=hashfunc) + vk = sk.get_verifying_key() + return self.insert_ecdsa_keypair(uuid, vk, sk) + def exists_signing_key(self, uuid: UUID): """Check whether this UUID has a signing key in the key store.""" return 'pke_' + uuid.hex in self._ks.private_keys def exists_verifying_key(self, uuid: UUID): """Check whether this UUID has a verifying key in the key store.""" - return uuid.hex in self._ks.certs + return uuid.hex in self._ks.certs or (uuid.hex + '_ecd') in self._ks.certs - def find_signing_key(self, uuid: UUID) -> SigningKey: + def find_signing_key(self, uuid: UUID) -> ed25519.SigningKey or ecdsa.SigningKey: """Find the signing key for this UUID.""" - sk = self._ks.private_keys['pke_' + uuid.hex] - return SigningKey(sk.pkey) - - def find_verifying_key(self, uuid: UUID) -> VerifyingKey: + # try to find a matching sk for the uuid + try: + sk : PrivateKeyEntry = self._ks.private_keys['pke_' + uuid.hex] + except KeyError as e: + # there is no sk for the given uuid + return None + + # check whether the entry is encrypted + if sk.is_decrypted() == False: + sk.decrypt(self._ks_password) + + # check the _OID to identify the key type + if sk._algorithm_oid == EDDSA_OID: + return ed25519.SigningKey(sk.pkey) + elif sk._algorithm_oid == ECDSA_OID: + # ==================================== IMPORTANT ==================================== + # The used curve as well as the used hash function have to be explicitly set here + # to match the ones used in create_ecdsa_keypair(), otherwise the ._from_string() + # function will throw exceptions because of unexpected/wrong keystr lengths (...) + # =================================================================================== + return ecdsa.SigningKey.from_string(sk.pkey, curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) + else: + raise Exception("stored key with unknown algorithm OID: '{}'".format(sk._algorithm_oid)) + + def _find_cert(self, uuid: UUID) -> ECDSACertificate or ED25519Certificate: + """ Find the stored cert for uuid """ + cert = None + + if self.exists_verifying_key(uuid) == True: + # try to get the edd key first + try: + cert = ED25519Certificate( + self._ks.certs[uuid.hex].alias, + + # the ED25519Certificate requires an ed25519.VerifyingKey + ed25519.VerifyingKey(self._ks.certs[uuid.hex].cert) + ) + except KeyError: + pass + + # no edd key found, try to get ecd + try: + cert = ECDSACertificate( + self._ks.certs[uuid.hex + '_ecd'].alias, + + # the ECDSACertifcate requires an ecdsa.VerifyingKey + ecdsa.VerifyingKey.from_string( + self._ks.certs[uuid.hex + '_ecd'].cert, + hashfunc=hashlib.sha256, curve=ecdsa.NIST256p + ) + ) + except KeyError: + pass + + return cert + + def find_verifying_key(self, uuid: UUID) -> ed25519.VerifyingKey or ecdsa.VerifyingKey: """Find the verifying key for this UUID.""" - cert = self._ks.certs[uuid.hex] - return VerifyingKey(cert.cert) + cert = self._find_cert(uuid) + + if type(cert) == ED25519Certificate: + return ed25519.VerifyingKey(cert.cert) + elif type(cert) == ECDSACertificate: + return ecdsa.VerifyingKey.from_string(cert.cert, curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) - def get_certificate(self, uuid: UUID) -> dict or None: + return None + + def get_certificate(self, uuid: UUID, validityInDays : int = 3650) -> dict or None: """Get the public key info for key registration""" - if uuid.hex not in self._ks.certs: + # try to find the cert + cert = self._find_cert(uuid) + + if cert == None: return None - cert = self._ks.certs[uuid.hex] - vk = VerifyingKey(cert.cert) + # set the timestamps (validity = +10 years) + # TODO set propper validity timestamp created = datetime.fromtimestamp(cert.timestamp) not_before = datetime.fromtimestamp(cert.timestamp) - # TODO fix handling of key validity - not_after = created + timedelta(days=365) + not_after = created + timedelta(days=validityInDays) + + # set the alogrithm + if type(cert) == ED25519Certificate: + algo = 'ECC_ED25519' + elif type(cert) == ECDSACertificate: + raise Exception("Certificate generation currently not supported for ECDSA keys!") + else: + raise Exception("Unexpected certificate class %s" % str(vk.__class__)) + return { "algorithm": 'ECC_ED25519', "created": int(created.timestamp()), From 8b6da7598ef7dab56f88fa24af5df67b1e8e6d45 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 13:41:42 +0200 Subject: [PATCH 61/68] add ecdsa key support to the upp generator example --- examples/upp-creator.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index d42adff..e40f251 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -19,7 +19,8 @@ DEFAULT_ISJSON = "False" DEFAULT_OUTPUT = "upp.bin" DEFAULT_NOSTDOUT = "False" -DEFAULT_ISHL = "False" +DEFAULT_ISHL = "False" +DEFAULT_ECDSA = "False" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) logger = logging.getLogger() @@ -78,6 +79,8 @@ def __init__(self): self.ishl : bool = None self.ishl_str : str = None self.ishash : bool = False + self.ecdsa_str : str = None + self.ecdsa : bool = None self.hasher : object = None self.keystore : ubirch.KeyStore = None @@ -137,6 +140,9 @@ def setup_argparse(self): self.argparser.add_argument("--ishl", "-l", metavar="ISHASHLINK", type=str, default=DEFAULT_ISHL, help="implied --isjson to be true; if set to true, the script will look for a hashlink list in the json object and use it to decide which fields to hash; true or false (default: %s)" % DEFAULT_ISHL ) + self.argparser.add_argument("--ecdsa", "-c", metavar="ECDSA", type=str, default=DEFAULT_ECDSA, + help="if set to true, the script will generate a ECDSA key (NIST256p, SHA256) instead of an ED25519 key in case no key was found for the UUID in the given keystore (default: %s)" % DEFAULT_ECDSA + ) def process_args(self) -> bool: # parse cli arguments (exists on err) @@ -155,6 +161,7 @@ def process_args(self) -> bool: self.output = self.args.output self.nostdout_str = self.args.nostdout self.ishl_str = self.args.ishl + self.ecdsa_str = self.args.ecdsa # get the keyreg value if self.keyreg_str.lower() in ["1", "yes", "y", "true"]: @@ -219,10 +226,16 @@ def process_args(self) -> bool: # check if ishl is true if (self.ishl == True and self.isjson == False): - logger.warning("Overwriting '--isjson false' because --ishl is true") + logger.warning("Overwriting '--isjson false' because '--ishl' is 'true'") self.isjson = True + # get the bool for ishl + if self.ecdsa_str.lower() in ["1", "yes", "y", "true"]: + self.ecdsa = True + else: + self.ecdsa = False + # success return True @@ -235,11 +248,26 @@ def init_keystore(self) -> bool: if self.nostdout == False: logger.info("No keys found for \"%s\" in \"%s\" - generating a keypair" % (self.uuid_str, self.keystore_path)) - self.keystore.create_ed25519_keypair(self.uuid) + if self.ecdsa == True: + logger.info("Generating a ECDSA keypair instead of ED25519!") + + if self.ecdsa == True: + self.keystore.create_ecdsa_keypair(self.uuid) + else: + self.keystore.create_ed25519_keypair(self.uuid) if self.nostdout == False: - logger.info("Public/Verifying key for \"%s\" [base64]: \"%s\"" % - (self.uuid_str, binascii.b2a_base64(self.keystore.find_verifying_key(self.uuid).to_bytes(), newline=False).decode())) + vk = self.keystore.find_verifying_key(self.uuid) + + if type(vk) == ubirch.ubirch_ks.ecdsa.VerifyingKey: + vk_b = vk.to_string() + k_t = "ECDSA" + else: + vk_b = vk.to_bytes() + k_t = "ED25519" + + logger.info("Public/Verifying key for \"%s\" [%s, base64]: \"%s\"" % + (self.uuid_str, k_t, binascii.b2a_base64(vk_b, newline=False).decode())) except Exception as e: logger.exception(e) From 31cbc69bf93a56f9c8f7a37d8ad6bcc25551ef3b Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 16:56:20 +0200 Subject: [PATCH 62/68] update the keystore tool to work with ecdsa --- examples/keystore-tool.py | 66 ++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/examples/keystore-tool.py b/examples/keystore-tool.py index 8b911c3..39a322c 100644 --- a/examples/keystore-tool.py +++ b/examples/keystore-tool.py @@ -5,11 +5,14 @@ import binascii import uuid import ed25519 +import ecdsa +import hashlib import ubirch DEFAULT_SHOW_SECRET = "False" +DEFAULT_ECDSA = "False" COMMAND_GET = "get" COMMAND_PUT = "put" COMMAND_DEL = "del" @@ -39,9 +42,11 @@ def __init__(self): # for put self.pubkey_str : str = None - self.pubkey : ed25519.VerifyingKey = None + self.pubkey : ed25519.VerifyingKey or ecdsa.VerifyingKey = None self.prvkey_str : str = None - self.prvkey : ed25519.SigningKey = None + self.prvkey : ed25519.SigningKey or ecdsa.SigningKey = None + self.ecdsa_str : str = None + self.ecdsa : bool = None self.keystore : ubirch.KeyStore = None @@ -88,6 +93,9 @@ def setup_argparse(self): self.put_argparser.add_argument("privkey", metavar="PRIVKEY", type=str, help="The HEX-encoded ED25519 PrivKey; e.g.: 9c7c43e122ae51e08a86e9bb89fe340bd4c7bd6665bf2b40004d4012f1523575127f8ac54a971765126a866428a6c74d4747d1b68e189f0fa3528a73e3f59714" ) + self.put_argparser.add_argument("--ecdsa", "-e", type=str, default=DEFAULT_ECDSA, + help="If set to 'true', the key is assumed to be an ECDSA key; e.g. 'true', 'false' (default: %s)" % DEFAULT_ECDSA + ) # subparser for the del-command self.del_argparser = subparsers.add_parser(COMMAND_DEL, help="Delete an entry from the KeyStore.") @@ -110,10 +118,22 @@ def process_args(self) -> bool: self.uuid_str = self.args.uuid self.pubkey_str = self.args.pubkey self.prvkey_str = self.args.privkey + self.ecdsa_str = self.args.ecdsa + + # get the bool for ecdsa + if self.ecdsa_str.lower() in ["1", "yes", "y", "true"]: + self.ecdsa = True + else: + self.ecdsa = False # load the keypair try: - self.pubkey = ed25519.VerifyingKey(binascii.unhexlify(self.pubkey_str)) + unhex_pubkey = binascii.unhexlify(self.pubkey_str) + + if self.ecdsa == True: + self.pubkey = ecdsa.VerifyingKey.from_string(unhex_pubkey, curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) + else: + self.pubkey = ed25519.VerifyingKey(unhex_pubkey) except Exception as e: logger.error("Error loading the PubKey!") logger.exception(e) @@ -121,7 +141,12 @@ def process_args(self) -> bool: return False try: - self.prvkey = ed25519.SigningKey(binascii.unhexlify(self.prvkey_str)) + unhex_prvkey = binascii.unhexlify(self.prvkey_str) + + if self.ecdsa == True: + self.prvkey = ecdsa.SigningKey.from_string(unhex_prvkey, curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) + else: + self.prvkey = ed25519.SigningKey(unhex_prvkey) except Exception as e: logger.error("Error loading the PrivKey!") logger.exception(e) @@ -172,12 +197,23 @@ def dump_keystore(self) -> bool: signing_keys = self.keystore._ks.private_keys # go trough the list of verifiying keys and print information for each entry - for vk_uuid in verifying_keys.keys(): + for vk_uuid_mod in verifying_keys.keys(): # check if a filtering uuid is set; if it is, filter if self.uuid != None: if self.uuid.hex != vk_uuid: continue + # check the key type + if vk_uuid_mod.find("_ecd") != -1: + vk_uuid = vk_uuid_mod[:-4] + + ktype = "ECDSA NIST256p SHA256" + else: + vk_uuid = vk_uuid_mod + + ktype = "ED25519" + + # get/show the private if the flag is set if self.show_sign == True: t = signing_keys.get("pke_" + vk_uuid) @@ -187,8 +223,9 @@ def dump_keystore(self) -> bool: print("=" * 134) print("UUID: %s" % str(uuid.UUID(hex=vk_uuid))) - print(" VK : %s" % binascii.hexlify(verifying_keys[vk_uuid].cert).decode()) + print(" VK : %s" % binascii.hexlify(verifying_keys[vk_uuid_mod].cert).decode()) print(" SK : %s" % sk) + print("TYPE: %s" % ktype) print("=" * 134) return True @@ -197,7 +234,10 @@ def put_keypair(self) -> bool: logger.info("Inserting keypair for %s with pubkey %s into %s!" % (self.uuid_str, self.pubkey_str, self.keystore_path)) try: - self.keystore.insert_ed25519_keypair(self.uuid, self.pubkey, self.prvkey) + if self.ecdsa == True: + self.keystore.insert_ecdsa_keypair(self.uuid, self.pubkey, self.prvkey) + else: + self.keystore.insert_ed25519_keypair(self.uuid, self.pubkey, self.prvkey) except Exception as e: logger.error("Error inserting the keypair into the KeyStore!") logger.exception(e) @@ -218,7 +258,17 @@ def del_keypair(self) -> bool: try: # direkt access to the entries variable is needed since .certs and .private_keys # are class properties which are only temporary (-> editing them has no effect) - self.keystore._ks.entries.pop(self.uuid.hex) + if self.keystore._ks.entries.get(self.uuid.hex, None) != None: + # suffix-less pubkey found, delete it + self.keystore._ks.entries.pop(self.uuid.hex) + else: + # check for ecdsa key + if self.keystore._ks.entries.get(self.uuid.hex + '_ecd', None) != None: + self.keystore._ks.entries.pop(self.uuid.hex + '_ecd') + else: + # key not found + raise(ValueError("No key found for UUID '%s'" % self.uuid_str)) + self.keystore._ks.entries.pop("pke_" + self.uuid.hex) except Exception as e: logger.error("Error deleting keys! No changes will be written!") From 28c08dd544a661a2f4bd76872754d33aace4861b Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 16:56:52 +0200 Subject: [PATCH 63/68] add not implemented exception when trying to generate keyreg UPPs with ecdsa keys --- examples/upp-creator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/upp-creator.py b/examples/upp-creator.py index e40f251..16432ac 100644 --- a/examples/upp-creator.py +++ b/examples/upp-creator.py @@ -374,6 +374,10 @@ def create_upp(self) -> bool: if self.nostdout == False: logger.info("Generating a key registration UPP for UUID \"%s\"" % self.uuid_str) + # check whether ecdsa is used (currently not supported) + if self.ecdsa == True: + raise(NotImplementedError("Generating KeyReg UPPs with ECDSA keys is currently not supported. Please use the X.509 registrator.")) + self.upp = self.proto.message_signed(self.uuid, ubirch.ubirch_protocol.UBIRCH_PROTOCOL_TYPE_REG, self.keystore.get_certificate(self.uuid)) pass else: From 5d3bc4634475774696649bafca1e2b9293d6f058 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 16:57:19 +0200 Subject: [PATCH 64/68] add a tool to (generate+)register ecdsa keys using X.509 certificates --- examples/x509-registrator.py | 313 +++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 examples/x509-registrator.py diff --git a/examples/x509-registrator.py b/examples/x509-registrator.py new file mode 100644 index 0000000..b2e7060 --- /dev/null +++ b/examples/x509-registrator.py @@ -0,0 +1,313 @@ +import binascii +import json +import logging +import sys +import argparse +import requests +import ecdsa +import OpenSSL +from uuid import UUID + +import ubirch + + +UBIRCH_REGISTRY_ENDPOINT = "https://identity.demo.ubirch.com/api/certs/v1/cert/register" + + +DEFAULT_OUTPUT = "x509.cert" +DEFAULT_NOSEND = "False" +DEFAULT_READ_CERT_FROM_OUPUT = "False" +DEFAULT_VALIDITY_TIME = "%d" % (365 * 24 * 60 * 60) + +X509_DEFAULT_COUNTRY = "DE" +X509_DEFAULT_STATE = "Berlin" +X509_DEFAULT_TOWN = "Berlin" + + +logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) +logger = logging.getLogger() + + +class Main: + def __init__(self): + self.argparser : argparse.ArgumentParser = None + self.args : argparse.Namespace = None + + + self.uuid : UUID = None + self.uuid_str : str = None + self.keystore_path : str = None + self.keystore_pass : str + self.output : str = None + self.nosend_str : str = None + self.nosend : bool = None + self.validity_time_str : str = None + self.validity_time : int = None + self.readfromoutput_str : str = None + self.readfromoutput : bool = None + + self.vk : ecdsa.VerifyingKey = None + self.sk : ecdsa.SigningKey = None + + self.keystore : ubirch.KeyStore = None + self.keystore_pass : str = None + + self.x509_cert : OpenSSL.crypto.X509 = None + self.x509_cert_str : str = None + + # initialize the argument parser + self.setup_argparse() + + return + + def setup_argparse(self): + self.argparser = argparse.ArgumentParser( + description="Create a X.509 certificate for a keypair and register it.", + epilog="This tool only supports ECDSA Keypairs with the NIST256p curve and Sha256 as hash function! If no keypair is found for the given UUID in the given keystore, a new keypair will be created and stored." + ) + self.argparser.add_argument("keystore", metavar="KEYSTORE", type=str, + help="keystore file path; e.g.: test.jks" + ) + self.argparser.add_argument("keystore_pass", metavar="KEYSTORE_PASS", type=str, + help="keystore password; e.g.: secret" + ) + self.argparser.add_argument("uuid", metavar="UUID", type=str, + help="UUID to work with; e.g.: 56bd9b85-6c6e-4a24-bf71-f2ac2de10183" + ) + self.argparser.add_argument("--output", "-o", metavar="OUTPUT", type=str, default=DEFAULT_OUTPUT, + help="path that sets where the X.509 certificate will be written to; e.g.: x509.cert (default: %s)" % DEFAULT_OUTPUT + ) + self.argparser.add_argument("--nosend", "-n", metavar="NOSEND", type=str, default=DEFAULT_NOSEND, + help="disables sending of the generated X.509 if set to 'true'; e.g.: 'true', 'false' (default: %s)" % DEFAULT_NOSEND + ) + self.argparser.add_argument("--validity-time", "-t", metavar="VALIDITY_TIME", type=str, default=DEFAULT_VALIDITY_TIME, + help="determines how long the key shall be valid (in seconds); e.g.: 36000 for 10 hours (default: %s)" % DEFAULT_VALIDITY_TIME + ) + self.argparser.add_argument("--read-cert-from-output", "-r", metavar="READ_CERT_FROM_OUTPUT", type=str, default=DEFAULT_READ_CERT_FROM_OUPUT, + help="if set to 'true', no certificate will be generated but one will be read from the set output file; e.g.: 'true', 'false' (default: %s)" % DEFAULT_READ_CERT_FROM_OUPUT + ) + + def process_args(self) -> bool: + # parse cli arguments (exists on err) + self.args = self.argparser.parse_args() + + # get all needed args + self.uuid_str = self.args.uuid + self.keystore_path = self.args.keystore + self.keystore_pass = self.args.keystore_pass + self.nosend_str = self.args.nosend + self.output = self.args.output + self.readfromoutput_str = self.args.read_cert_from_output + self.validity_time_str = self.args.validity_time + + # check the uuid argument + try: + self.uuid = UUID(self.uuid_str) + except ValueError as e: + logger.error("Invalud UUID input string: \"%s\"!" % self.uuid_str) + logger.exception(e) + + return False + + # get the validity time + try: + self.validity_time = int(self.validity_time_str) + except Exception as e: + logger.error("Can't convert validity time value '%s' to int!" % self.validity_time_str) + logger.exception(e) + + return False + + # get the nostdout value + if self.nosend_str.lower() in ["1", "yes", "y", "true"]: + self.nosend = True + else: + self.nosend = False + + # get the readfromoutput value + if self.readfromoutput_str.lower() in ["1", "yes", "y", "true"]: + self.readfromoutput = True + else: + self.readfromoutput = False + + # success + return True + + def init_keystore(self) -> bool: + try: + self.keystore = ubirch.KeyStore(self.keystore_path, self.keystore_pass) + + # check if the device already has keys or generate a new pair + if not self.keystore.exists_signing_key(self.uuid): + logger.info("No keys found for \"%s\" in \"%s\" - generating a ECDSA keypair" % (self.uuid_str, self.keystore_path)) + + self.keystore.create_ecdsa_keypair(self.uuid) + + self.sk = self.keystore.find_signing_key(self.uuid) + self.vk = self.keystore.find_verifying_key(self.uuid) + + if type(self.vk) != ubirch.ubirch_ks.ecdsa.VerifyingKey: + raise(NotImplementedError("X.509 certificate generation is currently only implemented for ECDSA keys!")) + except Exception as e: + logger.exception(e) + + return False + + return True + + def read_x509_cert(self) -> bool: + logger.info("Reading the X.509 certificate from '%s'" % self.output) + + try: + with open(self.output, "r") as file: + self.x509_cert_str = file.read() + + logger.info("Read certificate:\n%s" % self.x509_cert_str) + except Exception as e: + logger.error("Error reading the certificate!") + logger.exception(e) + + return True + + def create_x509_cert(self) -> bool: + logger.info("Creating a X.509 certificate for '%s' with a validity time of %d seconds" % (self.uuid_str, self.validity_time)) + + choice = input("Enter 'YES' to continue: ") + + if choice != 'YES': + logger.error("Aborting ...") + + return False + + try: + # dump the private key in PEM format + sk_pem : bytes = self.sk.to_pem() + + # load the PEM into OpenSLL + pkey : OpenSSL.crypto.PKey = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, sk_pem) + + # create the cert + self.x509_cert = OpenSSL.crypto.X509() + + self.x509_cert.get_subject().C = X509_DEFAULT_COUNTRY + self.x509_cert.get_subject().ST = X509_DEFAULT_STATE + self.x509_cert.get_subject().L = X509_DEFAULT_TOWN + self.x509_cert.get_subject().CN = self.uuid_str + self.x509_cert.gmtime_adj_notBefore(0) + self.x509_cert.gmtime_adj_notAfter(self.validity_time) + self.x509_cert.set_issuer(self.x509_cert.get_subject()) + self.x509_cert.set_pubkey(pkey) + self.x509_cert.sign(pkey, 'sha256') + + self.x509_cert_str : bytes = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, self.x509_cert) + self.x509_cert_str = self.x509_cert_str.decode("utf8").replace("\\n", "\n") + + logger.info("Generated certificate:\n%s" % self.x509_cert_str) + except Exception as e: + logger.error("Error generating the X.509 certificate!") + logger.exception(e) + + return True + + def store_x509_cert(self) -> bool: + logger.info("Writing the certificate to '%s' ..." % self.output) + + try: + with open(self.output, "w+") as file: + file.write(self.x509_cert_str) + except Exception as e: + logger.error("Error storing the certificate!") + logger.exception(e) + + return False + + return True + + def send_x509_cert(self) -> bool: + logger.info("Sending the certificate to '%s' ..." % UBIRCH_REGISTRY_ENDPOINT) + + try: + # send the cert + r = requests.post( + UBIRCH_REGISTRY_ENDPOINT, data=self.x509_cert_str, + headers={ + 'accept': 'application/json', + 'content-type': 'application/json' + } + ) + + # log the response + logger.info("Backend response:\n%s" % str(r.content)) + + # check if the request was successfull + if r.status_code == 200: + logger.info("Certificate accepted by the backend!") + else: + logger.error("Certificate rejected by the backend! Code: %d" % r.status_code) + + return False + except Exception as e: + logger.error("Error sending the certificate to the backend!") + logger.exception(e) + + return False + + return True + + def run(self) -> int: + # process all raw argument values + if self.process_args() != True: + logger.error("Errors occured during argument processing - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # initialize the keystore + if self.init_keystore() != True: + logger.error("Errors occured while initializing the uBirch Keystore - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check if the cert should just be read + if self.readfromoutput == True: + if self.read_x509_cert() != True: + logger.error("Errors occured while reading the certificate - exiting!\n") + + self.argparser.print_usage() + + return 1 + else: + # create the cert + if self.create_x509_cert() != True: + logger.error("Errors occured while generating the certificate - exiting!\n") + + self.argparser.print_usage() + + return 1 + + if self.store_x509_cert() != True: + logger.error("Errors occured while storing the certificate - exiting!\n") + + self.argparser.print_usage() + + return 1 + + # check whether the cert should be sent + if self.nosend != True: + # send the cert + if self.send_x509_cert() != True: + logger.error("Errors occured while sending the certificate the uBirch - exiting!\n") + + self.argparser.print_usage() + + return 1 + + return 0 + + +# initialize/start the main class +if __name__ == "__main__": + sys.exit(Main().run()) From 04717e470d52acf73ce4830b6cc6f75ca7094791 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 17:21:32 +0200 Subject: [PATCH 65/68] implement an option to set the ubirch environment --- examples/x509-registrator.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/x509-registrator.py b/examples/x509-registrator.py index b2e7060..458ec0c 100644 --- a/examples/x509-registrator.py +++ b/examples/x509-registrator.py @@ -11,7 +11,7 @@ import ubirch -UBIRCH_REGISTRY_ENDPOINT = "https://identity.demo.ubirch.com/api/certs/v1/cert/register" +UBIRCH_REGISTRY_ENDPOINT = "https://identity.%s.ubirch.com/api/certs/v1/cert/register" DEFAULT_OUTPUT = "x509.cert" @@ -45,6 +45,7 @@ def __init__(self): self.validity_time : int = None self.readfromoutput_str : str = None self.readfromoutput : bool = None + self.env : str = None self.vk : ecdsa.VerifyingKey = None self.sk : ecdsa.SigningKey = None @@ -65,6 +66,9 @@ def setup_argparse(self): description="Create a X.509 certificate for a keypair and register it.", epilog="This tool only supports ECDSA Keypairs with the NIST256p curve and Sha256 as hash function! If no keypair is found for the given UUID in the given keystore, a new keypair will be created and stored." ) + self.argparser.add_argument("env", metavar="ENV", type=str, + help="the uBirch environment to work on; one of 'dev', 'demo' or 'prod'" + ) self.argparser.add_argument("keystore", metavar="KEYSTORE", type=str, help="keystore file path; e.g.: test.jks" ) @@ -99,6 +103,7 @@ def process_args(self) -> bool: self.output = self.args.output self.readfromoutput_str = self.args.read_cert_from_output self.validity_time_str = self.args.validity_time + self.env = self.args.env # check the uuid argument try: @@ -224,12 +229,12 @@ def store_x509_cert(self) -> bool: return True def send_x509_cert(self) -> bool: - logger.info("Sending the certificate to '%s' ..." % UBIRCH_REGISTRY_ENDPOINT) + logger.info("Sending the certificate to '%s' ..." % (UBIRCH_REGISTRY_ENDPOINT % self.env)) try: # send the cert r = requests.post( - UBIRCH_REGISTRY_ENDPOINT, data=self.x509_cert_str, + (UBIRCH_REGISTRY_ENDPOINT % self.env), data=self.x509_cert_str, headers={ 'accept': 'application/json', 'content-type': 'application/json' From f5766a42563dc543d44fad7085a0f5d384e0c2fd Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 17:22:23 +0200 Subject: [PATCH 66/68] add documentation for the x509 registrator to --- examples/EXAMPLES.md | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/examples/EXAMPLES.md b/examples/EXAMPLES.md index e9cb19c..b979c6d 100644 --- a/examples/EXAMPLES.md +++ b/examples/EXAMPLES.md @@ -26,6 +26,7 @@ This file documents how to use the examples provided alongside the [uBirch-Proto - [Managing Keys](#managing-keys) - [Managing the local KeyStore](#managing-the-local-keystore) - [Managing keys inside the uBirch Identity Service](#managing-keys-inside-the-ubirch-identity-service) + - [Registering ECDSA Keys](#registering-ecdsa-keys) ## From measurement to blockchain-anchored UPP The process needed to get a UPP to be anchored in the blockchain can be cut down into multiple steps. For each of those steps there is an example in this directory, demonstrating how to handle them. There are also examples showing a full example-client implementation. @@ -473,4 +474,39 @@ For some operations a date string in a specific format will be needed (a specifi ``` $ TZ=UTC date "+%Y-%m-%dT%H:%M:%S.000Z" 2022-02-23T11:11:11.000Z -``` \ No newline at end of file +``` +### Registering ECDSA Keys +Currently the only way to register ECDSA Keys is by using X.509 certificates. This can be done by usign the `x509-registrator.py` script. It's able to generate an ECDSA keypair for a UUID + store it in a keystore, or read it from said keystore and generate a X.509 certificate for it. Additionally it will send the certificate the the uBirch backend to register they keypair. + +**Warning**: Only ECDSA keys using the `NIST256p` curve and `Sha256` as hash function are supported! Others **won't** be accepted by the backend! + +Below is a simple example call to register an ECDSA KeyPair to the backend. Note, that the keypair doesn't have to exist yet. If it doesn't it will be generated in the keystore (`devices.jks`). The first four arguments are positional. They are: + +` ` + +`ENV` is the uBirch environment and must be one of `dev`, `demo` or `prod`. The `KEYSTORE_FILE` must be a pfad to a valid JavaKeyStore file (normal extension: `.jks`). `KEYSTORE_PASS` must be the password needed to unlock the given keystore. `UUID` is the uuid of the identity to work with. +``` +python x509-registrator.py dev devices.jks secret_password 11a8ca3c-76a4-433d-bc5c-372a1a2292f6 +2022-04-08 17:11:35,033 root create_x509_cert() INFO Creating a X.509 certificate for '11a8ca3c-76a4-433d-bc5c-372a1a2292f6' with a validity time of 31536000 seconds +Enter 'YES' to continue: YES +2022-04-08 17:11:37,239 root create_x509_cert() INFO Generated certificate: +-----BEGIN CERTIFICATE----- +MIIBpTCCAUoCAQAwCgYIKoZIzj0EAwIwXjELMAkGA1UEBhMCREUxDzANBgNVBAgM +BkJlcmxnbjEPMA0GA1UEBwwGQmVybGluMS0wKwYDVQQDDCQxMWE4Y2EzYy03NmE0 +LTQzM2QtYmM1Yy0zNzJhMWEyMjkyZjYsHhcNMjIwNDA4MTUxMTM3WhcNMjMwNDA4 +MTUxMTM3WjBeMQswCQYDVQQGEwJERTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQH +DAZCZXJsaW4xLTArBgNVBAMMJDExYThjYTNjLTd2YTQtNDMzZC1iYzVjLTM3MmEx +YTIyOTJmNjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABLa/+lNVbYSuZ07f1rEG ++ozxRYlZ5TnuqHFc79vm9BUaN9wOsIDj0mGZf71VzwmTBJVjsMlXQeLORQNU311v +dn0wCgYIKoZIzj0EAwIDSQAwRgIhAK7w9LYwdV+nnv4o3otQeR+8p0cX79BhUyP4 +mPJdW100AiEA+igK3y1RGEdnqzXssiLYofIqmrZro413tFsJXLV2eM0= +-----END CERTIFICATE----- + +2022-04-08 17:11:37,239 root store_x509_cert() INFO Writing the certificate to 'x509.cert' ... +2022-04-08 17:11:37,239 root send_x509_cert() INFO Sending the certificate to 'https://identity.dev.ubirch.com/api/certs/v1/cert/register' ... +2022-04-08 17:11:38,432 root send_x509_cert() INFO Backend response: +b'{"algorithm":"ecdsa-p256v1","created":"2022-04-08T15:11:37.383Z","hwDeviceId":"11a8ca3c-76a4-433d-bc5c-372a1a2292f6","pubKey":"tr/6U1VthK5nTt/WsQb6jPFFiVnBOe6ocVzl2+b0FRo33A6wgOPsYZl/vVXPCZMElWOwyVdB4s5FA1TfXW92fQ==","pubKeyId":"tr/6U1VthK5nTt/8sQb6jPFFiVnBOe6ocVzv2+b0FRo33A6wgOPSYZl/vVXPCZMElWOwyVdB4s5FA1TfXW9afQ==","validNotAfter":"2023-04-08T15:11:37.000Z","validNotBefore":"2022-04-08T15:11:37.000Z"}' +2022-04-08 17:11:38,432 root send_x509_cert() INFO Certificate accepted by the backend! +``` + +If the certificate was already registered beforehand, generating a new one can be disabled by passing `-r true`. This will cause the script to read a certificate from the output file which can be specified with `-o [FILE]`, otherwise the default will be used. Sending the certificate to the backend can also be disabled by passing `-n true`. \ No newline at end of file From c01e58624114b342f7d6f8d6a4003018ec8619d9 Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Fri, 8 Apr 2022 18:19:41 +0200 Subject: [PATCH 67/68] make the verifier able to work with ecdsa --- examples/upp-verifier.py | 87 ++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/examples/upp-verifier.py b/examples/upp-verifier.py index 981b024..0360095 100644 --- a/examples/upp-verifier.py +++ b/examples/upp-verifier.py @@ -5,12 +5,15 @@ import binascii import uuid import ed25519 +import ecdsa +import hashlib import ubirch DEFAULT_INPUT = "/dev/stdin" DEFAULT_ISHEX = "false" +DEFAULT_ISECD = "false" logging.basicConfig(format='%(asctime)s %(name)20.20s %(funcName)20.20s() %(levelname)-8.8s %(message)s', level=logging.INFO) @@ -64,6 +67,8 @@ def __init__(self): self.ishex : bool = None self.ishex_str : str = None + self.isecd : bool = None + self.isecd_str : str = None # initialize the argument parser self.setup_argparse() @@ -87,6 +92,9 @@ def setup_argparse(self): self.argparser.add_argument("--ishex", "-x", metavar="ISHEX", type=str, default=DEFAULT_ISHEX, help="Sets whether the UPP input data is a hex string or binary; e.g. true, false (default: %s)" % DEFAULT_ISHEX ) + self.argparser.add_argument("--isecd", "-c", metavar="ISECD", type=str, default=DEFAULT_ISECD, + help="Sets whether the key provided with -k is a ECDSA NIST256p SHA256 key (true) or a ED25519 key (false) (default: %s)" % DEFAULT_ISECD + ) self.argparser.add_argument("--input", "-i", metavar="INPUT", type=str, default=DEFAULT_INPUT, help="UPP input file path; e.g. upp.bin or /dev/stdin (default: %s)" % DEFAULT_INPUT ) @@ -102,30 +110,7 @@ def process_args(self) -> bool: self.vk_uuid_str = self.args.verifying_key_uuid self.input = self.args.input self.ishex_str = self.args.ishex - - # check if a verifying key was supplied - if self.vk_str != "AUTO": - try: - # convert the string - self.vk = ed25519.VerifyingKey(self.vk_str, encoding="hex") - except Exception as e: - logger.error("Invalid verifying key supplied via --verifying-key/-k: \"%s\"" % self.vk_str) - logger.exception(e) - - return False - - if self.vk_uuid_str == "EMPTY": - logger.error("--verifying-key-uuid/-u must be specified when --verifying-key/-k is specified!") - - return False - - try: - self.vk_uuid = uuid.UUID(hex=self.vk_uuid_str) - except Exception as e: - logger.error("Invalid UUID supplied via --verifying-key-uuid/-u: \"%s\"" % self.vk_uuid_str) - logger.exception(e) - - return False + self.isecd_str = self.args.isecd # get the ishex value if self.ishex_str.lower() in ["1", "yes", "y", "true"]: @@ -133,6 +118,12 @@ def process_args(self) -> bool: else: self.ishex = False + # get the isecd value + if self.isecd_str.lower() in ["1", "yes", "y", "true"]: + self.isecd = True + else: + self.isecd = False + return True def read_upp(self) -> bool: @@ -140,8 +131,8 @@ def read_upp(self) -> bool: try: logger.info("Reading the input UPP from \"%s\"" % self.input) - with open(self.input, "rb") as file: - self.upp = file.read() + with open(self.input, "rb") as fd: + self.upp = fd.read() # check whether hex decoding is needed if self.ishex == True: @@ -164,15 +155,43 @@ def init_keystore(self) -> bool: return True def check_cli_vk(self) -> bool: - try: - if self.vk != None: - self.keystore.insert_ed25519_verifying_key(self.vk_uuid, self.vk) + # check if a verifying key was supplied + if self.vk_str != "AUTO": + # check if a uuid was supplied + if self.vk_uuid_str == "EMPTY": + logger.error("--verifying-key-uuid/-u must be specified when --verifying-key/-k is specified!") - logger.info("Inserted \"%s\": \"%s\" (UUID/VK) into the keystore" % (self.vk_uuid_str, self.vk_str)) - except Exception as e: - logger.exception(e) + return False + + # load the uuid + try: + self.vk_uuid = uuid.UUID(hex=self.vk_uuid_str) + except Exception as e: + logger.error("Invalid UUID supplied via --verifying-key-uuid/-u: \"%s\"" % self.vk_uuid_str) + logger.exception(e) + + return False + + # check the keytype, load it and insert it into the keystore + try: + if self.isecd == False: + logger.info("Loading the key as ED25519 verifying key") + + self.vk = ed25519.VerifyingKey(self.vk_str, encoding="hex") + self.keystore.insert_ed25519_verifying_key(self.vk_uuid, self.vk) + else: + logger.info("Loading the key as ECDSA NIST256p SHA256 verifying key") + + self.vk = ecdsa.VerifyingKey.from_string(binascii.unhexlify(self.vk_str), curve=ecdsa.NIST256p, hashfunc=hashlib.sha256) + self.keystore.insert_ecdsa_verifying_key(self.vk_uuid, self.vk) + except Exception as e: + logger.error("Error loading the verifying key and inserting it into the keystore") + logger.exception(e) + + return False + + logger.info("Inserted \"%s\": \"%s\" (UUID/VK) into the keystore" % (self.vk_uuid_str, self.vk_str)) - return False return True @@ -281,4 +300,4 @@ def run(self): if __name__ == "__main__": - sys.exit(Main().run()) + sys.exit(Main().run()) \ No newline at end of file From 20797431e3ed8356673d155c99a37a9dc3049bca Mon Sep 17 00:00:00 2001 From: UBOK19 <53264378+UBOK19@users.noreply.github.com> Date: Tue, 12 Apr 2022 15:44:13 +0200 Subject: [PATCH 68/68] add a debug logging statement --- examples/upp-sender.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/upp-sender.py b/examples/upp-sender.py index 31615ec..f3c92fd 100644 --- a/examples/upp-sender.py +++ b/examples/upp-sender.py @@ -121,6 +121,8 @@ def check_is_keyreg(self) -> bool: def init_api(self) -> bool: try: + logger.info("Configuring the API to use the '%s' environment!" % self.env) + # initialize the uBirch api self.api = ubirch.API(env=self.env, debug=True) self.api.set_authentication(self.uuid, self.auth)