From e785903a8aaf548a6579a0b369d7549cd532e598 Mon Sep 17 00:00:00 2001 From: Kevin <104317450+MaKeG0@users.noreply.github.com> Date: Tue, 6 Feb 2024 20:04:08 +0100 Subject: [PATCH 1/2] Added handling of list of timestamp In GelfTcp there is already the possibility to have list of messages, i've added also list of timestamp so that, only in case of list of messages, they can be sent each with their message. Is also possible to have a list of messages with only 1 timestamp so also that possibility is handled. In GelfHttp is a bit more complicated. There was not a list handling but, according to graylog documentation, they introduced a way to handle bulk http GELF post. https://go2docs.graylog.org/5-0/getting_in_log_data/ingest_gelf.html So i've added the same list handling from Tcp but, as the bulking is made appending dumped JSON with \n\r at the end, and, most important, must be avoided to have \n or \r inside the dumped messages, i had to make some cases that handle all the above. To keep the rest as it is i've jumped the single message before so only 1 variable needs to be handled. --- asyncgelf/asyncgelf.py | 53 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/asyncgelf/asyncgelf.py b/asyncgelf/asyncgelf.py index 7935579..4231275 100644 --- a/asyncgelf/asyncgelf.py +++ b/asyncgelf/asyncgelf.py @@ -129,6 +129,7 @@ async def tcp_handler(self, message, timestamp: Optional = None): :return: Exception """ messages = [] + timestamps = [] """ Input type checking """ @@ -137,6 +138,12 @@ async def tcp_handler(self, message, timestamp: Optional = None): else: messages = message + + if type(timestamp) is list: + timestamps = timestamp + + + try: if self.tls: @@ -157,7 +164,9 @@ async def tcp_handler(self, message, timestamp: Optional = None): return getattr(e, 'message', repr(e)) - for message in messages: + for i,message in enumerate(messages): + if len(timestamps) > 0: + timestamp = timestamps[i] gelf_message = GelfBase.make(self, message, timestamp) """ Transforming GELF dictionary into bytes """ bytes_msg = json.dumps(gelf_message).encode('utf-8') @@ -176,7 +185,7 @@ class GelfHttp(GelfBase): async def http_handler(self, message, timestamp: Optional = None): """ http handler for send logs to Graylog Input with type: gelf http - :param message: input message + :param message: message to send, can be list, str, dict :param timestamp: event timestamp in the format: seconds since UNIX epoch with optional decimal places for milliseconds :return: http status code @@ -185,10 +194,40 @@ async def http_handler(self, message, timestamp: Optional = None): 'Content-Type': 'application/json', } - if self.compress: - header.update({'Content-Encoding': 'gzip,deflate'}) + messages = [] + timestamps = [] + bulk_messages = "" + """ + Input type checking + """ + if type(message) is not list: + messages.append(message) - gelf_message = GelfBase.make(self, message, timestamp) + else: + messages = message + + if type(timestamp) is list: + timestamps = timestamp + + for i,message in enumerate(messages): + if len(timestamps) > 0: + timestamp = timestamps[i] + gelf_message = GelfBase.make(self, message, timestamp) + # In case of multiple messages send them in bulk + if len(messages) > 1: + # Convert the gelf_message dictionary to a JSON string and append it to bulk_message + """ + Warning: Individual GELF messages must be formatted as a valid JSON (containing no line breaks within). Attempts to post formatted JSON to this input will result in an error. + """ + bulk_messages+=json.dumps(gelf_message).replace('\n', ' ').replace('\r', '') + "\r\n" + + if len(messages) > 1: + # Remove the last "\r\n" from bulk_message + bulk_message = bulk_message.rstrip("\r\n") + gelf_message = bulk_messages + else: + # Dumping the gelf_message dictionary to a JSON string + gelf_message = json.dumps(gelf_message) if self.tls: ssl_contex = ssl.create_default_context() @@ -201,7 +240,7 @@ async def http_handler(self, message, timestamp: Optional = None): response = await client.post( gelf_endpoint, headers=header, - data=json.dumps(gelf_message), + data=gelf_message, ) return response.status_code @@ -219,7 +258,7 @@ async def http_handler(self, message, timestamp: Optional = None): response = await client.post( gelf_endpoint, headers=header, - data=json.dumps(gelf_message), + data=gelf_message, ) return response.status_code From 1d20fe21966d0ee3efd8ce3b90c742b17b9f8101 Mon Sep 17 00:00:00 2001 From: Kevin <104317450+MaKeG0@users.noreply.github.com> Date: Tue, 6 Feb 2024 20:17:31 +0100 Subject: [PATCH 2/2] small fix typo and direct cleaning into gelf_message variable --- asyncgelf/asyncgelf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/asyncgelf/asyncgelf.py b/asyncgelf/asyncgelf.py index 4231275..2b40033 100644 --- a/asyncgelf/asyncgelf.py +++ b/asyncgelf/asyncgelf.py @@ -223,8 +223,7 @@ async def http_handler(self, message, timestamp: Optional = None): if len(messages) > 1: # Remove the last "\r\n" from bulk_message - bulk_message = bulk_message.rstrip("\r\n") - gelf_message = bulk_messages + gelf_message = bulk_messages.rstrip("\r\n") else: # Dumping the gelf_message dictionary to a JSON string gelf_message = json.dumps(gelf_message)