From 20dbe908e0bb5931cfec4f7c66f72ace9cb5c40c Mon Sep 17 00:00:00 2001 From: Harish Navnit Date: Sat, 15 Apr 2023 14:58:58 +0530 Subject: [PATCH 1/4] Fix setup script --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b38d11b..66bcb69 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ +from os.path import dirname, join from setuptools import setup, find_packages -with open('README.rst') as r: + +with open(join(dirname(__file__), 'README.md')) as r: readme = r.read() setup(name='simpleais', From 282acf9fc3b41fd1abd377293b7ee27d70601ba5 Mon Sep 17 00:00:00 2001 From: Harish Navnit Date: Sat, 15 Apr 2023 14:58:32 +0530 Subject: [PATCH 2/4] Fetch timestamp from tcp streams --- simpleais/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 8bd2b94..514c1da 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -169,7 +169,13 @@ def parse_one(string, default_to_current_time=False): if default_to_current_time: sentence_time = time.time() else: - sentence_time = None + timestamp = None + ctag = re.compile('(c:[0-9])\w+').search(string) + if ctag: + timestamp = int(ctag.group()[2:]) + + sentence_time = timestamp + message = m.group(2) From 729f7b6f59acf8b46aee2b68f9eda81e3a2d32c9 Mon Sep 17 00:00:00 2001 From: Harish Navnit Date: Sun, 16 Apr 2023 21:00:18 +0530 Subject: [PATCH 3/4] Preserve original NMEA string and parse sentence source --- simpleais/__init__.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 514c1da..5198dec 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -176,6 +176,11 @@ def parse_one(string, default_to_current_time=False): sentence_time = timestamp + # Add the sentence source + sentence_source = None + stag = re.compile('s:[0-9a-zA-Z]+').search(string) + if stag: + sentence_source = stag.group()[2:] message = m.group(2) @@ -187,7 +192,8 @@ def parse_one(string, default_to_current_time=False): radio_channel = fields[4] payload = NmeaPayload(fields[5], int(fields[6])) if fragment_count == 1: - return Sentence(talker, sentence_type, radio_channel, payload, [checksum], sentence_time, [message]) + return Sentence(talker, sentence_type, radio_channel, payload, + [checksum], sentence_time, [message], sentence_source, string) else: fragment_number = int(fields[2]) message_id = fields[3] @@ -679,8 +685,11 @@ def _decoder_for_type(number): class SentenceFragment: - def __init__(self, talker, sentence_type, total_fragments, fragment_number, message_id, radio_channel, payload, - checksum, received_time=None, text=None): + def __init__(self, talker, sentence_type, total_fragments, fragment_number, + message_id, radio_channel, payload, checksum, received_time=None, + text=None, source=None, string=None): + self.string = string + self.source = source self.talker = talker self.sentence_type = sentence_type self.total_fragments = total_fragments @@ -737,7 +746,10 @@ def valid(self): class Sentence: - def __init__(self, talker, sentence_type, radio_channel, payload, checksums, received_time=None, text=None): + def __init__(self, talker, sentence_type, radio_channel, payload, + checksums, received_time=None, text=None, source=None, string=None): + self.string = string + self.source = source self.talker = talker self.sentence_type = sentence_type self.radio_channel = radio_channel @@ -789,7 +801,7 @@ def from_fragments(cls, matching_fragments): checksums, first.time, text) def __repr__(self): - return "Sentence({}, {})".format(self.time, self.text) + return "Sentence({}, <{}, {}>)".format(self.time, self.source, self.text) def __str__(self): return "Sentence(type {}, from {}, at {})".format(self.type_num, self['mmsi'], self.time) From c52da9d6ab25ae1314cc79235e925d50ce048adc Mon Sep 17 00:00:00 2001 From: Harish Navnit Date: Sun, 16 Apr 2023 21:00:50 +0530 Subject: [PATCH 4/4] Handle blob source inputs, viz s3 for now --- simpleais/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 5198dec..f67e991 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -8,6 +8,7 @@ import time from functools import reduce from io import TextIOBase +from smart_open import open as sopen aivdm_pattern = re.compile(r'([.0-9]+)?\s*(![A-Z]{5},\d,\d,.?,[AB12]?,[^,]+,[0-6]\*[0-9A-F]{2})') @@ -886,6 +887,8 @@ def lines_from_source(source): yield from _handle_serial_source(source) elif re.match("https?://.*", source): yield from _handle_url_source(source) + elif re.match("s3://", source): + yield from _handle_blob_source(source) else: # assume it's a file yield from _handle_file_source(source) @@ -959,3 +962,9 @@ def _handle_file_source(source): with source_reader as f: for line in f: yield line + + +def _handle_blob_source(source): + with sopen(source) as obj: + for line in obj: + yield line