From 2eef0f42475937af0b3a14d65854d927e4b17692 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Sun, 31 Mar 2024 17:51:37 -0700 Subject: [PATCH 01/10] fix example in readme --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ce185f5..4a19602 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,10 @@ Here's an example, a simplified version of the `aist` tool, which prints one line per complete AIS message: - for sentence in sentences_from_sources(sys.argv[1:): + import sys + from simpleais import * + + for sentence in sentences_from_source(sys.argv[1]): result = [] if sentence.time: result.append(sentence.time.strftime(TIME_FORMAT)) @@ -45,7 +48,8 @@ one line per complete AIS message: print(" ".join(result)) -The `sentence_from_sources()` function will pull from a wide variety of sources + +The `sentence_from_source()` function will pull from a wide variety of sources (local files, serial ports, HTTP URLs), yielding only complete sentences as they arrive. Each sentence has a wide variety of readable information. Documented fields can all be referred to by name. For example, `sentence['mmsi']` or From 03ebdeb036e584271f3184c88e5ca1ad03702e30 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Sun, 31 Mar 2024 17:55:16 -0700 Subject: [PATCH 02/10] update link to AIVDM protocol documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a19602..7b0ecc0 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ use aisgrep to get the relevant packets and aisinfo to plot the map: ## Sources -My main source for protocol information is here: http://catb.org/gpsd/AIVDM.html +My main source for protocol information is here: https://gpsd.gitlab.io/gpsd/AIVDM.html More protocol info is here: http://www.itu.int/dms_pubrec/itu-r/rec/m/R-REC-M.1371-5-201402-I!!PDF-E.pdf From be626953da752680b250cf96fb38c814bd792066 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Sun, 31 Mar 2024 17:56:20 -0700 Subject: [PATCH 03/10] add support for UDP as data source --- simpleais/__init__.py | 26 ++++++++++++++++++++++++++ tests/test_source_handling.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index d7b5d01..12dc514 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -882,6 +882,8 @@ def lines_from_source(source): yield from _handle_serial_source(source) elif re.match("https?://.*", source): yield from _handle_url_source(source) + elif re.match("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{1,5}$", source): + yield from _handle_udp_source(source) else: # assume it's a file yield from _handle_file_source(source) @@ -955,3 +957,27 @@ def _handle_file_source(source): with source_reader as f: for line in f: yield line + + +def _handle_udp_source(source): + import socket + import select + + ip, port = source.split(':') + if ip.endswith('.255'): + # use default IP for receiving UDP broadcast messages + ip = '' + port = int(port) + + while True: + # noinspection PyBroadException + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.bind((ip, port)) + # using select for timely detection of ctrl-c, even without AIS traffic + ready = select.select([s], [], [], 0.5) + if ready[0]: + yield s.recv(4096).decode('ascii') + except Exception: + logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True) + time.sleep(1) diff --git a/tests/test_source_handling.py b/tests/test_source_handling.py index 240e88d..bd254e0 100644 --- a/tests/test_source_handling.py +++ b/tests/test_source_handling.py @@ -67,7 +67,7 @@ def test_io_source_by_sentence(self): self.assertRaises(StopIteration, sentences.__next__) logs.check(('root', 'WARNING', 'skipped: "garbage data"')) - # TODO: figure out how to test serial and url sources effectively + # TODO: figure out how to test serial, url, and udp sources effectively def write_sample_data(self, file, compress=False): if compress: From 71e2bdb22edc023539891b96c621230c011e0cf4 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Mon, 1 Apr 2024 11:15:06 -0700 Subject: [PATCH 04/10] change udp source to only match for :port --- simpleais/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 12dc514..e45809d 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -882,8 +882,10 @@ def lines_from_source(source): yield from _handle_serial_source(source) elif re.match("https?://.*", source): yield from _handle_url_source(source) - elif re.match("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{1,5}$", source): + elif re.match("^:\\d{1,5}$", source): yield from _handle_udp_source(source) + elif re.match("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{1,5}$", source): + pass else: # assume it's a file yield from _handle_file_source(source) From a93ff35505b072d9778a76f4eff79e80be123be5 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Mon, 1 Apr 2024 11:27:03 -0700 Subject: [PATCH 05/10] add support for tcp client sources with ip:port --- simpleais/__init__.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index e45809d..578c31c 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -885,7 +885,7 @@ def lines_from_source(source): elif re.match("^:\\d{1,5}$", source): yield from _handle_udp_source(source) elif re.match("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{1,5}$", source): - pass + yield from _handle_tcp_client_source(source) else: # assume it's a file yield from _handle_file_source(source) @@ -983,3 +983,24 @@ def _handle_udp_source(source): except Exception: logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True) time.sleep(1) + + +def _handle_tcp_client_source(source): + import socket + import select + + ip, port = source.split(':') + port = int(port) + + while True: + # noinspection PyBroadException + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((ip, port)) + # using select for timely detection of ctrl-c, even without AIS traffic + ready = select.select([s], [], [], 0.5) + if ready[0]: + yield s.recv(4096).decode('ascii') + except Exception: + logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True) + time.sleep(1) From 0437bf01cfaaf86b707ca427c3e50755453c3556 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Mon, 1 Apr 2024 11:39:12 -0700 Subject: [PATCH 06/10] also support domain names for tcp client --- simpleais/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 578c31c..043fac7 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -884,7 +884,7 @@ def lines_from_source(source): yield from _handle_url_source(source) elif re.match("^:\\d{1,5}$", source): yield from _handle_udp_source(source) - elif re.match("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:\\d{1,5}$", source): + elif re.match(".*:\\d{1,5}$", source): yield from _handle_tcp_client_source(source) else: # assume it's a file From 57a66dd94aa22841df57f25c32d47c3ddbf22472 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Mon, 1 Apr 2024 12:56:14 -0700 Subject: [PATCH 07/10] support receiving multiple and partial messages as tcp client --- simpleais/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 043fac7..22c837e 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -991,16 +991,26 @@ def _handle_tcp_client_source(source): ip, port = source.split(':') port = int(port) + line_buffer = "" while True: # noinspection PyBroadException try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((ip, port)) + s.setblocking(0) # using select for timely detection of ctrl-c, even without AIS traffic ready = select.select([s], [], [], 0.5) if ready[0]: - yield s.recv(4096).decode('ascii') + line_buffer += s.recv(4096).decode('ascii') + lines = line_buffer.splitlines(True) + line_buffer = "" + for l in lines: + if l.find('\n') != -1: + yield l + else: + line_buffer += l except Exception: + print("error") logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True) time.sleep(1) From d95ce1ec805ab10bc85097bfd241854dce73a474 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Mon, 1 Apr 2024 21:09:01 -0700 Subject: [PATCH 08/10] fix tcp client to handle high data volumes --- simpleais/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 22c837e..e7ede4e 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -998,10 +998,7 @@ def _handle_tcp_client_source(source): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect((ip, port)) - s.setblocking(0) - # using select for timely detection of ctrl-c, even without AIS traffic - ready = select.select([s], [], [], 0.5) - if ready[0]: + while True: line_buffer += s.recv(4096).decode('ascii') lines = line_buffer.splitlines(True) line_buffer = "" From 1c018f31b88465a05e2650e7544c554262375610 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Mon, 1 Apr 2024 22:19:05 -0700 Subject: [PATCH 09/10] also support multi-line and partial data with UDP --- simpleais/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index e7ede4e..68e7542 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -970,16 +970,22 @@ def _handle_udp_source(source): # use default IP for receiving UDP broadcast messages ip = '' port = int(port) + line_buffer = "" while True: # noinspection PyBroadException try: with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: s.bind((ip, port)) - # using select for timely detection of ctrl-c, even without AIS traffic - ready = select.select([s], [], [], 0.5) - if ready[0]: - yield s.recv(4096).decode('ascii') + while True: + line_buffer += s.recv(4096).decode('ascii') + lines = line_buffer.splitlines(True) + line_buffer = "" + for l in lines: + if l.find('\n') != -1: + yield l + else: + line_buffer += l except Exception: logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True) time.sleep(1) From ceba1bf21636f3298bde5d300cb662e962c95be0 Mon Sep 17 00:00:00 2001 From: Adrian Studer Date: Fri, 5 Apr 2024 17:09:15 -0700 Subject: [PATCH 10/10] add timeout to UDP/TCP to allow users to stop script with CTRL-C --- simpleais/__init__.py | 47 +++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/simpleais/__init__.py b/simpleais/__init__.py index 68e7542..9ad4577 100644 --- a/simpleais/__init__.py +++ b/simpleais/__init__.py @@ -11,6 +11,9 @@ aivdm_pattern = re.compile(r'([.0-9]+)?\s*(![A-Z]{5},\d,\d,.?,[AB12]?,[^,]+,[0-6]\*[0-9A-F]{2})') +# Timeout in seconds for serial, TCP, and UDP. +# Allows users to stop a Python script with CTRL-C. +source_timeout = 10 class Bits: """ @@ -924,7 +927,7 @@ def _handle_serial_source(source): while True: # noinspection PyBroadException try: - with serial.Serial(source, 38400, timeout=10) as f: + with serial.Serial(source, 38400, timeout=source_timeout) as f: while True: raw_line = f.readline() try: @@ -976,16 +979,21 @@ def _handle_udp_source(source): # noinspection PyBroadException try: with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.settimeout(source_timeout) s.bind((ip, port)) while True: - line_buffer += s.recv(4096).decode('ascii') - lines = line_buffer.splitlines(True) - line_buffer = "" - for l in lines: - if l.find('\n') != -1: - yield l - else: - line_buffer += l + try: + line_buffer += s.recv(4096).decode('ascii') + lines = line_buffer.splitlines(True) + line_buffer = "" + for l in lines: + if l.find('\n') != -1: + yield l + else: + line_buffer += l + except socket.timeout: + # timeout gives the user a chance to CTRL-C even without AIS traffic + pass except Exception: logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True) time.sleep(1) @@ -1003,16 +1011,21 @@ def _handle_tcp_client_source(source): # noinspection PyBroadException try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(source_timeout) s.connect((ip, port)) while True: - line_buffer += s.recv(4096).decode('ascii') - lines = line_buffer.splitlines(True) - line_buffer = "" - for l in lines: - if l.find('\n') != -1: - yield l - else: - line_buffer += l + try: + line_buffer += s.recv(4096).decode('ascii') + lines = line_buffer.splitlines(True) + line_buffer = "" + for l in lines: + if l.find('\n') != -1: + yield l + else: + line_buffer += l + except socket.timeout: + # timeout gives the user a chance to CTRL-C even without AIS traffic + pass except Exception: print("error") logging.getLogger().error("unexpected failure in source {}".format(source), exc_info=True)