From eff6e67c3b600e5a3d4d6108ccd72ed00d52769f Mon Sep 17 00:00:00 2001 From: Paul Thevenon Date: Sun, 18 Jan 2026 01:00:17 +0100 Subject: [PATCH 1/3] - new source of BRDC NAV files - added test to check availability of correct files after 2016 --- src/prx/rinex_nav/nav_file_discovery.py | 100 ++++++------------ .../rinex_nav/test/test_nav_file_discovery.py | 8 ++ 2 files changed, 43 insertions(+), 65 deletions(-) diff --git a/src/prx/rinex_nav/nav_file_discovery.py b/src/prx/rinex_nav/nav_file_discovery.py index ac5f1bf..869fa46 100644 --- a/src/prx/rinex_nav/nav_file_discovery.py +++ b/src/prx/rinex_nav/nav_file_discovery.py @@ -1,13 +1,12 @@ import argparse import os import re -from ftplib import FTP from pathlib import Path import georinex import urllib.request import pandas as pd import prx.util -import requests +import ftplib from prx import converters, util from prx.converters import anything_to_rinex_3 @@ -16,85 +15,56 @@ log = util.get_logger(__name__) +IGS_FTP_SERVER = { + "gssc.esa.int": "/gnss/data/daily/", + "igs.ign.fr": "/pub/igs/data/", +} + def is_rinex_3_mixed_mgex_broadcast_ephemerides_file(file: Path): pattern = r"^[A-Za-z0-9]{9}_[A-Za-z]_\d{11}_[A-Za-z0-9]{3}_[A-Za-z]N\.rnx.*" return bool(re.match(pattern, file.name)) -def try_downloading_ephemerides_http(day: pd.Timestamp, local_destination_folder: Path): - # IGS BKG Rinex 3.04 mixed file paths follow this pattern: - # https://igs.bkg.bund.de/root_ftp/IGS/BRDC/2023/002/BRDC00_R_20230020000_01D_MN.rnx.gz - file_regex = f"BRDC00(?:IGS|WRD)_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" - remote_directory = ( - f"https://igs.bkg.bund.de/root_ftp/IGS/BRDC/{day.year}/{day.day_of_year:03}/" - ) - try: - # List available files whose names fit the pattern - directory_listing = requests.get(remote_directory, timeout=30).text - matches = list(set(re.findall(file_regex, directory_listing))) - if len(matches) == 0: - log.warning(f"Could not find broadcast ephemerides file for {day}") - return None - file = sorted(matches, key=lambda x: int("IGS" in x), reverse=True)[0] - local_compressed_file = local_destination_folder / file - url = remote_directory + file - urllib.request.urlretrieve(url, local_compressed_file) - local_file = converters.compressed_to_uncompressed(local_compressed_file) - os.remove(local_compressed_file) - log.info(f"Downloaded broadcast ephemerides file from {url}") - prx.util.try_repair_with_gfzrnx(local_file) - return local_file - except Exception as e: - log.warning(f"Could not download broadcast ephemerides file for {day}: {e}") - return None - - -def list_ftp_directory(server: str, folder: str): - ftp = FTP(server) - ftp.login() - ftp.cwd(folder) - dir_list = [] - ftp.dir(dir_list.append) - return [c.split()[-1].strip() for c in dir_list] +def check_online_availability(day: pd.Timestamp) -> bool: + """ + Check availability of NAV file on FTP server without downloading it. + """ + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + for server, remote_folder in IGS_FTP_SERVER.items(): + remote_folder_for_day = remote_folder + f"{day.year}/{day.day_of_year:03}" + ftp = ftplib.FTP(server) + ftp.login() + ftp.cwd(remote_folder_for_day) + try: + ftp.size(file) + return True + except ftplib.error_perm: + log.warning(f"{file} not available on {server}") + return False def try_downloading_ephemerides_ftp(day: pd.Timestamp, folder: Path): - server = "igs.ign.fr" - remote_folder = f"/pub/igs/data/{day.year}/{day.day_of_year:03}" - candidates = list_ftp_directory(server, remote_folder) - candidates = [ - c - for c in candidates - if f"_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" in c - ] - if len(candidates) == 0: - log.warning(f"Could not find broadcast ephemerides file for {day}") - return None - candidates = sorted( - candidates, - key=lambda x: int("BRDC00" in x) + int("IGS" in x), - reverse=True, - ) # - file = candidates[0] - ftp_file = f"ftp://{server}/{remote_folder}/{file}" + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" local_compressed_file = folder / file - urllib.request.urlretrieve(ftp_file, local_compressed_file) + for server, remote_folder in IGS_FTP_SERVER.items(): + remote_folder_for_day = remote_folder + f"{day.year}/{day.day_of_year:03}" + ftp_file = f"ftp://{server}/{remote_folder_for_day}/{file}" + urllib.request.urlretrieve(ftp_file, local_compressed_file) + if not local_compressed_file.exists(): + log.warning(f"Could not download {ftp_file}") + continue + local_file = converters.compressed_to_uncompressed(local_compressed_file) + os.remove(local_compressed_file) + log.info(f"Downloaded broadcast ephemerides file {ftp_file}") + prx.util.try_repair_with_gfzrnx(local_file) if not local_compressed_file.exists(): - log.warning(f"Could not download {ftp_file}") return None - local_file = converters.compressed_to_uncompressed(local_compressed_file) - os.remove(local_compressed_file) - log.info(f"Downloaded broadcast ephemerides file {ftp_file}") - prx.util.try_repair_with_gfzrnx(local_file) return local_file def try_downloading_ephemerides(mid_day: pd.Timestamp, folder: Path): - # Try downloading from HTTP server first, files on FTP server sometimes do not have all constellations - local_file = try_downloading_ephemerides_http(mid_day, folder) - if not local_file: - local_file = try_downloading_ephemerides_ftp(mid_day, folder) + local_file = try_downloading_ephemerides_ftp(mid_day, folder) if not local_file: log.warning(f"Could not download broadcast ephemerides for {mid_day}") return local_file diff --git a/src/prx/rinex_nav/test/test_nav_file_discovery.py b/src/prx/rinex_nav/test/test_nav_file_discovery.py index e251756..a54d555 100644 --- a/src/prx/rinex_nav/test/test_nav_file_discovery.py +++ b/src/prx/rinex_nav/test/test_nav_file_discovery.py @@ -56,6 +56,14 @@ def test_download_remote_ephemeris_files(set_up_test): assert isinstance(aux_files, dict) +def test_nav_file_remote_availability_after_2016(): + dates = [pd.Timestamp(y, 1, 1) for y in range(2016, pd.Timestamp.now().year)] + for date in dates: + assert aux.check_online_availability(date), ( + f"NAV file for day {date} not available" + ) + + def test_command_line_call(set_up_test): test_file = set_up_test["test_obs_file"] aux_file_script_path = ( From 4993412cf10cb0362ee4202a097c9a760b358901 Mon Sep 17 00:00:00 2001 From: Paul Thevenon Date: Sun, 18 Jan 2026 01:00:58 +0100 Subject: [PATCH 2/3] - bug fix when GLONASS not present in NAV file --- src/prx/rinex_nav/evaluate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/prx/rinex_nav/evaluate.py b/src/prx/rinex_nav/evaluate.py index 7ddbd2a..4f74dd1 100644 --- a/src/prx/rinex_nav/evaluate.py +++ b/src/prx/rinex_nav/evaluate.py @@ -524,7 +524,8 @@ def compute_ephemeris_and_clock_offset_reference_times(group): ) df = df.reset_index(drop=True) df = compute_gal_inav_fnav_indicators(df) - df["frequency_slot"] = df.FreqNum.where(df.sv.str[0] == "R", 1).astype(int) + if "R" in df.constellation.unique(): + df["frequency_slot"] = df.FreqNum.where(df.sv.str[0] == "R", 1).astype(int) df.attrs["ionospheric_corr_GPS"] = nav_ds.ionospheric_corr_GPS return df From 65b0148e7240a48473f568d1631da52ce9985ef8 Mon Sep 17 00:00:00 2001 From: Paul Thevenon Date: Wed, 4 Feb 2026 22:56:58 +0100 Subject: [PATCH 3/3] - re-add http BRDC NAV download --- src/prx/rinex_nav/nav_file_discovery.py | 59 ++++++++++++++++++- .../rinex_nav/test/test_nav_file_discovery.py | 12 +++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/prx/rinex_nav/nav_file_discovery.py b/src/prx/rinex_nav/nav_file_discovery.py index 869fa46..b35f599 100644 --- a/src/prx/rinex_nav/nav_file_discovery.py +++ b/src/prx/rinex_nav/nav_file_discovery.py @@ -4,6 +4,7 @@ from pathlib import Path import georinex import urllib.request +import urllib.error import pandas as pd import prx.util import ftplib @@ -26,7 +27,7 @@ def is_rinex_3_mixed_mgex_broadcast_ephemerides_file(file: Path): return bool(re.match(pattern, file.name)) -def check_online_availability(day: pd.Timestamp) -> bool: +def check_online_availability_ftp(day: pd.Timestamp) -> bool: """ Check availability of NAV file on FTP server without downloading it. """ @@ -44,9 +45,59 @@ def check_online_availability(day: pd.Timestamp) -> bool: return False -def try_downloading_ephemerides_ftp(day: pd.Timestamp, folder: Path): +def check_online_availability_http(day: pd.Timestamp) -> bool: + """ + Check if a remote file exists/accessible without downloading it. + """ + availability = False + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + remote_directory = ( + f"https://igs.bkg.bund.de/root_ftp/IGS/BRDC/{day.year}/{day.day_of_year:03}/" + ) + url = remote_directory + file + # Try HEAD first + try: + req = urllib.request.Request(url, method="HEAD") + _ = urllib.request.urlopen(req) + return True + except urllib.error.HTTPError: + availability = False + + # Fallback: request only the first byte so we avoid full download + try: + req = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + _ = urllib.request.urlopen(req) + return True + except urllib.error.HTTPError: + availability = False + + return availability + + +def try_downloading_ephemerides_http(day: pd.Timestamp, local_destination_folder: Path): + # IGS BKG Rinex 3.04 mixed file paths follow this pattern: + # https://igs.bkg.bund.de/root_ftp/IGS/BRDC/2023/002/BRDC00IGS_R_20230020000_01D_MN.rnx.gz file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" - local_compressed_file = folder / file + remote_directory = ( + f"https://igs.bkg.bund.de/root_ftp/IGS/BRDC/{day.year}/{day.day_of_year:03}/" + ) + try: + local_compressed_file = local_destination_folder / file + url = remote_directory + file + urllib.request.urlretrieve(url, local_compressed_file) + local_file = converters.compressed_to_uncompressed(local_compressed_file) + os.remove(local_compressed_file) + log.info(f"Downloaded broadcast ephemerides file from {url}") + prx.util.try_repair_with_gfzrnx(local_file) + return local_file + except Exception as e: + log.warning(f"Could not download broadcast ephemerides file for {day}: {e}") + return None + + +def try_downloading_ephemerides_ftp(day: pd.Timestamp, local_destination_folder: Path): + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + local_compressed_file = local_destination_folder / file for server, remote_folder in IGS_FTP_SERVER.items(): remote_folder_for_day = remote_folder + f"{day.year}/{day.day_of_year:03}" ftp_file = f"ftp://{server}/{remote_folder_for_day}/{file}" @@ -65,6 +116,8 @@ def try_downloading_ephemerides_ftp(day: pd.Timestamp, folder: Path): def try_downloading_ephemerides(mid_day: pd.Timestamp, folder: Path): local_file = try_downloading_ephemerides_ftp(mid_day, folder) + if not local_file: + local_file = try_downloading_ephemerides_http(mid_day, folder) if not local_file: log.warning(f"Could not download broadcast ephemerides for {mid_day}") return local_file diff --git a/src/prx/rinex_nav/test/test_nav_file_discovery.py b/src/prx/rinex_nav/test/test_nav_file_discovery.py index a54d555..681274b 100644 --- a/src/prx/rinex_nav/test/test_nav_file_discovery.py +++ b/src/prx/rinex_nav/test/test_nav_file_discovery.py @@ -56,10 +56,18 @@ def test_download_remote_ephemeris_files(set_up_test): assert isinstance(aux_files, dict) -def test_nav_file_remote_availability_after_2016(): +def test_nav_file_ftp_availability_after_2016(): dates = [pd.Timestamp(y, 1, 1) for y in range(2016, pd.Timestamp.now().year)] for date in dates: - assert aux.check_online_availability(date), ( + assert aux.check_online_availability_ftp(date), ( + f"NAV file for day {date} not available" + ) + + +def test_nav_file_http_availability_after_2022(): + dates = [pd.Timestamp(y, 1, 1) for y in range(2022, pd.Timestamp.now().year)] + for date in dates: + assert aux.check_online_availability_http(date), ( f"NAV file for day {date} not available" )