diff --git a/src/prx/rinex_nav/evaluate.py b/src/prx/rinex_nav/evaluate.py index 7ddbd2a..4f74dd1 100644 --- a/src/prx/rinex_nav/evaluate.py +++ b/src/prx/rinex_nav/evaluate.py @@ -524,7 +524,8 @@ def compute_ephemeris_and_clock_offset_reference_times(group): ) df = df.reset_index(drop=True) df = compute_gal_inav_fnav_indicators(df) - df["frequency_slot"] = df.FreqNum.where(df.sv.str[0] == "R", 1).astype(int) + if "R" in df.constellation.unique(): + df["frequency_slot"] = df.FreqNum.where(df.sv.str[0] == "R", 1).astype(int) df.attrs["ionospheric_corr_GPS"] = nav_ds.ionospheric_corr_GPS return df diff --git a/src/prx/rinex_nav/nav_file_discovery.py b/src/prx/rinex_nav/nav_file_discovery.py index ac5f1bf..b35f599 100644 --- a/src/prx/rinex_nav/nav_file_discovery.py +++ b/src/prx/rinex_nav/nav_file_discovery.py @@ -1,13 +1,13 @@ import argparse import os import re -from ftplib import FTP from pathlib import Path import georinex import urllib.request +import urllib.error import pandas as pd import prx.util -import requests +import ftplib from prx import converters, util from prx.converters import anything_to_rinex_3 @@ -16,27 +16,72 @@ log = util.get_logger(__name__) +IGS_FTP_SERVER = { + "gssc.esa.int": "/gnss/data/daily/", + "igs.ign.fr": "/pub/igs/data/", +} + def is_rinex_3_mixed_mgex_broadcast_ephemerides_file(file: Path): pattern = r"^[A-Za-z0-9]{9}_[A-Za-z]_\d{11}_[A-Za-z0-9]{3}_[A-Za-z]N\.rnx.*" return bool(re.match(pattern, file.name)) +def check_online_availability_ftp(day: pd.Timestamp) -> bool: + """ + Check availability of NAV file on FTP server without downloading it. + """ + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + for server, remote_folder in IGS_FTP_SERVER.items(): + remote_folder_for_day = remote_folder + f"{day.year}/{day.day_of_year:03}" + ftp = ftplib.FTP(server) + ftp.login() + ftp.cwd(remote_folder_for_day) + try: + ftp.size(file) + return True + except ftplib.error_perm: + log.warning(f"{file} not available on {server}") + return False + + +def check_online_availability_http(day: pd.Timestamp) -> bool: + """ + Check if a remote file exists/accessible without downloading it. + """ + availability = False + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + remote_directory = ( + f"https://igs.bkg.bund.de/root_ftp/IGS/BRDC/{day.year}/{day.day_of_year:03}/" + ) + url = remote_directory + file + # Try HEAD first + try: + req = urllib.request.Request(url, method="HEAD") + _ = urllib.request.urlopen(req) + return True + except urllib.error.HTTPError: + availability = False + + # Fallback: request only the first byte so we avoid full download + try: + req = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + _ = urllib.request.urlopen(req) + return True + except urllib.error.HTTPError: + availability = False + + return availability + + def try_downloading_ephemerides_http(day: pd.Timestamp, local_destination_folder: Path): # IGS BKG Rinex 3.04 mixed file paths follow this pattern: - # https://igs.bkg.bund.de/root_ftp/IGS/BRDC/2023/002/BRDC00_R_20230020000_01D_MN.rnx.gz - file_regex = f"BRDC00(?:IGS|WRD)_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + # https://igs.bkg.bund.de/root_ftp/IGS/BRDC/2023/002/BRDC00IGS_R_20230020000_01D_MN.rnx.gz + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" remote_directory = ( f"https://igs.bkg.bund.de/root_ftp/IGS/BRDC/{day.year}/{day.day_of_year:03}/" ) try: - # List available files whose names fit the pattern - directory_listing = requests.get(remote_directory, timeout=30).text - matches = list(set(re.findall(file_regex, directory_listing))) - if len(matches) == 0: - log.warning(f"Could not find broadcast ephemerides file for {day}") - return None - file = sorted(matches, key=lambda x: int("IGS" in x), reverse=True)[0] local_compressed_file = local_destination_folder / file url = remote_directory + file urllib.request.urlretrieve(url, local_compressed_file) @@ -50,51 +95,29 @@ def try_downloading_ephemerides_http(day: pd.Timestamp, local_destination_folder return None -def list_ftp_directory(server: str, folder: str): - ftp = FTP(server) - ftp.login() - ftp.cwd(folder) - dir_list = [] - ftp.dir(dir_list.append) - return [c.split()[-1].strip() for c in dir_list] - - -def try_downloading_ephemerides_ftp(day: pd.Timestamp, folder: Path): - server = "igs.ign.fr" - remote_folder = f"/pub/igs/data/{day.year}/{day.day_of_year:03}" - candidates = list_ftp_directory(server, remote_folder) - candidates = [ - c - for c in candidates - if f"_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" in c - ] - if len(candidates) == 0: - log.warning(f"Could not find broadcast ephemerides file for {day}") - return None - candidates = sorted( - candidates, - key=lambda x: int("BRDC00" in x) + int("IGS" in x), - reverse=True, - ) # - file = candidates[0] - ftp_file = f"ftp://{server}/{remote_folder}/{file}" - local_compressed_file = folder / file - urllib.request.urlretrieve(ftp_file, local_compressed_file) +def try_downloading_ephemerides_ftp(day: pd.Timestamp, local_destination_folder: Path): + file = f"BRDC00IGS_R_{day.year}{day.day_of_year:03}0000_01D_MN.rnx.gz" + local_compressed_file = local_destination_folder / file + for server, remote_folder in IGS_FTP_SERVER.items(): + remote_folder_for_day = remote_folder + f"{day.year}/{day.day_of_year:03}" + ftp_file = f"ftp://{server}/{remote_folder_for_day}/{file}" + urllib.request.urlretrieve(ftp_file, local_compressed_file) + if not local_compressed_file.exists(): + log.warning(f"Could not download {ftp_file}") + continue + local_file = converters.compressed_to_uncompressed(local_compressed_file) + os.remove(local_compressed_file) + log.info(f"Downloaded broadcast ephemerides file {ftp_file}") + prx.util.try_repair_with_gfzrnx(local_file) if not local_compressed_file.exists(): - log.warning(f"Could not download {ftp_file}") return None - local_file = converters.compressed_to_uncompressed(local_compressed_file) - os.remove(local_compressed_file) - log.info(f"Downloaded broadcast ephemerides file {ftp_file}") - prx.util.try_repair_with_gfzrnx(local_file) return local_file def try_downloading_ephemerides(mid_day: pd.Timestamp, folder: Path): - # Try downloading from HTTP server first, files on FTP server sometimes do not have all constellations - local_file = try_downloading_ephemerides_http(mid_day, folder) + local_file = try_downloading_ephemerides_ftp(mid_day, folder) if not local_file: - local_file = try_downloading_ephemerides_ftp(mid_day, folder) + local_file = try_downloading_ephemerides_http(mid_day, folder) if not local_file: log.warning(f"Could not download broadcast ephemerides for {mid_day}") return local_file diff --git a/src/prx/rinex_nav/test/test_nav_file_discovery.py b/src/prx/rinex_nav/test/test_nav_file_discovery.py index e251756..681274b 100644 --- a/src/prx/rinex_nav/test/test_nav_file_discovery.py +++ b/src/prx/rinex_nav/test/test_nav_file_discovery.py @@ -56,6 +56,22 @@ def test_download_remote_ephemeris_files(set_up_test): assert isinstance(aux_files, dict) +def test_nav_file_ftp_availability_after_2016(): + dates = [pd.Timestamp(y, 1, 1) for y in range(2016, pd.Timestamp.now().year)] + for date in dates: + assert aux.check_online_availability_ftp(date), ( + f"NAV file for day {date} not available" + ) + + +def test_nav_file_http_availability_after_2022(): + dates = [pd.Timestamp(y, 1, 1) for y in range(2022, pd.Timestamp.now().year)] + for date in dates: + assert aux.check_online_availability_http(date), ( + f"NAV file for day {date} not available" + ) + + def test_command_line_call(set_up_test): test_file = set_up_test["test_obs_file"] aux_file_script_path = (