From 16e9ff1d2b8f8ad4ab793f55ca4a7216b73b6a0b Mon Sep 17 00:00:00 2001 From: Callum McKenna Date: Wed, 7 Apr 2021 11:57:58 +1000 Subject: [PATCH 1/5] SEN-498: Add support for OpenSearch API --- fetch/load.py | 3 ++- fetch/opensearch.py | 58 +++++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 fetch/opensearch.py diff --git a/fetch/load.py b/fetch/load.py index 18b4c0d..0cf4435 100755 --- a/fetch/load.py +++ b/fetch/load.py @@ -13,7 +13,7 @@ import yaml.resolver from croniter import croniter -from . import ftp, http, ecmwf +from . import ftp, http, ecmwf, opensearch from ._core import RegexpOutputPathTransform, DateRangeSource, DateFilenameTransform, \ RsyncMirrorSource, SimpleObject, ShellFileProcessor from .util import remove_nones @@ -367,6 +367,7 @@ def add_item_constructor(source, type_annotation, attribute): add_default_constructor(http.HttpPostAction, '!http-post') add_default_constructor(http.HttpAuthAction, '!http-auth') add_default_constructor(ecmwf.EcmwfApiSource, '!ecmwf-api') + add_default_constructor(opensearch.OpenSearchApiSource, '!opensearch-api') add_item_constructor(RegexpOutputPathTransform, '!regexp-extract', 'pattern') add_item_constructor(DateFilenameTransform, '!date-pattern', 'format_') diff --git a/fetch/opensearch.py b/fetch/opensearch.py new file mode 100644 index 0000000..375d94d --- /dev/null +++ b/fetch/opensearch.py @@ -0,0 +1,58 @@ +from __future__ import absolute_import + +import logging +import os + +from sentinelsat import SentinelAPI + +from ._core import DataSource, fetch_file + +_log = logging.getLogger(__name__) + + +class OpenSearchApiSource(DataSource): + """ + Class for data retrievals using the OpenSearch API. + """ + + def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, timeout=None, filename_transform=None, override_existing=False): + self.target_dir = target_dir + self.filename_transform = filename_transform + self.override_existing = override_existing + self.api_url = api_url + self.username = username + self.password = password + self.query = query + self.show_progressbars = show_progressbars + self.timeout = timeout + + self.api = SentinelAPI(self.username, self.password, self.api_url, self.show_progressbars, self.timeout) + + def trigger(self, reporter): + """ + :type reporter: ResultHandler + """ + + query_results = self.api.query(**self.query) + + for (key, result) in query_results.items(): + _log.info('Found %s with uuid %s', result['filename'], key) + + def opensearch_fetch(target): + download = self.api.download(key) + + # Workaround for fixed filename + _log.debug('Renaming %s to %s', download['path'], target) + os.rename(download['path'], target) + + return True + + fetch_file( + result['link'].replace("'", '%27'), + opensearch_fetch, + reporter, + os.path.basename(result['filename']), + self.target_dir, + filename_transform=self.filename_transform, + override_existing=self.override_existing + ) diff --git a/setup.py b/setup.py index d953bd9..7d0c903 100755 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ 'croniter', 'feedparser', 'lxml', + 'sentinelsat', 'pathlib;python_version<"3.4"', 'pyyaml<5.1', 'requests>=2.21.0', From 81e4fac0d047741e30f45280ebfa83417313d7b3 Mon Sep 17 00:00:00 2001 From: Callum McKenna Date: Wed, 7 Apr 2021 15:20:10 +1000 Subject: [PATCH 2/5] SEN-498: Add opensearch-api to example YAML --- example-config.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/example-config.yaml b/example-config.yaml index db867af..cb604a9 100755 --- a/example-config.yaml +++ b/example-config.yaml @@ -290,3 +290,21 @@ rules: command: 'swfo-convert mcd43a1 h5-md --fname {brdf_base}/{collection}/{ymd}/{basename}{hdf_ext} --outdir /tmp/data/conversion/BRDF/{collection}/{ymd}/ --filter-opts ''{{"aggression": 6}}'' --compression BLOSC_ZSTANDARD' input_files: ['^(?P.*BRDF)/(?P.*)/(?P[0-9]{4}\.[0-9]{2}\.[0-9]{2})/(?P.*)(?P.hdf)(?P.xml)?', ['{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}', '{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}.xml']] expect_file: '/tmp/data/conversion/BRDF/{collection}/{ymd}/{basename}.h5' + +# +#------------------------------- +# Example using the OpenSearch API | +#------------------------------- +# + RESORB: + schedule: '0 */6 * * *' + source: !opensearch-api + username: + password: + api_url: https://scihub.copernicus.eu/gnss + query: + platformname: Sentinel-1 + producttype: AUX_RESORB + generationdate: '[NOW-5DAYS TO NOW]' + filename_transform: !regexp-extract '(?PS1[A,B]{1})_[A-Z].*_[A-Z].*_(?P[A-Z].*)_[A-Z].*_(?P[0-9,T]{15})_V(?P[0-9,T]{15})_(?P[0-9,T]{15}).EOF' + target_dir: /tmp/data/sensor-specific/Sentinel-1/{product}/{platform} \ No newline at end of file From b22e1b66fe43119b62067dd063b7d37239b65b7f Mon Sep 17 00:00:00 2001 From: Callum McKenna Date: Wed, 7 Apr 2021 16:01:43 +1000 Subject: [PATCH 3/5] SEN-498: Fix lint issues --- fetch/opensearch.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/fetch/opensearch.py b/fetch/opensearch.py index 375d94d..5581161 100644 --- a/fetch/opensearch.py +++ b/fetch/opensearch.py @@ -15,7 +15,8 @@ class OpenSearchApiSource(DataSource): Class for data retrievals using the OpenSearch API. """ - def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, timeout=None, filename_transform=None, override_existing=False): + def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, timeout=None, + filename_transform=None, override_existing=False): self.target_dir = target_dir self.filename_transform = filename_transform self.override_existing = override_existing @@ -35,21 +36,24 @@ def trigger(self, reporter): query_results = self.api.query(**self.query) - for (key, result) in query_results.items(): - _log.info('Found %s with uuid %s', result['filename'], key) + for (uuid, result) in query_results.items(): + _log.info('Found %s with uuid %s', result['filename'], uuid) - def opensearch_fetch(target): - download = self.api.download(key) + def create_fetch_function(key): + def opensearch_fetch(target): + download = self.api.download(key) - # Workaround for fixed filename - _log.debug('Renaming %s to %s', download['path'], target) - os.rename(download['path'], target) + # Workaround for fixed filename + _log.debug('Renaming %s to %s', download['path'], target) + os.rename(download['path'], target) - return True + return True + + return opensearch_fetch fetch_file( result['link'].replace("'", '%27'), - opensearch_fetch, + create_fetch_function(uuid), reporter, os.path.basename(result['filename']), self.target_dir, From 2ce839b53ed0d6f3d339d6486e85f1bb5daecd26 Mon Sep 17 00:00:00 2001 From: Callum McKenna Date: Thu, 22 Apr 2021 15:13:10 +1000 Subject: [PATCH 4/5] SEN-498: Add default connection timeout to OpenSearchApiSource --- fetch/opensearch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fetch/opensearch.py b/fetch/opensearch.py index 5581161..b10226a 100644 --- a/fetch/opensearch.py +++ b/fetch/opensearch.py @@ -7,6 +7,8 @@ from ._core import DataSource, fetch_file +DEFAULT_CONNECT_TIMEOUT_SECS = 100 + _log = logging.getLogger(__name__) @@ -15,7 +17,7 @@ class OpenSearchApiSource(DataSource): Class for data retrievals using the OpenSearch API. """ - def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, timeout=None, + def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, timeout=DEFAULT_CONNECT_TIMEOUT_SECS, filename_transform=None, override_existing=False): self.target_dir = target_dir self.filename_transform = filename_transform From a26e60ede494c24d00f6cefd06110ffd2bad007d Mon Sep 17 00:00:00 2001 From: Callum McKenna Date: Thu, 22 Apr 2021 15:17:04 +1000 Subject: [PATCH 5/5] SEN-498: Fix lint issues --- fetch/opensearch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fetch/opensearch.py b/fetch/opensearch.py index b10226a..5532f70 100644 --- a/fetch/opensearch.py +++ b/fetch/opensearch.py @@ -17,8 +17,8 @@ class OpenSearchApiSource(DataSource): Class for data retrievals using the OpenSearch API. """ - def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, timeout=DEFAULT_CONNECT_TIMEOUT_SECS, - filename_transform=None, override_existing=False): + def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False, + timeout=DEFAULT_CONNECT_TIMEOUT_SECS, filename_transform=None, override_existing=False): self.target_dir = target_dir self.filename_transform = filename_transform self.override_existing = override_existing