From b6a5ea890f0b3e5eb08aa3f46ec33b20c632db7d Mon Sep 17 00:00:00 2001 From: Guillaume Maze Date: Tue, 25 Apr 2023 16:46:15 +0200 Subject: [PATCH 1/6] Update url.py allow for login user/password --- erddapy/core/url.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 351b37ee..e5413be8 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import copy import functools import io @@ -61,12 +62,13 @@ def _sort_url(url: str) -> str: def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO: if "timeout" not in kwargs: kwargs["timeout"] = 60 - response = httpx.get( - quote_url(url), - follow_redirects=True, - auth=auth, - **kwargs, - ) + with httpx.Client() as client: + p = parse.urlparse(url) + protocol = 'tabledap' if 'tabledap' in p.path else 'griddap' + login_page = "%s://%s%s/login.html" % (p.scheme, p.netloc, p.path.split('/%s/' % protocol)[0]) + client.post(login_page, data={'user': os.getenv("ERDDAP_USERNAME"), 'password': os.getenv("ERDDAP_PASSWORD")}) + response = client.get(url, follow_redirects=True, auth=auth, **kwargs) + try: response.raise_for_status() except httpx.HTTPError as err: From 8a46f0b43155fdfe47180a1358c27a06a9cff20c Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 14:48:09 -0300 Subject: [PATCH 2/6] don't vcr simple tests --- tests/cassettes/test_urlopen_raise.yaml | 318 ------------------------ 1 file changed, 318 deletions(-) delete mode 100644 tests/cassettes/test_urlopen_raise.yaml diff --git a/tests/cassettes/test_urlopen_raise.yaml b/tests/cassettes/test_urlopen_raise.yaml deleted file mode 100644 index 64917aa1..00000000 --- a/tests/cassettes/test_urlopen_raise.yaml +++ /dev/null @@ -1,318 +0,0 @@ -interactions: -- request: - body: '' - headers: - accept: - - '*/*' - accept-encoding: - - gzip, deflate, br - connection: - - keep-alive - host: - - developer.mozilla.org - user-agent: - - python-httpx/0.23.0 - method: GET - uri: https://developer.mozilla.org/en-US/404 - response: - content: "\U0001F937\U0001F3FD\u200D\u2640\uFE0F - Page not found | null
MDN Web Docs
" - headers: - Cache-Control: - - no-store, must-revalidate - Connection: - - keep-alive - Content-Encoding: - - br - Content-Security-Policy: - - default-src 'self'; script-src 'report-sample' 'self' www.google-analytics.com/analytics.js - 'sha256-JEt9Nmc3BP88wxuTZm9aKNu87vEgGmKW1zzy/vb1KPs=' polyfill.io/v3/polyfill.min.js - assets.codepen.io production-assets.codepen.io 'sha256-x6Tv+AdV5e6dcolO0TEo+3BG4H2nG2ACjyG8mz6QCes=' - 'sha256-GA8+DpFnqAM/vwERTpb5zyLUaN5KnOhctfTsqWfhaUA='; script-src-elem 'report-sample' - 'self' www.google-analytics.com/analytics.js 'sha256-JEt9Nmc3BP88wxuTZm9aKNu87vEgGmKW1zzy/vb1KPs=' - polyfill.io/v3/polyfill.min.js assets.codepen.io production-assets.codepen.io - 'sha256-x6Tv+AdV5e6dcolO0TEo+3BG4H2nG2ACjyG8mz6QCes=' 'sha256-GA8+DpFnqAM/vwERTpb5zyLUaN5KnOhctfTsqWfhaUA='; - style-src 'report-sample' 'self' 'unsafe-inline'; object-src 'none'; base-uri - 'self'; connect-src 'self' updates.developer.allizom.org updates.developer.mozilla.org - www.google-analytics.com stats.g.doubleclick.net; font-src 'self'; frame-src - 'self' interactive-examples.mdn.mozilla.net interactive-examples.prod.mdn.mozilla.net - interactive-examples.stage.mdn.mozilla.net mdn.github.io yari-demos.prod.mdn.mozit.cloud - mdn.mozillademos.org yari-demos.stage.mdn.mozit.cloud jsfiddle.net www.youtube-nocookie.com - codepen.io www.surveygizmo.com; img-src 'self' *.githubusercontent.com *.googleusercontent.com - *.gravatar.com mozillausercontent.com firefoxusercontent.com profile.stage.mozaws.net - profile.accounts.firefox.com mdn.mozillademos.org media.prod.mdn.mozit.cloud - media.stage.mdn.mozit.cloud interactive-examples.mdn.mozilla.net interactive-examples.prod.mdn.mozilla.net - interactive-examples.stage.mdn.mozilla.net wikipedia.org www.google-analytics.com - www.gstatic.com; manifest-src 'self'; media-src 'self' archive.org videos.cdn.mozilla.net; - child-src 'self'; worker-src 'self'; - Content-Type: - - text/html; charset=utf-8 - Date: - - Wed, 27 Jul 2022 14:57:55 GMT - ETag: - - W/"eb71930fca1bf0f5a1c58df01562ab83" - Last-Modified: - - Wed, 27 Jul 2022 00:41:56 GMT - Server: - - AmazonS3 - Strict-Transport-Security: - - max-age=63072000 - Transfer-Encoding: - - chunked - Vary: - - Accept-Encoding - Via: - - 1.1 c59efe844fd614ff20a756172908fed0.cloudfront.net (CloudFront) - X-Amz-Cf-Id: - - OI75oo7LYqVH6UtsN-ZM_Npub4Dwab18TTS_VaLnbdwgwRjCJaY0uA== - X-Amz-Cf-Pop: - - GRU1-C1 - X-Cache: - - Error from cloudfront - X-Content-Type-Options: - - nosniff - X-Frame-Options: - - DENY - X-XSS-Protection: - - 1; mode=block - http_version: HTTP/1.1 - status_code: 404 -version: 1 From 1ce92922546386bfa7ea6a6af431c35dac1ffc07 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 14:48:25 -0300 Subject: [PATCH 3/6] run pre-commits --- erddapy/core/url.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index e5413be8..5c4606c0 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -6,6 +6,7 @@ import copy import functools import io +import os from collections import OrderedDict from typing import TYPE_CHECKING @@ -64,9 +65,16 @@ def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO: kwargs["timeout"] = 60 with httpx.Client() as client: p = parse.urlparse(url) - protocol = 'tabledap' if 'tabledap' in p.path else 'griddap' - login_page = "%s://%s%s/login.html" % (p.scheme, p.netloc, p.path.split('/%s/' % protocol)[0]) - client.post(login_page, data={'user': os.getenv("ERDDAP_USERNAME"), 'password': os.getenv("ERDDAP_PASSWORD")}) + protocol = "tabledap" if "tabledap" in p.path else "griddap" + base = p.path.split(f"/{protocol}/")[0] + login_page = f"{p.scheme}://{p.netloc}{base}/login.html" + client.post( + login_page, + data={ + "user": os.getenv("ERDDAP_USERNAME"), + "password": os.getenv("ERDDAP_PASSWORD"), + }, + ) response = client.get(url, follow_redirects=True, auth=auth, **kwargs) try: From ce1a71774a941b9f65d46975f1cd74036f507c16 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Nov 2023 16:45:38 -0300 Subject: [PATCH 4/6] don't use env vars --- erddapy/core/url.py | 19 +++++++++++-------- erddapy/erddapy.py | 7 ++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 5c4606c0..8f55fa58 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -63,18 +63,21 @@ def _sort_url(url: str) -> str: def _urlopen(url: str, auth: tuple | None = None, **kwargs: dict) -> BinaryIO: if "timeout" not in kwargs: kwargs["timeout"] = 60 + user = kwargs.pop("user", None) + password = kwargs.pop("password", None) with httpx.Client() as client: p = parse.urlparse(url) protocol = "tabledap" if "tabledap" in p.path else "griddap" base = p.path.split(f"/{protocol}/")[0] - login_page = f"{p.scheme}://{p.netloc}{base}/login.html" - client.post( - login_page, - data={ - "user": os.getenv("ERDDAP_USERNAME"), - "password": os.getenv("ERDDAP_PASSWORD"), - }, - ) + if user is not None and password is not None: + login_page = f"{p.scheme}://{p.netloc}{base}/login.html" + client.post( + login_page, + data={ + "user": f"{user}", + "password": f"{password}", + }, + ) response = client.get(url, follow_redirects=True, auth=auth, **kwargs) try: diff --git a/erddapy/erddapy.py b/erddapy/erddapy.py index ac8c4f66..d53cd248 100644 --- a/erddapy/erddapy.py +++ b/erddapy/erddapy.py @@ -417,11 +417,8 @@ def to_pandas( response = kw.pop("response", "csvp") distinct = kw.pop("distinct", False) url = self.get_download_url(response=response, distinct=distinct) - return to_pandas( - url, - requests_kwargs=requests_kwargs, - pandas_kwargs=dict(**kw), - ) + requests_kwargs = requests_kwargs if requests_kwargs else self.requests_kwargs + return to_pandas(url, requests_kwargs=requests_kwargs, pandas_kwargs=dict(**kw)) def to_ncCF( # noqa: N802 self: ERDDAP, From 828dd26791f23693c48f7ec9608c92061d7a9391 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 27 Feb 2024 14:14:40 -0300 Subject: [PATCH 5/6] fix lint --- erddapy/core/url.py | 1 - 1 file changed, 1 deletion(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 8f55fa58..2d510376 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -6,7 +6,6 @@ import copy import functools import io -import os from collections import OrderedDict from typing import TYPE_CHECKING From 40de1934e9664b9119e339fb3e8c21222f0cc333 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 5 Mar 2024 13:46:47 -0300 Subject: [PATCH 6/6] lint --- erddapy/core/url.py | 1 - erddapy/erddapy.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/erddapy/core/url.py b/erddapy/core/url.py index 2d510376..211f5603 100644 --- a/erddapy/core/url.py +++ b/erddapy/core/url.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import copy import functools import io diff --git a/erddapy/erddapy.py b/erddapy/erddapy.py index d53cd248..d4b233cc 100644 --- a/erddapy/erddapy.py +++ b/erddapy/erddapy.py @@ -417,8 +417,14 @@ def to_pandas( response = kw.pop("response", "csvp") distinct = kw.pop("distinct", False) url = self.get_download_url(response=response, distinct=distinct) - requests_kwargs = requests_kwargs if requests_kwargs else self.requests_kwargs - return to_pandas(url, requests_kwargs=requests_kwargs, pandas_kwargs=dict(**kw)) + requests_kwargs = ( + requests_kwargs if requests_kwargs else self.requests_kwargs + ) + return to_pandas( + url, + requests_kwargs=requests_kwargs, + pandas_kwargs=dict(**kw), + ) def to_ncCF( # noqa: N802 self: ERDDAP,