From 7ee23d2b380746d6dfdf3667de7690c4379a5cd7 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Mon, 7 Jul 2025 17:20:30 +0100 Subject: [PATCH 1/8] Make api.pvlive.uk the default domain URL Fixes https://github.com/SheffieldSolar/PV_Live-API/issues/55 --- pvlive_api/pvlive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlive_api/pvlive.py b/pvlive_api/pvlive.py index 1302432..0c0c211 100644 --- a/pvlive_api/pvlive.py +++ b/pvlive_api/pvlive.py @@ -54,10 +54,10 @@ def __init__( proxies: Optional[Dict] = None, ssl_verify: bool = True, domain_url: Literal[ - "api0.solar.sheffield.ac.uk", + "api.pvlive.uk", "api.solar.sheffield.ac.uk", - "api.pvlive.uk" - ] = "api.solar.sheffield.ac.uk" + "api0.solar.sheffield.ac.uk", + ] = "api.pvlive.uk" ): valid_domain_urls = [ "api0.solar.sheffield.ac.uk", From 4bbd04229fd01b4260ceb0dd0afd7a419ad7e6f9 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 14:06:33 +0100 Subject: [PATCH 2/8] Update README.md Add a note about domain_url --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e33dbd3..d76e4c4 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ A Python implementation of the PV_Live web API. See https://www.solar.sheffield. ## Usage +As of 2025-07-07, the production PV_Live API is hosted on Google Cloud Platform (GCP) at https://api.pvlive.uk. There is a also non-prod test/fix-on-fail (FOF) environment hosted on TUOS IT: https://api.solar.sheffield.ac.uk. To support switching between the two, the `pvlive-api` package exposes a parameter `domain_url`, which can be set to one of `["api.pvlive.uk", "api.solar.sheffield.ac.uk"]` but defaults to `api.pvlive.uk`. + There are three methods for extracting raw data from the PV_Live API: |Method|Description|Docs Link| From 148a7b9f739f912ec1fd450585b0fe381c978a73 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 17:46:56 +0100 Subject: [PATCH 3/8] Update class docstring with missing params --- pvlive_api/pvlive.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pvlive_api/pvlive.py b/pvlive_api/pvlive.py index 0c0c211..ae2f13f 100644 --- a/pvlive_api/pvlive.py +++ b/pvlive_api/pvlive.py @@ -47,6 +47,12 @@ class PVLive: proxies : Optional[Dict] Optionally specify a Dict of proxies for http and https requests in the format: {"http": "
", "https": "
"} + ssl_verify : Bool + Set to False to disable SSL cert checks when querying the API. Useful if e.g. your company + network/VPN has issues propagating SSL certs. Defaults to True. Not recommended to set to + False! + domain_url : Literal["api.pvlive.uk", "api.solar.sheffield.ac.uk", "api0.solar.sheffield.ac.uk"] + Used to switch between the production PV_Live API on GCP and the test/FOF server on-prem. """ def __init__( self, From eadd64996f1705ae23c747a6f3d2252a88b68f68 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 17:48:48 +0100 Subject: [PATCH 4/8] Update code to reflect changes to capacity API endpoint --- pvlive_api/pvlive.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pvlive_api/pvlive.py b/pvlive_api/pvlive.py index ae2f13f..e697a35 100644 --- a/pvlive_api/pvlive.py +++ b/pvlive_api/pvlive.py @@ -85,6 +85,7 @@ def __init__( self.pes_list = self._get_pes_list() self.gsp_ids = self.gsp_list.gsp_id.dropna().astype(int64).unique() self.pes_ids = self.pes_list.pes_id.dropna().astype(int64).unique() + self.deployment_datasets = None self.deployment_releases = None def _get_gsp_list(self): @@ -100,15 +101,15 @@ def _get_pes_list(self): return pd.DataFrame(response["data"], columns=response["meta"]) def _get_deployment_releases(self): - """Get a list of deployment releases as datestamps (YYYYMMDD).""" - if self.deployment_releases is None: + """ + Get the available deployment releases as a list of datestamps (YYYYMMDD) alongside a dict of + files available in each release. + """ + if self.deployment_datasets is None: url = f"{self.domain_url}/capacity/" - response = self._fetch_url(url, parse_json=False) - soup = BeautifulSoup(response.content, "html.parser") - releases = [r["href"].strip("/") for r in soup.find_all("a", href=True) - if re.match(r"[0-9]{8}/", r["href"])] - self.deployment_releases = sorted(releases, reverse=True) - return self.deployment_releases + self.deployment_datasets = self._fetch_url(url, parse_json=True) + self.deployment_releases = sorted(list(self.deployment_datasets.keys()), reverse=True) + return self.deployment_datasets, self.deployment_releases def _get_deployment_filenames(self, release): """Get a list of filenames for a given release.""" @@ -121,7 +122,7 @@ def _get_deployment_filenames(self, release): def _validate_deployment_inputs(self, region, include_history, by_system_size, release): """Validate input parameters to `deployment()`.""" - releases = self._get_deployment_releases() + deployment_datasets, releases = self._get_deployment_releases() if not isinstance(region, str): raise TypeError("`region` must be a string.") supported_regions = ["gsp", "llsoa"] @@ -171,15 +172,15 @@ def deployment(self, GSPs/llsoa and dc_capacity_mwp. """ self._validate_deployment_inputs(region, include_history, by_system_size, release) - releases = self._get_deployment_releases() + deployment_datasets, releases = self._get_deployment_releases() release = releases[release] if isinstance(release, int) else release - filenames = self._get_deployment_filenames(release) region_ = "20220314_GSP" if region == "gsp" else region + filenames = list(deployment_datasets[release].keys()) history_ = "_and_month" if include_history else "" system_size_ = "_and_system_size" if by_system_size else "" filename_ending = f"_capacity_by_{region_}{history_}{system_size_}.csv.gz" filename = [f for f in filenames if f.endswith(filename_ending)][0] - url = f"{self.domain_url}/capacity/{release}/{filename}" + url = deployment_datasets[release][filename] kwargs = dict(parse_dates=["install_month"]) if include_history else {} response = self._fetch_url(url, parse_json=False) mock_file = BytesIO(response.content) From 2bbdf197998acbdea78feb9dc90e6ad17f168303 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 17:49:55 +0100 Subject: [PATCH 5/8] Add support for specifying the GSP boundaries version Needed for backwards compatibility (some capacity files include it in the filename) --- pvlive_api/pvlive.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pvlive_api/pvlive.py b/pvlive_api/pvlive.py index e697a35..339da9c 100644 --- a/pvlive_api/pvlive.py +++ b/pvlive_api/pvlive.py @@ -53,6 +53,8 @@ class PVLive: False! domain_url : Literal["api.pvlive.uk", "api.solar.sheffield.ac.uk", "api0.solar.sheffield.ac.uk"] Used to switch between the production PV_Live API on GCP and the test/FOF server on-prem. + gsp_boundaries_version : str + Some capacity datasets (by GSP) are encoded with the version of the GSP boundaries used. """ def __init__( self, @@ -63,7 +65,8 @@ def __init__( "api.pvlive.uk", "api.solar.sheffield.ac.uk", "api0.solar.sheffield.ac.uk", - ] = "api.pvlive.uk" + ] = "api.pvlive.uk", + gsp_boundaries_version: str = "20250109", ): valid_domain_urls = [ "api0.solar.sheffield.ac.uk", @@ -87,6 +90,7 @@ def __init__( self.pes_ids = self.pes_list.pes_id.dropna().astype(int64).unique() self.deployment_datasets = None self.deployment_releases = None + self.gsp_boundaries_version = gsp_boundaries_version def _get_gsp_list(self): """Fetch the GSP list from the API and convert to Pandas DataFrame.""" @@ -174,8 +178,8 @@ def deployment(self, self._validate_deployment_inputs(region, include_history, by_system_size, release) deployment_datasets, releases = self._get_deployment_releases() release = releases[release] if isinstance(release, int) else release - region_ = "20220314_GSP" if region == "gsp" else region filenames = list(deployment_datasets[release].keys()) + region_ = f"{self.gsp_boundaries_version}_GSP" if region == "gsp" else region history_ = "_and_month" if include_history else "" system_size_ = "_and_system_size" if by_system_size else "" filename_ending = f"_capacity_by_{region_}{history_}{system_size_}.csv.gz" From 57223a77a40eb8752ecf223de69bfd4609b53c19 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 17:50:33 +0100 Subject: [PATCH 6/8] Remove support for period=5 Keeping the period param just in case we ever re-introduce multiple time-resolutions --- pvlive_api/pvlive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlive_api/pvlive.py b/pvlive_api/pvlive.py index 339da9c..2603e6f 100644 --- a/pvlive_api/pvlive.py +++ b/pvlive_api/pvlive.py @@ -543,7 +543,7 @@ def _validate_inputs(self, entity_type="gsp", entity_id=0, extra_fields="", peri elif entity_type == "gsp": if entity_id not in self.gsp_ids: raise PVLiveException(f"The gsp_id {entity_id} was not found.") - periods = [5, 30] + periods = [30] if period not in periods: raise ValueError("The period parameter must be one of: " f"{', '.join(map(str, periods))}.") From 773be4cc5a7fc533976c26640ed7faab31778ca0 Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 17:51:10 +0100 Subject: [PATCH 7/8] Drop any NaNs in the capacity dataset See https://github.com/SheffieldSolar/pvdeployment/issues/83 --- pvlive_api/pvlive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pvlive_api/pvlive.py b/pvlive_api/pvlive.py index 2603e6f..562cc34 100644 --- a/pvlive_api/pvlive.py +++ b/pvlive_api/pvlive.py @@ -192,6 +192,7 @@ def deployment(self, deployment_data.insert(0, "release", release) deployment_data.rename(columns={"dc_capacity_MWp": "dc_capacity_mwp"}, inplace=True) deployment_data.system_count = deployment_data.system_count.astype("Int64") + deployment_data.dropna(how="any", inplace=True) return deployment_data def latest(self, From 73b6e7cafde1ef19faecc8358355173f684c7dec Mon Sep 17 00:00:00 2001 From: Jamie Taylor Date: Thu, 10 Jul 2025 17:51:55 +0100 Subject: [PATCH 8/8] Update tests Remove tests with period=5 --- Tests/test_pvlive_api.py | 74 ---------------------------------------- requirements_dev.txt | 1 + 2 files changed, 1 insertion(+), 74 deletions(-) diff --git a/Tests/test_pvlive_api.py b/Tests/test_pvlive_api.py index 76758da..cff3ff1 100644 --- a/Tests/test_pvlive_api.py +++ b/Tests/test_pvlive_api.py @@ -133,25 +133,6 @@ def test_latest(self): ) self.check_df_columns(data) self.check_df_dtypes(data) - data = self.api.latest( - entity_type="pes", - entity_id=0, - period=5 - ) - self.check_pes_tuple(data) - self.check_pes_tuple_dtypes(data) - data = self.api.latest(entity_type="pes", entity_id=0, period=5, dataframe=True) - self.check_df_columns(data) - self.check_df_dtypes(data) - data = self.api.latest( - entity_type="pes", - entity_id=0, - extra_fields="ucl_mw,lcl_mw,installedcapacity_mwp,stats_error", - period=5, - dataframe=True - ) - self.check_df_columns(data) - self.check_df_dtypes(data) data = self.api.latest(entity_type="gsp", entity_id=103) self.check_gsp_tuple(data) self.check_gsp_tuple_dtypes(data) @@ -175,27 +156,6 @@ def test_day_peak(self): entity_id=0, dataframe=True ) - data = self.api.day_peak(d=test_date, entity_type="pes", entity_id=0, period=5) - self.check_pes_tuple(data) - self.check_pes_tuple_dtypes(data) - data = self.api.day_peak( - d=test_date, - entity_type="pes", - entity_id=0, - period=5, - dataframe=True - ) - self.check_df_columns(data) - self.check_df_dtypes(data) - data = self.api.day_peak( - d=test_date, - extra_fields="ucl_mw,lcl_mw,installedcapacity_mwp,stats_error", - entity_type="pes", - entity_id=0, - period=5, - dataframe=True - ) - self.check_df_dtypes(data) data = self.api.day_peak(d=test_date, entity_type="gsp", entity_id=54) self.check_gsp_tuple(data) self.check_gsp_tuple_dtypes(data) @@ -235,25 +195,6 @@ def test_between(self): ) self.check_df_columns(data) self.check_df_dtypes(data) - data = self.api.between( - start=get_test_time(0, 0), - end=get_test_time(14, 0), - entity_type="pes", - entity_id=0, - period=5 - ) - with self.subTest(): - assert isinstance(data, list) - data = self.api.between( - start=get_test_time(12, 20), - end=get_test_time(14, 0), - entity_type="pes", - entity_id=0, - period=5, - dataframe=True - ) - self.check_df_columns(data) - self.check_df_dtypes(data) def test_at_time(self): """Test the at_time function.""" @@ -272,21 +213,6 @@ def test_at_time(self): dataframe=True ) self.check_df_dtypes(data) - data = self.api.at_time(dt=test_time, entity_type="pes", entity_id=0, period=5) - self.check_pes_tuple(data) - self.check_pes_tuple_dtypes(data) - data = self.api.at_time(test_time, entity_type="pes", entity_id=0, period=5, dataframe=True) - self.check_df_columns(data) - self.check_df_dtypes(data) - data = self.api.at_time( - test_time, - entity_type="pes", - entity_id=0, - extra_fields="ucl_mw,lcl_mw,installedcapacity_mwp,stats_error", - period=5, - dataframe=True - ) - self.check_df_dtypes(data) data = self.api.at_time(dt=test_time, entity_type="gsp", entity_id=26) self.check_gsp_tuple(data) self.check_gsp_tuple_dtypes(data) diff --git a/requirements_dev.txt b/requirements_dev.txt index 7eb2584..cd646c6 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,3 +1,4 @@ sphinx sphinx_rtd_theme numpydoc +pytest