From ca92a966ce3400e51c3afe43cdf8785415088c54 Mon Sep 17 00:00:00 2001 From: Jan Pecinovsky Date: Thu, 21 Mar 2024 14:28:59 +0100 Subject: [PATCH 1/4] prepared base request --- entsoe/entsoe.py | 68 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/entsoe/entsoe.py b/entsoe/entsoe.py index af89081..2a25550 100644 --- a/entsoe/entsoe.py +++ b/entsoe/entsoe.py @@ -82,19 +82,11 @@ def _base_request(self, params: Dict, start: pd.Timestamp, ------- requests.Response """ - start_str = self._datetime_to_str(start) - end_str = self._datetime_to_str(end) - - base_params = { - 'securityToken': self.api_key, - 'periodStart': start_str, - 'periodEnd': end_str - } - params.update(base_params) + prepared_request = self.prepare_base_request(params=params, start=start, end=end) - logger.debug(f'Performing request to {URL} with params {params}') - response = self.session.get(url=URL, params=params, - proxies=self.proxies, timeout=self.timeout) + logger.debug(f'Performing request to {prepared_request.url}') + response = self.session.send(request=prepared_request, + proxies=self.proxies, timeout=self.timeout) try: response.raise_for_status() except requests.HTTPError as e: @@ -124,14 +116,52 @@ def _base_request(self, params: Dict, start: pd.Timestamp, f"documents and cannot be fulfilled as is.") raise e else: - # ENTSO-E has changed their server to also respond with 200 if there is no data but all parameters are valid - # this means we need to check the contents for this error even when status code 200 is returned - # to prevent parsing the full response do a text matching instead of full parsing - # also only do this when response type content is text and not for example a zip file - if response.headers.get('content-type', '') == 'application/xml': - if 'No matching data found' in response.text: - raise NoMatchingDataError + self.validate_base_response(response) return response + + def prepare_base_request(self, params: Dict, start: pd.Timestamp, + end: pd.Timestamp) -> requests.PreparedRequest: + """ + Parameters + ---------- + params : dict + start : pd.Timestamp + end : pd.Timestamp + + Returns + ------- + requests.PreparedRequest + """ + start_str = self._datetime_to_str(start) + end_str = self._datetime_to_str(end) + + base_params = { + 'securityToken': self.api_key, + 'periodStart': start_str, + 'periodEnd': end_str + } + params.update(base_params) + req = requests.Request( + method='GET', + url=URL, + params=params + ) + return req.prepare() + + @staticmethod + def validate_base_response(response: requests.Response) -> None: + """ + Parameters + ---------- + response : requests.Response + """ + # ENTSO-E has changed their server to also respond with 200 if there is no data but all parameters are valid + # this means we need to check the contents for this error even when status code 200 is returned + # to prevent parsing the full response do a text matching instead of full parsing + # also only do this when response type content is text and not for example a zip file + if response.headers.get('content-type', '') == 'application/xml': + if 'No matching data found' in response.text: + raise NoMatchingDataError @staticmethod def _datetime_to_str(dtm: pd.Timestamp) -> str: From 382c5600d34334845e5533bf882f26da0f450331 Mon Sep 17 00:00:00 2001 From: Jan Pecinovsky Date: Thu, 21 Mar 2024 14:53:47 +0100 Subject: [PATCH 2/4] prepare day ahead query --- entsoe/entsoe.py | 69 ++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/entsoe/entsoe.py b/entsoe/entsoe.py index 2a25550..45cacc1 100644 --- a/entsoe/entsoe.py +++ b/entsoe/entsoe.py @@ -83,7 +83,18 @@ def _base_request(self, params: Dict, start: pd.Timestamp, requests.Response """ prepared_request = self.prepare_base_request(params=params, start=start, end=end) - + return self._do_prepared_request(prepared_request) + + def _do_prepared_request(self, prepared_request: requests.PreparedRequest) -> requests.Response: + """ + Parameters + ---------- + prepared_request : requests.PreparedRequest + + Returns + ------- + requests.Response + """ logger.debug(f'Performing request to {prepared_request.url}') response = self.session.send(request=prepared_request, proxies=self.proxies, timeout=self.timeout) @@ -116,7 +127,13 @@ def _base_request(self, params: Dict, start: pd.Timestamp, f"documents and cannot be fulfilled as is.") raise e else: - self.validate_base_response(response) + # ENTSO-E has changed their server to also respond with 200 if there is no data but all parameters are valid + # this means we need to check the contents for this error even when status code 200 is returned + # to prevent parsing the full response do a text matching instead of full parsing + # also only do this when response type content is text and not for example a zip file + if response.headers.get('content-type', '') == 'application/xml': + if 'No matching data found' in response.text: + raise NoMatchingDataError return response def prepare_base_request(self, params: Dict, start: pd.Timestamp, @@ -147,22 +164,7 @@ def prepare_base_request(self, params: Dict, start: pd.Timestamp, params=params ) return req.prepare() - - @staticmethod - def validate_base_response(response: requests.Response) -> None: - """ - Parameters - ---------- - response : requests.Response - """ - # ENTSO-E has changed their server to also respond with 200 if there is no data but all parameters are valid - # this means we need to check the contents for this error even when status code 200 is returned - # to prevent parsing the full response do a text matching instead of full parsing - # also only do this when response type content is text and not for example a zip file - if response.headers.get('content-type', '') == 'application/xml': - if 'No matching data found' in response.text: - raise NoMatchingDataError - + @staticmethod def _datetime_to_str(dtm: pd.Timestamp) -> str: """ @@ -198,14 +200,31 @@ def query_day_ahead_prices(self, country_code: Union[Area, str], ------- str """ - area = lookup_area(country_code) - params = { - 'documentType': 'A44', - 'in_Domain': area.code, - 'out_Domain': area.code - } - response = self._base_request(params=params, start=start, end=end) + prepared_request = self.prepare_query_day_ahead_prices(country_code=country_code, + start=start, end=end) + response = self._do_prepared_request(prepared_request) return response.text + + def prepare_query_day_ahead_prices(self, country_code: Union[Area, str], + start: pd.Timestamp, end: pd.Timestamp) -> requests.PreparedRequest: + """ + Parameters + ---------- + country_code : Area|str + start : pd.Timestamp + end : pd.Timestamp + + Returns + ------- + requests.PreparedRequest + """ + area = lookup_area(country_code) + params = { + 'documentType': 'A44', + 'in_Domain': area.code, + 'out_Domain': area.code + } + return self.prepare_base_request(params=params, start=start, end=end) def query_aggregated_bids(self, country_code: Union[Area, str], process_type: str, From b0b891249262ddbf4af3eccf3962eae4da8e6e9e Mon Sep 17 00:00:00 2001 From: Jan Pecinovsky Date: Thu, 21 Mar 2024 14:54:59 +0100 Subject: [PATCH 3/4] make prepare base request private --- entsoe/entsoe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/entsoe/entsoe.py b/entsoe/entsoe.py index 45cacc1..bb00e28 100644 --- a/entsoe/entsoe.py +++ b/entsoe/entsoe.py @@ -82,7 +82,7 @@ def _base_request(self, params: Dict, start: pd.Timestamp, ------- requests.Response """ - prepared_request = self.prepare_base_request(params=params, start=start, end=end) + prepared_request = self._prepare_base_request(params=params, start=start, end=end) return self._do_prepared_request(prepared_request) def _do_prepared_request(self, prepared_request: requests.PreparedRequest) -> requests.Response: @@ -136,7 +136,7 @@ def _do_prepared_request(self, prepared_request: requests.PreparedRequest) -> re raise NoMatchingDataError return response - def prepare_base_request(self, params: Dict, start: pd.Timestamp, + def _prepare_base_request(self, params: Dict, start: pd.Timestamp, end: pd.Timestamp) -> requests.PreparedRequest: """ Parameters @@ -224,7 +224,7 @@ def prepare_query_day_ahead_prices(self, country_code: Union[Area, str], 'in_Domain': area.code, 'out_Domain': area.code } - return self.prepare_base_request(params=params, start=start, end=end) + return self._prepare_base_request(params=params, start=start, end=end) def query_aggregated_bids(self, country_code: Union[Area, str], process_type: str, From b2fbd8593ba51452a7c0abf9265298bcef8eb91c Mon Sep 17 00:00:00 2001 From: Jan Pecinovsky Date: Thu, 21 Mar 2024 15:21:11 +0100 Subject: [PATCH 4/4] move price parsing to `parsers` --- entsoe/entsoe.py | 15 +++++++-------- entsoe/parsers.py | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/entsoe/entsoe.py b/entsoe/entsoe.py index bb00e28..5fe6db8 100644 --- a/entsoe/entsoe.py +++ b/entsoe/entsoe.py @@ -1231,14 +1231,13 @@ def query_day_ahead_prices( start=start-pd.Timedelta(days=1), end=end+pd.Timedelta(days=1) ) - series = parse_prices(text)[resolution] - if len(series) == 0: - raise NoMatchingDataError - series = series.tz_convert(area.tz) - series = series.truncate(before=start, after=end) - # because of the above fix we need to check again if any valid data exists after truncating - if len(series) == 0: - raise NoMatchingDataError + series = parse_prices( + xml_text=text, + tz=area.tz, + resolution=resolution, + start=start, + end=end + ) return series @year_limited diff --git a/entsoe/parsers.py b/entsoe/parsers.py index eb07205..f2690a9 100644 --- a/entsoe/parsers.py +++ b/entsoe/parsers.py @@ -1,12 +1,13 @@ import sys import zipfile from io import BytesIO -from typing import Union +from typing import Literal, Union import warnings import bs4 from bs4.builder import XMLParsedAsHTMLWarning import pandas as pd +from .exceptions import NoMatchingDataError from .mappings import PSRTYPE_MAPPINGS, DOCSTATUS, BSNTYPE, Area from .series_parsers import _extract_timeseries, _resolution_to_timedelta, _parse_datetimeindex, _parse_timeseries_generic,\ _parse_timeseries_generic_whole @@ -18,11 +19,24 @@ -def parse_prices(xml_text): +def parse_prices( + xml_text: str, + tz: str, + resolution: Literal['15min', '30min', '60min'], + start: pd.Timestamp, + end: pd.Timestamp) -> pd.Series: """ + Parse day-ahead prices. + + Also performs tz conversion and truncation. + Parameters ---------- xml_text : str + tz : str + resolution : Literal['15min', '30min', '60min'] + start : pd.Timestamp + end : pd.Timestamp Returns ------- @@ -40,6 +54,14 @@ def parse_prices(xml_text): for freq, freq_series in series.items(): if len(freq_series) > 0: series[freq] = pd.concat(freq_series).sort_index() + series = series[resolution] + if len(series) == 0: + raise NoMatchingDataError + series = series.tz_convert(tz) + series = series.truncate(before=start, after=end) + # because of the above fix we need to check again if any valid data exists after truncating + if len(series) == 0: + raise NoMatchingDataError return series