From dcedbf67983e08345e4edf2eef6fb82766eb32c6 Mon Sep 17 00:00:00 2001 From: Gui-FernandesBR Date: Tue, 21 Jul 2026 08:58:01 -0300 Subject: [PATCH 1/5] ENH: Add native Meteomatics API support to the Environment class Adds a new "meteomatics" atmospheric model to Environment.set_atmospheric_model, porting and generalizing the implementation from the EuRoC-Dev repository. - fetchers.py: fetch_meteomatics_token + fetch_atmospheric_data_from_meteomatics authenticate with username/password (short-lived token), query temperature, pressure and wind components by height above ground level, grouping the parameters to respect the account's per-request limit. - environment.py: process_meteomatics_atmosphere converts the height-AGL data to above-sea-level profiles using the Environment elevation; set_atmospheric_model gains username/password kwargs (falling back to METEOMATICS_USERNAME / METEOMATICS_PASSWORD env vars); save/load handles the new model type. - Network requests use timeouts, do not retry deterministic 4xx failures, and surface actionable RuntimeError messages. - Tests fully mock the API (no real requests, no charges); docs and changelog updated. Closes #545 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + .../user/environment/3-further/other_apis.rst | 53 ++- rocketpy/environment/environment.py | 212 +++++++++++- rocketpy/environment/fetchers.py | 322 ++++++++++++++++++ tests/unit/environment/test_environment.py | 156 ++++++++- tests/unit/environment/test_fetchers.py | 220 ++++++++++++ 6 files changed, 955 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e68153f6e..af452f44b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Attention: The newest changes should be on top --> ### Added +- ENH: Support for Meteomatics API in the `Environment` class [#1079](https://github.com/RocketPy-Team/RocketPy/pull/1079) - ENH: update master with develop [#1081](https://github.com/RocketPy-Team/RocketPy/pull/1081) ### Changed diff --git a/docs/user/environment/3-further/other_apis.rst b/docs/user/environment/3-further/other_apis.rst index 37a9a0949..9aacd8273 100644 --- a/docs/user/environment/3-further/other_apis.rst +++ b/docs/user/environment/3-further/other_apis.rst @@ -159,6 +159,56 @@ For custom dictionaries, the canonical structure is: simulation workflow. +Meteomatics API +--------------- + +RocketPy can build an ``Environment`` directly from the +`Meteomatics `_ weather API. +Meteomatics authenticates with a personal **username** and **password** (a +short-lived access token is generated automatically under the hood), so you +need a Meteomatics account to use this feature. + +The API is queried for temperature, pressure and both wind components at +several altitudes above ground level around the launch site, which are then +converted to profiles above sea level using the ``Environment`` elevation. +Because of that, make sure a launch ``date`` and a reasonable ``elevation`` are +set before calling the method. + +.. code-block:: python + + from datetime import datetime, timedelta + from rocketpy import Environment + + env = Environment( + latitude=39.3897, + longitude=-8.28896, + elevation=113, + date=datetime.now() + timedelta(days=1), # forecast instant + ) + + env.set_atmospheric_model( + type="Meteomatics", + file="mix", # Meteomatics weather model + username="your_username", + password="your_password", + ) + + env.info() + +If you prefer not to hardcode the credentials, omit the ``username`` and +``password`` arguments and RocketPy will read them from the +``METEOMATICS_USERNAME`` and ``METEOMATICS_PASSWORD`` environment variables. + +.. note:: + + The altitude range and sampling resolution can be tuned by calling + :meth:`rocketpy.Environment.process_meteomatics_atmosphere` directly (for + example, to change ``min_altitude``, ``max_altitude`` or the number of + levels). The API returns an error if the requested altitude is outside the + range supported by the chosen model, and your account may not have access + to every model. + + Without OPeNDAP protocol ------------------------- @@ -166,7 +216,6 @@ On the other hand, one can also load data from APIs that do not support the OPeN In these cases, what we recommend is to download the data and then load it as a custom atmosphere. There are some efforts to natively support other APIs in RocketPy's -Environment class, for example: +Environment class, for example: -- `Meteomatics `_: `#545 `_ - `Open-Meteo `_: `#520 `_ diff --git a/rocketpy/environment/environment.py b/rocketpy/environment/environment.py index 389dd076e..4d9b26057 100644 --- a/rocketpy/environment/environment.py +++ b/rocketpy/environment/environment.py @@ -1,7 +1,8 @@ -# pylint: disable=too-many-public-methods, too-many-instance-attributes +# pylint: disable=too-many-public-methods, too-many-instance-attributes, too-many-lines import bisect import json import logging +import os import re import warnings from collections import namedtuple @@ -13,6 +14,7 @@ from rocketpy.environment.fetchers import ( fetch_aigfs_file_return_dataset, + fetch_atmospheric_data_from_meteomatics, fetch_atmospheric_data_from_windy, fetch_gefs_ensemble, fetch_gfs_file_return_dataset, @@ -145,8 +147,8 @@ class Environment: Environment.atmospheric_model_type : string Describes the atmospheric model which is being used. Can only assume the following values: ``standard_atmosphere``, ``custom_atmosphere``, - ``wyoming_sounding``, ``windy``, ``forecast``, ``reanalysis``, - ``ensemble``. + ``wyoming_sounding``, ``windy``, ``meteomatics``, ``forecast``, + ``reanalysis``, ``ensemble``. Environment.atmospheric_model_file : string Address of the file used for the atmospheric model being used. Only defined for ``wyoming_sounding``, ``windy``, ``forecast``, @@ -1188,6 +1190,8 @@ def set_atmospheric_model( # pylint: disable=too-many-statements wind_u=0, wind_v=0, pressure_conversion_factor=None, + username=None, + password=None, ): """Define the atmospheric model for this Environment. @@ -1196,8 +1200,8 @@ def set_atmospheric_model( # pylint: disable=too-many-statements type : string Atmospheric model selector (case-insensitive). Accepted values are ``"standard_atmosphere"``, ``"wyoming_sounding"``, ``"windy"``, - ``"forecast"``, ``"reanalysis"``, ``"ensemble"`` and - ``"custom_atmosphere"``. + ``"forecast"``, ``"reanalysis"``, ``"ensemble"``, + ``"custom_atmosphere"`` and ``"meteomatics"``. file : string | netCDF4.Dataset, optional Data source or model shortcut. Meaning depends on ``type``: @@ -1205,6 +1209,9 @@ def set_atmospheric_model( # pylint: disable=too-many-statements - ``"wyoming_sounding"``: URL of the sounding text page. - ``"windy"``: one of ``"ECMWF"``, ``"GFS"``, ``"ICON"`` or ``"ICONEU"``. + - ``"meteomatics"``: the Meteomatics weather model to query, such + as ``"mix"`` (the default when omitted). See the Meteomatics + documentation for the models available to your account. - ``"forecast"``: local path, OPeNDAP URL, open ``netCDF4.Dataset``, or one of ``"AIGFS"``, ``"GFS"``, ``"NAM"``, ``"RAP"``, ``"HRRR"`` or ``"HIRESW"`` for the @@ -1290,6 +1297,14 @@ def set_atmospheric_model( # pylint: disable=too-many-statements model name (e.g. ERA5/ECMWF/MERRA2 reanalysis files commonly use hPa, while online GFS/NAM/RAP/HRRR forecast models use Pa) or, if unavailable, by reading the pressure unit attribute from the file. + username : string, optional + Meteomatics account username. Only used when ``type`` is + ``"meteomatics"``. If None (the default), the value is read from the + ``METEOMATICS_USERNAME`` environment variable. + password : string, optional + Meteomatics account password. Only used when ``type`` is + ``"meteomatics"``. If None (the default), the value is read from the + ``METEOMATICS_PASSWORD`` environment variable. Returns ------- @@ -1338,6 +1353,10 @@ def set_atmospheric_model( # pylint: disable=too-many-statements self.process_custom_atmosphere(pressure, temperature, wind_u, wind_v) case "windy": self.process_windy_atmosphere(file) + case "meteomatics": + self.process_meteomatics_atmosphere( + model=file, username=username, password=password + ) case "forecast" | "reanalysis" | "ensemble": # Capture the user-supplied names before __validate_dictionary # converts them to dicts, so they can drive auto-detection. @@ -1736,6 +1755,181 @@ def __parse_windy_file(self, response, time_index, pressure_levels): wind_v_array, ) + def process_meteomatics_atmosphere( # pylint: disable=too-many-statements + self, + model="mix", + username=None, + password=None, + min_altitude=10, + max_altitude=12000, + wind_resolution=20, + temperature_pressure_resolution=10, + query_limit=10, + ): + """Process data from the Meteomatics API to retrieve a vertical + atmospheric profile at the launch site. + + The Meteomatics API is queried for temperature, pressure and both wind + components at several altitudes above ground level, which are then + converted to profiles above sea level using the ``Environment`` + elevation. Authentication uses a personal username and password; when + not provided, they are read from the ``METEOMATICS_USERNAME`` and + ``METEOMATICS_PASSWORD`` environment variables. + + Parameters + ---------- + model : str, optional + The Meteomatics weather model to query. Default is ``"mix"``. Your + account may not have access to every model. + username : str, optional + Meteomatics account username. Defaults to the + ``METEOMATICS_USERNAME`` environment variable. + password : str, optional + Meteomatics account password. Defaults to the + ``METEOMATICS_PASSWORD`` environment variable. + min_altitude : float, optional + Lowest altitude above ground level (in meters) to query. Default is + 10. + max_altitude : float, optional + Highest altitude above ground level (in meters) to query. Default + is 12000. The API errors if it lies outside the model's supported + range. + wind_resolution : int, optional + Number of altitude levels used for the wind components. Default is + 20. + temperature_pressure_resolution : int, optional + Number of altitude levels used for temperature and pressure. + Default is 10. + query_limit : int, optional + Maximum number of parameters requested at once. Parameters are + grouped accordingly to respect the account's per-request limit. + Default is 10. + + Raises + ------ + ValueError + If credentials are missing, if no launch date is set, or if the API + returns no usable data. + """ + model = model if isinstance(model, str) else "mix" + username = username or os.environ.get("METEOMATICS_USERNAME") + password = password or os.environ.get("METEOMATICS_PASSWORD") + if not username or not password: + raise ValueError( + "Meteomatics requires a username and password. Provide them via " + "the 'username' and 'password' arguments of set_atmospheric_model, " + "or set the METEOMATICS_USERNAME and METEOMATICS_PASSWORD " + "environment variables." + ) + if getattr(self, "datetime_date", None) is None: + raise ValueError( + "A launch date is required to use the Meteomatics atmospheric " + "model. Provide it when creating the Environment or via set_date()." + ) + + profiles = fetch_atmospheric_data_from_meteomatics( + username=username, + password=password, + latitude=self.latitude, + longitude=self.longitude, + date=self.datetime_date, + model=model, + min_altitude=min_altitude, + max_altitude=max_altitude, + wind_resolution=wind_resolution, + temperature_pressure_resolution=temperature_pressure_resolution, + query_limit=query_limit, + ) + + if self.elevation == 0: + warnings.warn( + "The Environment elevation is 0 m (possibly unset), so " + "Meteomatics heights above ground level are being treated as " + "heights above sea level. If the launch site is not at sea " + "level, set the elevation (e.g. Environment(elevation=...) or " + "set_elevation('Open-Elevation')) before this call for an " + "accurate profile.", + UserWarning, + stacklevel=2, + ) + + def to_profile_array(profile): + """Convert an {AGL height: value} mapping into a sorted + (ASL height, value) array, dropping any missing value.""" + heights = sorted(h for h, value in profile.items() if value is not None) + return np.array( + [(h + self.elevation, profile[h]) for h in heights], dtype=float + ) + + pressure_array = to_profile_array(profiles["pressure"]) + temperature_array = to_profile_array(profiles["temperature"]) + + # Wind u and v share the same altitude grid; keep only common levels. + wind_heights = sorted( + h + for h in set(profiles["wind_u"]) & set(profiles["wind_v"]) + if profiles["wind_u"][h] is not None and profiles["wind_v"][h] is not None + ) + # Each profile needs at least two levels: a single-point Function cannot + # be evaluated at its own node (it raises IndexError downstream), so a + # collapsed grid must fail here with an actionable message instead. + if min(len(pressure_array), len(temperature_array), len(wind_heights)) < 2: + raise ValueError( + "Meteomatics did not return enough usable atmospheric data: at " + "least two valid altitude levels are required for pressure, " + "temperature and wind. Check the requested model, the altitude " + "range (min_altitude and max_altitude must be far enough apart " + "that the sampled levels do not collapse to a single height), " + "and your account permissions." + ) + + wind_u_values = np.array([profiles["wind_u"][h] for h in wind_heights]) + wind_v_values = np.array([profiles["wind_v"][h] for h in wind_heights]) + wind_asl_heights = np.array(wind_heights, dtype=float) + self.elevation + wind_u_array = np.column_stack((wind_asl_heights, wind_u_values)) + wind_v_array = np.column_stack((wind_asl_heights, wind_v_values)) + + wind_speed_array = calculate_wind_speed(wind_u_values, wind_v_values) + wind_heading_array = calculate_wind_heading(wind_u_values, wind_v_values) + wind_direction_array = convert_wind_heading_to_direction(wind_heading_array) + + # Save atmospheric data + self.__set_pressure_function(pressure_array) + self.__set_barometric_height_function(pressure_array[:, (1, 0)]) + self.__set_temperature_function(temperature_array) + self.__set_wind_velocity_x_function(wind_u_array) + self.__set_wind_velocity_y_function(wind_v_array) + self.__set_wind_heading_function( + np.column_stack((wind_asl_heights, wind_heading_array)) + ) + self.__set_wind_direction_function( + np.column_stack((wind_asl_heights, wind_direction_array)) + ) + self.__set_wind_speed_function( + np.column_stack((wind_asl_heights, wind_speed_array)) + ) + + # Save maximum expected height + self._max_expected_height = float( + max(pressure_array[-1, 0], temperature_array[-1, 0], wind_asl_heights[-1]) + ) + + # Save model info metadata (single point in space and time) + self.atmospheric_model_init_date = self.datetime_date + self.atmospheric_model_end_date = self.datetime_date + self.atmospheric_model_interval = 0 + self.atmospheric_model_init_lat = self.latitude + self.atmospheric_model_end_lat = self.latitude + self.atmospheric_model_init_lon = self.longitude + self.atmospheric_model_end_lon = self.longitude + + # Save debugging data + self.wind_us = wind_u_values + self.wind_vs = wind_v_values + self.temperatures = temperature_array[:, 1] + self.pressures = pressure_array[:, 1] + self.height = wind_asl_heights + def process_wyoming_sounding(self, file): # pylint: disable=too-many-statements """Import and process the upper air sounding data from `Wyoming Upper Air Soundings` database given by the url in file. Sets @@ -3010,7 +3204,13 @@ def from_dict(cls, data): # pylint: disable=too-many-statements env.elevation = data["elevation"] env.max_expected_height = data["max_expected_height"] - if atmospheric_model in ("windy", "forecast", "reanalysis", "ensemble"): + if isinstance(atmospheric_model, str) and atmospheric_model.lower() in ( + "windy", + "meteomatics", + "forecast", + "reanalysis", + "ensemble", + ): env.atmospheric_model_init_date = data["atmospheric_model_init_date"] env.atmospheric_model_end_date = data["atmospheric_model_end_date"] env.atmospheric_model_interval = data["atmospheric_model_interval"] diff --git a/rocketpy/environment/fetchers.py b/rocketpy/environment/fetchers.py index 740e9818a..5f3d034eb 100644 --- a/rocketpy/environment/fetchers.py +++ b/rocketpy/environment/fetchers.py @@ -3,12 +3,14 @@ functions may be changed without notice in future feature releases. """ +import base64 import logging import re import time from datetime import datetime, timedelta, timezone import netCDF4 +import numpy as np import requests from rocketpy.tools import exponential_backoff @@ -17,6 +19,13 @@ MAX_RETRY_DELAY_SECONDS = 600 +METEOMATICS_BASE_URL = "https://api.meteomatics.com" +METEOMATICS_LOGIN_URL = "https://login.meteomatics.com/api/v1/token" +METEOMATICS_TIMEOUT_SECONDS = 30 +# Matches Meteomatics height-level parameters such as "t_500m:K", +# "pressure_1000m:Pa" or "wind_speed_u_120m:ms". +_METEOMATICS_PARAMETER_REGEX = re.compile(r"^(?P[a-z_]+)_(?P\d+)m:") + @exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) def fetch_open_elevation(lat, lon): @@ -444,3 +453,316 @@ def fetch_cmc_ensemble(): time.sleep(min(2**attempt_count, MAX_RETRY_DELAY_SECONDS)) if not success: raise RuntimeError("Unable to load latest weather data for CMC through " + file) + + +@exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) +def _meteomatics_get(url, headers=None, params=None): + """Performs a single Meteomatics GET request, retrying transient failures. + + Connection-level errors (and server-side 5xx responses) raise and are + retried by the decorator. Client-side 4xx responses are returned as-is so + the caller can turn them into an actionable, non-retried error, since + retrying a deterministic 4xx only wastes time and API quota. + """ + response = requests.get( + url, headers=headers, params=params, timeout=METEOMATICS_TIMEOUT_SECONDS + ) + if response.status_code >= 500: + # Server-side error: raise so the backoff decorator retries it. + response.raise_for_status() + return response + + +def fetch_meteomatics_token(username, password): + """Requests a short-lived access token from the Meteomatics login service. + + The Meteomatics API authenticates with a personal ``username`` and + ``password``. Instead of sending the credentials on every request, a token + is generated once and reused. Each token is valid for a couple of hours, + which is more than enough to build a single ``Environment``. + + Parameters + ---------- + username : str + The Meteomatics account username. + password : str + The Meteomatics account password. + + Returns + ------- + str + The access token to be used as the ``access_token`` query parameter in + subsequent data requests. + + Raises + ------ + RuntimeError + If the login service cannot be reached, rejects the credentials, or + does not return a token. + """ + credentials = f"{username}:{password}" + encoded_credentials = base64.b64encode(credentials.encode()).decode() + headers = {"Authorization": f"Basic {encoded_credentials}"} + try: + response = _meteomatics_get(METEOMATICS_LOGIN_URL, headers=headers) + except requests.exceptions.RequestException as e: + raise RuntimeError( + "Unable to reach the Meteomatics login service. Please try again later." + ) from e + + if response.status_code in (401, 403): + # Definitive authentication failure: do not retry. + raise RuntimeError( + f"Meteomatics rejected the credentials (HTTP {response.status_code}). " + "Check your username and password." + ) + if not response.ok: + raise RuntimeError( + f"Meteomatics login request failed (HTTP {response.status_code})." + ) + try: + token = response.json().get("access_token") + except requests.exceptions.JSONDecodeError as e: + raise RuntimeError( + "Meteomatics login service returned a malformed (non-JSON) response." + ) from e + if not token: + raise RuntimeError( + "Meteomatics login service did not return an access token. " + "Check your username and password." + ) + logger.info("Meteomatics access token generated successfully.") + return token + + +def _build_meteomatics_parameters( + min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution +): + """Builds the list of Meteomatics height-level parameters to query. + + Wind components are sampled on a finer altitude grid than temperature and + pressure, since the wind profile is usually the most variable one. + + Parameters + ---------- + min_altitude : float + Lowest altitude above ground level (in meters) to query. + max_altitude : float + Highest altitude above ground level (in meters) to query. + wind_resolution : int + Number of altitude levels used for the wind components. + temperature_pressure_resolution : int + Number of altitude levels used for temperature and pressure. + + Returns + ------- + list of str + Parameter strings in the ``"_m:"`` format. + """ + # Round to integer meters and drop duplicates that rounding may introduce + # for narrow bands, so we never request (and pay for) the same height twice. + fine_levels = np.unique( + np.linspace(min_altitude, max_altitude, wind_resolution).round().astype(int) + ) + coarse_levels = np.unique( + np.linspace(min_altitude, max_altitude, temperature_pressure_resolution) + .round() + .astype(int) + ) + + wind_parameters = [ + f"{var}_{height}m:{unit}" + for height in fine_levels + for var, unit in [("wind_speed_u", "ms"), ("wind_speed_v", "ms")] + ] + temperature_pressure_parameters = [ + f"{var}_{height}m:{unit}" + for height in coarse_levels + for var, unit in [("t", "K"), ("pressure", "Pa")] + ] + return wind_parameters + temperature_pressure_parameters + + +def _extract_meteomatics_json(data): + """Extracts (parameter, value) pairs from a Meteomatics JSON response. + + Only the first coordinate and first date of each parameter are used, since + the query is always issued for a single location and a single instant. + + Parameters + ---------- + data : dict + The JSON payload returned by the Meteomatics data endpoint. + + Returns + ------- + list of tuple + A list of ``(parameter, value)`` tuples. + + Raises + ------ + RuntimeError + If the payload does not have the expected Meteomatics structure. + """ + try: + return [ + (entry["parameter"], entry["coordinates"][0]["dates"][0]["value"]) + for entry in data["data"] + ] + except (KeyError, IndexError, TypeError) as e: + raise RuntimeError( + "Unexpected Meteomatics response structure; could not extract the " + "requested data." + ) from e + + +def _fetch_meteomatics_group(base_url, query_params): + """Performs a single Meteomatics data request and returns the JSON body. + + Raises + ------ + RuntimeError + If the API cannot be reached, returns an error status, or returns a + malformed (non-JSON) body. Client-side (4xx) errors are not retried. + """ + try: + response = _meteomatics_get(base_url, params=query_params) + except requests.exceptions.RequestException as e: + raise RuntimeError( + "Unable to reach the Meteomatics data API. Please try again later." + ) from e + if not response.ok: + raise RuntimeError( + f"Meteomatics data request failed (HTTP {response.status_code}). " + f"{response.text[:300]}".strip() + ) + try: + return response.json() + except requests.exceptions.JSONDecodeError as e: + raise RuntimeError( + "Meteomatics data API returned a malformed (non-JSON) response." + ) from e + + +def fetch_atmospheric_data_from_meteomatics( + username, + password, + latitude, + longitude, + date, + model="mix", + min_altitude=10, + max_altitude=12000, + wind_resolution=20, + temperature_pressure_resolution=10, + query_limit=10, +): + """Fetches a vertical atmospheric profile from the Meteomatics API. + + The data is retrieved for a single location and instant, sampling + temperature, pressure and both wind components at several altitudes above + ground level. To respect the account's per-request parameter limit, the + parameters are split into groups that are queried separately. + + Parameters + ---------- + username : str + The Meteomatics account username. + password : str + The Meteomatics account password. + latitude : float + Latitude of the launch site, in degrees. + longitude : float + Longitude of the launch site, in degrees. + date : datetime.datetime + The instant to query. It is formatted according to the Meteomatics + date-time specification (``%Y-%m-%dT%H:%M:%SZ``). + model : str, optional + The Meteomatics weather model to use. Default is ``"mix"``. Your + account may not have access to every model. See + https://www.meteomatics.com/en/api/request/optional-parameters/data-source/ + min_altitude : float, optional + Lowest altitude above ground level (in meters) to query. Default is 10. + max_altitude : float, optional + Highest altitude above ground level (in meters) to query. Default is + 12000. The API returns an error if the requested altitude is outside + the range supported by the chosen model. + wind_resolution : int, optional + Number of altitude levels used for the wind components. Default is 20. + temperature_pressure_resolution : int, optional + Number of altitude levels used for temperature and pressure. Default is + 10. + query_limit : int, optional + Maximum number of parameters requested at once. Parameters are grouped + accordingly to work around the account's per-request limit. Default is + 10. See https://api.meteomatics.com/user_stats for your own limits. + + Returns + ------- + dict + A dictionary with the keys ``"temperature"``, ``"pressure"``, + ``"wind_u"`` and ``"wind_v"``. Each value is a dictionary mapping the + altitude above ground level (in meters) to the corresponding value, in + SI units (K, Pa, m/s and m/s respectively). + + Raises + ------ + RuntimeError + If authentication fails, the API cannot be reached, returns an error + status, or returns a malformed response. + ValueError + If the altitude range is invalid or the response contains an + unrecognized parameter. + """ + if min_altitude < 0: + raise ValueError( + "min_altitude must be non-negative (heights are above ground level)." + ) + if max_altitude <= min_altitude: + raise ValueError("max_altitude must be greater than min_altitude.") + + token = fetch_meteomatics_token(username, password) + + date_string = date.strftime("%Y-%m-%dT%H:%M:%SZ") + parameters = _build_meteomatics_parameters( + min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution + ) + parameter_groups = [ + parameters[i : i + query_limit] for i in range(0, len(parameters), query_limit) + ] + + profiles = { + "temperature": {}, + "pressure": {}, + "wind_u": {}, + "wind_v": {}, + } + variable_to_profile = { + "t": "temperature", + "pressure": "pressure", + "wind_speed_u": "wind_u", + "wind_speed_v": "wind_v", + } + + for index, parameter_group in enumerate(parameter_groups): + logger.info( + "Fetching Meteomatics data for group %d/%d.", + index + 1, + len(parameter_groups), + ) + parameters_str = ",".join(parameter_group) + base_url = ( + f"{METEOMATICS_BASE_URL}/{date_string}/{parameters_str}/" + f"{latitude},{longitude}/json" + ) + query_params = {"model": model, "access_token": token} + data = _fetch_meteomatics_group(base_url, query_params) + + for parameter, value in _extract_meteomatics_json(data): + match = _METEOMATICS_PARAMETER_REGEX.match(parameter) + if match is None or match.group("var") not in variable_to_profile: + raise ValueError(f"Unrecognized Meteomatics parameter '{parameter}'.") + height = int(match.group("height")) + profiles[variable_to_profile[match.group("var")]][height] = value + + return profiles diff --git a/tests/unit/environment/test_environment.py b/tests/unit/environment/test_environment.py index 039521e84..11176304c 100644 --- a/tests/unit/environment/test_environment.py +++ b/tests/unit/environment/test_environment.py @@ -335,7 +335,8 @@ def test_environment_export_environment_exports_valid_environment_json( @pytest.mark.parametrize( - "atmospheric_model_type", ["windy", "forecast", "reanalysis", "ensemble"] + "atmospheric_model_type", + ["windy", "meteomatics", "forecast", "reanalysis", "ensemble"], ) def test_environment_to_dict_from_dict_round_trip_preserves_weather_metadata( example_plain_env, atmospheric_model_type @@ -429,6 +430,159 @@ def test_environment_to_dict_from_dict_round_trip_preserves_weather_metadata( assert restored_env.ensemble_member == env.ensemble_member == 1 +_METEOMATICS_FAKE_PROFILES = { + "temperature": {0: 288.15, 1000: 281.65, 5000: 255.65}, + "pressure": {0: 101325.0, 1000: 89876.0, 5000: 54048.0}, + "wind_u": {0: 1.0, 1000: 3.0, 5000: 8.0}, + "wind_v": {0: -1.0, 1000: -2.0, 5000: -4.0}, +} + + +def _patch_meteomatics_fetcher(monkeypatch, profiles=None, recorder=None): + """Replace the Meteomatics fetcher with an offline fake (no API calls).""" + profiles = _METEOMATICS_FAKE_PROFILES if profiles is None else profiles + + def fake_fetch(**kwargs): + if recorder is not None: + recorder.update(kwargs) + return profiles + + monkeypatch.setattr( + "rocketpy.environment.environment.fetch_atmospheric_data_from_meteomatics", + fake_fetch, + ) + + +def test_meteomatics_atmosphere_sets_profiles(example_euroc_env, monkeypatch): + """Build pressure, temperature and wind profiles from Meteomatics data. + + The fake profiles are indexed by height above ground level, so the + Environment elevation (100 m for the EuRoC fixture) must be added to obtain + heights above sea level. + """ + recorder = {} + _patch_meteomatics_fetcher(monkeypatch, recorder=recorder) + + example_euroc_env.set_atmospheric_model( + type="Meteomatics", file="mix", username="user", password="pass" + ) + + assert example_euroc_env.atmospheric_model_type == "Meteomatics" + # AGL 0 m -> ASL 100 m (the fixture elevation) + assert pytest.approx(101325.0, rel=1e-6) == example_euroc_env.pressure(100) + assert pytest.approx(288.15, rel=1e-6) == example_euroc_env.temperature(100) + assert pytest.approx(1.0) == example_euroc_env.wind_velocity_x(100) + assert pytest.approx(-1.0) == example_euroc_env.wind_velocity_y(100) + assert pytest.approx(np.sqrt(2.0)) == example_euroc_env.wind_speed(100) + assert example_euroc_env.max_expected_height == pytest.approx(5100.0) + # Credentials and model are forwarded to the fetcher. + assert recorder["username"] == "user" + assert recorder["password"] == "pass" + assert recorder["model"] == "mix" + + +def test_meteomatics_reads_credentials_from_environment(example_euroc_env, monkeypatch): + """Fall back to the METEOMATICS_* environment variables for credentials.""" + recorder = {} + _patch_meteomatics_fetcher(monkeypatch, recorder=recorder) + monkeypatch.setenv("METEOMATICS_USERNAME", "env-user") + monkeypatch.setenv("METEOMATICS_PASSWORD", "env-pass") + + example_euroc_env.set_atmospheric_model(type="Meteomatics") + + assert recorder["username"] == "env-user" + assert recorder["password"] == "env-pass" + assert recorder["model"] == "mix" # default model when file is omitted + assert pytest.approx(288.15, rel=1e-6) == example_euroc_env.temperature(100) + + +def test_meteomatics_missing_credentials_raises(example_euroc_env, monkeypatch): + """Raise a clear error when no credentials are available.""" + _patch_meteomatics_fetcher(monkeypatch) + monkeypatch.delenv("METEOMATICS_USERNAME", raising=False) + monkeypatch.delenv("METEOMATICS_PASSWORD", raising=False) + + with pytest.raises(ValueError, match="username and password"): + example_euroc_env.set_atmospheric_model(type="Meteomatics") + + +def test_meteomatics_missing_date_raises(example_plain_env, monkeypatch): + """Raise when the Environment has no launch date set.""" + _patch_meteomatics_fetcher(monkeypatch) + + with pytest.raises(ValueError, match="launch date"): + example_plain_env.set_atmospheric_model( + type="Meteomatics", username="user", password="pass" + ) + + +def test_meteomatics_drops_missing_values_and_intersects_wind_grid( + example_euroc_env, monkeypatch +): + """Drop ``None`` values and keep only wind levels present in both u and v. + + Temperature at 1000 m is ``None`` (dropped), and the wind grids disagree at + 5000 m (only ``wind_u`` has it), so the wind profile must keep only the + common, non-null levels {0, 1000} m AGL. + """ + profiles = { + "temperature": {0: 288.15, 1000: None, 5000: 255.65}, + "pressure": {0: 101325.0, 5000: 54048.0}, + "wind_u": {0: 1.0, 1000: 3.0, 5000: 8.0}, + "wind_v": {0: -1.0, 1000: -2.0}, # missing 5000 -> intersection drops it + } + _patch_meteomatics_fetcher(monkeypatch, profiles=profiles) + + example_euroc_env.set_atmospheric_model( + type="Meteomatics", username="user", password="pass" + ) + + # Wind kept only the two common non-null AGL levels {0, 1000} -> ASL {100, 1100}. + npt.assert_array_equal(example_euroc_env.height, [100.0, 1100.0]) + assert len(example_euroc_env.wind_us) == 2 + # Temperature dropped the None level: {0, 5000} AGL -> ASL {100, 5100}. + assert len(example_euroc_env.temperatures) == 2 + assert pytest.approx(255.65, rel=1e-6) == example_euroc_env.temperature(5100) + assert example_euroc_env.max_expected_height == pytest.approx(5100.0) + + +def test_meteomatics_no_usable_data_raises(example_euroc_env, monkeypatch): + """Raise a clear error when the API returns no usable wind data.""" + profiles = { + "temperature": {0: 288.15}, + "pressure": {0: 101325.0}, + "wind_u": {}, + "wind_v": {}, + } + _patch_meteomatics_fetcher(monkeypatch, profiles=profiles) + + with pytest.raises(ValueError, match="usable atmospheric data"): + example_euroc_env.set_atmospheric_model( + type="Meteomatics", username="user", password="pass" + ) + + +def test_meteomatics_single_level_profile_raises(example_euroc_env, monkeypatch): + """Reject a collapsed grid (one level per profile) up front. + + A single altitude level builds a Function that cannot be evaluated at its + own node, so ``set_atmospheric_model`` must fail immediately with a clear + message rather than succeed and crash later at ``pressure``/``density``. + """ + profiles = { + "temperature": {0: 288.15}, + "pressure": {0: 101325.0}, + "wind_u": {0: 1.0}, + "wind_v": {0: -1.0}, + } + _patch_meteomatics_fetcher(monkeypatch, profiles=profiles) + + with pytest.raises(ValueError, match="at least two valid altitude levels"): + example_euroc_env.set_atmospheric_model( + type="Meteomatics", username="user", password="pass" + ) + + class _DummyDataset: """Small test double that mimics a netCDF dataset variables mapping.""" diff --git a/tests/unit/environment/test_fetchers.py b/tests/unit/environment/test_fetchers.py index eea06f977..7e929aa9a 100644 --- a/tests/unit/environment/test_fetchers.py +++ b/tests/unit/environment/test_fetchers.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + import pytest from rocketpy.environment import fetchers @@ -81,3 +83,221 @@ def always_fails(_): fetchers.fetch_rap_file_return_dataset(max_attempts=2, base_delay=2) assert sleep_calls == [2, 4] + + +class _FakeResponse: + """Minimal stand-in for a ``requests.Response`` used in Meteomatics tests.""" + + def __init__(self, payload, status_code=200, text=""): + self._payload = payload + self.status_code = status_code + self.text = text + + @property + def ok(self): + return self.status_code < 400 + + def raise_for_status(self): + if self.status_code >= 400: + raise fetchers.requests.exceptions.HTTPError(f"status {self.status_code}") + + def json(self): + return self._payload + + +def _meteomatics_value_for(parameter): + """Return a deterministic fake value for a Meteomatics parameter string.""" + if parameter.startswith("t_"): + return 288.0 + if parameter.startswith("pressure_"): + return 90000.0 + if parameter.startswith("wind_speed_u_"): + return 4.0 + if parameter.startswith("wind_speed_v_"): + return -2.0 + raise AssertionError(f"unexpected parameter requested: {parameter}") + + +def _make_fake_meteomatics_get(calls, extra_bad_parameter=False, data_status=200): + """Build a fake ``requests.get`` that mimics the Meteomatics endpoints.""" + + def fake_get(url, headers=None, params=None, **_kwargs): + calls.append((url, params)) + if url == fetchers.METEOMATICS_LOGIN_URL: + assert headers is not None and "Authorization" in headers + return _FakeResponse({"access_token": "fake-token"}) + if data_status >= 400: + return _FakeResponse( + {}, status_code=data_status, text="validation error: altitude" + ) + # Data request: parameters are the 5th path segment. + parameters = url.split("/")[4].split(",") + data = [ + { + "parameter": parameter, + "coordinates": [ + {"dates": [{"value": _meteomatics_value_for(parameter)}]} + ], + } + for parameter in parameters + ] + if extra_bad_parameter: + data.append( + { + "parameter": "not_a_known_parameter:xx", + "coordinates": [{"dates": [{"value": 1.0}]}], + } + ) + return _FakeResponse({"data": data}) + + return fake_get + + +def test_fetch_meteomatics_token_success(monkeypatch): + """Return the access token when the login service responds with one.""" + monkeypatch.setattr( + fetchers.requests, "get", lambda *a, **k: _FakeResponse({"access_token": "tok"}) + ) + assert fetchers.fetch_meteomatics_token("user", "pass") == "tok" + + +def test_fetch_meteomatics_token_missing_token_raises(monkeypatch): + """Raise when the login service returns 200 but without a token.""" + monkeypatch.setattr(fetchers.requests, "get", lambda *a, **k: _FakeResponse({})) + with pytest.raises(RuntimeError, match="did not return an access token"): + fetchers.fetch_meteomatics_token("user", "pass") + + +def test_fetch_meteomatics_token_auth_failure_not_retried(monkeypatch): + """A 401/403 is a definitive auth failure: report clearly and do not retry.""" + calls = [] + + def fake_get(*args, **_kwargs): + calls.append(args) + return _FakeResponse({}, status_code=401, text="unauthorized") + + # If a retry happened it would sleep; make that observable instead of slow. + monkeypatch.setattr( + fetchers.time, "sleep", lambda *_: (_ for _ in ()).throw(AssertionError()) + ) + monkeypatch.setattr(fetchers.requests, "get", fake_get) + + with pytest.raises(RuntimeError, match="rejected the credentials"): + fetchers.fetch_meteomatics_token("user", "pass") + assert len(calls) == 1 # no retries + + +def test_fetch_meteomatics_data_groups_and_parses(monkeypatch): + """Group parameters within the query limit and parse the profiles.""" + # Arrange + calls = [] + monkeypatch.setattr(fetchers.requests, "get", _make_fake_meteomatics_get(calls)) + + # Act: distinct wind (fine) and temperature/pressure (coarse) resolutions so + # a fine-vs-coarse grid swap would be detectable. + profiles = fetchers.fetch_atmospheric_data_from_meteomatics( + username="user", + password="pass", + latitude=39.0, + longitude=-8.0, + date=datetime(2024, 1, 1, 12, tzinfo=timezone.utc), + model="mix", + min_altitude=10, + max_altitude=1000, + wind_resolution=3, + temperature_pressure_resolution=2, + query_limit=3, + ) + + # Assert + # 6 wind params (u,v at 3 levels) + 4 temp/pressure params (t,p at 2 levels) + # = 10 params, grouped by 3 -> ceil(10/3) = 4 groups. + data_calls = [c for c in calls if c[0] != fetchers.METEOMATICS_LOGIN_URL] + assert len(calls) == 5 # 1 token + 4 data groups + assert len(data_calls) == 4 + assert all(call[1]["access_token"] == "fake-token" for call in data_calls) + assert all(call[1]["model"] == "mix" for call in data_calls) + + # Wind uses the fine grid (3 levels); temperature/pressure the coarse (2). + assert profiles["temperature"] == {10: 288.0, 1000: 288.0} + assert profiles["pressure"] == {10: 90000.0, 1000: 90000.0} + assert profiles["wind_u"] == {10: 4.0, 505: 4.0, 1000: 4.0} + assert profiles["wind_v"] == {10: -2.0, 505: -2.0, 1000: -2.0} + + +def test_fetch_meteomatics_data_unrecognized_parameter_raises(monkeypatch): + """Raise a ValueError when the response contains an unknown parameter.""" + calls = [] + monkeypatch.setattr( + fetchers.requests, + "get", + _make_fake_meteomatics_get(calls, extra_bad_parameter=True), + ) + with pytest.raises(ValueError, match="Unrecognized Meteomatics parameter"): + fetchers.fetch_atmospheric_data_from_meteomatics( + username="user", + password="pass", + latitude=39.0, + longitude=-8.0, + date=datetime(2024, 1, 1, 12, tzinfo=timezone.utc), + wind_resolution=2, + temperature_pressure_resolution=2, + ) + + +def test_fetch_meteomatics_data_client_error_not_retried(monkeypatch): + """A 4xx data response yields an actionable RuntimeError and is not retried.""" + calls = [] + monkeypatch.setattr( + fetchers.time, "sleep", lambda *_: (_ for _ in ()).throw(AssertionError()) + ) + monkeypatch.setattr( + fetchers.requests, "get", _make_fake_meteomatics_get(calls, data_status=400) + ) + + with pytest.raises(RuntimeError, match="data request failed"): + fetchers.fetch_atmospheric_data_from_meteomatics( + username="user", + password="pass", + latitude=39.0, + longitude=-8.0, + date=datetime(2024, 1, 1, 12, tzinfo=timezone.utc), + wind_resolution=2, + temperature_pressure_resolution=2, + ) + # 1 token call + exactly 1 data call (the 400 was not retried). + data_calls = [c for c in calls if c[0] != fetchers.METEOMATICS_LOGIN_URL] + assert len(data_calls) == 1 + + +@pytest.mark.parametrize( + "payload", + [ + {}, # missing "data" + {"data": [{"parameter": "t_10m:K", "coordinates": []}]}, # empty coordinates + ], +) +def test_extract_meteomatics_json_bad_structure_raises(payload): + """Turn an unexpected 200 payload into a clear RuntimeError, not KeyError.""" + with pytest.raises(RuntimeError, match="Unexpected Meteomatics response"): + fetchers._extract_meteomatics_json(payload) + + +@pytest.mark.parametrize( + "altitudes", + [ + {"min_altitude": -1, "max_altitude": 1000}, # negative floor + {"min_altitude": 10, "max_altitude": 5}, # max below min + ], +) +def test_fetch_meteomatics_data_invalid_altitude_range_raises(altitudes): + """Reject invalid altitude ranges before making any request.""" + with pytest.raises(ValueError, match="altitude"): + fetchers.fetch_atmospheric_data_from_meteomatics( + username="user", + password="pass", + latitude=39.0, + longitude=-8.0, + date=datetime(2024, 1, 1, 12, tzinfo=timezone.utc), + **altitudes, + ) From 85b2a92be42e71ac562d03ccb9943e8cf33d8068 Mon Sep 17 00:00:00 2001 From: Gui-FernandesBR Date: Thu, 30 Jul 2026 13:26:18 -0300 Subject: [PATCH 2/5] MNT: simplify Meteomatics fetcher and profile assembly Follow-up cleanups from the code review of the Meteomatics support: - Collapse the duplicated request/error ladder of the login and data endpoints into a single `_meteomatics_request_json` helper. - Have `_build_meteomatics_parameters` return a {parameter: (profile, height)} mapping so the response parser no longer regex-parses back the strings this module just built. Drops the regex constant and the variable-to-profile table. - Generalize `to_profile_array` to several profiles at once, so the wind u/v grid intersection reuses it instead of repeating it inline. - Reuse the existing `__validate_datetime` helper for the launch-date check, and simplify the model default to `model or "mix"`. - Normalize `atmospheric_model_type` once in `from_dict`. The type is stored as the user spelled it, so the previously case-sensitive `match` and `== "ensemble"` branches silently dropped the ensemble arrays for an Environment built with `type="Ensemble"`. - Trim the elevation warning and raise it before the API call, so it is shown even when the request later fails. Co-Authored-By: Claude Opus 5 (1M context) --- rocketpy/environment/environment.py | 86 ++++++----- rocketpy/environment/fetchers.py | 191 ++++++++++++------------ tests/unit/environment/test_fetchers.py | 2 +- 3 files changed, 140 insertions(+), 139 deletions(-) diff --git a/rocketpy/environment/environment.py b/rocketpy/environment/environment.py index 4d9b26057..5c456fad0 100644 --- a/rocketpy/environment/environment.py +++ b/rocketpy/environment/environment.py @@ -1811,7 +1811,7 @@ def process_meteomatics_atmosphere( # pylint: disable=too-many-statements If credentials are missing, if no launch date is set, or if the API returns no usable data. """ - model = model if isinstance(model, str) else "mix" + model = model or "mix" username = username or os.environ.get("METEOMATICS_USERNAME") password = password or os.environ.get("METEOMATICS_PASSWORD") if not username or not password: @@ -1821,10 +1821,16 @@ def process_meteomatics_atmosphere( # pylint: disable=too-many-statements "or set the METEOMATICS_USERNAME and METEOMATICS_PASSWORD " "environment variables." ) - if getattr(self, "datetime_date", None) is None: - raise ValueError( - "A launch date is required to use the Meteomatics atmospheric " - "model. Provide it when creating the Environment or via set_date()." + self.__validate_datetime() + + if self.elevation == 0: + warnings.warn( + "The Environment elevation is 0 m (possibly unset), so Meteomatics " + "heights above ground level are being treated as heights above sea " + "level. Set the elevation before this call if the launch site is " + "not at sea level.", + UserWarning, + stacklevel=2, ) profiles = fetch_atmospheric_data_from_meteomatics( @@ -1841,39 +1847,34 @@ def process_meteomatics_atmosphere( # pylint: disable=too-many-statements query_limit=query_limit, ) - if self.elevation == 0: - warnings.warn( - "The Environment elevation is 0 m (possibly unset), so " - "Meteomatics heights above ground level are being treated as " - "heights above sea level. If the launch site is not at sea " - "level, set the elevation (e.g. Environment(elevation=...) or " - "set_elevation('Open-Elevation')) before this call for an " - "accurate profile.", - UserWarning, - stacklevel=2, + def to_profile_array(*names): + """Convert {AGL height: value} mappings into a sorted array whose + first column is the ASL height and the remaining ones the requested + profile values, keeping only the heights that carry a value in + every mapping.""" + common_heights = set.intersection(*(set(profiles[n]) for n in names)) + heights = sorted( + h + for h in common_heights + if all(profiles[n][h] is not None for n in names) ) - - def to_profile_array(profile): - """Convert an {AGL height: value} mapping into a sorted - (ASL height, value) array, dropping any missing value.""" - heights = sorted(h for h, value in profile.items() if value is not None) return np.array( - [(h + self.elevation, profile[h]) for h in heights], dtype=float + [ + (h + self.elevation, *(profiles[n][h] for n in names)) + for h in heights + ], + dtype=float, ) - pressure_array = to_profile_array(profiles["pressure"]) - temperature_array = to_profile_array(profiles["temperature"]) - + pressure_array = to_profile_array("pressure") + temperature_array = to_profile_array("temperature") # Wind u and v share the same altitude grid; keep only common levels. - wind_heights = sorted( - h - for h in set(profiles["wind_u"]) & set(profiles["wind_v"]) - if profiles["wind_u"][h] is not None and profiles["wind_v"][h] is not None - ) + wind_array = to_profile_array("wind_u", "wind_v") + # Each profile needs at least two levels: a single-point Function cannot # be evaluated at its own node (it raises IndexError downstream), so a # collapsed grid must fail here with an actionable message instead. - if min(len(pressure_array), len(temperature_array), len(wind_heights)) < 2: + if min(len(pressure_array), len(temperature_array), len(wind_array)) < 2: raise ValueError( "Meteomatics did not return enough usable atmospheric data: at " "least two valid altitude levels are required for pressure, " @@ -1883,11 +1884,11 @@ def to_profile_array(profile): "and your account permissions." ) - wind_u_values = np.array([profiles["wind_u"][h] for h in wind_heights]) - wind_v_values = np.array([profiles["wind_v"][h] for h in wind_heights]) - wind_asl_heights = np.array(wind_heights, dtype=float) + self.elevation - wind_u_array = np.column_stack((wind_asl_heights, wind_u_values)) - wind_v_array = np.column_stack((wind_asl_heights, wind_v_values)) + wind_asl_heights = wind_array[:, 0] + wind_u_values = wind_array[:, 1] + wind_v_values = wind_array[:, 2] + wind_u_array = wind_array[:, (0, 1)] + wind_v_array = wind_array[:, (0, 2)] wind_speed_array = calculate_wind_speed(wind_u_values, wind_v_values) wind_heading_array = calculate_wind_heading(wind_u_values, wind_v_values) @@ -3181,8 +3182,11 @@ def from_dict(cls, data): # pylint: disable=too-many-statements ) atmospheric_model = data["atmospheric_model_type"] env.atmospheric_model_type = atmospheric_model + # set_atmospheric_model stores the type as the user spelled it (e.g. + # "Meteomatics"), so the dispatch below must be case-insensitive. + model_type = atmospheric_model.lower() - match atmospheric_model: + match model_type: case "standard_atmosphere": env.set_atmospheric_model("standard_atmosphere") case "custom_atmosphere": @@ -3204,13 +3208,7 @@ def from_dict(cls, data): # pylint: disable=too-many-statements env.elevation = data["elevation"] env.max_expected_height = data["max_expected_height"] - if isinstance(atmospheric_model, str) and atmospheric_model.lower() in ( - "windy", - "meteomatics", - "forecast", - "reanalysis", - "ensemble", - ): + if model_type in ("windy", "meteomatics", "forecast", "reanalysis", "ensemble"): env.atmospheric_model_init_date = data["atmospheric_model_init_date"] env.atmospheric_model_end_date = data["atmospheric_model_end_date"] env.atmospheric_model_interval = data["atmospheric_model_interval"] @@ -3219,7 +3217,7 @@ def from_dict(cls, data): # pylint: disable=too-many-statements env.atmospheric_model_init_lon = data["atmospheric_model_init_lon"] env.atmospheric_model_end_lon = data["atmospheric_model_end_lon"] - if atmospheric_model == "ensemble": + if model_type == "ensemble": env.level_ensemble = data["level_ensemble"] env.height_ensemble = data["height_ensemble"] env.temperature_ensemble = data["temperature_ensemble"] diff --git a/rocketpy/environment/fetchers.py b/rocketpy/environment/fetchers.py index 5f3d034eb..ce88ba950 100644 --- a/rocketpy/environment/fetchers.py +++ b/rocketpy/environment/fetchers.py @@ -22,9 +22,6 @@ METEOMATICS_BASE_URL = "https://api.meteomatics.com" METEOMATICS_LOGIN_URL = "https://login.meteomatics.com/api/v1/token" METEOMATICS_TIMEOUT_SECONDS = 30 -# Matches Meteomatics height-level parameters such as "t_500m:K", -# "pressure_1000m:Pa" or "wind_speed_u_120m:ms". -_METEOMATICS_PARAMETER_REGEX = re.compile(r"^(?P[a-z_]+)_(?P\d+)m:") @exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) @@ -468,64 +465,96 @@ def _meteomatics_get(url, headers=None, params=None): url, headers=headers, params=params, timeout=METEOMATICS_TIMEOUT_SECONDS ) if response.status_code >= 500: - # Server-side error: raise so the backoff decorator retries it. response.raise_for_status() return response -def fetch_meteomatics_token(username, password): - """Requests a short-lived access token from the Meteomatics login service. - - The Meteomatics API authenticates with a personal ``username`` and - ``password``. Instead of sending the credentials on every request, a token - is generated once and reused. Each token is valid for a couple of hours, - which is more than enough to build a single ``Environment``. +def _meteomatics_request_json(url, endpoint, headers=None, params=None): + """Queries a Meteomatics endpoint and returns its parsed JSON body. Parameters ---------- - username : str - The Meteomatics account username. - password : str - The Meteomatics account password. + url : str + The endpoint address to query. + endpoint : str + Human-readable name of the endpoint (e.g. ``"login service"``), used to + build the error messages. + headers : dict, optional + Headers to send with the request. + params : dict, optional + Query parameters to send with the request. Returns ------- - str - The access token to be used as the ``access_token`` query parameter in - subsequent data requests. + dict + The parsed JSON body of the response. Raises ------ RuntimeError - If the login service cannot be reached, rejects the credentials, or - does not return a token. + If the endpoint cannot be reached, rejects the credentials, returns an + error status, or returns a malformed (non-JSON) body. Client-side (4xx) + errors are definitive and are not retried. """ - credentials = f"{username}:{password}" - encoded_credentials = base64.b64encode(credentials.encode()).decode() - headers = {"Authorization": f"Basic {encoded_credentials}"} try: - response = _meteomatics_get(METEOMATICS_LOGIN_URL, headers=headers) + response = _meteomatics_get(url, headers=headers, params=params) except requests.exceptions.RequestException as e: raise RuntimeError( - "Unable to reach the Meteomatics login service. Please try again later." + f"Unable to reach the Meteomatics {endpoint}. Please try again later." ) from e - if response.status_code in (401, 403): - # Definitive authentication failure: do not retry. raise RuntimeError( f"Meteomatics rejected the credentials (HTTP {response.status_code}). " "Check your username and password." ) if not response.ok: raise RuntimeError( - f"Meteomatics login request failed (HTTP {response.status_code})." + f"Meteomatics {endpoint} request failed " + f"(HTTP {response.status_code}). {response.text[:300]}".strip() ) try: - token = response.json().get("access_token") + return response.json() except requests.exceptions.JSONDecodeError as e: raise RuntimeError( - "Meteomatics login service returned a malformed (non-JSON) response." + f"Meteomatics {endpoint} returned a malformed (non-JSON) response." ) from e + + +def fetch_meteomatics_token(username, password): + """Requests a short-lived access token from the Meteomatics login service. + + The Meteomatics API authenticates with a personal ``username`` and + ``password``. Instead of sending the credentials on every request, a token + is generated once and reused for the handful of requests needed to build a + single ``Environment``. + + Parameters + ---------- + username : str + The Meteomatics account username. + password : str + The Meteomatics account password. + + Returns + ------- + str + The access token to be used as the ``access_token`` query parameter in + subsequent data requests. + + Raises + ------ + RuntimeError + If the login service cannot be reached, rejects the credentials, or + does not return a token. + """ + credentials = f"{username}:{password}" + encoded_credentials = base64.b64encode(credentials.encode()).decode() + payload = _meteomatics_request_json( + METEOMATICS_LOGIN_URL, + "login service", + headers={"Authorization": f"Basic {encoded_credentials}"}, + ) + token = payload.get("access_token") if not token: raise RuntimeError( "Meteomatics login service did not return an access token. " @@ -538,7 +567,7 @@ def fetch_meteomatics_token(username, password): def _build_meteomatics_parameters( min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution ): - """Builds the list of Meteomatics height-level parameters to query. + """Builds the Meteomatics height-level parameters to query. Wind components are sampled on a finer altitude grid than temperature and pressure, since the wind profile is usually the most variable one. @@ -556,31 +585,36 @@ def _build_meteomatics_parameters( Returns ------- - list of str - Parameter strings in the ``"_m:"`` format. + dict + Maps each parameter string, in the ``"_m:"`` + format, to the ``(profile name, height)`` pair it carries. Keeping this + mapping spares the caller from parsing the parameter strings back. """ - # Round to integer meters and drop duplicates that rounding may introduce - # for narrow bands, so we never request (and pay for) the same height twice. - fine_levels = np.unique( - np.linspace(min_altitude, max_altitude, wind_resolution).round().astype(int) - ) - coarse_levels = np.unique( - np.linspace(min_altitude, max_altitude, temperature_pressure_resolution) - .round() - .astype(int) - ) - wind_parameters = [ - f"{var}_{height}m:{unit}" - for height in fine_levels - for var, unit in [("wind_speed_u", "ms"), ("wind_speed_v", "ms")] - ] - temperature_pressure_parameters = [ - f"{var}_{height}m:{unit}" - for height in coarse_levels - for var, unit in [("t", "K"), ("pressure", "Pa")] + def levels(resolution): + # Round to integer meters and drop duplicates that rounding may + # introduce for narrow bands, so we never request (and pay for) the + # same height twice. + return np.unique( + np.linspace(min_altitude, max_altitude, resolution).round().astype(int) + ) + + grids = [ + ( + levels(wind_resolution), + [("wind_speed_u", "ms", "wind_u"), ("wind_speed_v", "ms", "wind_v")], + ), + ( + levels(temperature_pressure_resolution), + [("t", "K", "temperature"), ("pressure", "Pa", "pressure")], + ), ] - return wind_parameters + temperature_pressure_parameters + return { + f"{var}_{height}m:{unit}": (profile, int(height)) + for heights, variables in grids + for height in heights + for var, unit, profile in variables + } def _extract_meteomatics_json(data): @@ -616,34 +650,6 @@ def _extract_meteomatics_json(data): ) from e -def _fetch_meteomatics_group(base_url, query_params): - """Performs a single Meteomatics data request and returns the JSON body. - - Raises - ------ - RuntimeError - If the API cannot be reached, returns an error status, or returns a - malformed (non-JSON) body. Client-side (4xx) errors are not retried. - """ - try: - response = _meteomatics_get(base_url, params=query_params) - except requests.exceptions.RequestException as e: - raise RuntimeError( - "Unable to reach the Meteomatics data API. Please try again later." - ) from e - if not response.ok: - raise RuntimeError( - f"Meteomatics data request failed (HTTP {response.status_code}). " - f"{response.text[:300]}".strip() - ) - try: - return response.json() - except requests.exceptions.JSONDecodeError as e: - raise RuntimeError( - "Meteomatics data API returned a malformed (non-JSON) response." - ) from e - - def fetch_atmospheric_data_from_meteomatics( username, password, @@ -724,9 +730,10 @@ def fetch_atmospheric_data_from_meteomatics( token = fetch_meteomatics_token(username, password) date_string = date.strftime("%Y-%m-%dT%H:%M:%SZ") - parameters = _build_meteomatics_parameters( + parameter_map = _build_meteomatics_parameters( min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution ) + parameters = list(parameter_map) parameter_groups = [ parameters[i : i + query_limit] for i in range(0, len(parameters), query_limit) ] @@ -737,12 +744,6 @@ def fetch_atmospheric_data_from_meteomatics( "wind_u": {}, "wind_v": {}, } - variable_to_profile = { - "t": "temperature", - "pressure": "pressure", - "wind_speed_u": "wind_u", - "wind_speed_v": "wind_v", - } for index, parameter_group in enumerate(parameter_groups): logger.info( @@ -756,13 +757,15 @@ def fetch_atmospheric_data_from_meteomatics( f"{latitude},{longitude}/json" ) query_params = {"model": model, "access_token": token} - data = _fetch_meteomatics_group(base_url, query_params) + data = _meteomatics_request_json(base_url, "data API", params=query_params) for parameter, value in _extract_meteomatics_json(data): - match = _METEOMATICS_PARAMETER_REGEX.match(parameter) - if match is None or match.group("var") not in variable_to_profile: - raise ValueError(f"Unrecognized Meteomatics parameter '{parameter}'.") - height = int(match.group("height")) - profiles[variable_to_profile[match.group("var")]][height] = value + try: + profile, height = parameter_map[parameter] + except KeyError as e: + raise ValueError( + f"Unrecognized Meteomatics parameter '{parameter}'." + ) from e + profiles[profile][height] = value return profiles diff --git a/tests/unit/environment/test_fetchers.py b/tests/unit/environment/test_fetchers.py index 7e929aa9a..6ec074f83 100644 --- a/tests/unit/environment/test_fetchers.py +++ b/tests/unit/environment/test_fetchers.py @@ -255,7 +255,7 @@ def test_fetch_meteomatics_data_client_error_not_retried(monkeypatch): fetchers.requests, "get", _make_fake_meteomatics_get(calls, data_status=400) ) - with pytest.raises(RuntimeError, match="data request failed"): + with pytest.raises(RuntimeError, match="data API request failed"): fetchers.fetch_atmospheric_data_from_meteomatics( username="user", password="pass", From bda2adcc2af184ba3193f9bd23eddc1fde253623 Mon Sep 17 00:00:00 2001 From: Gui-FernandesBR Date: Thu, 30 Jul 2026 13:43:40 -0300 Subject: [PATCH 3/5] MNT: address Meteomatics review comments Two points raised in the PR review: - `fetch_atmospheric_data_from_meteomatics` stamped the instant with a trailing "Z" without normalizing the timezone, so an aware datetime in a non-UTC zone was sent as the wrong instant. Aware datetimes are now converted to UTC; naive ones are documented as assumed UTC. - Degenerate sampling arguments (`query_limit=0`, resolutions below 2) reached `range()`/`linspace()` and failed with an opaque low-level error after the login had already been paid for. They are now validated up front by `_validate_meteomatics_sampling`. - `process_meteomatics_atmosphere` silently coerced any non-string model to "mix", hiding a mistake such as passing a Dataset or a path as `file` and querying the wrong model. It now accepts None (default) or a string, and raises otherwise. Co-Authored-By: Claude Opus 5 (1M context) --- rocketpy/environment/environment.py | 14 +++-- rocketpy/environment/fetchers.py | 59 ++++++++++++++++++---- tests/unit/environment/test_environment.py | 14 +++++ tests/unit/environment/test_fetchers.py | 51 ++++++++++++++++++- 4 files changed, 125 insertions(+), 13 deletions(-) diff --git a/rocketpy/environment/environment.py b/rocketpy/environment/environment.py index 5c456fad0..c28184b35 100644 --- a/rocketpy/environment/environment.py +++ b/rocketpy/environment/environment.py @@ -1808,10 +1808,18 @@ def process_meteomatics_atmosphere( # pylint: disable=too-many-statements Raises ------ ValueError - If credentials are missing, if no launch date is set, or if the API - returns no usable data. + If ``model`` is not a string, if credentials are missing, if no + launch date is set, or if the API returns no usable data. """ - model = model or "mix" + if model is None: + model = "mix" + elif not isinstance(model, str): + # Coercing silently would hide a mistake such as passing a Dataset + # or a file path as 'file', and would query the wrong model. + raise ValueError( + f"Invalid Meteomatics model {model!r}: expected the model name as " + "a string (e.g. 'mix'), or None to use the default." + ) username = username or os.environ.get("METEOMATICS_USERNAME") password = password or os.environ.get("METEOMATICS_PASSWORD") if not username or not password: diff --git a/rocketpy/environment/fetchers.py b/rocketpy/environment/fetchers.py index ce88ba950..45863496a 100644 --- a/rocketpy/environment/fetchers.py +++ b/rocketpy/environment/fetchers.py @@ -617,6 +617,39 @@ def levels(resolution): } +def _validate_meteomatics_sampling( + min_altitude, + max_altitude, + wind_resolution, + temperature_pressure_resolution, + query_limit, +): + """Validates the sampling arguments before any request is issued. + + Catching these here keeps a degenerate input from reaching ``linspace`` or + ``range``, where it would surface as an opaque low-level error (or as an + empty request) after the account has already been charged for the login. + + Raises + ------ + ValueError + If the altitude range, the resolutions or the query limit are invalid. + """ + if min_altitude < 0: + raise ValueError( + "min_altitude must be non-negative (heights are above ground level)." + ) + if max_altitude <= min_altitude: + raise ValueError("max_altitude must be greater than min_altitude.") + if wind_resolution < 2 or temperature_pressure_resolution < 2: + raise ValueError( + "wind_resolution and temperature_pressure_resolution must be at least " + "2: a single altitude level is not enough to define a profile." + ) + if query_limit < 1: + raise ValueError("query_limit must be at least 1.") + + def _extract_meteomatics_json(data): """Extracts (parameter, value) pairs from a Meteomatics JSON response. @@ -682,7 +715,9 @@ def fetch_atmospheric_data_from_meteomatics( Longitude of the launch site, in degrees. date : datetime.datetime The instant to query. It is formatted according to the Meteomatics - date-time specification (``%Y-%m-%dT%H:%M:%SZ``). + date-time specification (``%Y-%m-%dT%H:%M:%SZ``). Timezone-aware + datetimes are converted to UTC; naive ones are assumed to be UTC + already. model : str, optional The Meteomatics weather model to use. Default is ``"mix"``. Your account may not have access to every model. See @@ -717,18 +752,24 @@ def fetch_atmospheric_data_from_meteomatics( If authentication fails, the API cannot be reached, returns an error status, or returns a malformed response. ValueError - If the altitude range is invalid or the response contains an - unrecognized parameter. + If the altitude range, the resolutions or the query limit are invalid, + or if the response contains an unrecognized parameter. """ - if min_altitude < 0: - raise ValueError( - "min_altitude must be non-negative (heights are above ground level)." - ) - if max_altitude <= min_altitude: - raise ValueError("max_altitude must be greater than min_altitude.") + _validate_meteomatics_sampling( + min_altitude, + max_altitude, + wind_resolution, + temperature_pressure_resolution, + query_limit, + ) token = fetch_meteomatics_token(username, password) + # The instant is sent with a trailing "Z", so an aware datetime must be + # converted to UTC instead of being formatted as-is. A naive datetime is + # assumed to already be in UTC. + if date.tzinfo is not None: + date = date.astimezone(timezone.utc) date_string = date.strftime("%Y-%m-%dT%H:%M:%SZ") parameter_map = _build_meteomatics_parameters( min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution diff --git a/tests/unit/environment/test_environment.py b/tests/unit/environment/test_environment.py index 11176304c..bee3decf1 100644 --- a/tests/unit/environment/test_environment.py +++ b/tests/unit/environment/test_environment.py @@ -481,6 +481,20 @@ def test_meteomatics_atmosphere_sets_profiles(example_euroc_env, monkeypatch): assert recorder["model"] == "mix" +def test_meteomatics_non_string_model_raises(example_euroc_env, monkeypatch): + """Reject a non-string model instead of silently querying the default. + + Passing a Dataset or a path as ``file`` by accident must not be coerced to + ``"mix"``, which would quietly query (and charge for) the wrong model. + """ + _patch_meteomatics_fetcher(monkeypatch) + + with pytest.raises(ValueError, match="Invalid Meteomatics model"): + example_euroc_env.set_atmospheric_model( + type="Meteomatics", file=123, username="user", password="pass" + ) + + def test_meteomatics_reads_credentials_from_environment(example_euroc_env, monkeypatch): """Fall back to the METEOMATICS_* environment variables for credentials.""" recorder = {} diff --git a/tests/unit/environment/test_fetchers.py b/tests/unit/environment/test_fetchers.py index 6ec074f83..a3c53d176 100644 --- a/tests/unit/environment/test_fetchers.py +++ b/tests/unit/environment/test_fetchers.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone import pytest @@ -301,3 +301,52 @@ def test_fetch_meteomatics_data_invalid_altitude_range_raises(altitudes): date=datetime(2024, 1, 1, 12, tzinfo=timezone.utc), **altitudes, ) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"wind_resolution": 1}, "at least"), + ({"temperature_pressure_resolution": 0}, "at least"), + ({"query_limit": 0}, "query_limit must be at least 1"), + ], +) +def test_fetch_meteomatics_data_invalid_sampling_raises(kwargs, message): + """Reject degenerate resolutions and query limits with a clear message. + + Without the up-front check these reach ``linspace``/``range`` and fail with + an opaque low-level error (or an empty request) instead. + """ + with pytest.raises(ValueError, match=message): + fetchers.fetch_atmospheric_data_from_meteomatics( + username="user", + password="pass", + latitude=39.0, + longitude=-8.0, + date=datetime(2024, 1, 1, 12, tzinfo=timezone.utc), + **kwargs, + ) + + +def test_fetch_meteomatics_data_converts_date_to_utc(monkeypatch): + """A non-UTC aware datetime must be converted, not stamped with a bare Z. + + The request path carries the instant with a trailing "Z", so 12:00 at + UTC+03:00 has to be sent as 09:00Z. + """ + calls = [] + monkeypatch.setattr(fetchers.requests, "get", _make_fake_meteomatics_get(calls)) + + fetchers.fetch_atmospheric_data_from_meteomatics( + username="user", + password="pass", + latitude=39.0, + longitude=-8.0, + date=datetime(2024, 1, 1, 12, tzinfo=timezone(timedelta(hours=3))), + wind_resolution=2, + temperature_pressure_resolution=2, + ) + + data_calls = [c for c in calls if c[0] != fetchers.METEOMATICS_LOGIN_URL] + assert data_calls, "expected at least one data request" + assert all("2024-01-01T09:00:00Z" in url for url, _ in data_calls) From 8f1a8190ae242c6754bd668654975dbc01786108 Mon Sep 17 00:00:00 2001 From: Gui-FernandesBR Date: Sat, 1 Aug 2026 10:56:36 -0300 Subject: [PATCH 4/5] ENH: Refactor Meteomatics integration into MeteomaticsFetcher class and clean up environment method --- rocketpy/environment/environment.py | 223 ++++---- rocketpy/environment/fetchers.py | 697 +++++++++++++----------- tests/unit/environment/test_fetchers.py | 2 +- 3 files changed, 493 insertions(+), 429 deletions(-) diff --git a/rocketpy/environment/environment.py b/rocketpy/environment/environment.py index c28184b35..bc8330c8c 100644 --- a/rocketpy/environment/environment.py +++ b/rocketpy/environment/environment.py @@ -1755,7 +1755,123 @@ def __parse_windy_file(self, response, time_index, pressure_levels): wind_v_array, ) - def process_meteomatics_atmosphere( # pylint: disable=too-many-statements + @staticmethod + def _validate_meteomatics_credentials_and_model(model, username, password): + """Validates model and credentials for Meteomatics requests.""" + if model is None: + model = "mix" + elif not isinstance(model, str): + # Coercing silently would hide a mistake such as passing a Dataset + # or a file path as 'file', and would query the wrong model. + raise ValueError( + f"Invalid Meteomatics model {model!r}: expected the model name as " + "a string (e.g. 'mix'), or None to use the default." + ) + username = username or os.environ.get("METEOMATICS_USERNAME") + password = password or os.environ.get("METEOMATICS_PASSWORD") + if not username or not password: + raise ValueError( + "Meteomatics requires a username and password. Provide them via " + "the 'username' and 'password' arguments of set_atmospheric_model, " + "or set the METEOMATICS_USERNAME and METEOMATICS_PASSWORD " + "environment variables." + ) + return model, username, password + + def _store_meteomatics_functions( + self, pressure_array, temperature_array, wind_array + ): + """Sets internal atmospheric functions for Meteomatics.""" + wind_asl_heights = wind_array[:, 0] + wind_u_values = wind_array[:, 1] + wind_v_values = wind_array[:, 2] + + wind_speed_array = calculate_wind_speed(wind_u_values, wind_v_values) + wind_heading_array = calculate_wind_heading(wind_u_values, wind_v_values) + wind_direction_array = convert_wind_heading_to_direction(wind_heading_array) + + # Save atmospheric data + self.__set_pressure_function(pressure_array) + self.__set_barometric_height_function(pressure_array[:, (1, 0)]) + self.__set_temperature_function(temperature_array) + self.__set_wind_velocity_x_function(wind_array[:, (0, 1)]) + self.__set_wind_velocity_y_function(wind_array[:, (0, 2)]) + self.__set_wind_heading_function( + np.column_stack((wind_asl_heights, wind_heading_array)) + ) + self.__set_wind_direction_function( + np.column_stack((wind_asl_heights, wind_direction_array)) + ) + self.__set_wind_speed_function( + np.column_stack((wind_asl_heights, wind_speed_array)) + ) + + # Save maximum expected height + self._max_expected_height = float( + max(pressure_array[-1, 0], temperature_array[-1, 0], wind_asl_heights[-1]) + ) + + def _store_meteomatics_metadata( + self, pressure_array, temperature_array, wind_array + ): + """Sets metadata attributes and debug data for Meteomatics.""" + wind_asl_heights = wind_array[:, 0] + self.atmospheric_model_init_date = self.datetime_date + self.atmospheric_model_end_date = self.datetime_date + self.atmospheric_model_interval = 0 + self.atmospheric_model_init_lat = self.latitude + self.atmospheric_model_end_lat = self.latitude + self.atmospheric_model_init_lon = self.longitude + self.atmospheric_model_end_lon = self.longitude + + # Save debugging data + self.wind_us = wind_array[:, 1] + self.wind_vs = wind_array[:, 2] + self.temperatures = temperature_array[:, 1] + self.pressures = pressure_array[:, 1] + self.height = wind_asl_heights + + def _process_meteomatics_profiles(self, profiles): + """Converts retrieved height-AGL profiles to ASL arrays and configures + the Environment atmospheric functions.""" + + def to_profile_array(*names): + common_heights = set.intersection(*(set(profiles[n]) for n in names)) + heights = sorted( + h + for h in common_heights + if all(profiles[n][h] is not None for n in names) + ) + return np.array( + [ + (h + self.elevation, *(profiles[n][h] for n in names)) + for h in heights + ], + dtype=float, + ) + + pressure_array = to_profile_array("pressure") + temperature_array = to_profile_array("temperature") + # Wind u and v share the same altitude grid; keep only common levels. + wind_array = to_profile_array("wind_u", "wind_v") + + # Each profile needs at least two levels: a single-point Function cannot + # be evaluated at its own node (it raises IndexError downstream), so a + # collapsed grid must fail here with an actionable message instead. + if min(len(pressure_array), len(temperature_array), len(wind_array)) < 2: + raise ValueError( + "Meteomatics did not return enough usable atmospheric data: at " + "least two valid altitude levels are required for pressure, " + "temperature and wind. Check the requested model, the altitude " + "range (min_altitude and max_altitude must be far enough apart " + "that the sampled levels do not collapse to a single height), " + "and your account permissions." + ) + + self._store_meteomatics_functions(pressure_array, temperature_array, wind_array) + self._store_meteomatics_metadata(pressure_array, temperature_array, wind_array) + + def process_meteomatics_atmosphere( self, model="mix", username=None, @@ -1811,24 +1927,9 @@ def process_meteomatics_atmosphere( # pylint: disable=too-many-statements If ``model`` is not a string, if credentials are missing, if no launch date is set, or if the API returns no usable data. """ - if model is None: - model = "mix" - elif not isinstance(model, str): - # Coercing silently would hide a mistake such as passing a Dataset - # or a file path as 'file', and would query the wrong model. - raise ValueError( - f"Invalid Meteomatics model {model!r}: expected the model name as " - "a string (e.g. 'mix'), or None to use the default." - ) - username = username or os.environ.get("METEOMATICS_USERNAME") - password = password or os.environ.get("METEOMATICS_PASSWORD") - if not username or not password: - raise ValueError( - "Meteomatics requires a username and password. Provide them via " - "the 'username' and 'password' arguments of set_atmospheric_model, " - "or set the METEOMATICS_USERNAME and METEOMATICS_PASSWORD " - "environment variables." - ) + model, username, password = self._validate_meteomatics_credentials_and_model( + model, username, password + ) self.__validate_datetime() if self.elevation == 0: @@ -1855,89 +1956,7 @@ def process_meteomatics_atmosphere( # pylint: disable=too-many-statements query_limit=query_limit, ) - def to_profile_array(*names): - """Convert {AGL height: value} mappings into a sorted array whose - first column is the ASL height and the remaining ones the requested - profile values, keeping only the heights that carry a value in - every mapping.""" - common_heights = set.intersection(*(set(profiles[n]) for n in names)) - heights = sorted( - h - for h in common_heights - if all(profiles[n][h] is not None for n in names) - ) - return np.array( - [ - (h + self.elevation, *(profiles[n][h] for n in names)) - for h in heights - ], - dtype=float, - ) - - pressure_array = to_profile_array("pressure") - temperature_array = to_profile_array("temperature") - # Wind u and v share the same altitude grid; keep only common levels. - wind_array = to_profile_array("wind_u", "wind_v") - - # Each profile needs at least two levels: a single-point Function cannot - # be evaluated at its own node (it raises IndexError downstream), so a - # collapsed grid must fail here with an actionable message instead. - if min(len(pressure_array), len(temperature_array), len(wind_array)) < 2: - raise ValueError( - "Meteomatics did not return enough usable atmospheric data: at " - "least two valid altitude levels are required for pressure, " - "temperature and wind. Check the requested model, the altitude " - "range (min_altitude and max_altitude must be far enough apart " - "that the sampled levels do not collapse to a single height), " - "and your account permissions." - ) - - wind_asl_heights = wind_array[:, 0] - wind_u_values = wind_array[:, 1] - wind_v_values = wind_array[:, 2] - wind_u_array = wind_array[:, (0, 1)] - wind_v_array = wind_array[:, (0, 2)] - - wind_speed_array = calculate_wind_speed(wind_u_values, wind_v_values) - wind_heading_array = calculate_wind_heading(wind_u_values, wind_v_values) - wind_direction_array = convert_wind_heading_to_direction(wind_heading_array) - - # Save atmospheric data - self.__set_pressure_function(pressure_array) - self.__set_barometric_height_function(pressure_array[:, (1, 0)]) - self.__set_temperature_function(temperature_array) - self.__set_wind_velocity_x_function(wind_u_array) - self.__set_wind_velocity_y_function(wind_v_array) - self.__set_wind_heading_function( - np.column_stack((wind_asl_heights, wind_heading_array)) - ) - self.__set_wind_direction_function( - np.column_stack((wind_asl_heights, wind_direction_array)) - ) - self.__set_wind_speed_function( - np.column_stack((wind_asl_heights, wind_speed_array)) - ) - - # Save maximum expected height - self._max_expected_height = float( - max(pressure_array[-1, 0], temperature_array[-1, 0], wind_asl_heights[-1]) - ) - - # Save model info metadata (single point in space and time) - self.atmospheric_model_init_date = self.datetime_date - self.atmospheric_model_end_date = self.datetime_date - self.atmospheric_model_interval = 0 - self.atmospheric_model_init_lat = self.latitude - self.atmospheric_model_end_lat = self.latitude - self.atmospheric_model_init_lon = self.longitude - self.atmospheric_model_end_lon = self.longitude - - # Save debugging data - self.wind_us = wind_u_values - self.wind_vs = wind_v_values - self.temperatures = temperature_array[:, 1] - self.pressures = pressure_array[:, 1] - self.height = wind_asl_heights + self._process_meteomatics_profiles(profiles) def process_wyoming_sounding(self, file): # pylint: disable=too-many-statements """Import and process the upper air sounding data from `Wyoming diff --git a/rocketpy/environment/fetchers.py b/rocketpy/environment/fetchers.py index 45863496a..8ad256063 100644 --- a/rocketpy/environment/fetchers.py +++ b/rocketpy/environment/fetchers.py @@ -452,235 +452,380 @@ def fetch_cmc_ensemble(): raise RuntimeError("Unable to load latest weather data for CMC through " + file) -@exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) -def _meteomatics_get(url, headers=None, params=None): - """Performs a single Meteomatics GET request, retrying transient failures. - - Connection-level errors (and server-side 5xx responses) raise and are - retried by the decorator. Client-side 4xx responses are returned as-is so - the caller can turn them into an actionable, non-retried error, since - retrying a deterministic 4xx only wastes time and API quota. +class MeteomaticsFetcher: + """Fetcher class to authenticate and query vertical atmospheric profiles + from the Meteomatics API. """ - response = requests.get( - url, headers=headers, params=params, timeout=METEOMATICS_TIMEOUT_SECONDS - ) - if response.status_code >= 500: - response.raise_for_status() - return response - - -def _meteomatics_request_json(url, endpoint, headers=None, params=None): - """Queries a Meteomatics endpoint and returns its parsed JSON body. - - Parameters - ---------- - url : str - The endpoint address to query. - endpoint : str - Human-readable name of the endpoint (e.g. ``"login service"``), used to - build the error messages. - headers : dict, optional - Headers to send with the request. - params : dict, optional - Query parameters to send with the request. - - Returns - ------- - dict - The parsed JSON body of the response. - Raises - ------ - RuntimeError - If the endpoint cannot be reached, rejects the credentials, returns an - error status, or returns a malformed (non-JSON) body. Client-side (4xx) - errors are definitive and are not retried. - """ - try: - response = _meteomatics_get(url, headers=headers, params=params) - except requests.exceptions.RequestException as e: - raise RuntimeError( - f"Unable to reach the Meteomatics {endpoint}. Please try again later." - ) from e - if response.status_code in (401, 403): - raise RuntimeError( - f"Meteomatics rejected the credentials (HTTP {response.status_code}). " - "Check your username and password." + @staticmethod + @exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) + def _get(url, headers=None, params=None): + """Performs a single Meteomatics GET request, retrying transient failures. + + Connection-level errors (and server-side 5xx responses) raise and are + retried by the decorator. Client-side 4xx responses are returned as-is so + the caller can turn them into an actionable, non-retried error, since + retrying a deterministic 4xx only wastes time and API quota. + """ + response = requests.get( + url, headers=headers, params=params, timeout=METEOMATICS_TIMEOUT_SECONDS ) - if not response.ok: - raise RuntimeError( - f"Meteomatics {endpoint} request failed " - f"(HTTP {response.status_code}). {response.text[:300]}".strip() - ) - try: - return response.json() - except requests.exceptions.JSONDecodeError as e: - raise RuntimeError( - f"Meteomatics {endpoint} returned a malformed (non-JSON) response." - ) from e - - -def fetch_meteomatics_token(username, password): - """Requests a short-lived access token from the Meteomatics login service. - - The Meteomatics API authenticates with a personal ``username`` and - ``password``. Instead of sending the credentials on every request, a token - is generated once and reused for the handful of requests needed to build a - single ``Environment``. - - Parameters - ---------- - username : str - The Meteomatics account username. - password : str - The Meteomatics account password. - - Returns - ------- - str - The access token to be used as the ``access_token`` query parameter in - subsequent data requests. + if response.status_code >= 500: + response.raise_for_status() + return response + + @classmethod + def _request_json(cls, url, endpoint, headers=None, params=None): + """Queries a Meteomatics endpoint and returns its parsed JSON body. + + Parameters + ---------- + url : str + The endpoint address to query. + endpoint : str + Human-readable name of the endpoint (e.g. ``"login service"``), used to + build the error messages. + headers : dict, optional + Headers to send with the request. + params : dict, optional + Query parameters to send with the request. + + Returns + ------- + dict + The parsed JSON body of the response. + + Raises + ------ + RuntimeError + If the endpoint cannot be reached, rejects the credentials, returns an + error status, or returns a malformed (non-JSON) body. Client-side (4xx) + errors are definitive and are not retried. + """ + try: + response = cls._get(url, headers=headers, params=params) + except requests.exceptions.RequestException as e: + raise RuntimeError( + f"Unable to reach the Meteomatics {endpoint}. Please try again later." + ) from e + if response.status_code in (401, 403): + raise RuntimeError( + f"Meteomatics rejected the credentials (HTTP {response.status_code}). " + "Check your username and password." + ) + if not response.ok: + raise RuntimeError( + f"Meteomatics {endpoint} request failed " + f"(HTTP {response.status_code}). {response.text[:300]}".strip() + ) + try: + return response.json() + except requests.exceptions.JSONDecodeError as e: + raise RuntimeError( + f"Meteomatics {endpoint} returned a malformed (non-JSON) response." + ) from e - Raises - ------ - RuntimeError - If the login service cannot be reached, rejects the credentials, or - does not return a token. - """ - credentials = f"{username}:{password}" - encoded_credentials = base64.b64encode(credentials.encode()).decode() - payload = _meteomatics_request_json( - METEOMATICS_LOGIN_URL, - "login service", - headers={"Authorization": f"Basic {encoded_credentials}"}, - ) - token = payload.get("access_token") - if not token: - raise RuntimeError( - "Meteomatics login service did not return an access token. " - "Check your username and password." + @classmethod + def fetch_token(cls, username, password): + """Requests a short-lived access token from the Meteomatics login service. + + The Meteomatics API authenticates with a personal ``username`` and + ``password``. Instead of sending the credentials on every request, a token + is generated once and reused for the handful of requests needed to build a + single ``Environment``. + + Parameters + ---------- + username : str + The Meteomatics account username. + password : str + The Meteomatics account password. + + Returns + ------- + str + The access token to be used as the ``access_token`` query parameter in + subsequent data requests. + + Raises + ------ + RuntimeError + If the login service cannot be reached, rejects the credentials, or + does not return a token. + """ + credentials = f"{username}:{password}" + encoded_credentials = base64.b64encode(credentials.encode()).decode() + payload = cls._request_json( + METEOMATICS_LOGIN_URL, + "login service", + headers={"Authorization": f"Basic {encoded_credentials}"}, ) - logger.info("Meteomatics access token generated successfully.") - return token - - -def _build_meteomatics_parameters( - min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution -): - """Builds the Meteomatics height-level parameters to query. - - Wind components are sampled on a finer altitude grid than temperature and - pressure, since the wind profile is usually the most variable one. + token = payload.get("access_token") + if not token: + raise RuntimeError( + "Meteomatics login service did not return an access token. " + "Check your username and password." + ) + logger.info("Meteomatics access token generated successfully.") + return token - Parameters - ---------- - min_altitude : float - Lowest altitude above ground level (in meters) to query. - max_altitude : float - Highest altitude above ground level (in meters) to query. - wind_resolution : int - Number of altitude levels used for the wind components. - temperature_pressure_resolution : int - Number of altitude levels used for temperature and pressure. + @staticmethod + def _build_parameters( + min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution + ): + """Builds the Meteomatics height-level parameters to query. + + Wind components are sampled on a finer altitude grid than temperature and + pressure, since the wind profile is usually the most variable one. + + Parameters + ---------- + min_altitude : float + Lowest altitude above ground level (in meters) to query. + max_altitude : float + Highest altitude above ground level (in meters) to query. + wind_resolution : int + Number of altitude levels used for the wind components. + temperature_pressure_resolution : int + Number of altitude levels used for temperature and pressure. + + Returns + ------- + dict + Maps each parameter string, in the ``"_m:"`` + format, to the ``(profile name, height)`` pair it carries. Keeping this + mapping spares the caller from parsing the parameter strings back. + """ + + def levels(resolution): + # Round to integer meters and drop duplicates that rounding may + # introduce for narrow bands, so we never request (and pay for) the + # same height twice. + return np.unique( + np.linspace(min_altitude, max_altitude, resolution).round().astype(int) + ) - Returns - ------- - dict - Maps each parameter string, in the ``"_m:"`` - format, to the ``(profile name, height)`` pair it carries. Keeping this - mapping spares the caller from parsing the parameter strings back. - """ + grids = [ + ( + levels(wind_resolution), + [("wind_speed_u", "ms", "wind_u"), ("wind_speed_v", "ms", "wind_v")], + ), + ( + levels(temperature_pressure_resolution), + [("t", "K", "temperature"), ("pressure", "Pa", "pressure")], + ), + ] + return { + f"{var}_{height}m:{unit}": (profile, int(height)) + for heights, variables in grids + for height in heights + for var, unit, profile in variables + } + + @staticmethod + def _validate_sampling( + min_altitude, + max_altitude, + wind_resolution, + temperature_pressure_resolution, + query_limit, + ): + """Validates the sampling arguments before any request is issued. + + Catching these here keeps a degenerate input from reaching ``linspace`` or + ``range``, where it would surface as an opaque low-level error (or as an + empty request) after the account has already been charged for the login. + + Raises + ------ + ValueError + If the altitude range, the resolutions or the query limit are invalid. + """ + if min_altitude < 0: + raise ValueError( + "min_altitude must be non-negative (heights are above ground level)." + ) + if max_altitude <= min_altitude: + raise ValueError("max_altitude must be greater than min_altitude.") + if wind_resolution < 2 or temperature_pressure_resolution < 2: + raise ValueError( + "wind_resolution and temperature_pressure_resolution must be at least " + "2: a single altitude level is not enough to define a profile." + ) + if query_limit < 1: + raise ValueError("query_limit must be at least 1.") + + @staticmethod + def _extract_json(data): + """Extracts (parameter, value) pairs from a Meteomatics JSON response. + + Only the first coordinate and first date of each parameter are used, since + the query is always issued for a single location and a single instant. + + Parameters + ---------- + data : dict + The JSON payload returned by the Meteomatics data endpoint. + + Returns + ------- + list of tuple + A list of ``(parameter, value)`` tuples. + + Raises + ------ + RuntimeError + If the payload does not have the expected Meteomatics structure. + """ + try: + return [ + (entry["parameter"], entry["coordinates"][0]["dates"][0]["value"]) + for entry in data["data"] + ] + except (KeyError, IndexError, TypeError) as e: + raise RuntimeError( + "Unexpected Meteomatics response structure; could not extract the " + "requested data." + ) from e - def levels(resolution): - # Round to integer meters and drop duplicates that rounding may - # introduce for narrow bands, so we never request (and pay for) the - # same height twice. - return np.unique( - np.linspace(min_altitude, max_altitude, resolution).round().astype(int) + @classmethod + def fetch_atmospheric_data( + cls, + username, + password, + latitude, + longitude, + date, + model="mix", + min_altitude=10, + max_altitude=12000, + wind_resolution=20, + temperature_pressure_resolution=10, + query_limit=10, + ): + """Fetches a vertical atmospheric profile from the Meteomatics API. + + The data is retrieved for a single location and instant, sampling + temperature, pressure and both wind components at several altitudes above + ground level. To respect the account's per-request parameter limit, the + parameters are split into groups that are queried separately. + + Parameters + ---------- + username : str + The Meteomatics account username. + password : str + The Meteomatics account password. + latitude : float + Latitude of the launch site, in degrees. + longitude : float + Longitude of the launch site, in degrees. + date : datetime.datetime + The instant to query. It is formatted according to the Meteomatics + date-time specification (``%Y-%m-%dT%H:%M:%SZ``). Timezone-aware + datetimes are converted to UTC; naive ones are assumed to be UTC + already. + model : str, optional + The Meteomatics weather model to use. Default is ``"mix"``. Your + account may not have access to every model. See + https://www.meteomatics.com/en/api/request/optional-parameters/data-source/ + min_altitude : float, optional + Lowest altitude above ground level (in meters) to query. Default is 10. + max_altitude : float, optional + Highest altitude above ground level (in meters) to query. Default is + 12000. The API returns an error if the requested altitude is outside + the range supported by the chosen model. + wind_resolution : int, optional + Number of altitude levels used for the wind components. Default is 20. + temperature_pressure_resolution : int, optional + Number of altitude levels used for temperature and pressure. Default is + 10. + query_limit : int, optional + Maximum number of parameters requested at once. Parameters are grouped + accordingly to work around the account's per-request limit. Default is + 10. See https://api.meteomatics.com/user_stats for your own limits. + + Returns + ------- + dict + A dictionary with the keys ``"temperature"``, ``"pressure"``, + ``"wind_u"`` and ``"wind_v"``. Each value is a dictionary mapping the + altitude above ground level (in meters) to the corresponding value, in + SI units (K, Pa, m/s and m/s respectively). + + Raises + ------ + RuntimeError + If authentication fails, the API cannot be reached, returns an error + status, or returns a malformed response. + ValueError + If the altitude range, the resolutions or the query limit are invalid, + or if the response contains an unrecognized parameter. + """ + cls._validate_sampling( + min_altitude, + max_altitude, + wind_resolution, + temperature_pressure_resolution, + query_limit, ) - grids = [ - ( - levels(wind_resolution), - [("wind_speed_u", "ms", "wind_u"), ("wind_speed_v", "ms", "wind_v")], - ), - ( - levels(temperature_pressure_resolution), - [("t", "K", "temperature"), ("pressure", "Pa", "pressure")], - ), - ] - return { - f"{var}_{height}m:{unit}": (profile, int(height)) - for heights, variables in grids - for height in heights - for var, unit, profile in variables - } - - -def _validate_meteomatics_sampling( - min_altitude, - max_altitude, - wind_resolution, - temperature_pressure_resolution, - query_limit, -): - """Validates the sampling arguments before any request is issued. - - Catching these here keeps a degenerate input from reaching ``linspace`` or - ``range``, where it would surface as an opaque low-level error (or as an - empty request) after the account has already been charged for the login. - - Raises - ------ - ValueError - If the altitude range, the resolutions or the query limit are invalid. - """ - if min_altitude < 0: - raise ValueError( - "min_altitude must be non-negative (heights are above ground level)." - ) - if max_altitude <= min_altitude: - raise ValueError("max_altitude must be greater than min_altitude.") - if wind_resolution < 2 or temperature_pressure_resolution < 2: - raise ValueError( - "wind_resolution and temperature_pressure_resolution must be at least " - "2: a single altitude level is not enough to define a profile." + token = cls.fetch_token(username, password) + + # The instant is sent with a trailing "Z", so an aware datetime must be + # converted to UTC instead of being formatted as-is. A naive datetime is + # assumed to already be in UTC. + if date.tzinfo is not None: + date = date.astimezone(timezone.utc) + date_string = date.strftime("%Y-%m-%dT%H:%M:%SZ") + parameter_map = cls._build_parameters( + min_altitude, + max_altitude, + wind_resolution, + temperature_pressure_resolution, ) - if query_limit < 1: - raise ValueError("query_limit must be at least 1.") + parameters = list(parameter_map) + parameter_groups = [ + parameters[i : i + query_limit] + for i in range(0, len(parameters), query_limit) + ] + profiles = { + "temperature": {}, + "pressure": {}, + "wind_u": {}, + "wind_v": {}, + } + + for index, parameter_group in enumerate(parameter_groups): + logger.info( + "Fetching Meteomatics data for group %d/%d.", + index + 1, + len(parameter_groups), + ) + parameters_str = ",".join(parameter_group) + base_url = ( + f"{METEOMATICS_BASE_URL}/{date_string}/{parameters_str}/" + f"{latitude},{longitude}/json" + ) + query_params = {"model": model, "access_token": token} + data = cls._request_json(base_url, "data API", params=query_params) -def _extract_meteomatics_json(data): - """Extracts (parameter, value) pairs from a Meteomatics JSON response. + for parameter, value in cls._extract_json(data): + try: + profile, height = parameter_map[parameter] + except KeyError as e: + raise ValueError( + f"Unrecognized Meteomatics parameter '{parameter}'." + ) from e + profiles[profile][height] = value - Only the first coordinate and first date of each parameter are used, since - the query is always issued for a single location and a single instant. + return profiles - Parameters - ---------- - data : dict - The JSON payload returned by the Meteomatics data endpoint. - - Returns - ------- - list of tuple - A list of ``(parameter, value)`` tuples. - Raises - ------ - RuntimeError - If the payload does not have the expected Meteomatics structure. - """ - try: - return [ - (entry["parameter"], entry["coordinates"][0]["dates"][0]["value"]) - for entry in data["data"] - ] - except (KeyError, IndexError, TypeError) as e: - raise RuntimeError( - "Unexpected Meteomatics response structure; could not extract the " - "requested data." - ) from e +def fetch_meteomatics_token(username, password): + """Requests a short-lived access token from the Meteomatics login service.""" + return MeteomaticsFetcher.fetch_token(username, password) def fetch_atmospheric_data_from_meteomatics( @@ -696,117 +841,17 @@ def fetch_atmospheric_data_from_meteomatics( temperature_pressure_resolution=10, query_limit=10, ): - """Fetches a vertical atmospheric profile from the Meteomatics API. - - The data is retrieved for a single location and instant, sampling - temperature, pressure and both wind components at several altitudes above - ground level. To respect the account's per-request parameter limit, the - parameters are split into groups that are queried separately. - - Parameters - ---------- - username : str - The Meteomatics account username. - password : str - The Meteomatics account password. - latitude : float - Latitude of the launch site, in degrees. - longitude : float - Longitude of the launch site, in degrees. - date : datetime.datetime - The instant to query. It is formatted according to the Meteomatics - date-time specification (``%Y-%m-%dT%H:%M:%SZ``). Timezone-aware - datetimes are converted to UTC; naive ones are assumed to be UTC - already. - model : str, optional - The Meteomatics weather model to use. Default is ``"mix"``. Your - account may not have access to every model. See - https://www.meteomatics.com/en/api/request/optional-parameters/data-source/ - min_altitude : float, optional - Lowest altitude above ground level (in meters) to query. Default is 10. - max_altitude : float, optional - Highest altitude above ground level (in meters) to query. Default is - 12000. The API returns an error if the requested altitude is outside - the range supported by the chosen model. - wind_resolution : int, optional - Number of altitude levels used for the wind components. Default is 20. - temperature_pressure_resolution : int, optional - Number of altitude levels used for temperature and pressure. Default is - 10. - query_limit : int, optional - Maximum number of parameters requested at once. Parameters are grouped - accordingly to work around the account's per-request limit. Default is - 10. See https://api.meteomatics.com/user_stats for your own limits. - - Returns - ------- - dict - A dictionary with the keys ``"temperature"``, ``"pressure"``, - ``"wind_u"`` and ``"wind_v"``. Each value is a dictionary mapping the - altitude above ground level (in meters) to the corresponding value, in - SI units (K, Pa, m/s and m/s respectively). - - Raises - ------ - RuntimeError - If authentication fails, the API cannot be reached, returns an error - status, or returns a malformed response. - ValueError - If the altitude range, the resolutions or the query limit are invalid, - or if the response contains an unrecognized parameter. - """ - _validate_meteomatics_sampling( - min_altitude, - max_altitude, - wind_resolution, - temperature_pressure_resolution, - query_limit, + """Fetches a vertical atmospheric profile from the Meteomatics API.""" + return MeteomaticsFetcher.fetch_atmospheric_data( + username=username, + password=password, + latitude=latitude, + longitude=longitude, + date=date, + model=model, + min_altitude=min_altitude, + max_altitude=max_altitude, + wind_resolution=wind_resolution, + temperature_pressure_resolution=temperature_pressure_resolution, + query_limit=query_limit, ) - - token = fetch_meteomatics_token(username, password) - - # The instant is sent with a trailing "Z", so an aware datetime must be - # converted to UTC instead of being formatted as-is. A naive datetime is - # assumed to already be in UTC. - if date.tzinfo is not None: - date = date.astimezone(timezone.utc) - date_string = date.strftime("%Y-%m-%dT%H:%M:%SZ") - parameter_map = _build_meteomatics_parameters( - min_altitude, max_altitude, wind_resolution, temperature_pressure_resolution - ) - parameters = list(parameter_map) - parameter_groups = [ - parameters[i : i + query_limit] for i in range(0, len(parameters), query_limit) - ] - - profiles = { - "temperature": {}, - "pressure": {}, - "wind_u": {}, - "wind_v": {}, - } - - for index, parameter_group in enumerate(parameter_groups): - logger.info( - "Fetching Meteomatics data for group %d/%d.", - index + 1, - len(parameter_groups), - ) - parameters_str = ",".join(parameter_group) - base_url = ( - f"{METEOMATICS_BASE_URL}/{date_string}/{parameters_str}/" - f"{latitude},{longitude}/json" - ) - query_params = {"model": model, "access_token": token} - data = _meteomatics_request_json(base_url, "data API", params=query_params) - - for parameter, value in _extract_meteomatics_json(data): - try: - profile, height = parameter_map[parameter] - except KeyError as e: - raise ValueError( - f"Unrecognized Meteomatics parameter '{parameter}'." - ) from e - profiles[profile][height] = value - - return profiles diff --git a/tests/unit/environment/test_fetchers.py b/tests/unit/environment/test_fetchers.py index a3c53d176..c226076db 100644 --- a/tests/unit/environment/test_fetchers.py +++ b/tests/unit/environment/test_fetchers.py @@ -280,7 +280,7 @@ def test_fetch_meteomatics_data_client_error_not_retried(monkeypatch): def test_extract_meteomatics_json_bad_structure_raises(payload): """Turn an unexpected 200 payload into a clear RuntimeError, not KeyError.""" with pytest.raises(RuntimeError, match="Unexpected Meteomatics response"): - fetchers._extract_meteomatics_json(payload) + fetchers.MeteomaticsFetcher._extract_json(payload) @pytest.mark.parametrize( From 94f388f9969622b97b8591b1e1e241fb6d45c60c Mon Sep 17 00:00:00 2001 From: Gui-FernandesBR Date: Sat, 1 Aug 2026 11:09:23 -0300 Subject: [PATCH 5/5] ENH: Decompose fetchers.py into rocketpy/environment/fetchers package with dedicated submodules --- rocketpy/environment/fetchers/__init__.py | 63 +++ rocketpy/environment/fetchers/base.py | 7 + .../environment/fetchers/elevation_fetcher.py | 43 ++ .../meteomatics_fetcher.py} | 450 +----------------- .../environment/fetchers/opendap_fetchers.py | 312 ++++++++++++ .../environment/fetchers/windy_fetcher.py | 49 ++ .../environment/fetchers/wyoming_fetcher.py | 46 ++ 7 files changed, 523 insertions(+), 447 deletions(-) create mode 100644 rocketpy/environment/fetchers/__init__.py create mode 100644 rocketpy/environment/fetchers/base.py create mode 100644 rocketpy/environment/fetchers/elevation_fetcher.py rename rocketpy/environment/{fetchers.py => fetchers/meteomatics_fetcher.py} (51%) create mode 100644 rocketpy/environment/fetchers/opendap_fetchers.py create mode 100644 rocketpy/environment/fetchers/windy_fetcher.py create mode 100644 rocketpy/environment/fetchers/wyoming_fetcher.py diff --git a/rocketpy/environment/fetchers/__init__.py b/rocketpy/environment/fetchers/__init__.py new file mode 100644 index 000000000..12adc57b3 --- /dev/null +++ b/rocketpy/environment/fetchers/__init__.py @@ -0,0 +1,63 @@ +"""This module contains auxiliary functions and classes for fetching data from +various third-party APIs. +""" + +import time + +import netCDF4 +import requests + +from rocketpy.environment.fetchers.base import ( + MAX_RETRY_DELAY_SECONDS, + logger, +) +from rocketpy.environment.fetchers.elevation_fetcher import fetch_open_elevation +from rocketpy.environment.fetchers.meteomatics_fetcher import ( + METEOMATICS_BASE_URL, + METEOMATICS_LOGIN_URL, + METEOMATICS_TIMEOUT_SECONDS, + MeteomaticsFetcher, + fetch_atmospheric_data_from_meteomatics, + fetch_meteomatics_token, +) +from rocketpy.environment.fetchers.opendap_fetchers import ( + fetch_aigfs_file_return_dataset, + fetch_cmc_ensemble, + fetch_gefs_ensemble, + fetch_gfs_file_return_dataset, + fetch_hiresw_file_return_dataset, + fetch_hrrr_file_return_dataset, + fetch_nam_file_return_dataset, + fetch_rap_file_return_dataset, +) +from rocketpy.environment.fetchers.windy_fetcher import ( + fetch_atmospheric_data_from_windy, +) +from rocketpy.environment.fetchers.wyoming_fetcher import ( + fetch_wyoming_sounding, +) + +__all__ = [ + "MAX_RETRY_DELAY_SECONDS", + "METEOMATICS_BASE_URL", + "METEOMATICS_LOGIN_URL", + "METEOMATICS_TIMEOUT_SECONDS", + "MeteomaticsFetcher", + "fetch_aigfs_file_return_dataset", + "fetch_atmospheric_data_from_meteomatics", + "fetch_atmospheric_data_from_windy", + "fetch_cmc_ensemble", + "fetch_gefs_ensemble", + "fetch_gfs_file_return_dataset", + "fetch_hiresw_file_return_dataset", + "fetch_hrrr_file_return_dataset", + "fetch_meteomatics_token", + "fetch_nam_file_return_dataset", + "fetch_open_elevation", + "fetch_rap_file_return_dataset", + "fetch_wyoming_sounding", + "logger", + "netCDF4", + "requests", + "time", +] diff --git a/rocketpy/environment/fetchers/base.py b/rocketpy/environment/fetchers/base.py new file mode 100644 index 000000000..b2001155d --- /dev/null +++ b/rocketpy/environment/fetchers/base.py @@ -0,0 +1,7 @@ +"""Base constants and logger for atmospheric data fetchers.""" + +import logging + +logger = logging.getLogger(__name__) + +MAX_RETRY_DELAY_SECONDS = 600 diff --git a/rocketpy/environment/fetchers/elevation_fetcher.py b/rocketpy/environment/fetchers/elevation_fetcher.py new file mode 100644 index 000000000..c7a379b08 --- /dev/null +++ b/rocketpy/environment/fetchers/elevation_fetcher.py @@ -0,0 +1,43 @@ +"""Fetch elevation data from third-party APIs.""" + +import requests + +from rocketpy.environment.fetchers.base import logger +from rocketpy.tools import exponential_backoff + + +@exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) +def fetch_open_elevation(lat, lon): + """Fetches elevation data from the Open-Elevation API at a given latitude + and longitude. + + Parameters + ---------- + lat : float + The latitude of the location. + lon : float + The longitude of the location. + + Returns + ------- + float + The elevation at the given latitude and longitude in meters. + + Raises + ------ + RuntimeError + If there is a problem reaching the Open-Elevation API servers. + """ + logger.debug( + "Fetching elevation from open-elevation.com for lat=%s, lon=%s", lat, lon + ) + request_url = f"https://api.open-elevation.com/api/v1/lookup?locations={lat},{lon}" + try: + response = requests.get(request_url) + results = response.json()["results"] + return results[0]["elevation"] + except ( + requests.exceptions.RequestException, + requests.exceptions.JSONDecodeError, + ) as e: + raise RuntimeError("Unable to reach Open-Elevation API servers.") from e diff --git a/rocketpy/environment/fetchers.py b/rocketpy/environment/fetchers/meteomatics_fetcher.py similarity index 51% rename from rocketpy/environment/fetchers.py rename to rocketpy/environment/fetchers/meteomatics_fetcher.py index 8ad256063..2d3e1e5b8 100644 --- a/rocketpy/environment/fetchers.py +++ b/rocketpy/environment/fetchers/meteomatics_fetcher.py @@ -1,457 +1,19 @@ -"""This module contains auxiliary functions for fetching data from various -third-party APIs. As this is a recent module (introduced in v1.2.0), some -functions may be changed without notice in future feature releases. -""" +"""Fetch weather data from the Meteomatics API.""" import base64 -import logging -import re -import time -from datetime import datetime, timedelta, timezone +from datetime import timezone -import netCDF4 import numpy as np import requests +from rocketpy.environment.fetchers.base import logger from rocketpy.tools import exponential_backoff -logger = logging.getLogger(__name__) - -MAX_RETRY_DELAY_SECONDS = 600 - METEOMATICS_BASE_URL = "https://api.meteomatics.com" METEOMATICS_LOGIN_URL = "https://login.meteomatics.com/api/v1/token" METEOMATICS_TIMEOUT_SECONDS = 30 -@exponential_backoff(max_attempts=3, base_delay=1, max_delay=60) -def fetch_open_elevation(lat, lon): - """Fetches elevation data from the Open-Elevation API at a given latitude - and longitude. - - Parameters - ---------- - lat : float - The latitude of the location. - lon : float - The longitude of the location. - - Returns - ------- - float - The elevation at the given latitude and longitude in meters. - - Raises - ------ - RuntimeError - If there is a problem reaching the Open-Elevation API servers. - """ - logger.debug( - "Fetching elevation from open-elevation.com for lat=%s, lon=%s", lat, lon - ) - request_url = f"https://api.open-elevation.com/api/v1/lookup?locations={lat},{lon}" - try: - response = requests.get(request_url) - results = response.json()["results"] - return results[0]["elevation"] - except ( - requests.exceptions.RequestException, - requests.exceptions.JSONDecodeError, - ) as e: - raise RuntimeError("Unable to reach Open-Elevation API servers.") from e - - -@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) -def fetch_atmospheric_data_from_windy(lat, lon, model): - """Fetches atmospheric data from Windy.com API for a given latitude and - longitude, using a specific model. - - Parameters - ---------- - lat : float - The latitude of the location. - lon : float - The longitude of the location. - model : str - The atmospheric model to use. Options are: ecmwf, GFS, ICON or ICONEU. - - Returns - ------- - dict - A dictionary containing the atmospheric data retrieved from the API. - """ - model = model.lower() - if model[-1] == "u": # case iconEu - model = "".join([model[:4], model[4].upper(), model[5:]]) - - url = ( - f"https://node.windy.com/forecast/meteogram/{model}/{lat}/{lon}/?step=undefined" - ) - - try: - response = requests.get(url).json() - if "data" not in response.keys(): # pragma: no cover - raise ValueError( - f"Could not get a valid response for '{model}' from Windy. " - "Check if the coordinates are set inside the model's domain." - ) - except requests.exceptions.RequestException as e: # pragma: no cover - if model == "iconEu": - raise ValueError( - "Could not get a valid response for Icon-EU from Windy. " - "Check if the coordinates are set inside Europe." - ) from e - - return response - - -def fetch_gfs_file_return_dataset(max_attempts=10, base_delay=2): - """Fetches the latest GFS (Global Forecast System) dataset from the UCAR - THREDDS data server using the OPeNDAP protocol. - - Parameters - ---------- - max_attempts : int, optional - The maximum number of attempts to fetch the dataset. Default is 10. - base_delay : int, optional - The base delay in seconds between attempts. Default is 2. - - Returns - ------- - netCDF4.Dataset - The GFS dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for GFS. - """ - file_url = ( - "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/GFS/Global_0p25deg/Best" - ) - attempt_count = 0 - while attempt_count < max_attempts: - try: - return netCDF4.Dataset(file_url) - except OSError: - attempt_count += 1 - time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) - - raise RuntimeError("Unable to load latest weather data for GFS through " + file_url) - - -def fetch_nam_file_return_dataset(max_attempts=10, base_delay=2): - """Fetches the latest NAM (North American Mesoscale) dataset from the UCAR - THREDDS data server using the OPeNDAP protocol. - - Parameters - ---------- - max_attempts : int, optional - The maximum number of attempts to fetch the dataset. Default is 10. - base_delay : int, optional - The base delay in seconds between attempts. Default is 2. - - Returns - ------- - netCDF4.Dataset - The NAM dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for NAM. - """ - file_url = "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/NAM/CONUS_12km/Best" - attempt_count = 0 - while attempt_count < max_attempts: - try: - return netCDF4.Dataset(file_url) - except OSError: - attempt_count += 1 - time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) - - raise RuntimeError("Unable to load latest weather data for NAM through " + file_url) - - -def fetch_rap_file_return_dataset(max_attempts=10, base_delay=2): - """Fetches the latest RAP (Rapid Refresh) dataset from the UCAR THREDDS - data server using the OPeNDAP protocol. - - Parameters - ---------- - max_attempts : int, optional - The maximum number of attempts to fetch the dataset. Default is 10. - base_delay : int, optional - The base delay in seconds between attempts. Default is 2. - - Returns - ------- - netCDF4.Dataset - The RAP dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for RAP. - """ - file_url = "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/RAP/CONUS_13km/Best" - attempt_count = 0 - while attempt_count < max_attempts: - try: - return netCDF4.Dataset(file_url) - except OSError: - attempt_count += 1 - time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) - - raise RuntimeError("Unable to load latest weather data for RAP through " + file_url) - - -def fetch_hrrr_file_return_dataset(max_attempts=10, base_delay=2): - """Fetches the latest HRRR (High-Resolution Rapid Refresh) dataset from - the NOAA's GrADS data server using the OpenDAP protocol. - - Parameters - ---------- - max_attempts : int, optional - The maximum number of attempts to fetch the dataset. Default is 10. - base_delay : int, optional - The base delay in seconds between attempts. Default is 2. - - Returns - ------- - netCDF4.Dataset - The HRRR dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for HRRR. - """ - file_url = "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/HRRR/CONUS_2p5km/Best" - attempt_count = 0 - while attempt_count < max_attempts: - try: - return netCDF4.Dataset(file_url) - except OSError: - attempt_count += 1 - time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) - - raise RuntimeError( - "Unable to load latest weather data for HRRR through " + file_url - ) - - -def fetch_aigfs_file_return_dataset(max_attempts=10, base_delay=2): - """Fetches the latest AIGFS (Artificial Intelligence GFS) dataset from - the NOAA's GrADS data server using the OpenDAP protocol. - - Parameters - ---------- - max_attempts : int, optional - The maximum number of attempts to fetch the dataset. Default is 10. - base_delay : int, optional - The base delay in seconds between attempts. Default is 2. - - Returns - ------- - netCDF4.Dataset - The AIGFS dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for AIGFS. - """ - file_url = ( - "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/AIGFS/Global_0p25deg/Best" - ) - attempt_count = 0 - while attempt_count < max_attempts: - try: - return netCDF4.Dataset(file_url) - except OSError: - attempt_count += 1 - time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) - - raise RuntimeError( - "Unable to load latest weather data for AIGFS through " + file_url - ) - - -def fetch_hiresw_file_return_dataset(max_attempts=10, base_delay=2): - """Fetches the latest HiResW (High-Resolution Window) dataset from the NOAA's - GrADS data server using the OpenDAP protocol. - - Parameters - ---------- - max_attempts : int, optional - The maximum number of attempts to fetch the dataset. Default is 10. - base_delay : int, optional - The base delay in seconds between attempts. Default is 2. - - Returns - ------- - netCDF4.Dataset - The HiResW dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for HiResW. - """ - # Attempt to get latest forecast - time_attempt = datetime.now(tz=timezone.utc) - attempt_count = 0 - dataset = None - - today = datetime.now(tz=timezone.utc) - date_info = (today.year, today.month, today.day, 12) # Hour given in UTC time - - while attempt_count < max_attempts: - time_attempt -= timedelta(hours=12) - date_info = ( - time_attempt.year, - time_attempt.month, - time_attempt.day, - 12, - ) # Hour given in UTC time - date_string = f"{date_info[0]:04d}{date_info[1]:02d}{date_info[2]:02d}" - file = ( - f"https://nomads.ncep.noaa.gov/dods/hiresw/hiresw{date_string}/" - "hiresw_conusarw_12z" - ) - try: - # Attempts to create a dataset from the file using OpenDAP protocol. - dataset = netCDF4.Dataset(file) - return dataset - except OSError: - attempt_count += 1 - time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) - - if dataset is None: - raise RuntimeError( - "Unable to load latest weather data for HiResW through " + file - ) - - -@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) -def fetch_wyoming_sounding(file): - """Fetches sounding data from a specified file using the Wyoming Weather - Web. - - Parameters - ---------- - file : str - The URL of the file to fetch. - - Returns - ------- - str - The content of the fetched file. - - Raises - ------ - ImportError - If unable to load the specified file. - ValueError - If the response indicates the specified station or date is invalid. - ValueError - If the response indicates the output format is invalid. - """ - response = requests.get(file) - if response.status_code != 200: # pragma: no cover - raise ImportError(f"Unable to load {file}.") - if len(re.findall("Can't get .+ Observations at", response.text)): - raise ValueError( - re.findall("Can't get .+ Observations at .+", response.text)[0] - + " Check station number and date." - ) - if response.text == "Invalid OUTPUT: specified\n": - raise ValueError( - "Invalid OUTPUT: specified. Make sure the output is Text: List." - ) - return response - - -@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) -def fetch_gefs_ensemble(): - """Fetches the latest GEFS (Global Ensemble Forecast System) dataset from - the NOAA's GrADS data server using the OpenDAP protocol. - - Returns - ------- - netCDF4.Dataset - The GEFS dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for GEFS. - """ - time_attempt = datetime.now(tz=timezone.utc) - success = False - attempt_count = 0 - while not success and attempt_count < 10: - time_attempt -= timedelta(hours=6 * attempt_count) # GEFS updates every 6 hours - file = ( - f"https://nomads.ncep.noaa.gov/dods/gens_bc/gens" - f"{time_attempt.year:04d}{time_attempt.month:02d}" - f"{time_attempt.day:02d}/" - f"gep_all_{6 * (time_attempt.hour // 6):02d}z" - ) - try: - dataset = netCDF4.Dataset(file) - success = True - return dataset - except OSError: - attempt_count += 1 - time.sleep(min(2**attempt_count, MAX_RETRY_DELAY_SECONDS)) - if not success: - raise RuntimeError( - "Unable to load latest weather data for GEFS through " + file - ) - - -@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) -def fetch_cmc_ensemble(): - """Fetches the latest CMC (Canadian Meteorological Centre) ensemble dataset - from the NOAA's GrADS data server using the OpenDAP protocol. - - Returns - ------- - netCDF4.Dataset - The CMC ensemble dataset. - - Raises - ------ - RuntimeError - If unable to load the latest weather data for CMC. - """ - # Attempt to get latest forecast - time_attempt = datetime.now(tz=timezone.utc) - success = False - attempt_count = 0 - while not success and attempt_count < 10: - time_attempt -= timedelta( - hours=12 * attempt_count - ) # CMC updates every 12 hours - file = ( - f"https://nomads.ncep.noaa.gov/dods/cmcens/" - f"cmcens{time_attempt.year:04d}{time_attempt.month:02d}" - f"{time_attempt.day:02d}/" - f"cmcensspr_{12 * (time_attempt.hour // 12):02d}z" - ) - try: - dataset = netCDF4.Dataset(file) - success = True - return dataset - except OSError: - attempt_count += 1 - time.sleep(min(2**attempt_count, MAX_RETRY_DELAY_SECONDS)) - if not success: - raise RuntimeError("Unable to load latest weather data for CMC through " + file) - - class MeteomaticsFetcher: """Fetcher class to authenticate and query vertical atmospheric profiles from the Meteomatics API. @@ -598,9 +160,6 @@ def _build_parameters( """ def levels(resolution): - # Round to integer meters and drop duplicates that rounding may - # introduce for narrow bands, so we never request (and pay for) the - # same height twice. return np.unique( np.linspace(min_altitude, max_altitude, resolution).round().astype(int) ) @@ -772,9 +331,6 @@ def fetch_atmospheric_data( token = cls.fetch_token(username, password) - # The instant is sent with a trailing "Z", so an aware datetime must be - # converted to UTC instead of being formatted as-is. A naive datetime is - # assumed to already be in UTC. if date.tzinfo is not None: date = date.astimezone(timezone.utc) date_string = date.strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/rocketpy/environment/fetchers/opendap_fetchers.py b/rocketpy/environment/fetchers/opendap_fetchers.py new file mode 100644 index 000000000..270c0d910 --- /dev/null +++ b/rocketpy/environment/fetchers/opendap_fetchers.py @@ -0,0 +1,312 @@ +"""Fetch weather datasets using OPeNDAP protocol (NOAA, UCAR, CMC, GEFS).""" + +import time +from datetime import datetime, timedelta, timezone + +import netCDF4 + +from rocketpy.environment.fetchers.base import MAX_RETRY_DELAY_SECONDS +from rocketpy.tools import exponential_backoff + + +def fetch_gfs_file_return_dataset(max_attempts=10, base_delay=2): + """Fetches the latest GFS (Global Forecast System) dataset from the UCAR + THREDDS data server using the OPeNDAP protocol. + + Parameters + ---------- + max_attempts : int, optional + The maximum number of attempts to fetch the dataset. Default is 10. + base_delay : int, optional + The base delay in seconds between attempts. Default is 2. + + Returns + ------- + netCDF4.Dataset + The GFS dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for GFS. + """ + file_url = ( + "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/GFS/Global_0p25deg/Best" + ) + attempt_count = 0 + while attempt_count < max_attempts: + try: + return netCDF4.Dataset(file_url) + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + raise RuntimeError("Unable to load latest weather data for GFS through " + file_url) + + +def fetch_nam_file_return_dataset(max_attempts=10, base_delay=2): + """Fetches the latest NAM (North American Mesoscale) dataset from the UCAR + THREDDS data server using the OPeNDAP protocol. + + Parameters + ---------- + max_attempts : int, optional + The maximum number of attempts to fetch the dataset. Default is 10. + base_delay : int, optional + The base delay in seconds between attempts. Default is 2. + + Returns + ------- + netCDF4.Dataset + The NAM dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for NAM. + """ + file_url = "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/NAM/CONUS_12km/Best" + attempt_count = 0 + while attempt_count < max_attempts: + try: + return netCDF4.Dataset(file_url) + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + raise RuntimeError("Unable to load latest weather data for NAM through " + file_url) + + +def fetch_rap_file_return_dataset(max_attempts=10, base_delay=2): + """Fetches the latest RAP (Rapid Refresh) dataset from the UCAR THREDDS + data server using the OPeNDAP protocol. + + Parameters + ---------- + max_attempts : int, optional + The maximum number of attempts to fetch the dataset. Default is 10. + base_delay : int, optional + The base delay in seconds between attempts. Default is 2. + + Returns + ------- + netCDF4.Dataset + The RAP dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for RAP. + """ + file_url = "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/RAP/CONUS_13km/Best" + attempt_count = 0 + while attempt_count < max_attempts: + try: + return netCDF4.Dataset(file_url) + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + raise RuntimeError("Unable to load latest weather data for RAP through " + file_url) + + +def fetch_hrrr_file_return_dataset(max_attempts=10, base_delay=2): + """Fetches the latest HRRR (High-Resolution Rapid Refresh) dataset from + the NOAA's GrADS data server using the OpenDAP protocol. + + Parameters + ---------- + max_attempts : int, optional + The maximum number of attempts to fetch the dataset. Default is 10. + base_delay : int, optional + The base delay in seconds between attempts. Default is 2. + + Returns + ------- + netCDF4.Dataset + The HRRR dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for HRRR. + """ + file_url = "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/HRRR/CONUS_2p5km/Best" + attempt_count = 0 + while attempt_count < max_attempts: + try: + return netCDF4.Dataset(file_url) + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + raise RuntimeError( + "Unable to load latest weather data for HRRR through " + file_url + ) + + +def fetch_aigfs_file_return_dataset(max_attempts=10, base_delay=2): + """Fetches the latest AIGFS (Artificial Intelligence GFS) dataset from + the NOAA's GrADS data server using the OpenDAP protocol. + + Parameters + ---------- + max_attempts : int, optional + The maximum number of attempts to fetch the dataset. Default is 10. + base_delay : int, optional + The base delay in seconds between attempts. Default is 2. + + Returns + ------- + netCDF4.Dataset + The AIGFS dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for AIGFS. + """ + file_url = ( + "https://thredds.ucar.edu/thredds/dodsC/grib/NCEP/AIGFS/Global_0p25deg/Best" + ) + attempt_count = 0 + while attempt_count < max_attempts: + try: + return netCDF4.Dataset(file_url) + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + raise RuntimeError( + "Unable to load latest weather data for AIGFS through " + file_url + ) + + +def fetch_hiresw_file_return_dataset(max_attempts=10, base_delay=2): + """Fetches the latest HiResW (High-Resolution Window) dataset from the NOAA's + GrADS data server using the OpenDAP protocol. + + Parameters + ---------- + max_attempts : int, optional + The maximum number of attempts to fetch the dataset. Default is 10. + base_delay : int, optional + The base delay in seconds between attempts. Default is 2. + + Returns + ------- + netCDF4.Dataset + The HiResW dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for HiResW. + """ + time_attempt = datetime.now(tz=timezone.utc) + attempt_count = 0 + dataset = None + + today = datetime.now(tz=timezone.utc) + date_info = (today.year, today.month, today.day, 12) + + while attempt_count < max_attempts: + time_attempt -= timedelta(hours=12) + date_info = ( + time_attempt.year, + time_attempt.month, + time_attempt.day, + 12, + ) + date_string = f"{date_info[0]:04d}{date_info[1]:02d}{date_info[2]:02d}" + file = ( + f"https://nomads.ncep.noaa.gov/dods/hiresw/hiresw{date_string}/" + "hiresw_conusarw_12z" + ) + try: + dataset = netCDF4.Dataset(file) + return dataset + except OSError: + attempt_count += 1 + time.sleep(min(base_delay**attempt_count, MAX_RETRY_DELAY_SECONDS)) + + if dataset is None: + raise RuntimeError( + "Unable to load latest weather data for HiResW through " + file + ) + + +@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) +def fetch_gefs_ensemble(): + """Fetches the latest GEFS (Global Ensemble Forecast System) dataset from + the NOAA's GrADS data server using the OpenDAP protocol. + + Returns + ------- + netCDF4.Dataset + The GEFS dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for GEFS. + """ + time_attempt = datetime.now(tz=timezone.utc) + success = False + attempt_count = 0 + while not success and attempt_count < 10: + time_attempt -= timedelta(hours=6 * attempt_count) + file = ( + f"https://nomads.ncep.noaa.gov/dods/gens_bc/gens" + f"{time_attempt.year:04d}{time_attempt.month:02d}" + f"{time_attempt.day:02d}/" + f"gep_all_{6 * (time_attempt.hour // 6):02d}z" + ) + try: + dataset = netCDF4.Dataset(file) + success = True + return dataset + except OSError: + attempt_count += 1 + time.sleep(min(2**attempt_count, MAX_RETRY_DELAY_SECONDS)) + if not success: + raise RuntimeError( + "Unable to load latest weather data for GEFS through " + file + ) + + +@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) +def fetch_cmc_ensemble(): + """Fetches the latest CMC (Canadian Meteorological Centre) ensemble dataset + from the NOAA's GrADS data server using the OpenDAP protocol. + + Returns + ------- + netCDF4.Dataset + The CMC ensemble dataset. + + Raises + ------ + RuntimeError + If unable to load the latest weather data for CMC. + """ + time_attempt = datetime.now(tz=timezone.utc) + success = False + attempt_count = 0 + while not success and attempt_count < 10: + time_attempt -= timedelta(hours=12 * attempt_count) + file = ( + f"https://nomads.ncep.noaa.gov/dods/cmcens/" + f"cmcens{time_attempt.year:04d}{time_attempt.month:02d}" + f"{time_attempt.day:02d}/" + f"cmcensspr_{12 * (time_attempt.hour // 12):02d}z" + ) + try: + dataset = netCDF4.Dataset(file) + success = True + return dataset + except OSError: + attempt_count += 1 + time.sleep(min(2**attempt_count, MAX_RETRY_DELAY_SECONDS)) + if not success: + raise RuntimeError("Unable to load latest weather data for CMC through " + file) diff --git a/rocketpy/environment/fetchers/windy_fetcher.py b/rocketpy/environment/fetchers/windy_fetcher.py new file mode 100644 index 000000000..ddb35bfb6 --- /dev/null +++ b/rocketpy/environment/fetchers/windy_fetcher.py @@ -0,0 +1,49 @@ +"""Fetch weather data from Windy.com API.""" + +import requests + +from rocketpy.tools import exponential_backoff + + +@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) +def fetch_atmospheric_data_from_windy(lat, lon, model): + """Fetches atmospheric data from Windy.com API for a given latitude and + longitude, using a specific model. + + Parameters + ---------- + lat : float + The latitude of the location. + lon : float + The longitude of the location. + model : str + The atmospheric model to use. Options are: ecmwf, GFS, ICON or ICONEU. + + Returns + ------- + dict + A dictionary containing the atmospheric data retrieved from the API. + """ + model = model.lower() + if model[-1] == "u": # case iconEu + model = "".join([model[:4], model[4].upper(), model[5:]]) + + url = ( + f"https://node.windy.com/forecast/meteogram/{model}/{lat}/{lon}/?step=undefined" + ) + + try: + response = requests.get(url).json() + if "data" not in response.keys(): # pragma: no cover + raise ValueError( + f"Could not get a valid response for '{model}' from Windy. " + "Check if the coordinates are set inside the model's domain." + ) + except requests.exceptions.RequestException as e: # pragma: no cover + if model == "iconEu": + raise ValueError( + "Could not get a valid response for Icon-EU from Windy. " + "Check if the coordinates are set inside Europe." + ) from e + + return response diff --git a/rocketpy/environment/fetchers/wyoming_fetcher.py b/rocketpy/environment/fetchers/wyoming_fetcher.py new file mode 100644 index 000000000..e7a4fb784 --- /dev/null +++ b/rocketpy/environment/fetchers/wyoming_fetcher.py @@ -0,0 +1,46 @@ +"""Fetch upper air sounding data from Wyoming Weather Web.""" + +import re + +import requests + +from rocketpy.tools import exponential_backoff + + +@exponential_backoff(max_attempts=5, base_delay=2, max_delay=60) +def fetch_wyoming_sounding(file): + """Fetches sounding data from a specified file using the Wyoming Weather + Web. + + Parameters + ---------- + file : str + The URL of the file to fetch. + + Returns + ------- + str + The content of the fetched file. + + Raises + ------ + ImportError + If unable to load the specified file. + ValueError + If the response indicates the specified station or date is invalid. + ValueError + If the response indicates the output format is invalid. + """ + response = requests.get(file) + if response.status_code != 200: # pragma: no cover + raise ImportError(f"Unable to load {file}.") + if len(re.findall("Can't get .+ Observations at", response.text)): + raise ValueError( + re.findall("Can't get .+ Observations at .+", response.text)[0] + + " Check station number and date." + ) + if response.text == "Invalid OUTPUT: specified\n": + raise ValueError( + "Invalid OUTPUT: specified. Make sure the output is Text: List." + ) + return response