From b609da62a35a5c4286e51dbbaec0d263026595a4 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 17 Sep 2025 12:43:00 +0200 Subject: [PATCH 01/28] Add Level 1 format option --- mwrpy/cli.py | 21 +++++++ mwrpy/level1/lev1_meta_nc.py | 31 +++++++++- mwrpy/level1/met_quality_control.py | 8 ++- mwrpy/level1/quality_control.py | 16 ++++-- mwrpy/level1/write_lev1_nc.py | 55 ++++++++++++++---- mwrpy/level2/get_ret_coeff.py | 4 +- mwrpy/plots/generate_plots.py | 8 ++- mwrpy/plots/plot_utils.py | 2 +- mwrpy/process_mwrpy.py | 63 +++++++++++++++++---- mwrpy/rpg_mwr.py | 87 +++++++++++++++++++++++++++-- mwrpy/site_config/hatpro.yaml | 62 ++++++++++---------- mwrpy/site_config/lhatpro.yaml | 62 ++++++++++---------- mwrpy/site_config/lhumpro_u90.yaml | 62 ++++++++++---------- mwrpy/utils.py | 66 ++++++++++++++++------ 14 files changed, 397 insertions(+), 150 deletions(-) diff --git a/mwrpy/cli.py b/mwrpy/cli.py index bdaa070..97dee7a 100755 --- a/mwrpy/cli.py +++ b/mwrpy/cli.py @@ -65,6 +65,27 @@ def _parse_args(args): metavar="YYYY-MM-DD", help="Single date to be processed.", ) + group.add_argument( + "-i", + "--instrument", + type=str, + help="Instrument to be processed (hatpro, lhatpro, lhumpro_u90).", + default="hatpro", + ) + group.add_argument( + "-f", + "--format", + type=str, + help="Data format to be used (cloudnet, e-profile).", + default="e-profile", + ) + group.add_argument( + "-a", + "--altitude", + type=float, + help="Altitude above mean sea level of site (m).", + default=0.0, + ) return parser.parse_args(args) diff --git a/mwrpy/level1/lev1_meta_nc.py b/mwrpy/level1/lev1_meta_nc.py index 0d2d920..e83371b 100644 --- a/mwrpy/level1/lev1_meta_nc.py +++ b/mwrpy/level1/lev1_meta_nc.py @@ -6,12 +6,13 @@ from mwrpy.utils import MetaData -def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: +def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) -> dict: """Adds Metadata for RPG MWR Level 1 variables for NetCDF file writing. Args: rpg_variables: RpgArray instances. data_type: Data type of the netCDF file. + data_format: Data format of the netCDF file (cloudnet, e-profile). Returns: Dictionary @@ -41,6 +42,9 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: attributes = dict( ATTRIBUTES_COM, **ATTRIBUTES_1B01, **ATTRIBUTES_1B11, **ATTRIBUTES_1B21 ) + if data_format == "cloudnet": + attributes.pop("time") + attributes = dict(ATTRIBUTES_CN, **attributes) for key in list(rpg_variables): if key in attributes: @@ -56,6 +60,18 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: return rpg_variables +ATTRIBUTES_CN = { + "time": MetaData( + units="hours since ", + long_name="Time UTC", + standard_name="time", + axis="T", + calendar="standard", + dimensions=("time",), + ), +} + + ATTRIBUTES_COM = { "time": MetaData( long_name="Time (UTC) of the measurement", @@ -185,6 +201,13 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: comment="0=horizon, 90=zenith", dimensions=("time",), ), + "zenith_angle": MetaData( + units="degree", + long_name="Zenith angle", + standard_name="zenith_angle", + comment="Angle to the local vertical. A value of zero is directly overhead.", + dimensions=("time",), + ), # "tb_accuracy": MetaData( # long_name="Total absolute calibration uncertainty of brightness temperature,\n" # "one standard deviation", @@ -303,6 +326,12 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: comment="0=horizon, 90=zenith", dimensions=("time",), ), + "ir_zenith_angle": MetaData( + units="degree", + long_name="Infrared sensor elevation angle", + comment="90=horizon, 0=zenith", + dimensions=("time",), + ), } diff --git a/mwrpy/level1/met_quality_control.py b/mwrpy/level1/met_quality_control.py index 40bc774..b0a2353 100644 --- a/mwrpy/level1/met_quality_control.py +++ b/mwrpy/level1/met_quality_control.py @@ -3,16 +3,18 @@ import metpy.calc as mpcalc import numpy as np from metpy.units import masked_array +from numpy import ma from mwrpy.utils import setbit -def apply_met_qc(data: dict, params: dict) -> None: +def apply_met_qc(data: dict, params: dict, altitude: float | None) -> None: """This function performs quality control of meteorological sensor data. Args: data: Level 1 data. params: Site specific parameters. + altitude: Altitude of the site in meters. Returns: None @@ -39,7 +41,9 @@ def apply_met_qc(data: dict, params: dict) -> None: if name not in data: continue if name == "air_pressure": - altitude = masked_array(params["altitude"], data_units="m") + alt = params.get("altitude", altitude) + alt = ma.masked if alt is None else alt + altitude = masked_array(alt, data_units="m") pressure = mpcalc.height_to_pressure_std(altitude).to("pascal").magnitude threshold_low = pressure - 10000 threshold_high = pressure + 10000 diff --git a/mwrpy/level1/quality_control.py b/mwrpy/level1/quality_control.py index e2e5cc9..2457b8c 100644 --- a/mwrpy/level1/quality_control.py +++ b/mwrpy/level1/quality_control.py @@ -74,7 +74,7 @@ def apply_qc( data["quality_flag_status"] = setbit(data["quality_flag_status"], 3) else: try: - ind = spectral_consistency(data, site, coeff_files) + ind = spectral_consistency(data, params, site, coeff_files) data["quality_flag"][ind] = setbit(data["quality_flag"][ind], 3) except MissingCoefficientsError as e: logging.error( @@ -184,7 +184,10 @@ def orbpos(data: dict, params: dict) -> np.ndarray: def spectral_consistency( - data: dict, site: str | None, coeff_files: Sequence[str | PathLike] | None + data: dict, + params: dict, + site: str | None, + coeff_files: Sequence[str | PathLike] | None, ) -> np.ndarray: """Applies spectral consistency coefficients for given frequency index, writes 2S02 product and returns indices to be flagged. @@ -205,11 +208,12 @@ def spectral_consistency( + 1.0 ) + coeff_dir = params.get("coeff_path", None) prefix = "ins" - c_list = get_coeff_list(site, prefix, coeff_files) + c_list = get_coeff_list(site, prefix, coeff_files, coeff_dir) if len(c_list) == 0: prefix = "spc" - c_list = get_coeff_list(site, prefix, coeff_files) + c_list = get_coeff_list(site, prefix, coeff_files, coeff_dir) if len(c_list) > 0: # pylint: disable=unbalanced-tuple-unpacking @@ -222,7 +226,7 @@ def spectral_consistency( weights1, weights2, factor, - ) = get_mvr_coeff(site, prefix, data["frequency"][:], coeff_files) + ) = get_mvr_coeff(site, prefix, data["frequency"][:], coeff_files, coeff_dir) ret_in = retrieval_input(data, coeff) ele_ind = np.where( (np.abs(data["elevation_angle"][:] - 90.0) < 0.5) @@ -342,7 +346,7 @@ def spectral_consistency( ] = True else: - c_list = get_coeff_list(site, "tbx", coeff_files) + c_list = get_coeff_list(site, "tbx", coeff_files, coeff_dir) if not c_list: raise MissingCoefficientsError("No coefficients found") diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 1a46d51..a8911be 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -19,6 +19,7 @@ from mwrpy.utils import ( add_interpol1d, add_time_bounds, + get_coeff_list, get_file_list, isbit, read_config, @@ -31,6 +32,7 @@ def lev1_to_nc( data_type: str, path_to_files: str | PathLike, + data_format: str, site: str | None = None, output_file: str | PathLike | None = None, lidar_path: str | PathLike | None = None, @@ -39,6 +41,7 @@ def lev1_to_nc( date: datetime.date | None = None, time_offset: datetime.timedelta | None = None, instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, + altitude: float | None = None, ) -> rpg_mwr.Rpg: """This function reads one day of RPG MWR binary files, adds attributes and writes it into netCDF file. @@ -46,6 +49,7 @@ def lev1_to_nc( Args: data_type: Data type of the netCDF file. path_to_files: Folder containing one day of RPG MWR binary files. + data_format: Data format of the netCDF file (cloudnet, e-profile). site: Name of site. output_file: Output file name. lidar_path: Path to (optional) lidar file @@ -54,6 +58,7 @@ def lev1_to_nc( date: Measurement date in UTC. time_offset: Time offset if instrument operated in local time. instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). + altitude: Altitude of the site in meters above mean sea level. Raises: MissingInputData: if required input file is missing. @@ -68,29 +73,52 @@ def lev1_to_nc( f"No coefficient files given, using files in repository for {site}." ) - if instrument_config is None: + if data_format == "e-profile" and instrument_config is None: logging.info( f"No instrument config given, using config file in repository for {site}." ) - params = read_config(site, instrument_type, "params") + params = ( + read_config(site, instrument_type, "params") + if data_format == "e-profile" + else read_config(None, instrument_type, "params") + ) if instrument_config is not None: params = {**params, **instrument_config} - rpg_bin = prepare_data(path_to_files, data_type, params, lidar_path, time_offset) + rpg_bin = prepare_data( + path_to_files, data_type, params, lidar_path, time_offset, altitude + ) if data_type in ("1B01", "1C01"): apply_qc(site, rpg_bin, params, coeff_files) if data_type in ("1B21", "1C01"): - apply_met_qc(rpg_bin.data, params) + apply_met_qc(rpg_bin.data, params, altitude) mwr = rpg_mwr.Rpg(rpg_bin.data, date) mwr.find_valid_times() - mwr.data = get_data_attributes(mwr.data, data_type) + mwr.data = get_data_attributes(mwr.data, data_type, data_format) if output_file is not None: - global_attributes = read_config(site, instrument_type, "global_specs") + if data_format == "cloudnet": + c_files = ( + get_coeff_list( + site, + ["spc", "ins", "lwp", "iwv", "hpt", "tpt", "tpb"], + None, + params.get("coeff_path", None), + ) + if (coeff_files is None) + else (coeff_files) + ) + global_attributes = { + "site": site, + "instrument": instrument_type, + "coeff_files": c_files, + } + else: + global_attributes = read_config(site, instrument_type, "global_specs") if data_type != "1C01": update_lev1_attributes(global_attributes, data_type) - rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type) + rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) return mwr @@ -100,6 +128,7 @@ def prepare_data( params: dict, lidar_path: str | PathLike | None, time_offset: datetime.timedelta | None = None, + altitude: float | None = None, ) -> RpgBin: """Load and prepare data for netCDF writing.""" if data_type in ("1B01", "1C01"): @@ -280,9 +309,9 @@ def prepare_data( file_list_hkd = get_file_list(path_to_files, "HKD") _append_hkd(file_list_hkd, rpg_bin, data_type, params, time_offset) - rpg_bin.data["altitude"] = ( - np.ones(len(rpg_bin.data["time"]), np.float32) * params["altitude"] - ) + alt = params.get("altitude", altitude) + alt = ma.masked if alt is None else alt + rpg_bin.data["altitude"] = np.ones(len(rpg_bin.data["time"]), np.float32) * alt return rpg_bin @@ -296,11 +325,13 @@ def _append_hkd( ) -> None: """Append hkd data on same time grid and perform TB sanity check.""" hkd = RpgBin(file_list_hkd, time_offset) + lat = params.get("latitude", ma.masked) + lon = params.get("longitude", ma.masked) if "latitude" not in hkd.data: add_interpol1d( rpg_bin.data, - np.ones(len(hkd.data["time"])) * params["latitude"], + np.ones(len(hkd.data["time"])) * lat, hkd.data["time"], "latitude", ) @@ -315,7 +346,7 @@ def _append_hkd( if "longitude" not in hkd.data: add_interpol1d( rpg_bin.data, - np.ones(len(hkd.data["time"])) * params["longitude"], + np.ones(len(hkd.data["time"])) * lon, hkd.data["time"], "longitude", ) diff --git a/mwrpy/level2/get_ret_coeff.py b/mwrpy/level2/get_ret_coeff.py index ec3c35a..cedca55 100644 --- a/mwrpy/level2/get_ret_coeff.py +++ b/mwrpy/level2/get_ret_coeff.py @@ -15,6 +15,7 @@ def get_mvr_coeff( prefix: str, freq: np.ndarray, coeff_files: Sequence[str | PathLike] | None, + coeff_dir: str | None = None, ): """This function extracts retrieval coefficients for given files. @@ -23,12 +24,13 @@ def get_mvr_coeff( prefix: Identifier for type of product. freq: Frequencies of observations. coeff_files: List of coefficient files. + coeff_dir: Directory where coefficient files are stored. Examples: >>> from mwrpy.level2.get_ret_coeff import get_mvr_coeff >>> get_mvr_coeff('site_name', 'lwp', np.array([22, 31.4])) """ - c_list = get_coeff_list(site, prefix, coeff_files) + c_list = get_coeff_list(site, prefix, coeff_files, coeff_dir) coeff: dict = {} diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index 10faa3a..aca5f81 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -387,7 +387,7 @@ def _read_time_vector(nc_file: str) -> ndarray: """Converts time vector to fraction hour.""" with netCDF4.Dataset(nc_file) as nc: time = nc.variables["time"][:] - return seconds2hours(time) + return seconds2hours(time) if time.max() > 24 else time def _screen_high_altitudes(data_field: ndarray, ax_values: tuple, max_y: int) -> tuple: @@ -439,7 +439,11 @@ def _read_date(nc_file: str) -> date: """Returns measurement date.""" locale.setlocale(locale.LC_TIME, "en_US.UTF-8") with netCDF4.Dataset(nc_file) as nc: - case_date = datetime.strptime(nc.date, "%Y-%m-%d") + case_date = ( + datetime.strptime(nc.date, "%Y-%m-%d") + if "date" in nc.ncattrs() + else datetime.strptime(f"{nc.year}-{nc.month}-{nc.day}", "%Y-%m-%d") + ) return case_date diff --git a/mwrpy/plots/plot_utils.py b/mwrpy/plots/plot_utils.py index 8aa7d38..de03fe4 100644 --- a/mwrpy/plots/plot_utils.py +++ b/mwrpy/plots/plot_utils.py @@ -149,7 +149,7 @@ def _calculate_rolling_mean(time: ndarray, data: ndarray, win: float = 0.5) -> n def _read_location(nc_file: str) -> str: """Returns site name.""" with netCDF4.Dataset(nc_file) as nc: - site_name = nc.site_location + site_name = nc.site_location if "site_location" in nc.ncattrs() else nc.location return site_name diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index c9b872f..7116cc7 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -5,6 +5,7 @@ import logging import os import time +from typing import Literal import matplotlib.pyplot as plt import netCDF4 as nc @@ -20,7 +21,7 @@ from mwrpy.plots.generate_plots import generate_figure from mwrpy.utils import ( _get_filename, - _read_site_config_yaml, + _get_filename_cloudnet, date_range, get_processing_dates, isodate2date, @@ -92,6 +93,7 @@ "ko_index", ] ) +IType = Literal["hatpro", "lhatpro", "lhumpro_u90"] def main(args): @@ -104,22 +106,44 @@ def main(args): if product not in PRODUCT_NAME: logging.error(f"Product {product} not recognised") continue + if args.format == "cloudnet": + if product not in ("1C01", "single", "multi"): + logging.error( + f"Product {product} not available in cloudnet format. Skipping." + ) + continue + if args.altitude is None: + logging.info("Site altitude not provided. Taking default of 0 m.") start = time.process_time() if args.command != "plot": logging.info(f"Processing {product} product, {args.site} {date}") if args.command == "reprocess": try: - process_product(product, date, args.site) + process_product( + product, + date, + args.site, + args.format, + args.instrument, + args.altitude, + ) except Exception as e: logging.error( f"Error in processing products: {e}. Incomplete or no processing for {date}." ) else: - process_product(product, date, args.site) + process_product( + product, + date, + args.site, + args.format, + args.instrument, + args.altitude, + ) if args.command != "no-plot": logging.info(f"Plotting {product} product, {args.site} {date}") try: - plot_product(product, date, args.site) + plot_product(product, date, args.site, args.format, args.instrument) except Exception as e: logging.error(f"Error in plotting product {product}: {e}.") finally: @@ -128,8 +152,19 @@ def main(args): logging.info(f"Processing took {elapsed_time:.1f} seconds") -def process_product(prod: str, date: datetime.date, site: str): - output_file = _get_filename(prod, date, site) +def process_product( + prod: str, + date: datetime.date, + site: str, + data_format: str, + instrument: IType, + altitude: float, +): + output_file = ( + _get_filename(prod, date, site) + if data_format == "e-profile" + else _get_filename_cloudnet(prod, date, site, instrument) + ) output_dir = os.path.dirname(output_file) if not os.path.isdir(output_dir): os.makedirs(output_dir) @@ -165,15 +200,17 @@ def process_product(prod: str, date: datetime.date, site: str): csv_off["date"] == xday[1].strftime("%m-%d"), "offset" ].values[0] - itype = _read_site_config_yaml(site)["type"] if prod[0] == "1": lev1_to_nc( prod, _get_raw_file_path(date, site), + data_format, site=site, output_file=output_file, lidar_path=_get_lidar_file_path(date, site), date=date, + instrument_type=instrument, + altitude=altitude, ) elif prod[0] == "2": if prod in ("2P04", "2P07", "2P08"): @@ -193,11 +230,11 @@ def process_product(prod: str, date: datetime.date, site: str): hum_file=hum_file, lwp_offset=lwp_offset, ) - elif prod == "single" and itype != "lhumpro_u90": + elif prod == "single" and instrument != "lhumpro_u90": generate_lev2_single( site, _get_filename("1C01", date, site), output_file, lwp_offset ) - elif itype == "lhumpro_u90": + elif instrument == "lhumpro_u90": generate_lev2_lhumpro( site, _get_filename("1C01", date, site), output_file, lwp_offset ) @@ -251,8 +288,12 @@ def process_product(prod: str, date: datetime.date, site: str): csv_off.to_csv(offset_current, index=False) -def plot_product(prod: str, date, site: str): - filename = _get_filename(prod, date, site) +def plot_product(prod: str, date, site: str, data_format: str, instrument: IType): + filename = ( + _get_filename(prod, date, site) + if data_format == "e-profile" + else _get_filename_cloudnet(prod, date, site, instrument) + ) if not os.path.isfile(filename): logging.warning("Nothing to plot for product " + prod) output_dir = f"{os.path.dirname(filename)}/" diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 7d966fe..4f0b67b 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -8,7 +8,7 @@ from numpy import ma from mwrpy import utils, version -from mwrpy.utils import MetaData +from mwrpy.utils import MetaData, seconds2hours class RpgArray: @@ -129,6 +129,29 @@ def find_valid_times(self): ind[time_i] = 1 self._screen(np.where(ind == 1)[0]) + def add_zenith_angle(self): + """Adds zenith angle to data if not present.""" + for key in ("elevation_angle", "ir_elevation_angle"): + if key not in self.data: + continue + zenith_angle = 90 - self.data[key][:] + new_key = key.replace("elevation", "zenith") + self.data[new_key] = RpgArray(zenith_angle, new_key) + self.data[new_key].dimensions = ("time",) + + def convert_time_to_hours(self): + """Converts time from seconds since epoch to hours since midnight.""" + time = self.data["time"].data[:] + time_hours = seconds2hours(time) + self.data["time"] = RpgArray( + time_hours, + "time", + "hours since " + self.date.strftime("%Y-%m-%d %H:%M:%S") + " +00:00", + ) + self.data["time"].dimensions = ("time",) + if "time_bnds" in self.data: + del self.data["time_bnds"] + def _screen(self, ind: np.ndarray): if len(ind) < 1: raise RuntimeError( @@ -144,8 +167,17 @@ def _screen(self, ind: np.ndarray): self.data[key].data = self.data[key].data[ind, :] -def save_rpg(rpg: Rpg, output_file: str | PathLike, att: dict, data_type: str) -> None: +def save_rpg( + rpg: Rpg, + output_file: str | PathLike, + att: dict, + data_type: str, + data_format: str = "e-profile", +) -> None: """Saves the RPG MWR file.""" + if data_format == "cloudnet": + Rpg.convert_time_to_hours(rpg) + Rpg.add_zenith_angle(rpg) if data_type == "1B01": dims = { "time": len(rpg.data["time"][:]), @@ -199,12 +231,22 @@ def save_rpg(rpg: Rpg, output_file: str | PathLike, att: dict, data_type: str) - ["Data type " + data_type + " not supported for file writing."] ) - with init_file(output_file, dims, rpg.data, att) as rootgrp: - setattr(rootgrp, "date", rpg.date.isoformat()) + with init_file(output_file, dims, rpg.data, att, data_format, data_type) as rootgrp: + if data_format == "e-profile": + setattr(rootgrp, "date", rpg.date.isoformat()) + else: + setattr(rootgrp, "year", rpg.date.strftime("%Y")) + setattr(rootgrp, "month", rpg.date.strftime("%m")) + setattr(rootgrp, "day", rpg.date.strftime("%d")) def init_file( - file_name: str | PathLike, dimensions: dict, rpg_arrays: dict, att_global: dict + file_name: str | PathLike, + dimensions: dict, + rpg_arrays: dict, + att_global: dict, + data_format: str, + data_type: str, ) -> netCDF4.Dataset: """Initializes an RPG MWR file for writing. @@ -213,12 +255,18 @@ def init_file( dimensions: Dictionary containing dimension for this file. rpg_arrays: Dictionary containing :class:`RpgArray` instances. att_global: Dictionary containing site specific global attributes + data_format: Data format to be used (cloudnet, e-profile). + data_type: Data type to be used (1B01, 1C01, 2I02, etc). """ nc_file = netCDF4.Dataset(file_name, "w", format="NETCDF4_CLASSIC") for key, dimension in dimensions.items(): nc_file.createDimension(key, dimension) _write_vars2nc(nc_file, rpg_arrays) - _add_standard_global_attributes(nc_file, att_global) + _add_cloudnet_global_attributes( + nc_file, att_global, data_type + ) if data_format == "cloudnet" else ( + _add_standard_global_attributes(nc_file, att_global) + ) return nc_file @@ -244,3 +292,30 @@ def _add_standard_global_attributes(nc_file: netCDF4.Dataset, att_global) -> Non if value is None: value = "" setattr(nc_file, name, value) + + +def _add_cloudnet_global_attributes( + nc_file: netCDF4.Dataset, add_global, data_type +) -> None: + t_zone = datetime.timezone.utc + form = "%Y-%m-%d %H:%M:%S" + instrument = add_global["instrument"].upper() + site = add_global["site"] + att_global = { + "Conventions": "CF-1.8", + "mwrpy_version": version.__version__, + "location": add_global["site"], + "source": f"RPG-Radiometer Physics {instrument}", + "references": "https://doi.org/10.21105/joss.06733", + "mwrpy_file_type": data_type, + "title": f"{instrument} microwave radiometer Level 1c from {site}", + "history": f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00" + + " - " + + data_type + + " file created", + } + for name, value in att_global.items(): + if value is None: + value = "" + setattr(nc_file, name, value) + nc_file.mwrpy_coefficients = ", ".join(add_global["coeff_files"]) diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index 28aa456..a4f2468 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -1,6 +1,38 @@ # Config file for all HATPRO instruments params: + # path to level1 data and path for processed files + data_in: /tmp/data/ + data_out: /tmp/data/ + + # path to retrieval coefficients + coeff_path: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. + # If you do not want to transform the coordinates set azi_cor to -999. + azi_cor: -999. + + const_azi: -999. + + # some default values: + # ------------------- receiver_nb: [1, 2] receiver: [1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2] @@ -71,36 +103,6 @@ params: [0., 100.], ] - # some default values: - # ------------------- - - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ - - # availability of IR - ir_flag: True - - # quality flag status for level 1 data; 0: flag active - # Bit 1: missing_tb - # Bit 2: tb_below_threshold - # Bit 3: tb_above_threshold - # Bit 4: spectral_consistency_above_threshold - # Bit 5: receiver_sanity_failed - # Bit 6: rain_detected - # Bit 7: sun_moon_in_beam - # Bit 8: tb_offset_above_threshold - flag_status: [0, 0, 0, 0, 0, 0, 0, 1] - - # integration time of measurements in seconds - int_time: 1 - - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - - const_azi: -999. - # Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index 7b616e7..224d51b 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -1,6 +1,38 @@ # Config file for all LHATPRO instruments params: + # path to level1 data and path for processed files + data_in: /tmp/data/ + data_out: /tmp/data/ + + # path to retrieval coefficients + coeff_path: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. + # If you do not want to transform the coordinates set azi_cor to -999. + azi_cor: -999. + + const_azi: -999. + + # some default values: + # ------------------- receiver_nb: [2, 1] receiver: [2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1] @@ -69,36 +101,6 @@ params: [0., 100.], ] - # some default values: - # ------------------- - - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ - - # availability of IR - ir_flag: True - - # quality flag status for level 1 data; 0: flag active - # Bit 1: missing_tb - # Bit 2: tb_below_threshold - # Bit 3: tb_above_threshold - # Bit 4: spectral_consistency_above_threshold - # Bit 5: receiver_sanity_failed - # Bit 6: rain_detected - # Bit 7: sun_moon_in_beam - # Bit 8: tb_offset_above_threshold - flag_status: [0, 0, 0, 0, 0, 0, 0, 1] - - # integration time of measurements in seconds - int_time: 1 - - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - - const_azi: -999. - # Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index 42c5dc4..c34b204 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -1,6 +1,38 @@ # Config file for all LHUMPRO U90 instruments params: + # path to level1 data and path for processed files + data_in: /tmp/data/ + data_out: /tmp/data/ + + # path to retrieval coefficients + coeff_path: + + # availability of IR + ir_flag: False + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. + # If you do not want to transform the coordinates set azi_cor to -999. + azi_cor: -999. + + const_azi: -999. + + # some default values: + # ------------------- receiver_nb: [2, 1] receiver: [2, 1, 1, 1, 1, 1, 1] @@ -54,36 +86,6 @@ params: [0., 100.], ] - # some default values: - # ------------------- - - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ - - # availability of IR - ir_flag: False - - # quality flag status for level 1 data; 0: flag active - # Bit 1: missing_tb - # Bit 2: tb_below_threshold - # Bit 3: tb_above_threshold - # Bit 4: spectral_consistency_above_threshold - # Bit 5: receiver_sanity_failed - # Bit 6: rain_detected - # Bit 7: sun_moon_in_beam - # Bit 8: tb_offset_above_threshold - flag_status: [0, 0, 0, 0, 0, 0, 0, 1] - - # integration time of measurements in seconds - int_time: 1 - - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - - const_azi: -999. - # Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 909db12..ddd1fe1 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -21,6 +21,7 @@ SECONDS_PER_HOUR = 3600 SECONDS_PER_DAY = 86400 Epoch = tuple[int, int, int] +IType = Literal["hatpro", "lhatpro", "lhumpro_u90"] class MetaData(NamedTuple): @@ -35,6 +36,8 @@ class MetaData(NamedTuple): retrieval_frequencies: str | None = None retrieval_auxiliary_input: str | None = None retrieval_description: str | None = None + axis: str | None = None + calendar: str | None = None def seconds2hours(time_in_seconds: np.ndarray) -> np.ndarray: @@ -276,10 +279,14 @@ def add_time_bounds(time_arr: np.ndarray, int_time: int) -> np.ndarray: def get_coeff_list( - site: str | None, prefix: str, coeff_files: Sequence[str | PathLike] | None + site: str | None, + prefix: str | list, + coeff_files: Sequence[str | PathLike] | None, + coeff_dir: str | None = None, ) -> list[str]: """Returns list of .nc coefficient file(s).""" if coeff_files is not None: + assert isinstance(prefix, str) c_list = [] for file in coeff_files: basename = os.path.basename(file) @@ -289,32 +296,38 @@ def get_coeff_list( return sorted(c_list) assert isinstance(site, str) - dir_path = os.path.dirname(os.path.realpath(__file__)) - c_list = glob.glob( - dir_path - + "/site_config/" - + site - + "/coefficients/" - + "*" - + prefix.lower() - + "*" - ) - if len(c_list) == 0: - c_list = glob.glob( - dir_path + if coeff_dir is not None: + dir_path = coeff_dir + else: + dir_path = ( + os.path.dirname(os.path.realpath(__file__)) + "/site_config/" + site + "/coefficients/" - + "*" - + prefix.upper() - + "*" ) + if isinstance(prefix, str): + prefix = [prefix] + c_list = [] + for p in prefix: + tmp = glob.glob(dir_path + "*" + p.lower() + "*") + if len(c_list) == 0: + tmp = glob.glob(dir_path + "*" + p.upper() + "*") + c_list = c_list + tmp if len(c_list) > 0: + if "spc" in c_list and "ins" in c_list: + c_list.remove("spc") return sorted(c_list) logging.warning( "No coefficient files for product " - + prefix + + str(prefix) + + " found in directory " + + "/site_config/" + + site + + "/coefficients/" + ) if len(prefix) == 1 else logging.warning( + "No coefficient files for products " + + ", ".join(prefix) + " found in directory " + "/site_config/" + site @@ -596,6 +609,23 @@ def _get_filename(prod: str, date_in: datetime.date, site: str) -> str: return os.path.join(data_out_dir, filename) +def _get_filename_cloudnet( + prod: str, date_in: datetime.date, site: str, instrument: IType +) -> str: + if np.char.isnumeric(prod[0]): + level = prod[0] + name = "l1c" + else: + level = "2" + name = prod + params = read_config(None, instrument, "params") + data_out_dir = os.path.join( + params["data_out"], f"level{level}", date_in.strftime("%Y/%m/%d") + ) + filename = f"{date_in.strftime('%Y%m%d')}_{site}_{instrument}-{name}.nc" + return os.path.join(data_out_dir, filename) + + def isodate2date(date_str: str) -> datetime.date: return datetime.datetime.strptime(date_str, "%Y-%m-%d").date() From 44265f2364660ac7394117a233c1ca5522bdec81 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Fri, 31 Oct 2025 14:57:25 +0100 Subject: [PATCH 02/28] Add Level 2 format option --- mwrpy/cli.py | 21 ++++++++---- mwrpy/level1/write_lev1_nc.py | 26 ++++++++++---- mwrpy/level2/lev2_collocated.py | 24 +++++++++++-- mwrpy/level2/lev2_meta_nc.py | 20 ++++++++++- mwrpy/level2/write_lev2_nc.py | 54 +++++++++++++++++++++++------- mwrpy/process_mwrpy.py | 31 +++++++++++++++-- mwrpy/rpg_mwr.py | 23 ++++++++++--- mwrpy/site_config/hatpro.yaml | 2 -- mwrpy/site_config/lhatpro.yaml | 2 -- mwrpy/site_config/lhumpro_u90.yaml | 2 -- mwrpy/utils.py | 7 ++-- tests/test_write_lev2_nc.py | 9 ++--- 12 files changed, 172 insertions(+), 49 deletions(-) diff --git a/mwrpy/cli.py b/mwrpy/cli.py index 97dee7a..863efd4 100755 --- a/mwrpy/cli.py +++ b/mwrpy/cli.py @@ -65,6 +65,13 @@ def _parse_args(args): metavar="YYYY-MM-DD", help="Single date to be processed.", ) + group.add_argument( + "-f", + "--format", + type=str, + help="Data format to be used (cloudnet, e-profile).", + default="cloudnet", + ) group.add_argument( "-i", "--instrument", @@ -72,13 +79,6 @@ def _parse_args(args): help="Instrument to be processed (hatpro, lhatpro, lhumpro_u90).", default="hatpro", ) - group.add_argument( - "-f", - "--format", - type=str, - help="Data format to be used (cloudnet, e-profile).", - default="e-profile", - ) group.add_argument( "-a", "--altitude", @@ -86,6 +86,13 @@ def _parse_args(args): help="Altitude above mean sea level of site (m).", default=0.0, ) + group.add_argument( + "-o", + "--azimuth_offset", + type=float, + help="Azimuth offset of the instrument (degrees).", + default=None, + ) return parser.parse_args(args) diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index a8911be..d90d7aa 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -42,6 +42,7 @@ def lev1_to_nc( time_offset: datetime.timedelta | None = None, instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, altitude: float | None = None, + azimuth_offset: float | None = None, ) -> rpg_mwr.Rpg: """This function reads one day of RPG MWR binary files, adds attributes and writes it into netCDF file. @@ -59,9 +60,10 @@ def lev1_to_nc( time_offset: Time offset if instrument operated in local time. instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). altitude: Altitude of the site in meters above mean sea level. + azimuth_offset: Azimuth offset to be added to azimuth angle. Raises: - MissingInputData: if required input file is missing. + MissingInputData: if input file is missing. """ if site is None: assert coeff_files is not None @@ -87,7 +89,13 @@ def lev1_to_nc( params = {**params, **instrument_config} rpg_bin = prepare_data( - path_to_files, data_type, params, lidar_path, time_offset, altitude + path_to_files, + data_type, + params, + lidar_path, + time_offset, + altitude, + azimuth_offset, ) if data_type in ("1B01", "1C01"): @@ -102,12 +110,12 @@ def lev1_to_nc( c_files = ( get_coeff_list( site, - ["spc", "ins", "lwp", "iwv", "hpt", "tpt", "tpb"], + ["spc", "ins"], None, params.get("coeff_path", None), ) - if (coeff_files is None) - else (coeff_files) + if coeff_files is None + else coeff_files ) global_attributes = { "site": site, @@ -129,6 +137,7 @@ def prepare_data( lidar_path: str | PathLike | None, time_offset: datetime.timedelta | None = None, altitude: float | None = None, + azimuth_offset: float | None = None, ) -> RpgBin: """Load and prepare data for netCDF writing.""" if data_type in ("1B01", "1C01"): @@ -205,9 +214,12 @@ def prepare_data( if params["azi_cor"] != -999.0: _azi_correction(rpg_bin.data, params) - if params["const_azi"] != -999.0: + azimuth_offset = ( + params["azimuth_offset"] if "azimuth_offset" in params else azimuth_offset + ) + if azimuth_offset is not None: rpg_bin.data["azimuth_angle"] = ( - rpg_bin.data["azimuth_angle"] + params["const_azi"] + rpg_bin.data["azimuth_angle"] + azimuth_offset ) % 360 if data_type == "1C01": diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 06f2a44..0af230e 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -2,6 +2,7 @@ from collections.abc import Sequence from os import PathLike from tempfile import NamedTemporaryFile +from typing import Literal import netCDF4 @@ -11,10 +12,12 @@ def generate_lev2_single( site: str | None, + data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, lwp_offset: list[float | None] = [None, None], coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as lwp_file, @@ -41,6 +44,7 @@ def generate_lev2_single( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=t_prof_file.name @@ -51,6 +55,7 @@ def generate_lev2_single( else None, lwp_offset=lwp_offset, coeff_files=coeff_files, + instrument_type=instrument_type, ) with ( @@ -65,7 +70,8 @@ def generate_lev2_single( ): nc_output.createDimension("height", len(nc_t_prof.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) - nc_output.createDimension("bnds", 2) + if data_format == "e-profile": + nc_output.createDimension("bnds", 2) for source, variables in ( ( @@ -152,6 +158,7 @@ def generate_lev2_single( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=t_prof_file.name @@ -162,6 +169,7 @@ def generate_lev2_single( else None, lwp_offset=[None, None], coeff_files=coeff_files, + instrument_type=instrument_type, ) with netCDF4.Dataset(stability_file.name, "r") as nc_sta: var_2I06 = ( @@ -186,10 +194,12 @@ def generate_lev2_single( def generate_lev2_lhumpro( site: str | None, + data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, lwp_offset: list[float | None] = [None, None], coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as lwp_file, @@ -208,12 +218,14 @@ def generate_lev2_lhumpro( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=None, hum_file=None, lwp_offset=lwp_offset, coeff_files=coeff_files, + instrument_type=instrument_type, ) with ( @@ -224,7 +236,8 @@ def generate_lev2_lhumpro( ): nc_output.createDimension("height", len(nc_abs_hum.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) - nc_output.createDimension("bnds", 2) + if data_format == "e-profile": + nc_output.createDimension("bnds", 2) for source, variables in ( ( @@ -276,9 +289,11 @@ def generate_lev2_lhumpro( def generate_lev2_multi( site: str | None, + data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as temperature_file, @@ -301,6 +316,7 @@ def generate_lev2_multi( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=temperature_file.name @@ -309,6 +325,7 @@ def generate_lev2_multi( hum_file=abs_hum_file.name if prod not in ("2P02", "2P03") else None, lwp_offset=[None, None], coeff_files=coeff_files, + instrument_type=instrument_type, ) with ( @@ -320,7 +337,8 @@ def generate_lev2_multi( ): nc_output.createDimension("time", len(nc_temp.variables["time"][:])) nc_output.createDimension("height", len(nc_temp.variables["height"][:])) - nc_output.createDimension("bnds", 2) + if data_format == "e-profile": + nc_output.createDimension("bnds", 2) for source, variables in ( ( diff --git a/mwrpy/level2/lev2_meta_nc.py b/mwrpy/level2/lev2_meta_nc.py index 10b5d65..7961447 100644 --- a/mwrpy/level2/lev2_meta_nc.py +++ b/mwrpy/level2/lev2_meta_nc.py @@ -6,13 +6,16 @@ from mwrpy.utils import MetaData -def get_data_attributes(rpg_variables: dict, data_type: str, coeff: dict) -> dict: +def get_data_attributes( + rpg_variables: dict, data_type: str, coeff: dict, data_format: str +) -> dict: """Adds Metadata for RPG MWR Level 2 variables for NetCDF file writing. Args: rpg_variables: RpgArray instances. data_type: Data type of the netCDF file. coeff: Coefficient data of variable + data_format: Data format of the netCDF file (cloudnet, e-profile). Returns: Dictionary @@ -61,6 +64,10 @@ def get_data_attributes(rpg_variables: dict, data_type: str, coeff: dict) -> dic else: del rpg_variables[key] + if data_format == "cloudnet": + attributes.pop("time") + attributes = dict(ATTRIBUTES_CN, **attributes) + index_map = {v: i for i, v in enumerate(attributes)} rpg_variables = dict( sorted(rpg_variables.items(), key=lambda pair: index_map[pair[0]]) @@ -69,6 +76,17 @@ def get_data_attributes(rpg_variables: dict, data_type: str, coeff: dict) -> dic return rpg_variables +ATTRIBUTES_CN = { + "time": MetaData( + comment="Time indication of samples is at end of integration-time", + units="hours since ", + long_name="Time UTC", + standard_name="time", + calendar="standard", + dimensions=("time",), + ), +} + DEFINITIONS_COM = { "quality_flag": ( "\n" diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 27c3227..e22f81f 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -3,6 +3,7 @@ from collections.abc import Sequence from datetime import datetime from os import PathLike +from typing import Literal import netCDF4 as nc import numpy as np @@ -18,6 +19,7 @@ from mwrpy.level2.lev2_meta_nc import get_data_attributes from mwrpy.level2.lwp_offset import correct_lwp_offset from mwrpy.utils import ( + get_coeff_list, interpol_2d, interpolate_2d, read_config, @@ -27,12 +29,14 @@ def lev2_to_nc( data_type: str, lev1_file: str | PathLike, + data_format: str, output_file: str | PathLike, site: str | None = None, temp_file: str | PathLike | None = None, hum_file: str | PathLike | None = None, lwp_offset: list[float | None] = [None, None], coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): """This function reads Level 1 files, applies retrieval coefficients for Level 2 products @@ -41,12 +45,14 @@ def lev2_to_nc( Args: data_type: Data type of the netCDF file. lev1_file: Path of Level 1 file. + data_format: Data format of the netCDF file (cloudnet, e-profile). output_file: Name of output file. site: Name of site. temp_file: Name of temperature product file. hum_file: Name of humidity product file. lwp_offset: Offset for LWP correction. coeff_files: List of coefficient files. + instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). """ if data_type not in ( @@ -62,8 +68,9 @@ def lev2_to_nc( ): raise ValueError(f"Data type {data_type} not recognised") - global_attributes = read_config(site, "hatpro", "global_specs") - params = read_config(site, "hatpro", "params") + assert instrument_type is not None + global_attributes = read_config(site, instrument_type, "global_specs") + params = read_config(site, instrument_type, "params") with nc.Dataset(lev1_file) as lev1: params["altitude"] = ma.median(lev1.variables["altitude"][:]) @@ -81,8 +88,26 @@ def lev2_to_nc( _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) _del_att(global_attributes) mwr = rpg_mwr.Rpg(rpg_dat) - mwr.data = get_data_attributes(mwr.data, data_type, coeff) - rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type) + mwr.data = get_data_attributes(mwr.data, data_type, coeff, data_format) + if data_format == "cloudnet": + c_files = ( + get_coeff_list( + site, + prefix=["tpb"] + if data_type in ("2P02", "2P04", "2P07", "2P08") + else ["lwp", "iwv", "hpt", "tpt"], + coeff_files=None, + coeff_dir=params.get("coeff_path", None), + ) + if coeff_files is None + else coeff_files + ) + global_attributes = { + "site": site, + "instrument": instrument_type, + "coeff_files": c_files, + } + rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) def get_products( @@ -107,16 +132,19 @@ def get_products( np.empty([0], np.int32), np.empty([0], np.int32), ) + coeff_path = params.get("coeff_path", None) if data_type in ("2I01", "2I02", "2I06"): product = ( "lwp" if data_type == "2I01" else "iwv" if data_type == "2I02" else "sta" ) - coeff = get_mvr_coeff(site, product, lev1["frequency"][:], coeff_files) + coeff = get_mvr_coeff( + site, product, lev1["frequency"][:], coeff_files, coeff_path + ) if coeff[0]["RT"] < 2: coeff, offset, lin, quad = get_mvr_coeff( - site, product, lev1["frequency"][:], coeff_files + site, product, lev1["frequency"][:], coeff_files, coeff_path ) else: # pylint: disable-next=unbalanced-tuple-unpacking @@ -254,10 +282,10 @@ def get_products( else: product, ret = "absolute_humidity", "hpt" - coeff = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files) + coeff = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files, coeff_path) if coeff[0]["RT"] < 2: coeff, offset, lin, quad = get_mvr_coeff( - site, ret, lev1["frequency"][:], coeff_files + site, ret, lev1["frequency"][:], coeff_files, coeff_path ) else: # pylint: disable-next=unbalanced-tuple-unpacking @@ -270,7 +298,7 @@ def get_products( weights1, weights2, factor, - ) = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files) + ) = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files, coeff_path) ret_in = retrieval_input(lev1, coeff) @@ -355,15 +383,17 @@ def get_products( _get_qf(rpg_dat, lev1, coeff, index, index_ret, product) elif data_type == "2P02": - coeff = get_mvr_coeff(site, "tpb", lev1["frequency"][:], coeff_files) + coeff = get_mvr_coeff( + site, "tpb", lev1["frequency"][:], coeff_files, coeff_path + ) if coeff[0]["RT"] < 2: coeff, offset, lin, quad = get_mvr_coeff( - site, "tpb", lev1["frequency"][:], coeff_files + site, "tpb", lev1["frequency"][:], coeff_files, coeff_path ) else: # pylint: disable-next=unbalanced-tuple-unpacking coeff, _, _, _, _, _, _, _ = get_mvr_coeff( - site, "tpb", lev1["frequency"][:], coeff_files + site, "tpb", lev1["frequency"][:], coeff_files, coeff_path ) coeff["AG"] = np.flip(np.sort(coeff["AG"])) diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index 7116cc7..c2e1a5c 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -126,6 +126,7 @@ def main(args): args.format, args.instrument, args.altitude, + args.azimuth_offset, ) except Exception as e: logging.error( @@ -139,6 +140,7 @@ def main(args): args.format, args.instrument, args.altitude, + args.azimuth_offset, ) if args.command != "no-plot": logging.info(f"Plotting {product} product, {args.site} {date}") @@ -159,6 +161,7 @@ def process_product( data_format: str, instrument: IType, altitude: float, + azimuth_offset: float | None, ): output_file = ( _get_filename(prod, date, site) @@ -211,6 +214,7 @@ def process_product( date=date, instrument_type=instrument, altitude=altitude, + azimuth_offset=azimuth_offset, ) elif prod[0] == "2": if prod in ("2P04", "2P07", "2P08"): @@ -224,22 +228,43 @@ def process_product( lev2_to_nc( prod, _get_filename("1C01", date, site), + data_format, output_file=output_file, site=site, temp_file=temp_file, hum_file=hum_file, lwp_offset=lwp_offset, + instrument_type=instrument, ) elif prod == "single" and instrument != "lhumpro_u90": generate_lev2_single( - site, _get_filename("1C01", date, site), output_file, lwp_offset + site, + data_format, + _get_filename("1C01", date, site), + output_file, + lwp_offset, + None, + instrument, ) elif instrument == "lhumpro_u90": generate_lev2_lhumpro( - site, _get_filename("1C01", date, site), output_file, lwp_offset + site, + data_format, + _get_filename("1C01", date, site), + output_file, + lwp_offset, + None, + instrument, ) elif prod == "multi": - generate_lev2_multi(site, _get_filename("1C01", date, site), output_file) + generate_lev2_multi( + site, + data_format, + _get_filename("1C01", date, site), + output_file, + None, + instrument, + ) offset_current = _get_filename("lwp_offset", date, site) if ( diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 4f0b67b..0f9d049 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -260,7 +260,10 @@ def init_file( """ nc_file = netCDF4.Dataset(file_name, "w", format="NETCDF4_CLASSIC") for key, dimension in dimensions.items(): - nc_file.createDimension(key, dimension) + if data_format == "cloudnet" and key == "bnds": + continue + else: + nc_file.createDimension(key, dimension) _write_vars2nc(nc_file, rpg_arrays) _add_cloudnet_global_attributes( nc_file, att_global, data_type @@ -301,17 +304,29 @@ def _add_cloudnet_global_attributes( form = "%Y-%m-%d %H:%M:%S" instrument = add_global["instrument"].upper() site = add_global["site"] + if data_type == "1C01": + level = "mwr-l1c" + title = f"{instrument} microwave radiometer Level 1c from {site}" + history = level + elif data_type == "2P02": + level = "mwr-multi" + title = f"MWR multiple-pointing from {site}" + history = "MWR multiple-pointing" + else: + level = "mwr-single" + title = f"MWR single-pointing from {site}" + history = "MWR single-pointing" att_global = { "Conventions": "CF-1.8", "mwrpy_version": version.__version__, "location": add_global["site"], "source": f"RPG-Radiometer Physics {instrument}", "references": "https://doi.org/10.21105/joss.06733", - "mwrpy_file_type": data_type, - "title": f"{instrument} microwave radiometer Level 1c from {site}", + "cloudnet_file_type": level, + "title": title, "history": f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00" + " - " - + data_type + + history + " file created", } for name, value in att_global.items(): diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index a4f2468..42c6de6 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -29,8 +29,6 @@ params: # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. - const_azi: -999. - # some default values: # ------------------- receiver_nb: [1, 2] diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index 224d51b..81b0010 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -29,8 +29,6 @@ params: # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. - const_azi: -999. - # some default values: # ------------------- receiver_nb: [2, 1] diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index c34b204..c77863a 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -29,8 +29,6 @@ params: # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. - const_azi: -999. - # some default values: # ------------------- receiver_nb: [2, 1] diff --git a/mwrpy/utils.py b/mwrpy/utils.py index ddd1fe1..46d7675 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -310,9 +310,12 @@ def get_coeff_list( c_list = [] for p in prefix: tmp = glob.glob(dir_path + "*" + p.lower() + "*") - if len(c_list) == 0: + if len(tmp) == 0: tmp = glob.glob(dir_path + "*" + p.upper() + "*") - c_list = c_list + tmp + if len(c_list) == 0: + c_list = tmp + elif len(tmp) > 0 and len(c_list) > 0: + c_list = c_list + tmp if len(c_list) > 0: if "spc" in c_list and "ins" in c_list: diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index a386fdd..525cc48 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -8,6 +8,7 @@ from mwrpy.level2.lev2_collocated import generate_lev2_multi, generate_lev2_single SITE = "hyytiala" +DATA_FORMAT = "e-profile" PACKAGE_DIR = os.path.abspath(os.path.dirname(__file__)) DATA_DIR = f"{PACKAGE_DIR}/data/{SITE}" @@ -31,25 +32,25 @@ def delete_file(): def test_generate_lev2_single_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(SITE, l1_file, path) + generate_lev2_single(SITE, DATA_FORMAT, l1_file, path) os.unlink(path) def test_generate_lev2_single_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(None, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_single(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) os.unlink(path) def test_generate_lev2_multi_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(SITE, l1_file, path) + generate_lev2_multi(SITE, DATA_FORMAT, l1_file, path) os.unlink(path) def test_generate_lev2_multi_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(None, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_multi(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) From bc35eab2c7199565b7d9e226f80cda151ce00750 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Fri, 31 Oct 2025 14:58:19 +0100 Subject: [PATCH 03/28] Update README and docs --- README.md | 30 ++-- docs/source/command_line_usage.rst | 105 ++++++++++++ docs/source/data_types.rst | 31 ++++ docs/source/fileformat.rst | 100 +++++++++++- docs/source/index.rst | 2 + docs/source/mwrpy_processing.rst | 247 +++++++++++++++++------------ 6 files changed, 395 insertions(+), 120 deletions(-) create mode 100644 docs/source/command_line_usage.rst create mode 100644 docs/source/data_types.rst diff --git a/README.md b/README.md index 92840c6..f233b7a 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Level 2 data products and visualization and is based on the IDL code [mwr_pro](https://zenodo.org/records/7973553). The netCDF data format including metadata information, variable names and file naming -is designed to be compliant with the data structure and naming convention -developed in the [EUMETNET Profiling Programme E-PROFILE](https://www.eumetnet.eu/). +is designed to be compliant with either the data structure and naming convention +developed in the [EUMETNET Profiling Programme E-PROFILE](https://www.eumetnet.eu/), or within ACTRIS. MWRpy documentation: @@ -58,24 +58,30 @@ For example, this is the [configuration file for RPG-HATPRO](mwrpy/site_config/h The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a site and instrument specific configuration file (`config.yaml`) and retrieval coefficients. For example, this is the [configuration file for Hyytiälä](mwrpy/site_config/hyytiala/config.yaml). +This site configuration file is not needed when using the Cloudnet file format. ## Command line usage MWRpy can be run using the command line tool `mwrpy/cli.py`: usage: mwrpy/cli.py [-h] -s SITE [-d YYYY-MM-DD] [--start YYYY-MM-DD] - [--stop YYYY-MM-DD] [-p ...] [{process,plot}] + [--stop YYYY-MM-DD] [-f ...] [-p ...] [{process,plot}] Arguments: -| Short | Long | Default | Description | -| :---- | :----------- | :------------------ | :--------------------------------------------------------------------------------- | -| `-h` | `--help` | | Show help and exit. | -| `-s` | `--site` | | Site to process data from, e.g, `hyytiala`. Required. | -| `-d` | `--date` | | Single date to be processed. Alternatively, `--start` and `--stop` can be defined. | -| | `--start` | `current day - 1` | Starting date. | -| | `--stop` | `current day ` | Stopping date. | -| `-p` | `--products` | 1C01, single, multi | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | +| Short | Long | Default | Description | +| :------------------------------------------------------------- | :----------------- | :------------------------ | :--------------------------------------------------------------------------------- | +| `-h` | `--help` | | Show help and exit. | +| `-s` | `--site` | | Site to process data from, e.g, `hyytiala`. Required. | +| `-d` | `--date` | | Single date to be processed. Alternatively, `--start` and `--stop` can be defined. | +| | `--start` | `current day - 1` | Starting date. | +| | `--stop` | `current day ` | Stopping date. | +| `-p` | `--products` | `1C01`, `single`, `multi` | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | +| `-f` | `--format` | `cloudnet` | Data format to be used (`cloudnet`, `e-profile`). | +| The following arguments are used for the Cloudnet file format: | +| `-i` | `--instrument` | `hatpro` | Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). | +| `-a` | `--altitude` | `0.0` | Altitude above mean sea level of site (m). | +| `-o` | `--azimuth_offset` | `None` | Azimuth offset of the instrument (degrees). Or `None`. | Commands: @@ -109,6 +115,8 @@ Commands: - single: Single pointing data product (including 2I01, 2I02, 2I06, 2P01, 2P03, and derived products) - multi: Multiple pointing data product (including 2P02, and derived products) +Only the `1C01`, `single`, and `multi` data types are available when using the Cloudnet file format. + ## Licence MIT diff --git a/docs/source/command_line_usage.rst b/docs/source/command_line_usage.rst new file mode 100644 index 0000000..56edfe6 --- /dev/null +++ b/docs/source/command_line_usage.rst @@ -0,0 +1,105 @@ +================== +Command line usage +================== + +After defining the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and site specific +information (``mwrpy/site_config/{site}.yaml``, only for E-PROFILE format) files, including input/output data +paths, MWRpy can also be run using the command line tool `mwrpy/cli.py`: + +.. code-block:: + + mwrpy/cli.py [-h] -s SITE [-d YYYY-MM-DD] [--start YYYY-MM-DD] + [--stop YYYY-MM-DD] [-p ...] [{process,plot}] + +.. list-table:: Arguments + :widths: 10 20 20 50 + :header-rows: 1 + + * - Short + - Long + - Default + - Description + * - `-h` + - `--help` + - + - Show help and exit. + * - `-s` + - `--site` + - + - Site to process data from, e.g, `hyytiala`. Required. + * - `-d` + - `--date` + - + - Single date to be processed. Alternatively, `--start` and `--stop` can be defined. + * - + - `--start` + - `current day - 1` + - Starting date. + * - + - `--stop` + - `current day` + - Stopping date. + * - `-p` + - `--products` + - 1C01, single, multi + - Processed products, e.g, `1C01, 2I02, 2P03, single`, see Data Types below. + * - `-f` + - `--format` + - cloudnet + - Data format to be used (`cloudnet`, `e-profile`). + +The following arguments are used for the Cloudnet file format: + +.. list-table:: Arguments + :widths: 10 20 20 50 + :header-rows: 1 + + * - Short + - Long + - Default + - Description + * - `-i` + - `--instrument` + - hatpro + - Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). + * - `-a` + - `--altitude` + - 0.0 + - Altitude above mean sea level of site (m). + * - `-o` + - `--azimuth_offset` + - None + - Azimuth offset of the instrument (degrees). Or `None`. + +These commands are available to select the processing mode: + +.. list-table:: Commands + :widths: 20 30 + :header-rows: 1 + + * - Command + - Description + * - `process` + - Process data and generate plots (default). + * - `plot` + - Only generate plots. + * - `no-plot` + - Only generate products. + * - `reprocess` + - Like `process`, but skips days when data processing fails. + +Example usage +------------- +To process and plot Level 1 & 2 data (1C01, single, multi) for the site `Hyytiala` (HATPRO instrument) for April 6, +2023, in the E-PROFILE format, run: + +.. code-block:: + + python mwrpy/cli.py -s hyytiala -d 2023-04-06 -f e-profile process + + +Run the following command for the Cloudnet format (with site altitude 150 m) and no plots: + +.. code-block:: + + python mwrpy/cli.py -s hyytiala -d 2023-04-06 -a 150 no-plot diff --git a/docs/source/data_types.rst b/docs/source/data_types.rst new file mode 100644 index 0000000..2b1702b --- /dev/null +++ b/docs/source/data_types.rst @@ -0,0 +1,31 @@ +========== +Data Types +========== + +The following data types (E-PROFILE naming convention) are available in MWRpy for Level 1 and Level 2 products: + +Level 1 +....... + +- 1B01: MWR brightness temperatures from .BRT and .BLB/.BLS files + retrieved spectrum +- 1B11: IR brightness temperatures from .IRT files +- 1B21: Weather station data from .MET files +- 1C01: Combined data type with time corresponding to 1B01 + +Level 2 +....... + +- 2I01: Liquid water path (LWP) +- 2I02: Integrated water vapor (IWV) +- 2I06: Stability Indices +- 2P01: Temperature profiles from single-pointing observations +- 2P02: Temperature profiles from multiple-pointing observations +- 2P03: Absolute humidity profiles +- 2P04: Relative humidity profiles (derived from 2P01/2P02 + 2P03) +- 2P07: Potential temperature (derived from 2P01/2P02 + 2P03) +- 2P08: Equivalent potential temperature (derived from 2P01/2P02 + 2P03) +- single: Single pointing data product (including 2I01, 2I02, 2I06, 2P01, 2P03, and derived products) +- multi: Multiple pointing data product (including 2P02, and derived products) + + +The data types 1C01, single, and multi are also available in the Cloudnet format. diff --git a/docs/source/fileformat.rst b/docs/source/fileformat.rst index f8b7af8..75eb98f 100644 --- a/docs/source/fileformat.rst +++ b/docs/source/fileformat.rst @@ -6,20 +6,28 @@ All MWRpy files use ``NETCDF4_CLASSIC`` data model, i.e., ``HDF5`` file format. **Dimensions** .. list-table:: - :widths: 25 + :widths: 25 25 :header-rows: 1 - * - Name + * - Name (E-PROFILE) + - Name (Cloudnet) * - time + - time * - bnds + - * - frequency + - frequency * - ir_wavelength + - ir_channel * - receiver_nb + - receiver_nb * - t_amb_nb + - t_amb_nb * - height + - height -**Variables (common to all files)** +**Variables (common to all E-PROFILE files)** .. list-table:: :widths: 25 50 25 25 25 25 @@ -62,6 +70,43 @@ All MWRpy files use ``NETCDF4_CLASSIC`` data model, i.e., ``HDF5`` file format. - float32 - altitude +**Variables (common to all Cloudnet files)** + +.. list-table:: + :widths: 25 50 25 25 25 25 + :header-rows: 1 + + * - Name + - Long name + - Dimensions + - Units + - Data type + - Standard name + * - time + - Time UTC + - time + - hours since YYYY-MM-DD 00:00:00 +00:00 + - double + - time + * - latitude + - Latitude of site + - time + - degree_north + - float32 + - latitude + * - longitude + - Longitude of site + - time + - degree_east + - float32 + - longitude + * - altitude + - Altitude of site + - time + - m + - float32 + - altitude + MWR-Level 1 files ................. @@ -70,7 +115,8 @@ MWR-Level 1 files ~~~~~~~~~ The Level 1 default file type ``1C01`` contains all variables from the file types -``1B01``, ``1B11`` (if an infrared radiometer is available), and ``1B21`` (if a weather station is available). +``1B01``, ``1B11`` (if an infrared radiometer is available), and ``1B21`` (if a weather station is available) and is +available for the E-PROFILE and Cloudnet data format. **Variables (MWR_1B01 specific)** @@ -193,6 +239,25 @@ The Level 1 default file type ``1C01`` contains all variables from the file type - int32 - +**Additional Cloudnet variable** + +.. list-table:: + :widths: 25 50 25 25 25 25 + :header-rows: 1 + + * - Name + - Long name + - Dimensions + - Units + - Data type + - Standard name + * - zenith_angle + - Zenith angle + - time + - degree + - float32 + - zenith_angle + **Variables (MWR_1B11 specific)** .. list-table:: @@ -242,6 +307,25 @@ The Level 1 default file type ``1C01`` contains all variables from the file type - float32 - +**Additional Cloudnet variable** + +.. list-table:: + :widths: 25 50 25 25 25 25 + :header-rows: 1 + + * - Name + - Long name + - Dimensions + - Units + - Data type + - Standard name + * - ir_zenith_angle + - Infrared sensor zenith angle + - time + - degree + - float32 + - + **Variables (MWR_1B21 specific)** .. list-table:: @@ -298,7 +382,7 @@ The Level 1 default file type ``1C01`` contains all variables from the file type - MWR-Level 2 files -............... +................. **Variables (common to all Level 2 files)** @@ -329,7 +413,8 @@ Single pointing file ~~~~~~~~~~~~~~~~~~~~ The Level 2 default file type ``single`` contains all variables from the file types -``2I01``, ``2I02``, ``2I06``, ``2P01``, and ``2P03`` (if the respective retrieval coefficients are available). +``2I01``, ``2I02``, ``2I06``, ``2P01``, and ``2P03`` (if the respective retrieval coefficients are available) and is +available for the E-PROFILE and Cloudnet data format. **Variables (MWR_2I01 specific)** @@ -545,7 +630,8 @@ Multiple pointing file ~~~~~~~~~~~~~~~~~~~~~~ The Level 2 default file type ``multi`` contains all variables from the file types -``2P02``, ``2P04``, ``2P07``, and ``2P08`` (if the respective retrieval coefficients are available). +``2P02``, ``2P04``, ``2P07``, and ``2P08`` (if the respective retrieval coefficients are available) and is +available for the E-PROFILE and Cloudnet data format. **Variables (MWR_2P02 specific)** diff --git a/docs/source/index.rst b/docs/source/index.rst index dfc952f..619c14b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,9 @@ Welcome to MWRpy's documentation! overview installation + data_types mwrpy_processing + command_line_usage fileformat guide diff --git a/docs/source/mwrpy_processing.rst b/docs/source/mwrpy_processing.rst index fd6b2a2..2a001e6 100644 --- a/docs/source/mwrpy_processing.rst +++ b/docs/source/mwrpy_processing.rst @@ -17,10 +17,27 @@ quality control and visualization. This example utilizes files taken from the AC .BRT and .HKD files are mandatory in MWRpy for processing +First steps for processing examples: + +First we define the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and site specific +information (``mwrpy/site_config/{site}.yaml``, only for E-PROFILE format) files, including input/output data paths. +Then the data path is specified: + +.. code-block:: python + + import os + + package_dir = os.getcwd() + site = "hyytiala" + data_path = f"{package_dir}/tests/data/{site}" + +E-PROFILE format +---------------- + Level 1c ~~~~~~~~~ -First we convert RPG microwave radiometer (MWR) binary files, including brightness temperature (TB) and +Now we convert RPG microwave radiometer (MWR) binary files, including brightness temperature (TB) and housekeeping data (\*.BRT, \*.HKD), into a Level 1c netCDF file. Data from optional elevation scans (\*.BLB, \*.BLS), weather station (\*.MET) and infrared radiometer (\*.IRT) are combined in this process and the following quality flags are derived: @@ -43,17 +60,13 @@ flag status variable contains information whether the flag is active. .. code-block:: python - import os from mwrpy.level1.write_lev1_nc import lev1_to_nc - SITE = "hyytiala" - PACKAGE_DIR = os.getcwd() - DATA_DIR = f"{PACKAGE_DIR}/tests/data/{SITE}" - mwr_raw = lev1_to_nc( - "1C01", - DATA_DIR, - site=SITE, + data_type="1C01", + path_to_files=data_path, + data_format="e-profile", + site=site, output_file="mwr_1c.nc", ) @@ -65,10 +78,9 @@ Variables such as brightness temperature can be plotted from the newly generated .. code-block:: python - import os from mwrpy.plots.generate_plots import generate_figure - PACKAGE_DIR = os.getcwd() - generate_figure('mwr_1c.nc', ['tb'], save_path=f"{PACKAGE_DIR}/") + + generate_figure('mwr_1c.nc', ['tb'], save_path=f"{package_dir}/") .. figure:: _static/20230406_hyytiala_tb.png @@ -82,7 +94,13 @@ are applied to generate the Level 2 single pointing product: .. code-block:: python from mwrpy.level2.lev2_collocated import generate_lev2_single - mwr_prod = generate_lev2_single("hyytiala", "mwr_1c.nc", "mwr-single.nc") + + mwr_prod = generate_lev2_single( + site="hyytiala", + data_format="e-profile", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-single.nc", + ) Variables such as integrated water vapor (`IWV `_) @@ -90,10 +108,9 @@ can be plotted from the newly generated file. .. code-block:: python - import os from mwrpy.plots.generate_plots import generate_figure - PACKAGE_DIR = os.getcwd() - generate_figure('mwr-single.nc', ['iwv'], save_path=f"{PACKAGE_DIR}/") + + generate_figure('mwr-single.nc', ['iwv'], save_path=f"{package_dir}/") .. figure:: _static/20230406_hyytiala_iwv.png @@ -107,98 +124,124 @@ product: .. code-block:: python from mwrpy.level2.lev2_collocated import generate_lev2_multi - mwr_prod = generate_lev2_multi("hyytiala", "mwr_1c.nc", "mwr-multi.nc") + + mwr_prod = generate_lev2_multi( + site="hyytiala", + data_format="e-profile", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-multi.nc", + ) Variables such as temperature profiles can be plotted from the newly generated file. .. code-block:: python - import os from mwrpy.plots.generate_plots import generate_figure - PACKAGE_DIR = os.getcwd() - generate_figure('mwr-multi.nc', ['temperature'], save_path=f"{PACKAGE_DIR}/") + + generate_figure('mwr-multi.nc', ['temperature'], save_path=f"{package_dir}/") .. figure:: _static/20230406_hyytiala_temperature.png -Command line usage -~~~~~~~~~~~~~~~~~~ - -After defining the instrument type and site specific configuration files (including input/output data paths) in -``mwrpy/site_config/``, MWRpy can also be run using the command line tool `mwrpy/cli.py`: - -.. code-block:: - - mwrpy/cli.py [-h] -s SITE [-d YYYY-MM-DD] [--start YYYY-MM-DD] - [--stop YYYY-MM-DD] [-p ...] [{process,plot}] - -.. list-table:: Arguments - :widths: 10 20 20 50 - :header-rows: 1 - - * - Short - - Long - - Default - - Description - * - `-h` - - `--help` - - - - Show help and exit. - * - `-s` - - `--site` - - - - Site to process data from, e.g, `hyytiala`. Required. - * - `-d` - - `--date` - - - - Single date to be processed. Alternatively, `--start` and `--stop` can be defined. - * - - - `--start` - - `current day - 1` - - Starting date. - * - - - `--stop` - - `current day` - - Stopping date. - * - `-p` - - `--products` - - 1C01, single, multi - - Processed products, e.g, `1C01, 2I02, 2P03, single`, see Data Types below. - -.. list-table:: Commands - :widths: 20 30 - :header-rows: 1 - - * - Command - - Description - * - `process` - - Process data and generate plots (default). - * - `plot` - - Only generate plots. - * - `no-plot` - - Only generate products. - * - `reprocess` - - Like `process`, but skips days when data processing fails. - -Data Types -~~~~~~~~~~ - -Level 1 - -- 1B01: MWR brightness temperatures from .BRT and .BLB/.BLS files + retrieved spectrum -- 1B11: IR brightness temperatures from .IRT files -- 1B21: Weather station data from .MET files -- 1C01: Combined data type with time corresponding to 1B01 - -Level 2 - -- 2I01: Liquid water path (LWP) -- 2I02: Integrated water vapor (IWV) -- 2I06: Stability Indices -- 2P01: Temperature profiles from single-pointing observations -- 2P02: Temperature profiles from multiple-pointing observations -- 2P03: Absolute humidity profiles -- 2P04: Relative humidity profiles (derived from 2P01/2P02 + 2P03) -- 2P07: Potential temperature (derived from 2P01/2P02 + 2P03) -- 2P08: Equivalent potential temperature (derived from 2P01/2P02 + 2P03) -- single: Single pointing data product (including 2I01, 2I02, 2I06, 2P01, 2P03, and derived products) -- multi: Multiple pointing data product (including 2P02, and derived products) +Cloudnet format +--------------- +In this example the Cloudnet API is used to fetch data and retrieval files and the Cloudnet data format is selected +for processing. More details can be found in the E-PROFILE example above. + +Using Cloudnet API to fetch data and retrieval files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download raw data (binary files): + +.. code-block:: python + + from cloudnet_api_client import APIClient + + date = "2023-04-06" + instrument_pid = "https://hdl.handle.net/21.12132/3.f360a2375f3e4e4f" # check https://cloudnet.fmi.fi/instruments to find the PID of your instrument + client = APIClient() + instruments = client.instruments() + instrument_type = [i.instrument_id for i in instruments if i.pid == instrument_pid][0] + files = client.raw_files(site_id=site, instrument_id=instrument_type, date=date) + binary_files = [f for f in files] + + binary_filepaths = await client.adownload(binary_files, data_path) + +Download retrieval files: + +.. code-block:: python + + import requests + + calibration = client.calibration(instrument_pid, date) + retrieval = calibration["data"] + + retrieval_files = [] + for file in retrieval["coefficientLinks"]: + filename = data_path + file.split("/")[-1] + response = requests.get(file) + with open(filename, "wb") as f: + f.write(response.content) + retrieval_files.append(str(filename)) + +Process and plot Level 1 data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In contrast to the E-PROFILE data format, no site specific information file is required, but metadata needs to be +defined. Also, the retrieval files are set as an argument. + +.. code-block:: python + + site_info = client.site(site_id=site) + site_meta = { + "name": site_info.id, + "altitude": site_info.altitude, + "latitude": site_info.latitude, + "longitude": site_info.longitude, + } + + from mwrpy.level1.write_lev1_nc import lev1_to_nc + mwr_raw = lev1_to_nc( + data_type="1C01", + path_to_files=data_path, + data_format="cloudnet", + instrument_type=instrument_type, + output_file="mwr_1c.nc", + coeff_files=retrieval_files, + instrument_config=site_meta, + ) + +For plotting, the instrument needs to be defined. In this example, the figure is only displayed and not saved. + +.. code-block:: python + + from mwrpy.plots.generate_plots import generate_figure + fig_name = generate_figure('mwr_1c.nc', ['tb'], show=True, instrument_type=instrument_type) + +Process and plot Level 2 data (single & multiple pointing) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The site name is set to ``None``, since no site specific information file is needed. Again, no plots are saved, only +displayed. + +.. code-block:: python + + from mwrpy.level2.lev2_collocated import generate_lev2_single + mwr_prod = generate_lev2_single( + site=None, + data_format="cloudnet", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-single.nc", + coeff_files=retrieval_files, + ) + + from mwrpy.plots.generate_plots import generate_figure + fig_name = generate_figure('mwr-single.nc', ['iwv'], show=True, instrument_type=instrument_type) + + from mwrpy.level2.lev2_collocated import generate_lev2_multi + mwr_prod = generate_lev2_multi( + site=None, + data_format="cloudnet", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-multi.nc", + coeff_files=retrieval_files, + ) + + from mwrpy.plots.generate_plots import generate_figure + fig_name = generate_figure('mwr-multi.nc', ['temperature'], show=True, instrument_type=instrument_type) From f3b826cfcef8732900fe6c93732854d113596a14 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 12 Nov 2025 14:16:14 +0100 Subject: [PATCH 04/28] Fixed reading in l1/spc files and time for Cloudnet format --- mwrpy/level1/write_lev1_nc.py | 2 +- mwrpy/level2/write_lev2_nc.py | 5 ++++- mwrpy/plots/generate_plots.py | 17 ++++++++++++----- mwrpy/plots/plot_utils.py | 6 +++++- mwrpy/process_mwrpy.py | 12 +++++++++--- mwrpy/rpg_mwr.py | 2 +- 6 files changed, 32 insertions(+), 12 deletions(-) diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index d90d7aa..627a29d 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -110,7 +110,7 @@ def lev1_to_nc( c_files = ( get_coeff_list( site, - ["spc", "ins"], + ["spc", "ins", "tbx"], None, params.get("coeff_path", None), ) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index e22f81f..69e1c58 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -87,7 +87,10 @@ def lev2_to_nc( ) _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) _del_att(global_attributes) - mwr = rpg_mwr.Rpg(rpg_dat) + mwr = rpg_mwr.Rpg( + rpg_dat, + date=num2pydate(lev1.variables["time"][:][0], lev1.variables["time"].units), + ) mwr.data = get_data_attributes(mwr.data, data_type, coeff, data_format) if data_format == "cloudnet": c_files = ( diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index aca5f81..5df2449 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -594,7 +594,12 @@ def _plot_colormesh_data( "potential_temperature", "equivalent_potential_temperature", ): - hum_time = seconds2hours(read_nc_fields(hum_file, "time")) + hum_time = read_nc_fields(hum_file, "time") + hum_time = ( + seconds2hours(read_nc_fields(hum_file, "time")) + if hum_time.max() > 24 + else hum_time + ) hum_flag = _get_ret_flag( hum_file, hum_time, "absolute_humidity", instrument_type=instrument_type ) @@ -738,7 +743,7 @@ def _plot_instrument_data( elif product == "sen": _plot_sen(ax, data, name, time, nc_file) elif product == "hkd": - _plot_hkd(ax, data, name, time) + _plot_hkd(ax, data, name, time, nc_file) pos = ax.get_position() ax.set_position([pos.x0, pos.y0, pos.width * 0.965, pos.height]) @@ -746,9 +751,11 @@ def _plot_instrument_data( return fig -def _plot_hkd(ax, data_in: ndarray, name: str, time: ndarray): +def _plot_hkd(ax, data_in: ndarray, name: str, time: ndarray, nc_file: str): """Plot for housekeeping data.""" time = _nan_time_gaps(time) + pointing_flag = read_nc_fields(nc_file, "pointing_flag") + data_in[pointing_flag == 1, :] = np.nan if name == "t_amb": data_in[data_in == -999.0] = np.nan if (data_in[:, 0].all() is ma.masked) | (data_in[:, 1].all() is ma.masked): @@ -1743,7 +1750,7 @@ def _plot_scan( fig.subplots_adjust(hspace=0.09) case_date = _read_date(nc_file) axt, ax1 = 0, 0 - data_g = None + var_pl = None for ind in range(len(angles)): ele_range = (angles[ind] - 1.0, angles[ind] + 1.0) elevation_f = _elevation_filter(nc_file, elevation, ele_range=ele_range) @@ -1873,7 +1880,7 @@ def _plot_scan( colorbar.set_label("scan deviation (" + clab + ")", fontsize=13) axi[ip].yaxis.set_tick_params(labelbottom=False) - if data_g is None: + if var_pl is None: ax.set_title("empty") else: axp = axs[axt, :] if len(angles) > 1 else axs diff --git a/mwrpy/plots/plot_utils.py b/mwrpy/plots/plot_utils.py index de03fe4..dbe17a7 100644 --- a/mwrpy/plots/plot_utils.py +++ b/mwrpy/plots/plot_utils.py @@ -28,7 +28,11 @@ def _get_ret_flag( """Returns quality flag for frequencies used in retrieval.""" file = netCDF4.Dataset(nc_file) quality_flag = file.variables[variable + "_quality_flag"] - time_variable = seconds2hours(file.variables["time"][:]) + time_variable = ( + seconds2hours(file.variables["time"][:]) + if np.max(file.variables["time"]) > 24 + else file.variables["time"][:] + ) _, index, _ = np.intersect1d( time_variable, time, assume_unique=True, return_indices=True ) diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index c2e1a5c..3c2929b 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -203,6 +203,12 @@ def process_product( csv_off["date"] == xday[1].strftime("%m-%d"), "offset" ].values[0] + l1_filename = ( + _get_filename("1C01", date, site) + if data_format == "e-profile" + else _get_filename_cloudnet("1C01", date, site, instrument) + ) + if prod[0] == "1": lev1_to_nc( prod, @@ -240,7 +246,7 @@ def process_product( generate_lev2_single( site, data_format, - _get_filename("1C01", date, site), + l1_filename, output_file, lwp_offset, None, @@ -250,7 +256,7 @@ def process_product( generate_lev2_lhumpro( site, data_format, - _get_filename("1C01", date, site), + l1_filename, output_file, lwp_offset, None, @@ -260,7 +266,7 @@ def process_product( generate_lev2_multi( site, data_format, - _get_filename("1C01", date, site), + l1_filename, output_file, None, instrument, diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 0f9d049..17854bc 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -142,7 +142,7 @@ def add_zenith_angle(self): def convert_time_to_hours(self): """Converts time from seconds since epoch to hours since midnight.""" time = self.data["time"].data[:] - time_hours = seconds2hours(time) + time_hours = seconds2hours(time) if time.max() > 24 else time self.data["time"] = RpgArray( time_hours, "time", From 3ac67e4474deeed8acc4b88255e5777511189c3e Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 1 Jul 2026 16:38:51 +0200 Subject: [PATCH 05/28] Fix test --- tests/test_write_lev1_nc.py | 5 +++-- tests/test_write_lev2_nc.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/test_write_lev1_nc.py b/tests/test_write_lev1_nc.py index 2571427..ad8654a 100644 --- a/tests/test_write_lev1_nc.py +++ b/tests/test_write_lev1_nc.py @@ -12,11 +12,12 @@ DATE = "2023-04-06" site = "hyytiala" product_list = ["1B01", "1B11", "1B21", "1C01"] +DATA_FORMAT = "e-profile" def test_lev1_to_nc(): for prod in product_list: - hatpro = lev1_to_nc(prod, DATA_DIR, site) + hatpro = lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site) assert str(hatpro.date) == DATE for t in hatpro.data["time"][:]: date = str( @@ -28,7 +29,7 @@ def test_lev1_to_nc(): def test_output_nc_file(): for prod in product_list: temp_file = "temp_file.nc" - lev1_to_nc(prod, DATA_DIR, site, output_file=temp_file) + lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site, output_file=temp_file) with netCDF4.Dataset(temp_file) as nc: # Write tests for the created netCDF file here: assert nc.date == DATE diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index 525cc48..ac2fbdb 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -20,7 +20,7 @@ def l1_file(request): fd, path = tempfile.mkstemp() os.close(fd) - lev1_to_nc("1C01", DATA_DIR, SITE, path) + lev1_to_nc("1C01", DATA_DIR, DATA_FORMAT, SITE, path) def delete_file(): os.unlink(path) @@ -39,7 +39,14 @@ def test_generate_lev2_single_site(l1_file): def test_generate_lev2_single_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_single( + None, + DATA_FORMAT, + l1_file, + path, + coeff_files=COEFF_FILES, + instrument_type="hatpro", + ) os.unlink(path) @@ -53,4 +60,11 @@ def test_generate_lev2_multi_site(l1_file): def test_generate_lev2_multi_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_multi( + None, + DATA_FORMAT, + l1_file, + path, + coeff_files=COEFF_FILES, + instrument_type="hatpro", + ) From 393040cb4d5cc6c551c03c98ab5d257bd46c73d6 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 1 Jul 2026 16:44:16 +0200 Subject: [PATCH 06/28] Fix type in plotting function --- mwrpy/plots/generate_plots.py | 48 +++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index 5df2449..7004079 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -191,47 +191,51 @@ def generate_figure( def _mark_gaps( time: ndarray, data: ma.MaskedArray, + min_x: float = 0, + max_x: float = 24, max_allowed_gap: float = 1, ) -> tuple: - """Mark gaps in time and data.""" - assert time[0] >= 0 - assert time[-1] <= 24 - max_gap = max_allowed_gap / 60 + if time[0] < min_x or time[-1] > max_x: + msg = f"x-axis values outside the range {min_x}-{max_x}." + raise ValueError(msg) + max_gap_fraction_hour = max_allowed_gap / 60 + + gap_indices = np.where(np.diff(time) > max_gap_fraction_hour)[0] + if not ma.is_masked(data): - mask_new = np.zeros(data.shape) + mask_new = np.zeros(data.shape, dtype=np.int32) elif ma.all(data.mask) is ma.masked: - mask_new = np.ones(data.shape) + mask_new = np.ones(data.shape, dtype=np.int32) else: mask_new = np.copy(data.mask) data_new = ma.copy(data) time_new = np.copy(time) - gap_indices = np.where(np.diff(time) > max_gap)[0] if data.ndim == 2: temp_array = np.zeros((2, data.shape[1])) temp_mask = np.ones((2, data.shape[1])) else: temp_array = np.zeros((2, 1)) temp_mask = np.ones((2, 1)) - time_delta = 0.0 - ind: np.int32 | np.int64 + time_delta = 0.001 for ind in np.sort(gap_indices)[::-1]: - ind += 1 - data_new = np.insert(data_new, ind, temp_array, axis=0) - mask_new = np.insert(mask_new, ind, temp_mask, axis=0) - time_new = np.insert(time_new, ind, time[ind] - time_delta) - time_new = np.insert(time_new, ind, time[ind - 1] + time_delta) - if (time[0] - 0) > max_gap: + ind_gap = ind + 1 + data_new = np.insert(data_new, ind_gap, temp_array, axis=0) + mask_new = np.insert(mask_new, ind_gap, temp_mask, axis=0) + time_new = np.insert(time_new, ind_gap, time[ind_gap] - time_delta) + time_new = np.insert(time_new, ind_gap, time[ind_gap - 1] + time_delta) + if (time[0] - min_x) > max_gap_fraction_hour: data_new = np.insert(data_new, 0, temp_array, axis=0) mask_new = np.insert(mask_new, 0, temp_mask, axis=0) time_new = np.insert(time_new, 0, time[0] - time_delta) time_new = np.insert(time_new, 0, time_delta) - if (24 - time[-1]) > max_gap: - ind = np.int32(len(mask_new.shape)) - data_new = np.insert(data_new, ind, temp_array, axis=0) - mask_new = np.insert(mask_new, ind, temp_mask, axis=0) - time_new = np.insert(time_new, ind, 24 - time_delta) - time_new = np.insert(time_new, ind, time[-1] + time_delta) - data_new.mask = mask_new + if (max_x - time[-1]) > max_gap_fraction_hour: + ind_gap = np.int32(len(mask_new)) + data_new = np.insert(data_new, ind_gap, temp_array, axis=0) + mask_new = np.insert(mask_new, ind_gap, temp_mask, axis=0) + time_new = np.insert(time_new, ind_gap, max_x - time_delta) + time_new = np.insert(time_new, ind_gap, time[-1] + time_delta) + data_new[mask_new] = ma.masked + data_new[data_new == 0.0] = ma.masked return time_new, data_new From 3ba076927b3b0ae109edde69b863bd882622f2ba Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 13:14:29 +0200 Subject: [PATCH 07/28] Implement final E-Profile data format (Level 2) --- mwrpy/level1/lev1_meta_nc.py | 46 +++++++++++++-- mwrpy/level1/quality_control.py | 4 +- mwrpy/level1/write_lev1_nc.py | 5 ++ mwrpy/level2/lev2_collocated.py | 52 ++++++++++++----- mwrpy/level2/lev2_meta_nc.py | 73 ++++++++---------------- mwrpy/level2/write_lev2_nc.py | 48 +++++++++++++--- mwrpy/rpg_mwr.py | 17 +++--- mwrpy/site_config/hatpro.yaml | 3 + mwrpy/site_config/hyytiala/config.yaml | 3 + mwrpy/site_config/juelich/config.yaml | 3 + mwrpy/site_config/lindenberg/config.yaml | 3 + mwrpy/site_config/palaiseau/config.yaml | 3 + mwrpy/utils.py | 17 +++++- 13 files changed, 185 insertions(+), 92 deletions(-) diff --git a/mwrpy/level1/lev1_meta_nc.py b/mwrpy/level1/lev1_meta_nc.py index e83371b..da15844 100644 --- a/mwrpy/level1/lev1_meta_nc.py +++ b/mwrpy/level1/lev1_meta_nc.py @@ -37,8 +37,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - if data_type in ("1B01", "1B11", "1B21"): read_att = att_reader[data_type] attributes = dict(ATTRIBUTES_COM, **read_att) - - elif data_type == "1C01": + else: attributes = dict( ATTRIBUTES_COM, **ATTRIBUTES_1B01, **ATTRIBUTES_1B11, **ATTRIBUTES_1B21 ) @@ -46,6 +45,13 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - attributes.pop("time") attributes = dict(ATTRIBUTES_CN, **attributes) + if data_format == "e-profile": + keys = ["latitude", "longitude", "altitude", "height"] + for key in keys: + if key in attributes: + attributes.pop(key) + attributes = dict(ATTRIBUTES_EP, **attributes) + for key in list(rpg_variables): if key in attributes: rpg_variables[key].set_attributes(attributes[key]) @@ -62,16 +68,44 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - ATTRIBUTES_CN = { "time": MetaData( + comment="Time indication of samples is at end of integration-time", units="hours since ", long_name="Time UTC", standard_name="time", - axis="T", calendar="standard", dimensions=("time",), ), } +ATTRIBUTES_EP = { + "station_latitude": MetaData( + long_name="Latitude of measurement station", + standard_name="latitude", + units="degree_north", + dimensions=("time",), + ), + "station_longitude": MetaData( + long_name="Longitude of measurement station", + standard_name="longitude", + units="degree_east", + dimensions=("time",), + ), + "station_altitude": MetaData( + long_name="Altitude above mean sea level of measurement station", + standard_name="altitude", + units="m", + dimensions=("time",), + ), + "altitude": MetaData( + long_name="Height above mean sea level", + standard_name="height_above_mean_sea_level", + units="m", + dimensions=("altitude",), + ), +} + + ATTRIBUTES_COM = { "time": MetaData( long_name="Time (UTC) of the measurement", @@ -105,7 +139,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - } -DEFINITIONS_1B01 = { +DEFINITIONS_QF = { "quality_flag": ( "\n" "Bit 1: missing_tb\n" @@ -230,7 +264,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - "quality_flag": MetaData( long_name="Quality flag", units="1", - definition=DEFINITIONS_1B01["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time", "frequency"), @@ -238,7 +272,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - "quality_flag_status": MetaData( long_name="Quality flag status", units="1", - definition=DEFINITIONS_1B01["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time", "frequency"), diff --git a/mwrpy/level1/quality_control.py b/mwrpy/level1/quality_control.py index 2457b8c..0007b6d 100644 --- a/mwrpy/level1/quality_control.py +++ b/mwrpy/level1/quality_control.py @@ -118,8 +118,8 @@ def orbpos(data: dict, params: dict) -> np.ndarray: for t in data["time"] ] ) - lat = data["latitude"] - lng = data["longitude"] + lat = data["latitude"] if "latitude" in data else data["station_latitude"] + lng = data["longitude"] if "longitude" in data else data["station_longitude"] sol = suncalc.get_position(time, lat=lat, lng=lng) lun = suncalc.suncalc.getMoonPosition(time, lat=lat, lng=lng) sun = { diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 627a29d..4f33914 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -98,6 +98,11 @@ def lev1_to_nc( azimuth_offset, ) + if data_format == "e-profile": + keys = ["altitude", "latitude", "longitude"] + for key in keys: + rpg_bin.data[f"station_{key}"] = rpg_bin.data.pop(key) + if data_type in ("1B01", "1C01"): apply_qc(site, rpg_bin, params, coeff_files) if data_type in ("1B21", "1C01"): diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 0af230e..768ce56 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -68,10 +68,16 @@ def generate_lev2_single( netCDF4.Dataset(t_pot_file.name, "r") as nc_t_pot, netCDF4.Dataset(eq_temp_file.name, "r") as nc_eq_temp, ): - nc_output.createDimension("height", len(nc_t_prof.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) if data_format == "e-profile": + nc_output.createDimension( + "altitude", len(nc_t_prof.variables["altitude"][:]) + ) nc_output.createDimension("bnds", 2) + else: + nc_output.createDimension( + "height", len(nc_t_prof.variables["height"][:]) + ) for source, variables in ( ( @@ -100,7 +106,7 @@ def generate_lev2_single( "temperature", "temperature_random_error", "temperature_systematic_error", - "height", + "height" if data_format == "cloudnet" else "altitude", "temperature_quality_flag", "temperature_quality_flag_status", ), @@ -110,9 +116,11 @@ def generate_lev2_single( ( "time", "time_bnds", - "latitude", - "longitude", - "altitude", + "latitude" if data_format == "cloudnet" else "station_latitude", + "longitude" + if data_format == "cloudnet" + else "station_longitude", + "altitude" if data_format == "cloudnet" else "station_altitude", "lwp", "lwp_offset", "lwp_random_error", @@ -234,10 +242,16 @@ def generate_lev2_lhumpro( netCDF4.Dataset(iwv_file.name, "r") as nc_iwv, netCDF4.Dataset(abs_hum_file.name, "r") as nc_abs_hum, ): - nc_output.createDimension("height", len(nc_abs_hum.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) if data_format == "e-profile": + nc_output.createDimension( + "altitude", len(nc_abs_hum.variables["altitude"][:]) + ) nc_output.createDimension("bnds", 2) + else: + nc_output.createDimension( + "height", len(nc_abs_hum.variables["height"][:]) + ) for source, variables in ( ( @@ -253,7 +267,7 @@ def generate_lev2_lhumpro( ( nc_abs_hum, ( - "height", + "height" if data_format == "cloudnet" else "altitude", "absolute_humidity", "absolute_humidity_random_error", "absolute_humidity_systematic_error", @@ -266,9 +280,11 @@ def generate_lev2_lhumpro( ( "time", "time_bnds", - "latitude", - "longitude", - "altitude", + "latitude" if data_format == "cloudnet" else "station_latitude", + "longitude" + if data_format == "cloudnet" + else "station_longitude", + "altitude" if data_format == "cloudnet" else "station_altitude", "lwp", "lwp_offset", "lwp_random_error", @@ -336,9 +352,13 @@ def generate_lev2_multi( netCDF4.Dataset(eq_temp_file.name, "r") as nc_eq_temp, ): nc_output.createDimension("time", len(nc_temp.variables["time"][:])) - nc_output.createDimension("height", len(nc_temp.variables["height"][:])) if data_format == "e-profile": + nc_output.createDimension( + "altitude", len(nc_temp.variables["altitude"][:]) + ) nc_output.createDimension("bnds", 2) + else: + nc_output.createDimension("height", len(nc_temp.variables["height"][:])) for source, variables in ( ( @@ -346,10 +366,12 @@ def generate_lev2_multi( ( "time", "time_bnds", - "height", - "latitude", - "longitude", - "altitude", + "height" if data_format == "cloudnet" else "altitude", + "latitude" if data_format == "cloudnet" else "station_latitude", + "longitude" + if data_format == "cloudnet" + else "station_longitude", + "altitude" if data_format == "cloudnet" else "station_altitude", "elevation_angle", "azimuth_angle", "temperature", diff --git a/mwrpy/level2/lev2_meta_nc.py b/mwrpy/level2/lev2_meta_nc.py index 7961447..384b4f8 100644 --- a/mwrpy/level2/lev2_meta_nc.py +++ b/mwrpy/level2/lev2_meta_nc.py @@ -3,6 +3,7 @@ from collections.abc import Callable from typing import TypeAlias +from mwrpy.level1.lev1_meta_nc import ATTRIBUTES_CN, ATTRIBUTES_EP, DEFINITIONS_QF from mwrpy.utils import MetaData @@ -52,6 +53,14 @@ def get_data_attributes( read_att = att_reader[data_type] attributes = dict(ATTRIBUTES_COM, **read_att) + if data_format == "e-profile": + keys = ["latitude", "longitude", "altitude", "height"] + for key in keys: + if key in attributes: + attributes.pop(key) + attributes = dict(ATTRIBUTES_EP, **attributes) + if "altitude" in rpg_variables: + rpg_variables["altitude"].set_attributes(attributes["altitude"]) for key in list(rpg_variables): if key in attributes: if getattr(attributes[key], "retrieval_type") is not None: @@ -61,6 +70,9 @@ def get_data_attributes( **{field: coeff[field]} ) rpg_variables[key].set_attributes(attributes[key]) + if data_format == "e-profile": + if getattr(attributes[key], "dimensions") == ("time", "height"): + setattr(rpg_variables[key], "dimensions", ("time", "altitude")) else: del rpg_variables[key] @@ -76,43 +88,6 @@ def get_data_attributes( return rpg_variables -ATTRIBUTES_CN = { - "time": MetaData( - comment="Time indication of samples is at end of integration-time", - units="hours since ", - long_name="Time UTC", - standard_name="time", - calendar="standard", - dimensions=("time",), - ), -} - -DEFINITIONS_COM = { - "quality_flag": ( - "\n" - "Bit 1: missing_tb\n" - "Bit 2: tb_below_threshold\n" - "Bit 3: tb_above_threshold\n" - "Bit 4: spectral_consistency_above_threshold\n" - "Bit 5: receiver_sanity_failed\n" - "Bit 6: rain_detected\n" - "Bit 7: sun_moon_in_beam\n" - "Bit 8: tb_offset_above_threshold" - ), - "quality_flag_status": ( - "\n" - "Bit 1: missing_tb_not_checked\n" - "Bit 2: tb_lower_threshold_not_checked\n" - "Bit 3: tb_upper_threshold_not_checked\n" - "Bit 4: spectral_consistency_not_checked\n" - "Bit 5: receiver_sanity_not_checked\n" - "Bit 6: rain_not_checked\n" - "Bit 7: sun_moon_in_beam_not_checked\n" - "Bit 8: tb_offset_not_checked" - ), -} - - ATTRIBUTES_COM = { "time": MetaData( long_name="Time (UTC) of the measurement", @@ -189,7 +164,7 @@ def get_data_attributes( "temperature_quality_flag": MetaData( long_name="Temperature quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -197,7 +172,7 @@ def get_data_attributes( "temperature_quality_flag_status": MetaData( long_name="Temperature quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -238,7 +213,7 @@ def get_data_attributes( "temperature_quality_flag": MetaData( long_name="Temperature quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -246,7 +221,7 @@ def get_data_attributes( "temperature_quality_flag_status": MetaData( long_name="Temperature quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -280,7 +255,7 @@ def get_data_attributes( "absolute_humidity_quality_flag": MetaData( long_name="Absolute humidity quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -288,7 +263,7 @@ def get_data_attributes( "absolute_humidity_quality_flag_status": MetaData( long_name="Absolute humidity quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -411,7 +386,7 @@ def get_data_attributes( "lwp_quality_flag": MetaData( long_name="Liquid water path quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -419,7 +394,7 @@ def get_data_attributes( "lwp_quality_flag_status": MetaData( long_name="Liquid water path quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -448,7 +423,7 @@ def get_data_attributes( "iwv_quality_flag": MetaData( long_name="Integrated water vapour quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -456,7 +431,7 @@ def get_data_attributes( "iwv_quality_flag_status": MetaData( long_name="Integrated water vapour quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -491,7 +466,7 @@ def get_data_attributes( "stability_quality_flag": MetaData( long_name="Quality flag for stability products", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -499,7 +474,7 @@ def get_data_attributes( "stability_quality_flag_status": MetaData( long_name="Quality flag status for stability products", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 69e1c58..bc3122a 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -73,7 +73,11 @@ def lev2_to_nc( params = read_config(site, instrument_type, "params") with nc.Dataset(lev1_file) as lev1: - params["altitude"] = ma.median(lev1.variables["altitude"][:]) + params["altitude"] = ( + ma.median(lev1.variables["altitude"][:]) + if data_format == "cloudnet" + else ma.median(lev1.variables["station_altitude"][:]) + ) rpg_dat, coeff, index, scan_time = get_products( site, @@ -87,6 +91,8 @@ def lev2_to_nc( ) _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) _del_att(global_attributes) + if data_format == "e-profile" and "height" in rpg_dat: + rpg_dat["altitude"] = rpg_dat.pop("height") mwr = rpg_mwr.Rpg( rpg_dat, date=num2pydate(lev1.variables["time"][:][0], lev1.variables["time"].units), @@ -110,6 +116,9 @@ def lev2_to_nc( "instrument": instrument_type, "coeff_files": c_files, } + else: + global_attributes["dependencies"] = str(lev1_file).split("/")[-1] + global_attributes["level1_quality_flag_status"] = str(params["flag_status"]) rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) @@ -420,7 +429,7 @@ def get_products( ibl, tb, scan_time = ( np.empty([0, len(coeff["AG"])], np.int32), ma.masked_all((len(freq_ind), len(coeff["AG"]), 0), np.float32), - np.empty([0], np.int32), + np.ma.empty([0], np.int32), ) for ix0v in ix0: @@ -446,7 +455,7 @@ def get_products( tb = np.concatenate( ( tb, - np.expand_dims( + np.ma.expand_dims( lev1["tb"][np.ix_(ix0v + np.flip(ind_ang), freq_ind)].T, 2 ), ), @@ -531,8 +540,18 @@ def get_products( hum_time = _read_time(hum_dat.variables["time"]) tem_time = _read_time(tem_dat.variables["time"]) + hum_height = ( + hum_dat.variables["height"][:] + if "height" in hum_dat.variables + else hum_dat.variables["altitude"][:] + ) + tem_height = ( + tem_dat.variables["height"][:] + if "height" in tem_dat.variables + else tem_dat.variables["altitude"][:] + ) - if len(hum_dat.variables["height"][:]) == len(tem_dat.variables["height"][:]): + if len(hum_height) == len(tem_height): hum_int = interpol_2d( hum_time, hum_dat.variables["absolute_humidity"][:, :], @@ -541,13 +560,13 @@ def get_products( else: hum_int = interpolate_2d( hum_time, - hum_dat.variables["height"][:], + hum_height, hum_dat.variables["absolute_humidity"][:, :], tem_time, - tem_dat.variables["height"][:], + tem_height, ) - rpg_dat["height"] = tem_dat.variables["height"][:] + rpg_dat["height"] = tem_height pres = np.interp(tem_time, lev1["time"][:], lev1["air_pressure"][:]) if data_type == "2P04": rpg_dat["relative_humidity"] = rel_hum( @@ -628,6 +647,9 @@ def _combine_lev1( "altitude", "latitude", "longitude", + "station_altitude", + "station_latitude", + "station_longitude", ] if index.any(): for ivars in lev1_vars: @@ -686,8 +708,16 @@ def retrieval_input(lev1: dict, coeff: dict) -> np.ndarray: ) bias = np.ones((len(lev1["time"][:]), 1), np.float32) - latitude = float(ma.median(lev1["latitude"])) - longitude = float(ma.median(lev1["longitude"])) + latitude = ( + float(ma.median(lev1["latitude"])) + if "latitude" in lev1 + else float(ma.median(lev1["station_latitude"])) + ) + longitude = ( + float(ma.median(lev1["longitude"])) + if "longitude" in lev1 + else float(ma.median(lev1["station_longitude"])) + ) if coeff["RT"] == -1: ret_in = lev1["tb"][:, :] diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 17854bc..d2a9b64 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -215,7 +215,9 @@ def save_rpg( dims = { "time": len(rpg.data["time"][:]), "bnds": 2, - "height": len(rpg.data["height"][:]), + "height": len(rpg.data["height"][:]) + if data_format == "cloudnet" + else len(rpg.data["altitude"][:]), } elif data_type in ("2I01", "2I02", "2I06"): dims = {"time": len(rpg.data["time"][:]), "bnds": 2} @@ -286,19 +288,16 @@ def _write_vars2nc(nc_file: netCDF4.Dataset, mwr_variables: dict) -> None: def _add_standard_global_attributes(nc_file: netCDF4.Dataset, att_global) -> None: - nc_file.mwrpy_version = version.__version__ - nc_file.processed = ( - datetime.datetime.now(tz=datetime.timezone.utc).strftime("%d %b %Y %H:%M:%S") - + " UTC" - ) for name, value in att_global.items(): + if name == "history": + value = f"{datetime.datetime.now(tz=datetime.timezone.utc).strftime('%d %b %Y %H:%M:%S')} UTC, mwrpy {version.__version__}" if value is None: value = "" setattr(nc_file, name, value) def _add_cloudnet_global_attributes( - nc_file: netCDF4.Dataset, add_global, data_type + nc_file: netCDF4.Dataset, add_global: dict, data_type: str ) -> None: t_zone = datetime.timezone.utc form = "%Y-%m-%d %H:%M:%S" @@ -333,4 +332,6 @@ def _add_cloudnet_global_attributes( if value is None: value = "" setattr(nc_file, name, value) - nc_file.mwrpy_coefficients = ", ".join(add_global["coeff_files"]) + nc_file.mwrpy_coefficients = ", ".join( + [file.split("/")[-1] for file in add_global["coeff_files"]] + ) diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index 42c6de6..2dea5fd 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -187,6 +187,9 @@ global_specs: # Logbook repair/replacement work performed instrument_history: + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: + # Manufacturer of the infrared radiometer ir_instrument_manufacturer: Heitronics diff --git a/mwrpy/site_config/hyytiala/config.yaml b/mwrpy/site_config/hyytiala/config.yaml index 9049c7c..780ca57 100644 --- a/mwrpy/site_config/hyytiala/config.yaml +++ b/mwrpy/site_config/hyytiala/config.yaml @@ -74,3 +74,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/site_config/juelich/config.yaml b/mwrpy/site_config/juelich/config.yaml index 9f8fd39..1a8b7f2 100644 --- a/mwrpy/site_config/juelich/config.yaml +++ b/mwrpy/site_config/juelich/config.yaml @@ -77,3 +77,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/site_config/lindenberg/config.yaml b/mwrpy/site_config/lindenberg/config.yaml index 9cdb7dd..f32afbc 100644 --- a/mwrpy/site_config/lindenberg/config.yaml +++ b/mwrpy/site_config/lindenberg/config.yaml @@ -74,3 +74,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/site_config/palaiseau/config.yaml b/mwrpy/site_config/palaiseau/config.yaml index d9e9273..c8a95fe 100644 --- a/mwrpy/site_config/palaiseau/config.yaml +++ b/mwrpy/site_config/palaiseau/config.yaml @@ -74,3 +74,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 46d7675..a2eb005 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -249,7 +249,7 @@ def add_interpol1d( interpolated_mask = ( np.interp(data0["time"], valid_time, valid_mask.astype(float)) < 0.5 ) - result = ma.masked_array(interpolated_values, mask=interpolated_mask) + result = np.ma.masked_array(interpolated_values, mask=interpolated_mask) interpolated_data = ( result if len(interpolated_data) == 0 @@ -607,8 +607,19 @@ def _get_filename(prod: str, date_in: datetime.date, site: str) -> str: data_out_dir = os.path.join( params["data_out"], f"level{level}", date_in.strftime("%Y/%m/%d") ) - wigos_id = global_attributes["wigos_station_id"] - filename = f"MWR_{prod}_{wigos_id}_{date_in.strftime('%Y%m%d')}.nc" + wigos_id = ( + global_attributes["wigos_station_id"] + if global_attributes["wigos_station_id"] is not None + else site + ) + instrument_id = ( + global_attributes["instrument_id"] + if global_attributes["instrument_id"] is not None + else "A" + ) + filename = ( + f"MWR_{prod}_{wigos_id}_{instrument_id}{date_in.strftime('%Y%m%d')}.nc" + ) return os.path.join(data_out_dir, filename) From 1cc4cca4240af4ca741ef8b4009c67b764c61360 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 13:23:33 +0200 Subject: [PATCH 08/28] Fix array type --- mwrpy/level2/write_lev2_nc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index bc3122a..1d93cee 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -452,11 +452,15 @@ def get_products( axis=0, ) ibl = np.append(ibl, [ix0v + np.flip(ind_ang)], axis=0) - tb = np.concatenate( + tb = np.ma.concatenate( ( tb, np.ma.expand_dims( - lev1["tb"][np.ix_(ix0v + np.flip(ind_ang), freq_ind)].T, 2 + np.ma.array( + lev1["tb"][np.ix_(ix0v + np.flip(ind_ang), freq_ind)].T, + np.float32, + ), + 2, ), ), axis=2, From 114a3551bafe56b9f9f21fb17e4f98d02a9b8388 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 13:51:46 +0200 Subject: [PATCH 09/28] Account for different height variable --- mwrpy/rpg_mwr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index d2a9b64..fb30c51 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -215,10 +215,12 @@ def save_rpg( dims = { "time": len(rpg.data["time"][:]), "bnds": 2, - "height": len(rpg.data["height"][:]) - if data_format == "cloudnet" - else len(rpg.data["altitude"][:]), } + dims = ( + dict(dims, **{"height": len(rpg.data["height"][:])}) + if data_format == "cloudnet" + else dict(dims, **{"altitude": len(rpg.data["altitude"][:])}) + ) elif data_type in ("2I01", "2I02", "2I06"): dims = {"time": len(rpg.data["time"][:]), "bnds": 2} elif data_type == "2S02": From 9bc402d48da7d93a92aa37ba398210ce7d4ecb02 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 14:40:07 +0200 Subject: [PATCH 10/28] Provide instrument type with site name --- mwrpy/level2/write_lev2_nc.py | 3 +++ mwrpy/utils.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 1d93cee..dbfa130 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -23,6 +23,7 @@ interpol_2d, interpolate_2d, read_config, + read_site_config_yaml, ) @@ -68,6 +69,8 @@ def lev2_to_nc( ): raise ValueError(f"Data type {data_type} not recognised") + if instrument_type is None and site is not None: + instrument_type = read_site_config_yaml(site)["type"] assert instrument_type is not None global_attributes = read_config(site, instrument_type, "global_specs") params = read_config(site, instrument_type, "params") diff --git a/mwrpy/utils.py b/mwrpy/utils.py index a2eb005..7add9be 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -360,14 +360,14 @@ def read_config( key: Literal["global_specs", "params"], ) -> dict: if site is not None: - itype = _read_site_config_yaml(site)["type"] + itype = read_site_config_yaml(site)["type"] elif instrument_type is not None: itype = instrument_type else: raise ValueError("site or instrument_type is required") data = _read_itype_config_yaml(itype)[key] if site is not None: - data.update(_read_site_config_yaml(site)[key]) + data.update(read_site_config_yaml(site)[key]) return data @@ -383,7 +383,7 @@ def _read_itype_config_yaml(itype: str) -> dict: return yaml.load(f, Loader=SafeLoader) -def _read_site_config_yaml(site: str) -> dict: +def read_site_config_yaml(site: str) -> dict: """Reads configuration file for specific site.""" dir_name = os.path.dirname(os.path.realpath(__file__)) site_file = os.path.join(dir_name, "site_config", site, "config.yaml") From 2531d757b867d0ae5b1c91b34f5b6c9e2f08db06 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 17 Sep 2025 12:43:00 +0200 Subject: [PATCH 11/28] Add Level 1 format option --- mwrpy/cli.py | 21 +++++++ mwrpy/level1/lev1_meta_nc.py | 45 +++++++++++++-- mwrpy/level1/met_quality_control.py | 7 ++- mwrpy/level1/quality_control.py | 16 ++++-- mwrpy/level1/write_lev1_nc.py | 55 ++++++++++++++---- mwrpy/level2/get_ret_coeff.py | 4 +- mwrpy/plots/generate_plots.py | 8 ++- mwrpy/plots/plot_utils.py | 2 +- mwrpy/process_mwrpy.py | 78 +++++++++++++++++++++----- mwrpy/rpg_mwr.py | 87 +++++++++++++++++++++++++++-- mwrpy/site_config/hatpro.yaml | 62 ++++++++++---------- mwrpy/site_config/lhatpro.yaml | 62 ++++++++++---------- mwrpy/site_config/lhumpro_u90.yaml | 62 ++++++++++---------- mwrpy/utils.py | 66 ++++++++++++++++------ 14 files changed, 419 insertions(+), 156 deletions(-) diff --git a/mwrpy/cli.py b/mwrpy/cli.py index bdaa070..97dee7a 100755 --- a/mwrpy/cli.py +++ b/mwrpy/cli.py @@ -65,6 +65,27 @@ def _parse_args(args): metavar="YYYY-MM-DD", help="Single date to be processed.", ) + group.add_argument( + "-i", + "--instrument", + type=str, + help="Instrument to be processed (hatpro, lhatpro, lhumpro_u90).", + default="hatpro", + ) + group.add_argument( + "-f", + "--format", + type=str, + help="Data format to be used (cloudnet, e-profile).", + default="e-profile", + ) + group.add_argument( + "-a", + "--altitude", + type=float, + help="Altitude above mean sea level of site (m).", + default=0.0, + ) return parser.parse_args(args) diff --git a/mwrpy/level1/lev1_meta_nc.py b/mwrpy/level1/lev1_meta_nc.py index b432fb8..eae7309 100644 --- a/mwrpy/level1/lev1_meta_nc.py +++ b/mwrpy/level1/lev1_meta_nc.py @@ -6,12 +6,13 @@ from mwrpy.utils import MetaData -def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: +def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) -> dict: """Adds Metadata for RPG MWR Level 1 variables for NetCDF file writing. Args: rpg_variables: RpgArray instances. data_type: Data type of the netCDF file. + data_format: Data format of the netCDF file (cloudnet, e-profile). Returns: Dictionary @@ -23,6 +24,16 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: from level1.lev1_meta_nc import get_data_attributes att = get_data_attributes('data','data_type') """ + if data_type not in ( + "1B01", + "1B11", + "1B21", + "1C01", + ): + raise RuntimeError( + ["Data type " + data_type + " not supported for file writing."] + ) + if data_type in ("1B01", "1B11", "1B21"): read_att = att_reader[data_type] attributes = dict(ATTRIBUTES_COM, **read_att) @@ -31,10 +42,9 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: attributes = dict( ATTRIBUTES_COM, **ATTRIBUTES_1B01, **ATTRIBUTES_1B11, **ATTRIBUTES_1B21 ) - else: - raise RuntimeError( - ["Data type " + data_type + " not supported for file writing."] - ) + if data_format == "cloudnet": + attributes.pop("time") + attributes = dict(ATTRIBUTES_CN, **attributes) for key in list(rpg_variables): if key in attributes: @@ -50,6 +60,18 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: return rpg_variables +ATTRIBUTES_CN = { + "time": MetaData( + units="hours since ", + long_name="Time UTC", + standard_name="time", + axis="T", + calendar="standard", + dimensions=("time",), + ), +} + + ATTRIBUTES_COM = { "time": MetaData( long_name="Time (UTC) of the measurement", @@ -179,6 +201,13 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: comment="0=horizon, 90=zenith", dimensions=("time",), ), + "zenith_angle": MetaData( + units="degree", + long_name="Zenith angle", + standard_name="zenith_angle", + comment="Angle to the local vertical. A value of zero is directly overhead.", + dimensions=("time",), + ), "tb_cov_amb": MetaData( long_name="Error covariance matrix of brightness temperature channels on ambient target.", units="K*K", @@ -290,6 +319,12 @@ def get_data_attributes(rpg_variables: dict, data_type: str) -> dict: comment="0=horizon, 90=zenith", dimensions=("time",), ), + "ir_zenith_angle": MetaData( + units="degree", + long_name="Infrared sensor elevation angle", + comment="90=horizon, 0=zenith", + dimensions=("time",), + ), } diff --git a/mwrpy/level1/met_quality_control.py b/mwrpy/level1/met_quality_control.py index aca726c..8cf7682 100644 --- a/mwrpy/level1/met_quality_control.py +++ b/mwrpy/level1/met_quality_control.py @@ -6,12 +6,13 @@ from mwrpy.utils import setbit -def apply_met_qc(data: dict, params: dict) -> None: +def apply_met_qc(data: dict, params: dict, altitude: float | None) -> None: """This function performs quality control of meteorological sensor data. Args: data: Level 1 data. params: Site specific parameters. + altitude: Altitude of the site in meters. Returns: None @@ -38,7 +39,9 @@ def apply_met_qc(data: dict, params: dict) -> None: if name not in data: continue if name == "air_pressure": - pressure = atmoslib.isa_pressure(params["altitude"]) + alt = params.get("altitude", altitude) + alt = ma.masked if alt is None else alt + pressure = atmoslib.isa_pressure(alt) threshold_low = pressure - 10000 threshold_high = pressure + 10000 else: diff --git a/mwrpy/level1/quality_control.py b/mwrpy/level1/quality_control.py index fcc65d8..f7d3fa6 100644 --- a/mwrpy/level1/quality_control.py +++ b/mwrpy/level1/quality_control.py @@ -74,7 +74,7 @@ def apply_qc( data["quality_flag_status"] = setbit(data["quality_flag_status"], 3) else: try: - ind = spectral_consistency(data, site, coeff_files) + ind = spectral_consistency(data, params, site, coeff_files) data["quality_flag"][ind] = setbit(data["quality_flag"][ind], 3) except MissingCoefficientsError as e: logging.error( @@ -184,7 +184,10 @@ def orbpos(data: dict, params: dict) -> np.ndarray: def spectral_consistency( - data: dict, site: str | None, coeff_files: Sequence[str | PathLike] | None + data: dict, + params: dict, + site: str | None, + coeff_files: Sequence[str | PathLike] | None, ) -> np.ndarray: """Applies spectral consistency coefficients for given frequency index, writes 2S02 product and returns indices to be flagged. @@ -205,11 +208,12 @@ def spectral_consistency( + 1.0 ) + coeff_dir = params.get("coeff_path", None) prefix = "ins" - c_list = get_coeff_list(site, prefix, coeff_files) + c_list = get_coeff_list(site, prefix, coeff_files, coeff_dir) if len(c_list) == 0: prefix = "spc" - c_list = get_coeff_list(site, prefix, coeff_files) + c_list = get_coeff_list(site, prefix, coeff_files, coeff_dir) if len(c_list) > 0: # pylint: disable=unbalanced-tuple-unpacking @@ -222,7 +226,7 @@ def spectral_consistency( weights1, weights2, factor, - ) = get_mvr_coeff(site, prefix, data["frequency"][:], coeff_files) + ) = get_mvr_coeff(site, prefix, data["frequency"][:], coeff_files, coeff_dir) ret_in = retrieval_input(data, coeff) ele_ind = np.where( (np.abs(data["elevation_angle"][:] - 90.0) < 0.5) @@ -344,7 +348,7 @@ def spectral_consistency( ] = True else: - c_list = get_coeff_list(site, "tbx", coeff_files) + c_list = get_coeff_list(site, "tbx", coeff_files, coeff_dir) if not c_list: raise MissingCoefficientsError("No coefficients found") diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index c544111..27bb6c2 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -20,6 +20,7 @@ from mwrpy.utils import ( add_interpol1d, add_time_bounds, + get_coeff_list, get_file_list, isbit, read_config, @@ -32,6 +33,7 @@ def lev1_to_nc( data_type: str, path_to_files: str | PathLike, + data_format: str, site: str | None = None, output_file: str | PathLike | None = None, lidar_path: str | PathLike | None = None, @@ -40,6 +42,7 @@ def lev1_to_nc( date: datetime.date | None = None, time_offset: datetime.timedelta | None = None, instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, + altitude: float | None = None, ) -> rpg_mwr.Rpg: """This function reads one day of RPG MWR binary files, adds attributes and writes it into netCDF file. @@ -47,6 +50,7 @@ def lev1_to_nc( Args: data_type: Data type of the netCDF file. path_to_files: Folder containing one day of RPG MWR binary files. + data_format: Data format of the netCDF file (cloudnet, e-profile). site: Name of site. output_file: Output file name. lidar_path: Path to (optional) lidar file @@ -55,6 +59,7 @@ def lev1_to_nc( date: Measurement date in UTC. time_offset: Time offset if instrument operated in local time. instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). + altitude: Altitude of the site in meters above mean sea level. Raises: MissingInputData: if required input file is missing. @@ -69,31 +74,54 @@ def lev1_to_nc( f"No coefficient files given, using files in repository for {site}." ) - if instrument_config is None: + if data_format == "e-profile" and instrument_config is None: logging.info( f"No instrument config given, using config file in repository for {site}." ) - params = read_config(site, instrument_type, "params") + params = ( + read_config(site, instrument_type, "params") + if data_format == "e-profile" + else read_config(None, instrument_type, "params") + ) if instrument_config is not None: params = {**params, **instrument_config} - rpg_bin = prepare_data(path_to_files, data_type, params, lidar_path, time_offset) + rpg_bin = prepare_data( + path_to_files, data_type, params, lidar_path, time_offset, altitude + ) assert isinstance(rpg_bin, RpgBin) if data_type in ("1B01", "1C01"): apply_qc(site, rpg_bin, params, coeff_files) if data_type in ("1B21", "1C01"): - apply_met_qc(rpg_bin.data, params) + apply_met_qc(rpg_bin.data, params, altitude) mwr = rpg_mwr.Rpg(rpg_bin.data, date) mwr.find_valid_times() - mwr.data = get_data_attributes(mwr.data, data_type) + mwr.data = get_data_attributes(mwr.data, data_type, data_format) if output_file is not None: - global_attributes = read_config(site, instrument_type, "global_specs") + if data_format == "cloudnet": + c_files = ( + get_coeff_list( + site, + ["spc", "ins", "lwp", "iwv", "hpt", "tpt", "tpb"], + None, + params.get("coeff_path", None), + ) + if (coeff_files is None) + else (coeff_files) + ) + global_attributes = { + "site": site, + "instrument": instrument_type, + "coeff_files": c_files, + } + else: + global_attributes = read_config(site, instrument_type, "global_specs") _update_calibration_attributes(rpg_bin, global_attributes) if data_type != "1C01": update_lev1_attributes(global_attributes, data_type) - rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type) + rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) return mwr @@ -103,6 +131,7 @@ def prepare_data( params: dict, lidar_path: str | PathLike | None, time_offset: datetime.timedelta | None = None, + altitude: float | None = None, date: float | None = None, ) -> RpgBin | dict: """Load and prepare data for netCDF writing.""" @@ -396,9 +425,9 @@ def prepare_data( file_list_hkd = get_file_list(path_to_files, "HKD") _append_hkd(file_list_hkd, rpg_bin, data_type, params, time_offset) - rpg_bin.data["altitude"] = ( - np.ones(len(rpg_bin.data["time"]), np.float32) * params["altitude"] - ) + alt = params.get("altitude", altitude) + alt = ma.masked if alt is None else alt + rpg_bin.data["altitude"] = np.ones(len(rpg_bin.data["time"]), np.float32) * alt return rpg_bin @@ -412,11 +441,13 @@ def _append_hkd( ) -> None: """Append hkd data on same time grid and perform TB sanity check.""" hkd = RpgBin(file_list_hkd, time_offset) + lat = params.get("latitude", ma.masked) + lon = params.get("longitude", ma.masked) if "latitude" not in hkd.data: add_interpol1d( rpg_bin.data, - np.ones(len(hkd.data["time"])) * params["latitude"], + np.ones(len(hkd.data["time"])) * lat, hkd.data["time"], "latitude", ) @@ -431,7 +462,7 @@ def _append_hkd( if "longitude" not in hkd.data: add_interpol1d( rpg_bin.data, - np.ones(len(hkd.data["time"])) * params["longitude"], + np.ones(len(hkd.data["time"])) * lon, hkd.data["time"], "longitude", ) diff --git a/mwrpy/level2/get_ret_coeff.py b/mwrpy/level2/get_ret_coeff.py index ec3c35a..cedca55 100644 --- a/mwrpy/level2/get_ret_coeff.py +++ b/mwrpy/level2/get_ret_coeff.py @@ -15,6 +15,7 @@ def get_mvr_coeff( prefix: str, freq: np.ndarray, coeff_files: Sequence[str | PathLike] | None, + coeff_dir: str | None = None, ): """This function extracts retrieval coefficients for given files. @@ -23,12 +24,13 @@ def get_mvr_coeff( prefix: Identifier for type of product. freq: Frequencies of observations. coeff_files: List of coefficient files. + coeff_dir: Directory where coefficient files are stored. Examples: >>> from mwrpy.level2.get_ret_coeff import get_mvr_coeff >>> get_mvr_coeff('site_name', 'lwp', np.array([22, 31.4])) """ - c_list = get_coeff_list(site, prefix, coeff_files) + c_list = get_coeff_list(site, prefix, coeff_files, coeff_dir) coeff: dict = {} diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index a2daf49..b67dcec 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -479,7 +479,7 @@ def _read_time_vector(nc_file: str) -> ndarray: """Converts time vector to fraction hour.""" with netCDF4.Dataset(nc_file) as nc: time = nc.variables["time"][:] - return seconds2hours(time) + return seconds2hours(time) if time.max() > 24 else time def _screen_high_altitudes(data_field: ndarray, ax_values: tuple, max_y: int) -> tuple: @@ -531,7 +531,11 @@ def _read_date(nc_file: str) -> date: """Returns measurement date.""" locale.setlocale(locale.LC_TIME, "en_US.UTF-8") with netCDF4.Dataset(nc_file) as nc: - case_date = datetime.strptime(nc.date, "%Y-%m-%d") + case_date = ( + datetime.strptime(nc.date, "%Y-%m-%d") + if "date" in nc.ncattrs() + else datetime.strptime(f"{nc.year}-{nc.month}-{nc.day}", "%Y-%m-%d") + ) return case_date diff --git a/mwrpy/plots/plot_utils.py b/mwrpy/plots/plot_utils.py index 79980b8..954b315 100644 --- a/mwrpy/plots/plot_utils.py +++ b/mwrpy/plots/plot_utils.py @@ -168,7 +168,7 @@ def _dir_avg( def _read_location(nc_file: str) -> str: """Returns site name.""" with netCDF4.Dataset(nc_file) as nc: - site_name = nc.site_location + site_name = nc.site_location if "site_location" in nc.ncattrs() else nc.location return site_name diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index 491304c..6f7c075 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -5,7 +5,9 @@ import logging import os import time +from typing import Literal +import matplotlib.pyplot as plt import netCDF4 as nc import pandas as pd @@ -19,7 +21,7 @@ from mwrpy.plots.generate_plots import generate_figure from mwrpy.utils import ( _get_filename, - _read_site_config_yaml, + _get_filename_cloudnet, date_range, get_processing_dates, isodate2date, @@ -93,6 +95,7 @@ "ko_index", ] ) +IType = Literal["hatpro", "lhatpro", "lhumpro_u90"] def main(args): @@ -107,27 +110,60 @@ def main(args): if product not in PRODUCT_NAME: logging.error(f"Product {product} not recognised") continue + if args.format == "cloudnet": + if product not in ("1C01", "single", "multi"): + logging.error( + f"Product {product} not available in cloudnet format. Skipping." + ) + continue + if args.altitude is None: + logging.info("Site altitude not provided. Taking default of 0 m.") start = time.process_time() if args.command != "plot": logging.info(f"Processing {product} product, {args.site} {date}") if args.command == "reprocess": try: - process_product(product, date, args.site) + process_product( + product, + date, + args.site, + args.format, + args.instrument, + args.altitude, + ) except Exception as e: logging.error( f"Error in processing products: {e}. Incomplete or no processing for {date}." ) else: - process_product(product, date, args.site) + process_product( + product, + date, + args.site, + args.format, + args.instrument, + args.altitude, + ) if args.command != "no-plot": logging.info(f"Plotting {product} product, {args.site} {date}") - plot_product(product, date, args.site) - + try: + plot_product(product, date, args.site, args.format, args.instrument) + except Exception as e: + logging.error(f"Error in plotting product {product}: {e}.") + finally: + plt.close() elapsed_time = time.process_time() - start logging.info(f"Processing took {elapsed_time:.1f} seconds") -def process_product(prod: str, date: datetime.date, site: str): +def process_product( + prod: str, + date: datetime.date, + site: str, + data_format: str, + instrument: IType, + altitude: float, +): """Process a given product for a specific date and site. This function handles the processing of different products based on their type (level 1, level 2, single, multi) and manages the necessary file @@ -137,11 +173,18 @@ def process_product(prod: str, date: datetime.date, site: str): prod: Product code (e.g., '1C01', '2I01', 'single', 'multi'). date: Date for which the product is to be processed. site: Site identifier. + data_format: Data format of the netCDF file (cloudnet, e-profile). + instrument: Specific instrument type (hatpro, lhatpro, etc.). + altitude: Altitude of the site in meters above mean sea level. Returns: None """ - output_file = _get_filename(prod, date, site) + output_file = ( + _get_filename(prod, date, site) + if data_format == "e-profile" + else _get_filename_cloudnet(prod, date, site, instrument) + ) output_dir = os.path.dirname(output_file) if not os.path.isdir(output_dir): os.makedirs(output_dir) @@ -179,16 +222,18 @@ def process_product(prod: str, date: datetime.date, site: str): ].values[0] lwp_offset_tuple = (lwp_offset[0], lwp_offset[1]) - itype = _read_site_config_yaml(site)["type"] # Process level 1 data if prod[0] == "1": lev1_to_nc( prod, _get_raw_file_path(date, site), + data_format, site=site, output_file=output_file, lidar_path=_get_lidar_file_path(date, site), date=date, + instrument_type=instrument, + altitude=altitude, ) # Process level 2 single products @@ -212,11 +257,11 @@ def process_product(prod: str, date: datetime.date, site: str): ) # Process level 2 combined products - elif prod == "single" and itype != "lhumpro_u90": + elif prod == "single" and instrument != "lhumpro_u90": generate_lev2_single( site, _get_filename("1C01", date, site), output_file, lwp_offset_tuple ) - elif itype == "lhumpro_u90": + elif instrument == "lhumpro_u90": generate_lev2_lhumpro( site, _get_filename("1C01", date, site), output_file, lwp_offset_tuple ) @@ -271,7 +316,7 @@ def process_product(prod: str, date: datetime.date, site: str): csv_off.to_csv(offset_current, index=False) -def plot_product(prod: str, date: datetime.date, site: str): +def plot_product(prod: str, date, site: str, data_format: str, instrument: IType): """Plot a given product for a specific date and site. Plotting covariance data without 1C01 file is supported. @@ -279,12 +324,19 @@ def plot_product(prod: str, date: datetime.date, site: str): prod: Product code (e.g., '1C01', '2I01', 'single', 'multi'). date: Date for which the product is to be plotted. site: Site identifier. + data_format: Data format of the netCDF file (cloudnet, e-profile). + instrument: Specific instrument type (hatpro, lhatpro, etc.). Returns: None """ - filename = _get_filename(prod, date, site) - params = read_config(site, None, "params") + filename = ( + _get_filename(prod, date, site) + if data_format == "e-profile" + else _get_filename_cloudnet(prod, date, site, instrument) + ) + if not os.path.isfile(filename): + logging.warning("Nothing to plot for product " + prod) output_dir = f"{os.path.dirname(filename)}/" # Plot level 1 data diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 7d966fe..4f0b67b 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -8,7 +8,7 @@ from numpy import ma from mwrpy import utils, version -from mwrpy.utils import MetaData +from mwrpy.utils import MetaData, seconds2hours class RpgArray: @@ -129,6 +129,29 @@ def find_valid_times(self): ind[time_i] = 1 self._screen(np.where(ind == 1)[0]) + def add_zenith_angle(self): + """Adds zenith angle to data if not present.""" + for key in ("elevation_angle", "ir_elevation_angle"): + if key not in self.data: + continue + zenith_angle = 90 - self.data[key][:] + new_key = key.replace("elevation", "zenith") + self.data[new_key] = RpgArray(zenith_angle, new_key) + self.data[new_key].dimensions = ("time",) + + def convert_time_to_hours(self): + """Converts time from seconds since epoch to hours since midnight.""" + time = self.data["time"].data[:] + time_hours = seconds2hours(time) + self.data["time"] = RpgArray( + time_hours, + "time", + "hours since " + self.date.strftime("%Y-%m-%d %H:%M:%S") + " +00:00", + ) + self.data["time"].dimensions = ("time",) + if "time_bnds" in self.data: + del self.data["time_bnds"] + def _screen(self, ind: np.ndarray): if len(ind) < 1: raise RuntimeError( @@ -144,8 +167,17 @@ def _screen(self, ind: np.ndarray): self.data[key].data = self.data[key].data[ind, :] -def save_rpg(rpg: Rpg, output_file: str | PathLike, att: dict, data_type: str) -> None: +def save_rpg( + rpg: Rpg, + output_file: str | PathLike, + att: dict, + data_type: str, + data_format: str = "e-profile", +) -> None: """Saves the RPG MWR file.""" + if data_format == "cloudnet": + Rpg.convert_time_to_hours(rpg) + Rpg.add_zenith_angle(rpg) if data_type == "1B01": dims = { "time": len(rpg.data["time"][:]), @@ -199,12 +231,22 @@ def save_rpg(rpg: Rpg, output_file: str | PathLike, att: dict, data_type: str) - ["Data type " + data_type + " not supported for file writing."] ) - with init_file(output_file, dims, rpg.data, att) as rootgrp: - setattr(rootgrp, "date", rpg.date.isoformat()) + with init_file(output_file, dims, rpg.data, att, data_format, data_type) as rootgrp: + if data_format == "e-profile": + setattr(rootgrp, "date", rpg.date.isoformat()) + else: + setattr(rootgrp, "year", rpg.date.strftime("%Y")) + setattr(rootgrp, "month", rpg.date.strftime("%m")) + setattr(rootgrp, "day", rpg.date.strftime("%d")) def init_file( - file_name: str | PathLike, dimensions: dict, rpg_arrays: dict, att_global: dict + file_name: str | PathLike, + dimensions: dict, + rpg_arrays: dict, + att_global: dict, + data_format: str, + data_type: str, ) -> netCDF4.Dataset: """Initializes an RPG MWR file for writing. @@ -213,12 +255,18 @@ def init_file( dimensions: Dictionary containing dimension for this file. rpg_arrays: Dictionary containing :class:`RpgArray` instances. att_global: Dictionary containing site specific global attributes + data_format: Data format to be used (cloudnet, e-profile). + data_type: Data type to be used (1B01, 1C01, 2I02, etc). """ nc_file = netCDF4.Dataset(file_name, "w", format="NETCDF4_CLASSIC") for key, dimension in dimensions.items(): nc_file.createDimension(key, dimension) _write_vars2nc(nc_file, rpg_arrays) - _add_standard_global_attributes(nc_file, att_global) + _add_cloudnet_global_attributes( + nc_file, att_global, data_type + ) if data_format == "cloudnet" else ( + _add_standard_global_attributes(nc_file, att_global) + ) return nc_file @@ -244,3 +292,30 @@ def _add_standard_global_attributes(nc_file: netCDF4.Dataset, att_global) -> Non if value is None: value = "" setattr(nc_file, name, value) + + +def _add_cloudnet_global_attributes( + nc_file: netCDF4.Dataset, add_global, data_type +) -> None: + t_zone = datetime.timezone.utc + form = "%Y-%m-%d %H:%M:%S" + instrument = add_global["instrument"].upper() + site = add_global["site"] + att_global = { + "Conventions": "CF-1.8", + "mwrpy_version": version.__version__, + "location": add_global["site"], + "source": f"RPG-Radiometer Physics {instrument}", + "references": "https://doi.org/10.21105/joss.06733", + "mwrpy_file_type": data_type, + "title": f"{instrument} microwave radiometer Level 1c from {site}", + "history": f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00" + + " - " + + data_type + + " file created", + } + for name, value in att_global.items(): + if value is None: + value = "" + setattr(nc_file, name, value) + nc_file.mwrpy_coefficients = ", ".join(add_global["coeff_files"]) diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index db4eff8..6f751eb 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -1,6 +1,38 @@ # Config file for all HATPRO instruments params: + # path to level1 data and path for processed files + data_in: /tmp/data/ + data_out: /tmp/data/ + + # path to retrieval coefficients + coeff_path: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. + # If you do not want to transform the coordinates set azi_cor to -999. + azi_cor: -999. + + const_azi: -999. + + # some default values: + # ------------------- receiver_nb: [1, 2] receiver: [1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2] @@ -71,36 +103,6 @@ params: [0., 100.], ] - # some default values: - # ------------------- - - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ - - # availability of IR - ir_flag: True - - # quality flag status for level 1 data; 0: flag active - # Bit 1: missing_tb - # Bit 2: tb_below_threshold - # Bit 3: tb_above_threshold - # Bit 4: spectral_consistency_above_threshold - # Bit 5: receiver_sanity_failed - # Bit 6: rain_detected - # Bit 7: sun_moon_in_beam - # Bit 8: tb_offset_above_threshold - flag_status: [0, 0, 0, 0, 0, 0, 0, 1] - - # integration time of measurements in seconds - int_time: 1 - - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - - const_azi: -999. - # Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index 1519571..d281370 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -1,6 +1,38 @@ # Config file for all LHATPRO instruments params: + # path to level1 data and path for processed files + data_in: /tmp/data/ + data_out: /tmp/data/ + + # path to retrieval coefficients + coeff_path: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. + # If you do not want to transform the coordinates set azi_cor to -999. + azi_cor: -999. + + const_azi: -999. + + # some default values: + # ------------------- receiver_nb: [2, 1] receiver: [2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1] @@ -69,36 +101,6 @@ params: [0., 100.], ] - # some default values: - # ------------------- - - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ - - # availability of IR - ir_flag: True - - # quality flag status for level 1 data; 0: flag active - # Bit 1: missing_tb - # Bit 2: tb_below_threshold - # Bit 3: tb_above_threshold - # Bit 4: spectral_consistency_above_threshold - # Bit 5: receiver_sanity_failed - # Bit 6: rain_detected - # Bit 7: sun_moon_in_beam - # Bit 8: tb_offset_above_threshold - flag_status: [0, 0, 0, 0, 0, 0, 0, 1] - - # integration time of measurements in seconds - int_time: 1 - - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - - const_azi: -999. - # Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index 779e5f8..6282bc1 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -1,6 +1,38 @@ # Config file for all LHUMPRO U90 instruments params: + # path to level1 data and path for processed files + data_in: /tmp/data/ + data_out: /tmp/data/ + + # path to retrieval coefficients + coeff_path: + + # availability of IR + ir_flag: False + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. + # If you do not want to transform the coordinates set azi_cor to -999. + azi_cor: -999. + + const_azi: -999. + + # some default values: + # ------------------- receiver_nb: [2, 1] receiver: [2, 1, 1, 1, 1, 1, 1] @@ -54,36 +86,6 @@ params: [0., 100.], ] - # some default values: - # ------------------- - - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ - - # availability of IR - ir_flag: False - - # quality flag status for level 1 data; 0: flag active - # Bit 1: missing_tb - # Bit 2: tb_below_threshold - # Bit 3: tb_above_threshold - # Bit 4: spectral_consistency_above_threshold - # Bit 5: receiver_sanity_failed - # Bit 6: rain_detected - # Bit 7: sun_moon_in_beam - # Bit 8: tb_offset_above_threshold - flag_status: [0, 0, 0, 0, 0, 0, 0, 1] - - # integration time of measurements in seconds - int_time: 1 - - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - - const_azi: -999. - # Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 70423a7..e163606 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -21,6 +21,7 @@ SECONDS_PER_HOUR = 3600 SECONDS_PER_DAY = 86400 Epoch = tuple[int, int, int] +IType = Literal["hatpro", "lhatpro", "lhumpro_u90"] class MetaData(NamedTuple): @@ -35,6 +36,8 @@ class MetaData(NamedTuple): retrieval_frequencies: str | None = None retrieval_auxiliary_input: str | None = None retrieval_description: str | None = None + axis: str | None = None + calendar: str | None = None def seconds2hours(time_in_seconds: np.ndarray) -> np.ndarray: @@ -276,10 +279,14 @@ def add_time_bounds(time_arr: np.ndarray, int_time: int) -> np.ndarray: def get_coeff_list( - site: str | None, prefix: str, coeff_files: Sequence[str | PathLike] | None + site: str | None, + prefix: str | list, + coeff_files: Sequence[str | PathLike] | None, + coeff_dir: str | None = None, ) -> list[str]: """Returns list of .nc coefficient file(s).""" if coeff_files is not None: + assert isinstance(prefix, str) c_list = [] for file in coeff_files: basename = os.path.basename(file) @@ -289,32 +296,38 @@ def get_coeff_list( return sorted(c_list) assert isinstance(site, str) - dir_path = os.path.dirname(os.path.realpath(__file__)) - c_list = glob.glob( - dir_path - + "/site_config/" - + site - + "/coefficients/" - + "*" - + prefix.lower() - + "*" - ) - if len(c_list) == 0: - c_list = glob.glob( - dir_path + if coeff_dir is not None: + dir_path = coeff_dir + else: + dir_path = ( + os.path.dirname(os.path.realpath(__file__)) + "/site_config/" + site + "/coefficients/" - + "*" - + prefix.upper() - + "*" ) + if isinstance(prefix, str): + prefix = [prefix] + c_list = [] + for p in prefix: + tmp = glob.glob(dir_path + "*" + p.lower() + "*") + if len(c_list) == 0: + tmp = glob.glob(dir_path + "*" + p.upper() + "*") + c_list = c_list + tmp if len(c_list) > 0: + if "spc" in c_list and "ins" in c_list: + c_list.remove("spc") return sorted(c_list) logging.warning( "No coefficient files for product " - + prefix + + str(prefix) + + " found in directory " + + "/site_config/" + + site + + "/coefficients/" + ) if len(prefix) == 1 else logging.warning( + "No coefficient files for products " + + ", ".join(prefix) + " found in directory " + "/site_config/" + site @@ -596,6 +609,23 @@ def _get_filename(prod: str, date_in: datetime.date, site: str) -> str: return os.path.join(data_out_dir, filename) +def _get_filename_cloudnet( + prod: str, date_in: datetime.date, site: str, instrument: IType +) -> str: + if np.char.isnumeric(prod[0]): + level = prod[0] + name = "l1c" + else: + level = "2" + name = prod + params = read_config(None, instrument, "params") + data_out_dir = os.path.join( + params["data_out"], f"level{level}", date_in.strftime("%Y/%m/%d") + ) + filename = f"{date_in.strftime('%Y%m%d')}_{site}_{instrument}-{name}.nc" + return os.path.join(data_out_dir, filename) + + def isodate2date(date_str: str) -> datetime.date: return datetime.datetime.strptime(date_str, "%Y-%m-%d").date() From 0772902344525b703d99d7d0e01b89ccfcecbe1b Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Fri, 31 Oct 2025 14:57:25 +0100 Subject: [PATCH 12/28] Add Level 2 format option --- mwrpy/cli.py | 21 ++++++++---- mwrpy/level1/write_lev1_nc.py | 24 +++++++++---- mwrpy/level2/lev2_collocated.py | 24 +++++++++++-- mwrpy/level2/lev2_meta_nc.py | 20 ++++++++++- mwrpy/level2/write_lev2_nc.py | 54 +++++++++++++++++++++++------- mwrpy/process_mwrpy.py | 32 ++++++++++++++++-- mwrpy/rpg_mwr.py | 23 ++++++++++--- mwrpy/site_config/hatpro.yaml | 2 -- mwrpy/site_config/lhatpro.yaml | 2 -- mwrpy/site_config/lhumpro_u90.yaml | 2 -- mwrpy/utils.py | 7 ++-- tests/test_write_lev2_nc.py | 9 ++--- 12 files changed, 172 insertions(+), 48 deletions(-) diff --git a/mwrpy/cli.py b/mwrpy/cli.py index 97dee7a..863efd4 100755 --- a/mwrpy/cli.py +++ b/mwrpy/cli.py @@ -65,6 +65,13 @@ def _parse_args(args): metavar="YYYY-MM-DD", help="Single date to be processed.", ) + group.add_argument( + "-f", + "--format", + type=str, + help="Data format to be used (cloudnet, e-profile).", + default="cloudnet", + ) group.add_argument( "-i", "--instrument", @@ -72,13 +79,6 @@ def _parse_args(args): help="Instrument to be processed (hatpro, lhatpro, lhumpro_u90).", default="hatpro", ) - group.add_argument( - "-f", - "--format", - type=str, - help="Data format to be used (cloudnet, e-profile).", - default="e-profile", - ) group.add_argument( "-a", "--altitude", @@ -86,6 +86,13 @@ def _parse_args(args): help="Altitude above mean sea level of site (m).", default=0.0, ) + group.add_argument( + "-o", + "--azimuth_offset", + type=float, + help="Azimuth offset of the instrument (degrees).", + default=None, + ) return parser.parse_args(args) diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 27bb6c2..9c668a9 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -43,6 +43,7 @@ def lev1_to_nc( time_offset: datetime.timedelta | None = None, instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, altitude: float | None = None, + azimuth_offset: float | None = None, ) -> rpg_mwr.Rpg: """This function reads one day of RPG MWR binary files, adds attributes and writes it into netCDF file. @@ -60,9 +61,10 @@ def lev1_to_nc( time_offset: Time offset if instrument operated in local time. instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). altitude: Altitude of the site in meters above mean sea level. + azimuth_offset: Azimuth offset to be added to azimuth angle. Raises: - MissingInputData: if required input file is missing. + MissingInputData: if input file is missing. """ if site is None: assert coeff_files is not None @@ -88,7 +90,13 @@ def lev1_to_nc( params = {**params, **instrument_config} rpg_bin = prepare_data( - path_to_files, data_type, params, lidar_path, time_offset, altitude + path_to_files, + data_type, + params, + lidar_path, + time_offset, + altitude, + azimuth_offset, ) assert isinstance(rpg_bin, RpgBin) @@ -108,8 +116,8 @@ def lev1_to_nc( None, params.get("coeff_path", None), ) - if (coeff_files is None) - else (coeff_files) + if coeff_files is None + else coeff_files ) global_attributes = { "site": site, @@ -133,6 +141,7 @@ def prepare_data( time_offset: datetime.timedelta | None = None, altitude: float | None = None, date: float | None = None, + azimuth_offset: float | None = None, ) -> RpgBin | dict: """Load and prepare data for netCDF writing.""" if data_type in ("1B01", "1C01"): @@ -209,9 +218,12 @@ def prepare_data( if params["azi_cor"] != -999.0: _azi_correction(rpg_bin.data, params) - if params["const_azi"] != -999.0: + azimuth_offset = ( + params["azimuth_offset"] if "azimuth_offset" in params else azimuth_offset + ) + if azimuth_offset is not None: rpg_bin.data["azimuth_angle"] = ( - rpg_bin.data["azimuth_angle"] + params["const_azi"] + rpg_bin.data["azimuth_angle"] + azimuth_offset ) % 360 file_list_abscal = ( diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 1399604..6dc6f41 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -2,6 +2,7 @@ from collections.abc import Sequence from os import PathLike from tempfile import NamedTemporaryFile +from typing import Literal import netCDF4 @@ -11,10 +12,12 @@ def generate_lev2_single( site: str | None, + data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as lwp_file, @@ -41,6 +44,7 @@ def generate_lev2_single( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=t_prof_file.name @@ -51,6 +55,7 @@ def generate_lev2_single( else None, lwp_offset=lwp_offset, coeff_files=coeff_files, + instrument_type=instrument_type, ) with ( @@ -65,7 +70,8 @@ def generate_lev2_single( ): nc_output.createDimension("height", len(nc_t_prof.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) - nc_output.createDimension("bnds", 2) + if data_format == "e-profile": + nc_output.createDimension("bnds", 2) for source, variables in ( ( @@ -152,6 +158,7 @@ def generate_lev2_single( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=t_prof_file.name @@ -162,6 +169,7 @@ def generate_lev2_single( else None, lwp_offset=(None, None), coeff_files=coeff_files, + instrument_type=instrument_type, ) with netCDF4.Dataset(stability_file.name, "r") as nc_sta: var_2I06 = ( @@ -186,10 +194,12 @@ def generate_lev2_single( def generate_lev2_lhumpro( site: str | None, + data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as lwp_file, @@ -208,12 +218,14 @@ def generate_lev2_lhumpro( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=None, hum_file=None, lwp_offset=lwp_offset, coeff_files=coeff_files, + instrument_type=instrument_type, ) with ( @@ -224,7 +236,8 @@ def generate_lev2_lhumpro( ): nc_output.createDimension("height", len(nc_abs_hum.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) - nc_output.createDimension("bnds", 2) + if data_format == "e-profile": + nc_output.createDimension("bnds", 2) for source, variables in ( ( @@ -276,9 +289,11 @@ def generate_lev2_lhumpro( def generate_lev2_multi( site: str | None, + data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as temperature_file, @@ -301,6 +316,7 @@ def generate_lev2_multi( lev2_to_nc( prod, mwr_l1c_file, + data_format=data_format, output_file=file, site=site, temp_file=temperature_file.name @@ -309,6 +325,7 @@ def generate_lev2_multi( hum_file=abs_hum_file.name if prod not in ("2P02", "2P03") else None, lwp_offset=(None, None), coeff_files=coeff_files, + instrument_type=instrument_type, ) with ( @@ -320,7 +337,8 @@ def generate_lev2_multi( ): nc_output.createDimension("time", len(nc_temp.variables["time"][:])) nc_output.createDimension("height", len(nc_temp.variables["height"][:])) - nc_output.createDimension("bnds", 2) + if data_format == "e-profile": + nc_output.createDimension("bnds", 2) for source, variables in ( ( diff --git a/mwrpy/level2/lev2_meta_nc.py b/mwrpy/level2/lev2_meta_nc.py index 10b5d65..7961447 100644 --- a/mwrpy/level2/lev2_meta_nc.py +++ b/mwrpy/level2/lev2_meta_nc.py @@ -6,13 +6,16 @@ from mwrpy.utils import MetaData -def get_data_attributes(rpg_variables: dict, data_type: str, coeff: dict) -> dict: +def get_data_attributes( + rpg_variables: dict, data_type: str, coeff: dict, data_format: str +) -> dict: """Adds Metadata for RPG MWR Level 2 variables for NetCDF file writing. Args: rpg_variables: RpgArray instances. data_type: Data type of the netCDF file. coeff: Coefficient data of variable + data_format: Data format of the netCDF file (cloudnet, e-profile). Returns: Dictionary @@ -61,6 +64,10 @@ def get_data_attributes(rpg_variables: dict, data_type: str, coeff: dict) -> dic else: del rpg_variables[key] + if data_format == "cloudnet": + attributes.pop("time") + attributes = dict(ATTRIBUTES_CN, **attributes) + index_map = {v: i for i, v in enumerate(attributes)} rpg_variables = dict( sorted(rpg_variables.items(), key=lambda pair: index_map[pair[0]]) @@ -69,6 +76,17 @@ def get_data_attributes(rpg_variables: dict, data_type: str, coeff: dict) -> dic return rpg_variables +ATTRIBUTES_CN = { + "time": MetaData( + comment="Time indication of samples is at end of integration-time", + units="hours since ", + long_name="Time UTC", + standard_name="time", + calendar="standard", + dimensions=("time",), + ), +} + DEFINITIONS_COM = { "quality_flag": ( "\n" diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 57db45a..78d55e7 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -4,6 +4,7 @@ from collections.abc import Sequence from datetime import datetime, timedelta, timezone from os import PathLike +from typing import Literal import atmoslib import atmoslib.constants as ac @@ -18,6 +19,7 @@ from mwrpy.level2.lev2_meta_nc import get_data_attributes from mwrpy.level2.lwp_offset import correct_lwp_offset from mwrpy.utils import ( + get_coeff_list, interpol_2d, interpolate_2d, isbit, @@ -35,12 +37,14 @@ def _local_solar_time(unix_seconds: float, longitude: float) -> datetime: def lev2_to_nc( data_type: str, lev1_file: str | PathLike, + data_format: str, output_file: str | PathLike, site: str | None = None, temp_file: str | PathLike | None = None, hum_file: str | PathLike | None = None, lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, + instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): """This function reads Level 1 files, applies retrieval coefficients for Level 2 products @@ -49,12 +53,14 @@ def lev2_to_nc( Args: data_type: Data type of the netCDF file. lev1_file: Path of Level 1 file. + data_format: Data format of the netCDF file (cloudnet, e-profile). output_file: Name of output file. site: Name of site. temp_file: Name of temperature product file. hum_file: Name of humidity product file. lwp_offset: LWP offset with the previous day's last and next day's first reliable values. coeff_files: List of coefficient files. + instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). """ if data_type not in ( @@ -70,8 +76,9 @@ def lev2_to_nc( ): raise ValueError(f"Data type {data_type} not recognised") - global_attributes = read_config(site, "hatpro", "global_specs") - params = read_config(site, "hatpro", "params") + assert instrument_type is not None + global_attributes = read_config(site, instrument_type, "global_specs") + params = read_config(site, instrument_type, "params") with nc.Dataset(lev1_file) as lev1: params["altitude"] = ma.median(lev1.variables["altitude"][:]) @@ -89,8 +96,26 @@ def lev2_to_nc( _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) _del_att(global_attributes) mwr = rpg_mwr.Rpg(rpg_dat) - mwr.data = get_data_attributes(mwr.data, data_type, coeff) - rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type) + mwr.data = get_data_attributes(mwr.data, data_type, coeff, data_format) + if data_format == "cloudnet": + c_files = ( + get_coeff_list( + site, + prefix=["tpb"] + if data_type in ("2P02", "2P04", "2P07", "2P08") + else ["lwp", "iwv", "hpt", "tpt"], + coeff_files=None, + coeff_dir=params.get("coeff_path", None), + ) + if coeff_files is None + else coeff_files + ) + global_attributes = { + "site": site, + "instrument": instrument_type, + "coeff_files": c_files, + } + rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) def get_products( @@ -115,16 +140,19 @@ def get_products( np.empty([0], np.int32), np.empty([0], np.int32), ) + coeff_path = params.get("coeff_path", None) if data_type in ("2I01", "2I02", "2I06"): product = ( "lwp" if data_type == "2I01" else "iwv" if data_type == "2I02" else "sta" ) - coeff = get_mvr_coeff(site, product, lev1["frequency"][:], coeff_files) + coeff = get_mvr_coeff( + site, product, lev1["frequency"][:], coeff_files, coeff_path + ) if coeff[0]["RT"] < 2: coeff, offset, lin, quad = get_mvr_coeff( - site, product, lev1["frequency"][:], coeff_files + site, product, lev1["frequency"][:], coeff_files, coeff_path ) else: # pylint: disable-next=unbalanced-tuple-unpacking @@ -278,10 +306,10 @@ def get_products( else: product, ret = "absolute_humidity", "hpt" - coeff = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files) + coeff = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files, coeff_path) if coeff[0]["RT"] < 2: coeff, offset, lin, quad = get_mvr_coeff( - site, ret, lev1["frequency"][:], coeff_files + site, ret, lev1["frequency"][:], coeff_files, coeff_path ) else: # pylint: disable-next=unbalanced-tuple-unpacking @@ -294,7 +322,7 @@ def get_products( weights1, weights2, factor, - ) = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files) + ) = get_mvr_coeff(site, ret, lev1["frequency"][:], coeff_files, coeff_path) ret_in = retrieval_input(lev1, coeff) @@ -379,15 +407,17 @@ def get_products( _get_qf(rpg_dat, lev1, coeff, index, index_ret, product) elif data_type == "2P02": - coeff = get_mvr_coeff(site, "tpb", lev1["frequency"][:], coeff_files) + coeff = get_mvr_coeff( + site, "tpb", lev1["frequency"][:], coeff_files, coeff_path + ) if coeff[0]["RT"] < 2: coeff, offset, lin, quad = get_mvr_coeff( - site, "tpb", lev1["frequency"][:], coeff_files + site, "tpb", lev1["frequency"][:], coeff_files, coeff_path ) else: # pylint: disable-next=unbalanced-tuple-unpacking coeff, _, _, _, _, _, _, _ = get_mvr_coeff( - site, "tpb", lev1["frequency"][:], coeff_files + site, "tpb", lev1["frequency"][:], coeff_files, coeff_path ) coeff["AG"] = np.flip(np.sort(coeff["AG"])) diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index 6f7c075..b109939 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -130,6 +130,7 @@ def main(args): args.format, args.instrument, args.altitude, + args.azimuth_offset, ) except Exception as e: logging.error( @@ -143,6 +144,7 @@ def main(args): args.format, args.instrument, args.altitude, + args.azimuth_offset, ) if args.command != "no-plot": logging.info(f"Plotting {product} product, {args.site} {date}") @@ -163,6 +165,7 @@ def process_product( data_format: str, instrument: IType, altitude: float, + azimuth_offset: float | None, ): """Process a given product for a specific date and site. This function handles the processing of different products based on their type @@ -176,6 +179,7 @@ def process_product( data_format: Data format of the netCDF file (cloudnet, e-profile). instrument: Specific instrument type (hatpro, lhatpro, etc.). altitude: Altitude of the site in meters above mean sea level. + azimuth_offset: Azimuth offset to be added to azimuth angle. Returns: None @@ -234,6 +238,7 @@ def process_product( date=date, instrument_type=instrument, altitude=altitude, + azimuth_offset=azimuth_offset, ) # Process level 2 single products @@ -249,24 +254,45 @@ def process_product( lev2_to_nc( prod, _get_filename("1C01", date, site), + data_format, output_file=output_file, site=site, temp_file=temp_file, hum_file=hum_file, lwp_offset=lwp_offset_tuple, + instrument_type=instrument, ) # Process level 2 combined products elif prod == "single" and instrument != "lhumpro_u90": generate_lev2_single( - site, _get_filename("1C01", date, site), output_file, lwp_offset_tuple + site, + data_format, + _get_filename("1C01", date, site), + output_file, + lwp_offset_tuple, + None, + instrument, ) elif instrument == "lhumpro_u90": generate_lev2_lhumpro( - site, _get_filename("1C01", date, site), output_file, lwp_offset_tuple + site, + data_format, + _get_filename("1C01", date, site), + output_file, + lwp_offset_tuple, + None, + instrument, ) elif prod == "multi": - generate_lev2_multi(site, _get_filename("1C01", date, site), output_file) + generate_lev2_multi( + site, + data_format, + _get_filename("1C01", date, site), + output_file, + None, + instrument, + ) # Update LWP offset file if necessary offset_current = _get_filename("lwp_offset", date, site) diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 4f0b67b..0f9d049 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -260,7 +260,10 @@ def init_file( """ nc_file = netCDF4.Dataset(file_name, "w", format="NETCDF4_CLASSIC") for key, dimension in dimensions.items(): - nc_file.createDimension(key, dimension) + if data_format == "cloudnet" and key == "bnds": + continue + else: + nc_file.createDimension(key, dimension) _write_vars2nc(nc_file, rpg_arrays) _add_cloudnet_global_attributes( nc_file, att_global, data_type @@ -301,17 +304,29 @@ def _add_cloudnet_global_attributes( form = "%Y-%m-%d %H:%M:%S" instrument = add_global["instrument"].upper() site = add_global["site"] + if data_type == "1C01": + level = "mwr-l1c" + title = f"{instrument} microwave radiometer Level 1c from {site}" + history = level + elif data_type == "2P02": + level = "mwr-multi" + title = f"MWR multiple-pointing from {site}" + history = "MWR multiple-pointing" + else: + level = "mwr-single" + title = f"MWR single-pointing from {site}" + history = "MWR single-pointing" att_global = { "Conventions": "CF-1.8", "mwrpy_version": version.__version__, "location": add_global["site"], "source": f"RPG-Radiometer Physics {instrument}", "references": "https://doi.org/10.21105/joss.06733", - "mwrpy_file_type": data_type, - "title": f"{instrument} microwave radiometer Level 1c from {site}", + "cloudnet_file_type": level, + "title": title, "history": f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00" + " - " - + data_type + + history + " file created", } for name, value in att_global.items(): diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index 6f751eb..a76206c 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -29,8 +29,6 @@ params: # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. - const_azi: -999. - # some default values: # ------------------- receiver_nb: [1, 2] diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index d281370..a9e7565 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -29,8 +29,6 @@ params: # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. - const_azi: -999. - # some default values: # ------------------- receiver_nb: [2, 1] diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index 6282bc1..02c5527 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -29,8 +29,6 @@ params: # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. - const_azi: -999. - # some default values: # ------------------- receiver_nb: [2, 1] diff --git a/mwrpy/utils.py b/mwrpy/utils.py index e163606..32bfd53 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -310,9 +310,12 @@ def get_coeff_list( c_list = [] for p in prefix: tmp = glob.glob(dir_path + "*" + p.lower() + "*") - if len(c_list) == 0: + if len(tmp) == 0: tmp = glob.glob(dir_path + "*" + p.upper() + "*") - c_list = c_list + tmp + if len(c_list) == 0: + c_list = tmp + elif len(tmp) > 0 and len(c_list) > 0: + c_list = c_list + tmp if len(c_list) > 0: if "spc" in c_list and "ins" in c_list: diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index a386fdd..525cc48 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -8,6 +8,7 @@ from mwrpy.level2.lev2_collocated import generate_lev2_multi, generate_lev2_single SITE = "hyytiala" +DATA_FORMAT = "e-profile" PACKAGE_DIR = os.path.abspath(os.path.dirname(__file__)) DATA_DIR = f"{PACKAGE_DIR}/data/{SITE}" @@ -31,25 +32,25 @@ def delete_file(): def test_generate_lev2_single_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(SITE, l1_file, path) + generate_lev2_single(SITE, DATA_FORMAT, l1_file, path) os.unlink(path) def test_generate_lev2_single_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(None, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_single(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) os.unlink(path) def test_generate_lev2_multi_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(SITE, l1_file, path) + generate_lev2_multi(SITE, DATA_FORMAT, l1_file, path) os.unlink(path) def test_generate_lev2_multi_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(None, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_multi(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) From b8b793d0659e95ce78554ce5267c7007abc730d1 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Fri, 31 Oct 2025 14:58:19 +0100 Subject: [PATCH 13/28] Update README and docs --- README.md | 30 ++-- docs/source/command_line_usage.rst | 105 ++++++++++++ docs/source/data_types.rst | 31 ++++ docs/source/fileformat.rst | 100 +++++++++++- docs/source/index.rst | 2 + docs/source/mwrpy_processing.rst | 247 +++++++++++++++++------------ 6 files changed, 395 insertions(+), 120 deletions(-) create mode 100644 docs/source/command_line_usage.rst create mode 100644 docs/source/data_types.rst diff --git a/README.md b/README.md index 92840c6..f233b7a 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Level 2 data products and visualization and is based on the IDL code [mwr_pro](https://zenodo.org/records/7973553). The netCDF data format including metadata information, variable names and file naming -is designed to be compliant with the data structure and naming convention -developed in the [EUMETNET Profiling Programme E-PROFILE](https://www.eumetnet.eu/). +is designed to be compliant with either the data structure and naming convention +developed in the [EUMETNET Profiling Programme E-PROFILE](https://www.eumetnet.eu/), or within ACTRIS. MWRpy documentation: @@ -58,24 +58,30 @@ For example, this is the [configuration file for RPG-HATPRO](mwrpy/site_config/h The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a site and instrument specific configuration file (`config.yaml`) and retrieval coefficients. For example, this is the [configuration file for Hyytiälä](mwrpy/site_config/hyytiala/config.yaml). +This site configuration file is not needed when using the Cloudnet file format. ## Command line usage MWRpy can be run using the command line tool `mwrpy/cli.py`: usage: mwrpy/cli.py [-h] -s SITE [-d YYYY-MM-DD] [--start YYYY-MM-DD] - [--stop YYYY-MM-DD] [-p ...] [{process,plot}] + [--stop YYYY-MM-DD] [-f ...] [-p ...] [{process,plot}] Arguments: -| Short | Long | Default | Description | -| :---- | :----------- | :------------------ | :--------------------------------------------------------------------------------- | -| `-h` | `--help` | | Show help and exit. | -| `-s` | `--site` | | Site to process data from, e.g, `hyytiala`. Required. | -| `-d` | `--date` | | Single date to be processed. Alternatively, `--start` and `--stop` can be defined. | -| | `--start` | `current day - 1` | Starting date. | -| | `--stop` | `current day ` | Stopping date. | -| `-p` | `--products` | 1C01, single, multi | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | +| Short | Long | Default | Description | +| :------------------------------------------------------------- | :----------------- | :------------------------ | :--------------------------------------------------------------------------------- | +| `-h` | `--help` | | Show help and exit. | +| `-s` | `--site` | | Site to process data from, e.g, `hyytiala`. Required. | +| `-d` | `--date` | | Single date to be processed. Alternatively, `--start` and `--stop` can be defined. | +| | `--start` | `current day - 1` | Starting date. | +| | `--stop` | `current day ` | Stopping date. | +| `-p` | `--products` | `1C01`, `single`, `multi` | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | +| `-f` | `--format` | `cloudnet` | Data format to be used (`cloudnet`, `e-profile`). | +| The following arguments are used for the Cloudnet file format: | +| `-i` | `--instrument` | `hatpro` | Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). | +| `-a` | `--altitude` | `0.0` | Altitude above mean sea level of site (m). | +| `-o` | `--azimuth_offset` | `None` | Azimuth offset of the instrument (degrees). Or `None`. | Commands: @@ -109,6 +115,8 @@ Commands: - single: Single pointing data product (including 2I01, 2I02, 2I06, 2P01, 2P03, and derived products) - multi: Multiple pointing data product (including 2P02, and derived products) +Only the `1C01`, `single`, and `multi` data types are available when using the Cloudnet file format. + ## Licence MIT diff --git a/docs/source/command_line_usage.rst b/docs/source/command_line_usage.rst new file mode 100644 index 0000000..56edfe6 --- /dev/null +++ b/docs/source/command_line_usage.rst @@ -0,0 +1,105 @@ +================== +Command line usage +================== + +After defining the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and site specific +information (``mwrpy/site_config/{site}.yaml``, only for E-PROFILE format) files, including input/output data +paths, MWRpy can also be run using the command line tool `mwrpy/cli.py`: + +.. code-block:: + + mwrpy/cli.py [-h] -s SITE [-d YYYY-MM-DD] [--start YYYY-MM-DD] + [--stop YYYY-MM-DD] [-p ...] [{process,plot}] + +.. list-table:: Arguments + :widths: 10 20 20 50 + :header-rows: 1 + + * - Short + - Long + - Default + - Description + * - `-h` + - `--help` + - + - Show help and exit. + * - `-s` + - `--site` + - + - Site to process data from, e.g, `hyytiala`. Required. + * - `-d` + - `--date` + - + - Single date to be processed. Alternatively, `--start` and `--stop` can be defined. + * - + - `--start` + - `current day - 1` + - Starting date. + * - + - `--stop` + - `current day` + - Stopping date. + * - `-p` + - `--products` + - 1C01, single, multi + - Processed products, e.g, `1C01, 2I02, 2P03, single`, see Data Types below. + * - `-f` + - `--format` + - cloudnet + - Data format to be used (`cloudnet`, `e-profile`). + +The following arguments are used for the Cloudnet file format: + +.. list-table:: Arguments + :widths: 10 20 20 50 + :header-rows: 1 + + * - Short + - Long + - Default + - Description + * - `-i` + - `--instrument` + - hatpro + - Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). + * - `-a` + - `--altitude` + - 0.0 + - Altitude above mean sea level of site (m). + * - `-o` + - `--azimuth_offset` + - None + - Azimuth offset of the instrument (degrees). Or `None`. + +These commands are available to select the processing mode: + +.. list-table:: Commands + :widths: 20 30 + :header-rows: 1 + + * - Command + - Description + * - `process` + - Process data and generate plots (default). + * - `plot` + - Only generate plots. + * - `no-plot` + - Only generate products. + * - `reprocess` + - Like `process`, but skips days when data processing fails. + +Example usage +------------- +To process and plot Level 1 & 2 data (1C01, single, multi) for the site `Hyytiala` (HATPRO instrument) for April 6, +2023, in the E-PROFILE format, run: + +.. code-block:: + + python mwrpy/cli.py -s hyytiala -d 2023-04-06 -f e-profile process + + +Run the following command for the Cloudnet format (with site altitude 150 m) and no plots: + +.. code-block:: + + python mwrpy/cli.py -s hyytiala -d 2023-04-06 -a 150 no-plot diff --git a/docs/source/data_types.rst b/docs/source/data_types.rst new file mode 100644 index 0000000..2b1702b --- /dev/null +++ b/docs/source/data_types.rst @@ -0,0 +1,31 @@ +========== +Data Types +========== + +The following data types (E-PROFILE naming convention) are available in MWRpy for Level 1 and Level 2 products: + +Level 1 +....... + +- 1B01: MWR brightness temperatures from .BRT and .BLB/.BLS files + retrieved spectrum +- 1B11: IR brightness temperatures from .IRT files +- 1B21: Weather station data from .MET files +- 1C01: Combined data type with time corresponding to 1B01 + +Level 2 +....... + +- 2I01: Liquid water path (LWP) +- 2I02: Integrated water vapor (IWV) +- 2I06: Stability Indices +- 2P01: Temperature profiles from single-pointing observations +- 2P02: Temperature profiles from multiple-pointing observations +- 2P03: Absolute humidity profiles +- 2P04: Relative humidity profiles (derived from 2P01/2P02 + 2P03) +- 2P07: Potential temperature (derived from 2P01/2P02 + 2P03) +- 2P08: Equivalent potential temperature (derived from 2P01/2P02 + 2P03) +- single: Single pointing data product (including 2I01, 2I02, 2I06, 2P01, 2P03, and derived products) +- multi: Multiple pointing data product (including 2P02, and derived products) + + +The data types 1C01, single, and multi are also available in the Cloudnet format. diff --git a/docs/source/fileformat.rst b/docs/source/fileformat.rst index f8b7af8..75eb98f 100644 --- a/docs/source/fileformat.rst +++ b/docs/source/fileformat.rst @@ -6,20 +6,28 @@ All MWRpy files use ``NETCDF4_CLASSIC`` data model, i.e., ``HDF5`` file format. **Dimensions** .. list-table:: - :widths: 25 + :widths: 25 25 :header-rows: 1 - * - Name + * - Name (E-PROFILE) + - Name (Cloudnet) * - time + - time * - bnds + - * - frequency + - frequency * - ir_wavelength + - ir_channel * - receiver_nb + - receiver_nb * - t_amb_nb + - t_amb_nb * - height + - height -**Variables (common to all files)** +**Variables (common to all E-PROFILE files)** .. list-table:: :widths: 25 50 25 25 25 25 @@ -62,6 +70,43 @@ All MWRpy files use ``NETCDF4_CLASSIC`` data model, i.e., ``HDF5`` file format. - float32 - altitude +**Variables (common to all Cloudnet files)** + +.. list-table:: + :widths: 25 50 25 25 25 25 + :header-rows: 1 + + * - Name + - Long name + - Dimensions + - Units + - Data type + - Standard name + * - time + - Time UTC + - time + - hours since YYYY-MM-DD 00:00:00 +00:00 + - double + - time + * - latitude + - Latitude of site + - time + - degree_north + - float32 + - latitude + * - longitude + - Longitude of site + - time + - degree_east + - float32 + - longitude + * - altitude + - Altitude of site + - time + - m + - float32 + - altitude + MWR-Level 1 files ................. @@ -70,7 +115,8 @@ MWR-Level 1 files ~~~~~~~~~ The Level 1 default file type ``1C01`` contains all variables from the file types -``1B01``, ``1B11`` (if an infrared radiometer is available), and ``1B21`` (if a weather station is available). +``1B01``, ``1B11`` (if an infrared radiometer is available), and ``1B21`` (if a weather station is available) and is +available for the E-PROFILE and Cloudnet data format. **Variables (MWR_1B01 specific)** @@ -193,6 +239,25 @@ The Level 1 default file type ``1C01`` contains all variables from the file type - int32 - +**Additional Cloudnet variable** + +.. list-table:: + :widths: 25 50 25 25 25 25 + :header-rows: 1 + + * - Name + - Long name + - Dimensions + - Units + - Data type + - Standard name + * - zenith_angle + - Zenith angle + - time + - degree + - float32 + - zenith_angle + **Variables (MWR_1B11 specific)** .. list-table:: @@ -242,6 +307,25 @@ The Level 1 default file type ``1C01`` contains all variables from the file type - float32 - +**Additional Cloudnet variable** + +.. list-table:: + :widths: 25 50 25 25 25 25 + :header-rows: 1 + + * - Name + - Long name + - Dimensions + - Units + - Data type + - Standard name + * - ir_zenith_angle + - Infrared sensor zenith angle + - time + - degree + - float32 + - + **Variables (MWR_1B21 specific)** .. list-table:: @@ -298,7 +382,7 @@ The Level 1 default file type ``1C01`` contains all variables from the file type - MWR-Level 2 files -............... +................. **Variables (common to all Level 2 files)** @@ -329,7 +413,8 @@ Single pointing file ~~~~~~~~~~~~~~~~~~~~ The Level 2 default file type ``single`` contains all variables from the file types -``2I01``, ``2I02``, ``2I06``, ``2P01``, and ``2P03`` (if the respective retrieval coefficients are available). +``2I01``, ``2I02``, ``2I06``, ``2P01``, and ``2P03`` (if the respective retrieval coefficients are available) and is +available for the E-PROFILE and Cloudnet data format. **Variables (MWR_2I01 specific)** @@ -545,7 +630,8 @@ Multiple pointing file ~~~~~~~~~~~~~~~~~~~~~~ The Level 2 default file type ``multi`` contains all variables from the file types -``2P02``, ``2P04``, ``2P07``, and ``2P08`` (if the respective retrieval coefficients are available). +``2P02``, ``2P04``, ``2P07``, and ``2P08`` (if the respective retrieval coefficients are available) and is +available for the E-PROFILE and Cloudnet data format. **Variables (MWR_2P02 specific)** diff --git a/docs/source/index.rst b/docs/source/index.rst index dfc952f..619c14b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,9 @@ Welcome to MWRpy's documentation! overview installation + data_types mwrpy_processing + command_line_usage fileformat guide diff --git a/docs/source/mwrpy_processing.rst b/docs/source/mwrpy_processing.rst index fd6b2a2..2a001e6 100644 --- a/docs/source/mwrpy_processing.rst +++ b/docs/source/mwrpy_processing.rst @@ -17,10 +17,27 @@ quality control and visualization. This example utilizes files taken from the AC .BRT and .HKD files are mandatory in MWRpy for processing +First steps for processing examples: + +First we define the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and site specific +information (``mwrpy/site_config/{site}.yaml``, only for E-PROFILE format) files, including input/output data paths. +Then the data path is specified: + +.. code-block:: python + + import os + + package_dir = os.getcwd() + site = "hyytiala" + data_path = f"{package_dir}/tests/data/{site}" + +E-PROFILE format +---------------- + Level 1c ~~~~~~~~~ -First we convert RPG microwave radiometer (MWR) binary files, including brightness temperature (TB) and +Now we convert RPG microwave radiometer (MWR) binary files, including brightness temperature (TB) and housekeeping data (\*.BRT, \*.HKD), into a Level 1c netCDF file. Data from optional elevation scans (\*.BLB, \*.BLS), weather station (\*.MET) and infrared radiometer (\*.IRT) are combined in this process and the following quality flags are derived: @@ -43,17 +60,13 @@ flag status variable contains information whether the flag is active. .. code-block:: python - import os from mwrpy.level1.write_lev1_nc import lev1_to_nc - SITE = "hyytiala" - PACKAGE_DIR = os.getcwd() - DATA_DIR = f"{PACKAGE_DIR}/tests/data/{SITE}" - mwr_raw = lev1_to_nc( - "1C01", - DATA_DIR, - site=SITE, + data_type="1C01", + path_to_files=data_path, + data_format="e-profile", + site=site, output_file="mwr_1c.nc", ) @@ -65,10 +78,9 @@ Variables such as brightness temperature can be plotted from the newly generated .. code-block:: python - import os from mwrpy.plots.generate_plots import generate_figure - PACKAGE_DIR = os.getcwd() - generate_figure('mwr_1c.nc', ['tb'], save_path=f"{PACKAGE_DIR}/") + + generate_figure('mwr_1c.nc', ['tb'], save_path=f"{package_dir}/") .. figure:: _static/20230406_hyytiala_tb.png @@ -82,7 +94,13 @@ are applied to generate the Level 2 single pointing product: .. code-block:: python from mwrpy.level2.lev2_collocated import generate_lev2_single - mwr_prod = generate_lev2_single("hyytiala", "mwr_1c.nc", "mwr-single.nc") + + mwr_prod = generate_lev2_single( + site="hyytiala", + data_format="e-profile", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-single.nc", + ) Variables such as integrated water vapor (`IWV `_) @@ -90,10 +108,9 @@ can be plotted from the newly generated file. .. code-block:: python - import os from mwrpy.plots.generate_plots import generate_figure - PACKAGE_DIR = os.getcwd() - generate_figure('mwr-single.nc', ['iwv'], save_path=f"{PACKAGE_DIR}/") + + generate_figure('mwr-single.nc', ['iwv'], save_path=f"{package_dir}/") .. figure:: _static/20230406_hyytiala_iwv.png @@ -107,98 +124,124 @@ product: .. code-block:: python from mwrpy.level2.lev2_collocated import generate_lev2_multi - mwr_prod = generate_lev2_multi("hyytiala", "mwr_1c.nc", "mwr-multi.nc") + + mwr_prod = generate_lev2_multi( + site="hyytiala", + data_format="e-profile", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-multi.nc", + ) Variables such as temperature profiles can be plotted from the newly generated file. .. code-block:: python - import os from mwrpy.plots.generate_plots import generate_figure - PACKAGE_DIR = os.getcwd() - generate_figure('mwr-multi.nc', ['temperature'], save_path=f"{PACKAGE_DIR}/") + + generate_figure('mwr-multi.nc', ['temperature'], save_path=f"{package_dir}/") .. figure:: _static/20230406_hyytiala_temperature.png -Command line usage -~~~~~~~~~~~~~~~~~~ - -After defining the instrument type and site specific configuration files (including input/output data paths) in -``mwrpy/site_config/``, MWRpy can also be run using the command line tool `mwrpy/cli.py`: - -.. code-block:: - - mwrpy/cli.py [-h] -s SITE [-d YYYY-MM-DD] [--start YYYY-MM-DD] - [--stop YYYY-MM-DD] [-p ...] [{process,plot}] - -.. list-table:: Arguments - :widths: 10 20 20 50 - :header-rows: 1 - - * - Short - - Long - - Default - - Description - * - `-h` - - `--help` - - - - Show help and exit. - * - `-s` - - `--site` - - - - Site to process data from, e.g, `hyytiala`. Required. - * - `-d` - - `--date` - - - - Single date to be processed. Alternatively, `--start` and `--stop` can be defined. - * - - - `--start` - - `current day - 1` - - Starting date. - * - - - `--stop` - - `current day` - - Stopping date. - * - `-p` - - `--products` - - 1C01, single, multi - - Processed products, e.g, `1C01, 2I02, 2P03, single`, see Data Types below. - -.. list-table:: Commands - :widths: 20 30 - :header-rows: 1 - - * - Command - - Description - * - `process` - - Process data and generate plots (default). - * - `plot` - - Only generate plots. - * - `no-plot` - - Only generate products. - * - `reprocess` - - Like `process`, but skips days when data processing fails. - -Data Types -~~~~~~~~~~ - -Level 1 - -- 1B01: MWR brightness temperatures from .BRT and .BLB/.BLS files + retrieved spectrum -- 1B11: IR brightness temperatures from .IRT files -- 1B21: Weather station data from .MET files -- 1C01: Combined data type with time corresponding to 1B01 - -Level 2 - -- 2I01: Liquid water path (LWP) -- 2I02: Integrated water vapor (IWV) -- 2I06: Stability Indices -- 2P01: Temperature profiles from single-pointing observations -- 2P02: Temperature profiles from multiple-pointing observations -- 2P03: Absolute humidity profiles -- 2P04: Relative humidity profiles (derived from 2P01/2P02 + 2P03) -- 2P07: Potential temperature (derived from 2P01/2P02 + 2P03) -- 2P08: Equivalent potential temperature (derived from 2P01/2P02 + 2P03) -- single: Single pointing data product (including 2I01, 2I02, 2I06, 2P01, 2P03, and derived products) -- multi: Multiple pointing data product (including 2P02, and derived products) +Cloudnet format +--------------- +In this example the Cloudnet API is used to fetch data and retrieval files and the Cloudnet data format is selected +for processing. More details can be found in the E-PROFILE example above. + +Using Cloudnet API to fetch data and retrieval files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download raw data (binary files): + +.. code-block:: python + + from cloudnet_api_client import APIClient + + date = "2023-04-06" + instrument_pid = "https://hdl.handle.net/21.12132/3.f360a2375f3e4e4f" # check https://cloudnet.fmi.fi/instruments to find the PID of your instrument + client = APIClient() + instruments = client.instruments() + instrument_type = [i.instrument_id for i in instruments if i.pid == instrument_pid][0] + files = client.raw_files(site_id=site, instrument_id=instrument_type, date=date) + binary_files = [f for f in files] + + binary_filepaths = await client.adownload(binary_files, data_path) + +Download retrieval files: + +.. code-block:: python + + import requests + + calibration = client.calibration(instrument_pid, date) + retrieval = calibration["data"] + + retrieval_files = [] + for file in retrieval["coefficientLinks"]: + filename = data_path + file.split("/")[-1] + response = requests.get(file) + with open(filename, "wb") as f: + f.write(response.content) + retrieval_files.append(str(filename)) + +Process and plot Level 1 data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In contrast to the E-PROFILE data format, no site specific information file is required, but metadata needs to be +defined. Also, the retrieval files are set as an argument. + +.. code-block:: python + + site_info = client.site(site_id=site) + site_meta = { + "name": site_info.id, + "altitude": site_info.altitude, + "latitude": site_info.latitude, + "longitude": site_info.longitude, + } + + from mwrpy.level1.write_lev1_nc import lev1_to_nc + mwr_raw = lev1_to_nc( + data_type="1C01", + path_to_files=data_path, + data_format="cloudnet", + instrument_type=instrument_type, + output_file="mwr_1c.nc", + coeff_files=retrieval_files, + instrument_config=site_meta, + ) + +For plotting, the instrument needs to be defined. In this example, the figure is only displayed and not saved. + +.. code-block:: python + + from mwrpy.plots.generate_plots import generate_figure + fig_name = generate_figure('mwr_1c.nc', ['tb'], show=True, instrument_type=instrument_type) + +Process and plot Level 2 data (single & multiple pointing) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The site name is set to ``None``, since no site specific information file is needed. Again, no plots are saved, only +displayed. + +.. code-block:: python + + from mwrpy.level2.lev2_collocated import generate_lev2_single + mwr_prod = generate_lev2_single( + site=None, + data_format="cloudnet", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-single.nc", + coeff_files=retrieval_files, + ) + + from mwrpy.plots.generate_plots import generate_figure + fig_name = generate_figure('mwr-single.nc', ['iwv'], show=True, instrument_type=instrument_type) + + from mwrpy.level2.lev2_collocated import generate_lev2_multi + mwr_prod = generate_lev2_multi( + site=None, + data_format="cloudnet", + mwr_l1c_file="mwr_1c.nc", + output_file="mwr-multi.nc", + coeff_files=retrieval_files, + ) + + from mwrpy.plots.generate_plots import generate_figure + fig_name = generate_figure('mwr-multi.nc', ['temperature'], show=True, instrument_type=instrument_type) From 7c95dfa863fa80b775c131c547fe4e31822c8336 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 12 Nov 2025 14:16:14 +0100 Subject: [PATCH 14/28] Fixed reading in l1/spc files and time for Cloudnet format --- mwrpy/level1/write_lev1_nc.py | 2 +- mwrpy/level2/write_lev2_nc.py | 5 ++++- mwrpy/plots/generate_plots.py | 14 ++++++++++---- mwrpy/plots/plot_utils.py | 6 +++++- mwrpy/process_mwrpy.py | 12 +++++++++--- mwrpy/rpg_mwr.py | 2 +- 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 9c668a9..74354e9 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -112,7 +112,7 @@ def lev1_to_nc( c_files = ( get_coeff_list( site, - ["spc", "ins", "lwp", "iwv", "hpt", "tpt", "tpb"], + ["spc", "ins", "lwp", "iwv", "hpt", "tpt", "tpb", "tbx"], None, params.get("coeff_path", None), ) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 78d55e7..b65803c 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -95,7 +95,10 @@ def lev2_to_nc( ) _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) _del_att(global_attributes) - mwr = rpg_mwr.Rpg(rpg_dat) + mwr = rpg_mwr.Rpg( + rpg_dat, + date=num2pydate(lev1.variables["time"][:][0], lev1.variables["time"].units), + ) mwr.data = get_data_attributes(mwr.data, data_type, coeff, data_format) if data_format == "cloudnet": c_files = ( diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index b67dcec..569f0b7 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -2,7 +2,6 @@ import glob import locale -import logging from datetime import date, datetime, timezone import atmoslib @@ -700,7 +699,12 @@ def _plot_colormesh_data( "potential_temperature", "equivalent_potential_temperature", ): - hum_time = seconds2hours(read_nc_fields(hum_file, "time")) + hum_time = read_nc_fields(hum_file, "time") + hum_time = ( + seconds2hours(read_nc_fields(hum_file, "time")) + if hum_time.max() > 24 + else hum_time + ) hum_flag = _get_ret_flag( hum_file, hum_time, "absolute_humidity", instrument_type=instrument_type ) @@ -844,7 +848,7 @@ def _plot_instrument_data( elif product == "sen": _plot_sen(ax, data, name, time, nc_file) elif product == "hkd": - _plot_hkd(ax, data, name, time) + _plot_hkd(ax, data, name, time, nc_file) elif product == "cov": _plot_covariance(ax, data, name, nc_file) @@ -854,9 +858,11 @@ def _plot_instrument_data( return fig -def _plot_hkd(ax, data_in: ndarray, name: str, time: ndarray): +def _plot_hkd(ax, data_in: ndarray, name: str, time: ndarray, nc_file: str): """Plot for housekeeping data.""" time = _nan_time_gaps(time) + pointing_flag = read_nc_fields(nc_file, "pointing_flag") + data_in[pointing_flag == 1, :] = np.nan if name == "t_amb": data_in[data_in == -999.0] = np.nan if (data_in[:, 0].all() is ma.masked) | (data_in[:, 1].all() is ma.masked): diff --git a/mwrpy/plots/plot_utils.py b/mwrpy/plots/plot_utils.py index 954b315..5b0e9cf 100644 --- a/mwrpy/plots/plot_utils.py +++ b/mwrpy/plots/plot_utils.py @@ -28,7 +28,11 @@ def _get_ret_flag( """Returns quality flag for frequencies used in retrieval.""" file = netCDF4.Dataset(nc_file) quality_flag = file.variables[variable + "_quality_flag"] - time_variable = seconds2hours(file.variables["time"][:]) + time_variable = ( + seconds2hours(file.variables["time"][:]) + if np.max(file.variables["time"]) > 24 + else file.variables["time"][:] + ) _, index, _ = np.intersect1d( time_variable, time, assume_unique=True, return_indices=True ) diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index b109939..e56d44c 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -226,6 +226,12 @@ def process_product( ].values[0] lwp_offset_tuple = (lwp_offset[0], lwp_offset[1]) + l1_filename = ( + _get_filename("1C01", date, site) + if data_format == "e-profile" + else _get_filename_cloudnet("1C01", date, site, instrument) + ) + # Process level 1 data if prod[0] == "1": lev1_to_nc( @@ -268,7 +274,7 @@ def process_product( generate_lev2_single( site, data_format, - _get_filename("1C01", date, site), + l1_filename, output_file, lwp_offset_tuple, None, @@ -278,7 +284,7 @@ def process_product( generate_lev2_lhumpro( site, data_format, - _get_filename("1C01", date, site), + l1_filename, output_file, lwp_offset_tuple, None, @@ -288,7 +294,7 @@ def process_product( generate_lev2_multi( site, data_format, - _get_filename("1C01", date, site), + l1_filename, output_file, None, instrument, diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 0f9d049..17854bc 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -142,7 +142,7 @@ def add_zenith_angle(self): def convert_time_to_hours(self): """Converts time from seconds since epoch to hours since midnight.""" time = self.data["time"].data[:] - time_hours = seconds2hours(time) + time_hours = seconds2hours(time) if time.max() > 24 else time self.data["time"] = RpgArray( time_hours, "time", From 5e0c567c0fde54c285d6cc874db3249c97993aa3 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 1 Jul 2026 16:38:51 +0200 Subject: [PATCH 15/28] Fix test --- tests/test_write_lev1_nc.py | 5 +++-- tests/test_write_lev2_nc.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/test_write_lev1_nc.py b/tests/test_write_lev1_nc.py index 2571427..ad8654a 100644 --- a/tests/test_write_lev1_nc.py +++ b/tests/test_write_lev1_nc.py @@ -12,11 +12,12 @@ DATE = "2023-04-06" site = "hyytiala" product_list = ["1B01", "1B11", "1B21", "1C01"] +DATA_FORMAT = "e-profile" def test_lev1_to_nc(): for prod in product_list: - hatpro = lev1_to_nc(prod, DATA_DIR, site) + hatpro = lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site) assert str(hatpro.date) == DATE for t in hatpro.data["time"][:]: date = str( @@ -28,7 +29,7 @@ def test_lev1_to_nc(): def test_output_nc_file(): for prod in product_list: temp_file = "temp_file.nc" - lev1_to_nc(prod, DATA_DIR, site, output_file=temp_file) + lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site, output_file=temp_file) with netCDF4.Dataset(temp_file) as nc: # Write tests for the created netCDF file here: assert nc.date == DATE diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index 525cc48..ac2fbdb 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -20,7 +20,7 @@ def l1_file(request): fd, path = tempfile.mkstemp() os.close(fd) - lev1_to_nc("1C01", DATA_DIR, SITE, path) + lev1_to_nc("1C01", DATA_DIR, DATA_FORMAT, SITE, path) def delete_file(): os.unlink(path) @@ -39,7 +39,14 @@ def test_generate_lev2_single_site(l1_file): def test_generate_lev2_single_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_single( + None, + DATA_FORMAT, + l1_file, + path, + coeff_files=COEFF_FILES, + instrument_type="hatpro", + ) os.unlink(path) @@ -53,4 +60,11 @@ def test_generate_lev2_multi_site(l1_file): def test_generate_lev2_multi_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(None, DATA_FORMAT, l1_file, path, coeff_files=COEFF_FILES) + generate_lev2_multi( + None, + DATA_FORMAT, + l1_file, + path, + coeff_files=COEFF_FILES, + instrument_type="hatpro", + ) From dc4f48727d8eb9d20ae4cc3eb4b3fdb6f868d15e Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 13:14:29 +0200 Subject: [PATCH 16/28] Implement final E-Profile data format (Level 2) --- mwrpy/level1/lev1_meta_nc.py | 46 +++++++++++++-- mwrpy/level1/quality_control.py | 4 +- mwrpy/level1/write_lev1_nc.py | 5 ++ mwrpy/level2/lev2_collocated.py | 52 ++++++++++++----- mwrpy/level2/lev2_meta_nc.py | 73 ++++++++---------------- mwrpy/level2/write_lev2_nc.py | 48 +++++++++++++--- mwrpy/rpg_mwr.py | 17 +++--- mwrpy/site_config/hatpro.yaml | 3 + mwrpy/site_config/hyytiala/config.yaml | 3 + mwrpy/site_config/juelich/config.yaml | 3 + mwrpy/site_config/lindenberg/config.yaml | 3 + mwrpy/site_config/palaiseau/config.yaml | 3 + mwrpy/utils.py | 17 +++++- 13 files changed, 185 insertions(+), 92 deletions(-) diff --git a/mwrpy/level1/lev1_meta_nc.py b/mwrpy/level1/lev1_meta_nc.py index eae7309..3c1fc0f 100644 --- a/mwrpy/level1/lev1_meta_nc.py +++ b/mwrpy/level1/lev1_meta_nc.py @@ -37,8 +37,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - if data_type in ("1B01", "1B11", "1B21"): read_att = att_reader[data_type] attributes = dict(ATTRIBUTES_COM, **read_att) - - elif data_type == "1C01": + else: attributes = dict( ATTRIBUTES_COM, **ATTRIBUTES_1B01, **ATTRIBUTES_1B11, **ATTRIBUTES_1B21 ) @@ -46,6 +45,13 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - attributes.pop("time") attributes = dict(ATTRIBUTES_CN, **attributes) + if data_format == "e-profile": + keys = ["latitude", "longitude", "altitude", "height"] + for key in keys: + if key in attributes: + attributes.pop(key) + attributes = dict(ATTRIBUTES_EP, **attributes) + for key in list(rpg_variables): if key in attributes: rpg_variables[key].set_attributes(attributes[key]) @@ -62,16 +68,44 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - ATTRIBUTES_CN = { "time": MetaData( + comment="Time indication of samples is at end of integration-time", units="hours since ", long_name="Time UTC", standard_name="time", - axis="T", calendar="standard", dimensions=("time",), ), } +ATTRIBUTES_EP = { + "station_latitude": MetaData( + long_name="Latitude of measurement station", + standard_name="latitude", + units="degree_north", + dimensions=("time",), + ), + "station_longitude": MetaData( + long_name="Longitude of measurement station", + standard_name="longitude", + units="degree_east", + dimensions=("time",), + ), + "station_altitude": MetaData( + long_name="Altitude above mean sea level of measurement station", + standard_name="altitude", + units="m", + dimensions=("time",), + ), + "altitude": MetaData( + long_name="Height above mean sea level", + standard_name="height_above_mean_sea_level", + units="m", + dimensions=("altitude",), + ), +} + + ATTRIBUTES_COM = { "time": MetaData( long_name="Time (UTC) of the measurement", @@ -105,7 +139,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - } -DEFINITIONS_1B01 = { +DEFINITIONS_QF = { "quality_flag": ( "\n" "Bit 1: missing_tb\n" @@ -223,7 +257,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - "quality_flag": MetaData( long_name="Quality flag", units="1", - definition=DEFINITIONS_1B01["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time", "frequency"), @@ -231,7 +265,7 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - "quality_flag_status": MetaData( long_name="Quality flag status", units="1", - definition=DEFINITIONS_1B01["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time", "frequency"), diff --git a/mwrpy/level1/quality_control.py b/mwrpy/level1/quality_control.py index f7d3fa6..2fbf45d 100644 --- a/mwrpy/level1/quality_control.py +++ b/mwrpy/level1/quality_control.py @@ -118,8 +118,8 @@ def orbpos(data: dict, params: dict) -> np.ndarray: for t in data["time"] ] ) - lat = data["latitude"] - lng = data["longitude"] + lat = data["latitude"] if "latitude" in data else data["station_latitude"] + lng = data["longitude"] if "longitude" in data else data["station_longitude"] sol = suncalc.get_position(time, lat=lat, lng=lng) lun = suncalc.suncalc.getMoonPosition(time, lat=lat, lng=lng) sun = { diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 74354e9..c0fff07 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -100,6 +100,11 @@ def lev1_to_nc( ) assert isinstance(rpg_bin, RpgBin) + if data_format == "e-profile": + keys = ["altitude", "latitude", "longitude"] + for key in keys: + rpg_bin.data[f"station_{key}"] = rpg_bin.data.pop(key) + if data_type in ("1B01", "1C01"): apply_qc(site, rpg_bin, params, coeff_files) if data_type in ("1B21", "1C01"): diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 6dc6f41..99601d4 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -68,10 +68,16 @@ def generate_lev2_single( netCDF4.Dataset(t_pot_file.name, "r") as nc_t_pot, netCDF4.Dataset(eq_temp_file.name, "r") as nc_eq_temp, ): - nc_output.createDimension("height", len(nc_t_prof.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) if data_format == "e-profile": + nc_output.createDimension( + "altitude", len(nc_t_prof.variables["altitude"][:]) + ) nc_output.createDimension("bnds", 2) + else: + nc_output.createDimension( + "height", len(nc_t_prof.variables["height"][:]) + ) for source, variables in ( ( @@ -100,7 +106,7 @@ def generate_lev2_single( "temperature", "temperature_random_error", "temperature_systematic_error", - "height", + "height" if data_format == "cloudnet" else "altitude", "temperature_quality_flag", "temperature_quality_flag_status", ), @@ -110,9 +116,11 @@ def generate_lev2_single( ( "time", "time_bnds", - "latitude", - "longitude", - "altitude", + "latitude" if data_format == "cloudnet" else "station_latitude", + "longitude" + if data_format == "cloudnet" + else "station_longitude", + "altitude" if data_format == "cloudnet" else "station_altitude", "lwp", "lwp_offset", "lwp_random_error", @@ -234,10 +242,16 @@ def generate_lev2_lhumpro( netCDF4.Dataset(iwv_file.name, "r") as nc_iwv, netCDF4.Dataset(abs_hum_file.name, "r") as nc_abs_hum, ): - nc_output.createDimension("height", len(nc_abs_hum.variables["height"][:])) nc_output.createDimension("time", len(nc_lwp.variables["time"][:])) if data_format == "e-profile": + nc_output.createDimension( + "altitude", len(nc_abs_hum.variables["altitude"][:]) + ) nc_output.createDimension("bnds", 2) + else: + nc_output.createDimension( + "height", len(nc_abs_hum.variables["height"][:]) + ) for source, variables in ( ( @@ -253,7 +267,7 @@ def generate_lev2_lhumpro( ( nc_abs_hum, ( - "height", + "height" if data_format == "cloudnet" else "altitude", "absolute_humidity", "absolute_humidity_random_error", "absolute_humidity_systematic_error", @@ -266,9 +280,11 @@ def generate_lev2_lhumpro( ( "time", "time_bnds", - "latitude", - "longitude", - "altitude", + "latitude" if data_format == "cloudnet" else "station_latitude", + "longitude" + if data_format == "cloudnet" + else "station_longitude", + "altitude" if data_format == "cloudnet" else "station_altitude", "lwp", "lwp_offset", "lwp_random_error", @@ -336,9 +352,13 @@ def generate_lev2_multi( netCDF4.Dataset(eq_temp_file.name, "r") as nc_eq_temp, ): nc_output.createDimension("time", len(nc_temp.variables["time"][:])) - nc_output.createDimension("height", len(nc_temp.variables["height"][:])) if data_format == "e-profile": + nc_output.createDimension( + "altitude", len(nc_temp.variables["altitude"][:]) + ) nc_output.createDimension("bnds", 2) + else: + nc_output.createDimension("height", len(nc_temp.variables["height"][:])) for source, variables in ( ( @@ -346,10 +366,12 @@ def generate_lev2_multi( ( "time", "time_bnds", - "height", - "latitude", - "longitude", - "altitude", + "height" if data_format == "cloudnet" else "altitude", + "latitude" if data_format == "cloudnet" else "station_latitude", + "longitude" + if data_format == "cloudnet" + else "station_longitude", + "altitude" if data_format == "cloudnet" else "station_altitude", "elevation_angle", "azimuth_angle", "temperature", diff --git a/mwrpy/level2/lev2_meta_nc.py b/mwrpy/level2/lev2_meta_nc.py index 7961447..384b4f8 100644 --- a/mwrpy/level2/lev2_meta_nc.py +++ b/mwrpy/level2/lev2_meta_nc.py @@ -3,6 +3,7 @@ from collections.abc import Callable from typing import TypeAlias +from mwrpy.level1.lev1_meta_nc import ATTRIBUTES_CN, ATTRIBUTES_EP, DEFINITIONS_QF from mwrpy.utils import MetaData @@ -52,6 +53,14 @@ def get_data_attributes( read_att = att_reader[data_type] attributes = dict(ATTRIBUTES_COM, **read_att) + if data_format == "e-profile": + keys = ["latitude", "longitude", "altitude", "height"] + for key in keys: + if key in attributes: + attributes.pop(key) + attributes = dict(ATTRIBUTES_EP, **attributes) + if "altitude" in rpg_variables: + rpg_variables["altitude"].set_attributes(attributes["altitude"]) for key in list(rpg_variables): if key in attributes: if getattr(attributes[key], "retrieval_type") is not None: @@ -61,6 +70,9 @@ def get_data_attributes( **{field: coeff[field]} ) rpg_variables[key].set_attributes(attributes[key]) + if data_format == "e-profile": + if getattr(attributes[key], "dimensions") == ("time", "height"): + setattr(rpg_variables[key], "dimensions", ("time", "altitude")) else: del rpg_variables[key] @@ -76,43 +88,6 @@ def get_data_attributes( return rpg_variables -ATTRIBUTES_CN = { - "time": MetaData( - comment="Time indication of samples is at end of integration-time", - units="hours since ", - long_name="Time UTC", - standard_name="time", - calendar="standard", - dimensions=("time",), - ), -} - -DEFINITIONS_COM = { - "quality_flag": ( - "\n" - "Bit 1: missing_tb\n" - "Bit 2: tb_below_threshold\n" - "Bit 3: tb_above_threshold\n" - "Bit 4: spectral_consistency_above_threshold\n" - "Bit 5: receiver_sanity_failed\n" - "Bit 6: rain_detected\n" - "Bit 7: sun_moon_in_beam\n" - "Bit 8: tb_offset_above_threshold" - ), - "quality_flag_status": ( - "\n" - "Bit 1: missing_tb_not_checked\n" - "Bit 2: tb_lower_threshold_not_checked\n" - "Bit 3: tb_upper_threshold_not_checked\n" - "Bit 4: spectral_consistency_not_checked\n" - "Bit 5: receiver_sanity_not_checked\n" - "Bit 6: rain_not_checked\n" - "Bit 7: sun_moon_in_beam_not_checked\n" - "Bit 8: tb_offset_not_checked" - ), -} - - ATTRIBUTES_COM = { "time": MetaData( long_name="Time (UTC) of the measurement", @@ -189,7 +164,7 @@ def get_data_attributes( "temperature_quality_flag": MetaData( long_name="Temperature quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -197,7 +172,7 @@ def get_data_attributes( "temperature_quality_flag_status": MetaData( long_name="Temperature quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -238,7 +213,7 @@ def get_data_attributes( "temperature_quality_flag": MetaData( long_name="Temperature quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -246,7 +221,7 @@ def get_data_attributes( "temperature_quality_flag_status": MetaData( long_name="Temperature quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -280,7 +255,7 @@ def get_data_attributes( "absolute_humidity_quality_flag": MetaData( long_name="Absolute humidity quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -288,7 +263,7 @@ def get_data_attributes( "absolute_humidity_quality_flag_status": MetaData( long_name="Absolute humidity quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -411,7 +386,7 @@ def get_data_attributes( "lwp_quality_flag": MetaData( long_name="Liquid water path quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -419,7 +394,7 @@ def get_data_attributes( "lwp_quality_flag_status": MetaData( long_name="Liquid water path quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -448,7 +423,7 @@ def get_data_attributes( "iwv_quality_flag": MetaData( long_name="Integrated water vapour quality flag", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -456,7 +431,7 @@ def get_data_attributes( "iwv_quality_flag_status": MetaData( long_name="Integrated water vapour quality flag status", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), @@ -491,7 +466,7 @@ def get_data_attributes( "stability_quality_flag": MetaData( long_name="Quality flag for stability products", units="1", - definition=DEFINITIONS_COM["quality_flag"], + definition=DEFINITIONS_QF["quality_flag"], comment="0 indicates data with good quality according to applied tests.\n" "The list of (not) applied tests is encoded in quality_flag_status", dimensions=("time",), @@ -499,7 +474,7 @@ def get_data_attributes( "stability_quality_flag_status": MetaData( long_name="Quality flag status for stability products", units="1", - definition=DEFINITIONS_COM["quality_flag_status"], + definition=DEFINITIONS_QF["quality_flag_status"], comment="Checks not executed in determination of quality_flag.\n" "0 indicates quality check has been applied.", dimensions=("time",), diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index b65803c..881d1d3 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -81,7 +81,11 @@ def lev2_to_nc( params = read_config(site, instrument_type, "params") with nc.Dataset(lev1_file) as lev1: - params["altitude"] = ma.median(lev1.variables["altitude"][:]) + params["altitude"] = ( + ma.median(lev1.variables["altitude"][:]) + if data_format == "cloudnet" + else ma.median(lev1.variables["station_altitude"][:]) + ) rpg_dat, coeff, index, scan_time = get_products( site, @@ -95,6 +99,8 @@ def lev2_to_nc( ) _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) _del_att(global_attributes) + if data_format == "e-profile" and "height" in rpg_dat: + rpg_dat["altitude"] = rpg_dat.pop("height") mwr = rpg_mwr.Rpg( rpg_dat, date=num2pydate(lev1.variables["time"][:][0], lev1.variables["time"].units), @@ -118,6 +124,9 @@ def lev2_to_nc( "instrument": instrument_type, "coeff_files": c_files, } + else: + global_attributes["dependencies"] = str(lev1_file).split("/")[-1] + global_attributes["level1_quality_flag_status"] = str(params["flag_status"]) rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) @@ -444,7 +453,7 @@ def get_products( ibl, tb, scan_time = ( np.empty([0, len(coeff["AG"])], np.int32), ma.masked_all((len(freq_ind), len(coeff["AG"]), 0), np.float32), - np.empty([0], np.int32), + np.ma.empty([0], np.int32), ) for ix0v in ix0: @@ -470,7 +479,7 @@ def get_products( tb = np.concatenate( ( tb, - np.expand_dims( + np.ma.expand_dims( lev1["tb"][np.ix_(ix0v + np.flip(ind_ang), freq_ind)].T, 2 ), ), @@ -555,8 +564,18 @@ def get_products( hum_time = _read_time(hum_dat.variables["time"]) tem_time = _read_time(tem_dat.variables["time"]) + hum_height = ( + hum_dat.variables["height"][:] + if "height" in hum_dat.variables + else hum_dat.variables["altitude"][:] + ) + tem_height = ( + tem_dat.variables["height"][:] + if "height" in tem_dat.variables + else tem_dat.variables["altitude"][:] + ) - if len(hum_dat.variables["height"][:]) == len(tem_dat.variables["height"][:]): + if len(hum_height) == len(tem_height): hum_int = interpol_2d( hum_time, hum_dat.variables["absolute_humidity"][:, :], @@ -565,13 +584,13 @@ def get_products( else: hum_int = interpolate_2d( hum_time, - hum_dat.variables["height"][:], + hum_height, hum_dat.variables["absolute_humidity"][:, :], tem_time, - tem_dat.variables["height"][:], + tem_height, ) - rpg_dat["height"] = tem_dat.variables["height"][:] + rpg_dat["height"] = tem_height pres = np.interp(tem_time, lev1["time"][:], lev1["air_pressure"][:]) T = tem_dat.variables["temperature"][:, :] # hum_int is absolute humidity (kg m-3) from the 2P03 product; vapor @@ -659,6 +678,9 @@ def _combine_lev1( "altitude", "latitude", "longitude", + "station_altitude", + "station_latitude", + "station_longitude", ] if index.any(): for ivars in lev1_vars: @@ -717,8 +739,16 @@ def retrieval_input(lev1: dict, coeff: dict) -> np.ndarray: ) bias = np.ones((len(lev1["time"][:]), 1), np.float32) - latitude = float(ma.median(lev1["latitude"])) - longitude = float(ma.median(lev1["longitude"])) + latitude = ( + float(ma.median(lev1["latitude"])) + if "latitude" in lev1 + else float(ma.median(lev1["station_latitude"])) + ) + longitude = ( + float(ma.median(lev1["longitude"])) + if "longitude" in lev1 + else float(ma.median(lev1["station_longitude"])) + ) if coeff["RT"] == -1: ret_in = lev1["tb"][:, :] diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index 17854bc..d2a9b64 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -215,7 +215,9 @@ def save_rpg( dims = { "time": len(rpg.data["time"][:]), "bnds": 2, - "height": len(rpg.data["height"][:]), + "height": len(rpg.data["height"][:]) + if data_format == "cloudnet" + else len(rpg.data["altitude"][:]), } elif data_type in ("2I01", "2I02", "2I06"): dims = {"time": len(rpg.data["time"][:]), "bnds": 2} @@ -286,19 +288,16 @@ def _write_vars2nc(nc_file: netCDF4.Dataset, mwr_variables: dict) -> None: def _add_standard_global_attributes(nc_file: netCDF4.Dataset, att_global) -> None: - nc_file.mwrpy_version = version.__version__ - nc_file.processed = ( - datetime.datetime.now(tz=datetime.timezone.utc).strftime("%d %b %Y %H:%M:%S") - + " UTC" - ) for name, value in att_global.items(): + if name == "history": + value = f"{datetime.datetime.now(tz=datetime.timezone.utc).strftime('%d %b %Y %H:%M:%S')} UTC, mwrpy {version.__version__}" if value is None: value = "" setattr(nc_file, name, value) def _add_cloudnet_global_attributes( - nc_file: netCDF4.Dataset, add_global, data_type + nc_file: netCDF4.Dataset, add_global: dict, data_type: str ) -> None: t_zone = datetime.timezone.utc form = "%Y-%m-%d %H:%M:%S" @@ -333,4 +332,6 @@ def _add_cloudnet_global_attributes( if value is None: value = "" setattr(nc_file, name, value) - nc_file.mwrpy_coefficients = ", ".join(add_global["coeff_files"]) + nc_file.mwrpy_coefficients = ", ".join( + [file.split("/")[-1] for file in add_global["coeff_files"]] + ) diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index a76206c..24394fb 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -184,6 +184,9 @@ global_specs: # Logbook repair/replacement work performed instrument_history: + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: + # Manufacturer of the infrared radiometer ir_instrument_manufacturer: Heitronics diff --git a/mwrpy/site_config/hyytiala/config.yaml b/mwrpy/site_config/hyytiala/config.yaml index 9049c7c..780ca57 100644 --- a/mwrpy/site_config/hyytiala/config.yaml +++ b/mwrpy/site_config/hyytiala/config.yaml @@ -74,3 +74,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/site_config/juelich/config.yaml b/mwrpy/site_config/juelich/config.yaml index 9f8fd39..1a8b7f2 100644 --- a/mwrpy/site_config/juelich/config.yaml +++ b/mwrpy/site_config/juelich/config.yaml @@ -77,3 +77,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/site_config/lindenberg/config.yaml b/mwrpy/site_config/lindenberg/config.yaml index 9cdb7dd..f32afbc 100644 --- a/mwrpy/site_config/lindenberg/config.yaml +++ b/mwrpy/site_config/lindenberg/config.yaml @@ -74,3 +74,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/site_config/palaiseau/config.yaml b/mwrpy/site_config/palaiseau/config.yaml index d9e9273..c8a95fe 100644 --- a/mwrpy/site_config/palaiseau/config.yaml +++ b/mwrpy/site_config/palaiseau/config.yaml @@ -74,3 +74,6 @@ global_specs: # Logbook repair/replacement work performed met_instrument_history: + + # Checks not executed in determination of level 1 quality flag + level1_quality_flag_status: diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 32bfd53..3d2526a 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -249,7 +249,7 @@ def add_interpol1d( interpolated_mask = ( np.interp(data0["time"], valid_time, valid_mask.astype(float)) < 0.5 ) - result = ma.masked_array(interpolated_values, mask=interpolated_mask) + result = np.ma.masked_array(interpolated_values, mask=interpolated_mask) interpolated_data = ( result if len(interpolated_data) == 0 @@ -607,8 +607,19 @@ def _get_filename(prod: str, date_in: datetime.date, site: str) -> str: data_out_dir = os.path.join( params["data_out"], f"level{level}", date_in.strftime("%Y/%m/%d") ) - wigos_id = global_attributes["wigos_station_id"] - filename = f"MWR_{prod}_{wigos_id}_{date_in.strftime('%Y%m%d')}.nc" + wigos_id = ( + global_attributes["wigos_station_id"] + if global_attributes["wigos_station_id"] is not None + else site + ) + instrument_id = ( + global_attributes["instrument_id"] + if global_attributes["instrument_id"] is not None + else "A" + ) + filename = ( + f"MWR_{prod}_{wigos_id}_{instrument_id}{date_in.strftime('%Y%m%d')}.nc" + ) return os.path.join(data_out_dir, filename) From 7b2660f9e6885bff28e0ecfb2e1dc42c9733bca8 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 13:23:33 +0200 Subject: [PATCH 17/28] Fix array type --- mwrpy/level2/write_lev2_nc.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 881d1d3..382431f 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -476,11 +476,15 @@ def get_products( axis=0, ) ibl = np.append(ibl, [ix0v + np.flip(ind_ang)], axis=0) - tb = np.concatenate( + tb = np.ma.concatenate( ( tb, np.ma.expand_dims( - lev1["tb"][np.ix_(ix0v + np.flip(ind_ang), freq_ind)].T, 2 + np.ma.array( + lev1["tb"][np.ix_(ix0v + np.flip(ind_ang), freq_ind)].T, + np.float32, + ), + 2, ), ), axis=2, From c1727c4f8f2f2c0aaf8a145571bb45d60bd8829b Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 13:51:46 +0200 Subject: [PATCH 18/28] Account for different height variable --- mwrpy/rpg_mwr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index d2a9b64..fb30c51 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -215,10 +215,12 @@ def save_rpg( dims = { "time": len(rpg.data["time"][:]), "bnds": 2, - "height": len(rpg.data["height"][:]) - if data_format == "cloudnet" - else len(rpg.data["altitude"][:]), } + dims = ( + dict(dims, **{"height": len(rpg.data["height"][:])}) + if data_format == "cloudnet" + else dict(dims, **{"altitude": len(rpg.data["altitude"][:])}) + ) elif data_type in ("2I01", "2I02", "2I06"): dims = {"time": len(rpg.data["time"][:]), "bnds": 2} elif data_type == "2S02": From 63b0a89494fd4120cf98c0fda4b6772ad89c5350 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 21 Jul 2026 14:40:07 +0200 Subject: [PATCH 19/28] Provide instrument type with site name --- mwrpy/level2/write_lev2_nc.py | 3 +++ mwrpy/utils.py | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 382431f..76b071d 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -24,6 +24,7 @@ interpolate_2d, isbit, read_config, + read_site_config_yaml, ) @@ -76,6 +77,8 @@ def lev2_to_nc( ): raise ValueError(f"Data type {data_type} not recognised") + if instrument_type is None and site is not None: + instrument_type = read_site_config_yaml(site)["type"] assert instrument_type is not None global_attributes = read_config(site, instrument_type, "global_specs") params = read_config(site, instrument_type, "params") diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 3d2526a..5c1e256 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -360,14 +360,14 @@ def read_config( key: Literal["global_specs", "params"], ) -> dict: if site is not None: - itype = _read_site_config_yaml(site)["type"] + itype = read_site_config_yaml(site)["type"] elif instrument_type is not None: itype = instrument_type else: raise ValueError("site or instrument_type is required") data = _read_itype_config_yaml(itype)[key] if site is not None: - data.update(_read_site_config_yaml(site)[key]) + data.update(read_site_config_yaml(site)[key]) return data @@ -383,7 +383,7 @@ def _read_itype_config_yaml(itype: str) -> dict: return yaml.load(f, Loader=SafeLoader) -def _read_site_config_yaml(site: str) -> dict: +def read_site_config_yaml(site: str) -> dict: """Reads configuration file for specific site.""" dir_name = os.path.dirname(os.path.realpath(__file__)) site_file = os.path.join(dir_name, "site_config", site, "config.yaml") From d3c252f940a372edd0ed8e8f1d640af6fcafd261 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Tue, 25 Aug 2026 15:39:09 +0200 Subject: [PATCH 20/28] Improve usage of different data formats and update docs --- README.md | 8 +- docs/source/command_line_usage.rst | 29 +---- docs/source/fileformat.rst | 24 +++- docs/source/mwrpy_processing.rst | 71 +++++------ docs/source/overview.rst | 7 +- mwrpy/level1/droplet_mwrpy.py | 2 +- mwrpy/level1/lev1_meta_nc.py | 60 ++++------ mwrpy/level1/write_lev1_nc.py | 16 ++- mwrpy/level2/lev2_collocated.py | 6 + mwrpy/level2/lev2_meta_nc.py | 16 +++ mwrpy/level2/write_lev2_nc.py | 88 +++++++++----- mwrpy/plots/generate_plots.py | 182 ++++++++++++++--------------- mwrpy/plots/plot_utils.py | 7 +- mwrpy/process_mwrpy.py | 141 ++++++++++++---------- mwrpy/rpg_mwr.py | 18 ++- mwrpy/site_config/hatpro.yaml | 15 ++- mwrpy/site_config/lhatpro.yaml | 13 +++ mwrpy/site_config/lhumpro_u90.yaml | 10 ++ mwrpy/utils.py | 48 +++++--- 19 files changed, 435 insertions(+), 326 deletions(-) diff --git a/README.md b/README.md index f233b7a..d8eb6c5 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,10 @@ The folder `mwrpy/site_config/` contains configuration files for each instrument type, which defines the input and output data paths etc. For example, this is the [configuration file for RPG-HATPRO](mwrpy/site_config/hatpro.yaml). -The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a -site and instrument specific configuration file (`config.yaml`) and retrieval coefficients. -For example, this is the [configuration file for Hyytiälä](mwrpy/site_config/hyytiala/config.yaml). -This site configuration file is not needed when using the Cloudnet file format. +The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a folder with retrieval coefficients +(`mwrpy/site_config/hyytiala/coefficients/`) and a site and instrument specific configuration file (`config.yaml`). +For example, this is the [configuration file for Hyytiälä](mwrpy/site_config/hyytiala/config.yaml), which is optional +and helps with configuring multiple instruments of the same type. ## Command line usage diff --git a/docs/source/command_line_usage.rst b/docs/source/command_line_usage.rst index 56edfe6..d941470 100644 --- a/docs/source/command_line_usage.rst +++ b/docs/source/command_line_usage.rst @@ -2,9 +2,9 @@ Command line usage ================== -After defining the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and site specific -information (``mwrpy/site_config/{site}.yaml``, only for E-PROFILE format) files, including input/output data -paths, MWRpy can also be run using the command line tool `mwrpy/cli.py`: +After defining the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and optional site +specific information (``mwrpy/site_config/{site}/config.yaml``, only for E-PROFILE format) files, MWRpy can also be +run using the command line tool `mwrpy/cli.py`: .. code-block:: @@ -42,34 +42,15 @@ paths, MWRpy can also be run using the command line tool `mwrpy/cli.py`: * - `-p` - `--products` - 1C01, single, multi - - Processed products, e.g, `1C01, 2I02, 2P03, single`, see Data Types below. + - Processed products, e.g, `1C01, 2I02, 2P03, single`, see Data Types. * - `-f` - `--format` - cloudnet - Data format to be used (`cloudnet`, `e-profile`). - -The following arguments are used for the Cloudnet file format: - -.. list-table:: Arguments - :widths: 10 20 20 50 - :header-rows: 1 - - * - Short - - Long - - Default - - Description * - `-i` - `--instrument` - hatpro - Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). - * - `-a` - - `--altitude` - - 0.0 - - Altitude above mean sea level of site (m). - * - `-o` - - `--azimuth_offset` - - None - - Azimuth offset of the instrument (degrees). Or `None`. These commands are available to select the processing mode: @@ -98,7 +79,7 @@ To process and plot Level 1 & 2 data (1C01, single, multi) for the site `Hyytial python mwrpy/cli.py -s hyytiala -d 2023-04-06 -f e-profile process -Run the following command for the Cloudnet format (with site altitude 150 m) and no plots: +Run the following command for the Cloudnet format and no plots: .. code-block:: diff --git a/docs/source/fileformat.rst b/docs/source/fileformat.rst index 75eb98f..6647270 100644 --- a/docs/source/fileformat.rst +++ b/docs/source/fileformat.rst @@ -51,19 +51,19 @@ All MWRpy files use ``NETCDF4_CLASSIC`` data model, i.e., ``HDF5`` file format. - seconds since 1970-01-01 00:00:00.000 - int32 - - * - latitude + * - station_latitude - Latitude of measurement station - time - degree_north - float32 - latitude - * - longitude + * - station_longitude - Longitude of measurement station - time - degree_east - float32 - longitude - * - altitude + * - station_altitude - Altitude above mean sea level of measurement station - time - m @@ -408,13 +408,26 @@ MWR-Level 2 files - degree - float32 - sensor_elevation_angle + * - quality_flag + - General quality flag + - time + - 1 + - int32 + - + * - quality_flag_status + - General quality flag status + - time + - 1 + - int32 + - Single pointing file ~~~~~~~~~~~~~~~~~~~~ The Level 2 default file type ``single`` contains all variables from the file types ``2I01``, ``2I02``, ``2I06``, ``2P01``, and ``2P03`` (if the respective retrieval coefficients are available) and is -available for the E-PROFILE and Cloudnet data format. +available for the E-PROFILE and Cloudnet data format. The variable / dimension ``height`` is named ``altitude`` in +the E-PROFILE data format. **Variables (MWR_2I01 specific)** @@ -631,7 +644,8 @@ Multiple pointing file The Level 2 default file type ``multi`` contains all variables from the file types ``2P02``, ``2P04``, ``2P07``, and ``2P08`` (if the respective retrieval coefficients are available) and is -available for the E-PROFILE and Cloudnet data format. +available for the E-PROFILE and Cloudnet data format. The variable / dimension ``height`` is named ``altitude`` in +the E-PROFILE data format. **Variables (MWR_2P02 specific)** diff --git a/docs/source/mwrpy_processing.rst b/docs/source/mwrpy_processing.rst index 2a001e6..09b30c8 100644 --- a/docs/source/mwrpy_processing.rst +++ b/docs/source/mwrpy_processing.rst @@ -19,17 +19,17 @@ quality control and visualization. This example utilizes files taken from the AC First steps for processing examples: -First we define the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and site specific -information (``mwrpy/site_config/{site}.yaml``, only for E-PROFILE format) files, including input/output data paths. -Then the data path is specified: +First we define the instrument type configuration file (``mwrpy/site_config/{i_type}.yaml``), including instrument +specific information. An optional site specific configuration file (e.g. ``mwrpy/site_config/{site_name}/config +.yaml``) can be configured, when dealing with multiple instruments of the same type. Then the data path is specified: .. code-block:: python import os package_dir = os.getcwd() - site = "hyytiala" - data_path = f"{package_dir}/tests/data/{site}" + site_name = "hyytiala" + data_path = f"{package_dir}/tests/data/{site_name}" E-PROFILE format ---------------- @@ -53,10 +53,10 @@ flags are derived: Quality flags are stored as bits and Bit 1-3 include checks for missing brightness temperature values and their valid range (2.7 - 330 K). The spectral consistency flag (Bit 4) compares measured and retrieved TB. For this flag, it is expected to have the corresponding RPG retrieval coefficient file (``SPC*.RET``) in -``/mwrpy/site_config/{site}/coefficients/``. Data from HKD files are used to determine the stability of the receiver -components (Bit 5). The sensor from the attached weather station detects rain for quality Bit 6 and the sun and moon -orbits are calculated and compared to the measurement geometry to detect potential interferences (Bit 7). A quality -flag status variable contains information whether the flag is active. +``/mwrpy/site_config/{site_name}/coefficients/``. Data from HKD files are used to determine the stability of the +receiver components (Bit 5). The sensor from the attached weather station detects rain for quality Bit 6 and the sun +and moon orbits are calculated and compared to the measurement geometry to detect potential interferences (Bit 7). A +quality flag status variable contains information whether the flag is active. .. code-block:: python @@ -66,8 +66,9 @@ flag status variable contains information whether the flag is active. data_type="1C01", path_to_files=data_path, data_format="e-profile", - site=site, - output_file="mwr_1c.nc", + site=site_name, + instrument_type="hatpro", + output_file=f"{data_path}/mwr_1c.nc", ) The data format of the generated ``mwr_1c.nc`` file, including metadata information and variable names, is @@ -80,7 +81,7 @@ Variables such as brightness temperature can be plotted from the newly generated from mwrpy.plots.generate_plots import generate_figure - generate_figure('mwr_1c.nc', ['tb'], save_path=f"{package_dir}/") + generate_figure(f"{data_path}/mwr_1c.nc", ['tb'], save_path=f"{data_path}/") .. figure:: _static/20230406_hyytiala_tb.png @@ -96,10 +97,11 @@ are applied to generate the Level 2 single pointing product: from mwrpy.level2.lev2_collocated import generate_lev2_single mwr_prod = generate_lev2_single( - site="hyytiala", + site=site_name, + instrument_type="hatpro", data_format="e-profile", - mwr_l1c_file="mwr_1c.nc", - output_file="mwr-single.nc", + mwr_l1c_file=f"{data_path}/mwr_1c.nc", + output_file=f"{data_path}/mwr-single.nc", ) Variables such as integrated water vapor @@ -110,7 +112,7 @@ can be plotted from the newly generated file. from mwrpy.plots.generate_plots import generate_figure - generate_figure('mwr-single.nc', ['iwv'], save_path=f"{package_dir}/") + generate_figure(f"{data_path}/mwr-single.nc", ['iwv'], save_path=f"{data_path}/") .. figure:: _static/20230406_hyytiala_iwv.png @@ -126,10 +128,11 @@ product: from mwrpy.level2.lev2_collocated import generate_lev2_multi mwr_prod = generate_lev2_multi( - site="hyytiala", + site=site_name, + instrument_type="hatpro", data_format="e-profile", - mwr_l1c_file="mwr_1c.nc", - output_file="mwr-multi.nc", + mwr_l1c_file=f"{data_path}/mwr_1c.nc", + output_file=f"{data_path}/mwr-multi.nc", ) Variables such as temperature profiles can be plotted from the newly generated file. @@ -138,7 +141,7 @@ Variables such as temperature profiles can be plotted from the newly generated f from mwrpy.plots.generate_plots import generate_figure - generate_figure('mwr-multi.nc', ['temperature'], save_path=f"{package_dir}/") + generate_figure(f"{data_path}/mwr-multi.nc", ['temperature'], save_path=f"{data_path}/") .. figure:: _static/20230406_hyytiala_temperature.png @@ -159,8 +162,8 @@ Download raw data (binary files): instrument_pid = "https://hdl.handle.net/21.12132/3.f360a2375f3e4e4f" # check https://cloudnet.fmi.fi/instruments to find the PID of your instrument client = APIClient() instruments = client.instruments() - instrument_type = [i.instrument_id for i in instruments if i.pid == instrument_pid][0] - files = client.raw_files(site_id=site, instrument_id=instrument_type, date=date) + i_type = [i.instrument_id for i in instruments if i.pid == instrument_pid][0] + files = client.raw_files(site_id=site_name, instrument_id=i_type, date=date) binary_files = [f for f in files] binary_filepaths = await client.adownload(binary_files, data_path) @@ -176,7 +179,7 @@ Download retrieval files: retrieval_files = [] for file in retrieval["coefficientLinks"]: - filename = data_path + file.split("/")[-1] + filename = f"{data_path}/{file.split('/')[-1]}" response = requests.get(file) with open(filename, "wb") as f: f.write(response.content) @@ -189,7 +192,7 @@ defined. Also, the retrieval files are set as an argument. .. code-block:: python - site_info = client.site(site_id=site) + site_info = client.site(site_id=site_name) site_meta = { "name": site_info.id, "altitude": site_info.altitude, @@ -202,8 +205,8 @@ defined. Also, the retrieval files are set as an argument. data_type="1C01", path_to_files=data_path, data_format="cloudnet", - instrument_type=instrument_type, - output_file="mwr_1c.nc", + instrument_type=i_type, + output_file=f"{data_path}/mwr_1c_cn.nc", coeff_files=retrieval_files, instrument_config=site_meta, ) @@ -213,7 +216,7 @@ For plotting, the instrument needs to be defined. In this example, the figure is .. code-block:: python from mwrpy.plots.generate_plots import generate_figure - fig_name = generate_figure('mwr_1c.nc', ['tb'], show=True, instrument_type=instrument_type) + fig_name = generate_figure(f"{data_path}/mwr_1c.nc", ['tb'], show=True, instrument_type=i_type) Process and plot Level 2 data (single & multiple pointing) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -226,22 +229,24 @@ displayed. mwr_prod = generate_lev2_single( site=None, data_format="cloudnet", - mwr_l1c_file="mwr_1c.nc", - output_file="mwr-single.nc", + mwr_l1c_file=f"{data_path}/mwr_1c.nc", + output_file=f"{data_path}/mwr-single.nc", coeff_files=retrieval_files, + instrument_type=i_type, ) from mwrpy.plots.generate_plots import generate_figure - fig_name = generate_figure('mwr-single.nc', ['iwv'], show=True, instrument_type=instrument_type) + fig_name = generate_figure(f"{data_path}/mwr-single.nc", ['iwv'], show=True, instrument_type=i_type) from mwrpy.level2.lev2_collocated import generate_lev2_multi mwr_prod = generate_lev2_multi( site=None, data_format="cloudnet", - mwr_l1c_file="mwr_1c.nc", - output_file="mwr-multi.nc", + mwr_l1c_file=f"{data_path}/mwr_1c.nc", + output_file=f"{data_path}/mwr-multi.nc", coeff_files=retrieval_files, + instrument_type=i_type, ) from mwrpy.plots.generate_plots import generate_figure - fig_name = generate_figure('mwr-multi.nc', ['temperature'], show=True, instrument_type=instrument_type) + fig_name = generate_figure(f"{data_path}/mwr-multi.nc", ['temperature'], show=True, instrument_type=i_type) diff --git a/docs/source/overview.rst b/docs/source/overview.rst index 5be8cce..8cb5025 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -10,9 +10,10 @@ framework of `ACTRIS`_ (Aerosol, Clouds and Trace Gases Research Infrastructure, gain information on the vertical structure of the atmosphere, especially in the lower troposphere, and profiles of temperature and humidity are retrieved together with integrated quantities of water vapor and the cloud liquid water path (LWP). The code is an advancement of the IDL based processing software `mwr_pro`_ and is able to handle raw data -from instruments of the manufacturer Radiometer Physics GmbH (RPG, https://www.radiometer-physics.de/). The output -format, including metadata information, variable names, and file naming of is designed to be compliant with the data -structure and naming convention developed together with the EUMETNET Profiling Programme E-PROFILE (`Rüfenacht 2021`_). +from instruments of the manufacturer Radiometer Physics GmbH (RPG, https://www.radiometer-physics.de/). An additional +output format, including metadata information, variable names, and file naming of is included and designed to be +compliant with the data structure and naming convention developed together with the EUMETNET Profiling Programme +E-PROFILE (`Rüfenacht 2021`_). One of the key components within the ACTRIS center for cloud remote sensing (CCRES) is the synergistic algorithm Cloudnet (`Illingworth 2007`_), which classifies hydrometeors in the atmosphere by combining several ground-based remote diff --git a/mwrpy/level1/droplet_mwrpy.py b/mwrpy/level1/droplet_mwrpy.py index 0de6209..7400185 100644 --- a/mwrpy/level1/droplet_mwrpy.py +++ b/mwrpy/level1/droplet_mwrpy.py @@ -259,7 +259,7 @@ def find_lwcl_free( index_rem = np.array(range(len(lev1["time"]))) if path_to_lidar: # Use lidar data (Cloudnet format) to identify liquid water clouds - lidar = read_lidar(path_to_lidar) + lidar, _ = read_lidar(path_to_lidar) mwr_ind = [ i for i, tt in enumerate(lev1["time"]) diff --git a/mwrpy/level1/lev1_meta_nc.py b/mwrpy/level1/lev1_meta_nc.py index 3c1fc0f..f0c5325 100644 --- a/mwrpy/level1/lev1_meta_nc.py +++ b/mwrpy/level1/lev1_meta_nc.py @@ -35,22 +35,14 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - ) if data_type in ("1B01", "1B11", "1B21"): - read_att = att_reader[data_type] - attributes = dict(ATTRIBUTES_COM, **read_att) + attributes = att_reader[data_type] else: - attributes = dict( - ATTRIBUTES_COM, **ATTRIBUTES_1B01, **ATTRIBUTES_1B11, **ATTRIBUTES_1B21 - ) - if data_format == "cloudnet": - attributes.pop("time") - attributes = dict(ATTRIBUTES_CN, **attributes) - - if data_format == "e-profile": - keys = ["latitude", "longitude", "altitude", "height"] - for key in keys: - if key in attributes: - attributes.pop(key) - attributes = dict(ATTRIBUTES_EP, **attributes) + attributes = dict(ATTRIBUTES_1B01, **ATTRIBUTES_1B11, **ATTRIBUTES_1B21) + attributes = ( + dict(ATTRIBUTES_CN, **attributes) + if data_format == "cloudnet" + else dict(ATTRIBUTES_EP, **attributes) + ) for key in list(rpg_variables): if key in attributes: @@ -75,38 +67,28 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - calendar="standard", dimensions=("time",), ), -} - - -ATTRIBUTES_EP = { - "station_latitude": MetaData( + "latitude": MetaData( long_name="Latitude of measurement station", standard_name="latitude", - units="degree_north", + units="degrees_north", dimensions=("time",), ), - "station_longitude": MetaData( + "longitude": MetaData( long_name="Longitude of measurement station", standard_name="longitude", - units="degree_east", + units="degrees_east", dimensions=("time",), ), - "station_altitude": MetaData( + "altitude": MetaData( long_name="Altitude above mean sea level of measurement station", standard_name="altitude", units="m", dimensions=("time",), ), - "altitude": MetaData( - long_name="Height above mean sea level", - standard_name="height_above_mean_sea_level", - units="m", - dimensions=("altitude",), - ), } -ATTRIBUTES_COM = { +ATTRIBUTES_EP = { "time": MetaData( long_name="Time (UTC) of the measurement", units="seconds since 1970-01-01 00:00:00.000", @@ -118,24 +100,30 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - units="seconds since 1970-01-01 00:00:00.000", dimensions=("time", "bnds"), ), - "latitude": MetaData( + "station_latitude": MetaData( long_name="Latitude of measurement station", standard_name="latitude", - units="degrees_north", + units="degree_north", dimensions=("time",), ), - "longitude": MetaData( + "station_longitude": MetaData( long_name="Longitude of measurement station", standard_name="longitude", - units="degrees_east", + units="degree_east", dimensions=("time",), ), - "altitude": MetaData( + "station_altitude": MetaData( long_name="Altitude above mean sea level of measurement station", standard_name="altitude", units="m", dimensions=("time",), ), + "altitude": MetaData( + long_name="Height above mean sea level", + standard_name="height_above_mean_sea_level", + units="m", + dimensions=("altitude",), + ), } diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index c0fff07..dcf2657 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -24,6 +24,7 @@ get_file_list, isbit, read_config, + read_lidar, update_lev1_attributes, ) @@ -117,20 +118,31 @@ def lev1_to_nc( c_files = ( get_coeff_list( site, - ["spc", "ins", "lwp", "iwv", "hpt", "tpt", "tpb", "tbx"], + ["spc", "ins", "tbx"], None, params.get("coeff_path", None), ) if coeff_files is None else coeff_files ) + i_gen = read_config(None, instrument_type, "global_specs")[ + "instrument_generation" + ] + _, lidar_meta = read_lidar(lidar_path) if lidar_path else (None, None) global_attributes = { "site": site, "instrument": instrument_type, "coeff_files": c_files, + "instrument_generation": i_gen, + "history": lidar_meta["history"] if lidar_meta else None, + "source": lidar_meta["source"] if lidar_meta else None, } else: global_attributes = read_config(site, instrument_type, "global_specs") + global_attributes["site_location"] = site + global_attributes["dependencies"] = ( + str(lidar_path).split("/")[-1] if lidar_path else None + ) _update_calibration_attributes(rpg_bin, global_attributes) if data_type != "1C01": update_lev1_attributes(global_attributes, data_type) @@ -145,8 +157,8 @@ def prepare_data( lidar_path: str | PathLike | None, time_offset: datetime.timedelta | None = None, altitude: float | None = None, - date: float | None = None, azimuth_offset: float | None = None, + date: float | None = None, ) -> RpgBin | dict: """Load and prepare data for netCDF writing.""" if data_type in ("1B01", "1C01"): diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 99601d4..4109610 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -121,6 +121,8 @@ def generate_lev2_single( if data_format == "cloudnet" else "station_longitude", "altitude" if data_format == "cloudnet" else "station_altitude", + "quality_flag", + "quality_flag_status", "lwp", "lwp_offset", "lwp_random_error", @@ -285,6 +287,8 @@ def generate_lev2_lhumpro( if data_format == "cloudnet" else "station_longitude", "altitude" if data_format == "cloudnet" else "station_altitude", + "quality_flag", + "quality_flag_status", "lwp", "lwp_offset", "lwp_random_error", @@ -372,6 +376,8 @@ def generate_lev2_multi( if data_format == "cloudnet" else "station_longitude", "altitude" if data_format == "cloudnet" else "station_altitude", + "quality_flag", + "quality_flag_status", "elevation_angle", "azimuth_angle", "temperature", diff --git a/mwrpy/level2/lev2_meta_nc.py b/mwrpy/level2/lev2_meta_nc.py index 384b4f8..b0624c1 100644 --- a/mwrpy/level2/lev2_meta_nc.py +++ b/mwrpy/level2/lev2_meta_nc.py @@ -131,6 +131,22 @@ def get_data_attributes( comment="0=horizon, 90=zenith", dimensions=("time",), ), + "quality_flag": MetaData( + long_name="General quality flag", + units="1", + definition=DEFINITIONS_QF["quality_flag"], + comment="0 indicates data with good quality according to applied tests.\n" + "The list of (not) applied tests is encoded in quality_flag_status", + dimensions=("time",), + ), + "quality_flag_status": MetaData( + long_name="General quality flag status", + units="1", + definition=DEFINITIONS_QF["quality_flag_status"], + comment="Checks not executed in determination of quality_flag.\n" + "0 indicates quality check has been applied.", + dimensions=("time",), + ), } diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 76b071d..62e114b 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -77,11 +77,9 @@ def lev2_to_nc( ): raise ValueError(f"Data type {data_type} not recognised") - if instrument_type is None and site is not None: - instrument_type = read_site_config_yaml(site)["type"] - assert instrument_type is not None - global_attributes = read_config(site, instrument_type, "global_specs") params = read_config(site, instrument_type, "params") + global_attributes = read_config(site, instrument_type, "global_specs") + global_attributes["site_location"] = site with nc.Dataset(lev1_file) as lev1: params["altitude"] = ( @@ -104,9 +102,12 @@ def lev2_to_nc( _del_att(global_attributes) if data_format == "e-profile" and "height" in rpg_dat: rpg_dat["altitude"] = rpg_dat.pop("height") + l2_date = num2pydate( + lev1.variables["time"][:][0], lev1.variables["time"].units + ).strftime("%Y%m%d") mwr = rpg_mwr.Rpg( rpg_dat, - date=num2pydate(lev1.variables["time"][:][0], lev1.variables["time"].units), + date=datetime.strptime(l2_date, "%Y%m%d").date(), ) mwr.data = get_data_attributes(mwr.data, data_type, coeff, data_format) if data_format == "cloudnet": @@ -126,9 +127,15 @@ def lev2_to_nc( "site": site, "instrument": instrument_type, "coeff_files": c_files, + "history": lev1.history, + "source": lev1.source, } else: - global_attributes["dependencies"] = str(lev1_file).split("/")[-1] + global_attributes["dependencies"] = ( + (f"{lev1.dependencies}\n" f"{str(lev1_file).split('/')[-1]}") + if lev1.dependencies + else str(lev1_file).split("/")[-1] + ) global_attributes["level1_quality_flag_status"] = str(params["flag_status"]) rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) @@ -159,7 +166,11 @@ def get_products( if data_type in ("2I01", "2I02", "2I06"): product = ( - "lwp" if data_type == "2I01" else "iwv" if data_type == "2I02" else "sta" + "lwp" + if data_type == "2I01" + else "iwv" + if data_type == "2I02" + else "stability" ) coeff = get_mvr_coeff( @@ -226,7 +237,7 @@ def get_products( hidden_layer[:, 1:] = np.tanh( fac[:] * np.einsum("ijk,ij->ik", c_w1, ret_in[index, :]) ) - if product == "sta": + if product == "stability": tmp_product = np.squeeze( np.tanh(fac[:] * np.einsum("ij,ikj->ik", hidden_layer, c_w2)) * op_sc @@ -260,10 +271,10 @@ def get_products( axis=1, ) )[0] # type: ignore + _get_qf(rpg_dat, lev1, coeff, index, index_ret, product) if product in ("lwp", "iwv"): ret_product = ma.masked_all(len(index), np.float32) ret_product[index_ret] = tmp_product[index_ret] - _get_qf(rpg_dat, lev1, coeff, index, index_ret, product) if product == "lwp": if params["flag_status"][3] == 0 and np.all( isbit(lev1["met_quality_flag"][index], 3) @@ -313,8 +324,6 @@ def get_products( ret_product[index_ret] = tmp_product[index_ret, ind] rpg_dat[prd] = ret_product - _get_qf(rpg_dat, lev1, coeff, index, index_ret, "stability") - elif data_type in ("2P01", "2P03"): if data_type == "2P01": product, ret = "temperature", "tpt" @@ -625,6 +634,14 @@ def get_products( atmoslib.equivalent_potential_temperature(T, p_baro, q_moist) ) + _get_qf( + rpg_dat, + lev1, + coeff, + np.array(range(len(tem_time))), + np.array(range(len(tem_time))), + "derived", + ) _combine_lev1( tem_dat, rpg_dat, @@ -644,28 +661,37 @@ def _get_qf( product: str, scan: np.ndarray = np.empty([0], np.int32), ) -> None: - rpg_dat[product + "_quality_flag"] = ma.masked_all((len(index)), np.int32) - rpg_dat[product + "_quality_flag_status"] = ma.masked_all((len(index)), np.int32) - - _, freq_ind, _ = np.intersect1d( - lev1["frequency"][:], - coeff["FR"][:], - assume_unique=False, - return_indices=True, + rpg_dat["quality_flag"] = np.bitwise_or.reduce( + lev1["quality_flag"][index[index_ret], :], axis=1 ) - if scan.any(): - for ind, _ in enumerate(scan[:, 0]): - flg = np.bitwise_or.reduce( - lev1["quality_flag"][np.ix_(scan[ind, :], freq_ind)], axis=1 - ) - rpg_dat[product + "_quality_flag"][ind] = np.bitwise_or.reduce(flg) - else: - rpg_dat[product + "_quality_flag"][index_ret] = np.bitwise_or.reduce( - lev1["quality_flag"][np.ix_(index[index_ret], freq_ind)], axis=1 + rpg_dat["quality_flag_status"] = np.bitwise_or.reduce( + lev1["quality_flag_status"][index[index_ret], :], axis=1 + ) + if product != "derived": + rpg_dat[product + "_quality_flag"] = ma.masked_all((len(index)), np.int32) + rpg_dat[product + "_quality_flag_status"] = ma.masked_all( + (len(index)), np.int32 ) - rpg_dat[product + "_quality_flag_status"][index_ret] = lev1["quality_flag_status"][ - :, freq_ind[0] - ][index[index_ret]] + + _, freq_ind, _ = np.intersect1d( + lev1["frequency"][:], + coeff["FR"][:], + assume_unique=False, + return_indices=True, + ) + if scan.any(): + for ind, _ in enumerate(scan[:, 0]): + flg = np.bitwise_or.reduce( + lev1["quality_flag"][np.ix_(scan[ind, :], freq_ind)], axis=1 + ) + rpg_dat[product + "_quality_flag"][ind] = np.bitwise_or.reduce(flg) + else: + rpg_dat[product + "_quality_flag"][index_ret] = np.bitwise_or.reduce( + lev1["quality_flag"][np.ix_(index[index_ret], freq_ind)], axis=1 + ) + rpg_dat[product + "_quality_flag_status"][index_ret] = lev1[ + "quality_flag_status" + ][:, freq_ind[0]][index[index_ret]] def _combine_lev1( diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index 6d2bba3..cf8b877 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -2,7 +2,6 @@ import glob import locale -import logging from datetime import date, datetime, timezone import atmoslib @@ -187,85 +186,80 @@ def generate_figure( valid_fields, valid_names = _find_valid_fields(nc_file, field_names) if len(valid_fields) == 0: return None - file_name = None - try: - fig, axes = _initialize_figure(len(valid_fields), dpi) - time = _read_time_vector(nc_file) - for ax, field, name in zip(axes, valid_fields, valid_names): - ax.set_facecolor(_COLORS["lightgray"]) - is_height = _is_height_dimension(nc_file, name) - if image_name and "_scan" in image_name: - name = image_name - pl_source = ATTRIBUTES[name].source - if "angle" not in name and pl_source not in ( - "met", - "met2", - "irt", - "qf", - "mqf", - "hkd", - "scan", - "cov", - ): - if pointing == 0: - if ax == axes[0]: - time = _elevation_azimuth_filter(nc_file, time, ele_range) - field = _elevation_azimuth_filter(nc_file, field, ele_range) - elif pl_source in ("met", "met2", "irt", "qf", "mqf", "hkd"): + fig, axes = _initialize_figure(len(valid_fields), dpi) + time = _read_time_vector(nc_file) + + for ax, field, name in zip(axes, valid_fields, valid_names): + ax.set_facecolor(_COLORS["lightgray"]) + is_height = _is_height_dimension(nc_file, name) + if image_name and "_scan" in image_name: + name = image_name + pl_source = ATTRIBUTES[name].source + if "angle" not in name and pl_source not in ( + "met", + "met2", + "irt", + "qf", + "mqf", + "hkd", + "scan", + "cov", + ): + if pointing == 0: if ax == axes[0]: - time = _elevation_filter(nc_file, time, ele_range) - field = _elevation_filter(nc_file, field, ele_range) - if title: - _set_title(ax, name, nc_file, "") - if not is_height: - fig = _plot_instrument_data( - ax, - field, - name, - pl_source, - time, - fig, - nc_file, - ele_range, - pointing, - instrument_type, + time = _elevation_azimuth_filter(nc_file, time, ele_range) + field = _elevation_azimuth_filter(nc_file, field, ele_range) + elif pl_source in ("met", "met2", "irt", "qf", "mqf", "hkd"): + if ax == axes[0]: + time = _elevation_filter(nc_file, time, ele_range) + field = _elevation_filter(nc_file, field, ele_range) + if title: + _set_title(ax, name, nc_file, "") + if not is_height: + fig = _plot_instrument_data( + ax, + field, + name, + pl_source, + time, + fig, + nc_file, + ele_range, + pointing, + instrument_type, + ) + else: + height = _read_height_vector(nc_file) + ax_value = (time, height) + field, ax_value = _screen_high_altitudes(field, ax_value, max_y) + _set_ax(ax, max_y) + + plot_type = ATTRIBUTES[name].plot_type + if plot_type == "mesh": + _plot_colormesh_data( + ax, field, name, ax_value, nc_file, instrument_type ) - else: - ax_value = _read_ax_values(nc_file) - ax_value = (time, ax_value[1]) - field, ax_value = _screen_high_altitudes(field, ax_value, max_y) - _set_ax(ax, max_y) - - plot_type = ATTRIBUTES[name].plot_type - if plot_type == "mesh": - _plot_colormesh_data( - ax, field, name, ax_value, nc_file, instrument_type - ) - if axes[-1].get_title() != "empty": - if {"tb_cov_ln2", "tb_cov_amb"} & set(field_names): - case_date = _get_cal_date(nc_file) - site_name = _read_location(nc_file) - _add_subtitle(fig, case_date, site_name) - fig.set_size_inches(9.0, 7.0 * len(axes)) - else: - case_date = ( - _read_date(nc_file) - if image_name and "_scan" in image_name - else _set_labels(fig, axes[-1], nc_file, sub_title, instrument_type) - ) - file_name = handle_saving( - nc_file, image_name, save_path, show, case_date, valid_names - ) + if axes[-1].get_title() != "empty": + if {"tb_cov_ln2", "tb_cov_amb"} & set(field_names): + case_date = _get_cal_date(nc_file) + site_name = _read_location(nc_file) + _add_subtitle(fig, case_date, site_name) + fig.set_size_inches(9.0, 7.0 * len(axes)) else: - return None + case_date = ( + _read_date(nc_file) + if image_name and "_scan" in image_name + else _set_labels(fig, axes[-1], nc_file, sub_title, instrument_type) + ) + file_name = handle_saving( + nc_file, image_name, save_path, show, case_date, valid_names + ) + else: + return None - except Exception as e: - logging.error(f"Error in plotting: {e}.") - finally: - plt.close() - return file_name + return file_name def _mark_gaps( @@ -397,7 +391,9 @@ def _find_valid_fields(nc_file: str, names: list) -> tuple[list, list]: def _is_height_dimension(full_path: str, var_name: str) -> bool: """Checks for height dimension in netCDF file.""" with netCDF4.Dataset(full_path) as nc: - is_height = "height" in nc.variables[var_name].dimensions + is_height = bool( + set(nc.variables[var_name].dimensions) & {"height", "altitude"} + ) return is_height @@ -467,12 +463,15 @@ def _initialize_figure(n_subplots: int, dpi) -> tuple[Figure, list[Axes]]: return fig, axes_list -def _read_ax_values(full_path: str) -> tuple[ndarray, ndarray]: - """Returns time and height arrays.""" - time = read_nc_fields(full_path, "time") - height = read_nc_fields(full_path, "height") - height_km = height / 1000 - return time, height_km +def _read_height_vector(nc_file: str) -> ndarray: + """Converts height vector to km.""" + with netCDF4.Dataset(nc_file) as nc: + height = ( + nc.variables["height"][:] + if "height" in nc.variables + else nc.variables["altitude"][:] + ) + return height / 1000.0 def _read_time_vector(nc_file: str) -> ndarray: @@ -619,11 +618,8 @@ def _plot_segment_data( colorbar = _init_colorbar(pl, ax) colorbar.set_ticks(np.arange(len(clabel))) if name == "quality_flag_3": - if instrument_type is None: - site = _read_location(nc_file) - params = read_config(site, None, "params") - else: - params = read_config(None, instrument_type, "params") + site = _read_location(nc_file) + params = read_config(site, instrument_type, "params") clabel[2] = clabel[2] + " (" + str(params["TB_threshold"][1]) + " K)" clabel[1] = clabel[1] + " (" + str(params["TB_threshold"][0]) + " K)" colorbar.ax.set_yticklabels(clabel, fontsize=13) @@ -1076,11 +1072,8 @@ def _plot_qf( instrument_type: str | None = None, ): """Plot for Level 1 quality flags.""" - if instrument_type is None: - site = _read_location(nc_file) - params = read_config(site, None, "params") - else: - params = read_config(None, instrument_type, "params") + site = _read_location(nc_file) + params = read_config(site, instrument_type, "params") plt.close(fig) nsub = 4 if params["flag_status"][3] == 0 else 3 @@ -1203,11 +1196,8 @@ def _plot_tb( instrument_type: str | None = None, ): """Plot for microwave brightness temperatures.""" - if instrument_type is None: - site = _read_location(nc_file) - params = read_config(site, None, "params") - else: - params = read_config(None, instrument_type, "params") + site = _read_location(nc_file) + params = read_config(site, instrument_type, "params") frequency = read_nc_fields(nc_file, "frequency") quality_flag = read_nc_fields(nc_file, "quality_flag") if name == "tb_spectrum": diff --git a/mwrpy/plots/plot_utils.py b/mwrpy/plots/plot_utils.py index 5b0e9cf..747cc9a 100644 --- a/mwrpy/plots/plot_utils.py +++ b/mwrpy/plots/plot_utils.py @@ -38,11 +38,8 @@ def _get_ret_flag( ) quality_flag = quality_flag[index] flag = np.zeros(len(time), np.int32) - if instrument_type is None: - site = _read_location(nc_file) - params = read_config(site, None, "params") - else: - params = read_config(None, instrument_type, "params") + site = _read_location(nc_file) + params = read_config(site, instrument_type, "params") if params["flag_status"][3] == 0 and bits == 0: flag[isbit(quality_flag[:], 3) > 0] = 1 diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index 6dbb484..ee30fb3 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -7,10 +7,10 @@ import time from typing import Literal -import matplotlib.pyplot as plt import netCDF4 as nc import pandas as pd +import mwrpy.utils from mwrpy.level1.write_lev1_nc import lev1_to_nc, prepare_data from mwrpy.level2.lev2_collocated import ( generate_lev2_lhumpro, @@ -19,14 +19,6 @@ ) from mwrpy.level2.write_lev2_nc import lev2_to_nc from mwrpy.plots.generate_plots import generate_figure -from mwrpy.utils import ( - _get_filename, - _get_filename_cloudnet, - date_range, - get_processing_dates, - isodate2date, - read_config, -) PRODUCT_NAME = { "1B01": [ @@ -101,11 +93,11 @@ def main(args): """Main function for processing and plotting MWR data.""" logging.basicConfig(level="INFO") - _start_date, _stop_date = get_processing_dates(args) - start_date = isodate2date(_start_date) - stop_date = isodate2date(_stop_date) + _start_date, _stop_date = mwrpy.utils.get_processing_dates(args) + start_date = mwrpy.utils.isodate2date(_start_date) + stop_date = mwrpy.utils.isodate2date(_stop_date) - for date in date_range(start_date, stop_date): + for date in mwrpy.utils.date_range(start_date, stop_date): for product in args.products: if product not in PRODUCT_NAME: logging.error(f"Product {product} not recognised") @@ -123,7 +115,7 @@ def main(args): logging.info(f"Processing {product} product, {args.site} {date}") if args.command == "reprocess": try: - process_product( + output_file = process_product( product, date, args.site, @@ -136,8 +128,9 @@ def main(args): logging.error( f"Error in processing products: {e}. Incomplete or no processing for {date}." ) + output_file = None else: - process_product( + output_file = process_product( product, date, args.site, @@ -146,14 +139,13 @@ def main(args): args.altitude, args.azimuth_offset, ) + if output_file: + logging.info("Processed %s: %s", product, output_file) + if args.command != "no-plot": logging.info(f"Plotting {product} product, {args.site} {date}") - try: - plot_product(product, date, args.site, args.format, args.instrument) - except Exception as e: - logging.error(f"Error in plotting product {product}: {e}.") - finally: - plt.close() + plot_product(product, date, args.site, args.format, args.instrument) + elapsed_time = time.process_time() - start logging.info(f"Processing took {elapsed_time:.1f} seconds") @@ -182,13 +174,13 @@ def process_product( azimuth_offset: Azimuth offset to be added to azimuth angle. Returns: - None + output_file: Name of output file. """ - output_file = ( - _get_filename(prod, date, site) - if data_format == "e-profile" - else _get_filename_cloudnet(prod, date, site, instrument) + filename = getattr( + mwrpy.utils, + "_get_filename_cloudnet" if data_format == "cloudnet" else "_get_filename", ) + output_file = filename(prod, date, site, instrument) output_dir = os.path.dirname(output_file) if not os.path.isdir(output_dir): os.makedirs(output_dir) @@ -201,8 +193,8 @@ def process_product( date + datetime.timedelta(days=iday + 1), ] offset_file = [ - _get_filename("lwp_offset", xday[0], site), - _get_filename("lwp_offset", xday[1], site), + filename("lwp_offset", xday[0], site, instrument), + filename("lwp_offset", xday[1], site, instrument), ] if ( (prod in ("2I01", "single")) @@ -226,21 +218,29 @@ def process_product( ].values[0] lwp_offset_tuple = (lwp_offset[0], lwp_offset[1]) - l1_filename = ( - _get_filename("1C01", date, site) - if data_format == "e-profile" - else _get_filename_cloudnet("1C01", date, site, instrument) - ) - + l1_filename = filename("1C01", date, site, instrument) # Process level 1 data if prod[0] == "1": + params = mwrpy.utils.read_config(site, instrument, "params") + if data_format == "e-profile": + altitude = params["altitude"] + if altitude is None: + altitude = 0.0 + logging.info("Site altitude not provided. Taking default of 0 m.") + azimuth_offset = ( + params["azimuth_offset"] + if "azimuth_offset" in params and params["azimuth_offset"] is not None + else 0.0 + ) lev1_to_nc( prod, - _get_raw_file_path(date, site), + _get_raw_file_path(date, site, instrument) + if instrument is None + else _get_raw_file_path(date, None, instrument), data_format, site=site, output_file=output_file, - lidar_path=_get_lidar_file_path(date, site), + lidar_path=_get_lidar_file_path(date, site, params), date=date, instrument_type=instrument, altitude=altitude, @@ -250,16 +250,16 @@ def process_product( # Process level 2 single products elif prod[0] == "2": if prod in ("2P04", "2P07", "2P08"): - temp_file = _get_filename("2P02", date, site) + temp_file = filename("2P02", date, site, instrument) if len(temp_file) == 0: - temp_file = _get_filename("2P01", date, site) - hum_file = _get_filename("2P03", date, site) + temp_file = filename("2P01", date, site, instrument) + hum_file = filename("2P03", date, site, instrument) else: temp_file = None hum_file = None lev2_to_nc( prod, - _get_filename("1C01", date, site), + filename("1C01", date, site, instrument), data_format, output_file=output_file, site=site, @@ -301,7 +301,7 @@ def process_product( ) # Update LWP offset file if necessary - offset_current = _get_filename("lwp_offset", date, site) + offset_current = filename("lwp_offset", date, site, instrument) if ( (prod in ("2I01", "single")) and (os.path.isfile(output_file)) @@ -347,6 +347,8 @@ def process_product( ) csv_off.to_csv(offset_current, index=False) + return output_file + def plot_product(prod: str, date, site: str, data_format: str, instrument: IType): """Plot a given product for a specific date and site. @@ -362,18 +364,18 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType Returns: None """ - filename = ( - _get_filename(prod, date, site) - if data_format == "e-profile" - else _get_filename_cloudnet(prod, date, site, instrument) + filename = getattr( + mwrpy.utils, + "_get_filename_cloudnet" if data_format == "cloudnet" else "_get_filename", ) - if not os.path.isfile(filename): + input_file = filename(prod, date, site, instrument) + if not os.path.isfile(input_file): logging.warning("Nothing to plot for product " + prod) - params = read_config(site, None, "params") - output_dir = f"{os.path.dirname(filename)}/" + params = mwrpy.utils.read_config(site, instrument, "params") + output_dir = f"{os.path.dirname(input_file)}/" # Plot level 1 data - if os.path.isfile(filename) and prod[0] == "1": + if os.path.isfile(input_file) and prod[0] == "1": keymap = { "tb": ["tb"], "tb_spectrum": ["tb_spectrum"], @@ -410,26 +412,28 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType else ["Gain"], save_path=params["path_to_cal"], image_name=key, + instrument_type=instrument, cov_data=his_data, site=site, ) else: - logging.warning("No to plot for product " + prod) + logging.warning("Nothing to plot for product " + prod) else: output_dir = params["path_to_cal"] if key == "cov" else output_dir if output_dir is not None: generate_figure( - filename, + input_file, variables, ele_range=ele_range, save_path=output_dir + "COVARIANCE/" if key == "cov" else output_dir, image_name=key, + instrument_type=instrument, ) # Plot level 2 single products - elif os.path.isfile(filename) and (prod[0] == "2"): + elif os.path.isfile(input_file) and (prod[0] == "2"): for key in PRODUCT_NAME[prod]: elevation = ( ( @@ -446,34 +450,37 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType if prod == "2I06": f_names = f_names_stability generate_figure( - filename, + input_file, f_names, ele_range=elevation, save_path=output_dir, image_name=PRODUCT_NAME[prod][0], title=False, + instrument_type=instrument, ) elif key in ("lwp_scan", "iwv_scan"): generate_figure( - filename, + input_file, [key.rstrip("_scan")], ele_range=elevation, save_path=output_dir, image_name=key, title=False, + instrument_type=instrument, ) else: generate_figure( - filename, + input_file, [key], ele_range=elevation, save_path=output_dir, image_name=key, pointing=pointing, + instrument_type=instrument, ) # Plot level 2 combined products - elif os.path.isfile(filename) and (prod in ("single", "multi")): + elif os.path.isfile(input_file) and (prod in ("single", "multi")): for var_name in PRODUCT_NAME[prod]: elevation = ( ( @@ -504,26 +511,28 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType for key, variables in keymap.items(): if key in ("lwp_scan", "iwv_scan"): generate_figure( - filename, + input_file, [key.rstrip("_scan")], ele_range=elevation, save_path=output_dir, image_name=key, title=False, + instrument_type=instrument, ) else: generate_figure( - filename, + input_file, variables, ele_range=elevation, save_path=output_dir, image_name=key, title=title, pointing=pointing, + instrument_type=instrument, ) # Plot covariance data and calibration history even if 1C01 file is not available - elif prod == "1C01" and not os.path.isfile(filename): + elif prod == "1C01" and not os.path.isfile(input_file): output_dir = params["path_to_cal"] cov_data = prepare_data( "", "cov", params, None, date=time.mktime(date.timetuple()) @@ -534,6 +543,7 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType "", ["tb_cov_ln2", "tb_cov_amb"], save_path=output_dir + "COVARIANCE/", + instrument_type=instrument, image_name="cov", cov_data=cov_data, site=site, @@ -551,6 +561,7 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType if "tb_cov_ln2" in his_data else ["Gain"], save_path=output_dir, + instrument_type=instrument, image_name="his", cov_data=his_data, site=site, @@ -561,31 +572,35 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType logging.warning("Nothing to plot for product " + prod) -def _get_raw_file_path(date_in: datetime.date, site: str) -> str: +def _get_raw_file_path( + date_in: datetime.date, site: str | None, instrument: str | None +) -> str: """Get the raw file path for a given date and site. Args: date_in: Date for which the raw file path is needed. site: Site identifier. + instrument: Instrument identifier. Returns: The raw file path as a string. """ - params = read_config(site, None, "params") + params = mwrpy.utils.read_config(site, instrument, "params") return os.path.join(params["data_in"], date_in.strftime("%Y/%m/%d/")) -def _get_lidar_file_path(date_in: datetime.date, site: str) -> str | None: +def _get_lidar_file_path(date_in: datetime.date, site: str, params: dict) -> str | None: """Get the lidar file path for a given date and site. Args: date_in: Date for which the lidar file path is needed. site: Site identifier. + params: Configuration parameters. Returns: The lidar file path as a string or None if not found. """ - params, path = read_config(site, None, "params"), "" + path = "" lidar_model = params.get("lidar_model", "unknown") lidar_model = "unknown" if lidar_model is None else lidar_model.lower() if "path_to_lidar" in params and params["path_to_lidar"] is not None: diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index fb30c51..a0994f6 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -306,9 +306,8 @@ def _add_cloudnet_global_attributes( instrument = add_global["instrument"].upper() site = add_global["site"] if data_type == "1C01": - level = "mwr-l1c" + level = history = "mwr-l1c" title = f"{instrument} microwave radiometer Level 1c from {site}" - history = level elif data_type == "2P02": level = "mwr-multi" title = f"MWR multiple-pointing from {site}" @@ -325,11 +324,18 @@ def _add_cloudnet_global_attributes( "references": "https://doi.org/10.21105/joss.06733", "cloudnet_file_type": level, "title": title, - "history": f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00" - + " - " - + history - + " file created", + "history": f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00 - {history} file created", } + if "history" in add_global and add_global["history"]: + att_global["source"] = ( + (f"{att_global['source']}\n" f"{add_global['source']}") + if data_type == "1C01" + else f"{add_global['source']}" + ) + att_global["history"] = ( + f"{datetime.datetime.now(tz=t_zone).strftime(form)} +00:00 - {history} file created\n" + f"{add_global['history']}" + ) for name, value in att_global.items(): if value is None: value = "" diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index 24394fb..7ca7604 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -8,6 +8,13 @@ params: # path to retrieval coefficients coeff_path: + # path to Cloudnet lidar file and lidar model (optional) + path_to_lidar: + lidar_model: + + # path to ABSCAL.HIS file + path_to_cal: /tmp/data/ + # availability of IR ir_flag: True @@ -22,12 +29,18 @@ params: # Bit 8: tb_offset_above_threshold flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + # integration time for BL scans in seconds + scan_time: 100. + # integration time of measurements in seconds int_time: 1 # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. + # Azimuth correction: + # Set az_cor to the angle that RPG software gives when instrument is pointing to the North. + const_azi: -999. # some default values: # ------------------- @@ -101,7 +114,7 @@ params: [0., 100.], ] -# Missing entries are filled with site specific config file +# Missing entries can also be filled with site specific config file global_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index a9e7565..6d13282 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -8,6 +8,13 @@ params: # path to retrieval coefficients coeff_path: + # path to Cloudnet lidar file and lidar model (optional) + path_to_lidar: + lidar_model: + + # path to ABSCAL.HIS file + path_to_cal: /tmp/data/ + # availability of IR ir_flag: True @@ -22,12 +29,18 @@ params: # Bit 8: tb_offset_above_threshold flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + # integration time for BL scans in seconds + scan_time: 100. + # integration time of measurements in seconds int_time: 1 # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. + # Azimuth correction: + # Set az_cor to the angle that RPG software gives when instrument is pointing to the North. + const_azi: -999. # some default values: # ------------------- diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index 02c5527..5aa87e7 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -8,6 +8,13 @@ params: # path to retrieval coefficients coeff_path: + # path to Cloudnet lidar file and lidar model (optional) + path_to_lidar: + lidar_model: + + # path to ABSCAL.HIS file + path_to_cal: /tmp/data/ + # availability of IR ir_flag: False @@ -28,6 +35,9 @@ params: # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. # If you do not want to transform the coordinates set azi_cor to -999. azi_cor: -999. + # Azimuth correction: + # Set az_cor to the angle that RPG software gives when instrument is pointing to the North. + const_azi: -999. # some default values: # ------------------- diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 5c1e256..06297a5 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -359,15 +359,18 @@ def read_config( instrument_type: str | None, key: Literal["global_specs", "params"], ) -> dict: - if site is not None: - itype = read_site_config_yaml(site)["type"] + site_config = read_site_config_yaml(site) + if site is not None and len(site_config) == 0 and instrument_type is None: + raise ValueError(f"site_config file or instrument_type is required") + if site is not None and len(site_config) > 0: + itype = site_config["type"] elif instrument_type is not None: itype = instrument_type else: raise ValueError("site or instrument_type is required") data = _read_itype_config_yaml(itype)[key] - if site is not None: - data.update(read_site_config_yaml(site)[key]) + if len(site_config) > 0: + data.update(site_config[key]) return data @@ -383,12 +386,13 @@ def _read_itype_config_yaml(itype: str) -> dict: return yaml.load(f, Loader=SafeLoader) -def read_site_config_yaml(site: str) -> dict: +def read_site_config_yaml(site: str | None) -> dict: """Reads configuration file for specific site.""" dir_name = os.path.dirname(os.path.realpath(__file__)) + site = "" if site is None else site site_file = os.path.join(dir_name, "site_config", site, "config.yaml") if not os.path.isfile(site_file): - raise NotImplementedError(f"Error: site config file {site_file} not found") + return dict() with open(site_file, "r", encoding="utf8") as f: return yaml.load(f, Loader=SafeLoader) @@ -466,10 +470,14 @@ def read_nc_fields(nc_file: str, name: str) -> np.ndarray: return nc.variables[name][:] -def read_lidar(path_to_lidar: str | PathLike) -> dict: +def read_lidar(path_to_lidar: str | PathLike) -> tuple[dict, dict]: """Reads lidar data.""" data, names = {}, ["time", "height", "beta"] with netCDF4.Dataset(path_to_lidar) as nc: + meta = { + "history": nc.history, + "source": nc.source, + } for key in names: data[key] = nc.variables[key][:].data if key == "time": @@ -484,7 +492,7 @@ def read_lidar(path_to_lidar: str | PathLike) -> dict: if key == "beta": data[key][data[key] == nc.variables[key].get_fill_value()] = np.nan - return data + return data, meta def n_elements(array: np.ndarray, dist: float, var: str | None = None) -> int: @@ -591,9 +599,11 @@ def get_processing_dates(args) -> tuple[str, str]: return start_date, stop_date -def _get_filename(prod: str, date_in: datetime.date, site: str) -> str: - global_attributes = read_config(site, None, "global_specs") - params = read_config(site, None, "params") +def _get_filename( + prod: str, date_in: datetime.date, site: str, instrument: IType +) -> str: + params = read_config(site, instrument, "params") + global_attributes = read_config(site, instrument, "global_specs") if np.char.isnumeric(prod[0]): level = prod[0] else: @@ -626,17 +636,23 @@ def _get_filename(prod: str, date_in: datetime.date, site: str) -> str: def _get_filename_cloudnet( prod: str, date_in: datetime.date, site: str, instrument: IType ) -> str: + params = read_config(None, instrument, "params") if np.char.isnumeric(prod[0]): level = prod[0] name = "l1c" else: level = "2" name = prod - params = read_config(None, instrument, "params") - data_out_dir = os.path.join( - params["data_out"], f"level{level}", date_in.strftime("%Y/%m/%d") - ) - filename = f"{date_in.strftime('%Y%m%d')}_{site}_{instrument}-{name}.nc" + if name == "lwp_offset": + data_out_dir = os.path.join( + params["data_out"], f"level{level}", date_in.strftime("%Y") + ) + filename = f"{site}_{prod}_{date_in.strftime('%Y')}.csv" + else: + data_out_dir = os.path.join( + params["data_out"], f"level{level}", date_in.strftime("%Y/%m/%d") + ) + filename = f"{date_in.strftime('%Y%m%d')}_{site}_{instrument}-{name}.nc" return os.path.join(data_out_dir, filename) From 98ea6f893cddea1142d818ba2704cc81dc5308c2 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Thu, 27 Aug 2026 16:00:16 +0200 Subject: [PATCH 21/28] Fix general quality flag --- mwrpy/level2/write_lev2_nc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 62e114b..a57473a 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -24,7 +24,6 @@ interpolate_2d, isbit, read_config, - read_site_config_yaml, ) @@ -661,10 +660,12 @@ def _get_qf( product: str, scan: np.ndarray = np.empty([0], np.int32), ) -> None: - rpg_dat["quality_flag"] = np.bitwise_or.reduce( + rpg_dat["quality_flag"] = ma.masked_all((len(index)), np.int32) + rpg_dat["quality_flag_status"] = ma.masked_all((len(index)), np.int32) + rpg_dat["quality_flag"][index_ret] = np.bitwise_or.reduce( lev1["quality_flag"][index[index_ret], :], axis=1 ) - rpg_dat["quality_flag_status"] = np.bitwise_or.reduce( + rpg_dat["quality_flag_status"][index_ret] = np.bitwise_or.reduce( lev1["quality_flag_status"][index[index_ret], :], axis=1 ) if product != "derived": From 6a3b59edcfe6671ad24961891b3aa431f95a8f41 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Thu, 27 Aug 2026 16:03:38 +0200 Subject: [PATCH 22/28] Make site config optional for all data formats --- README.md | 25 ++++---- docs/source/command_line_usage.rst | 13 ++-- mwrpy/cli.py | 14 ----- mwrpy/level1/write_lev1_nc.py | 29 ++++----- mwrpy/process_mwrpy.py | 78 +----------------------- mwrpy/site_config/hatpro.yaml | 30 +++++---- mwrpy/site_config/hyytiala/config.yaml | 40 +++++++++++- mwrpy/site_config/juelich/config.yaml | 39 ++++++++++-- mwrpy/site_config/lhatpro.yaml | 30 +++++---- mwrpy/site_config/lhumpro_u90.yaml | 30 +++++---- mwrpy/site_config/lindenberg/config.yaml | 40 +++++++++++- mwrpy/site_config/palaiseau/config.yaml | 40 +++++++++++- mwrpy/utils.py | 57 +++++++++++++++-- 13 files changed, 279 insertions(+), 186 deletions(-) diff --git a/README.md b/README.md index d8eb6c5..88240ba 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ MWRpy requires Python 3.10 or newer. ## Configuration The folder `mwrpy/site_config/` contains configuration files for each instrument -type, which defines the input and output data paths etc. +type, which also define the input and output data paths etc. For example, this is the [configuration file for RPG-HATPRO](mwrpy/site_config/hatpro.yaml). The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a folder with retrieval coefficients @@ -69,19 +69,16 @@ MWRpy can be run using the command line tool `mwrpy/cli.py`: Arguments: -| Short | Long | Default | Description | -| :------------------------------------------------------------- | :----------------- | :------------------------ | :--------------------------------------------------------------------------------- | -| `-h` | `--help` | | Show help and exit. | -| `-s` | `--site` | | Site to process data from, e.g, `hyytiala`. Required. | -| `-d` | `--date` | | Single date to be processed. Alternatively, `--start` and `--stop` can be defined. | -| | `--start` | `current day - 1` | Starting date. | -| | `--stop` | `current day ` | Stopping date. | -| `-p` | `--products` | `1C01`, `single`, `multi` | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | -| `-f` | `--format` | `cloudnet` | Data format to be used (`cloudnet`, `e-profile`). | -| The following arguments are used for the Cloudnet file format: | -| `-i` | `--instrument` | `hatpro` | Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). | -| `-a` | `--altitude` | `0.0` | Altitude above mean sea level of site (m). | -| `-o` | `--azimuth_offset` | `None` | Azimuth offset of the instrument (degrees). Or `None`. | +| Short | Long | Default | Description | +| :---- | :------------- | :------------------------ | :--------------------------------------------------------------------------------- | +| `-h` | `--help` | | Show help and exit. | +| `-s` | `--site` | | Site to process data from, e.g, `hyytiala`. Required. | +| `-d` | `--date` | | Single date to be processed. Alternatively, `--start` and `--stop` can be defined. | +| | `--start` | `current day - 1` | Starting date. | +| | `--stop` | `current day ` | Stopping date. | +| `-p` | `--products` | `1C01`, `single`, `multi` | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | +| `-f` | `--format` | `cloudnet` | Data format to be used (`cloudnet`, `e-profile`). | +| `-i` | `--instrument` | `hatpro` | Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). | Commands: diff --git a/docs/source/command_line_usage.rst b/docs/source/command_line_usage.rst index d941470..c5eede2 100644 --- a/docs/source/command_line_usage.rst +++ b/docs/source/command_line_usage.rst @@ -2,8 +2,8 @@ Command line usage ================== -After defining the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and optional site -specific information (``mwrpy/site_config/{site}/config.yaml``, only for E-PROFILE format) files, MWRpy can also be +With the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and optional site +specific information (``mwrpy/site_config/{site}/config.yaml``) files, MWRpy can also be run using the command line tool `mwrpy/cli.py`: .. code-block:: @@ -72,15 +72,14 @@ These commands are available to select the processing mode: Example usage ------------- To process and plot Level 1 & 2 data (1C01, single, multi) for the site `Hyytiala` (HATPRO instrument) for April 6, -2023, in the E-PROFILE format, run: +2023, in the Cloudnet format, run: .. code-block:: - python mwrpy/cli.py -s hyytiala -d 2023-04-06 -f e-profile process + python mwrpy/cli.py -s hyytiala -d 2023-04-06 - -Run the following command for the Cloudnet format and no plots: +Run the following command for the E-Profile format and no plots: .. code-block:: - python mwrpy/cli.py -s hyytiala -d 2023-04-06 -a 150 no-plot + python mwrpy/cli.py -s hyytiala -d 2023-04-06 -f e-profile no-plot diff --git a/mwrpy/cli.py b/mwrpy/cli.py index 863efd4..2908ad8 100755 --- a/mwrpy/cli.py +++ b/mwrpy/cli.py @@ -79,20 +79,6 @@ def _parse_args(args): help="Instrument to be processed (hatpro, lhatpro, lhumpro_u90).", default="hatpro", ) - group.add_argument( - "-a", - "--altitude", - type=float, - help="Altitude above mean sea level of site (m).", - default=0.0, - ) - group.add_argument( - "-o", - "--azimuth_offset", - type=float, - help="Azimuth offset of the instrument (degrees).", - default=None, - ) return parser.parse_args(args) diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index dcf2657..64769c9 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -77,16 +77,7 @@ def lev1_to_nc( f"No coefficient files given, using files in repository for {site}." ) - if data_format == "e-profile" and instrument_config is None: - logging.info( - f"No instrument config given, using config file in repository for {site}." - ) - - params = ( - read_config(site, instrument_type, "params") - if data_format == "e-profile" - else read_config(None, instrument_type, "params") - ) + params = read_config(site, instrument_type, "params") if instrument_config is not None: params = {**params, **instrument_config} @@ -114,6 +105,7 @@ def lev1_to_nc( mwr.find_valid_times() mwr.data = get_data_attributes(mwr.data, data_type, data_format) if output_file is not None: + global_attributes = read_config(site, instrument_type, "global_specs") if data_format == "cloudnet": c_files = ( get_coeff_list( @@ -125,10 +117,8 @@ def lev1_to_nc( if coeff_files is None else coeff_files ) - i_gen = read_config(None, instrument_type, "global_specs")[ - "instrument_generation" - ] _, lidar_meta = read_lidar(lidar_path) if lidar_path else (None, None) + i_gen = global_attributes["instrument_generation"] global_attributes = { "site": site, "instrument": instrument_type, @@ -138,7 +128,6 @@ def lev1_to_nc( "source": lidar_meta["source"] if lidar_meta else None, } else: - global_attributes = read_config(site, instrument_type, "global_specs") global_attributes["site_location"] = site global_attributes["dependencies"] = ( str(lidar_path).split("/")[-1] if lidar_path else None @@ -232,13 +221,15 @@ def prepare_data( rpg_blb = RpgBin(file_list_blb, time_offset) _add_blb(rpg_bin, rpg_blb, rpg_hkd, params) - if params["azi_cor"] != -999.0: + if params["azi_cor"]: + logging.info("Performing azimuth angle correction.") _azi_correction(rpg_bin.data, params) azimuth_offset = ( - params["azimuth_offset"] if "azimuth_offset" in params else azimuth_offset + params.get("const_azi") if azimuth_offset is None else azimuth_offset ) if azimuth_offset is not None: + logging.info(f"Adding azimuth offset of {azimuth_offset} degrees.") rpg_bin.data["azimuth_angle"] = ( rpg_bin.data["azimuth_angle"] + azimuth_offset ) % 360 @@ -454,8 +445,10 @@ def prepare_data( file_list_hkd = get_file_list(path_to_files, "HKD") _append_hkd(file_list_hkd, rpg_bin, data_type, params, time_offset) - alt = params.get("altitude", altitude) - alt = ma.masked if alt is None else alt + alt = params.get("altitude") if altitude is None else altitude + if alt is None: + alt = 0.0 + logging.info("Site altitude not provided. Taking default of 0 m.") rpg_bin.data["altitude"] = np.ones(len(rpg_bin.data["time"]), np.float32) * alt return rpg_bin diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index ee30fb3..98856aa 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -1,7 +1,6 @@ """Module for processing.""" import datetime -import glob import logging import os import time @@ -108,8 +107,6 @@ def main(args): f"Product {product} not available in cloudnet format. Skipping." ) continue - if args.altitude is None: - logging.info("Site altitude not provided. Taking default of 0 m.") start = time.process_time() if args.command != "plot": logging.info(f"Processing {product} product, {args.site} {date}") @@ -121,8 +118,6 @@ def main(args): args.site, args.format, args.instrument, - args.altitude, - args.azimuth_offset, ) except Exception as e: logging.error( @@ -136,8 +131,6 @@ def main(args): args.site, args.format, args.instrument, - args.altitude, - args.azimuth_offset, ) if output_file: logging.info("Processed %s: %s", product, output_file) @@ -156,8 +149,6 @@ def process_product( site: str, data_format: str, instrument: IType, - altitude: float, - azimuth_offset: float | None, ): """Process a given product for a specific date and site. This function handles the processing of different products based on their type @@ -170,15 +161,13 @@ def process_product( site: Site identifier. data_format: Data format of the netCDF file (cloudnet, e-profile). instrument: Specific instrument type (hatpro, lhatpro, etc.). - altitude: Altitude of the site in meters above mean sea level. - azimuth_offset: Azimuth offset to be added to azimuth angle. Returns: output_file: Name of output file. """ filename = getattr( mwrpy.utils, - "_get_filename_cloudnet" if data_format == "cloudnet" else "_get_filename", + "get_filename_cloudnet" if data_format == "cloudnet" else "get_filename", ) output_file = filename(prod, date, site, instrument) output_dir = os.path.dirname(output_file) @@ -222,29 +211,15 @@ def process_product( # Process level 1 data if prod[0] == "1": params = mwrpy.utils.read_config(site, instrument, "params") - if data_format == "e-profile": - altitude = params["altitude"] - if altitude is None: - altitude = 0.0 - logging.info("Site altitude not provided. Taking default of 0 m.") - azimuth_offset = ( - params["azimuth_offset"] - if "azimuth_offset" in params and params["azimuth_offset"] is not None - else 0.0 - ) lev1_to_nc( prod, - _get_raw_file_path(date, site, instrument) - if instrument is None - else _get_raw_file_path(date, None, instrument), + mwrpy.utils.get_raw_file_path(date, site, instrument), data_format, site=site, output_file=output_file, - lidar_path=_get_lidar_file_path(date, site, params), + lidar_path=mwrpy.utils.get_lidar_file_path(date, site, params), date=date, instrument_type=instrument, - altitude=altitude, - azimuth_offset=azimuth_offset, ) # Process level 2 single products @@ -570,50 +545,3 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType logging.warning("Nothing to plot for product " + prod) else: logging.warning("Nothing to plot for product " + prod) - - -def _get_raw_file_path( - date_in: datetime.date, site: str | None, instrument: str | None -) -> str: - """Get the raw file path for a given date and site. - - Args: - date_in: Date for which the raw file path is needed. - site: Site identifier. - instrument: Instrument identifier. - - Returns: - The raw file path as a string. - """ - params = mwrpy.utils.read_config(site, instrument, "params") - return os.path.join(params["data_in"], date_in.strftime("%Y/%m/%d/")) - - -def _get_lidar_file_path(date_in: datetime.date, site: str, params: dict) -> str | None: - """Get the lidar file path for a given date and site. - - Args: - date_in: Date for which the lidar file path is needed. - site: Site identifier. - params: Configuration parameters. - - Returns: - The lidar file path as a string or None if not found. - """ - path = "" - lidar_model = params.get("lidar_model", "unknown") - lidar_model = "unknown" if lidar_model is None else lidar_model.lower() - if "path_to_lidar" in params and params["path_to_lidar"] is not None: - path = os.path.join( - params["path_to_lidar"], - date_in.strftime("%Y/%m/%d/"), - ) - file = glob.glob( - path + date_in.strftime("%Y%m%d") + "_" + site + "_" + lidar_model + "*.nc" - ) - if len(file) == 0: - logging.info( - "No lidar file of type " + lidar_model + " found in directory " + str(path) - ) - return None - return file[0] diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index 7ca7604..5230c6f 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -1,19 +1,24 @@ # Config file for all HATPRO instruments +# Missing entries can also be filled with site specific config file params: - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ + altitude: + longitude: + latitude: + + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ - # path to retrieval coefficients + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) coeff_path: # path to Cloudnet lidar file and lidar model (optional) path_to_lidar: lidar_model: - # path to ABSCAL.HIS file - path_to_cal: /tmp/data/ + # path to ABSCAL.HIS file (optional) + path_to_cal: # availability of IR ir_flag: True @@ -35,12 +40,12 @@ params: # integration time of measurements in seconds int_time: 1 - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - # Azimuth correction: - # Set az_cor to the angle that RPG software gives when instrument is pointing to the North. - const_azi: -999. + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: # some default values: # ------------------- @@ -114,7 +119,6 @@ params: [0., 100.], ] -# Missing entries can also be filled with site specific config file global_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/hyytiala/config.yaml b/mwrpy/site_config/hyytiala/config.yaml index 780ca57..9c2703f 100644 --- a/mwrpy/site_config/hyytiala/config.yaml +++ b/mwrpy/site_config/hyytiala/config.yaml @@ -7,13 +7,47 @@ params: longitude: 24.288 latitude: 61.844 - # path to ABSCAL.HIS file - path_to_cal: + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ + + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) + coeff_path: # path to Cloudnet lidar file and lidar model (optional) - path_to_lidar: /tmp/data/ + path_to_lidar: lidar_model: cl61d + # path to ABSCAL.HIS file (optional) + path_to_cal: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time for BL scans in seconds + scan_time: 50. + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: + global_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Hyytiala, Finland diff --git a/mwrpy/site_config/juelich/config.yaml b/mwrpy/site_config/juelich/config.yaml index 1a8b7f2..8b9a207 100644 --- a/mwrpy/site_config/juelich/config.yaml +++ b/mwrpy/site_config/juelich/config.yaml @@ -7,15 +7,46 @@ params: longitude: 6.407 latitude: 50.906 - # path to ABSCAL.HIS file + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ + + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) + coeff_path: + + # path to Cloudnet lidar file and lidar model (optional) + path_to_lidar: + lidar_model: chm15k + + # path to ABSCAL.HIS file (optional) path_to_cal: + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + # integration time for BL scans in seconds scan_time: 50. - # path to Cloudnet lidar file and lidar model (optional) - path_to_lidar: /tmp/data/ - lidar_model: chm15k + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: global_specs: # A succinct description of what is in the dataset, composed of instrument type and site name diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index 6d13282..076562d 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -1,19 +1,24 @@ # Config file for all LHATPRO instruments +# Missing entries can also be filled with site specific config file params: - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ + altitude: + longitude: + latitude: - # path to retrieval coefficients + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ + + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) coeff_path: # path to Cloudnet lidar file and lidar model (optional) path_to_lidar: lidar_model: - # path to ABSCAL.HIS file - path_to_cal: /tmp/data/ + # path to ABSCAL.HIS file (optional) + path_to_cal: # availability of IR ir_flag: True @@ -35,12 +40,12 @@ params: # integration time of measurements in seconds int_time: 1 - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - # Azimuth correction: - # Set az_cor to the angle that RPG software gives when instrument is pointing to the North. - const_azi: -999. + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: # some default values: # ------------------- @@ -112,7 +117,6 @@ params: [0., 100.], ] -# Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index 5aa87e7..0af0d4c 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -1,19 +1,24 @@ # Config file for all LHUMPRO U90 instruments +# Missing entries can also be filled with site specific config file params: - # path to level1 data and path for processed files - data_in: /tmp/data/ - data_out: /tmp/data/ + altitude: + longitude: + latitude: - # path to retrieval coefficients + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ + + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) coeff_path: # path to Cloudnet lidar file and lidar model (optional) path_to_lidar: lidar_model: - # path to ABSCAL.HIS file - path_to_cal: /tmp/data/ + # path to ABSCAL.HIS file (optional) + path_to_cal: # availability of IR ir_flag: False @@ -32,12 +37,12 @@ params: # integration time of measurements in seconds int_time: 1 - # Azimuth angle is transformed to geographical coordinates (E=90 and W=270), currently only for RPG scanners. - # If you do not want to transform the coordinates set azi_cor to -999. - azi_cor: -999. - # Azimuth correction: - # Set az_cor to the angle that RPG software gives when instrument is pointing to the North. - const_azi: -999. + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: # some default values: # ------------------- @@ -94,7 +99,6 @@ params: [0., 100.], ] -# Missing entries are filled with site specific config file global_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/lindenberg/config.yaml b/mwrpy/site_config/lindenberg/config.yaml index f32afbc..07b8c43 100644 --- a/mwrpy/site_config/lindenberg/config.yaml +++ b/mwrpy/site_config/lindenberg/config.yaml @@ -7,13 +7,47 @@ params: longitude: 14.118 latitude: 52.208 - # path to ABSCAL.HIS file - path_to_cal: + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ + + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) + coeff_path: # path to Cloudnet lidar file and lidar model (optional) - path_to_lidar: /tmp/data/ + path_to_lidar: lidar_model: chm15k + # path to ABSCAL.HIS file (optional) + path_to_cal: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time for BL scans in seconds + scan_time: 50. + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: + global_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Lindenberg, Germany diff --git a/mwrpy/site_config/palaiseau/config.yaml b/mwrpy/site_config/palaiseau/config.yaml index c8a95fe..db563b5 100644 --- a/mwrpy/site_config/palaiseau/config.yaml +++ b/mwrpy/site_config/palaiseau/config.yaml @@ -7,13 +7,47 @@ params: longitude: 14.118 latitude: 52.208 - # path to ABSCAL.HIS file - path_to_cal: /tmp/data/ + # path to raw input data and processed files + data_in: /path/to/raw/data/ + data_out: /path/to/processed/files/ + + # path to retrieval coefficients (optional, default: mwrpy/site_config/'site'/coefficients/) + coeff_path: # path to Cloudnet lidar file and lidar model (optional) - path_to_lidar: /tmp/data/ + path_to_lidar: lidar_model: chm15k + # path to ABSCAL.HIS file (optional) + path_to_cal: + + # availability of IR + ir_flag: True + + # quality flag status for level 1 data; 0: flag active + # Bit 1: missing_tb + # Bit 2: tb_below_threshold + # Bit 3: tb_above_threshold + # Bit 4: spectral_consistency_above_threshold + # Bit 5: receiver_sanity_failed + # Bit 6: rain_detected + # Bit 7: sun_moon_in_beam + # Bit 8: tb_offset_above_threshold + flag_status: [0, 0, 0, 0, 0, 0, 0, 1] + + # integration time for BL scans in seconds + scan_time: 50. + + # integration time of measurements in seconds + int_time: 1 + + # Azimuth angle is transformed to geographical coordinates (E=90 and W=270). + # This is needed in older RPG software versions. + azi_cor: False + # Azimuth offset correction; needed if the instrument is not aligned to the North. + # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. + const_azi: + global_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Palaiseau, France diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 06297a5..9ae29dd 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -360,14 +360,12 @@ def read_config( key: Literal["global_specs", "params"], ) -> dict: site_config = read_site_config_yaml(site) - if site is not None and len(site_config) == 0 and instrument_type is None: - raise ValueError(f"site_config file or instrument_type is required") - if site is not None and len(site_config) > 0: + if len(site_config) > 0: itype = site_config["type"] elif instrument_type is not None: itype = instrument_type else: - raise ValueError("site or instrument_type is required") + raise ValueError("site_config file or instrument_type is required") data = _read_itype_config_yaml(itype)[key] if len(site_config) > 0: data.update(site_config[key]) @@ -599,7 +597,7 @@ def get_processing_dates(args) -> tuple[str, str]: return start_date, stop_date -def _get_filename( +def get_filename( prod: str, date_in: datetime.date, site: str, instrument: IType ) -> str: params = read_config(site, instrument, "params") @@ -633,7 +631,7 @@ def _get_filename( return os.path.join(data_out_dir, filename) -def _get_filename_cloudnet( +def get_filename_cloudnet( prod: str, date_in: datetime.date, site: str, instrument: IType ) -> str: params = read_config(None, instrument, "params") @@ -656,6 +654,53 @@ def _get_filename_cloudnet( return os.path.join(data_out_dir, filename) +def get_raw_file_path( + date_in: datetime.date, site: str | None, instrument: str | None +) -> str: + """Get the raw file path for a given date and site. + + Args: + date_in: Date for which the raw file path is needed. + site: Site identifier. + instrument: Instrument identifier. + + Returns: + The raw file path as a string. + """ + params = read_config(site, instrument, "params") + return os.path.join(params["data_in"], date_in.strftime("%Y/%m/%d/")) + + +def get_lidar_file_path(date_in: datetime.date, site: str, params: dict) -> str | None: + """Get the lidar file path for a given date and site. + + Args: + date_in: Date for which the lidar file path is needed. + site: Site identifier. + params: Configuration parameters. + + Returns: + The lidar file path as a string or None if not found. + """ + path = "" + lidar_model = params.get("lidar_model", "unknown") + lidar_model = "unknown" if lidar_model is None else lidar_model.lower() + if "path_to_lidar" in params and params["path_to_lidar"] is not None: + path = os.path.join( + params["path_to_lidar"], + date_in.strftime("%Y/%m/%d/"), + ) + file = glob.glob( + path + date_in.strftime("%Y%m%d") + "_" + site + "_" + lidar_model + "*.nc" + ) + if len(file) == 0: + logging.info( + "No lidar file of type " + lidar_model + " found in directory " + str(path) + ) + return None + return file[0] + + def isodate2date(date_str: str) -> datetime.date: return datetime.datetime.strptime(date_str, "%Y-%m-%d").date() From 10ab6972e579a80a8a1b6b30dc1c9c726082644b Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 2 Sep 2026 16:03:35 +0200 Subject: [PATCH 23/28] Infer instrument type from lev1; data format specific metadata --- mwrpy/level1/write_lev1_nc.py | 28 ++--- mwrpy/level2/lev2_collocated.py | 37 ++----- mwrpy/level2/write_lev2_nc.py | 24 ++--- mwrpy/plots/generate_plots.py | 16 ++- mwrpy/plots/plot_utils.py | 131 ----------------------- mwrpy/process_mwrpy.py | 29 ++--- mwrpy/site_config/hatpro.yaml | 3 +- mwrpy/site_config/hyytiala/config.yaml | 3 +- mwrpy/site_config/juelich/config.yaml | 3 +- mwrpy/site_config/lhatpro.yaml | 3 +- mwrpy/site_config/lhumpro_u90.yaml | 3 +- mwrpy/site_config/lindenberg/config.yaml | 3 +- mwrpy/site_config/palaiseau/config.yaml | 3 +- mwrpy/utils.py | 7 +- tests/test_write_lev1_nc.py | 4 +- tests/test_write_lev2_nc.py | 14 +-- 16 files changed, 81 insertions(+), 230 deletions(-) diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 64769c9..9d2e9a3 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -29,12 +29,13 @@ ) FuncType: TypeAlias = Callable[[str], np.ndarray] +IType = Literal["hatpro", "lhatpro", "lhumpro_u90"] def lev1_to_nc( - data_type: str, path_to_files: str | PathLike, - data_format: str, + data_type: str = "1C01", + data_format: str = "cloudnet", site: str | None = None, output_file: str | PathLike | None = None, lidar_path: str | PathLike | None = None, @@ -42,7 +43,7 @@ def lev1_to_nc( instrument_config: dict | None = None, date: datetime.date | None = None, time_offset: datetime.timedelta | None = None, - instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, + instrument_type: IType | None = "hatpro", altitude: float | None = None, azimuth_offset: float | None = None, ) -> rpg_mwr.Rpg: @@ -50,8 +51,8 @@ def lev1_to_nc( adds attributes and writes it into netCDF file. Args: - data_type: Data type of the netCDF file. path_to_files: Folder containing one day of RPG MWR binary files. + data_type: Data type of the netCDF file (1C01, 1B01, etc.). data_format: Data format of the netCDF file (cloudnet, e-profile). site: Name of site. output_file: Output file name. @@ -80,6 +81,7 @@ def lev1_to_nc( params = read_config(site, instrument_type, "params") if instrument_config is not None: params = {**params, **instrument_config} + site = params.get("site") if site is None else site rpg_bin = prepare_data( path_to_files, @@ -105,7 +107,6 @@ def lev1_to_nc( mwr.find_valid_times() mwr.data = get_data_attributes(mwr.data, data_type, data_format) if output_file is not None: - global_attributes = read_config(site, instrument_type, "global_specs") if data_format == "cloudnet": c_files = ( get_coeff_list( @@ -118,23 +119,22 @@ def lev1_to_nc( else coeff_files ) _, lidar_meta = read_lidar(lidar_path) if lidar_path else (None, None) - i_gen = global_attributes["instrument_generation"] global_attributes = { "site": site, - "instrument": instrument_type, + "instrument": params["type"], "coeff_files": c_files, - "instrument_generation": i_gen, "history": lidar_meta["history"] if lidar_meta else None, "source": lidar_meta["source"] if lidar_meta else None, } else: + global_attributes = read_config(site, instrument_type, "e-profile_specs") global_attributes["site_location"] = site global_attributes["dependencies"] = ( str(lidar_path).split("/")[-1] if lidar_path else None ) - _update_calibration_attributes(rpg_bin, global_attributes) - if data_type != "1C01": - update_lev1_attributes(global_attributes, data_type) + _update_calibration_attributes(rpg_bin, global_attributes) + if data_type != "1C01": + update_lev1_attributes(global_attributes, data_type) rpg_mwr.save_rpg(mwr, output_file, global_attributes, data_type, data_format) return mwr @@ -779,9 +779,9 @@ def _add_blb(brt: RpgBin, blb: RpgBin, hkd: RpgBin, params: dict) -> None: def _update_calibration_attributes(rpg_bin: RpgBin, global_attributes: dict) -> None: global_attributes["type_of_automatic_calibrations"] = ( - "calibration with ambient temperature target and noise diode with high-frequency noise switching" - if global_attributes["instrument_generation"] == "G5" - else "calibration with ambient temperature target and noise diode" + "calibration with ambient temperature target and noise diode" + if global_attributes["instrument_generation"] != "G5" + else "calibration with ambient temperature target and noise diode with high-frequency noise switching" ) if "date_of_last_covariance_matrix" in rpg_bin.data: diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 4109610..3e37825 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -2,7 +2,6 @@ from collections.abc import Sequence from os import PathLike from tempfile import NamedTemporaryFile -from typing import Literal import netCDF4 @@ -11,13 +10,11 @@ def generate_lev2_single( - site: str | None, - data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, + data_format: str = "cloudnet", lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, - instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as lwp_file, @@ -44,9 +41,8 @@ def generate_lev2_single( lev2_to_nc( prod, mwr_l1c_file, - data_format=data_format, - output_file=file, - site=site, + file, + data_format, temp_file=t_prof_file.name if prod in ("2P04", "2P07", "2P08") else None, @@ -55,7 +51,6 @@ def generate_lev2_single( else None, lwp_offset=lwp_offset, coeff_files=coeff_files, - instrument_type=instrument_type, ) with ( @@ -168,9 +163,8 @@ def generate_lev2_single( lev2_to_nc( prod, mwr_l1c_file, - data_format=data_format, - output_file=file, - site=site, + file, + data_format, temp_file=t_prof_file.name if prod in ("2P04", "2P07", "2P08") else None, @@ -179,7 +173,6 @@ def generate_lev2_single( else None, lwp_offset=(None, None), coeff_files=coeff_files, - instrument_type=instrument_type, ) with netCDF4.Dataset(stability_file.name, "r") as nc_sta: var_2I06 = ( @@ -203,13 +196,11 @@ def generate_lev2_single( def generate_lev2_lhumpro( - site: str | None, - data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, + data_format: str = "cloudnet", lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, - instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as lwp_file, @@ -228,14 +219,12 @@ def generate_lev2_lhumpro( lev2_to_nc( prod, mwr_l1c_file, - data_format=data_format, - output_file=file, - site=site, + file, + data_format, temp_file=None, hum_file=None, lwp_offset=lwp_offset, coeff_files=coeff_files, - instrument_type=instrument_type, ) with ( @@ -308,12 +297,10 @@ def generate_lev2_lhumpro( def generate_lev2_multi( - site: str | None, - data_format: str, mwr_l1c_file: str | PathLike, output_file: str | PathLike, + data_format: str = "cloudnet", coeff_files: Sequence[str | PathLike] | None = None, - instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): with ( NamedTemporaryFile() as temperature_file, @@ -336,16 +323,14 @@ def generate_lev2_multi( lev2_to_nc( prod, mwr_l1c_file, - data_format=data_format, - output_file=file, - site=site, + file, + data_format, temp_file=temperature_file.name if prod not in ("2P02", "2P03") else None, hum_file=abs_hum_file.name if prod not in ("2P02", "2P03") else None, lwp_offset=(None, None), coeff_files=coeff_files, - instrument_type=instrument_type, ) with ( diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index a57473a..8311311 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -4,7 +4,6 @@ from collections.abc import Sequence from datetime import datetime, timedelta, timezone from os import PathLike -from typing import Literal import atmoslib import atmoslib.constants as ac @@ -37,14 +36,12 @@ def _local_solar_time(unix_seconds: float, longitude: float) -> datetime: def lev2_to_nc( data_type: str, lev1_file: str | PathLike, - data_format: str, output_file: str | PathLike, - site: str | None = None, + data_format: str, temp_file: str | PathLike | None = None, hum_file: str | PathLike | None = None, lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, - instrument_type: Literal["hatpro", "lhatpro", "lhumpro_u90"] | None = None, ): """This function reads Level 1 files, applies retrieval coefficients for Level 2 products @@ -53,14 +50,12 @@ def lev2_to_nc( Args: data_type: Data type of the netCDF file. lev1_file: Path of Level 1 file. - data_format: Data format of the netCDF file (cloudnet, e-profile). output_file: Name of output file. - site: Name of site. + data_format: Data format of the netCDF file (cloudnet, e-profile). temp_file: Name of temperature product file. hum_file: Name of humidity product file. lwp_offset: LWP offset with the previous day's last and next day's first reliable values. coeff_files: List of coefficient files. - instrument_type: Specific instrument type (HATPRO, LHATPRO, etc.). """ if data_type not in ( @@ -76,11 +71,14 @@ def lev2_to_nc( ): raise ValueError(f"Data type {data_type} not recognised") - params = read_config(site, instrument_type, "params") - global_attributes = read_config(site, instrument_type, "global_specs") - global_attributes["site_location"] = site - with nc.Dataset(lev1_file) as lev1: + site = lev1.location if data_format == "cloudnet" else lev1.site_location + instrument_type = ( + lev1.source.split()[2].lower() + if data_format == "cloudnet" + else lev1.instrument_model.lower() + ) + params = read_config(site, instrument_type, "params") params["altitude"] = ( ma.median(lev1.variables["altitude"][:]) if data_format == "cloudnet" @@ -98,7 +96,6 @@ def lev2_to_nc( lwp_offset=lwp_offset, ) _combine_lev1(lev1, rpg_dat, index, data_type, scan_time) - _del_att(global_attributes) if data_format == "e-profile" and "height" in rpg_dat: rpg_dat["altitude"] = rpg_dat.pop("height") l2_date = num2pydate( @@ -130,6 +127,9 @@ def lev2_to_nc( "source": lev1.source, } else: + global_attributes = read_config(site, instrument_type, "e-profile_specs") + _del_att(global_attributes) + global_attributes["site_location"] = site global_attributes["dependencies"] = ( (f"{lev1.dependencies}\n" f"{str(lev1_file).split('/')[-1]}") if lev1.dependencies diff --git a/mwrpy/plots/generate_plots.py b/mwrpy/plots/generate_plots.py index cf8b877..19b9925 100644 --- a/mwrpy/plots/generate_plots.py +++ b/mwrpy/plots/generate_plots.py @@ -89,7 +89,6 @@ def generate_figure( 91.0, ), pointing: int = 0, - instrument_type: str | None = None, dpi: int = 120, image_name: str | None = None, sub_title: bool = True, @@ -108,7 +107,6 @@ def generate_figure( max_y (int, optional): Upper limit in the plots (km). Default is 12. ele_range (tuple, optional): Range of elevation angles to be plotted. pointing (int, optional): Type of observation (0: single pointing, 1: BL scan). - instrument_type (str, optional): Type of instrument (hatpro, lhatpro, lhumpro). dpi (int, optional): Figure quality (if saved). Higher value means more pixels, i.e., better image quality. Default is 120. image_name (str, optional): Name (and full path) of the output image. @@ -124,7 +122,7 @@ def generate_figure( Examples: >>> from mwrpy.plots import generate_figure - >>> generate_figure('lev2_file.nc', ['lwp'], instrument_type='hatpro') + >>> generate_figure('lev2_file.nc', ['lwp']) """ if ( nc_file == "" @@ -189,6 +187,7 @@ def generate_figure( fig, axes = _initialize_figure(len(valid_fields), dpi) time = _read_time_vector(nc_file) + instrument_type = _read_instrument_type(nc_file) for ax, field, name in zip(axes, valid_fields, valid_names): ax.set_facecolor(_COLORS["lightgray"]) @@ -481,6 +480,17 @@ def _read_time_vector(nc_file: str) -> ndarray: return seconds2hours(time) if time.max() > 24 else time +def _read_instrument_type(nc_file: str) -> str: + """Reads instrument type from netCDF file.""" + with netCDF4.Dataset(nc_file) as nc: + instrument_type = ( + nc.instrument_model.lower() + if "instrument_model" in nc.ncattrs() + else nc.source.split()[2].lower() + ) + return instrument_type + + def _screen_high_altitudes(data_field: ndarray, ax_values: tuple, max_y: int) -> tuple: """Removes altitudes from 2D data that are not visible in the figure. Bug in pcolorfast causing effect to axis not noticing limitation while diff --git a/mwrpy/plots/plot_utils.py b/mwrpy/plots/plot_utils.py index 747cc9a..b1b957f 100644 --- a/mwrpy/plots/plot_utils.py +++ b/mwrpy/plots/plot_utils.py @@ -6,7 +6,6 @@ import netCDF4 import numpy as np import pandas as pd -from matplotlib import ticker from numpy import ma, ndarray from mwrpy.utils import ( @@ -48,24 +47,6 @@ def _get_ret_flag( return flag -def _get_lev1(nc_file: str) -> str: - """Returns name of lev1 file.""" - site = _read_location(nc_file) - global_attributes = read_config(site, None, "global_specs") - params = read_config(site, None, "params") - datef = datetime.strptime(nc_file[-11:-3], "%Y%m%d") - data_out_l1 = params["data_out"] + "level1/" + datef.strftime("%Y/%m/%d/") - lev1_file = ( - data_out_l1 - + "MWR_1C01_" - + global_attributes["wigos_station_id"] - + "_" - + datef.strftime("%Y%m%d") - + ".nc" - ) - return lev1_file - - def _get_freq_flag(data: ndarray, bits: ndarray) -> ndarray: """Returns array of flag values for each frequency.""" flag = np.ones(data.shape) * np.nan @@ -171,115 +152,3 @@ def _read_location(nc_file: str) -> str: with netCDF4.Dataset(nc_file) as nc: site_name = nc.site_location if "site_location" in nc.ncattrs() else nc.location return site_name - - -def heatmap( - data, row_labels, col_labels, ax=None, cbar_kw=None, cbarlabel="", **kwargs -): - """Create a heatmap from a numpy array and two lists of labels. - - Parameters - ---------- - data - A 2D numpy array of shape (M, N). - row_labels - A list or array of length M with the labels for the rows. - col_labels - A list or array of length N with the labels for the columns. - ax - A `matplotlib.axes.Axes` instance to which the heatmap is plotted. If - not provided, use current axes or create a new one. Optional. - cbar_kw - A dictionary with arguments to `matplotlib.Figure.colorbar`. Optional. - cbarlabel - The label for the colorbar. Optional. - **kwargs - All other arguments are forwarded to `imshow`. - """ - if cbar_kw is None: - cbar_kw = {} - - # Plot the heatmap - im = ax.imshow(data, **kwargs) - - # Create colorbar - cbar = ax.figure.colorbar(im, ax=ax, anchor=(1.2, 0.5), **cbar_kw) - cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom") - - # Show all ticks and label them with the respective list entries. - ax.set_xticks(np.arange(data.shape[1]), labels=col_labels) - ax.set_yticks(np.arange(data.shape[0]), labels=row_labels) - - # Turn spines off and create white grid. - ax.spines[:].set_visible(False) - - ax.set_xticks(np.arange(data.shape[1] + 1) - 0.5, minor=True) - ax.set_yticks(np.arange(data.shape[0] + 1) - 0.5, minor=True) - ax.grid(which="minor", color="w", linestyle="-", linewidth=3) - ax.tick_params(which="minor", bottom=False, left=False) - - return im, cbar - - -def annotate_heatmap( - im, - data=None, - valfmt="{x:.2f}", - textcolors=("black", "white"), - threshold=None, - **textkw, -): - """A function to annotate a heatmap. - - Parameters - ---------- - im - The AxesImage to be labeled. - data - Data used to annotate. If None, the image's data is used. Optional. - valfmt - The format of the annotations inside the heatmap. This should either - use the string format method, e.g. "$ {x:.2f}", or be a - `matplotlib.ticker.Formatter`. Optional. - textcolors - A pair of colors. The first is used for values below a threshold, - the second for those above. Optional. - threshold - Value in data units according to which the colors from textcolors are - applied. If None (the default) uses the middle of the colormap as - separation. Optional. - **kwargs - All other arguments are forwarded to each call to `text` used to create - the text labels. - """ - if not isinstance(data, (list, np.ndarray)): - data = im.get_array() - - im_dat = im.get_array() - - # Normalize the threshold to the images color range. - if threshold is not None: - threshold = im.norm(threshold) - else: - threshold = im.norm(np.max(data)) / 2.0 - - # Set default alignment to center, but allow it to be - # overwritten by textkw. - kw = {"horizontalalignment": "center", "verticalalignment": "center"} - kw.update(textkw) - - # Get the formatter in case a string is supplied - if isinstance(valfmt, str): - valfmt = ticker.StrMethodFormatter(valfmt) - - # Loop over the data and create a `Text` for each "pixel". - # Change the text's color depending on the data. - texts = [] - for i in range(data.shape[0]): - for j in range(data.shape[1]): - if data[i, j] > 0.0: - kw.update(color=textcolors[int(im.norm(im_dat[i, j]) < threshold)]) - text = im.axes.text(j, i, valfmt(data[i, j], None), **kw) - texts.append(text) - - return texts diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index 98856aa..5cd7eea 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -212,8 +212,8 @@ def process_product( if prod[0] == "1": params = mwrpy.utils.read_config(site, instrument, "params") lev1_to_nc( - prod, mwrpy.utils.get_raw_file_path(date, site, instrument), + prod, data_format, site=site, output_file=output_file, @@ -235,44 +235,36 @@ def process_product( lev2_to_nc( prod, filename("1C01", date, site, instrument), + output_file, data_format, - output_file=output_file, - site=site, temp_file=temp_file, hum_file=hum_file, lwp_offset=lwp_offset_tuple, - instrument_type=instrument, ) # Process level 2 combined products elif prod == "single" and instrument != "lhumpro_u90": generate_lev2_single( - site, - data_format, l1_filename, output_file, + data_format, lwp_offset_tuple, None, - instrument, ) elif instrument == "lhumpro_u90": generate_lev2_lhumpro( - site, - data_format, l1_filename, output_file, + data_format, lwp_offset_tuple, None, - instrument, ) elif prod == "multi": generate_lev2_multi( - site, - data_format, l1_filename, output_file, + data_format, None, - instrument, ) # Update LWP offset file if necessary @@ -341,7 +333,7 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType """ filename = getattr( mwrpy.utils, - "_get_filename_cloudnet" if data_format == "cloudnet" else "_get_filename", + "get_filename_cloudnet" if data_format == "cloudnet" else "get_filename", ) input_file = filename(prod, date, site, instrument) if not os.path.isfile(input_file): @@ -387,7 +379,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType else ["Gain"], save_path=params["path_to_cal"], image_name=key, - instrument_type=instrument, cov_data=his_data, site=site, ) @@ -404,7 +395,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType if key == "cov" else output_dir, image_name=key, - instrument_type=instrument, ) # Plot level 2 single products @@ -431,7 +421,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType save_path=output_dir, image_name=PRODUCT_NAME[prod][0], title=False, - instrument_type=instrument, ) elif key in ("lwp_scan", "iwv_scan"): generate_figure( @@ -441,7 +430,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType save_path=output_dir, image_name=key, title=False, - instrument_type=instrument, ) else: generate_figure( @@ -451,7 +439,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType save_path=output_dir, image_name=key, pointing=pointing, - instrument_type=instrument, ) # Plot level 2 combined products @@ -492,7 +479,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType save_path=output_dir, image_name=key, title=False, - instrument_type=instrument, ) else: generate_figure( @@ -503,7 +489,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType image_name=key, title=title, pointing=pointing, - instrument_type=instrument, ) # Plot covariance data and calibration history even if 1C01 file is not available @@ -518,7 +503,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType "", ["tb_cov_ln2", "tb_cov_amb"], save_path=output_dir + "COVARIANCE/", - instrument_type=instrument, image_name="cov", cov_data=cov_data, site=site, @@ -536,7 +520,6 @@ def plot_product(prod: str, date, site: str, data_format: str, instrument: IType if "tb_cov_ln2" in his_data else ["Gain"], save_path=output_dir, - instrument_type=instrument, image_name="his", cov_data=his_data, site=site, diff --git a/mwrpy/site_config/hatpro.yaml b/mwrpy/site_config/hatpro.yaml index 5230c6f..a6fbb5b 100644 --- a/mwrpy/site_config/hatpro.yaml +++ b/mwrpy/site_config/hatpro.yaml @@ -119,7 +119,8 @@ params: [0., 100.], ] -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/hyytiala/config.yaml b/mwrpy/site_config/hyytiala/config.yaml index 9c2703f..a2852e6 100644 --- a/mwrpy/site_config/hyytiala/config.yaml +++ b/mwrpy/site_config/hyytiala/config.yaml @@ -48,7 +48,8 @@ params: # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. const_azi: -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Hyytiala, Finland diff --git a/mwrpy/site_config/juelich/config.yaml b/mwrpy/site_config/juelich/config.yaml index 8b9a207..4594b1c 100644 --- a/mwrpy/site_config/juelich/config.yaml +++ b/mwrpy/site_config/juelich/config.yaml @@ -48,7 +48,8 @@ params: # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. const_azi: -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Juelich, Germany diff --git a/mwrpy/site_config/lhatpro.yaml b/mwrpy/site_config/lhatpro.yaml index 076562d..0d3539d 100644 --- a/mwrpy/site_config/lhatpro.yaml +++ b/mwrpy/site_config/lhatpro.yaml @@ -117,7 +117,8 @@ params: [0., 100.], ] -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/lhumpro_u90.yaml b/mwrpy/site_config/lhumpro_u90.yaml index 0af0d4c..02823be 100644 --- a/mwrpy/site_config/lhumpro_u90.yaml +++ b/mwrpy/site_config/lhumpro_u90.yaml @@ -99,7 +99,8 @@ params: [0., 100.], ] -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # Name of the conventions followed by the dataset conventions: CF-1.8 diff --git a/mwrpy/site_config/lindenberg/config.yaml b/mwrpy/site_config/lindenberg/config.yaml index 07b8c43..40707f6 100644 --- a/mwrpy/site_config/lindenberg/config.yaml +++ b/mwrpy/site_config/lindenberg/config.yaml @@ -48,7 +48,8 @@ params: # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. const_azi: -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Lindenberg, Germany diff --git a/mwrpy/site_config/palaiseau/config.yaml b/mwrpy/site_config/palaiseau/config.yaml index db563b5..563ac78 100644 --- a/mwrpy/site_config/palaiseau/config.yaml +++ b/mwrpy/site_config/palaiseau/config.yaml @@ -48,7 +48,8 @@ params: # Set const_azi to the angle that RPG software gives when instrument is pointing to the North. const_azi: -global_specs: +# Metadata for E-Profile data format +e-profile_specs: # A succinct description of what is in the dataset, composed of instrument type and site name title: HATPRO G5 MWR at Palaiseau, France diff --git a/mwrpy/utils.py b/mwrpy/utils.py index 9ae29dd..d208582 100644 --- a/mwrpy/utils.py +++ b/mwrpy/utils.py @@ -357,7 +357,7 @@ def get_file_list(path_to_files: str | PathLike, extension: str) -> list[str]: def read_config( site: str | None, instrument_type: str | None, - key: Literal["global_specs", "params"], + key: Literal["e-profile_specs", "params"], ) -> dict: site_config = read_site_config_yaml(site) if len(site_config) > 0: @@ -367,6 +367,7 @@ def read_config( else: raise ValueError("site_config file or instrument_type is required") data = _read_itype_config_yaml(itype)[key] + data["type"] = itype if len(site_config) > 0: data.update(site_config[key]) return data @@ -601,7 +602,7 @@ def get_filename( prod: str, date_in: datetime.date, site: str, instrument: IType ) -> str: params = read_config(site, instrument, "params") - global_attributes = read_config(site, instrument, "global_specs") + global_attributes = read_config(site, instrument, "e-profile_specs") if np.char.isnumeric(prod[0]): level = prod[0] else: @@ -634,7 +635,7 @@ def get_filename( def get_filename_cloudnet( prod: str, date_in: datetime.date, site: str, instrument: IType ) -> str: - params = read_config(None, instrument, "params") + params = read_config(site, instrument, "params") if np.char.isnumeric(prod[0]): level = prod[0] name = "l1c" diff --git a/tests/test_write_lev1_nc.py b/tests/test_write_lev1_nc.py index ad8654a..7478079 100644 --- a/tests/test_write_lev1_nc.py +++ b/tests/test_write_lev1_nc.py @@ -17,7 +17,7 @@ def test_lev1_to_nc(): for prod in product_list: - hatpro = lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site) + hatpro = lev1_to_nc(DATA_DIR, prod, DATA_FORMAT, site) assert str(hatpro.date) == DATE for t in hatpro.data["time"][:]: date = str( @@ -29,7 +29,7 @@ def test_lev1_to_nc(): def test_output_nc_file(): for prod in product_list: temp_file = "temp_file.nc" - lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site, output_file=temp_file) + lev1_to_nc(DATA_DIR, prod, DATA_FORMAT, site, output_file=temp_file) with netCDF4.Dataset(temp_file) as nc: # Write tests for the created netCDF file here: assert nc.date == DATE diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index ac2fbdb..f09f3a9 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -20,7 +20,7 @@ def l1_file(request): fd, path = tempfile.mkstemp() os.close(fd) - lev1_to_nc("1C01", DATA_DIR, DATA_FORMAT, SITE, path) + lev1_to_nc(DATA_DIR, "1C01", DATA_FORMAT, SITE, path) def delete_file(): os.unlink(path) @@ -32,7 +32,7 @@ def delete_file(): def test_generate_lev2_single_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(SITE, DATA_FORMAT, l1_file, path) + generate_lev2_single(DATA_FORMAT, l1_file, path) os.unlink(path) @@ -40,12 +40,10 @@ def test_generate_lev2_single_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) generate_lev2_single( - None, - DATA_FORMAT, l1_file, path, + DATA_FORMAT, coeff_files=COEFF_FILES, - instrument_type="hatpro", ) os.unlink(path) @@ -53,7 +51,7 @@ def test_generate_lev2_single_no_site(l1_file): def test_generate_lev2_multi_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(SITE, DATA_FORMAT, l1_file, path) + generate_lev2_multi(l1_file, path, DATA_FORMAT) os.unlink(path) @@ -61,10 +59,8 @@ def test_generate_lev2_multi_no_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) generate_lev2_multi( - None, - DATA_FORMAT, l1_file, path, + DATA_FORMAT, coeff_files=COEFF_FILES, - instrument_type="hatpro", ) From 2d082f2a9ad3ce315525bf78d6e98493de2eb566 Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 2 Sep 2026 16:04:33 +0200 Subject: [PATCH 24/28] Update README and docs --- README.md | 16 ++++---- docs/source/command_line_usage.rst | 7 ++-- docs/source/mwrpy_processing.rst | 63 ++++++++++++------------------ 3 files changed, 35 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 88240ba..a76880a 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Level 2 data products and visualization and is based on the IDL code The netCDF data format including metadata information, variable names and file naming is designed to be compliant with either the data structure and naming convention -developed in the [EUMETNET Profiling Programme E-PROFILE](https://www.eumetnet.eu/), or within ACTRIS. +developed in the [EUMETNET Profiling Programme E-PROFILE](https://www.eumetnet.eu/), or within [ACTRIS Cloudnet](https://cloudnet.fmi.fi/). MWRpy documentation: @@ -51,14 +51,14 @@ MWRpy requires Python 3.10 or newer. ## Configuration -The folder `mwrpy/site_config/` contains configuration files for each instrument -type, which also define the input and output data paths etc. +The folder `mwrpy/site_config/` contains mandatory configuration files for each instrument +type, which also define the input and output data paths, metadata, etc. For example, this is the [configuration file for RPG-HATPRO](mwrpy/site_config/hatpro.yaml). -The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a folder with retrieval coefficients -(`mwrpy/site_config/hyytiala/coefficients/`) and a site and instrument specific configuration file (`config.yaml`). -For example, this is the [configuration file for Hyytiälä](mwrpy/site_config/hyytiala/config.yaml), which is optional -and helps with configuring multiple instruments of the same type. +The folders for each site, e.g. `mwrpy/site_config/hyytiala/`, contain a folder with the required retrieval coefficients +(`mwrpy/site_config/hyytiala/coefficients/`) and an optional site and instrument specific configuration file (`config. +yaml`). For example, this is the [configuration file for Hyytiälä](mwrpy/site_config/hyytiala/config.yaml), which can +help with configuring multiple instruments of the same type. ## Command line usage @@ -78,7 +78,7 @@ Arguments: | | `--stop` | `current day ` | Stopping date. | | `-p` | `--products` | `1C01`, `single`, `multi` | Processed products, e.g, `1C01, 2I02, 2P03, single`, see below. | | `-f` | `--format` | `cloudnet` | Data format to be used (`cloudnet`, `e-profile`). | -| `-i` | `--instrument` | `hatpro` | Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). | +| `-i` | `--instrument` | `hatpro` | Instrument type to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). | Commands: diff --git a/docs/source/command_line_usage.rst b/docs/source/command_line_usage.rst index c5eede2..07265d5 100644 --- a/docs/source/command_line_usage.rst +++ b/docs/source/command_line_usage.rst @@ -2,9 +2,8 @@ Command line usage ================== -With the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and optional site -specific information (``mwrpy/site_config/{site}/config.yaml``) files, MWRpy can also be -run using the command line tool `mwrpy/cli.py`: +With the instrument type configuration (``mwrpy/site_config/{instrument_type}.yaml``) and retrieval files in +``mwrpy/site_config/{site}/coefficients/``, MWRpy can also be run using the command line tool `mwrpy/cli.py`: .. code-block:: @@ -50,7 +49,7 @@ run using the command line tool `mwrpy/cli.py`: * - `-i` - `--instrument` - hatpro - - Instrument to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). + - Instrument type to be processed (`hatpro`, `lhatpro`, `lhumpro_u90`). These commands are available to select the processing mode: diff --git a/docs/source/mwrpy_processing.rst b/docs/source/mwrpy_processing.rst index 09b30c8..6519ed3 100644 --- a/docs/source/mwrpy_processing.rst +++ b/docs/source/mwrpy_processing.rst @@ -4,7 +4,7 @@ MWRpy processing In this tutorial `MWRpy `_ products are generated from raw data, including quality control and visualization. This example utilizes files taken from the ACTRIS site -`Hyytiala `_: +`Hyytiala `_ equipped with a RPG-HATPRO instrument: - RPG microwave radiometer: - Brightness temperatures: `230406.BRT `_ @@ -17,30 +17,31 @@ quality control and visualization. This example utilizes files taken from the AC .BRT and .HKD files are mandatory in MWRpy for processing -First steps for processing examples: - -First we define the instrument type configuration file (``mwrpy/site_config/{i_type}.yaml``), including instrument -specific information. An optional site specific configuration file (e.g. ``mwrpy/site_config/{site_name}/config -.yaml``) can be configured, when dealing with multiple instruments of the same type. Then the data path is specified: +First step for the processing examples is to specify the site name and data path: .. code-block:: python import os - package_dir = os.getcwd() site_name = "hyytiala" + package_dir = os.getcwd() data_path = f"{package_dir}/tests/data/{site_name}" E-PROFILE format ---------------- +First E-Profile specific metadata can be configured in the instrument type configuration file +(``mwrpy/site_config/hatpro.yaml``), which also includes instrument specific information. An optional site specific +configuration file (e.g. ``mwrpy/site_config/{site_name}/config.yaml``) can be configured, when dealing with multiple + instruments of the same type. + Level 1c ~~~~~~~~~ Now we convert RPG microwave radiometer (MWR) binary files, including brightness temperature (TB) and -housekeeping data (\*.BRT, \*.HKD), into a Level 1c netCDF file. Data from optional elevation scans (\*.BLB, \*.BLS), -weather station (\*.MET) and infrared radiometer (\*.IRT) are combined in this process and the following quality -flags are derived: +housekeeping data (\*.BRT, \*.HKD), into a Level 1c netCDF file (1C01, default). Data from optional elevation scans (\* +.BLB, \*.BLS), weather station (\*.MET) and infrared radiometer (\*.IRT) are combined in this process and the +following quality flags are derived: - Bit 1: missing_tb - Bit 2: tb_below_threshold @@ -63,11 +64,9 @@ quality flag status variable contains information whether the flag is active. from mwrpy.level1.write_lev1_nc import lev1_to_nc mwr_raw = lev1_to_nc( - data_type="1C01", path_to_files=data_path, data_format="e-profile", site=site_name, - instrument_type="hatpro", output_file=f"{data_path}/mwr_1c.nc", ) @@ -97,11 +96,9 @@ are applied to generate the Level 2 single pointing product: from mwrpy.level2.lev2_collocated import generate_lev2_single mwr_prod = generate_lev2_single( - site=site_name, - instrument_type="hatpro", - data_format="e-profile", mwr_l1c_file=f"{data_path}/mwr_1c.nc", output_file=f"{data_path}/mwr-single.nc", + data_format="e-profile", ) Variables such as integrated water vapor @@ -128,11 +125,9 @@ product: from mwrpy.level2.lev2_collocated import generate_lev2_multi mwr_prod = generate_lev2_multi( - site=site_name, - instrument_type="hatpro", - data_format="e-profile", mwr_l1c_file=f"{data_path}/mwr_1c.nc", output_file=f"{data_path}/mwr-multi.nc", + data_format="e-profile", ) Variables such as temperature profiles can be plotted from the newly generated file. @@ -147,8 +142,8 @@ Variables such as temperature profiles can be plotted from the newly generated f Cloudnet format --------------- -In this example the Cloudnet API is used to fetch data and retrieval files and the Cloudnet data format is selected -for processing. More details can be found in the E-PROFILE example above. +In this example the `Cloudnet API client `_ is used to fetch +data and retrieval files and the Cloudnet data format is selected for processing (default). Using Cloudnet API to fetch data and retrieval files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -187,14 +182,14 @@ Download retrieval files: Process and plot Level 1 data ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In contrast to the E-PROFILE data format, no site specific information file is required, but metadata needs to be -defined. Also, the retrieval files are set as an argument. +In contrast to the E-PROFILE data format, no additional metadata needs to be defined in the configuration file. The +fetched retrieval files are set as an argument together with site information (e.g. site name, altitude, etc.). .. code-block:: python site_info = client.site(site_id=site_name) site_meta = { - "name": site_info.id, + "site": site_info.id, "altitude": site_info.altitude, "latitude": site_info.latitude, "longitude": site_info.longitude, @@ -202,51 +197,41 @@ defined. Also, the retrieval files are set as an argument. from mwrpy.level1.write_lev1_nc import lev1_to_nc mwr_raw = lev1_to_nc( - data_type="1C01", path_to_files=data_path, - data_format="cloudnet", - instrument_type=i_type, - output_file=f"{data_path}/mwr_1c_cn.nc", + output_file=f"{data_path}/mwr_1c.nc", coeff_files=retrieval_files, instrument_config=site_meta, ) -For plotting, the instrument needs to be defined. In this example, the figure is only displayed and not saved. +In this example, the figure is only displayed and not saved. .. code-block:: python from mwrpy.plots.generate_plots import generate_figure - fig_name = generate_figure(f"{data_path}/mwr_1c.nc", ['tb'], show=True, instrument_type=i_type) + fig_name = generate_figure(f"{data_path}/mwr_1c.nc", ['tb'], show=True) Process and plot Level 2 data (single & multiple pointing) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The site name is set to ``None``, since no site specific information file is needed. Again, no plots are saved, only -displayed. +Again, no plots are saved, only displayed. .. code-block:: python from mwrpy.level2.lev2_collocated import generate_lev2_single mwr_prod = generate_lev2_single( - site=None, - data_format="cloudnet", mwr_l1c_file=f"{data_path}/mwr_1c.nc", output_file=f"{data_path}/mwr-single.nc", coeff_files=retrieval_files, - instrument_type=i_type, ) from mwrpy.plots.generate_plots import generate_figure - fig_name = generate_figure(f"{data_path}/mwr-single.nc", ['iwv'], show=True, instrument_type=i_type) + fig_name = generate_figure(f"{data_path}/mwr-single.nc", ['iwv'], show=True) from mwrpy.level2.lev2_collocated import generate_lev2_multi mwr_prod = generate_lev2_multi( - site=None, - data_format="cloudnet", mwr_l1c_file=f"{data_path}/mwr_1c.nc", output_file=f"{data_path}/mwr-multi.nc", coeff_files=retrieval_files, - instrument_type=i_type, ) from mwrpy.plots.generate_plots import generate_figure - fig_name = generate_figure(f"{data_path}/mwr-multi.nc", ['temperature'], show=True, instrument_type=i_type) + fig_name = generate_figure(f"{data_path}/mwr-multi.nc", ['temperature'], show=True) From 836b32c0468109cfc279078a931b4d0c18b3546e Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 2 Sep 2026 16:19:24 +0200 Subject: [PATCH 25/28] Fix test --- tests/test_write_lev2_nc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index f09f3a9..d5e828c 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -32,7 +32,7 @@ def delete_file(): def test_generate_lev2_single_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(DATA_FORMAT, l1_file, path) + generate_lev2_single(l1_file, path, DATA_FORMAT) os.unlink(path) From 01ca741b4502ab381f38ae177e5d847b3a72d17e Mon Sep 17 00:00:00 2001 From: tobiasmarke Date: Wed, 9 Sep 2026 15:54:54 +0200 Subject: [PATCH 26/28] Fix height variable for interpolation --- mwrpy/level2/write_lev2_nc.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mwrpy/level2/write_lev2_nc.py b/mwrpy/level2/write_lev2_nc.py index 52de288..1968936 100644 --- a/mwrpy/level2/write_lev2_nc.py +++ b/mwrpy/level2/write_lev2_nc.py @@ -577,7 +577,7 @@ def get_products( if "height" in hum_dat.variables else hum_dat.variables["altitude"][:] ) - tem_height = ( + rpg_dat["height"] = ( tem_dat.variables["height"][:] if "height" in tem_dat.variables else tem_dat.variables["altitude"][:] @@ -585,13 +585,12 @@ def get_products( hum_int = interpolate_2d_nearest( hum_time, - hum_dat.variables["height"][:], + hum_height, hum_dat.variables["absolute_humidity"][:, :], tem_time, - tem_dat.variables["height"][:], + rpg_dat["height"], ) - rpg_dat["height"] = tem_height pres = np.interp(tem_time, lev1["time"][:], lev1["air_pressure"][:]) T = tem_dat.variables["temperature"][:, :] # hum_int is absolute humidity (kg m-3) from the 2P03 product; vapor From c2dcac291aa832152084b5f3ee3a481098c12bca Mon Sep 17 00:00:00 2001 From: Simo Tukiainen Date: Thu, 17 Sep 2026 11:58:51 +0300 Subject: [PATCH 27/28] Restore lev1_to_nc argument order and keep returned Rpg object intact --- docs/source/mwrpy_processing.rst | 2 ++ mwrpy/level1/lev1_meta_nc.py | 6 ++++++ mwrpy/level1/write_lev1_nc.py | 4 ++-- mwrpy/level2/lev2_collocated.py | 6 +++--- mwrpy/process_mwrpy.py | 11 ++++------- mwrpy/rpg_mwr.py | 11 ++++++++--- tests/test_write_lev1_nc.py | 4 ++-- tests/test_write_lev2_nc.py | 10 +++++----- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/source/mwrpy_processing.rst b/docs/source/mwrpy_processing.rst index 6519ed3..1949178 100644 --- a/docs/source/mwrpy_processing.rst +++ b/docs/source/mwrpy_processing.rst @@ -64,6 +64,7 @@ quality flag status variable contains information whether the flag is active. from mwrpy.level1.write_lev1_nc import lev1_to_nc mwr_raw = lev1_to_nc( + data_type="1C01", path_to_files=data_path, data_format="e-profile", site=site_name, @@ -197,6 +198,7 @@ fetched retrieval files are set as an argument together with site information (e from mwrpy.level1.write_lev1_nc import lev1_to_nc mwr_raw = lev1_to_nc( + data_type="1C01", path_to_files=data_path, output_file=f"{data_path}/mwr_1c.nc", coeff_files=retrieval_files, diff --git a/mwrpy/level1/lev1_meta_nc.py b/mwrpy/level1/lev1_meta_nc.py index f0c5325..ae8bc57 100644 --- a/mwrpy/level1/lev1_meta_nc.py +++ b/mwrpy/level1/lev1_meta_nc.py @@ -67,6 +67,12 @@ def get_data_attributes(rpg_variables: dict, data_type: str, data_format: str) - calendar="standard", dimensions=("time",), ), + # Not written to file, kept for the returned object + "time_bnds": MetaData( + long_name="Start and end time (UTC) of the measurements", + units="seconds since 1970-01-01 00:00:00.000", + dimensions=("time", "bnds"), + ), "latitude": MetaData( long_name="Latitude of measurement station", standard_name="latitude", diff --git a/mwrpy/level1/write_lev1_nc.py b/mwrpy/level1/write_lev1_nc.py index 9d2e9a3..14427d2 100644 --- a/mwrpy/level1/write_lev1_nc.py +++ b/mwrpy/level1/write_lev1_nc.py @@ -33,8 +33,8 @@ def lev1_to_nc( + data_type: str, path_to_files: str | PathLike, - data_type: str = "1C01", data_format: str = "cloudnet", site: str | None = None, output_file: str | PathLike | None = None, @@ -51,8 +51,8 @@ def lev1_to_nc( adds attributes and writes it into netCDF file. Args: - path_to_files: Folder containing one day of RPG MWR binary files. data_type: Data type of the netCDF file (1C01, 1B01, etc.). + path_to_files: Folder containing one day of RPG MWR binary files. data_format: Data format of the netCDF file (cloudnet, e-profile). site: Name of site. output_file: Output file name. diff --git a/mwrpy/level2/lev2_collocated.py b/mwrpy/level2/lev2_collocated.py index 3e37825..2151209 100644 --- a/mwrpy/level2/lev2_collocated.py +++ b/mwrpy/level2/lev2_collocated.py @@ -12,9 +12,9 @@ def generate_lev2_single( mwr_l1c_file: str | PathLike, output_file: str | PathLike, - data_format: str = "cloudnet", lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, + data_format: str = "cloudnet", ): with ( NamedTemporaryFile() as lwp_file, @@ -198,9 +198,9 @@ def generate_lev2_single( def generate_lev2_lhumpro( mwr_l1c_file: str | PathLike, output_file: str | PathLike, - data_format: str = "cloudnet", lwp_offset: tuple[float | None, float | None] = (None, None), coeff_files: Sequence[str | PathLike] | None = None, + data_format: str = "cloudnet", ): with ( NamedTemporaryFile() as lwp_file, @@ -299,8 +299,8 @@ def generate_lev2_lhumpro( def generate_lev2_multi( mwr_l1c_file: str | PathLike, output_file: str | PathLike, - data_format: str = "cloudnet", coeff_files: Sequence[str | PathLike] | None = None, + data_format: str = "cloudnet", ): with ( NamedTemporaryFile() as temperature_file, diff --git a/mwrpy/process_mwrpy.py b/mwrpy/process_mwrpy.py index 5cd7eea..151ccc2 100644 --- a/mwrpy/process_mwrpy.py +++ b/mwrpy/process_mwrpy.py @@ -212,8 +212,8 @@ def process_product( if prod[0] == "1": params = mwrpy.utils.read_config(site, instrument, "params") lev1_to_nc( - mwrpy.utils.get_raw_file_path(date, site, instrument), prod, + mwrpy.utils.get_raw_file_path(date, site, instrument), data_format, site=site, output_file=output_file, @@ -247,24 +247,21 @@ def process_product( generate_lev2_single( l1_filename, output_file, - data_format, lwp_offset_tuple, - None, + data_format=data_format, ) elif instrument == "lhumpro_u90": generate_lev2_lhumpro( l1_filename, output_file, - data_format, lwp_offset_tuple, - None, + data_format=data_format, ) elif prod == "multi": generate_lev2_multi( l1_filename, output_file, - data_format, - None, + data_format=data_format, ) # Update LWP offset file if necessary diff --git a/mwrpy/rpg_mwr.py b/mwrpy/rpg_mwr.py index a0994f6..69ee6eb 100644 --- a/mwrpy/rpg_mwr.py +++ b/mwrpy/rpg_mwr.py @@ -1,7 +1,9 @@ """RpgArray Class.""" +import copy import datetime from os import PathLike +from pathlib import Path import netCDF4 import numpy as np @@ -176,8 +178,11 @@ def save_rpg( ) -> None: """Saves the RPG MWR file.""" if data_format == "cloudnet": - Rpg.convert_time_to_hours(rpg) - Rpg.add_zenith_angle(rpg) + # Work on a copy to keep the caller's object intact + rpg = copy.copy(rpg) + rpg.data = dict(rpg.data) + rpg.convert_time_to_hours() + rpg.add_zenith_angle() if data_type == "1B01": dims = { "time": len(rpg.data["time"][:]), @@ -341,5 +346,5 @@ def _add_cloudnet_global_attributes( value = "" setattr(nc_file, name, value) nc_file.mwrpy_coefficients = ", ".join( - [file.split("/")[-1] for file in add_global["coeff_files"]] + Path(file).name for file in add_global["coeff_files"] ) diff --git a/tests/test_write_lev1_nc.py b/tests/test_write_lev1_nc.py index 7478079..ad8654a 100644 --- a/tests/test_write_lev1_nc.py +++ b/tests/test_write_lev1_nc.py @@ -17,7 +17,7 @@ def test_lev1_to_nc(): for prod in product_list: - hatpro = lev1_to_nc(DATA_DIR, prod, DATA_FORMAT, site) + hatpro = lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site) assert str(hatpro.date) == DATE for t in hatpro.data["time"][:]: date = str( @@ -29,7 +29,7 @@ def test_lev1_to_nc(): def test_output_nc_file(): for prod in product_list: temp_file = "temp_file.nc" - lev1_to_nc(DATA_DIR, prod, DATA_FORMAT, site, output_file=temp_file) + lev1_to_nc(prod, DATA_DIR, DATA_FORMAT, site, output_file=temp_file) with netCDF4.Dataset(temp_file) as nc: # Write tests for the created netCDF file here: assert nc.date == DATE diff --git a/tests/test_write_lev2_nc.py b/tests/test_write_lev2_nc.py index d5e828c..e956683 100644 --- a/tests/test_write_lev2_nc.py +++ b/tests/test_write_lev2_nc.py @@ -20,7 +20,7 @@ def l1_file(request): fd, path = tempfile.mkstemp() os.close(fd) - lev1_to_nc(DATA_DIR, "1C01", DATA_FORMAT, SITE, path) + lev1_to_nc("1C01", DATA_DIR, DATA_FORMAT, SITE, path) def delete_file(): os.unlink(path) @@ -32,7 +32,7 @@ def delete_file(): def test_generate_lev2_single_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_single(l1_file, path, DATA_FORMAT) + generate_lev2_single(l1_file, path, data_format=DATA_FORMAT) os.unlink(path) @@ -42,7 +42,7 @@ def test_generate_lev2_single_no_site(l1_file): generate_lev2_single( l1_file, path, - DATA_FORMAT, + data_format=DATA_FORMAT, coeff_files=COEFF_FILES, ) os.unlink(path) @@ -51,7 +51,7 @@ def test_generate_lev2_single_no_site(l1_file): def test_generate_lev2_multi_site(l1_file): fd, path = tempfile.mkstemp() os.close(fd) - generate_lev2_multi(l1_file, path, DATA_FORMAT) + generate_lev2_multi(l1_file, path, data_format=DATA_FORMAT) os.unlink(path) @@ -61,6 +61,6 @@ def test_generate_lev2_multi_no_site(l1_file): generate_lev2_multi( l1_file, path, - DATA_FORMAT, + data_format=DATA_FORMAT, coeff_files=COEFF_FILES, ) From 8c2d0f8b95ddae6e1f96c20c72d892c973d33393 Mon Sep 17 00:00:00 2001 From: Simo Tukiainen Date: Thu, 17 Sep 2026 12:40:26 +0300 Subject: [PATCH 28/28] Add test for processing without site config --- tests/test_no_config.py | 73 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tests/test_no_config.py diff --git a/tests/test_no_config.py b/tests/test_no_config.py new file mode 100644 index 0000000..9468eb5 --- /dev/null +++ b/tests/test_no_config.py @@ -0,0 +1,73 @@ +import datetime +import glob +import os +import sys + +import netCDF4 +import pytest + +from mwrpy.level1.write_lev1_nc import lev1_to_nc +from mwrpy.level2.lev2_collocated import generate_lev2_multi, generate_lev2_single + +PACKAGE_DIR = os.path.abspath(os.path.dirname(__file__)) +DATA_DIR = f"{PACKAGE_DIR}/data/hyytiala" +COEFF_FILES = glob.glob( + f"{PACKAGE_DIR}/../mwrpy/site_config/hyytiala/coefficients/*.ret" +) +DATE = datetime.date(2023, 4, 6) +# Site without config file in the repository +INSTRUMENT_CONFIG = { + "site": "nowhere", + "latitude": 61.844, + "longitude": 24.287, + "altitude": 150, +} + + +@pytest.fixture(scope="module") +def l1_file(tmp_path_factory): + path = tmp_path_factory.mktemp("no_config") / "l1c.nc" + hatpro = lev1_to_nc( + "1C01", + DATA_DIR, + output_file=path, + coeff_files=COEFF_FILES, + instrument_config=INSTRUMENT_CONFIG, + instrument_type="hatpro", + ) + # Returned object keeps epoch seconds even though file has hours + t0 = float(hatpro.data["time"][:][0]) + assert datetime.datetime.fromtimestamp(t0, tz=datetime.timezone.utc).date() == DATE + assert "time_bnds" in hatpro.data + return path + + +def test_lev1(l1_file): + with netCDF4.Dataset(l1_file) as nc: + assert nc.location == "nowhere" + assert nc.cloudnet_file_type == "mwr-l1c" + assert (nc.year, nc.month, nc.day) == ("2023", "04", "06") + assert nc.variables["time"].units.startswith("hours since 2023-04-06") + assert 0 <= nc.variables["time"][:].min() < nc.variables["time"][:].max() <= 24 + assert "time_bnds" not in nc.variables + assert "zenith_angle" in nc.variables + assert nc.variables["altitude"][:].max() == 150 + + +# Level 2 writing is not tested on Windows +@pytest.mark.skipif(sys.platform == "win32", reason="Not supported on Windows") +@pytest.mark.parametrize( + "fun, file_type, variable", + [ + (generate_lev2_single, "mwr-single", "lwp"), + (generate_lev2_multi, "mwr-multi", "temperature"), + ], +) +def test_lev2(l1_file, tmp_path, fun, file_type, variable): + path = tmp_path / "l2.nc" + fun(l1_file, path, coeff_files=COEFF_FILES) + with netCDF4.Dataset(path) as nc: + assert nc.location == "nowhere" + assert nc.cloudnet_file_type == file_type + assert variable in nc.variables + assert nc.variables["time"].units.startswith("hours since 2023-04-06")