From f319aee8f596b8a804987a8bb8d9935da06e4344 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 24 Sep 2026 16:24:36 -0600 Subject: [PATCH 01/12] Refactor ISMIP7 forcing archive resolution Resolve forcing files directly from the native AIS and GIS archive layouts instead of requiring manually rearranged inputs. * centralize forcing path discovery in a shared archive resolver * select the latest available version independently for each dataset * support explicit version overrides and numeric version ordering * make forcing product and resolution configurable * support ESM, OCX, and fracture-data directory layouts * include source-grid information in mapping filenames * remove hard-coded versions and resolutions from ice-sheet parameters * update configuration, documentation, and resolver tests Co-authored-by: Codex [codex@openai.com](mailto:codex@openai.com) --- compass/landice/ismip7/archive.py | 312 ++++++++++++++++++ compass/landice/ismip7/ice_sheet_params.py | 17 +- .../atmosphere/process_runoff.py | 38 +-- .../ismip7_forcing/atmosphere/process_smb.py | 38 +-- .../atmosphere/process_smb_gradient.py | 38 +-- .../atmosphere/process_temperature.py | 38 +-- .../process_temperature_gradient.py | 38 +-- .../fracture/process_excess_melt.py | 21 +- .../fracture/process_lake_properties.py | 21 +- .../fracture/process_shelf_collapse.py | 21 +- .../tests/ismip7_forcing/ismip7_forcing.cfg | 26 +- .../ismip7_forcing/ismip7_forcing_ocx_ais.cfg | 13 +- .../ismip7_forcing/ismip7_forcing_ocx_gis.cfg | 13 +- .../ismip7_forcing/ismip7_forcing_test.cfg | 15 +- .../ocean_thermal/process_thermal_forcing.py | 84 ++--- docs/developers_guide/landice/api.rst | 6 + docs/developers_guide/landice/framework.rst | 10 +- .../landice/test_groups/ismip7_forcing.rst | 24 +- .../landice/test_groups/ismip7_forcing.rst | 111 +++---- tests/landice/ismip7/test_archive.py | 167 ++++++++++ 20 files changed, 716 insertions(+), 335 deletions(-) create mode 100644 compass/landice/ismip7/archive.py create mode 100644 tests/landice/ismip7/test_archive.py diff --git a/compass/landice/ismip7/archive.py b/compass/landice/ismip7/archive.py new file mode 100644 index 0000000000..bebfe2ec91 --- /dev/null +++ b/compass/landice/ismip7/archive.py @@ -0,0 +1,312 @@ +"""Resolve forcing files in the native ISMIP7 archive hierarchy.""" + +import glob +import os +import re +from dataclasses import dataclass + +from compass.landice.ismip7.ice_sheet_params import get_params + +_ICE_SHEET_DIRECTORIES = { + 'ais': 'AIS', + 'gis': 'GIS', +} + +_DEFAULT_ATMOSPHERE_PRODUCTS = { + 'CESM2-WACCM': 'SDBN1', + 'MRI-ESM2-0': 'GEMB-SDBN1', + 'RACMO2.3p2-ERA': 'SDBN1', +} + +_VERSION_PATTERN = re.compile(r'^v(\d+(?:\.\d+)*)$') +_GRID_PATTERN = re.compile(r'^(.+)-(\d+)m$') + + +@dataclass(frozen=True) +class ForcingSource: + """A resolved collection of ISMIP7 forcing files.""" + + directory: str + files: tuple[str, ...] + version: str + model: str + scenario: str + forcing_group: str + label: str + source_grid: str + product: str | None = None + resolution: str | None = None + + +def resolve_atmosphere_source(config, variable): + """ + Resolve one atmosphere variable from the native archive hierarchy. + + ``latest`` is evaluated independently for each variable because archive + versions are not necessarily synchronized across variables. + """ + params = get_params(config) + scenario = config.get('ismip7', 'scenario') + configured_model = config.get('ismip7', 'model') + + if params['atm_model'] is None: + model = configured_model + forcing_group = f'{model}_{scenario}' + base_path = os.path.join(_ice_sheet_path(config), model, scenario) + else: + model = params['atm_model'] + forcing_group = scenario + base_path = os.path.join(_ice_sheet_path(config), scenario, model) + + section = 'ismip7_atmosphere' + product = config.get(section, 'product', fallback='auto') + resolution = config.get(section, 'resolution', fallback='auto') + grid_name, product, resolution = _resolve_atmosphere_grid( + base_path, model, product, resolution) + + variable_path = os.path.join(base_path, grid_name, variable) + version = _requested_version(config, section, variable) + version_path, version, files = resolve_version_directory( + variable_path, version, f'{variable}_*.nc') + + return ForcingSource( + directory=version_path, + files=files, + version=version, + model=model, + scenario=scenario, + forcing_group=forcing_group, + label=f'{model}_{scenario}', + source_grid=f'atmosphere_{grid_name}', + product=product, + resolution=resolution, + ) + + +def resolve_ocean_source(config, choice=None): + """Resolve ocean thermal forcing from the native archive hierarchy.""" + params = get_params(config) + ice_sheet = config.get('ismip7', 'ice_sheet') + scenario = config.get('ismip7', 'scenario') + configured_model = config.get('ismip7', 'model') + section = 'ismip7_ocean_thermal' + requested_version = config.get(section, 'version', fallback='latest') + + if params.get('ocean_choice_layout', False): + if choice is None: + raise ValueError('An ocean choice is required for AIS OCX.') + model = configured_model + grid_name = 'ocean' + base_path = os.path.join( + _ice_sheet_path(config), scenario, 'ocean', choice) + forcing_group = f'{scenario}_{choice}' + label = forcing_group + resolution = None + else: + if params['ocean_model'] is None: + model = configured_model + base_path = os.path.join( + _ice_sheet_path(config), model, scenario) + forcing_group = f'{model}_{scenario}' + else: + model = params['ocean_model'] + base_path = os.path.join( + _ice_sheet_path(config), scenario, model) + forcing_group = scenario + + if ice_sheet == 'ais': + grid_name = 'ocean' + resolution = None + else: + requested_resolution = config.get( + section, 'resolution', fallback='auto') + grid_name, resolution = _resolve_ocean_grid( + base_path, requested_resolution) + base_path = os.path.join(base_path, grid_name, 'tf') + label = f'{model}_{scenario}' + + version_path, version, files = resolve_version_directory( + base_path, requested_version, 'tf_*.nc') + + return ForcingSource( + directory=version_path, + files=files, + version=version, + model=model, + scenario=scenario, + forcing_group=forcing_group, + label=label, + source_grid=grid_name, + resolution=resolution, + ) + + +def resolve_fracture_source(config, file_pattern): + """Resolve one fracture product, keeping all pathways on one version.""" + model = config.get('ismip7', 'model') + scenario = config.get('ismip7', 'scenario') + base_path = os.path.join( + _ice_sheet_path(config), model, scenario, 'fracture') + requested_version = config.get( + 'ismip7_fracture', 'version', fallback='latest') + version_path, version, files = resolve_version_directory( + base_path, requested_version, file_pattern) + + return ForcingSource( + directory=version_path, + files=files, + version=version, + model=model, + scenario=scenario, + forcing_group=f'{model}_{scenario}', + label=f'{model}_{scenario}', + source_grid='fracture', + ) + + +def resolve_version_directory(base_path, requested_version='latest', + file_pattern='*.nc'): + """ + Resolve an explicit version or the numerically latest version directory. + + Version names may contain any number of numeric components, so ``v2.10`` + correctly sorts after ``v2.9``. If the latest directory exists but does + not contain the expected files, an error is raised instead of silently + falling back to an older version. + """ + versions = [] + if os.path.isdir(base_path): + for name in os.listdir(base_path): + path = os.path.join(base_path, name) + match = _VERSION_PATTERN.fullmatch(name) + if match is not None and os.path.isdir(path): + key = tuple(int(value) for value in match.group(1).split('.')) + versions.append((key, name)) + + if requested_version.lower() == 'latest': + if not versions: + raise FileNotFoundError( + f'No version directories were found in:\n {base_path}') + _, version = max(versions) + else: + version = requested_version + if _VERSION_PATTERN.fullmatch(version) is None: + raise ValueError( + f"Invalid version '{version}'. Use 'latest' or a version " + f"such as 'v2' or 'v2.1'.") + + version_path = os.path.join(base_path, version) + if not os.path.isdir(version_path): + available = ', '.join(name for _, name in sorted(versions)) or 'none' + raise FileNotFoundError( + f"Version '{version}' was not found in:\n {base_path}\n" + f'Available versions: {available}') + + files = tuple(sorted(glob.glob(os.path.join(version_path, file_pattern)))) + if not files: + raise FileNotFoundError( + f"Version '{version}' contains no files matching:\n" + f' {os.path.join(version_path, file_pattern)}') + + return version_path, version, files + + +def mapping_file_name(config, component, source_grid, method_remap): + """Build a mapping filename that identifies the selected source grid.""" + ice_sheet = config.get('ismip7', 'ice_sheet') + mali_mesh_name = config.get('ismip7', 'mali_mesh_name') + grid = re.sub(r'[^A-Za-z0-9]+', '_', source_grid).strip('_').lower() + return (f'map_ismip7_{ice_sheet}_{component}_{grid}_to_' + f'{mali_mesh_name}_{method_remap}.nc') + + +def _ice_sheet_path(config): + ice_sheet = config.get('ismip7', 'ice_sheet') + if ice_sheet not in _ICE_SHEET_DIRECTORIES: + raise ValueError( + f"Unknown ice_sheet '{ice_sheet}'. Must be one of: " + f'{list(_ICE_SHEET_DIRECTORIES)}') + archive_root = config.get('ismip7', 'base_path_ismip7') + return os.path.join(archive_root, _ICE_SHEET_DIRECTORIES[ice_sheet]) + + +def _requested_version(config, section, variable): + option = f'{variable}_version' + if config.has_option(section, option): + return config.get(section, option) + return config.get(section, 'version', fallback='latest') + + +def _resolve_atmosphere_grid(base_path, model, requested_product, + requested_resolution): + candidates = [] + if os.path.isdir(base_path): + for name in os.listdir(base_path): + match = _GRID_PATTERN.fullmatch(name) + path = os.path.join(base_path, name) + if match is not None and os.path.isdir(path): + product, metres = match.groups() + candidates.append((product, int(metres), name)) + + product = requested_product + if product.lower() == 'auto': + preferred = _DEFAULT_ATMOSPHERE_PRODUCTS.get(model) + products = sorted({candidate[0] for candidate in candidates}) + if preferred in products: + product = preferred + elif len(products) == 1: + product = products[0] + else: + choices = ', '.join(products) or 'none' + raise ValueError( + f"Could not select an atmosphere product automatically in:\n" + f' {base_path}\nAvailable products: {choices}. Set ' + f"'product' in [ismip7_atmosphere].") + + product_candidates = [candidate for candidate in candidates + if candidate[0] == product] + resolution = _select_resolution( + base_path, product_candidates, requested_resolution, 'atmosphere') + grid_name = f'{product}-{resolution}' + return grid_name, product, resolution + + +def _resolve_ocean_grid(base_path, requested_resolution): + candidates = [] + if os.path.isdir(base_path): + for name in os.listdir(base_path): + match = re.fullmatch(r'ocean-(\d+)m', name) + path = os.path.join(base_path, name) + if match is not None and os.path.isdir(path): + candidates.append(('ocean', int(match.group(1)), name)) + + resolution = _select_resolution( + base_path, candidates, requested_resolution, 'ocean') + return f'ocean-{resolution}', resolution + + +def _select_resolution(base_path, candidates, requested_resolution, + component): + if requested_resolution.lower() == 'auto': + if not candidates: + raise FileNotFoundError( + f'No {component} resolution directories were found in:\n' + f' {base_path}') + metres = min(candidate[1] for candidate in candidates) + else: + normalized = requested_resolution.lower().removesuffix('m') + if not normalized.isdigit(): + raise ValueError( + f"Invalid {component} resolution '{requested_resolution}'. " + f"Use 'auto' or a value such as '1000m'.") + metres = int(normalized) + + matches = [candidate for candidate in candidates + if candidate[1] == metres] + if not matches: + available = ', '.join( + f'{candidate[1]}m' for candidate in sorted(candidates)) or 'none' + raise FileNotFoundError( + f"Resolution '{metres}m' was not found in:\n {base_path}\n" + f'Available resolutions: {available}') + return f'{metres}m' diff --git a/compass/landice/ismip7/ice_sheet_params.py b/compass/landice/ismip7/ice_sheet_params.py index d991eae45c..a6e05fe937 100644 --- a/compass/landice/ismip7/ice_sheet_params.py +++ b/compass/landice/ismip7/ice_sheet_params.py @@ -7,10 +7,6 @@ 'ais': { 'projection': 'ais-bedmap2', 'prefix': 'AIS', - 'atm_resolution': '2000m', - 'atm_version': 'v2', - 'ocean_version': 'v3', - 'ocean_grid': 'ocean', 'ocean_3d': True, 'ocean_temporal': 'decade', 'atm_model': None, @@ -19,10 +15,6 @@ 'gis': { 'projection': 'gis-bamber', 'prefix': 'GrIS', - 'atm_resolution': '1000m', - 'atm_version': 'v2', - 'ocean_version': 'v2', - 'ocean_grid': 'ocean', 'ocean_3d': False, 'ocean_temporal': 'yearly', 'atm_model': None, @@ -32,19 +24,14 @@ # Overrides applied for the OCX (reanalysis) scenario. OCX has no distinct # ESM model: it uses fixed reanalysis products (RACMO for the atmosphere and -# EN4 for the ocean) at data version v1. When scenario is 'OCX' the [ismip7] -# model option is ignored and these sources are used instead. +# EN4 for the ocean). When scenario is 'OCX' the [ismip7] model option is +# ignored and these sources are used instead. _OCX_OVERRIDES = { 'gis': { - 'atm_version': 'v1', - 'ocean_version': 'v1', - 'ocean_grid': 'ocean-1000m', 'atm_model': 'RACMO2.3p2-ERA', 'ocean_model': 'EN4', }, 'ais': { - 'atm_version': 'v1', - 'ocean_version': 'v1', 'atm_model': 'RACMO2.3p2-ERA', 'ocean_model': None, # AIS OCX ocean files have no model token and live in per-choice diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index 8125abe250..11d020ad22 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,7 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -49,40 +51,22 @@ def run(self): """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_atmosphere"] method_remap = section.get("method_remap") start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "mrro", version) - file_pattern = (f"mrro_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No runoff files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "mrro") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -104,8 +88,8 @@ def run(self): f"{start_year}-{end_year}") # Build mapping file (reuse if already created by other atm steps) - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file...") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index 9701265638..5093a7f13a 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,7 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -49,13 +51,9 @@ def run(self): """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -64,24 +62,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "acabf", version) - file_pattern = (f"acabf_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No SMB files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "acabf") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -103,9 +88,8 @@ def run(self): f"{start_year}-{end_year}") # Build mapping file using the first input file as the grid template - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file...") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index e35755e9b6..508cd6e7d4 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,7 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -50,13 +52,9 @@ def run(self): """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -65,24 +63,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "dacabfdz", version) - file_pattern = (f"dacabfdz_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No SMB gradient files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "dacabfdz") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -104,9 +89,8 @@ def run(self): f"{start_year}-{end_year}") # Build mapping file (reuse if already created by process_smb) - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file...") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index 56b13403be..bff8db75fb 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,7 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -49,13 +51,9 @@ def run(self): """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -64,24 +62,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "ts", version) - file_pattern = (f"ts_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No temperature files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "ts") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -103,9 +88,8 @@ def run(self): f"{start_year}-{end_year}") # Build mapping file (reuse if already created by process_smb) - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file...") diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index 41cf9f30aa..31ae46b1eb 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,7 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -51,13 +53,9 @@ def run(self): """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -66,24 +64,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "dtsdz", version) - file_pattern = (f"dtsdz_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No temperature gradient files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "dtsdz") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -105,9 +90,8 @@ def run(self): f"for years {start_year}-{end_year}") # Build mapping file (reuse if already created by other steps) - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file...") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index 0f291792f0..057dca5010 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -7,6 +7,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, @@ -62,17 +66,14 @@ def run(self): config = self.config section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_fracture"] method_remap = section.get("method_remap_excess_melt") - version = section.get("version") start_year = section.getint("start_year") end_year = section.getint("end_year") @@ -83,14 +84,10 @@ def run(self): return # Discover the excess melt file - input_path = os.path.join(base_path_ismip7, "fracture", version) file_pattern = "excess_melt_*.nc" - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No excess melt file found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_fracture_source(config, file_pattern) + input_path = source.directory + all_files = source.files if len(all_files) > 1: raise ValueError( f"Expected a single excess melt file but found " @@ -107,8 +104,8 @@ def run(self): # Build mapping file. Excess melt is a flux, so conservative # remapping is appropriate by default. - mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "fracture", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file for the excess melt grid...") diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index 2b5d5ac1d5..7895425ae1 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -1,9 +1,12 @@ -import glob import os import shutil from mpas_tools.logging import check_call +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, @@ -67,17 +70,14 @@ def run(self): config = self.config section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_fracture"] method_remap = section.get("method_remap_lake_properties") - version = section.get("version") start_year = section.getint("start_year") end_year = section.getint("end_year") @@ -88,14 +88,9 @@ def run(self): return # Discover the lake properties file - input_path = os.path.join(base_path_ismip7, "fracture", version) file_pattern = "lake_properties_*.nc" - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No lake properties file found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_fracture_source(config, file_pattern) + all_files = source.files if len(all_files) > 1: raise ValueError( f"Expected a single lake properties file but found " @@ -107,8 +102,8 @@ def run(self): # Build mapping file. Lake properties are continuous fields, so # bilinear remapping is appropriate by default. - mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "fracture", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file for the lake properties " diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index bf6537438e..7e47058b60 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -1,9 +1,12 @@ -import glob import os import shutil from mpas_tools.logging import check_call +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, @@ -54,17 +57,14 @@ def run(self): config = self.config section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_fracture"] method_remap = section.get("method_remap_shelf_collapse") - version = section.get("version") start_year = section.getint("start_year") end_year = section.getint("end_year") @@ -75,14 +75,9 @@ def run(self): return # Discover the ice shelf collapse mask file - input_path = os.path.join(base_path_ismip7, "fracture", version) file_pattern = "ice_shelf_collapse_mask_*.nc" - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No ice shelf collapse mask file found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_fracture_source(config, file_pattern) + all_files = source.files if len(all_files) > 1: raise ValueError( f"Expected a single ice shelf collapse mask file but found " @@ -93,8 +88,8 @@ def run(self): logger.info(f"Processing ice shelf collapse mask: {basename}") # Build mapping file. neareststod preserves the 0/1 mask values. - mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "fracture", source.source_grid, method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file for the collapse mask grid...") diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg index be7d195aa4..7bc0836ab7 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg @@ -4,7 +4,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = NotAvailable -# Base path to the input ISMIP7 forcing files. User has to supply. +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +# User has to supply. base_path_ismip7 = NotAvailable # Base path to the MALI mesh. User has to supply. @@ -39,6 +40,17 @@ process_ocean_climatology = true # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +# Atmosphere product. 'auto' selects SDBN1 for CESM2-WACCM and OCX, and +# GEMB-SDBN1 for MRI-ESM2-0. Other products (e.g. dEBM2) can be named here. +product = auto + +# Source-grid resolution. 'auto' selects the finest available resolution. +resolution = auto + +# Dataset version. 'latest' is resolved independently for each variable. +# A variable-specific override such as acabf_version = v2 takes precedence. +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -51,6 +63,13 @@ end_year = 2014 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +# Source-grid resolution for GrIS. 'auto' selects the finest available +# resolution. AIS ocean forcing has a single archive grid. +resolution = auto + +# Dataset version. Use an explicit version such as v2 for reproducibility. +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -70,6 +89,9 @@ end_year = 2014 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +# Dataset version +version = latest + # Base path to observational climatology data # (directory containing tf/, so/, thetao/ subdirs) base_path_climatology = /path/to/ISMIP7/forcing/AIS/obs/zhou_annual_06_nov @@ -91,7 +113,7 @@ method_remap_excess_melt = conserve method_remap_lake_properties = bilinear # Version subdirectory of the fracture forcing data -version = v2 +version = latest # Start year for processing start_year = 1850 diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg index 975cb89b85..2c41d4ae52 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg @@ -14,8 +14,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = ais -# Base path to the input ISMIP7 forcing files. User has to supply. -base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/AIS/OCX +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing # Base path to the MALI mesh. User has to supply. base_path_mali = /global/cfs/cdirs/fanssie/MALI_input_files/AIS_4to20km_r01 @@ -48,6 +48,10 @@ process_ocean_climatology = false # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +product = auto +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -60,6 +64,9 @@ end_year = 2025 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -80,5 +87,7 @@ end_year = 2025 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +version = latest + # Base path to observational climatology data base_path_climatology = None diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg index d3658027db..d876b05c03 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg @@ -12,8 +12,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = gis -# Base path to the input ISMIP7 forcing files. User has to supply. -base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/GIS/OCX +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing # Base path to the MALI mesh. User has to supply. base_path_mali = /global/cfs/cdirs/fanssie/MALI_input_files/GIS_1to10km_r02/ @@ -46,6 +46,10 @@ process_ocean_climatology = false # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +product = auto +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -58,6 +62,9 @@ end_year = 2025 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -73,5 +80,7 @@ end_year = 2025 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +version = latest + # Base path to observational climatology data base_path_climatology = None diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg index ab282b2e06..53eaf517e2 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg @@ -4,8 +4,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = ais -# Base path to the input ISMIP7 forcing files. User has to supply. -base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/AIS/CESM2-WACCM/ssp585 +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing # Base path to the MALI mesh. User has to supply. base_path_mali = /global/cfs/cdirs/fanssie/MALI_input_files/AIS_4to20km_r01 @@ -39,6 +39,10 @@ process_ocean_climatology = true # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +product = auto +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -51,6 +55,9 @@ end_year = 2301 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -66,6 +73,8 @@ end_year = 2301 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +version = latest + # Base path to observational climatology data # (directory containing tf/, so/, thetao/ subdirs) base_path_climatology = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/AIS/obs/zhou_annual_06_nov @@ -87,7 +96,7 @@ method_remap_excess_melt = conserve method_remap_lake_properties = bilinear # Version subdirectory of the fracture forcing data -version = v2 +version = latest # Start year for processing start_year = 2015 diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index 595d90b964..30e96ed6da 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,6 +5,11 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_ocean_source, + resolve_version_directory, +) from compass.landice.ismip7.ice_sheet_params import get_params from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source @@ -72,22 +76,16 @@ def _run_scenario(self): params = get_params(config) section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_ocean_thermal"] method_remap = section.get("method_remap") start_year = section.getint("start_year") end_year = section.getint("end_year") - prefix = params['prefix'] - ocean_version = params['ocean_version'] - ocean_grid = params['ocean_grid'] ocean_3d = params['ocean_3d'] # Assemble the list of forcing sources to process. AIS OCX has several @@ -97,38 +95,14 @@ def _run_scenario(self): jobs = [] if params.get('ocean_choice_layout', False): for choice in self._get_ocean_choices(config): - input_path = os.path.join(base_path_ismip7, "ocean", choice, - ocean_version) - file_pattern = (f"tf_{prefix}_{scenario}_ocean_{choice}_" - f"{ocean_version}_*.nc") - jobs.append({ - 'input_path': input_path, - 'file_pattern': file_pattern, - 'forcing_group': f"{scenario}_{choice}", - 'label': f"{scenario}_{choice}", - }) + jobs.append(resolve_ocean_source(config, choice=choice)) else: - if params['ocean_model'] is not None: - forcing_group = scenario - model = params['ocean_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "ocean", "tf", - ocean_version) - file_pattern = (f"tf_{prefix}_{model}_{scenario}_" - f"{ocean_grid}_{ocean_version}_*.nc") - jobs.append({ - 'input_path': input_path, - 'file_pattern': file_pattern, - 'forcing_group': forcing_group, - 'label': f"{model}_{scenario}", - }) - - # Mapping file is shared across sources (identical ocean source grid). - mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" - f"{mali_mesh_name}_{method_remap}.nc") + jobs.append(resolve_ocean_source(config)) for job in jobs: + self.logger.info(f"Using ocean source {job.directory}") + mapping_file = mapping_file_name( + config, "ocean", job.source_grid, method_remap) self._process_ocean_forcing( job, mapping_file, ocean_3d, method_remap, start_year, end_year, mali_mesh_name, mali_mesh_file, @@ -141,7 +115,7 @@ def _run_scenario(self): if params.get('ocean_choice_layout', False): for job in jobs: self._link_atmosphere_outputs( - output_base_path, scenario, job['forcing_group']) + output_base_path, scenario, job.forcing_group) def _get_ocean_choices(self, config): """ @@ -232,9 +206,8 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, Parameters ---------- - job : dict - Source description with keys ``input_path``, ``file_pattern``, - ``forcing_group``, and ``label`` + job : compass.landice.ismip7.archive.ForcingSource + Resolved ocean forcing source mapping_file : str Path of the shared ocean-to-MALI mapping file (built on demand) ocean_3d : bool @@ -251,17 +224,9 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, logger = self.logger config = self.config - input_path = job['input_path'] - file_pattern = job['file_pattern'] - forcing_group = job['forcing_group'] - label = job['label'] - - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No ocean thermal forcing files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + forcing_group = job.forcing_group + label = job.label + all_files = job.files # Filter to files that overlap with the requested year range. # AIS files are named with decade or multi-decade ranges (e.g., @@ -364,21 +329,16 @@ def _run_climatology(self): mali_mesh_name = section.get("mali_mesh_name") mali_mesh_file = section.get("mali_mesh_file") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_ocean_climatology"] method_remap = section.get("method_remap") base_path_climatology = section.get("base_path_climatology") - version = 'v3' + requested_version = section.get("version", fallback="latest") # Discover climatology TF file - input_path = os.path.join(base_path_climatology, "tf", version) - all_files = sorted(glob.glob(os.path.join(input_path, "tf_*.nc"))) - - if not all_files: - raise FileNotFoundError( - f"No ocean climatology TF files found in:\n" - f" {input_path}") + _, version, all_files = resolve_version_directory( + os.path.join(base_path_climatology, "tf"), requested_version, + "tf_*.nc") # Use the first (and likely only) file input_file = all_files[0] @@ -386,8 +346,8 @@ def _run_climatology(self): f"{os.path.basename(input_file)}") # Build mapping file using the climatology file as grid template. - mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" - f"{mali_mesh_name}_{method_remap}.nc") + mapping_file = mapping_file_name( + config, "ocean", "climatology", method_remap) if not os.path.exists(mapping_file): logger.info("Building mapping file for ocean grid...") diff --git a/docs/developers_guide/landice/api.rst b/docs/developers_guide/landice/api.rst index b0ecad439e..3170a05f38 100644 --- a/docs/developers_guide/landice/api.rst +++ b/docs/developers_guide/landice/api.rst @@ -27,6 +27,12 @@ ISMIP7 framework :toctree: generated/ ice_sheet_params.get_params + archive.ForcingSource + archive.resolve_atmosphere_source + archive.resolve_ocean_source + archive.resolve_fracture_source + archive.resolve_version_directory + archive.mapping_file_name mapping.build_mapping_file remap.extrapolate_source remap.open_rename_and_trim diff --git a/docs/developers_guide/landice/framework.rst b/docs/developers_guide/landice/framework.rst index 3d25a70a31..7fe14e3e08 100644 --- a/docs/developers_guide/landice/framework.rst +++ b/docs/developers_guide/landice/framework.rst @@ -29,8 +29,14 @@ rather than in any one test group. :py:func:`compass.landice.ismip7.ice_sheet_params.get_params()` returns the parameters that differ between the Antarctic and Greenland ice sheets: the -projection, the filename prefix, dataset versions and resolutions, and whether -the ocean forcing is 3-D. +projection, fixed OCX sources, and whether the ocean forcing is 3-D. + +:py:mod:`compass.landice.ismip7.archive` resolves atmosphere, ocean, and +fracture files from the native archive hierarchy. It selects products and +resolutions from config, compares dotted version numbers numerically, and +resolves ``latest`` separately for datasets that are versioned independently. +Mapping filenames include the selected source grid to prevent reuse across +different resolutions. :py:func:`compass.landice.ismip7.mapping.build_mapping_file()` builds an ESMF mapping file from an ISMIP7 polar stereographic grid to a MALI mesh, using diff --git a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst index 6f88904b99..fb268dd12e 100644 --- a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst @@ -22,17 +22,18 @@ in :ref:`landice_ismip7_forcing` in the User's Guide. Code shared with the other ISMIP7 test groups lives in the landice framework package :py:mod:`compass.landice.ismip7`, described in :ref:`dev_landice_framework`. This test group uses -:py:func:`compass.landice.ismip7.ice_sheet_params.get_params` for the -ice-sheet-specific parameters (projection, file naming prefix, grid -resolution, data version, ocean dimensionality), +:py:func:`compass.landice.ismip7.ice_sheet_params.get_params` for invariant +ice-sheet-specific parameters (projection, file naming prefix, and ocean +dimensionality), :py:mod:`compass.landice.ismip7.archive` for native archive +path, resolution, and version discovery, :py:func:`compass.landice.ismip7.mapping.build_mapping_file` to create the SCRIP and ESMF mapping files, and the remapping helpers in :py:mod:`compass.landice.ismip7.remap`. -When ``scenario = OCX``, ``get_params`` applies a set of OCX overrides on top -of the ice-sheet defaults: data version ``v1``, the ocean file-name grid token -(e.g. ``ocean-1000m``), and the fixed reanalysis sources (``atm_model`` = -``RACMO2.3p2-ERA`` and ``ocean_model`` = ``EN4``). The processing steps use +When ``scenario = OCX``, ``get_params`` supplies the fixed reanalysis sources +(``atm_model`` = ``RACMO2.3p2-ERA`` and ``ocean_model`` = ``EN4``). The +archive resolver discovers their native paths, selected resolutions, and +versions. The processing steps use ``atm_model`` / ``ocean_model`` in place of the ``[ismip7] model`` option when they are set, so the OCX ``model`` option is ignored. This keeps OCX handling centralized and lets a single config file drive both test cases. @@ -65,7 +66,8 @@ test case processes the ISMIP7 atmosphere forcing fields. It contains five steps: SMB, temperature, their respective gradients, and runoff. Each step discovers input files matching the ice-sheet-specific naming pattern, builds or reuses a mapping file, remaps each input file with ``ncremap``, and -combines/renames the results to MALI conventions. +combines/renames the results to MALI conventions. ``latest`` is resolved per +variable because atmosphere datasets can have different current versions. Steps: @@ -99,7 +101,6 @@ options ``process_ocean_thermal`` and ``process_ocean_climatology`` in the scenario combination). Uses config from ``[ismip7_ocean_thermal]``. * ``_run_climatology()``: Processes the static observational climatology (Zhou et al., AIS only). Uses config from ``[ismip7_ocean_climatology]``. - The TF version (currently v3) is hard-coded. For AIS scenario data, the step: @@ -128,8 +129,9 @@ fracture The :py:class:`compass.landice.tests.ismip7_forcing.fracture.Fracture` test case processes the ISMIP7 surface-melt-driven ice shelf collapse forcing (AIS only). It implements the three ISMIP7 pathways as independent -steps, each discovering its source file from the ``fracture/{version}/`` -subdirectory of ``base_path_ismip7``, building or reusing a mapping file, +steps, each discovering its source file from the native +``AIS/{model}/{scenario}/fracture/{version}/`` hierarchy, building or reusing +a mapping file, remapping with ``ncremap``, and renaming the result to MALI conventions with an accompanying ``xtime`` variable. Per-pathway remapping methods are set in the ``[ismip7_fracture]`` config section. Setting a pathway's remapping-method diff --git a/docs/users_guide/landice/test_groups/ismip7_forcing.rst b/docs/users_guide/landice/test_groups/ismip7_forcing.rst index 4a5ca9ea82..6393efa8ed 100644 --- a/docs/users_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip7_forcing.rst @@ -44,7 +44,8 @@ To use this test group, users need to: 2. Set the ``ice_sheet`` config option to either ``ais`` or ``gis``. -3. Provide the path to the ISMIP7 forcing data (``base_path_ismip7``). +3. Set ``base_path_ismip7`` to the root of the native ISMIP7 archive, which + contains the ``AIS`` and ``GIS`` directories. 4. Run the ``atmosphere`` test case for each model and scenario combination. @@ -110,73 +111,43 @@ expects exactly one file per forcing field there. Input Data ---------- -ISMIP7 forcing data is organized by variable and version. The expected -directory structure under ``base_path_ismip7`` is: - -For AIS atmosphere (2km, polar stereographic EPSG:3031): - -.. code-block:: none - - acabf/v2/acabf_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - ts/v2/ts_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - dacabfdz/v2/dacabfdz_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - dtsdz/v2/dtsdz_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - mrro/v2/mrro_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - -For AIS ocean thermal (8km, 30 depth levels, decade files): - -.. code-block:: none - - ocean/tf/v3/tf_AIS_{model}_{scenario}_ocean_v3_{start_year}-{end_year}.nc - -For AIS ocean thermal climatology (8km, 30 depth levels, static): - -.. code-block:: none - - {base_path_climatology}/tf/v3/tf_AIS_obs_ocean_climatology_*.nc - -For AIS fracture / ice shelf collapse mask (8km, annual, Path C): - -.. code-block:: none - - fracture/v2/ice_shelf_collapse_mask_*.nc - -For GrIS atmosphere (1km, polar stereographic EPSG:3413): - -.. code-block:: none - - acabf/v2/acabf_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - ts/v2/ts_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - dacabfdz/v2/dacabfdz_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - dtsdz/v2/dtsdz_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - mrro/v2/mrro_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - -For GrIS ocean thermal (same 1km grid, 2D, yearly files): +``base_path_ismip7`` is the unmodified archive root. The resolver selects +files from these native layouts: .. code-block:: none - ocean/tf/v2/tf_GrIS_{model}_{scenario}_ocean_v2_{year}.nc - -The OCX (reanalysis) scenario follows the same directory layout as the ESM -scenarios, but uses fixed reanalysis sources, data version ``v1``, and a -named grid resolution in the ocean file names. For GrIS OCX the atmosphere -source is ``RACMO2.3p2-ERA`` and the ocean source is ``EN4``: - -.. code-block:: none - - acabf/v1/acabf_GrIS_RACMO2.3p2-ERA_OCX_SDBN1-1000m_v1_{year}.nc - ocean/tf/v1/tf_GrIS_EN4_OCX_ocean-1000m_v1_{year}.nc - -Set ``base_path_ismip7`` to the ``OCX`` directory and ``scenario = OCX``. -The sources, version, and ocean grid token are selected automatically for -OCX, and the ``[ismip7] model`` option is ignored (set it to ``None``). + {root}/{AIS|GIS}/{model}/{scenario}/{product}-{resolution}/{variable}/{version}/*.nc + {root}/AIS/{model}/{scenario}/ocean/tf/{version}/*.nc + {root}/GIS/{model}/{scenario}/ocean-{resolution}/tf/{version}/*.nc + {root}/AIS/{model}/{scenario}/fracture/{version}/*.nc + {root}/AIS/OCX/RACMO2.3p2-ERA/SDBN1-{resolution}/{variable}/{version}/*.nc + {root}/GIS/OCX/RACMO2.3p2-ERA/SDBN1-{resolution}/{variable}/{version}/*.nc + {root}/AIS/OCX/ocean/{choice}/{version}/*.nc + {root}/GIS/OCX/EN4/ocean-{resolution}/tf/{version}/*.nc + +The atmosphere ``product`` and atmosphere and GrIS ocean ``resolution`` are +configurable. ``auto`` selects the primary product for the requested model +and the finest available resolution. Set an explicit value to use another +archive product or grid. + +Versions default to ``latest`` and are compared numerically, including dotted +versions such as ``v2.1``. Atmosphere versions are resolved separately for +each variable because the archive does not version all fields in lockstep. +An explicit section-wide version may be set for reproducibility; an option +such as ``acabf_version = v2`` overrides it for one atmosphere variable. The +newest directory must contain matching files, so an incomplete transfer is +reported rather than silently replaced with an older version. + +The observational climatology remains configured separately with +``base_path_climatology``. Its version also defaults to ``latest`` below the +``tf`` directory. .. _landice_ismip7_forcing_config: config options -------------- -The ``ismip7_forcing`` test group uses four config sections. The default +The ``ismip7_forcing`` test group uses five config sections. The default values are: .. code-block:: cfg @@ -187,7 +158,7 @@ values are: # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = NotAvailable - # Base path to the input ISMIP7 forcing files + # Root of the native archive (the directory containing AIS/ and GIS/) base_path_ismip7 = NotAvailable # Base path to the MALI mesh @@ -220,6 +191,13 @@ values are: # config options for ismip7 atmosphere forcing [ismip7_atmosphere] + # Product and source resolution + product = auto + resolution = auto + + # Resolve separately for each variable; explicit versions are also valid + version = latest + # Remapping method: bilinear, neareststod, conserve method_remap = conserve @@ -232,9 +210,16 @@ values are: # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] + # GrIS source resolution and dataset version + resolution = auto + version = latest + # Remapping method: bilinear, neareststod, conserve method_remap = bilinear + # Dataset version + version = latest + # Start year for processing start_year = 1850 @@ -267,7 +252,7 @@ values are: method_remap_lake_properties = bilinear # Version subdirectory of the fracture forcing data - version = v2 + version = latest # Start year for processing start_year = 1850 @@ -356,8 +341,8 @@ surface-melt-driven ice shelf collapse forcing (AIS only). It implements the three ISMIP7 pathways as separate steps, each remapping annual fields from the native 8km polar stereographic grid onto the MALI unstructured mesh. -All three source files are discovered from the ``fracture/{version}/`` -subdirectory of ``base_path_ismip7``. +All three source files are discovered from +``AIS/{model}/{scenario}/fracture/{version}/`` below ``base_path_ismip7``. Each pathway is run independently and can be skipped by setting its remapping-method config option to ``None`` in the ``[ismip7_fracture]`` diff --git a/tests/landice/ismip7/test_archive.py b/tests/landice/ismip7/test_archive.py new file mode 100644 index 0000000000..e8d7726d49 --- /dev/null +++ b/tests/landice/ismip7/test_archive.py @@ -0,0 +1,167 @@ +"""Tests for ISMIP7 native-archive path resolution.""" + +from configparser import ConfigParser + +import pytest + +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, + resolve_fracture_source, + resolve_ocean_source, + resolve_version_directory, +) + + +def _config(root, ice_sheet='gis', model='CESM2-WACCM', + scenario='ssp585'): + config = ConfigParser() + config['ismip7'] = { + 'base_path_ismip7': str(root), + 'ice_sheet': ice_sheet, + 'model': model, + 'scenario': scenario, + 'mali_mesh_name': 'mesh', + } + config['ismip7_atmosphere'] = { + 'product': 'auto', + 'resolution': 'auto', + 'version': 'latest', + } + config['ismip7_ocean_thermal'] = { + 'resolution': 'auto', + 'version': 'latest', + } + config['ismip7_fracture'] = {'version': 'latest'} + return config + + +def _forcing_file(root, relative_path): + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + return path + + +def test_latest_version_uses_numeric_components(tmp_path): + """Dotted version components are sorted numerically, not as strings.""" + _forcing_file(tmp_path, 'v2.9/field.nc') + expected = _forcing_file(tmp_path, 'v2.10/field.nc') + + _, version, files = resolve_version_directory(tmp_path) + + assert version == 'v2.10' + assert files == (str(expected),) + + +def test_latest_does_not_fall_back_from_incomplete_version(tmp_path): + """An incomplete newest transfer is reported instead of hidden.""" + _forcing_file(tmp_path, 'v1/field.nc') + (tmp_path / 'v2').mkdir() + + with pytest.raises(FileNotFoundError, match="Version 'v2' contains no"): + resolve_version_directory(tmp_path) + + +def test_atmosphere_latest_is_resolved_per_variable(tmp_path): + """Variables can select different latest versions on the same grid.""" + base = 'GIS/CESM2-WACCM/ssp585' + _forcing_file( + tmp_path, f'{base}/SDBN1-4000m/acabf/v4/acabf_2000.nc') + acabf = _forcing_file( + tmp_path, f'{base}/SDBN1-1000m/acabf/v3/acabf_2000.nc') + ts = _forcing_file( + tmp_path, f'{base}/SDBN1-1000m/ts/v2/ts_2000.nc') + config = _config(tmp_path) + + acabf_source = resolve_atmosphere_source(config, 'acabf') + ts_source = resolve_atmosphere_source(config, 'ts') + + assert acabf_source.version == 'v3' + assert acabf_source.files == (str(acabf),) + assert ts_source.version == 'v2' + assert ts_source.files == (str(ts),) + assert acabf_source.resolution == '1000m' + + +def test_variable_version_override_takes_precedence(tmp_path): + """A variable-specific version can pin one atmosphere dataset.""" + base = 'GIS/CESM2-WACCM/ssp585/SDBN1-1000m/acabf' + expected = _forcing_file(tmp_path, f'{base}/v2/acabf_2000.nc') + _forcing_file(tmp_path, f'{base}/v3/acabf_2000.nc') + config = _config(tmp_path) + config['ismip7_atmosphere']['acabf_version'] = 'v2' + + source = resolve_atmosphere_source(config, 'acabf') + + assert source.version == 'v2' + assert source.files == (str(expected),) + + +def test_auto_product_uses_model_default(tmp_path): + """Auto distinguishes the primary model product from dEBM2.""" + base = 'AIS/MRI-ESM2-0/ssp585' + expected = _forcing_file( + tmp_path, f'{base}/GEMB-SDBN1-2000m/ts/v2/ts_2000.nc') + _forcing_file(tmp_path, f'{base}/dEBM2-8000m/ts/v1/ts_2000.nc') + config = _config( + tmp_path, ice_sheet='ais', model='MRI-ESM2-0') + + source = resolve_atmosphere_source(config, 'ts') + + assert source.product == 'GEMB-SDBN1' + assert source.files == (str(expected),) + + +def test_gis_ocean_resolution_and_version(tmp_path): + """GrIS ocean resolution and version are selected from the archive.""" + base = 'GIS/CESM2-WACCM/ssp585' + expected = _forcing_file( + tmp_path, f'{base}/ocean-1000m/tf/v2/tf_2000.nc') + _forcing_file( + tmp_path, f'{base}/ocean-4000m/tf/v3/tf_2000.nc') + config = _config(tmp_path) + + source = resolve_ocean_source(config) + + assert source.resolution == '1000m' + assert source.version == 'v2' + assert source.files == (str(expected),) + + +def test_ais_ocx_ocean_choice_layout(tmp_path): + """AIS OCX choices resolve below OCX/ocean rather than a variable dir.""" + expected = _forcing_file( + tmp_path, 'AIS/OCX/ocean/main/v2/tf_AIS_OCX_main.nc') + config = _config( + tmp_path, ice_sheet='ais', model='None', scenario='OCX') + + source = resolve_ocean_source(config, choice='main') + + assert source.version == 'v2' + assert source.forcing_group == 'OCX_main' + assert source.files == (str(expected),) + + +def test_fracture_uses_native_model_scenario_path(tmp_path): + """Fracture data resolves below the selected model and scenario.""" + expected = _forcing_file( + tmp_path, + 'AIS/CESM2-WACCM/ssp585/fracture/v2.1/excess_melt_data.nc') + config = _config(tmp_path, ice_sheet='ais') + + source = resolve_fracture_source(config, 'excess_melt_*.nc') + + assert source.version == 'v2.1' + assert source.files == (str(expected),) + + +def test_mapping_name_identifies_source_grid(tmp_path): + """Mapping files cannot collide across source resolutions.""" + config = _config(tmp_path) + + name = mapping_file_name( + config, 'atm', 'atmosphere_SDBN1-1000m', 'conserve') + + assert name == ( + 'map_ismip7_gis_atm_atmosphere_sdbn1_1000m_to_mesh_conserve.nc') From f30c82566c871b4ed04a525db170f11aeb666a96 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 24 Sep 2026 23:15:02 -0600 Subject: [PATCH 02/12] Refactor ISMIP7 forcing mapping workflow Add dedicated mapping steps for atmospheric, ocean, and fracture forcings. Reuse existing compatible mapping files and pass mapping outputs to the processing steps instead of generating them on demand. Update configuration, documentation, and tests for the new workflow. --- .../ismip7_forcing/atmosphere/__init__.py | 4 + .../atmosphere/build_mapping_file.py | 30 +++++++ .../atmosphere/process_runoff.py | 19 ++-- .../ismip7_forcing/atmosphere/process_smb.py | 19 ++-- .../atmosphere/process_smb_gradient.py | 19 ++-- .../atmosphere/process_temperature.py | 19 ++-- .../process_temperature_gradient.py | 19 ++-- .../tests/ismip7_forcing/fracture/__init__.py | 4 + .../fracture/build_mapping_file.py | 52 +++++++++++ .../fracture/process_excess_melt.py | 21 +++-- .../fracture/process_lake_properties.py | 22 +++-- .../fracture/process_shelf_collapse.py | 20 ++--- .../tests/ismip7_forcing/ismip7_forcing.cfg | 4 + .../ismip7_forcing/ismip7_forcing_ocx_ais.cfg | 3 + .../ismip7_forcing/ismip7_forcing_ocx_gis.cfg | 3 + .../ismip7_forcing/ismip7_forcing_test.cfg | 3 + .../tests/ismip7_forcing/mapping_step.py | 55 ++++++++++++ .../ismip7_forcing/ocean_thermal/__init__.py | 4 + .../ocean_thermal/build_mapping_file.py | 76 ++++++++++++++++ .../ocean_thermal/process_thermal_forcing.py | 77 +++++++++------- .../landice/test_groups/ismip7_forcing.rst | 40 +++++---- .../landice/test_groups/ismip7_forcing.rst | 39 +++++++-- tests/landice/ismip7/test_archive.py | 36 ++++++++ tests/landice/ismip7/test_mapping_step.py | 87 +++++++++++++++++++ 24 files changed, 536 insertions(+), 139 deletions(-) create mode 100644 compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py create mode 100644 compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py create mode 100644 compass/landice/tests/ismip7_forcing/mapping_step.py create mode 100644 compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py create mode 100644 tests/landice/ismip7/test_mapping_step.py diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py index 7e577b2026..6c49cb98ad 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py @@ -1,3 +1,6 @@ +from compass.landice.tests.ismip7_forcing.atmosphere.build_mapping_file import ( # noqa: E501 + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.atmosphere.process_runoff import ( ProcessRunoff, ) @@ -40,6 +43,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessSmb(test_case=self)) self.add_step(ProcessTemperature(test_case=self)) self.add_step(ProcessSmbGradient(test_case=self)) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py new file mode 100644 index 0000000000..95b314d982 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py @@ -0,0 +1,30 @@ +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) +from compass.landice.tests.ismip7_forcing.mapping_step import ( + build_and_cache_mapping, + setup_mapping_step, +) +from compass.step import Step + + +class BuildMappingFile(Step): + """Build weights from the selected atmosphere grid to the MALI mesh.""" + + def __init__(self, test_case): + super().__init__(test_case=test_case, name='build_mapping_file') + + def setup(self): + setup_mapping_step(self) + + def run(self): + config = self.config + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'acabf') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + + self.logger.info(f'Using atmosphere grid from {source.files[0]}') + build_and_cache_mapping( + self, source.files[0], mapping_file, method_remap) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index 11d020ad22..94db1faf9c 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -9,7 +9,6 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -45,6 +44,14 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'mrro') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -53,7 +60,6 @@ def run(self): config = self.config section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -87,17 +93,10 @@ def run(self): logger.info(f"Found {len(input_files)} runoff files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by other atm steps) + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index 5093a7f13a..e84ea3c0c0 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -9,7 +9,6 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -45,6 +44,14 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'acabf') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -53,7 +60,6 @@ def run(self): config = self.config section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -87,17 +93,10 @@ def run(self): logger.info(f"Found {len(input_files)} SMB files for years " f"{start_year}-{end_year}") - # Build mapping file using the first input file as the grid template + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index 508cd6e7d4..9f38bde8d7 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -9,7 +9,6 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -46,6 +45,14 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'dacabfdz') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -54,7 +61,6 @@ def run(self): config = self.config section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -88,17 +94,10 @@ def run(self): logger.info(f"Found {len(input_files)} SMB gradient files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by process_smb) + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index bff8db75fb..dfd145fe7c 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -9,7 +9,6 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -45,6 +44,14 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'ts') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -53,7 +60,6 @@ def run(self): config = self.config section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -87,17 +93,10 @@ def run(self): logger.info(f"Found {len(input_files)} temperature files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by process_smb) + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index 31ae46b1eb..47541a0798 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -9,7 +9,6 @@ mapping_file_name, resolve_atmosphere_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -47,6 +46,14 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'dtsdz') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -55,7 +62,6 @@ def run(self): config = self.config section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -89,17 +95,10 @@ def run(self): logger.info(f"Found {len(input_files)} temperature gradient files " f"for years {start_year}-{end_year}") - # Build mapping file (reuse if already created by other steps) + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "atm", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each year file remapped_files = [] for input_file in input_files: diff --git a/compass/landice/tests/ismip7_forcing/fracture/__init__.py b/compass/landice/tests/ismip7_forcing/fracture/__init__.py index b6a7e63c6a..ba34980e6a 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/__init__.py +++ b/compass/landice/tests/ismip7_forcing/fracture/__init__.py @@ -1,6 +1,9 @@ from compass.landice.tests.ismip7_forcing.configure import ( configure as configure_testgroup, ) +from compass.landice.tests.ismip7_forcing.fracture.build_mapping_file import ( + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.fracture.process_excess_melt import ( ProcessExcessMelt, ) @@ -40,6 +43,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessExcessMelt(test_case=self)) self.add_step(ProcessLakeProperties(test_case=self)) self.add_step(ProcessShelfCollapse(test_case=self)) diff --git a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py new file mode 100644 index 0000000000..5116eaecc6 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py @@ -0,0 +1,52 @@ +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) +from compass.landice.tests.ismip7_forcing.mapping_step import ( + build_and_cache_mapping, + setup_mapping_step, +) +from compass.step import Step + + +class BuildMappingFile(Step): + """Build weights from the fracture grid to the MALI mesh.""" + + def __init__(self, test_case): + super().__init__(test_case=test_case, name='build_mapping_file') + + def setup(self): + setup_mapping_step(self) + + def run(self): + config = self.config + section = config['ismip7_fracture'] + source = _resolve_grid_source(config) + methods = { + section.get('method_remap_shelf_collapse'), + section.get('method_remap_excess_melt'), + section.get('method_remap_lake_properties'), + } + + for method_remap in sorted(methods): + if method_remap.lower() == 'none': + continue + mapping_file = mapping_file_name( + config, 'fracture', source.source_grid, method_remap) + self.logger.info(f'Using fracture grid from {source.files[0]}') + build_and_cache_mapping( + self, source.files[0], mapping_file, method_remap) + + +def _resolve_grid_source(config): + """Select a fracture file with native x/y coordinates as grid donor.""" + errors = [] + for pattern in ('lake_properties_*.nc', + 'ice_shelf_collapse_mask_*.nc'): + try: + return resolve_fracture_source(config, pattern) + except FileNotFoundError as exc: + errors.append(str(exc)) + raise FileNotFoundError( + 'No fracture source with x/y coordinates was found.\n' + + '\n'.join(errors)) diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index 057dca5010..9a68fcd753 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -11,7 +11,6 @@ mapping_file_name, resolve_fracture_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, extrapolate_source, @@ -58,6 +57,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get( + 'ismip7_fracture', 'method_remap_excess_melt') + if method_remap.lower() != 'none': + mapping_file = mapping_file_name( + config, 'fracture', 'fracture', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -67,7 +75,6 @@ def run(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -102,18 +109,10 @@ def run(self): self._prepare_source_grid(input_file, input_path, gridded_file, logger) - # Build mapping file. Excess melt is a flux, so conservative - # remapping is appropriate by default. + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "fracture", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the excess melt grid...") - build_mapping_file(config, logger, - gridded_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index 7895425ae1..839af87ea7 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -7,7 +7,6 @@ mapping_file_name, resolve_fracture_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, extrapolate_source, @@ -62,6 +61,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get( + 'ismip7_fracture', 'method_remap_lake_properties') + if method_remap.lower() != 'none': + mapping_file = mapping_file_name( + config, 'fracture', 'fracture', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -71,7 +79,6 @@ def run(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -100,19 +107,10 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing lake properties: {basename}") - # Build mapping file. Lake properties are continuous fields, so - # bilinear remapping is appropriate by default. + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "fracture", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the lake properties " - "grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation extrap_file = f"extrap_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index 7e47058b60..df5d1b6d7c 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -7,7 +7,6 @@ mapping_file_name, resolve_fracture_source, ) -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import ( add_xtime_and_write, open_rename_and_trim, @@ -49,6 +48,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get( + 'ismip7_fracture', 'method_remap_shelf_collapse') + if method_remap.lower() != 'none': + mapping_file = mapping_file_name( + config, 'fracture', 'fracture', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -58,7 +66,6 @@ def run(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -87,17 +94,10 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing ice shelf collapse mask: {basename}") - # Build mapping file. neareststod preserves the 0/1 mask values. + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "fracture", source.source_grid, method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the collapse mask grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap the collapse mask onto the MALI mesh remapped_file = f"remapped_{basename}" if not os.path.exists(remapped_file): diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg index 7bc0836ab7..2dcec7e33c 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg @@ -31,6 +31,10 @@ mali_mesh_file = NotAvailable # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 128 +# Optional directory containing mapping files from a previous run. Missing +# files are built and saved under output_base_path/mapping_files. +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg index 2c41d4ae52..c686cb6796 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg @@ -39,6 +39,9 @@ mali_mesh_file = AIS_4to20km_r01_20220907.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional directory containing mapping files from a previous run +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg index d876b05c03..2636c392d9 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg @@ -37,6 +37,9 @@ mali_mesh_file = GIS_1to10km_r02_20230202.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional directory containing mapping files from a previous run +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg index 53eaf517e2..a6d2ca5d82 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg @@ -30,6 +30,9 @@ mali_mesh_file = AIS_4to20km_r01_20220907.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional directory containing mapping files from a previous run +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = false diff --git a/compass/landice/tests/ismip7_forcing/mapping_step.py b/compass/landice/tests/ismip7_forcing/mapping_step.py new file mode 100644 index 0000000000..ab17e230b2 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/mapping_step.py @@ -0,0 +1,55 @@ +"""Shared support for ISMIP7 forcing mapping-file steps.""" + +import os +import shutil + +from compass.landice.ismip7.mapping import build_mapping_file + + +def setup_mapping_step(step): + """Add the MALI mesh input and request the ESMF task allocation.""" + section = step.config['ismip7'] + base_path_mali = section.get('base_path_mali') + mali_mesh_file = section.get('mali_mesh_file') + + step.add_input_file( + filename=mali_mesh_file, + target=os.path.join(base_path_mali, mali_mesh_file)) + + step.ntasks = section.getint('esmf_ntasks') + step.min_tasks = step.ntasks + + +def build_and_cache_mapping(step, source_file, mapping_file, method_remap): + """Reuse or build a mapping file, then cache a newly built file.""" + config = step.config + logger = step.logger + section = config['ismip7'] + mapping_files_path = section.get('mapping_files_path') + output_base_path = section.get('output_base_path') + mali_mesh_file = section.get('mali_mesh_file') + + reused = False + if mapping_files_path != 'NotAvailable': + cached_file = os.path.abspath( + os.path.join(mapping_files_path, mapping_file)) + if os.path.isfile(cached_file): + logger.info(f'Reusing mapping file {cached_file}') + if os.path.lexists(mapping_file): + os.remove(mapping_file) + os.symlink(cached_file, mapping_file) + reused = True + + build_mapping_file( + config, logger, source_file, mapping_file, + mali_mesh_file=mali_mesh_file, method_remap=method_remap) + + if reused: + return + + mapping_files_dir = os.path.join(output_base_path, 'mapping_files') + os.makedirs(mapping_files_dir, exist_ok=True) + destination = os.path.join(mapping_files_dir, mapping_file) + if os.path.realpath(mapping_file) != os.path.realpath(destination): + logger.info(f'Caching mapping file in {mapping_files_dir}') + shutil.copy2(mapping_file, destination) diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py index 2a97700428..e384db2cb4 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py @@ -1,6 +1,9 @@ from compass.landice.tests.ismip7_forcing.configure import ( configure as configure_testgroup, ) +from compass.landice.tests.ismip7_forcing.ocean_thermal.build_mapping_file import ( # noqa: E501 + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing import ( # noqa: E501 ProcessThermalForcing, ) @@ -29,6 +32,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessThermalForcing(test_case=self)) def configure(self): diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py new file mode 100644 index 0000000000..bb9ef1f0a7 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py @@ -0,0 +1,76 @@ +import os + +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_ocean_source, + resolve_version_directory, +) +from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.tests.ismip7_forcing.mapping_step import ( + build_and_cache_mapping, + setup_mapping_step, +) +from compass.step import Step + + +class BuildMappingFile(Step): + """Build weights from selected ocean grids to the MALI mesh.""" + + def __init__(self, test_case): + super().__init__(test_case=test_case, name='build_mapping_file') + + def setup(self): + setup_mapping_step(self) + + def run(self): + config = self.config + section = config['ismip7'] + + if section.getboolean('process_ocean_thermal'): + self._build_scenario_mapping() + if section.getboolean('process_ocean_climatology'): + self._build_climatology_mapping() + + def _build_scenario_mapping(self): + config = self.config + params = get_params(config) + section = config['ismip7_ocean_thermal'] + method_remap = section.get('method_remap') + + if params.get('ocean_choice_layout', False): + choice = _first_ocean_choice(section.get('ocean_choice')) + source = resolve_ocean_source(config, choice=choice) + else: + source = resolve_ocean_source(config) + + mapping_file = mapping_file_name( + config, 'ocean', source.source_grid, method_remap) + self.logger.info(f'Using ocean grid from {source.files[0]}') + build_and_cache_mapping( + self, source.files[0], mapping_file, method_remap) + + def _build_climatology_mapping(self): + config = self.config + section = config['ismip7_ocean_climatology'] + method_remap = section.get('method_remap') + base_path = section.get('base_path_climatology') + version = section.get('version', fallback='latest') + _, _, files = resolve_version_directory( + os.path.join(base_path, 'tf'), version, 'tf_*.nc') + mapping_file = mapping_file_name( + config, 'ocean', 'climatology', method_remap) + + self.logger.info(f'Using climatology grid from {files[0]}') + build_and_cache_mapping( + self, files[0], mapping_file, method_remap) + + +def _first_ocean_choice(raw_choices): + """Return one representative AIS OCX choice for mapping generation.""" + choices = [choice.strip() for choice in raw_choices.split(',') + if choice.strip()] + if any(choice.lower() == 'all' for choice in choices): + return 'main' + if not choices: + raise ValueError('No ocean_choice specified for AIS OCX forcing.') + return choices[0] diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index 30e96ed6da..c56213f7bf 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -11,7 +11,6 @@ resolve_version_directory, ) from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -51,6 +50,30 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + if section.getboolean('process_ocean_thermal'): + params = get_params(config) + if params.get('ocean_choice_layout', False): + choice = self._get_ocean_choices(config)[0] + source = resolve_ocean_source(config, choice=choice) + else: + source = resolve_ocean_source(config) + method_remap = config.get( + 'ismip7_ocean_thermal', 'method_remap') + mapping_file = mapping_file_name( + config, 'ocean', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + + if section.getboolean('process_ocean_climatology'): + method_remap = config.get( + 'ismip7_ocean_climatology', 'method_remap') + mapping_file = mapping_file_name( + config, 'ocean', 'climatology', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -77,7 +100,6 @@ def _run_scenario(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -104,9 +126,8 @@ def _run_scenario(self): mapping_file = mapping_file_name( config, "ocean", job.source_grid, method_remap) self._process_ocean_forcing( - job, mapping_file, ocean_3d, method_remap, - start_year, end_year, mali_mesh_name, mali_mesh_file, - output_base_path) + job, mapping_file, ocean_3d, start_year, end_year, + mali_mesh_name, output_base_path) # AIS OCX writes one ocean_thermal_forcing dir per ocean choice # (OCX_), but atmosphere output is written once, under OCX. @@ -184,21 +205,31 @@ def _link_atmosphere_outputs(self, output_base_path, atm_forcing_group, dst_dir = os.path.join(output_base_path, choice_forcing_group, "atmosphere") + + if os.path.realpath(dst_dir) == os.path.realpath(src_dir): + logger.warning( + f"Skipping atmosphere mirror for {choice_forcing_group}: " + f"destination {dst_dir} resolves to the source directory.") + return + os.makedirs(dst_dir, exist_ok=True) for fname in os.listdir(src_dir): src = os.path.join(src_dir, fname) - if not os.path.isfile(src): + if os.path.islink(src) or not os.path.isfile(src): continue dst = os.path.join(dst_dir, fname) + if os.path.realpath(dst) == os.path.realpath(src): + logger.warning( + f"Skipping {fname}: destination resolves to source.") + continue if os.path.lexists(dst): os.remove(dst) os.symlink(src, dst) logger.info(f" Linked {dst} -> {src}") def _process_ocean_forcing(self, job, mapping_file, ocean_3d, - method_remap, start_year, end_year, - mali_mesh_name, mali_mesh_file, + start_year, end_year, mali_mesh_name, output_base_path): """ Discover, remap, combine, and save the thermal forcing for a single @@ -209,21 +240,17 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, job : compass.landice.ismip7.archive.ForcingSource Resolved ocean forcing source mapping_file : str - Path of the shared ocean-to-MALI mapping file (built on demand) + Path of the shared ocean-to-MALI mapping file ocean_3d : bool Whether the thermal forcing is 3D (AIS) or 2D (GrIS) - method_remap : str - Remapping method passed to the mapping-file builder start_year, end_year : int Inclusive year range to process - mali_mesh_name, mali_mesh_file : str - MALI mesh name and file + mali_mesh_name : str + MALI mesh name output_base_path : str Base path under which output is written """ logger = self.logger - config = self.config - forcing_group = job.forcing_group label = job.label all_files = job.files @@ -249,14 +276,6 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, logger.info(f"Found {len(input_files)} ocean thermal forcing files " f"overlapping years {start_year}-{end_year}") - # Build mapping file using the first input file as grid template. - if not os.path.exists(mapping_file): - logger.info("Building mapping file for ocean grid...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each file remapped_files = [] for input_file in input_files: @@ -327,7 +346,6 @@ def _run_climatology(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") output_base_path = section.get("output_base_path") section = config["ismip7_ocean_climatology"] @@ -336,7 +354,7 @@ def _run_climatology(self): requested_version = section.get("version", fallback="latest") # Discover climatology TF file - _, version, all_files = resolve_version_directory( + _, _, all_files = resolve_version_directory( os.path.join(base_path_climatology, "tf"), requested_version, "tf_*.nc") @@ -345,17 +363,10 @@ def _run_climatology(self): logger.info(f"Processing ocean TF climatology: " f"{os.path.basename(input_file)}") - # Build mapping file using the climatology file as grid template. + # The mapping file is supplied by the build_mapping_file step. mapping_file = mapping_file_name( config, "ocean", "climatology", method_remap) - if not os.path.exists(mapping_file): - logger.info("Building mapping file for ocean grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Extrapolate and remap basename = os.path.basename(input_file) remapped_file = f"remapped_{basename}" diff --git a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst index fb268dd12e..5acec9efab 100644 --- a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst @@ -27,7 +27,8 @@ ice-sheet-specific parameters (projection, file naming prefix, and ocean dimensionality), :py:mod:`compass.landice.ismip7.archive` for native archive path, resolution, and version discovery, :py:func:`compass.landice.ismip7.mapping.build_mapping_file` to create the -SCRIP and ESMF mapping files, and the remapping helpers in +SCRIP and ESMF mapping files from dedicated ``BuildMappingFile`` steps, and +the remapping helpers in :py:mod:`compass.landice.ismip7.remap`. When ``scenario = OCX``, ``get_params`` supplies the fixed reanalysis sources @@ -62,15 +63,18 @@ atmosphere ~~~~~~~~~~ The :py:class:`compass.landice.tests.ismip7_forcing.atmosphere.Atmosphere` -test case processes the ISMIP7 atmosphere forcing fields. It contains five -steps: SMB, temperature, their respective gradients, and runoff. Each step -discovers input files matching the ice-sheet-specific naming pattern, builds -or reuses a mapping file, remaps each input file with ``ncremap``, and -combines/renames the results to MALI conventions. ``latest`` is resolved per -variable because atmosphere datasets can have different current versions. +test case processes the ISMIP7 atmosphere forcing fields. It contains a +mapping step followed by five processing steps for SMB, temperature, their +respective gradients, and runoff. The mapping step uses the archive resolver +to select the configured source grid and runs with ``esmf_ntasks``. Each +processing step consumes those weights and remaps its input files with +``ncremap``. ``latest`` is resolved per variable because atmosphere datasets +can have different current versions. Steps: +* :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.build_mapping_file.BuildMappingFile` — + builds and caches the shared atmosphere mapping file * :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.process_smb.ProcessSmb` — ``acabf`` → ``sfcMassBal`` * :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.process_temperature.ProcessTemperature` — @@ -88,10 +92,13 @@ ocean_thermal ~~~~~~~~~~~~~ The :py:class:`compass.landice.tests.ismip7_forcing.ocean_thermal.OceanThermal` -test case processes the ISMIP7 ocean thermal forcing. It contains a single step, +test case processes the ISMIP7 ocean thermal forcing. It contains a mapping +step followed by the processing step, :py:class:`~compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing.ProcessThermalForcing`, which handles both AIS (3D, decade-spanning files) and GrIS (2D, yearly files) -by branching on the ``ocean_3d`` parameter from ``ice_sheet_params``. +by branching on the ``ocean_3d`` parameter from ``ice_sheet_params``. The +mapping step builds each enabled scenario or climatology mapping with +``esmf_ntasks``. The ``run()`` method dispatches to two sub-methods based on the boolean config options ``process_ocean_thermal`` and ``process_ocean_climatology`` in the @@ -128,18 +135,21 @@ fracture The :py:class:`compass.landice.tests.ismip7_forcing.fracture.Fracture` test case processes the ISMIP7 surface-melt-driven ice shelf collapse -forcing (AIS only). It implements the three ISMIP7 pathways as independent -steps, each discovering its source file from the native -``AIS/{model}/{scenario}/fracture/{version}/`` hierarchy, building or reusing -a mapping file, -remapping with ``ncremap``, and renaming the result to MALI conventions with -an accompanying ``xtime`` variable. Per-pathway remapping methods are set in +forcing (AIS only). It contains a mapping step followed by three independent +pathway-processing steps. The mapping step uses a lake-properties or +shelf-collapse file with native ``x``/``y`` coordinates and builds one file +per unique enabled remapping method. Each processing step discovers its source +file from the native ``AIS/{model}/{scenario}/fracture/{version}/`` hierarchy, +remaps with ``ncremap``, and renames the result to MALI conventions with an +accompanying ``xtime`` variable. Per-pathway remapping methods are set in the ``[ismip7_fracture]`` config section. Setting a pathway's remapping-method option to ``None`` causes that step to return early without processing its file, which is useful when only some pathway source files are available. Steps: +* :py:class:`~compass.landice.tests.ismip7_forcing.fracture.build_mapping_file.BuildMappingFile` + — builds and caches the enabled fracture mapping files * :py:class:`~compass.landice.tests.ismip7_forcing.fracture.process_excess_melt.ProcessExcessMelt` (Path A) — ``excess_melt`` → ``ismip7ExcessMelt``. The excess melt file lacks ``x``/``y`` coordinate variables and its array is flipped along the diff --git a/docs/users_guide/landice/test_groups/ismip7_forcing.rst b/docs/users_guide/landice/test_groups/ismip7_forcing.rst index 6393efa8ed..397dafcdc5 100644 --- a/docs/users_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip7_forcing.rst @@ -11,20 +11,22 @@ The test group supports both the Antarctic Ice Sheet (AIS) and the Greenland Ice Sheet (GrIS), controlled by a single ``ice_sheet`` config option. The test group includes three test cases: ``atmosphere``, ``ocean_thermal``, -and ``fracture``. +and ``fracture``. Each begins with a ``build_mapping_file`` step that runs +ESMF with a separate resource allocation. -* The ``atmosphere`` test case has five steps: +* The ``atmosphere`` test case has six steps: ``build_mapping_file``, ``process_smb``, ``process_temperature``, ``process_smb_gradient``, ``process_temperature_gradient``, and ``process_runoff``. -* The ``ocean_thermal`` test case has one step: ``process_thermal_forcing``. +* The ``ocean_thermal`` test case has two steps: ``build_mapping_file`` and + ``process_thermal_forcing``. For AIS this produces 3D thermal forcing (with 30 ocean depth layers); for GrIS it produces 2D (depth-averaged) thermal forcing. The step can also process the observational ocean thermal forcing climatology (Zhou et al.) for AIS, controlled by the ``process_ocean_climatology`` config option. -* The ``fracture`` test case has three steps: ``process_excess_melt`` - (Path A), ``process_lake_properties`` (Path B), and +* The ``fracture`` test case has four steps: ``build_mapping_file``, + ``process_excess_melt`` (Path A), ``process_lake_properties`` (Path B), and ``process_shelf_collapse`` (Path C). It processes the ISMIP7 surface-melt-driven ice shelf collapse forcing (AIS only). @@ -55,6 +57,12 @@ To use this test group, users need to: combination to process the surface-melt-driven ice shelf collapse pathways (excess melt, lake properties, and the ice shelf collapse mask). +The mapping step can be run first on a large allocation, followed by the +processing steps on a single node. Newly built weights are saved in +``{output_base_path}/mapping_files``. Set ``mapping_files_path`` to that +directory in later runs to reuse weights for the same source grid, MALI mesh, +and remapping method. + Example user config files are provided in the source tree for local testing: * ``compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg`` @@ -182,6 +190,9 @@ values are: # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 128 + # Optional directory containing compatible mapping files from another run + mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true @@ -217,9 +228,6 @@ values are: # Remapping method: bilinear, neareststod, conserve method_remap = bilinear - # Dataset version - version = latest - # Start year for processing start_year = 1850 @@ -232,6 +240,9 @@ values are: # Remapping method: bilinear, neareststod, conserve method_remap = bilinear + # Dataset version + version = latest + # Base path to observational climatology data base_path_climatology = /path/to/ISMIP7/forcing/AIS/obs/zhou_annual_06_nov @@ -279,6 +290,10 @@ grid to the MALI unstructured mesh. Steps: +* **build_mapping_file**: Builds and caches weights for the selected + atmosphere product and resolution. All five processing steps share these + weights. + * **process_smb**: Remaps the surface mass balance (``acabf``) field. The output variable is ``sfcMassBal``. @@ -305,6 +320,10 @@ The ``landice/ismip7_forcing/ocean_thermal`` test case processes the ISMIP7 ocean thermal forcing (``tf``) and remaps it from the native polar stereographic grid to the MALI unstructured mesh. +The **build_mapping_file** step builds and caches mappings for each enabled +source grid. Scenario and climatology mappings are distinct when their source +grids differ. The **process_thermal_forcing** step consumes those mappings. + The step supports two processing modes, controlled by boolean config options in the ``[ismip7]`` section: @@ -349,6 +368,10 @@ remapping-method config option to ``None`` in the ``[ismip7_fracture]`` section (for example, ``method_remap_excess_melt = None`` skips Path A). This is useful when only some of the pathway source files are available. +The **build_mapping_file** step uses a fracture file with native ``x``/``y`` +coordinates as the common grid template and creates one mapping file per +unique enabled remapping method. + * **process_excess_melt** (Path A): Remaps the excess meltwater field (melt + rain after firn air content depletion), matching ``excess_melt_*.nc``. The output variable is ``ismip7ExcessMelt`` diff --git a/tests/landice/ismip7/test_archive.py b/tests/landice/ismip7/test_archive.py index e8d7726d49..b84942ce88 100644 --- a/tests/landice/ismip7/test_archive.py +++ b/tests/landice/ismip7/test_archive.py @@ -129,6 +129,42 @@ def test_gis_ocean_resolution_and_version(tmp_path): assert source.files == (str(expected),) +def test_gis_ocx_atmosphere_layout(tmp_path): + """GrIS OCX atmosphere uses the RACMO product below the OCX tree.""" + base = 'GIS/OCX/RACMO2.3p2-ERA' + expected = _forcing_file( + tmp_path, f'{base}/SDBN1-1000m/ts/v2/ts_2000.nc') + _forcing_file( + tmp_path, f'{base}/SDBN1-4000m/ts/v3/ts_2000.nc') + config = _config( + tmp_path, ice_sheet='gis', model='None', scenario='OCX') + + source = resolve_atmosphere_source(config, 'ts') + + assert source.model == 'RACMO2.3p2-ERA' + assert source.resolution == '1000m' + assert source.version == 'v2' + assert source.files == (str(expected),) + + +def test_gis_ocx_ocean_layout(tmp_path): + """GrIS OCX ocean uses EN4 and the selected native resolution.""" + base = 'GIS/OCX/EN4' + expected = _forcing_file( + tmp_path, f'{base}/ocean-1000m/tf/v1/tf_2000.nc') + _forcing_file( + tmp_path, f'{base}/ocean-4000m/tf/v2/tf_2000.nc') + config = _config( + tmp_path, ice_sheet='gis', model='None', scenario='OCX') + + source = resolve_ocean_source(config) + + assert source.model == 'EN4' + assert source.resolution == '1000m' + assert source.version == 'v1' + assert source.files == (str(expected),) + + def test_ais_ocx_ocean_choice_layout(tmp_path): """AIS OCX choices resolve below OCX/ocean rather than a variable dir.""" expected = _forcing_file( diff --git a/tests/landice/ismip7/test_mapping_step.py b/tests/landice/ismip7/test_mapping_step.py new file mode 100644 index 0000000000..b42196fac7 --- /dev/null +++ b/tests/landice/ismip7/test_mapping_step.py @@ -0,0 +1,87 @@ +"""Tests for shared ISMIP7 mapping-step behavior.""" + +from configparser import ConfigParser +from types import SimpleNamespace + +from compass.landice.tests.ismip7_forcing import mapping_step +from compass.landice.tests.ismip7_forcing.fracture.build_mapping_file import ( + _resolve_grid_source, +) + + +class _Logger: + def info(self, message): + pass + + +def _step(tmp_path, mapping_files_path): + config = ConfigParser() + config['ismip7'] = { + 'mapping_files_path': str(mapping_files_path), + 'output_base_path': str(tmp_path / 'output'), + 'mali_mesh_file': 'mesh.nc', + } + return SimpleNamespace(config=config, logger=_Logger()) + + +def test_mapping_step_reuses_cached_file(tmp_path, monkeypatch): + """An existing cached mapping is linked rather than rebuilt.""" + cache = tmp_path / 'cache' + cache.mkdir() + expected = cache / 'map.nc' + expected.touch() + step = _step(tmp_path, cache) + calls = [] + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + mapping_step, 'build_mapping_file', + lambda *args, **kwargs: calls.append((args, kwargs))) + + mapping_step.build_and_cache_mapping( + step, 'source.nc', 'map.nc', 'bilinear') + + assert (tmp_path / 'map.nc').is_symlink() + assert (tmp_path / 'map.nc').resolve() == expected + assert len(calls) == 1 + + +def test_mapping_step_caches_new_file(tmp_path, monkeypatch): + """A newly built mapping is copied to the reusable cache directory.""" + step = _step(tmp_path, 'NotAvailable') + + def _build(config, logger, source_file, mapping_file, **kwargs): + (tmp_path / mapping_file).write_text('weights') + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(mapping_step, 'build_mapping_file', _build) + + mapping_step.build_and_cache_mapping( + step, 'source.nc', 'map.nc', 'bilinear') + + cached = tmp_path / 'output' / 'mapping_files' / 'map.nc' + assert cached.read_text() == 'weights' + + +def test_fracture_mapping_uses_xy_grid_donor(tmp_path): + """The mapping source is not the ungridded excess-melt file.""" + version_path = ( + tmp_path / 'AIS' / 'CESM2-WACCM' / 'ssp585' / + 'fracture' / 'v2.1') + version_path.mkdir(parents=True) + (version_path / 'excess_melt_data.nc').touch() + expected = version_path / 'lake_properties_data.nc' + expected.touch() + + config = ConfigParser() + config['ismip7'] = { + 'base_path_ismip7': str(tmp_path), + 'ice_sheet': 'ais', + 'model': 'CESM2-WACCM', + 'scenario': 'ssp585', + } + config['ismip7_fracture'] = {'version': 'latest'} + + source = _resolve_grid_source(config) + + assert source.files == (str(expected),) From 436dbc9672c0e7fe438fb91d0bc45c01a47f3207 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 24 Sep 2026 21:20:25 -0700 Subject: [PATCH 03/12] Fix bug in config parser calls --- compass/landice/ismip7/archive.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/compass/landice/ismip7/archive.py b/compass/landice/ismip7/archive.py index bebfe2ec91..1e4d773f85 100644 --- a/compass/landice/ismip7/archive.py +++ b/compass/landice/ismip7/archive.py @@ -58,14 +58,15 @@ def resolve_atmosphere_source(config, variable): forcing_group = scenario base_path = os.path.join(_ice_sheet_path(config), scenario, model) - section = 'ismip7_atmosphere' - product = config.get(section, 'product', fallback='auto') - resolution = config.get(section, 'resolution', fallback='auto') + section_name = 'ismip7_atmosphere' + section = config[section_name] + product = section.get('product', fallback='auto') + resolution = section.get('resolution', fallback='auto') grid_name, product, resolution = _resolve_atmosphere_grid( base_path, model, product, resolution) variable_path = os.path.join(base_path, grid_name, variable) - version = _requested_version(config, section, variable) + version = _requested_version(config, section_name, variable) version_path, version, files = resolve_version_directory( variable_path, version, f'{variable}_*.nc') @@ -89,8 +90,9 @@ def resolve_ocean_source(config, choice=None): ice_sheet = config.get('ismip7', 'ice_sheet') scenario = config.get('ismip7', 'scenario') configured_model = config.get('ismip7', 'model') - section = 'ismip7_ocean_thermal' - requested_version = config.get(section, 'version', fallback='latest') + section_name = 'ismip7_ocean_thermal' + section = config[section_name] + requested_version = section.get('version', fallback='latest') if params.get('ocean_choice_layout', False): if choice is None: @@ -118,8 +120,8 @@ def resolve_ocean_source(config, choice=None): grid_name = 'ocean' resolution = None else: - requested_resolution = config.get( - section, 'resolution', fallback='auto') + requested_resolution = section.get( + 'resolution', fallback='auto') grid_name, resolution = _resolve_ocean_grid( base_path, requested_resolution) base_path = os.path.join(base_path, grid_name, 'tf') @@ -234,7 +236,7 @@ def _requested_version(config, section, variable): option = f'{variable}_version' if config.has_option(section, option): return config.get(section, option) - return config.get(section, 'version', fallback='latest') + return config[section].get('version', fallback='latest') def _resolve_atmosphere_grid(base_path, model, requested_product, From 4ee9a913ecf8926cb754288f6a80c960492f855b Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Thu, 24 Sep 2026 22:37:19 -0700 Subject: [PATCH 04/12] Fix bug in determining ocean version for file naming --- .../ismip7_forcing/ocean_thermal/process_thermal_forcing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index c56213f7bf..90e4c80bde 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -391,6 +391,7 @@ def _run_climatology(self): # Rename to MALI conventions logger.info("Renaming variables to MALI conventions...") + version = os.path.basename(os.path.dirname(input_file)) output_file = (f"{mali_mesh_name}_thermal_forcing_climatology_" f"{version}.nc") From d7c22d23dda585cc45235c31d4f707bc4b048924 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Mon, 21 Sep 2026 16:27:05 -0600 Subject: [PATCH 05/12] Allow different numbers of processors between steps Set min_tasks to 1 to prevent jobs from failing when the number of processors was reduced between running build_mapping_files and the remaining processing steps. --- compass/landice/tests/ismip7_forcing/mapping_step.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/mapping_step.py b/compass/landice/tests/ismip7_forcing/mapping_step.py index ab17e230b2..1e900149c9 100644 --- a/compass/landice/tests/ismip7_forcing/mapping_step.py +++ b/compass/landice/tests/ismip7_forcing/mapping_step.py @@ -17,7 +17,7 @@ def setup_mapping_step(step): target=os.path.join(base_path_mali, mali_mesh_file)) step.ntasks = section.getint('esmf_ntasks') - step.min_tasks = step.ntasks + step.min_tasks = 1 def build_and_cache_mapping(step, source_file, mapping_file, method_remap): @@ -42,7 +42,9 @@ def build_and_cache_mapping(step, source_file, mapping_file, method_remap): build_mapping_file( config, logger, source_file, mapping_file, - mali_mesh_file=mali_mesh_file, method_remap=method_remap) + mali_mesh_file=mali_mesh_file, + method_remap=method_remap, + ntasks=step.ntasks) if reused: return From fc9923e3ec134f048bd4975840115aa90dc50475 Mon Sep 17 00:00:00 2001 From: Trevor Hillebrand Date: Fri, 25 Sep 2026 07:10:33 -0700 Subject: [PATCH 06/12] Add ntasks to mapping step test Add ntasks to mapping step test, which was failing during CI. --- tests/landice/ismip7/test_mapping_step.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/landice/ismip7/test_mapping_step.py b/tests/landice/ismip7/test_mapping_step.py index b42196fac7..3a16029161 100644 --- a/tests/landice/ismip7/test_mapping_step.py +++ b/tests/landice/ismip7/test_mapping_step.py @@ -21,7 +21,7 @@ def _step(tmp_path, mapping_files_path): 'output_base_path': str(tmp_path / 'output'), 'mali_mesh_file': 'mesh.nc', } - return SimpleNamespace(config=config, logger=_Logger()) + return SimpleNamespace(config=config, logger=_Logger(), ntasks=16) def test_mapping_step_reuses_cached_file(tmp_path, monkeypatch): From 465ede56b4b322fa4548be9c409be19bddfe462f Mon Sep 17 00:00:00 2001 From: Matt Hoffman Date: Fri, 25 Sep 2026 12:47:57 -0600 Subject: [PATCH 07/12] Skip fracture mappings if no methods are specified Add check to skip fracture mappings if none are requested. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../tests/ismip7_forcing/fracture/build_mapping_file.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py index 5116eaecc6..2a9b6e87c1 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py @@ -21,12 +21,15 @@ def setup(self): def run(self): config = self.config section = config['ismip7_fracture'] - source = _resolve_grid_source(config) methods = { section.get('method_remap_shelf_collapse'), section.get('method_remap_excess_melt'), section.get('method_remap_lake_properties'), } + if all(method.lower() == 'none' for method in methods): + self.logger.info('No fracture mappings requested; skipping.') + return + source = _resolve_grid_source(config) for method_remap in sorted(methods): if method_remap.lower() == 'none': From 38f004515455162cb82ec5a8b8a2b85812e06739 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 25 Sep 2026 12:57:49 -0600 Subject: [PATCH 08/12] Handle None values in fracture method checks When fracture method config options are not set, section.get() returns None instead of a string. Check for None before calling .lower() to avoid AttributeError and properly skip fracture mappings when no methods are configured. Addresses review comment on PR #997. --- .../tests/ismip7_forcing/fracture/build_mapping_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py index 2a9b6e87c1..7e65ad9e56 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py @@ -26,13 +26,13 @@ def run(self): section.get('method_remap_excess_melt'), section.get('method_remap_lake_properties'), } - if all(method.lower() == 'none' for method in methods): + if all(method is None or method.lower() == 'none' for method in methods): self.logger.info('No fracture mappings requested; skipping.') return source = _resolve_grid_source(config) for method_remap in sorted(methods): - if method_remap.lower() == 'none': + if method_remap is None or method_remap.lower() == 'none': continue mapping_file = mapping_file_name( config, 'fracture', source.source_grid, method_remap) From 0f94381c4493a3d4364a56f2f8cd8b3ac9640ce9 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 25 Sep 2026 12:58:05 -0600 Subject: [PATCH 09/12] Clarify EN4 is only used for GrIS OCX Update comment to specify that EN4 ocean forcing is used for GrIS in the OCX scenario, not AIS. AIS OCX deliberately leaves ocean_model unset and uses per-choice ocean directories (main/cold/warm/vary). Addresses review comment on PR #997. --- compass/landice/ismip7/ice_sheet_params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compass/landice/ismip7/ice_sheet_params.py b/compass/landice/ismip7/ice_sheet_params.py index a6e05fe937..f59b7cee5f 100644 --- a/compass/landice/ismip7/ice_sheet_params.py +++ b/compass/landice/ismip7/ice_sheet_params.py @@ -24,8 +24,8 @@ # Overrides applied for the OCX (reanalysis) scenario. OCX has no distinct # ESM model: it uses fixed reanalysis products (RACMO for the atmosphere and -# EN4 for the ocean). When scenario is 'OCX' the [ismip7] model option is -# ignored and these sources are used instead. +# EN4 for the ocean in GrIS). When scenario is 'OCX' the [ismip7] model option +# is ignored and these sources are used instead. _OCX_OVERRIDES = { 'gis': { 'atm_model': 'RACMO2.3p2-ERA', From 539ecf309d36d4a4c1ff26dd28da171b5c1345bf Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 25 Sep 2026 12:58:54 -0600 Subject: [PATCH 10/12] Add BuildMappingFile classes to API documentation Add the three new BuildMappingFile classes (atmosphere, ocean_thermal, and fracture) to the ismip7_forcing autosummary in the API documentation so their API pages will be generated. Addresses review comment on PR #997. --- docs/developers_guide/landice/api.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/developers_guide/landice/api.rst b/docs/developers_guide/landice/api.rst index 3170a05f38..cf40250e8e 100644 --- a/docs/developers_guide/landice/api.rst +++ b/docs/developers_guide/landice/api.rst @@ -493,6 +493,9 @@ ismip7_forcing atmosphere.Atmosphere atmosphere.Atmosphere.configure + atmosphere.build_mapping_file.BuildMappingFile + atmosphere.build_mapping_file.BuildMappingFile.setup + atmosphere.build_mapping_file.BuildMappingFile.run atmosphere.process_smb.ProcessSmb atmosphere.process_smb.ProcessSmb.setup atmosphere.process_smb.ProcessSmb.run @@ -511,12 +514,18 @@ ismip7_forcing ocean_thermal.OceanThermal ocean_thermal.OceanThermal.configure + ocean_thermal.build_mapping_file.BuildMappingFile + ocean_thermal.build_mapping_file.BuildMappingFile.setup + ocean_thermal.build_mapping_file.BuildMappingFile.run ocean_thermal.process_thermal_forcing.ProcessThermalForcing ocean_thermal.process_thermal_forcing.ProcessThermalForcing.setup ocean_thermal.process_thermal_forcing.ProcessThermalForcing.run fracture.Fracture fracture.Fracture.configure + fracture.build_mapping_file.BuildMappingFile + fracture.build_mapping_file.BuildMappingFile.setup + fracture.build_mapping_file.BuildMappingFile.run fracture.process_excess_melt.ProcessExcessMelt fracture.process_excess_melt.ProcessExcessMelt.setup fracture.process_excess_melt.ProcessExcessMelt.run From 4b200c4ac95fdcf489a22c2320d63773b2bc99ee Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 25 Sep 2026 12:59:21 -0600 Subject: [PATCH 11/12] Clarify OCX ocean source differs between GrIS and AIS Update documentation to distinguish that GrIS OCX uses EN4 as the ocean source, while AIS OCX leaves ocean_model unset and uses per-choice ocean directories (main/cold/warm/vary). Addresses review comment on PR #997. --- .../landice/test_groups/ismip7_forcing.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst index 5acec9efab..c07eb07b6a 100644 --- a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst @@ -32,12 +32,14 @@ the remapping helpers in :py:mod:`compass.landice.ismip7.remap`. When ``scenario = OCX``, ``get_params`` supplies the fixed reanalysis sources -(``atm_model`` = ``RACMO2.3p2-ERA`` and ``ocean_model`` = ``EN4``). The -archive resolver discovers their native paths, selected resolutions, and -versions. The processing steps use -``atm_model`` / ``ocean_model`` in place of the ``[ismip7] model`` option when -they are set, so the OCX ``model`` option is ignored. This keeps OCX handling -centralized and lets a single config file drive both test cases. +(``atm_model`` = ``RACMO2.3p2-ERA`` for both ice sheets and ``ocean_model`` = +``EN4`` for GrIS). For AIS OCX, ``ocean_model`` remains unset and the archive +resolver uses per-choice ocean directories (main/cold/warm/vary). The archive +resolver discovers native paths, selected resolutions, and versions. The +processing steps use ``atm_model`` / ``ocean_model`` in place of the +``[ismip7] model`` option when they are set, so the OCX ``model`` option is +ignored. This keeps OCX handling centralized and lets a single config file drive +both test cases. configure ~~~~~~~~~ From 04db5d8bd630a3e9e4afd7e5ba34f6bc72989dc7 Mon Sep 17 00:00:00 2001 From: Matthew Hoffman Date: Fri, 25 Sep 2026 13:13:39 -0600 Subject: [PATCH 12/12] Fix flake8 line length error in fracture build_mapping_file Break long line into multiple lines to satisfy 79 character limit. Co-Authored-By: Claude Sonnet 4.5 --- .../tests/ismip7_forcing/fracture/build_mapping_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py index 7e65ad9e56..3ca2dc4df2 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py +++ b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py @@ -26,7 +26,8 @@ def run(self): section.get('method_remap_excess_melt'), section.get('method_remap_lake_properties'), } - if all(method is None or method.lower() == 'none' for method in methods): + if all(method is None or method.lower() == 'none' + for method in methods): self.logger.info('No fracture mappings requested; skipping.') return source = _resolve_grid_source(config)