diff --git a/compass/landice/ismip7/archive.py b/compass/landice/ismip7/archive.py new file mode 100644 index 0000000000..1e4d773f85 --- /dev/null +++ b/compass/landice/ismip7/archive.py @@ -0,0 +1,314 @@ +"""Resolve forcing files in the native ISMIP7 archive hierarchy.""" + +import glob +import os +import re +from dataclasses import dataclass + +from compass.landice.ismip7.ice_sheet_params import get_params + +_ICE_SHEET_DIRECTORIES = { + 'ais': 'AIS', + 'gis': 'GIS', +} + +_DEFAULT_ATMOSPHERE_PRODUCTS = { + 'CESM2-WACCM': 'SDBN1', + 'MRI-ESM2-0': 'GEMB-SDBN1', + 'RACMO2.3p2-ERA': 'SDBN1', +} + +_VERSION_PATTERN = re.compile(r'^v(\d+(?:\.\d+)*)$') +_GRID_PATTERN = re.compile(r'^(.+)-(\d+)m$') + + +@dataclass(frozen=True) +class ForcingSource: + """A resolved collection of ISMIP7 forcing files.""" + + directory: str + files: tuple[str, ...] + version: str + model: str + scenario: str + forcing_group: str + label: str + source_grid: str + product: str | None = None + resolution: str | None = None + + +def resolve_atmosphere_source(config, variable): + """ + Resolve one atmosphere variable from the native archive hierarchy. + + ``latest`` is evaluated independently for each variable because archive + versions are not necessarily synchronized across variables. + """ + params = get_params(config) + scenario = config.get('ismip7', 'scenario') + configured_model = config.get('ismip7', 'model') + + if params['atm_model'] is None: + model = configured_model + forcing_group = f'{model}_{scenario}' + base_path = os.path.join(_ice_sheet_path(config), model, scenario) + else: + model = params['atm_model'] + forcing_group = scenario + base_path = os.path.join(_ice_sheet_path(config), scenario, model) + + section_name = 'ismip7_atmosphere' + section = config[section_name] + product = section.get('product', fallback='auto') + resolution = section.get('resolution', fallback='auto') + grid_name, product, resolution = _resolve_atmosphere_grid( + base_path, model, product, resolution) + + variable_path = os.path.join(base_path, grid_name, variable) + version = _requested_version(config, section_name, variable) + version_path, version, files = resolve_version_directory( + variable_path, version, f'{variable}_*.nc') + + return ForcingSource( + directory=version_path, + files=files, + version=version, + model=model, + scenario=scenario, + forcing_group=forcing_group, + label=f'{model}_{scenario}', + source_grid=f'atmosphere_{grid_name}', + product=product, + resolution=resolution, + ) + + +def resolve_ocean_source(config, choice=None): + """Resolve ocean thermal forcing from the native archive hierarchy.""" + params = get_params(config) + ice_sheet = config.get('ismip7', 'ice_sheet') + scenario = config.get('ismip7', 'scenario') + configured_model = config.get('ismip7', 'model') + section_name = 'ismip7_ocean_thermal' + section = config[section_name] + requested_version = section.get('version', fallback='latest') + + if params.get('ocean_choice_layout', False): + if choice is None: + raise ValueError('An ocean choice is required for AIS OCX.') + model = configured_model + grid_name = 'ocean' + base_path = os.path.join( + _ice_sheet_path(config), scenario, 'ocean', choice) + forcing_group = f'{scenario}_{choice}' + label = forcing_group + resolution = None + else: + if params['ocean_model'] is None: + model = configured_model + base_path = os.path.join( + _ice_sheet_path(config), model, scenario) + forcing_group = f'{model}_{scenario}' + else: + model = params['ocean_model'] + base_path = os.path.join( + _ice_sheet_path(config), scenario, model) + forcing_group = scenario + + if ice_sheet == 'ais': + grid_name = 'ocean' + resolution = None + else: + requested_resolution = section.get( + 'resolution', fallback='auto') + grid_name, resolution = _resolve_ocean_grid( + base_path, requested_resolution) + base_path = os.path.join(base_path, grid_name, 'tf') + label = f'{model}_{scenario}' + + version_path, version, files = resolve_version_directory( + base_path, requested_version, 'tf_*.nc') + + return ForcingSource( + directory=version_path, + files=files, + version=version, + model=model, + scenario=scenario, + forcing_group=forcing_group, + label=label, + source_grid=grid_name, + resolution=resolution, + ) + + +def resolve_fracture_source(config, file_pattern): + """Resolve one fracture product, keeping all pathways on one version.""" + model = config.get('ismip7', 'model') + scenario = config.get('ismip7', 'scenario') + base_path = os.path.join( + _ice_sheet_path(config), model, scenario, 'fracture') + requested_version = config.get( + 'ismip7_fracture', 'version', fallback='latest') + version_path, version, files = resolve_version_directory( + base_path, requested_version, file_pattern) + + return ForcingSource( + directory=version_path, + files=files, + version=version, + model=model, + scenario=scenario, + forcing_group=f'{model}_{scenario}', + label=f'{model}_{scenario}', + source_grid='fracture', + ) + + +def resolve_version_directory(base_path, requested_version='latest', + file_pattern='*.nc'): + """ + Resolve an explicit version or the numerically latest version directory. + + Version names may contain any number of numeric components, so ``v2.10`` + correctly sorts after ``v2.9``. If the latest directory exists but does + not contain the expected files, an error is raised instead of silently + falling back to an older version. + """ + versions = [] + if os.path.isdir(base_path): + for name in os.listdir(base_path): + path = os.path.join(base_path, name) + match = _VERSION_PATTERN.fullmatch(name) + if match is not None and os.path.isdir(path): + key = tuple(int(value) for value in match.group(1).split('.')) + versions.append((key, name)) + + if requested_version.lower() == 'latest': + if not versions: + raise FileNotFoundError( + f'No version directories were found in:\n {base_path}') + _, version = max(versions) + else: + version = requested_version + if _VERSION_PATTERN.fullmatch(version) is None: + raise ValueError( + f"Invalid version '{version}'. Use 'latest' or a version " + f"such as 'v2' or 'v2.1'.") + + version_path = os.path.join(base_path, version) + if not os.path.isdir(version_path): + available = ', '.join(name for _, name in sorted(versions)) or 'none' + raise FileNotFoundError( + f"Version '{version}' was not found in:\n {base_path}\n" + f'Available versions: {available}') + + files = tuple(sorted(glob.glob(os.path.join(version_path, file_pattern)))) + if not files: + raise FileNotFoundError( + f"Version '{version}' contains no files matching:\n" + f' {os.path.join(version_path, file_pattern)}') + + return version_path, version, files + + +def mapping_file_name(config, component, source_grid, method_remap): + """Build a mapping filename that identifies the selected source grid.""" + ice_sheet = config.get('ismip7', 'ice_sheet') + mali_mesh_name = config.get('ismip7', 'mali_mesh_name') + grid = re.sub(r'[^A-Za-z0-9]+', '_', source_grid).strip('_').lower() + return (f'map_ismip7_{ice_sheet}_{component}_{grid}_to_' + f'{mali_mesh_name}_{method_remap}.nc') + + +def _ice_sheet_path(config): + ice_sheet = config.get('ismip7', 'ice_sheet') + if ice_sheet not in _ICE_SHEET_DIRECTORIES: + raise ValueError( + f"Unknown ice_sheet '{ice_sheet}'. Must be one of: " + f'{list(_ICE_SHEET_DIRECTORIES)}') + archive_root = config.get('ismip7', 'base_path_ismip7') + return os.path.join(archive_root, _ICE_SHEET_DIRECTORIES[ice_sheet]) + + +def _requested_version(config, section, variable): + option = f'{variable}_version' + if config.has_option(section, option): + return config.get(section, option) + return config[section].get('version', fallback='latest') + + +def _resolve_atmosphere_grid(base_path, model, requested_product, + requested_resolution): + candidates = [] + if os.path.isdir(base_path): + for name in os.listdir(base_path): + match = _GRID_PATTERN.fullmatch(name) + path = os.path.join(base_path, name) + if match is not None and os.path.isdir(path): + product, metres = match.groups() + candidates.append((product, int(metres), name)) + + product = requested_product + if product.lower() == 'auto': + preferred = _DEFAULT_ATMOSPHERE_PRODUCTS.get(model) + products = sorted({candidate[0] for candidate in candidates}) + if preferred in products: + product = preferred + elif len(products) == 1: + product = products[0] + else: + choices = ', '.join(products) or 'none' + raise ValueError( + f"Could not select an atmosphere product automatically in:\n" + f' {base_path}\nAvailable products: {choices}. Set ' + f"'product' in [ismip7_atmosphere].") + + product_candidates = [candidate for candidate in candidates + if candidate[0] == product] + resolution = _select_resolution( + base_path, product_candidates, requested_resolution, 'atmosphere') + grid_name = f'{product}-{resolution}' + return grid_name, product, resolution + + +def _resolve_ocean_grid(base_path, requested_resolution): + candidates = [] + if os.path.isdir(base_path): + for name in os.listdir(base_path): + match = re.fullmatch(r'ocean-(\d+)m', name) + path = os.path.join(base_path, name) + if match is not None and os.path.isdir(path): + candidates.append(('ocean', int(match.group(1)), name)) + + resolution = _select_resolution( + base_path, candidates, requested_resolution, 'ocean') + return f'ocean-{resolution}', resolution + + +def _select_resolution(base_path, candidates, requested_resolution, + component): + if requested_resolution.lower() == 'auto': + if not candidates: + raise FileNotFoundError( + f'No {component} resolution directories were found in:\n' + f' {base_path}') + metres = min(candidate[1] for candidate in candidates) + else: + normalized = requested_resolution.lower().removesuffix('m') + if not normalized.isdigit(): + raise ValueError( + f"Invalid {component} resolution '{requested_resolution}'. " + f"Use 'auto' or a value such as '1000m'.") + metres = int(normalized) + + matches = [candidate for candidate in candidates + if candidate[1] == metres] + if not matches: + available = ', '.join( + f'{candidate[1]}m' for candidate in sorted(candidates)) or 'none' + raise FileNotFoundError( + f"Resolution '{metres}m' was not found in:\n {base_path}\n" + f'Available resolutions: {available}') + return f'{metres}m' diff --git a/compass/landice/ismip7/ice_sheet_params.py b/compass/landice/ismip7/ice_sheet_params.py index d991eae45c..f59b7cee5f 100644 --- a/compass/landice/ismip7/ice_sheet_params.py +++ b/compass/landice/ismip7/ice_sheet_params.py @@ -7,10 +7,6 @@ 'ais': { 'projection': 'ais-bedmap2', 'prefix': 'AIS', - 'atm_resolution': '2000m', - 'atm_version': 'v2', - 'ocean_version': 'v3', - 'ocean_grid': 'ocean', 'ocean_3d': True, 'ocean_temporal': 'decade', 'atm_model': None, @@ -19,10 +15,6 @@ 'gis': { 'projection': 'gis-bamber', 'prefix': 'GrIS', - 'atm_resolution': '1000m', - 'atm_version': 'v2', - 'ocean_version': 'v2', - 'ocean_grid': 'ocean', 'ocean_3d': False, 'ocean_temporal': 'yearly', 'atm_model': None, @@ -32,19 +24,14 @@ # Overrides applied for the OCX (reanalysis) scenario. OCX has no distinct # ESM model: it uses fixed reanalysis products (RACMO for the atmosphere and -# EN4 for the ocean) at data version v1. When scenario is 'OCX' the [ismip7] -# model option is ignored and these sources are used instead. +# EN4 for the ocean in GrIS). When scenario is 'OCX' the [ismip7] model option +# is ignored and these sources are used instead. _OCX_OVERRIDES = { 'gis': { - 'atm_version': 'v1', - 'ocean_version': 'v1', - 'ocean_grid': 'ocean-1000m', 'atm_model': 'RACMO2.3p2-ERA', 'ocean_model': 'EN4', }, 'ais': { - 'atm_version': 'v1', - 'ocean_version': 'v1', 'atm_model': 'RACMO2.3p2-ERA', 'ocean_model': None, # AIS OCX ocean files have no model token and live in per-choice diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py b/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py index 7e577b2026..6c49cb98ad 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/__init__.py @@ -1,3 +1,6 @@ +from compass.landice.tests.ismip7_forcing.atmosphere.build_mapping_file import ( # noqa: E501 + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.atmosphere.process_runoff import ( ProcessRunoff, ) @@ -40,6 +43,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessSmb(test_case=self)) self.add_step(ProcessTemperature(test_case=self)) self.add_step(ProcessSmbGradient(test_case=self)) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py new file mode 100644 index 0000000000..95b314d982 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/atmosphere/build_mapping_file.py @@ -0,0 +1,30 @@ +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) +from compass.landice.tests.ismip7_forcing.mapping_step import ( + build_and_cache_mapping, + setup_mapping_step, +) +from compass.step import Step + + +class BuildMappingFile(Step): + """Build weights from the selected atmosphere grid to the MALI mesh.""" + + def __init__(self, test_case): + super().__init__(test_case=test_case, name='build_mapping_file') + + def setup(self): + setup_mapping_step(self) + + def run(self): + config = self.config + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'acabf') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + + self.logger.info(f'Using atmosphere grid from {source.files[0]}') + build_and_cache_mapping( + self, source.files[0], mapping_file, method_remap) diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py index 8125abe250..94db1faf9c 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_runoff.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,8 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -43,46 +44,35 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'mrro') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_atmosphere"] method_remap = section.get("method_remap") start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "mrro", version) - file_pattern = (f"mrro_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No runoff files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "mrro") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -103,16 +93,9 @@ def run(self): logger.info(f"Found {len(input_files)} runoff files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by other atm steps) - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) # Remap each year file remapped_files = [] diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py index 9701265638..e84ea3c0c0 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,8 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -43,19 +44,22 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'acabf') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -64,24 +68,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "acabf", version) - file_pattern = (f"acabf_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No SMB files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "acabf") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -102,17 +93,9 @@ def run(self): logger.info(f"Found {len(input_files)} SMB files for years " f"{start_year}-{end_year}") - # Build mapping file using the first input file as the grid template - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) # Remap each year file remapped_files = [] diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py index e35755e9b6..9f38bde8d7 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_smb_gradient.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,8 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -44,19 +45,22 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'dacabfdz') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -65,24 +69,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "dacabfdz", version) - file_pattern = (f"dacabfdz_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No SMB gradient files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "dacabfdz") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -103,17 +94,9 @@ def run(self): logger.info(f"Found {len(input_files)} SMB gradient files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by process_smb) - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) # Remap each year file remapped_files = [] diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py index 56b13403be..dfd145fe7c 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,8 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -43,19 +44,22 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'ts') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -64,24 +68,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "ts", version) - file_pattern = (f"ts_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No temperature files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "ts") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -102,17 +93,9 @@ def run(self): logger.info(f"Found {len(input_files)} temperature files for years " f"{start_year}-{end_year}") - # Build mapping file (reuse if already created by process_smb) - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) # Remap each year file remapped_files = [] diff --git a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py index 41cf9f30aa..47541a0798 100644 --- a/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py +++ b/compass/landice/tests/ismip7_forcing/atmosphere/process_temperature_gradient.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,8 +5,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, +) from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -45,19 +46,22 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get('ismip7_atmosphere', 'method_remap') + source = resolve_atmosphere_source(config, 'dtsdz') + mapping_file = mapping_file_name( + config, 'atm', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case """ logger = self.logger config = self.config - params = get_params(config) - section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") @@ -66,24 +70,11 @@ def run(self): start_year = section.getint("start_year") end_year = section.getint("end_year") - # Discover input files - prefix = params['prefix'] - resolution = params['atm_resolution'] - version = params['atm_version'] - if params['atm_model'] is not None: - forcing_group = scenario - model = params['atm_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "dtsdz", version) - file_pattern = (f"dtsdz_{prefix}_{model}_{scenario}_" - f"SDBN1-{resolution}_{version}_*.nc") - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No temperature gradient files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_atmosphere_source(config, "dtsdz") + all_files = source.files + model = source.model + forcing_group = source.forcing_group + logger.info(f"Using atmosphere source {source.directory}") # Filter to requested year range input_files = [] @@ -104,17 +95,9 @@ def run(self): logger.info(f"Found {len(input_files)} temperature gradient files " f"for years {start_year}-{end_year}") - # Build mapping file (reuse if already created by other steps) - ice_sheet = config.get("ismip7", "ice_sheet") - mapping_file = (f"map_ismip7_{ice_sheet}_atm_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "atm", source.source_grid, method_remap) # Remap each year file remapped_files = [] diff --git a/compass/landice/tests/ismip7_forcing/fracture/__init__.py b/compass/landice/tests/ismip7_forcing/fracture/__init__.py index b6a7e63c6a..ba34980e6a 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/__init__.py +++ b/compass/landice/tests/ismip7_forcing/fracture/__init__.py @@ -1,6 +1,9 @@ from compass.landice.tests.ismip7_forcing.configure import ( configure as configure_testgroup, ) +from compass.landice.tests.ismip7_forcing.fracture.build_mapping_file import ( + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.fracture.process_excess_melt import ( ProcessExcessMelt, ) @@ -40,6 +43,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessExcessMelt(test_case=self)) self.add_step(ProcessLakeProperties(test_case=self)) self.add_step(ProcessShelfCollapse(test_case=self)) diff --git a/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py new file mode 100644 index 0000000000..3ca2dc4df2 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/fracture/build_mapping_file.py @@ -0,0 +1,56 @@ +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) +from compass.landice.tests.ismip7_forcing.mapping_step import ( + build_and_cache_mapping, + setup_mapping_step, +) +from compass.step import Step + + +class BuildMappingFile(Step): + """Build weights from the fracture grid to the MALI mesh.""" + + def __init__(self, test_case): + super().__init__(test_case=test_case, name='build_mapping_file') + + def setup(self): + setup_mapping_step(self) + + def run(self): + config = self.config + section = config['ismip7_fracture'] + methods = { + section.get('method_remap_shelf_collapse'), + section.get('method_remap_excess_melt'), + section.get('method_remap_lake_properties'), + } + if all(method is None or method.lower() == 'none' + for method in methods): + self.logger.info('No fracture mappings requested; skipping.') + return + source = _resolve_grid_source(config) + + for method_remap in sorted(methods): + if method_remap is None or method_remap.lower() == 'none': + continue + mapping_file = mapping_file_name( + config, 'fracture', source.source_grid, method_remap) + self.logger.info(f'Using fracture grid from {source.files[0]}') + build_and_cache_mapping( + self, source.files[0], mapping_file, method_remap) + + +def _resolve_grid_source(config): + """Select a fracture file with native x/y coordinates as grid donor.""" + errors = [] + for pattern in ('lake_properties_*.nc', + 'ice_shelf_collapse_mask_*.nc'): + try: + return resolve_fracture_source(config, pattern) + except FileNotFoundError as exc: + errors.append(str(exc)) + raise FileNotFoundError( + 'No fracture source with x/y coordinates was found.\n' + + '\n'.join(errors)) diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py index 0f291792f0..9a68fcd753 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_excess_melt.py @@ -7,7 +7,10 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) from compass.landice.ismip7.remap import ( add_xtime_and_write, extrapolate_source, @@ -54,6 +57,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get( + 'ismip7_fracture', 'method_remap_excess_melt') + if method_remap.lower() != 'none': + mapping_file = mapping_file_name( + config, 'fracture', 'fracture', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -62,17 +74,13 @@ def run(self): config = self.config section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_fracture"] method_remap = section.get("method_remap_excess_melt") - version = section.get("version") start_year = section.getint("start_year") end_year = section.getint("end_year") @@ -83,14 +91,10 @@ def run(self): return # Discover the excess melt file - input_path = os.path.join(base_path_ismip7, "fracture", version) file_pattern = "excess_melt_*.nc" - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No excess melt file found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_fracture_source(config, file_pattern) + input_path = source.directory + all_files = source.files if len(all_files) > 1: raise ValueError( f"Expected a single excess melt file but found " @@ -105,17 +109,9 @@ def run(self): self._prepare_source_grid(input_file, input_path, gridded_file, logger) - # Build mapping file. Excess melt is a flux, so conservative - # remapping is appropriate by default. - mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the excess melt grid...") - build_mapping_file(config, logger, - gridded_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "fracture", source.source_grid, method_remap) # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py index 2b5d5ac1d5..839af87ea7 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_lake_properties.py @@ -1,10 +1,12 @@ -import glob import os import shutil from mpas_tools.logging import check_call -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) from compass.landice.ismip7.remap import ( add_xtime_and_write, extrapolate_source, @@ -59,6 +61,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get( + 'ismip7_fracture', 'method_remap_lake_properties') + if method_remap.lower() != 'none': + mapping_file = mapping_file_name( + config, 'fracture', 'fracture', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -67,17 +78,13 @@ def run(self): config = self.config section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_fracture"] method_remap = section.get("method_remap_lake_properties") - version = section.get("version") start_year = section.getint("start_year") end_year = section.getint("end_year") @@ -88,14 +95,9 @@ def run(self): return # Discover the lake properties file - input_path = os.path.join(base_path_ismip7, "fracture", version) file_pattern = "lake_properties_*.nc" - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No lake properties file found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_fracture_source(config, file_pattern) + all_files = source.files if len(all_files) > 1: raise ValueError( f"Expected a single lake properties file but found " @@ -105,18 +107,9 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing lake properties: {basename}") - # Build mapping file. Lake properties are continuous fields, so - # bilinear remapping is appropriate by default. - mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the lake properties " - "grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "fracture", source.source_grid, method_remap) # Extrapolate fill values on the source grid before remapping so # they don't pollute neighboring cells during interpolation diff --git a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py index bf6537438e..df5d1b6d7c 100644 --- a/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py +++ b/compass/landice/tests/ismip7_forcing/fracture/process_shelf_collapse.py @@ -1,10 +1,12 @@ -import glob import os import shutil from mpas_tools.logging import check_call -from compass.landice.ismip7.mapping import build_mapping_file +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_fracture_source, +) from compass.landice.ismip7.remap import ( add_xtime_and_write, open_rename_and_trim, @@ -46,6 +48,15 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + method_remap = config.get( + 'ismip7_fracture', 'method_remap_shelf_collapse') + if method_remap.lower() != 'none': + mapping_file = mapping_file_name( + config, 'fracture', 'fracture', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -54,17 +65,13 @@ def run(self): config = self.config section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_fracture"] method_remap = section.get("method_remap_shelf_collapse") - version = section.get("version") start_year = section.getint("start_year") end_year = section.getint("end_year") @@ -75,14 +82,9 @@ def run(self): return # Discover the ice shelf collapse mask file - input_path = os.path.join(base_path_ismip7, "fracture", version) file_pattern = "ice_shelf_collapse_mask_*.nc" - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No ice shelf collapse mask file found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + source = resolve_fracture_source(config, file_pattern) + all_files = source.files if len(all_files) > 1: raise ValueError( f"Expected a single ice shelf collapse mask file but found " @@ -92,16 +94,9 @@ def run(self): basename = os.path.basename(input_file) logger.info(f"Processing ice shelf collapse mask: {basename}") - # Build mapping file. neareststod preserves the 0/1 mask values. - mapping_file = (f"map_ismip7_{ice_sheet}_fracture_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file for the collapse mask grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "fracture", source.source_grid, method_remap) # Remap the collapse mask onto the MALI mesh remapped_file = f"remapped_{basename}" diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg index be7d195aa4..2dcec7e33c 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing.cfg @@ -4,7 +4,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = NotAvailable -# Base path to the input ISMIP7 forcing files. User has to supply. +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +# User has to supply. base_path_ismip7 = NotAvailable # Base path to the MALI mesh. User has to supply. @@ -30,6 +31,10 @@ mali_mesh_file = NotAvailable # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 128 +# Optional directory containing mapping files from a previous run. Missing +# files are built and saved under output_base_path/mapping_files. +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true @@ -39,6 +44,17 @@ process_ocean_climatology = true # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +# Atmosphere product. 'auto' selects SDBN1 for CESM2-WACCM and OCX, and +# GEMB-SDBN1 for MRI-ESM2-0. Other products (e.g. dEBM2) can be named here. +product = auto + +# Source-grid resolution. 'auto' selects the finest available resolution. +resolution = auto + +# Dataset version. 'latest' is resolved independently for each variable. +# A variable-specific override such as acabf_version = v2 takes precedence. +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -51,6 +67,13 @@ end_year = 2014 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +# Source-grid resolution for GrIS. 'auto' selects the finest available +# resolution. AIS ocean forcing has a single archive grid. +resolution = auto + +# Dataset version. Use an explicit version such as v2 for reproducibility. +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -70,6 +93,9 @@ end_year = 2014 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +# Dataset version +version = latest + # Base path to observational climatology data # (directory containing tf/, so/, thetao/ subdirs) base_path_climatology = /path/to/ISMIP7/forcing/AIS/obs/zhou_annual_06_nov @@ -91,7 +117,7 @@ method_remap_excess_melt = conserve method_remap_lake_properties = bilinear # Version subdirectory of the fracture forcing data -version = v2 +version = latest # Start year for processing start_year = 1850 diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg index 975cb89b85..c686cb6796 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_ais.cfg @@ -14,8 +14,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = ais -# Base path to the input ISMIP7 forcing files. User has to supply. -base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/AIS/OCX +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing # Base path to the MALI mesh. User has to supply. base_path_mali = /global/cfs/cdirs/fanssie/MALI_input_files/AIS_4to20km_r01 @@ -39,6 +39,9 @@ mali_mesh_file = AIS_4to20km_r01_20220907.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional directory containing mapping files from a previous run +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true @@ -48,6 +51,10 @@ process_ocean_climatology = false # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +product = auto +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -60,6 +67,9 @@ end_year = 2025 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -80,5 +90,7 @@ end_year = 2025 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +version = latest + # Base path to observational climatology data base_path_climatology = None diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg index d3658027db..2636c392d9 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_ocx_gis.cfg @@ -12,8 +12,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = gis -# Base path to the input ISMIP7 forcing files. User has to supply. -base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/GIS/OCX +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing # Base path to the MALI mesh. User has to supply. base_path_mali = /global/cfs/cdirs/fanssie/MALI_input_files/GIS_1to10km_r02/ @@ -37,6 +37,9 @@ mali_mesh_file = GIS_1to10km_r02_20230202.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional directory containing mapping files from a previous run +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true @@ -46,6 +49,10 @@ process_ocean_climatology = false # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +product = auto +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -58,6 +65,9 @@ end_year = 2025 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -73,5 +83,7 @@ end_year = 2025 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +version = latest + # Base path to observational climatology data base_path_climatology = None diff --git a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg index ab282b2e06..a6d2ca5d82 100644 --- a/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg +++ b/compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg @@ -4,8 +4,8 @@ # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = ais -# Base path to the input ISMIP7 forcing files. User has to supply. -base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/AIS/CESM2-WACCM/ssp585 +# Root of the native ISMIP7 archive (the directory containing AIS/ and GIS/). +base_path_ismip7 = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing # Base path to the MALI mesh. User has to supply. base_path_mali = /global/cfs/cdirs/fanssie/MALI_input_files/AIS_4to20km_r01 @@ -30,6 +30,9 @@ mali_mesh_file = AIS_4to20km_r01_20220907.nc # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 512 +# Optional directory containing mapping files from a previous run +mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = false @@ -39,6 +42,10 @@ process_ocean_climatology = true # config options for ismip7 atmosphere forcing [ismip7_atmosphere] +product = auto +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = conserve @@ -51,6 +58,9 @@ end_year = 2301 # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] +resolution = auto +version = latest + # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear @@ -66,6 +76,8 @@ end_year = 2301 # Remapping method. Options: bilinear, neareststod, conserve method_remap = bilinear +version = latest + # Base path to observational climatology data # (directory containing tf/, so/, thetao/ subdirs) base_path_climatology = /global/cfs/cdirs/m4288/users/trhille/ISMIP7/forcing/AIS/obs/zhou_annual_06_nov @@ -87,7 +99,7 @@ method_remap_excess_melt = conserve method_remap_lake_properties = bilinear # Version subdirectory of the fracture forcing data -version = v2 +version = latest # Start year for processing start_year = 2015 diff --git a/compass/landice/tests/ismip7_forcing/mapping_step.py b/compass/landice/tests/ismip7_forcing/mapping_step.py new file mode 100644 index 0000000000..1e900149c9 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/mapping_step.py @@ -0,0 +1,57 @@ +"""Shared support for ISMIP7 forcing mapping-file steps.""" + +import os +import shutil + +from compass.landice.ismip7.mapping import build_mapping_file + + +def setup_mapping_step(step): + """Add the MALI mesh input and request the ESMF task allocation.""" + section = step.config['ismip7'] + base_path_mali = section.get('base_path_mali') + mali_mesh_file = section.get('mali_mesh_file') + + step.add_input_file( + filename=mali_mesh_file, + target=os.path.join(base_path_mali, mali_mesh_file)) + + step.ntasks = section.getint('esmf_ntasks') + step.min_tasks = 1 + + +def build_and_cache_mapping(step, source_file, mapping_file, method_remap): + """Reuse or build a mapping file, then cache a newly built file.""" + config = step.config + logger = step.logger + section = config['ismip7'] + mapping_files_path = section.get('mapping_files_path') + output_base_path = section.get('output_base_path') + mali_mesh_file = section.get('mali_mesh_file') + + reused = False + if mapping_files_path != 'NotAvailable': + cached_file = os.path.abspath( + os.path.join(mapping_files_path, mapping_file)) + if os.path.isfile(cached_file): + logger.info(f'Reusing mapping file {cached_file}') + if os.path.lexists(mapping_file): + os.remove(mapping_file) + os.symlink(cached_file, mapping_file) + reused = True + + build_mapping_file( + config, logger, source_file, mapping_file, + mali_mesh_file=mali_mesh_file, + method_remap=method_remap, + ntasks=step.ntasks) + + if reused: + return + + mapping_files_dir = os.path.join(output_base_path, 'mapping_files') + os.makedirs(mapping_files_dir, exist_ok=True) + destination = os.path.join(mapping_files_dir, mapping_file) + if os.path.realpath(mapping_file) != os.path.realpath(destination): + logger.info(f'Caching mapping file in {mapping_files_dir}') + shutil.copy2(mapping_file, destination) diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py index 2a97700428..e384db2cb4 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/__init__.py @@ -1,6 +1,9 @@ from compass.landice.tests.ismip7_forcing.configure import ( configure as configure_testgroup, ) +from compass.landice.tests.ismip7_forcing.ocean_thermal.build_mapping_file import ( # noqa: E501 + BuildMappingFile, +) from compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing import ( # noqa: E501 ProcessThermalForcing, ) @@ -29,6 +32,7 @@ def __init__(self, test_group): subdir = name super().__init__(test_group=test_group, name=name, subdir=subdir) + self.add_step(BuildMappingFile(test_case=self)) self.add_step(ProcessThermalForcing(test_case=self)) def configure(self): diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py new file mode 100644 index 0000000000..bb9ef1f0a7 --- /dev/null +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/build_mapping_file.py @@ -0,0 +1,76 @@ +import os + +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_ocean_source, + resolve_version_directory, +) +from compass.landice.ismip7.ice_sheet_params import get_params +from compass.landice.tests.ismip7_forcing.mapping_step import ( + build_and_cache_mapping, + setup_mapping_step, +) +from compass.step import Step + + +class BuildMappingFile(Step): + """Build weights from selected ocean grids to the MALI mesh.""" + + def __init__(self, test_case): + super().__init__(test_case=test_case, name='build_mapping_file') + + def setup(self): + setup_mapping_step(self) + + def run(self): + config = self.config + section = config['ismip7'] + + if section.getboolean('process_ocean_thermal'): + self._build_scenario_mapping() + if section.getboolean('process_ocean_climatology'): + self._build_climatology_mapping() + + def _build_scenario_mapping(self): + config = self.config + params = get_params(config) + section = config['ismip7_ocean_thermal'] + method_remap = section.get('method_remap') + + if params.get('ocean_choice_layout', False): + choice = _first_ocean_choice(section.get('ocean_choice')) + source = resolve_ocean_source(config, choice=choice) + else: + source = resolve_ocean_source(config) + + mapping_file = mapping_file_name( + config, 'ocean', source.source_grid, method_remap) + self.logger.info(f'Using ocean grid from {source.files[0]}') + build_and_cache_mapping( + self, source.files[0], mapping_file, method_remap) + + def _build_climatology_mapping(self): + config = self.config + section = config['ismip7_ocean_climatology'] + method_remap = section.get('method_remap') + base_path = section.get('base_path_climatology') + version = section.get('version', fallback='latest') + _, _, files = resolve_version_directory( + os.path.join(base_path, 'tf'), version, 'tf_*.nc') + mapping_file = mapping_file_name( + config, 'ocean', 'climatology', method_remap) + + self.logger.info(f'Using climatology grid from {files[0]}') + build_and_cache_mapping( + self, files[0], mapping_file, method_remap) + + +def _first_ocean_choice(raw_choices): + """Return one representative AIS OCX choice for mapping generation.""" + choices = [choice.strip() for choice in raw_choices.split(',') + if choice.strip()] + if any(choice.lower() == 'all' for choice in choices): + return 'main' + if not choices: + raise ValueError('No ocean_choice specified for AIS OCX forcing.') + return choices[0] diff --git a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py index 595d90b964..90e4c80bde 100644 --- a/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py +++ b/compass/landice/tests/ismip7_forcing/ocean_thermal/process_thermal_forcing.py @@ -1,4 +1,3 @@ -import glob import os import shutil @@ -6,8 +5,12 @@ from mpas_tools.io import write_netcdf from mpas_tools.logging import check_call +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_ocean_source, + resolve_version_directory, +) from compass.landice.ismip7.ice_sheet_params import get_params -from compass.landice.ismip7.mapping import build_mapping_file from compass.landice.ismip7.remap import extrapolate_source from compass.step import Step @@ -47,6 +50,30 @@ def setup(self): target=os.path.join(base_path_mali, mali_mesh_file)) + if section.getboolean('process_ocean_thermal'): + params = get_params(config) + if params.get('ocean_choice_layout', False): + choice = self._get_ocean_choices(config)[0] + source = resolve_ocean_source(config, choice=choice) + else: + source = resolve_ocean_source(config) + method_remap = config.get( + 'ismip7_ocean_thermal', 'method_remap') + mapping_file = mapping_file_name( + config, 'ocean', source.source_grid, method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + + if section.getboolean('process_ocean_climatology'): + method_remap = config.get( + 'ismip7_ocean_climatology', 'method_remap') + mapping_file = mapping_file_name( + config, 'ocean', 'climatology', method_remap) + self.add_input_file( + filename=mapping_file, + target=f'../build_mapping_file/{mapping_file}') + def run(self): """ Run this step of the test case @@ -72,22 +99,15 @@ def _run_scenario(self): params = get_params(config) section = config["ismip7"] - base_path_ismip7 = section.get("base_path_ismip7") mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") - model = section.get("model") scenario = section.get("scenario") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_ocean_thermal"] method_remap = section.get("method_remap") start_year = section.getint("start_year") end_year = section.getint("end_year") - prefix = params['prefix'] - ocean_version = params['ocean_version'] - ocean_grid = params['ocean_grid'] ocean_3d = params['ocean_3d'] # Assemble the list of forcing sources to process. AIS OCX has several @@ -97,42 +117,17 @@ def _run_scenario(self): jobs = [] if params.get('ocean_choice_layout', False): for choice in self._get_ocean_choices(config): - input_path = os.path.join(base_path_ismip7, "ocean", choice, - ocean_version) - file_pattern = (f"tf_{prefix}_{scenario}_ocean_{choice}_" - f"{ocean_version}_*.nc") - jobs.append({ - 'input_path': input_path, - 'file_pattern': file_pattern, - 'forcing_group': f"{scenario}_{choice}", - 'label': f"{scenario}_{choice}", - }) + jobs.append(resolve_ocean_source(config, choice=choice)) else: - if params['ocean_model'] is not None: - forcing_group = scenario - model = params['ocean_model'] - else: - forcing_group = f"{model}_{scenario}" - input_path = os.path.join(base_path_ismip7, "ocean", "tf", - ocean_version) - file_pattern = (f"tf_{prefix}_{model}_{scenario}_" - f"{ocean_grid}_{ocean_version}_*.nc") - jobs.append({ - 'input_path': input_path, - 'file_pattern': file_pattern, - 'forcing_group': forcing_group, - 'label': f"{model}_{scenario}", - }) - - # Mapping file is shared across sources (identical ocean source grid). - mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" - f"{mali_mesh_name}_{method_remap}.nc") + jobs.append(resolve_ocean_source(config)) for job in jobs: + self.logger.info(f"Using ocean source {job.directory}") + mapping_file = mapping_file_name( + config, "ocean", job.source_grid, method_remap) self._process_ocean_forcing( - job, mapping_file, ocean_3d, method_remap, - start_year, end_year, mali_mesh_name, mali_mesh_file, - output_base_path) + job, mapping_file, ocean_3d, start_year, end_year, + mali_mesh_name, output_base_path) # AIS OCX writes one ocean_thermal_forcing dir per ocean choice # (OCX_), but atmosphere output is written once, under OCX. @@ -141,7 +136,7 @@ def _run_scenario(self): if params.get('ocean_choice_layout', False): for job in jobs: self._link_atmosphere_outputs( - output_base_path, scenario, job['forcing_group']) + output_base_path, scenario, job.forcing_group) def _get_ocean_choices(self, config): """ @@ -210,21 +205,31 @@ def _link_atmosphere_outputs(self, output_base_path, atm_forcing_group, dst_dir = os.path.join(output_base_path, choice_forcing_group, "atmosphere") + + if os.path.realpath(dst_dir) == os.path.realpath(src_dir): + logger.warning( + f"Skipping atmosphere mirror for {choice_forcing_group}: " + f"destination {dst_dir} resolves to the source directory.") + return + os.makedirs(dst_dir, exist_ok=True) for fname in os.listdir(src_dir): src = os.path.join(src_dir, fname) - if not os.path.isfile(src): + if os.path.islink(src) or not os.path.isfile(src): continue dst = os.path.join(dst_dir, fname) + if os.path.realpath(dst) == os.path.realpath(src): + logger.warning( + f"Skipping {fname}: destination resolves to source.") + continue if os.path.lexists(dst): os.remove(dst) os.symlink(src, dst) logger.info(f" Linked {dst} -> {src}") def _process_ocean_forcing(self, job, mapping_file, ocean_3d, - method_remap, start_year, end_year, - mali_mesh_name, mali_mesh_file, + start_year, end_year, mali_mesh_name, output_base_path): """ Discover, remap, combine, and save the thermal forcing for a single @@ -232,36 +237,23 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, Parameters ---------- - job : dict - Source description with keys ``input_path``, ``file_pattern``, - ``forcing_group``, and ``label`` + job : compass.landice.ismip7.archive.ForcingSource + Resolved ocean forcing source mapping_file : str - Path of the shared ocean-to-MALI mapping file (built on demand) + Path of the shared ocean-to-MALI mapping file ocean_3d : bool Whether the thermal forcing is 3D (AIS) or 2D (GrIS) - method_remap : str - Remapping method passed to the mapping-file builder start_year, end_year : int Inclusive year range to process - mali_mesh_name, mali_mesh_file : str - MALI mesh name and file + mali_mesh_name : str + MALI mesh name output_base_path : str Base path under which output is written """ logger = self.logger - config = self.config - - input_path = job['input_path'] - file_pattern = job['file_pattern'] - forcing_group = job['forcing_group'] - label = job['label'] - - all_files = sorted(glob.glob(os.path.join(input_path, file_pattern))) - - if not all_files: - raise FileNotFoundError( - f"No ocean thermal forcing files found matching pattern:\n" - f" {os.path.join(input_path, file_pattern)}") + forcing_group = job.forcing_group + label = job.label + all_files = job.files # Filter to files that overlap with the requested year range. # AIS files are named with decade or multi-decade ranges (e.g., @@ -284,14 +276,6 @@ def _process_ocean_forcing(self, job, mapping_file, ocean_3d, logger.info(f"Found {len(input_files)} ocean thermal forcing files " f"overlapping years {start_year}-{end_year}") - # Build mapping file using the first input file as grid template. - if not os.path.exists(mapping_file): - logger.info("Building mapping file for ocean grid...") - build_mapping_file(config, logger, - input_files[0], mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) - # Remap each file remapped_files = [] for input_file in input_files: @@ -362,39 +346,26 @@ def _run_climatology(self): section = config["ismip7"] mali_mesh_name = section.get("mali_mesh_name") - mali_mesh_file = section.get("mali_mesh_file") output_base_path = section.get("output_base_path") - ice_sheet = section.get("ice_sheet") section = config["ismip7_ocean_climatology"] method_remap = section.get("method_remap") base_path_climatology = section.get("base_path_climatology") - version = 'v3' + requested_version = section.get("version", fallback="latest") # Discover climatology TF file - input_path = os.path.join(base_path_climatology, "tf", version) - all_files = sorted(glob.glob(os.path.join(input_path, "tf_*.nc"))) - - if not all_files: - raise FileNotFoundError( - f"No ocean climatology TF files found in:\n" - f" {input_path}") + _, _, all_files = resolve_version_directory( + os.path.join(base_path_climatology, "tf"), requested_version, + "tf_*.nc") # Use the first (and likely only) file input_file = all_files[0] logger.info(f"Processing ocean TF climatology: " f"{os.path.basename(input_file)}") - # Build mapping file using the climatology file as grid template. - mapping_file = (f"map_ismip7_{ice_sheet}_ocean_to_" - f"{mali_mesh_name}_{method_remap}.nc") - - if not os.path.exists(mapping_file): - logger.info("Building mapping file for ocean grid...") - build_mapping_file(config, logger, - input_file, mapping_file, - mali_mesh_file=mali_mesh_file, - method_remap=method_remap) + # The mapping file is supplied by the build_mapping_file step. + mapping_file = mapping_file_name( + config, "ocean", "climatology", method_remap) # Extrapolate and remap basename = os.path.basename(input_file) @@ -420,6 +391,7 @@ def _run_climatology(self): # Rename to MALI conventions logger.info("Renaming variables to MALI conventions...") + version = os.path.basename(os.path.dirname(input_file)) output_file = (f"{mali_mesh_name}_thermal_forcing_climatology_" f"{version}.nc") diff --git a/docs/developers_guide/landice/api.rst b/docs/developers_guide/landice/api.rst index b0ecad439e..cf40250e8e 100644 --- a/docs/developers_guide/landice/api.rst +++ b/docs/developers_guide/landice/api.rst @@ -27,6 +27,12 @@ ISMIP7 framework :toctree: generated/ ice_sheet_params.get_params + archive.ForcingSource + archive.resolve_atmosphere_source + archive.resolve_ocean_source + archive.resolve_fracture_source + archive.resolve_version_directory + archive.mapping_file_name mapping.build_mapping_file remap.extrapolate_source remap.open_rename_and_trim @@ -487,6 +493,9 @@ ismip7_forcing atmosphere.Atmosphere atmosphere.Atmosphere.configure + atmosphere.build_mapping_file.BuildMappingFile + atmosphere.build_mapping_file.BuildMappingFile.setup + atmosphere.build_mapping_file.BuildMappingFile.run atmosphere.process_smb.ProcessSmb atmosphere.process_smb.ProcessSmb.setup atmosphere.process_smb.ProcessSmb.run @@ -505,12 +514,18 @@ ismip7_forcing ocean_thermal.OceanThermal ocean_thermal.OceanThermal.configure + ocean_thermal.build_mapping_file.BuildMappingFile + ocean_thermal.build_mapping_file.BuildMappingFile.setup + ocean_thermal.build_mapping_file.BuildMappingFile.run ocean_thermal.process_thermal_forcing.ProcessThermalForcing ocean_thermal.process_thermal_forcing.ProcessThermalForcing.setup ocean_thermal.process_thermal_forcing.ProcessThermalForcing.run fracture.Fracture fracture.Fracture.configure + fracture.build_mapping_file.BuildMappingFile + fracture.build_mapping_file.BuildMappingFile.setup + fracture.build_mapping_file.BuildMappingFile.run fracture.process_excess_melt.ProcessExcessMelt fracture.process_excess_melt.ProcessExcessMelt.setup fracture.process_excess_melt.ProcessExcessMelt.run diff --git a/docs/developers_guide/landice/framework.rst b/docs/developers_guide/landice/framework.rst index 3d25a70a31..7fe14e3e08 100644 --- a/docs/developers_guide/landice/framework.rst +++ b/docs/developers_guide/landice/framework.rst @@ -29,8 +29,14 @@ rather than in any one test group. :py:func:`compass.landice.ismip7.ice_sheet_params.get_params()` returns the parameters that differ between the Antarctic and Greenland ice sheets: the -projection, the filename prefix, dataset versions and resolutions, and whether -the ocean forcing is 3-D. +projection, fixed OCX sources, and whether the ocean forcing is 3-D. + +:py:mod:`compass.landice.ismip7.archive` resolves atmosphere, ocean, and +fracture files from the native archive hierarchy. It selects products and +resolutions from config, compares dotted version numbers numerically, and +resolves ``latest`` separately for datasets that are versioned independently. +Mapping filenames include the selected source grid to prevent reuse across +different resolutions. :py:func:`compass.landice.ismip7.mapping.build_mapping_file()` builds an ESMF mapping file from an ISMIP7 polar stereographic grid to a MALI mesh, using diff --git a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst index 6f88904b99..c07eb07b6a 100644 --- a/docs/developers_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/developers_guide/landice/test_groups/ismip7_forcing.rst @@ -22,20 +22,24 @@ in :ref:`landice_ismip7_forcing` in the User's Guide. Code shared with the other ISMIP7 test groups lives in the landice framework package :py:mod:`compass.landice.ismip7`, described in :ref:`dev_landice_framework`. This test group uses -:py:func:`compass.landice.ismip7.ice_sheet_params.get_params` for the -ice-sheet-specific parameters (projection, file naming prefix, grid -resolution, data version, ocean dimensionality), +:py:func:`compass.landice.ismip7.ice_sheet_params.get_params` for invariant +ice-sheet-specific parameters (projection, file naming prefix, and ocean +dimensionality), :py:mod:`compass.landice.ismip7.archive` for native archive +path, resolution, and version discovery, :py:func:`compass.landice.ismip7.mapping.build_mapping_file` to create the -SCRIP and ESMF mapping files, and the remapping helpers in +SCRIP and ESMF mapping files from dedicated ``BuildMappingFile`` steps, and +the remapping helpers in :py:mod:`compass.landice.ismip7.remap`. -When ``scenario = OCX``, ``get_params`` applies a set of OCX overrides on top -of the ice-sheet defaults: data version ``v1``, the ocean file-name grid token -(e.g. ``ocean-1000m``), and the fixed reanalysis sources (``atm_model`` = -``RACMO2.3p2-ERA`` and ``ocean_model`` = ``EN4``). The processing steps use -``atm_model`` / ``ocean_model`` in place of the ``[ismip7] model`` option when -they are set, so the OCX ``model`` option is ignored. This keeps OCX handling -centralized and lets a single config file drive both test cases. +When ``scenario = OCX``, ``get_params`` supplies the fixed reanalysis sources +(``atm_model`` = ``RACMO2.3p2-ERA`` for both ice sheets and ``ocean_model`` = +``EN4`` for GrIS). For AIS OCX, ``ocean_model`` remains unset and the archive +resolver uses per-choice ocean directories (main/cold/warm/vary). The archive +resolver discovers native paths, selected resolutions, and versions. The +processing steps use ``atm_model`` / ``ocean_model`` in place of the +``[ismip7] model`` option when they are set, so the OCX ``model`` option is +ignored. This keeps OCX handling centralized and lets a single config file drive +both test cases. configure ~~~~~~~~~ @@ -61,14 +65,18 @@ atmosphere ~~~~~~~~~~ The :py:class:`compass.landice.tests.ismip7_forcing.atmosphere.Atmosphere` -test case processes the ISMIP7 atmosphere forcing fields. It contains five -steps: SMB, temperature, their respective gradients, and runoff. Each step -discovers input files matching the ice-sheet-specific naming pattern, builds -or reuses a mapping file, remaps each input file with ``ncremap``, and -combines/renames the results to MALI conventions. +test case processes the ISMIP7 atmosphere forcing fields. It contains a +mapping step followed by five processing steps for SMB, temperature, their +respective gradients, and runoff. The mapping step uses the archive resolver +to select the configured source grid and runs with ``esmf_ntasks``. Each +processing step consumes those weights and remaps its input files with +``ncremap``. ``latest`` is resolved per variable because atmosphere datasets +can have different current versions. Steps: +* :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.build_mapping_file.BuildMappingFile` — + builds and caches the shared atmosphere mapping file * :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.process_smb.ProcessSmb` — ``acabf`` → ``sfcMassBal`` * :py:class:`~compass.landice.tests.ismip7_forcing.atmosphere.process_temperature.ProcessTemperature` — @@ -86,10 +94,13 @@ ocean_thermal ~~~~~~~~~~~~~ The :py:class:`compass.landice.tests.ismip7_forcing.ocean_thermal.OceanThermal` -test case processes the ISMIP7 ocean thermal forcing. It contains a single step, +test case processes the ISMIP7 ocean thermal forcing. It contains a mapping +step followed by the processing step, :py:class:`~compass.landice.tests.ismip7_forcing.ocean_thermal.process_thermal_forcing.ProcessThermalForcing`, which handles both AIS (3D, decade-spanning files) and GrIS (2D, yearly files) -by branching on the ``ocean_3d`` parameter from ``ice_sheet_params``. +by branching on the ``ocean_3d`` parameter from ``ice_sheet_params``. The +mapping step builds each enabled scenario or climatology mapping with +``esmf_ntasks``. The ``run()`` method dispatches to two sub-methods based on the boolean config options ``process_ocean_thermal`` and ``process_ocean_climatology`` in the @@ -99,7 +110,6 @@ options ``process_ocean_thermal`` and ``process_ocean_climatology`` in the scenario combination). Uses config from ``[ismip7_ocean_thermal]``. * ``_run_climatology()``: Processes the static observational climatology (Zhou et al., AIS only). Uses config from ``[ismip7_ocean_climatology]``. - The TF version (currently v3) is hard-coded. For AIS scenario data, the step: @@ -127,17 +137,21 @@ fracture The :py:class:`compass.landice.tests.ismip7_forcing.fracture.Fracture` test case processes the ISMIP7 surface-melt-driven ice shelf collapse -forcing (AIS only). It implements the three ISMIP7 pathways as independent -steps, each discovering its source file from the ``fracture/{version}/`` -subdirectory of ``base_path_ismip7``, building or reusing a mapping file, -remapping with ``ncremap``, and renaming the result to MALI conventions with -an accompanying ``xtime`` variable. Per-pathway remapping methods are set in +forcing (AIS only). It contains a mapping step followed by three independent +pathway-processing steps. The mapping step uses a lake-properties or +shelf-collapse file with native ``x``/``y`` coordinates and builds one file +per unique enabled remapping method. Each processing step discovers its source +file from the native ``AIS/{model}/{scenario}/fracture/{version}/`` hierarchy, +remaps with ``ncremap``, and renames the result to MALI conventions with an +accompanying ``xtime`` variable. Per-pathway remapping methods are set in the ``[ismip7_fracture]`` config section. Setting a pathway's remapping-method option to ``None`` causes that step to return early without processing its file, which is useful when only some pathway source files are available. Steps: +* :py:class:`~compass.landice.tests.ismip7_forcing.fracture.build_mapping_file.BuildMappingFile` + — builds and caches the enabled fracture mapping files * :py:class:`~compass.landice.tests.ismip7_forcing.fracture.process_excess_melt.ProcessExcessMelt` (Path A) — ``excess_melt`` → ``ismip7ExcessMelt``. The excess melt file lacks ``x``/``y`` coordinate variables and its array is flipped along the diff --git a/docs/users_guide/landice/test_groups/ismip7_forcing.rst b/docs/users_guide/landice/test_groups/ismip7_forcing.rst index 4a5ca9ea82..397dafcdc5 100644 --- a/docs/users_guide/landice/test_groups/ismip7_forcing.rst +++ b/docs/users_guide/landice/test_groups/ismip7_forcing.rst @@ -11,20 +11,22 @@ The test group supports both the Antarctic Ice Sheet (AIS) and the Greenland Ice Sheet (GrIS), controlled by a single ``ice_sheet`` config option. The test group includes three test cases: ``atmosphere``, ``ocean_thermal``, -and ``fracture``. +and ``fracture``. Each begins with a ``build_mapping_file`` step that runs +ESMF with a separate resource allocation. -* The ``atmosphere`` test case has five steps: +* The ``atmosphere`` test case has six steps: ``build_mapping_file``, ``process_smb``, ``process_temperature``, ``process_smb_gradient``, ``process_temperature_gradient``, and ``process_runoff``. -* The ``ocean_thermal`` test case has one step: ``process_thermal_forcing``. +* The ``ocean_thermal`` test case has two steps: ``build_mapping_file`` and + ``process_thermal_forcing``. For AIS this produces 3D thermal forcing (with 30 ocean depth layers); for GrIS it produces 2D (depth-averaged) thermal forcing. The step can also process the observational ocean thermal forcing climatology (Zhou et al.) for AIS, controlled by the ``process_ocean_climatology`` config option. -* The ``fracture`` test case has three steps: ``process_excess_melt`` - (Path A), ``process_lake_properties`` (Path B), and +* The ``fracture`` test case has four steps: ``build_mapping_file``, + ``process_excess_melt`` (Path A), ``process_lake_properties`` (Path B), and ``process_shelf_collapse`` (Path C). It processes the ISMIP7 surface-melt-driven ice shelf collapse forcing (AIS only). @@ -44,7 +46,8 @@ To use this test group, users need to: 2. Set the ``ice_sheet`` config option to either ``ais`` or ``gis``. -3. Provide the path to the ISMIP7 forcing data (``base_path_ismip7``). +3. Set ``base_path_ismip7`` to the root of the native ISMIP7 archive, which + contains the ``AIS`` and ``GIS`` directories. 4. Run the ``atmosphere`` test case for each model and scenario combination. @@ -54,6 +57,12 @@ To use this test group, users need to: combination to process the surface-melt-driven ice shelf collapse pathways (excess melt, lake properties, and the ice shelf collapse mask). +The mapping step can be run first on a large allocation, followed by the +processing steps on a single node. Newly built weights are saved in +``{output_base_path}/mapping_files``. Set ``mapping_files_path`` to that +directory in later runs to reuse weights for the same source grid, MALI mesh, +and remapping method. + Example user config files are provided in the source tree for local testing: * ``compass/landice/tests/ismip7_forcing/ismip7_forcing_test.cfg`` @@ -110,73 +119,43 @@ expects exactly one file per forcing field there. Input Data ---------- -ISMIP7 forcing data is organized by variable and version. The expected -directory structure under ``base_path_ismip7`` is: - -For AIS atmosphere (2km, polar stereographic EPSG:3031): - -.. code-block:: none - - acabf/v2/acabf_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - ts/v2/ts_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - dacabfdz/v2/dacabfdz_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - dtsdz/v2/dtsdz_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - mrro/v2/mrro_AIS_{model}_{scenario}_SDBN1-2000m_v2_{year_range}.nc - -For AIS ocean thermal (8km, 30 depth levels, decade files): - -.. code-block:: none - - ocean/tf/v3/tf_AIS_{model}_{scenario}_ocean_v3_{start_year}-{end_year}.nc - -For AIS ocean thermal climatology (8km, 30 depth levels, static): - -.. code-block:: none - - {base_path_climatology}/tf/v3/tf_AIS_obs_ocean_climatology_*.nc - -For AIS fracture / ice shelf collapse mask (8km, annual, Path C): - -.. code-block:: none - - fracture/v2/ice_shelf_collapse_mask_*.nc - -For GrIS atmosphere (1km, polar stereographic EPSG:3413): +``base_path_ismip7`` is the unmodified archive root. The resolver selects +files from these native layouts: .. code-block:: none - acabf/v2/acabf_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - ts/v2/ts_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - dacabfdz/v2/dacabfdz_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - dtsdz/v2/dtsdz_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - mrro/v2/mrro_GrIS_{model}_{scenario}_SDBN1-1000m_v2_{year}.nc - -For GrIS ocean thermal (same 1km grid, 2D, yearly files): - -.. code-block:: none - - ocean/tf/v2/tf_GrIS_{model}_{scenario}_ocean_v2_{year}.nc - -The OCX (reanalysis) scenario follows the same directory layout as the ESM -scenarios, but uses fixed reanalysis sources, data version ``v1``, and a -named grid resolution in the ocean file names. For GrIS OCX the atmosphere -source is ``RACMO2.3p2-ERA`` and the ocean source is ``EN4``: - -.. code-block:: none - - acabf/v1/acabf_GrIS_RACMO2.3p2-ERA_OCX_SDBN1-1000m_v1_{year}.nc - ocean/tf/v1/tf_GrIS_EN4_OCX_ocean-1000m_v1_{year}.nc - -Set ``base_path_ismip7`` to the ``OCX`` directory and ``scenario = OCX``. -The sources, version, and ocean grid token are selected automatically for -OCX, and the ``[ismip7] model`` option is ignored (set it to ``None``). + {root}/{AIS|GIS}/{model}/{scenario}/{product}-{resolution}/{variable}/{version}/*.nc + {root}/AIS/{model}/{scenario}/ocean/tf/{version}/*.nc + {root}/GIS/{model}/{scenario}/ocean-{resolution}/tf/{version}/*.nc + {root}/AIS/{model}/{scenario}/fracture/{version}/*.nc + {root}/AIS/OCX/RACMO2.3p2-ERA/SDBN1-{resolution}/{variable}/{version}/*.nc + {root}/GIS/OCX/RACMO2.3p2-ERA/SDBN1-{resolution}/{variable}/{version}/*.nc + {root}/AIS/OCX/ocean/{choice}/{version}/*.nc + {root}/GIS/OCX/EN4/ocean-{resolution}/tf/{version}/*.nc + +The atmosphere ``product`` and atmosphere and GrIS ocean ``resolution`` are +configurable. ``auto`` selects the primary product for the requested model +and the finest available resolution. Set an explicit value to use another +archive product or grid. + +Versions default to ``latest`` and are compared numerically, including dotted +versions such as ``v2.1``. Atmosphere versions are resolved separately for +each variable because the archive does not version all fields in lockstep. +An explicit section-wide version may be set for reproducibility; an option +such as ``acabf_version = v2`` overrides it for one atmosphere variable. The +newest directory must contain matching files, so an incomplete transfer is +reported rather than silently replaced with an older version. + +The observational climatology remains configured separately with +``base_path_climatology``. Its version also defaults to ``latest`` below the +``tf`` directory. .. _landice_ismip7_forcing_config: config options -------------- -The ``ismip7_forcing`` test group uses four config sections. The default +The ``ismip7_forcing`` test group uses five config sections. The default values are: .. code-block:: cfg @@ -187,7 +166,7 @@ values are: # Ice sheet: ais (Antarctic) or gis (Greenland) ice_sheet = NotAvailable - # Base path to the input ISMIP7 forcing files + # Root of the native archive (the directory containing AIS/ and GIS/) base_path_ismip7 = NotAvailable # Base path to the MALI mesh @@ -211,6 +190,9 @@ values are: # Number of MPI tasks for ESMF_RegridWeightGen esmf_ntasks = 128 + # Optional directory containing compatible mapping files from another run + mapping_files_path = NotAvailable + # Whether to process time-varying ocean thermal forcing (ESM scenario data) process_ocean_thermal = true @@ -220,6 +202,13 @@ values are: # config options for ismip7 atmosphere forcing [ismip7_atmosphere] + # Product and source resolution + product = auto + resolution = auto + + # Resolve separately for each variable; explicit versions are also valid + version = latest + # Remapping method: bilinear, neareststod, conserve method_remap = conserve @@ -232,6 +221,10 @@ values are: # config options for ismip7 ocean thermal forcing [ismip7_ocean_thermal] + # GrIS source resolution and dataset version + resolution = auto + version = latest + # Remapping method: bilinear, neareststod, conserve method_remap = bilinear @@ -247,6 +240,9 @@ values are: # Remapping method: bilinear, neareststod, conserve method_remap = bilinear + # Dataset version + version = latest + # Base path to observational climatology data base_path_climatology = /path/to/ISMIP7/forcing/AIS/obs/zhou_annual_06_nov @@ -267,7 +263,7 @@ values are: method_remap_lake_properties = bilinear # Version subdirectory of the fracture forcing data - version = v2 + version = latest # Start year for processing start_year = 1850 @@ -294,6 +290,10 @@ grid to the MALI unstructured mesh. Steps: +* **build_mapping_file**: Builds and caches weights for the selected + atmosphere product and resolution. All five processing steps share these + weights. + * **process_smb**: Remaps the surface mass balance (``acabf``) field. The output variable is ``sfcMassBal``. @@ -320,6 +320,10 @@ The ``landice/ismip7_forcing/ocean_thermal`` test case processes the ISMIP7 ocean thermal forcing (``tf``) and remaps it from the native polar stereographic grid to the MALI unstructured mesh. +The **build_mapping_file** step builds and caches mappings for each enabled +source grid. Scenario and climatology mappings are distinct when their source +grids differ. The **process_thermal_forcing** step consumes those mappings. + The step supports two processing modes, controlled by boolean config options in the ``[ismip7]`` section: @@ -356,14 +360,18 @@ surface-melt-driven ice shelf collapse forcing (AIS only). It implements the three ISMIP7 pathways as separate steps, each remapping annual fields from the native 8km polar stereographic grid onto the MALI unstructured mesh. -All three source files are discovered from the ``fracture/{version}/`` -subdirectory of ``base_path_ismip7``. +All three source files are discovered from +``AIS/{model}/{scenario}/fracture/{version}/`` below ``base_path_ismip7``. Each pathway is run independently and can be skipped by setting its remapping-method config option to ``None`` in the ``[ismip7_fracture]`` section (for example, ``method_remap_excess_melt = None`` skips Path A). This is useful when only some of the pathway source files are available. +The **build_mapping_file** step uses a fracture file with native ``x``/``y`` +coordinates as the common grid template and creates one mapping file per +unique enabled remapping method. + * **process_excess_melt** (Path A): Remaps the excess meltwater field (melt + rain after firn air content depletion), matching ``excess_melt_*.nc``. The output variable is ``ismip7ExcessMelt`` diff --git a/tests/landice/ismip7/test_archive.py b/tests/landice/ismip7/test_archive.py new file mode 100644 index 0000000000..b84942ce88 --- /dev/null +++ b/tests/landice/ismip7/test_archive.py @@ -0,0 +1,203 @@ +"""Tests for ISMIP7 native-archive path resolution.""" + +from configparser import ConfigParser + +import pytest + +from compass.landice.ismip7.archive import ( + mapping_file_name, + resolve_atmosphere_source, + resolve_fracture_source, + resolve_ocean_source, + resolve_version_directory, +) + + +def _config(root, ice_sheet='gis', model='CESM2-WACCM', + scenario='ssp585'): + config = ConfigParser() + config['ismip7'] = { + 'base_path_ismip7': str(root), + 'ice_sheet': ice_sheet, + 'model': model, + 'scenario': scenario, + 'mali_mesh_name': 'mesh', + } + config['ismip7_atmosphere'] = { + 'product': 'auto', + 'resolution': 'auto', + 'version': 'latest', + } + config['ismip7_ocean_thermal'] = { + 'resolution': 'auto', + 'version': 'latest', + } + config['ismip7_fracture'] = {'version': 'latest'} + return config + + +def _forcing_file(root, relative_path): + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + return path + + +def test_latest_version_uses_numeric_components(tmp_path): + """Dotted version components are sorted numerically, not as strings.""" + _forcing_file(tmp_path, 'v2.9/field.nc') + expected = _forcing_file(tmp_path, 'v2.10/field.nc') + + _, version, files = resolve_version_directory(tmp_path) + + assert version == 'v2.10' + assert files == (str(expected),) + + +def test_latest_does_not_fall_back_from_incomplete_version(tmp_path): + """An incomplete newest transfer is reported instead of hidden.""" + _forcing_file(tmp_path, 'v1/field.nc') + (tmp_path / 'v2').mkdir() + + with pytest.raises(FileNotFoundError, match="Version 'v2' contains no"): + resolve_version_directory(tmp_path) + + +def test_atmosphere_latest_is_resolved_per_variable(tmp_path): + """Variables can select different latest versions on the same grid.""" + base = 'GIS/CESM2-WACCM/ssp585' + _forcing_file( + tmp_path, f'{base}/SDBN1-4000m/acabf/v4/acabf_2000.nc') + acabf = _forcing_file( + tmp_path, f'{base}/SDBN1-1000m/acabf/v3/acabf_2000.nc') + ts = _forcing_file( + tmp_path, f'{base}/SDBN1-1000m/ts/v2/ts_2000.nc') + config = _config(tmp_path) + + acabf_source = resolve_atmosphere_source(config, 'acabf') + ts_source = resolve_atmosphere_source(config, 'ts') + + assert acabf_source.version == 'v3' + assert acabf_source.files == (str(acabf),) + assert ts_source.version == 'v2' + assert ts_source.files == (str(ts),) + assert acabf_source.resolution == '1000m' + + +def test_variable_version_override_takes_precedence(tmp_path): + """A variable-specific version can pin one atmosphere dataset.""" + base = 'GIS/CESM2-WACCM/ssp585/SDBN1-1000m/acabf' + expected = _forcing_file(tmp_path, f'{base}/v2/acabf_2000.nc') + _forcing_file(tmp_path, f'{base}/v3/acabf_2000.nc') + config = _config(tmp_path) + config['ismip7_atmosphere']['acabf_version'] = 'v2' + + source = resolve_atmosphere_source(config, 'acabf') + + assert source.version == 'v2' + assert source.files == (str(expected),) + + +def test_auto_product_uses_model_default(tmp_path): + """Auto distinguishes the primary model product from dEBM2.""" + base = 'AIS/MRI-ESM2-0/ssp585' + expected = _forcing_file( + tmp_path, f'{base}/GEMB-SDBN1-2000m/ts/v2/ts_2000.nc') + _forcing_file(tmp_path, f'{base}/dEBM2-8000m/ts/v1/ts_2000.nc') + config = _config( + tmp_path, ice_sheet='ais', model='MRI-ESM2-0') + + source = resolve_atmosphere_source(config, 'ts') + + assert source.product == 'GEMB-SDBN1' + assert source.files == (str(expected),) + + +def test_gis_ocean_resolution_and_version(tmp_path): + """GrIS ocean resolution and version are selected from the archive.""" + base = 'GIS/CESM2-WACCM/ssp585' + expected = _forcing_file( + tmp_path, f'{base}/ocean-1000m/tf/v2/tf_2000.nc') + _forcing_file( + tmp_path, f'{base}/ocean-4000m/tf/v3/tf_2000.nc') + config = _config(tmp_path) + + source = resolve_ocean_source(config) + + assert source.resolution == '1000m' + assert source.version == 'v2' + assert source.files == (str(expected),) + + +def test_gis_ocx_atmosphere_layout(tmp_path): + """GrIS OCX atmosphere uses the RACMO product below the OCX tree.""" + base = 'GIS/OCX/RACMO2.3p2-ERA' + expected = _forcing_file( + tmp_path, f'{base}/SDBN1-1000m/ts/v2/ts_2000.nc') + _forcing_file( + tmp_path, f'{base}/SDBN1-4000m/ts/v3/ts_2000.nc') + config = _config( + tmp_path, ice_sheet='gis', model='None', scenario='OCX') + + source = resolve_atmosphere_source(config, 'ts') + + assert source.model == 'RACMO2.3p2-ERA' + assert source.resolution == '1000m' + assert source.version == 'v2' + assert source.files == (str(expected),) + + +def test_gis_ocx_ocean_layout(tmp_path): + """GrIS OCX ocean uses EN4 and the selected native resolution.""" + base = 'GIS/OCX/EN4' + expected = _forcing_file( + tmp_path, f'{base}/ocean-1000m/tf/v1/tf_2000.nc') + _forcing_file( + tmp_path, f'{base}/ocean-4000m/tf/v2/tf_2000.nc') + config = _config( + tmp_path, ice_sheet='gis', model='None', scenario='OCX') + + source = resolve_ocean_source(config) + + assert source.model == 'EN4' + assert source.resolution == '1000m' + assert source.version == 'v1' + assert source.files == (str(expected),) + + +def test_ais_ocx_ocean_choice_layout(tmp_path): + """AIS OCX choices resolve below OCX/ocean rather than a variable dir.""" + expected = _forcing_file( + tmp_path, 'AIS/OCX/ocean/main/v2/tf_AIS_OCX_main.nc') + config = _config( + tmp_path, ice_sheet='ais', model='None', scenario='OCX') + + source = resolve_ocean_source(config, choice='main') + + assert source.version == 'v2' + assert source.forcing_group == 'OCX_main' + assert source.files == (str(expected),) + + +def test_fracture_uses_native_model_scenario_path(tmp_path): + """Fracture data resolves below the selected model and scenario.""" + expected = _forcing_file( + tmp_path, + 'AIS/CESM2-WACCM/ssp585/fracture/v2.1/excess_melt_data.nc') + config = _config(tmp_path, ice_sheet='ais') + + source = resolve_fracture_source(config, 'excess_melt_*.nc') + + assert source.version == 'v2.1' + assert source.files == (str(expected),) + + +def test_mapping_name_identifies_source_grid(tmp_path): + """Mapping files cannot collide across source resolutions.""" + config = _config(tmp_path) + + name = mapping_file_name( + config, 'atm', 'atmosphere_SDBN1-1000m', 'conserve') + + assert name == ( + 'map_ismip7_gis_atm_atmosphere_sdbn1_1000m_to_mesh_conserve.nc') diff --git a/tests/landice/ismip7/test_mapping_step.py b/tests/landice/ismip7/test_mapping_step.py new file mode 100644 index 0000000000..3a16029161 --- /dev/null +++ b/tests/landice/ismip7/test_mapping_step.py @@ -0,0 +1,87 @@ +"""Tests for shared ISMIP7 mapping-step behavior.""" + +from configparser import ConfigParser +from types import SimpleNamespace + +from compass.landice.tests.ismip7_forcing import mapping_step +from compass.landice.tests.ismip7_forcing.fracture.build_mapping_file import ( + _resolve_grid_source, +) + + +class _Logger: + def info(self, message): + pass + + +def _step(tmp_path, mapping_files_path): + config = ConfigParser() + config['ismip7'] = { + 'mapping_files_path': str(mapping_files_path), + 'output_base_path': str(tmp_path / 'output'), + 'mali_mesh_file': 'mesh.nc', + } + return SimpleNamespace(config=config, logger=_Logger(), ntasks=16) + + +def test_mapping_step_reuses_cached_file(tmp_path, monkeypatch): + """An existing cached mapping is linked rather than rebuilt.""" + cache = tmp_path / 'cache' + cache.mkdir() + expected = cache / 'map.nc' + expected.touch() + step = _step(tmp_path, cache) + calls = [] + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + mapping_step, 'build_mapping_file', + lambda *args, **kwargs: calls.append((args, kwargs))) + + mapping_step.build_and_cache_mapping( + step, 'source.nc', 'map.nc', 'bilinear') + + assert (tmp_path / 'map.nc').is_symlink() + assert (tmp_path / 'map.nc').resolve() == expected + assert len(calls) == 1 + + +def test_mapping_step_caches_new_file(tmp_path, monkeypatch): + """A newly built mapping is copied to the reusable cache directory.""" + step = _step(tmp_path, 'NotAvailable') + + def _build(config, logger, source_file, mapping_file, **kwargs): + (tmp_path / mapping_file).write_text('weights') + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(mapping_step, 'build_mapping_file', _build) + + mapping_step.build_and_cache_mapping( + step, 'source.nc', 'map.nc', 'bilinear') + + cached = tmp_path / 'output' / 'mapping_files' / 'map.nc' + assert cached.read_text() == 'weights' + + +def test_fracture_mapping_uses_xy_grid_donor(tmp_path): + """The mapping source is not the ungridded excess-melt file.""" + version_path = ( + tmp_path / 'AIS' / 'CESM2-WACCM' / 'ssp585' / + 'fracture' / 'v2.1') + version_path.mkdir(parents=True) + (version_path / 'excess_melt_data.nc').touch() + expected = version_path / 'lake_properties_data.nc' + expected.touch() + + config = ConfigParser() + config['ismip7'] = { + 'base_path_ismip7': str(tmp_path), + 'ice_sheet': 'ais', + 'model': 'CESM2-WACCM', + 'scenario': 'ssp585', + } + config['ismip7_fracture'] = {'version': 'latest'} + + source = _resolve_grid_source(config) + + assert source.files == (str(expected),)